Repository: etcd-io/etcd Branch: main Commit: 5791f2b80ecf Files: 1424 Total size: 9.6 MB Directory structure: gitextract_dhrjhetk/ ├── .devcontainer/ │ └── devcontainer.json ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug-report.yml │ │ ├── config.yml │ │ ├── feature-request.yml │ │ └── test-flake.yml │ ├── OWNERS │ ├── PULL_REQUEST_TEMPLATE.md │ ├── SECURITY.md │ ├── dependabot.yml │ └── workflows/ │ ├── OWNERS │ ├── antithesis-test.yml │ ├── antithesis-verify.yml │ ├── antithesis.debugger.yml │ ├── cherrypick-bot-ok-to-test.yaml │ ├── codeql-analysis.yml │ ├── gh-workflow-approve.yaml │ ├── measure-testgrid-flakiness.yaml │ ├── scorecards.yml │ ├── stale.yaml │ └── verify-released-assets.yaml ├── .gitignore ├── .go-version ├── .header ├── ADOPTERS.md ├── CHANGELOG/ │ ├── CHANGELOG-2.3.md │ ├── CHANGELOG-3.0.md │ ├── CHANGELOG-3.1.md │ ├── CHANGELOG-3.2.md │ ├── CHANGELOG-3.3.md │ ├── CHANGELOG-3.4.md │ ├── CHANGELOG-3.5.md │ ├── CHANGELOG-3.6.md │ ├── CHANGELOG-3.7.md │ ├── CHANGELOG-4.0.md │ └── README.md ├── CONTRIBUTING.md ├── DCO ├── Dockerfile ├── Documentation/ │ ├── OWNERS │ ├── README.md │ ├── contributor-guide/ │ │ ├── branch_management.md │ │ ├── bump_etcd_version_k8s.md │ │ ├── community-membership.md │ │ ├── dependency_management.md │ │ ├── exit_codes.md │ │ ├── features.md │ │ ├── local_cluster.md │ │ ├── logging.md │ │ ├── modules.md │ │ ├── prow_jobs.md │ │ ├── release.md │ │ ├── reporting_bugs.md │ │ ├── roadmap.md │ │ ├── triage_issues.md │ │ └── triage_prs.md │ ├── dev-guide/ │ │ └── apispec/ │ │ └── swagger/ │ │ ├── rpc.swagger.json │ │ ├── v3election.swagger.json │ │ └── v3lock.swagger.json │ ├── etcd-internals/ │ │ └── diagrams/ │ │ ├── consistent_read_workflow.drawio │ │ ├── etcd_internal_parts.drawio │ │ ├── write_workflow_follower.drawio │ │ └── write_workflow_leader.drawio │ └── postmortems/ │ └── v3.5-data-inconsistency.md ├── GOVERNANCE.md ├── LICENSE ├── Makefile ├── OWNERS ├── OWNERS_ALIASES ├── Procfile ├── README.md ├── api/ │ ├── .gomodguard.yaml │ ├── LICENSE │ ├── authpb/ │ │ ├── auth.pb.go │ │ ├── auth.proto │ │ └── deprecated.go │ ├── etcdserverpb/ │ │ ├── etcdserver.pb.go │ │ ├── etcdserver.proto │ │ ├── gw/ │ │ │ └── rpc.pb.gw.go │ │ ├── raft_internal.pb.go │ │ ├── raft_internal.proto │ │ ├── raft_internal_stringer.go │ │ ├── raft_internal_stringer_test.go │ │ ├── rpc.pb.go │ │ ├── rpc.proto │ │ └── rpc_grpc.pb.go │ ├── go.mod │ ├── go.sum │ ├── membershippb/ │ │ ├── membership.pb.go │ │ └── membership.proto │ ├── mvccpb/ │ │ ├── deprecated.go │ │ ├── kv.pb.go │ │ └── kv.proto │ ├── v3rpc/ │ │ └── rpctypes/ │ │ ├── doc.go │ │ ├── error.go │ │ ├── error_test.go │ │ ├── md.go │ │ └── metadatafields.go │ ├── version/ │ │ ├── version.go │ │ └── version_test.go │ └── versionpb/ │ ├── version.pb.go │ └── version.proto ├── bill-of-materials.json ├── bill-of-materials.override.json ├── cache/ │ ├── LICENSE │ ├── OWNERS │ ├── README.md │ ├── cache.go │ ├── cache_test.go │ ├── config.go │ ├── demux.go │ ├── demux_test.go │ ├── go.mod │ ├── go.sum │ ├── predicate.go │ ├── ready.go │ ├── ready_test.go │ ├── ringbuffer.go │ ├── ringbuffer_test.go │ ├── snapshot.go │ ├── store.go │ ├── store_test.go │ └── watcher.go ├── client/ │ ├── pkg/ │ │ ├── .gomodguard.yaml │ │ ├── LICENSE │ │ ├── fileutil/ │ │ │ ├── dir_unix.go │ │ │ ├── dir_windows.go │ │ │ ├── doc.go │ │ │ ├── filereader.go │ │ │ ├── filereader_test.go │ │ │ ├── fileutil.go │ │ │ ├── fileutil_test.go │ │ │ ├── lock.go │ │ │ ├── lock_flock.go │ │ │ ├── lock_linux.go │ │ │ ├── lock_linux_test.go │ │ │ ├── lock_plan9.go │ │ │ ├── lock_solaris.go │ │ │ ├── lock_test.go │ │ │ ├── lock_unix.go │ │ │ ├── lock_windows.go │ │ │ ├── preallocate.go │ │ │ ├── preallocate_darwin.go │ │ │ ├── preallocate_test.go │ │ │ ├── preallocate_unix.go │ │ │ ├── preallocate_unsupported.go │ │ │ ├── purge.go │ │ │ ├── purge_test.go │ │ │ ├── read_dir.go │ │ │ ├── read_dir_test.go │ │ │ ├── sync.go │ │ │ ├── sync_darwin.go │ │ │ └── sync_linux.go │ │ ├── go.mod │ │ ├── go.sum │ │ ├── logutil/ │ │ │ ├── doc.go │ │ │ ├── log_format.go │ │ │ ├── log_format_test.go │ │ │ ├── log_level.go │ │ │ ├── zap.go │ │ │ ├── zap_journal.go │ │ │ ├── zap_journal_test.go │ │ │ └── zap_test.go │ │ ├── pathutil/ │ │ │ ├── path.go │ │ │ └── path_test.go │ │ ├── srv/ │ │ │ ├── srv.go │ │ │ └── srv_test.go │ │ ├── systemd/ │ │ │ ├── doc.go │ │ │ └── journal.go │ │ ├── testutil/ │ │ │ ├── assert.go │ │ │ ├── before.go │ │ │ ├── leak.go │ │ │ ├── leak_test.go │ │ │ ├── pauseable_handler.go │ │ │ ├── recorder.go │ │ │ ├── testingtb.go │ │ │ ├── testutil.go │ │ │ └── var.go │ │ ├── tlsutil/ │ │ │ ├── cipher_suites.go │ │ │ ├── cipher_suites_test.go │ │ │ ├── doc.go │ │ │ ├── tlsutil.go │ │ │ ├── versions.go │ │ │ └── versions_test.go │ │ ├── transport/ │ │ │ ├── doc.go │ │ │ ├── keepalive_listener.go │ │ │ ├── keepalive_listener_openbsd.go │ │ │ ├── keepalive_listener_test.go │ │ │ ├── keepalive_listener_unix.go │ │ │ ├── limit_listen.go │ │ │ ├── listener.go │ │ │ ├── listener_opts.go │ │ │ ├── listener_test.go │ │ │ ├── listener_tls.go │ │ │ ├── sockopt.go │ │ │ ├── sockopt_solaris.go │ │ │ ├── sockopt_unix.go │ │ │ ├── sockopt_wasm.go │ │ │ ├── sockopt_windows.go │ │ │ ├── timeout_conn.go │ │ │ ├── timeout_dialer.go │ │ │ ├── timeout_dialer_test.go │ │ │ ├── timeout_listener.go │ │ │ ├── timeout_listener_test.go │ │ │ ├── timeout_transport.go │ │ │ ├── timeout_transport_test.go │ │ │ ├── tls.go │ │ │ ├── tls_test.go │ │ │ ├── transport.go │ │ │ ├── transport_test.go │ │ │ └── unix_listener.go │ │ ├── types/ │ │ │ ├── doc.go │ │ │ ├── id.go │ │ │ ├── id_test.go │ │ │ ├── set.go │ │ │ ├── set_test.go │ │ │ ├── slice.go │ │ │ ├── slice_test.go │ │ │ ├── urls.go │ │ │ ├── urls_test.go │ │ │ ├── urlsmap.go │ │ │ └── urlsmap_test.go │ │ └── verify/ │ │ └── verify.go │ └── v3/ │ ├── .gomodguard.yaml │ ├── LICENSE │ ├── OWNERS │ ├── README.md │ ├── auth.go │ ├── client.go │ ├── client_test.go │ ├── clientv3util/ │ │ ├── example_key_test.go │ │ └── util.go │ ├── cluster.go │ ├── compact_op.go │ ├── compact_op_test.go │ ├── compare.go │ ├── concurrency/ │ │ ├── doc.go │ │ ├── election.go │ │ ├── key.go │ │ ├── main_test.go │ │ ├── mutex.go │ │ ├── session.go │ │ ├── stm.go │ │ └── stm_test.go │ ├── config.go │ ├── config_test.go │ ├── credentials/ │ │ ├── credentials.go │ │ └── credentials_test.go │ ├── ctx.go │ ├── ctx_test.go │ ├── doc.go │ ├── experimental/ │ │ └── recipes/ │ │ ├── barrier.go │ │ ├── client.go │ │ ├── doc.go │ │ ├── double_barrier.go │ │ ├── grpc_gateway/ │ │ │ └── user_add.sh │ │ ├── key.go │ │ ├── priority_queue.go │ │ ├── queue.go │ │ ├── rwmutex.go │ │ └── watch.go │ ├── go.mod │ ├── go.sum │ ├── internal/ │ │ ├── endpoint/ │ │ │ ├── endpoint.go │ │ │ └── endpoint_test.go │ │ └── resolver/ │ │ └── resolver.go │ ├── kubernetes/ │ │ ├── client.go │ │ └── interface.go │ ├── kv.go │ ├── lease.go │ ├── leasing/ │ │ ├── cache.go │ │ ├── doc.go │ │ ├── kv.go │ │ ├── txn.go │ │ └── util.go │ ├── logger.go │ ├── main_test.go │ ├── maintenance.go │ ├── mirror/ │ │ └── syncer.go │ ├── mock/ │ │ └── mockserver/ │ │ ├── doc.go │ │ └── mockserver.go │ ├── namespace/ │ │ ├── doc.go │ │ ├── kv.go │ │ ├── lease.go │ │ ├── util.go │ │ ├── util_test.go │ │ └── watch.go │ ├── naming/ │ │ ├── doc.go │ │ ├── endpoints/ │ │ │ ├── endpoints.go │ │ │ ├── endpoints_impl.go │ │ │ └── internal/ │ │ │ └── update.go │ │ └── resolver/ │ │ └── resolver.go │ ├── op.go │ ├── op_test.go │ ├── options.go │ ├── ordering/ │ │ ├── doc.go │ │ ├── kv.go │ │ ├── kv_test.go │ │ └── util.go │ ├── retry.go │ ├── retry_interceptor.go │ ├── retry_interceptor_test.go │ ├── snapshot/ │ │ ├── doc.go │ │ └── v3_snapshot.go │ ├── sort.go │ ├── txn.go │ ├── txn_test.go │ ├── utils.go │ ├── watch.go │ ├── watch_test.go │ └── yaml/ │ ├── config.go │ └── config_test.go ├── code-of-conduct.md ├── codecov.yml ├── contrib/ │ ├── OWNERS │ ├── README.md │ ├── lock/ │ │ └── README.md │ ├── mixin/ │ │ ├── .gitignore │ │ ├── .lint │ │ ├── Makefile │ │ ├── OWNERS │ │ ├── README.md │ │ ├── alerts/ │ │ │ └── alerts.libsonnet │ │ ├── config.libsonnet │ │ ├── dashboards/ │ │ │ ├── dashboards.libsonnet │ │ │ ├── etcd-grafana7x.libsonnet │ │ │ ├── etcd.libsonnet │ │ │ ├── g.libsonnet │ │ │ ├── panels.libsonnet │ │ │ ├── targets.libsonnet │ │ │ └── variables.libsonnet │ │ ├── jsonnetfile.json │ │ ├── jsonnetfile.lock.json │ │ ├── mixin.libsonnet │ │ └── test.yaml │ ├── raftexample/ │ │ ├── Procfile │ │ ├── README.md │ │ ├── doc.go │ │ ├── httpapi.go │ │ ├── kvstore.go │ │ ├── kvstore_test.go │ │ ├── listener.go │ │ ├── main.go │ │ ├── raft.go │ │ ├── raft_test.go │ │ └── raftexample_test.go │ └── systemd/ │ ├── etcd.service │ ├── etcd3-multinode/ │ │ └── README.md │ └── sysusers.d/ │ └── 20-etcd.conf ├── dummy.go ├── etcd.conf.yml.sample ├── etcdctl/ │ ├── .gomodguard.yaml │ ├── LICENSE │ ├── OWNERS │ ├── README.md │ ├── READMEv2.md │ ├── ctlv3/ │ │ ├── command/ │ │ │ ├── alarm_command.go │ │ │ ├── auth_command.go │ │ │ ├── check.go │ │ │ ├── compaction_command.go │ │ │ ├── completion_command.go │ │ │ ├── defrag_command.go │ │ │ ├── del_command.go │ │ │ ├── diagnosis/ │ │ │ │ ├── engine/ │ │ │ │ │ ├── diagnosis.go │ │ │ │ │ └── intf/ │ │ │ │ │ └── plugin.go │ │ │ │ ├── examples/ │ │ │ │ │ └── etcd_diagnosis_report.json │ │ │ │ └── plugins/ │ │ │ │ ├── common/ │ │ │ │ │ ├── checker.go │ │ │ │ │ └── client.go │ │ │ │ ├── epstatus/ │ │ │ │ │ └── plugin.go │ │ │ │ ├── membership/ │ │ │ │ │ └── plugin.go │ │ │ │ ├── metrics/ │ │ │ │ │ └── plugin.go │ │ │ │ └── read/ │ │ │ │ └── plugin.go │ │ │ ├── diagnosis_command.go │ │ │ ├── doc.go │ │ │ ├── downgrade_command.go │ │ │ ├── elect_command.go │ │ │ ├── ep_command.go │ │ │ ├── get_command.go │ │ │ ├── global.go │ │ │ ├── groups.go │ │ │ ├── help_command.go │ │ │ ├── lease_command.go │ │ │ ├── lock_command.go │ │ │ ├── make_mirror_command.go │ │ │ ├── member_command.go │ │ │ ├── move_leader_command.go │ │ │ ├── options_command.go │ │ │ ├── printer.go │ │ │ ├── printer_fields.go │ │ │ ├── printer_json.go │ │ │ ├── printer_json_test.go │ │ │ ├── printer_protobuf.go │ │ │ ├── printer_simple.go │ │ │ ├── printer_table.go │ │ │ ├── put_command.go │ │ │ ├── role_command.go │ │ │ ├── snapshot_command.go │ │ │ ├── txn_command.go │ │ │ ├── user_command.go │ │ │ ├── util.go │ │ │ ├── util_test.go │ │ │ ├── version_command.go │ │ │ ├── watch_command.go │ │ │ └── watch_command_test.go │ │ └── ctl.go │ ├── doc/ │ │ └── mirror_maker.md │ ├── go.mod │ ├── go.sum │ ├── main.go │ └── util/ │ └── normalizer.go ├── etcdutl/ │ ├── .gomodguard.yaml │ ├── LICENSE │ ├── OWNERS │ ├── README.md │ ├── ctl.go │ ├── etcdutl/ │ │ ├── bucket_command.go │ │ ├── common.go │ │ ├── common_test.go │ │ ├── completion_commmand.go │ │ ├── defrag_command.go │ │ ├── hashkv_command.go │ │ ├── hashkv_command_test.go │ │ ├── migrate_command.go │ │ ├── printer.go │ │ ├── printer_fields.go │ │ ├── printer_json.go │ │ ├── printer_protobuf.go │ │ ├── printer_simple.go │ │ ├── printer_table.go │ │ ├── snapshot_command.go │ │ └── version_command.go │ ├── go.mod │ ├── go.sum │ ├── main.go │ └── snapshot/ │ ├── doc.go │ ├── v3_snapshot.go │ └── v3_snapshot_test.go ├── go.mod ├── go.sum ├── go.work ├── go.work.sum ├── hack/ │ ├── README.md │ ├── benchmark/ │ │ ├── README.md │ │ └── bench.sh │ ├── insta-discovery/ │ │ ├── Procfile │ │ ├── README.md │ │ └── discovery │ ├── kubernetes-deploy/ │ │ ├── README.md │ │ ├── etcd.yml │ │ └── vulcand.yml │ ├── patch/ │ │ ├── README.md │ │ └── cherrypick.sh │ └── tls-setup/ │ ├── Makefile │ ├── Procfile │ ├── README.md │ └── config/ │ ├── ca-config.json │ ├── ca-csr.json │ └── req-csr.json ├── pkg/ │ ├── .gomodguard.yaml │ ├── LICENSE │ ├── README.md │ ├── adt/ │ │ ├── README.md │ │ ├── adt.go │ │ ├── example_test.go │ │ ├── interval_tree.go │ │ └── interval_tree_test.go │ ├── cobrautl/ │ │ ├── error.go │ │ └── help.go │ ├── contention/ │ │ ├── contention.go │ │ └── doc.go │ ├── cpuutil/ │ │ ├── doc.go │ │ └── endian.go │ ├── crc/ │ │ ├── crc.go │ │ └── crc_test.go │ ├── debugutil/ │ │ ├── doc.go │ │ └── pprof.go │ ├── expect/ │ │ ├── expect.go │ │ └── expect_test.go │ ├── featuregate/ │ │ ├── feature_gate.go │ │ └── feature_gate_test.go │ ├── flags/ │ │ ├── flag.go │ │ ├── flag_test.go │ │ ├── ignored.go │ │ ├── selective_string.go │ │ ├── selective_string_test.go │ │ ├── strings.go │ │ ├── strings_test.go │ │ ├── uint32.go │ │ ├── uint32_test.go │ │ ├── unique_strings.go │ │ ├── unique_strings_test.go │ │ ├── unique_urls.go │ │ ├── unique_urls_test.go │ │ ├── urls.go │ │ └── urls_test.go │ ├── go.mod │ ├── go.sum │ ├── grpctesting/ │ │ ├── recorder.go │ │ └── stub_server.go │ ├── httputil/ │ │ ├── httputil.go │ │ └── httputil_test.go │ ├── idutil/ │ │ ├── id.go │ │ └── id_test.go │ ├── ioutil/ │ │ ├── pagewriter.go │ │ ├── pagewriter_test.go │ │ ├── readcloser.go │ │ ├── readcloser_test.go │ │ ├── reader.go │ │ ├── reader_test.go │ │ └── util.go │ ├── netutil/ │ │ ├── doc.go │ │ ├── host_normalize.go │ │ ├── host_normalize_test.go │ │ ├── netutil.go │ │ ├── netutil_test.go │ │ ├── routes.go │ │ ├── routes_linux.go │ │ └── routes_linux_test.go │ ├── notify/ │ │ └── notify.go │ ├── osutil/ │ │ ├── interrupt_unix.go │ │ ├── interrupt_windows.go │ │ ├── osutil.go │ │ ├── osutil_test.go │ │ ├── signal.go │ │ └── signal_linux.go │ ├── pbutil/ │ │ ├── pbutil.go │ │ └── pbutil_test.go │ ├── proxy/ │ │ ├── doc.go │ │ ├── fixtures/ │ │ │ ├── ca-csr.json │ │ │ ├── ca.crt │ │ │ ├── gencert.json │ │ │ ├── gencerts.sh │ │ │ ├── server-ca-csr.json │ │ │ ├── server.crt │ │ │ └── server.key.insecure │ │ ├── server.go │ │ └── server_test.go │ ├── report/ │ │ ├── doc.go │ │ ├── perfdash.go │ │ ├── report.go │ │ ├── report_test.go │ │ ├── timeseries.go │ │ ├── timeseries_test.go │ │ └── weighted.go │ ├── runtime/ │ │ ├── fds_linux.go │ │ └── fds_other.go │ ├── schedule/ │ │ ├── doc.go │ │ ├── schedule.go │ │ └── schedule_test.go │ ├── stringutil/ │ │ ├── doc.go │ │ ├── rand.go │ │ └── rand_test.go │ ├── traceutil/ │ │ ├── trace.go │ │ └── trace_test.go │ └── wait/ │ ├── wait.go │ ├── wait_test.go │ ├── wait_time.go │ └── wait_time_test.go ├── scripts/ │ ├── OWNERS │ ├── README │ ├── benchmark_test.sh │ ├── build-binary.sh │ ├── build-docker.sh │ ├── build-release.sh │ ├── build.sh │ ├── build_lib.sh │ ├── build_tools.sh │ ├── codecov_upload.sh │ ├── etcd_version_annotations.txt │ ├── fix/ │ │ ├── bom.sh │ │ ├── mod-tidy.sh │ │ ├── shell_ws.sh │ │ └── yamllint.sh │ ├── fuzzing.sh │ ├── genproto.sh │ ├── markdown_diff_lint.sh │ ├── measure-testgrid-flakiness.sh │ ├── release.sh │ ├── release_mod.sh │ ├── release_notes.tpl.txt │ ├── sync_go_toolchain_directive.sh │ ├── test.sh │ ├── test_images.sh │ ├── test_lib.sh │ ├── test_utils.sh │ ├── update_dep.sh │ ├── update_go_workspace.sh │ ├── update_proto_annotations.sh │ ├── verify_genproto.sh │ ├── verify_go_versions.sh │ ├── verify_golangci-lint_version.sh │ ├── verify_grpc_experimental.sh │ └── verify_proto_annotations.sh ├── security/ │ ├── OWNERS │ ├── README.md │ ├── email-templates.md │ └── security-release-process.md ├── server/ │ ├── .gomodguard.yaml │ ├── LICENSE │ ├── auth/ │ │ ├── doc.go │ │ ├── jwt.go │ │ ├── jwt_test.go │ │ ├── main_test.go │ │ ├── metrics.go │ │ ├── nop.go │ │ ├── options.go │ │ ├── range_perm_cache.go │ │ ├── range_perm_cache_test.go │ │ ├── simple_token.go │ │ ├── simple_token_test.go │ │ ├── store.go │ │ ├── store_mock_test.go │ │ └── store_test.go │ ├── config/ │ │ ├── config.go │ │ ├── config_test.go │ │ ├── v2_deprecation.go │ │ └── v2_deprecation_test.go │ ├── embed/ │ │ ├── auth_test.go │ │ ├── config.go │ │ ├── config_logging.go │ │ ├── config_logging_journal_unix.go │ │ ├── config_logging_journal_windows.go │ │ ├── config_test.go │ │ ├── config_tracing.go │ │ ├── config_tracing_test.go │ │ ├── doc.go │ │ ├── etcd.go │ │ ├── etcd_test.go │ │ ├── serve.go │ │ ├── serve_test.go │ │ └── util.go │ ├── etcdmain/ │ │ ├── config.go │ │ ├── config_test.go │ │ ├── doc.go │ │ ├── etcd.go │ │ ├── gateway.go │ │ ├── grpc_proxy.go │ │ ├── grpc_proxy_logger.go │ │ ├── grpc_proxy_logger_test.go │ │ ├── help.go │ │ ├── main.go │ │ └── util.go │ ├── etcdserver/ │ │ ├── adapters.go │ │ ├── api/ │ │ │ ├── capability.go │ │ │ ├── cluster.go │ │ │ ├── doc.go │ │ │ ├── etcdhttp/ │ │ │ │ ├── debug.go │ │ │ │ ├── doc.go │ │ │ │ ├── health.go │ │ │ │ ├── health_test.go │ │ │ │ ├── metrics.go │ │ │ │ ├── peer.go │ │ │ │ ├── peer_test.go │ │ │ │ ├── types/ │ │ │ │ │ ├── errors.go │ │ │ │ │ └── errors_test.go │ │ │ │ ├── utils.go │ │ │ │ ├── version.go │ │ │ │ └── version_test.go │ │ │ ├── membership/ │ │ │ │ ├── cluster.go │ │ │ │ ├── cluster_opts.go │ │ │ │ ├── cluster_test.go │ │ │ │ ├── doc.go │ │ │ │ ├── errors.go │ │ │ │ ├── member.go │ │ │ │ ├── member_test.go │ │ │ │ ├── membership_test.go │ │ │ │ ├── metrics.go │ │ │ │ ├── store.go │ │ │ │ ├── storev2.go │ │ │ │ └── storev2_test.go │ │ │ ├── rafthttp/ │ │ │ │ ├── coder.go │ │ │ │ ├── doc.go │ │ │ │ ├── fake_roundtripper_test.go │ │ │ │ ├── functional_test.go │ │ │ │ ├── http.go │ │ │ │ ├── http_test.go │ │ │ │ ├── metrics.go │ │ │ │ ├── msg_codec.go │ │ │ │ ├── msg_codec_test.go │ │ │ │ ├── msgappv2_codec.go │ │ │ │ ├── msgappv2_codec_test.go │ │ │ │ ├── peer.go │ │ │ │ ├── peer_status.go │ │ │ │ ├── peer_test.go │ │ │ │ ├── pipeline.go │ │ │ │ ├── pipeline_test.go │ │ │ │ ├── probing_status.go │ │ │ │ ├── remote.go │ │ │ │ ├── snapshot_sender.go │ │ │ │ ├── snapshot_test.go │ │ │ │ ├── stream.go │ │ │ │ ├── stream_test.go │ │ │ │ ├── transport.go │ │ │ │ ├── transport_bench_test.go │ │ │ │ ├── transport_test.go │ │ │ │ ├── urlpick.go │ │ │ │ ├── urlpick_test.go │ │ │ │ ├── util.go │ │ │ │ └── util_test.go │ │ │ ├── snap/ │ │ │ │ ├── db.go │ │ │ │ ├── doc.go │ │ │ │ ├── message.go │ │ │ │ ├── metrics.go │ │ │ │ ├── snappb/ │ │ │ │ │ ├── snap.pb.go │ │ │ │ │ └── snap.proto │ │ │ │ ├── snapshotter.go │ │ │ │ └── snapshotter_test.go │ │ │ ├── v2error/ │ │ │ │ ├── error.go │ │ │ │ └── error_test.go │ │ │ ├── v2stats/ │ │ │ │ ├── leader.go │ │ │ │ ├── queue.go │ │ │ │ └── server.go │ │ │ ├── v2store/ │ │ │ │ ├── doc.go │ │ │ │ ├── event.go │ │ │ │ ├── event_history.go │ │ │ │ ├── event_queue.go │ │ │ │ ├── event_test.go │ │ │ │ ├── heap_test.go │ │ │ │ ├── metrics.go │ │ │ │ ├── node.go │ │ │ │ ├── node_extern.go │ │ │ │ ├── node_extern_test.go │ │ │ │ ├── node_test.go │ │ │ │ ├── stats.go │ │ │ │ ├── stats_test.go │ │ │ │ ├── store.go │ │ │ │ ├── store_bench_test.go │ │ │ │ ├── store_ttl_test.go │ │ │ │ ├── ttl_key_heap.go │ │ │ │ ├── watcher.go │ │ │ │ ├── watcher_hub.go │ │ │ │ ├── watcher_hub_test.go │ │ │ │ └── watcher_test.go │ │ │ ├── v3alarm/ │ │ │ │ └── alarms.go │ │ │ ├── v3client/ │ │ │ │ ├── doc.go │ │ │ │ └── v3client.go │ │ │ ├── v3compactor/ │ │ │ │ ├── compactor.go │ │ │ │ ├── compactor_test.go │ │ │ │ ├── doc.go │ │ │ │ ├── periodic.go │ │ │ │ ├── periodic_test.go │ │ │ │ ├── revision.go │ │ │ │ └── revision_test.go │ │ │ ├── v3discovery/ │ │ │ │ ├── discovery.go │ │ │ │ └── discovery_test.go │ │ │ ├── v3election/ │ │ │ │ ├── doc.go │ │ │ │ ├── election.go │ │ │ │ └── v3electionpb/ │ │ │ │ ├── gw/ │ │ │ │ │ └── v3election.pb.gw.go │ │ │ │ ├── v3election.pb.go │ │ │ │ ├── v3election.proto │ │ │ │ └── v3election_grpc.pb.go │ │ │ ├── v3lock/ │ │ │ │ ├── doc.go │ │ │ │ ├── lock.go │ │ │ │ └── v3lockpb/ │ │ │ │ ├── gw/ │ │ │ │ │ └── v3lock.pb.gw.go │ │ │ │ ├── v3lock.pb.go │ │ │ │ ├── v3lock.proto │ │ │ │ └── v3lock_grpc.pb.go │ │ │ └── v3rpc/ │ │ │ ├── auth.go │ │ │ ├── codec.go │ │ │ ├── grpc.go │ │ │ ├── header.go │ │ │ ├── health.go │ │ │ ├── interceptor.go │ │ │ ├── key.go │ │ │ ├── key_test.go │ │ │ ├── lease.go │ │ │ ├── maintenance.go │ │ │ ├── member.go │ │ │ ├── metrics.go │ │ │ ├── quota.go │ │ │ ├── util.go │ │ │ ├── util_test.go │ │ │ ├── validationfuzz_test.go │ │ │ ├── watch.go │ │ │ └── watch_test.go │ │ ├── apply/ │ │ │ ├── apply.go │ │ │ ├── auth.go │ │ │ ├── auth_test.go │ │ │ ├── backend.go │ │ │ ├── capped.go │ │ │ ├── corrupt.go │ │ │ ├── interface.go │ │ │ ├── metrics.go │ │ │ ├── quota.go │ │ │ ├── uber_applier.go │ │ │ └── uber_applier_test.go │ │ ├── bootstrap.go │ │ ├── bootstrap_test.go │ │ ├── cindex/ │ │ │ ├── cindex.go │ │ │ ├── cindex_test.go │ │ │ └── doc.go │ │ ├── cluster_util.go │ │ ├── cluster_util_test.go │ │ ├── corrupt.go │ │ ├── corrupt_test.go │ │ ├── doc.go │ │ ├── errors/ │ │ │ └── errors.go │ │ ├── metrics.go │ │ ├── raft.go │ │ ├── raft_test.go │ │ ├── server.go │ │ ├── server_access_control.go │ │ ├── server_access_control_test.go │ │ ├── server_test.go │ │ ├── snapshot_merge.go │ │ ├── tracing.go │ │ ├── txn/ │ │ │ ├── delete.go │ │ │ ├── metrics.go │ │ │ ├── metrics_test.go │ │ │ ├── put.go │ │ │ ├── range.go │ │ │ ├── txn.go │ │ │ ├── txn_test.go │ │ │ ├── util.go │ │ │ ├── util_bench_test.go │ │ │ └── util_test.go │ │ ├── util.go │ │ ├── util_test.go │ │ ├── v3_server.go │ │ ├── version/ │ │ │ ├── doc.go │ │ │ ├── downgrade.go │ │ │ ├── downgrade_test.go │ │ │ ├── errors.go │ │ │ ├── monitor.go │ │ │ ├── monitor_test.go │ │ │ ├── version.go │ │ │ └── version_test.go │ │ ├── zap_raft.go │ │ └── zap_raft_test.go │ ├── features/ │ │ └── etcd_features.go │ ├── go.mod │ ├── go.sum │ ├── lease/ │ │ ├── doc.go │ │ ├── lease.go │ │ ├── lease_queue.go │ │ ├── lease_queue_test.go │ │ ├── leasehttp/ │ │ │ ├── doc.go │ │ │ ├── http.go │ │ │ └── http_test.go │ │ ├── leasepb/ │ │ │ ├── lease.pb.go │ │ │ └── lease.proto │ │ ├── lessor.go │ │ ├── lessor_bench_test.go │ │ ├── lessor_test.go │ │ └── metrics.go │ ├── main.go │ ├── mock/ │ │ ├── mockstorage/ │ │ │ ├── doc.go │ │ │ └── storage_recorder.go │ │ ├── mockstore/ │ │ │ ├── doc.go │ │ │ └── store_recorder.go │ │ └── mockwait/ │ │ ├── doc.go │ │ └── wait_recorder.go │ ├── proxy/ │ │ ├── grpcproxy/ │ │ │ ├── adapter/ │ │ │ │ ├── auth_client_adapter.go │ │ │ │ ├── chan_stream.go │ │ │ │ ├── cluster_client_adapter.go │ │ │ │ ├── doc.go │ │ │ │ ├── election_client_adapter.go │ │ │ │ ├── kv_client_adapter.go │ │ │ │ ├── lease_client_adapter.go │ │ │ │ ├── lock_client_adapter.go │ │ │ │ ├── maintenance_client_adapter.go │ │ │ │ └── watch_client_adapter.go │ │ │ ├── auth.go │ │ │ ├── cache/ │ │ │ │ └── store.go │ │ │ ├── cluster.go │ │ │ ├── doc.go │ │ │ ├── election.go │ │ │ ├── health.go │ │ │ ├── kv.go │ │ │ ├── leader.go │ │ │ ├── lease.go │ │ │ ├── lock.go │ │ │ ├── maintenance.go │ │ │ ├── metrics.go │ │ │ ├── register.go │ │ │ ├── util.go │ │ │ ├── watch.go │ │ │ ├── watch_broadcast.go │ │ │ ├── watch_broadcasts.go │ │ │ ├── watch_ranges.go │ │ │ └── watcher.go │ │ └── tcpproxy/ │ │ ├── doc.go │ │ ├── userspace.go │ │ └── userspace_test.go │ ├── storage/ │ │ ├── backend/ │ │ │ ├── backend.go │ │ │ ├── backend_bench_test.go │ │ │ ├── backend_test.go │ │ │ ├── batch_tx.go │ │ │ ├── batch_tx_test.go │ │ │ ├── config_default.go │ │ │ ├── config_linux.go │ │ │ ├── config_windows.go │ │ │ ├── doc.go │ │ │ ├── export_test.go │ │ │ ├── hooks.go │ │ │ ├── hooks_test.go │ │ │ ├── metrics.go │ │ │ ├── read_tx.go │ │ │ ├── testing/ │ │ │ │ └── betesting.go │ │ │ ├── tx_buffer.go │ │ │ ├── tx_buffer_test.go │ │ │ ├── verify.go │ │ │ └── verify_test.go │ │ ├── backend.go │ │ ├── datadir/ │ │ │ ├── datadir.go │ │ │ ├── datadir_test.go │ │ │ └── doc.go │ │ ├── hooks.go │ │ ├── metrics.go │ │ ├── mvcc/ │ │ │ ├── doc.go │ │ │ ├── hash.go │ │ │ ├── hash_test.go │ │ │ ├── index.go │ │ │ ├── index_bench_test.go │ │ │ ├── index_test.go │ │ │ ├── key_index.go │ │ │ ├── key_index_test.go │ │ │ ├── kv.go │ │ │ ├── kv_test.go │ │ │ ├── kv_view.go │ │ │ ├── kvstore.go │ │ │ ├── kvstore_bench_test.go │ │ │ ├── kvstore_compaction.go │ │ │ ├── kvstore_compaction_test.go │ │ │ ├── kvstore_test.go │ │ │ ├── kvstore_txn.go │ │ │ ├── metrics.go │ │ │ ├── metrics_txn.go │ │ │ ├── revision.go │ │ │ ├── store.go │ │ │ ├── store_test.go │ │ │ ├── testutil/ │ │ │ │ └── hash.go │ │ │ ├── watchable_store.go │ │ │ ├── watchable_store_bench_test.go │ │ │ ├── watchable_store_test.go │ │ │ ├── watchable_store_txn.go │ │ │ ├── watcher.go │ │ │ ├── watcher_bench_test.go │ │ │ ├── watcher_group.go │ │ │ └── watcher_test.go │ │ ├── quota.go │ │ ├── schema/ │ │ │ ├── actions.go │ │ │ ├── actions_test.go │ │ │ ├── alarm.go │ │ │ ├── auth.go │ │ │ ├── auth_roles.go │ │ │ ├── auth_roles_test.go │ │ │ ├── auth_test.go │ │ │ ├── auth_users.go │ │ │ ├── auth_users_test.go │ │ │ ├── bucket.go │ │ │ ├── changes.go │ │ │ ├── changes_test.go │ │ │ ├── cindex.go │ │ │ ├── confstate.go │ │ │ ├── confstate_test.go │ │ │ ├── lease.go │ │ │ ├── lease_test.go │ │ │ ├── membership.go │ │ │ ├── migration.go │ │ │ ├── migration_test.go │ │ │ ├── schema.go │ │ │ ├── schema_test.go │ │ │ ├── version.go │ │ │ └── version_test.go │ │ ├── storage.go │ │ ├── util.go │ │ └── wal/ │ │ ├── decoder.go │ │ ├── doc.go │ │ ├── encoder.go │ │ ├── file_pipeline.go │ │ ├── file_pipeline_test.go │ │ ├── metrics.go │ │ ├── record_test.go │ │ ├── repair.go │ │ ├── repair_test.go │ │ ├── testdata/ │ │ │ └── TestNew.wal │ │ ├── testing/ │ │ │ └── waltesting.go │ │ ├── util.go │ │ ├── version.go │ │ ├── version_test.go │ │ ├── wal.go │ │ ├── wal_bench_test.go │ │ ├── wal_test.go │ │ └── walpb/ │ │ ├── record.go │ │ ├── record.pb.go │ │ ├── record.proto │ │ └── record_test.go │ └── verify/ │ ├── doc.go │ └── verify.go ├── tests/ │ ├── LICENSE │ ├── OWNERS │ ├── antithesis/ │ │ ├── Makefile │ │ ├── README.md │ │ ├── config/ │ │ │ ├── Dockerfile │ │ │ ├── docker-compose-1-node.yml │ │ │ ├── docker-compose-3-node.yml │ │ │ └── manifests/ │ │ │ └── default-etcd-3-replicas.yaml │ │ ├── server/ │ │ │ ├── Dockerfile │ │ │ └── inject/ │ │ │ └── verify.patch │ │ └── test-template/ │ │ ├── Dockerfile │ │ ├── entrypoint/ │ │ │ └── main.go │ │ └── robustness/ │ │ ├── common/ │ │ │ └── path.go │ │ ├── finally/ │ │ │ └── main.go │ │ └── traffic/ │ │ └── main.go │ ├── common/ │ │ ├── alarm_test.go │ │ ├── auth_test.go │ │ ├── auth_util.go │ │ ├── compact_test.go │ │ ├── defrag_test.go │ │ ├── e2e_test.go │ │ ├── endpoint_test.go │ │ ├── grpc_test.go │ │ ├── hashkv_test.go │ │ ├── integration_test.go │ │ ├── kv_test.go │ │ ├── lease_test.go │ │ ├── main_test.go │ │ ├── maintenance_auth_test.go │ │ ├── member_test.go │ │ ├── role_test.go │ │ ├── status_test.go │ │ ├── txn_test.go │ │ ├── unit_test.go │ │ ├── user_test.go │ │ ├── wait_leader_test.go │ │ └── watch_test.go │ ├── e2e/ │ │ ├── cluster_downgrade_test.go │ │ ├── cmux_test.go │ │ ├── corrupt_test.go │ │ ├── ctl_v3_auth_cluster_test.go │ │ ├── ctl_v3_auth_no_proxy_test.go │ │ ├── ctl_v3_auth_security_test.go │ │ ├── ctl_v3_auth_test.go │ │ ├── ctl_v3_completion_test.go │ │ ├── ctl_v3_defrag_test.go │ │ ├── ctl_v3_elect_test.go │ │ ├── ctl_v3_kv_test.go │ │ ├── ctl_v3_lease_test.go │ │ ├── ctl_v3_lock_test.go │ │ ├── ctl_v3_make_mirror_test.go │ │ ├── ctl_v3_member_no_proxy_test.go │ │ ├── ctl_v3_member_test.go │ │ ├── ctl_v3_move_leader_test.go │ │ ├── ctl_v3_role_test.go │ │ ├── ctl_v3_snapshot_test.go │ │ ├── ctl_v3_test.go │ │ ├── ctl_v3_watch_test.go │ │ ├── defrag_no_space_test.go │ │ ├── discovery_v3_test.go │ │ ├── doc.go │ │ ├── etcd_config_test.go │ │ ├── etcd_grpcproxy_test.go │ │ ├── etcd_mix_versions_test.go │ │ ├── etcd_release_upgrade_test.go │ │ ├── failover_test.go │ │ ├── force_new_cluster_test.go │ │ ├── gateway_test.go │ │ ├── graceful_shutdown_test.go │ │ ├── http_health_check_test.go │ │ ├── leader_snapshot_no_proxy_test.go │ │ ├── logging_test.go │ │ ├── main_test.go │ │ ├── member_no_proxy_test.go │ │ ├── metrics_test.go │ │ ├── promote_experimental_flag_test.go │ │ ├── reproduce_17780_test.go │ │ ├── reproduce_18667_test.go │ │ ├── reproduce_19406_test.go │ │ ├── reproduce_20271_test.go │ │ ├── runtime_reconfiguration_test.go │ │ ├── utils.go │ │ ├── utl_migrate_test.go │ │ ├── v2store_deprecation_test.go │ │ ├── v3_cipher_suite_test.go │ │ ├── v3_curl_auth_test.go │ │ ├── v3_curl_cluster_test.go │ │ ├── v3_curl_election_test.go │ │ ├── v3_curl_kv_test.go │ │ ├── v3_curl_lease_test.go │ │ ├── v3_curl_lock_test.go │ │ ├── v3_curl_maintenance_test.go │ │ ├── v3_curl_maxstream_test.go │ │ ├── v3_curl_watch_test.go │ │ ├── v3_lease_no_proxy_test.go │ │ ├── watch_test.go │ │ └── zap_logging_test.go │ ├── fixtures/ │ │ ├── CommonName-root.crt │ │ ├── CommonName-root.key │ │ ├── ca-csr.json │ │ ├── ca.crt │ │ ├── client-ca-csr-nocn.json │ │ ├── client-clientusage.crt │ │ ├── client-clientusage.key.insecure │ │ ├── client-nocn.crt │ │ ├── client-nocn.key.insecure │ │ ├── ed25519-private-key.pem │ │ ├── ed25519-public-key.pem │ │ ├── gencert.json │ │ ├── gencerts.sh │ │ ├── revoke.crl │ │ ├── server-ca-csr-ecdsa.json │ │ ├── server-ca-csr-ip.json │ │ ├── server-ca-csr-ipv6.json │ │ ├── server-ca-csr-wildcard.json │ │ ├── server-ca-csr.json │ │ ├── server-ca-csr2.json │ │ ├── server-ca-csr3.json │ │ ├── server-ecdsa.crt │ │ ├── server-ecdsa.key.insecure │ │ ├── server-ip.crt │ │ ├── server-ip.key.insecure │ │ ├── server-ipv6.crt │ │ ├── server-ipv6.key.insecure │ │ ├── server-revoked.crt │ │ ├── server-revoked.key.insecure │ │ ├── server-serverusage.crt │ │ ├── server-serverusage.key.insecure │ │ ├── server-wildcard.crt │ │ ├── server-wildcard.key.insecure │ │ ├── server.crt │ │ ├── server.key.insecure │ │ ├── server2.crt │ │ ├── server2.key.insecure │ │ ├── server3.crt │ │ └── server3.key.insecure │ ├── framework/ │ │ ├── config/ │ │ │ ├── client.go │ │ │ └── cluster.go │ │ ├── e2e/ │ │ │ ├── cluster.go │ │ │ ├── cluster_direct.go │ │ │ ├── cluster_proxy.go │ │ │ ├── cluster_test.go │ │ │ ├── config.go │ │ │ ├── curl.go │ │ │ ├── downgrade.go │ │ │ ├── e2e.go │ │ │ ├── etcd_process.go │ │ │ ├── etcd_spawn.go │ │ │ ├── etcdctl.go │ │ │ ├── etcdctl_test.go │ │ │ ├── flags.go │ │ │ ├── lazyfs.go │ │ │ ├── metrics.go │ │ │ ├── testing.go │ │ │ └── util.go │ │ ├── integration/ │ │ │ ├── bridge.go │ │ │ ├── cluster.go │ │ │ ├── cluster_direct.go │ │ │ ├── cluster_proxy.go │ │ │ ├── config.go │ │ │ ├── integration.go │ │ │ └── testing.go │ │ ├── interfaces/ │ │ │ └── interface.go │ │ ├── testrunner.go │ │ ├── testutils/ │ │ │ ├── execute.go │ │ │ ├── helpters.go │ │ │ ├── log_observer.go │ │ │ ├── log_observer_test.go │ │ │ └── path.go │ │ └── unit/ │ │ └── unit.go │ ├── go.mod │ ├── go.sum │ ├── integration/ │ │ ├── cache_test.go │ │ ├── clientv3/ │ │ │ ├── cluster_test.go │ │ │ ├── concurrency/ │ │ │ │ ├── election_test.go │ │ │ │ ├── example_election_test.go │ │ │ │ ├── example_mutex_test.go │ │ │ │ ├── example_stm_test.go │ │ │ │ ├── main_test.go │ │ │ │ ├── mutex_test.go │ │ │ │ └── session_test.go │ │ │ ├── connectivity/ │ │ │ │ ├── black_hole_test.go │ │ │ │ ├── dial_test.go │ │ │ │ ├── doc.go │ │ │ │ ├── main_test.go │ │ │ │ ├── network_partition_test.go │ │ │ │ └── server_shutdown_test.go │ │ │ ├── doc.go │ │ │ ├── examples/ │ │ │ │ ├── example_auth_test.go │ │ │ │ ├── example_cluster_test.go │ │ │ │ ├── example_kv_test.go │ │ │ │ ├── example_lease_test.go │ │ │ │ ├── example_maintenance_test.go │ │ │ │ ├── example_metrics_test.go │ │ │ │ ├── example_test.go │ │ │ │ ├── example_watch_test.go │ │ │ │ └── main_test.go │ │ │ ├── experimental/ │ │ │ │ └── recipes/ │ │ │ │ ├── v3_barrier_test.go │ │ │ │ ├── v3_double_barrier_test.go │ │ │ │ ├── v3_lock_test.go │ │ │ │ └── v3_queue_test.go │ │ │ ├── kv_test.go │ │ │ ├── lease/ │ │ │ │ ├── doc.go │ │ │ │ ├── lease_test.go │ │ │ │ ├── leasing_test.go │ │ │ │ └── main_test.go │ │ │ ├── main_test.go │ │ │ ├── maintenance_test.go │ │ │ ├── metrics_test.go │ │ │ ├── mirror_auth_test.go │ │ │ ├── mirror_test.go │ │ │ ├── namespace_test.go │ │ │ ├── naming/ │ │ │ │ ├── endpoints_test.go │ │ │ │ ├── main_test.go │ │ │ │ └── resolver_test.go │ │ │ ├── ordering_kv_test.go │ │ │ ├── ordering_util_test.go │ │ │ ├── snapshot/ │ │ │ │ └── v3_snapshot_test.go │ │ │ ├── txn_test.go │ │ │ ├── user_test.go │ │ │ ├── util.go │ │ │ └── watch/ │ │ │ ├── v3_watch_restore_test.go │ │ │ ├── v3_watch_test.go │ │ │ ├── watch_fragment_test.go │ │ │ └── watch_test.go │ │ ├── cluster_test.go │ │ ├── corrupt_test.go │ │ ├── doc.go │ │ ├── embed/ │ │ │ ├── embed_proxy_test.go │ │ │ └── embed_test.go │ │ ├── fixtures-expired/ │ │ │ ├── README │ │ │ ├── ca-csr.json │ │ │ ├── ca.crt │ │ │ ├── gencert.json │ │ │ ├── gencerts.sh │ │ │ ├── server-ca-csr-ip.json │ │ │ ├── server-ca-csr.json │ │ │ ├── server-ip.crt │ │ │ ├── server-ip.key.insecure │ │ │ ├── server.crt │ │ │ └── server.key.insecure │ │ ├── lazy_cluster.go │ │ ├── main_test.go │ │ ├── member_test.go │ │ ├── metrics_test.go │ │ ├── network_partition_test.go │ │ ├── proxy/ │ │ │ └── grpcproxy/ │ │ │ ├── cluster_test.go │ │ │ ├── kv_test.go │ │ │ └── register_test.go │ │ ├── revision_test.go │ │ ├── snapshot/ │ │ │ ├── member_test.go │ │ │ └── v3_snapshot_test.go │ │ ├── testing_test.go │ │ ├── tracing_test.go │ │ ├── util_test.go │ │ ├── utl_wal_version_test.go │ │ ├── v2store/ │ │ │ ├── main_test.go │ │ │ ├── store_tag_test.go │ │ │ └── store_test.go │ │ ├── v3_alarm_test.go │ │ ├── v3_auth_test.go │ │ ├── v3_election_test.go │ │ ├── v3_failover_test.go │ │ ├── v3_grpc_inflight_test.go │ │ ├── v3_grpc_test.go │ │ ├── v3_kv_test.go │ │ ├── v3_leadership_test.go │ │ ├── v3_lease_test.go │ │ ├── v3_stm_test.go │ │ ├── v3_tls_test.go │ │ ├── v3election_grpc_test.go │ │ └── v3lock_grpc_test.go │ ├── robustness/ │ │ ├── Makefile │ │ ├── OWNERS │ │ ├── README.md │ │ ├── client/ │ │ │ ├── client.go │ │ │ ├── kvhash.go │ │ │ └── watch.go │ │ ├── coverage/ │ │ │ ├── README.md │ │ │ ├── apiserver-shared-conf/ │ │ │ │ └── tracing.yaml │ │ │ ├── collect_kind_traces.sh │ │ │ ├── contract_test.go │ │ │ ├── coverage_test.go │ │ │ ├── key_pattern_test.go │ │ │ ├── kind-with-tracing.yaml │ │ │ ├── matchers_test.go │ │ │ ├── patches/ │ │ │ │ └── kubernetes/ │ │ │ │ ├── 0001-Add-Open-Telemetry-trace-event-when-passing-through-.patch │ │ │ │ ├── 0002-Add-kubernetesEtcdContractTracker.patch │ │ │ │ ├── 0003-Add-1m-timeout-to-Watch-to-ensure-Spans-are-exported.patch │ │ │ │ └── 0004-Configure-cmd-tests.patch │ │ │ ├── sort_test.go │ │ │ └── testdata/ │ │ │ └── .gitignore │ │ ├── failpoint/ │ │ │ ├── cluster.go │ │ │ ├── failpoint.go │ │ │ ├── gofail.go │ │ │ ├── kill.go │ │ │ ├── network.go │ │ │ └── trigger.go │ │ ├── identity/ │ │ │ ├── id.go │ │ │ └── lease_ids.go │ │ ├── main_test.go │ │ ├── model/ │ │ │ ├── describe.go │ │ │ ├── describe_test.go │ │ │ ├── deterministic.go │ │ │ ├── deterministic_test.go │ │ │ ├── history.go │ │ │ ├── history_test.go │ │ │ ├── non_deterministic.go │ │ │ ├── non_deterministic_test.go │ │ │ ├── replay.go │ │ │ ├── types.go │ │ │ ├── types_test.go │ │ │ └── watch.go │ │ ├── options/ │ │ │ ├── cluster_options.go │ │ │ ├── cluster_options_test.go │ │ │ └── server_config_options.go │ │ ├── patches/ │ │ │ ├── beforeSendWatchResponse/ │ │ │ │ ├── build.patch │ │ │ │ └── watch.patch │ │ │ └── compactBeforeSetFinishedCompact/ │ │ │ └── kvstore_compaction.patch │ │ ├── random/ │ │ │ └── random.go │ │ ├── report/ │ │ │ ├── client.go │ │ │ ├── client_test.go │ │ │ ├── failpoint.go │ │ │ ├── report.go │ │ │ ├── wal.go │ │ │ └── wal_test.go │ │ ├── scenarios/ │ │ │ └── scenarios.go │ │ ├── testdata/ │ │ │ └── .gitignore │ │ ├── traffic/ │ │ │ ├── etcd.go │ │ │ ├── key_store.go │ │ │ ├── kubernetes.go │ │ │ ├── limiter.go │ │ │ ├── limiter_test.go │ │ │ └── traffic.go │ │ └── validate/ │ │ ├── operations.go │ │ ├── operations_test.go │ │ ├── patch_history.go │ │ ├── patch_history_test.go │ │ ├── result.go │ │ ├── validate.go │ │ ├── validate_test.go │ │ └── watch.go │ └── semaphore.test.bash └── tools/ ├── .golangci.yaml ├── .markdownlint.jsonc ├── .yamlfmt ├── .yamllint ├── OWNERS ├── benchmark/ │ ├── OWNERS │ ├── README.md │ ├── cmd/ │ │ ├── doc.go │ │ ├── lease.go │ │ ├── mvcc-put.go │ │ ├── mvcc.go │ │ ├── put.go │ │ ├── range.go │ │ ├── root.go │ │ ├── stm.go │ │ ├── txn_mixed.go │ │ ├── txn_put.go │ │ ├── util.go │ │ ├── watch.go │ │ ├── watch_get.go │ │ └── watch_latency.go │ ├── doc.go │ └── main.go ├── check-grpc-experimental/ │ ├── allowlist.txt │ ├── doc.go │ └── main.go ├── etcd-dump-db/ │ ├── OWNERS │ ├── README.md │ ├── backend.go │ ├── doc.go │ ├── main.go │ ├── meta.go │ ├── page.go │ ├── scan.go │ └── utils.go ├── etcd-dump-logs/ │ ├── OWNERS │ ├── README.md │ ├── doc.go │ ├── etcd-dump-log_test.go │ ├── expectedoutput/ │ │ ├── decoder_correctoutputformat.output │ │ ├── decoder_wrongoutputformat.output │ │ ├── listAll.output │ │ ├── listConfigChange.output │ │ ├── listConfigChangeIRRCompaction.output │ │ ├── listIRRCompaction.output │ │ ├── listIRRDeleteRange.output │ │ ├── listIRRLeaseGrant.output │ │ ├── listIRRLeaseRevoke.output │ │ ├── listIRRPut.output │ │ ├── listIRRRange.output │ │ ├── listIRRTxn.output │ │ ├── listInternalRaftRequest.output │ │ ├── listNormal.output │ │ └── listRequest.output │ ├── main.go │ ├── raw.go │ ├── raw_test.go │ └── testdecoder/ │ ├── decoder_correctoutputformat.sh │ └── decoder_wrongoutputformat.sh ├── etcd-dump-metrics/ │ ├── OWNERS │ ├── README.md │ ├── etcd.go │ ├── install_darwin.go │ ├── install_linux.go │ ├── install_windows.go │ ├── main.go │ ├── metrics.go │ └── utils.go ├── local-tester/ │ ├── OWNERS │ ├── Procfile │ ├── README.md │ ├── bridge/ │ │ ├── bridge.go │ │ └── dispatch.go │ ├── bridge.sh │ └── faults.sh ├── mod/ │ ├── doc.go │ ├── go.mod │ ├── go.sum │ ├── install_all.sh │ ├── libs.go │ └── tools.go ├── proto-annotations/ │ ├── cmd/ │ │ ├── etcd_version.go │ │ └── root.go │ └── main.go └── testgrid-analysis/ ├── OWNERS ├── cmd/ │ ├── data.go │ ├── flaky.go │ ├── github.go │ └── root.go ├── go.mod ├── go.sum └── main.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .devcontainer/devcontainer.json ================================================ // For format details, see https://aka.ms/devcontainer.json. For config options, see the // README at: https://github.com/devcontainers/templates/tree/main/src/go { "name": "Go", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile "image": "mcr.microsoft.com/devcontainers/go:1.25-bookworm", // Features to add to the dev container. More info: https://containers.dev/features. "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": {}, "ghcr.io/devcontainers/features/github-cli:1": {}, "ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {} }, // Use 'forwardPorts' to make a list of ports inside the container available locally. "forwardPorts": [ 2379, 2380 ], // Use 'postCreateCommand' to run commands after the container is created. "postCreateCommand": "make build" // Configure tool-specific properties. // "customizations": {}, } ================================================ FILE: .github/ISSUE_TEMPLATE/bug-report.yml ================================================ --- name: Bug Report description: Report a bug encountered while operating etcd labels: - type/bug body: - type: checkboxes id: confirmations attributes: label: Bug report criteria description: Please confirm this bug report meets the following criteria. options: - label: This bug report is not security related, security issues should be disclosed privately via security@etcd.io. - label: This is not a support request or question, support requests or questions should be raised in the etcd [discussion forums](https://github.com/etcd-io/etcd/discussions). - label: You have read the etcd [bug reporting guidelines](https://github.com/etcd-io/etcd/blob/main/Documentation/contributor-guide/reporting_bugs.md). - label: Existing open issues along with etcd [frequently asked questions](https://etcd.io/docs/latest/faq) have been checked and this is not a duplicate. - type: markdown attributes: value: | Please fill the form below and provide as much information as possible. Not doing so may result in your bug not being addressed in a timely manner. - type: textarea id: problem attributes: label: What happened? validations: required: true - type: textarea id: expected attributes: label: What did you expect to happen? validations: required: true - type: textarea id: repro attributes: label: How can we reproduce it (as minimally and precisely as possible)? validations: required: true - type: textarea id: additional attributes: label: Anything else we need to know? - type: textarea id: etcdVersion attributes: label: Etcd version (please run commands below) value: |
```console $ etcd --version # paste output here $ etcdctl version # paste output here ```
validations: required: true - type: textarea id: config attributes: label: Etcd configuration (command line flags or environment variables) value: |
# paste your configuration here
- type: textarea id: etcdDebugInformation attributes: label: Etcd debug information (please run commands below, feel free to obfuscate the IP address or FQDN in the output) value: |
```console $ etcdctl member list -w table # paste output here $ etcdctl --endpoints= endpoint status -w table # paste output here ```
- type: textarea id: logs attributes: label: Relevant log output description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. render: Shell ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ --- blank_issues_enabled: false contact_links: - name: Question url: https://github.com/etcd-io/etcd/discussions about: Question relating to Etcd ================================================ FILE: .github/ISSUE_TEMPLATE/feature-request.yml ================================================ --- name: Feature request description: Provide idea for a new feature labels: - type/feature body: - type: textarea id: feature attributes: label: What would you like to be added? validations: required: true - type: textarea id: rationale attributes: label: Why is this needed? validations: required: true ================================================ FILE: .github/ISSUE_TEMPLATE/test-flake.yml ================================================ --- name: Flaking Test description: Report flaky tests labels: - type/flake - area/testing body: - type: textarea id: workflows attributes: label: Which Github Action / Prow Jobs are flaking? validations: required: true - type: textarea id: tests attributes: label: Which tests are flaking? validations: required: true - type: input id: link attributes: label: Github Action / Prow Job link - type: textarea id: reason attributes: label: Reason for failure (if possible) - type: textarea id: additional attributes: label: Anything else we need to know? ================================================ FILE: .github/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners approvers: - ivanvc # Ivan Valdes ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ ================================================ FILE: .github/SECURITY.md ================================================ Please read https://github.com/etcd-io/etcd/blob/main/security/README.md. ================================================ FILE: .github/dependabot.yml ================================================ --- version: 2 updates: - package-ecosystem: github-actions directory: / schedule: interval: weekly - package-ecosystem: gomod directory: / schedule: interval: weekly allow: - dependency-type: all - package-ecosystem: gomod directory: /tools/mod # Not linked from /go.mod schedule: interval: weekly allow: - dependency-type: direct - package-ecosystem: docker directory: / schedule: interval: weekly - package-ecosystem: docker directory: / target-branch: "release-3.4" schedule: interval: monthly - package-ecosystem: docker directory: / target-branch: "release-3.5" schedule: interval: monthly - package-ecosystem: docker directory: / target-branch: "release-3.6" schedule: interval: monthly ================================================ FILE: .github/workflows/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - github_actions ================================================ FILE: .github/workflows/antithesis-test.yml ================================================ --- name: Build and trigger Antithesis exploration on: # pull_request: # branches: [main] schedule: - cron: "0 0 * * *" # run every day at midnight workflow_dispatch: inputs: test: description: 'Test name' required: false type: string duration: description: 'Duration (exploration hours)' required: true type: int description: description: 'Description (avoid quotes, please!)' required: true type: string etcd_ref: description: 'etcd version to build etcd-server from' required: false type: string email: description: 'Additional email notification recipient (separate with ;)' required: true type: string # Declare default permissions as read only. permissions: read-all env: REGISTRY: us-central1-docker.pkg.dev REPOSITORY: molten-verve-216720/linuxfoundation-repository jobs: build-and-push-and-test: runs-on: ubuntu-latest environment: Antithesis steps: - name: Checkout the code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Login to Antithesis Docker Registry uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ${{ env.REGISTRY }} username: _json_key password: ${{ secrets.ANTITHESIS_CONTAINER_REGISTRY_TOKEN }} - name: Build and push config image working-directory: ./tests/antithesis run: | make antithesis-build-config-image IMAGE_TAG=${{ inputs.etcd_ref || 'main' }}_${{ github.sha }} export IMAGE="${{ env.REGISTRY }}/${{ env.REPOSITORY }}/etcd-config:${{ inputs.etcd_ref || 'main' }}_${{ github.sha }}" docker tag etcd-config:latest $IMAGE docker push $IMAGE - name: Build and push client image working-directory: ./tests/antithesis run: | make antithesis-build-client-docker-image export IMAGE="${{ env.REGISTRY }}/${{ env.REPOSITORY }}/etcd-client:${{ inputs.etcd_ref || 'main' }}_${{ github.sha }}" docker tag etcd-client:latest $IMAGE docker push $IMAGE - name: Build and push etcd image working-directory: ./tests/antithesis run: | make antithesis-build-etcd-image REF=${{ inputs.etcd_ref || 'main' }} export IMAGE="${{ env.REGISTRY }}/${{ env.REPOSITORY }}/etcd-server:${{ inputs.etcd_ref || 'main' }}_${{ github.sha }}" docker tag etcd-server:latest $IMAGE docker push $IMAGE - name: Run Antithesis Tests uses: antithesishq/antithesis-trigger-action@f6221e2ba819fe0ac3e36bd67a281fa439a03fba # v0.10 with: notebook_name: etcd tenant: linuxfoundation username: ${{ secrets.ANTITHESIS_WEBHOOK_USERNAME }} password: ${{ secrets.ANTITHESIS_WEBHOOK_PASSWORD }} github_token: ${{ secrets.GH_PAT }} config_image: us-central1-docker.pkg.dev/molten-verve-216720/linuxfoundation-repository/etcd-config:${{ inputs.etcd_ref || 'main' }}_${{ github.sha }} images: us-central1-docker.pkg.dev/molten-verve-216720/linuxfoundation-repository/etcd-client:${{ inputs.etcd_ref || 'main' }}_${{ github.sha }};us-central1-docker.pkg.dev/molten-verve-216720/linuxfoundation-repository/etcd-server:${{ inputs.etcd_ref || 'main' }}_${{ github.sha }};docker.io/library/ubuntu:latest;us-central1-docker.pkg.dev/molten-verve-216720/linuxfoundation-repository/etcd-config:${{ inputs.etcd_ref || 'main' }}_${{ github.sha }} description: ${{ inputs.description || 'etcd nightly antithesis run' }} email_recipients: ${{ inputs.email || 'siarkowicz@google.com' }} test_name: ${{ inputs.test || 'etcd nightly antithesis run' }} additional_parameters: |- custom.duration = ${{ inputs.duration || 12 }} antithesis.source = ${{ inputs.etcd_ref || 'main' }} ================================================ FILE: .github/workflows/antithesis-verify.yml ================================================ --- name: Verify Antithesis Docker Compose Pipeline on: push: branches: - main paths: - 'tests/antithesis/**' - '.github/workflows/antithesis-verify.yml' pull_request: paths: - 'tests/antithesis/**' - '.github/workflows/antithesis-verify.yml' jobs: test-docker-compose: strategy: matrix: node-count: [1, 3] name: Test ${{ matrix.node-count }}-node cluster runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Build etcd-server and etcd-client images run: | make -C tests/antithesis antithesis-build-etcd-image make -C tests/antithesis antithesis-build-client-docker-image - name: Run docker-compose up working-directory: ./tests/antithesis run: | make antithesis-docker-compose-up CFG_NODE_COUNT=${{ matrix.node-count }} & - name: Check for healthy cluster working-directory: ./tests/antithesis run: | timeout=120 interval=10 end_time=$(( $(date +%s) + timeout )) while [ $(date +%s) -lt $end_time ]; do # The client container might not be running yet, so ignore errors from docker compose logs if docker compose -f config/docker-compose-${{ matrix.node-count }}-node.yml logs client 2>/dev/null | grep -q "Client \[entrypoint\]: cluster is healthy!"; then echo "Cluster is healthy!" exit 0 fi echo "Waiting for cluster to become healthy..." sleep $interval done echo "Cluster did not become healthy in ${timeout} seconds." docker compose -f config/docker-compose-${{ matrix.node-count }}-node.yml logs exit 1 - name: Run traffic working-directory: ./tests/antithesis run: make antithesis-run-container-traffic CFG_NODE_COUNT=${{ matrix.node-count }} - name: Run validation working-directory: ./tests/antithesis run: make antithesis-run-container-validation CFG_NODE_COUNT=${{ matrix.node-count }} - name: Clean up if: always() working-directory: ./tests/antithesis run: make antithesis-clean CFG_NODE_COUNT=${{ matrix.node-count }} ================================================ FILE: .github/workflows/antithesis.debugger.yml ================================================ --- name: Trigger Antithesis debugger on: workflow_dispatch: inputs: session_id: description: "The session_id of a test. Found at the bottom of a report." required: true type: string input_hash: description: "The input hash of a moment." required: true type: string vtime: description: "The vtime of a moment." required: true type: string email: description: 'Email notification recipient(s) (separate with ;)' required: false type: string # Declare default permissions as read only. permissions: read-all jobs: trigger-debugger: runs-on: ubuntu-latest environment: Antithesis steps: - name: Trigger Antithesis Debugger uses: antithesishq/antithesis-trigger-action@f6221e2ba819fe0ac3e36bd67a281fa439a03fba # v0.10 with: notebook_name: debugging tenant: linuxfoundation username: ${{ secrets.ANTITHESIS_WEBHOOK_USERNAME }} password: ${{ secrets.ANTITHESIS_WEBHOOK_PASSWORD }} github_token: ${{ secrets.GH_PAT }} email_recipients: ${{ inputs.email || 'siarkowicz@google.com' }} additional_parameters: |- antithesis.debugging.session_id = ${{ inputs.session_id }} antithesis.debugging.input_hash = ${{ inputs.input_hash }} antithesis.debugging.vtime = ${{ inputs.vtime }} ================================================ FILE: .github/workflows/cherrypick-bot-ok-to-test.yaml ================================================ --- name: Auto-label ok-to-test (cherrypick bot) permissions: read-all on: pull_request_target: types: - labeled branches: - release-3.6 - release-3.5 - release-3.4 jobs: add-label: name: Add label # 90416843 = k8s-infra-cherrypick-robot account ID. if: | github.event.pull_request.user.id == 90416843 && github.event.label.name == 'needs-ok-to-test' runs-on: ubuntu-latest permissions: pull-requests: write steps: - name: Update PR uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} debug: ${{ secrets.ACTIONS_RUNNER_DEBUG == 'true' }} script: | try { await github.rest.issues.removeLabel({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, name: 'needs-ok-to-test' }); } catch (e) { if (e.status !== 404) throw e; } await github.rest.issues.addLabels({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, labels: ['ok-to-test'] }); ================================================ FILE: .github/workflows/codeql-analysis.yml ================================================ --- # For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: [main, release-3.4, release-3.5, release-3.6] pull_request: # The branches below must be a subset of the branches above branches: [main] schedule: - cron: '20 14 * * 5' permissions: read-all jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] # Learn more: # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed language: ['go'] steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6 with: # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main languages: ${{ matrix.language }} # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6 ================================================ FILE: .github/workflows/gh-workflow-approve.yaml ================================================ --- name: Approve GitHub Workflows permissions: read-all on: pull_request_target: types: - labeled - synchronize branches: - main - release-3.6 - release-3.5 - release-3.4 jobs: approve: name: Approve ok-to-test if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') runs-on: ubuntu-latest permissions: actions: write steps: - name: Update PR uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 continue-on-error: true with: github-token: ${{ secrets.GITHUB_TOKEN }} debug: ${{ secrets.ACTIONS_RUNNER_DEBUG == 'true' }} script: | const result = await github.rest.actions.listWorkflowRunsForRepo({ owner: context.repo.owner, repo: context.repo.repo, event: "pull_request", status: "action_required", head_sha: context.payload.pull_request.head.sha, per_page: 100 }); for (var run of result.data.workflow_runs) { await github.rest.actions.approveWorkflowRun({ owner: context.repo.owner, repo: context.repo.repo, run_id: run.id }); } ================================================ FILE: .github/workflows/measure-testgrid-flakiness.yaml ================================================ --- name: Measure TestGrid Flakiness on: schedule: - cron: "0 0 * * *" # run every day at midnight permissions: read-all jobs: measure-testgrid-flakiness: name: Measure TestGrid Flakiness runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - id: goversion run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version: ${{ steps.goversion.outputs.goversion }} - env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail ./scripts/measure-testgrid-flakiness.sh ================================================ FILE: .github/workflows/scorecards.yml ================================================ --- name: Scorecards supply-chain security on: # Only the default branch is supported. branch_protection_rule: schedule: - cron: '45 1 * * 0' push: branches: ["main"] # Declare default permissions as read only. permissions: read-all jobs: analysis: name: Scorecards analysis runs-on: ubuntu-latest permissions: # Needed to upload the results to code-scanning dashboard. security-events: write # Used to receive a badge. id-token: write steps: - name: "Checkout code" uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: "Run analysis" uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 with: results_file: results.sarif results_format: sarif # Publish the results for public repositories to enable scorecard badges. For more details, see # https://github.com/ossf/scorecard-action#publishing-results. # For private repositories, `publish_results` will automatically be set to `false`, regardless # of the value entered here. publish_results: true # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: SARIF file path: results.sarif retention-days: 5 # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" uses: github/codeql-action/upload-sarif@0d579ffd059c29b07949a3cce3983f0780820c98 # v4.32.6 with: sarif_file: results.sarif ================================================ FILE: .github/workflows/stale.yaml ================================================ --- name: Mark and close stale issues and PRs on: schedule: - cron: '0 0 * * *' permissions: issues: write pull-requests: write jobs: stale: runs-on: ubuntu-latest steps: - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f #v10.2.0 with: days-before-stale: 90 days-before-close: 21 stale-issue-label: 'stale' stale-pr-label: 'stale' exempt-issue-labels: 'stage/tracked,help wanted' exempt-pr-labels: 'stage/tracked,help wanted' exempt-milestones: '' exempt-assignees: '' stale-issue-message: 'This issue has been automatically marked as stale because it has not had recent activity. It will be closed after 21 days if no further activity occurs. Thank you for your contributions.' stale-pr-message: 'This pull request has been automatically marked as stale because it has not had recent activity. It will be closed after 21 days if no further activity occurs. Thank you for your contributions.' close-issue-message: '' close-pr-message: '' operations-per-run: 30 ================================================ FILE: .github/workflows/verify-released-assets.yaml ================================================ --- name: Verify released binary assets permissions: read-all on: release: types: [published] jobs: verify-assets: name: Verify released binary assets runs-on: ubuntu-latest steps: - name: Verify binary assets env: GH_TOKEN: ${{ github.token }} RELEASE: ${{ github.event.release.tag_name }} REPOSITORY: ${{ github.repository }} run: | mkdir github-assets pushd github-assets gh --repo "${REPOSITORY}" release download "${RELEASE}" test_assets() { if [ "$(wc -l is unhealthy: failed to connect: \". This change unified the error message, all error types now have the same output "\ is unhealthy: failed to commit proposal: \". ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Fix bug where [db_compaction_total_duration_milliseconds metric incorrectly measured duration as 0](https://github.com/etcd-io/etcd/pull/10646). --- ## [v3.1.20](https://github.com/etcd-io/etcd/releases/tag/v3.1.20) (2018-10-10) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.19...v3.1.20) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### Improved - Improve ["became inactive" warning log](https://github.com/etcd-io/etcd/pull/10024), which indicates message send to a peer failed. - Improve [read index wait timeout warning log](https://github.com/etcd-io/etcd/pull/10026), which indicates that local node might have slow network. - Add [gRPC interceptor for debugging logs](https://github.com/etcd-io/etcd/pull/9990); enable `etcd --debug` flag to see per-request debug information. - Add [consistency check in snapshot status](https://github.com/etcd-io/etcd/pull/10109). If consistency check on snapshot file fails, `snapshot status` returns `"snapshot file integrity check failed..."` error. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Improve [`etcd_network_peer_round_trip_time_seconds`](https://github.com/etcd-io/etcd/pull/10155) Prometheus metric to track leader heartbeats. - Previously, it only samples the TCP connection for snapshot messages. - Display all registered [gRPC metrics at start](https://github.com/etcd-io/etcd/pull/10034). - Add [`etcd_snap_db_fsync_duration_seconds_count`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_snap_db_save_total_duration_seconds_bucket`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_server_id`](https://github.com/etcd-io/etcd/pull/9998) Prometheus metric. - Add [`etcd_server_health_success`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. - Add [`etcd_server_health_failures`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. - Add [`etcd_server_read_indexes_failed_total`](https://github.com/etcd-io/etcd/pull/10094) Prometheus metric. ### client v3 - Fix logic on [release lock key if cancelled](https://github.com/etcd-io/etcd/pull/10153) in `clientv3/concurrency` package. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.1.19](https://github.com/etcd-io/etcd/releases/tag/v3.1.19) (2018-07-24) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.18...v3.1.19) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### Improved - Improve [Raft Read Index timeout warning messages](https://github.com/etcd-io/etcd/pull/9897). ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_server_go_version`](https://github.com/etcd-io/etcd/pull/9957) Prometheus metric. - Add [`etcd_server_slow_read_indexes_total`](https://github.com/etcd-io/etcd/pull/9897) Prometheus metric. - Add [`etcd_server_quota_backend_bytes`](https://github.com/etcd-io/etcd/pull/9820) Prometheus metric. - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. - Add [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819) Prometheus metric. - In addition to [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819). - Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/etcd-io/etcd/pull/9256) Prometheus metric. - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. ### client v3 - Fix [lease keepalive interval updates when response queue is full](https://github.com/etcd-io/etcd/pull/9952). - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/etcd-io/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.1.18](https://github.com/etcd-io/etcd/releases/tag/v3.1.18) (2018-06-15) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.17...v3.1.18) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_server_version`](https://github.com/etcd-io/etcd/pull/8960) Prometheus metric. - To replace [Kubernetes `etcd-version-monitor`](https://github.com/etcd-io/etcd/issues/8948). ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.1.17](https://github.com/etcd-io/etcd/releases/tag/v3.1.17) (2018-06-06) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.16...v3.1.17) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### etcd server - Fix [v3 snapshot recovery](https://github.com/etcd-io/etcd/issues/7628). - A follower receives a leader snapshot to be persisted as a `[SNAPSHOT-INDEX].snap.db` file on disk. - Now, server [ensures that the incoming snapshot be persisted on disk before loading it](https://github.com/etcd-io/etcd/pull/7876). - Otherwise, index mismatch happens and triggers server-side panic (e.g. newer WAL entry with outdated snapshot index). ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.1.16](https://github.com/etcd-io/etcd/releases/tag/v3.1.16) (2018-05-31) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.15...v3.1.16) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### etcd server - Fix [`mvcc` server panic from restore operation](https://github.com/etcd-io/etcd/pull/9775). - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X, **etcd server panicked** during snapshot restore operation. - Now, this server-side panic has been fixed. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.1.15](https://github.com/etcd-io/etcd/releases/tag/v3.1.15) (2018-05-09) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.14...v3.1.15) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### etcd server - Purge old [`*.snap.db` snapshot files](https://github.com/etcd-io/etcd/pull/7967). - Previously, etcd did not respect `--max-snapshots` flag to purge old `*.snap.db` files. - Now, etcd purges old `*.snap.db` files to keep maximum `--max-snapshots` number of files on disk. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.1.14](https://github.com/etcd-io/etcd/releases/tag/v3.1.14) (2018-04-24) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.13...v3.1.14) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_server_is_leader`](https://github.com/etcd-io/etcd/pull/9587) Prometheus metric. ### etcd server - Add [`--initial-election-tick-advance`](https://github.com/etcd-io/etcd/pull/9591) flag to configure initial election tick fast-forward. - By default, `--initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger. - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election. - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout. - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities. - Now, this can be disabled by setting `--initial-election-tick-advance=false`. - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `--initial-election-tick-advance` at the cost of slow initial bootstrap. - If single-node, it advances ticks regardless. - Address [disruptive rejoining follower node](https://github.com/etcd-io/etcd/issues/9333). ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.1.13](https://github.com/etcd-io/etcd/releases/tag/v3.1.13) (2018-03-29) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.12...v3.1.13) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### Improved - Adjust [election timeout on server restart](https://github.com/etcd-io/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/etcd-io/etcd/issues/9333). - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node. - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add missing [`etcd_network_peer_sent_failures_total` count](https://github.com/etcd-io/etcd/pull/9437). ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.1.12](https://github.com/etcd-io/etcd/releases/tag/v3.1.12) (2018-03-08) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.11...v3.1.12) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### etcd server - Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/etcd-io/etcd/pull/9297). - "unsynced" watcher is watcher that needs to be in sync with events that have happened. - That is, "unsynced" watcher is the slow watcher that was requested on old revision. - "unsynced" watcher restore operation was not correctly populating its underlying watcher group. - Which possibly causes [missing events from "unsynced" watchers](https://github.com/etcd-io/etcd/issues/9086). - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.1.11](https://github.com/etcd-io/etcd/releases/tag/v3.1.11) (2017-11-28) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.10...v3.1.11) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### etcd server - [#8411](https://github.com/etcd-io/etcd/issues/8411),[#8806](https://github.com/etcd-io/etcd/pull/8806) backport "mvcc: sending events after restore" - [#8009](https://github.com/etcd-io/etcd/issues/8009),[#8902](https://github.com/etcd-io/etcd/pull/8902) backport coreos/bbolt v1.3.1-coreos.5 ### Go - Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.1.10](https://github.com/etcd-io/etcd/releases/tag/v3.1.10) (2017-07-14) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.9...v3.1.10) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### Added - Tag docker images with minor versions. - e.g. `docker pull quay.io/coreos/etcd:v3.1` to fetch latest v3.1 versions. ### Go - Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). - Fix panic on `net/http.CloseNotify` --- ## [v3.1.9](https://github.com/etcd-io/etcd/releases/tag/v3.1.9) (2017-06-09) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.8...v3.1.9) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### etcd server - Allow v2 snapshot over 512MB. ### Go - Compile with [*Go 1.7.6*](https://golang.org/doc/devel/release.html#go1.7). --- ## [v3.1.8](https://github.com/etcd-io/etcd/releases/tag/v3.1.8) (2017-05-19) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.7...v3.1.8) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### Go - Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). --- ## [v3.1.7](https://github.com/etcd-io/etcd/releases/tag/v3.1.7) (2017-04-28) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.6...v3.1.7) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### Go - Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). --- ## [v3.1.6](https://github.com/etcd-io/etcd/releases/tag/v3.1.6) (2017-04-19) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.5...v3.1.6) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### etcd server - Fill in Auth API response header. - Remove auth check in Status API. ### Go - Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). --- ## [v3.1.5](https://github.com/etcd-io/etcd/releases/tag/v3.1.5) (2017-03-27) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.4...v3.1.5) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### etcd server - Fix raft memory leak issue. - Fix Windows file path issues. ### Other - Add `/etc/nsswitch.conf` file to alpine-based Docker image. ### Go - Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). --- ## [v3.1.4](https://github.com/etcd-io/etcd/releases/tag/v3.1.4) (2017-03-22) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.3...v3.1.4) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### Go - Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). --- ## [v3.1.3](https://github.com/etcd-io/etcd/releases/tag/v3.1.3) (2017-03-10) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.2...v3.1.3) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### etcd gateway - Fix `etcd gateway` schema handling in DNS discovery. - Fix sd_notify behaviors in `gateway`, `grpc-proxy`. ### gRPC Proxy - Fix sd_notify behaviors in `gateway`, `grpc-proxy`. ### Other - Use machine default host when advertise URLs are default values(`localhost:2379,2380`) AND if listen URL is `0.0.0.0`. ### Go - Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). --- ## [v3.1.2](https://github.com/etcd-io/etcd/releases/tag/v3.1.2) (2017-02-24) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.1...v3.1.2) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### etcd gateway - Fix `etcd gateway` with multiple endpoints. ### Other - Use IPv4 default host, by default (when IPv4 and IPv6 are available). ### Go - Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). --- ## [v3.1.1](https://github.com/etcd-io/etcd/releases/tag/v3.1.1) (2017-02-17) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.0...v3.1.1) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### Go - Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7). --- ## [v3.1.0](https://github.com/etcd-io/etcd/releases/tag/v3.1.0) (2017-01-20) See [code changes](https://github.com/etcd-io/etcd/compare/v3.0.0...v3.1.0) and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_1/).** ### Improved - Faster linearizable reads (implements Raft [read-index](https://github.com/etcd-io/etcd/pull/6212)). - v3 authentication API is now stable. ### Breaking Changes - Deprecated following gRPC metrics in favor of [go-grpc-prometheus](https://github.com/grpc-ecosystem/go-grpc-prometheus). - `etcd_grpc_requests_total` - `etcd_grpc_requests_failed_total` - `etcd_grpc_active_streams` - `etcd_grpc_unary_requests_duration_seconds` ### Dependency - Upgrade [`github.com/ugorji/go/codec`](https://github.com/ugorji/go) to [**`ugorji/go@9c7f9b7`**](https://github.com/ugorji/go/commit/9c7f9b7a2bc3a520f7c7b30b34b7f85f47fe27b6), and [regenerate v2 `client`](https://github.com/etcd-io/etcd/pull/6945). ### Security, Authentication See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - SRV records (e.g., infra1.example.com) must match the discovery domain (i.e., example.com) if no custom certificate authority is given. - `TLSConfig.ServerName` is ignored with user-provided certificates for backwards compatibility; to be deprecated. - For example, `etcd --discovery-srv=example.com` will only authenticate peers/clients when the provided certs have root domain `example.com` as an entry in Subject Alternative Name (SAN) field. ### etcd server - Automatic leadership transfer when leader steps down. - etcd flags - `--strict-reconfig-check` flag is set by default. - Add `--log-output` flag. - Add `--metrics` flag. - etcd uses default route IP if advertise URL is not given. - Cluster rejects removing members if quorum will be lost. - Discovery now has upper limit for waiting on retries. - Warn on binding listeners through domain names; to be deprecated. - v3.0 and v3.1 with `--auto-compaction-retention=10` run periodic compaction on v3 key-value store for every 10-hour. - Compactor only supports periodic compaction. - Compactor records latest revisions every 5-minute, until it reaches the first compaction period (e.g. 10-hour). - In order to retain key-value history of last compaction period, it uses the last revision that was fetched before compaction period, from the revision records that were collected every 5-minute. - When `--auto-compaction-retention=10`, compactor uses revision 100 for compact revision where revision 100 is the latest revision fetched from 10 hours ago. - If compaction succeeds or requested revision has already been compacted, it resets period timer and starts over with new historical revision records (e.g. restart revision collect and compact for the next 10-hour period). - If compaction fails, it retries in 5 minutes. ### client v3 - Add `SetEndpoints` method; update endpoints at runtime. - Add `Sync` method; auto-update endpoints at runtime. - Add `Lease TimeToLive` API; fetch lease information. - replace Config.Logger field with global logger. - Get API responses are sorted in ascending order by default. ### etcdctl v3 - Add `lease timetolive` command. - Add `--print-value-only` flag to get command. - Add `--dest-prefix` flag to make-mirror command. - `get` command responses are sorted in ascending order by default. ### gRPC Proxy - Experimental gRPC proxy feature. ### Other - `recipes` now conform to sessions defined in `clientv3/concurrency`. - ACI has symlinks to `/usr/local/bin/etcd*`. ### Go - Compile with [*Go 1.7.4*](https://golang.org/doc/devel/release.html#go1.7). --- ================================================ FILE: CHANGELOG/CHANGELOG-3.2.md ================================================ Previous change logs can be found at [CHANGELOG-3.1](https://github.com/etcd-io/etcd/blob/main/CHANGELOG/CHANGELOG-3.1.md). ## v3.2.33 (TBD) --- ## [v3.2.32](https://github.com/etcd-io/etcd/releases/tag/v3.2.32) (2021-03-28) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.31...v3.2.32) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. ### Package `wal` - add wal slice bound check to make sure entry index is not greater than the number of entries - check slice size in decodeRecord - fix panic when decoder not set ### Package `fileutil` - fix constant for linux locking ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.2.31](https://github.com/etcd-io/etcd/releases/tag/v3.2.31) (2020-08-18) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.30...v3.2.31) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. ### auth, etcdserver - Improve [`runtime.FDUsage` call pattern to reduce objects malloc of Memory Usage and CPU Usage](https://github.com/etcd-io/etcd/pull/11986). - [attaching a fake root token when calling `LeaseRevoke`](https://github.com/etcd-io/etcd/pull/11691). - fix a data corruption bug caused by lease expiration when authentication is enabled and upgrading cluster from etcd-3.2 to etcd-3.3 ### Package `runtime` - Optimize [`runtime.FDUsage` by removing unnecessary sorting](https://github.com/etcd-io/etcd/pull/12214). ### Metrics, Monitoring - Add [`os_fd_used` and `os_fd_limit` to monitor current OS file descriptors](https://github.com/etcd-io/etcd/pull/12214). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.2.30](https://github.com/etcd-io/etcd/releases/tag/v3.2.30) (2020-04-01) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.29...v3.2.30) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. ### Package `wal` - Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). ### Metrics, Monitoring - Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.2.29](https://github.com/etcd-io/etcd/releases/tag/v3.2.29) (2020-03-18) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.28...v3.2.29) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. ### etcd server - [Fix corruption bug in defrag](https://github.com/etcd-io/etcd/pull/11613). - Log [`[CLIENT-PORT]/health` check in server side](https://github.com/etcd-io/etcd/pull/11704). ### client v3 - Fix [`"hasleader"` metadata embedding](https://github.com/etcd-io/etcd/pull/11687). - Previously, `clientv3.WithRequireLeader(ctx)` was overwriting existing context keys. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - Add [`etcd_server_client_requests_total` with `"type"` and `"client_api_version"` labels](https://github.com/etcd-io/etcd/pull/11687). ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.28](https://github.com/etcd-io/etcd/releases/tag/v3.2.28) (2019-11-10) ### Improved - Add `etcd --experimental-peer-skip-client-san-verification` to [skip verification of peer client address](https://github.com/etcd-io/etcd/pull/11195). ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_cluster_version`](https://github.com/etcd-io/etcd/pull/11271) Prometheus metric. ### etcdserver - Fix [`wait purge file loop during shutdown`](https://github.com/etcd-io/etcd/pull/11308). - Previously, during shutdown etcd could accidentally remove needed wal files, resulting in catastrophic error `etcdserver: open wal error: wal: file not found.` during startup. - Now, etcd makes sure the purge file loop exits before server signals stop of the raft node. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.27](https://github.com/etcd-io/etcd/releases/tag/v3.2.27) (2019-09-17) ### etcdctl v3 - [Strip out insecure endpoints from DNS SRV records when using discovery](https://github.com/etcd-io/etcd/pull/10443) with etcdctl v2 - Add [`etcdctl endpoint health --write-out` support](https://github.com/etcd-io/etcd/pull/9540). - Previously, [`etcdctl endpoint health --write-out json` did not work](https://github.com/etcd-io/etcd/issues/9532). - The command output is changed. Previously, if endpoint is unreachable, the command output is "\ is unhealthy: failed to connect: \". This change unified the error message, all error types now have the same output "\ is unhealthy: failed to commit proposal: \". - Fix [`etcdctl snapshot status` to not modify snapshot file](https://github.com/etcd-io/etcd/pull/11157). - For example, start etcd `v3.3.10` - Write some data - Use etcdctl `v3.3.10` to save snapshot - Somehow, upgrading Kubernetes fails, thus rolling back to previous version etcd `v3.2.24` - Run etcdctl `v3.2.24` `snapshot status` against the snapshot file saved from `v3.3.10` server - Run etcdctl `v3.2.24` `snapshot restore` fails with `"expected sha256 [12..."` ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Fix bug where [db_compaction_total_duration_milliseconds metric incorrectly measured duration as 0](https://github.com/etcd-io/etcd/pull/10646). - Add [`etcd_debugging_mvcc_current_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. - Add [`etcd_debugging_mvcc_compact_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.26](https://github.com/etcd-io/etcd/releases/tag/v3.2.26) (2019-01-11) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.25...v3.2.26) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### gRPC Proxy - Fix [memory leak in cache layer](https://github.com/etcd-io/etcd/pull/10327). ### Security, Authentication - Disable [CommonName authentication for gRPC-gateway](https://github.com/etcd-io/etcd/pull/10366) gRPC-gateway proxy requests to etcd server use the etcd client server TLS certificate. If that certificate contains CommonName we do not want to use that for authentication as it could lead to permission escalation. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.25](https://github.com/etcd-io/etcd/releases/tag/v3.2.25) (2018-10-10) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.24...v3.2.25) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Improved - Improve ["became inactive" warning log](https://github.com/etcd-io/etcd/pull/10024), which indicates message send to a peer failed. - Improve [read index wait timeout warning log](https://github.com/etcd-io/etcd/pull/10026), which indicates that local node might have slow network. - Add [gRPC interceptor for debugging logs](https://github.com/etcd-io/etcd/pull/9990); enable `etcd --debug` flag to see per-request debug information. - Add [consistency check in snapshot status](https://github.com/etcd-io/etcd/pull/10109). If consistency check on snapshot file fails, `snapshot status` returns `"snapshot file integrity check failed..."` error. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Improve [`etcd_network_peer_round_trip_time_seconds`](https://github.com/etcd-io/etcd/pull/10155) Prometheus metric to track leader heartbeats. - Previously, it only samples the TCP connection for snapshot messages. - Display all registered [gRPC metrics at start](https://github.com/etcd-io/etcd/pull/10032). - Add [`etcd_snap_db_fsync_duration_seconds_count`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_snap_db_save_total_duration_seconds_bucket`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_server_id`](https://github.com/etcd-io/etcd/pull/9998) Prometheus metric. - Add [`etcd_server_health_success`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. - Add [`etcd_server_health_failures`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. - Add [`etcd_server_read_indexes_failed_total`](https://github.com/etcd-io/etcd/pull/10094) Prometheus metric. ### client v3 - Fix logic on [release lock key if cancelled](https://github.com/etcd-io/etcd/pull/10153) in `clientv3/concurrency` package. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.24](https://github.com/etcd-io/etcd/releases/tag/v3.2.24) (2018-07-24) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.23...v3.2.24) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Improved - Improve [Raft Read Index timeout warning messages](https://github.com/etcd-io/etcd/pull/9897). ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_server_go_version`](https://github.com/etcd-io/etcd/pull/9957) Prometheus metric. - Add [`etcd_server_heartbeat_send_failures_total`](https://github.com/etcd-io/etcd/pull/9942) Prometheus metric. - Add [`etcd_server_slow_apply_total`](https://github.com/etcd-io/etcd/pull/9942) Prometheus metric. - Add [`etcd_disk_backend_defrag_duration_seconds`](https://github.com/etcd-io/etcd/pull/9942) Prometheus metric. - Add [`etcd_mvcc_hash_duration_seconds`](https://github.com/etcd-io/etcd/pull/9942) Prometheus metric. - Add [`etcd_server_slow_read_indexes_total`](https://github.com/etcd-io/etcd/pull/9897) Prometheus metric. - Add [`etcd_server_quota_backend_bytes`](https://github.com/etcd-io/etcd/pull/9820) Prometheus metric. - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. - Add [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819) Prometheus metric. - In addition to [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819). - Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/etcd-io/etcd/pull/9256) Prometheus metric. - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_server_quota_backend_bytes`. - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. ### gRPC Proxy - Add [flags for specifying TLS for connecting to proxy](https://github.com/etcd-io/etcd/pull/9894): - Add `grpc-proxy start --cert-file`, `grpc-proxy start --key-file` and `grpc-proxy start --trusted-ca-file` flags. - Add [`grpc-proxy start --metrics-addr` flag for specifying a separate metrics listen address](https://github.com/etcd-io/etcd/pull/9894). ### client v3 - Fix [lease keepalive interval updates when response queue is full](https://github.com/etcd-io/etcd/pull/9952). - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/etcd-io/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.23](https://github.com/etcd-io/etcd/releases/tag/v3.2.23) (2018-06-15) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.22...v3.2.23) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Improved - Improve [slow request apply warning log](https://github.com/etcd-io/etcd/pull/9288). - e.g. `read-only range request "key:\"/a\" range_end:\"/b\" " with result "range_response_count:3 size:96" took too long (97.966µs) to execute`. - Redact [request value field](https://github.com/etcd-io/etcd/pull/9822). - Provide [response size](https://github.com/etcd-io/etcd/pull/9826). - Add [backoff on watch retries on transient errors](https://github.com/etcd-io/etcd/pull/9840). ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_server_version`](https://github.com/etcd-io/etcd/pull/8960) Prometheus metric. - To replace [Kubernetes `etcd-version-monitor`](https://github.com/etcd-io/etcd/issues/8948). ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.22](https://github.com/etcd-io/etcd/releases/tag/v3.2.22) (2018-06-06) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.21...v3.2.22) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Security, Authentication - Support TLS cipher suite whitelisting. - To block [weak cipher suites](https://github.com/etcd-io/etcd/issues/8320). - TLS handshake fails when client hello is requested with invalid cipher suites. - Add [`etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/9801) flag. - If empty, Go auto-populates the list. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.21](https://github.com/etcd-io/etcd/releases/tag/v3.2.21) (2018-05-31) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.20...v3.2.21) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### etcd server - Fix [auth storage panic when simple token provider is disabled](https://github.com/etcd-io/etcd/pull/8695). - Fix [`mvcc` server panic from restore operation](https://github.com/etcd-io/etcd/pull/9775). - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X, **etcd server panicked** during snapshot restore operation. - Now, this server-side panic has been fixed. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.20](https://github.com/etcd-io/etcd/releases/tag/v3.2.20) (2018-05-09) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.19...v3.2.20) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### etcd server - Purge old [`*.snap.db` snapshot files](https://github.com/etcd-io/etcd/pull/7967). - Previously, etcd did not respect `--max-snapshots` flag to purge old `*.snap.db` files. - Now, etcd purges old `*.snap.db` files to keep maximum `--max-snapshots` number of files on disk. ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.19](https://github.com/etcd-io/etcd/releases/tag/v3.2.19) (2018-04-24) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.18...v3.2.19) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Fix [`etcd_debugging_server_lease_expired_total`](https://github.com/etcd-io/etcd/pull/9557) Prometheus metric. - Fix [race conditions in v2 server stat collecting](https://github.com/etcd-io/etcd/pull/9562). - Add [`etcd_server_is_leader`](https://github.com/etcd-io/etcd/pull/9587) Prometheus metric. ### Security, Authentication - Fix [TLS reload](https://github.com/etcd-io/etcd/pull/9570) when [certificate SAN field only includes IP addresses but no domain names](https://github.com/etcd-io/etcd/issues/9541). - In Go, server calls `(*tls.Config).GetCertificate` for TLS reload if and only if server's `(*tls.Config).Certificates` field is not empty, or `(*tls.ClientHelloInfo).ServerName` is not empty with a valid SNI from the client. Previously, etcd always populates `(*tls.Config).Certificates` on the initial client TLS handshake, as non-empty. Thus, client was always expected to supply a matching SNI in order to pass the TLS verification and to trigger `(*tls.Config).GetCertificate` to reload TLS assets. - However, a certificate whose SAN field does [not include any domain names but only IP addresses](https://github.com/etcd-io/etcd/issues/9541) would request `*tls.ClientHelloInfo` with an empty `ServerName` field, thus failing to trigger the TLS reload on initial TLS handshake; this becomes a problem when expired certificates need to be replaced online. - Now, `(*tls.Config).Certificates` is created empty on initial TLS client handshake, first to trigger `(*tls.Config).GetCertificate`, and then to populate rest of the certificates on every new TLS connection, even when client SNI is empty (e.g. cert only includes IPs). ### etcd server - Add [`etcd --initial-election-tick-advance`](https://github.com/etcd-io/etcd/pull/9591) flag to configure initial election tick fast-forward. - By default, `etcd --initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger. - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election. - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout. - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities. - Now, this can be disabled by setting `--initial-election-tick-advance=false`. - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `--initial-election-tick-advance` at the cost of slow initial bootstrap. - If single-node, it advances ticks regardless. - Address [disruptive rejoining follower node](https://github.com/etcd-io/etcd/issues/9333). ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.18](https://github.com/etcd-io/etcd/releases/tag/v3.2.18) (2018-03-29) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.17...v3.2.18) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Improved - Adjust [election timeout on server restart](https://github.com/etcd-io/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/etcd-io/etcd/issues/9333). - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node. - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add missing [`etcd_network_peer_sent_failures_total` count](https://github.com/etcd-io/etcd/pull/9437). ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.17](https://github.com/etcd-io/etcd/releases/tag/v3.2.17) (2018-03-08) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.16...v3.2.17) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### etcd server - Fix [server panic on invalid Election Proclaim/Resign HTTP(S) requests](https://github.com/etcd-io/etcd/pull/9379). - Previously, wrong-formatted HTTP requests to Election API could trigger panic in etcd server. - e.g. `curl -L http://localhost:2379/v3/election/proclaim -X POST -d '{"value":""}'`, `curl -L http://localhost:2379/v3/election/resign -X POST -d '{"value":""}'`. - Prevent [overflow by large `TTL` values for `Lease` `Grant`](https://github.com/etcd-io/etcd/pull/9399). - `TTL` parameter to `Grant` request is unit of second. - Leases with too large `TTL` values exceeding `math.MaxInt64` [expire in unexpected ways](https://github.com/etcd-io/etcd/issues/9374). - Server now returns `rpctypes.ErrLeaseTTLTooLarge` to client, when the requested `TTL` is larger than *9,000,000,000 seconds* (which is >285 years). - Again, etcd `Lease` is meant for short-periodic keepalives or sessions, in the range of seconds or minutes. Not for hours or days! - Enable etcd server [`raft.Config.CheckQuorum` when starting with `ForceNewCluster`](https://github.com/etcd-io/etcd/pull/9347). ### Proxy v2 - Fix [v2 proxy leaky HTTP requests](https://github.com/etcd-io/etcd/pull/9336). ### Go - Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.16](https://github.com/etcd-io/etcd/releases/tag/v3.2.16) (2018-02-12) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.15...v3.2.16) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### etcd server - Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/etcd-io/etcd/pull/9297). - "unsynced" watcher is watcher that needs to be in sync with events that have happened. - That is, "unsynced" watcher is the slow watcher that was requested on old revision. - "unsynced" watcher restore operation was not correctly populating its underlying watcher group. - Which possibly causes [missing events from "unsynced" watchers](https://github.com/etcd-io/etcd/issues/9086). - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node. ### Go - Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.15](https://github.com/etcd-io/etcd/releases/tag/v3.2.15) (2018-01-22) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.14...v3.2.15) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### etcd server - Prevent [server panic from member update/add](https://github.com/etcd-io/etcd/pull/9174) with [wrong scheme URLs](https://github.com/etcd-io/etcd/issues/9173). - Log [user context cancel errors on stream APIs in debug level with TLS](https://github.com/etcd-io/etcd/pull/9178). ### Go - Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.14](https://github.com/etcd-io/etcd/releases/tag/v3.2.14) (2018-01-11) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.13...v3.2.14) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Improved - Log [user context cancel errors on stream APIs in debug level](https://github.com/etcd-io/etcd/pull/9105). ### etcd server - Fix [`mvcc/backend.defragdb` nil-pointer dereference on create bucket failure](https://github.com/etcd-io/etcd/pull/9119). ### Go - Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.13](https://github.com/etcd-io/etcd/releases/tag/v3.2.13) (2018-01-02) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.12...v3.2.13) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### etcd server - Remove [verbose error messages on stream cancel and gRPC info-level logs](https://github.com/etcd-io/etcd/pull/9080) in server-side. - Fix [gRPC server panic on `GracefulStop` TLS-enabled server](https://github.com/etcd-io/etcd/pull/8987). ### Go - Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.12](https://github.com/etcd-io/etcd/releases/tag/v3.2.12) (2017-12-20) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.11...v3.2.12) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Dependency - Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases/tag) from [**`v1.7.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.4) to [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5). - Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.3`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3) to [**`v1.3.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3.0). ### etcd server - Fix [error message of `Revision` compactor](https://github.com/etcd-io/etcd/pull/8999) in server-side. ### client v3 - Add [`MaxCallSendMsgSize` and `MaxCallRecvMsgSize`](https://github.com/etcd-io/etcd/pull/9047) fields to [`clientv3.Config`](https://godoc.org/github.com/etcd-io/etcd/clientv3#Config). - Fix [exceeded response size limit error in client-side](https://github.com/etcd-io/etcd/issues/9043). - Address [kubernetes#51099](https://github.com/kubernetes/kubernetes/issues/51099). - In previous versions(v3.2.10, v3.2.11), client response size was limited to only 4 MiB. - `MaxCallSendMsgSize` default value is 2 MiB, if not configured. - `MaxCallRecvMsgSize` default value is `math.MaxInt32`, if not configured. ### Go - Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.11](https://github.com/etcd-io/etcd/releases/tag/v3.2.11) (2017-12-05) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.10...v3.2.11) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Dependency - Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases/tag) from [**`v1.7.3`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.3) to [**`v1.7.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.4). ### Security, Authentication See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - Log [more details on TLS handshake failures](https://github.com/etcd-io/etcd/pull/8952/files). ### client v3 - Fix racey grpc-go's server handler transport `WriteStatus` call to prevent [TLS-enabled etcd server crash](https://github.com/etcd-io/etcd/issues/8904). - Add [gRPC RPC failure warnings](https://github.com/etcd-io/etcd/pull/8939) to help debug such issues in the future. ### Documentation - Remove `--listen-metrics-urls` flag in monitoring document (non-released in `v3.2.x`, planned for `v3.3.x`). ### Go - Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.10](https://github.com/etcd-io/etcd/releases/tag/v3.2.10) (2017-11-16) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.9...v3.2.10) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Dependency - Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases/tag) from [**`v1.2.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.2.1) to [**`v1.7.3`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.3). - Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.2.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.2.0) to [**`v1.3`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3). ### Security, Authentication See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - Revert [discovery SRV auth `ServerName` with `*.{ROOT_DOMAIN}`](https://github.com/etcd-io/etcd/pull/8651) to support non-wildcard subject alternative names in the certs (see [issue #8445](https://github.com/etcd-io/etcd/issues/8445) for more contexts). - For instance, `etcd --discovery-srv=etcd.local` will only authenticate peers/clients when the provided certs have root domain `etcd.local` (**not `*.etcd.local`**) as an entry in Subject Alternative Name (SAN) field. ### etcd server - Replace backend key-value database `boltdb/bolt` with [`coreos/bbolt`](https://github.com/coreos/bbolt/releases) to address [backend database size issue](https://github.com/etcd-io/etcd/issues/8009). ### client v3 - Rewrite balancer to handle [network partitions](https://github.com/etcd-io/etcd/issues/8711). ### Go - Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.9](https://github.com/etcd-io/etcd/releases/tag/v3.2.9) (2017-10-06) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.8...v3.2.9) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Security, Authentication See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - Update `golang.org/x/crypto/bcrypt` (see [golang/crypto@6c586e1](https://github.com/golang/crypto/commit/6c586e17d90a7d08bbbc4069984180dce3b04117)). - Fix discovery SRV bootstrapping to [authenticate `ServerName` with `*.{ROOT_DOMAIN}`](https://github.com/etcd-io/etcd/pull/8651), in order to support sub-domain wildcard matching (see [issue #8445](https://github.com/etcd-io/etcd/issues/8445) for more contexts). - For instance, `etcd --discovery-srv=etcd.local` will only authenticate peers/clients when the provided certs have root domain `*.etcd.local` as an entry in Subject Alternative Name (SAN) field. ### Go - Compile with [*Go 1.8.4*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.8](https://github.com/etcd-io/etcd/releases/tag/v3.2.8) (2017-09-29) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.7...v3.2.8) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### client v2 - Fix v2 client failover to next endpoint on mutable operation. ### gRPC Proxy - Handle [`KeysOnly` flag](https://github.com/etcd-io/etcd/pull/8552). ### Go - Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.7](https://github.com/etcd-io/etcd/releases/tag/v3.2.7) (2017-09-01) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.6...v3.2.7) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Security, Authentication - Fix [server-side auth so concurrent auth operations do not return old revision error](https://github.com/etcd-io/etcd/pull/8306). ### client v3 - Fix [`concurrency/stm` Put with serializable snapshot](https://github.com/etcd-io/etcd/pull/8439). - Use store revision from first fetch to resolve write conflicts instead of modified revision. ### Go - Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.6](https://github.com/etcd-io/etcd/releases/tag/v3.2.6) (2017-08-21) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.5...v3.2.6) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### etcd server - Fix watch restore from snapshot. - Fix multiple URLs for `--listen-peer-urls` flag. - Add `--enable-pprof` flag to etcd configuration file format. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Fix `etcd_debugging_mvcc_keys_total` inconsistency. ### Go - Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.5](https://github.com/etcd-io/etcd/releases/tag/v3.2.5) (2017-08-04) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.4...v3.2.5) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### etcdctl v3 - Return non-zero exit code on unhealthy `endpoint health`. ### Security, Authentication See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - [Server supports reverse-lookup on wildcard DNS `SAN`](https://github.com/etcd-io/etcd/pull/8281). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server first reverse-lookups the remote IP address to get a list of names mapping to that address (e.g. `nslookup IPADDR`). Then accepts the connection if those names have a matching name with peer cert's DNS names (either by exact or wildcard match). If none is matched, server forward-lookups each DNS entry in peer cert (e.g. look up `example.default.svc` when the entry is `*.example.default.svc`), and accepts connection only when the host's resolved addresses have the matching IP address with the peer's remote IP address. For example, peer B's CSR (with `cfssl`) SAN field is `["*.example.default.svc", "*.example.default.svc.cluster.local"]` when peer B's remote IP address is `10.138.0.2`. When peer B tries to join the cluster, peer A reverse-lookup the IP `10.138.0.2` to get the list of host names. And either exact or wildcard match the host names with peer B's cert DNS names in Subject Alternative Name (SAN) field. If none of reverse/forward lookups worked, it returns an error `"tls: "10.138.0.2" does not match any of DNSNames ["*.example.default.svc","*.example.default.svc.cluster.local"]`. See [issue#8268](https://github.com/etcd-io/etcd/issues/8268) for more detail. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Fix unreachable `/metrics` endpoint when `--enable-v2=false`. ### gRPC Proxy - Handle [`PrevKv` flag](https://github.com/etcd-io/etcd/pull/8366). ### Other - Add container registry `gcr.io/etcd-development/etcd`. ### Go - Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.4](https://github.com/etcd-io/etcd/releases/tag/v3.2.4) (2017-07-19) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.3...v3.2.4) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### etcd server - Do not block on active client stream when stopping server ### gRPC proxy - Fix gRPC proxy Snapshot RPC error handling ### Go - Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.3](https://github.com/etcd-io/etcd/releases/tag/v3.2.3) (2017-07-14) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.2...v3.2.3) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### client v3 - Let clients establish unlimited streams ### Other - Tag docker images with minor versions - e.g. `docker pull quay.io/coreos/etcd:v3.2` to fetch latest v3.2 versions ### Go - Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.2](https://github.com/etcd-io/etcd/releases/tag/v3.2.2) (2017-07-07) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.1...v3.2.2) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Improved - Rate-limit lease revoke on expiration. - Extend leases on promote to avoid queueing effect on lease expiration. ### Security, Authentication See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - [Server accepts connections if IP matches, without checking DNS entries](https://github.com/etcd-io/etcd/pull/8223). For instance, if peer cert contains IP addresses and DNS names in Subject Alternative Name (SAN) field, and the remote IP address matches one of those IP addresses, server just accepts connection without further checking the DNS names. For example, peer B's CSR (with `cfssl`) SAN field is `["invalid.domain", "10.138.0.2"]` when peer B's remote IP address is `10.138.0.2` and `invalid.domain` is a invalid host. When peer B tries to join the cluster, peer A successfully authenticates B, since Subject Alternative Name (SAN) field has a valid matching IP address. See [issue#8206](https://github.com/etcd-io/etcd/issues/8206) for more detail. ### etcd server - Accept connection with matched IP SAN but no DNS match. - Don't check DNS entries in certs if there's a matching IP. ### gRPC gateway - Use user-provided listen address to connect to gRPC gateway. - `net.Listener` rewrites IPv4 0.0.0.0 to IPv6 [::], breaking IPv6 disabled hosts. - Only v3.2.0, v3.2.1 are affected. ### Go - Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.1](https://github.com/etcd-io/etcd/releases/tag/v3.2.1) (2017-06-23) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.0...v3.2.1) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### etcd server - Fix backend database in-memory index corruption issue on restore (only 3.2.0 is affected). ### gRPC gateway - Fix Txn marshaling. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Fix backend database size debugging metrics. ### Go - Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). --- ## [v3.2.0](https://github.com/etcd-io/etcd/releases/tag/v3.2.0) (2017-06-09) See [code changes](https://github.com/etcd-io/etcd/compare/v3.1.0...v3.2.0) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_2/).** ### Improved - Improve backend read concurrency. ### Breaking Changes - Increased [`--snapshot-count` default value from 10,000 to 100,000](https://github.com/etcd-io/etcd/pull/7160). - Higher snapshot count means it holds Raft entries in memory for longer before discarding old entries. - It is a trade-off between less frequent snapshotting and [higher memory usage](https://github.com/kubernetes/kubernetes/issues/60589#issuecomment-371977156). - User lower `--snapshot-count` value for lower memory usage. - User higher `--snapshot-count` value for better availabilities of slow followers (less frequent snapshots from leader). - `clientv3.Lease.TimeToLive` returns `LeaseTimeToLiveResponse.TTL == -1` on lease not found. - `clientv3.NewFromConfigFile` is moved to `clientv3/yaml.NewConfig`. - `embed.Etcd.Peers` field is now `[]*peerListener`. - Rejects domains names for `--listen-peer-urls` and `--listen-client-urls` (3.1 only prints out warnings), since [domain name is invalid for network interface binding](https://github.com/etcd-io/etcd/issues/6336). ### Dependency - Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.0.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.0.4) to [**`v1.2.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.2.1). - Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) to [**`v1.2.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.2.0). ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_disk_backend_snapshot_duration_seconds`](https://github.com/etcd-io/etcd/pull/7892) - Add `etcd_debugging_server_lease_expired_total` metrics. ### Security, Authentication See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - [TLS certificates get reloaded on every client connection](https://github.com/etcd-io/etcd/pull/7829). This is useful when replacing expiry certs without stopping etcd servers; it can be done by overwriting old certs with new ones. Refreshing certs for every connection should not have too much overhead, but can be improved in the future, with caching layer. Example tests can be found [here](https://github.com/etcd-io/etcd/blob/b041ce5d514a4b4aaeefbffb008f0c7570a18986/integration/v3_grpc_test.go#L1601-L1757). - [Server denies incoming peer certs with wrong IP `SAN`](https://github.com/etcd-io/etcd/pull/7687). For instance, if peer cert contains any IP addresses in Subject Alternative Name (SAN) field, server authenticates a peer only when the remote IP address matches one of those IP addresses. This is to prevent unauthorized endpoints from joining the cluster. For example, peer B's CSR (with `cfssl`) SAN field is `["*.example.default.svc", "*.example.default.svc.cluster.local", "10.138.0.27"]` when peer B's actual IP address is `10.138.0.2`, not `10.138.0.27`. When peer B tries to join the cluster, peer A will reject B with the error `x509: certificate is valid for 10.138.0.27, not 10.138.0.2`, because B's remote IP address does not match the one in Subject Alternative Name (SAN) field. - [Server resolves TLS `DNSNames` when checking `SAN`](https://github.com/etcd-io/etcd/pull/7767). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server authenticates a peer only when forward-lookups (`dig b.com`) on those DNS names have matching IP with the remote IP address. For example, peer B's CSR (with `cfssl`) SAN field is `["b.com"]` when peer B's remote IP address is `10.138.0.2`. When peer B tries to join the cluster, peer A looks up the incoming host `b.com` to get the list of IP addresses (e.g. `dig b.com`). And rejects B if the list does not contain the IP `10.138.0.2`, with the error `tls: 10.138.0.2 does not match any of DNSNames ["b.com"]`. - Auth support JWT token. ### etcd server - RPCs - Add Election, Lock service. - Native client `etcdserver/api/v3client` - client "embedded" in the server. - Logging, monitoring - Server warns large snapshot operations. - Add `etcd --enable-v2` flag to enable v2 API server. - `etcd --enable-v2=true` by default. - Add `etcd --auth-token` flag. - v3.2 compactor runs [every hour](https://github.com/etcd-io/etcd/pull/7875). - Compactor only supports periodic compaction. - Compactor continues to record latest revisions every 5-minute. - For every hour, it uses the last revision that was fetched before compaction period, from the revision records that were collected every 5-minute. - That is, for every hour, compactor discards historical data created before compaction period. - The retention window of compaction period moves to next hour. - For instance, when hourly writes are 100 and `--auto-compaction-retention=10`, v3.1 compacts revision 1000, 2000, and 3000 for every 10-hour, while v3.2 compacts revision 1000, 1100, and 1200 for every 1-hour. - If compaction succeeds or requested revision has already been compacted, it resets period timer and removes used compacted revision from historical revision records (e.g. start next revision collect and compaction from previously collected revisions). - If compaction fails, it retries in 5 minutes. - Allow snapshot over 512MB. ### client v3 - STM prefetching. - Add namespace feature. - Add `ErrOldCluster` with server version checking. - Translate `WithPrefix()` into `WithFromKey()` for empty key. ### etcdctl v3 - Add `check perf` command. - Add `etcdctl --from-key` flag to role grant-permission command. - `lock` command takes an optional command to execute. ### gRPC Proxy - Proxy endpoint discovery. - Namespaces. - Coalesce lease requests. ### etcd gateway - Support [DNS SRV priority](https://github.com/etcd-io/etcd/pull/7882) for [smart proxy routing](https://github.com/etcd-io/etcd/issues/4378). ### Other - v3 client - concurrency package's elections updated to match RPC interfaces. - let client dial endpoints not in the balancer. - Release - Annotate acbuild with supports-systemd-notify. - Add `nsswitch.conf` to Docker container image. - Add ppc64le, arm64(experimental) builds. ### Go - Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8). --- ================================================ FILE: CHANGELOG/CHANGELOG-3.3.md ================================================ Previous change logs can be found at [CHANGELOG-3.2](https://github.com/etcd-io/etcd/blob/main/CHANGELOG/CHANGELOG-3.2.md). --- ## v3.3.27 (2021-10-15) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.26...v3.3.27) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. ### Other - Updated [base image](https://github.com/etcd-io/etcd/pull/13386) from `debian:buster-v1.4.0` to `debian:bullseye-20210927` to fix the following critical CVEs: - [CVE-2021-3711](https://nvd.nist.gov/vuln/detail/CVE-2021-3711): miscalculation of a buffer size in openssl's SM2 decryption - [CVE-2021-35942](https://nvd.nist.gov/vuln/detail/CVE-2021-35942): integer overflow flaw in glibc - [CVE-2019-9893](https://nvd.nist.gov/vuln/detail/CVE-2019-9893): incorrect syscall argument generation in libseccomp - [CVE-2021-36159](https://nvd.nist.gov/vuln/detail/CVE-2021-36159): libfetch in apk-tools mishandles numeric strings in FTP and HTTP protocols to allow out of bound reads. --- ## v3.3.26 (2021-10-03) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.25...v3.3.26) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. ### Package `clientv3` - Fix [auth token invalid after watch reconnects](https://github.com/etcd-io/etcd/pull/12264). Get AuthToken automatically when clientConn is ready. ### Package `fileutil` - Fix [constant](https://github.com/etcd-io/etcd/pull/12440) for linux locking. ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## v3.3.25 (2020-08-24) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.23...v3.3.25) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. ### Security - A [log warning](https://github.com/etcd-io/etcd/pull/12242) is added when etcd use any existing directory that has a permission different than 700 on Linux and 777 on Windows. ## [v3.3.24](https://github.com/etcd-io/etcd/releases/tag/v3.3.24) (2020-08-18) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.23...v3.3.24) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. ### Package `etcd server` - Fix [`int64` convert panic in raft logger](https://github.com/etcd-io/etcd/pull/12106). - Fix [kubernetes/kubernetes#91937](https://github.com/kubernetes/kubernetes/issues/91937). ### Package `runtime` - Optimize [`runtime.FDUsage` by removing unnecessary sorting](https://github.com/etcd-io/etcd/pull/12214). ### Metrics, Monitoring - Add [`os_fd_used` and `os_fd_limit` to monitor current OS file descriptors](https://github.com/etcd-io/etcd/pull/12214). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.3.23](https://github.com/etcd-io/etcd/releases/tag/v3.3.23) (2020-07-16) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.22...v3.3.23) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. ### Breaking Changes - Fix [incorrect package dependency when etcd clientv3 used as libary](https://github.com/etcd-io/etcd/issues/12068). - Changed behavior on [existing dir permission](https://github.com/etcd-io/etcd/pull/11798). - Previously, the permission was not checked on existing data directory and the directory used for automatically generating self-signed certificates for TLS connections with clients. Now a check is added to make sure those directories, if already exist, has a desired permission of 700 on Linux and 777 on Windows. ### Package `wal` ### etcd server - Fix [watch stream got closed if one watch request is not permitted](https://github.com/etcd-io/etcd/pull/11758). - Add [etcd --auth-token-ttl](https://github.com/etcd-io/etcd/pull/11980) flag to customize `simpleTokenTTL` settings. - Improve [runtime.FDUsage objects malloc of Memory Usage and CPU Usage](https://github.com/etcd-io/etcd/pull/11986). - Improve [mvcc.watchResponse channel Memory Usage](https://github.com/etcd-io/etcd/pull/11987). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.3.22](https://github.com/etcd-io/etcd/releases/tag/v3.3.22) (2020-05-20) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.21...v3.3.22) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. ### Package `wal` - Add [missing CRC checksum check in WAL validate method otherwise causes panic](https://github.com/etcd-io/etcd/pull/11924). - See https://github.com/etcd-io/etcd/issues/11918. ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.3.21](https://github.com/etcd-io/etcd/releases/tag/v3.3.21) (2020-05-18) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.20...v3.3.21) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. ### `etcdctl` - Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). ### Package `clientv3` - Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). ### etcd server - Improve logging around snapshot send and receive. - [Add log when etcdserver failed to apply command](https://github.com/etcd-io/etcd/pull/11670). - [Fix deadlock bug in mvcc](https://github.com/etcd-io/etcd/pull/11817). - Fix [inconsistency between WAL and server snapshot](https://github.com/etcd-io/etcd/pull/11888). - Previously, server restore fails if it had crashed after persisting raft hard state but before saving snapshot. - See https://github.com/etcd-io/etcd/issues/10219 for more. ### Package `auth` - [Fix a data corruption bug by saving consistent index](https://github.com/etcd-io/etcd/pull/11652). ### Metrics, Monitoring - Add [`etcd_debugging_auth_revision`](https://github.com/etcd-io/etcd/commit/f14d2a087f7b0fd6f7980b95b5e0b945109c95f3). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.3.20](https://github.com/etcd-io/etcd/releases/tag/v3.3.20) (2020-04-01) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.19...v3.3.20) and [v3.2 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. ### Package `wal` - Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). ### Metrics, Monitoring - Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.3.19](https://github.com/etcd-io/etcd/releases/tag/v3.3.19) (2020-03-18) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.18...v3.3.19) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. ### client v3 - Fix [`"hasleader"` metadata embedding](https://github.com/etcd-io/etcd/pull/11687). - Previously, `clientv3.WithRequireLeader(ctx)` was overwriting existing context keys. ### etcd server - [Fix corruption bug in defrag](https://github.com/etcd-io/etcd/pull/11613). - Log [`[CLIENT-PORT]/health` check in server side](https://github.com/etcd-io/etcd/pull/11704). ### etcdctl v3 - Fix [`etcdctl member add`](https://github.com/etcd-io/etcd/pull/11638) command to prevent potential timeout. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. - Add [`etcd_server_client_requests_total` with `"type"` and `"client_api_version"` labels](https://github.com/etcd-io/etcd/pull/11687). ### gRPC Proxy - Fix [`panic on error`](https://github.com/etcd-io/etcd/pull/11694) for metrics handler. ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.3.18](https://github.com/etcd-io/etcd/releases/tag/v3.3.18) (2019-11-26) ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_cluster_version`](https://github.com/etcd-io/etcd/pull/11261) Prometheus metric. - Add [`etcd_debugging_mvcc_total_put_size_in_bytes`](https://github.com/etcd-io/etcd/pull/11374) Prometheus metric. ### etcdserver - Fix [`wait purge file loop during shutdown`](https://github.com/etcd-io/etcd/pull/11308). - Previously, during shutdown etcd could accidentally remove needed wal files, resulting in catastrophic error `etcdserver: open wal error: wal: file not found.` during startup. - Now, etcd makes sure the purge file loop exits before server signals stop of the raft node. --- ## [v3.3.17](https://github.com/etcd-io/etcd/releases/tag/v3.3.17) (2019-10-11) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.16...v3.3.17) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. ### Release details This release replaces 3.3.16. Due to the etcd 3.3.16 release being incorrectly released (see details below), please use this release instead. --- ## [v3.3.16](https://github.com/etcd-io/etcd/releases/tag/v3.3.16) (2019-10-10) **WARNING: This is a bad release! Please use etcd 3.3.17 instead. See https://github.com/etcd-io/etcd/issues/11241 for details.** ### Issues with release - go mod for 'v3.3.16' may return a different hash if retrieved from a go mod proxy than if retrieved directly from github. Depending on this version is unsafe. See https://github.com/etcd-io/etcd/issues/11241 for details. - The binaries and docker image for this release have been published and will be left as-is, but will not be signed since this is a bad release. See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.15...v3.3.16) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### Improved - Add `etcd --experimental-peer-skip-client-san-verification` to [skip verification of peer client address](https://github.com/etcd-io/etcd/pull/11196). ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_debugging_mvcc_current_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. - Add [`etcd_debugging_mvcc_compact_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. ### Dependency - Upgrade [`github.com/coreos/bbolt`](https://github.com/etcd-io/bbolt/releases) from [**`v1.3.1-coreos.6`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.1-coreos.6) to [**`v1.3.3`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.3). ### etcdctl v3 - Fix [`etcdctl member add`](https://github.com/etcd-io/etcd/pull/11194) command to prevent potential timeout. ### Go - Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. ### client v3 - Fix [client balancer failover against multiple endpoints](https://github.com/etcd-io/etcd/pull/11184). - Fix ["kube-apiserver: failover on multi-member etcd cluster fails certificate check on DNS mismatch" (kubernetes#83028)](https://github.com/kubernetes/kubernetes/issues/83028). - Fix [IPv6 endpoint parsing in client](https://github.com/etcd-io/etcd/pull/11211). - Fix ["1.16: etcd client does not parse IPv6 addresses correctly when members are joining" (kubernetes#83550)](https://github.com/kubernetes/kubernetes/issues/83550). --- ## [v3.3.15](https://github.com/etcd-io/etcd/releases/tag/v3.3.15) (2019-08-19) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.14...v3.3.15) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** NOTE: This patch release had to include some new features from 3.4, while trying to minimize the difference between client balancer implementation. This release fixes ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). ### Breaking Changes - Revert "Migrate dependency management tool from `glide` to [Go module](https://github.com/etcd-io/etcd/pull/10063)". - Now, etcd >= v3.3.15 uses `glide` for dependency management. - See [kubernetes#81434](https://github.com/kubernetes/kubernetes/pull/81434) for more contexts. ### Go - Require [*Go 1.12+*](https://github.com/etcd-io/etcd/pull/10045). - Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. --- ## [v3.3.14](https://github.com/etcd-io/etcd/releases/tag/v3.3.14) (2019-08-16) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.13...v3.3.14) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - [v3.3.14-rc.0](https://github.com/etcd-io/etcd/releases/tag/v3.3.14-rc.0) (2019-08-15), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.14-beta.0...v3.3.14-rc.0). - [v3.3.14-beta.0](https://github.com/etcd-io/etcd/releases/tag/v3.3.14-beta.0) (2019-08-14), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.13...v3.3.14-beta.0). **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** NOTE: This patch release had to include some new features from 3.4, while trying to minimize the difference between client balancer implementation. This release fixes ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). ### Breaking Changes - Rewrite [client balancer](https://github.com/etcd-io/etcd/pull/9860) with [new gRPC balancer interface](https://github.com/etcd-io/etcd/issues/9106). - Upgrade [gRPC to v1.23.0](https://github.com/etcd-io/etcd/pull/10911). - Improve [client balancer failover against secure endpoints](https://github.com/etcd-io/etcd/pull/10911). - Fix ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). - [The new client balancer](https://etcd.io/docs/latest/learning/design-client/) uses an asynchronous resolver to pass endpoints to the gRPC dial function. to block until the underlying connection is up, pass `grpc.WithBlock()` to `clientv3.Config.DialOptions`. - Require [*Go 1.12+*](https://github.com/etcd-io/etcd/pull/10045). - Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. - Migrate dependency management tool from `glide` to [Go module](https://github.com/etcd-io/etcd/pull/10063). - <= 3.3 puts `vendor` directory under `cmd/vendor` directory to [prevent conflicting transitive dependencies](https://github.com/etcd-io/etcd/issues/4913). - 3.4 moves `cmd/vendor` directory to `vendor` at repository root. - Remove recursive symlinks in `cmd` directory. - Now `go get/install/build` on `etcd` packages (e.g. `clientv3`, `tools/benchmark`) enforce builds with etcd `vendor` directory. - Deprecated `latest` [release container](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) tag. - **`docker pull gcr.io/etcd-development/etcd:latest` would not be up-to-date**. - Deprecated [minor](https://semver.org/) version [release container](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) tags. - `docker pull gcr.io/etcd-development/etcd:v3.3` would still work but may be stale. - **`docker pull gcr.io/etcd-development/etcd:v3.4` would not work**. - Use **`docker pull gcr.io/etcd-development/etcd:v3.3.14`** instead, with the exact patch version. - Deprecated [ACIs from official release](https://github.com/etcd-io/etcd/pull/9059). - [AppC was officially suspended](https://github.com/appc/spec#-disclaimer-), as of late 2016. - [`acbuild`](https://github.com/containers/build#this-project-is-currently-unmaintained) is not maintained anymore. - `*.aci` files are not available from `v3.4` release. ### etcd server - Add [`rpctypes.ErrLeaderChanged`](https://github.com/etcd-io/etcd/pull/10094). - Now linearizable requests with read index would fail fast when there is a leadership change, instead of waiting until context timeout. - Fix [race condition in `rafthttp` transport pause/resume](https://github.com/etcd-io/etcd/pull/10826). ### API - Add [`watch_id` field to `etcdserverpb.WatchCreateRequest`](https://github.com/etcd-io/etcd/pull/9065) to allow user-provided watch ID to `mvcc`. - Corresponding `watch_id` is returned via `etcdserverpb.WatchResponse`, if any. - Add [`fragment` field to `etcdserverpb.WatchCreateRequest`](https://github.com/etcd-io/etcd/pull/9291) to request etcd server to [split watch events](https://github.com/etcd-io/etcd/issues/9294) when the total size of events exceeds `etcd --max-request-bytes` flag value plus gRPC-overhead 512 bytes. - The default server-side request bytes limit is `embed.DefaultMaxRequestBytes` which is 1.5 MiB plus gRPC-overhead 512 bytes. - If watch response events exceed this server-side request limit and watch request is created with `fragment` field `true`, the server will split watch events into a set of chunks, each of which is a subset of watch events below server-side request limit. - Useful when client-side has limited bandwidths. - For example, watch response contains 10 events, where each event is 1 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB. Then, server will send 10 separate fragmented events to the client. - For example, watch response contains 5 events, where each event is 2 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB and `clientv3.Config.MaxCallRecvMsgSize` is 1 MiB. Then, server will try to send 5 separate fragmented events to the client, and the client will error with `"code = ResourceExhausted desc = grpc: received message larger than max (...)"`. - Client must implement fragmented watch event merge (which `clientv3` does in etcd v3.4). - Add [`WatchRequest.WatchProgressRequest`](https://github.com/etcd-io/etcd/pull/9869). - To manually trigger broadcasting watch progress event (empty watch response with latest header) to all associated watch streams. - Think of it as `WithProgressNotify` that can be triggered manually. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_network_snapshot_send_inflights_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. - Add [`etcd_network_snapshot_receive_inflights_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. - Add [`etcd_server_snapshot_apply_in_progress_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. ### client v3 - Fix [gRPC panic "send on closed channel](https://github.com/etcd-io/etcd/issues/9956) by upgrading [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5) to [**`v1.23.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.0). - Rewrite [client balancer](https://github.com/etcd-io/etcd/pull/9860) with [new gRPC balancer interface](https://github.com/etcd-io/etcd/issues/9106). - Upgrade [gRPC to v1.23.0](https://github.com/etcd-io/etcd/pull/10911). - Improve [client balancer failover against secure endpoints](https://github.com/etcd-io/etcd/pull/10911). - Fix ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). - [The new client balancer](https://etcd.io/docs/latest/learning/design-client/) uses an asynchronous resolver to pass endpoints to the gRPC dial function. to block until the underlying connection is up, pass `grpc.WithBlock()` to `clientv3.Config.DialOptions`. ### etcdctl v3 - Add [`etcdctl endpoint health --write-out` support](https://github.com/etcd-io/etcd/pull/9540). - Previously, [`etcdctl endpoint health --write-out json` did not work](https://github.com/etcd-io/etcd/issues/9532). - The command output is changed. Previously, if endpoint is unreachable, the command output is "\ is unhealthy: failed to connect: \". This change unified the error message, all error types now have the same output "\ is unhealthy: failed to commit proposal: \". - Add [missing newline in `etcdctl endpoint health`](https://github.com/etcd-io/etcd/pull/10793). ### Package `pkg/adt` - Change [`pkg/adt.IntervalTree` from `struct` to `interface`](https://github.com/etcd-io/etcd/pull/10959). - See [`pkg/adt` README](https://github.com/etcd-io/etcd/tree/main/pkg/adt) and [`pkg/adt` godoc](https://godoc.org/go.etcd.io/etcd/pkg/adt). - Improve [`pkg/adt.IntervalTree` test coverage](https://github.com/etcd-io/etcd/pull/10959). - See [`pkg/adt` README](https://github.com/etcd-io/etcd/tree/main/pkg/adt) and [`pkg/adt` godoc](https://godoc.org/go.etcd.io/etcd/pkg/adt). - Fix [Red-Black tree to maintain black-height property](https://github.com/etcd-io/etcd/pull/10978). - Previously, delete operation violates [black-height property](https://github.com/etcd-io/etcd/issues/10965). ### Go - Require [*Go 1.12+*](https://github.com/etcd-io/etcd/pull/10045). - Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. --- ## [v3.3.13](https://github.com/etcd-io/etcd/releases/tag/v3.3.13) (2019-05-02) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.12...v3.3.13) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### Improved - Improve [heartbeat send failure logging](https://github.com/etcd-io/etcd/pull/10663). - Add [`Verify` function to perform corruption check on WAL contents](https://github.com/etcd-io/etcd/pull/10603). ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Fix bug where [db_compaction_total_duration_milliseconds metric incorrectly measured duration as 0](https://github.com/etcd-io/etcd/pull/10646). ### client v3 - Fix [`(*Client).Endpoints()` method race condition](https://github.com/etcd-io/etcd/pull/10595). ### Package `wal` - Add [`Verify` function to perform corruption check on WAL contents](https://github.com/etcd-io/etcd/pull/10603). ### Dependency - Migrate [`github.com/ugorji/go/codec`](https://github.com/ugorji/go/releases) to [**`github.com/json-iterator/go`**](https://github.com/json-iterator/go) (See [#10667](https://github.com/etcd-io/etcd/pull/10667) for more). - Migrate [`github.com/ghodss/yaml`](https://github.com/ghodss/yaml/releases) to [**`sigs.k8s.io/yaml`**](https://github.com/kubernetes-sigs/yaml) (See [#10718](https://github.com/etcd-io/etcd/pull/10718) for more). ### Go - Compile with [*Go 1.10.8*](https://golang.org/doc/devel/release.html#go1.10). --- ## [v3.3.12](https://github.com/etcd-io/etcd/releases/tag/v3.3.12) (2019-02-07) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.11...v3.3.12) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### etcdctl v3 - [Strip out insecure endpoints from DNS SRV records when using discovery](https://github.com/etcd-io/etcd/pull/10443) with etcdctl v2 ### Go - Compile with [*Go 1.10.8*](https://golang.org/doc/devel/release.html#go1.10). --- ## [v3.3.11](https://github.com/etcd-io/etcd/releases/tag/v3.3.11) (2019-01-11) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.10...v3.3.11) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### gRPC Proxy - Fix [memory leak in cache layer](https://github.com/etcd-io/etcd/pull/10327). ### Security, Authentication - Disable [CommonName authentication for gRPC-gateway](https://github.com/etcd-io/etcd/pull/10366) gRPC-gateway proxy requests to etcd server use the etcd client server TLS certificate. If that certificate contains CommonName we do not want to use that for authentication as it could lead to permission escalation. ### Go - Compile with [*Go 1.10.7*](https://golang.org/doc/devel/release.html#go1.10). --- ## [v3.3.10](https://github.com/etcd-io/etcd/releases/tag/v3.3.10) (2018-10-10) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.9...v3.3.10) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### Improved - Improve ["became inactive" warning log](https://github.com/etcd-io/etcd/pull/10024), which indicates message send to a peer failed. - Improve [read index wait timeout warning log](https://github.com/etcd-io/etcd/pull/10026), which indicates that local node might have slow network. - Add [gRPC interceptor for debugging logs](https://github.com/etcd-io/etcd/pull/9990); enable `etcd --debug` flag to see per-request debug information. - Add [consistency check in snapshot status](https://github.com/etcd-io/etcd/pull/10109). If consistency check on snapshot file fails, `snapshot status` returns `"snapshot file integrity check failed..."` error. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Improve [`etcd_network_peer_round_trip_time_seconds`](https://github.com/etcd-io/etcd/pull/10155) Prometheus metric to track leader heartbeats. - Previously, it only samples the TCP connection for snapshot messages. - Add [`etcd_snap_db_fsync_duration_seconds_count`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_snap_db_save_total_duration_seconds_bucket`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_server_id`](https://github.com/etcd-io/etcd/pull/9998) Prometheus metric. - Add [`etcd_server_health_success`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. - Add [`etcd_server_health_failures`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. - Add [`etcd_server_read_indexes_failed_total`](https://github.com/etcd-io/etcd/pull/10094) Prometheus metric. ### client v3 - Fix logic on [release lock key if cancelled](https://github.com/etcd-io/etcd/pull/10153) in `clientv3/concurrency` package. ### Go - Compile with [*Go 1.10.4*](https://golang.org/doc/devel/release.html#go1.10). --- ## [v3.3.9](https://github.com/etcd-io/etcd/releases/tag/v3.3.9) (2018-07-24) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.8...v3.3.9) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### Improved - Improve [Raft Read Index timeout warning messages](https://github.com/etcd-io/etcd/pull/9897). ### Security, Authentication - Compile with [*Go 1.10.3*](https://golang.org/doc/devel/release.html#go1.10) to support [crypto/x509 "Name Constraints"](https://github.com/etcd-io/etcd/issues/9912). ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_server_go_version`](https://github.com/etcd-io/etcd/pull/9957) Prometheus metric. - Add [`etcd_server_heartbeat_send_failures_total`](https://github.com/etcd-io/etcd/pull/9940) Prometheus metric. - Add [`etcd_server_slow_apply_total`](https://github.com/etcd-io/etcd/pull/9940) Prometheus metric. - Add [`etcd_disk_backend_defrag_duration_seconds`](https://github.com/etcd-io/etcd/pull/9940) Prometheus metric. - Add [`etcd_mvcc_hash_duration_seconds`](https://github.com/etcd-io/etcd/pull/9940) Prometheus metric. - Add [`etcd_mvcc_hash_rev_duration_seconds`](https://github.com/etcd-io/etcd/pull/9940) Prometheus metric. - Add [`etcd_server_slow_read_indexes_total`](https://github.com/etcd-io/etcd/pull/9897) Prometheus metric. - Add [`etcd_server_quota_backend_bytes`](https://github.com/etcd-io/etcd/pull/9820) Prometheus metric. - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. - Add [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819) Prometheus metric. - In addition to [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819). - Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/etcd-io/etcd/pull/9256) Prometheus metric. - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. ### client v3 - Fix [lease keepalive interval updates when response queue is full](https://github.com/etcd-io/etcd/pull/9952). - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/etcd-io/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration. ### Go - Compile with [*Go 1.10.3*](https://golang.org/doc/devel/release.html#go1.10). --- ## [v3.3.8](https://github.com/etcd-io/etcd/releases/tag/v3.3.8) (2018-06-15) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.7...v3.3.8) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### Improved - Improve [slow request apply warning log](https://github.com/etcd-io/etcd/pull/9288). - e.g. `read-only range request "key:\"/a\" range_end:\"/b\" " with result "range_response_count:3 size:96" took too long (97.966µs) to execute`. - Redact [request value field](https://github.com/etcd-io/etcd/pull/9822). - Provide [response size](https://github.com/etcd-io/etcd/pull/9826). - Add [backoff on watch retries on transient errors](https://github.com/etcd-io/etcd/pull/9840). ### Go - Compile with [*Go 1.9.7*](https://golang.org/doc/devel/release.html#go1.9). --- ## [v3.3.7](https://github.com/etcd-io/etcd/releases/tag/v3.3.7) (2018-06-06) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.6...v3.3.7) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### Security, Authentication - Support TLS cipher suite whitelisting. - To block [weak cipher suites](https://github.com/etcd-io/etcd/issues/8320). - TLS handshake fails when client hello is requested with invalid cipher suites. - Add [`etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/9801) flag. - If empty, Go auto-populates the list. ### etcdctl v3 - Fix [`etcdctl move-leader` command for TLS-enabled endpoints](https://github.com/etcd-io/etcd/pull/9807). ### Go - Compile with [*Go 1.9.6*](https://golang.org/doc/devel/release.html#go1.9). --- ## [v3.3.6](https://github.com/etcd-io/etcd/releases/tag/v3.3.6) (2018-05-31) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.5...v3.3.6) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### etcd server - Allow [empty auth token](https://github.com/etcd-io/etcd/pull/9369). - Previously, when auth token is an empty string, it returns [`failed to initialize the etcd server: auth: invalid auth options` error](https://github.com/etcd-io/etcd/issues/9349). - Fix [auth storage panic on server lease revoke routine with JWT token](https://github.com/etcd-io/etcd/issues/9695). - Fix [`mvcc` server panic from restore operation](https://github.com/etcd-io/etcd/pull/9775). - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X, **etcd server panicked** during snapshot restore operation. - Now, this server-side panic has been fixed. ### Go - Compile with [*Go 1.9.6*](https://golang.org/doc/devel/release.html#go1.9). --- ## [v3.3.5](https://github.com/etcd-io/etcd/releases/tag/v3.3.5) (2018-05-09) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.4...v3.3.5) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### etcdctl v3 - Fix [`etcdctl watch [key] [range_end] -- [exec-command…]`](https://github.com/etcd-io/etcd/pull/9688) parsing. - Previously, `ETCDCTL_API=3 ./bin/etcdctl watch foo -- echo watch event received` panicked. ### Go - Compile with [*Go 1.9.6*](https://golang.org/doc/devel/release.html#go1.9). --- ## [v3.3.4](https://github.com/etcd-io/etcd/releases/tag/v3.3.4) (2018-04-24) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.3...v3.3.4) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_server_is_leader`](https://github.com/etcd-io/etcd/pull/9587) Prometheus metric. - Fix [`etcd_debugging_server_lease_expired_total`](https://github.com/etcd-io/etcd/pull/9557) Prometheus metric. - Fix [race conditions in v2 server stat collecting](https://github.com/etcd-io/etcd/pull/9562). ### Security, Authentication - Fix [TLS reload](https://github.com/etcd-io/etcd/pull/9570) when [certificate SAN field only includes IP addresses but no domain names](https://github.com/etcd-io/etcd/issues/9541). - In Go, server calls `(*tls.Config).GetCertificate` for TLS reload if and only if server's `(*tls.Config).Certificates` field is not empty, or `(*tls.ClientHelloInfo).ServerName` is not empty with a valid SNI from the client. Previously, etcd always populates `(*tls.Config).Certificates` on the initial client TLS handshake, as non-empty. Thus, client was always expected to supply a matching SNI in order to pass the TLS verification and to trigger `(*tls.Config).GetCertificate` to reload TLS assets. - However, a certificate whose SAN field does [not include any domain names but only IP addresses](https://github.com/etcd-io/etcd/issues/9541) would request `*tls.ClientHelloInfo` with an empty `ServerName` field, thus failing to trigger the TLS reload on initial TLS handshake; this becomes a problem when expired certificates need to be replaced online. - Now, `(*tls.Config).Certificates` is created empty on initial TLS client handshake, first to trigger `(*tls.Config).GetCertificate`, and then to populate rest of the certificates on every new TLS connection, even when client SNI is empty (e.g. cert only includes IPs). ### etcd server - Add [`etcd --initial-election-tick-advance`](https://github.com/etcd-io/etcd/pull/9591) flag to configure initial election tick fast-forward. - By default, `etcd --initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger. - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election. - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout. - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities. - Now, this can be disabled by setting `--initial-election-tick-advance=false`. - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `etcd --initial-election-tick-advance` at the cost of slow initial bootstrap. - If single-node, it advances ticks regardless. - Address [disruptive rejoining follower node](https://github.com/etcd-io/etcd/issues/9333). ### Package `embed` - Add [`embed.Config.InitialElectionTickAdvance`](https://github.com/etcd-io/etcd/pull/9591) to enable/disable initial election tick fast-forward. - `embed.NewConfig()` would return `*embed.Config` with `InitialElectionTickAdvance` as true by default. ### Go - Compile with [*Go 1.9.5*](https://golang.org/doc/devel/release.html#go1.9). --- ## [v3.3.3](https://github.com/etcd-io/etcd/releases/tag/v3.3.3) (2018-03-29) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.2...v3.3.3) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### Improved - Adjust [election timeout on server restart](https://github.com/etcd-io/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/etcd-io/etcd/issues/9333). - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node. - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart. - Adjust [periodic compaction retention window](https://github.com/etcd-io/etcd/pull/9485). - e.g. `etcd --auto-compaction-mode=revision --auto-compaction-retention=1000` automatically `Compact` on `"latest revision" - 1000` every 5-minute (when latest revision is 30000, compact on revision 29000). - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=72h` automatically `Compact` with 72-hour retention windown for every 7.2-hour. **Now, `Compact` happens, for every 1-hour but still with 72-hour retention window.** - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` automatically `Compact` with 30-minute retention windown for every 3-minute. **Now, `Compact` happens, for every 30-minute but still with 30-minute retention window.** - Periodic compactor keeps recording latest revisions for every compaction period when given period is less than 1-hour, or for every 1-hour when given compaction period is greater than 1-hour (e.g. 1-hour when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`). - For every compaction period or 1-hour, compactor uses the last revision that was fetched before compaction period, to discard historical data. - The retention window of compaction period moves for every given compaction period or hour. - For instance, when hourly writes are 100 and `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`, `v3.2.x`, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 2400, 2640, and 2880 for every 2.4-hour, while `v3.3.3` *or later* compacts revision 2400, 2500, 2600 for every 1-hour. - Furthermore, when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` and writes per minute are about 1000, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 30000, 33000, and 36000, for every 3-minute, while `v3.3.3` *or later* compacts revision 30000, 60000, and 90000, for every 30-minute. ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add missing [`etcd_network_peer_sent_failures_total` count](https://github.com/etcd-io/etcd/pull/9437). ### Go - Compile with [*Go 1.9.5*](https://golang.org/doc/devel/release.html#go1.9). --- ## [v3.3.2](https://github.com/etcd-io/etcd/releases/tag/v3.3.2) (2018-03-08) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.1...v3.3.2) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### etcd server - Fix [server panic on invalid Election Proclaim/Resign HTTP(S) requests](https://github.com/etcd-io/etcd/pull/9379). - Previously, wrong-formatted HTTP requests to Election API could trigger panic in etcd server. - e.g. `curl -L http://localhost:2379/v3/election/proclaim -X POST -d '{"value":""}'`, `curl -L http://localhost:2379/v3/election/resign -X POST -d '{"value":""}'`. - Fix [revision-based compaction retention parsing](https://github.com/etcd-io/etcd/pull/9339). - Previously, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` was [translated to revision retention 3600000000000](https://github.com/etcd-io/etcd/issues/9337). - Now, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` is correctly parsed as revision retention 1. - Prevent [overflow by large `TTL` values for `Lease` `Grant`](https://github.com/etcd-io/etcd/pull/9399). - `TTL` parameter to `Grant` request is unit of second. - Leases with too large `TTL` values exceeding `math.MaxInt64` [expire in unexpected ways](https://github.com/etcd-io/etcd/issues/9374). - Server now returns `rpctypes.ErrLeaseTTLTooLarge` to client, when the requested `TTL` is larger than *9,000,000,000 seconds* (which is >285 years). - Again, etcd `Lease` is meant for short-periodic keepalives or sessions, in the range of seconds or minutes. Not for hours or days! - Enable etcd server [`raft.Config.CheckQuorum` when starting with `ForceNewCluster`](https://github.com/etcd-io/etcd/pull/9347). ### Proxy v2 - Fix [v2 proxy leaky HTTP requests](https://github.com/etcd-io/etcd/pull/9336). ### Go - Compile with [*Go 1.9.4*](https://golang.org/doc/devel/release.html#go1.9). --- ## [v3.3.1](https://github.com/etcd-io/etcd/releases/tag/v3.3.1) (2018-02-12) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0...v3.3.1) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### Improved - Add [warnings on requests taking too long](https://github.com/etcd-io/etcd/pull/9288). - e.g. `etcdserver: read-only range request "key:\"\\000\" range_end:\"\\000\" " took too long [3.389041388s] to execute` ### etcd server - Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/etcd-io/etcd/pull/9281). - "unsynced" watcher is watcher that needs to be in sync with events that have happened. - That is, "unsynced" watcher is the slow watcher that was requested on old revision. - "unsynced" watcher restore operation was not correctly populating its underlying watcher group. - Which possibly causes [missing events from "unsynced" watchers](https://github.com/etcd-io/etcd/issues/9086). - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node. ### Go - Compile with [*Go 1.9.4*](https://golang.org/doc/devel/release.html#go1.9). --- ## [v3.3.0](https://github.com/etcd-io/etcd/releases/tag/v3.3.0) (2018-02-01) See [code changes](https://github.com/etcd-io/etcd/compare/v3.2.0...v3.3.0) and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/) for any breaking changes. - [v3.3.0](https://github.com/etcd-io/etcd/releases/tag/v3.3.0) (2018-02-01), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0-rc.4...v3.3.0). - [v3.3.0-rc.4](https://github.com/etcd-io/etcd/releases/tag/v3.3.0-rc.4) (2018-01-22), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0-rc.3...v3.3.0-rc.4). - [v3.3.0-rc.3](https://github.com/etcd-io/etcd/releases/tag/v3.3.0-rc.3) (2018-01-17), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0-rc.2...v3.3.0-rc.3). - [v3.3.0-rc.2](https://github.com/etcd-io/etcd/releases/tag/v3.3.0-rc.2) (2018-01-11), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0-rc.1...v3.3.0-rc.2). - [v3.3.0-rc.1](https://github.com/etcd-io/etcd/releases/tag/v3.3.0-rc.1) (2018-01-02), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0-rc.0...v3.3.0-rc.1). - [v3.3.0-rc.0](https://github.com/etcd-io/etcd/releases/tag/v3.3.0-rc.0) (2017-12-20), see [code changes](https://github.com/etcd-io/etcd/compare/v3.2.0...v3.3.0-rc.0). **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_3/).** ### Improved - Use [`coreos/bbolt`](https://github.com/coreos/bbolt/releases) to replace [`boltdb/bolt`](https://github.com/boltdb/bolt#project-status). - Fix [etcd database size grows until `mvcc: database space exceeded`](https://github.com/etcd-io/etcd/issues/8009). - [Support database size larger than 8GiB](https://github.com/etcd-io/etcd/pull/7525) (8GiB is now a suggested maximum size for normal environments) - [Reduce memory allocation](https://github.com/etcd-io/etcd/pull/8428) on [Range operations](https://github.com/etcd-io/etcd/pull/8475). - [Rate limit](https://github.com/etcd-io/etcd/pull/8099) and [randomize](https://github.com/etcd-io/etcd/pull/8101) lease revoke on restart or leader elections. - Prevent [spikes in Raft proposal rate](https://github.com/etcd-io/etcd/issues/8096). - Support `clientv3` balancer failover under [network faults/partitions](https://github.com/etcd-io/etcd/issues/8711). - Better warning on [mismatched `etcd --initial-cluster`](https://github.com/etcd-io/etcd/pull/8083) flag. - etcd compares `etcd --initial-advertise-peer-urls` against corresponding `etcd --initial-cluster` URLs with forward-lookup. - If resolved IP addresses of `etcd --initial-advertise-peer-urls` and `etcd --initial-cluster` do not match (e.g. [due to DNS error](https://github.com/etcd-io/etcd/pull/9210)), etcd will exit with errors. - v3.2 error: `etcd --initial-cluster must include s1=https://s1.test:2380 given --initial-advertise-peer-urls=https://s1.test:2380`. - v3.3 error: `failed to resolve https://s1.test:2380 to match --initial-cluster=s1=https://s1.test:2380 (failed to resolve "https://s1.test:2380" (error ...))`. ### Breaking Changes - Require [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) [**`v1.7.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.4) or [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5). - Deprecate [`metadata.Incoming/OutgoingContext`](https://github.com/etcd-io/etcd/pull/7896). - Deprecate `grpclog.Logger`, upgrade to [`grpclog.LoggerV2`](https://github.com/etcd-io/etcd/pull/8533). - Deprecate [`grpc.ErrClientConnTimeout`](https://github.com/etcd-io/etcd/pull/8505) errors in `clientv3`. - Use [`MaxRecvMsgSize` and `MaxSendMsgSize`](https://github.com/etcd-io/etcd/pull/8437) to limit message size, in etcd server. - Translate [gRPC status error in v3 client `Snapshot` API](https://github.com/etcd-io/etcd/pull/9038). - v3 `etcdctl` [`lease timetolive LEASE_ID`](https://github.com/etcd-io/etcd/issues/9028) on expired lease now prints [`"lease LEASE_ID already expired"`](https://github.com/etcd-io/etcd/pull/9047). - <=3.2 prints `"lease LEASE_ID granted with TTL(0s), remaining(-1s)"`. - Replace [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3alpha` with [`/v3beta`](https://github.com/etcd-io/etcd/pull/8880). - To deprecate [`/v3alpha`](https://github.com/etcd-io/etcd/issues/8125) in v3.4. - In v3.3, `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.4. Use `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. - Change `etcd --auto-compaction-retention` flag to [accept string values](https://github.com/etcd-io/etcd/pull/8563) with [finer granularity](https://github.com/etcd-io/etcd/issues/8503). - Now that `etcd --auto-compaction-retention` accepts string values, etcd configuration YAML file `auto-compaction-retention` field must be changed to `string` type. - Previously, `--config-file etcd.config.yaml` can have `auto-compaction-retention: 24` field, now must be `auto-compaction-retention: "24"` or `auto-compaction-retention: "24h"`. - If configured as `etcd --auto-compaction-mode periodic --auto-compaction-retention "24h"`, the time duration value for `etcd --auto-compaction-retention` flag must be valid for [`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration) function in Go. ### Dependency - Upgrade [`boltdb/bolt`](https://github.com/boltdb/bolt#project-status) from [**`v1.3.0`**](https://github.com/boltdb/bolt/releases/tag/v1.3.0) to [`coreos/bbolt`](https://github.com/coreos/bbolt/releases) [**`v1.3.1-coreos.6`**](https://github.com/coreos/bbolt/releases/tag/v1.3.1-coreos.6). - Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.2.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.2.1) to [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5). - Upgrade [`github.com/ugorji/go/codec`](https://github.com/ugorji/go) to [**`v1.1`**](https://github.com/ugorji/go/releases/tag/v1.1), and [regenerate v2 `client`](https://github.com/etcd-io/etcd/pull/8721). - Upgrade [`github.com/ugorji/go/codec`](https://github.com/ugorji/go) to [**`ugorji/go@54210f4e0`**](https://github.com/ugorji/go/commit/54210f4e076c57f351166f0ed60e67d3fca57a36), and [regenerate v2 `client`](https://github.com/etcd-io/etcd/pull/8574). - Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.2.2`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.2.2) to [**`v1.3.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3.0). - Upgrade [`golang.org/x/crypto/bcrypt`](https://github.com/golang/crypto) to [**`golang/crypto@6c586e17d`**](https://github.com/golang/crypto/commit/6c586e17d90a7d08bbbc4069984180dce3b04117). ### Metrics, Monitoring See [List of metrics](https://github.com/etcd-io/etcd/tree/main/Documentation/metrics) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd --listen-metrics-urls`](https://github.com/etcd-io/etcd/pull/8242) flag for additional `/metrics` and `/health` endpoints. - Useful for [bypassing critical APIs when monitoring etcd](https://github.com/etcd-io/etcd/issues/8060). - Add [`etcd_server_version`](https://github.com/etcd-io/etcd/pull/8960) Prometheus metric. - To replace [Kubernetes `etcd-version-monitor`](https://github.com/etcd-io/etcd/issues/8948). - Add [`etcd_debugging_mvcc_db_compaction_keys_total`](https://github.com/etcd-io/etcd/pull/8280) Prometheus metric. - Add [`etcd_debugging_server_lease_expired_total`](https://github.com/etcd-io/etcd/pull/8064) Prometheus metric. - To improve [lease revoke monitoring](https://github.com/etcd-io/etcd/issues/8050). - Document [Prometheus 2.0 rules](https://github.com/etcd-io/etcd/pull/8879). - Initialize gRPC server [metrics with zero values](https://github.com/etcd-io/etcd/pull/8878). - Fix [range/put/delete operation metrics](https://github.com/etcd-io/etcd/pull/8054) with transaction. - `etcd_debugging_mvcc_range_total` - `etcd_debugging_mvcc_put_total` - `etcd_debugging_mvcc_delete_total` - `etcd_debugging_mvcc_txn_total` - Fix [`etcd_debugging_mvcc_keys_total`](https://github.com/etcd-io/etcd/pull/8390) on restore. - Fix [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/8120) on restore. - Also change to [`prometheus.NewGaugeFunc`](https://github.com/etcd-io/etcd/pull/8150). ### Security, Authentication See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - Add [CRL based connection rejection](https://github.com/etcd-io/etcd/pull/8124) to manage [revoked certs](https://github.com/etcd-io/etcd/issues/4034). - Document [TLS authentication changes](https://github.com/etcd-io/etcd/pull/8895). - [Server accepts connections if IP matches, without checking DNS entries](https://github.com/etcd-io/etcd/pull/8223). For instance, if peer cert contains IP addresses and DNS names in Subject Alternative Name (SAN) field, and the remote IP address matches one of those IP addresses, server just accepts connection without further checking the DNS names. - [Server supports reverse-lookup on wildcard DNS `SAN`](https://github.com/etcd-io/etcd/pull/8281). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server first reverse-lookups the remote IP address to get a list of names mapping to that address (e.g. `nslookup IPADDR`). Then accepts the connection if those names have a matching name with peer cert's DNS names (either by exact or wildcard match). If none is matched, server forward-lookups each DNS entry in peer cert (e.g. look up `example.default.svc` when the entry is `*.example.default.svc`), and accepts connection only when the host's resolved addresses have the matching IP address with the peer's remote IP address. - Add [`etcd --peer-cert-allowed-cn`](https://github.com/etcd-io/etcd/pull/8616) flag. - To support [CommonName(CN) based auth](https://github.com/etcd-io/etcd/issues/8262) for inter peer connection. - [Swap priority](https://github.com/etcd-io/etcd/pull/8594) of cert CommonName(CN) and username + password. - To address ["username and password specified in the request should take priority over CN in the cert"](https://github.com/etcd-io/etcd/issues/8584). - Protect [lease revoke with auth](https://github.com/etcd-io/etcd/pull/8031). - Provide user's role on [auth permission error](https://github.com/etcd-io/etcd/pull/8164). - Fix [auth store panic with disabled token](https://github.com/etcd-io/etcd/pull/8695). ### etcd server - Add [`etcd --experimental-initial-corrupt-check`](https://github.com/etcd-io/etcd/pull/8554) flag to [check cluster database hashes before serving client/peer traffic](https://github.com/etcd-io/etcd/issues/8313). - `etcd --experimental-initial-corrupt-check=false` by default. - v3.4 will enable `--initial-corrupt-check=true` by default. - Add [`etcd --experimental-corrupt-check-time`](https://github.com/etcd-io/etcd/pull/8420) flag to [raise corrupt alarm monitoring](https://github.com/etcd-io/etcd/issues/7125). - `etcd --experimental-corrupt-check-time=0s` disabled by default. - Add [`etcd --experimental-enable-v2v3`](https://github.com/etcd-io/etcd/pull/8407) flag to [emulate v2 API with v3](https://github.com/etcd-io/etcd/issues/6925). - `etcd --experimental-enable-v2v3=false` by default. - Add [`etcd --max-txn-ops`](https://github.com/etcd-io/etcd/pull/7976) flag to [configure maximum number operations in transaction](https://github.com/etcd-io/etcd/issues/7826). - Add [`etcd --max-request-bytes`](https://github.com/etcd-io/etcd/pull/7968) flag to [configure maximum client request size](https://github.com/etcd-io/etcd/issues/7923). - If not configured, it defaults to 1.5 MiB. - Add [`etcd --client-crl-file`, `--peer-crl-file`](https://github.com/etcd-io/etcd/pull/8124) flags for [Certificate revocation list](https://github.com/etcd-io/etcd/issues/4034). - Add [`etcd --peer-cert-allowed-cn`](https://github.com/etcd-io/etcd/pull/8616) flag to support [CN-based auth for inter-peer connection](https://github.com/etcd-io/etcd/issues/8262). - Add [`etcd --listen-metrics-urls`](https://github.com/etcd-io/etcd/pull/8242) flag for additional `/metrics` and `/health` endpoints. - Support [additional (non) TLS `/metrics` endpoints for a TLS-enabled cluster](https://github.com/etcd-io/etcd/pull/8282). - e.g. `etcd --listen-metrics-urls=https://localhost:2378,http://localhost:9379` to serve `/metrics` and `/health` on secure port 2378 and insecure port 9379. - Useful for [bypassing critical APIs when monitoring etcd](https://github.com/etcd-io/etcd/issues/8060). - Add [`etcd --auto-compaction-mode`](https://github.com/etcd-io/etcd/pull/8123) flag to [support revision-based compaction](https://github.com/etcd-io/etcd/issues/8098). - Change `etcd --auto-compaction-retention` flag to [accept string values](https://github.com/etcd-io/etcd/pull/8563) with [finer granularity](https://github.com/etcd-io/etcd/issues/8503). - Now that `etcd --auto-compaction-retention` accepts string values, etcd configuration YAML file `auto-compaction-retention` field must be changed to `string` type. - Previously, `etcd --config-file etcd.config.yaml` can have `auto-compaction-retention: 24` field, now must be `auto-compaction-retention: "24"` or `auto-compaction-retention: "24h"`. - If configured as `--auto-compaction-mode periodic --auto-compaction-retention "24h"`, the time duration value for `etcd --auto-compaction-retention` flag must be valid for [`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration) function in Go. - e.g. `etcd --auto-compaction-mode=revision --auto-compaction-retention=1000` automatically `Compact` on `"latest revision" - 1000` every 5-minute (when latest revision is 30000, compact on revision 29000). - e.g. `etcd --auto-compaction-mode=periodic --auto-compaction-retention=72h` automatically `Compact` with 72-hour retention windown, for every 7.2-hour. - e.g. `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` automatically `Compact` with 30-minute retention windown, for every 3-minute. - Periodic compactor continues to record latest revisions for every 1/10 of given compaction period (e.g. 1-hour when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=10h`). - For every 1/10 of given compaction period, compactor uses the last revision that was fetched before compaction period, to discard historical data. - The retention window of compaction period moves for every 1/10 of given compaction period. - For instance, when hourly writes are 100 and `--auto-compaction-retention=10`, v3.1 compacts revision 1000, 2000, and 3000 for every 10-hour, while v3.2.x, v3.3.0, v3.3.1, and v3.3.2 compact revision 1000, 1100, and 1200 for every 1-hour. Furthermore, when writes per minute are 1000, v3.3.0, v3.3.1, and v3.3.2 with `--auto-compaction-mode=periodic --auto-compaction-retention=30m` compact revision 30000, 33000, and 36000, for every 3-minute with more finer granularity. - Whether compaction succeeds or not, this process repeats for every 1/10 of given compaction period. If compaction succeeds, it just removes compacted revision from historical revision records. - Add [`etcd --grpc-keepalive-min-time`, `etcd --grpc-keepalive-interval`, `etcd --grpc-keepalive-timeout`](https://github.com/etcd-io/etcd/pull/8535) flags to configure server-side keepalive policies. - Serve [`/health` endpoint as unhealthy](https://github.com/etcd-io/etcd/pull/8272) when [alarm (e.g. `NOSPACE`) is raised or there's no leader](https://github.com/etcd-io/etcd/issues/8207). - Define [`etcdhttp.Health`](https://godoc.org/github.com/coreos/etcd/etcdserver/api/etcdhttp#Health) struct with JSON encoder. - Note that `"health"` field is [`string` type, not `bool`](https://github.com/etcd-io/etcd/pull/9143). - e.g. `{"health":"false"}`, `{"health":"true"}` - [Remove `"errors"` field](https://github.com/etcd-io/etcd/pull/9162) since `v3.3.0-rc.3` (did exist only in `v3.3.0-rc.0`, `v3.3.0-rc.1`, `v3.3.0-rc.2`). - Move [logging setup to embed package](https://github.com/etcd-io/etcd/pull/8810) - Disable gRPC server info-level logs by default (can be enabled with `etcd --debug` flag). - Use [monotonic time in Go 1.9](https://github.com/etcd-io/etcd/pull/8507) for `lease` package. - Warn on [empty hosts in advertise URLs](https://github.com/etcd-io/etcd/pull/8384). - Address [advertise client URLs accepts empty hosts](https://github.com/etcd-io/etcd/issues/8379). - etcd v3.4 will exit on this error. - e.g. `etcd --advertise-client-urls=http://:2379`. - Warn on [shadowed environment variables](https://github.com/etcd-io/etcd/pull/8385). - Address [error on shadowed environment variables](https://github.com/etcd-io/etcd/issues/8380). - etcd v3.4 will exit on this error. ### API - Support [ranges in transaction comparisons](https://github.com/etcd-io/etcd/pull/8025) for [disconnected linearized reads](https://github.com/etcd-io/etcd/issues/7924). - Add [nested transactions](https://github.com/etcd-io/etcd/pull/8102) to extend [proxy use cases](https://github.com/etcd-io/etcd/issues/7857). - Add [lease comparison target in transaction](https://github.com/etcd-io/etcd/pull/8324). - Add [lease list](https://github.com/etcd-io/etcd/pull/8358). - Add [hash by revision](https://github.com/etcd-io/etcd/pull/8263) for [better corruption checking against boltdb](https://github.com/etcd-io/etcd/issues/8016). ### client v3 - Add [health balancer](https://github.com/etcd-io/etcd/pull/8545) to fix [watch API hangs](https://github.com/etcd-io/etcd/issues/7247), improve [endpoint switch under network faults](https://github.com/etcd-io/etcd/issues/7941). - [Refactor balancer](https://github.com/etcd-io/etcd/pull/8840) and add [client-side keepalive pings](https://github.com/etcd-io/etcd/pull/8199) to handle [network partitions](https://github.com/etcd-io/etcd/issues/8711). - Add [`MaxCallSendMsgSize` and `MaxCallRecvMsgSize`](https://github.com/etcd-io/etcd/pull/9047) fields to [`clientv3.Config`](https://godoc.org/github.com/coreos/etcd/clientv3#Config). - Fix [exceeded response size limit error in client-side](https://github.com/etcd-io/etcd/issues/9043). - Address [kubernetes#51099](https://github.com/kubernetes/kubernetes/issues/51099). - In previous versions(v3.2.10, v3.2.11), client response size was limited to only 4 MiB. - `MaxCallSendMsgSize` default value is 2 MiB, if not configured. - `MaxCallRecvMsgSize` default value is `math.MaxInt32`, if not configured. - Accept [`Compare_LEASE`](https://github.com/etcd-io/etcd/pull/8324) in [`clientv3.Compare`](https://godoc.org/github.com/coreos/etcd/clientv3#Compare). - Add [`LeaseValue` helper](https://github.com/etcd-io/etcd/pull/8488) to `Cmp` `LeaseID` values in `Txn`. - Add [`MoveLeader`](https://github.com/etcd-io/etcd/pull/8153) to `Maintenance`. - Add [`HashKV`](https://github.com/etcd-io/etcd/pull/8351) to `Maintenance`. - Add [`Leases`](https://github.com/etcd-io/etcd/pull/8358) to `Lease`. - Add [`clientv3/ordering`](https://github.com/etcd-io/etcd/pull/8092) for enforce [ordering in serialized requests](https://github.com/etcd-io/etcd/issues/7623). - Fix ["put at-most-once" violation](https://github.com/etcd-io/etcd/pull/8335). - Fix [`WatchResponse.Canceled`](https://github.com/etcd-io/etcd/pull/8283) on [compacted watch request](https://github.com/etcd-io/etcd/issues/8231). - Fix [`concurrency/stm` `Put` with serializable snapshot](https://github.com/etcd-io/etcd/pull/8439). - Use store revision from first fetch to resolve write conflicts instead of modified revision. ### etcdctl v3 - Add [`etcdctl --discovery-srv`](https://github.com/etcd-io/etcd/pull/8462) flag. - Add [`etcdctl --keepalive-time`, `--keepalive-timeout`](https://github.com/etcd-io/etcd/pull/8663) flags. - Add [`etcdctl lease list`](https://github.com/etcd-io/etcd/pull/8358) command. - Add [`etcdctl lease keep-alive --once`](https://github.com/etcd-io/etcd/pull/8775) flag. - Make [`lease timetolive LEASE_ID`](https://github.com/etcd-io/etcd/issues/9028) on expired lease print [`lease LEASE_ID already expired`](https://github.com/etcd-io/etcd/pull/9047). - <=3.2 prints `lease LEASE_ID granted with TTL(0s), remaining(-1s)`. - Add [`etcdctl snapshot restore --wal-dir`](https://github.com/etcd-io/etcd/pull/9124) flag. - Add [`etcdctl defrag --data-dir`](https://github.com/etcd-io/etcd/pull/8367) flag. - Add [`etcdctl move-leader`](https://github.com/etcd-io/etcd/pull/8153) command. - Add [`etcdctl endpoint hashkv`](https://github.com/etcd-io/etcd/pull/8351) command. - Add [`etcdctl endpoint --cluster`](https://github.com/etcd-io/etcd/pull/8143) flag, equivalent to [v2 `etcdctl cluster-health`](https://github.com/etcd-io/etcd/issues/8117). - Make `etcdctl endpoint health` command terminate with [non-zero exit code on unhealthy status](https://github.com/etcd-io/etcd/pull/8342). - Add [`etcdctl lock --ttl`](https://github.com/etcd-io/etcd/pull/8370) flag. - Support [`etcdctl watch [key] [range_end] -- [exec-command…]`](https://github.com/etcd-io/etcd/pull/8919), equivalent to [v2 `etcdctl exec-watch`](https://github.com/etcd-io/etcd/issues/8814). - Make `etcdctl watch -- [exec-command]` set environmental variables [`ETCD_WATCH_REVISION`, `ETCD_WATCH_EVENT_TYPE`, `ETCD_WATCH_KEY`, `ETCD_WATCH_VALUE`](https://github.com/etcd-io/etcd/pull/9142) for each event. - Support [`etcdctl watch` with environmental variables `ETCDCTL_WATCH_KEY` and `ETCDCTL_WATCH_RANGE_END`](https://github.com/etcd-io/etcd/pull/9142). - Enable [`clientv3.WithRequireLeader(context.Context)` for `watch`](https://github.com/etcd-io/etcd/pull/8672) command. - Print [`"del"` instead of `"delete"`](https://github.com/etcd-io/etcd/pull/8297) in `txn` interactive mode. - Print [`ETCD_INITIAL_ADVERTISE_PEER_URLS` in `member add`](https://github.com/etcd-io/etcd/pull/8332). - Fix [`etcdctl snapshot status` to not modify snapshot file](https://github.com/etcd-io/etcd/pull/8815). - For example, start etcd `v3.3.10` - Write some data - Use etcdctl `v3.3.10` to save snapshot - Somehow, upgrading Kubernetes fails, thus rolling back to previous version etcd `v3.2.24` - Run etcdctl `v3.2.24` `snapshot status` against the snapshot file saved from `v3.3.10` server - Run etcdctl `v3.2.24` `snapshot restore` fails with `"expected sha256 [12..."` ### etcdctl v3 - Handle [empty key permission](https://github.com/etcd-io/etcd/pull/8514) in `etcdctl`. ### etcdctl v2 - Add [`etcdctl backup --with-v3`](https://github.com/etcd-io/etcd/pull/8479) flag. ### gRPC Proxy - Add [`grpc-proxy start --experimental-leasing-prefix`](https://github.com/etcd-io/etcd/pull/8341) flag. - For disconnected linearized reads. - Based on [V system leasing](https://github.com/etcd-io/etcd/issues/6065). - See ["Disconnected consistent reads with etcd" blog post](https://coreos.com/blog/coreos-labs-disconnected-consistent-reads-with-etcd). - Add [`grpc-proxy start --experimental-serializable-ordering`](https://github.com/etcd-io/etcd/pull/8315) flag. - To ensure serializable reads have monotonically increasing store revisions across endpoints. - Add [`grpc-proxy start --metrics-addr`](https://github.com/etcd-io/etcd/pull/8242) flag for an additional `/metrics` endpoint. - Set `--metrics-addr=http://[HOST]:9379` to serve `/metrics` in insecure port 9379. - Serve [`/health` endpoint in grpc-proxy](https://github.com/etcd-io/etcd/pull/8322). - Add [`grpc-proxy start --debug`](https://github.com/etcd-io/etcd/pull/8994) flag. - Add [`grpc-proxy start --max-send-bytes`](https://github.com/etcd-io/etcd/pull/9250) flag to [configure maximum client request size](https://github.com/etcd-io/etcd/issues/7923). - Add [`grpc-proxy start --max-recv-bytes`](https://github.com/etcd-io/etcd/pull/9250) flag to [configure maximum client request size](https://github.com/etcd-io/etcd/issues/7923). - Fix [Snapshot API error handling](https://github.com/etcd-io/etcd/commit/dbd16d52fbf81e5fd806d21ff5e9148d5bf203ab). - Fix [KV API `PrevKv` flag handling](https://github.com/etcd-io/etcd/pull/8366). - Fix [KV API `KeysOnly` flag handling](https://github.com/etcd-io/etcd/pull/8552). ### gRPC gateway - Replace [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3alpha` with [`/v3beta`](https://github.com/etcd-io/etcd/pull/8880). - To deprecate [`/v3alpha`](https://github.com/etcd-io/etcd/issues/8125) in v3.4. - In v3.3, `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.4. Use `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. - Support ["authorization" token](https://github.com/etcd-io/etcd/pull/7999). - Support [websocket for bi-directional streams](https://github.com/etcd-io/etcd/pull/8257). - Fix [`Watch` API with gRPC gateway](https://github.com/etcd-io/etcd/issues/8237). - Upgrade gRPC gateway to [v1.3.0](https://github.com/etcd-io/etcd/issues/8838). ### etcd server - Fix [backend database in-memory index corruption](https://github.com/etcd-io/etcd/pull/8127) issue on restore (only 3.2.0 is affected). - Fix [watch restore from snapshot](https://github.com/etcd-io/etcd/pull/8427). - Fix [`mvcc/backend.defragdb` nil-pointer dereference on create bucket failure](https://github.com/etcd-io/etcd/pull/9119). - Fix [server crash](https://github.com/etcd-io/etcd/pull/8010) on [invalid transaction request from gRPC gateway](https://github.com/etcd-io/etcd/issues/7889). - Prevent [server panic from member update/add](https://github.com/etcd-io/etcd/pull/9174) with [wrong scheme URLs](https://github.com/etcd-io/etcd/issues/9173). - Make [peer dial timeout longer](https://github.com/etcd-io/etcd/pull/8599). - See [coreos/etcd-operator#1300](https://github.com/etcd-io/etcd-operator/issues/1300) for more detail. - Make server [wait up to request time-out](https://github.com/etcd-io/etcd/pull/8267) with [pending RPCs](https://github.com/etcd-io/etcd/issues/8224). - Fix [`grpc.Server` panic on `GracefulStop`](https://github.com/etcd-io/etcd/pull/8987) with [TLS-enabled server](https://github.com/etcd-io/etcd/issues/8916). - Fix ["multiple peer URLs cannot start" issue](https://github.com/etcd-io/etcd/issues/8383). - Fix server-side auth so [concurrent auth operations do not return old revision error](https://github.com/etcd-io/etcd/pull/8442). - Handle [WAL renaming failure on Windows](https://github.com/etcd-io/etcd/pull/8286). - Upgrade [`coreos/go-systemd`](https://github.com/coreos/go-systemd/releases) to `v15` (see https://github.com/coreos/go-systemd/releases/tag/v15). - [Put back `/v2/machines`](https://github.com/etcd-io/etcd/pull/8062) endpoint for python-etcd wrapper. ### client v2 - [Fail-over v2 client](https://github.com/etcd-io/etcd/pull/8519) to next endpoint on [oneshot failure](https://github.com/etcd-io/etcd/issues/8515). ### Package `raft` - Add [non-voting member](https://github.com/etcd-io/etcd/pull/8751). - To implement [Raft thesis 4.2.1 Catching up new servers](https://github.com/etcd-io/etcd/issues/8568). - `Learner` node does not vote or promote itself. ### Other - Support previous two minor versions (see our [new release policy](https://github.com/etcd-io/etcd/pull/8805)). - `v3.3.x` is the last release cycle that supports `ACI`. - [AppC was officially suspended](https://github.com/appc/spec#-disclaimer-), as of late 2016. - [`acbuild`](https://github.com/containers/build#this-project-is-currently-unmaintained) is not maintained anymore. - `*.aci` files won't be available from etcd v3.4 release. - Add container registry [`gcr.io/etcd-development/etcd`](https://gcr.io/etcd-development/etcd). - [quay.io/coreos/etcd](https://quay.io/coreos/etcd) is still supported as secondary. ### Go - Require [*Go 1.9+*](https://github.com/etcd-io/etcd/issues/6174). - Compile with [*Go 1.9.3*](https://golang.org/doc/devel/release.html#go1.9). - Deprecate [`golang.org/x/net/context`](https://github.com/etcd-io/etcd/pull/8511). --- ================================================ FILE: CHANGELOG/CHANGELOG-3.4.md ================================================ Previous change logs can be found at [CHANGELOG-3.3](https://github.com/etcd-io/etcd/blob/main/CHANGELOG/CHANGELOG-3.3.md). --- ## v3.4.42 (TBC) ### etcd server - Fix [Race between read index and leader change](https://github.com/etcd-io/etcd/pull/21385) - Fix [Stale reads caused by process pausing](https://github.com/etcd-io/etcd/pull/21423) ### Dependencies - Compile binaries using [go 1.25.7](https://github.com/etcd-io/etcd/pull/21406) - [Bump golang.org/x/net to v0.51.0 to resolve GO-2026-4559](https://github.com/etcd-io/etcd/pull/21444) --- ## v3.4.41 (2026-02-13) ### Package `clientv3` - [Remove the use of grpc-go's Metadata field](https://github.com/etcd-io/etcd/pull/21243) ### Dependencies - Compile binaries using [go 1.24.13](https://github.com/etcd-io/etcd/pull/21266). This addresses [CVE-2025-61726](https://github.com/advisories/GHSA-gm9r-q53w-2gh4), [CVE-2025-61731](https://github.com/advisories/GHSA-xvqr-69v8-f3gv), and [CVE-2025-61732](https://github.com/advisories/GHSA-8jvr-vh7g-f8gx). --- ## v3.4.40 (2025-12-17) ### etcd server - [Print token fingerprint instead of the original tokens in log messages](https://github.com/etcd-io/etcd/pull/20943) ### Dependencies - [Scripts/build-binary.sh: use `buildvcs=false` to avoid having a pseudo-version reported by `go version`](https://github.com/etcd-io/etcd/pull/20950) - Compile binaries using [go 1.24.11](https://github.com/etcd-io/etcd/pull/21000). - [Use buildvcs=false in release script](https://github.com/etcd-io/etcd/pull/21028) - Bump [golang.org/x/crypto to 0.45.0 to address CVE-2025-47914, and CVE-2025-58181](https://github.com/etcd-io/etcd/pull/21022). --- ## v3.4.39 (2025-11-11) ### Dependencies - [Compile binaries with `buildvcs=false` to avoid having a pseudo-version reported by `go version`](https://github.com/etcd-io/etcd/pull/20847). - Compile binaries using [go 1.24.10](https://github.com/etcd-io/etcd/pull/20903). --- ## v3.4.38 (2025-10-21) ### etcd server - Fix [mvcc: avoid double decrement of watcher gauge on close/cancel race](https://github.com/etcd-io/etcd/pull/20065) - Fix [Watch on future revision returns old events or notifications](https://github.com/etcd-io/etcd/pull/20291) - Improve [help message for --quota-backend-bytes](https://github.com/etcd-io/etcd/pull/20379) - Fix [potential data corruption when applySnapshot and defragment happen concurrently](https://github.com/etcd-io/etcd/pull/20659) - [Reject watch request with -1 revision to prevent invalid resync behavior on uncompacted etcd](https://github.com/etcd-io/etcd/pull/20711) - Fix [etcd may return success for leaseRenew request even when the lease is revoked](https://github.com/etcd-io/etcd/pull/20813) ### Dependencies - Compile binaries using [go 1.24.9](https://github.com/etcd-io/etcd/pull/20807). - [Bump bbolt to v1.3.12](https://github.com/etcd-io/etcd/pull/20515). --- ## v3.4.37 (2025-04-15) ### Dependencies - Bump [golang.org/x/net to v0.36.0 to address CVE-2025-22870](https://github.com/etcd-io/etcd/pull/19529). - Compile binaries using [go 1.23.8](https://github.com/etcd-io/etcd/pull/19726) --- ## v3.4.36 (2025-02-25) ### etcd server - [Avoid deadlock in etcd.Close when stopping during bootstrapping](https://github.com/etcd-io/etcd/pull/19166) - Fix [missing delete event on watch opened on same revision as compaction request](https://github.com/etcd-io/etcd/pull/19251) ### Package `clientv3` - Fix [runtime panic that occurs when KeepAlive is called with a Context implemented by an uncomparable type](https://github.com/etcd-io/etcd/pull/18936) ### Dependencies - Compile binaries using [go 1.23.6](https://github.com/etcd-io/etcd/pull/19429) - Bump golang.org/x/crypto to v0.35.0 to address [CVE-2024-45337](https://github.com/etcd-io/etcd/pull/19197) and [CVE-2025-22869](https://github.com/etcd-io/etcd/pull/19477). - Bump golang.org/x/net to v0.34.0 to address [CVE-2024-45338](https://github.com/etcd-io/etcd/pull/19197). --- ## v3.4.35 (2024-11-12) ### etcd server - Fix [watchserver related goroutine leakage](https://github.com/etcd-io/etcd/pull/18785) - Fix [panicking occurred due to improper error handling during defragmentation](https://github.com/etcd-io/etcd/pull/18843) - Fix [close temp file(s) in case an error happens during defragmentation](https://github.com/etcd-io/etcd/pull/18855) ### Dependencies - Compile binaries using [go 1.22.9](https://github.com/etcd-io/etcd/pull/18850). --- ## v3.4.34 (2024-09-11) ### etcd server - Fix [performance regression issue caused by the `ensureLeadership` in lease renew](https://github.com/etcd-io/etcd/pull/18440). - [Keep the tombstone during compaction if it happens to be the compaction revision](https://github.com/etcd-io/etcd/pull/18475) ### Package clientv3 - [Print gRPC metadata in guaranteed order using the official go fmt pkg](https://github.com/etcd-io/etcd/pull/18311). ### Dependencies - Compile binaries using [go 1.22.7](https://github.com/etcd-io/etcd/pull/18549). - Upgrade [bbolt to 1.3.11](https://github.com/etcd-io/etcd/pull/18488). --- ## v3.4.33 (2024-06-13) ### etcd grpc-proxy - Fix [Memberlist results not updated when proxy node down](https://github.com/etcd-io/etcd/pull/17896). ### Dependencies - Compile binaries using go [1.21.11](https://github.com/etcd-io/etcd/pull/18130). - Upgrade [bbolt to 1.3.10](https://github.com/etcd-io/etcd/pull/17945). --- ## v3.4.32 (2024-04-25) ### etcd server - Fix [LeaseTimeToLive returns error if leader changed](https://github.com/etcd-io/etcd/pull/17705). - Fix [ignore raft messages if member id mismatch](https://github.com/etcd-io/etcd/pull/17814). - Update [the compaction log when bootstrap](https://github.com/etcd-io/etcd/pull/17831). - [Allow new server to join 3.5 cluster if `next-cluster-version-compatible=true`](https://github.com/etcd-io/etcd/pull/17665) - [Allow updating the cluster version when downgrading from 3.5](https://github.com/etcd-io/etcd/pull/17821). - Fix [Revision decreasing after panic during compaction](https://github.com/etcd-io/etcd/pull/17864) ### Package `clientv3` - Add [requests retry when receiving ErrGPRCNotSupportedForLearner and endpoints > 1](https://github.com/etcd-io/etcd/pull/17692). - Fix [initialization for epMu in client context](https://github.com/etcd-io/etcd/pull/17714). ### Dependencies - Compile binaries using [go 1.21.9](https://github.com/etcd-io/etcd/pull/17709). --- ## v3.4.31 (2024-03-21) ### etcd server - Add [mvcc: print backend database size and size in use in compaction logs](https://github.com/etcd-io/etcd/pull/17436). - Fix leases wrongly revoked by the leader by [ignoring old leader's leases revoking request](https://github.com/etcd-io/etcd/pull/17465). - Fix [no progress notification being sent for watch that doesn't get any events](https://github.com/etcd-io/etcd/pull/17567). - Fix [watch event loss after compaction](https://github.com/etcd-io/etcd/pull/17610). - Add `next-cluster-version-compatible` flag to [allow downgrade from 3.5](https://github.com/etcd-io/etcd/pull/17330). ### Package `clientv3` - Add [client backoff and retry config options](https://github.com/etcd-io/etcd/pull/17369). ### Dependencies - Upgrade [bbolt to 1.3.9](https://github.com/etcd-io/etcd/pull/17484). - Compile binaries using [go 1.21.8](https://github.com/etcd-io/etcd/pull/17538). - Upgrade [google.golang.org/protobuf to v1.33.0 to address CVE-2024-24786](https://github.com/etcd-io/etcd/pull/17554). - Upgrade github.com/sirupsen/logrus to v1.9.3 to address [PRISMA-2023-0056](https://github.com/etcd-io/etcd/pull/17580). ### Others - [Make CGO_ENABLED configurable](https://github.com/etcd-io/etcd/pull/17422). --- ## v3.4.30 (2024-01-31) ### etcd server - Fix [nil pointer panicking due to using the wrong log library](https://github.com/etcd-io/etcd/pull/17270) ### Dependencies - Compile binaries using go [1.20.13](https://github.com/etcd-io/etcd/pull/17276). - Upgrade [golang.org/x/crypto to v0.17+ to address CVE-2023-48795](https://github.com/etcd-io/etcd/pull/17347). --- ## v3.4.29 (2024-01-09) ### etcd server - [Disable following HTTP redirects in peer communication](https://github.com/etcd-io/etcd/pull/17112) - [Add livez/readyz HTTP endpoints](https://github.com/etcd-io/etcd/pull/17128) - Fix [Check if be is nil to avoid panic when be is overriden with nil](https://github.com/etcd-io/etcd/pull/17154) - Fix [Add missing experimental-enable-lease-checkpoint-persist flag in etcd help](https://github.com/etcd-io/etcd/pull/17189) - Fix [Don't flock snapshot files](https://github.com/etcd-io/etcd/pull/17208) ### Dependencies - Compile binaries using go [1.20.12](https://github.com/etcd-io/etcd/pull/17076). --- ## v3.4.28 (2023-11-23) ### etcd server - Improve [Skip getting authInfo from incoming context when auth is disabled](https://github.com/etcd-io/etcd/pull/16240) - Use [the default write scheduler](https://github.com/etcd-io/etcd/pull/16782) since golang.org/x/net@v0.11.0 started using round-robin scheduler. - Add [cluster ID check during data corruption detection to prevent false alarm](https://github.com/etcd-io/etcd/issues/15548). - Add [Learner support Snapshot RPC](https://github.com/etcd-io/etcd/pull/16990/). ### Package `clientv3` - Fix [Reset auth token when failing to authenticate due to auth being disabled](https://github.com/etcd-io/etcd/pull/16240). - [Simplify grpc dialer usage](https://github.com/etcd-io/etcd/issues/11519). - [Replace balancer with upstream grpc solution](https://github.com/etcd-io/etcd/pull/16844). - Fix [race condition when accessing cfg.Endpoints in dial()](https://github.com/etcd-io/etcd/pull/16857). - Fix [invalid authority header issue in single endpoint scenario](https://github.com/etcd-io/etcd/pull/16988). ### Dependencies - Compile binaries using [go 1.20.11](https://github.com/etcd-io/etcd/pull/16916). - Upgrade [bbolt to 1.3.8](https://github.com/etcd-io/etcd/pull/16834). - Upgrade gRPC to 1.58.3 in https://github.com/etcd-io/etcd/pull/16997 and https://github.com/etcd-io/etcd/pull/16999. Note that gRPC server will reject requests with connection header (refer to https://github.com/grpc/grpc-go/pull/4803). --- ## v3.4.27 (2023-07-11) ### etcd server - Fix [corruption check may get a `ErrCompacted` error when server has just been compacted](https://github.com/etcd-io/etcd/pull/16047) - Improve [Lease put performance for the case that auth is disabled or the user is admin](https://github.com/etcd-io/etcd/pull/16020) - Fix [embed: nil pointer dereference when stopServer](https://github.com/etcd-io/etcd/pull/16195) ### etcdctl v3 - Add [optional --bump-revision and --mark-compacted flag to etcdctl snapshot restore operation](https://github.com/etcd-io/etcd/pull/16193). ### Dependencies - Compile binaries using [go 1.19.10](https://github.com/etcd-io/etcd/pull/16038). --- ## v3.4.26 (2023-05-12) ### etcd server - Fix [LeaseTimeToLive API may return keys to clients which have no read permission on the keys](https://github.com/etcd-io/etcd/pull/15814). ### Dependencies - Compile binaries using [go 1.19.9](https://github.com/etcd-io/etcd/pull/15823) --- ## v3.4.25 (2023-04-14) ### etcd server - Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15486) to enable support for TLS 1.3. - Add [`etcd --listen-client-http-urls`](https://github.com/etcd-io/etcd/pull/15620) flag to support separating http server from grpc one, thus giving full immunity to [watch stream starvation under high read load](https://github.com/etcd-io/etcd/issues/15402). - Change [http2 frame scheduler to random algorithm](https://github.com/etcd-io/etcd/pull/15478) - Fix [server/embed: fix data race when starting both secure & insecure gRPC servers on the same address](https://github.com/etcd-io/etcd/pull/15518) - Fix [server/auth: disallow creating empty permission ranges](https://github.com/etcd-io/etcd/pull/15621) - Fix [wsproxy did not print log in JSON format](https://github.com/etcd-io/etcd/pull/15662). - Fix [CVE-2021-28235](https://nvd.nist.gov/vuln/detail/CVE-2021-28235) by [clearing password after authenticating the user](https://github.com/etcd-io/etcd/pull/15655). - Fix [etcdserver may panic when parsing a JWT token without username or revision](https://github.com/etcd-io/etcd/pull/15677). - Fix [Watch response traveling back in time when reconnecting member downloads snapshot from the leader](https://github.com/etcd-io/etcd/pull/15520). - Fix [Requested watcher progress notifications are not synchronised with stream](https://github.com/etcd-io/etcd/pull/15697). ### Package `clientv3` - Reverted the fix to [auth invalid token and old revision errors in watch](https://github.com/etcd-io/etcd/pull/15542). ### Dependencies - Recommend [Go 1.19+](https://github.com/etcd-io/etcd/pull/15337). - Compile binaries using [Go 1.19.8](https://github.com/etcd-io/etcd/pull/15652). - Upgrade [golang.org/x/net to v0.7.0](https://github.com/etcd-io/etcd/pull/15333). ### Docker image - Fix [etcd docker images all tagged with amd64 architecture](https://github.com/etcd-io/etcd/pull/15681) --- ## v3.4.24 (2023-02-16) ### etcd server - Fix [etcdserver might promote a non-started learner](https://github.com/etcd-io/etcd/pull/15097). - Improve [mvcc: reduce count-only range overhead](https://github.com/etcd-io/etcd/pull/15099) - Improve [mvcc: push down RangeOptions.limit argv into index tree to reduce memory overhead](https://github.com/etcd-io/etcd/pull/15137) - Improve [server: set multiple concurrentReadTx instances share one txReadBuffer](https://github.com/etcd-io/etcd/pull/15195) - Fix [aligning zap log timestamp resolution to microseconds](https://github.com/etcd-io/etcd/pull/15241). Etcd now uses zap timestamp format: `2006-01-02T15:04:05.999999Z0700` (microsecond instead of milliseconds precision). - Fix [consistently format IPv6 addresses for comparison](https://github.com/etcd-io/etcd/pull/15188) ### Package `clientv3` - Fix [etcd might send duplicated events to watch clients](https://github.com/etcd-io/etcd/pull/15275). ### Dependencies - Upgrade [bbolt to v1.3.7](https://github.com/etcd-io/etcd/pull/15223). - Upgrade [github.com/grpc-ecosystem/grpc-gateway](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [v1.9.5](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.9.5) to [v1.11.0](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.11.0). ### Docker image - Updated [base image from base-debian11 to static-debian11 and removed dependency on busybox](https://github.com/etcd-io/etcd/pull/15038). --- ## v3.4.23 (2022-12-21) ### etcd server - Fix [Remove memberID from data corrupt alarm](https://github.com/etcd-io/etcd/pull/14853). - Fix [nil pointer panic for readonly txn due to nil response](https://github.com/etcd-io/etcd/pull/14900). - Bumped [some dependencies](https://github.com/etcd-io/etcd/pull/15019) to address some HIGH Vulnerabilities. ### Package `clientv3` - Fix [Refreshing token on CommonName based authentication causes segmentation violation in client](https://github.com/etcd-io/etcd/pull/14792). ### Dependencies - Recommend [Go 1.17+](https://github.com/etcd-io/etcd/pull/15019). - Compile binaries using [Go 1.17.13](https://github.com/etcd-io/etcd/pull/15019). ### Docker image - Use [distroless base image](https://github.com/etcd-io/etcd/pull/15017) to address critical Vulnerabilities. --- ## v3.4.22 (2022-11-02) ### etcd server - Fix [memberID equals zero in corruption alarm](https://github.com/etcd-io/etcd/pull/14530) - Fix [auth invalid token and old revision errors in watch](https://github.com/etcd-io/etcd/pull/14548) - Fix [avoid closing a watch with ID 0 incorrectly](https://github.com/etcd-io/etcd/pull/14562) - Fix [auth: fix data consistency issue caused by recovery from snapshot](https://github.com/etcd-io/etcd/pull/14649) ### Package `netutil` - Fix [netutil: add url comparison without resolver to URLStringsEqual](https://github.com/etcd-io/etcd/pull/14577) ### Package `clientv3` - Fix [Add backoff before retry when watch stream returns unavailable](https://github.com/etcd-io/etcd/pull/14581). ### etcd grpc-proxy - Add [`etcd grpc-proxy start --listen-cipher-suites`](https://github.com/etcd-io/etcd/pull/14601) flag to support adding configurable cipher list. --- ## v3.4.21 (2022-09-15) ### etcd server - Fix [Durability API guarantee broken in single node cluster](https://github.com/etcd-io/etcd/pull/14423) - Fix [Panic due to nil log object](https://github.com/etcd-io/etcd/pull/14420) - Fix [authentication data not loaded on member startup](https://github.com/etcd-io/etcd/pull/14410) ### etcdctl v3 - Fix [etcdctl move-leader may fail for multiple endpoints](https://github.com/etcd-io/etcd/pull/14441) --- ## v3.4.20 (2022-08-06) ### Package `clientv3` - Fix [filter learners members during autosync](https://github.com/etcd-io/etcd/pull/14236). ### etcd server - Add [`etcd --max-concurrent-streams`](https://github.com/etcd-io/etcd/pull/14251) flag to configure the max concurrent streams each client can open at a time, and defaults to math.MaxUint32. - Add [`etcd --experimental-enable-lease-checkpoint-persist`](https://github.com/etcd-io/etcd/pull/14253) flag to enable checkpoint persisting. - Fix [Lease checkpoints don't prevent to reset ttl on leader change](https://github.com/etcd-io/etcd/pull/14253), requires enabling checkpoint persisting. - Fix [Protect rangePermCache with a RW lock correctly](https://github.com/etcd-io/etcd/pull/14230) - Fix [raft: postpone MsgReadIndex until first commit in the term](https://github.com/etcd-io/etcd/pull/14258) - Fix [etcdserver: resend ReadIndex request on empty apply request](https://github.com/etcd-io/etcd/pull/14269) - Fix [remove temp files in snap dir when etcdserver starting](https://github.com/etcd-io/etcd/pull/14246) - Fix [Etcdserver is still in progress of processing LeaseGrantRequest when it receives a LeaseKeepAliveRequest on the same leaseID](https://github.com/etcd-io/etcd/pull/14177) - Fix [Grant lease with negative ID can possibly cause db out of sync](https://github.com/etcd-io/etcd/pull/14239) - Fix [Allow non mutating requests pass through quotaKVServer when NOSPACE](https://github.com/etcd-io/etcd/pull/14254) --- ## v3.4.19 (2022-07-12) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.18...v3.4.19) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### etcd server - Fix [exclude the same alarm type activated by multiple peers](https://github.com/etcd-io/etcd/pull/13475). - Fix [Defrag unsets backend options](https://github.com/etcd-io/etcd/pull/13713). - Fix [lease leak issue due to tokenProvider isn't enabled when restoring auth store from a snapshot](https://github.com/etcd-io/etcd/pull/13206). - Fix [the race condition between goroutine and channel on the same leases to be revoked](https://github.com/etcd-io/etcd/pull/14150). - Fix [lessor may continue to schedule checkpoint after stepping down leader role](https://github.com/etcd-io/etcd/pull/14150). ### Package `clientv3` - Fix [a bug of not refreshing expired tokens](https://github.com/etcd-io/etcd/pull/13999). ### Dependency - Upgrade [go.etcd.io/bbolt](https://github.com/etcd-io/bbolt/releases) from [v1.3.3](https://github.com/etcd-io/bbolt/releases/tag/v1.3.3) to [v1.3.6](https://github.com/etcd-io/bbolt/releases/tag/v1.3.6). ### Security - Upgrade [golang.org/x/crypto](https://github.com/etcd-io/etcd/pull/14179) to v0.0.0-20220411220226-7b82a4e95df4 to address [CVE-2022-27191 ](https://github.com/advisories/GHSA-8c26-wmh5-6g9v). - Upgrade [gopkg.in/yaml.v2](https://github.com/etcd-io/etcd/pull/14192) to v2.4.0 to address [CVE-2019-11254](https://github.com/advisories/GHSA-wxc4-f4m6-wwqv). ### Go - Require [Go 1.16+](https://github.com/etcd-io/etcd/pull/14136). - Compile with [Go 1.16+](https://go.dev/doc/devel/release#go1.16). - etcd uses [go modules](https://github.com/etcd-io/etcd/pull/14136) (instead of vendor dir) to track dependencies. --- ## v3.4.18 (2021-10-15) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.17...v3.4.18) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### Metrics, Monitoring See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. - Add [`etcd_disk_defrag_inflight`](https://github.com/etcd-io/etcd/pull/13397). ### Other - Updated [base image](https://github.com/etcd-io/etcd/pull/13386) from `debian:buster-v1.4.0` to `debian:bullseye-20210927` to fix the following critical CVEs: - [CVE-2021-3711](https://nvd.nist.gov/vuln/detail/CVE-2021-3711): miscalculation of a buffer size in openssl's SM2 decryption - [CVE-2021-35942](https://nvd.nist.gov/vuln/detail/CVE-2021-35942): integer overflow flaw in glibc - [CVE-2019-9893](https://nvd.nist.gov/vuln/detail/CVE-2019-9893): incorrect syscall argument generation in libseccomp - [CVE-2021-36159](https://nvd.nist.gov/vuln/detail/CVE-2021-36159): libfetch in apk-tools mishandles numeric strings in FTP and HTTP protocols to allow out of bound reads. --- ## v3.4.17 (2021-10-03) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.16...v3.4.17) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### `etcdctl` - Fix [etcdctl check datascale command](https://github.com/etcd-io/etcd/pull/11896) to work with https endpoints. ### gRPC gateway - Add [`MaxCallRecvMsgSize`](https://github.com/etcd-io/etcd/pull/13077) support for http client. ### Dependency - Replace [`github.com/dgrijalva/jwt-go with github.com/golang-jwt/jwt'](https://github.com/etcd-io/etcd/pull/13378). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## v3.4.16 (2021-05-11) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.15...v3.4.16) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### etcd server - Add [`--experimental-warning-apply-duration`](https://github.com/etcd-io/etcd/pull/12448) flag which allows apply duration threshold to be configurable. - Fix [`--unsafe-no-fsync`](https://github.com/etcd-io/etcd/pull/12751) to still write-out data avoiding corruption (most of the time). - Reduce [around 30% memory allocation by logging range response size without marshal](https://github.com/etcd-io/etcd/pull/12871). - Add [exclude alarms from health check conditionally](https://github.com/etcd-io/etcd/pull/12880). ### Metrics - Fix [incorrect metrics generated when clients cancel watches](https://github.com/etcd-io/etcd/pull/12803) back-ported from (https://github.com/etcd-io/etcd/pull/12196). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.15](https://github.com/etcd-io/etcd/releases/tag/v3.4.15) (2021-02-26) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.14...v3.4.15) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### etcd server - Log [successful etcd server-side health check in debug level](https://github.com/etcd-io/etcd/pull/12677). - Fix [64 KB websocket notification message limit](https://github.com/etcd-io/etcd/pull/12402). ### Package `fileutil` - Fix [`F_OFD_` constants](https://github.com/etcd-io/etcd/pull/12444). ### Dependency - Bump up [`gorilla/websocket` to v1.4.2](https://github.com/etcd-io/etcd/pull/12645). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.14](https://github.com/etcd-io/etcd/releases/tag/v3.4.14) (2020-11-25) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.13...v3.4.14) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### Package `clientv3` - Fix [auth token invalid after watch reconnects](https://github.com/etcd-io/etcd/pull/12264). Get AuthToken automatically when clientConn is ready. ### etcd server - [Fix server panic](https://github.com/etcd-io/etcd/pull/12288) when force-new-cluster flag is enabled in a cluster which had learner node. ### Package `netutil` - Remove [`netutil.DropPort/RecoverPort/SetLatency/RemoveLatency`](https://github.com/etcd-io/etcd/pull/12491). - These are not used anymore. They were only used for older versions of functional testing. - Removed to adhere to best security practices, minimize arbitrary shell invocation. ### `tools/etcd-dump-metrics` - Implement [input validation to prevent arbitrary shell invocation](https://github.com/etcd-io/etcd/pull/12491). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.13](https://github.com/etcd-io/etcd/releases/tag/v3.4.13) (2020-8-24) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.12...v3.4.13) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### Security - A [log warning](https://github.com/etcd-io/etcd/pull/12242) is added when etcd use any existing directory that has a permission different than 700 on Linux and 777 on Windows. ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.12](https://github.com/etcd-io/etcd/releases/tag/v3.4.12) (2020-08-19) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.11...v3.4.12) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### etcd server - Fix [server panic in slow writes warnings](https://github.com/etcd-io/etcd/issues/12197). - Fixed via [PR#12238](https://github.com/etcd-io/etcd/pull/12238). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.11](https://github.com/etcd-io/etcd/releases/tag/v3.4.11) (2020-08-18) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.10...v3.4.11) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### etcd server - Improve [`runtime.FDUsage` call pattern to reduce objects malloc of Memory Usage and CPU Usage](https://github.com/etcd-io/etcd/pull/11986). - Add [`etcd --experimental-watch-progress-notify-interval`](https://github.com/etcd-io/etcd/pull/12216) flag to make watch progress notify interval configurable. ### Package `clientv3` - Remove [excessive watch cancel logging messages](https://github.com/etcd-io/etcd/pull/12187). - See [kubernetes/kubernetes#93450](https://github.com/kubernetes/kubernetes/issues/93450). ### Package `runtime` - Optimize [`runtime.FDUsage` by removing unnecessary sorting](https://github.com/etcd-io/etcd/pull/12214). ### Metrics, Monitoring - Add [`os_fd_used` and `os_fd_limit` to monitor current OS file descriptors](https://github.com/etcd-io/etcd/pull/12214). - Add [`etcd_disk_defrag_inflight`](https://github.com/etcd-io/etcd/pull/13397). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.10](https://github.com/etcd-io/etcd/releases/tag/v3.4.10) (2020-07-16) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.9...v3.4.10) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### Package `etcd server` - Add [`--unsafe-no-fsync`](https://github.com/etcd-io/etcd/pull/11946) flag. - Setting the flag disables all uses of fsync, which is unsafe and will cause data loss. This flag makes it possible to run an etcd node for testing and development without placing lots of load on the file system. - Add [etcd --auth-token-ttl](https://github.com/etcd-io/etcd/pull/11980) flag to customize `simpleTokenTTL` settings. - Improve [runtime.FDUsage objects malloc of Memory Usage and CPU Usage](https://github.com/etcd-io/etcd/pull/11986). - Improve [mvcc.watchResponse channel Memory Usage](https://github.com/etcd-io/etcd/pull/11987). - Fix [`int64` convert panic in raft logger](https://github.com/etcd-io/etcd/pull/12106). - Fix [kubernetes/kubernetes#91937](https://github.com/kubernetes/kubernetes/issues/91937). ### Breaking Changes - Changed behavior on [existing dir permission](https://github.com/etcd-io/etcd/pull/11798). - Previously, the permission was not checked on existing data directory and the directory used for automatically generating self-signed certificates for TLS connections with clients. Now a check is added to make sure those directories, if already exist, has a desired permission of 700 on Linux and 777 on Windows. ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.9](https://github.com/etcd-io/etcd/releases/tag/v3.4.9) (2020-05-20) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.8...v3.4.9) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### Package `wal` - Add [missing CRC checksum check in WAL validate method otherwise causes panic](https://github.com/etcd-io/etcd/pull/11924). - See https://github.com/etcd-io/etcd/issues/11918. ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.8](https://github.com/etcd-io/etcd/releases/tag/v3.4.8) (2020-05-18) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.7...v3.4.8) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### `etcdctl` - Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). ### Package `clientv3` - Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). ### etcd server - Improve logging around snapshot send and receive. - [Add log when etcdserver failed to apply command](https://github.com/etcd-io/etcd/pull/11670). - [Fix deadlock bug in mvcc](https://github.com/etcd-io/etcd/pull/11817). - Fix [inconsistency between WAL and server snapshot](https://github.com/etcd-io/etcd/pull/11888). - Previously, server restore fails if it had crashed after persisting raft hard state but before saving snapshot. - See https://github.com/etcd-io/etcd/issues/10219 for more. ### Package Auth - [Fix a data corruption bug by saving consistent index](https://github.com/etcd-io/etcd/pull/11652). ### Metrics, Monitoring - Add [`etcd_debugging_auth_revision`](https://github.com/etcd-io/etcd/commit/f14d2a087f7b0fd6f7980b95b5e0b945109c95f3). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.7](https://github.com/etcd-io/etcd/releases/tag/v3.4.7) (2020-04-01) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.6...v3.4.7) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### etcd server - Improve [compaction performance when latest index is greater than 1-million](https://github.com/etcd-io/etcd/pull/11734). ### Package `wal` - Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). ### Metrics, Monitoring - Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.6](https://github.com/etcd-io/etcd/releases/tag/v3.4.6) (2020-03-29) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.5...v3.4.6) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. ### Package `lease` - Fix [memory leak in follower nodes](https://github.com/etcd-io/etcd/pull/11731). - https://github.com/etcd-io/etcd/issues/11495 - https://github.com/etcd-io/etcd/issues/11730 ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.5](https://github.com/etcd-io/etcd/releases/tag/v3.4.5) (2020-03-18) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.4...v3.4.5) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** ### etcd server - Log [`[CLIENT-PORT]/health` check in server side](https://github.com/etcd-io/etcd/pull/11704). ### client v3 - Fix [`"hasleader"` metadata embedding](https://github.com/etcd-io/etcd/pull/11687). - Previously, `clientv3.WithRequireLeader(ctx)` was overwriting existing context keys. ### etcdctl v3 - Fix [`etcdctl member add`](https://github.com/etcd-io/etcd/pull/11638) command to prevent potential timeout. ### Metrics, Monitoring See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. - Add [`etcd_server_client_requests_total` with `"type"` and `"client_api_version"` labels](https://github.com/etcd-io/etcd/pull/11687). ### gRPC Proxy - Fix [`panic on error`](https://github.com/etcd-io/etcd/pull/11694) for metrics handler. ### Go - Compile with [*Go 1.12.17*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.4](https://github.com/etcd-io/etcd/releases/tag/v3.4.4) (2020-02-24) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.3...v3.4.4) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** ### etcd server - Fix [`wait purge file loop during shutdown`](https://github.com/etcd-io/etcd/pull/11308). - Previously, during shutdown etcd could accidentally remove needed wal files, resulting in catastrophic error `etcdserver: open wal error: wal: file not found.` during startup. - Now, etcd makes sure the purge file loop exits before server signals stop of the raft node. - [Fix corruption bug in defrag](https://github.com/etcd-io/etcd/pull/11613). - Fix [quorum protection logic when promoting a learner](https://github.com/etcd-io/etcd/pull/11640). - Improve [peer corruption checker](https://github.com/etcd-io/etcd/pull/11621) to work when peer mTLS is enabled. ### Metrics, Monitoring See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_debugging_mvcc_total_put_size_in_bytes`](https://github.com/etcd-io/etcd/pull/11374) Prometheus metric. - Fix bug where [etcd_debugging_mvcc_db_compaction_keys_total is always 0](https://github.com/etcd-io/etcd/pull/11400). ### Auth - Fix [NoPassword check when adding user through GRPC gateway](https://github.com/etcd-io/etcd/pull/11418) ([issue#11414](https://github.com/etcd-io/etcd/issues/11414)) - Fix bug where [some auth related messages are logged at wrong level](https://github.com/etcd-io/etcd/pull/11586) --- ## [v3.4.3](https://github.com/etcd-io/etcd/releases/tag/v3.4.3) (2019-10-24) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.2...v3.4.3) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** ### Metrics, Monitoring See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Change [`etcd_cluster_version`](https://github.com/etcd-io/etcd/pull/11254) Prometheus metrics to include only major and minor version. ### Go - Compile with [*Go 1.12.12*](https://golang.org/doc/devel/release.html#go1.12). --- ## [v3.4.2](https://github.com/etcd-io/etcd/releases/tag/v3.4.2) (2019-10-11) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.1...v3.4.2) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** ### etcdctl v3 - Fix [`etcdctl member add`](https://github.com/etcd-io/etcd/pull/11194) command to prevent potential timeout. ### etcdserver - Add [`tracing`](https://github.com/etcd-io/etcd/pull/11179) to range, put and compact requests in etcdserver. ### Go - Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. ### client v3 - Fix [client balancer failover against multiple endpoints](https://github.com/etcd-io/etcd/pull/11184). - Fix ["kube-apiserver: failover on multi-member etcd cluster fails certificate check on DNS mismatch" (kubernetes#83028)](https://github.com/kubernetes/kubernetes/issues/83028). - Fix [IPv6 endpoint parsing in client](https://github.com/etcd-io/etcd/pull/11211). - Fix ["1.16: etcd client does not parse IPv6 addresses correctly when members are joining" (kubernetes#83550)](https://github.com/kubernetes/kubernetes/issues/83550). --- ## [v3.4.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.1) (2019-09-17) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0...v3.4.1) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** ### Metrics, Monitoring See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_debugging_mvcc_current_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. - Add [`etcd_debugging_mvcc_compact_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. ### etcd server - Fix [secure server logging message](https://github.com/etcd-io/etcd/commit/8b053b0f44c14ac0d9f39b9b78c17c57d47966eb). - Remove [redundant `%` characters in file descriptor warning message](https://github.com/etcd-io/etcd/commit/d5f79adc9cea9ec8c93669526464b0aa19ed417b). ### Package `embed` - Add [`embed.Config.ZapLoggerBuilder`](https://github.com/etcd-io/etcd/pull/11148) to allow creating a custom zap logger. ### Dependency - Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.23.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.0) to [**`v1.23.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.1). ### Go - Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. --- ## v3.4.0 (2019-08-30) See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0...v3.4.0) and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/) for any breaking changes. - [v3.4.0](https://github.com/etcd-io/etcd/releases/tag/v3.4.0) (2019-08-30), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0-rc.4...v3.4.0). - [v3.4.0-rc.4](https://github.com/etcd-io/etcd/releases/tag/v3.4.0-rc.4) (2019-08-29), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0-rc.3...v3.4.0-rc.4). - [v3.4.0-rc.3](https://github.com/etcd-io/etcd/releases/tag/v3.4.0-rc.3) (2019-08-27), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0-rc.2...v3.4.0-rc.3). - [v3.4.0-rc.2](https://github.com/etcd-io/etcd/releases/tag/v3.4.0-rc.2) (2019-08-23), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0-rc.1...v3.4.0-rc.2). - [v3.4.0-rc.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.0-rc.1) (2019-08-15), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0-rc.0...v3.4.0-rc.1). - [v3.4.0-rc.0](https://github.com/etcd-io/etcd/releases/tag/v3.4.0-rc.0) (2019-08-12), see [code changes](https://github.com/etcd-io/etcd/compare/v3.3.0...v3.4.0-rc.0). **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_4/).** ### Documentation - etcd now has a new website! Please visit https://etcd.io. ### Improved - Add Raft learner: [etcd#10725](https://github.com/etcd-io/etcd/pull/10725), [etcd#10727](https://github.com/etcd-io/etcd/pull/10727), [etcd#10730](https://github.com/etcd-io/etcd/pull/10730). - User guide: [runtime-configuration document](https://etcd.io/docs/latest/op-guide/runtime-configuration/#add-a-new-member-as-learner). - API change: [API reference document](https://etcd.io/docs/latest/dev-guide/api_reference_v3/). - More details on implementation: [learner design document](https://etcd.io/docs/latest/learning/design-learner/) and [implementation task list](https://github.com/etcd-io/etcd/issues/10537). - Rewrite [client balancer](https://github.com/etcd-io/etcd/pull/9860) with [new gRPC balancer interface](https://github.com/etcd-io/etcd/issues/9106). - Upgrade [gRPC to v1.23.0](https://github.com/etcd-io/etcd/pull/10911). - Improve [client balancer failover against secure endpoints](https://github.com/etcd-io/etcd/pull/10911). - Fix ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). - Fix [gRPC panic "send on closed channel](https://github.com/etcd-io/etcd/issues/9956). - [The new client balancer](https://etcd.io/docs/latest/learning/design-client/) uses an asynchronous resolver to pass endpoints to the gRPC dial function. To block until the underlying connection is up, pass `grpc.WithBlock()` to `clientv3.Config.DialOptions`. - Add [backoff on watch retries on transient errors](https://github.com/etcd-io/etcd/pull/9840). - Add [jitter to watch progress notify](https://github.com/etcd-io/etcd/pull/9278) to prevent [spikes in `etcd_network_client_grpc_sent_bytes_total`](https://github.com/etcd-io/etcd/issues/9246). - Improve [read index wait timeout warning log](https://github.com/etcd-io/etcd/pull/10026), which indicates that local node might have slow network. - Improve [slow request apply warning log](https://github.com/etcd-io/etcd/pull/9288). - e.g. `read-only range request "key:\"/a\" range_end:\"/b\" " with result "range_response_count:3 size:96" took too long (97.966µs) to execute`. - Redact [request value field](https://github.com/etcd-io/etcd/pull/9822). - Provide [response size](https://github.com/etcd-io/etcd/pull/9826). - Improve ["became inactive" warning log](https://github.com/etcd-io/etcd/pull/10024), which indicates message send to a peer failed. - Improve [TLS setup error logging](https://github.com/etcd-io/etcd/pull/9518) to help debug [TLS-enabled cluster configuring issues](https://github.com/etcd-io/etcd/issues/9400). - Improve [long-running concurrent read transactions under light write workloads](https://github.com/etcd-io/etcd/pull/9296). - Previously, periodic commit on pending writes blocks incoming read transactions, even if there is no pending write. - Now, periodic commit operation does not block concurrent read transactions, thus improves long-running read transaction performance. - Make [backend read transactions fully concurrent](https://github.com/etcd-io/etcd/pull/10523). - Previously, ongoing long-running read transactions block writes and future reads. - With this change, write throughput is increased by 70% and P99 write latency is reduced by 90% in the presence of long-running reads. - Improve [Raft Read Index timeout warning messages](https://github.com/etcd-io/etcd/pull/9897). - Adjust [election timeout on server restart](https://github.com/etcd-io/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/etcd-io/etcd/issues/9333). - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node. - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart. - Add [Raft Pre-Vote feature](https://github.com/etcd-io/etcd/pull/9352) to reduce [disruptive rejoining servers](https://github.com/etcd-io/etcd/issues/9333). - For instance, a flaky(or rejoining) member may drop in and out, and start campaign. This member will end up with a higher term, and ignore all incoming messages with lower term. In this case, a new leader eventually need to get elected, thus disruptive to cluster availability. Raft implements Pre-Vote phase to prevent this kind of disruptions. If enabled, Raft runs an additional phase of election to check if pre-candidate can get enough votes to win an election. - Adjust [periodic compaction retention window](https://github.com/etcd-io/etcd/pull/9485). - e.g. `etcd --auto-compaction-mode=revision --auto-compaction-retention=1000` automatically `Compact` on `"latest revision" - 1000` every 5-minute (when latest revision is 30000, compact on revision 29000). - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h` automatically `Compact` with 24-hour retention windown for every 2.4-hour. Now, `Compact` happens for every 1-hour. - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` automatically `Compact` with 30-minute retention windown for every 3-minute. Now, `Compact` happens for every 30-minute. - Periodic compactor keeps recording latest revisions for every compaction period when given period is less than 1-hour, or for every 1-hour when given compaction period is greater than 1-hour (e.g. 1-hour when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`). - For every compaction period or 1-hour, compactor uses the last revision that was fetched before compaction period, to discard historical data. - The retention window of compaction period moves for every given compaction period or hour. - For instance, when hourly writes are 100 and `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`, `v3.2.x`, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 2400, 2640, and 2880 for every 2.4-hour, while `v3.3.3` *or later* compacts revision 2400, 2500, 2600 for every 1-hour. - Furthermore, when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` and writes per minute are about 1000, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 30000, 33000, and 36000, for every 3-minute, while `v3.3.3` *or later* compacts revision 30000, 60000, and 90000, for every 30-minute. - Improve [lease expire/revoke operation performance](https://github.com/etcd-io/etcd/pull/9418), address [lease scalability issue](https://github.com/etcd-io/etcd/issues/9496). - Make [Lease `Lookup` non-blocking with concurrent `Grant`/`Revoke`](https://github.com/etcd-io/etcd/pull/9229). - Make etcd server return `raft.ErrProposalDropped` on internal Raft proposal drop in [v3 applier](https://github.com/etcd-io/etcd/pull/9549) and [v2 applier](https://github.com/etcd-io/etcd/pull/9558). - e.g. a node is removed from cluster, or [`raftpb.MsgProp` arrives at current leader while there is an ongoing leadership transfer](https://github.com/etcd-io/etcd/issues/8975). - Add [`snapshot`](https://github.com/etcd-io/etcd/pull/9118) package for easier snapshot workflow (see [`godoc.org/github.com/etcd/clientv3/snapshot`](https://godoc.org/github.com/etcd-io/etcd/clientv3/snapshot) for more). - Improve [functional tester](https://github.com/etcd-io/etcd/tree/main/functional) coverage: [proxy layer to run network fault tests in CI](https://github.com/etcd-io/etcd/pull/9081), [TLS is enabled both for server and client](https://github.com/etcd-io/etcd/pull/9534), [liveness mode](https://github.com/etcd-io/etcd/issues/9230), [shuffle test sequence](https://github.com/etcd-io/etcd/issues/9381), [membership reconfiguration failure cases](https://github.com/etcd-io/etcd/pull/9564), [disastrous quorum loss and snapshot recover from a seed member](https://github.com/etcd-io/etcd/pull/9565), [embedded etcd](https://github.com/etcd-io/etcd/pull/9572). - Improve [index compaction blocking](https://github.com/etcd-io/etcd/pull/9511) by using a copy on write clone to avoid holding the lock for the traversal of the entire index. - Update [JWT methods](https://github.com/etcd-io/etcd/pull/9883) to allow for use of any supported signature method/algorithm. - Add [Lease checkpointing](https://github.com/etcd-io/etcd/pull/9924) to persist remaining TTLs to the consensus log periodically so that long lived leases progress toward expiry in the presence of leader elections and server restarts. - Enabled by experimental flag "--experimental-enable-lease-checkpoint". - Add [gRPC interceptor for debugging logs](https://github.com/etcd-io/etcd/pull/9990); enable `etcd --debug` flag to see per-request debug information. - Add [consistency check in snapshot status](https://github.com/etcd-io/etcd/pull/10109). If consistency check on snapshot file fails, `snapshot status` returns `"snapshot file integrity check failed..."` error. - Add [`Verify` function to perform corruption check on WAL contents](https://github.com/etcd-io/etcd/pull/10603). - Improve [heartbeat send failure logging](https://github.com/etcd-io/etcd/pull/10663). - Support [users with no password](https://github.com/etcd-io/etcd/pull/9817) for reducing security risk introduced by leaked password. The users can only be authenticated with `CommonName` based auth. - Add `etcd --experimental-peer-skip-client-san-verification` to [skip verification of peer client address](https://github.com/etcd-io/etcd/pull/10524). - Add `etcd --experimental-compaction-batch-limit` to [sets the maximum revisions deleted in each compaction batch](https://github.com/etcd-io/etcd/pull/11034). - Reduced default compaction batch size from 10k revisions to 1k revisions to improve p99 latency during compactions and reduced wait between compactions from 100ms to 10ms. ### Breaking Changes - Rewrite [client balancer](https://github.com/etcd-io/etcd/pull/9860) with [new gRPC balancer interface](https://github.com/etcd-io/etcd/issues/9106). - Upgrade [gRPC to v1.23.0](https://github.com/etcd-io/etcd/pull/10911). - Improve [client balancer failover against secure endpoints](https://github.com/etcd-io/etcd/pull/10911). - Fix ["kube-apiserver 1.13.x refuses to work when first etcd-server is not available" (kubernetes#72102)](https://github.com/kubernetes/kubernetes/issues/72102). - Fix [gRPC panic "send on closed channel](https://github.com/etcd-io/etcd/issues/9956). - [The new client balancer](https://etcd.io/docs/latest/learning/design-client/) uses an asynchronous resolver to pass endpoints to the gRPC dial function. To block until the underlying connection is up, pass `grpc.WithBlock()` to `clientv3.Config.DialOptions`. - Require [*Go 1.12+*](https://github.com/etcd-io/etcd/pull/10045). - Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. - Migrate dependency management tool from `glide` to [Go module](https://github.com/etcd-io/etcd/pull/10063). - <= 3.3 puts `vendor` directory under `cmd/vendor` directory to [prevent conflicting transitive dependencies](https://github.com/etcd-io/etcd/issues/4913). - 3.4 moves `cmd/vendor` directory to `vendor` at repository root. - Remove recursive symlinks in `cmd` directory. - Now `go get/install/build` on `etcd` packages (e.g. `clientv3`, `tools/benchmark`) enforce builds with etcd `vendor` directory. - Deprecated `latest` [release container](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) tag. - **`docker pull gcr.io/etcd-development/etcd:latest` would not be up-to-date**. - Deprecated [minor](https://semver.org/) version [release container](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) tags. - `docker pull gcr.io/etcd-development/etcd:v3.3` would still work. - **`docker pull gcr.io/etcd-development/etcd:v3.4` would not work**. - Use **`docker pull gcr.io/etcd-development/etcd:v3.4.x`** instead, with the exact patch version. - Deprecated [ACIs from official release](https://github.com/etcd-io/etcd/pull/9059). - [AppC was officially suspended](https://github.com/appc/spec#-disclaimer-), as of late 2016. - [`acbuild`](https://github.com/containers/build#this-project-is-currently-unmaintained) is not maintained anymore. - `*.aci` files are not available from `v3.4` release. - Move [`"github.com/coreos/etcd"`](https://github.com/etcd-io/etcd/issues/9965) to [`"github.com/etcd-io/etcd"`](https://github.com/etcd-io/etcd/issues/9965). - Change import path to `"go.etcd.io/etcd"`. - e.g. `import "go.etcd.io/etcd/raft"`. - Make [`ETCDCTL_API=3 etcdctl` default](https://github.com/etcd-io/etcd/issues/9600). - Now, `etcdctl set foo bar` must be `ETCDCTL_API=2 etcdctl set foo bar`. - Now, `ETCDCTL_API=3 etcdctl put foo bar` could be just `etcdctl put foo bar`. - Make [`etcd --enable-v2=false` default](https://github.com/etcd-io/etcd/pull/10935). - Make [`embed.DefaultEnableV2` `false` default](https://github.com/etcd-io/etcd/pull/10935). - **Deprecated `etcd --ca-file` flag**. Use [`etcd --trusted-ca-file`](https://github.com/etcd-io/etcd/pull/9470) instead (`etcd --ca-file` flag has been marked deprecated since v2.1). - **Deprecated `etcd --peer-ca-file` flag**. Use [`etcd --peer-trusted-ca-file`](https://github.com/etcd-io/etcd/pull/9470) instead (`etcd --peer-ca-file` flag has been marked deprecated since v2.1). - **Deprecated `pkg/transport.TLSInfo.CAFile` field**. Use [`pkg/transport.TLSInfo.TrustedCAFile`](https://github.com/etcd-io/etcd/pull/9470) instead (`CAFile` field has been marked deprecated since v2.1). - Exit on [empty hosts in advertise URLs](https://github.com/etcd-io/etcd/pull/8786). - Address [advertise client URLs accepts empty hosts](https://github.com/etcd-io/etcd/issues/8379). - e.g. exit with error on `--advertise-client-urls=http://:2379`. - e.g. exit with error on `--initial-advertise-peer-urls=http://:2380`. - Exit on [shadowed environment variables](https://github.com/etcd-io/etcd/pull/9382). - Address [error on shadowed environment variables](https://github.com/etcd-io/etcd/issues/8380). - e.g. exit with error on `ETCD_NAME=abc etcd --name=def`. - e.g. exit with error on `ETCD_INITIAL_CLUSTER_TOKEN=abc etcd --initial-cluster-token=def`. - e.g. exit with error on `ETCDCTL_ENDPOINTS=abc.com ETCDCTL_API=3 etcdctl endpoint health --endpoints=def.com`. - Change [`etcdserverpb.AuthRoleRevokePermissionRequest/key,range_end` fields type from `string` to `bytes`](https://github.com/etcd-io/etcd/pull/9433). - Deprecating `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819) instead. - Deprecating `etcd_debugging_mvcc_put_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_put_total`](https://github.com/etcd-io/etcd/pull/10962) instead. - Deprecating `etcd_debugging_mvcc_delete_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_delete_total`](https://github.com/etcd-io/etcd/pull/10962) instead. - Deprecating `etcd_debugging_mvcc_range_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_range_total`](https://github.com/etcd-io/etcd/pull/10968) instead. - Deprecating `etcd_debugging_mvcc_txn_total`Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_txn_total`](https://github.com/etcd-io/etcd/pull/10968) instead. - Rename `etcdserver.ServerConfig.SnapCount` field to `etcdserver.ServerConfig.SnapshotCount`, to be consistent with the flag name `etcd --snapshot-count`. - Rename `embed.Config.SnapCount` field to [`embed.Config.SnapshotCount`](https://github.com/etcd-io/etcd/pull/9745), to be consistent with the flag name `etcd --snapshot-count`. - Change [`embed.Config.CorsInfo` in `*cors.CORSInfo` type to `embed.Config.CORS` in `map[string]struct{}` type](https://github.com/etcd-io/etcd/pull/9490). - Deprecated [`embed.Config.SetupLogging`](https://github.com/etcd-io/etcd/pull/9572). - Now logger is set up automatically based on [`embed.Config.Logger`, `embed.Config.LogOutputs`, `embed.Config.Debug` fields](https://github.com/etcd-io/etcd/pull/9572). - Rename [`etcd --log-output` to `etcd --log-outputs`](https://github.com/etcd-io/etcd/pull/9624) to support multiple log outputs. - **`etcd --log-output`** will be deprecated in v3.5. - Rename [**`embed.Config.LogOutput`** to **`embed.Config.LogOutputs`**](https://github.com/etcd-io/etcd/pull/9624) to support multiple log outputs. - Change [**`embed.Config.LogOutputs`** type from `string` to `[]string`](https://github.com/etcd-io/etcd/pull/9579) to support multiple log outputs. - Now that `etcd --log-outputs` accepts multiple writers, etcd configuration YAML file `log-outputs` field must be changed to `[]string` type. - Previously, `etcd --config-file etcd.config.yaml` can have `log-outputs: default` field, now must be `log-outputs: [default]`. - Deprecating [`etcd --debug`](https://github.com/etcd-io/etcd/pull/10947) flag. Use `etcd --log-level=debug` flag instead. - v3.5 will deprecate `etcd --debug` flag in favor of `etcd --log-level=debug`. - Change v3 `etcdctl snapshot` exit codes with [`snapshot` package](https://github.com/etcd-io/etcd/pull/9118/commits/df689f4280e1cce4b9d61300be13ca604d41670a). - Exit on error with exit code 1 (no more exit code 5 or 6 on `snapshot save/restore` commands). - Deprecated [`grpc.ErrClientConnClosing`](https://github.com/etcd-io/etcd/pull/10981). - `clientv3` and `proxy/grpcproxy` now does not return `grpc.ErrClientConnClosing`. - `grpc.ErrClientConnClosing` has been [deprecated in gRPC >= 1.10](https://github.com/grpc/grpc-go/pull/1854). - Use `clientv3.IsConnCanceled(error)` or `google.golang.org/grpc/status.FromError(error)` instead. - Deprecated [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3beta` with [`/v3`](https://github.com/etcd-io/etcd/pull/9298). - Deprecated [`/v3alpha`](https://github.com/etcd-io/etcd/pull/9298). - To deprecate [`/v3beta`](https://github.com/etcd-io/etcd/issues/9189) in v3.5. - In v3.4, `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. - Change [`wal` package function signatures](https://github.com/etcd-io/etcd/pull/9572) to support [structured logger and logging to file](https://github.com/etcd-io/etcd/issues/9438) in server-side. - Previously, `Open(dirpath string, snap walpb.Snapshot) (*WAL, error)`, now `Open(lg *zap.Logger, dirpath string, snap walpb.Snapshot) (*WAL, error)`. - Previously, `OpenForRead(dirpath string, snap walpb.Snapshot) (*WAL, error)`, now `OpenForRead(lg *zap.Logger, dirpath string, snap walpb.Snapshot) (*WAL, error)`. - Previously, `Repair(dirpath string) bool`, now `Repair(lg *zap.Logger, dirpath string) bool`. - Previously, `Create(dirpath string, metadata []byte) (*WAL, error)`, now `Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error)`. - Remove [`pkg/cors` package](https://github.com/etcd-io/etcd/pull/9490). - Move internal packages to `etcdserver`. - `"github.com/coreos/etcd/alarm"` to `"go.etcd.io/etcd/etcdserver/api/v3alarm"`. - `"github.com/coreos/etcd/compactor"` to `"go.etcd.io/etcd/etcdserver/api/v3compactor"`. - `"github.com/coreos/etcd/discovery"` to `"go.etcd.io/etcd/etcdserver/api/v2discovery"`. - `"github.com/coreos/etcd/etcdserver/auth"` to `"go.etcd.io/etcd/etcdserver/api/v2auth"`. - `"github.com/coreos/etcd/etcdserver/membership"` to `"go.etcd.io/etcd/etcdserver/api/membership"`. - `"github.com/coreos/etcd/etcdserver/stats"` to `"go.etcd.io/etcd/etcdserver/api/v2stats"`. - `"github.com/coreos/etcd/error"` to `"go.etcd.io/etcd/etcdserver/api/v2error"`. - `"github.com/coreos/etcd/rafthttp"` to `"go.etcd.io/etcd/etcdserver/api/rafthttp"`. - `"github.com/coreos/etcd/snap"` to `"go.etcd.io/etcd/etcdserver/api/snap"`. - `"github.com/coreos/etcd/store"` to `"go.etcd.io/etcd/etcdserver/api/v2store"`. - Change [snapshot file permissions](https://github.com/etcd-io/etcd/pull/9977): On Linux, the snapshot file changes from readable by all (mode 0644) to readable by the user only (mode 0600). - Change [`pkg/adt.IntervalTree` from `struct` to `interface`](https://github.com/etcd-io/etcd/pull/10959). - See [`pkg/adt` README](https://github.com/etcd-io/etcd/tree/main/pkg/adt) and [`pkg/adt` godoc](https://godoc.org/go.etcd.io/etcd/pkg/adt). - Release branch `/version` defines version `3.4.x-pre`, instead of `3.4.y+git`. - Use `3.4.5-pre`, instead of `3.4.4+git`. ### Dependency - Upgrade [`github.com/coreos/bbolt`](https://github.com/etcd-io/bbolt/releases) from [**`v1.3.1-coreos.6`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.1-coreos.6) to [`go.etcd.io/bbolt`](https://github.com/etcd-io/bbolt/releases) [**`v1.3.3`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.3). - Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5) to [**`v1.23.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.0). - Migrate [`github.com/ugorji/go/codec`](https://github.com/ugorji/go/releases) to [**`github.com/json-iterator/go`**](https://github.com/json-iterator/go), to [regenerate v2 `client`](https://github.com/etcd-io/etcd/pull/9494) (See [#10667](https://github.com/etcd-io/etcd/pull/10667) for more). - Migrate [`github.com/ghodss/yaml`](https://github.com/ghodss/yaml/releases) to [**`sigs.k8s.io/yaml`**](https://github.com/kubernetes-sigs/yaml) (See [#10687](https://github.com/etcd-io/etcd/pull/10687) for more). - Upgrade [`golang.org/x/crypto`](https://github.com/golang/crypto) from [**`crypto@9419663f5`**](https://github.com/golang/crypto/commit/9419663f5a44be8b34ca85f08abc5fe1be11f8a3) to [**`crypto@0709b304e793`**](https://github.com/golang/crypto/commit/0709b304e793a5edb4a2c0145f281ecdc20838a4). - Upgrade [`golang.org/x/net`](https://github.com/golang/net) from [**`net@66aacef3d`**](https://github.com/golang/net/commit/66aacef3dd8a676686c7ae3716979581e8b03c47) to [**`net@adae6a3d119a`**](https://github.com/golang/net/commit/adae6a3d119ae4890b46832a2e88a95adc62b8e7). - Upgrade [`golang.org/x/sys`](https://github.com/golang/sys) from [**`sys@ebfc5b463`**](https://github.com/golang/sys/commit/ebfc5b4631820b793c9010c87fd8fef0f39eb082) to [**`sys@c7b8b68b1456`**](https://github.com/golang/sys/commit/c7b8b68b14567162c6602a7c5659ee0f26417c18). - Upgrade [`golang.org/x/text`](https://github.com/golang/text) from [**`text@b19bf474d`**](https://github.com/golang/text/commit/b19bf474d317b857955b12035d2c5acb57ce8b01) to [**`v0.3.0`**](https://github.com/golang/text/releases/tag/v0.3.0). - Upgrade [`golang.org/x/time`](https://github.com/golang/time) from [**`time@c06e80d93`**](https://github.com/golang/time/commit/c06e80d9300e4443158a03817b8a8cb37d230320) to [**`time@fbb02b229`**](https://github.com/golang/time/commit/fbb02b2291d28baffd63558aa44b4b56f178d650). - Upgrade [`github.com/golang/protobuf`](https://github.com/golang/protobuf/releases) from [**`golang/protobuf@1e59b77b5`**](https://github.com/golang/protobuf/commit/1e59b77b52bf8e4b449a57e6f79f21226d571845) to [**`v1.3.2`**](https://github.com/golang/protobuf/releases/tag/v1.3.2). - Upgrade [`gopkg.in/yaml.v2`](https://github.com/go-yaml/yaml/releases) from [**`yaml@cd8b52f82`**](https://github.com/go-yaml/yaml/commit/cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b) to [**`yaml@5420a8b67`**](https://github.com/go-yaml/yaml/commit/5420a8b6744d3b0345ab293f6fcba19c978f1183). - Upgrade [`github.com/dgrijalva/jwt-go`](https://github.com/dgrijalva/jwt-go/releases) from [**`v3.0.0`**](https://github.com/dgrijalva/jwt-go/releases/tag/v3.0.0) to [**`v3.2.0`**](https://github.com/dgrijalva/jwt-go/releases/tag/v3.2.0). - Upgrade [`github.com/soheilhy/cmux`](https://github.com/soheilhy/cmux/releases) from [**`v0.1.3`**](https://github.com/soheilhy/cmux/releases/tag/v0.1.3) to [**`v0.1.4`**](https://github.com/soheilhy/cmux/releases/tag/v0.1.4). - Upgrade [`github.com/google/btree`](https://github.com/google/btree/releases) from [**`google/btree@925471ac9`**](https://github.com/google/btree/commit/925471ac9e2131377a91e1595defec898166fe49) to [**`v1.0.0`**](https://github.com/google/btree/releases/tag/v1.0.0). - Upgrade [`github.com/spf13/cobra`](https://github.com/spf13/cobra/releases) from [**`spf13/cobra@1c44ec8d3`**](https://github.com/spf13/cobra/commit/1c44ec8d3f1552cac48999f9306da23c4d8a288b) to [**`v0.0.3`**](https://github.com/spf13/cobra/releases/tag/v0.0.3). - Upgrade [`github.com/spf13/pflag`](https://github.com/spf13/pflag/releases) from [**`v1.0.0`**](https://github.com/spf13/pflag/releases/tag/v1.0.0) to [**`spf13/pflag@1ce0cc6db`**](https://github.com/spf13/pflag/commit/1ce0cc6db4029d97571db82f85092fccedb572ce). - Upgrade [`github.com/coreos/go-systemd`](https://github.com/coreos/go-systemd/releases) from [**`v15`**](https://github.com/coreos/go-systemd/releases/tag/v15) to [**`v17`**](https://github.com/coreos/go-systemd/releases/tag/v17). - Upgrade [`github.com/prometheus/client_golang`](https://github.com/prometheus/client_golang/releases) from [**``prometheus/client_golang@5cec1d042``**](https://github.com/prometheus/client_golang/commit/5cec1d0429b02e4323e042eb04dafdb079ddf568) to [**`v1.0.0`**](https://github.com/prometheus/client_golang/releases/tag/v1.0.0). - Upgrade [`github.com/grpc-ecosystem/go-grpc-prometheus`](https://github.com/grpc-ecosystem/go-grpc-prometheus/releases) from [**``grpc-ecosystem/go-grpc-prometheus@0dafe0d49``**](https://github.com/grpc-ecosystem/go-grpc-prometheus/commit/0dafe0d496ea71181bf2dd039e7e3f44b6bd11a7) to [**`v1.2.0`**](https://github.com/grpc-ecosystem/go-grpc-prometheus/releases/tag/v1.2.0). - Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.3.1`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3.1) to [**`v1.4.1`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.4.1). - Migrate [`github.com/kr/pty`](https://github.com/kr/pty/releases) to [**`github.com/creack/pty`**](https://github.com/creack/pty/releases/tag/v1.1.7), as the later has replaced the original module. - Upgrade [`github.com/gogo/protobuf`](https://github.com/gogo/protobuf/releases) from [**`v1.0.0`**](https://github.com/gogo/protobuf/releases/tag/v1.0.0) to [**`v1.2.1`**](https://github.com/gogo/protobuf/releases/tag/v1.2.1). ### Metrics, Monitoring See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Add [`etcd_snap_db_fsync_duration_seconds_count`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_snap_db_save_total_duration_seconds_bucket`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_send_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_success`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_failures`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_snapshot_receive_total_duration_seconds`](https://github.com/etcd-io/etcd/pull/9997) Prometheus metric. - Add [`etcd_network_active_peers`](https://github.com/etcd-io/etcd/pull/9762) Prometheus metric. - Let's say `"7339c4e5e833c029"` server `/metrics` returns `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="729934363faa4a24"} 1` and `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="b548c2511513015"} 1`. This indicates that the local node `"7339c4e5e833c029"` currently has two active remote peers `"729934363faa4a24"` and `"b548c2511513015"` in a 3-node cluster. If the node `"b548c2511513015"` is down, the local node `"7339c4e5e833c029"` will show `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="729934363faa4a24"} 1` and `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="b548c2511513015"} 0`. - Add [`etcd_network_disconnected_peers_total`](https://github.com/etcd-io/etcd/pull/9762) Prometheus metric. - If a remote peer `"b548c2511513015"` is down, the local node `"7339c4e5e833c029"` server `/metrics` would return `etcd_network_disconnected_peers_total{Local="7339c4e5e833c029",Remote="b548c2511513015"} 1`, while active peer metrics will show `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="729934363faa4a24"} 1` and `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="b548c2511513015"} 0`. - Add [`etcd_network_server_stream_failures_total`](https://github.com/etcd-io/etcd/pull/9760) Prometheus metric. - e.g. `etcd_network_server_stream_failures_total{API="lease-keepalive",Type="receive"} 1` - e.g. `etcd_network_server_stream_failures_total{API="watch",Type="receive"} 1` - Improve [`etcd_network_peer_round_trip_time_seconds`](https://github.com/etcd-io/etcd/pull/10155) Prometheus metric to track leader heartbeats. - Previously, it only samples the TCP connection for snapshot messages. - Increase [`etcd_network_peer_round_trip_time_seconds`](https://github.com/etcd-io/etcd/pull/9762) Prometheus metric histogram upper-bound. - Previously, highest bucket only collects requests taking 0.8192 seconds or more. - Now, highest buckets collect 0.8192 seconds, 1.6384 seconds, and 3.2768 seconds or more. - Add [`etcd_server_is_leader`](https://github.com/etcd-io/etcd/pull/9587) Prometheus metric. - Add [`etcd_server_id`](https://github.com/etcd-io/etcd/pull/9998) Prometheus metric. - Add [`etcd_cluster_version`](https://github.com/etcd-io/etcd/pull/10257) Prometheus metric. - Add [`etcd_server_version`](https://github.com/etcd-io/etcd/pull/8960) Prometheus metric. - To replace [Kubernetes `etcd-version-monitor`](https://github.com/etcd-io/etcd/issues/8948). - Add [`etcd_server_go_version`](https://github.com/etcd-io/etcd/pull/9957) Prometheus metric. - Add [`etcd_server_health_success`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. - Add [`etcd_server_health_failures`](https://github.com/etcd-io/etcd/pull/10156) Prometheus metric. - Add [`etcd_server_read_indexes_failed_total`](https://github.com/etcd-io/etcd/pull/10094) Prometheus metric. - Add [`etcd_server_heartbeat_send_failures_total`](https://github.com/etcd-io/etcd/pull/9761) Prometheus metric. - Add [`etcd_server_slow_apply_total`](https://github.com/etcd-io/etcd/pull/9761) Prometheus metric. - Add [`etcd_server_slow_read_indexes_total`](https://github.com/etcd-io/etcd/pull/9897) Prometheus metric. - Add [`etcd_server_quota_backend_bytes`](https://github.com/etcd-io/etcd/pull/9820) Prometheus metric. - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. - Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/etcd-io/etcd/pull/9256) Prometheus metric. - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`. - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB. - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB. - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete. - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation. - Add [`etcd_mvcc_db_open_read_transactions`](https://github.com/etcd-io/etcd/pull/10523/commits/ad80752715aaed449629369687c5fd30eb1bda76) Prometheus metric. - Add [`etcd_snap_fsync_duration_seconds`](https://github.com/etcd-io/etcd/pull/9762) Prometheus metric. - Add [`etcd_disk_backend_defrag_duration_seconds`](https://github.com/etcd-io/etcd/pull/9761) Prometheus metric. - Add [`etcd_mvcc_hash_duration_seconds`](https://github.com/etcd-io/etcd/pull/9761) Prometheus metric. - Add [`etcd_mvcc_hash_rev_duration_seconds`](https://github.com/etcd-io/etcd/pull/9761) Prometheus metric. - Add [`etcd_debugging_disk_backend_commit_rebalance_duration_seconds`](https://github.com/etcd-io/etcd/pull/9834) Prometheus metric. - Add [`etcd_debugging_disk_backend_commit_spill_duration_seconds`](https://github.com/etcd-io/etcd/pull/9834) Prometheus metric. - Add [`etcd_debugging_disk_backend_commit_write_duration_seconds`](https://github.com/etcd-io/etcd/pull/9834) Prometheus metric. - Add [`etcd_debugging_lease_granted_total`](https://github.com/etcd-io/etcd/pull/9778) Prometheus metric. - Add [`etcd_debugging_lease_revoked_total`](https://github.com/etcd-io/etcd/pull/9778) Prometheus metric. - Add [`etcd_debugging_lease_renewed_total`](https://github.com/etcd-io/etcd/pull/9778) Prometheus metric. - Add [`etcd_debugging_lease_ttl_total`](https://github.com/etcd-io/etcd/pull/9778) Prometheus metric. - Add [`etcd_network_snapshot_send_inflights_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. - Add [`etcd_network_snapshot_receive_inflights_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. - Add [`etcd_server_snapshot_apply_in_progress_total`](https://github.com/etcd-io/etcd/pull/11009) Prometheus metric. - Add [`etcd_server_is_learner`](https://github.com/etcd-io/etcd/pull/10731) Prometheus metric. - Add [`etcd_server_learner_promote_failures`](https://github.com/etcd-io/etcd/pull/10731) Prometheus metric. - Add [`etcd_server_learner_promote_successes`](https://github.com/etcd-io/etcd/pull/10731) Prometheus metric. - Increase [`etcd_debugging_mvcc_index_compaction_pause_duration_milliseconds`](https://github.com/etcd-io/etcd/pull/9762) Prometheus metric histogram upper-bound. - Previously, highest bucket only collects requests taking 1.024 seconds or more. - Now, highest buckets collect 1.024 seconds, 2.048 seconds, and 4.096 seconds or more. - Fix missing [`etcd_network_peer_sent_failures_total`](https://github.com/etcd-io/etcd/pull/9437) Prometheus metric count. - Fix [`etcd_debugging_server_lease_expired_total`](https://github.com/etcd-io/etcd/pull/9557) Prometheus metric. - Fix [race conditions in v2 server stat collecting](https://github.com/etcd-io/etcd/pull/9562). - Change [gRPC proxy to expose etcd server endpoint /metrics](https://github.com/etcd-io/etcd/pull/10618). - The metrics that were exposed via the proxy were not etcd server members but instead the proxy itself. - Fix bug where [db_compaction_total_duration_milliseconds metric incorrectly measured duration as 0](https://github.com/etcd-io/etcd/pull/10646). - Deprecating `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/etcd-io/etcd/pull/9819) instead. - Deprecating `etcd_debugging_mvcc_put_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_put_total`](https://github.com/etcd-io/etcd/pull/10962) instead. - Deprecating `etcd_debugging_mvcc_delete_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_delete_total`](https://github.com/etcd-io/etcd/pull/10962) instead. - Deprecating `etcd_debugging_mvcc_range_total` Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_range_total`](https://github.com/etcd-io/etcd/pull/10968) instead. - Deprecating `etcd_debugging_mvcc_txn_total`Prometheus metric (to be removed in v3.5). Use [`etcd_mvcc_txn_total`](https://github.com/etcd-io/etcd/pull/10968) instead. ### Security, Authentication See [security doc](https://etcd.io/docs/latest/op-guide/security/) for more details. - Support TLS cipher suite whitelisting. - To block [weak cipher suites](https://github.com/etcd-io/etcd/issues/8320). - TLS handshake fails when client hello is requested with invalid cipher suites. - Add [`etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/9801) flag. - If empty, Go auto-populates the list. - Add [`etcd --host-whitelist`](https://github.com/etcd-io/etcd/pull/9372) flag, [`etcdserver.Config.HostWhitelist`](https://github.com/etcd-io/etcd/pull/9372), and [`embed.Config.HostWhitelist`](https://github.com/etcd-io/etcd/pull/9372), to prevent ["DNS Rebinding"](https://en.wikipedia.org/wiki/DNS_rebinding) attack. - Any website can simply create an authorized DNS name, and direct DNS to `"localhost"` (or any other address). Then, all HTTP endpoints of etcd server listening on `"localhost"` becomes accessible, thus vulnerable to [DNS rebinding attacks (CVE-2018-5702)](https://bugs.chromium.org/p/project-zero/issues/detail?id=1447#c2). - Client origin enforce policy works as follow: - If client connection is secure via HTTPS, allow any hostnames.. - If client connection is not secure and `"HostWhitelist"` is not empty, only allow HTTP requests whose Host field is listed in whitelist. - By default, `"HostWhitelist"` is `"*"`, which means insecure server allows all client HTTP requests. - Note that the client origin policy is enforced whether authentication is enabled or not, for tighter controls. - When specifying hostnames, loopback addresses are not added automatically. To allow loopback interfaces, add them to whitelist manually (e.g. `"localhost"`, `"127.0.0.1"`, etc.). - e.g. `etcd --host-whitelist example.com`, then the server will reject all HTTP requests whose Host field is not `example.com` (also rejects requests to `"localhost"`). - Support [`etcd --cors`](https://github.com/etcd-io/etcd/pull/9490) in v3 HTTP requests (gRPC gateway). - Support [`ttl` field for `etcd` Authentication JWT token](https://github.com/etcd-io/etcd/pull/8302). - e.g. `etcd --auth-token jwt,pub-key=,priv-key=,sign-method=,ttl=5m`. - Allow empty token provider in [`etcdserver.ServerConfig.AuthToken`](https://github.com/etcd-io/etcd/pull/9369). - Fix [TLS reload](https://github.com/etcd-io/etcd/pull/9570) when [certificate SAN field only includes IP addresses but no domain names](https://github.com/etcd-io/etcd/issues/9541). - In Go, server calls `(*tls.Config).GetCertificate` for TLS reload if and only if server's `(*tls.Config).Certificates` field is not empty, or `(*tls.ClientHelloInfo).ServerName` is not empty with a valid SNI from the client. Previously, etcd always populates `(*tls.Config).Certificates` on the initial client TLS handshake, as non-empty. Thus, client was always expected to supply a matching SNI in order to pass the TLS verification and to trigger `(*tls.Config).GetCertificate` to reload TLS assets. - However, a certificate whose SAN field does [not include any domain names but only IP addresses](https://github.com/etcd-io/etcd/issues/9541) would request `*tls.ClientHelloInfo` with an empty `ServerName` field, thus failing to trigger the TLS reload on initial TLS handshake; this becomes a problem when expired certificates need to be replaced online. - Now, `(*tls.Config).Certificates` is created empty on initial TLS client handshake, first to trigger `(*tls.Config).GetCertificate`, and then to populate rest of the certificates on every new TLS connection, even when client SNI is empty (e.g. cert only includes IPs). ### etcd server - Add [`rpctypes.ErrLeaderChanged`](https://github.com/etcd-io/etcd/pull/10094). - Now linearizable requests with read index would fail fast when there is a leadership change, instead of waiting until context timeout. - Add [`etcd --initial-election-tick-advance`](https://github.com/etcd-io/etcd/pull/9591) flag to configure initial election tick fast-forward. - By default, `etcd --initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger. - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election. - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout. - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities. - Now, this can be disabled by setting `etcd --initial-election-tick-advance=false`. - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `etcd --initial-election-tick-advance` at the cost of slow initial bootstrap. - If single-node, it advances ticks regardless. - Address [disruptive rejoining follower node](https://github.com/etcd-io/etcd/issues/9333). - Add [`etcd --pre-vote`](https://github.com/etcd-io/etcd/pull/9352) flag to enable to run an additional Raft election phase. - For instance, a flaky(or rejoining) member may drop in and out, and start campaign. This member will end up with a higher term, and ignore all incoming messages with lower term. In this case, a new leader eventually need to get elected, thus disruptive to cluster availability. Raft implements Pre-Vote phase to prevent this kind of disruptions. If enabled, Raft runs an additional phase of election to check if pre-candidate can get enough votes to win an election. - `etcd --pre-vote=false` by default. - v3.5 will enable `etcd --pre-vote=true` by default. - Add `etcd --experimental-compaction-batch-limit` to [sets the maximum revisions deleted in each compaction batch](https://github.com/etcd-io/etcd/pull/11034). - Reduced default compaction batch size from 10k revisions to 1k revisions to improve p99 latency during compactions and reduced wait between compactions from 100ms to 10ms. - Add [`etcd --discovery-srv-name`](https://github.com/etcd-io/etcd/pull/8690) flag to support custom DNS SRV name with discovery. - If not given, etcd queries `_etcd-server-ssl._tcp.[YOUR_HOST]` and `_etcd-server._tcp.[YOUR_HOST]`. - If `etcd --discovery-srv-name="foo"`, then query `_etcd-server-ssl-foo._tcp.[YOUR_HOST]` and `_etcd-server-foo._tcp.[YOUR_HOST]`. - Useful for operating multiple etcd clusters under the same domain. - Support TLS cipher suite whitelisting. - To block [weak cipher suites](https://github.com/etcd-io/etcd/issues/8320). - TLS handshake fails when client hello is requested with invalid cipher suites. - Add [`etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/9801) flag. - If empty, Go auto-populates the list. - Support [`etcd --cors`](https://github.com/etcd-io/etcd/pull/9490) in v3 HTTP requests (gRPC gateway). - Rename [`etcd --log-output` to `etcd --log-outputs`](https://github.com/etcd-io/etcd/pull/9624) to support multiple log outputs. - **`etcd --log-output` will be deprecated in v3.5**. - Add [`etcd --logger`](https://github.com/etcd-io/etcd/pull/9572) flag to support [structured logger and multiple log outputs](https://github.com/etcd-io/etcd/issues/9438) in server-side. - **`etcd --logger=capnslog` will be deprecated in v3.5**. - Main motivation is to promote automated etcd monitoring, rather than looking back server logs when it starts breaking. Future development will make etcd log as few as possible, and make etcd easier to monitor with metrics and alerts. - `etcd --logger=capnslog --log-outputs=default` is the default setting and same as previous etcd server logging format. - `etcd --logger=zap --log-outputs=default` is not supported when `etcd --logger=zap`. - Use `etcd --logger=zap --log-outputs=stderr` instead. - Or, use `etcd --logger=zap --log-outputs=systemd/journal` to send logs to the local systemd journal. - Previously, if etcd parent process ID (PPID) is 1 (e.g. run with systemd), `etcd --logger=capnslog --log-outputs=default` redirects server logs to local systemd journal. And if write to journald fails, it writes to `os.Stderr` as a fallback. - However, even with PPID 1, it can fail to dial systemd journal (e.g. run embedded etcd with Docker container). Then, [every single log write will fail](https://github.com/etcd-io/etcd/pull/9729) and fall back to `os.Stderr`, which is inefficient. - To avoid this problem, systemd journal logging must be configured manually. - `etcd --logger=zap --log-outputs=stderr` will log server operations in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig) and writes logs to `os.Stderr`. Use this to override journald log redirects. - `etcd --logger=zap --log-outputs=stdout` will log server operations in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig) and writes logs to `os.Stdout` Use this to override journald log redirects. - `etcd --logger=zap --log-outputs=a.log` will log server operations in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig) and writes logs to the specified file `a.log`. - `etcd --logger=zap --log-outputs=a.log,b.log,c.log,stdout` [writes server logs to multiple files `a.log`, `b.log` and `c.log` at the same time](https://github.com/etcd-io/etcd/pull/9579) and outputs to `os.Stderr`, in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig). - `etcd --logger=zap --log-outputs=/dev/null` will discard all server logs. - Add [`etcd --log-level`](https://github.com/etcd-io/etcd/pull/10947) flag to support log level. - v3.5 will deprecate `etcd --debug` flag in favor of `etcd --log-level=debug`. - Add [`etcd --backend-batch-limit`](https://github.com/etcd-io/etcd/pull/10283) flag. - Add [`etcd --backend-batch-interval`](https://github.com/etcd-io/etcd/pull/10283) flag. - Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/etcd-io/etcd/pull/9281). - "unsynced" watcher is watcher that needs to be in sync with events that have happened. - That is, "unsynced" watcher is the slow watcher that was requested on old revision. - "unsynced" watcher restore operation was not correctly populating its underlying watcher group. - Which possibly causes [missing events from "unsynced" watchers](https://github.com/etcd-io/etcd/issues/9086). - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node. - Fix [`mvcc` server panic from restore operation](https://github.com/etcd-io/etcd/pull/9775). - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X, **etcd server panicked** during snapshot restore operation. - Now, this server-side panic has been fixed. - Fix [server panic on invalid Election Proclaim/Resign HTTP(S) requests](https://github.com/etcd-io/etcd/pull/9379). - Previously, wrong-formatted HTTP requests to Election API could trigger panic in etcd server. - e.g. `curl -L http://localhost:2379/v3/election/proclaim -X POST -d '{"value":""}'`, `curl -L http://localhost:2379/v3/election/resign -X POST -d '{"value":""}'`. - Fix [revision-based compaction retention parsing](https://github.com/etcd-io/etcd/pull/9339). - Previously, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` was [translated to revision retention 3600000000000](https://github.com/etcd-io/etcd/issues/9337). - Now, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` is correctly parsed as revision retention 1. - Prevent [overflow by large `TTL` values for `Lease` `Grant`](https://github.com/etcd-io/etcd/pull/9399). - `TTL` parameter to `Grant` request is unit of second. - Leases with too large `TTL` values exceeding `math.MaxInt64` [expire in unexpected ways](https://github.com/etcd-io/etcd/issues/9374). - Server now returns `rpctypes.ErrLeaseTTLTooLarge` to client, when the requested `TTL` is larger than *9,000,000,000 seconds* (which is >285 years). - Again, etcd `Lease` is meant for short-periodic keepalives or sessions, in the range of seconds or minutes. Not for hours or days! - Fix [expired lease revoke](https://github.com/etcd-io/etcd/pull/10693). - Fix ["the key is not deleted when the bound lease expires"](https://github.com/etcd-io/etcd/issues/10686). - Enable etcd server [`raft.Config.CheckQuorum` when starting with `ForceNewCluster`](https://github.com/etcd-io/etcd/pull/9347). - Allow [non-WAL files in `etcd --wal-dir` directory](https://github.com/etcd-io/etcd/pull/9743). - Previously, existing files such as [`lost+found`](https://github.com/etcd-io/etcd/issues/7287) in WAL directory prevent etcd server boot. - Now, WAL directory that contains only `lost+found` or a file that's not suffixed with `.wal` is considered non-initialized. - Fix [`ETCD_CONFIG_FILE` env variable parsing in `etcd`](https://github.com/etcd-io/etcd/pull/10762). - Fix [race condition in `rafthttp` transport pause/resume](https://github.com/etcd-io/etcd/pull/10826). - Fix [server crash from creating an empty role](https://github.com/etcd-io/etcd/pull/10907). - Previously, creating a role with an empty name crashed etcd server with an error code `Unavailable`. - Now, creating a role with an empty name is not allowed with an error code `InvalidArgument`. ### API - Add `isLearner` field to `etcdserverpb.Member`, `etcdserverpb.MemberAddRequest` and `etcdserverpb.StatusResponse` as part of [raft learner implementation](https://github.com/etcd-io/etcd/pull/10725). - Add `MemberPromote` rpc to `etcdserverpb.Cluster` interface and the corresponding `MemberPromoteRequest` and `MemberPromoteResponse` as part of [raft learner implementation](https://github.com/etcd-io/etcd/pull/10725). - Add [`snapshot`](https://github.com/etcd-io/etcd/pull/9118) package for snapshot restore/save operations (see [`godoc.org/github.com/etcd/clientv3/snapshot`](https://godoc.org/github.com/coreos/etcd/clientv3/snapshot) for more). - Add [`watch_id` field to `etcdserverpb.WatchCreateRequest`](https://github.com/etcd-io/etcd/pull/9065) to allow user-provided watch ID to `mvcc`. - Corresponding `watch_id` is returned via `etcdserverpb.WatchResponse`, if any. - Add [`fragment` field to `etcdserverpb.WatchCreateRequest`](https://github.com/etcd-io/etcd/pull/9291) to request etcd server to [split watch events](https://github.com/etcd-io/etcd/issues/9294) when the total size of events exceeds `etcd --max-request-bytes` flag value plus gRPC-overhead 512 bytes. - The default server-side request bytes limit is `embed.DefaultMaxRequestBytes` which is 1.5 MiB plus gRPC-overhead 512 bytes. - If watch response events exceed this server-side request limit and watch request is created with `fragment` field `true`, the server will split watch events into a set of chunks, each of which is a subset of watch events below server-side request limit. - Useful when client-side has limited bandwidths. - For example, watch response contains 10 events, where each event is 1 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB. Then, server will send 10 separate fragmented events to the client. - For example, watch response contains 5 events, where each event is 2 MiB. And server `etcd --max-recv-bytes` flag value is 1 MiB and `clientv3.Config.MaxCallRecvMsgSize` is 1 MiB. Then, server will try to send 5 separate fragmented events to the client, and the client will error with `"code = ResourceExhausted desc = grpc: received message larger than max (...)"`. - Client must implement fragmented watch event merge (which `clientv3` does in etcd v3.4). - Add [`raftAppliedIndex` field to `etcdserverpb.StatusResponse`](https://github.com/etcd-io/etcd/pull/9176) for current Raft applied index. - Add [`errors` field to `etcdserverpb.StatusResponse`](https://github.com/etcd-io/etcd/pull/9206) for server-side error. - e.g. `"etcdserver: no leader", "NOSPACE", "CORRUPT"` - Add [`dbSizeInUse` field to `etcdserverpb.StatusResponse`](https://github.com/etcd-io/etcd/pull/9256) for actual DB size after compaction. - Add [`WatchRequest.WatchProgressRequest`](https://github.com/etcd-io/etcd/pull/9869). - To manually trigger broadcasting watch progress event (empty watch response with latest header) to all associated watch streams. - Think of it as `WithProgressNotify` that can be triggered manually. Note: **v3.5 will deprecate `etcd --log-package-levels` flag for `capnslog`**; `etcd --logger=zap --log-outputs=stderr` will the default. **v3.5 will deprecate `[CLIENT-URL]/config/local/log` endpoint.** ### Package `embed` - Add [`embed.Config.CipherSuites`](https://github.com/etcd-io/etcd/pull/9801) to specify a list of supported cipher suites for TLS handshake between client/server and peers. - If empty, Go auto-populates the list. - Both `embed.Config.ClientTLSInfo.CipherSuites` and `embed.Config.CipherSuites` cannot be non-empty at the same time. - If not empty, specify either `embed.Config.ClientTLSInfo.CipherSuites` or `embed.Config.CipherSuites`. - Add [`embed.Config.InitialElectionTickAdvance`](https://github.com/etcd-io/etcd/pull/9591) to enable/disable initial election tick fast-forward. - `embed.NewConfig()` would return `*embed.Config` with `InitialElectionTickAdvance` as true by default. - Define [`embed.CompactorModePeriodic`](https://godoc.org/github.com/etcd-io/etcd/embed#pkg-variables) for `compactor.ModePeriodic`. - Define [`embed.CompactorModeRevision`](https://godoc.org/github.com/etcd-io/etcd/embed#pkg-variables) for `compactor.ModeRevision`. - Change [`embed.Config.CorsInfo` in `*cors.CORSInfo` type to `embed.Config.CORS` in `map[string]struct{}` type](https://github.com/etcd-io/etcd/pull/9490). - Remove [`embed.Config.SetupLogging`](https://github.com/etcd-io/etcd/pull/9572). - Now logger is set up automatically based on [`embed.Config.Logger`, `embed.Config.LogOutputs`, `embed.Config.Debug` fields](https://github.com/etcd-io/etcd/pull/9572). - Add [`embed.Config.Logger`](https://github.com/etcd-io/etcd/pull/9518) to support [structured logger `zap`](https://github.com/uber-go/zap) in server-side. - Add [`embed.Config.LogLevel`](https://github.com/etcd-io/etcd/pull/10947). - Rename `embed.Config.SnapCount` field to [`embed.Config.SnapshotCount`](https://github.com/etcd-io/etcd/pull/9745), to be consistent with the flag name `etcd --snapshot-count`. - Rename [**`embed.Config.LogOutput`** to **`embed.Config.LogOutputs`**](https://github.com/etcd-io/etcd/pull/9624) to support multiple log outputs. - Change [**`embed.Config.LogOutputs`** type from `string` to `[]string`](https://github.com/etcd-io/etcd/pull/9579) to support multiple log outputs. - Add [`embed.Config.BackendBatchLimit`](https://github.com/etcd-io/etcd/pull/10283) field. - Add [`embed.Config.BackendBatchInterval`](https://github.com/etcd-io/etcd/pull/10283) field. - Make [`embed.DefaultEnableV2` `false` default](https://github.com/etcd-io/etcd/pull/10935). ### Package `pkg/adt` - Change [`pkg/adt.IntervalTree` from `struct` to `interface`](https://github.com/etcd-io/etcd/pull/10959). - See [`pkg/adt` README](https://github.com/etcd-io/etcd/tree/main/pkg/adt) and [`pkg/adt` godoc](https://godoc.org/go.etcd.io/etcd/pkg/adt). - Improve [`pkg/adt.IntervalTree` test coverage](https://github.com/etcd-io/etcd/pull/10959). - See [`pkg/adt` README](https://github.com/etcd-io/etcd/tree/main/pkg/adt) and [`pkg/adt` godoc](https://godoc.org/go.etcd.io/etcd/pkg/adt). - Fix [Red-Black tree to maintain black-height property](https://github.com/etcd-io/etcd/pull/10978). - Previously, delete operation violates [black-height property](https://github.com/etcd-io/etcd/issues/10965). ### Package `integration` - Add [`CLUSTER_DEBUG` to enable test cluster logging](https://github.com/etcd-io/etcd/pull/9678). - Deprecated `capnslog` in integration tests. ### client v3 - Add [`MemberAddAsLearner`](https://github.com/etcd-io/etcd/pull/10725) to `Clientv3.Cluster` interface. This API is used to add a learner member to etcd cluster. - Add [`MemberPromote`](https://github.com/etcd-io/etcd/pull/10727) to `Clientv3.Cluster` interface. This API is used to promote a learner member in etcd cluster. - Client may receive [`rpctypes.ErrLeaderChanged`](https://github.com/etcd-io/etcd/pull/10094) from server. - Now linearizable requests with read index would fail fast when there is a leadership change, instead of waiting until context timeout. - Add [`WithFragment` `OpOption`](https://github.com/etcd-io/etcd/pull/9291) to support [watch events fragmentation](https://github.com/etcd-io/etcd/issues/9294) when the total size of events exceeds `etcd --max-request-bytes` flag value plus gRPC-overhead 512 bytes. - Watch fragmentation is disabled by default. - The default server-side request bytes limit is `embed.DefaultMaxRequestBytes` which is 1.5 MiB plus gRPC-overhead 512 bytes. - If watch response events exceed this server-side request limit and watch request is created with `fragment` field `true`, the server will split watch events into a set of chunks, each of which is a subset of watch events below server-side request limit. - Useful when client-side has limited bandwidths. - For example, watch response contains 10 events, where each event is 1 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB. Then, server will send 10 separate fragmented events to the client. - For example, watch response contains 5 events, where each event is 2 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB and `clientv3.Config.MaxCallRecvMsgSize` is 1 MiB. Then, server will try to send 5 separate fragmented events to the client, and the client will error with `"code = ResourceExhausted desc = grpc: received message larger than max (...)"`. - Add [`Watcher.RequestProgress` method](https://github.com/etcd-io/etcd/pull/9869). - To manually trigger broadcasting watch progress event (empty watch response with latest header) to all associated watch streams. - Think of it as `WithProgressNotify` that can be triggered manually. - Fix [lease keepalive interval updates when response queue is full](https://github.com/etcd-io/etcd/pull/9952). - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/etcd-io/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration. - Change [snapshot file permissions](https://github.com/etcd-io/etcd/pull/9977): On Linux, the snapshot file changes from readable by all (mode 0644) to readable by the user only (mode 0600). - Client may choose to send keepalive pings to server using [`PermitWithoutStream`](https://github.com/etcd-io/etcd/pull/10146). - By setting `PermitWithoutStream` to true, client can send keepalive pings to server without any active streams(RPCs). In other words, it allows sending keepalive pings with unary or simple RPC calls. - `PermitWithoutStream` is set to false by default. - Fix logic on [release lock key if cancelled](https://github.com/etcd-io/etcd/pull/10153) in `clientv3/concurrency` package. - Fix [`(*Client).Endpoints()` method race condition](https://github.com/etcd-io/etcd/pull/10595). - Deprecated [`grpc.ErrClientConnClosing`](https://github.com/etcd-io/etcd/pull/10981). - `clientv3` and `proxy/grpcproxy` now does not return `grpc.ErrClientConnClosing`. - `grpc.ErrClientConnClosing` has been [deprecated in gRPC >= 1.10](https://github.com/grpc/grpc-go/pull/1854). - Use `clientv3.IsConnCanceled(error)` or `google.golang.org/grpc/status.FromError(error)` instead. ### etcdctl v3 - Make [`ETCDCTL_API=3 etcdctl` default](https://github.com/etcd-io/etcd/issues/9600). - Now, `etcdctl set foo bar` must be `ETCDCTL_API=2 etcdctl set foo bar`. - Now, `ETCDCTL_API=3 etcdctl put foo bar` could be just `etcdctl put foo bar`. - Add [`etcdctl member add --learner` and `etcdctl member promote`](https://github.com/etcd-io/etcd/pull/10725) to add and promote raft learner member in etcd cluster. - Add [`etcdctl --password`](https://github.com/etcd-io/etcd/pull/9730) flag. - To support [`:` character in user name](https://github.com/etcd-io/etcd/issues/9691). - e.g. `etcdctl --user user --password password get foo` - Add [`etcdctl user add --new-user-password`](https://github.com/etcd-io/etcd/pull/9730) flag. - Add [`etcdctl check datascale`](https://github.com/etcd-io/etcd/pull/9185) command. - Add [`etcdctl check datascale --auto-compact, --auto-defrag`](https://github.com/etcd-io/etcd/pull/9351) flags. - Add [`etcdctl check perf --auto-compact, --auto-defrag`](https://github.com/etcd-io/etcd/pull/9330) flags. - Add [`etcdctl defrag --cluster`](https://github.com/etcd-io/etcd/pull/9390) flag. - Add ["raft applied index" field to `endpoint status`](https://github.com/etcd-io/etcd/pull/9176). - Add ["errors" field to `endpoint status`](https://github.com/etcd-io/etcd/pull/9206). - Add [`etcdctl endpoint health --write-out` support](https://github.com/etcd-io/etcd/pull/9540). - Previously, [`etcdctl endpoint health --write-out json` did not work](https://github.com/etcd-io/etcd/issues/9532). - Add [missing newline in `etcdctl endpoint health`](https://github.com/etcd-io/etcd/pull/10793). - Fix [`etcdctl watch [key] [range_end] -- [exec-command…]`](https://github.com/etcd-io/etcd/pull/9688) parsing. - Previously, `ETCDCTL_API=3 etcdctl watch foo -- echo watch event received` panicked. - Fix [`etcdctl move-leader` command for TLS-enabled endpoints](https://github.com/etcd-io/etcd/pull/9807). - Add [`progress` command to `etcdctl watch --interactive`](https://github.com/etcd-io/etcd/pull/9869). - To manually trigger broadcasting watch progress event (empty watch response with latest header) to all associated watch streams. - Think of it as `WithProgressNotify` that can be triggered manually. - Add [timeout](https://github.com/etcd-io/etcd/pull/10301) to `etcdctl snapshot save`. - User can specify timeout of `etcdctl snapshot save` command using flag `--command-timeout`. - Fix etcdctl to [strip out insecure endpoints from DNS SRV records when using discovery](https://github.com/etcd-io/etcd/pull/10443) ### gRPC proxy - Fix [etcd server panic from restore operation](https://github.com/etcd-io/etcd/pull/9775). - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X, **etcd server panicked** during snapshot restore operation. - Especially, gRPC proxy was affected, since it detects a leader loss with a key `"proxy-namespace__lostleader"` and a watch revision `"int64(math.MaxInt64 - 2)"`. - Now, this server-side panic has been fixed. - Fix [memory leak in cache layer](https://github.com/etcd-io/etcd/pull/10327). - Change [gRPC proxy to expose etcd server endpoint /metrics](https://github.com/etcd-io/etcd/pull/10618). - The metrics that were exposed via the proxy were not etcd server members but instead the proxy itself. ### gRPC gateway - Replace [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3beta` with [`/v3`](https://github.com/etcd-io/etcd/pull/9298). - Deprecated [`/v3alpha`](https://github.com/etcd-io/etcd/pull/9298). - To deprecate [`/v3beta`](https://github.com/etcd-io/etcd/issues/9189) in v3.5. - In v3.4, `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. - Add API endpoints [`/{v3beta,v3}/lease/leases, /{v3beta,v3}/lease/revoke, /{v3beta,v3}/lease/timetolive`](https://github.com/etcd-io/etcd/pull/9450). - To deprecate [`/{v3beta,v3}/kv/lease/leases, /{v3beta,v3}/kv/lease/revoke, /{v3beta,v3}/kv/lease/timetolive`](https://github.com/etcd-io/etcd/issues/9430) in v3.5. - Support [`etcd --cors`](https://github.com/etcd-io/etcd/pull/9490) in v3 HTTP requests (gRPC gateway). ### Package `raft` - Fix [deadlock during PreVote migration process](https://github.com/etcd-io/etcd/pull/8525). - Add [`raft.ErrProposalDropped`](https://github.com/etcd-io/etcd/pull/9067). - Now [`(r *raft) Step` returns `raft.ErrProposalDropped`](https://github.com/etcd-io/etcd/pull/9137) if a proposal has been ignored. - e.g. a node is removed from cluster, or [`raftpb.MsgProp` arrives at current leader while there is an ongoing leadership transfer](https://github.com/etcd-io/etcd/issues/8975). - Improve [Raft `becomeLeader` and `stepLeader`](https://github.com/etcd-io/etcd/pull/9073) by keeping track of latest `pb.EntryConfChange` index. - Previously record `pendingConf` boolean field scanning the entire tail of the log, which can delay heartbeat send. - Fix [missing learner nodes on `(n *node) ApplyConfChange`](https://github.com/etcd-io/etcd/pull/9116). - Add [`raft.Config.MaxUncommittedEntriesSize`](https://github.com/etcd-io/etcd/pull/10167) to limit the total size of the uncommitted entries in bytes. - Once exceeded, raft returns `raft.ErrProposalDropped` error. - Prevent [unbounded Raft log growth](https://github.com/cockroachdb/cockroach/issues/27772). - There was a bug in [PR#10167](https://github.com/etcd-io/etcd/pull/10167) but fixed via [PR#10199](https://github.com/etcd-io/etcd/pull/10199). - Add [`raft.Ready.CommittedEntries` pagination using `raft.Config.MaxSizePerMsg`](https://github.com/etcd-io/etcd/pull/9982). - This prevents out-of-memory errors if the raft log has become very large and commits all at once. - Fix [correctness bug in CommittedEntries pagination](https://github.com/etcd-io/etcd/pull/10063). - Optimize [message send flow control](https://github.com/etcd-io/etcd/pull/9985). - Leader now sends more append entries if it has more non-empty entries to send after updating flow control information. - Now, Raft allows multiple in-flight append messages. - Optimize [memory allocation when boxing slice in `maybeCommit`](https://github.com/etcd-io/etcd/pull/10679). - By boxing a heap-allocated slice header instead of the slice header on the stack, we can avoid an allocation when passing through the sort.Interface interface. - Avoid [memory allocation in Raft entry `String` method](https://github.com/etcd-io/etcd/pull/10680). - Avoid [multiple memory allocations when merging stable and unstable log](https://github.com/etcd-io/etcd/pull/10684). - Extract [progress tracking into own component](https://github.com/etcd-io/etcd/pull/10683). - Add [package `raft/tracker`](https://github.com/etcd-io/etcd/pull/10807). - Optimize [string representation of `Progress`](https://github.com/etcd-io/etcd/pull/10882). - Make [relationship between `node` and `RawNode` explicit](https://github.com/etcd-io/etcd/pull/10803). - Prevent [learners from becoming leader](https://github.com/etcd-io/etcd/pull/10822). - Add [package `raft/quorum` to reason about committed indexes as well as vote outcomes for both majority and joint quorums](https://github.com/etcd-io/etcd/pull/10779). - Bundle [Voters and Learner into `raft/tracker.Config` struct](https://github.com/etcd-io/etcd/pull/10865). - Use [membership sets in progress tracking](https://github.com/etcd-io/etcd/pull/10779). - Implement [joint quorum computation](https://github.com/etcd-io/etcd/pull/10779). - Refactor [`raft/node.go` to centralize configuration change application](https://github.com/etcd-io/etcd/pull/10865). - Allow [voter to become learner through snapshot](https://github.com/etcd-io/etcd/pull/10864). - Add [package `raft/confchange` to internally support joint consensus](https://github.com/etcd-io/etcd/pull/10779). - Use [`RawNode` for node's event loop](https://github.com/etcd-io/etcd/pull/10892). - Add [`RawNode.Bootstrap` method](https://github.com/etcd-io/etcd/pull/10892). - Add [`raftpb.ConfChangeV2` to use joint quorums](https://github.com/etcd-io/etcd/pull/10914). - `raftpb.ConfChange` continues to work as today: it allows carrying out a single configuration change. A `pb.ConfChange` proposal gets added to the Raft log as such and is thus also observed by the app during Ready handling, and fed back to ApplyConfChange. - `raftpb.ConfChangeV2` allows joint configuration changes but will continue to carry out configuration changes in "one phase" (i.e. without ever entering a joint config) when this is possible. - `raftpb.ConfChangeV2` messages initiate configuration changes. They support both the simple "one at a time" membership change protocol and full Joint Consensus allowing for arbitrary changes in membership. - Change [`raftpb.ConfState.Nodes` to `raftpb.ConfState.Voters`](https://github.com/etcd-io/etcd/pull/10914). - Allow [learners to vote, but still learners do not count in quorum](https://github.com/etcd-io/etcd/pull/10998). - necessary in the situation in which a learner has been promoted (i.e. is now a voter) but has not learned about this yet. - Fix [restoring joint consensus](https://github.com/etcd-io/etcd/pull/11003). - Visit [`Progress` in stable order](https://github.com/etcd-io/etcd/pull/11004). - Proactively [probe newly added followers](https://github.com/etcd-io/etcd/pull/11037). - The general expectation in `tracker.Progress.Next == c.LastIndex` is that the follower has no log at all (and will thus likely need a snapshot), though the app may have applied a snapshot out of band before adding the replica (thus making the first index the better choice). - Previously, when the leader applied a new configuration that added voters, it would not immediately probe these voters, delaying when they would be caught up. ### Package `wal` - Add [`Verify` function to perform corruption check on WAL contents](https://github.com/etcd-io/etcd/pull/10603). - Fix [`wal` directory cleanup on creation failures](https://github.com/etcd-io/etcd/pull/10689). ### Tooling - Add [`etcd-dump-logs --entry-type`](https://github.com/etcd-io/etcd/pull/9628) flag to support WAL log filtering by entry type. - Add [`etcd-dump-logs --stream-decoder`](https://github.com/etcd-io/etcd/pull/9790) flag to support custom decoder. - Add [`SHA256SUMS`](https://github.com/etcd-io/etcd/pull/11087) file to release assets. - etcd maintainers are a distributed team, this change allows for releases to be cut and validation provided without requiring a signing key. ### Go - Require [*Go 1.12+*](https://github.com/etcd-io/etcd/pull/10045). - Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes. ### Dockerfile - [Rebase etcd image from Alpine to Debian](https://github.com/etcd-io/etcd/pull/10805) to improve security and maintenance effort for etcd release. --- ================================================ FILE: CHANGELOG/CHANGELOG-3.5.md ================================================ Previous change logs can be found at [CHANGELOG-3.4](https://github.com/etcd-io/etcd/blob/main/CHANGELOG/CHANGELOG-3.4.md). --- ## v3.5.28 (TBC) ### etcd server - [Ensure the metrics interceptor runs before other interceptors so that metrics remain up to date](https://github.com/etcd-io/etcd/pull/21336) - Fix [Race between read index and leader change](https://github.com/etcd-io/etcd/pull/21387) - Fix [Stale reads caused by process pausing](https://github.com/etcd-io/etcd/pull/21421) ### Package `clientv3` - [Print the endpoint the grpc request was actually sent to in unary interceptor](https://github.com/etcd-io/etcd/pull/21380) ### etcd grpc-proxy - [server/etcdmain: fix startup deadlock in grpcproxy](https://github.com/etcd-io/etcd/pull/21356) ### etcdctl - Fix [slice bounds trimming single-quoted args in Argify](https://github.com/etcd-io/etcd/pull/21403) ### Dependencies - [Bump go.opentelemetry.io/otel/sdk to v1.40.0 to resolve https://pkg.go.dev/vuln/GO-2026-4394](https://github.com/etcd-io/etcd/pull/21338) - Compile binaries using [go 1.25.7](https://github.com/etcd-io/etcd/pull/21405) - [Bump golang.org/x/net to v0.51.0 to resolve GO-2026-4559](https://github.com/etcd-io/etcd/pull/21441) --- ## v3.5.27 (2026-02-13) ### Package `clientv3` - [Remove the use of grpc-go's Metadata field](https://github.com/etcd-io/etcd/pull/21242) ### Dependencies - Compile binaries using [go 1.24.13](https://github.com/etcd-io/etcd/pull/21266). This addresses [CVE-2025-61726](https://github.com/advisories/GHSA-gm9r-q53w-2gh4), [CVE-2025-61731](https://github.com/advisories/GHSA-xvqr-69v8-f3gv), and [CVE-2025-61732](https://github.com/advisories/GHSA-8jvr-vh7g-f8gx). --- ## v3.5.26 (2025-12-17) ### etcd server - [Print token fingerprint instead of the original tokens in log messages](https://github.com/etcd-io/etcd/pull/20942) - Fix [zombie members in v3store](https://github.com/etcd-io/etcd/pull/20995) ### etcdctl - [Fix a typo of 'etcdctl snapshot restore' command](https://github.com/etcd-io/etcd/pull/20948). ### Dependencies - Compile binaries using [go 1.24.11](https://github.com/etcd-io/etcd/pull/20999). - Bump [golang.org/x/crypto to 0.45.0 to address CVE-2025-47914, and CVE-2025-58181](https://github.com/etcd-io/etcd/pull/21023). --- ## v3.5.25 (2025-11-11) ### etcd server - Fix [`--force-new-cluster can't clean up learners after creating snapshot`](https://github.com/etcd-io/etcd/pull/20896) ### etcdutl - Add [flag `--wal-dir` to `etcdutl check v2store` command to support dedicated WAL directory](https://github.com/etcd-io/etcd/pull/20886) ### Dependencies - Compile binaries using [go 1.24.10](https://github.com/etcd-io/etcd/pull/20902). --- ## v3.5.24 (2025-10-22) ### etcd server - [Reject watch request with -1 revision to prevent invalid resync behavior on uncompacted etcd](https://github.com/etcd-io/etcd/pull/20709) - [Change the TLS handshake 'EOF' errors to DEBUG not to spam logs](https://github.com/etcd-io/etcd/pull/20751) - Fix [Learner promotion not being persisted into v3store may be propagated across multiple upgrades](https://github.com/etcd-io/etcd/pull/20797) ### Dependencies - Compile binaries using [go 1.24.9](https://github.com/etcd-io/etcd/pull/20806). --- ## v3.5.23 (2025-09-19) ### etcd server - Fix [etcd may return success for leaseRenew request even when the lease is revoked](https://github.com/etcd-io/etcd/pull/20616) - Fix [potential data corruption when applySnapshot and defragment happen concurrently](https://github.com/etcd-io/etcd/pull/20653) ### Dependencies - Compile binaries using [go 1.24.7](https://github.com/etcd-io/etcd/pull/20665). - [Bump bbolt to v1.3.12](https://github.com/etcd-io/etcd/pull/20514). --- ## v3.5.22 (2025-07-22) ### etcd server - Fix [the compaction pause duration metric is not emitted for every compaction batch](https://github.com/etcd-io/etcd/pull/19771) - Fix [mvcc: avoid double decrement of watcher gauge on close/cancel race](https://github.com/etcd-io/etcd/pull/20066) - Fix [Watch on future revision returns old events or notifications](https://github.com/etcd-io/etcd/pull/20290) - Fix [`--force-new-cluster` can't remove all other members in a corner case](https://github.com/etcd-io/etcd/pull/20339) - Fix [v2store check (IsMetaStoreOnly) returns wrong result even there is no any auth data](https://github.com/etcd-io/etcd/pull/20357) - Improve [help message for --quota-backend-bytes](https://github.com/etcd-io/etcd/pull/20380) ### Package `clientv3` - [Replace `resolver.State.Addresses` with `resolver.State.Endpoint.Addresses`](https://github.com/etcd-io/etcd/pull/19783). - [Deprecated the Metadata field in the Endpoint struct from the client/v3/naming/endpoints package](https://github.com/etcd-io/etcd/pull/19846). ### Dependencies - Compile binaries using [go 1.23.11](https://github.com/etcd-io/etcd/pull/20321) --- ## v3.5.21 (2025-03-27) ### Dependencies - Bump [github.com/golang-jwt/jwt/v4 from 4.5.1 to 4.5.2 to address CVE-2025-30204](https://github.com/etcd-io/etcd/pull/19646). - Bump [bump golang.org/x/net from v0.36.0 to v0.38.0 to address CVE-2025-22870 and CVE-2025-22872](https://github.com/etcd-io/etcd/pull/19686). --- ## v3.5.20 (2025-03-21) ### etcd server - Fix [the learner promotion changes not being persisted into v3store (bbolt)](https://github.com/etcd-io/etcd/pull/19563) - Update [the RLock in Demoted method for read-only access to expiry](https://github.com/etcd-io/etcd/pull/19445) ### etcdctl - Fix [command `etcdctl member promote` doesn't support json output](https://github.com/etcd-io/etcd/pull/19602) ### etcd grpc-proxy - Fix [grpcproxy can get stuck in and endless loop causing high CPU usage](https://github.com/etcd-io/etcd/pull/19562) --- ## v3.5.19 (2025-03-05) ### etcd server - Backport [add learner status check to readyz endpoint](https://github.com/etcd-io/etcd/pull/19280). - Fix [performance regression due to uncertain compaction sleep interval](https://github.com/etcd-io/etcd/pull/19405). ### `tools/benchmark` - Backport [add mixed read-write performance evaluation scripts](https://github.com/etcd-io/etcd/pull/19275). ### Dependencies - Compile binaries using [go 1.23.7](https://github.com/etcd-io/etcd/pull/19528). - Bump [golang.org/x/crypto to v0.35.0 to address CVE-2025-22869](https://github.com/etcd-io/etcd/pull/19478). - Bump [golang.org/x/net to v0.36.0 to address CVE-2025-22870](https://github.com/etcd-io/etcd/pull/19530). --- ## v3.5.18 (2025-01-24) ### etcd server - Avoid deadlock in etcd.Close when stopping during bootstrapping, see https://github.com/etcd-io/etcd/pull/19167 and https://github.com/etcd-io/etcd/pull/19258. - [Print warning messages if any of the deprecated v2store related flags is set](https://github.com/etcd-io/etcd/pull/18999) - Fix [missing delete event on watch opened on same revision as compaction request](https://github.com/etcd-io/etcd/pull/19249) ### Package `clientv3` - Fix [runtime panic that occurs when KeepAlive is called with a Context implemented by an uncomparable type](https://github.com/etcd-io/etcd/pull/18937) ### etcdutl v3 - Add [command `etcdutl check v2store` to offline check whether v2store contains custom content](https://github.com/etcd-io/etcd/pull/19113) ### etcd grpc-proxy - Add [`tls min/max version to grpc proxy`](https://github.com/etcd-io/etcd/pull/18829) to support setting TLS min and max version. ### Dependencies - Bump [golang-jwt/jwt to 4.5.1 to address GO-2024-3250](https://github.com/etcd-io/etcd/pull/18899). - Compile binaries using [go 1.22.11](https://github.com/etcd-io/etcd/pull/19211). - Bump [golang.org/x/crypto to 0.32.0 to address CVE-2024-45337](https://github.com/etcd-io/etcd/pull/19154). - Bump [golang.org/x/net to 0.34.0 to address CVE-2024-45338](https://github.com/etcd-io/etcd/pull/19158). --- ## v3.5.17 (2024-11-12) ### etcd server - Fix [watchserver related goroutine leakage](https://github.com/etcd-io/etcd/pull/18784) - Fix [risk of a partial write txn being applied](https://github.com/etcd-io/etcd/pull/18799) - Fix [panicking occurred due to improper error handling during defragmentation](https://github.com/etcd-io/etcd/pull/18842) - Fix [close temp file(s) in case an error happens during defragmentation](https://github.com/etcd-io/etcd/pull/18854) ### Dependencies - Compile binaries using [go 1.22.9](https://github.com/etcd-io/etcd/pull/18849). --- ## v3.5.16 (2024-09-10) ### etcd server - Fix [performance regression issue caused by the `ensureLeadership` in lease renew](https://github.com/etcd-io/etcd/pull/18439). - [Keep the tombstone during compaction if it happens to be the compaction revision](https://github.com/etcd-io/etcd/pull/18474) - Add [`etcd --experimental-compaction-sleep-interval`](https://github.com/etcd-io/etcd/pull/18514) flag to control the sleep interval between each compaction batch. ### Dependencies - Compile binaries using [go 1.22.7](https://github.com/etcd-io/etcd/pull/18550). - Upgrade [bbolt to v1.3.11](https://github.com/etcd-io/etcd/pull/18489). --- ## v3.5.15 (2024-07-19) ### etcd server - Fix [add prometheus metric registration for metric `etcd_disk_wal_write_duration_seconds`](https://github.com/etcd-io/etcd/pull/18174). - Add [Support multiple values for allowed client and peer TLS identities](https://github.com/etcd-io/etcd/pull/18160) - Fix [noisy logs from simple auth token expiration by reducing log level to debug](https://github.com/etcd-io/etcd/pull/18245) - [Differentiate the warning message for rejected client and peer connections](https://github.com/etcd-io/etcd/pull/18319) ### Package clientv3 - [Print gRPC metadata in guaranteed order using the official go fmt pkg](https://github.com/etcd-io/etcd/pull/18312). ### Dependencies - Compile binaries using [go 1.21.12](https://github.com/etcd-io/etcd/pull/18271). - [Fully address CVE-2023-45288 and fix govulncheck CI check](https://github.com/etcd-io/etcd/pull/18170) ## v3.5.14 (2024-05-29) ### etcd server - Fix [LeaseTimeToLive returns error if leader changed](https://github.com/etcd-io/etcd/pull/17704). - Add [metrics `etcd_disk_wal_write_duration_seconds`](https://github.com/etcd-io/etcd/pull/17616). - Fix [ignore raft messages if member id mismatch](https://github.com/etcd-io/etcd/pull/17813). - Update [the compaction log when bootstrap](https://github.com/etcd-io/etcd/pull/17830). - Fix [Revision decreasing after panic during compaction](https://github.com/etcd-io/etcd/pull/17865) - Add [`etcd --experimental-stop-grpc-service-on-defrag`](https://github.com/etcd-io/etcd/pull/17914) to enable client failover on defrag. - Add [support for `AllowedCN` and `AllowedHostname` through config file](https://github.com/etcd-io/etcd/pull/18063) ### etcdutl v3 - Add [`--initial-memory-map-size` to `snapshot restore` to avoid memory allocation issues](https://github.com/etcd-io/etcd/pull/17977) ### Package `clientv3` - Add [requests retry when receiving ErrGPRCNotSupportedForLearner and endpoints > 1](https://github.com/etcd-io/etcd/pull/17641). - Fix [initialization for mu in client context](https://github.com/etcd-io/etcd/pull/17699). ### Dependencies - Compile binaries using [go 1.21.10](https://github.com/etcd-io/etcd/pull/17980). - Upgrade [bbolt to v1.3.10](https://github.com/etcd-io/etcd/pull/17943). --- ## v3.5.13 (2024-03-29) ### etcd server - Fix leases wrongly revoked by the leader by [ignoring old leader's leases revoking request](https://github.com/etcd-io/etcd/pull/17425). - Fix [no progress notification being sent for watch that doesn't get any events](https://github.com/etcd-io/etcd/pull/17566). - Fix [watch event loss after compaction](https://github.com/etcd-io/etcd/pull/17612). ### Package `clientv3` - Add [client backoff and retry config options](https://github.com/etcd-io/etcd/pull/17363). - [Ignore SetKeepAlivePeriod errors on OpenBSD](https://github.com/etcd-io/etcd/pull/17387). - [Support unix/unixs socket in client or peer URLs](https://github.com/etcd-io/etcd/pull/15940) ### gRPC Proxy - Add [three flags (see below) for grpc-proxy](https://github.com/etcd-io/etcd/pull/17447) - `--dial-keepalive-time` - `--dial-keepalive-timeout` - `--permit-without-stream` ### Dependencies - Upgrade [bbolt to v1.3.9](https://github.com/etcd-io/etcd/pull/17483). - Compile binaries using [go 1.21.8](https://github.com/etcd-io/etcd/pull/17537). - Upgrade [google.golang.org/protobuf to v1.33.0 to address CVE-2024-24786](https://github.com/etcd-io/etcd/pull/17553). - Upgrade github.com/sirupsen/logrus to v1.9.3 to address [PRISMA-2023-0056](https://github.com/etcd-io/etcd/pull/17482). ### Others - [Make CGO_ENABLED configurable](https://github.com/etcd-io/etcd/pull/17421). --- ## v3.5.12 (2024-01-31) ### etcd server - Fix [not validating database consistent index, and panicking on nil backend](https://github.com/etcd-io/etcd/pull/17151) - Document [`experimental-enable-lease-checkpoint-persist` flag in etcd help](https://github.com/etcd-io/etcd/pull/17190) - Fix [needlessly flocking snapshot files when deleting](https://github.com/etcd-io/etcd/pull/17206) - Add [digest for etcd base image](https://github.com/etcd-io/etcd/pull/17205) - Fix [delete inconsistencies in read buffer](https://github.com/etcd-io/etcd/pull/17230) - Add [mvcc: print backend database size and size in use in compaction logs](https://github.com/etcd-io/etcd/pull/17291) ### Dependencies - Compile binaries using [go 1.20.13](https://github.com/etcd-io/etcd/pull/17275) - Upgrade [golang.org/x/crypto to v0.17+ to address CVE-2023-48795](https://github.com/etcd-io/etcd/pull/17346) ## v3.5.11 (2023-12-07) ### etcd server - Fix distributed tracing by ensuring `--experimental-distributed-tracing-sampling-rate` configuration option is available to [set tracing sample rate](https://github.com/etcd-io/etcd/pull/16951). - Fix [url redirects while checking peer urls during new member addition](https://github.com/etcd-io/etcd/pull/16986) - Add [livez/readyz HTTP endpoints](https://github.com/etcd-io/etcd/pull/17039) ### Dependencies - Compile binaries using [go 1.20.12](https://github.com/etcd-io/etcd/pull/17077) - Fix [CVE-2023-47108](https://github.com/advisories/GHSA-8pgv-569h-w5rw) by [bumping go.opentelemetry.io/otel to 1.20.0 and go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc to 0.46.0](https://github.com/etcd-io/etcd/pull/16946). --- ## v3.5.10 (2023-10-27) ### etcd server - Fix [`--socket-reuse-port` and `--socket-reuse-address` not able to be set in configuration file](https://github.com/etcd-io/etcd/pull/16435). - Fix [corruption check may get a `ErrCompacted` error when server has just been compacted](https://github.com/etcd-io/etcd/pull/16048) - Improve [Lease put performance for the case that auth is disabled or the user is admin](https://github.com/etcd-io/etcd/pull/16019) - Improve [Skip getting authInfo from incoming context when auth is disabled](https://github.com/etcd-io/etcd/pull/16241) - Fix [Hash and HashKV have duplicated RESTful API](https://github.com/etcd-io/etcd/pull/16490) ### etcdutl v3 - Add [optional --bump-revision and --mark-compacted flag to etcdutl snapshot restore operation](https://github.com/etcd-io/etcd/pull/16165). ### etcdctl v3 - Add [optional --bump-revision and --mark-compacted flag to etcdctl snapshot restore operation](https://github.com/etcd-io/etcd/pull/16165). ### etcd grpc-proxy - Fix [Memberlist results not updated when proxy node down](https://github.com/etcd-io/etcd/pull/15907). ### Package `clientv3` - Fix [Multiple endpoints with same prefix got mixed up](https://github.com/etcd-io/etcd/pull/15939) - Fix [Unexpected blocking when barrier waits on a nonexistent key](https://github.com/etcd-io/etcd/pull/16188) - Fix [Reset auth token when failing to authenticate due to auth being disabled](https://github.com/etcd-io/etcd/pull/16241) - Fix [panic in etcd validate secure endpoints](https://github.com/etcd-io/etcd/pull/16565) ### Dependencies - Compile binaries using [go 1.20.10](https://github.com/etcd-io/etcd/pull/16745). - Upgrade gRPC to 1.58.3 in https://github.com/etcd-io/etcd/pull/16625, https://github.com/etcd-io/etcd/pull/16781 and https://github.com/etcd-io/etcd/pull/16790. Note that gRPC server will reject requests with connection header (refer to https://github.com/grpc/grpc-go/pull/4803). - Upgrade [bbolt to v1.3.8](https://github.com/etcd-io/etcd/pull/16833) --- ## v3.5.9 (2023-05-11) ### etcd server - Fix [LeaseTimeToLive API may return keys to clients which have no read permission on the keys](https://github.com/etcd-io/etcd/pull/15815). ### Dependencies - Compile binaries using [go 1.19.9](https://github.com/etcd-io/etcd/pull/15822). --- ## v3.5.8 (2023-04-13) ### etcd server - Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15483) to enable support for TLS 1.3. - Add [`etcd --listen-client-http-urls`](https://github.com/etcd-io/etcd/pull/15589) flag to support separating http server from grpc one, thus giving full immunity to [watch stream starvation under high read load](https://github.com/etcd-io/etcd/issues/15402). - Change [http2 frame scheduler to random algorithm](https://github.com/etcd-io/etcd/pull/15452) - Fix [Watch response traveling back in time when reconnecting member downloads snapshot from the leader](https://github.com/etcd-io/etcd/pull/15515) - Fix [race when starting both secure & insecure gRPC servers on the same address](https://github.com/etcd-io/etcd/pull/15517) - Fix [server/auth: disallow creating empty permission ranges](https://github.com/etcd-io/etcd/pull/15619) - Fix [aligning zap log timestamp resolution to microseconds](https://github.com/etcd-io/etcd/pull/15240). Etcd now uses zap timestamp format: `2006-01-02T15:04:05.999999Z0700` (microsecond instead of milliseconds precision). - Fix [wsproxy did not print log in JSON format](https://github.com/etcd-io/etcd/pull/15661). - Fix [CVE-2021-28235](https://nvd.nist.gov/vuln/detail/CVE-2021-28235) by [clearing password after authenticating the user](https://github.com/etcd-io/etcd/pull/15653). - Fix [etcdserver may panic when parsing a JWT token without username or revision](https://github.com/etcd-io/etcd/pull/15676). - Fix [Requested watcher progress notifications are not synchronised with stream](https://github.com/etcd-io/etcd/pull/15695). ### Package `netutil` - Fix [consistently format IPv6 addresses for comparison](https://github.com/etcd-io/etcd/pull/15187). ### Package `clientv3` - Fix [etcd might send duplicated events to watch clients](https://github.com/etcd-io/etcd/pull/15274). ### Dependencies - Recommend [Go 1.19+](https://github.com/etcd-io/etcd/pull/15337). - Compile binaries using [go to 1.19.8](https://github.com/etcd-io/etcd/pull/15651) - Upgrade [golang.org/x/net to v0.7.0](https://github.com/etcd-io/etcd/pull/15337) - Upgrade [bbolt to v1.3.7](https://github.com/etcd-io/etcd/pull/15222). ### Docker image - [Remove nsswitch.conf from docker image](https://github.com/etcd-io/etcd/pull/15161) - Fix [etcd docker images all tagged with amd64 architecture](https://github.com/etcd-io/etcd/pull/15612) --- ## v3.5.7 (2023-01-20) ### etcd server - Fix [Remove memberID from data corrupt alarm](https://github.com/etcd-io/etcd/pull/14852). - Fix [Allow non mutating requests pass through quotaKVServer when NOSPACE](https://github.com/etcd-io/etcd/pull/14884). - Fix [nil pointer panic for readonly txn due to nil response](https://github.com/etcd-io/etcd/pull/14899). - Fix [The last record which was partially synced to disk isn't automatically repaired](https://github.com/etcd-io/etcd/pull/15069). - Fix [etcdserver might promote a non-started learner](https://github.com/etcd-io/etcd/pull/15096). ### Package `clientv3` - Reverted the fix to [auth invalid token and old revision errors in watch](https://github.com/etcd-io/etcd/pull/14995). ### Dependencies - Recommend [Go 1.17+](https://github.com/etcd-io/etcd/pull/15019). - Compile binaries using [Go 1.17.13](https://github.com/etcd-io/etcd/pull/15019) - Bumped [some dependencies](https://github.com/etcd-io/etcd/pull/15018) to address some HIGH Vulnerabilities. ### Docker image - Use [distroless base image](https://github.com/etcd-io/etcd/pull/15016) to address critical Vulnerabilities. - Updated [base image from base-debian11 to static-debian11 and removed dependency on busybox](https://github.com/etcd-io/etcd/pull/15037). --- ## v3.5.6 (2022-11-21) ### etcd server - Fix [auth invalid token and old revision errors in watch](https://github.com/etcd-io/etcd/pull/14547) - Fix [avoid closing a watch with ID 0 incorrectly](https://github.com/etcd-io/etcd/pull/14563) - Fix [auth: fix data consistency issue caused by recovery from snapshot](https://github.com/etcd-io/etcd/pull/14648) - Fix [revision might be inconsistency between members when etcd crashes during processing defragmentation operation](https://github.com/etcd-io/etcd/pull/14733) - Fix [timestamp in inconsistent format](https://github.com/etcd-io/etcd/pull/14799) - Fix [Failed resolving host due to lost DNS record](https://github.com/etcd-io/etcd/pull/14573) ### Package `clientv3` - Fix [Add backoff before retry when watch stream returns unavailable](https://github.com/etcd-io/etcd/pull/14582). - Fix [stack overflow error in double barrier](https://github.com/etcd-io/etcd/pull/14658) - Fix [Refreshing token on CommonName based authentication causes segmentation violation in client](https://github.com/etcd-io/etcd/pull/14790). ### etcd grpc-proxy - Add [`etcd grpc-proxy start --listen-cipher-suites`](https://github.com/etcd-io/etcd/pull/14500) flag to support adding configurable cipher list. --- ## v3.5.5 (2022-09-15) ### Deprecations - Deprecated [SetKeepAlive and SetKeepAlivePeriod in limitListenerConn](https://github.com/etcd-io/etcd/pull/14366). ### Package `clientv3` - Fix [do not overwrite authTokenBundle on dial](https://github.com/etcd-io/etcd/pull/14132). - Fix [IsOptsWithPrefix returns false even if WithPrefix() is included](https://github.com/etcd-io/etcd/pull/14187). ### etcd server - [Build official darwin/arm64 artifacts](https://github.com/etcd-io/etcd/pull/14436). - Add [`etcd --max-concurrent-streams`](https://github.com/etcd-io/etcd/pull/14219) flag to configure the max concurrent streams each client can open at a time, and defaults to math.MaxUint32. - Add [`etcd --experimental-compact-hash-check-enabled --experimental-compact-hash-check-time`](https://github.com/etcd-io/etcd/issues/14039) flags to support enabling reliable corruption detection on compacted revisions. - Fix [unexpected error during txn](https://github.com/etcd-io/etcd/issues/14110). - Fix [lease leak issue due to tokenProvider isn't enabled when restoring auth store from a snapshot](https://github.com/etcd-io/etcd/pull/13205). - Fix [the race condition between goroutine and channel on the same leases to be revoked](https://github.com/etcd-io/etcd/pull/14087). - Fix [lessor may continue to schedule checkpoint after stepping down leader role](https://github.com/etcd-io/etcd/pull/14087). - Fix [Restrict the max size of each WAL entry to the remaining size of the WAL file](https://github.com/etcd-io/etcd/pull/14127). - Fix [Protect rangePermCache with a RW lock correctly](https://github.com/etcd-io/etcd/pull/14227) - Fix [memberID equals zero in corruption alarm](https://github.com/etcd-io/etcd/pull/14272) - Fix [Durability API guarantee broken in single node cluster](https://github.com/etcd-io/etcd/pull/14424) - Fix [etcd fails to start after performing alarm list operation and then power off/on](https://github.com/etcd-io/etcd/pull/14429) - Fix [authentication data not loaded on member startup](https://github.com/etcd-io/etcd/pull/14409) ### etcdctl v3 - Fix [etcdctl move-leader may fail for multiple endpoints](https://github.com/etcd-io/etcd/pull/14434) ### Other - [Bump golang.org/x/crypto to latest version](https://github.com/etcd-io/etcd/pull/13996) to address [CVE-2022-27191](https://github.com/advisories/GHSA-8c26-wmh5-6g9v). - [Bump OpenTelemetry to 1.0.1 and gRPC to 1.41.0](https://github.com/etcd-io/etcd/pull/14312). --- ## v3.5.4 (2022-04-24) ### etcd server - Fix [etcd panic on startup (auth enabled)](https://github.com/etcd-io/etcd/pull/13946) ### package `client/pkg/v3` - [Revert the change of trimming the trailing dot from SRV.Target](https://github.com/etcd-io/etcd/pull/13950) returned by DNS lookup --- ## v3.5.3 (2022-04-13) ### etcd server - Fix [Provide a better liveness probe for when etcd runs as a Kubernetes pod](https://github.com/etcd-io/etcd/pull/13706) - Fix [inconsistent log format](https://github.com/etcd-io/etcd/pull/13864) - Fix [Inconsistent revision and data occurs](https://github.com/etcd-io/etcd/pull/13908) - Fix [Etcdserver is still in progress of processing LeaseGrantRequest when it receives a LeaseKeepAliveRequest on the same leaseID](https://github.com/etcd-io/etcd/pull/13932) - Fix [consistent_index coming from snapshot is overwritten by the old local value](https://github.com/etcd-io/etcd/pull/13933) - [Update container base image snapshot](https://github.com/etcd-io/etcd/pull/13862) - Fix [Defrag unsets backend options](https://github.com/etcd-io/etcd/pull/13701). ### package `client/pkg/v3` - [Trim the suffix dot from the target](https://github.com/etcd-io/etcd/pull/13714) in SRV records returned by DNS lookup ### etcdctl v3 - [Always print the raft_term in decimal](https://github.com/etcd-io/etcd/pull/13727) when displaying member list in json. --- ## [v3.5.2](https://github.com/etcd-io/etcd/releases/tag/v3.5.2) (2022-02-01) See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.1...v3.5.2) and [v3.5 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_5/) for any breaking changes. ### etcd server - Fix [exclude the same alarm type activated by multiple peers](https://github.com/etcd-io/etcd/pull/13476). - Add [`etcd --experimental-enable-lease-checkpoint-persist`](https://github.com/etcd-io/etcd/pull/13508) flag to enable checkpoint persisting. - Fix [Lease checkpoints don't prevent to reset ttl on leader change](https://github.com/etcd-io/etcd/pull/13508), requires enabling checkpoint persisting. - Fix [assertion failed due to tx closed when recovering v3 backend from a snapshot db](https://github.com/etcd-io/etcd/pull/13501) - Fix [segmentation violation(SIGSEGV) error due to premature unlocking of watchableStore](https://github.com/etcd-io/etcd/pull/13541) --- ## [v3.5.1](https://github.com/etcd-io/etcd/releases/tag/v3.5.1) (2021-10-15) See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.5.1) and [v3.5 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_5/) for any breaking changes. ### etcd server - Fix [self-signed-cert-validity parameter cannot be specified in the config file](https://github.com/etcd-io/etcd/pull/13237). - Fix [ensure that cluster members stored in v2store and backend are in sync](https://github.com/etcd-io/etcd/pull/13348) ### etcd client - [Fix etcd client sends invalid :authority header](https://github.com/etcd-io/etcd/issues/13192) ### package clientv3 - Endpoints self identify now as `etcd-endpoints://{id}/{authority}` where authority is based on first endpoint passed, for example `etcd-endpoints://0xc0009d8540/localhost:2079` ### Other - Updated [base image](https://github.com/etcd-io/etcd/pull/13386) from `debian:buster-v1.4.0` to `debian:bullseye-20210927` to fix the following critical CVEs: - [CVE-2021-3711](https://nvd.nist.gov/vuln/detail/CVE-2021-3711): miscalculation of a buffer size in openssl's SM2 decryption - [CVE-2021-35942](https://nvd.nist.gov/vuln/detail/CVE-2021-35942): integer overflow flaw in glibc - [CVE-2019-9893](https://nvd.nist.gov/vuln/detail/CVE-2019-9893): incorrect syscall argument generation in libseccomp - [CVE-2021-36159](https://nvd.nist.gov/vuln/detail/CVE-2021-36159): libfetch in apk-tools mishandles numeric strings in FTP and HTTP protocols to allow out of bound reads. --- ## v3.5.0 (2021-06) See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0...v3.5.0) and [v3.5 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_5/) for any breaking changes. - [v3.5.0](https://github.com/etcd-io/etcd/releases/tag/v3.5.0) (2021 TBD), see [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0-rc.1...v3.5.0). - [v3.5.0-rc.1](https://github.com/etcd-io/etcd/releases/tag/v3.5.0-rc.1) (2021-06-10), see [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0-rc.0...v3.5.0-rc.1). - [v3.5.0-rc.0](https://github.com/etcd-io/etcd/releases/tag/v3.5.0-rc.0) (2021-06-04), see [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0-beta.4...v3.5.0-rc.0). - [v3.5.0-beta.4](https://github.com/etcd-io/etcd/releases/tag/v3.5.0-beta.4) (2021-05-26), see [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0-beta.3...v3.5.0-beta.4). - [v3.5.0-beta.3](https://github.com/etcd-io/etcd/releases/tag/v3.5.0-beta.3) (2021-05-18), see [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0-beta.2...v3.5.0-beta.3). - [v3.5.0-beta.2](https://github.com/etcd-io/etcd/releases/tag/v3.5.0-beta.2) (2021-05-18), see [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0-beta.1...v3.5.0-beta.2). - [v3.5.0-beta.1](https://github.com/etcd-io/etcd/releases/tag/v3.5.0-beta.1) (2021-05-18), see [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0...v3.5.0-beta.1). **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.5 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_3_5/).** ### Breaking Changes - `go.etcd.io/etcd` Go packages have moved to `go.etcd.io/etcd/{api,pkg,raft,client,etcdctl,server,raft,tests}/v3` to follow the [Go modules](https://github.com/golang/go/wiki/Modules) conventions - `go.etcd.io/clientv3/snapshot` SnapshotManager class have moved to `go.etcd.io/clientv3/etcdctl`. The method `snapshot.Save` to download a snapshot from the remote server was preserved in 'go.etcd.io/clientv3/snapshot`. - `go.etcd.io/client' package got migrated to 'go.etcd.io/client/v2'. - Changed behavior of clientv3 API [MemberList](https://github.com/etcd-io/etcd/pull/11639). - Previously, it is directly served with server's local data, which could be stale. - Now, it is served with linearizable guarantee. If the server is disconnected from quorum, `MemberList` call will fail. - [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) only supports [`/v3`](TODO) endpoint. - Deprecated [`/v3beta`](https://github.com/etcd-io/etcd/pull/9298). - `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` doesn't work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. - **`etcd --experimental-enable-v2v3` flag remains experimental and to be deprecated.** - v2 storage emulation feature will be deprecated in the next release. - etcd 3.5 is the last version that supports V2 API. Flags `--enable-v2` and `--experimental-enable-v2v3` [are now deprecated](https://github.com/etcd-io/etcd/pull/12940) and will be removed in etcd v3.6 release. - **`etcd --experimental-backend-bbolt-freelist-type` flag has been deprecated.** Use **`etcd --backend-bbolt-freelist-type`** instead. The default type is hashmap and it is stable now. - **`etcd --debug` flag has been deprecated.** Use **`etcd --log-level=debug`** instead. - Remove [`embed.Config.Debug`](https://github.com/etcd-io/etcd/pull/10947). - **`etcd --log-output` flag has been deprecated.** Use **`etcd --log-outputs`** instead. - **`etcd --logger=zap --log-outputs=stderr`** is now the default. - **`etcd --logger=capnslog` flag value has been deprecated.** - **`etcd --logger=zap --log-outputs=default` flag value is not supported.**. - Use `etcd --logger=zap --log-outputs=stderr`. - Or, use `etcd --logger=zap --log-outputs=systemd/journal` to send logs to the local systemd journal. - Previously, if etcd parent process ID (PPID) is 1 (e.g. run with systemd), `etcd --logger=capnslog --log-outputs=default` redirects server logs to local systemd journal. And if write to journald fails, it writes to `os.Stderr` as a fallback. - However, even with PPID 1, it can fail to dial systemd journal (e.g. run embedded etcd with Docker container). Then, [every single log write will fail](https://github.com/etcd-io/etcd/pull/9729) and fall back to `os.Stderr`, which is inefficient. - To avoid this problem, systemd journal logging must be configured manually. - **`etcd --log-outputs=stderr`** is now the default. - **`etcd --log-package-levels` flag for `capnslog` has been deprecated.** Now, **`etcd --logger=zap --log-outputs=stderr`** is the default. - **`[CLIENT-URL]/config/local/log` endpoint has been deprecated, as is `etcd --log-package-levels` flag.** - `curl http://127.0.0.1:2379/config/local/log -XPUT -d '{"Level":"DEBUG"}'` won't work. - Please use `etcd --logger=zap --log-outputs=stderr` instead. - Deprecated `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metric. Use `etcd_mvcc_db_total_size_in_bytes` instead. - Deprecated `etcd_debugging_mvcc_put_total` Prometheus metric. Use `etcd_mvcc_put_total` instead. - Deprecated `etcd_debugging_mvcc_delete_total` Prometheus metric. Use `etcd_mvcc_delete_total` instead. - Deprecated `etcd_debugging_mvcc_txn_total` Prometheus metric. Use `etcd_mvcc_txn_total` instead. - Deprecated `etcd_debugging_mvcc_range_total` Prometheus metric. Use `etcd_mvcc_range_total` instead. - Main branch `/version` outputs `3.5.0-pre`, instead of `3.4.0+git`. - Changed `proxy` package function signature to [support structured logger](https://github.com/etcd-io/etcd/pull/11614). - Previously, `NewClusterProxy(c *clientv3.Client, advaddr string, prefix string) (pb.ClusterServer, <-chan struct{})`, now `NewClusterProxy(lg *zap.Logger, c *clientv3.Client, advaddr string, prefix string) (pb.ClusterServer, <-chan struct{})`. - Previously, `Register(c *clientv3.Client, prefix string, addr string, ttl int)`, now `Register(lg *zap.Logger, c *clientv3.Client, prefix string, addr string, ttl int) <-chan struct{}`. - Previously, `NewHandler(t *http.Transport, urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterval time.Duration) http.Handler`, now `NewHandler(lg *zap.Logger, t *http.Transport, urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterval time.Duration) http.Handler`. - Changed `pkg/flags` function signature to [support structured logger](https://github.com/etcd-io/etcd/pull/11616). - Previously, `SetFlagsFromEnv(prefix string, fs *flag.FlagSet) error`, now `SetFlagsFromEnv(lg *zap.Logger, prefix string, fs *flag.FlagSet) error`. - Previously, `SetPflagsFromEnv(prefix string, fs *pflag.FlagSet) error`, now `SetPflagsFromEnv(lg *zap.Logger, prefix string, fs *pflag.FlagSet) error`. - ClientV3 supports [grpc resolver API](https://github.com/etcd-io/etcd/blob/main/client/v3/naming/resolver/resolver.go). - Endpoints can be managed using [endpoints.Manager](https://github.com/etcd-io/etcd/blob/main/client/v3/naming/endpoints/endpoints.go) - Previously supported [GRPCResolver was decomissioned](https://github.com/etcd-io/etcd/pull/12675). Use [resolver](https://github.com/etcd-io/etcd/blob/main/client/v3/naming/resolver/resolver.go) instead. - Turned on [--pre-vote by default](https://github.com/etcd-io/etcd/pull/12770). Should prevent disrupting RAFT leader by an individual member. - [ETCD_CLIENT_DEBUG env](https://github.com/etcd-io/etcd/pull/12786): Now supports log levels (debug, info, warn, error, dpanic, panic, fatal). Only when set, overrides application-wide grpc logging settings. - [Embed Etcd.Close()](https://github.com/etcd-io/etcd/pull/12828) needs to called exactly once and closes Etcd.Err() stream. - [Embed Etcd does not override global/grpc logger](https://github.com/etcd-io/etcd/pull/12861) be default any longer. If desired, please call `embed.Config::SetupGlobalLoggers()` explicitly. - [Embed Etcd custom logger should be configured using simpler builder `NewZapLoggerBuilder`](https://github.com/etcd-io/etcd/pull/12973). - Client errors of `context cancelled` or `context deadline exceeded` are exposed as `codes.Canceled` and `codes.DeadlineExceeded`, instead of `codes.Unknown`. ### Storage format changes - [WAL log's snapshots persists raftpb.ConfState](https://github.com/etcd-io/etcd/pull/12735) - [Backend persists raftpb.ConfState](https://github.com/etcd-io/etcd/pull/12962) in the `meta` bucket `confState` key. - [Backend persists applied term](https://github.com/etcd-io/etcd/pull/) in the `meta` bucket. - Backend persists `downgrade` in the `cluster` bucket ### Security - Add [`TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256` and `TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256` to `etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/11864). - Changed [the format of WAL entries related to auth for not keeping password as a plain text](https://github.com/etcd-io/etcd/pull/11943). - Add third party [Security Audit Report](https://github.com/etcd-io/etcd/pull/12201). - A [log warning](https://github.com/etcd-io/etcd/pull/12242) is added when etcd uses any existing directory that has a permission different than 700 on Linux and 777 on Windows. - Add optional [`ClientCertFile` and `ClientKeyFile`](https://github.com/etcd-io/etcd/pull/12705) options for peer and client tls configuration when split certificates are used. ### Metrics, Monitoring See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. Note that any `etcd_debugging_*` metrics are experimental and subject to change. - Deprecated `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metric. Use `etcd_mvcc_db_total_size_in_bytes` instead. - Deprecated `etcd_debugging_mvcc_put_total` Prometheus metric. Use `etcd_mvcc_put_total` instead. - Deprecated `etcd_debugging_mvcc_delete_total` Prometheus metric. Use `etcd_mvcc_delete_total` instead. - Deprecated `etcd_debugging_mvcc_txn_total` Prometheus metric. Use `etcd_mvcc_txn_total` instead. - Deprecated `etcd_debugging_mvcc_range_total` Prometheus metric. Use `etcd_mvcc_range_total` instead. - Add [`etcd_debugging_mvcc_current_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. - Add [`etcd_debugging_mvcc_compact_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric. - Change [`etcd_cluster_version`](https://github.com/etcd-io/etcd/pull/11254) Prometheus metrics to include only major and minor version. - Add [`etcd_debugging_mvcc_total_put_size_in_bytes`](https://github.com/etcd-io/etcd/pull/11374) Prometheus metric. - Add [`etcd_server_client_requests_total` with `"type"` and `"client_api_version"` labels](https://github.com/etcd-io/etcd/pull/11687). - Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). - Add [`etcd_debugging_auth_revision`](https://github.com/etcd-io/etcd/commit/f14d2a087f7b0fd6f7980b95b5e0b945109c95f3). - Add [`os_fd_used` and `os_fd_limit` to monitor current OS file descriptors](https://github.com/etcd-io/etcd/pull/12214). - Add [`etcd_disk_defrag_inflight`](https://github.com/etcd-io/etcd/pull/13395). ### etcd server - Add [don't attempt to grant nil permission to a role](https://github.com/etcd-io/etcd/pull/13086). - Add [don't activate alarms w/missing AlarmType](https://github.com/etcd-io/etcd/pull/13084). - Add [`TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256` and `TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256` to `etcd --cipher-suites`](https://github.com/etcd-io/etcd/pull/11864). - Automatically [create parent directory if it does not exist](https://github.com/etcd-io/etcd/pull/9626) (fix [issue#9609](https://github.com/etcd-io/etcd/issues/9609)). - v4.0 will configure `etcd --enable-v2=true --enable-v2v3=/aaa` to enable v2 API server that is backed by **v3 storage**. - [`etcd --backend-bbolt-freelist-type`] flag is now stable. - `etcd --experimental-backend-bbolt-freelist-type` has been deprecated. - Support [downgrade API](https://github.com/etcd-io/etcd/pull/11715). - Deprecate v2 apply on cluster version. [Use v3 request to set cluster version and recover cluster version from v3 backend](https://github.com/etcd-io/etcd/pull/11427). - [Use v2 api to update cluster version to support mixed version cluster during upgrade](https://github.com/etcd-io/etcd/pull/12988). - [Fix corruption bug in defrag](https://github.com/etcd-io/etcd/pull/11613). - Fix [quorum protection logic when promoting a learner](https://github.com/etcd-io/etcd/pull/11640). - Improve [peer corruption checker](https://github.com/etcd-io/etcd/pull/11621) to work when peer mTLS is enabled. - Log [`[CLIENT-PORT]/health` check in server side](https://github.com/etcd-io/etcd/pull/11704). - Log [successful etcd server-side health check in debug level](https://github.com/etcd-io/etcd/pull/12677). - Improve [compaction performance when latest index is greater than 1-million](https://github.com/etcd-io/etcd/pull/11734). - [Refactor consistentindex](https://github.com/etcd-io/etcd/pull/11699). - [Add log when etcdserver failed to apply command](https://github.com/etcd-io/etcd/pull/11670). - Improve [count-only range performance](https://github.com/etcd-io/etcd/pull/11771). - Remove [redundant storage restore operation to shorten the startup time](https://github.com/etcd-io/etcd/pull/11779). - With 40 million key test data,it can shorten the startup time from 5 min to 2.5 min. - [Fix deadlock bug in mvcc](https://github.com/etcd-io/etcd/pull/11817). - Fix [inconsistency between WAL and server snapshot](https://github.com/etcd-io/etcd/pull/11888). - Previously, server restore fails if it had crashed after persisting raft hard state but before saving snapshot. - See https://github.com/etcd-io/etcd/issues/10219 for more. - Add [missing CRC checksum check in WAL validate method otherwise causes panic](https://github.com/etcd-io/etcd/pull/11924). - See https://github.com/etcd-io/etcd/issues/11918. - Improve logging around snapshot send and receive. - [Push down RangeOptions.limit argv into index tree to reduce memory overhead](https://github.com/etcd-io/etcd/pull/11990). - Add [reason field for /health response](https://github.com/etcd-io/etcd/pull/11983). - Add [exclude alarms from health check conditionally](https://github.com/etcd-io/etcd/pull/12880). - Add [`etcd --unsafe-no-fsync`](https://github.com/etcd-io/etcd/pull/11946) flag. - Setting the flag disables all uses of fsync, which is unsafe and will cause data loss. This flag makes it possible to run an etcd node for testing and development without placing lots of load on the file system. - Add [`etcd --auth-token-ttl`](https://github.com/etcd-io/etcd/pull/11980) flag to customize `simpleTokenTTL` settings. - Improve [`runtime.FDUsage` call pattern to reduce objects malloc of Memory Usage and CPU Usage](https://github.com/etcd-io/etcd/pull/11986). - Improve [mvcc.watchResponse channel Memory Usage](https://github.com/etcd-io/etcd/pull/11987). - Log [expensive request info in UnaryInterceptor](https://github.com/etcd-io/etcd/pull/12086). - [Fix invalid Go type in etcdserverpb](https://github.com/etcd-io/etcd/pull/12000). - [Improve healthcheck by using v3 range request and its corresponding timeout](https://github.com/etcd-io/etcd/pull/12195). - Add [`etcd --experimental-watch-progress-notify-interval`](https://github.com/etcd-io/etcd/pull/12216) flag to make watch progress notify interval configurable. - Fix [server panic in slow writes warnings](https://github.com/etcd-io/etcd/issues/12197). - Fixed via [PR#12238](https://github.com/etcd-io/etcd/pull/12238). - [Fix server panic](https://github.com/etcd-io/etcd/pull/12288) when force-new-cluster flag is enabled in a cluster which had learner node. - Add [`etcd --self-signed-cert-validity`](https://github.com/etcd-io/etcd/pull/12429) flag to support setting certificate expiration time. - Notice, certificates generated by etcd are valid for 1 year by default when specifying the auto-tls or peer-auto-tls option. - Add [`etcd --experimental-warning-apply-duration`](https://github.com/etcd-io/etcd/pull/12448) flag which allows apply duration threshold to be configurable. - Add [`etcd --experimental-memory-mlock`](https://github.com/etcd-io/etcd/pull/TODO) flag which prevents etcd memory pages to be swapped out. - Add [`etcd --socket-reuse-port`](https://github.com/etcd-io/etcd/pull/12702) flag - Setting this flag enables `SO_REUSEPORT` which allows rebind of a port already in use. User should take caution when using this flag to ensure flock is properly enforced. - Add [`etcd --socket-reuse-address`](https://github.com/etcd-io/etcd/pull/12702) flag - Setting this flag enables `SO_REUSEADDR` which allows binding to an address in `TIME_WAIT` state, improving etcd restart time. - Reduce [around 30% memory allocation by logging range response size without marshal](https://github.com/etcd-io/etcd/pull/12871). - `ETCD_VERIFY="all"` environment triggers [additional verification of consistency](https://github.com/etcd-io/etcd/pull/12901) of etcd data-dir files. - Add [`etcd --enable-log-rotation`](https://github.com/etcd-io/etcd/pull/12774) boolean flag which enables log rotation if true. - Add [`etcd --log-rotation-config-json`](https://github.com/etcd-io/etcd/pull/12774) flag which allows passthrough of JSON config to configure log rotation for a file output target. - Add experimental distributed tracing boolean flag [`--experimental-enable-distributed-tracing`](https://github.com/etcd-io/etcd/pull/12919) which enables tracing. - Add [`etcd --experimental-distributed-tracing-address`](https://github.com/etcd-io/etcd/pull/12919) string flag which allows configuring the OpenTelemetry collector address. - Add [`etcd --experimental-distributed-tracing-service-name`](https://github.com/etcd-io/etcd/pull/12919) string flag which allows changing the default "etcd" service name. - Add [`etcd --experimental-distributed-tracing-instance-id`](https://github.com/etcd-io/etcd/pull/12919) string flag which configures an instance ID, which must be unique per etcd instance. - Add [`--experimental-bootstrap-defrag-threshold-megabytes`](https://github.com/etcd-io/etcd/pull/12941) which configures a threshold for the unused db size and etcdserver will automatically perform defragmentation on bootstrap when it exceeds this value. The functionality is disabled if the value is 0. ### Package `runtime` - Optimize [`runtime.FDUsage` by removing unnecessary sorting](https://github.com/etcd-io/etcd/pull/12214). ### Package `embed` - Remove [`embed.Config.Debug`](https://github.com/etcd-io/etcd/pull/10947). - Use `embed.Config.LogLevel` instead. - Add [`embed.Config.ZapLoggerBuilder`](https://github.com/etcd-io/etcd/pull/11147) to allow creating a custom zap logger. - Replace [global `*zap.Logger` with etcd server logger object](https://github.com/etcd-io/etcd/pull/12212). - Add [`embed.Config.EnableLogRotation`](https://github.com/etcd-io/etcd/pull/12774) which enables log rotation if true. - Add [`embed.Config.LogRotationConfigJSON`](https://github.com/etcd-io/etcd/pull/12774) to allow passthrough of JSON config to configure log rotation for a file output target. - Add [`embed.Config.ExperimentalEnableDistributedTracing`](https://github.com/etcd-io/etcd/pull/12919) which enables experimental distributed tracing if true. - Add [`embed.Config.ExperimentalDistributedTracingAddress`](https://github.com/etcd-io/etcd/pull/12919) which allows overriding default collector address. - Add [`embed.Config.ExperimentalDistributedTracingServiceName`](https://github.com/etcd-io/etcd/pull/12919) which allows overriding default "etcd" service name. - Add [`embed.Config.ExperimentalDistributedTracingServiceInstanceID`](https://github.com/etcd-io/etcd/pull/12919) which allows configuring an instance ID, which must be uniquer per etcd instance. ### Package `clientv3` - Remove [excessive watch cancel logging messages](https://github.com/etcd-io/etcd/pull/12187). - See [kubernetes/kubernetes#93450](https://github.com/kubernetes/kubernetes/issues/93450). - Add [`TryLock`](https://github.com/etcd-io/etcd/pull/11104) method to `clientv3/concurrency/Mutex`. A non-blocking method on `Mutex` which does not wait to get lock on the Mutex, returns immediately if Mutex is locked by another session. - Fix [client balancer failover against multiple endpoints](https://github.com/etcd-io/etcd/pull/11184). - Fix [`"kube-apiserver: failover on multi-member etcd cluster fails certificate check on DNS mismatch"`](https://github.com/kubernetes/kubernetes/issues/83028). - Fix [IPv6 endpoint parsing in client](https://github.com/etcd-io/etcd/pull/11211). - Fix ["1.16: etcd client does not parse IPv6 addresses correctly when members are joining" (kubernetes#83550)](https://github.com/kubernetes/kubernetes/issues/83550). - Fix [errors caused by grpc changing balancer/resolver API](https://github.com/etcd-io/etcd/pull/11564). This change is compatible with grpc >= [v1.26.0](https://github.com/grpc/grpc-go/releases/tag/v1.26.0), but is not compatible with < v1.26.0 version. - Use [ServerName as the authority](https://github.com/etcd-io/etcd/pull/11574) after bumping to grpc v1.26.0. Remove workaround in [#11184](https://github.com/etcd-io/etcd/pull/11184). - Fix [`"hasleader"` metadata embedding](https://github.com/etcd-io/etcd/pull/11687). - Previously, `clientv3.WithRequireLeader(ctx)` was overwriting existing context keys. - Fix [watch leak caused by lazy cancellation](https://github.com/etcd-io/etcd/pull/11850). When clients cancel their watches, a cancel request will now be immediately sent to the server instead of waiting for the next watch event. - Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). - Fix [auth token invalid after watch reconnects](https://github.com/etcd-io/etcd/pull/12264). Get AuthToken automatically when clientConn is ready. - Improve [clientv3:get AuthToken gracefully without extra connection](https://github.com/etcd-io/etcd/pull/12165). - Changed [clientv3 dialing code](https://github.com/etcd-io/etcd/pull/12671) to use grpc resolver API instead of custom balancer. - Endpoints self identify now as `etcd-endpoints://{id}/#initially={list of endpoints}` e.g. `etcd-endpoints://0xc0009d8540/#initially=[localhost:2079]` - Make sure [save snapshot downloads checksum for integrity checks](https://github.com/etcd-io/etcd/pull/11896). ### Package `lease` - Fix [memory leak in follower nodes](https://github.com/etcd-io/etcd/pull/11731). - https://github.com/etcd-io/etcd/issues/11495 - https://github.com/etcd-io/etcd/issues/11730 - Make sure [grant/revoke won't be applied repeatedly after restarting etcd](https://github.com/etcd-io/etcd/pull/11935). ### Package `wal` - Add [`etcd_wal_write_bytes_total`](https://github.com/etcd-io/etcd/pull/11738). - Handle [out-of-range slice bound in `ReadAll` and entry limit in `decodeRecord`](https://github.com/etcd-io/etcd/pull/11793). ### etcdctl v3 - Fix `etcdctl member add` command to prevent potential timeout. ([PR#11194](https://github.com/etcd-io/etcd/pull/11194) and [PR#11638](https://github.com/etcd-io/etcd/pull/11638)) - Add [`etcdctl watch --progress-notify`](https://github.com/etcd-io/etcd/pull/11462) flag. - Add [`etcdctl auth status`](https://github.com/etcd-io/etcd/pull/11536) command to check if authentication is enabled - Add [`etcdctl get --count-only`](https://github.com/etcd-io/etcd/pull/11743) flag for output type `fields`. - Add [`etcdctl member list -w=json --hex`](https://github.com/etcd-io/etcd/pull/11812) flag to print memberListResponse in hex format json. - Changed [`etcdctl lock exec-command`](https://github.com/etcd-io/etcd/pull/12829) to return exit code of exec-command. - [New tool: `etcdutl`](https://github.com/etcd-io/etcd/pull/12971) incorporated functionality of: `etcdctl snapshot status|restore`, `etcdctl backup`, `etcdctl defrag --data-dir ...`. - [ETCDCTL_API=3 `etcdctl migrate`](https://github.com/etcd-io/etcd/pull/12971) has been decommissioned. Use etcd <=v3.4 to restore v2 storage. ### gRPC gateway - [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) only supports [`/v3`](TODO) endpoint. - Deprecated [`/v3beta`](https://github.com/etcd-io/etcd/pull/9298). - `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` does work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead. - Set [`enable-grpc-gateway`](https://github.com/etcd-io/etcd/pull/12297) flag to true when using a config file to keep the defaults the same as the command line configuration. ### gRPC Proxy - Fix [`panic on error`](https://github.com/etcd-io/etcd/pull/11694) for metrics handler. - Add [gRPC keepalive related flags](https://github.com/etcd-io/etcd/pull/11711) `grpc-keepalive-min-time`, `grpc-keepalive-interval` and `grpc-keepalive-timeout`. - [Fix grpc watch proxy hangs when failed to cancel a watcher](https://github.com/etcd-io/etcd/pull/12030) . - Add [metrics handler for grpcproxy self](https://github.com/etcd-io/etcd/pull/12107). - Add [health handler for grpcproxy self](https://github.com/etcd-io/etcd/pull/12114). ### Auth - Fix [NoPassword check when adding user through GRPC gateway](https://github.com/etcd-io/etcd/pull/11418) ([issue#11414](https://github.com/etcd-io/etcd/issues/11414)) - Fix bug where [some auth related messages are logged at wrong level](https://github.com/etcd-io/etcd/pull/11586) - [Fix a data corruption bug by saving consistent index](https://github.com/etcd-io/etcd/pull/11652). - [Improve checkPassword performance](https://github.com/etcd-io/etcd/pull/11735). - [Add authRevision field in AuthStatus](https://github.com/etcd-io/etcd/pull/11659). - Fix [a bug of not refreshing expired tokens](https://github.com/etcd-io/etcd/pull/13308). - ### API - Add [`/v3/auth/status`](https://github.com/etcd-io/etcd/pull/11536) endpoint to check if authentication is enabled - [Add `Linearizable` field to `etcdserverpb.MemberListRequest`](https://github.com/etcd-io/etcd/pull/11639). - [Learner support Snapshot RPC](https://github.com/etcd-io/etcd/pull/12890/). ### Package `netutil` - Remove [`netutil.DropPort/RecoverPort/SetLatency/RemoveLatency`](https://github.com/etcd-io/etcd/pull/12491). - These are not used anymore. They were only used for older versions of functional testing. - Removed to adhere to best security practices, minimize arbitrary shell invocation. ### `tools/etcd-dump-metrics` - Implement [input validation to prevent arbitrary shell invocation](https://github.com/etcd-io/etcd/pull/12491). ### Dependency - Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.23.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.0) to [**`v1.37.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.37.0). - Upgrade [`go.uber.org/zap`](https://github.com/uber-go/zap/releases) from [**`v1.14.1`**](https://github.com/uber-go/zap/releases/tag/v1.14.1) to [**`v1.16.0`**](https://github.com/uber-go/zap/releases/tag/v1.16.0). ### Platforms - etcd now [officially supports `arm64`](https://github.com/etcd-io/etcd/pull/12929). - See https://github.com/etcd-io/etcd/pull/12928 for adding automated tests with `arm64` EC2 instances (Graviton 2). - See https://github.com/etcd-io/website/pull/273 for new platform support tier policies. ### Release - Add s390x build support ([PR#11548](https://github.com/etcd-io/etcd/pull/11548) and [PR#11358](https://github.com/etcd-io/etcd/pull/11358)) ### Go - Require [*Go 1.16+*](https://github.com/etcd-io/etcd/pull/11110). - Compile with [*Go 1.16+*](https://golang.org/doc/devel/release.html#go1.16) - etcd uses [go modules](https://github.com/etcd-io/etcd/pull/12279) (instead of vendor dir) to track dependencies. ### Project Governance - The etcd team has added, a well defined and openly discussed, project [governance](https://github.com/etcd-io/etcd/pull/11175). --- ================================================ FILE: CHANGELOG/CHANGELOG-3.6.md ================================================ Previous change logs can be found at [CHANGELOG-3.5](https://github.com/etcd-io/etcd/blob/main/CHANGELOG/CHANGELOG-3.5.md). --- ## v3.6.9 (TBC) ### etcd server - [Ensure the metrics interceptor runs before other interceptors so that metrics remain up to date](https://github.com/etcd-io/etcd/pull/21329) - Fix [Race between read index and leader change](https://github.com/etcd-io/etcd/pull/21378) - Fix [Stale reads caused by process pausing](https://github.com/etcd-io/etcd/pull/21417) ### Package `clientv3` - [Print the endpoint the grpc request was actually sent to in unary interceptor](https://github.com/etcd-io/etcd/pull/21382) ### etcd grpc-proxy - [server/etcdmain: fix startup deadlock in grpcproxy](https://github.com/etcd-io/etcd/pull/21354) ### etcdctl - Fix [slice bounds trimming single-quoted args in Argify](https://github.com/etcd-io/etcd/pull/21402) ### Dependencies - [Bump go.opentelemetry.io/otel/sdk to v1.40.0 to resolve https://pkg.go.dev/vuln/GO-2026-4394](https://github.com/etcd-io/etcd/pull/21340) - Compile binaries using [go 1.25.7](https://github.com/etcd-io/etcd/pull/21393) - [Bump golang.org/x/net to v0.51.0 to resolve GO-2026-4559](https://github.com/etcd-io/etcd/pull/21440) --- ## v3.6.8 (2026-02-13) ### etcd server - [Postpone removal of the --max-snapshots flag from v3.7 to v3.8](https://github.com/etcd-io/etcd/pull/21161) - [Revoke the deprecation of the `--snapshot-count` flag](https://github.com/etcd-io/etcd/pull/21163) ### Package `clientv3` - [Remove the use of grpc-go's Metadata field](https://github.com/etcd-io/etcd/pull/21241) ### Dependencies - Bump [golang.org/x/crypto to 0.45.0 to address CVE-2025-47914, and CVE-2025-58181](https://github.com/etcd-io/etcd/pull/21037). - Compile binaries using [go 1.24.13](https://github.com/etcd-io/etcd/pull/21266). This addresses [CVE-2025-61726](https://github.com/advisories/GHSA-gm9r-q53w-2gh4), [CVE-2025-61731](https://github.com/advisories/GHSA-xvqr-69v8-f3gv), and [CVE-2025-61732](https://github.com/advisories/GHSA-8jvr-vh7g-f8gx). --- ## v3.6.7 (2025-12-17) ### etcd server - [Print token fingerprint instead of the original tokens in log messages](https://github.com/etcd-io/etcd/pull/20941) ### Dependencies - Compile binaries using [go 1.24.11](https://github.com/etcd-io/etcd/pull/20998). --- ## v3.6.6 (2025-11-11) ### etcd server - [Reject watch request with -1 revision to prevent invalid resync behavior on uncompacted etcd](https://github.com/etcd-io/etcd/pull/20707) - [Change the TLS handshake 'EOF' errors to DEBUG not to spam logs](https://github.com/etcd-io/etcd/pull/20749) - Fix [endpoint status not retuning the correct storage quota](https://github.com/etcd-io/etcd/pull/20790) - Fix [`--force-new-cluster can't clean up learners after creating snapshot`](https://github.com/etcd-io/etcd/pull/20896) - Fix [duplicate metrics collector registration that caused warning messages](https://github.com/etcd-io/etcd/pull/20905) ### Dependencies - Compile binaries using [go 1.24.10](https://github.com/etcd-io/etcd/pull/20901). --- ## v3.6.5 (2025-09-19) ### etcd server - [Remove the flag `--experimental-snapshot-catch-up-entries` from `etcd --help` output](https://github.com/etcd-io/etcd/pull/20422) - Fix [etcd repeatedly log the error "cannot detect storage schema version: missing confstate information"](https://github.com/etcd-io/etcd/pull/20496) - Fix [etcd may return success for leaseRenew request even when the lease is revoked](https://github.com/etcd-io/etcd/pull/20615) - Fix [potential data corruption when applySnapshot and defragment happen concurrently](https://github.com/etcd-io/etcd/pull/20650) ### Dependencies - Compile binaries using [go 1.24.7](https://github.com/etcd-io/etcd/pull/20664). - [Bump bbolt to v1.4.3](https://github.com/etcd-io/etcd/pull/20513). --- ## v3.6.4 (2025-07-25) ### etcd server - Fix [etcdserver bootstrap failure when replaying learner promotion operation due to not exist in v3store](https://github.com/etcd-io/etcd/pull/20387) --- ## v3.6.3 (2025-07-22) ### etcd server - Fix [v2store check (IsMetaStoreOnly) returns wrong result even there is no any auth data](https://github.com/etcd-io/etcd/pull/20370) - Improve [help message for --quota-backend-bytes](https://github.com/etcd-io/etcd/pull/20352) --- ## v3.6.2 (2025-07-09) ### etcd server - Fix [Watch on future revision returns old events or notifications](https://github.com/etcd-io/etcd/pull/20286) ### Dependencies - [Bump bbolt to v1.4.2](https://github.com/etcd-io/etcd/pull/20267) - Compile binaries using [go 1.23.11](https://github.com/etcd-io/etcd/pull/20314). --- ## v3.6.1 (2025-06-06) ### etcd server - [Replaced the deprecated/removed `UnaryServerInterceptor` and `StreamServerInterceptor` in otelgrpc with `NewServerHandler`](https://github.com/etcd-io/etcd/pull/20043) - [Add protection on `PromoteMember` and `UpdateRaftAttributes` to prevent panicking](https://github.com/etcd-io/etcd/pull/20051) - [Fix the issue that `--force-new-cluster` can't remove all other members in a corner case](https://github.com/etcd-io/etcd/pull/20071) - Fix [mvcc: avoid double decrement of watcher gauge on close/cancel race](https://github.com/etcd-io/etcd/pull/20067) - [Add validation to ensure there is no empty v3discovery endpoint](https://github.com/etcd-io/etcd/pull/20113) ### etcdctl - Fix [command `etcdctl endpoint health` doesn't work when options are set via environment variables](https://github.com/etcd-io/etcd/pull/20121) ### Dependencies - Compile binaries using [go 1.23.10](https://github.com/etcd-io/etcd/pull/20128). --- ## v3.6.0 (2025-05-15) There isn't any production code change since v3.6.0-rc.5. --- ## v3.6.0-rc.5 (2025-05-08) ### etcd server - Fix [the compaction pause duration metric is not emitted for every compaction batch](https://github.com/etcd-io/etcd/pull/19770) ### Package `clientv3` - [Replace `resolver.State.Addresses` with `resolver.State.Endpoint.Addresses`](https://github.com/etcd-io/etcd/pull/19782). - [Deprecated the Metadata field in the Endpoint struct from the client/v3/naming/endpoints package](https://github.com/etcd-io/etcd/pull/19842). ### Dependencies - Compile binaries using [go 1.23.9](https://github.com/etcd-io/etcd/pull/19867). --- ## v3.6.0-rc.4 (2025-04-15) ### etcd server - [Switch to validating v3 when v2 and v3 are synchronized](https://github.com/etcd-io/etcd/pull/19703). ### Dependencies - Compile binaries using [go 1.23.8](https://github.com/etcd-io/etcd/pull/19724) --- ## v3.6.0-rc.3 (2025-03-27) ### etcd server - [Auto sync members in v3store for the issues which have already been affected by #19557](https://github.com/etcd-io/etcd/pull/19636). - [Move `client/internal/v2` into `server/internel/clientv2`](https://github.com/etcd-io/etcd/pull/19673). - [Replace ExperimentalMaxLearners with a Feature Gate](https://github.com/etcd-io/etcd/pull/19560). ### etcd grpc-proxy - Fix [grpcproxy can get stuck in and endless loop causing high CPU usage](https://github.com/etcd-io/etcd/pull/19562) ### Dependencies - Bump [github.com/golang-jwt/jwt/v5 from 5.2.1 to 5.2.2 to address CVE-2025-30204](https://github.com/etcd-io/etcd/pull/19647). - Bump [bump golang.org/x/net from v0.37.0 to v0.38.0 to address CVE-2025-22872](https://github.com/etcd-io/etcd/pull/19687). --- ## v3.6.0-rc.2 (2025-03-05) ### etcd server - Add [Prometheus metric to query server feature gates](https://github.com/etcd-io/etcd/pull/19495). ### Dependencies - Compile binaries using [go 1.23.7](https://github.com/etcd-io/etcd/pull/19527). - Bump [golang.org/x/net to v0.36.0 to address CVE-2025-22870](https://github.com/etcd-io/etcd/pull/19531). - Bump [github.com/grpc-ecosystem/grpc-gateway/v2 to v2.26.3 to fix the issue of etcdserver crashing on receiving REST watch stream requests](https://github.com/etcd-io/etcd/pull/19522). --- ## v3.6.0-rc.1 (2025-02-25) ### etcdctl v3 - Add [`DowngradeInfo` in result of endpoint status](https://github.com/etcd-io/etcd/pull/19471) ### etcd server - Add [`DowngradeInfo` to endpoint status response](https://github.com/etcd-io/etcd/pull/19471) ### Dependencies - Bump [golang.org/x/crypto to v0.35.0 to address CVE-2025-22869](https://github.com/etcd-io/etcd/pull/19480). --- ## v3.6.0-rc.0 (2025-02-13) See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0). ### Breaking Changes - `etcd` will no longer start on data dir created by newer versions (for example etcd v3.6 will not run on v3.7+ data dir). To downgrade data dir please check out `etcdutl migrate` command. - `etcd` doesn't support serving client requests on the peer listen endpoints (--listen-peer-urls). See [pull/13565](https://github.com/etcd-io/etcd/pull/13565). - `etcdctl` will sleep(2s) in case of range delete without `--range` flag. See [pull/13747](https://github.com/etcd-io/etcd/pull/13747) - Applications which depend on etcd v3.6 packages must be built with go version >= v1.18. #### Flags Removed - The following flags have been removed: - `--enable-v2` - `--experimental-enable-v2v3` - `--proxy` - `--proxy-failure-wait` - `--proxy-refresh-interval` - `--proxy-dial-timeout` - `--proxy-write-timeout` - `--proxy-read-timeout` ### Deprecations - Deprecated [V2 discovery](https://etcd.io/docs/v3.5/dev-internal/discovery_protocol/). - Deprecated [SetKeepAlive and SetKeepAlivePeriod in limitListenerConn](https://github.com/etcd-io/etcd/pull/14356). - Removed [etcdctl defrag --data-dir](https://github.com/etcd-io/etcd/pull/13793). - Removed [etcdctl snapshot status](https://github.com/etcd-io/etcd/pull/13809). - Removed [etcdctl snapshot restore](https://github.com/etcd-io/etcd/pull/13809). - Removed [NewZapCoreLoggerBuilder in server/embed](https://github.com/etcd-io/etcd/pull/19404) ### etcdctl v3 - Add command to generate [shell completion](https://github.com/etcd-io/etcd/pull/13133). - When print endpoint status, [show db size in use](https://github.com/etcd-io/etcd/pull/13639) - [Always print the raft_term in decimal](https://github.com/etcd-io/etcd/pull/13711) when displaying member list in json. - [Add one more field `storageVersion`](https://github.com/etcd-io/etcd/pull/13773) into the response of command `etcdctl endpoint status`. - Add [`--max-txn-ops`](https://github.com/etcd-io/etcd/pull/14340) flag to make-mirror command. - Add [`--consistency`](https://github.com/etcd-io/etcd/pull/15261) flag to member list command. - Display [field `hash_revision`](https://github.com/etcd-io/etcd/pull/14812) for `etcdctl endpoint hash` command. - Add [`--max-request-bytes` and `--max-recv-bytes`](https://github.com/etcd-io/etcd/pull/18718) global flags. ### etcdutl v3 - Add command to generate [shell completion](https://github.com/etcd-io/etcd/pull/13142). - Add `migrate` command for downgrading/upgrading etcd data dir files. - Add [optional --bump-revision and --mark-compacted flag to etcdutl snapshot restore operation](https://github.com/etcd-io/etcd/pull/16029). - Add [hashkv](https://github.com/etcd-io/etcd/pull/15965) command to print hash of keys and values up to given revision - Removed [legacy etcdutl backup](https://github.com/etcd-io/etcd/pull/16662) - [Count the number of keys from users perspective](https://github.com/etcd-io/etcd/pull/19344) ### Package `clientv3` - [Support serializable `MemberList` operation](https://github.com/etcd-io/etcd/pull/15261). ### Package `server` - Package `mvcc` was moved to `storage/mvcc` - Package `mvcc/backend` was moved to `storage/backend` - Package `mvcc/buckets` was moved to `storage/schema` - Package `wal` was moved to `storage/wal` - Package `datadir` was moved to `storage/datadir` ### Package `raft` - [Decouple raft from etcd](https://github.com/etcd-io/etcd/issues/14713). Migrated raft to a separate [repository](https://github.com/etcd-io/raft), and renamed raft module to `go.etcd.io/raft/v3`. ### etcd server - Add [`etcd --log-format`](https://github.com/etcd-io/etcd/pull/13339) flag to support log format. - Add [`etcd --experimental-max-learners`](https://github.com/etcd-io/etcd/pull/13377) flag to allow configuration of learner max membership. - Add [`etcd --experimental-enable-lease-checkpoint-persist`](https://github.com/etcd-io/etcd/pull/13508) flag to handle upgrade from v3.5.2 clusters with this feature enabled. - Add [`etcdctl make-mirror --rev`](https://github.com/etcd-io/etcd/pull/13519) flag to support incremental mirror. - Add [v3 discovery](https://github.com/etcd-io/etcd/pull/13635) to bootstrap a new etcd cluster. - Add [field `storage`](https://github.com/etcd-io/etcd/pull/13772) into the response body of endpoint `/version`. - Add [`etcd --max-concurrent-streams`](https://github.com/etcd-io/etcd/pull/14169) flag to configure the max concurrent streams each client can open at a time, and defaults to math.MaxUint32. - Add [`etcd grpc-proxy --experimental-enable-grpc-logging`](https://github.com/etcd-io/etcd/pull/14266) flag to logging all grpc requests and responses. - Add [`etcd --experimental-compact-hash-check-enabled --experimental-compact-hash-check-time`](https://github.com/etcd-io/etcd/issues/14039) flags to support enabling reliable corruption detection on compacted revisions. - Add [Protection on maintenance request when auth is enabled](https://github.com/etcd-io/etcd/pull/14663). - Graduated [`--experimental-warning-unary-request-duration` to `--warning-unary-request-duration`](https://github.com/etcd-io/etcd/pull/14414). Note the experimental flag is deprecated and will be decommissioned in v3.7. - Add [field `hash_revision` into `HashKVResponse`](https://github.com/etcd-io/etcd/pull/14537). - Add [`etcd --experimental-snapshot-catch-up-entries`](https://github.com/etcd-io/etcd/pull/15033) flag to configure number of entries for a slow follower to catch up after compacting the raft storage entries and defaults to 5k. - Decreased [`--snapshot-count` default value from 100,000 to 10,000](https://github.com/etcd-io/etcd/pull/15408) - Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15156) to enable support for TLS 1.3. - Add [quota to endpoint status response](https://github.com/etcd-io/etcd/pull/17877) - Add [feature gate `SetMemberLocalAddr`](https://github.com/etcd-io/etcd/pull/19413) to [enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer]((https://github.com/etcd-io/etcd/pull/17661)) - Add [Support multiple values for allowed client and peer TLS identities](https://github.com/etcd-io/etcd/pull/18015) - Add [`embed.Config.GRPCAdditionalServerOptions`](https://github.com/etcd-io/etcd/pull/14066) to support updating the default internal gRPC configuration for embedded use cases. ### etcd grpc-proxy - Add [`etcd grpc-proxy start --endpoints-auto-sync-interval`](https://github.com/etcd-io/etcd/pull/14354) flag to enable and configure interval of auto sync of endpoints with server. - Add [`etcd grpc-proxy start --listen-cipher-suites`](https://github.com/etcd-io/etcd/pull/14308) flag to support adding configurable cipher list. - Add [`tls min/max version to grpc proxy`](https://github.com/etcd-io/etcd/pull/18816) to support setting TLS min and max version. ### tools/benchmark - [Add etcd client autoSync flag](https://github.com/etcd-io/etcd/pull/13416) ### Metrics, Monitoring See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. - Add [`etcd_disk_defrag_inflight`](https://github.com/etcd-io/etcd/pull/13371). - Add [`etcd_debugging_server_alarms`](https://github.com/etcd-io/etcd/pull/14276). - Add [`etcd_server_range_duration_seconds`](https://github.com/etcd-io/etcd/pull/17983). ### Go - Require [Go 1.23+](https://github.com/etcd-io/etcd/pull/16594). - Compile with [Go 1.23+](https://go.dev/doc/devel/release#go1.21.minor). Please refer to [gc-guide](https://go.dev/doc/gc-guide) to configure `GOGC` and `GOMEMLIMIT` properly. ### Other - Use Distroless as base image to make the image less vulnerable and reduce image size. - [Upgrade grpc-gateway from v1 to v2](https://github.com/etcd-io/etcd/pull/16595). - [Switch from grpc-ecosystem/go-grpc-prometheus to grpc-ecosystem/go-grpc-middleware/providers/prometheus](https://github.com/etcd-io/etcd/pull/19195). --- ================================================ FILE: CHANGELOG/CHANGELOG-3.7.md ================================================ Previous change logs can be found at [CHANGELOG-3.6](https://github.com/etcd-io/etcd/blob/main/CHANGELOG/CHANGELOG-3.6.md). --- ## v3.7.0 (TBD) ### Breaking Changes - [Removed all deprecated experimental flags](https://github.com/etcd-io/etcd/pull/19959) - [Removed v2discovery](https://github.com/etcd-io/etcd/pull/20109) - [Removed client/v2](https://github.com/etcd-io/etcd/pull/20117) - [Removed v2 request and apply_v2.go](https://github.com/etcd-io/etcd/pull/21263) ### etcd server - [Update go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc to v0.61.0 and replaced the deprecated `UnaryServerInterceptor` and `StreamServerInterceptor` with `NewServerHandler`](https://github.com/etcd-io/etcd/pull/20017) - [Add Support for Unix Socket endpoints](https://github.com/etcd-io/etcd/pull/19760) - [Improves performance of lease and user/role operations (up to 2x) by updating `(*readView) Rev()` to use `SharedBufReadTxMode`](https://github.com/etcd-io/etcd/pull/20411) - [Allow client to retrieve AuthStatus without authentication](https://github.com/etcd-io/etcd/pull/20802) - [Add FastLeaseKeepAlive feature to enable faster lease renewal by skipping the wait for the applied index](https://github.com/etcd-io/etcd/pull/20589) - [Bootstrap etcd from v3store](https://github.com/etcd-io/etcd/issues/20187), see changes below, - [Stop loading v2 snapshot files](https://github.com/etcd-io/etcd/pull/21107) - [Initialize confState from v3 store on bootstrap](https://github.com/etcd-io/etcd/pull/21138) - [Remove flag `--max-snapshots` in 3.8 rather than 3.7](https://github.com/etcd-io/etcd/pull/21160) - [Keep the `--snapshot-count` flag](https://github.com/etcd-io/etcd/pull/21162) ### Package `clientv3` - Allow setting JWT directly by users, see https://github.com/etcd-io/etcd/pull/16803 and https://github.com/etcd-io/etcd/pull/20747. - [Function etcdClientDebugLevel is renamed to ClientLogLevel and made it public](https://github.com/etcd-io/etcd/pull/20006) ### Package `pkg` - [Optimize find performance by splitting intervals with the same left endpoint by their right endpoints](https://github.com/etcd-io/etcd/pull/19768) - [netutil: Refactor IPv6 address comparison logic](https://github.com/etcd-io/etcd/pull/20365) ### Dependencies - Compile binaries with [Go 1.26](https://go.dev/doc/devel/release#go1.26.minor). - [Migrate the deprecated go-grpc-middleware v1 logging and tags libraries to v2 interceptors](https://github.com/etcd-io/etcd/pull/20420) ### Deprecations - Deprecated [UsageFunc in pkg/cobrautl](https://github.com/etcd-io/etcd/pull/18356). ### etcdctl - [Organize etcdctl commands](https://github.com/etcd-io/etcd/pull/20162) to make them more concise and easier to understand. - [Hide the global flags](https://github.com/etcd-io/etcd/pull/20493) to make the output of `etcdctl --help` looks cleaner and is consistent with kubectl. ### etcdutl - [Add a timeout flag to all etcdutl commands](https://github.com/etcd-io/etcd/pull/20708) when waiting to acquire a file lock on the database file. ### Metrics, Monitoring See [List of metrics](https://etcd.io/docs/latest/metrics/) for all metrics per release. - Add [`etcd_server_request_duration_seconds`](https://github.com/etcd-io/etcd/pull/21038). - Add [the following metrics related to watch send loop](https://github.com/etcd-io/etcd/pull/21030), - etcd_debugging_server_watch_send_loop_watch_stream_duration_seconds - etcd_debugging_server_watch_send_loop_watch_stream_duration_per_event_seconds - etcd_debugging_server_watch_send_loop_control_stream_duration_seconds - etcd_debugging_server_watch_send_loop_progress_duration_seconds ================================================ FILE: CHANGELOG/CHANGELOG-4.0.md ================================================ Previous change logs can be found at [CHANGELOG-3.x](https://github.com/etcd-io/etcd/blob/main/CHANGELOG/CHANGELOG-3.x.md). --- ## v4.0.0 (TBD) See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v4.0.0) and [v4.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_4_0/) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v4.0 upgrade guide](https://etcd.io/docs/latest/upgrades/upgrade_4_0/).** ### Breaking Changes - [Secure etcd by default](https://github.com/etcd-io/etcd/issues/9475)? - Deprecate [`etcd --proxy*`](TODO) flags; **no more v2 proxy**. - Deprecate [v2 storage backend](https://github.com/etcd-io/etcd/issues/9232); **no more v2 store**. - v2 API is still supported via [v2 emulation](TODO). - Deprecate [`etcdctl backup`](TODO) command. - `clientv3.Client.KeepAlive(ctx context.Context, id LeaseID) (<-chan *LeaseKeepAliveResponse, error)` is now [`clientv4.Client.KeepAlive(ctx context.Context, id LeaseID) <-chan *LeaseKeepAliveResponse`](TODO). - Similar to `Watch`, [`KeepAlive` does not return errors](https://github.com/etcd-io/etcd/issues/7488). - If there's an unknown server error, kill all open channels and create a new stream on the next `KeepAlive` call. - Rename `github.com/coreos/client` to `github.com/coreos/clientv2`. - [`etcd --experimental-initial-corrupt-check`](TODO) has been deprecated. - Use [`etcd --initial-corrupt-check`](TODO) instead. - [`etcd --experimental-corrupt-check-time`](TODO) has been deprecated. - Use [`etcd --corrupt-check-time`](TODO) instead. - Enable TLS 1.13, deprecate TLS cipher suites. ### etcd server - [`etcd --initial-corrupt-check`](TODO) flag is now stable (`etcd --experimental-initial-corrupt-check` has been deprecated). - `etcd --initial-corrupt-check=true` by default, to check cluster database hashes before serving client/peer traffic. - [`etcd --corrupt-check-time`](TODO) flag is now stable (`etcd --experimental-corrupt-check-time` has been deprecated). - `etcd --corrupt-check-time=12h` by default, to check cluster database hashes for every 12-hour. - Enable TLS 1.13, deprecate TLS cipher suites. ### Go - Require [*Go 2*](https://blog.golang.org/go2draft). --- ================================================ FILE: CHANGELOG/README.md ================================================ # Change logs ## Production recommendation The minimum recommended etcd versions to run in **production** are v3.4.22+ and v3.5.6+. Refer to the [versioning policy](https://etcd.io/docs/v3.5/op-guide/versioning/) for more details. ### v3.5 data corruption issue Running etcd v3.5.2, v3.5.1 and v3.5.0 under high load can cause a data corruption issue. If etcd process is killed, occasionally some committed transactions are not reflected on all the members. Recommendation is to upgrade to v3.5.4+. If you have encountered data corruption, please follow instructions on https://etcd.io/docs/v3.5/op-guide/data_corruption/. ## Change log rules 1. Each patch release only includes changes against previous patch release. For example, the change log of v3.5.5 should only include items which are new to v3.5.4. 2. For the first release (e.g. 3.4.0, 3.5.0, 3.6.0, 4.0.0 etc.) for each minor or major version, it only includes changes which are new to the first release of previous minor or major version. For example, v3.5.0 should only include items which are new to v3.4.0, and v3.6.0 should only include items which are new to v3.5.0. ================================================ FILE: CONTRIBUTING.md ================================================ # How to contribute etcd is Apache 2.0 licensed and accepts contributions via GitHub pull requests. This document outlines the basics of contributing to etcd. This is a rough outline of what a contributor's workflow looks like: * [Find something to work on](#Find-something-to-work-on) * [Check for flaky tests](#Check-for-flaky-tests) * [Set up development environment](#Set-up-development-environment) * [Implement your change](#Implement-your-change) * [Commit your change](#Commit-your-change) * [Create a pull request](#Create-a-pull-request) * [Get your pull request reviewed](#Get-your-pull-request-reviewed) If you have any questions, please reach out using one of the methods listed in [contact]. [contact]: ./README.md#Contact ## Learn more about etcd Before making a change please look through the resources below to learn more about etcd and tools used for development. * Please learn about [Git](https://github.com/git-guides) version control system used in etcd. * Read the [etcd learning resources](https://etcd.io/docs/v3.5/learning/) * Read the [etcd community membership](/Documentation/contributor-guide/community-membership.md) * Watch [etcd deep dive](https://www.youtube.com/watch?v=D2pm6ufIt98&t=927s) * Watch [etcd code walkthrough](https://www.youtube.com/watch?v=H3XaSF6wF7w) ## Find something to work on All the work in the etcd project is tracked in [GitHub issue tracker]. Issues should be properly labeled making it easy to find something for you. Depending on your interest and experience you should check different labels: * If you are just starting, check issues labeled with [good first issue]. * When you feel more comfortable in your contributions, check out [help wanted]. * Advanced contributors can try to help with issues labeled [priority/important] covering the most relevant work at the time. If any of the aforementioned labels don't have unassigned issues, please [contact] one of the [maintainers] asking to triage more issues. [github issue tracker]: https://github.com/etcd-io/etcd/issues [good first issue]: https://github.com/search?type=issues&q=org%3Aetcd-io+state%3Aopen++label%3A%22good+first+issue%22 [help wanted]: https://github.com/search?type=issues&q=org%3Aetcd-io+state%3Aopen++label%3A%22help+wanted%22 [maintainers]: https://github.com/etcd-io/etcd/blob/main/OWNERS [priority/important]: https://github.com/search?type=issues&q=org%3Aetcd-io+state%3Aopen++label%3A%22priority%2Fimportant%22 ### Check for flaky tests The project could always use some help to deflake tests. [These](https://github.com/etcd-io/etcd/issues?q=is%3Aissue+is%3Aopen+label%3Atype%2Fflake) are the currently open flaky test issues. For more, because etcd uses Kubernetes' prow infrastructure to run CI jobs, the past test results can be viewed at [testgrid](https://testgrid.k8s.io/sig-etcd). | Tests | Status | | ----- | ------ | | periodics e2e-amd64 | [![sig-etcd-periodics/ci-etcd-e2e-amd64](https://testgrid.k8s.io/q/summary/sig-etcd-periodics/ci-etcd-e2e-amd64/tests_status?style=svg)](https://testgrid.k8s.io/q/summary/sig-etcd-periodics/ci-etcd-e2e-amd64) | | presubmit build | [![sig-etcd-presubmits/pull-etcd-build](https://testgrid.k8s.io/q/summary/sig-etcd-presubmits/pull-etcd-build/tests_status?style=svg)](https://testgrid.k8s.io/q/summary/sig-etcd-presubmits/pull-etcd-build) | | presubmit e2e-amd64 | [![sig-etcd-presubmits/pull-etcd-e2e-amd64](https://testgrid.k8s.io/q/summary/sig-etcd-presubmits/pull-etcd-e2e-amd64/tests_status?style=svg)](https://testgrid.k8s.io/q/summary/sig-etcd-presubmits/pull-etcd-e2e-amd64) | | presubmit unit-test-amd64 | [![sig-etcd-presubmits/pull-etcd-unit-test-amd64](https://testgrid.k8s.io/q/summary/sig-etcd-presubmits/pull-etcd-unit-test-amd64/tests_status?style=svg)](https://testgrid.k8s.io/q/summary/sig-etcd-presubmits/pull-etcd-unit-test-amd64) | | presubmit verify | [![sig-etcd-presubmits/pull-etcd-verify](https://testgrid.k8s.io/q/summary/sig-etcd-presubmits/pull-etcd-verify/tests_status?style=svg)](https://testgrid.k8s.io/q/summary/sig-etcd-presubmits/pull-etcd-verify) | | postsubmit build | [![sig-etcd-postsubmits/post-etcd-build](https://testgrid.k8s.io/q/summary/sig-etcd-postsubmits/post-etcd-build/tests_status?style=svg)](https://testgrid.k8s.io/q/summary/sig-etcd-postsubmits/post-etcd-build) | If you find any flaky tests on testgrid, please 1. Check [existing issues](https://github.com/etcd-io/etcd/issues?q=is%3Aissue+is%3Aopen+label%3Atype%2Fflake) to see if an issue has already been opened for this test. If not, open an issue with the `type/flake` label. 2. Try to reproduce the flaky test on your machine via [`stress`](https://pkg.go.dev/golang.org/x/tools/cmd/stress), for example, to reproduce the failure of `TestPeriodicSkipRevNotChange`: ```bash # install the stress utility go install golang.org/x/tools/cmd/stress@latest cd server/etcdserver/api/v3compactor # compile the test go test -v -c -count 1 # run the compiled test file using stress stress -p=8 ./v3compactor.test -test.run “^TestPeriodicSkipRevNotChange$” ``` 3. Fix it. ## Set up development environment The etcd project supports two options for development: 1. Manually set up the local environment. 2. Automatically set up [devcontainer](https://containers.dev). For both options, the only supported architecture is `linux-amd64`. Bug reports for other environments will generally be ignored. Supporting new environments requires the introduction of proper tests and maintainer support that is currently lacking in the etcd project. If you would like etcd to support your preferred environment you can [file an issue]. ### Option 1 - Manually set up the local environment This is the original etcd development environment, is most supported, and is backward compatible for the development of older etcd versions. Follow the steps below to set up the environment: - [Clone the repository](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) - Install Go by following [installation](https://go.dev/doc/install). Please check the minimal go version in [go.mod file](./go.mod#L3). - Install build tools: - [`make`](https://www.gnu.org/software/make/): For Debian-based distributions you can run `sudo apt-get install build-essential` - [`protoc`](https://protobuf.dev/): You can download it for your os. Use version [`v3.20.3`](https://github.com/protocolbuffers/protobuf/releases/tag/v3.20.3). - [`yamllint`](https://www.yamllint.com/): For Debian-based distribution you can run `sudo apt-get install yamllint` - [`jq`](https://jqlang.github.io/jq/): For Debian-based distribution you can run `sudo apt-get install jq` - [`xz`](https://tukaani.org/xz/): For Debian-based distribution you can run `sudo apt-get install xz-utils` - Verify that everything is installed by running `make build` Note: `make build` runs with `-v`. Other build flags can be added through env `GO_BUILD_FLAGS`, **if required**. Eg., ```console GO_BUILD_FLAGS="-buildmode=pie" make build ``` ### Option 2 - Automatically set up devcontainer This is a more recently added environment that aims to make it faster for new contributors to get started with etcd. This option is supported for etcd versions 3.6 onwards. This option can be [used locally](https://code.visualstudio.com/docs/devcontainers/tutorial) on a system running Visual Studio Code and Docker, or in a remote cloud-based [Codespaces](https://github.com/features/codespaces) environment. To get started, create a codespace for this repository by clicking this 👇 [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=11225014) A codespace will open in a web-based version of Visual Studio Code. The [dev container](.devcontainer/devcontainer.json) is fully configured with the software needed for this project. **Note**: Dev containers is an open spec which is supported by [GitHub Codespaces](https://github.com/codespaces) and [other tools](https://containers.dev/supporting). [file an issue]: https://github.com/etcd-io/etcd/issues/new/choose ## Implement your change etcd code should follow the coding style suggested by the Golang community. See the [style doc](https://go.dev/wiki/CodeReviewComments) for details. Please ensure that your change passes static analysis (requires [golangci-lint](https://golangci-lint.run/welcome/install/)): - `make verify` to verify if all checks pass. - `make verify-*` to verify a single check, for example, `make verify-bom` to verify if `bill-of-materials.json` file is up-to-date. - `make fix` to fix all checks. - `make fix-*` to fix a single check, for example, `make fix-bom` to update `bill-of-materials.json`. Please ensure that your change passes tests. - `make test-unit` to run unit tests. - `make test-integration` to run integration tests. - `make test-e2e` to run e2e tests. All changes are expected to come with a unit test. All new features are expected to have either e2e or integration tests. ## Commit your change etcd follows a rough convention for commit messages: * First line: * Should start with the name of the package (for example `etcdserver`, `etcdctl`) followed by the `:` character. * Describe the `what` behind the change * Optionally, the author might provide the `why` behind the change in the main commit message body. * Last line should be `Signed-off-by: firstname lastname ` (can be automatically generate by providing `--signoff` to git commit command). Example of commit message: ``` etcdserver: add grpc interceptor to log info on incoming requests To improve debuggability of etcd v3. Added a grpc interceptor to log info on incoming requests to etcd server. The log output includes remote client info, request content (with value field redacted), request handling latency, response size, etc. Uses zap logger if available, otherwise uses capnslog. Signed-off-by: FirstName LastName ``` ## Create a pull request Please follow the [making a pull request](https://docs.github.com/en/get-started/quickstart/contributing-to-projects#making-a-pull-request) guide. If you are still working on the pull request, you can convert it to a draft by clicking `Convert to draft` link just below the list of reviewers. Multiple small PRs are preferred over single large ones (>500 lines of code). Please make sure there is an associated issue for each PR you submit. Create one if it doesn't exist yet, and close the issue once the PR gets merged and has been backported to previous stable releases, if necessary. If there are multiple PRs linked to the same issue, refrain from closing the issue until all PRs have been merged and, if needed, backported to previous stable releases. ## Get your pull request reviewed Before requesting review please ensure that all GitHub and Prow checks are successful. In some cases your pull request may have the label `needs-ok-to-test`. If so an `etcd-io` organisation member will leave a comment on your pull request with `/ok-to-test` to trigger all checks to be run. It might happen that some unrelated tests on your PR are failing, due to their flakiness. In such cases please [file an issue] to deflake the problematic test and ask one of [maintainers] to rerun the tests. If all checks were successful feel free to reach out for review from people that were involved in the original discussion or [maintainers]. Depending on the complexity of the PR it might require between 1 and 2 maintainers to approve your change before merging. Thanks for contributing! ================================================ FILE: DCO ================================================ Developer Certificate of Origin Version 1.1 Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 660 York Street, Suite 102, San Francisco, CA 94110 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. ================================================ FILE: Dockerfile ================================================ ARG ARCH=amd64 FROM --platform=linux/${ARCH} gcr.io/distroless/static-debian12@sha256:20bc6c0bc4d625a22a8fde3e55f6515709b32055ef8fb9cfbddaa06d1760f838 ADD etcd /usr/local/bin/ ADD etcdctl /usr/local/bin/ ADD etcdutl /usr/local/bin/ WORKDIR /var/etcd/ WORKDIR /var/lib/etcd/ EXPOSE 2379 2380 # Define default command. CMD ["/usr/local/bin/etcd"] ================================================ FILE: Documentation/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/documentation ================================================ FILE: Documentation/README.md ================================================ This directory includes etcd project internal documentation for new and existing contributors. For user and developer documentation please go to [etcd.io](https://etcd.io/), which is developed in [website](https://github.com/etcd-io/website/) repo. ================================================ FILE: Documentation/contributor-guide/branch_management.md ================================================ # Branch management ## Guide * New development occurs on the [main branch][main]. * The main branch should always have a green build! * Backwards-compatible bug fixes should target the main branch and subsequently be ported to stable branches. * Once the main branch is ready for release, it will be tagged and become the new stable branch. The etcd team has adopted a *rolling release model* and supports two stable versions of etcd. ### Main branch The `main` branch is our development branch. All new features land here first. To try new and experimental features, pull `main` and play with it. Note that `main` may not be stable because new features may introduce bugs. Before the release of the next stable version, feature PRs will be frozen. A [release manager](./release.md#release-management) will be assigned to the major/minor version and will lead the etcd community in testing, bug-fix, and documentation of the release for one to two weeks. ### Stable branches All branches with the prefix `release-` are considered _stable_ branches. After every minor release ([semver.org](https://semver.org/)), we will have a new stable branch for that release, managed by a [patch release manager](./release.md#release-management). We will keep fixing the backward-compatible bugs for the latest two stable releases. A _patch_ release to each supported release branch, incorporating any bug fixes, will be once every two weeks, given any patches. [main]: https://github.com/etcd-io/etcd/tree/main ================================================ FILE: Documentation/contributor-guide/bump_etcd_version_k8s.md ================================================ # Bump etcd Version in Kubernetes This guide will walk through the update of etcd in Kubernetes to a new version (`kubernetes/kubernetes` repository). > Currently we bump etcd v3.5.x for K8s release-1.33 and lower versions, and we bump etcd v3.6.x for K8s release-1.34 and higher versions. You can use this [issue](https://github.com/kubernetes/kubernetes/issues/131101) as a reference when updating the etcd version in Kubernetes. Bumping the etcd version in Kubernetes consists of two steps. * Bump etcd client SDK * Bump etcd image > The commented lines in this document signifies the line to be changed ## Bump etcd client SDK > Reference: [link](https://github.com/kubernetes/kubernetes/pull/131103) You can refer to the guide [here](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/vendor.md) under the **Adding or updating a dependency** section. 1. Get all the etcd modules used in Kubernetes. ```bash $ grep 'go.etcd.io/etcd/' go.mod | awk '{print $1}' go.etcd.io/etcd/api/v3 go.etcd.io/etcd/client/pkg/v3 go.etcd.io/etcd/client/v3 go.etcd.io/etcd/client/v2 go.etcd.io/etcd/pkg/v3 go.etcd.io/etcd/raft/v3 go.etcd.io/etcd/server/v3 ``` 2. For each module, in the root directory of the `kubernetes/kubernetes` repository, fetch the new version in `go.mod` using the following command (using `client/v3` as an example): ```bash hack/pin-dependency.sh go.etcd.io/etcd/client/v3 NEW_VERSION ``` 3. Rebuild the `vendor` directory and update the `go.mod` files for all staging repositories using the command below. This automatically updates the licenses. ```bash hack/update-vendor.sh ``` 4. Check if the new dependency requires newer versions of existing dependencies we have pinned. You can check this by: * Running `hack/lint-dependencies.sh` against your branch and against `master` and comparing the results. * Checking if any new `replace` directives were added to `go.mod` files of components inside the staging directory. ## Bump etcd image ### Build etcd image > Reference: [link 1](https://github.com/kubernetes/kubernetes/pull/131105) [link 2](https://github.com/kubernetes/kubernetes/pull/131126) 1. In `build/dependencies.yaml`, update the `version` of `etcd-image` to the new version. Update `golang: etcd release version` if necessary. ```yaml - name: "etcd-image" # version: 3.5.17 version: 3.5.21 refPaths: - path: cluster/images/etcd/Makefile match: BUNDLED_ETCD_VERSIONS\?| --- - name: "golang: etcd release version" # version: 1.22.9 version: 1.23.7 # https://github.com/etcd-io/etcd/blob/main/CHANGELOG/CHANGELOG-3.6.md ``` 2. In `cluster/images/etcd/Makefile`, include the new version in `BUNDLED_ETCD_VERSIONS` and update the `LATEST_ETCD_VERSION` as well (the image tag will be generated from the `LATEST_ETCD_VERSION`). Update `GOLANG_VERSION` according to the version used to compile that release version (`"golang: etcd release version"` in step 1). ```Makefile # BUNDLED_ETCD_VERSIONS?=3.4.18 3.5.17 BUNDLED_ETCD_VERSIONS?=3.4.18 3.5.21 # LATEST_ETCD_VERSION?=3.5.17 LATEST_ETCD_VERSION?=3.5.21 # GOLANG_VERSION := 1.22.9 GOLANG_VERSION := 1.23.7 ``` 3. In `cluster/images/etcd/migrate/options.go`, include the new version in the `supportedEtcdVersions` slice. ```go var ( // supportedEtcdVersions = []string{"3.4.18", "3.5.17"} supportedEtcdVersions = []string{"3.4.18", "3.5.21"} ) ``` ### Publish etcd image > Reference: [link](https://github.com/kubernetes/k8s.io/pull/7957) 1. When the previous step is merged, a post-commit job will run to build the image. You can find the newly built image in the [registry](https://gcr.io/k8s-staging-etcd/etcd). 2. Locate the newly built image and copy its SHA256 digest. 3. Inside the `kubernetes/k8s.io` repository, in `registry.k8s.io/images/k8s-staging-etcd/images.yaml`, create a new entry for the desired version and copy the SHA256 digest. ```yaml "sha256:b4a9e4a7e1cf08844c7c4db6a19cab380fbf0aad702b8c01e578e9543671b9f9": ["3.5.17-0"] # ADD: "sha256:d58c035df557080a27387d687092e3fc2b64c6d0e3162dc51453a115f847d121": ["3.5.21-0"] ``` ### Update to use the new etcd image > Reference: [link](https://github.com/kubernetes/kubernetes/pull/131144) 1. In `build/dependencies.yaml`, change the `version` of `etcd` to the new version. ```yaml # etcd - name: "etcd" # version: 3.5.17 version: 3.5.21 refPaths: - path: cluster/gce/manifests/etcd.manifest match: etcd_docker_tag|etcd_version ``` 2. In `cluster/gce/manifests/etcd.manifest`, change the image tag to the new image tag and `TARGET_VERSION` to the new version. ```manifest // "image": "{{ pillar.get('etcd_docker_repository', 'registry.k8s.io/etcd') }}:{{ pillar.get('etcd_docker_tag', '3.5.17-0') }}", "image": "{{ pillar.get('etcd_docker_repository', 'registry.k8s.io/etcd') }}:{{ pillar.get('etcd_docker_tag', '3.5.21-0') }}", --- { "name": "TARGET_VERSION", // "value": "{{ pillar.get('etcd_version', '3.5.17') }}" "value": "{{ pillar.get('etcd_version', '3.5.21') }}" }, ``` 3. In `cluster/gce/upgrade-aliases.sh`, update the exports for `ETCD_IMAGE` to the new image tag and `ETCD_VERSION` to the new version. ```sh # export ETCD_IMAGE=3.5.17-0 export ETCD_IMAGE=3.5.21-0 # export ETCD_VERSION=3.5.17 export ETCD_VERSION=3.5.21 ``` 4. In `cmd/kubeadm/app/constants/constants.go`, change the `DefaultEtcdVersion` to the new version. In the same file, update `SupportedEtcdVersion` accordingly. ```go // DefaultEtcdVersion = "3.5.17-0" DefaultEtcdVersion = "3.5.21-0" --- SupportedEtcdVersion = map[uint8]string{ // 30: "3.5.17-0", // 31: "3.5.17-0", // 32: "3.5.17-0", // 33: "3.5.17-0", 30: "3.5.21-0", 31: "3.5.21-0", 32: "3.5.21-0", 33: "3.5.21-0", } ``` 5. In `hack/lib/etcd.sh`, update the `ETCD_VERSION`. ```sh # ETCD_VERSION=${ETCD_VERSION:-3.5.17} ETCD_VERSION=${ETCD_VERSION:-3.5.21} ``` 6. In `staging/src/k8s.io/sample-apiserver/artifacts/example/deployment.yaml`, update the etcd image used. ```yaml - name: etcd # image: gcr.io/etcd-development/etcd:v3.5.17 image: gcr.io/etcd-development/etcd:v3.5.21 ``` 7. In `test/utils/image/manifest.go`, update the etcd image tag. ```go // configs[Etcd] = Config{list.GcEtcdRegistry, "etcd", "3.5.17-0"} configs[Etcd] = Config{list.GcEtcdRegistry, "etcd", "3.5.21-0"} ``` ================================================ FILE: Documentation/contributor-guide/community-membership.md ================================================ # Community membership This doc outlines the various responsibilities of contributor roles in etcd. | Role | Responsibilities | Requirements | Defined by | |------------|----------------------------------------------|---------------------------------------------------------------|-------------------------------| | Member | Active contributor in the community | Sponsored by 2 reviewers and multiple contributions | etcd GitHub org member | | Reviewer | Review contributions from other members | History of review and authorship | [OWNERS] file reviewer entry | | Maintainer | Set direction and priorities for the project | Demonstrated responsibility and excellent technical judgement | [OWNERS] file approver entry | ## New contributors New contributors should be welcomed to the community by existing members, helped with PR workflow, and directed to relevant documentation and communication channels. ## Established community members Established community members are expected to demonstrate their adherence to the principles in this document, familiarity with project organization, roles, policies, procedures, conventions, etc., and technical and/or writing ability. Role-specific expectations, responsibilities, and requirements are enumerated below. ## Member Members are continuously active contributors to the community. They can have issues and PRs assigned to them. Members are expected to remain active contributors to the community. **Defined by:** Member of the etcd GitHub organization. ### Member requirements - Enabled [two-factor authentication] on their GitHub account - Have made multiple contributions to the project or community. Contribution may include, but is not limited to: - Authoring or reviewing PRs on GitHub. At least one PR must be **merged**. - Filing or commenting on issues on GitHub - Contributing to community discussions (e.g. meetings, Slack, email discussion forums, Stack Overflow) - Subscribed to [etcd-dev@googlegroups.com](https://groups.google.com/g/etcd-dev) - Have read the [contributor guide] - Sponsored by two active maintainers or reviewers. - Sponsors must be from multiple member companies to demonstrate integration across the community. - With no objections from other maintainers - Open a [membership nomination] issue against the `kubernetes/org` repo - Ensure your sponsors are @mentioned on the issue - Make sure that the list of contributions included is representative of your work on the project. - Members can be removed by a supermajority of the maintainers or can resign by notifying the maintainers. ### Member responsibilities and privileges - Responsive to issues and PRs assigned to them - Granted "triage access" to etcd project - Active owner of code they have contributed (unless ownership is explicitly transferred) - Code is well-tested - Tests consistently pass - Addresses bugs or issues discovered after code is accepted **Note:** Members who frequently contribute code are expected to proactively perform code reviews and work towards becoming a *reviewer*. ## Reviewers Reviewers are contributors who have demonstrated greater skill in reviewing the code from other contributors. They are knowledgeable about both the codebase and software engineering principles. Their LGTM counts towards merging a code change into the project. A reviewer is generally on the ladder towards maintainership. **Defined by:** *reviewers* entry in the [OWNERS] file. ### Reviewer requirements - member for at least 3 months. - Primary reviewer for at least 5 PRs to the codebase. - Reviewed or contributed at least 20 substantial PRs to the codebase. - Knowledgeable about the codebase. - Sponsored by two active maintainers. - Sponsors must be from multiple member companies to demonstrate integration across the community. - With no objections from other maintainers - Reviewers can be removed by a supermajority of the maintainers or can resign by notifying the maintainers. ### Reviewer responsibilities and privileges - Code reviewer status may be a precondition to accepting large code contributions - Responsible for project quality control via code reviews - Focus on code quality and correctness, including testing and factoring - May also review for more holistic issues, but not a requirement - Expected to be responsive to review requests - Assigned PRs to review related to area of expertise - Assigned test bugs related to area of expertise - Granted "triage access" to etcd project ## Maintainers Maintainers are first and foremost contributors who have shown they are committed to the long-term success of a project. Maintainership is about building trust with the current maintainers and being a person that they can depend on to make decisions in the best interest of the project in a consistent manner. **Defined by:** *approvers* entry in the [OWNERS] file. ### Maintainer requirements - Deep understanding of the technical goals and direction of the project - Deep understanding of the technical domain of the project - Sustained contributions to design and direction by doing all of: - Authoring and reviewing proposals - Initiating, contributing, and resolving discussions (emails, GitHub issues, meetings) - Identifying subtle or complex issues in the designs and implementation of PRs - Directly contributed to the project through implementation and/or review - Sponsored by two active maintainers and elected by supermajority - Sponsors must be from multiple member companies to demonstrate integration across the community. - To become a maintainer send an email with your candidacy to - Ensure your sponsors are @mentioned in the email - Include a list of contributions representative of your work on the project. - Existing maintainers vote will privately and respond to the email with either acceptance or feedback for suggested improvement. - With your membership approved you are expected to: - Open a PR and add an entry to the [OWNERS] file - Request to be added to the and mailing lists - Request to join [etcd-maintainer teams of the etcd-io organization in GitHub](https://github.com/orgs/etcd-io/teams/maintainers-etcd) - Request to join the private slack channel for etcd maintainers on [kubernetes slack](http://slack.kubernetes.io/) - Request access to `etcd-development` GCP project where we publish releases - Request access to passwords shared between maintainers - Request cncf service desk access by emailing - Raise cncf service desk ticket to be addded to [cncf-etcd-maintainers mailing list](https://lists.cncf.io/g/cncf-etcd-maintainers/directory) ### Maintainer responsibilities and privileges - Make and approve technical design decisions - Set technical direction and priorities - Define milestones and releases - Mentor and guide reviewers, and contributors to the project. - Participate when called upon in the [security disclosure and release process] - Ensure the continued health of the project - Adequate test coverage to confidently release - Tests are passing reliably (i.e. not flaky) and are fixed when they fail - Ensure a healthy process for discussion and decision-making is in place. - Work with other maintainers to maintain the project's overall health and success holistically ### Retiring Life priorities, interests, and passions can change. Maintainers can retire and move to [emeritus maintainers]. If a maintainer needs to step down, they should inform other maintainers and, if possible, help find someone to pick up the related work. At the very least, ensure the related work can be continued. If a maintainer has not been performing their duties for 12 months, they can be removed by other maintainers. In that case, the inactive maintainer will be first notified via an email. If the situation doesn't improve, they will be removed. If an emeritus maintainer wants to regain an active role, they can do so by renewing their contributions. Active maintainers should welcome such a move. Retiring other maintainers or regaining the status should require the approval of at least two active maintainers. Retiring maintainers must: - Open a PR and move to emeritus approvers in the [OWNERS] file - Open a PR to be removed from the [etcd-maintainer teams of the etcd-io organization in GitHub](https://github.com/orgs/etcd-io/teams/maintainers-etcd) - Remove their access to `etcd-development` GCP project where we publish releases - Raise cncf service desk ticket to be removed as a [cncf-etcd-maintainers mailing list](https://lists.cncf.io/g/cncf-etcd-maintainers/directory) admin - Request to be removed as a member of the [etcd-maintainers](https://groups.google.com/g/etcd-maintainers) and [etcd-maintainers-private](https://groups.google.com/g/etcd-maintainers-private) Google groups ## Acknowledgements Contributor roles and responsibilities were written based on [Kubernetes community membership] [OWNERS]: /OWNERS [contributor guide]: /CONTRIBUTING.md [membership nomination]: https://github.com/kubernetes/org/issues/new?assignees=&labels=area%2Fgithub-membership&projects=&template=membership.yml&title=REQUEST%3A+New+membership+for+%3Cyour-GH-handle%3E [Kubernetes community membership]: https://github.com/kubernetes/community/blob/master/community-membership.md [emeritus maintainers]: /README.md#etcd-emeritus-maintainers [security disclosure and release process]: /security/README.md [two-factor authentication]: https://docs.github.com/en/authentication/securing-your-account-with-two-factor-authentication-2fa/about-two-factor-authentication ================================================ FILE: Documentation/contributor-guide/dependency_management.md ================================================ # Dependency management ## Table of Contents - **[Main branch](#main-branch)** - [Dependencies used in workflows](#dependencies-used-in-workflows) - [Bumping order](#bumping-order) - [Steps to bump a dependency](#steps-to-bump-a-dependency) - [Alternative: Using the update_dep.sh script](#alternative-using-the-update_depsh-script) - [Indirect dependencies](#indirect-dependencies) - [Known incompatible dependency updates](#known-incompatible-dependency-updates) - [arduino/setup-protoc](#arduinosetup-protoc) - [Rotation worksheet](#rotation-worksheet) - **[Stable branches](#stable-branches)** - **[Golang versions](#golang-versions)** - **[Core dependencies mappings](#core-dependencies-mappings)** ## Main branch The dependabot is enabled & [configured](https://github.com/etcd-io/etcd/blob/main/.github/dependabot.yml) to manage dependencies for etcd `main` branch. But dependabot doesn't work well for multi-module repository like `etcd`, see [dependabot-core/issues/6678](https://github.com/dependabot/dependabot-core/issues/6678). Usually, human intervention is required each time when dependabot automatically opens some PRs to bump dependencies. Please see the guidance below. ### Dependencies used in workflows The PRs that automatically bump dependencies (see examples below) used in workflows are fine and can be approved & merged directly as long as all checks are successful. - [build(deps): bump github/codeql-action from 2.2.11 to 2.2.12](https://github.com/etcd-io/etcd/pull/15736) - [build(deps): bump actions/checkout from 3.5.0 to 3.5.2](https://github.com/etcd-io/etcd/pull/15735) - [build(deps): bump ossf/scorecard-action from 2.1.2 to 2.1.3](https://github.com/etcd-io/etcd/pull/15607) ### Bumping order When multiple etcd modules depend on the same package, please bump the package version for all the modules in the correct order. The rule is simple: if module A depends on module B, then bump the dependency for module B before module A. If the two modules do not depend on each other, then it doesn't matter to bump which module first. For example, multiple modules depend on `github.com/spf13/cobra`, so we need to bump the dependency in the following order, - go.etcd.io/etcd/pkg/v3 - go.etcd.io/etcd/server/v3 - go.etcd.io/etcd/etcdctl/v3 - go.etcd.io/etcd/etcdutl/v3 - go.etcd.io/etcd/tests/v3 - go.etcd.io/etcd/v3 - go.etcd.io/etcd/tools/v3 For more details about etcd Golang modules, please check Note the module `go.etcd.io/etcd/tools/v3` doesn't depend on any other modules, nor by any other modules, so it doesn't matter when to bump dependencies for it. ### Steps to bump a dependency Use the `github.com/spf13/cobra` as an example, follow the steps below to bump it from 1.6.1 to 1.7.0 for module `go.etcd.io/etcd/etcdctl/v3`, ```bash cd ${ETCD_ROOT_DIR}/etcdctl go get github.com/spf13/cobra@v1.7.0 go mod tidy cd .. make fix # This will update the bill of materials, Go modules and workspace, etc. ``` Execute the same steps for all other modules. When you finish bumping the dependency for all modules, then commit the change, ```bash git add . git commit --signoff -m "dependency: bump github.com/spf13/cobra from 1.6.1 to 1.7.0" ``` Please close the related PRs which were automatically opened by dependabot. When you bump multiple dependencies in one PR, it's recommended to create a separate commit for each dependency. But it isn't a must; for example, you can get all dependencies bumping for the module `go.etcd.io/etcd/tools/v3` included in one commit. #### Alternative: Using the update_dep.sh script > Note: Please use bash shell version 5.x or higher. As an alternative to the manual steps above, you can use the `update_dep.sh` script to automate the dependency bump process across all modules: ```bash # Update to a specific version ./scripts/update_dep.sh github.com/spf13/cobra v1.7.0 # Update to the latest version ./scripts/update_dep.sh github.com/spf13/cobra ``` The script will: 1. Display the current version of the dependency across all go.mod files 2. Warn and prompt for confirmation if the dependency is purely indirect 3. Update the dependency in all modules that depend on it 4. Run `make fix verify-dep` to ensure consistency across all modules 5. Display the updated versions for verification This script handles the correct bumping order automatically and ensures version consistency across all modules. #### Troubleshooting In an event of bumping the version of protoc, protoc plugins or grpc-gateway, it might change `*.proto` file which can result in the following error: ```bash [0;31mFAIL: 'genproto' FAILED at Wed Jul 31 07:09:08 UTC 2024 make: *** [Makefile:134: verify-genproto] Error 255 ``` To fix the above error, run the following script from the root of etcd repository: ```bash ./scripts/genproto.sh ``` ### Indirect dependencies Usually, we don't bump a dependency if all modules just indirectly depend on it, such as `github.com/go-logr/logr`. If an indirect dependency (e.g. `D1`) causes any CVE or bugs that affect etcd, usually the module (e.g. `M1`, not part of etcd, but used by etcd) which depends on it should bump the dependency (`D1`), and then etcd just needs to bump `M1`. However, if the module (`M1`) somehow doesn't bump the problematic dependency, then etcd can still bump it (`D1`) directly following the same steps above. But as a long-term solution, etcd should try to remove the dependency on such module (`M1`) that lack maintenance. For mixed cases, in which some modules directly while others indirectly depend on a dependency, we have multiple options, - Bump the dependency for all modules, no matter it's direct or indirect dependency. - Bump the dependency only for modules that directly depend on it. We should try to follow the first way, and temporarily fall back to the second one if we run into any issue on the first way. Eventually we should fix the issue and ensure all modules depend on the same version of the dependency. ### Known incompatible dependency updates #### arduino/setup-protoc Please refer to [build(deps): bump arduino/setup-protoc from 1.3.0 to 2.0.0](https://github.com/etcd-io/etcd/pull/16016) ### Rotation worksheet The dependabot scheduling interval is weekly; it means dependabot will automatically raise a bunch of PRs per week. Usually, human intervention is required each time. We have a [rotation worksheet](https://docs.google.com/spreadsheets/d/1jodHIO7Dk2VWTs1IRnfMFaRktS9IH8XRyifOnPdSY8I/edit#gid=1394774387), and everyone is welcome to participate; you just need to register your name in the worksheet. ## Stable branches Usually, we don't proactively bump dependencies for stable releases unless there are any CVEs or bugs that affect etcd. If we have to do it, then follow the same guidance above. Note that there is no `./scripts/fix.sh`/`make fix` in release-3.4, so no need to execute it for 3.4. ## Golang versions For all libraries that exist as independent subprojects (e.g., bbolt, raft, gofail), we should always stick to the oldest supported Go minor version for all branches, including main. It's up to the users of these libraries to choose which [Go version](https://go.dev/dl) they want to use in their own projects. For other subprojects that produce binaries or images (e.g. etcd, etcd-operator, auger), the main branches should use the latest Go minor version for development, while stable releases should use the latest patch of the previous supported Go minor version to ensure stability. Suggested steps for performing a minor version upgrade for the etcd development branch: 1. Carefully review new Go version release notes and potentially related blog posts for any deprecations, performance impacts, or other considerations. 2. Create a GitHub issue to signal intent to upgrade and invite discussion, for example, . 3. Complete the upgrade locally in your development environment by editing `.go-version` and running `make fix`. 4. Run performance benchmarks locally to compare before and after. 5. Raise a pull request for the changes, for example, . Stable etcd release branches will be maintained to stay on the latest patch release of a supported Go version. Upgrading minor versions will be completed before the minor version in use currently is no longer supported. Refer to the [Go release policy](https://go.dev/doc/devel/release). For an example of how to update etcd to a new patch release of Go refer to issue and the linked pull requests. References: - ## Core dependencies mappings [bbolt](https://github.com/etcd-io/bbolt) and [raft](https://github.com/etcd-io/raft) are two core dependencies of etcd. Both etcd 3.4.x and 3.5.x depend on bbolt 1.3.x, and etcd 3.6.x depends on bbolt 1.4.x. raft is included in the etcd repository for release-3.4 and release-3.5 branches, so etcd 3.4.x and 3.5.x do not depend on any external raft module. We moved raft into [a separate repository](https://github.com/etcd-io/raft) starting from 3.6, and the first raft release is v3.6.0, so etcd 3.6.0 depends on raft v3.6.0. Please see the table below: | etcd versions | bbolt versions | raft versions | |---------------|----------------|---------------| | 3.4.x | v1.3.x | N/A | | 3.5.x | v1.3.x | N/A | | 3.6.x | v1.4.x | v3.6.x | ================================================ FILE: Documentation/contributor-guide/exit_codes.md ================================================ # Exit Codes Reference This document provides a reference of exit codes returned by the etcd server. etcd server explicitly uses three exit codes: **0** (success), **1** (general errors), and **2** (argument errors). When terminated by signals (SIGTERM/SIGINT) on Linux/Unix systems, the exit code depends on the process type: PID 1 processes exit with code 0, while non-PID 1 processes re-raise the signal, resulting in exit codes 143 (SIGTERM) or 130 (SIGINT). ## Exit Code 0 - Success | Scenario | Code Reference | | -------- | -------------- | | Help flag (`--help`) | [`server/etcdmain/config.go#L131`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/config.go#L131) | | Version flag (`--version`) | [`server/etcdmain/config.go#L144`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/config.go#L144) | | Normal shutdown | [`server/etcdmain/etcd.go#L176`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/etcd.go#L176) | | Graceful shutdown on signal (PID 1) | [`pkg/osutil/interrupt_unix.go#L74`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/pkg/osutil/interrupt_unix.go#L74) | ## Signal Termination (128 + signal number) **Note:** This behavior is specific to Linux platform. On other platforms, etcd typically returns exit code 0 if it exits without error, and 1 otherwise. For non-PID 1 processes on Linux/Unix, the signal handler re-raises the signal to the process itself ([`pkg/osutil/interrupt_unix.go#L77`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/pkg/osutil/interrupt_unix.go#L77)), which results in the kernel setting the exit code to 128 + signal number. | Signal | Exit Code | Code Reference | | ------ | --------- | -------------- | | SIGINT (Ctrl-C) | 130 (128 + 2) | [`pkg/osutil/interrupt_unix.go#L53`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/pkg/osutil/interrupt_unix.go#L53) | | SIGTERM | 143 (128 + 15) | [`pkg/osutil/interrupt_unix.go#L53`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/pkg/osutil/interrupt_unix.go#L53) | ## Exit Code 1 - General Errors All server errors exit with code 1: | Scenario | Code Reference | | -------- | -------------- | | Failed to create logger | [`server/etcdmain/etcd.go#L60`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/etcd.go#L60) | | Failed to verify flags | [`server/etcdmain/etcd.go#L69`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/etcd.go#L69) | | Discovery token already used | [`server/etcdmain/etcd.go#L141`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/etcd.go#L141) | | Initial cluster configuration error | [`server/etcdmain/etcd.go#L155`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/etcd.go#L155) | | Discovery failed | [`server/etcdmain/etcd.go#L157`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/etcd.go#L157) | | Listener failed | [`server/etcdmain/etcd.go#L172`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/etcd.go#L172) | | Failed to list data directory | [`server/etcdmain/etcd.go#L201`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/etcd.go#L201) | | Invalid datadir (member + proxy exist) | [`server/etcdmain/etcd.go#L221`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/etcd.go#L221) | | Unsupported architecture | [`server/etcdmain/etcd.go#L252`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/etcd.go#L252) | | Generic fatal error | [`server/etcdmain/util.go#L34`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/util.go#L34) | | Invalid listen-peer-urls | [`server/embed/config.go#L821`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/embed/config.go#L821) | | Invalid listen-client-urls | [`server/embed/config.go#L830`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/embed/config.go#L830) | | Invalid listen-client-http-urls | [`server/embed/config.go#L839`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/embed/config.go#L839) | | Invalid initial-advertise-peer-urls | [`server/embed/config.go#L848`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/embed/config.go#L848) | | Invalid advertise-client-urls | [`server/embed/config.go#L857`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/embed/config.go#L857) | | Invalid listen-metrics-urls | [`server/embed/config.go#L866`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/embed/config.go#L866) | ## Exit Code 2 - Argument Errors | Scenario | Code Reference | | -------- | -------------- | | Flag parsing error | [`server/etcdmain/config.go#L133`](https://github.com/etcd-io/etcd/blob/e0a72cf470756149f4f602bf89284038e6397549/server/etcdmain/config.go#L133) | ================================================ FILE: Documentation/contributor-guide/features.md ================================================ # Features This document provides an overview of etcd features and general development guidelines for adding and deprecating them. The project maintainers can override these guidelines per the need of the project following the project governance. ## Overview The etcd features fall into three stages: Alpha, Beta, and GA. ### Alpha Any new feature is usually added as an Alpha feature. An Alpha feature is characterized as below: - Might be buggy due to a lack of user testing. Enabling the feature may not work as expected. - Disabled by default. - Support for such a feature may be dropped at any time without notice - Feature-related issues may be given lower priorities. - It can be removed in the next minor or major release without following the feature deprecation policy unless it graduates to a more stable stage. ### Beta A Beta feature is characterized as below: - Supported as part of the supported releases of etcd. - Enabled by default. - Discontinuation of support must follow the feature deprecation policy. ### GA A GA feature is characterized as below: - Supported as part of the supported releases of etcd. - Always enabled; you cannot disable it. The corresponding feature gate is no longer needed. - Discontinuation of support must follow the feature deprecation policy. ## Development Guidelines ### Adding a new feature Any new enhancements to the etcd are typically added as an Alpha feature. etcd follows the Kubernetes [KEP process](https://github.com/kubernetes/enhancements/blob/master/keps/sig-architecture/0000-kep-process/README.md) for new enhancements. The general development requirements are listed below. They can be somewhat flexible depending on the scope of the feature and review discussions and will evolve over time. - Open a [KEP](https://github.com/kubernetes/enhancements/issues) issue - It must provide a clear need for the proposed feature. - It should list development work items as checkboxes. There must be one work item towards future graduation to Beta. - Label the issue with `/sig etcd`. - Keep the issue open for tracking purposes until a decision is made on graduation. - Open a [KEP](https://github.com/kubernetes/enhancements) Pull Request (PR). - The KEP template can be simplified for etcd. - It must provide clear graduation criteria for each stage. - The KEP doc should reside in [keps/sig-etcd](https://github.com/kubernetes/enhancements/tree/master/keps/sig-etcd/) - Open Pull Requests (PRs) in [etcd](https://github.com/etcd-io/etcd) - Provide unit tests. Integration tests are also recommended as possible. - Provide robust e2e test coverage. If the feature being added is complicated or quickly needed, maintainers can decide to go with e2e tests for basic coverage initially and have robust coverage added at a later time before the feature graduation to the stable feature. - Provide logs for proper debugging. - Provide metrics and benchmarks as needed. - Add an Alpha [feature gate](https://etcd.io/docs/v3.6/feature-gates/). - Any code changes or configuration flags related to the implementation of the feature must be gated with the feature gate e.g. `if cfg.ServerFeatureGate.Enabled(features.FeatureName)`. - Add a CHANGELOG entry. - At least two maintainers must approve the KEP and related code changes. ### Graduating a feature to the next stage It is important that features don't get stuck in one stage. They should be revisited and moved to the next stage once they meet the graduation criteria listed in the KEP. A feature should stay at one stage for at least one release before being promoted. #### Provide implementation If a feature is found ready for graduation to the next stage, open a Pull Request (PR) with the following changes. - Update the feature `PreRelease` stage in `server/features/etcd_features.go`. - Update the status in the original KEP issue. At least two maintainers must approve the work. Patch releases should not be considered for graduation. ### Deprecating a feature #### Alpha Alpha features can be removed without going through the deprecation process. - Remove the feature gate in `server/features/etcd_features.go`, and clean up all relevant code. - Close the original KEP issue with reasons to drop the feature. #### Beta and GA As the project evolves, a Beta/GA feature may sometimes need to be deprecated and removed. Such a situation should be handled using the steps below: - A Beta/GA feature can only be deprecated after at least 2 minor or major releases. - Update original KEP issue if it has not been closed or create a new etcd issue with reasons and steps to deprecate the feature. - Add the feature deprecation documentation in the release notes and feature gates documentation of the next minor/major release. - In the next minor/major release, set the feature gate to `{Default: false, PreRelease: featuregate.Deprecated, LockedToDefault: false}` in `server/features/etcd_features.go`. Deprecated feature gates must respond with a warning when used. - If the feature has GAed, and the original gated codes has been cleaned up, add the disablement codes back with the feature gate. - In the minor/major release after the next, set the feature gate to `{Default: false, PreRelease: featuregate.Deprecated, LockedToDefault: true}` in `server/features/etcd_features.go`, and start cleaning the code. At least two maintainers must approve the work. Patch releases should not be considered for deprecation. ================================================ FILE: Documentation/contributor-guide/local_cluster.md ================================================ # Set up the local cluster For testing and development deployments, the quickest and easiest way is to configure a local cluster. For a production deployment, refer to the [clustering][clustering] section. ## Local standalone cluster ### Starting a cluster Run the following to deploy an etcd cluster as a standalone cluster: ``` $ ./etcd ... ``` If the `etcd` binary is not present in the current working directory, it might be located either at `$GOPATH/bin/etcd` or at `/usr/local/bin/etcd`. Run the command appropriately. The running etcd member listens on `localhost:2379` for client requests. ### Interacting with the cluster Use `etcdctl` to interact with the running cluster: 1. Store an example key-value pair in the cluster: ``` $ ./etcdctl put foo bar OK ``` If OK is printed, storing the key-value pair is successful. 2. Retrieve the value of `foo`: ``` $ ./etcdctl get foo bar ``` If `bar` is returned, interaction with the etcd cluster is working as expected. ## Local multi-member cluster ### Starting a cluster A `Procfile` at the base of the etcd git repository is provided to easily configure a local multi-member cluster. To start a multi-member cluster, navigate to the root of the etcd source tree and perform the following: 1. Install `goreman` to control Procfile-based applications: ``` $ go install github.com/mattn/goreman@latest ``` The installation will place executables in the $GOPATH/bin. If $GOPATH environment variable is not set, the tool will be installed into the $HOME/go/bin. Make sure that $PATH is set accordingly in your environment. 2. Start a cluster with `goreman` using etcd's stock Procfile: ``` $ goreman -f Procfile start ``` The members start running. They listen on `localhost:2379`, `localhost:22379`, and `localhost:32379` respectively for client requests. ### Interacting with the cluster Use `etcdctl` to interact with the running cluster: 1. Print the list of members: ``` $ etcdctl --write-out=table --endpoints=localhost:2379 member list ``` The list of etcd members is displayed as follows: ``` +------------------+---------+--------+------------------------+------------------------+ | ID | STATUS | NAME | PEER ADDRS | CLIENT ADDRS | +------------------+---------+--------+------------------------+------------------------+ | 8211f1d0f64f3269 | started | infra1 | http://127.0.0.1:2380 | http://127.0.0.1:2379 | | 91bc3c398fb3c146 | started | infra2 | http://127.0.0.1:22380 | http://127.0.0.1:22379 | | fd422379fda50e48 | started | infra3 | http://127.0.0.1:32380 | http://127.0.0.1:32379 | +------------------+---------+--------+------------------------+------------------------+ ``` 2. Store an example key-value pair in the cluster: ``` $ etcdctl put foo bar OK ``` If OK is printed, storing the key-value pair is successful. ### Testing fault tolerance To exercise etcd's fault tolerance, kill a member and attempt to retrieve the key. 1. Identify the process name of the member to be stopped. The `Procfile` lists the properties of the multi-member cluster. For example, consider the member with the process name, `etcd2`. 2. Stop the member: ``` # kill etcd2 $ goreman run stop etcd2 ``` 3. Store a key: ``` $ etcdctl put key hello OK ``` 4. Retrieve the key that is stored in the previous step: ``` $ etcdctl get key hello ``` 5. Retrieve a key from the stopped member: ``` $ etcdctl --endpoints=localhost:22379 get key ``` The command should display an error caused by connection failure: ``` 2017/06/18 23:07:35 grpc: Conn.resetTransport failed to create client transport: connection error: desc = "transport: dial tcp 127.0.0.1:22379: getsockopt: connection refused"; Reconnecting to "localhost:22379" Error: grpc: timed out trying to connect ``` 6. Restart the stopped member: ``` $ goreman run restart etcd2 ``` 7. Get the key from the restarted member: ``` $ etcdctl --endpoints=localhost:22379 get key hello ``` Restarting the member re-establishs the connection. `etcdctl` will now be able to retrieve the key successfully. To learn more about interacting with etcd, read [interacting with etcd section][interacting]. [clustering]: https://etcd.io/docs/latest/op-guide/clustering/ [interacting]: https://etcd.io/docs/latest/dev-guide/interacting_v3/ ================================================ FILE: Documentation/contributor-guide/logging.md ================================================ # Logging Conventions etcd uses the [zap][zap] library for logging application output categorized into *levels*. A log message's level is determined according to these conventions: * Debug: Everything is still fine, but even common operations may be logged, and less helpful but more quantity of notices. Usually not used in production. * Examples: * Send a normal message to a remote peer * Write a log entry to disk * Info: Normal, working log information, everything is fine, but helpful notices for auditing or common operations. Should rather not be logged more frequently than once per a few seconds in a normal server's operation. * Examples: * Startup configuration * Start to do a snapshot * Warning: (Hopefully) Temporary conditions that may cause errors, but may work fine. A replica disappearing (that may reconnect) is a warning. * Examples: * Failure to send a raft message to a remote peer * Failure to receive heartbeat message within the configured election timeout * Error: Data has been lost, a request has failed for a bad reason, or a required resource has been lost. * Examples: * Failure to allocate disk space for WAL * Panic: Unrecoverable or unexpected error situation that requires stopping execution. * Examples: * Failure to create the database * Fatal: Unrecoverable or unexpected error situation that requires immediate exit. Mostly used in the test. * Examples: * Failure to find the data directory * Failure to run a test function [zap]: https://github.com/uber-go/zap ================================================ FILE: Documentation/contributor-guide/modules.md ================================================ # Golang modules The etcd project (since version 3.5) is organized into multiple [golang modules](https://golang.org/ref/mod) hosted in a [single repository](https://golang.org/ref/mod#vcs-dir). ![modules graph](modules.svg) There are the following modules: - **go.etcd.io/etcd/api/v3** - contains API definitions (like protos & proto-generated libraries) that defines communication protocol between etcd clients and servers. - **go.etcd.io/etcd/pkg/v3** - a collection of utility packages used by etcd without being specific to etcd itself. A package belongs here only if it could possibly be moved out into its own repository in the future. Please avoid adding here code that has a lot of dependencies on its own, as they automatically become dependencies of the client library (that we want to keep lightweight). - **go.etcd.io/etcd/client/v3** - client library used to contact etcd over the network (grpc). Recommended for all new usage of etcd. - **go.etcd.io/raft/v3** - implementation of distributed consensus protocol. Should have no etcd specific code. Hosted in a separate repository: https://github.com/etcd-io/raft. - **go.etcd.io/etcd/server/v3** - etcd implementation. The code in this package is internal to etcd and should not be consumed by external projects. The package layout and API can change within the minor versions. - **go.etcd.io/etcd/etcdctl/v3** - a command line tool to access and manage etcd. - **go.etcd.io/etcd/tests/v3** - a module that contains all integration tests of etcd. Notice: All unit tests (fast and not requiring cross-module dependencies) should be kept in the local modules of the code under the test. - **go.etcd.io/bbolt** - implementation of persistent b-tree. Hosted in a separate repository: https://github.com/etcd-io/bbolt. ### Operations 1. All etcd modules should be released in the same versions, e.g. `go.etcd.io/etcd/client/v3@v3.5.10` must depend on `go.etcd.io/etcd/api/v3@v3.5.10`. The consistent updating of versions can be performed using: ```shell script % DRY_RUN=false TARGET_VERSION="v3.5.10" ./scripts/release_mod.sh update_versions ``` 2. The released modules should be tagged according to https://golang.org/ref/mod#vcs-version rules, i.e. each module should get its own tag. The tagging can be performed using: ```shell script % DRY_RUN=false REMOTE_REPO="origin" ./scripts/release_mod.sh push_mod_tags ``` 3. All etcd modules should depend on the same versions of underlying dependencies. This can be verified using: ```shell script % PASSES="dep" ./test.sh ``` 4. The go.mod files must not contain dependencies not being used and must conform to `go mod tidy` format. This is being verified by: ``` % PASSES="mod_tidy" ./test.sh ``` 5. To trigger actions across all modules (e.g. auto-format all files), please use/expand the following script: ```shell script % make fix ``` ### Future As a North Star, we would like to evaluate etcd modules towards the following model: ![modules graph](modules-future.svg) This assumes: - Splitting etcdmigrate/etcdadm out of etcdctl binary. Thanks to this etcdctl would become clearly a command-line wrapper around network client API, while etcdmigrate/etcdadm would support direct physical operations on the etcd storage files. - Splitting etcd-proxy out of ./etcd binary, as it contains more experimental code so carries additional risk & dependencies. - Deprecation of support for v2 protocol. ================================================ FILE: Documentation/contributor-guide/prow_jobs.md ================================================ # Analyzing Prow Job Resource Usage ## 1. Introduction to Prow [Prow](https://docs.prow.k8s.io/docs/) is a Kubernetes based CI/CD system. Jobs can be triggered by various types of events and report their status to many different services. Prow provides GitHub automation through policy enforcement and chat-ops via `/command` interactions on pull requests (e.g., `/test`, `/approve`, `/retest`), enabling contributors to trigger jobs and manage workflows directly from GitHub comments. When a user comments `/ok-to-test`or `/retest,` on a Pull Request, GitHub sends a webhook to Prow's Kubernetes cluster. Visit this [site](https://docs.prow.k8s.io/docs/life-of-a-prow-job/) to further understand the lifecycle of a Prow job. This is where you can find all etcd Prow jobs [status](https://prow.k8s.io/?repo=etcd-io%2Fetcd) ## 2. How Prow is used for etcd Testing etcd's CI is managed by [kubernetes/test-infra](https://github.com/kubernetes/test-infra), running Prow. When a pull request is submitted, or a `/command` is issued, the CI of etcd which managed by [kubernetes/test-infra](https://github.com/kubernetes/test-infra) uses Prow to run the tests. You can view all supported Prow [commands](https://prow.k8s.io/command-help). ### Jobs Types The jobs [configuration](https://github.com/kubernetes/test-infra/tree/master/config/jobs/etcd) for etcd. Please see [ProwJob](https://docs.prow.k8s.io/docs/jobs/) docs for more info. There are 3 different job types: - Recurring jobs that regularly run etcd performance benchmarks, specifically targeting the put API, for the amd64 architecture.[etcd-benchmarks-periodics.yaml](https://github.com/kubernetes/test-infra/blob/master/config/jobs/etcd/etcd-benchmarks-periodic.yaml) - Presubmits jobs: Run on pull requests before code is merged, ensuring new changes do not break the build or tests. [etcd-operator-presubmits.yaml](https://github.com/kubernetes/test-infra/blob/master/config/jobs/etcd/etcd-presubmits.yaml) - Postsubmits run after merging the etcd-io/etcd-operator code: [etcd-operator-postsubmits.yaml](https://github.com/kubernetes/test-infra/blob/master/config/jobs/etcd/etcd-operator-postsubmits.yaml) - Periodic jobs are jobs that run automatically on a fixed schedule (such as every 4 hours, once a day, etc.), regardless of code changes or pull requests. They’re designed to continuously check the stability, compatibility, or performance of the project over time. [etcd-periodics.yaml](https://github.com/kubernetes/test-infra/blob/master/config/jobs/etcd/etcd-periodics.yaml) - Builds the etcd project for all main and release branches before merging any PR. [etcd-presubmits.yaml](https://github.com/kubernetes/test-infra/blob/master/config/jobs/etcd/etcd-presubmits.yaml) - This file defines jobs that run automatically after code is merged (postsubmit) into the main or release branches of the etcd repository (etcd-io/etcd). [etcd-postsubmits.yaml](https://github.com/kubernetes/test-infra/blob/master/config/jobs/etcd/etcd-postsubmits.yaml) - Test pull requests for the etcd-io/raft repository on certain branches (main and release-3.6)[etcd-raft-presubmits.yaml](https://github.com/kubernetes/test-infra/blob/master/config/jobs/etcd/etcd-raft-presubmits.yaml) - Checks markdown formatting for website changes [etcd-website-presubmits.yaml](https://github.com/kubernetes/test-infra/blob/master/config/jobs/etcd/etcd-website-presubmits.yaml). - This file contains jobs that run after code is merged into the etcd-io/protodoc repository. [protodoc-presubmit.yaml](https://github.com/kubernetes/test-infra/blob/master/config/jobs/etcd/protodoc-postsubmits.yaml) - This file defines jobs that run on pull requests (before merge) for the etcd-io/protodoc repository.[protodoc-postsubmit.yaml](https://github.com/kubernetes/test-infra/blob/master/config/jobs/etcd/protodoc-presubmits.yaml) As an example, `pull-etcd-e2e-amd64` is one of the [presubmits](https://github.com/kubernetes/test-infra/blob/b21a1d3a72d5715ea7c9234cade21751847cfbe5/config/jobs/etcd/etcd-presubmits.yaml#L193). The job automatically runs end-to-end (e2e) tests on the amd64 architecture for every pull request to the etcd repository targeting the main, release-3.6, release-3.5, or release-3.4 branches. This is an example to its dashboard result [graph](https://prow.k8s.io/?repo=etcd-io%2Fetcd&type=presubmit&job=pull-etcd-e2e-amd64). Refer to [the test-infra Job Types documentation](https://github.com/kubernetes/test-infra/tree/master/config/jobs#job-types) to learn more about them. ### How to Trigger Prow Running Tests These tests can be triggered when you leave a comment, like `/ok-to-test` (only triggered by an etcd-io member) or `/retest`, in PR [example](https://github.com/etcd-io/etcd/pull/20733#issuecomment-3341443205). `/ok-to-test` allows Prow to run tests on a pull request from a first-time contributor. `/retest` tells Prow to rerun any failed or flaky joobs on the pull request, useful if a previous test failed due to a transient issue. You can find all supported [commands](https://prow.k8s.io/command-help). ## 3. Navigating Performance Dashboard (Grafana) Test-infra's Prow exposes Grafana dashboards to provide visibility into build resource usage (CPU, memory, number of running builds, etc.) for the Prow build cluster’s Kubernetes jobs. It is scoped via organization, repository, build identifier and time range filters. - GKE Dashboards: [https://monitoring-gke.prow.k8s.io/d/96Q8oOOZk/builds?orgId=1&refresh=30s&var-org=etcd-io&var-repo=etcd&var-build=All&from=now-7d&to=now](https://monitoring-gke.prow.k8s.io/d/96Q8oOOZk/builds?orgId=1&refresh=30s&var-org=etcd-io&var-repo=etcd&var-build=All&from=now-7d&to=now) - EKS Dashboards: [https://monitoring-eks.prow.k8s.io/d/96Q8oOOZk/builds?orgId=1&refresh=30s&var-org=etcd-io&var-repo=etcd&var-build=All&from=now-7d&to=now](https://monitoring-eks.prow.k8s.io/d/96Q8oOOZk/builds?orgId=1&refresh=30s&var-org=etcd-io&var-repo=etcd&var-build=All&from=now-7d&to=now) It is useful for a few reasons: 1. Tuning resources: By drilling into each build-run, you can determine realistic memory & CPU requests and limits for that job‑type. This helps avoid waste or avoid failed builds hitting resource limits. 2. Spotting anomalies: If one build suddenly used 8 GiB while normally this job uses 1 GiB, it may indicate a regression or mis‑configuration. 3. Capacity planning: Seeing typical and peak usage helps cluster operators plan node sizes, scheduling, concurrency of builds, etc. 4. Debugging performance issues: A build with unexpectedly high CPU or memory might be stuck, looping, or consuming resources inefficiently. ### Panel: “Running / Pending Builds” Shows the number of builds that are in Running vs Pending states over time. Use it to track build backlog or concurrency — e.g., if the “Pending” line rises, builds may be waiting for resources. If the “Running” line fluctuates a lot or remains at some steady value, you can infer how many builds typically run in parallel. ### Panel: “Memory Usage per Build” Shows memory usage over time for each build ID (each build listed in the legend at the bottom). The y‑axis shows memory use (e.g., in MiB / GiB). Use this to spot builds with unusually high memory usage — a spike indicates one build consumed many resources. ### Panel: “CPU Usage per Build” Similar to the memory panel but shows CPU usage per build over time. Spikes in CPU usage may indicate heavy compute jobs, inefficiencies, or need for resource tuning. ### Panel: "Resources" - Memory panel Green line (“used”): how much memory this build’s pod was using at each time point. Orange/Yellow line (“requested”): how much memory was requested (i.e., Kubernetes requests.memory) for that pod. Red line (“limit”): how much memory was limited (i.e., Kubernetes limits.memory) for that pod. Y‑axis: shows memory (GiB, MiB) over the build runtime. X‑axis: time of day/date. If the green “used” line is close to or hits the red “limit”, it means the build came close to its memory cap (risking OOM). If “used” is much lower than “requested”, you may be over‑allocating memory (waste). If the “requested” line is much higher than “used”, it suggests the job’s request could be tuned downward. - CPU panel Similar structure: green = actual usage, orange/yellow = requested CPU, red = CPU limit (if set). Y‑axis often in number of CPU cores or fraction thereof (e.g., 1.0 = one core). A green line with spikes may show bursts of CPU usage (e.g., build or compile phases) while idle periods show low usage. If CPU usage consistently saturates the limit, the job may be throttled or delayed. If usage is consistently far below request, tuning may reduce cost. ## 3.1 Prow job categories (robustness, integration, static checks) - Static check: - Description: Fast, deterministic checks (build, unit tests, linters, go vet/staticcheck, formatting, license/header checks, generated-code verification) that catch style, correctness and packaging problems early. - When to run: Every PR as presubmits; quick feedback loop before running expensive tests. - Example job patterns: pull-etcd-verify, pull-etcd-lint, pull-etcd-unit - Tests: - Robustness: - Description: Long-running, fault-injection and chaos-style end-to-end tests that validate etcd correctness and availability under failures (node crashes, network partitions, resource exhaustion, upgrades). - When to run: Periodics for continuous coverage; run for PRs that touch consensus, storage, recovery, or upgrade paths. - Example job patterns: pull-etcd-robustness, periodic-robustness - Integration: - Description: Functional end-to-end and cross-component tests that exercise real client/server interactions, snapshots/restore, upgrades and compatibility across OS/arch. - When to run: Presubmits for PRs that change APIs, client behavior, or integration points; periodics for broad platform coverage. - Example job patterns: pull-etcd-e2e-amd64, pull-etcd-integration ## 4. Interpreting Metrics Some Prow components expose Prometheus metrics that can be used for monitoring and alerting. You can find metrics like the number of PRs in each Tide pool, a histogram of the number of PRs in each merge and various other metrics to this [site](https://github.com/kubernetes-sigs/prow/blob/main/site/content/en/docs/metrics/_index.md). ================================================ FILE: Documentation/contributor-guide/release.md ================================================ # Release The guide talks about how to release a new version of etcd. The procedure includes some manual steps for sanity checking, but it can probably be further scripted. Please keep this document up-to-date if making changes to the release process. ## Release management Under the leadership of **James Blair** [@jmhbnz](https://github.com/jmhbnz) and **Ivan Valdes Castillo** [@ivanvc](https://github.com/ivanvc), the following pool of release candidates manages the release of each etcd major/minor version as well as manages patches to each stable release branch. They are responsible for communicating the timelines and status of each release and for ensuring the stability of the release branch. - Benjamin Wang [@ahrtr](https://github.com/ahrtr) - Fu Wei [@fuweid](https://github.com/fuweid) - James Blair [@jmhbnz](https://github.com/jmhbnz) - Ivan Valdes Castillo [@ivanvc](https://github.com/ivanvc) - Marek Siarkowicz [@serathius](https://github.com/serathius) - Sahdev Zala [@spzala](https://github.com/spzala) - Siyuan Zhang [@siyuanfoundation](https://github.com/siyuanfoundation) All release version numbers follow the format of [semantic versioning 2.0.0](http://semver.org/). ### Major, minor version release, or its pre-release - Ensure the relevant [milestone](https://github.com/etcd-io/etcd/milestones) on GitHub is complete. All referenced issues should be closed or moved elsewhere. - Ensure the latest [upgrade documentation](https://etcd.io/docs/next/upgrades) is available. - Bump [hardcoded MinClusterVerion in the repository](https://github.com/etcd-io/etcd/blob/v3.4.15/version/version.go#L29), if necessary. - Add feature capability maps for the new version, if necessary. ### Patch version release - To request a backport, developers submit cherry-pick PRs targeting the release branch. The commits should not include merge commits. The commits should be restricted to bug fixes and security patches. - The cherrypick PRs should target the appropriate release branch (`base:release--`). The k8s infra cherry pick robot `/cherrypick ` PR chatops command may be used to automatically generate cherrypick PRs. - The release patch manager reviews the cherrypick PRs. Please discuss carefully what is backported to the patch release. Each patch release should be strictly better than its predecessor. - The release patch manager will cherry-pick these commits starting from the oldest one into stable branch. ## Write a release note - Write an introduction for the new release. For example, what major bug we fix, what new features we introduce, or what performance improvement we make. - Put `[GH XXXX]` at the head of the change line to reference the Pull Request that introduces the change. Moreover, add a link on it to jump to the Pull Request. - Find PRs with the `release-note` label and explain them in the `NEWS` file, as a straightforward summary of changes for end-users. ## Patch release criteria The etcd project aims to release a new patch version if any of the following conditions are met: - Fixed one or more major CVEs (>=7.5). - Fixed one or more critical bugs. - Fixed three or more major bugs. - Fixed five or more minor bugs. ## Release guide ### Prerequisites There are some prerequisites, which should be done before the release process. These are one-time operations, which don't need to be executed before releasing each version. 1. Generate a GPG key and add it to your GitHub account. Refer to the links on [settings](https://github.com/settings/keys). 2. Ensure you have a Linux machine, on which the git, Golang, and docker have been installed. - Ensure the Golang version matches the version defined in `.go-version` file. - Ensure non-privileged users can run docker commands, refer to the [Linux postinstall](https://docs.docker.com/engine/install/linux-postinstall/). - Ensure there is at least 5GB of free space on your Linux machine. 3. Install gsutil, refer to [gsutil_install](https://cloud.google.com/storage/docs/gsutil_install). When asked about cloud project to use, pick `etcd-development`. 4. Authenticate the image registry, refer to [Authentication methods](https://cloud.google.com/container-registry/docs/advanced-authentication). - `gcloud auth login` - `gcloud auth configure-docker` 5. Install gh, refer to [GitHub's documentation](https://github.com/cli/cli#installation). Ensure that running `gh auth login` succeeds for the GitHub account you use to contribute to etcd, and that `gh auth status` has a clean exit and doesn't show any issues. ### Release steps At least one day before the release: 1. Raise an issue to publish the release plan, e.g. [issues/17350](https://github.com/etcd-io/etcd/issues/17350). 2. Raise a `kubernetes/org` pull request ([example PR](https://github.com/kubernetes/org/pull/5582)) to ensure members of the release team are added to the [release github team](https://github.com/orgs/etcd-io/teams/release-etcd). On the day of the release: 1. Verify you can pass the authentication to the image registries, - `docker login gcr.io` - `docker login quay.io` - If the release person doesn't have access to 1password, one of the owners (@ahrtr, @ivanvc, @jmhbnz, @serathius) needs to share the password with them per [this guide](https://support.1password.com/share-items/). See rough steps below, - [Sign in](https://team-etcd.1password.com/home) to your account on 1password.com. - Click `Your Vault Items` on the right side. - Select `Password of quay.io`. - Click `Share` on the top right, and set expiration as `1 hour` and only available to the release person using his/her email. - Click `Copy Link` then send the link to the release person via slack or email. 2. Clone the etcd repository and checkout the target branch, - `git clone --branch release-3.X git@github.com:etcd-io/etcd.git` 3. Run the release script under the repository's root directory, replacing `${VERSION}` with a value without the `v` prefix, i.e. `3.5.13`. - `DRY_RUN=false ./scripts/release.sh ${VERSION}` - **NOTE:** When doing a pre-release (i.e., a version from the main branch, 3.6.0-alpha.2), you will need to explicitly set the branch to main: ```bash DRY_RUN=false BRANCH=main ./scripts/release.sh ${VERSION} ``` It generates all release binaries under the directory `/tmp/etcd-release-${VERSION}/etcd/release/` and images. Binaries are pushed to the Google Cloud bucket under project `etcd-development`, and images are pushed to `quay.io` and `gcr.io`. - It is advisable to do a dry run before the actual release. This will create a `/tmp` directory. Do **NOT** forget to remove this directory before the actual release. ```bash DRY_RUN=true BRANCH=${BRANCH} ./scripts/release.sh ${VERSION} ``` 4. Publish the release page on GitHub - Open the **draft** release URL shown by the release script - Click the pen button at the top right to edit the release - Review that it looks correct, reviewing that the bottom checkboxes are checked depending on the release version (v3.4 & v3.5 no checkboxes, v3.6 has the set as latest release checkbox checked, v3.7 has the set as pre-release checkbox checked) - Then, publish the release 5. Announce to the etcd-dev googlegroup Follow the format of previous release emails sent to etcd-dev@googlegroups.com, see an example below. After sending out the email, ask one of the mailing list maintainers to approve the email from the pending list. Additionally, label the release email as `Release`. ```text Hello, etcd v3.4.30 is now public! https://github.com/etcd-io/etcd/releases/tag/v3.4.30 Thanks to everyone who contributed to the release! etcd team ``` 6. Update the changelog to reflect the correct release date. 7. Paste the release link to the issue raised in Step 1 and close the issue. 8. Raise a follow-up `kubernetes/org` pull request to return the GitHub release team to empty, least privilege state. 9. Crease a new stable branch through `git push origin release-${VERSION_MAJOR}.${VERSION_MINOR}` if this is a new major or minor stable release. 10. Re-generate a new password for quay.io if needed (e.g. shared to a contributor who isn't in the release team, and we should rotate the password at least once every 3 months). 11. Bump the new etcd release in Kubernetes, refer to [Bump etcd Version in Kubernetes](bump_etcd_version_k8s.md). - For etcd 3.6 patches, bump it to Kubernetes 1.34 and all newer minor versions (including `master` branch) - For etcd 3.5 patches, bump it to Kubernetes 1.33 and all older supported versions #### Release known issues 1. Timeouts pushing binaries - If binaries fail to fully upload to Google Cloud storage, the script must be re-run using the same command. Any artifacts that are already pushed will be overwritten to ensure they are correct. The storage bucket does not use object versioning so incorrect files cannot remain. 2. Timeouts pushing images - It is rare, although possible for connection timeouts to occur when publishing etcd release images to `quay.io` or `gcr.io`. If this occurs, it is known to be safe to rerun the release script command appending the `--no-upload` flag, and image uploads will gracefully resume. 3. GPG vs SSH signing - The release scripts assume that git tags will be signed with a GPG key. Since 2022 GitHub has supported [signing commits and tags using ssh](https://github.blog/changelog/2022-08-23-ssh-commit-verification-now-supported). Until further release script updates are completed you will need to disable this feature in your `~/.gitconfig` and revert to signing via GPG to perform etcd releases. ================================================ FILE: Documentation/contributor-guide/reporting_bugs.md ================================================ # Reporting bugs If any part of the etcd project has bugs or documentation mistakes, please let us know by [opening an issue][etcd-issue]. We treat bugs and mistakes very seriously and believe no issue is too small. Before creating a bug report, please check that an issue reporting the same problem does not already exist. To make the bug report accurate and easy to understand, please try to create bug reports that are: - Specific. Include as many details as possible: which version, what environment, what configuration, etc. If the bug is related to running the etcd server, please attach the etcd log (the starting log with the etcd configuration is especially important). - Reproducible. Include the steps to reproduce the problem. We understand some issues might be hard to reproduce, please include the steps that might lead to the problem. If possible, please attach the affected etcd data dir and stack trace to the bug report. - Isolated. Please try to isolate and reproduce the bug with minimum dependencies. It would significantly slow down the speed to fix a bug if too many dependencies are involved in a bug report. Debugging external systems that rely on etcd is out of scope, but we are happy to provide guidance in the right direction or help with using etcd itself. - Unique. Do not duplicate existing bug reports. - Scoped. One bug per report. Do not follow up with another bug inside one report. It may be worthwhile to read [Elika Etemad’s article on filing good bug reports][filing-good-bugs] before creating a bug report. We might ask for further information to locate a bug. A duplicated bug report will be closed. ## Frequently asked questions ### How to get a stack trace ``` bash $ kill -QUIT $PID ``` ### How to get the etcd version ``` bash $ etcd --version ``` ### How to get etcd configuration and log when it runs as systemd service ‘etcd2.service’ ``` bash $ sudo systemctl cat etcd2 $ sudo journalctl -u etcd2 ``` Due to an upstream systemd bug, journald may miss the last few log lines when its processes exit. If journalctl says etcd stopped without a fatal or panic message, try `sudo journalctl -f -t etcd2` to get the full log. [etcd-issue]: https://github.com/etcd-io/etcd/issues/new [filing-good-bugs]: http://fantasai.inkedblade.net/style/talks/filing-good-bugs/ ================================================ FILE: Documentation/contributor-guide/roadmap.md ================================================ # Roadmap etcd uses GitHub milestones to track all tasks in each major or minor release. The `roadmap.md` file only records the most important tasks for each release. The list is based on the current maintainer capacity that may shift over time. Proposed milestones are what we think we can deliver with the people we have. If we have more support on the important stuff, we could pick up more items from the backlog. Note that etcd will continue to mainly focus on technical debt over the next few major or minor releases. Each item has an assigned priority. Refer to [priority definitions](https://github.com/etcd-io/etcd/blob/main/Documentation/contributor-guide/triage_issues.md#step-5---prioritise-the-issue). ## v3.6.0 For a full list of tasks in `v3.6.0`, please see [milestone etcd-v3.6](https://github.com/etcd-io/etcd/milestone/38). | Title | Priority | Status | Note | |--------------------------------------------------------------------------------------------------------------------|-----------------------------|-------------|--------------------------------------------------------------------------------------------------------------| | [Support downgrade](https://github.com/etcd-io/etcd/issues/11716) | priority/important-soon | Completed | etcd will support downgrade starting from 3.6.0. But it will also support offline downgrade from 3.5 to 3.4. | | [StoreV2 deprecation](https://github.com/etcd-io/etcd/issues/12913) | priority/important-soon | In progress | This task will be covered in both 3.6 and 3.7. | | [Release raft 3.6.0](https://github.com/etcd-io/raft/issues/89) | priority/important-soon | Completed | etcd 3.6.0 will depends on raft 3.6.0 | | [Release bbolt 1.4.0](https://github.com/etcd-io/bbolt/issues/553) | priority/important-soon | Completed | etcd 3.6.0 will depends on bbolt 1.4.0 | | [Support /livez and /readyz endpoints](https://github.com/etcd-io/etcd/issues/16007) | priority/important-longterm | Completed | It provides clearer APIs, and can also work around the stalled writes issue | | [Bump gRPC](https://github.com/etcd-io/etcd/issues/16290) | priority/important-longterm | Completed | It isn't guaranteed to be resolved in 3.6, and might be postponed to 3.7 depending on the effort and risk. | | [Deprecate grpc-gateway or bump it](https://github.com/etcd-io/etcd/issues/14499) | priority/important-longterm | Completed | It isn't guaranteed to be resolved in 3.6, and might be postponed to 3.7 depending on the effort and risk. | | [bbolt: Add logger into bbolt](https://github.com/etcd-io/bbolt/issues/509) | priority/important-longterm | Completed | It's important to diagnose bbolt issues | | [bbolt: Add surgery commands](https://github.com/etcd-io/bbolt/issues/370) | priority/important-longterm | Completed | Surgery commands are important for fixing corrupted db files | | [Evaluate and (Gradulate or deprecate/remove) experimental features](https://github.com/etcd-io/etcd/issues/16292) | priority/backlog | Not started | This task will be covered in both 3.6 and 3.7. | ## v3.7.0 For a full list of tasks in `v3.7.0`, please see [milestone etcd-v3.7](https://github.com/etcd-io/etcd/milestone/39). | Title | Priority | Note | |-------------------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------------------------------------| | [StoreV2 deprecation](https://github.com/etcd-io/etcd/issues/12913) | P0 | Finish the remaining tasks 3.7. | | [Support range stream](https://github.com/etcd-io/etcd/issues/12342) | P0 | to be investigated & discussed. | | [Refactor lease: Lease might be revoked by mistake by old leader](https://github.com/etcd-io/etcd/issues/15247) | P1 | to be investigated & discussed | | [Integrate raft's new feature (async write) into etcd](https://github.com/etcd-io/etcd/issues/16291) | P1 | It should improve the performance | | [bbolt: Support customizing the bbolt rebalance threshold](https://github.com/etcd-io/bbolt/issues/422) | P2 | It may get rid of etcd's defragmentation. Both bbolt and etcd need to be changed. | | [Evaluate and (graduate or deprecate/remove) experimental features](https://github.com/etcd-io/etcd/issues/16292) | P2 | Finish the remaining tasks 3.7. | ## Backlog (future releases) | Title | Priority | Note | |----------------------------------------------------------------------------------------------------------|----------|------| | [Remove the dependency on grpc-go's experimental API](https://github.com/etcd-io/etcd/issues/15145) | | | | [Protobuf: cleanup both golang/protobuf and gogo/protobuf](https://github.com/etcd-io/etcd/issues/14533) | | | | [Proposals should include a merkle root](https://github.com/etcd-io/etcd/issues/13839) | | | | [Add Distributed Tracing using OpenTelemetry](https://github.com/etcd-io/etcd/issues/12460) | | | | [Support CA rotation](https://github.com/etcd-io/etcd/issues/11555) | | | | [bbolt: Migrate all commands to cobra style commands](https://github.com/etcd-io/bbolt/issues/472) | | | | [raft: enhance the configuration change validation](https://github.com/etcd-io/raft/issues/80) | | | ================================================ FILE: Documentation/contributor-guide/triage_issues.md ================================================ # Issue triage guidelines ## Purpose Speed up issue management. The `etcd` issues are listed at and are identified with labels. For example, an issue that is identified as a bug will be set to the label `type/bug`. The etcd project uses labels to indicate common attributes such as `area`, `type`, and `priority` of incoming issues. New issues will often start without any labels, but typically `etcd` maintainers, reviewers, and members will add labels by following these triage guidelines. The detailed list of labels can be found at . ## Scope This document serves as the primary guidelines for triaging incoming issues in `etcd`. All contributors are encouraged and welcome to help manage issues which will help reduce the burden on project maintainers, though the work and responsibilities discussed in this document are created with `etcd` project reviewers and members in mind as these individuals will have triage access to the etcd project which is a requirement for actions like applying labels or closing issues. Refer to [etcd community membership](https://github.com/etcd-io/etcd/blob/main/Documentation/contributor-guide/community-membership.md) for guidance on becoming an etcd project member or reviewer. ## Step 1 - Find an issue to triage To get started you can use the following recommended issue searches to identify issues that are in need of triage: * [Issues that have no labels](https://github.com/etcd-io/etcd/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated+no%3Alabel) * [Issues created recently](https://github.com/etcd-io/etcd/issues?q=is%3Aissue+is%3Aopen+) * [Issues not assigned but linked pr](https://github.com/etcd-io/etcd/issues?q=is%3Aopen+is%3Aissue+no%3Aassignee+linked%3Apr) * [Issues with no comments](https://github.com/etcd-io/etcd/issues?q=is%3Aopen+is%3Aissue+comments%3A0+) * [Issues with help wanted](https://github.com/etcd-io/etcd/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22+) ## Step 2 - Check the issue is valid Before we start adding labels or trying to work out a priority, our first triage step needs to be working out if the issue actually belongs to the etcd project and is not a duplicate. ### Issues that don't belong to etcd Sometimes issues are reported that belong to other projects that `etcd` use. For example, `grpc` or `golang` issues. Such issues should be addressed by asking the reporter to open issues in the appropriate other projects. These issues can generally be closed unless a maintainer and issue reporter see a need to keep it open for tracking purposes. If you have triage permissions please close it, alternatively mention the @etcd-io/members group to request a member with triage access to close the issue. ### Duplicate issues If an issue is a duplicate, add a comment stating so along with a reference for the original issue and if you have triage permissions please close it, alternatively mention the @etcd-io/members group to request a member with triage access close the issue. ## Step 3 - Apply the appropriate type of label Adding a `type` label to an issue helps create visibility on the health of the project and helps contributors identify potential priorities, i.e. addressing existing bugs or test flakes before implementing new features. ### Support requests As a general rule, the focus for etcd support is to address common themes in a broad way that helps all users, i.e. through channels like known issues, frequently asked questions, and high-quality documentation. To make the best use of project members time we should avoid providing 1:1 support if a broad approach is available. Some people mistakenly use our GitHub bug report or feature request templates to file support requests. Usually, they are asking for help operating or configuring some aspect of etcd. Support requests for etcd should instead be raised as [discussions](https://github.com/etcd-io/etcd/discussions). Common types of support requests are: 1. Questions about configuring or operating existing well-documented etcd features, for example, . Note - If an existing feature is not well documented please apply the `area/documentation` label and propose documentation improvements that would prevent future users from stumbling on the problem again. 2. Bug reports or questions about unsupported versions of etcd, for example . When responding to these issues please refer to our [supported versions documentation](https://etcd.io/docs/latest/op-guide/versioning) and encourage the reporter to upgrade to a recent patch release of a supported version as soon as possible. We should limit the effort supporting users that do not make the effort to run a supported version of etcd or ensure their version is patched. 3. Bug reports that do not provide a complete list of steps to reproduce the issue and/or contributors are not able to reproduce the issue, for example, . We should limit the effort we put into reproducing issues ourselves and motivate users to provide the necessary information to accept the bug report. 4. General questions that are filed using feature request or bug report issue templates, for example, . Note - These types of requests may surface good additions to our [frequently asked questions](https://etcd.io/docs/v3.5/faq). If you identify that an issue is a support request please: 1. Add the `type/support` or `type/question` label. 2. Add the following comment to inform the issue creator that discussions should be used instead and that this issue will be converted to a discussion. > Thank you for your question, this support issue will be moved to our [Discussion Forums](https://github.com/etcd-io/etcd/discussions). > > We are trying to consolidate the channels to which questions for help/support are posted so that we can improve our efficiency in responding to your requests, and make it easier for you to find answers to frequently asked questions and how to address common use cases. > > We regularly see messages posted in multiple forums, with the full response thread only in one place or, worse, spread across multiple forums. Also, the large volume of support issues on GitHub is making it difficult for us to use issues to identify real bugs. > > Members of the etcd community use Discussion Forums to field support requests. Before posting a new question, please search these for answers to similar questions, and also familiarize yourself with: > > 1. [user documentation](https://etcd.io/docs/latest) > 2. [frequently asked questions](https://etcd.io/docs/v3.5/faq) > > Again, thanks for using etcd and raising this question. > > The etcd team 3. Finally, click `Convert to discussion` on the right-hand panel, selecting the appropriate discussion category. ### Bug reports If an issue has been raised as a bug it should already have the `type/bug` label, however, if this is missing for an issue you determine to be a bug please add the label manually. The next step is to validate if the issue is indeed a bug. If not, add a comment with the findings and close the trivial issue. For non-trivial issues, wait to hear back from the issue reporter and see if there is any objection. If the issue reporter does not reply in 30 days, close the issue. If the problem can not be reproduced or requires more information, leave a comment for the issue reporter as soon as possible while the issue is fresh for the issue reporter. ### Feature requests New feature requests should be created via the etcd feature request template and in theory already have the `type/feature` label, however, if this is missing for an issue you determine to be a feature please add the label manually. ### Test flakes Test flakes are a specific type of bug that the etcd project tracks separately as these are a priority to address. These should be created via the test flake template and in theory already have the `type/flake` label, however, if this is missing for an issue you determine to be related to a flaking test please add the label manually. ## Step 4 - Define the areas impacted Adding an `area` label to an issue helps create visibility on which areas of the etcd project require attention and helps contributors find issues to work on relating to their particular skills or knowledge of the etcd codebase. If an issue crosses multiple domains please add additional `area` labels to reflect that. Below is a brief summary of the area labels in active use by the etcd project along with any notes on their use: | Label | Notes | | --- | --- | | area/external | Tracking label for issues raised that are external to etcd. | | area/community | | | area/raft | | | area/clientv3 | | | area/performance | | | area/security | | | area/tls | | | area/auth | | | area/etcdctl | | | area/etcdutl | | | area/contrib | Not to be confused with `area/community` this label is specifically used for issues relating to community-maintained scripts or files in the `contrib/` directory which aren't part of the core etcd project. | | area/documentation | | | area/tooling | Generally used in relation to the third party / external utilities or tools that are used in various stages of the etcd build, test, or release process, for example, tooling to create sboms. | | area/testing | | | area/robustness-testing | | ## Step 5 - Prioritise the issue If an issue lacks a priority label it has not been formally prioritized yet. Adding a `priority` label helps the etcd project understand what is important and should be worked on now, and conversely, what is not as important and is on the project backlog. |Priority label|What it means|Examples| |---|---|---| | `priority/critical-urgent` | Maintainers are responsible for making sure that these issues (in their area) are being actively worked on—i.e., drop what you're doing. The stuff is burning. These should be fixed before the next release. | user-visible critical bugs in core features
broken builds on tier1 supported platforms
tests and critical security issues | | `priority/important-soon` | Must be staffed and worked on either currently or very soon—ideally in time for the next release. | | | `priority/important-longterm` | Important over the long term, but may not be currently staffed and/or may require multiple releases to complete. | | | `priority/backlog` | General agreement that this is a nice-to-have, but no one's available to work on it anytime soon. Community contributions would be most welcome in the meantime, though it might take a while to get them reviewed if reviewers are fully occupied with higher-priority issues—for example, immediately before a release.| | | `priority/awaiting-more-evidence` | Possibly useful, but not yet enough support to actually get it done. | Mostly placeholders for potentially good ideas, so that they don't get completely forgotten, and can be referenced or deduped every time they come up | ## Step 6 - Support new contributors As part of the `etcd` triage process once the `kind` and `area` have been determined, please consider if the issue would be suitable for a less experienced contributor. The `good first issue` label is a subset of the `help wanted` label, indicating that members have committed to providing extra assistance for new contributors. All `good first issue` items also have the `help wanted` label. ### Help wanted Items marked with the `help wanted` label need to ensure that they meet these criteria: * **Low Barrier to Entry** - It should be easy for new contributors. * **Clear** - The task is agreed upon and does not require further discussions in the community. * **Goldilocks priority** - The priority should not be so high that a core contributor should do it, but not too low that it isn’t useful enough for a core contributor to spend time reviewing it, answering questions, helping get it into a release, etc. ### Good first issue Items marked with `good first issue` are intended for first-time contributors. It indicates that members will keep an eye out for these pull requests and shepherd it through our processes. New contributors should not be left to find an approver, ping for reviews, decipher test commands, or identify that their build failed due to a flake. It is important to make new contributors feel welcome and valued. We should assure them that they will have an extra level of help with their first contribution. After a contributor has successfully completed one or two `good first issue` items, they should be ready to move on to `help wanted` items. * **No Barrier to Entry** - The task is something that a new contributor can tackle without advanced setup or domain knowledge. * **Solution Explained** - The recommended solution is clearly described in the issue. * **Gives Examples** - Link to examples of similar implementations so new contributors have a reference guide for their changes. * **Identifies Relevant Code** - The relevant code and tests to be changed should be linked in the issue. * **Ready to Test** - There should be existing tests that can be modified, or existing test cases fit to be copied. If the area of code doesn’t have tests, before labeling the issue, add a test fixture. This prep often makes a great help wanted task! ## Step 7 - Follow up Once initial triage has been completed, issues need to be re-evaluated over time to ensure they don't become stale incorrectly. ### Track important issues If an issue is at risk of being closed by the stale bot in the future, but is an important issue for the etcd project, then please apply the `stage/tracked` label and remove any `stale` labels that exist. This will ensure the project does not lose sight of the issue. ### Close incomplete issues Issues that lack enough information from the issue reporter should be closed if the issue reporter does not provide information in 30 days. Issues can always be re-opened at a later date if new information is provided. ### Check for incomplete work If an issue owned by a developer has no pull request created in 30 days, contact the issue owner and kindly ask about the status of their work, or to release ownership on the issue if needed. ================================================ FILE: Documentation/contributor-guide/triage_prs.md ================================================ # PR management ## Purpose Speed up PR management. The `etcd` PRs are listed at https://github.com/etcd-io/etcd/pulls A PR can have various labels, milestones, reviewers, etc. The detailed list of labels can be found at https://github.com/kubernetes/kubernetes/labels Following are a few example searches on PR for convenience: * [Open PRS for milestone etcd-v3.6](https://github.com/etcd-io/etcd/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+milestone%3Aetcd-v3.6) * [PRs under investigation](https://github.com/etcd-io/etcd/labels/Investigating) ## Scope These guidelines serve as a primary document for managing PRs and review policy in `etcd`. Everyone is welcome to help manage PRs but the work and responsibilities discussed in this document are created with `etcd` maintainers and active contributors in mind. ## Ensure tests are run The etcd project use Kubernetes Prow and GitHub Actions to run tests. To ensure all required tests run if a pull request is ready for testing and still has the `needs-ok-to-test` label then please comment on the pull request `/ok-to-test`. ## Handle inactive PRs Poke PR owner if review comments are not addressed in 15 days. If the PR owner does not reply in 90 days, update the PR with a new commit if possible. If not, inactive PR should be closed after 180 days. ## Poke reviewer if needed Reviewers are responsive in a timely fashion, but considering everyone is busy, give them some time after requesting a review if a quick response is not provided. If the response is not provided in 10 days, feel free to contact them via adding a comment in the PR or sending an email or message on Slack. ## Verify important labels are in place Make sure that appropriate reviewers are added to the PR. Also, make sure that a milestone is identified. If any of these or other important labels are missing, add them. If a correct label cannot be decided, leave a comment for the maintainers to do so as needed. ## Review policy To ensure code quality and shared ownership, this review policy applies to all pull requests (PRs). ### Default rule PRs should get at least two approvals (/lgtm or GitHub review approval) before merging. Notes: * Approvals should come from a maintainer, reviewer, or submodule owner familiar with the relevant code or area. * If there’s disagreement, maintainers should discuss and agree before merging. ### Exceptions for Less Impactful PRs For low-risk changes — such as: * CI workflows * Documentation * Comments The rule can be relaxed: * One approval is generally enough. However: * If the author is a maintainer, they should still get approval from another maintainer, reviewer, or submodule owner, even for minor changes. ================================================ FILE: Documentation/dev-guide/apispec/swagger/rpc.swagger.json ================================================ { "swagger": "2.0", "info": { "title": "api/etcdserverpb/rpc.proto", "version": "version not set" }, "tags": [ { "name": "KV" }, { "name": "Watch" }, { "name": "Lease" }, { "name": "Cluster" }, { "name": "Maintenance" }, { "name": "Auth" } ], "consumes": [ "application/json" ], "produces": [ "application/json" ], "paths": { "/v3/auth/authenticate": { "post": { "summary": "Authenticate processes an authenticate request.", "operationId": "Auth_Authenticate", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthenticateResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthenticateRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/disable": { "post": { "summary": "AuthDisable disables authentication.", "operationId": "Auth_AuthDisable", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthDisableResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthDisableRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/enable": { "post": { "summary": "AuthEnable enables authentication.", "operationId": "Auth_AuthEnable", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthEnableResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthEnableRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/role/add": { "post": { "summary": "RoleAdd adds a new role. Role name cannot be empty.", "operationId": "Auth_RoleAdd", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleAddResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleAddRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/role/delete": { "post": { "summary": "RoleDelete deletes a specified role.", "operationId": "Auth_RoleDelete", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleDeleteResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleDeleteRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/role/get": { "post": { "summary": "RoleGet gets detailed role information.", "operationId": "Auth_RoleGet", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleGetResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleGetRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/role/grant": { "post": { "summary": "RoleGrantPermission grants a permission of a specified key or range to a specified role.", "operationId": "Auth_RoleGrantPermission", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleGrantPermissionResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleGrantPermissionRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/role/list": { "post": { "summary": "RoleList gets lists of all roles.", "operationId": "Auth_RoleList", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleListResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleListRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/role/revoke": { "post": { "summary": "RoleRevokePermission revokes a key or range permission of a specified role.", "operationId": "Auth_RoleRevokePermission", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleRevokePermissionResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthRoleRevokePermissionRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/status": { "post": { "summary": "AuthStatus displays authentication status.", "operationId": "Auth_AuthStatus", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthStatusResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthStatusRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/user/add": { "post": { "summary": "UserAdd adds a new user. User name cannot be empty.", "operationId": "Auth_UserAdd", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthUserAddResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthUserAddRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/user/changepw": { "post": { "summary": "UserChangePassword changes the password of a specified user.", "operationId": "Auth_UserChangePassword", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthUserChangePasswordResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthUserChangePasswordRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/user/delete": { "post": { "summary": "UserDelete deletes a specified user.", "operationId": "Auth_UserDelete", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthUserDeleteResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthUserDeleteRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/user/get": { "post": { "summary": "UserGet gets detailed user information.", "operationId": "Auth_UserGet", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthUserGetResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthUserGetRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/user/grant": { "post": { "summary": "UserGrantRole grants a role to a specified user.", "operationId": "Auth_UserGrantRole", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthUserGrantRoleResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthUserGrantRoleRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/user/list": { "post": { "summary": "UserList gets a list of all users.", "operationId": "Auth_UserList", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthUserListResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthUserListRequest" } } ], "tags": [ "Auth" ] } }, "/v3/auth/user/revoke": { "post": { "summary": "UserRevokeRole revokes a role of specified user.", "operationId": "Auth_UserRevokeRole", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAuthUserRevokeRoleResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAuthUserRevokeRoleRequest" } } ], "tags": [ "Auth" ] } }, "/v3/cluster/member/add": { "post": { "summary": "MemberAdd adds a member into the cluster.", "operationId": "Cluster_MemberAdd", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbMemberAddResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbMemberAddRequest" } } ], "tags": [ "Cluster" ] } }, "/v3/cluster/member/list": { "post": { "summary": "MemberList lists all the members in the cluster.", "operationId": "Cluster_MemberList", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbMemberListResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbMemberListRequest" } } ], "tags": [ "Cluster" ] } }, "/v3/cluster/member/promote": { "post": { "summary": "MemberPromote promotes a member from raft learner (non-voting) to raft voting member.", "operationId": "Cluster_MemberPromote", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbMemberPromoteResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbMemberPromoteRequest" } } ], "tags": [ "Cluster" ] } }, "/v3/cluster/member/remove": { "post": { "summary": "MemberRemove removes an existing member from the cluster.", "operationId": "Cluster_MemberRemove", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbMemberRemoveResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbMemberRemoveRequest" } } ], "tags": [ "Cluster" ] } }, "/v3/cluster/member/update": { "post": { "summary": "MemberUpdate updates the member configuration.", "operationId": "Cluster_MemberUpdate", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbMemberUpdateResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbMemberUpdateRequest" } } ], "tags": [ "Cluster" ] } }, "/v3/kv/compaction": { "post": { "summary": "Compact compacts the event history in the etcd key-value store. The key-value\nstore should be periodically compacted or the event history will continue to grow\nindefinitely.", "operationId": "KV_Compact", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbCompactionResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "description": "CompactionRequest compacts the key-value store up to a given revision. All superseded keys\nwith a revision less than the compaction revision will be removed.", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbCompactionRequest" } } ], "tags": [ "KV" ] } }, "/v3/kv/deleterange": { "post": { "summary": "DeleteRange deletes the given range from the key-value store.\nA delete request increments the revision of the key-value store\nand generates a delete event in the event history for every deleted key.", "operationId": "KV_DeleteRange", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbDeleteRangeResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbDeleteRangeRequest" } } ], "tags": [ "KV" ] } }, "/v3/kv/lease/leases": { "post": { "summary": "LeaseLeases lists all existing leases.", "operationId": "Lease_LeaseLeases2", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbLeaseLeasesResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbLeaseLeasesRequest" } } ], "tags": [ "Lease" ] } }, "/v3/kv/lease/revoke": { "post": { "summary": "LeaseRevoke revokes a lease. All keys attached to the lease will expire and be deleted.", "operationId": "Lease_LeaseRevoke2", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbLeaseRevokeResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbLeaseRevokeRequest" } } ], "tags": [ "Lease" ] } }, "/v3/kv/lease/timetolive": { "post": { "summary": "LeaseTimeToLive retrieves lease information.", "operationId": "Lease_LeaseTimeToLive2", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbLeaseTimeToLiveResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbLeaseTimeToLiveRequest" } } ], "tags": [ "Lease" ] } }, "/v3/kv/put": { "post": { "summary": "Put puts the given key into the key-value store.\nA put request increments the revision of the key-value store\nand generates one event in the event history.", "operationId": "KV_Put", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbPutResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbPutRequest" } } ], "tags": [ "KV" ] } }, "/v3/kv/range": { "post": { "summary": "Range gets the keys in the range from the key-value store.", "operationId": "KV_Range", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbRangeResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbRangeRequest" } } ], "tags": [ "KV" ] } }, "/v3/kv/txn": { "post": { "summary": "Txn processes multiple requests in a single transaction.\nA txn request increments the revision of the key-value store\nand generates events with the same revision for every completed request.\nIt is not allowed to modify the same key several times within one txn.", "operationId": "KV_Txn", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbTxnResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "description": "From google paxosdb paper:\nOur implementation hinges around a powerful primitive which we call MultiOp. All other database\noperations except for iteration are implemented as a single call to MultiOp. A MultiOp is applied atomically\nand consists of three components:\n1. A list of tests called guard. Each test in guard checks a single entry in the database. It may check\nfor the absence or presence of a value, or compare with a given value. Two different tests in the guard\nmay apply to the same or different entries in the database. All tests in the guard are applied and\nMultiOp returns the results. If all tests are true, MultiOp executes t op (see item 2 below), otherwise\nit executes f op (see item 3 below).\n2. A list of database operations called t op. Each operation in the list is either an insert, delete, or\nlookup operation, and applies to a single database entry. Two different operations in the list may apply\nto the same or different entries in the database. These operations are executed\nif guard evaluates to\ntrue.\n3. A list of database operations called f op. Like t op, but executed if guard evaluates to false.", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbTxnRequest" } } ], "tags": [ "KV" ] } }, "/v3/lease/grant": { "post": { "summary": "LeaseGrant creates a lease which expires if the server does not receive a keepAlive\nwithin a given time to live period. All keys attached to the lease will be expired and\ndeleted if the lease expires. Each expired key generates a delete event in the event history.", "operationId": "Lease_LeaseGrant", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbLeaseGrantResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbLeaseGrantRequest" } } ], "tags": [ "Lease" ] } }, "/v3/lease/keepalive": { "post": { "summary": "LeaseKeepAlive keeps the lease alive by streaming keep alive requests from the client\nto the server and streaming keep alive responses from the server to the client.", "operationId": "Lease_LeaseKeepAlive", "responses": { "200": { "description": "A successful response.(streaming responses)", "schema": { "type": "object", "properties": { "result": { "$ref": "#/definitions/etcdserverpbLeaseKeepAliveResponse" }, "error": { "$ref": "#/definitions/googleRpcStatus" } }, "title": "Stream result of etcdserverpbLeaseKeepAliveResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "description": " (streaming inputs)", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbLeaseKeepAliveRequest" } } ], "tags": [ "Lease" ] } }, "/v3/lease/leases": { "post": { "summary": "LeaseLeases lists all existing leases.", "operationId": "Lease_LeaseLeases", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbLeaseLeasesResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbLeaseLeasesRequest" } } ], "tags": [ "Lease" ] } }, "/v3/lease/revoke": { "post": { "summary": "LeaseRevoke revokes a lease. All keys attached to the lease will expire and be deleted.", "operationId": "Lease_LeaseRevoke", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbLeaseRevokeResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbLeaseRevokeRequest" } } ], "tags": [ "Lease" ] } }, "/v3/lease/timetolive": { "post": { "summary": "LeaseTimeToLive retrieves lease information.", "operationId": "Lease_LeaseTimeToLive", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbLeaseTimeToLiveResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbLeaseTimeToLiveRequest" } } ], "tags": [ "Lease" ] } }, "/v3/maintenance/alarm": { "post": { "summary": "Alarm activates, deactivates, and queries alarms regarding cluster health.", "operationId": "Maintenance_Alarm", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbAlarmResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbAlarmRequest" } } ], "tags": [ "Maintenance" ] } }, "/v3/maintenance/defragment": { "post": { "summary": "Defragment defragments a member's backend database to recover storage space.", "operationId": "Maintenance_Defragment", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbDefragmentResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbDefragmentRequest" } } ], "tags": [ "Maintenance" ] } }, "/v3/maintenance/downgrade": { "post": { "summary": "Downgrade requests downgrades, verifies feasibility or cancels downgrade\non the cluster version.\nSupported since etcd 3.5.", "operationId": "Maintenance_Downgrade", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbDowngradeResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbDowngradeRequest" } } ], "tags": [ "Maintenance" ] } }, "/v3/maintenance/hash": { "post": { "summary": "Hash computes the hash of whole backend keyspace,\nincluding key, lease, and other buckets in storage.\nThis is designed for testing ONLY!\nDo not rely on this in production with ongoing transactions,\nsince Hash operation does not hold MVCC locks.\nUse \"HashKV\" API instead for \"key\" bucket consistency checks.", "operationId": "Maintenance_Hash", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbHashResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbHashRequest" } } ], "tags": [ "Maintenance" ] } }, "/v3/maintenance/hashkv": { "post": { "summary": "HashKV computes the hash of all MVCC keys up to a given revision.\nIt only iterates \"key\" bucket in backend storage.", "operationId": "Maintenance_HashKV", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbHashKVResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbHashKVRequest" } } ], "tags": [ "Maintenance" ] } }, "/v3/maintenance/snapshot": { "post": { "summary": "Snapshot sends a snapshot of the entire backend from a member over a stream to a client.", "operationId": "Maintenance_Snapshot", "responses": { "200": { "description": "A successful response.(streaming responses)", "schema": { "type": "object", "properties": { "result": { "$ref": "#/definitions/etcdserverpbSnapshotResponse" }, "error": { "$ref": "#/definitions/googleRpcStatus" } }, "title": "Stream result of etcdserverpbSnapshotResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbSnapshotRequest" } } ], "tags": [ "Maintenance" ] } }, "/v3/maintenance/status": { "post": { "summary": "Status gets the status of the member.", "operationId": "Maintenance_Status", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbStatusResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbStatusRequest" } } ], "tags": [ "Maintenance" ] } }, "/v3/maintenance/transfer-leadership": { "post": { "summary": "MoveLeader requests current leader node to transfer its leadership to transferee.", "operationId": "Maintenance_MoveLeader", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/etcdserverpbMoveLeaderResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbMoveLeaderRequest" } } ], "tags": [ "Maintenance" ] } }, "/v3/watch": { "post": { "summary": "Watch watches for events happening or that have happened. Both input and output\nare streams; the input stream is for creating and canceling watchers and the output\nstream sends events. One watch RPC can watch on multiple key ranges, streaming events\nfor several watches at once. The entire event history can be watched starting from the\nlast compaction revision.", "operationId": "Watch_Watch", "responses": { "200": { "description": "A successful response.(streaming responses)", "schema": { "type": "object", "properties": { "result": { "$ref": "#/definitions/etcdserverpbWatchResponse" }, "error": { "$ref": "#/definitions/googleRpcStatus" } }, "title": "Stream result of etcdserverpbWatchResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/googleRpcStatus" } } }, "parameters": [ { "name": "body", "description": " (streaming inputs)", "in": "body", "required": true, "schema": { "$ref": "#/definitions/etcdserverpbWatchRequest" } } ], "tags": [ "Watch" ] } } }, "definitions": { "AlarmRequestAlarmAction": { "type": "string", "enum": [ "GET", "ACTIVATE", "DEACTIVATE" ], "default": "GET" }, "CompareCompareResult": { "type": "string", "enum": [ "EQUAL", "GREATER", "LESS", "NOT_EQUAL" ], "default": "EQUAL" }, "CompareCompareTarget": { "type": "string", "enum": [ "VERSION", "CREATE", "MOD", "VALUE", "LEASE" ], "default": "VERSION" }, "DowngradeRequestDowngradeAction": { "type": "string", "enum": [ "VALIDATE", "ENABLE", "CANCEL" ], "default": "VALIDATE" }, "EventEventType": { "type": "string", "enum": [ "PUT", "DELETE" ], "default": "PUT" }, "RangeRequestSortOrder": { "type": "string", "enum": [ "NONE", "ASCEND", "DESCEND" ], "default": "NONE", "title": "- NONE: default, no sorting\n - ASCEND: lowest target value first\n - DESCEND: highest target value first" }, "RangeRequestSortTarget": { "type": "string", "enum": [ "KEY", "VERSION", "CREATE", "MOD", "VALUE" ], "default": "KEY" }, "WatchCreateRequestFilterType": { "type": "string", "enum": [ "NOPUT", "NODELETE" ], "default": "NOPUT", "description": " - NOPUT: filter out put event.\n - NODELETE: filter out delete event." }, "authpbPermission": { "type": "object", "properties": { "permType": { "$ref": "#/definitions/authpbPermissionType" }, "key": { "type": "string", "format": "byte" }, "range_end": { "type": "string", "format": "byte" } }, "title": "Permission is a single entity" }, "authpbPermissionType": { "type": "string", "enum": [ "READ", "WRITE", "READWRITE" ], "default": "READ" }, "authpbUserAddOptions": { "type": "object", "properties": { "no_password": { "type": "boolean" } } }, "etcdserverpbAlarmMember": { "type": "object", "properties": { "memberID": { "type": "string", "format": "uint64", "description": "memberID is the ID of the member associated with the raised alarm." }, "alarm": { "$ref": "#/definitions/etcdserverpbAlarmType", "description": "alarm is the type of alarm which has been raised." } } }, "etcdserverpbAlarmRequest": { "type": "object", "properties": { "action": { "$ref": "#/definitions/AlarmRequestAlarmAction", "description": "action is the kind of alarm request to issue. The action\nmay GET alarm statuses, ACTIVATE an alarm, or DEACTIVATE a\nraised alarm." }, "memberID": { "type": "string", "format": "uint64", "description": "memberID is the ID of the member associated with the alarm. If memberID is 0, the\nalarm request covers all members." }, "alarm": { "$ref": "#/definitions/etcdserverpbAlarmType", "description": "alarm is the type of alarm to consider for this request." } } }, "etcdserverpbAlarmResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "alarms": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/etcdserverpbAlarmMember" }, "description": "alarms is a list of alarms associated with the alarm request." } } }, "etcdserverpbAlarmType": { "type": "string", "enum": [ "NONE", "NOSPACE", "CORRUPT" ], "default": "NONE", "title": "- NONE: default, used to query if any alarm is active\n - NOSPACE: space quota is exhausted\n - CORRUPT: kv store corruption detected" }, "etcdserverpbAuthDisableRequest": { "type": "object" }, "etcdserverpbAuthDisableResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbAuthEnableRequest": { "type": "object" }, "etcdserverpbAuthEnableResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbAuthRoleAddRequest": { "type": "object", "properties": { "name": { "type": "string", "description": "name is the name of the role to add to the authentication system." } } }, "etcdserverpbAuthRoleAddResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbAuthRoleDeleteRequest": { "type": "object", "properties": { "role": { "type": "string" } } }, "etcdserverpbAuthRoleDeleteResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbAuthRoleGetRequest": { "type": "object", "properties": { "role": { "type": "string" } } }, "etcdserverpbAuthRoleGetResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "perm": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/authpbPermission" } } } }, "etcdserverpbAuthRoleGrantPermissionRequest": { "type": "object", "properties": { "name": { "type": "string", "description": "name is the name of the role which will be granted the permission." }, "perm": { "$ref": "#/definitions/authpbPermission", "description": "perm is the permission to grant to the role." } } }, "etcdserverpbAuthRoleGrantPermissionResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbAuthRoleListRequest": { "type": "object" }, "etcdserverpbAuthRoleListResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "roles": { "type": "array", "items": { "type": "string" } } } }, "etcdserverpbAuthRoleRevokePermissionRequest": { "type": "object", "properties": { "role": { "type": "string" }, "key": { "type": "string", "format": "byte" }, "range_end": { "type": "string", "format": "byte" } } }, "etcdserverpbAuthRoleRevokePermissionResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbAuthStatusRequest": { "type": "object" }, "etcdserverpbAuthStatusResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "enabled": { "type": "boolean" }, "authRevision": { "type": "string", "format": "uint64", "title": "authRevision is the current revision of auth store" } } }, "etcdserverpbAuthUserAddRequest": { "type": "object", "properties": { "name": { "type": "string" }, "password": { "type": "string" }, "options": { "$ref": "#/definitions/authpbUserAddOptions" }, "hashedPassword": { "type": "string" } } }, "etcdserverpbAuthUserAddResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbAuthUserChangePasswordRequest": { "type": "object", "properties": { "name": { "type": "string", "description": "name is the name of the user whose password is being changed." }, "password": { "type": "string", "description": "password is the new password for the user. Note that this field will be removed in the API layer." }, "hashedPassword": { "type": "string", "description": "hashedPassword is the new password for the user. Note that this field will be initialized in the API layer." } } }, "etcdserverpbAuthUserChangePasswordResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbAuthUserDeleteRequest": { "type": "object", "properties": { "name": { "type": "string", "description": "name is the name of the user to delete." } } }, "etcdserverpbAuthUserDeleteResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbAuthUserGetRequest": { "type": "object", "properties": { "name": { "type": "string" } } }, "etcdserverpbAuthUserGetResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "roles": { "type": "array", "items": { "type": "string" } } } }, "etcdserverpbAuthUserGrantRoleRequest": { "type": "object", "properties": { "user": { "type": "string", "description": "user is the name of the user which should be granted a given role." }, "role": { "type": "string", "description": "role is the name of the role to grant to the user." } } }, "etcdserverpbAuthUserGrantRoleResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbAuthUserListRequest": { "type": "object" }, "etcdserverpbAuthUserListResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "users": { "type": "array", "items": { "type": "string" } } } }, "etcdserverpbAuthUserRevokeRoleRequest": { "type": "object", "properties": { "name": { "type": "string" }, "role": { "type": "string" } } }, "etcdserverpbAuthUserRevokeRoleResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbAuthenticateRequest": { "type": "object", "properties": { "name": { "type": "string" }, "password": { "type": "string" } } }, "etcdserverpbAuthenticateResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "token": { "type": "string", "title": "token is an authorized token that can be used in succeeding RPCs" } } }, "etcdserverpbCompactionRequest": { "type": "object", "properties": { "revision": { "type": "string", "format": "int64", "description": "revision is the key-value store revision for the compaction operation." }, "physical": { "type": "boolean", "description": "physical is set so the RPC will wait until the compaction is physically\napplied to the local database such that compacted entries are totally\nremoved from the backend database." } }, "description": "CompactionRequest compacts the key-value store up to a given revision. All superseded keys\nwith a revision less than the compaction revision will be removed." }, "etcdserverpbCompactionResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbCompare": { "type": "object", "properties": { "result": { "$ref": "#/definitions/CompareCompareResult", "description": "result is logical comparison operation for this comparison." }, "target": { "$ref": "#/definitions/CompareCompareTarget", "description": "target is the key-value field to inspect for the comparison." }, "key": { "type": "string", "format": "byte", "description": "key is the subject key for the comparison operation." }, "version": { "type": "string", "format": "int64", "title": "version is the version of the given key" }, "create_revision": { "type": "string", "format": "int64", "title": "create_revision is the creation revision of the given key" }, "mod_revision": { "type": "string", "format": "int64", "description": "mod_revision is the last modified revision of the given key." }, "value": { "type": "string", "format": "byte", "description": "value is the value of the given key, in bytes." }, "lease": { "type": "string", "format": "int64", "description": "lease is the lease id of the given key.\n\nleave room for more target_union field tags, jump to 64" }, "range_end": { "type": "string", "format": "byte", "description": "range_end compares the given target to all keys in the range [key, range_end).\nSee RangeRequest for more details on key ranges.\n\nTODO: fill out with most of the rest of RangeRequest fields when needed." } } }, "etcdserverpbDefragmentRequest": { "type": "object" }, "etcdserverpbDefragmentResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbDeleteRangeRequest": { "type": "object", "properties": { "key": { "type": "string", "format": "byte", "description": "key is the first key to delete in the range." }, "range_end": { "type": "string", "format": "byte", "description": "range_end is the key following the last key to delete for the range [key, range_end).\nIf range_end is not given, the range is defined to contain only the key argument.\nIf range_end is one bit larger than the given key, then the range is all the keys\nwith the prefix (the given key).\nIf range_end is '\\0', the range is all keys greater than or equal to the key argument." }, "prev_kv": { "type": "boolean", "description": "If prev_kv is set, etcd gets the previous key-value pairs before deleting it.\nThe previous key-value pairs will be returned in the delete response." } } }, "etcdserverpbDeleteRangeResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "deleted": { "type": "string", "format": "int64", "description": "deleted is the number of keys deleted by the delete range request." }, "prev_kvs": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/mvccpbKeyValue" }, "description": "if prev_kv is set in the request, the previous key-value pairs will be returned." } } }, "etcdserverpbDowngradeInfo": { "type": "object", "properties": { "enabled": { "type": "boolean", "description": "enabled indicates whether the cluster is enabled to downgrade." }, "targetVersion": { "type": "string", "description": "targetVersion is the target downgrade version." } } }, "etcdserverpbDowngradeRequest": { "type": "object", "properties": { "action": { "$ref": "#/definitions/DowngradeRequestDowngradeAction", "description": "action is the kind of downgrade request to issue. The action may\nVALIDATE the target version, DOWNGRADE the cluster version,\nor CANCEL the current downgrading job." }, "version": { "type": "string", "description": "version is the target version to downgrade." } } }, "etcdserverpbDowngradeResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "version": { "type": "string", "description": "version is the current cluster version." } } }, "etcdserverpbHashKVRequest": { "type": "object", "properties": { "revision": { "type": "string", "format": "int64", "description": "revision is the key-value store revision for the hash operation." } } }, "etcdserverpbHashKVResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "hash": { "type": "integer", "format": "int64", "description": "hash is the hash value computed from the responding member's MVCC keys up to a given revision." }, "compact_revision": { "type": "string", "format": "int64", "description": "compact_revision is the compacted revision of key-value store when hash begins." }, "hash_revision": { "type": "string", "format": "int64", "description": "hash_revision is the revision up to which the hash is calculated." } } }, "etcdserverpbHashRequest": { "type": "object" }, "etcdserverpbHashResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "hash": { "type": "integer", "format": "int64", "description": "hash is the hash value computed from the responding member's KV's backend." } } }, "etcdserverpbLeaseGrantRequest": { "type": "object", "properties": { "TTL": { "type": "string", "format": "int64", "description": "TTL is the advisory time-to-live in seconds. Expired lease will return -1." }, "ID": { "type": "string", "format": "int64", "description": "ID is the requested ID for the lease. If ID is set to 0, the lessor chooses an ID." } } }, "etcdserverpbLeaseGrantResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "ID": { "type": "string", "format": "int64", "description": "ID is the lease ID for the granted lease." }, "TTL": { "type": "string", "format": "int64", "description": "TTL is the server chosen lease time-to-live in seconds." }, "error": { "type": "string" } } }, "etcdserverpbLeaseKeepAliveRequest": { "type": "object", "properties": { "ID": { "type": "string", "format": "int64", "description": "ID is the lease ID for the lease to keep alive." } } }, "etcdserverpbLeaseKeepAliveResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "ID": { "type": "string", "format": "int64", "description": "ID is the lease ID from the keep alive request." }, "TTL": { "type": "string", "format": "int64", "description": "TTL is the new time-to-live for the lease." } } }, "etcdserverpbLeaseLeasesRequest": { "type": "object" }, "etcdserverpbLeaseLeasesResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "leases": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/etcdserverpbLeaseStatus" } } } }, "etcdserverpbLeaseRevokeRequest": { "type": "object", "properties": { "ID": { "type": "string", "format": "int64", "description": "ID is the lease ID to revoke. When the ID is revoked, all associated keys will be deleted." } } }, "etcdserverpbLeaseRevokeResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbLeaseStatus": { "type": "object", "properties": { "ID": { "type": "string", "format": "int64", "title": "TODO: int64 TTL = 2;" } } }, "etcdserverpbLeaseTimeToLiveRequest": { "type": "object", "properties": { "ID": { "type": "string", "format": "int64", "description": "ID is the lease ID for the lease." }, "keys": { "type": "boolean", "description": "keys is true to query all the keys attached to this lease." } } }, "etcdserverpbLeaseTimeToLiveResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "ID": { "type": "string", "format": "int64", "description": "ID is the lease ID from the keep alive request." }, "TTL": { "type": "string", "format": "int64", "description": "TTL is the remaining TTL in seconds for the lease; the lease will expire in under TTL+1 seconds." }, "grantedTTL": { "type": "string", "format": "int64", "description": "GrantedTTL is the initial granted time in seconds upon lease creation/renewal." }, "keys": { "type": "array", "items": { "type": "string", "format": "byte" }, "description": "Keys is the list of keys attached to this lease." } } }, "etcdserverpbMember": { "type": "object", "properties": { "ID": { "type": "string", "format": "uint64", "description": "ID is the member ID for this member." }, "name": { "type": "string", "description": "name is the human-readable name of the member. If the member is not started, the name will be an empty string." }, "peerURLs": { "type": "array", "items": { "type": "string" }, "description": "peerURLs is the list of URLs the member exposes to the cluster for communication." }, "clientURLs": { "type": "array", "items": { "type": "string" }, "description": "clientURLs is the list of URLs the member exposes to clients for communication. If the member is not started, clientURLs will be empty." }, "isLearner": { "type": "boolean", "description": "isLearner indicates if the member is raft learner." } } }, "etcdserverpbMemberAddRequest": { "type": "object", "properties": { "peerURLs": { "type": "array", "items": { "type": "string" }, "description": "peerURLs is the list of URLs the added member will use to communicate with the cluster." }, "isLearner": { "type": "boolean", "description": "isLearner indicates if the added member is raft learner." } } }, "etcdserverpbMemberAddResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "member": { "$ref": "#/definitions/etcdserverpbMember", "description": "member is the member information for the added member." }, "members": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/etcdserverpbMember" }, "description": "members is a list of all members after adding the new member." } } }, "etcdserverpbMemberListRequest": { "type": "object", "properties": { "linearizable": { "type": "boolean" } } }, "etcdserverpbMemberListResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "members": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/etcdserverpbMember" }, "description": "members is a list of all members associated with the cluster." } } }, "etcdserverpbMemberPromoteRequest": { "type": "object", "properties": { "ID": { "type": "string", "format": "uint64", "description": "ID is the member ID of the member to promote." } } }, "etcdserverpbMemberPromoteResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "members": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/etcdserverpbMember" }, "description": "members is a list of all members after promoting the member." } } }, "etcdserverpbMemberRemoveRequest": { "type": "object", "properties": { "ID": { "type": "string", "format": "uint64", "description": "ID is the member ID of the member to remove." } } }, "etcdserverpbMemberRemoveResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "members": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/etcdserverpbMember" }, "description": "members is a list of all members after removing the member." } } }, "etcdserverpbMemberUpdateRequest": { "type": "object", "properties": { "ID": { "type": "string", "format": "uint64", "description": "ID is the member ID of the member to update." }, "peerURLs": { "type": "array", "items": { "type": "string" }, "description": "peerURLs is the new list of URLs the member will use to communicate with the cluster." } } }, "etcdserverpbMemberUpdateResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "members": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/etcdserverpbMember" }, "description": "members is a list of all members after updating the member." } } }, "etcdserverpbMoveLeaderRequest": { "type": "object", "properties": { "targetID": { "type": "string", "format": "uint64", "description": "targetID is the node ID for the new leader." } } }, "etcdserverpbMoveLeaderResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "etcdserverpbPutRequest": { "type": "object", "properties": { "key": { "type": "string", "format": "byte", "description": "key is the key, in bytes, to put into the key-value store." }, "value": { "type": "string", "format": "byte", "description": "value is the value, in bytes, to associate with the key in the key-value store." }, "lease": { "type": "string", "format": "int64", "description": "lease is the lease ID to associate with the key in the key-value store. A lease\nvalue of 0 indicates no lease." }, "prev_kv": { "type": "boolean", "description": "If prev_kv is set, etcd gets the previous key-value pair before changing it.\nThe previous key-value pair will be returned in the put response." }, "ignore_value": { "type": "boolean", "description": "If ignore_value is set, etcd updates the key using its current value.\nReturns an error if the key does not exist." }, "ignore_lease": { "type": "boolean", "description": "If ignore_lease is set, etcd updates the key using its current lease.\nReturns an error if the key does not exist." } } }, "etcdserverpbPutResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "prev_kv": { "$ref": "#/definitions/mvccpbKeyValue", "description": "if prev_kv is set in the request, the previous key-value pair will be returned." } } }, "etcdserverpbRangeRequest": { "type": "object", "properties": { "key": { "type": "string", "format": "byte", "description": "key is the first key for the range. If range_end is not given, the request only looks up key." }, "range_end": { "type": "string", "format": "byte", "description": "range_end is the upper bound on the requested range [key, range_end).\nIf range_end is '\\0', the range is all keys \u003e= key.\nIf range_end is key plus one (e.g., \"aa\"+1 == \"ab\", \"a\\xff\"+1 == \"b\"),\nthen the range request gets all keys prefixed with key.\nIf both key and range_end are '\\0', then the range request returns all keys." }, "limit": { "type": "string", "format": "int64", "description": "limit is a limit on the number of keys returned for the request. When limit is set to 0,\nit is treated as no limit." }, "revision": { "type": "string", "format": "int64", "description": "revision is the point-in-time of the key-value store to use for the range.\nIf revision is less or equal to zero, the range is over the newest key-value store.\nIf the revision has been compacted, ErrCompacted is returned as a response." }, "sort_order": { "$ref": "#/definitions/RangeRequestSortOrder", "description": "sort_order is the order for returned sorted results." }, "sort_target": { "$ref": "#/definitions/RangeRequestSortTarget", "description": "sort_target is the key-value field to use for sorting." }, "serializable": { "type": "boolean", "description": "serializable sets the range request to use serializable member-local reads.\nRange requests are linearizable by default; linearizable requests have higher\nlatency and lower throughput than serializable requests but reflect the current\nconsensus of the cluster. For better performance, in exchange for possible stale reads,\na serializable range request is served locally without needing to reach consensus\nwith other nodes in the cluster." }, "keys_only": { "type": "boolean", "description": "keys_only when set returns only the keys and not the values." }, "count_only": { "type": "boolean", "description": "count_only when set returns only the count of the keys in the range." }, "min_mod_revision": { "type": "string", "format": "int64", "description": "min_mod_revision is the lower bound for returned key mod revisions; all keys with\nlesser mod revisions will be filtered away." }, "max_mod_revision": { "type": "string", "format": "int64", "description": "max_mod_revision is the upper bound for returned key mod revisions; all keys with\ngreater mod revisions will be filtered away." }, "min_create_revision": { "type": "string", "format": "int64", "description": "min_create_revision is the lower bound for returned key create revisions; all keys with\nlesser create revisions will be filtered away." }, "max_create_revision": { "type": "string", "format": "int64", "description": "max_create_revision is the upper bound for returned key create revisions; all keys with\ngreater create revisions will be filtered away." } } }, "etcdserverpbRangeResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "kvs": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/mvccpbKeyValue" }, "description": "kvs is the list of key-value pairs matched by the range request.\nkvs is empty when count is requested." }, "more": { "type": "boolean", "description": "more indicates if there are more keys to return in the requested range." }, "count": { "type": "string", "format": "int64", "description": "count is set to the actual number of keys within the range when requested.\nUnlike Kvs, it is unaffected by limits and filters (e.g., Min/Max, Create/Modify, Revisions)\nand reflects the full count within the specified range." } } }, "etcdserverpbRequestOp": { "type": "object", "properties": { "request_range": { "$ref": "#/definitions/etcdserverpbRangeRequest" }, "request_put": { "$ref": "#/definitions/etcdserverpbPutRequest" }, "request_delete_range": { "$ref": "#/definitions/etcdserverpbDeleteRangeRequest" }, "request_txn": { "$ref": "#/definitions/etcdserverpbTxnRequest" } } }, "etcdserverpbResponseHeader": { "type": "object", "properties": { "cluster_id": { "type": "string", "format": "uint64", "description": "cluster_id is the ID of the cluster which sent the response." }, "member_id": { "type": "string", "format": "uint64", "description": "member_id is the ID of the member which sent the response." }, "revision": { "type": "string", "format": "int64", "description": "revision is the key-value store revision when the request was applied, and it's\nunset (so 0) in case of calls not interacting with key-value store.\nFor watch progress responses, the header.revision indicates progress. All future events\nreceived in this stream are guaranteed to have a higher revision number than the\nheader.revision number." }, "raft_term": { "type": "string", "format": "uint64", "description": "raft_term is the raft term when the request was applied." } } }, "etcdserverpbResponseOp": { "type": "object", "properties": { "response_range": { "$ref": "#/definitions/etcdserverpbRangeResponse" }, "response_put": { "$ref": "#/definitions/etcdserverpbPutResponse" }, "response_delete_range": { "$ref": "#/definitions/etcdserverpbDeleteRangeResponse" }, "response_txn": { "$ref": "#/definitions/etcdserverpbTxnResponse" } } }, "etcdserverpbSnapshotRequest": { "type": "object" }, "etcdserverpbSnapshotResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader", "description": "header has the current key-value store information. The first header in the snapshot\nstream indicates the point in time of the snapshot." }, "remaining_bytes": { "type": "string", "format": "uint64", "title": "remaining_bytes is the number of blob bytes to be sent after this message" }, "blob": { "type": "string", "format": "byte", "description": "blob contains the next chunk of the snapshot in the snapshot stream." }, "version": { "type": "string", "description": "local version of server that created the snapshot.\nIn cluster with binaries with different version, each cluster can return different result.\nInforms which etcd server version should be used when restoring the snapshot." } } }, "etcdserverpbStatusRequest": { "type": "object" }, "etcdserverpbStatusResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "version": { "type": "string", "description": "version is the cluster protocol version used by the responding member." }, "dbSize": { "type": "string", "format": "int64", "description": "dbSize is the size of the backend database physically allocated, in bytes, of the responding member." }, "leader": { "type": "string", "format": "uint64", "description": "leader is the member ID which the responding member believes is the current leader." }, "raftIndex": { "type": "string", "format": "uint64", "description": "raftIndex is the current raft committed index of the responding member." }, "raftTerm": { "type": "string", "format": "uint64", "description": "raftTerm is the current raft term of the responding member." }, "raftAppliedIndex": { "type": "string", "format": "uint64", "description": "raftAppliedIndex is the current raft applied index of the responding member." }, "errors": { "type": "array", "items": { "type": "string" }, "description": "errors contains alarm/health information and status." }, "dbSizeInUse": { "type": "string", "format": "int64", "description": "dbSizeInUse is the size of the backend database logically in use, in bytes, of the responding member." }, "isLearner": { "type": "boolean", "description": "isLearner indicates if the member is raft learner." }, "storageVersion": { "type": "string", "description": "storageVersion is the version of the db file. It might be updated with delay in relationship to the target cluster version." }, "dbSizeQuota": { "type": "string", "format": "int64", "title": "dbSizeQuota is the configured etcd storage quota in bytes (the value passed to etcd instance by flag --quota-backend-bytes)" }, "downgradeInfo": { "$ref": "#/definitions/etcdserverpbDowngradeInfo", "description": "downgradeInfo indicates if there is downgrade process." } } }, "etcdserverpbTxnRequest": { "type": "object", "properties": { "compare": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/etcdserverpbCompare" }, "description": "compare is a list of predicates representing a conjunction of terms.\nIf the comparisons succeed, then the success requests will be processed in order,\nand the response will contain their respective responses in order.\nIf the comparisons fail, then the failure requests will be processed in order,\nand the response will contain their respective responses in order." }, "success": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/etcdserverpbRequestOp" }, "description": "success is a list of requests which will be applied when compare evaluates to true." }, "failure": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/etcdserverpbRequestOp" }, "description": "failure is a list of requests which will be applied when compare evaluates to false." } }, "description": "From google paxosdb paper:\nOur implementation hinges around a powerful primitive which we call MultiOp. All other database\noperations except for iteration are implemented as a single call to MultiOp. A MultiOp is applied atomically\nand consists of three components:\n1. A list of tests called guard. Each test in guard checks a single entry in the database. It may check\nfor the absence or presence of a value, or compare with a given value. Two different tests in the guard\nmay apply to the same or different entries in the database. All tests in the guard are applied and\nMultiOp returns the results. If all tests are true, MultiOp executes t op (see item 2 below), otherwise\nit executes f op (see item 3 below).\n2. A list of database operations called t op. Each operation in the list is either an insert, delete, or\nlookup operation, and applies to a single database entry. Two different operations in the list may apply\nto the same or different entries in the database. These operations are executed\nif guard evaluates to\ntrue.\n3. A list of database operations called f op. Like t op, but executed if guard evaluates to false." }, "etcdserverpbTxnResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "succeeded": { "type": "boolean", "description": "succeeded is set to true if the compare evaluated to true or false otherwise." }, "responses": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/etcdserverpbResponseOp" }, "description": "responses is a list of responses corresponding to the results from applying\nsuccess if succeeded is true or failure if succeeded is false." } } }, "etcdserverpbWatchCancelRequest": { "type": "object", "properties": { "watch_id": { "type": "string", "format": "int64", "description": "watch_id is the watcher id to cancel so that no more events are transmitted." } } }, "etcdserverpbWatchCreateRequest": { "type": "object", "properties": { "key": { "type": "string", "format": "byte", "description": "key is the key to register for watching." }, "range_end": { "type": "string", "format": "byte", "description": "range_end is the end of the range [key, range_end) to watch. If range_end is not given,\nonly the key argument is watched. If range_end is equal to '\\0', all keys greater than\nor equal to the key argument are watched.\nIf the range_end is one bit larger than the given key,\nthen all keys with the prefix (the given key) will be watched." }, "start_revision": { "type": "string", "format": "int64", "description": "start_revision is an optional revision to watch from (inclusive). No start_revision is \"now\"." }, "progress_notify": { "type": "boolean", "description": "progress_notify is set so that the etcd server will periodically send a WatchResponse with\nno events to the new watcher if there are no recent events. It is useful when clients\nwish to recover a disconnected watcher starting from a recent known revision.\nThe etcd server may decide how often it will send notifications based on current load." }, "filters": { "type": "array", "items": { "$ref": "#/definitions/WatchCreateRequestFilterType" }, "description": "filters filter the events at server side before it sends back to the watcher." }, "prev_kv": { "type": "boolean", "description": "If prev_kv is set, created watcher gets the previous KV before the event happens.\nIf the previous KV is already compacted, nothing will be returned." }, "watch_id": { "type": "string", "format": "int64", "description": "If watch_id is provided and non-zero, it will be assigned to this watcher.\nSince creating a watcher in etcd is not a synchronous operation,\nthis can be used ensure that ordering is correct when creating multiple\nwatchers on the same stream. Creating a watcher with an ID already in\nuse on the stream will cause an error to be returned." }, "fragment": { "type": "boolean", "description": "fragment enables splitting large revisions into multiple watch responses." } } }, "etcdserverpbWatchProgressRequest": { "type": "object", "description": "Requests the a watch stream progress status be sent in the watch response stream as soon as\npossible." }, "etcdserverpbWatchRequest": { "type": "object", "properties": { "create_request": { "$ref": "#/definitions/etcdserverpbWatchCreateRequest" }, "cancel_request": { "$ref": "#/definitions/etcdserverpbWatchCancelRequest" }, "progress_request": { "$ref": "#/definitions/etcdserverpbWatchProgressRequest" } } }, "etcdserverpbWatchResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "watch_id": { "type": "string", "format": "int64", "description": "watch_id is the ID of the watcher that corresponds to the response." }, "created": { "type": "boolean", "description": "created is set to true if the response is for a create watch request.\nThe client should record the watch_id and expect to receive events for\nthe created watcher from the same stream.\nAll events sent to the created watcher will attach with the same watch_id." }, "canceled": { "type": "boolean", "description": "canceled is set to true if the response is for a cancel watch request\nor if the start_revision has already been compacted.\nNo further events will be sent to the canceled watcher." }, "compact_revision": { "type": "string", "format": "int64", "description": "compact_revision is set to the minimum index if a watcher tries to watch\nat a compacted index.\n\nThis happens when creating a watcher at a compacted revision or the watcher cannot\ncatch up with the progress of the key-value store.\n\nThe client should treat the watcher as canceled and should not try to create any\nwatcher with the same start_revision again." }, "cancel_reason": { "type": "string", "description": "cancel_reason indicates the reason for canceling the watcher." }, "fragment": { "type": "boolean", "description": "framgment is true if large watch response was split over multiple responses." }, "events": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/mvccpbEvent" } } } }, "googleRpcStatus": { "type": "object", "properties": { "code": { "type": "integer", "format": "int32" }, "message": { "type": "string" }, "details": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/protobufAny" } } } }, "mvccpbEvent": { "type": "object", "properties": { "type": { "$ref": "#/definitions/EventEventType", "description": "type is the kind of event. If type is a PUT, it indicates\nnew data has been stored to the key. If type is a DELETE,\nit indicates the key was deleted." }, "kv": { "$ref": "#/definitions/mvccpbKeyValue", "description": "kv holds the KeyValue for the event.\nA PUT event contains current kv pair.\nA PUT event with kv.Version=1 indicates the creation of a key.\nA DELETE/EXPIRE event contains the deleted key with\nits modification revision set to the revision of deletion." }, "prev_kv": { "$ref": "#/definitions/mvccpbKeyValue", "description": "prev_kv holds the key-value pair before the event happens." } } }, "mvccpbKeyValue": { "type": "object", "properties": { "key": { "type": "string", "format": "byte", "description": "key is the key in bytes. An empty key is not allowed." }, "create_revision": { "type": "string", "format": "int64", "description": "create_revision is the revision of last creation on this key." }, "mod_revision": { "type": "string", "format": "int64", "description": "mod_revision is the revision of last modification on this key." }, "version": { "type": "string", "format": "int64", "description": "version is the version of the key. A deletion resets\nthe version to zero and any modification of the key\nincreases its version." }, "value": { "type": "string", "format": "byte", "description": "value is the value held by the key, in bytes." }, "lease": { "type": "string", "format": "int64", "description": "lease is the ID of the lease that attached to key.\nWhen the attached lease expires, the key will be deleted.\nIf lease is 0, then no lease is attached to the key." } } }, "protobufAny": { "type": "object", "properties": { "@type": { "type": "string" } }, "additionalProperties": {} } }, "securityDefinitions": { "ApiKey": { "type": "apiKey", "name": "Authorization", "in": "header" } }, "security": [ { "ApiKey": [] } ] } ================================================ FILE: Documentation/dev-guide/apispec/swagger/v3election.swagger.json ================================================ { "swagger": "2.0", "info": { "title": "server/etcdserver/api/v3election/v3electionpb/v3election.proto", "version": "version not set" }, "tags": [ { "name": "Election" } ], "consumes": [ "application/json" ], "produces": [ "application/json" ], "paths": { "/v3/election/campaign": { "post": { "summary": "Campaign waits to acquire leadership in an election, returning a LeaderKey\nrepresenting the leadership if successful. The LeaderKey can then be used\nto issue new values on the election, transactionally guard API requests on\nleadership still being held, and resign from the election.", "operationId": "Election_Campaign", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/v3electionpbCampaignResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/rpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/v3electionpbCampaignRequest" } } ], "tags": [ "Election" ] } }, "/v3/election/leader": { "post": { "summary": "Leader returns the current election proclamation, if any.", "operationId": "Election_Leader", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/v3electionpbLeaderResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/rpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/v3electionpbLeaderRequest" } } ], "tags": [ "Election" ] } }, "/v3/election/observe": { "post": { "summary": "Observe streams election proclamations in-order as made by the election's\nelected leaders.", "operationId": "Election_Observe", "responses": { "200": { "description": "A successful response.(streaming responses)", "schema": { "type": "object", "properties": { "result": { "$ref": "#/definitions/v3electionpbLeaderResponse" }, "error": { "$ref": "#/definitions/rpcStatus" } }, "title": "Stream result of v3electionpbLeaderResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/rpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/v3electionpbLeaderRequest" } } ], "tags": [ "Election" ] } }, "/v3/election/proclaim": { "post": { "summary": "Proclaim updates the leader's posted value with a new value.", "operationId": "Election_Proclaim", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/v3electionpbProclaimResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/rpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/v3electionpbProclaimRequest" } } ], "tags": [ "Election" ] } }, "/v3/election/resign": { "post": { "summary": "Resign releases election leadership so other campaigners may acquire\nleadership on the election.", "operationId": "Election_Resign", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/v3electionpbResignResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/rpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/v3electionpbResignRequest" } } ], "tags": [ "Election" ] } } }, "definitions": { "etcdserverpbResponseHeader": { "type": "object", "properties": { "cluster_id": { "type": "string", "format": "uint64", "description": "cluster_id is the ID of the cluster which sent the response." }, "member_id": { "type": "string", "format": "uint64", "description": "member_id is the ID of the member which sent the response." }, "revision": { "type": "string", "format": "int64", "description": "revision is the key-value store revision when the request was applied, and it's\nunset (so 0) in case of calls not interacting with key-value store.\nFor watch progress responses, the header.revision indicates progress. All future events\nreceived in this stream are guaranteed to have a higher revision number than the\nheader.revision number." }, "raft_term": { "type": "string", "format": "uint64", "description": "raft_term is the raft term when the request was applied." } } }, "mvccpbKeyValue": { "type": "object", "properties": { "key": { "type": "string", "format": "byte", "description": "key is the key in bytes. An empty key is not allowed." }, "create_revision": { "type": "string", "format": "int64", "description": "create_revision is the revision of last creation on this key." }, "mod_revision": { "type": "string", "format": "int64", "description": "mod_revision is the revision of last modification on this key." }, "version": { "type": "string", "format": "int64", "description": "version is the version of the key. A deletion resets\nthe version to zero and any modification of the key\nincreases its version." }, "value": { "type": "string", "format": "byte", "description": "value is the value held by the key, in bytes." }, "lease": { "type": "string", "format": "int64", "description": "lease is the ID of the lease that attached to key.\nWhen the attached lease expires, the key will be deleted.\nIf lease is 0, then no lease is attached to the key." } } }, "protobufAny": { "type": "object", "properties": { "@type": { "type": "string" } }, "additionalProperties": {} }, "rpcStatus": { "type": "object", "properties": { "code": { "type": "integer", "format": "int32" }, "message": { "type": "string" }, "details": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/protobufAny" } } } }, "v3electionpbCampaignRequest": { "type": "object", "properties": { "name": { "type": "string", "format": "byte", "description": "name is the election's identifier for the campaign." }, "lease": { "type": "string", "format": "int64", "description": "lease is the ID of the lease attached to leadership of the election. If the\nlease expires or is revoked before resigning leadership, then the\nleadership is transferred to the next campaigner, if any." }, "value": { "type": "string", "format": "byte", "description": "value is the initial proclaimed value set when the campaigner wins the\nelection." } } }, "v3electionpbCampaignResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "leader": { "$ref": "#/definitions/v3electionpbLeaderKey", "description": "leader describes the resources used for holding leadereship of the election." } } }, "v3electionpbLeaderKey": { "type": "object", "properties": { "name": { "type": "string", "format": "byte", "description": "name is the election identifier that corresponds to the leadership key." }, "key": { "type": "string", "format": "byte", "description": "key is an opaque key representing the ownership of the election. If the key\nis deleted, then leadership is lost." }, "rev": { "type": "string", "format": "int64", "description": "rev is the creation revision of the key. It can be used to test for ownership\nof an election during transactions by testing the key's creation revision\nmatches rev." }, "lease": { "type": "string", "format": "int64", "description": "lease is the lease ID of the election leader." } } }, "v3electionpbLeaderRequest": { "type": "object", "properties": { "name": { "type": "string", "format": "byte", "description": "name is the election identifier for the leadership information." } } }, "v3electionpbLeaderResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "kv": { "$ref": "#/definitions/mvccpbKeyValue", "description": "kv is the key-value pair representing the latest leader update." } } }, "v3electionpbProclaimRequest": { "type": "object", "properties": { "leader": { "$ref": "#/definitions/v3electionpbLeaderKey", "description": "leader is the leadership hold on the election." }, "value": { "type": "string", "format": "byte", "description": "value is an update meant to overwrite the leader's current value." } } }, "v3electionpbProclaimResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } }, "v3electionpbResignRequest": { "type": "object", "properties": { "leader": { "$ref": "#/definitions/v3electionpbLeaderKey", "description": "leader is the leadership to relinquish by resignation." } } }, "v3electionpbResignResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } } } } ================================================ FILE: Documentation/dev-guide/apispec/swagger/v3lock.swagger.json ================================================ { "swagger": "2.0", "info": { "title": "server/etcdserver/api/v3lock/v3lockpb/v3lock.proto", "version": "version not set" }, "tags": [ { "name": "Lock" } ], "consumes": [ "application/json" ], "produces": [ "application/json" ], "paths": { "/v3/lock/lock": { "post": { "summary": "Lock acquires a distributed shared lock on a given named lock.\nOn success, it will return a unique key that exists so long as the\nlock is held by the caller. This key can be used in conjunction with\ntransactions to safely ensure updates to etcd only occur while holding\nlock ownership. The lock is held until Unlock is called on the key or the\nlease associate with the owner expires.", "operationId": "Lock_Lock", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/v3lockpbLockResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/rpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/v3lockpbLockRequest" } } ], "tags": [ "Lock" ] } }, "/v3/lock/unlock": { "post": { "summary": "Unlock takes a key returned by Lock and releases the hold on lock. The\nnext Lock caller waiting for the lock will then be woken up and given\nownership of the lock.", "operationId": "Lock_Unlock", "responses": { "200": { "description": "A successful response.", "schema": { "$ref": "#/definitions/v3lockpbUnlockResponse" } }, "default": { "description": "An unexpected error response.", "schema": { "$ref": "#/definitions/rpcStatus" } } }, "parameters": [ { "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/v3lockpbUnlockRequest" } } ], "tags": [ "Lock" ] } } }, "definitions": { "etcdserverpbResponseHeader": { "type": "object", "properties": { "cluster_id": { "type": "string", "format": "uint64", "description": "cluster_id is the ID of the cluster which sent the response." }, "member_id": { "type": "string", "format": "uint64", "description": "member_id is the ID of the member which sent the response." }, "revision": { "type": "string", "format": "int64", "description": "revision is the key-value store revision when the request was applied, and it's\nunset (so 0) in case of calls not interacting with key-value store.\nFor watch progress responses, the header.revision indicates progress. All future events\nreceived in this stream are guaranteed to have a higher revision number than the\nheader.revision number." }, "raft_term": { "type": "string", "format": "uint64", "description": "raft_term is the raft term when the request was applied." } } }, "protobufAny": { "type": "object", "properties": { "@type": { "type": "string" } }, "additionalProperties": {} }, "rpcStatus": { "type": "object", "properties": { "code": { "type": "integer", "format": "int32" }, "message": { "type": "string" }, "details": { "type": "array", "items": { "type": "object", "$ref": "#/definitions/protobufAny" } } } }, "v3lockpbLockRequest": { "type": "object", "properties": { "name": { "type": "string", "format": "byte", "description": "name is the identifier for the distributed shared lock to be acquired." }, "lease": { "type": "string", "format": "int64", "description": "lease is the ID of the lease that will be attached to ownership of the\nlock. If the lease expires or is revoked and currently holds the lock,\nthe lock is automatically released. Calls to Lock with the same lease will\nbe treated as a single acquisition; locking twice with the same lease is a\nno-op." } } }, "v3lockpbLockResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" }, "key": { "type": "string", "format": "byte", "description": "key is a key that will exist on etcd for the duration that the Lock caller\nowns the lock. Users should not modify this key or the lock may exhibit\nundefined behavior." } } }, "v3lockpbUnlockRequest": { "type": "object", "properties": { "key": { "type": "string", "format": "byte", "description": "key is the lock ownership key granted by Lock." } } }, "v3lockpbUnlockResponse": { "type": "object", "properties": { "header": { "$ref": "#/definitions/etcdserverpbResponseHeader" } } } } } ================================================ FILE: Documentation/etcd-internals/diagrams/consistent_read_workflow.drawio ================================================ 7LzH0uTK1S32NBySAW+G8N4UPDC5Ae9twT+9gK/7kOeQ/K8UkkIjtfsKWUDavddea2ei/wYz/Sks8VRpY5Z3f4OA7PwbzP4NgmAYQ54fb8n1qwRCYehXSbnU2a8y8F8Fdn3nvwuB36VbneXfv9y4jmO31tNfC9NxGPJ0/UtZvCzj8dfbirH7a6tTXOb/UWCncfefpX6drdWvUgIF/lUu5nVZ/dEyCPz+po//uPl3wbeKs/H4UxHM/Q1mlnFcf33qTybv3tn7Y15+Pcf/D9/+s2NLPqz/Vx5YiYmwbpcdL7ekkpDjUWL6O/mrlj3utt8D/t3Z9fpjBsZt7eohZ/45wcDfYLoYh5UZu3H5uQd+fvNvq3S5xFmd/+u7YRzy9/a66/50O0tgGAg/5d91Gdv8327O4m+VZ78b2vNlrZ/lUOMk78zxW6/1ODzfJeO6jv2fbqC6uny/WMfpKY1/X6VPX/Knbrpa++65Bn/3/behgdAf17/H+zYZf6dfAy3q8+0HPY31Wwu3P5V9f1fyrOb0PtCf5Wv5/4iPL/KPLPlf4/C/6uG7xkOav3X/nt6nj/n5P64b+E9rePwoH/t8Xa7nlt8P/B3HsV/PXH8UkOg/0F9Fx3+xyepP9vjPwvi3H5T/rP9fpvJ8+G0t/91yKA3++xD9va/K3SHHK3RmQf078v9bzv8HlvMfZvJfjOl/tBwS+Kvd/NMY/mQ1EP5frOafhf+vWw0I/p+bzTJuQ/bPZRyXtRrLcXgWcnwX6GcOm3xdr98rEW/r+Ndl+mUbf8A19G8LhzzX+VmvwdvAPzAM/X0dPtd/B/4B/HHNnr978HNx/enCzJf6mY7XOn6VDc/U/K4OIP4oeOsD/wGA+B8F/6rw5+r689W/V/k/rv533JY0/9/N8O8JXeOlzNf/3Y2/PTjP/hIB/9OalryL13r/a0D8b5bx+1HzNfo/4Rf6OOxf7BAj/828fg3q93N/DmH/WdW/mTQJ/FtVv4b9H1X9GOs/x/T/wH6h/3P7/cPB6/6HW9A/P6k/oOG/4sT/Taz6Kwg+8AjDJFkU/+EC2L8B6R/Y+ievwP/o6HORxWv8N5j6dQnx01D+DWJqjzasA1CEcqSeX7rtVtxDJCjK/T7/MB5Dhc9P1iL1Cnw+KIvQsR+Qlj+AVrqivEd9940+FGVLvdcdmev+DaJbuzRJHAB0xn6uypVkIklGGBfgjIBtIoarGVegJPqUFP60vJS5xnpk6sujvGBKq/gbs/U4M1Thcj7le5mqV5/cShFaLIr86Tqd6ffwrvM7DGMIyP75uT9/U8NPGUoeFK6s7dBvKXdvpqOndg+cWo/96PohP+MsqZbyIVWM45LmXFRKiTIYyAweChqjoiLLWvmZqV9VPq3t0tespoWjS0n6NJvffj3tY47Q14MuWqMBymO8GiRQjSfuIwvt6TNbVPX4Om+m5pE6xHngzNUAifZUNqb8xjDacMh6qAocQTGf7ORwNdTSJnWbjZH303CO587jW94pN8l+CtKC1qosTRVUKt1E3p8w4ctP/eCu+SKv8fWa36I+9JZPGBept9oZEzD5RJOf/vud4FgFhZCRzNmVjtO1HIn49nxLA/1UaIXeUIc69GkekdHzSEfLPAyx1w2QV42iA5vuqfcsp0HI+2DIyOCexzBYLXaTB9CgNMrPiMFvucLh44lObRdo5WKXowOdBuKIIoFLzqcWBwORXPHpZ+yOzV6OT/MJxCAuHe7LQ3t4hQzo4srHxRwmFYKDDnVMMScl65M834qR/9nB9F11rYqqqc3V2OBPuDn8QlOFp1zSQW/gto/IOBB6RXidg5Ngiw/CDRKN94cOVESFtN8Bo1F6wynqeaaszHOUKDShmKlkfYRNHXaEn7Wnw4T2sYX2N243ay6jRxZzpK1Ju6G/sNqPqsinpAPAH0sknATCDZJhBkFdK3ssmeNyxfLtkqLfH6MErWrqQk3VG2AGRryi5HsJw40TAgPqg5AROIpzR0rpJAoDqFA4KCuk6L2k+OtTDyAlUSXv+lQXShrXUgon0caXQldKrEtqI55x0C6lKofHfygEY1uM4gfKDw42+DA5RgU7JbESXRG0rJVmVjKjVIqcLSgyZU+UOH2+8Yd2bXFgKPOiCKWalVJgyhxptTqmIYC2Z4qZORujKvdFV4AZwmeanbxMtpLWqVr/+K+b3zelyN+3o7BfngWbyeXjA/RTIUGNiNhi/Pe55NOcylKWL6jX/uj+/UdNmaJg3vWg1ZyWeJeWxjz69Fdp8RbeV1QnWjNbu5zq0l1IzJ/OL4WOXmcKd4PsQ8XVDTIhyLskBazUEFM5pLOtcQPxMLJ5s2K0DDyYGxUfj+RvSBra1zkPlDrnqmoaFhLqcOBOQbu0oA+l9vCmsrv5zFCFNCZOSDmJriIXBvaFc+Od1N5OHxfF7wcKa7LecJ6FuDZl55zTiBBFSvlU+QNTSxKmxFTLDjEoz10oUktF2ODgh8osWDtvHpFGayqhBshr1s0LZlTzlDE9oz+2EXKEJIuUFB8lzkQ+p3/l9Wjoqq4rFmRSjGsNuU4t/RTk+p07biYc8VxCm6vkmBtSy2YinLPGrq4tzQYvV5AdzIq4TpDZwk4nRVol3XWSScpbGrOv2CHTckOF6jKGVtyiiOiL64FznhcecG/nK3Ja24ikrh2hB9f5SNonqbaNTtFiy7d7T+XdPmuXcdVtVG/12Mkmvbdf2/io0xsXJBLYTbtjW0cImMiS26zd3Cu7P1Dk8nW+PJD20OgLFfXOAeYc7KBaUur4gVFFuBW+HTvUnrpemjusMyKQs+YnRCBr+MUkO12iyw3UKe28a4KdHn28w99l4nUOB00KJSDtvf0FiF9EianWoGec1kdXqKynTGDHca7rSw4077k83YQPiESznbRjFbebVlBRgUrvVlWxiE7tHMNjH2rgnaA/KiUyJ7JHdJE2s6BXebbphboPzm3mJU9X57lf8riVQQobAy9MPDQLMpA3Pf1paSk6lHQko0ZihwbIekldsaFdyDhnqgOQafKg0unnOYYf1XHbRhxEk+PvUxx6c7JaMzj0pbPmQ5xmE75mGyjDHyAzTxDcBhe7b1fOGzuzdOA1txXCRPwDkLkK6NsMYhI8z29QuknHBJ9PfFBAJ6UPFGSz/qip1VCzfitcYSQ2NhEen8S3vo8BuTHsxELjQ1qehmKTfIPQSiI1l/a5a9wEkJ4nqrbX7YSVF+UNz3OUNNGqBRDoC9lJGmpW31+/uI1bb7/QfFt4xKAL3htPd6Dk1l/0hPj2EzxcyamHOoiAVbtOXPWIxfe+VhBgE+sV/qasb3gAve0ZG0ZA68Mv+iNONO+bQWeTaOtDGGljcnNLJ439Gf83MMAL5/A3whcy/A2f7+ECe23DIwy5RGV2cI0R2IEOZgKPQaJ0ta6IbfPUOo+xGXiDBiFhYP1MCFHvfoE+JXkISSp3C4+DpXu5EQzNw9B+adVAXlMz2aPtocfbNpvpnt03nHyJiIUZkxuJYZjmPI1wcFiNnO/wpfCxZ2FOcg9yPINNTzCuPgk0xiKTl/W8nuTBp7zXT9Sj5QQ337m6iUpEn6Hyy4vRD4zwxAsoREpvpUiUckRxWqnBtInI3NtjhQ1GEAvCyeVVG30AiGbv0AyqIuVcSfYtz+bfap9ykPWAhxbQdaNJ2uF6Ff5gIELsbExQhIVXAs99DKu+zMCCI0Fxi3Ps5ibNxArwAqslvkS3gYm+3I/0HnJiQV7/qrfLW89wTo4rz33U85TFV1bj9Ola6tU5XLCD6lpv0bow6s8Ip5ykSaDihR21OdZBFXINe+FKggUjDb3zoRO8YBDBXp+nknLIaJ9Dt+1PQMEAZz9NlWOBnrYzTwaI1rpCYHonzw47Vif8AHkBjZhmMvOiGf/yDykgDpeZi/x1D5DqqwWnZtadkjaLY6f1hRH81hTJPaSe0ybktpt3qiIk6odvb19voOthsKpXKIl6Wf/eT8nzJwcxNfiuObzoerb6TY4Mi/iUX1Q0ft6fAlnd66BfKRKCuQB9Ie4BoKSY1i859McX3tfW+sbp6l0kFD6+wtsseLz2n0cyeixvM8sHTS6Y5N4RNfjX3xI0CVkIx7s3zMg7P5UeT3sElOJ5sWsD8ZAvWinQJeeSvImxQr9bA/+qO7K9KgEyfnV9U6LPdDNICm1vuxPc7on/C08f4ox7G4S7j9l4KlZsbyFQYGy44A1YahgdfF7a3jG2S8ox1jWxP0fRqjyPr0crGgeKsuoYRtdaKAzx0gQxD4tdfKMaBnRX5YE8wItf4NG2dHNfO6s009sJk8x8xUs9T4RWtEtNkHKCQOnBJYZWF4MT//NiWv3OitjUj6bksy1HfgbEiz49GYKEwFfzcYeDGqzael3mNG87I1U4Mr+vjcy9AODzcKHj6BSD7ppGjI69/8RYGpwbH29eVh+RSyLqc1w0AbkW8xddBzuF1SGbkGH4EqQJA9XLhdTWSkWecPezQ5LYw9/wulXnuwIZ5k1eji37lO8BrhPk8gTKRKzyzQCfKMbnX5U80Toho3YbnAc51XxNnOSHsZOJG2WmV+g7lG3fbLvXFXY37H6RCSHBC8NgHIYauN+fNS91fNtflKB/zQOsvmKj2F4MwEyz/emoOVXn15is81uEJz6Y6Luk7m65pD/wOPJip37gISsfSfbD8kj7bqFYq8iVVU1S/VU7q7yVojexCxUXYhYECiqAy80AyUBwgYi2nzYx0Iv5OpnZ37gc0DNp/LKq5wYMki9pqMCHIwHQGxh20MNeo4MNrIPx6GF575qmE479NCSSLzh2zV0X2ITAJoeiwmOzTxmiEl7BJm8dRWGS5FBiUTDFB61cUZvSvKqWG1tScPdhP1zUM98xo6RGRftk9kPxDWBloXF7ZZpsBt/IqFOqL/VVT1JNyLhGzNwMK7Vt4t98zTVcf0yxh9zprxn+4ibWcGXxMnX2wF97ATL81yifQHQ4TzPIW7y8gyDiWnknxCkItVxd7sXzbzc9PJeiaD/yKjlqQZ1ifvS6RFGCGH8o7r2gfm75KX9+GXZVUuL7iXv1PGeXj/D9133uVlLHn65l99dHT2dtt30rckJY7iIfvWOBBCA2POP3viyaPL4CfA7UjF7fExutIoGsIxtFkuB30+yJGEO1pgLYZQJX5gL4TQYNy1mgDn1rD3sXe68T3wNCm6glscSee45M+JJSV+kuQ7PJ07bEhpvGIIfUIIrEUKUpym3UTLbFhf+sM+2t3rTlMROtw6iJPYOzx+/SW+3JK7qIR0u3qHpTl3pL1xOL26gG7yfgAGFQrs/zzR91/6l+Lgr0Ju27pz/dntT0FQkhFvryngUfUnoo6R/3//H3j75EQTVFDVDHogWk7LirEPr0g28TSO7Un7kiu1zoevXmnnERtVmOf66H+PX89ImDqkv4jot9sEsGfUqgF7I054OazK8ajI6ecsH7+q41PbO4pdDT0z+N5Gc0gofEfoi5QodE/hlaQrf+rKZPbmaj3Zpz7H9tsfq3Fpn6T+Nkz98te3cUyGwCgW9tgNm9c+W9T40J/CFU6FerXqB3afuM+afGz59rY4/9XTWjtboU0q84oIGfPv2aDUKF/7nK71PNsefs9NgWvUaBVZnlrEQ9FT32+dh0x308C3n0XkanVGehwR5ftxwJkpNdxYa3NoUY2LEZNxMuu/SO6YWRyg+qo1CoLmUuVZ0Rmv8ydliLF+NwZqRrMA8MQvDjnzz8EkO6yHcfT16YNtHdJd+gsuKT7LwadnHnfJmxfjCL/Y1cHmJPAh9/HyPuTw52yMR6YxUzfp6hXqv0qe2z31XDni++67HCq7dy0vobWR29KDwKlUJxEN6+9vYROJY5z0ch9sukNA7KQrP9Tdnl22j5I9wBY5rNegaIPaZ0Jlfl2TAUxDpmNOKSllMt0H8czkXlXP4IZTkJ/ToptJconf64QFRTgdjPeZt0L6/EXZW94tVRSSEXOPDQA4ycJJNypoWWKVmyS6BlBwTrs6+Ji3UXtZnjhKQodyQMpNMUQuT9xWJKuS1Cdjxb/Eb998gcg3xDZJy2ZOvwYOMnTBXaVnMy9HVWBZd+kY9i6snkdKwWXX3pidMSphtL9YPx9fc3epOvNOyNjxBMl7iJRJMqaCAIq2Eq5ycaz2/A4VWMf9DojN0r4BuM2uvpenzxdC3hGf8arfsR3Ve1qwF3jODNcAIO9fFj+As92mjgwelrGgIdAgPEdDvbCDIMVWFsVeUPlYpny2yUXZZ1xmpVvdBJhZekWDirLm20c5JBy1y53Y1cfHlCKO9QMWiVF7cjrbL24xYzfKabmdVOD5hcHfvpxYVrjx86P109U3+I20qTCGBiXbF576yjY1YxvfIiKQIf4W96RjKBuq3utPJhDONkH66sbeV1etcTqGDRiyWNtFdAioLPtX/T133ojZp2ocPL6ENGdwinljRB7DE3s5a/yQa+Ea0bQJ0MhYhtCtRem0ZEeSwP7b41D97jE6EGwz8JFLJ66bMNj7gULECzb2Y+PUxP1K2Pw8NRRFmdQciyql3CJ43XIECd7bjp0OaVlW1z1W+0Y8pU6z4k5zbS2WuGnzq+CYDjK2tuj2QhWISATgHcqVN5OExT9LtpI3ezp6s5t+nCQeEx6IQrLu3xoPuwwo5PKiFoZnUSXBr6MsuZ9eD2adikV10pbxxm/mSfR/VhHujnfCX0ZApbLxH++Ior6gOB0+URltckKEs746QS25RXKah/OupguIRDIO8KxfB3BUGL+khRs2If71PhV8e5AOdb7/hOOvL0CJkBw753AX3TMfGJzAs9V70laVGJDkjSB25Y+22Z6juZKFMpcZ96Gmfj4Q2AG3k2cuzfcxYiF+MLeim0zbumpg5EYLISf59HBHTz9CX7do1pyAieMmZG18IyzQnIRWMFp+pyidpfvpv6v/tAAEKxfu5HuueDkyghUJI0YcNoqZfBLI4y3Ft79S1R9rtcybcqHFR1B6QeJlnAXPGTfEU7INqJsgKyLmXS8PgO8Nh6876hM8Sx3tGD+dncbxx/8bAxiIYfXPDVE40U+LQdaTw7BWK38hYtI29uWYExf8lozE2FyOwhgUjy2eHhVFNvFas9NnM8Yy/KeTmBGXCvhKpa5sFNrTroZRxFCtE1zZEBFWUU8i7Ol9GFRzdo3FEzA9ur25dYsVcYRxa80I5q6o70oF7bFb4Sx+in2COCEBW0YSLlJMU5rezM7tAPeTYa0GR6IActMK+T4RXnjqtY96C8wknagMVnSIhVu9DWDe51+wjhDvQMqFwBvcv0TJwFJdDaGHPvbmIyrYxN5ie3MpxIiKZHJAo8j7a4AJONKlezzU2PcL5zQdkzQY2QSxY/UVfDgQZfwcdTNirlbNYb2cqqAhx0oXSI5XN2DG8vKeEDSxY6khKzajLdakmZ8/3F4cuSCc1Njxh/VUc5ZArmn5a2OBMlGgI9kcVB4qKmINfQpcqbgdetN3uCTTcXvckoIAXy3tMebQe3p0vBFJvUpkH04aqT4hl1tEN+MEZ5NxYAVuLLiD04TdaMyqiNu3PcaQocX4jsz8mPuApi3e0cusbaxMirfcPyC84757KN7uWDk8V7jnpsLThuLwZ6NBOWCs/HCJCeh1LoOvJZBT3pjPX9njMak4X4hb6Qeby/VJ9LqMHCk+/3hvzl9QTEVhgqaFU4x4O3/PHjwbuHPQTJb+hBDM22r8NkW7HlgvxQsyhfRb6dj2o/uQ9o7l3U9ubWCX1HCXw1KgmIp3OEaTHMUFIGBMZ4XBAIOPYVbyvdityp9km8TTwlApZEa7saDI2XMozJxWSMh9Cq+W6rvkW31q/xZKHDKG5a/MG6mQzV3pgfZArUOXMv3YljbSNvH5HVfc667eR3MrVN0G0GkbiT4HTPXJjsoU22ZQHVrZuIMdapBmbfJJcTqSXvRrefhXGKIr31fJ6LZU3AljYroXSILgQZ+2K3Hw200qpnJvdIZeFrBCJRSV5sWLb4UAj+ZgJ1nJOSj0Vq4FOfRVqoZSY2X0GjfyRqlHyZG7PBVKKFO8H8zX/spYJkv5/32q21Fc5FO3LAqqosUnszR7giCWBTCd4F2q9cnRq6PYpPVRodw5VaKX2+VIP1/UVLG10Yfi5TbvXqoTppGE9mw/0jYfR3vgTEn5v0/M4Vqc3YKjLLmy8vWSGVDziXF+RKgXJJhGxoomhvkI+3D8gTLoyMFwrBEw+xdGCzY2allDbBy8X6QTVtnph8VMVPIE+qqKczcgx5cU7vRE1kP2CAS0qvkVA2qp/zl3I+n7GBQOnwmr5M+6LOKIEYrL5yYEKDV7asB/AGLOmxZSGxrUqWtDcBSgsa4H9LwRN07g0pWixP2VfbLEcgnUYz0hLWHotjLdfau8SnFIYB4pP0Qdt5H7cupcaiOEQHBwNMEzRXjSDslxdBsgoCA3d71+7SlAuja27XZtDl0Fc/Ye0T4y6JdzDzZs+c1Yczrc6VUdZKZm5nH3WrWHkMKllUn+cA/nuQprSuZd2nmmmxT/RQnBdP7wsPRhGHMi16U8n88lUdu4I8yFFQTSZGt33IaWIMmrg1ra8Y0lKg3NFv60aVNWmi70pNN3vAvMQvlCTyFrU/EeV0zvZVuIiH0TE6mxFTXQn+fUnfrVvBoyhXi1UvlE67QchUeLGWuHaClPseTrLYtMkMHwG8bAHB3oyDyc7XyrD5DhDkaq9p52YOGhspHSoe6CheIPIGti29H4HtyH4v9xFSpINx6RkdNvtmHvxaXo7ZAVcqO5ZQchQa6DiHzwN7QZ9pwdXuQuSfbMQLgj/EXml3yP187oapvlR2jktWtSMqNGXqfzKVolPGJz4VqQyWSBIqUeCwAizaV1v4Fog/jjSWH29bkducY5ZOL6hyhNFYJ3RE4nfpM+b7yK3OHzbFuK3Fmg3PA11WKcLLlUNJZzS8/snhrSh/35ENCmEYXDMv59QIfXruzUY2TH0otNVxOmow5J5pMeQjWfTmeo6gq89clMM1sKYeDjzzQEpIqfCJ9z5+x7fkooBKgCnwpZz0IecnNE7rpFkmZbD5SAOe1VSSrL8u4kAE1oBflR+S3BAdIekWidTspoKLZ0jjuPNcAjllbtseZ0V3zeeryuQllPOp+y1srNIX4pvZr3vMUL16cOFxGoHN3zcBFjpOlt4fzdNIPtXRZfs+IkX5xjR0P1YziHD08c2lPSq6VeQ0zXs6XE1eCNPelhOl6fu1ag+Xbx+5KJ4hetOk17L6aCVRbfiqPbPOm6ZD0XkK2JoNBplo1rsnpbWFCpLbYnof9dUJNx+qrFnyRvdo5vm7aGH7RI61vQEZz0Piuy454g6kqkFIqUXLo1FzK9BR2CtE90ZtdRzBAbK/t0urwOdIdFoRchbjqraVQnbiCFK6lotntmJv6DATHQcoMnc9bTZ1C1Ooi68xojahpvmvHBscdIxNYpIgXvJXYCTEqiiPvThKdn3Pn/Mmpd4tuNgK2iKNF4dtX3sQg9fWwY+Px5Rpve68xlsJqknIK1fQ3kaZpFbHV0NdHZjvefp21K7graNCgjOA1f2omexDzBqhzw34Mhn5GBLp20VeLO6ANiedBzaCJSHZqMq7bFFe+Pa3FgDsUx1JXwXiI6h9d2RRpSq0mXj34KQgbl/H3yY8xJPEgHrxrBFCn0HyyqMGxGhNECghgk33zsnAOypX5UeHJxqE7LXR+y4pgkbfLcGLqcVW0tYHTae9QYqB2YaW1FhfuEzmD5GCAU1523JV9cnwx6nnan8aRnIDda1ALcCf221A4o7JboDb3x2zYaUvRrQxIz68TJkAGejwIjXyNqJ3PPOE+esRx0mlub6P3rggK73EFHxmmYll3lIjBArrZI+5waciZPsGgOEHiS4iM9Tb3LkPuCEZ3/+Oxi9pSLI3B3iddzO/SQYTEaeeVbCXW6epFyO4FCb2V1cfF58tmwNUXUxLXqp8gAd5sDiD5Q20RWIoHd/Fat04QLwCbwZyGPi6MS3j83DuR7E08yhVPAukuPDIV/VH7fz6g9WZmQmj1EdpS0Tvo7gJtc7P9hH2EdEo9BRwJGZi4Did8Ub/DXPQwkSjmjMwu4PUZLaiOLiNcKjDsbGhabJbhTgPYVfwjq8VPbfG4Iw/v8C0u94sBEpVw6dKN796XCpCnIZc24eILxC0h9Qlzts4uyUmBDMe1ApJGI5zgMUvluXtPdZ9UyInZDh498roT/pKS5kvPeOLcGnCKKCsuud6+1K8QQropVp8nkf0czShENEkTUX9S3qePdc/uvUbRzxoXrauXd/ptJs8tMrdFARcLExY82nMmapgTpJxC2BRbbcvnqLh+e4lfvZ7MD16yt8Y8Wk4VHF0fn6nKOnQIDA/UW9ycp6EHZ97gsn7ngpXAYpEXxzq9VrVwc89CQ+s7dwReuKWHRrqaaFdAu+2yOAXRaptVcx9YYhbNTFj3+0lsxhO5126gxj6VQCY/BEF9WICk7FGSG0/dOce9QMQ24buppHiURB1GACxoO168+rWPCNedN8I4OiGe0w3SZagOdtrnumm7lozauk77aeDHQrwG6KbSg12RicE2YaX3IK4RVjt6dI8Qnm+bsVMoPpTVDaSszfXpHgRIvzqG7PXwVDIICZr850GSKsNBeiP+d02ePdcPvWbYZNQnveid3dzNG4ENLBhDbc3xVZMZHfwzLnQxRtzBGBMcmzAfYC+uzfWKW6NYkqKOr2bLxMJjVf3mkcevVHyIMDFGtGPEwaJLCSNi3ZxX72mVxS8idzDMFQl0M2Qa5siK/snyA3LzKZjZ1t2GGTTLL7pvG4CQWUKtHVamxtUj2sNFf2DCN/kJlXv7pBVZd+0UcvSJBwEQPe6s6yJcL+oLHQUm3iizoBF69YMnmvHLnNDWGOPwba2zDtISyZiGzd2EKczOSh6bjCIrwyUY847j/UwCWMBL6fyVyYF2Dyzt7JNeIdGCZaJDWR7zVcVPMZeuJ2JY8dbFRsDsBiMwiDFHz3YoiB2CeMHLg/2WtGAUPUEhip9RMELZvNWf4JJPyRDubQe5WMPGeFey5O92y+EQfc+VYtwHX6SnkIQMhe/W6jaeKYUGNklXJhhBXiMYmf6iyi2pB9eXhiOSRAgilco7Pvfr9afnfIQCZ+TwxkHjuj7bobiaYAjmOEHVcAeRS+ip5fz7Bs+cPhwXi3oq9tDAY5Ie9+N4M/jUoIRGGLQ7d/wgTsl7QvtfEnQRxt3pu1wDR5pa2Dm5gZAJZ8gN0QJNJOv8zUg3g4la0I/UB09MiH7HIjWXMvQ3kWmDTYLBYjUZRx66rtt0y4hSKE1JULMk5J1Iwl1ju/SdmYMsQnRiLFpcqwPQO6pROy7Key2HkHzYMPL4XT0lSWtd5ZIJnjscZJYMQQH1/013V5Ach0EAfjmwaVJWAHXu4rPOwzbWpwPdwAxXRkrMPYZg/8I/XdLKmHLe9kZMikysXqwy83etLHJDY4a5/yJxQvH9WxO9wVg9bH7ZrIBvOCi2pIpkIyXghZaJVHtaIJO84VAT3pXmNJ91MJ7nzUCQRiRDezMXXakyzPjFq344ZoqMhij64RjwW0PBB8TFAn14GEtM4lLJZWDAGC5V08R4cJFt/ChIz6Foi93CzvjXjOiGqEKgeAxFsUKLouSZUpLoucyLdW1cuid71s9f2cSDaZVHofYv2agPL3Vf/kwVZiDleeb0fW2y9jHw0MFI942NwmLXVsrAWvH9QFEKgxzja0bKT/yZyG/ImsBZPgRX4xAAPYSdypaTAH/EmX8iSD0dhC9W+oSmuTSyr5QG1k57Uw5VeGmCnbfAOSKTl1UoAgVB6OjAZVG2lcqNGuOPpbSIA/YxrAwwSAKHsd049DK8LhaEH/aMAUM9LxQ3rBdF5c7WXphjyAgWFclfgBXDMPPMMAjdqvvfi45sJWe5TyDYS9srOe1vue6acVVVqD4aGKKj59mTVyZLVwJhXp/n4WpPycVA8/C0pvJrnjZ8OdvyiB24WFCJGNvxdONg8oHauc3Ln4MtBNfnGgI/z18rwIGOo82WEuJ2N4jJ0PkClHxw65jab5wHHlYceaKCFJ4Nr7SWVZiRUEWSVmfVV98xZU80n1IMtLnSTO3I9kRbSOmrxlq4tALutd1iy9BqI1Okl9xeCkFDWoMFjB0uXJHQ1bvCQaenTIcW19WgFdyorfYMcj8GzkrZxjQnMIDeJiWzRQhg5Abrf48Ft5SsFWp+hiW8IUmyem/xxbIr5kvbmM6BlDwZw4chmnSy8YZAAyL/LAZmO8jgDiJ3QEVDQnJGAmB9rrKDzSZXgsv3+5NdVlDkvBoriMvK0ICliSPJlHVfGuB7pYpbFCUDKlr4N1oDiSJeSLI1F2YAXpCL2Qvoc3VY+2DwEJyBHPdKhWRacbI490XoWSHWIQB/HLrnqAZo0/B5mLQG5CCrNLEOlgUsaMOlPGyyF4uvSLjPSWPEpMtov+ApVG/loEmz+y75sCiqP+IvIC0qz2czMcJbkSkOylH+/vW2KAE3W9055yAzOOIvVsEA4NoxxMsLVQMN/5GsL3ohrfKTRAr0PWq9wwdLRWwvCT5uxWCv6qx6KqfY1dBy6Vdmda1/kbFMbzZ3TSalPiWDKu4vWLWbkLkMH5Rcae886cURpk/bPcNuHqROB9elz7gR3qPpx1LeZH6QhVFrsqp4ek6Olw3/tj2bjKzuhX0SOW9jknmZtGjiUbnR2PxuVB9XRozcprt+wkbsjaAdrbRByC4ABsLYYOLA8pdZLexQlbkrY9g6rIclbRWtgYAKBJr7mZYRn/ljfVB33TEqAIj4IxoKGGJbSiJPC1F3b7nMDwRY17t1HmiWCEGqA9T+Wa2R77Ip9LdK1Lc16p6w1UuNuM3LViRwBIeRknwUZIOU+RE8SHMiHTM0HuPVybHnLvoI9CmHEWulXy3ZUYA7lr4PdtQKkxidrH4TGnDHpXi06enBZfVlHwBzg4Us7EbDYrMEI30HoIqVAIjNFHFv8BC0+xAfhLzcz1BILDZMRGJ/mqb/E3oNE4qzTd8+XyVBSMORsz460gC3/2c78RqJFJpCgAR5osfAz76V8Fm+deikeTubEL32BsJm3VC2psWq+lChxsOemvvYx1+1qooFiKANMxD4uZiDl3PG8PEkFWvlXRy2Bvisi+FvW6qHhqILD+HbZS2vTsHP6ZOe7x4sSxDSIoiLGc+ugDXOv06wLQbRoQ0DDyQ6SmGxOU38+B3i/rw8k2E3pknVAdaxkfSV2d0/rwvEor1xrMpUuqCOkpmeRbB9TWC4R4/rpcmxjMdznrn+rqufYvk70kM8sBNGHLX1zIfoR41MqHDF/+SAxaY1Yj2okwntvyGSqj6aDxbl++K7d8vZyI/sJ2LJI6VW87fTpEeIqOxsB1ncen3tDpjooh6sgduES0IhyawyA65/IrAo4j3qPlaIRNVdItZn80uet3kROAbE0pWpqJwcRFGGF8cLe1LwLt+2xDtmTHNrAF9Fz6OXiMJbyxaihlkp6hZod34o5BJufeWUvh1zKR/UGg5HHQv3oWPmuHk9SamEZHYafEj+Bppfrg6dNcER9ZbVk9Q8cq2QUOKzrmemVkm6rDXUpoTGdtVLwcVBAl4UcahfHmEMr7+3hkR8uB9OSqjWn7M4pERDnmPUnKQVzRDpfIZuKvPRHYQortX86wA3s1vnlxQzu+lmNaOzbQlwOjyCxJrDHqwISXtKCxWJrHoyXYnNOpw9nu/r2sUF6ARpuk0X1LVA+kY+E3xDqv000gAP02be+y+ONqLYDzyvhf0BGop04K5M3R3fk+dznMbhPIpZI8K4syKvgpSi45XmFuKVKHYyXY8PMJiHOsQinjAXgQcEr10LNyc97Zm0LykDRPmE2x0SfAJrWnq3jrFanUZiaF3d01AFv+SrO/eAJKqutB4ARbuGPRh9TiL7jk0UzvAk7NvVVaLopoR630lTCgMd5Zdk+0Tbt8Kot49fTaMftJjL04fnXPoLMOSVj49gIYcSXK5cMCFZ9N371slWVHA+BO7BFEniRRdR8eiYOd0xNwAyRtfzwMGg6DqdsNcCwSU9kPI53uieBN/Af49IBeicQi2c/6+tMHliTlaToRSBP0mzHXFLKDpzkFXH6kRbAQ/ZQdOwxiWe6O/SjtLIaQVi+15RMfsrxmY5531nhDxwitbjq/v6FvKa2xhq+wNh6zhxSkYw03p34GftkGAgueIGTjW8TWh1XembW2kUSqNcLfd+7zxnrPIcvFzDJkvt36qvsrl10HGHyX4zLk28zxctL01mGOrHhh9uTMIPiS8r1LXT0Q3/JSZCfIC3OxBDNMLmRK3u/T81/6UF0Ewxw5WlBdCvccTIbicM19wEyOZPzD24gcgSAXNasdJavzHA+8tyfVcC9IYrrddB2XaAJK2y5J8Ami7q1y+nUuvf9MWbTEmvo04BdQoEk19zPJVrq7WwTemmD2XejnnzX5F+sC627mCIcaKgve+Bfvp0TZFnZWbCj3zRF8eRjqMT1qMAWbSmqmXBcrS6NVZlckVIT6qYVwP+DEGRfx9SPUmlAJfiLqdTcmGzSyyqJ8DHEeed8ASx58jtIPzlQtEKIbvpoK5qDIJ1UuKfwwDfgFY7F8Co/pmUVsENu3F13aB+SOqLgPu5Uck78wOLhucQM4vRYnDGwpHlgYcnL05MJBwhgV+SF4kiDTo8/5dpnyFUeHXo6hHGmjvQefkUdi7+s1zliImKcMdhTjuhHGzOb1YeR3mC/pu5OOmi1M8amE9UNyEmuQgtMbXlBUCTiib4VLExyM+XqE2EK+FI/WdKveYVNuVp5AeldM6ZVakWXBhN4E5wXAg5p3XKsEgFZs1OVWvfS2bXRg6Twt+Ccl9MBlHAwpc12zgPsSyQ9r3EDoZuzzy0LJj94clCcHKSHRKVKTHbE74PW7kDKmyCQi2zs2uyffH3fhdNmuE+7a5WzA1XW+b4i8Ogt+6GC8ZsO/Oo5IWyXfLqXHk1zm5z/7wiakUxAaD+RLkRFcQ1vFrikxByB8XBQ0cp5X8BUuhT0NovHb5g/3s6I7GrFQnYQhQZbCQX/UxMX/gVymAWS4Y5PnxSq2wt4tk74evvgyHN6iz77gTJ9TkCBBw7iqsvK1kyykAoCc9o76HVhvo3TsTkOAfgKLRl0NmOry/bRqb4dhrCVNXHSkYTr0HsemzwhxHL0ODRT2es2LKeh8rxRcuJITkPqDEam8W27j5jr3AhQETvy6DFuR7urF09dMM0TtA8eML0JLh0a+DmLSKH0T8hnPTO/ELAnSoBT5vVssxVXzVEeIOoRaFhdyE9vtO1Z8DXdoaFpquN9QuWkOn85QQzELSUdqPGD323UjiVEO7n2PtFZzCFZf4p9ML6EApxK3leaGrz1S+QfddoHH5bp4gkBNrG9G9kh3AIVuj9o1E2OGbitcYEQFg0azz71Ysj657t+gfUnGXN4s4UNcyUBxDHpNlPoeck6ScBjOs8lkfbj9U/UuAePI4PscMJTIHXR1FyGW1Y0gKw5hcPXp2h+LEC1mluqa7jRoFeOTCe1LaD4bBMjYuM6fYv3XpAjkQCspg25dyivZENOkp50/20+bqaeIbUn3DFqyAonJfK8mlxObn3PQ/4qoSt/dr0pVzUpA0dpLj/om3ONAODC9so6BVlvZxWVppFJcFTzJIGjwQexfQLeEjwAse0ZJ0vJ4hoihh781q5DmIUmEQYu0275DmUzYftK7AIQYN4qz8uPkS5FkfZtm6HgieJH23BFo7XT1yd57QHkkZfIPiCdo6kjwMDHSbR3y+Nif2xGWu4TsOqGaBhwT6HXiyd++6awGVwRhBjTQYAvwmmws867oFwCi+JFpWFJrCoUh2WAKe7zkYSDDWZBKbMYB4P7+bMe3mvhcZ8ivo0zCYvT5wGZ0b1+uuERyCe+OJmeSG5yTkjAiIuHMro4JP6IvJca+GFJ6w598OPSGFaJf6eyTZ8YwXfjUAYtGaKTmrz67+MWw4SJQ4mmGz1tLV+G6I92gVWzDT+7tYw+wRRYIJxhsJvPfFSZZzo0qTxMWaTAgRq7sP88osIfFQPIzasRCcsHVyBxgrZAYy27RE+srnXT1aEZb9sfobmQ+SPMERkTyHcc3CSe6BXtXZw8rpB1Y3c9sImcmrzcx7oA7nU9K0C58Tx5q4/bVwAvKqpsI2QzRxE1tr5MOyM/VFbqqDqDfPFx0PYyk8C7luFIfOD3P7edU9zF2b7MAs5MMXc8+LgaZnuPzWExxHyLwIIMQCvt0sEJ8P0LFW10WGeGv7sAW5pJml41yB2KyZN4p59EjjOum6b0BjJvPoLeDlDLFp9RdvF4bxpSHQhsbgzZnQmNvBiWoWufBiVlFw3JurQX7yFKfYvhxq3+z8uxZz+lVFHyW4XXhzT95hf97DNnT95raDJNaKX5GXF7tK87RHGMHtSnJhndiiX88vt0TlLJDXb6di9DC+ZoTJDLs45ymOlCqu+djiaGjj9bgibxM32fQsdcSJ4pBaYYTqmmD+5z4wKyu4DUVdU85yDDqGKWGMAJjYEosNs2Y6fGan+t1BUHSMju19Deb3LY3vuSsvBXmD+rtW5TNXtsY0tATIiIgeXkeorz2lhCz33ZtUtZPmOqfdXekwehYr6o/ZXu/z+ugtqYtd3s5sRiQL+cyRllVxm2C79+aD9rH6LLul7IjVDMiOmz7wiL/U7HXmuPYHiVcJE69sbgSg6u+GTbOIZYNDYOswfYT4vNIStds3WtCTNHxfzvlQ9roOH3VruKgrlnhPswZ0+mNyzfcGAnzPaJcm09SbHZ3l1PS/9juUww8qvb16TZOUQs/qeOY6MuKNCgns+w31tlXV+rvLEdlGA5Uz4kt6QaaO6DgkFqlAIgDku30CIgV+k6bjRi6IxSmV40+zXWMAMFX85F/Q2X7fOMbJNZXSRaKibWFnmxsklj3Lu0VDIOf7d9bDDOG9Mt9PBdrZWTRCXvFX1gWLx91zVjcTWAqDh7ZxDTcejKRNHhgaTXpulMCv56YsaCC4ZfmmqbA5SZqoiQs3LD/YazKwmfZgALxH1oyesnW8mpcgbkUMJ4MTtqHsLJu69JV0Hhwx9gtLR/YYJd9oal37Csem266azj3z2kIk2mIB++7pqjdCI1luooaE/nofnddweGVxIN++bupin0xYDmJXSHjYkrIsJAKMwI0pXh4N66FnG1uQFBG0Vx3cDJP/oUW0DWAo7yLETfQhV6a+vu5CeV1VTOaTFrimmYqJQTTUCSFmjaTQoZ6LoTVhgRTSdyq5kJd8a5psO25hUNnoXH63NTuCTy/xaIfN++52cKO8/qa54D6o1n4Oz8/rpWyPMck5Vwj3xvRUUY+tDwI6MUBTLHgQCZeKyTGiLRee2WuFT51P0l7+tjHHD1BgomUodUWzmSNo4obynS+0a+z+nNeVHvgtckUWnpV/4rEhXMBw9fVm+qmeY1t/9yzjh8s2C+J3trIs7MldN7Ufp/z2U9M7FORZJVo/pHJhrGk0RcU/E03WLXFnZRLReYE4hyIZ3q0SmMRsALDATNxuBv5KRq8C+Aek3v8ggr4pZhvpVPjZ78ImcG9cTKJoo6Bhmo1+BffgKPAvCaPjARl5QuvDSvUG4SC7IVm4ew50+W4ragmIAngR9MtrEgdG4H1ufIQWXh/0Sh0wiENxo+UtXUuGqUDWl9pzUmM3tjqtj8y5zsJ3cr5rWa0PQBoiKuTJ8eh6dmJpmUE81XjodeQml/8at0sl9wQXqTjhPrURZW/u+nbZX7OWH8eacKY59EkR0+yFBxJxErDb7fDdTOtJMnNz8HPP3rQnroh6Zvi+fUUvC4okPMoTgaCCs8jlVGA3un8ukhERJc2edlz9zOjOBT2zZxi4x88ymNlIC5832/AGfOCwug/m2l+kDK9xjL0kkUJ2FOQ7D2cgmLxUP8CPYD6VfPvj7B6xnMfCQLA8aL5byqL4djiirckXGedWAG3I5Xd+Aatp3U50H3jwgEZUwpvRf0vrl5Ylj0u6+pyLUpnYK/KeXfq+RLPQjPz+6jjVSNUTnCL3me4ZM0aGHKO2McA0fTO1gYd6NXoEVorcSh0gyT7A9UVEzYWcXN0B0UcjZLHmdoAwjK0FCPO63cdo46CHa+5waGLZXkBVZ570kq8ChUZajoLQfJerlSywXtbJ3OqvhBBGf5ExVyIIiSnH2crkF8qlw/p4/OWv+1EgCr9i2itG74CL7isahqrqsHq/XdLJufMW3IrFl1ryRRLZS+JT40Yyr13M2agufvafF8BlxHHUBQN0hUJZNmNek+xWHkUu4GYxNtbrh8KGrIdACx1knTmkJIx83ndr6V5yIgyQ4i08COLgKDOuS/pYPoJygGH6YYjsQT9PHKiXI+57cBcmtzxk/G6NpKnxMYA+pkmJosEGJ4LjN0EtX9gWb16zb5x9HI9A7y2WZqiMJwPNLwTX/Pd1ow/DyoiO4i+Ami1fIKsCi6iyyYa5fL2k3ypxS81KPBN44YEP0Je7VBlHeXl6R97TUjUmYDD0V4Z3mVGCb7fWb8rvaOUoZyHrez3QisHm1XzToYDUR1NN7whOWEYe8d2VuPtRlFiKbK9Su5flRCV3sgkjfS5RgKRPnkDOUrwBFHPfB4kjF3lahJ8laMb0Nl6LAzubtQ92AOG+n06TYVMDAXsubz9bw/Jv4hQIof0QXv6UcmprwcOwqIV3xBWokniVY7CDiO9995u/BGgB9prJ1ClTi99dYy/wt0VlKlXlvMMb6/+DpqtYehxrlk9z92JYWmwx007MaPHTX52v55+IiemYaMuyVJWVWXQCXPeL+cSuazS1/zahGDHqB3SGfYgdJISIELumQzvcSJ0KNQ5O9MTSmPtQLB9NGl4Z48gIAcHBu7AyrSd+FADgNIy/dAePUhzLjjB5muHQk08fJHpwlIgXNvBsoPBptCvQEjfWL8Svv6vIUyd82KSG+xmRjnw9A6NXNX/fz2eh95v4oHYWeR6GdiuSVp+Y5SqDe0OUKce73Z3nL/yTmzwz3h/DGWccrlv3i4o8BxtyYkNiHBImBAdiKBWmrO1mqcqoJUVUsBZD+f1RUYWXc5c5SiTTXuTyIEVL+4P0fRpqCM0vue4JlmkV/uN13BlvRRrcxpznjgKjiiaoS5YNbl0saQt/oRZ282UDnt2WtfIPaxi5AO9fUwcZRw3NM4Z5kYNnizuJrZJ10aDhl5O0/lJR9Bu737T7zc7gmPYmaqDyl1PmS5foooAQjFgFCTxvP8nG98U1APu8QyNAljB4IZgyguxR6YE5Xgp+beNui/4JK+N+g3w/n5kpHs3HpyFIu+Zkv+NV0jmViyMsli9NCLoqvJpQ0Zs9Jw6CtB9R9NOJ8md3aSFAP9cK/dp12LNDh7lbf6U6cer3h4gzhHDXNKbCSJ7EuA7PauBqMec1w/DgIpt+SOpWh4ry00fmmIT8S0v3KukjNfPcFSOyNF6WP5ahKAsMouj9yPzwJ1xf70rJMBzy7Fql7awTshxHGNAaDY7Le8X7MUNOYjn8p3rAK9hl4YMa+o/yLrhqNMuGk8zjZjtq+/0HL1Hw2q8/mGX2/eu9e6xuMwuj4iw6yTdvxfrrFZ8s/UzRM7IrBP+MnWoQOJxsp1Igb6c420RdT4krkAn2licZVYNKr7aex+PC2xfSF27rqhnL4Glg8PfCZMfxLXPSuw2yicJlzGIVn0mc3L9WseAI8tprTBpaXYiX0cC5v/8UkL12sxgJgK6RBp7XXxFzHXfxvHdkmLVNyBN3nv9cePFVsoAHJJ5ixnUtbqLieXTiF0xRPCn/qjbf572XT3pCR5smjJ/haDNSnn+TaMWme5M7mkE0hfbEWlR+zfGkIMB/Fw4b62Mvy0NspMNcVkO9kay6n9FvjdkYrrbBruG7ddqLv9ZFUYwRVlFGJPxR/VRvLv9rnMvJRYB7ESilvioi4nt3EhfGHhrn2wOFuDvFiO/MFLlIefRPUjEY/4IgQj0qd4qsl+G31lZjwtRZRw1ipTtq1pwvyn56ZvmqgRDt4d8kvL8+t919Ep03EKWvP5533Mo5yGgFQtptz46nHDNYFAX0JYLCVGoWdP2+A9xM3GU3De4Cy08ESuaeu1YxYQ6ORvAPdlOORm9nJY7lHFJT0VjcrSCeNe2LUsx+fzuHbvBJV253zxI54JxuHsu7JfXOa95QwhHQ5n1n7XtVUcjpxOcbzbFkVRsWBVKaIGVq2f+emKbENL+IxjzWpAl/1SfNSL/qQ73wx0IQY6SNL9y3aHYXcdID4C9Vkgwvq8Y1+twlVWlKevlCsNZhHrnhcOeHzPLQUeQWCO+0Ji3EeyxOkKM2ptL9IipUd3NRkZoylX4stzONxtlwnfFalOqVbfsbQgR0IfDe005QWBYybt33g2YaaHCiiWA/cmTR9pTB7QZs7qcm085ynxKB69UIid7UQl5Lz6BanXYprgTFmjFCPuMoncgH0eQKA/sXkkgL0SVLPcGAsBo/BDCz9TowNhE3e7Ck5ci84v662z3jC8TQtq/3Ldbcee9hUoYe6WqivP2Nn4NV6lqxGqkqBJBUGK6p0aHaj0GT3sf/lPKRiAQtkY2UZgUyAtyBvRvTNmwqcq49izcaYamCw+r2jcnG4QpcyHWGUp7e+7pwYnGztrBmbpWk4Dl5Doq16mgifMJZM5HKgwnRrNOW6NjiboK916RslN2xnFzLxz3SgaVwhex9ln3Y+HO0CIgpeCU1Ic5K/XPv+44Nu1HcWTYjMwFTyS1ZptaxE3dtlr7Uh628/0Ddq8nnbn9VvAI3+vIw8nCL9U4rOQ75LKH3nie5lgIJ4fMbogmKo/qv1sdJnMJ+zzR0A30d2Bd5OMODEoYPUr9AkIYtX3be3xe2FMjttdQSFDmN9SgORX3O+XPSBs74m1+yVkbpOOpQmadsIOFSPSsCNfTfI7MI9BD967Ffe+xTJNM1xFDohhR4k+mCgLqspcs4aYXLhvwGzGiInJFkGoX9dc47/jN+j1iovzDfYHZZs8/RW3+tczHBYHhkvrcldCfpSL3mJR5A9BTpa7t0CJdLNt5V8Tu/7Yx8ZWZ9/DEkufNWHySnZFoBCbTlbycDhCkXocnzpDUJMjRhp6u6GQxeSM3id3U4tfqet3aMClzvVxi8iqLLkivG+9i6Re5nJj7qPlGJ9oaluc7qKIq/nlk9O/zyE+QyscjQbm4GqkVa06oINfg4u9er2EWjkClNgL9Am/iJ4krxl+UDKcAoT0yw1yyeAL6wmUjeAHee+Ef5cvbsGsrnpjQVZSxH17bpR5XEVIHmoNM672jb9oMketvCwitebE0UJSZS0D4Llm/1o5OqUUGOVKHewBkDr+YDsZZB+i0aDqd2gSHn+l7Swo8o3HPiUnSkSC5nDiscw7sOd7RDttjEj3PBTNwLCBXnZeiuTWveDT31SdDJYtGPCxD5+JZrfo/Ngf6wW8OFAgwljNil/LYLVcagEDT+wk2FbFKOGOJoZb/QNi8LRIjOwSjDslc6Q/ZSt9SBUuufJv+eaOAETYT/IlsG5sCdJFc7chLgh77+lzdEFRf1ujKFxiFbxhk2QqiIIaTRA1XGLfUA1S361SdoqP7IhAafgoWxHfsSn98XXDy4bmZgIRKzjRIetZF01VJalDmNQBDcnvYEuH18nqe7Xty6CFRg+qchkMT04217IBWJmP7Sn87hqia05z1b+JmcZ3KWC/nPeXwTkQ2yh0CIZyZimQRtzWslogfZFrRbS2BvmxQGTxH+DUIjpT/JR3XqV096E5cb1tar36f1PXsN5L/6kWxtFxEO22nHyi1iJ4IQyBDA0M/sRAIrcOR3+5w/xqcciPd5kKQpUyvubtwAcVvB7jul6zkN+QfSZ+fns2sKUsOwU8BdB8YlBPWel1Tzv7doI/uhOeXv90MDu8bEyVmz7ChooYIP+H16cdCl/onAaSwlFztHR/Dlx9iN3XpLf9orqLR6znHQRSXkyo55SB3B4Xd3pdyidgHeQKJr8N+7RG0nk1q0IMVBd9ksbS5dvszPpuTG6cLxg+WjMSKJvuqx4+nltX7b9ba+PORf+uBZE80LPq4YmmzVw49ous/qJvjBCBKT2oCfk6Hh7D5TK2dDpikTGLDyV5UCdjt/04OBWIZk/7ylEHAautHAw2kS91/p/1k+IPBqXr/lelgex6G8f46BzUe/9sUAEv+Zdwkbx9jMLTtYa/gA12ov6YccqldHJ3qscHOefCOaJPIjLmg1d4/AjzAgXOW+vIamjA+nOruo9IgGxv88PCQoFIKvcAe2uTrn3F6zG4pV/4IJMuxS+bcs5tUGTMHgPp/Q5Q9MSCSIgUzrl7d0aGb1LyH6kMgWq6mXSnYR+i97MAF74bTOWGtkkNgciN313Ea3FJW0QH+n/msuX9w8KiZp+mz2oowAkNrktfF0FVyh2DX9PXuVaLYPUk/MqQ3eX/5fUc/7Mc1fDaeK/E3MhJz5Ms2zqARiZjCMZFnDlE6KjOBjB+FG6BSLmFTK8bUx+XAJbBk2yolqjsNbTxie1x6Q9Ta6AxRcb3erVZo9TbnPYueIDkNIyzOKBmxTq03V/73OeElTUux8ltpyOB52H3M+GV4UK8MlhjhiXT3eT1k4d25xZ3EO0Ffc79eXXgF8T+krF2jPuTA3wruH80Ty90VxVghibZaGy1kPLcYm1lqZrPjsBlaih2kLeC8Qn3DeiUYrbXirOxzjvafNQWyLt+ZvFQ6N1JsubYCdAidmniN9oL+uFWNDVGr4/mCCM14d/sxht1agWvgUlsJCARwC6KMkqTP/yQAwIkM343eUpMZjBkfY0AA3B+HcSOJ4wqOLuiWJcznfIaK1Yhqjru1X1uBbYV0yEgLUrb6p70nYqf6tqf3c6NPO80RgzQd6A2PtUjNahakgPx9+vvVFi50AOSAzTwrsb6SKkNuXNKbtXnnoPRp6owpGMUSy/GVxSmB9cI8OfExu0w3Ot//ol0U5Kf/v3u+Dc86TuGDd/uC/dbl8nq3wbpKHsWI4HMm3U5k9Iaa4tKSpJUQ6lwQJPW9LuZRyi+MTmA09YpK6pYgUUGxBahEzbN8PWn4kFNSM6j0rIlVZNNtfhx4pivQ1lWsvQ+hw5Qz0qjEQaBlkSu8uxbwna7K9Mgd8dGT+dqB6zvk0oXu6zgFywEb1GIeM+RPpLYJKJ1CN13EoyIc3v/QKfGJxVnWSmbbk2jBt6dDjImLtxDBvB9d0bLZm9Gm4eeWRhkz0Tv66YOm4weopwUwM3cUbh5YpxL+LjZmsoRseqZsntMHN8ar7T0BsPb0jKFuir1GCWtSuehRg4DZh6lf78auQO5y57n4pYz2FQ33Aq31VXa+ewwGkdDRhUOBLxsem3DIrh+zlvcuasjjaf6Objm7xDpJRCv+6kER3RUnyygvhWwKVmibZs1ZQeRKQsGminO5sx21cOec9Me2A1jD53SrQwpuD9JTD87f4RnfIcNNBcvjirfVxOABRPRL7sKgb4mtWOdNFhCbmALPLCcKXsswoknxhcIxk59oMiioVHCEP8MF8JvOS0WOZ4Cma/daXtBLOiNfHFBiAC7DUcc4cM8l2bUQmDxJmM2jJEcyXIAFfc+/dHx1j8gm9wL3seNzJohQj3EcsNI5Sla4wnljEG8n2b3cH8upD1ASF4u1FDVpd2+VzJjT4ZSpKs5rnyBWtfWIEkAQhAF4bvuxhULbDTqdYl/66XL8/nhQrYxmhdfPAnJBbxrdG4i3qO+fEm0U5rn34t9FZJsNuGdZQrosg5W+Y48w2xIz8XksXmtvGErhv2YmUXUKPLQU/2puXz+Hfv/rzbIaEAUShcEElZyPOuYcEKpuPMXnwCne4QcJqXBMk8MjXtVDWPGbCTYo30Au3G9Yp/Vf7a7dW08QxfBVCaxGIQseouCe6PXcI3RMfMcWE0bWRk0SwENqOxeSgEeC7SKZm+zuv3+hc35jBgE0cz/0AuBGALablpBFT4RE/gbyJxzpoez18P8j8ZZH6TTokS8SRwqW7lob/9uhJCBdKDbJJexKeLiGVTkdflrdiyyTq1VGKBAFP9yZYArpNdWHu0z2b4tdT9aSfvORe2zps+j/v/P6OhP+uEDakop+dwuHj0UlEZ89OqUMm6sGCtOh+AHEB1VN2zzZRseB5+n8jbe4zwyOHuIqaGoqrdkwWmJ969VDQBdph5WGMGWUEwWLizZCVUo//i0eKoAPSVlbWNS7nwN1B3kgNWdiNQqkvdnTLnAS3bi6RFCghaXc/e8RwV/aq0yJ4NXYRuCoH6QiEeYttUiH9ev/9qChhthj3b0/V1qY+JJV5LPHgN0LXWtBE/UbdXx6DLAMKAc8BU8RevaZDU6aarq+Mia5/Ay84fcYvOmVfPbDJY/12h20u20GDPZqMtDfja5vri6EwHIq6Hh903avHrOoI6sXoYeET1iHl60o2cR0HRVh3v6VpE5hrMrdUCSyKXA+Z82FPIX4zjnsQWgPHBMXTUkimUVwpe2U+Vd5vWbKBWDlt80e9DsNuG8nMHpSKy/EIjHw+6SBA2pvBHcsoAK37/NUa9A1L5sawalM2dWtRZGHHZOdvpyvtfhIpiUdAqSS+L3+obZ9r+QIu6umpJ3ksZX3PDjoRCifvQBoIHGhmyMLYELw9I7CRvNPlGDVlINsv1TUyspTHq9yJPIUx0ESXDPNrKdqBh9Q4qfrwCK6k6CushtzwhfuKfFhalZYgJ5sCH8rQTmLgiQeWoieJEgawr407SvNvkBMJ/1YxCvAbZahdTC3VZTt9ac3txYHZYSCTB78mNOkCNXXa22M6u3o5sylQCnjoo+C3NOS1xteYQBQ9Rph00OBkvMGPUn2Qo37ZvQ13xrGbenyxHe6nHvdCcW8l2qAx8GFYs/5bPlD5dDBJxeE4dlghcSuKwHhHtuKD7AqzzetscJjE29bBr4V29HheO6LnMcfSNEr+A6POE8PgVGad68nQB6EzkRPktliO35YWab/9V4r4v//agRkl1Xl7278IGbvAiF3FfjiWx8yfwpM6js9Sho1Ue3dP8CfrEvSom1+Iim4/SjQI02MaitocvxxbxyT7cm34//7X+Mq17qX60N/iQSDF8KsfzIzgGnEKd1uYT4N8PkJo+/YFW0pHxdE0V1/c9iVJjI+Ltl7OqVQjuJTzwctylBBD9FalA9/7Ub+/VKmLrqFdQx5Cf/YcRhmJ5nPMnXmAJOvsDiiIwjj5lZ5dpekZfWZkLnwt+wQIWAZ+S6a2Q8VM/M0/a6k54Uw/Ct+NdkOAYbq2oHFyyvLarhPqrWdG++Ih2iYYsBQkiqWUsERnCjt1b9akVbNaGtppqCJUN61hjiFv2hwa6ep+8B84m+Bzt1zELr87OGI9Kzz14uNjcCGnH7md97whq3GcZjKGF7Gp6UoscWGof08DWgZeuixDwcswI/N+0HPl5W70Zo4NlI+1nIt60sgcsxVq907Ygz4aFaou92nRUtC0eB8zj2VPVGAHPNdXEWLTWjTzWe4NpcIzSga9OKUBw1ihA54DpdhfGzcIvjegKCvFr3W+Y/04L61VlG0UBAVb+zClGSeDqSduJQ6XLabkYnTVjfWq7Xt2El/pbwG7haQ3l57X9J9y0JCXUeMY1IwAAjjT0PbHcdefvs++vCB1m34Uor8mCltY89RKipW8BNdSW2v+VoWbGYQJhjw6sHmIayafHhkKqjivzQ+zbWxJ1eB2+l4V288QD1MUfiMHBRXv5OX6IEe1KhI+3csB7S9TfLVq8TclVuGiaNX47AtlCGq3Jkhqq5E2qsLL4kQTj0MKBYhA/hGIQvHjfVL22nY3dZQsRvRGSC/rCaq9ImriTslQ3mioSDlEqLHAnlsyQCjzeFoIlGsEXl3R8RO9f3+KNb1s8iczKFucL2TGKbiXWuegX+3x/qC89pr3XQJnyrktxZR4rOpAmriNmup2MaKsV6oJnm/Z1NxphgXu0NQzz9QSY7d2DES4MNmubc9pBJn87HuA8jZLDE4l5to3deDs7juEcpifO7EXDnI3i0hV3w0MiKw7CVGQBfXZGdJdpElvkJBIK6ML8qKOg21/a1kygac2WfRG6i5eW7azFACF77sP78cdHVxKQ0Xt4N+YxL/WQj9CD0EEpP+iX0QvDoXHOEl/UpkrREohKMOnaSq7jSPTqHDMsgHRqvtujjZOMM30o0hE6N/FuL35KvvA5owXnX7Os0mgMwOmj0Qzd/ToSwwvnpWipJAcSEfVIywa26K9kJ1RabTJvMvPdDwUqr3P/CzasaWqqW4LQU4h45oDC0/9sBErxI5V1wpRkyOU64M05fxhwbVImRWDAM7TfzsK+vL6ScyBjNplaCxzw8B/depwY9wB02FkK7z++sD59FXnFEmBKIo5P0ntZMYZB5mWCXO+WX2JGfrc5/k1RzRIglUHLoSiKOJ3c1JfPyWsmO+HqDQG/3nJAcPp1i4Z+vK2FZjVv8l9QKTef6OrDn4A8d5nAxBR+FRhAlkmlMdc1TUokGfEbX8f69Y957PBpPGdqvTz0/W0UrKTjIiGf5bMhEP9Qh8Y9x+G2zPIPcopxiQn/6SgdclYdJNce5azPj4Yq7ltswgU+ZdKNUF+JFIMoVOxoEn+rvgEjKuI570oKlNHs7k4UhFq33jCyjxBOfdfd+xaJTAVH0mlPUgUPa4FzaiP2iT8+2RFJkO5nykmDVgX4uW6AMP0OLB7vyO+3wdcIQmoqf1RlpevWMaXglVUvOi2lDr2xqDX04fTOjg3HCwpbnvE8G5X+5kUPZFvUD72JsnyTRKSgdV1LPhhYTaD/fUQtoeJUD7zqa4yiHZ+aDit+zxxVFUXr3/E8ozg7odJ3hcUrporlTi//YUH40o/+SD7vFxLCefA6gFz9343dCvvvdtcGfbVvTSLVN3R5RTy+bAVxrASRSm9YCNr74KUmcpmshuT1lWrPXi9UkRNd2+NI1cHPOSOK45RFAZdrtiDQTOQCmL4fuyzmsTGdrYnYARljuJAvrw85S8Z+wkj4cG0aRi/nikgo6UGc5+B1l9q5nriWBP6G+Qae5XmARMiCYjjlqcJU6g8Epst8xfb8jL5IrMVUFoS/aADl47oHxvoQSaCajqRYYj0YyzZi7OE1JpfqghaFOONqKSP7cXtrZE3VGk1aeRy+H1HOcHGEtoNrt7VJ9kK93145a4zX1PQ2PWj9SVFli+MpB9WqLKnLrhqdcg8KPtt+rMMIxNh7gRPgYezI0cZz+0O/6LOgHPQz/Kgrv/JJFWbAMG9T98NIT7JoMinAlXUD9AL8JLEQ1nioxyf7HjFc7Af2GBkVC//xJm3yiUea2RoMccOuYxYFtiIMV6lJlemt+GanZHe+/pI3GZiQwc6fd22wTO/Lziu+uyTroKeYRALFWL3lyHFC769KhywkoR79MThj2uUyMfNVPVZoUjYqnAK9FPNoM8h+3iCttDNXeqYBXBq7Hnr0Z5HzjKH0CRq5XHhUu2jiByN29xXXkN/DAsvAj9s8cWWMDSIquAVphxorW2K24X08gKTQvNYXoHcqkG7RbTfpBEoJOETEsZ4KhgwGBNSVvsbnoz5aE+xHcf6Gq9pgT9nZCF9U9Wt4dIp/11kuf6IYxdxtUJ5bjaDJL/6VxtZqzwm/jYfCnq+EzvBT86Pvn+yXSxFzi7rbQhPuHTk1bT5/EpGVztzc6n7JhvigpPwux5yjLgf9sIqLMWhgGDNPXRoPHrlq4JUhQsYr0MVjgZF0Z6cd9YghlxvoNUdzaC4aPsMHrp8IoGGtHqKxmNtZIxmZtdbMeyUyV3xcIjLXrSXo9BoacYQOTYq0YTSZh1o6v8cyFcKKT7x6qxftfbE1Ik4L2uDTd7q8XRdn7RFhDanTysat8yYIanaOCf0d9dIy3ze8Dh308XnZ+fqFnvZ+mBDBNGDGDGLqZXZPM4InXOqfl0t0UuNZ3woJyOHF0CTPWoP1JIG1fodf23qoxsA98JqFz019q6HeYwwaI2Q3zvT1IoU/fPW6Run6rlK8EbaztAlIJ+CvjEN/PeAhnGirYvevSw5Mgr7dFyxZ89jrJ9+q4rpSFc5F+Qbk8rkiO/nMBh+DzN2o69pglbzqsOBqn9AH2DmFTc08sYmpvF/P1cZRtpO2rTaDdEw7rP283UYTclICtcgklHRkSaD2IlCU1cslraqfiLfFyd6+9+/DMvOuZEpmcV/kjRTd+ogQmG9ymDQJohU2zyGUgLFiCWVRDT0IpWklv0lf8JAUNs//knAMgItN4O0nTEunexVuUD0cAjPL9sCBMJN/duhnBNaDmVm0PBqqE9JUk17yKRyqYpbh42nYPxia3tMFKE8O/L+KPCQmL95Xf/VO4Qql/0wGoOfYzfCX2UD5Ac+uUpbmt3LRxgjxjOK/xSZHNCS9c1SrMaH/1YV0b3vWFM6FN76lD6K4/3d7vevmFzfh5TeTaiJatsMFWb4bKfv31hwGIbt9jsGPlu+n907Px/t1GWbF7ymL+A+fbLJzIv0JMbqXDskKyG0QqaCME/WXZeEH2yFvi9dnJvw77wlLmyfjlmN/Xr5uqzIJ4wiH1PYD+7aVw3/sybmA8N+fWlTJbZTPDzy8Yp2UcADncPVhtcxj9mNWPpNHKBKUi2VZb5CsPysr9BVWSwnx/zlAyJ84fT8gcdvB0FDHg8fFAI01d/cXoI0a/ZLLVgKDnr1/fOhYCereEA2WukYazo0iwlvSxl08KDTefkHIKDMkiEdG2AUFmev9AbAhcr7eW2KX2Vw/9q2YzE3owcUj8uasniE9YZL3sTIZ4IoRG/dc1dfYubIPtGROKi9cJAIJAQtcpmh/a0D9CeHtkHuJ6l3YJSGKXTkX4b7Z/K0jrffam1QwM1aPLJ0HCjCuLJPuSZs5ai9JHF5RZR7JFGLXyQTWtyG3QO6CeXezpAZQw9AiVQSJ9cdBgNPk+uWixNRifiic8dh/T4rwlQ1MGwLv7iEf44Mi3zuVvKxGWHSASHzIsn0N0C1zfbPQrdlBtzeGeiNuLSD4ZXLN8029xnEiKiIfL47nBKa1P598yIx22GR3tgtGefXta0PI4Y0IOmUnBAMX/kkizi8piiFiVsR55meVPYnyYUz23uxu70UCpohW47mIbl2qSfcUGtbHGP6xBjpROEFKpQB2EcxZDqMR4b3HR7RFBxYzzrM0lp+pGGz9vofmbRKKTt6Yv/kuqX5kVd8+BxaqdQWbaaOn4Mk7zO0DZtePl3WR+LFSYBsz9pJaNc4jrJc8M5RHn/bPyhopV6yV9AFZjF0u7XqLPWdUptcEDY5sdsAT6HJfDUerxjkYteQu8SmQoRxPowlrIeDg8aGQVGsQUzl/ot3ylniyN4oRVKobImG7nj63cyhkVnaTq7QDJ/lV/po8zXRY8PgX0gGKTR8Rjjv/X+TeiqDvYfwq9lAgTKnSbcl368WfTzL31jeL2Pd4nYPvZ4KBtdAUYChsu0uf8+c2w0oV0J58Q9YlgDNLdqloRgtNGAIstK3XliEQ54PntQAm36+tmGAc1uZGntyYx3Q9uVopFECmkhaA4RZ1z4BQpBT51O3ziWT3qRov3rvGW1f7HGxdTAu6wTSorNfr+g3hOqp3lIzc6W/ZiHdRmIdLGs9OHmiW5beZPdx/BOPB/L9PYOgjJ/ik2PmV20PLV3aAv0r12BpKS/aIVpn51v6uOs5VSEtkC3tS3JvkpNPBDsY0BkmkLaI0bT7M0ifu6jmm98EOf0VRLD0xEapKcVX9wD8p08yN8aw1rDqooSyytrnTy110cQA8fcLJXku9UNnbNBgOIAeiW1DEPTH4r8dZHjFxiZqjZi5VaBpJdGeM37MMm5tQbOU83Mai8NFgcbrCadvz6SZHWQi5vUNMMW7vWs/slIz4Sep/vrTGLTk794PwOMWKUIfcX+6JBtlCtxgbm8YFqL+O9cDJW3wrtOL7pfHiF6WrVI4PaqX/jnvE/6TTuzNxbj2kty/M3voPCO8eK93buOnT61z5DeL/w4wqUPeLSWi/Bz3CjEEpEAj10rCq+oJSEUb/G/C82qJ9auZKW6UteG3NHRP+mVuTVNxfaR01/mtv8YLkJ/zkViFJx4vSEihKhLhHj+GRrYWSX+RgMGoZ6dwkriOly3V3E5BD9Hs0t+eYH0HFKZ4cvDTePqTetpWmOF+oORjPEbmrubTf2MOVUBDfSGFRH4IBlC19ux50j+Z8cWo5Fl3HpBTY35K7HwpHHjZAsVdIZs3r7lWj3osxPF1hdQVdLVGi7gd1LBHR9RCfwO4YuRjqXT/PvbvTqjk+OqD2HpbzEbEz/WHyrLg7hnZCmJnKnNrxPyW1YO9trUEvMsl4dx9Obf4G2AjivOHrxqvfHql+8SHL91Oz9nhbPJJNRkpGOpKQJpy/lZnKELDDW79e7zSKTbjtiGuZLAfETRZVWPGHkcRBvKGk+pcSt/0OMSKdHW6oF8wDEdXzJdC4inkmcJEN9f7KPmemrwEpExDGbgH8btJ7JgP1vZtNRxF5XyjTaXwYV69YKQ5glZyY/lv36egeNeSzFLCdtAwVQrJxy6Us7U+vAwyEIo7tWJt/kZOjbp5Aul2UE4jQ7pb9OOal+D2MoPPS8J9059xaBTGwROVZZT6t0JiZ6TsHzSJXPVV3Kl6HpjKUvbpN4nEFREeLkcjPu0hhdwORk+YqaqNvyqXMr0onRgVEuYSz+kcCkaSVv6n/ebrAns3OzMNV3QQzMhVoefIpz/1+c11TJMds0b3Ou1O8nQVujeJi8IevSTm9kI7r+ggJn2/80c4Q2noL90hGZIML63/uUW7fbNMtn7L3KM4jLD8i/N4nnegX9KbLIAyhsYdt90suE43AeI6trD5cJXK2mISlpC3UVi1c7ko9mSy03whgoBPeYGL60uMfPrCOigcuNxEEthVnBOjYV1ZOanikKE5QmEwkKE8vEnKKzNOfi9bnfLD2U7SIZiQwMzBlkv3VsvExXs4InzEcVdc3xhJCWAcWwWIbM9UsgR86B5h8woD8X035vxfB1ruUb75qqgtEB0H0jmgoVP4jjN+DeNrEup0YbgDah4f4hVIgVHe98oiqLJslOtJyrZA8J6DNXRC8dXCbJPpmujOw/iy4DBqprpD7mzMsAwFyDWzv31QtkFLNF+1lc3GoF9ionVq/fKHeivuZiIvyxXWzgHrdgWnJ3A3PIzYHPd89UHffXp7OEqLS+UkMhW+3BCk/ep8Qqy49rgzeWkBYYVIo6dyozAaVO4a+EmI/gzCTR7q5eAf+nWbQ/MCcW1fjyUg6X1MLGnUxbZRgtpDR7Iq+tUb0PYhrcvWsvVsjybToU3D0JZxb6FFHyCPLVInCt3j5UwsvhSFLwQeH2000HgFE+ZQ2iZVg7h7/mbQ0ix0eJh2B1HluTtYdg//HUur1qNPaEV5lgg6uQcHinZy9TgSR89NIOqyZxAC6xdf0O/Uf/5W8I0kSFx8a20mFBxamExuNPzSpBQ9ava0L1o2Y986JfV0VYb6IcFY7K88NZTtxjRFsgzZGZZdvtqvXvxEbbr5skE5dlBe86ZWMaa+JfT7Ftam8BABNUcdpXM7Imc7//bP86AnnDCOnkV3q3mvWSZy0tTxe2W8IUACAwphYZw9R/5x/h/WocTjswWrrRO2VLL+Oj0d9PSMqGfqIhS5IwgUmoedrPng85hq1zJxsS4W88j29LHwwJ0dNBGgGZLSaIS5sS92gvB8lkLzd9BAO8/EjaxaOOxk+beOJUnRgS6PB0O/soTOpLlzzC1nedm1UZb9tnpQJkmK0Dd6R8tMo8i/kTZmqjNts7LwpuZqPfcgv8z2rO3oZQqAL6n6FsJ8MoH9JLMRRX+nID+x9nJKHgeIBYjbTVMMZybcKHEMcvo5ko5hGGvVt2l7btRq1uSz4IO015R0bM8imj2a5bOuGDEqK0hRxKhZ5BCIwxgWHhSHtej9FWH/S0YDnPthJ4kdYuVetEulQh4JhZrs/T2xP1J4TMCV/Swafg+92OQ4TN+KKoZv69BK6cBwX4t2yN8MVFSfTNDixylYBeEwz5rjr8Su0F8mlyZ022JAN2+WOe1HBpSfwF/7/xcoOtvUyactFi2bOPMN2HUeiLleaOSejCwnyMe6fWUkMj5SOMfSztEEBS4RWL8wyNiWJtEFD179v+x+mxTI+a3caRe16v6h49+evtdJU120072ap5npLajoxFx2C+BS/nX+0U5ATMerd7TDvi56ZmCaauADqvNvTmpGmlgi4GyfoyzbBV+/IGiEfga06xttzggBMjPSzTEBFHgt/op8f4ns0DD0FmZ7KI2P+ghqCwvSaGRoTUdAqkSMcO6kXq3EIKMTFzKUbKDEzBX8crIhKUpM+5KnO6QGpvJWzGhHm3hECEUdm7IO7WnwdHocM0dwdhbPDsZyl27LMknpD4spJoqGv86Zq+WyXl+IyUllC53/BoCyg91yVxeGcDpjYcVhiv6QNDMqAoNgTax8GF3U2Du4kAphchQFlCRfyMc0yjb1oaLQRQvxcfvox780uQRFkzWnmK4wXKeFOHgiNF6gLYKOe7K2ys8tvVV1za+eyDoSNOLfpJgeqGGAGejsZiAtrD+XGaJGz/01m1OYYWgx2LDzVy+TnrosKF2k8Py9q+zlttVQfGXVJrVAUxVd/7pGVuL3Yw+5XlZYZsL1awmHo4CbKbHSrSbTNJeJK0Idgy5L81L1fEZkMtGNdbiC662nFMM9w8bBEuE89JUBhbYzy/52kakMd4LyME0NXqvFbgWKj2buqYhMkx/p8hQa0/oQlGz6tczo8eejI5dHOt7shik9mtAd3O5jHOfIswRH2QRlGQg8qVP9nc+qioYkoXXPPZE0mpsmPub2ERAqNTEdK/Z/wDJAuIsPLTZFghE+yxhQ6vqsC+Imvy3CVBkJv1W4JIWsxZR6EiNM04gtSnVYLBaEjCx5tTI4bzuMFBDSyHJ8OOnmJhV2Te2ri95ZguNX8yxq4NS3azXCxwB92IwqeOsc8SzF8L+JcvBOmi90zck0GfwrUFrWSdGn0ArhAFeuCAKFK0VbeKxway98ygFQVipJQEcA8lKUNT1yPWU/TcFNH2ANyLxl6JMZ9tVKWvedh1v5SHatXSA+C0kGvpd7AaYcw3spIhiPsZS6/+ZYzeMh4PPKSKL6TFK2oYu3fC8sgRodISXAEEKQWHSETU9jeFe+/V7mMxBtGWItpKNy2e94WvoxufRHUbCK3SZUeAz6Gb/4gST8MWIDTVf7Zdv43VAP2z7N9w3Ef/NgXFBUJ99h1Hcd6ldsH7vgXlrpUHm2wQaKona5v9hR3x2GP6AVn8wjPNcw1wlYubHJEwddV6dOkc0rcHIOuiidzC5VbQPh5JhAiazY5QCDgr0s51PyPkuwbdi5J+jkNXwxdmHCV2QhYLrf7y9pHc59DvLe5Shx1JU3jUfQ1CFLa2EHn0+TgvAwJN8W92hwHIUSj6Ienw2WbqXZzkpSSAayB783AAGsKMl823nhwu9IG8ix+YEDlSfTjUNvCLphlL9mL73RtyJaQf658bim1IU1S+6/mq3CZPhCGwlEccGxhNM3/nJZzlYlbsk0HX7RkGU2JkgVwToCnG0c44VyjSqyK05xVd/lUJ7rpVp6zWGiv4lk2DT9N8vAFTCJYEM1CUActJEJctAMX5QF93eudn+UnAf+QKcOJ+UTnsIZzlWw4hshr8fJFaMF8CuwE+6Ko0XDqLpCUA2N8em5IucG9RYQcYCVKeQVoW/w9GT8Y3SlzIlDrcL73doDSA4y/AT8mOev0iECr8RXcAofc0j3TXDXI4fk+ctKNojzIj28iFN/wHi0cv3YyuRGjoVBfPqDELb95+v6FGdowqq1UYpU88uUT1fNeMD28i82yR5FlG6bmoXSKDq2QamCpYN7dp9Ly5kPyAXsap6fH0sza9k8wPMh0YqCSvg6NGVMOTlUfpDp/bfijziKV9CD0CX+jJEgXleW23YV+fhUjAQlrAQjPh+xABfetpMu4VzfieeCORjHNXXymtCEloCmdfpBKVbrW+46/uZKq74t1lCix1SVyiqmDQVjz0Nmx9ez4xVh9lK84mv/OwY3xAWDd1ImuVG6X8VLfmT2Y/fYPq58+LIji0zSuDgpoxeqCMIHID80MRf4v86dK7Lk1nXDBb/uc71H736CURtKJKmiFLR070ypA1aAkeCXGs9dp0UwtU8rsWhb5TqybVj4bLbIKtgdkqhjZN2IqXAFWTt7Z1apZo1MiuNAuW2i4S66h/vxq2XNXHOZOr4zhpDoi+HIG6Pj3/TXWd5aJQudhi4W/tjRE+zjlPtsRvg9oeOhOU70W+uRiiQcr4hk2D+z5aCQHKt4/H1g66AJbjkNq2aDUt7OL3GhJYYAii8wvP9Y0r7W64tNxmfCk29wvFFoN4I8L+auqLVTJwXj4zlKd5gCeddSlH90qRIkNywXERZNQudnrYkS/zfF/dlTK2L87VBPsgy3YYlcS885F+2RXsWxITDcWtmRIN1Xh62q2ra/Uecw7HNiFkEGpPzbO8TmtZGcJIAsReehXSOG25FYX5Z3b1NOrBlOvwl6pmlvSJm+IDlllEY5fP+zRbhke5Buv/xCj4IBPxiah1HJMbT+MTiz/Hc88ksxtvRbY+jsh00xhgmCVmGZ6OmFK4XCaaM+PCcVJ+GXaEjA2p6nXgaS4QZn+fACjC+KMV6L+T4yClPw80AMoPfv7++njj9NK7hDTXVqYu2K0YMH1pRjZudoBtwN6ZDEYL5FArYoNmr3H4/ceXTa9N9U4VZAvb/z333qLHf+NToYNF0MlI2d+i4LBoMiFqEBjJgsiAqO6v6CBk6OVF6JpPIMfU4FuXXUoUQrVpYjsNlPO9Ab6CLC9iIlmJ/as5SLmWqdun8HoD9f8ALCqsaKIBcVQM/MscnMTteRVUYXBWoEqFVhv4EhjNNvnEnQ0ppO3eXCJ2GZo/xx48sMp+avqKlZBeHXvdw3fB6ajc4VEZbJBhgoY4Sj3wCPN5qQe+mrsXvJZ1lnPlmpuoU2RmhIvmNfSldKFsXdZx9No4B88nroWfEjV+orh8gKYCAFMkyRiRIrHJF70GYQHzisrE3spUMgy76We/LX7Q5Vt/Q+V2B00lZXA8hVD98PeM5f0gzE0WSD5YeWd6VXFr/WAaJMesLzKyd9xZ9OlQnBIwAKQx7gDe8SdU9mzXtnf+dhTobiWZmKVmyLEDROvzahp+Ibz3Eb32//gT+1TFGkOaoqeuKMq9frxmdDOK/kq1ZmbMHzU3ty3hU41ILFHxk54jA1xYyGSahi+Q+8D41HUpPApM66yUIsyo7gxHuD+mfkJ9Ly2snU63l8A3j7FykRI1NBXm3+zY4cbKbvLVb0qbL+6H9AQA5DlFD5WIcIUgqxvj+oqtvCEQwp1brnLJjCSQXtYbZ/e9pByoloOyEu17rLS59MRycQlAizHfcRu89U1PV3Uy4WIae7XnrfYNmTGe9yv0FJ6hoLOv+GBB5K0Q2rvadglvZ8nKzxrmKrSZuWfOtjVKkyy9CQsbim2yF+IfNYKwB46ToT0kc6dil2vtQBiZz+vYfahgmJ9tBrPeqXYKeftfGCZ/kFBnvQ4F1TjpMefV16dziYTmtUL/AeUihks1Am1A+0OOlnk9zU6e7ZQMJf7Ps+gQjDqOWztgsfOcMy/u6r/fy1JVDzlHbUf+djmeGlcbD0QnYvWTQK60TW6vG8yFwqnLZlMqfUXqVCM2h4zt6I6GtgMr0SIpv5A3OJrXsEAfJ7X8yVB5D52+I1aVeQo6RMhfDq5w2ahFMeILaUZvnXyIe5Md3MI4kyRI978cXZZLFEDLqmYpeNNlvEK4uAbI+kWavumK3HWxamAe5IjJf+M3eNW1FCSJPOhJqKJwLstbI/1RW+NMFTJMtyVfmU+jPgG1MlkOvJ53JBTZWrin6jACq1naTtLM1lWc4cjbkwxPEFzeShRWvd5xX0Jws/F8hoYDTo6+T/iD0ZQvYhYr5S1YeUkyhku6mEzzRuO5ErZ9Y+KXybg2TqjL4SxyWF31BdwwfTxTbDdePZjstdb6Mqc0XH+vYP0/4H+5Q2j9JFFlTx2li1W9l5W/NTVFfzQ2P976zlsbNqtFRO61ehVqR9JdbK6b+28U86p9sNvhnvMl6foHqub0zo/PGTGylOShAFsQoukdV8OzbaxeQgjNBdJ3yCBl92iMXZ64IZ16VgntddyE6KHb6s7IBmVpe9pOrypdeRUAqnVJdqly+adZr8g+FJca+qyzKTHE2sl/zas2iQl0NdvC/4/K9sOr50IaugDE1FCa9ZOFQGFoHkrPkbBkyMFu1UU8W8GEpdVwqilzGmb+CtHfw0jZBtkDMCRJTJagwL8YJLxeUw4R84jqanN7i6weggOR6yqGW7a6mpR+mTOM4gsSW4P/kIq2x4jpTyGRwEI2ABO4ycAH/N1Cw9nhRmeC26uviM/d/2aWSnPA1vRXsxZR40P+S//OUJHPjOifFr8ntBfFIkxL+rxelZhRSuHo+0onhbzzIGHcbnxGxam6d5Oc0p36vRxfJSMi9cDAsi29rPV6QKZFwCn4Np8Wf9NX7ryPNNWzExFvGDI+wHGzIxXZUjjmOjPb8NlCKDQaYeZCl22BGXnZ06Pr+4x+sggocYpuc9dZcpFcbey2Sk1b03IlC8RhJgs03m2jFDWcmU2J8INul2P57KuidSBk6nSjAmQMLqfvRRDxJ0qTcEF0DFZ4dRLKdQQAC2bDSRRYdCQQF/WaMlFSR+zP0/Pj3a+y6pjJsl0lIlTr9LCZQx2OcqCe+AK/xeVcBZB8su0/MDHS4RYRd9Y0/ICcfR9UK/2ZABId/roS9Wgl2lXfWF+uuw/o3mph8EUgjwdZ352JiFRtLfCkTDk4+H4VBtMMxiMyO0Y1VG0WVeRsSp+TBOJCT5dxiAWYZQZn6FkoyaWe0172o9MgidQiFX7HhD5QuTmYPegbgrQpLceCWYE1zzEz9fF/LFnv7SpW2Vb1PkIfep8YVXGdDboHJ0T+sP6qObUNxD9qpXfD8r3VmNbd9pALsFpTgj+iuLrVF2R7bns5ebj+RyguKTQeM7oQnPv8uerMACrCcIyFDszqKK+xR8hUMjiTa8brcbIIfen+cZfgpI4rorO0QItp9DkgpU+Uifu8gyFMeWMBGXajFwoizoND23H02hbYKs2RJelDC8cDU/WHVgDTie4kUV11lonjd/cqLwqZkohVYJYfQLCL+io/AjUYYMYxpjDemkfhIFwE70R/EGk55cwsinMN9MGIZ0m2MaghQR5U9CPrTr+zNhn50coAKqPnO2lmO4F6uHIvVTdDgOLeKXD7bPecBcqBDtRBRyUjGxFuFkxobN/P/c/dfWrDqTLYo+TV1WNby5xEOSmMTDHZB4783TH5RzzqrfrVpt77Nq79POaN8Y30BJCiFF9OghhRRe77QuLmrIxGOv9WoVTIeO6+z08F11S/CmVhpbA3Q3ZHNffueSijqM76a3Bus8dGPQVXXC+HGRT4D2gFyGok/QW5+RcLFcvZbVtMRx8ADNTZqTUhhc50pBgYZiAeWZ5SF/v6x5icBZF1t1rbgHytGGqVC5/0P6cxXMZBLno6kbYpvwh6WM/s8YJgwi3lNqJhQanfcudTz5UcJj1H/OYihgx2OemiSjNE53LANB5ZL3oVHIQEqygZ5PyoxBg/UGAkSOt/KQ+mps8gpsrqV0k0S1a3K2/U39YvWlfToeasYviYdPxUZO9976CtsQjZAx5/cLlX+0uNnvAuDroxR2mZlRltGTA8Ax6mhP9LgSB2Epb53k3YZ6lyk1P6RILLD80uMxQ3jsM4WK95YlJEKa/O1uXAgDa7rEuGwJGyu/u93UDeQDXvNkLEs06y69NkacnFbkxe3x5AMkm+kv6eKy5gtk4GLWVkte1h7b6l8y6tOEDhLvsTTWeQRwaA7c8cHc9JzJamrCqQAMtTi0ue2+iGn9TcFv47651vlaxlC2H6RbHiYkSNjHonGF5BdszX5a3XNMQeyVF9mEaFFRn7sfD5NzOo5JFoGJro8ruOPpeah0sziL46t/ITsM6hpsb8iTdT2u3459BPE/Tna/DWvdaFYnh63Jau1zN4/527hXPm6fdzA3uXvnxwQ6nFBoBszGN++z42+Fsd+aUV8YdWGxd6OOOJUPDKvuvRB220fCoMFefrDW6zhVGiz5B6kaMnz0Na8+nT53CThx+f4Y45GbYmh26PRt4d/Z6QXhl/7Gz0QQ+fLjlg78mxEp8WZCfR51wb6VMwrEqXvrd/KIwDCmI2Gqb96IM1zt1+8IX56OwTN77kRK386yw5kFd3C29I+rtrdS/TJG/eCg+dRpsADAvunOHuwYzF/q2vvQMS4AuFwOuWYpaKbLc3QIOrMHy10kdmdswew8GHBE26rPogspHatWknfPInC4SK3p109CUDnHwF1xJalnD/C1wYr+EDZ4hyWp8Vob4oiVsnja+61DZJ4YPKgpNcqjFtkxDeQdTFq+H1lbUdd8rtstDcqkBgZjNY7+whFNSF46RV+h9sYioe+XyiF10nFC6S1TuYfpcTmhtd3B6XAaIkoaMbWVdc+nVr3dcO/MmEfQ/nlZq7Y+RuCPVe5FbfL62vL1+kXdG8vmpg9Dd1J4/YTUyfYUrfPQIOj65sRNrjr6EOWDCCXr4ab5gunBd9q41b6hJnWqk0RDWYxinu5fOgTDsw6OxhWFw51OxBbZ4hDFuOA+o7d2IhAS1PdV25pvoXq8eyvM19XzPOl4TVMrbeY3lfjA4zs3Zm31EB+XNWNfL03QQhHplJLIJNod3voAFsYQMwLnSrM6VrlFvEY8F4nqWVf3mLC/lCIxNPtWkisnx6Pc5IIYjc1WwPxAnFARLNXVRS/n5gq13rnfcf34AoibEN0JOdTF5i3aKRwQt7EW65wz4ncWYubmFqTna9+TvngcaNDyqudl5AMdnAPMSjtdiigyDsvL9124j0OQgk4MlyKsxRLnJu19BkqIzL/QT/zqPttjMH4NA5i83zPmgEOrxGTOTpqYgLxqt3SqMvHKdOXsPaOIGy4EtlZvlYfkXVKvvX7nmF4dKxya6y0Dxm3hLgic2WtpbbVgC4CC6tqFSWoXwapItOeboVmPgES5Di4lB6FPi1XTvQUwbEEP+nk9CS6tt18dc5liSAxrnPPUN+hJKIHJtFcL1ArRh2NXrq8twbrdzzkciGBdf/hW8Rbe4+VVHMdw8zQN7T5d03bs6L2+04YQv0PJDNCZxFQ3PQgWHzGRsdkj9VOPm6F9zYTpPHbp/maA5YpccDvvCCG5rzx7AvZKSTC7Isdq5bUj8Dpdo/oE8spO+athssw7UBEwZ/irlKnEJg0UB3sd2ZX8Ahs6YnAmgygtslqHWUmUJ/meWmRUmS/LTwfXoJXdHEc/QZUkBQPUTucIndMtVcMDxBVfdp9Jqaih28qyqWHK54pTXqINQeD54IXDbBvD+tRQUMFyuJt2+Pk6H9FnjtQ6vIVEJXFfX2TEUVuf48vXFctaih5/gD1NBBPvASVZq5+dg+pzsY+4MhtTSTQek18NUxhVSs9FtMC0cWK33zkivrWdGIR/d0zycTBSHr+htmsPtUbZh+R3YVrnNeLdFetm2/xLS7W8pR7RKLgw5kE/dDFMwcSPoK8qscXzwzRRwReFcHRk3n0JiNzKW24hF0JgK5wsLHW5jiQmJoq/8IKpeJUmULuJGMe2xYCbBTCNSNMK3H6qNwyVwyVAHtvEU8O0rB/FcwqNJUjgytIP9k6c5RQuXlNT2MXSEh+X06St3ccjik7645PsBByP9ck7srVsFnFDX0nXDc2nb87Povc33JOdyZHdE3OrfCuqdkAf87R35NHOj4Q78reHHtIXV4baAPmGPleLkYalt/s3w/K1/nw078SpNPnAExtVFH3TqLNse5T6e1EYu6zgBs/h8xa+Y+XmqdJA4O5qoVo30UTTgUlrU618l87X7Di4Jd5kOn7lMh1BgB59vDYlszqaYsfuarINu5I3Hc0eKn+G97WVtjCHNleUYl/DjerwOsjq0pqaKi5zEXtOKDe8jud4KLHbez0ilRayIc45m5aDQZc0OFCmLeJwV8WWmPHOJHp3oxu0w51s6H4oYkZyviwjQ9LNyjKU1+N747Opde8Z8JK9Af9yE/rRAMs6VTCViB/temvXO1eHVvxk6Tmk7RjjAKSasc1DdC9H1rFnGYeVwUUebnWJY+IIL5OfJg//SHVdDHFc1MHXqVCorQlu7bz5+epnRibt8mldXE7ajsXQiiMIE7SAl95j3bO7PqZDJp9Y630GZKuv7wrVGLbJ1enuvDsT6DszR+miGsRedtTmmirEK2nN9W+6L2XEIAV+uGdVs6wZyfjDq35rKRSON9a0KLXqJLaN8mAp8i385nF2qHuIjE1CjI7dd0IuNSq/UdONnA/O1FYMHFmMKzKLyJjF6KL4IFy5R1T0fny0obULZ2H7vR7pItLrKQrx/W00x8M2n++97qoRP1LwKHsdvGv/a7bRPGH7+frs0DE303dMs4XphlizzaHcxTRcYkdzKGfx/LJoyLc3PqjXKLaF6e9Phw2oXZ/J/g3M3Qno37kmAaaR0UzTZIYwlKOGyfJHqiS29XeYXfQJXusrkOSZNc0eyd0xDCElyXYbtycCEPgLI2jjXcOeVn6K2hrO/n2NS1BH0MwClM+BfcG+xBBm105x5H0wZb2wbEvwvGYA9DUsFm1Kfk4C6ZIE68G55asCB5ky1VkD1mqevREon+UTy1TnEnRb45JGbWZ7r8w/ire4zyYpLhzTm1GkZM6qzm+v8GsOLBcZCrKASacsNEC+L/YiPq7UNLTamF83B1OJAv9KVT40reDxnUCot3vjMpkx+T4ujz6uZwnmTZivsb+D2gcOSZtcBPTLldR/XS9Qr0fgRTRTEI1N0aLjqTaPSA0MojWImNCebYOgifvSaup8Q/fng1ONKzF8558lJ/fqRNQcOXyQh1KtYB5t6cUGwYdEGx5oGzL6oQ7HlWDbLMmGycF29wVHC7Az06jpJN1s/2sJ/UDrjiJYGPq9GEKcqzj4q1jHjpYHuLVheNrkAz/kFHnJHffnmmhClzhBoJr68kn4qp0RFOLrzJdGb/I1ROFwOGFCbl7DXZJysM/IBawqccG7NhOto34NSgoxyjAchzTNAOezT7fIuMncy5o4g+uYSLaXWvi4nxivFJ0wjLrycWKJWYL+jrGxA3MuXgG4xmiJvqKe/fcTbgN190YnrYwE1wV4w9TIRAVm21KKdlEvVEqAs86T1EnwJEQxk+iD1AwXirPnytKmdsQwJOVkhLydtRySl50JJnlev8wZE0MP3mtyGk5dVbclrlztR0f9CJc+Syt8E9URQbAYpezXvVcQQgtGG92vYFEowwWRYLxzHB+DJuF055x23aWchCMQxvwRu830kUbzkvyer3YlY/JrK295CPYleaCt/jgMmNKzWYavspWGyZ4kiUyN26+vDRcV2qRLkR43AzitTp+zV8keCxVi4z2cwBS8NU5v85VYLswai9KEiOctTDjNRem8u+g1IUsqadSAQAlsqKqsf16DPFfSw1TZ98LHnpgVj/MaEc16VNGFrBiHMrrOZlSZNtBu/dISiq3fmEunbqUuXR+79DhVQVrxAkBOswX7HiwrZI1GBU5sn/CkXHaC+JakF6zkXfmRJZcot7izdOibrZgVnyliYGcYJm1L7w4D2WMCWSF+GfCaD48PSO8oSRr0DsFooopTFYFQLC2rVihUEb4rnBkcJh5XfIF2n5UkgXbnc0N+llUl7T5jm52bfwEqJsgUFMRH6pH3eSOLQd1Jgen5ywKBXyxTOccYua1lVlujDRUOM+4ugNznkwBcEEn2rOiko+69LhmnWMwvrLj/9iaDxQCJ1sJfqGLMmO0LUYGwPl3XIt29vlCr1MniXssK/SVdRT+YpZ+fkJN3zlPLccUef3sz3jQ3lMltbLe/FIj/ZtrX5wJzGDM4EkcsrDxCN6c2JHLt2izl8uy4I2mfO+BriyRpCQbSQQEfICBgWwHesskW5cM70+hoxpR/K7wJt7y/sH554aV/mi2QpbVmvKNG+k10SLRMMuOXaRaKQ/ZzHhTDZA5rUu0dU1FM2Fs7D7R280a5nI2FD4QIYBT1XHMn5zdiN0EgZ19t/LxBTJRyInHDGNTKBLJYaQzr49awwozUYRlRB6kw8ymGSnFQBSdpk2/X9a3pm1+kz5rtbKUJhpiq+fkUq90Jn881b9/Fkn2i65Mjo4m82Cp3v/ov/2Kr2zTcIis8QQgFSccx/l1YRTRARbn94vryzMhvJ6PxjS2lB8hkrTgb2PJU6ZDVL3QRxOt13qfllYcFo537uCsgZRarr5JBFPm+T+u6A08lVWNpy0vs0UgAy6mJiVhgAujF5NLWCbbYFpU3o3mDzm0ppgWzmz9monakYFwN14d40NgHtvbAOz51EEsOlmtiiH2QfE2u5tB3vE7zh4drRT+mN9oVcc4MVR3tE3yX/u2dTF+XU7jFTOGnlzwzoKH+NzRVdDNf3HkrcswQgEfWXHRoEvmMJw0l4ZhJ/G1r4REqf+QoWy4SPSUE6ava4OwueKnmzlh7LGFQlexr3+9r9rgPV7P9zAJJkvou5/KXqhwitKJq3Nws295a/nIWOUnyaOxk+Z183af6OOzNC9fi0GDeJ8x0WfYyQC+lYAFFghDJuhew5439BIwP3CcvsVr/O2dMTVpNEuHEqdaJMdrWCe0MMlDqED3uUumVLuqOXyF2qJPaR41AF20KkkiBQMUamvEllA4XL2f4EWYT7uXLWOPKR8kY8iMek48CB1lO+sEa32qhfOkGhkLa3x6rCsV4EX4EWuM5cdNgMu76kcQRfc9eiHTsa/ReY4maEG8amXiJRwjb75vv3eMNiNOU5CojINwvjqCrTRz/ZboWmxPXXUPokXVMoW1wiLXvSMexI920GwR5tXsnGsv4BW4OiR8Y3Q+CvHCv1L+kwBlbq7fKKYFZNZx+QQXNVW9dMqMPm6Zl+k1PuGh5JGy/hxttiXoavnhLtDn/HkvvrdDal713ww9Ceay7IIM0/EgRfNqoocDFE6zCm0HPieHDONA6igXtIkhTv+mD0UsYQZB1ow3mgLoPsGOBA58TxpwSM6+aRzzoFhyaGeY1xrP29nJgfpBwVEizHHrl176d+yzeZTmWOZbYFNtW2OFmR2BXFuUrL/vPExWre1KUiKor7dViLbl/T3QCs0tbeEV1m7w3NUMAslOf0Nw83lY7qxZxVBbu84uZ2QvMCz8j1+QKF/ZMQe1JwOGycnxM+0GoLhiS2+Xh90FlR7XMYDGDLYTziirU4tpLtm7eB248ipdZ/oZhuDj3LCdOLPNkWQSh9dRqa2PfGKjdjr8ThAm5zjCyEByQO0P6HSgto8vHSo7EtalbSElFyb/Etsjv2hPQzP6q8pquFETbbHYJg+6IplU7y40R75qwunPVzLFPpli6d+Qxz/3d+1zHS5pMJekA4uZ5XnH5G36b+6lOgBedXsWFKlVp0+u6Ckd+dYvTIJjaXCPjtzyeie9M5MaiaTDI94WsLPpKp7NuI36H1BUl2V0JzQ44zh+uAe+/bbiF7L9Wfpi0JQngH71Zg+B3jHX2BREorC9n/MeXw1KVsomRNa6NoMp+bO3dodt1XExiRKKCHmg9XB7j0hSaaagrCxfRvDcmf1GbETr2uehoGtH2PUmFz+yzYhy2yS8sDBlbmSnUR8RNnwJ22Vr2GqxLviZG0sZQeh2yH+gu84XfnzHqz5iXQjLlaozxLt38THohmK95QMG8XBgDwL7IE8pOvH+TSYa+Ms/1NluGTI2r5cV0FNy8ze/MAIC75rlo+iBg2c2cOv86ROWY4qS1HZI1J6E8ScrGTn6DtAP7VHP1baJXoe7h+FiP9ri5O6Med9BYucbg4h13JROa9ocwCAEELy7ffDD15sl7dYyrvAsnJ4qTBHsWg72zs9F3o0+1vdb17uTkdy7QmhurpSdJi8a+e9hch7ijBfGVMGnS1Ktw+wK7DgbpPUFB1/WfdSo9yTPNBMRzFROZJCuMyA+BSlXDR45MX+mjgNhUB7EX7FgRWWsarPYluS0jfZ/SPO6MBQJoHo4+fnSBaYlX567qYQ5tzdm3S48RiwIkglJUrjnprl2fYgtsQgWTjDYVSyopPr3Hj06uyeOm8kVTVw4cZmsS0ZkD4S7vbm5LGTZZRy456zpecbNOYfAu7sB/TfB3N72PZnYyWnNwaEPZ47M6BDXr9rZJRDDNaPHVEI/pSo2PRiIjlvm+XwP8gbdxtnofdr4XvIZnRD7WdZqJjr815gZNIPenyWce/1ZPIkbRaED6TUZQesoI7vfrHged1WCIEdYE7/dY8xCin7w5cTrtG7069L5bTd4gd4XOExJ1PJ64bh3P8q37305+3OZW7cyIgPk0eDq+4GfUOD6ype3z46ykZxVIb15g5uZs2Vr70KkmiQqHRtI1CGB7qQLZ0mDsAeGVBa1J+yfQWnvAXAvEnORf3v5iwkw3V/LbzQ4ZJiIJLvaBQnKzB/5OXLcJ0UbtUaRVpmZ1bdMTqlrCSgaf4cK715S2gz4/RlehK03+dIr85aYu9aHMMNsodwQlY9mALUSTLY0QxTuLwQ91sOX+apnHPEfzoh+0U0WdHimlsDtSMy3yq52acpIO35rLTJz9k550zf/0RqV9kJaewhWTWrMpR+MX0/DL3kXodi/c2miX4ksYm1kqNV1c7rtZjPwtGcXFRKpkRHCHpAmq2487+0qSwJa+b6wZQizCiqBA43Afq6zU4JF2xndfOLwdRsp53Cc7jE6kt/qKQMY8MCv/aoRkj0JOpyHtGeHToctGJpQN7v2Msuhh/SRJ17TxCwBrZKb+YLJCf7l1x6gzWMRKJYjBB2wkX8u6mejhD3wvbHa5y2kysEb/CtkqIXKJDB8uiMWYhkOPVg46mEuYwqdlRkzAt9FGEaR6ieuRqBY6w3ApfrbFshHGx0JTsbwsojRtQj6QsKEAmTi0q9i7G307s2sqFiJoXjHOdqlmYA1SiCuFYmIomftvUGLnStW28EvF60QhOMgYTKakdf5mV+17MJ3F4Xe53sHHHq5Mi2MvkZrE9yvg+CuD/Hg+0IAXMC2HhbdU76rVldp9K1f8yEWHLmQ2ZZPUf+434fMhKRARC9YxDYj8sC0l6VUialzse/UcfFynnhKr24qzHINPCjMcUmmEMab10TNJ+N0/8rQtkRWFzvKam0a0At0cWhyfX8FjCHi27q1YjixpEF2WnyBY4fyy1Wh6GSrgepWLPn4hmi9T3T0cmVjSwpVx3U6kZY48cRlZ/rz74pqgKC2ts8VOw0K2E+0NY9Wt8sNmgFCoq63qaa4rHSIhgs94q2moU6v154cXZM++5DVQ8+EiB80N3PZONEIOr7c8zdDDS9zpa/HzsIwRphlrBhZa3tFrVXfI/9gR4TJQCT8u3RZzDe0ow2VXNbVZErTEwYrNuFzF6Fc0JXq+lcYg9SuJTS53mEOdeyLCbwuf7JUiQlE4eezohlU4tX4rCLxbClRmeLEj0aVn4DuntnW2IPTdsvj4UCMtvqe2M6VW3izkYRWrcCTawcuEhoqFT7aZlG3LSpv+9p7ggx2aMySgXEc5XhLoRIFDSlNfvSXW9P3WtM9C3ZGdqiz+mxz6KjWtkCLcYVpVk0mjUfTIT6rK7nBr3ZZf8MHKdl98nHLbGgeDbBwXSZcJLRBeG+Wkuu71hGfrE5ScrIWW7vlQZ3BElJTiW5y0EaP9R4UaocwKY0AffuCctZ5djx/n1tDdSWrI/VYXYFjD6KrpYtgo4Hmkn/qDfNWQ5Kht0TGqpmivhdAkp3jqfgvL6L+YETH6+Tt0hFbLD+4REExCjcYPtvUagA/SxzATw05vLu/Xq8o9xIDFKg4F5KExkpSy8gOJiNSheDq5UPSKz7Qw0GOlcYsNEK/pt29Ie1b3cMskHpmOOxyf/XgSWBumVOVyb6ftJvnhJRGIbA3ti+oNDoPH5nGS9VaRdvyQKvsO1sfLgxHgw+MXpObFvU6W1GxJmcdxVWuPi8c37zp44S1cVc20qsyoQ6yOncVXFHzR8QOiaeJECB5FJHuPGx7VYSxkhY+O662equXsZXGPxX6vSxW9POljNVGJoKqvamDHgfU2rrs3H6K3ImG3LOkLTSDsRbN770JPBbZltq96+9zj6yPWGJj9pgWHk4a6lUFeUpHgDK7h1RhMTCmlsYFd7Ozs6583jauJY0Seh/WLdmLizMUAJMHq8YmaW1IYF+Hg6Apz+RV9b+YNH3XaO9vco8su0jzbOHBpTj0tvI3VWxV1o2eC4LL37mdgfeQYgun7/YWj4Jut8q/542pvpZZ8r5JqSC6Zw8HWu0AGiaG9F9VP0RUMyJhoGaPQsWdUfVxX/u/Q3uwFjSbFieT4eL+3jhI2oTdYYKBFggMZrKxvplNrbWmlfA+nI0ModjXngCobOSs0sbwy+TrqqW4B2BRt5dYkpJPJF+EIMJ1cTNNXXoXBXdhO+p2AdT8CHm4KqflGuB/63jFRlxctP/IBFGK6dd/H+6Ii+4Xd72WhpQaE4hf18Fs0ojKbqC2CoDKLbaxWO7J2zTwlwZQ37rAFzJ7sERPc6rwopNslV3Jxz3fHTCTPpM6/0SQS7yJtzvPBFj6EAP2WDKGQ2j743I5qBPAj6U56aEFKMvFKoFF5OvcvNq1kSkMSf/Hb2sE8f9iX5eLC3LyKovg3lP/9sP+GPNwSGuM561dQgiDwH0V7Nq/Z+TdFqPBvKNedUjZ02Tpfzy1/fvrvOIn/B/7n164/ywiK+g8Q1g7Kjuq7ln+U0/R/0MQfpWVWFeWfz8T+vDNe/rgu/vMhYLH5j0cDD/7ksrb9qyW//yNQ9f3jO4yG/nsf/XtXFrtDD1foTNL732H0z7eJ2y37474/Cpb1av8syL5FZv95OcxrORRDH7fCf5Wy87D13ww8B3qu/uue9zCMTyH8FNbZuj7W5AaVxNs6PEXl2rV/fpoP/frnhzD5XC/rPDSZ/2e/PH3LZme1BuAB/4Ehf12Hv2ua/uuaP/9swe/i+psLM5urp8Oy+a+y/um8P6pDKfyvAlDfv0P/ASHUXyX/VePv6vrbq3+s838pKcuwzWn23w0C9seNazwX2frf3fjn4IAR+W8lb87aeK327O/a8a+E58+vmkP1NPq/JJaEoH8QV/ofJPCPtv75vX8Qwv9syP8Xcon97+VyBA//NQdnn59n4B5h5v769W/4cw/3K8d/139fBuK0//le+FeI/et7SQz/FxXD/1D2ZyP+6ea/rxkH2AIgpEofNYmTx60dlmqthv55n2RY16H7ew35616mrQpwzwo0i42XMUuBxOTVCfSPjf/8OH2EEEgm++ukbBb27I++AnUtZTyCPuzOYo7H8j/SakkHmP6PY5ibZY3/aAWbV23LDe0w/3obhSAcelz/v3Tzr0/6oc/+hQL/n4BNAoL+AyQ++1s5hCEY+ifQhHHonyETRqD/KczE/0k2ubbK/lSEv5XQ503Xvx/GfxqffxzXrvp+/8DUbKnuOPlVBf01kP8p7UB8AIwuf/b5/9AQ4Dj+L4YA++chIP7FCKD/YwNA/f+E0fpnI/W3Q4D9vZmhEfpvzcxjd/4wW/9XrUy6zfuv0fDvji8zz8PxX3r4lIgV6M3fzY8yz+tfd6RtvCxV+lfxn7fB/2Bb/7r+yxb+8R7/l4zr/31D+JfK/m8NIYr/P2QIUfQfDCEO/0Mlf7zUPxnCf1EVTP4HieDQX39Q4h9qpv9BX/4XJvYZz/j6m9v+BIb/5h3+Aqz/NOYI9Q86+Eed/0ctOIn8n1DSvxP3/wMa+zf68KfG/K0yQH+vU/+lMf+lVv9a8f9Lg1AI+wcNgsj/XoP+BiNo/O8xgkD/70DE/7/rJEb/IzmF/qc0B0X/4Ul/+kv/o5rzV4//d5rzF4Wruhj0Nvv7zfzFBqF/RQ3/t2zzX1PMv6d7DxFEUZrO839SA+IfKOP/ghj+2WD+G6/xv6HMH5eIOPbFQ40rjzWsA1KlYgAOuW67peAWz//AX0buOSZ8fvMGlHlfhhFitWmFj2dhyHbTLH62Puz2NVjARs1N6q65mzdive8d4uxP9LEOX5AcWSPY4vPNqxkzqqFjDKnLhb1cBetrf4aB0WxRLihLUxx5B7F/okf2skiiBFzgMAb1pE73M2oas7eh5IRs84SnBLX1me7BZA33PRo8wLaRcIuzxSIWGlto5zjwwsFIBfVcvwoGlLGDyYbMmwl58BPz3IfjD585JObomOfX/6v3KveoiY3Aa4l6TPmOuiuC8G6Cxboo9bFO1bCjljI6eC8QqFQ0Zc0W1BsRQqjlPsf8+mUPAmcwpO0dQGAOFg1eLsrKL65iD60N4+cfbsK4PhATLKR2Fd/H7xK/Sb7pwsOrU6HCeNYXBwq91/PgeZ0vykXrdL/Czfc39WahQLKCiLj3TL+4yP181sZe2vAVfraWV303tIpu5OSekqb56mmPG8n+yFOKdQ1C2LmyEcF5Kj0ccZkSlXQ7nhROBN8rol4ZrsT5x0vHjhy7PlPJun7fdwavCe/gf/UYy/WvYSBN/ftF89SR7peIvkklhOHiTbF7NyOFN2P5qkJvoQAnTDJgyX2HGMJTVHvxVAlpYfHjzavDvZmRa02s9jkHyiEQplnMIujZNy6HqEvft1AsIxM2LxBu+tHB5udgO0f1BGFc7UK2c0ZVBL2qcdciKV9QglBoYvh1wCReLyOtmqL8vlEZPQaxYss+d3lxrhbERUZKGEwZWE7x5igj5KuF1ol/5aviC7ZiMM1rOEBiBNaDBra039VmHxbL8UIlJiOd8ZkQeghbwgbMZftIftqzQ1LzrxbQfY0Qq0sMJHmb8OSnrsXKT9+uRVccN69371gzwZK3/Bin09vAFpKAdAoQcWxxYHIUoxsmbkSZSS3aEBVwAN+IQG+8vXgR7MMQ777voDELx75rNLbUGLbTX1QAFlBZM1c709OhUpdb7bOprp2sHQGLGU2ac88qyJZp4Ve0hVFxPodWUBVbbivOuhw9sY5xiKZpoOQWwKQN91U8pl6hMUCnQ8GhqTSOr19ygetT7CfKbbilhtBadj33JdEqr/MHfDQf7YyiCiVXYy5+bSjVqYS65IteifC71y+nJhPfTDs7rLmQeTQxpdQXlh2dmoEID3HkdrpHUc1Z/JiOpd1YCCvbsiU4lcL5aJKuMnG0KxFqMMKWNVEAphzY0O35OwVhXFjXno/yM49eq5saRQjWN/2a5JQDU+JQUko5SsYGj1mDJ6vMRcRRfCqmHSWZ0xnI5Qafo1ZXfOUcpKQgWFr2CSU9D/Y3xm/JT4OjoSn42J3vC0IrmHrLySpB9WKyEfZLmfvpeEYTUAINzK9iBwIYNj696VGvwEbVRcDes/i8ucMw4QSTI08keSLUtYGihRdH5aowE/buGopvwwSDlqSTqUKSQkq/uPHiTfV3klXRv8/FzeOGpgvOkX59OR12X6L6ThMg3tOGBCoZ286udtI2mhvh1+eJfmclbliyKlpKLTWPPAjxWfR6TsAmq7N8Vb8xF8PwHg4iQQ3vj+A4VKVOAiajahUdzEMQYe5vgdfZ5niHe1dlC44+bc8L4nf+MM47Ybcv/ef3lrXkqw1liPphZvy1iR5dIYmzGgtYuTXtkTqj+1X4A+OmpokbBPHpkQ66Qcjy1/XzUy+kH1ov3w+G/nlIkz3W/n1M7V5pfnVl38+IxTDJ2xF1BB5boMvbLYs0LuRVwAqOAirPuuXJ+b6gaC9GMU5brqdcpPUg2KcvhILF7w0Pb30Kr9MbHLAg4pngoBMWs5ciKuolaWqmLf3gzBZwRkQpM/0xlJvaWMXRMMUt795aY5my5g16J50l77CrDt7ZSUz8NP23T+YcfgfF4J92idNS+y4ilWsMWmVgjYZn3/UfWIL5mr/Dcspl/MYkh1sE1IlJWt1kinKJfZIxYZEI+xILqBHFKPRrT8IYmPuRorD7tQfoU1GBHW1fxsTexYPBOfqlysjgJaugPmfzrhVdt736GuwoVh5TlL3SD9Ysi/sbuLje1tfO5LzMvLH3R0DgG4byC9EGEnofb8ldGPFqF1Xb6ikODRA40O+fd5EfTYkulB1VJ2qzIRueApAiMp1HB3sPvx39PIX8zlwM/BE5MQwcRSSexVuzHBR3JNDrLYgLlZyC311ohtUYgGF9BTqwSL6x/bJWe+pkaeCrRcB8qRvJbpFXJDd2GCGkDtxUA8dDmBWvvJDZidhVq6pFF8iOboYN1QM+AW469tQnzAsTUumzgWEvFd9kcDKkzpsGW53RFzes4S1ur0P8vClZGmnX7tDmRbEFs7DhHYIjWmZ3Go2Dg/DELMOU9EReKwzRFn/bbG4BqaL8t0s4MVmJeWeuO9L2U0v9ejgKqMX51UI+tfgHADVu4Yks6orPxEvNfOjHq5R5jOUKBHqwl88Yg4vdbXiNX8vPq0eoC05r4xvLofApMh/ZlMLv+e2Lh3l/DyGELY9wtVt2MRApTlwmpeZtQuQfSxCq8ELeY3YjjAgpFdhmwuYvyTiPBPfORgedwICw49SW3Lzq7V9U932cTQhSsIlKp74hMUVe0rIyve1bD5ZO2LW2y5vDzZY/QHZzECstceXRByBijw7d7aWkqIKja9wz+WULhlKZiCAJBy+lbQCWqk75kX0G28o6gLAduVxhW7VYwYsHEY73LJRxDLaHxEsVyzUQCKcPlLsJsvPhGXeAXMvv+PWlPMG+jvZFwX/09AA2BzhwTdCmLO57Eg2f+9o35bVR8ZoPKGxb+9ysEd/7qXhzWnCY1OezH76JNr/2uUCtHiAMHpTdOJXQ8JpvMocu4MQMZF4Rv3QgCzXvelv9kbCHLlSgnx6d4TX9LpQGnI0o/qRPVrfqD0jD4lYWvhpXYEKgmaqJeK9p/UUSjr8V2BJhdlhXe08HMW09gz04JmmnaNKmqgYEY95cWlaPfeiRAN/vZ1RjoV8SoVu/ovlDLHENHbVyogOwX7ZWLHD8owJNY3TwFEpyqRo9XPkVLYX1ZsghxDhEsZPDOj5XD7PXoQLGcfYHQDCS4ij+YtqLQJdPY4kyW84sgtmdSAbR7ZhEzrM7Y4xf+x26ofGzn7zYnznN21O7MRL9nTtDs90eRojUzRkQlStqvBu+A922ByaK34beFiGcCK+neeMIzowSpstgCzMRq5PTYwibKJ7iAbCKKNHWfA/QLvromNoWOV3DYg6IZY3dIJcrS41U+VKOL8R79QBdzh62nGOfeOGdhUS5R31af73jjgDjj1IyxdM8WK1Wp4IL+eFwgb10OtlWVIGRN9MjEri/lZbu+QKF70NW+ICzBclnPz+NTBcQt8CkvNwSangjaoVxe9otZa2BU0MeI/Qy6ZfLBw+QzgNS6z5xaUYo8RSTcxUQi9INfiy8AvkeReOQKbydCJx0Zowah3abufR6s3YlPJRH7l56AEf34SYFi40WX72HjXM/HXUk4WNrKWT8OvgaJ5SQTHZ92736qjUcGrkLBDpuzMw1oQFe3XxoBabgNjHQkQi8Kel35lhOVYWyMPSEt0Z2TTWhjdH4gi8PhKD+Au5VcmXSt08c1EgD7/R12kRbAd8y/RaiJu7INoIuWajlj4yFY2q3kH1L3I3vj4zxlvRZIreBkCKZ3ox2uuhbOxe1ZAweLS0wy3AQL8BMw/2HmPfnl0BZYGgb0SaT8VMpnT5r2oFTFdlmySfX7TbNN0bNB3o/rOaMNaVZ+GJh8DDFe6D0/mOU0OWyp5BwCjf4snT92dmP6NivRlrXQPRNYX9/QmC/XsUHG7WFq0KmfvlA6uQUYI2YMh3cxDSVi419tpzJvJd3re6XXg7ELkWdsAAoulrBUHufOWcBulkPy6/83hibuKkI9BRsR9vNlBGwqEHunWKNv0EDqVF8T6yysUB7o1+uGueQMQ0lfB4LCQRjV9vAykWo1RwybavJufB5onOell33qa8UD2KYrWriuj01CaOTDDQWSdcdDslYoW8Wavy29K+ahNQvSF7TTOkDRlEaOoY7xDq7APDpjIg37R/EkB++GLI9EPmpqINZS//8KY538pk8haaGA0SJS6qn9sLM7heLZapnXCzCsAAsartL4otYzPfpql9/b+UBrjAFtBNXga71THKJLwfgpab6BbxIWhkKJauP2C8PYjs8tt99MOxLMgNw+ts2ORM61mSNqd8KjOcF2AQjXpSSMz4EfMhXzucF/yCT/IzA40KVb2DOwFnLLMW8FVvvNTGRdsXjNDK/AxvF3ucRHIzNLPnPzKDpYx7tHIJe8VQewsIyR+naHPgIWV/z8yaOX7TKdKq/uYcIxNUGlHEAMHp6IDRmgKDFchb+YQ6yWyX7HD+AxpjKw3Oy65FvYKeMH/kWFw5d7I8vtoeaPRr+PtmaQCkYg7oWBkT/kMBTjCigfDp4dIkHvJptl5wJ30ldnLP2fmxorcIpmTfpwzrShQVya7bA5RooxmC8QoLWOnW4lKbYxwbcUm9K94PT5u9WnsmcXzIjSWFrRlSWOg3YzdDKXxIz/xDev42jvLqG791thm90C48PxmtBDuxXCXZ+iNj2MraFFQs2VPzpEPujZ6TlZY3fT/R4HA6jxbz6S13D4QCBHtZ4+juvMBQ/AU2WPqkDagsHtCzE12RzHVw92Mgq7M+QAv8FP/jsHl9hINU8bY5YPYHZgzL/MMzT2GpOWbjEYiY6PXuCa9d5WKBo/XLxAKSP/dN411db+EhdtnTyg4TzUBW0M9lERlxmL5b2PXn2oQJsJ1KggRd45usV5uLxcGNpwWQZzAFRXugrFuPQUfmJz6+ws3totTewLtbQXz8pWqrUR6xBmIe7eSgEOrnaN8ykYnTIInsevCTBI0u5930rCVC3CLvEJsfzI2BUuLGrY1tLEQtvT3eNFoBQBcKiqGZIBHQJBQ7RV+ymXkpOWGfKCMXr16+w9KHRO7WdJbz5vam3+LXXztFq764r63jjg8NfcvSxYiA6v3uEa+aQFnhje3563K4oj2zxH2/HwyPVGXQRMSVg6IUGsC4zxXEx20eXezkN0do8ltJcXo8QqcerZ3H9zy0lwLDN5J1pjpGoAi28BruUmkHtXy+Cc4tpA0B/zmXqiQgqGqzu+yr1LhSxTWKoDdsR27JYbKR2Alt7Kq6XvbTvh4d0qt7euFT00TpZoA0TPCmOcgwl5QrRlJzO8ofo99WeZn0amlHMtmE4FJBwiUEOqTvwSoBe5LOTPHivHQpzsDyzL2GTSFETn2iuYA5mABROuencTVoAOPQwK+ChKA5i9RU+fiOq16K8jtGro5pfxvLKYsBsOk1KDwtZuG8xZzpZWkweAm3xKs/gwf5sNocF05TmLrDjMhWL9sUwPaJxEiNgr4F5HcbMV5m8JHQFOtzZ8OQVi4fPuoWaunFz8B0yLo9F59nkhVDOUJ7SNwJTsl7hGfUN9Jh3Z3rKFmyVcz3k2Y4uokFZOX8R2V0iGczsON35KHCRbsEXG+iMHCMJ1FDbBrC2wDDBD5SDLb5ffDaPqGbShmkV5VQt5vGXDt6V2bmaIXGojqXiufscOffNLEfBfgII7jLI6cHknxgEhF24oOu8nKdDWnkVqMFh5pAqDJTxryFU29GCApNOZIzipBwq5h+gzlRWBqjb8yazyqkJafEw0hBb/zKa5Al2ExSEYVbZZN7tSArHYFJlcoU4emF4beqYOWb9kJyyM474y+BCUVpmVtwEsM84pHnzKdIBQK43N/we9zi2Zz9iWNIsylneNHeDBRK2yVb+0AVoBvdam+2VYG8I54IhvTIC+uUjLefK/lhjfh0ftjOMiGFaye40PNfTk1G+K6Ihuf6l8a68qc1dYhvoC2miLEY0avnqd5YGSGaBjne0kH9+FCsyD9NPXxgFncASm1BbpilPGgdEZt32QXPVC//z/rTR6DaMep+AsEcwwVahMRWEUAsKWOa3vuDsffmuztloTwu3S/Lv5fEA5bvBm32hHyLuFAZgOqLyvUbMd4n7DiJfOKTHWhy4yjhM3MU5GlehrPkPB6oOw2CigneAcW360X4Y/0jVvUlL4YOxrVoQsi2MsR5kZVxyk9mKu4bA/MthX874DD69SBQJlYD4mBp90Y9NVPEjvU3CC5P3Fd2cL4y4HStT9KBaclIEwb7mOlqChQmoKLIcrTV8MA1Cf2wdF55GvoCcrUkwOUIrdWeNeJ9gpB/6NWRgo51hBcke7xjeG6zER5hZfujyqyHoMANL3NbaKzT8qd9jtyxiGNFEmeZvgSVR7iSYS6Gg8XF425GE3I6ZiRgljsqdXbGOhyFxoZ8PuPo+M64zoCT27Rs6nIf+4y2Ky4wte8W9PullFXtDm1orBh6acdG9PD3/I9hgoaENrl8c7aJzJiAzj1fC+a7ZovC3GtbPCZgFera6kt8K3KbyAltYNCqUa3msrzduvIprrpDNmUwzr87+1p2sZFx48Jl38sPc8At7ZRHJQ1yuP2OdCHcEZjDB3jKcnD5uH6eShn0xuDdZiR1bYD2fVm6/dEA/L4CsVRkzzq/OEHvtbT8F6Mtbexy7Wtl63BOBZQ1nYhlZPe4TThnhyMb9ShcSD+U+TW4PG2GmZN9/KsFESCUzbglIcrABxX80q+2wC6HCU3FtnUQNQsZ0a1+5QL1zeoXGjZJm1eURxrM6b6dgwuwbBucU8U5hpFdxe+Mr+8x8ywbDIAS/bFXubDh4aKquhjpfP+OUkEsI59hqJLbSHCW9Ylr0Cld57JaB+vB4kX92it6lR1ash1wDCnQEyVpnZL2Yz7jwKrCaLEIp1yH36AfQGrb6wGDNg/ts8NgI6vJVORLzYSIyJEub3JHjIRT+eKwlu433FSuaRAtyvr4vzMYtYZg37m5oCmxy3GOw4EFLLQZdAvbgHUsU0d7Zop7f74fy46z6BhtMvAC5TLCr9nH1z4IMHaTU/PNTYp5vjyfs7WBjRA5/t3LWv0N3eadniCI4M2OKwyoIyq+v21BtpNwpfARNREoCAOLzRnrHWhCPh58zwsQxLfvlZnUKgGImLTAtkTRMHavB4Af7wr5Aka8l++UbsLPhDQ6PrbUQD6f+rpWhUg+UQPiUSjhYb6MlFg0VO8xRBbmDWJTlw4DV+dlBYDHgDQoV5wGcEF6vrQncSOJuLkwbp5pEA+9USspTlH/7IyFrV5LqbbOx3hneOoVdpXSe5rl1bWDR67RqTvx5ziI63yIHOmP+JQgIu8b/Pni5LEQmC5VzOPVH1OzwcM0Bw4M0o1NA4+rYV8kwZtLy4/R2z71t4GiZsVHBnUajr6uhIiq7VoY21zgeflOFEW15IaHBZguiXNndDM791VwphcCeasbQ9ys/Y9XcR0VYyv7Ry52zVlsvZQUc2S1+HSv/NKjfXWKkcrYMCJWKZrT5y176XtseyAfwR0P5pUyVhh+JvNPb9Qaqx/JC6D2y9sB5bK9x8bBsVnH93vWZEEzNPVJUiyHmOAIPGjdwUUYSXGlsFjUUNrBA3mjUe/SZsKfLXTHkaxV9o2CHQG7BD1dUi91VbPrwPV02j+fpIvrgPEulHeT9cWatOD3G+7YyI65zdV9sq5VG1h56mRDNn8XiXFHciKHGzWuvJYu/B9Nq4P0zdgMkww+95PvHd0DOq38rgj+18ZBBUM0WV3WC7ePjUqAYxPbKpk247PXWjaoOc2K6btuK31wPL+eDHjZuBVAo4FXw96cl+DFmfou5Aacc2Xzoa9Oju0qdRfZo9efuUK9zGFFukSFk6DptK0wCC6tzP+BE3k5DZ796KTbEI75cGAGU2aI7JfI2oqvxbSU280a0jxhqr3vt/RodzZat97LXpPistJK5PCp8/BeuxQo3fBzDgpWIg/TsOgJzP3fYUNkHV2HXu23mYQZ3cvWVIk6sx8C7fsY1+13fnLlOlqJFWHQPYaP4VQuOH2FnMfe9xbpg6kYfKXXeGlV8OhE5FWF+aZAFhs9fZ1okpS56vQalF+t5EJBS8dORANthLmBKE1Jm6oIn0gunx8rbwnFtQ47vIb6HwcKUjV041PlFl0vUUAH/cC0nGC2+88QkgXumC73P2OZ4PqRofr47Wb8ry6lBhrMSbWWsFVO5vTIH7mNV1Td0qjCs+baU24oMmKdGoOv2UR4mMp9fTOciuWXK18CF5C8pY00bR1VsLsF3P+wzcQfQQcB5v2RR0jtK4QvEV35qeTKbIynQ9E9YCEBhyvIZBzkmNHzpbt1l5Rns3pheAaJYCYHojif/IP7nH+jnB60CcZyl9kErt+ZeUEDtB3A3wF1L9TvovrljiFFZt/a7nM/B1p3uWnLI2x6HZKsFI6u0lgQ5p7iaNPjid0p5jMUbWq3d45JZWx/aFO4R8f4gxLtBw1pKFOeCAUk1sxsbT/yD75RH0nK310YyfAmzkbK5R6L0vVYNpC/IB3WY5IZG+ahe7+A0bpD1Rrrfo7eibYrp+NHWEMcnLv4eqbtPa0ajbioHGzZEzA/IuWN4zYAwA6/ppTOjjd/HWpNCObEE486YuC7eHq2p+TeHAX6Q/x+armNJbmVXfs3b05slPbvpvdnRNr33X/9Yo3MXCkVoRs1mFSqRCaCA3a277+8x5Q+boHbAptFW/ViHHp1l0fS1nWE4gBXPl6oEC+P2HqyD9DsTpMHF4rpKz76Yr98rmbj+qFbVbGxq8m+fClgwa1tqziBgvkD67+8SEHP38zF8ovHaeamoYLn6zsY2dkdV8HalcFlWhadhrYCGGnn0gM5SYmm+ipTCYvck+1LvX52KSnH4yhh6qs7y05C43HZnzWFCkN81wTg881OWTf8E0ZaOU6LAkNs88vqkc6NmJlVEycOXrJdy/SeHP6fAJGIYPYSdLyhZtHUtfcbXd+tsFRzfka2d+bI3LPrFJpc0WBo4onvmTj3c4+rW4gFaq35O88+7LLm4OTOQpXgIlojdTRb7hqvk5+7yNdnwrxXzd+6HlWMVshVW7Dhy4WHw5xiEVEeX+dUCvv9oMYpJ0Qjar4c62/jfco/64JILEHJEaeM9ELrfdqsuP6bQ8IGc15q5CgcmagyLJbO/kBn0+bT3QWt29Xj45DfnZq3sxXH0Xlo5OsLAwqVQnTqloMQMRIynIDofZ0efLl4G3hqv6aVPz6OYUqgkUwzDv38eQZrUC+QzEwcicHQ0i7Y6ApPkuv4rZ0o6h5g4ii4cwuruP0ZmuVRelDJwdGy10dz3xz7X+sqnT6bdNNln/d+gll5nOU0RMFrm0Mp65ntOlQ+yH94IFDl6Lx5IhrVYh0JIcg+eyVLIQPx+zeDXW8V42VBOcbFxarYwS4f71l8vIRqPu0E7Pi6X8XRC0ckwo0QIufnaMpOMil8B1yItA2mz796wca/T8WdbNTGHYtPlE/fn/zPH7smpHKHil0MaHJul57/gCPr6DlMsqe2pGur1FlLTQSwkgaocV2R/gW/b6j2WixmyyzFhV8aRLBwSbbCS8a8hjvTxFpUo/LE2N8UaDULyWP0yADaZGXzV7YwHpvcXi4hhJFP1VpZ8p6VUpyOnIqMIIcgYSYHmbC4fwR2sngwE6JCAtlgU1OmiNSSYOSj6ksgAmujzKsCrswZKZ9G+44RCBSoMnUfI+T0fiJTaaFfKVKtzSiWrLfvzcerF6nX+rqN71ooXFZO5oQNZjRZ4+HbfRxgn6RNVj+LSSkiqcnUtx49gNFNkU4CSasTd5ysqmR3pZWgrIWwgWiPqi0bQAT7Do//9Ify8fkVuKHajwGyZp8btxQa4hmB9jN7PlN7TA1wbvFq/tVFS7E6J7HW4gs43snaTiVbjtJnzj9cpQe4ECu4Jr4a6Og+o8R2fbqEK6I8mwUBBaAE04iZ5FJy3hsyxdJxC8+oI93OYctfVxi6qOGMZIBs4Jttaw9sniP3NdNC+vjWclGitNR94pWV0JmDKwhhFUgNJrgb6+KS3R6wQsWtbFGq9CuqIPp7ouLVNuuAddHyu9/NwCKNbKrW3kF2m7TL57KkoOlMWCphDAeOiMXPZx0YUns3k+7AsCWuiEpPaGoHtWd/g/yhN2jZ8vp3cc1pSRGBtMaVKUB8qyHXetX9wzyHmATcH8Li8lmLlWypPJOis79zHhjqU6/qgsxPbjXI/qaOjL6PDgBNhPkX2tb/WMtgq8wfkOfyynltekMDRt7pYk3x1+orPXpb76WFRX6NpmoQ/hrLtwWqAk11JJi1AoaKJBjPEzb3j1a160K7+rmH2vtU92dALcaM2O8eRrpWWhZtDr0XacUgh5PmgPPQThCWcKLgRY4MLabjmvNAK8501N/XqNBi3HUu02zgEGix2OHneXWsafmrsxtSYeMDH8PSuoZQUhj7/lfkAx9SlJtrN8HdgF5Dhe8V3ANbBvZFXJDWg4skNkOr34nFLWbKHAYl4yhB7PjXSoUZeyp4m6PWv+1oFPi5O48Qx2m4fNe7VzzT1HIZpaBZ7GNtVYkQKwiMXfGDzL8e/9x88xR7u4enyqgVygZT5hKUnylYlFxwNW4NNjX+61333CxOZAj9c8fsXOT6aKlDFE2GpcgGZetN+tXMhB7swhFCZ/5XHGNayXwpSWLYUD9WxWBw5z9fsX6C3q49TL8NIT1QFtTbHneRG89jhcfQbOzKNE2VEOeLQr8/TpK1aMVAU+lKLgnJIOvTW9OXPS/0dWqNoX8JheXo/KZTsC/ZYqgdRrChyR/SLjGUVcn8uHAA3TvOdZ4/Sx3AZKIsjBWjK2GW4GduafI3hHn2Bsvt+Mjqjfw8LJ443VvLwq7zsT4ok0wjFEbgr3Bi5Dn8K7CcdH2pcU9aTpskvvsgcvbSLoqp41PZvKCng4E1w+m7Wst/P5DlKERFVExB29ohCokAHrNyXadILuf8VON0UXhTZE78qCdqFn4oCU2EFqgbN2B2jT+NnNv2zc9tTpOtLEl8HWX5H8BwVOHw5NX6Ns13xjmDHe8oC6aT67JQrVZK/gGTWtU9+qYPpjhIGEdmeon6SVymVVS8lFLwic9OC10dojDFpf4OYN2rgQH9O1t7x97jeT/Uic6G5GohhTLR8mgXIN6AtOoCCGMsrUunu6GdPURualJGapLXQHqSQ3oUPRfOLu9a3yJHP+b8gpYG3jKuZ15kB2Y9iC/hHcvBUzGRnxbHUEW0drTsWVZjEVgi20+HIaH/trSuY0epS19ixhZzvs8FLBudulf452W18FCo29T00eyvGJge8jT4Q8cNEnc3Bi47mrfK6HKbJyQEHYQqvwIxpbl6Ykx+/+j08tSJYehQQlSfeRWnQznMWX+FCsw8iH10niPSBmOb7l1cz/jHpmq1tWnSAIgzIjmaUwl56mIYUYrFrFRWzPXtr1WU9qP6Dpuclyqkk1dmrN/P7ACrk+Rc+HGRK+ptU+LMldp59Cv6GnLlGt9/jzOlKZp0JxtrNwVX9BuYu5DWuq4xC0TLSgc2AauaFQaP8dxf4A18TB3abnVIpV277I/CX4AidglKg7jmy7p9HNexGvDjfSG5x6d/od7o/XOb3CTlJuqmAiviLDqfJD9ZGhgcKa+B8+3TtgnRzLK2dkgK3gsRpuEugomLe/fJfOecm3MWH4noeyKxa1aV4z4l5PdPVxSCfdl7jYTGI57kwxDCW0iaAjMS3tWhi4Pz2jIJn1PaH4JWCnw3lU+0lw42AXswzNNpkK4Q3yyM1tA0DbqSK9L64ZnnXhyLVf/UeIHWgkCAquDVU8WdxgBVAEZJgjjUpgpevyNdpf4fAnzrtsjI0l5GHfYCSBhMiRKJnVpGhxMt87v2LQ774iy8LkZLRs9RyYtL0E3RLohixLl7+GYab9uG+niAD5OES7+fKaMliupKMTrZAmpWimDmxuoM9sqRVC1v11ZFHo0bx28vKqr2kLYZAOt4GQYd0D6oLChiuHT8GjCsfd3OJzpt5x+q+euOC4C6b7eKryfLwc4e0raPFuuAm/KRTPwpa6gzj0VnYNVd4zWZSoLe1oZAy/nFu/57X8y9Lc/1OtTXrihSlypgmu5TjCia5GyPVNSia40ZVNPgM6/1SZNvXKDG++XqaPMMg5HUMElWOnX1pyN44PRFD330Z+qkPWjw5ze8naA0nu4K8km6k4WikW4Mj7gN/TqSLDB+SlcqHptBKglDTzyBJNwk5qwddLwr9Hnb7YxGcRSPU0AiF+JTff/R3yHlTne8JKPoA8w7LFSj5Uf2MK17L7+uz+EAmXuPfQdTgv3Wy8Hr6DvjLkdZ1LrLbboq15LBHcTL3NwhJSFRFhjoTroLuF2xdXmtHwi5OBeeIX1Vu0XzZmFxBG9V9tPS9fjwNhTdxuLccRIfsYWMunZMc/5ya4+N4rdd8Rxe/ZZm8ZGvjBxtFTDCDkfXP11t3gFwfDn9RzQWXUIokn7lGfIJHfUeoYrMq7CMrDnEDr6iRgoPj/lfdxsFqditnEqWdWv+ZSlrtGqycB5gyvzT4TMtMptOsDhBo7wovLZui2KWnLsaTkLb5XflKZp/uPIfBn5Z+gBQ3chj/bmXzLirMX+mLDc+xe3oOxEU2Yy2mErecH164og8NwM0vxRXKUYLgVldjyhypiELxQGexdrLf9Xkf1PfqjBuEZSU/GEQaCTX4bPJeszK4bhS/8Wq/0bRF+mqzAWKkzJON892LT2TnlT4fFqamVndhkPLo2NLG6yohG8TP9Np8XyxFFjTIem4eoiwu9NBPTne1eJlEyrz+qleXYYMfSLvk+WKwtbygDbaCghsYHlYpSFmqAzRMdAnNF3WYKjVStqq2/2Rm+WTzbyaJO7Gy9LWBsyvi1KSRJcjrUM35VG6cBR21BXwc6SIXFw7dHcqNNd0KDJP3RP9NqxF9LJ6fIo7GnUhWSrm8qvS+Yp7K1BEc9EsXNeIq4RxvDuQyblV9iiBE5RCVdAGOWi4nANbSRCWCBIryegvZyY7wG3Pkxx0oX+//7qy01tqM6nL8TY2mWOLFgyr5nXzqdiimZ4axlt/h79bNbKGnMzA/88q71Zq/P0aitBV0uGSN4UbAPZcL1IXwmOY9MRhqAahFlUpKbb3PLs0SspL4B2E4wFEW3JLBfYKxfrKjydRK78+kgpxmJQdDOiD1x8XM1QO/e4J7NtJ4M4zHsk4SLcB1Rqdy9hPxlxQlxJb9rv/5GiZp5hk/Q/uvTRS3pw0ovyLpxVtZ1IBRoSFn6yhjvJEcGCTlHtJI1zQKdVhyWlDlwQZT67gQggj1siYUJfSLIGFSdq8M2qAWhyy+Fy/fETrfr6rCUHhsIzfcIJoiBUdH2uFZRdZBe7RVavkpx/XfDFwK5n4hEYZjoYQlUR8fHmTtQTAP+BqPsoQZ60b4NZ8Ej5nZkBOjsxwVG0SLNQK2PsUKXmCm5pL/MnUg2FecKGDXPTO8QB8udWgeq2ReJsy8hnENCyN1yxUgUfZZcV28aciD0LRWXiA9PrJ82HHdMA9zYxe3M//iryJFSmTOYvxwtos3bgou5xvUx/xManqCcy9Xed9oTP/md7YJq2RROljl7E4flQ1U2cLFfhxjEpgfPseMcln59yZMzvHhs95pio4mnxWnmOZfuhlCsnxLrgNWoppXKMTGE/u8SotJPOFrwGuRQ5jnFVb/E9NnYCaM55j2CKKcQs6k/RsBI2AaSUt6Tcm2CbsuSzfPqfFaoviRchn88zfWVTZ/MXqArPe5sQi1fspATWl++Wtyk6z20MtMe0YJO8wnEb/AZiq6SjEJ+QtW6XUO3z2gDOYSLM9pTw3lATgpESJD3gaTz5DWJCCiLCjTN3GrlIKPmHhSUw8GoLyv05C4DKTcF7e7nqpw0sN9vkGTBocakPz4iTbpJoozSw7zM7xGVwBuGdvB8eXVj9zL2C/Pbv0UoO5AsHJMy4tF6ermCCtNmfOR6k4R/ngbeYO7oyJfyEjGrYJqlYdKhXj/12I3AYpRKl42WdrabWcQpOYRS708ijAfIDHZKmzM11ddQd24B/jtZW4/jiyLlW+npEopK8PKdIKTRVl/uN6VNBJtM58mob9+w/G7uIiH8S9jGyTu+ACt9mWNB6kJgaDx/MN8T20GLL5Dq8KKQaYWu31g0vBrqCDAPxwHcSOlTUeA1NobiscEv36/Vl7hX0m3tOw4VOCfKDnTIYjn8p1L4+1zcXdUjdbUePz3b1YLKWc5jw6SrTzNR4RvbKl2aPaT2WFxARG+Qa11L0/VXlAj24vnJX7BnL+LBA1PSEdyHas1ETxrt8vFbD/wpQ1QPkbnT6FRJqgQpTfwAlVKwW6py74iMxpeXJ8KvMh2pHrJpulwQLc+kDPz8C74DMbdbNfzKZSDz6X0z/nbf15gGmZIp4tqdBK2l1C25b7jqNAcAtN6mb9H16W4CiQT/nU4l8EbvutkRPmkangjkK3G36/9VBg7kotgS5ZXMRO44oc1s0UEgnYcim6YEJ1iguvCH+IzGheqQozTAoD8chgSD+VxrJkQqUNNG4Nn9IHJgcTFcAAnItWjvgRRbTaIPIgP8aVr4Ko68Xleg/H1hDUV5AgpXNIQuvGgyNLJCBlnqBUu9BKXg5BLCxZ6vmPghuIIEGaBy+h6+a9A6p+ELDNd4HmnFQTfPL3bquzH/5INBW75itITGeY5vEr7F53dAoWnLFHAx3g5Whr0q8kDED78d52I3dWy775gc8gHfeCK9QGp9jLm5A2SvEQ1BmKKo8eGvlGBqHsnDWv4ijEPhZT9yeFRMbqhvi4UpyEEhCApz7Z31qp+n0n/D7MZrPilssb/iF/4FJVrCdPX9cVL6SLh1KKC8VFscIFnLd/3RVTojOKbXtXPx0qWF4KHV40vE/U9ZZVvgWUFeeh6tHAw8b6Xq6/+eidyohuoZhHUBIgS6YUDVoQzYILgjxpw+auQnvJfOpClBQLSt8D9K708aIfM3g0CZvP9OQWO6O1nPNIjMzD4NQFb885GmQomHswP2GfvZ4Npfe8R1nSyvLof2wThynZG/n//5l2Tr+AAIQdW4uKxML3X8i/eNOUC4Pf7jb/WKuneYZTWjlKqBrI1/Np//CVENl/1JDKo95Mi9nyxIHjiVBRMvettsRBWS/pDSrF/bmYHKf53k3basiSlqzwf41dQ3DCYE6gQeuE99Lcq23LjalSEef0P4uwxw53Pwm944n3GF/MCEzFwC+q/v76raNooQeY2AW7674aQ2xLqX6EBYA5RmnzVZE8hcXiFR71VHigIVpCfEeAfJs/D4n1JhsMz8o7u6Gpc0DlHrApwMYr94cY/q+OyiNRHDjwnrK3o6QgrPEWdTDOUH5pHY4N32Uw0nBro+ZuwDSQY8IjwOJWcqoJ7deJMrsU/+wKMJpzV53zZ3eDjYEs3EqHHlFJ2t4DHRYAtSCasdslvkkatScvhrEwTM78xBEBNqr3Ug2Hpcz0aAoQ1WT4VUb7doncvpGJiVZu/f69o7X8vEyceRJmcA9vnTHQQyf5+yh8i3cvKvipSA+sBYI3oVItSUQnT+u/rd6jOpJa+rYaKlIeVplgJNkv7Vuqj0txie0zwra/+E4QkcNYcPEZ/Ge08saOJq2TvW/xKmdJ9F77nXmAWs4qN+YRNsM/cFvU/Zr0ORG5ii0pivu4WTFZAzIgfsrXWXUW4tN8v2o1m1dVm1XI5CqL+Y+oxsiT2VcSK53etTL20Igtciv1MhdZ6UAWlrQSvdppUXD4kxm47W8jWVMI9I4jWiexH+J7RHF+k8FLDksRokv8zy4LLWEf5tzuNiOL+taMljoPKiMYIMWnpAy4xrIvAjtJKFxDTOH/Vl2UrmZosh1tQqkB4B8I2WWuOEJCGFy8+VWV7IMbPqg+MSBg6uNxvm3B3ogjDx/C6LH4hgP7t5gRXlNCNYLd1b73VIMQa+dH5qUw9nFkECekZATafkOgfIsg067Q8f/K0Tty1z0u3F7lIiTR/oBorFPswOdGtGZo26vgcxuhUabieq9Sm2uDpuYzrrOLen/IGZm1/myIcWZ6XizKoDtk2mK+XyrO3o1P5QTMNxJGuQeFifLKOkcof9PyGMY3xr2TkwcsG/RxBtzGjrnsj+aP5JCpnWywoE+7/Zb/Dy/YgTpaL5EsxkUw5MIAmylkWdNVTvv+kXKjKD//VXg3HVrxF0TlsuuhoyLPhAlZKQ+GHMNsorjjzY8cIlzgRSxdsLn/Aj98Hk/Fuo8toYvCGRKjLx+MN5wFHjfHQRstvL8466DFFA8E1uc1AgKjIUffPnYBTrEDDQmInGYNFxFElHjrQnVzclO+lMnvqXi9DBIqQEkn530EXGSfPNY/rBnoAqJOHARptZPD0Gjju3cx+LS4OT6MwQ+tyuCRTOP6DNj0AXIqgJjNkDcpiPXfrme9tJ+QGWmKwu17+9fCP4+vl7RfqZFTnL8QVRYakfn3y3vTwKGx2/2S14vGYkvro38yUNMRecbRQX8GC9Uj7tPpQeFeRAAW0h+EBGbgOvOp9gF/PSsnr+J+g1jwXSJ/xU7CRPs1EipjmRRPqfpjmCZLIgZ888UKgwL9uoyBvyFMnL2/M4KPJFEs4Msk+eK0v9vNR+htapzPaAZsuTXg7zleyFCPvvQsHYbatKPzyyOZYmvPjqPDJvpBxzsVeZ6C0t7YBgPKw/Dn5bOXVp97SBlEBSgmFqC3qJ3eHV+C/wvYqSf6gDBAajF/liGjgvKJGKPOSkGEkCINfncInRxZSW/PsHv1DUwCYhsjcU4O+joYP/hCMzG0YSZFZXkhu/h1Pz22GZcgvp9g8TdOmRKpW93waFHQXp1wahFMNCadhcOOQ7ToQUbTvd9UyPzWbtUIknRZMse3dl+lmmM0glmcE8Qxir+KcXmO9N2hR3OcsOldpqtvf4PZRHciHQORrW5k7AjGnvpyLknj6G0eFnvlsOiPgwHRUq8LDbFarhQt4ZTFRHs1xmJab9V/A4luc9q/8XfjHR7MhIUBcYnUL1S7jzR6wyR1x2HIjR7Hai6Zez2P68iNX3vzbAgyXtstICDD+RByI3ZpixiKd/sWSv86vz1P1NPOR6wdRyb/5HPJe0xRvpAJnXphbcPPgDRXxD01f2WQ0ryypqXW6kPORv9GoN4TjtZwyF90PsSsB8nLVhE2dMjpAOTBF/e2sO1gsFJgg+df/uITx8lCAfoS63h5n95EH7Hcr6v2vK2wFW9UqbV1cZEc8WedwQ//IzStHoZMRKymPX6mwmt9BVWRfxqnopATAGh4vaR5KNGnP2qcIDLtjOYPPQW50jIK1nKHPJqgf3NrYiYQzJgo455IvywXWdSQ4UCju38VOIWX0C4GiOl01JE6q82xv1vc+Ci+YL/MO013Gj+g7hpaSyFkB7s6JvxefC5hdItYZ7F/g59NfLcQBUlU74A2/rxT/WNp8toqpoxFsb0j8KGNzaN0MfOFItSjyVuqk/L/RqaCgNlTjtg/BNU12xQoOU4cv/jfqLZN3IzZ1JTaZWPo2XHHn4uW6jhaEdth9vGzQR+k9sUAXNEVWfK/vKX5mZz0coCKH5ohKtGQo9yf4AKWIVn0pWiJWoLI1cm4WBpzaqqngIjPrQUwPPc49WqOxK4WtnD5EUBwTHfO3GioiA8/Eu2rB13ba+QiDCrgKMekeug+QenQTDjfHIKi6hQhtTfzbLM05PCW5em1CTexcJBVCPmxjWuOyPH++j68hP9fSn/b5Rd9uRIhTPMF3pwHJLn/S8V2keQS2s2qYT/RLjajOzoHxOJ3k1qxHMykOuIwQK5/qQqHiPTbVT2naXYjkH9XkKw1c5q8cRrYiJOMmtoTNWASb/en1WX9zBFV8ernhz1vmly89mUF+hsuyf23K77ACGO9YXoF7wI1qJOpSo2Dj1hPLgfC5ItemqcQ8k27rRRcqAmMCOeylwuZvsXQsO1ZJas4JGjYFoifSPn+IFcnafhHtuBk59Peg5GbK3cdZg1nNiQIsKyUPJrHLRMQYegEpH6AO6Qv49cZCyvLvRtgFUxVef8/73u95yaGtZ5/4eR91Kep0MGXH8FPxJO0kxQr8D2cgDCIPzVQM5xGcLj25oCYnNTyuDZxbPLWrzJWaaeggit9KGN06MtHzQzvFKFKdfDdZ3l8BgWAYsHKaHakdiDHtpfHgxF5cVACQkQMkIvJUWRhYH4uVKAQys1DT4fIx4JELUhPKPTaWPUp4TPgbKygzdzpL0/GLt8b1xzS5xNT9FqiRF+Q+sQq5DOp2tldSfi1ZuLLjE5Vaj2Lb76+EytJgSu5b4C6i3MwGrwpCBYLqdpDSvs43or+l2PvVzL2zghDqnWlHgFIkuvd3p5PyTiMeg71uPuSvEOUQM5u/D3aZqPuNyG/qFeAhN5hczeWLcpfBsF463NFWVrZzMHymRs55ltxQ9XWEhE+tq4B30xJlEyn1NWEx/RBW9jKU8KZaJyjqVrNwRYzx1ul60TLLC3u4ryimup4NiErJt4HfRqrCTHgAKGdvO7Bud4OGrIJlsdfxwPhKvxQ/2q9b2t/UgA8LyWMGJvRr3sOFSZvYjCy3QzBbcn67PRN8G2nHiYdhWVOAEJlw+ZHDTVQvz4J89K9VAm6oX/JDLkkPVRL6m4K6UTQnbXwNcx5gSbjzI6pJsxX8ZXWovHLXH8MGP/qm4ldiJlGJtw9MjYck/9oS/ovTfDSnNMj9utPvzobgf5AGXqvMXb++fXmZlPDHyQD/BvD8cvBQJyLqb+ANmkPCqK5wciKb6nH1tQcks/7FSsetVH2uwW8cIkBuqSb6jugdEWVnf+QQzDSMGrWQhpAunYll6LKZrt4x3FCCcVDPXkw+xfgQ+7SSAdGwLUuGWecoxXE22NVSdPl0a/Nj7Mwn3ker5/lTaBhUZwIlcJetX5FJNQY4DPullf3dPf0nRLPj+NTFmYpTZn/uAfIJCMJBhAFERGh1bhQmoGTVIDPdQbG/O+dOGTRogreM+nXhv0mS9XAefPld58xkyggRvAMxY+dl1tFKDyY2I9ODs3ggqyCDlXjKlm4SvZ8goroSyfB5utgIppKvsNzEbPUHfzQ9Ymb0V63ZveuAHzU6yWbe1Wat33lZS1ofSfZ2nqGPMDpm8SVag0CHP3gshxWmcN1DWfffPm16eSUGccmskV7P55JqOK/Uj/crO7UidgC875ml+gWW8C/Cj3/X5FmdJqn1PBwLEy6KMq8ptF9ZB4INPcOsAj8Z4fJbfh4G7LM3Nf15+EnD1wIcV4jlgZFQp/D5FmyIvATCEwCp+RvcKHTlRT1/XV2iLgehqPow7G1l+25qsxOTLlodpWviaYoZFV9373ogieg3C3nUzsb2S52/wabc1alZ/jdQLoR0Pkf1tC5svLXBwI9nH/tbRVxC7azHbQ0LLMWhXgrpbLrmu5qemDVgXnRBVjDGS5E651+VWJxGC39GgR/09cQUhPWMaWjwKw3Q3kmp/U+2aUUEFE7leM0Uw4T3emWkMjENh1nNa5X+N0emZ0Yibzz5D2Td5kCT8vIJ7svxBlAZ87J03E5Tq6s/nj//3d7dPfdIuSHEagb6a7uSOHctfYRmf5JaQZeAekmQff36P7sBprv/fdQej6ZImke8E5RjNAG8LFPB+ceNFlh6kWqbrhzcKOu6Tqb85/Gx8cvYB+uc/aSIYPmEA30le3K+/HbYwKmgVvWV1f1fWEU+ZcQoNilk1433YAyiZRQ6iPYvOeW/lvan2Lp7TU2iuacq/Luvji63C9HQhRFklqs/70Z+wooWNGu1N4/SkkIimDbnk1qzK14eUMHGhjfFVihb8Ks+LzLfn8GTkfn7qkrLGIYa2uEpsPguE2/T5KjDJGu/ro0N9jHlTy8ARpJb8GeYO+p9MfnzF8NvVqAmCHCGqqt63/rZTQFr2wzv+JHKUCDiEi5jXhg028l1vO60nmCN+bPRtWxqy9L8lK9ZuyRWlKlSE8+/1X89z5QQA4vhZ6zMpd5nvLzYPqAbC/KqeDHf6e6jwMRaop7/ErXccXxRDV5Oaj6KgVNkOTGzMCV4k5bw0h5f3SZjN1+/9V+997IG4cO6r2Ksxl47XoJgcR1kaNDEMc42wA2BTV8C3j4NRvXZlybx4RHzdR6AzxdNSN26L3Z2yTfp612buCDSBzxkQACOXcips2KPP34k8ndFGjsFjMWttYyVhz4cs93zg3rkfrK59ZDyBT+F8AAfDBVnNez8pmJK3VFLT06e/MmRtD5nSliUVwk8O4HLL4n3AxTybp8InZ0Hq8LigtZHR49VpGRbwYc7yD9HDfC5xUY6/EbdKIYmqbhQ9QlPHCrElJzkqhF1iixGVf5y3Xp4zpCHPry94hX/kWST2Zc1Gj+9Tna0GOSe2KAJ1HJnuLiEQEH+dUlQyEge9oOnsKXg6QgMFwVeaN77mbnFgov0Ged4LG2gIBEpQxUwU9u5GK5Okq1oi5h/P1EGXoNVvr84ET9Z8Ze2Q6bo/Jzh6b6QHTbC/hmB2oa7R/eSzDlTvkRzxES/PCFfiFFW8GMbgM4fLmiXIy6s5bd8f5UAqv16xB/i7hLY4qf0I4XEuUHs5ycTwFtNpEqD89j7RIMHdZW0/rUihXUwUrcVBb9aE0UhBul9DU11fl/oMy49mp/EOm2oAeWf8nMtr4eDPfOc5cEBj1+Rlj5K2m8uH5/VEH12oVEBx2Rt8/cVimvywJ4HYVYYt68ZZaTfwYmrIcefNZTg9SfOlVGyO470mIPUTgxrNd4x3eIXy+op3KaHCT1Qn8yeer4mJ3W7jCBSpH5ksJg9EiHqRyXAB/MO+um21CUQgyMFbBK/HtX1uyd8fl7jn9CuH8RwtCdPXUxYfeAvGCkJiSErg56jIAHHGqyguvMTyGGH4wyOgC//K1NMocMdfj9RoRI6oHMb8Wua7Px3lQSydO78C/qTiEv+njTtPnhdvYGnEzNL6Vm0UUx7K1GNn+0rN4Zl/QKRWnyKrLM/OL2V21TUoICxSE+dhUWZ31xQtmkcNKsHj+R/DiyHCZPgfjmaKsz90u/7qn1doJaUhMjZ7CRHbxocRb54rFT+FttjX0ZGU+jLROC6C3YEKsv6MlTy7v+hj6StUMycX/9lxrJ5KUXkczju//YarQhmkSHDmAmS4HFUNsYjPBV7iDhPxK1GAFbk0fYeeI49y45x1+0R+d2KEroB0IvN5kWZnj9HTeB8vDJIs59EBWICdNWoNtxuJZOLqAKJ7XVvGkTny7UXWFO8D+Qylo8ONL5JSv6by6UDxU2d8g8IrkSBV7Ns9dQcMBRztqLT/Pn4ieRBbAS8Xym9EogSx1AKZXuyDuvQZYcuveuI8u8Ppxh8iBmEzfRL4DEUBDFR3fo2vNyU0VsagrG+nGu57I1HnwVD91HJYErRXhlCfHw4e4kOUh8lOoP9RWzxCuHUwLpPHyUtgQJ64yN3jctjQYYIebRl+/Xzs4Y9N8yLBvnwztrOiazZzbdHVV23enDevEdqZ1Mv/c+Ef7PMJaQSvwIjbDOyn4YfG8tRpgMXTy5pQ/Vb4ReWIJiCyXVnNGGCTTDzYEeOLbOTPBNQuj3J9kEXTMrQU9Cmolet4MY4iFQ0qxVbVlRYz2KIvz43HJFdQ6LJ8u+bglAdpBfNgvxivoVCE1A5vGAc+SdByY7T/M6yBlNB7jBHeziqjGouACClVI8szAFUPQslsjsi+gR+ocnHBZxXYSDuHSE2P5i6SPnjV9s4JSecLa00Ju2wE2G439W3hj4ioEacYk46gY/jEGEzk2Dn11O6rEFMR2NHo9spFUI4Y1oWJVf/xCC64wZpiiFPEBWkNMoToEJcZohOCie+krfTLVn26N+5v8eSj67eyR279O3J+rYL8e3s9PDGhBHa6lY3u8cWO6OL1gJIz8f6ubM1TyjEHbd19IDQAKZju9MFkGjPldLNYBFQGoAIxeL41l4lo0EoQ5H8hTIthKq3Ug2meV4EJpDNShdQEleMN9+v0Nyd0OFeJTrbKaOfHNQdKFYQC4pxaXURQzLx72ml/kZbCVgJBtOCHrBiyQ5/1PXlHplXmhSSw3A3BDYIVmFPPJvYnSPlg6jH6xbYZ6Wpj8hTr8GTIwni2GkQ7sh2FT5D+As30aSp6uHfNeegaqOv9a8mtbiO+HgplQSRIkjK0LfG9sdFYfmvw5nJ0GzWVOg7lPOboxNEz86ZqMnIX7ZnvSIvRROpnqunSvFWyt3QawhxiFgLv24IsZQlAFKeqZoMPhWIE2n7Ykx7cmq7EaXG/esvVw96FQwAIY+Xt924gqB79dfAly8NOFLBqTv/vETRJ0nzQSGOI2jaLa/zJZ0nyiiI/PRG8OCXuVoESemxU05yMuwr+HhdVTJkgxTgz4kA6n2vrdQus8PfXrjmTcbbSYYgJiDzmWFGOGYu9IrcnHp+TSx9YJleCFBxkkEhOWPLC2cgYIQD/hN3ky/nhjBkW33hgPiDWwW+WTIs55j1FWuB6HQHjSxfOjao/vsClV2sCLHGyIRm0RdQ/7LaDQL+9BssG78lS0KZvqWLVxRqoCNh5v7KKABFzRDhlkd2vn9mSS8K7UtXvy3jnB1lSdrU32TZ8HbbV//ffm2DVwpXWHO8Z1zmuadFJTk1V4g/r/cCMsupDUYqkRolffWelqQ1JHpNZl35fUd+ZrKKsSUGWo+Wu3xYi00dIrdUCupPSkEe5wq6I58+3eQq+ZptcSQ08ddgUFfz5ryY/BfX5Q8Ujo1zsYb7F99BWQublOstTjsGvm76NaXRpAL0hXuQB1SKl2IabOg68IgOkNvXOCxtDeSLRTkWk1sbsjx2z0EyLmLUq90HxA2WMLWE+T3yFPBJIp05EZNRDqHh2c6BQ7VFJdib8fpmEGEzU/Enym/lr7q5p9sv7mYx9YrgiBZHcie87+A80Gbt/mLPDsa2pYsLYy6/Z1QuFwFnlPEhdfR+RkXsrpdcNUm2PL6GvUYNE+0r9j/zRe9k/8J+Duc0Bm/9X8epmyw/KUhNxnirx+Qc5nyiF1zz9UwTyNyp7vl3Q5HKdnDqg5wg6fBtnVkM9MqIZSGV3v0klHE5gVrJXXOSZUeknzJYeX2x0HF+EXP/4Z4fUKCwAsQbf4Tm4t/lkOyz+8ygMF6gnGwT2f5Er/71BRcRDmR0lD8dRJeZ709MinJCoi0BzqFyY4sWkDtdgmJxPRhm2GnmXziVlcC24w6f2NZzYn3vMuuTQjLbRn3GYkZVfnZBnYhxeh4HZgx+AFtgACcnE0DaQ4Ddyym8qMvhXs6GSYH84dpX+gBbX5P+Ovbs777jeN1FlL2M34jdF+2Wi6b4gvyF4azs2PoJ9gBH02CRhxXKYxEO6p0Mu3FHHg0vjuhIy8xoGWH1GbjnNQtM1AVmEmrGotuwDi7EyFNL56PnkDpvNgjQichC/IKP+AeWKN4vOSoNIEMV9eGcUnkjVtPSItU+KxLBaz0WFveqaJp31b/BEtDp4F39w5mtHGecIJRuEN1anHT9745k1hr3nnhqRdm1MKIWArjLcyV2XWsTJY1T/gh/wAlsO32SLqy5ij0VX0UoxDMEZwgxXvpaaAVVYARELRc1X1kYLj908TIm3jubS8/Ucm7t2/k67Q13r3roK5oRTkLAuyvLWPqCiYXC3M2YMVCHFTAJWmDxuQAXIKBN0Tjz9VHi9Vx3/SS6DaTBQ+AEFQTLCqfTHnhFyJidJIibwO15L0MPbXMZphAkaHgOCkArMKAcpFwaQ60lKEUWkr55MNtApFysTWFilnZusLqcDhBtY4vKT9CxAQ9zaIUsiczsehcVKjXNXwkGL1MFyjUPp97waXmKVdbF66So0o2jneLULGt2cj+n1MDn7695pe3X4+f0SxxPlR/0VSAu4kykbyvZdPZS2WK67vWQrtvWeSsbJfRx57kLdstAhCmRwEepcoUePbftKQ/jl22zHlSHjUls2mA2gBh1C8IJiuoH/DAdcX2tikq/rM2BnHJ53KgNxmRncSRXw5f7EDpHfrcJ+CL5iARbmUeov5EwEa8E+/NCrdlXeE7aoVAICh6chA9bSaOIWS34bmOj3EioxWk0u4yjv8+Xuf8AzpfYTuOVRuJfVjE9+C+mhWnBr2fwTPzoul+lRARP3AsefU/GqDHq6VpaRXnUvGQotjaaliyYQumBXP+alYDoZx78TPIVAmlvhN3dtYqL7c+c1L+5RznVq6k5Pyphtift2HVdgYzl6T8JC8Ll+frZyiHYf1D1MD+SWodlUknjNzMBTQcNXq4oNN0P62c0g5gjCVRxZRILiC3do7kq6xKu1azatwWmWAm4G0aj5+4u52NrVtl0PNbK2s21T/s/osYJVku/ypf8iM7YrzkomGfp6ft9ZVFD5DekMrmumWH5Zx7Hglw//9Xw6I7lujngvY8Bh6iEQ4DoGx9UAwZQuiI9oc+ZWgL5fnb7zX9VKyi3FHcTrZt0L2VEfwEdYE8WYDXW5RdgblRzEx/zNzYiJjegC5eY5cixsd8fY/5ML1zu3b9T/gYPMc/tq6Jm8dBnXQL1ErcmE8XinjW++LmKndMXNJkdiKBCKn0a7OhV/R7qQ3ytuyELOMeFmoynewn9xMOVUF3iJjrCAlN+OhF+JLC0NhxFaX7mU3HzyYJDk9HQzveLlZ0oB/AAk96F4Oer2xu67Uuf2iDcIRWB1Vs8rVbX3nNgEjY98UuxaWvKBtL5LY2RqMNLj16PD3jPFhuvXCQ9U1fNT5eflOk9KsKsQ7ITlS3Yr0Z7X+/zrGjUwQs4x0LRIRr1l770e61p50LtnwJnP/sWbA7U475hYcpfWbbT/43ujlAviZhu4hCO2jngQjQXfcVB6WbuT+xrsmzikOTrGmgMUEMBPD3MUWhLoZFGEHQ+8Qm9phhJGcY0/0St6ETj+wLk+BdtWp3YMzYtFCBNeXGJLA8BeMEEzUIPmiKfxUqDClKpOma5LmALA0rg4P2PUP69BZQtLHV5ef9+efsbdUFZT8Z3mZkBqGMHhfQgPpK1M6pGLJyQ08VJQVHywO1B7ZCYirD284PknMAFC4bvZIS/e9yMedSSAfCd3BcDf93fHEOJXFTDv2y/SgfEFOzwjVBVxi073UTyLxMREhK4ftk2v3KZu6Ef/zjHv767jxgtywalsdRVH3UlH6DeoG6x5L6tcaY6uuQ54HI8G2DWWX9F9l88nqQF6jHKClxXmkGvyJfclc2laNVB0jGMqP30oWpZ7fbog4JanG28jZC8v79chLRspktZXNPk7owDu254JvMi91fmRbhKntn8B8VLpG7qtd6Es3VosNG2S/n7YkxIdadIcCuxInL1i11Rq8lAu5WQtzgyQetXJ5b93QSS//uM80utS3NIO7ZzxvUFKnAIjp3FOl1kEav0NuTb3TntWuX1LVAth5fQw+hTpQmQD7Jvzxy60q3Nlxddn0O6Bm5NS0rmwzWY1J8FBKK8fp5g2rO9NA4rArEB8WQJJkj4B9c8qAPfbivpLDJ1Q8jZewYD68TK+xef4ddKA9daxUQ2xjHQAjg3F2WorzR1fiOZHXkJ4hoJWZJ3DB0zbvHoQm+m7UnUllM93LG/kH3fsYc/nTTbmgIdZrwGXhmLswuVqT9jtd/2/HCfE4ihhmpTmIn382V597bnC0s8Oytmq/wF+kaGB8VVqYNsubqAmOpaaoe45A155FTeo/v5xTY52XoDOtRIYtvfEtGmQPu0/HxCdo7d9LhIHXgwAcf8AFcKZn25QjrsO7q/yh2evj8etfI9BjdBWcg1dleZtq5Jq4DXiBMIRVGi7uIkzr+WSINeszJBTfNMlo/6Sc9XJTV+6Ou1/cf5xeLUMFBl4Bl7Qi2Tl04Z5a0gMb389U1PoeOXOwX3+RzHwq3e3u5Sct+LQxndMmehG76H7lruFO/sRDAGtON35OrIPTtSrj81Ys/bhb4R9wF7lsTDtCev2KEwPoLgJCuMH2vdNdtjq9MFsF2eKm4Olj+ItY1PsfaouT48HS6jNItdDsRihfvTVbfaAS0izcCdpQqckUZlK0vo8LE8fssAa4xB8M2LQShCAbYXpJ2hnDGMNX6KeI9hNjZNyDiYbCgyM9L3ZarPXJsdyfcI9cI8lxHyNP89X9ofJMluh4QaCvEUxa/rrUDvgxIcVksfS/9yc7pdjvlDL2ADHvilSJfU+Y4qouFIPueHXBHhJ/28tV4uaXWuaNSawD+J47vdYzSqSNMCn6cCFgumCyiOUUV95GbI1+0IynV6e8qs6eXQxLOTEcLHwzc5N3QjKs29SLIN/m7rgrHWYidaAwqqgYJnSv0Oted5MDnEskIVzR3UG5Ue+nLgvViv+V+pMEjSWeCA92TKKJXn5eNXdeRHr4ei5QN0jxntEkAN18ElPvJLSjWV/fTwQpL9IWFTf9ECvLtIXZkxV44+at14hcaIrgjkEjSODm6oPd6fB0IVUKeDocyGKhEytt16zvvdi1QB4pMqON5ioNVKNxFo5MH5DqHElfi2UeSGglM8QvtWyGjc/PFwdvlUbBXJYCfVNtdrwoljmWbzVxZC2sbveOTCRAO1rJ1GYnHkgQxeog2KSWRCeUp4RvbRTeQa/MNiMeuFCpkv27GPNqWFS6E7PjV78rYx5CsU7AX/awhWgyvG1wt03ow+iH6Yf9djpEjwAVFffVTeKD/bktc71lpZ4QfQmhC6QShOPUxSc9endQcLqss2G68U/Cf/ETMIU3ygpVTGRClr8cCKCgNRH/2UUy9l/GsX3OexI33rzaPGIxtbwe4qMqFA6Kc9FAVaNDIo/iJUMxGA3aX0P91sa9vINx1COBLtQY3U7Xjxw0l98Sw7POcXqv351BQxFc4oXMy0ulKM89K6wxv1LPSsxfgeL2XaEWfp6xNkh24IJTw4wF8t8UKG68C9zUq2t962lBr5TXyEtJmBUjwR3BW2nHuxmRxpVfqzJHsKdzZqPcMTaf7U2P/lFZW2pV0fV0672U4yQP/O534cVxrZWTDRbPba6njHWFOADIHJKlxCA2sCoZCWiu+DUt2DTidwh0fsuDr6LhEAiGcCd+dNpLTcERi8Kis+KLxo9Q4t4bYO/HoseiI+x8oZ0Vm4XliDQKjymlLYzYQTArVVnDOZLftYFJIARmrzWuophSEb4KLNDq+tR2urc0BZUG1nYmQSsuel+0r8vwBsYkpjbHP+OL0+lgIhe3JDfUdUIQtb8IInEOz7fPDSRDU6ZAt3wl2BdmW6x3oemmEy4j7dF7xG+Zf418aEKv/ujbjFnqx/nfyKeT6wb0EpxBxdLhcLQKsmOEp6ABGkZYJxEGCiKuwKoDUirTL+p1PIHWUPGJNFwhF+39gWAPkrqqPYI0a0GkbKdhMRRjpcRCXiFn7Eu8/jEdVHA6QN4NYIst6ZXigdYYkOEr+Q8P4DLeehb0HTpNzwX+sZDG0xwOOgQrTqCe4BEBH7jIB/WwlSJzMvDWESnvluyzGjLcVWdTqf67C6vhTOmDNED4tdWZA2MZJ/X74kbDC+jq3C3ft0Cwh2RjVSL/GWKCEewdmk8qGrl+4woMCrZf/P0nVsS4rswK95e6AwxRLvvWeHL7y3X//I27OYmTN9qm9xSSkUISmljpZXyc54+wzBQ6qvxHA9WUm1627Or75s5j6pVMTHPl9x3CFUMao2u+wWNrnGv9k9kCmmPzWmf3oF+fkNmn7Xu67hr9we5NyFv5JCEnUY0A74ZQlySHu8NdY43KNoHnpaorBW5kYdU5mEQH93EDytoU88iyzZqi0mM7bv1hVN9Y9/8378lzwojWhKzaMSITPOnPrJagCWa33ytxRgpaenjlPcczSVKj5/Xok9B/05ZmX20XH3vA10f1Uhw5WYbf5NgsrZeAvtSZhSV5oFWjNtIgevr6f5nLy3E9UTsF4I0ApVf4TkWbn5b/Jn96nzjcwLvnRcTzSCDeQ1Du+bbS0UZ0jAyjoF4Otah4/r6N3DI6TDlk8MH7o4hanU63N6fCzCkjz1YCLQ/rkRs1cNnizo3H745sPKrNV9dtP/xRjHqsF6XPe3OFrlHkz7CK2XdIgxkBxTmNPefiy/FMK46GA98q+zJCI9tKcEfzREAol4KZngAVsuHO3GsMVZyihL1vSSAsS9aRHu34ew6g0k29E4IPQjCP6kQeAARvYU9fEx5kbtN28iv9C8RKmb2RvSb/mHTM65NirxdFwsUA4fR2p+DzyTeJEfDrSZtJBXqZTHC9QujGCuA2jsTNug7JNf08x9KshFUiVmjSEqV/rH49SCoQSj+KsDIVIpMcRXtUN7t/MHzg4d1+8nJxTyPV+avsPOnaNE6L/sx64bzV7RyoMZt35/Ll8zr2XvJpTfIj82ltTXpAkT4qPC0hzfPmsF0QCZpL58y4sAB71Ua8bSIkfhc19If/W6Iu8Ihi6JahABjg3Rz02qL/GPOhzMfJTrwm57QtZgGwKwBIqlm68wp8qrhIqV3xagI3+4jOkz9+WLbLa/f238YzaFvPwD5Hodc6lavd6BFuUmSUIW8h5VjTwQCqM7yBY8HD0QJNqSD94FS2zgmFrajkHsaL0H61KI3li2XPvVVV2sR+mXsnpm9GRBYOIcEvZHF0pQIuEJGdlKZ3hctABEiVe1cM70apKv/UMhigeGlPIaxIY9PuzWFoLLUTz4szlMslkgdDpEzIuxJpXoIfADZo9uWqiv+543QHlQ7iUk+E2OTWHpGwImJgYk1H71qKrqw0IWnPUq+FHMz0Ymps1gA5v6JMfxmIgKvovLn6L82PR+VvNzatobTviF7tUyE4wW9L7SEfRStw2cQEh2p0qvsIRq6NcsD/v4xMtXWJJkGknfEHXdqMDtfHLyI9zn+c3j55OVJTbc7yIpZuexZkfVsdA1u9g4zJfTA3fQgzSHz1kUjU+KO3U19skJs+GJpbjvGUmsN3FDJOqy4V2kpfcs0uTfqNciR8j9DVxWIm3TmFx2WgjlEcafu7v7s0vuYsoIPoytRk3ZcbJ1LkUHL4uZxPEZ2+U0K95KN8C+6n9Bfc2Ngnm/x0zqRsonpRYcz1ld69M5LyTHuDLYnJrYatCcxxvKq3z/WoUlzUstKj8s1fQPAXDopsOAKCo50nlTEtdL/+uTj7NszyhulC1qUCfIvgFSm8GXXrgHhUZ6cYwLIDhIZwNfTf56erNmDstVadqDd7LZzwNnYsdL8zTiQY0ft3PZptyYZ7lCIKl/OxMUdF/b+6vnyp0T8V/BGAg4Lr7RQjaSmbcUedOVLNAJ6T0kvC8uYfh3HfC2xpSD4ijTGiSb6QPMD6DTl7Y9m//11xWa6/K0CyP0LLwnyMMHxISpXiLcVxayNYHjPKiEseZUjkp1toxM45mCVLzYhh97WKCyaDSIsy8Tg6p9vPQYX/4ue4BfV+wGnpMk9an9jtBxKfhby1IW5JcWv138OZCyL5DFB4VFErykndwNTaEro6fNH0M8dYRtlMw4AfQTTyGqaQqnrMxRMUrTwhxRiTl5lsnncDJH+o5I1wdQ3E8RI3EQIDEMTQievxLU3Pv2k3cYAKgc+y8v9Tcop31ccGPi8rZ+k5hdt+t0EreIVl+Q3RAtNPqudoxvlmkcZQo+XBjYq+fLa6TDtEei55JZNL91AgfxMKIp9g3f/XfO+BWbVkw2QPAFqQrJ1+Te1vPbTFMSZczUqHHXX+Bj8bGPOtqfTtEWuWbU5+V4j6c5/H1q3lq08AR5RDm2mdmY4D3l3bQJgvspP8/zao7xAwQ/79rMttGZiR64c1fs+gxSK1S4zNhOp0OkTgufhrCEA3s037gPBRKUjyVPncc0/PLK1wJqkkLgfXOFKlV59h/tX2n302pm8x7ynkCf/FWxlMZ37vNUlEmdoLGhxCiqztf0lB/R/nwjg6otCbYeVlo5eppmWV0jgamOu40vt5npbgnXKxA+ovU3XcM0L5rXaEK/0LgR0bUpG3RDjup1dW32x6dnKNq8pkIsSQeTKEuF76eD8eHbAfRX05W8i408ExBVXkn2d0kUFN2q8i+KILa0YY3GJFqsqo73odviDP3wNho1MqkIse9ut7UKl6IfScEjnsGg/Bb3xsI8+1Qnm+pCiSk77PyhJZ0eEUiqteDctM7cLtr5BFltYYtXyhuPFg69G63VfAcx4h2ppH9T2J4784lBpYXT1MkhYM7FEAmSytx1qchqecTo/a4W85/sm86a0dRj45xM9bRFVcVwkY8ImHEfPxCkNstCjgndTdG201IUgfdHakyW+14YT9xaLbkQY2Ff3ONggqaeqoK9KafYlmg6/R/RPV+MR6cgbhwLw1pSAOG22G7ow1isB6r95BeHvrppFr38b/MIfy3DN662m17mC70z83zfXe8zX/tjJga+qzXHG/XvJ0L7HKovDcIv2Fc8YiixLDb7aOxpAQraE8LgO+sxtP22QxXpvyfuQFm4vrFmQPyWVPdmryodK/s0bmbGyPVX9OjVvfdGfxL2KbFBisr+VJM98UAXkanXtWLnASrzB1SxoBN8jO5XLy2Hi+GB9YYiiOr2mb9kLBxNFaXKGF04258GajR6NMRVbwkiGvEOR61wV998OnXuh77HCrSG2a8/vP8xLnZVHsun6GEN5LO4ghD4uB8y4EOONNTPWtVtpx6VUP+RMvo0iNJE7xpgDWS1DZSvPS0Jpj9MW3clLzAQajy6g7zJxge9Ehztg/z2FYW711dW/PVvB97n9xUqUreLTT4z1Nw/TJ1Ee/FGLC/8jrLOoYD0cDkf7wIH8VNM6TAJNO0sbLFJoquTl757fNsEootnK9rUsxeLm3g/CQrezgZCgJZQB4jC+FVqKWKeA2vjZQ6kX/WHZL2KSfxB+X2nJ1yD9hIIZdysviuweWTEMD1Y1f2iZa/mAArwSoPg6ot+yl+Vw4AFI1Kn0YMeKBUwv892CCQCSNhr5xTF1H8ijqEVuHfJ8gOXVqLJeKKziPk5QlSHV9qoVdf7IT3AUATWU2t62tXKMy1aXEM3nZG0IgmZfyxjRw8XZZ+upxfd8vyg6naHhMYP4TNsDrMTRzU5ZxEIBVXkZr0y++/q5M6LGE6Lamk2mPbymT4BD/XZ4TcmkmQQemrXbPHG6k6OzeEenVxff110ozYsDSHZVQ1JwL+AAFtUKXHWNcc9UzKam33sL4R7Qs5K0yA9Yu6I66Z0Tfez4E3VLcOw1JazYqp1QL/EG1tBlLep3gkKj0PHi6t38VsifviGQMikVNXQN7LX7LHs4NwdGh5TXXHGbvE2DbDuia9NkKQ5tN/NJ2K6OYdhr8Fd3IuKfQZZLofGCgmcHIJwsEyltdbVS4ixYdJkIiX98HMoavRH7M6x7i+Hup6iTx7Kyz/+JLkf+rLNm5KoYRRkqP7WRtGM0Pt1vyYK/Z2pZ3XVndbXPCplpOXJNku+usyHai5lezL0yxX5WyMt6L0qnfIQ8n9DQhHWnVy1+prHVEi9b2wNkc7nJBPCDlPMyMoDiR5jm1B/KoqP5EVYP5/y7Fkyfb1OW7+8Q0kt2/wIrdmR47PAC74ivWBnjoaq/WDDH9EbnuiqJ+gfnFmxowIGJPaystcgYMRj4AsNF5Ks7olVLc74Vgo/ubbJY2Y7Je7Z+gBFQlc/M+FHGHa3qf7o+pJkL4WG9VYfpZ/UncU7ztAOeOBvnlyiVaahqe1Xy9q5e7BnfKG99Hnp6DDuNoK7hiptoUGSiQ8B5rD9gsoO0E2esMPD5d/f1HH3+/KjUfkeYvOKgC76jSj+NQf/VTy1I0+sboXxpZZ3kquDZTS6Lim/okkzZhnRMzTOePL/NtA87ZSjIEfg78b4sjNwFJv3/TA3HXswe2Mg92NvH8j8GcHucqtjYCaMXx9teflpSRBdfqri5zbEtkIo/D5eVKVr0HdMPyWkximBioyWcGJJJFhKp7NuJuJ5Tm5z1fzUuWJaWyNFKKwk2hFDq2kjxbUgakkT2xDG5nNj0ch69yud4Gcz900bbNaZ3F0o7SuClk5mGo84wHN48V02gd0JfN9LZWhD+5Bbl+mh2ZeKE8uF7GVhG8cbm+oTfT8OLlxkr2gEQs4Hglrx62BEOZRvHJHzfVPxNWpSwtqsOZmbEro/LsSkbFN/x+vhi7hrHTjyVq26xWHYo/WvH07j4KbUyuJim9gPq44AVLJxjRQteSyHoBIR/Ss5DSrDyqRIXp+2GChtQTmEZ6InO2ZiLeHhgzrwcSHvF4h8LAwM5y5imHtCW5ujK3j51+Z7nTuFKlf6OYzVXlm/V5C8oRaLQc5mSWOJpmZCQMgI/7369VR3RK8pQG6bgJVfKva5wqoFYuGNpVc+BMcsnUyCnBhJYse9+rS0jTMIY6yLP1jLGlsZNDFS1VFuHbOjX4C3IstkD9iSxcHsgUTpeqGl25QwbsZ7ovd2Gnv860TxrsixSKXR86OGE/9bD+RTY632Sej+kA+SV0yVBKL11w2mcWZTECU5wrAQ3CvSHFfJ9SB1/SHJUFFpikG9cyIAslmZIh7tBJdzR0sg5jHXwCQP95LZBKTFxR3KbEaGP1t7/jrk+4peDaKtVoYNjKMmvdo6zrB1jQ0pUWFTFC+xr9cPDplXa904L+8EMrH5+Pd0w0aDY2gM8UCsGbcyHuo3F3Han6H9fb09ZcPd+7KC9Q4A3JrkS0IILYYq5TTgD8CVIJpNKbxFHglHusFVUvIF8FEyYfO0yUwIn0r0IwTqptmVBuPl8JWKsQvmbrjK2PQ8FN+rbBi5UpWoVUDEJY1MAZfsSb/xF9FZ1AURA84XZIeViqrQz7OQkVQFsgqkD7IRjnuboyZI3/VFJZPwc4TJQGN5u/C7GGmM/r7fVxF1jT2x0Y8P3DFhx1WHfhD0+8XCT3mEbzmDs2d7iJSQZhdmkCi9zgNVZFUkCx+DDeHILH4lXxDzy70AaeKF6znUFS2DDoIOEfWrcLu+2srlVRxAKmSFiV0txiOQbHbTLJWeU4s6ZvvJ7yYBAm3hL6oZWabd8MsjyeBBj7iQb7A2ibT/eJrbCo7c0V1IVe6rbtrWb3/UvTpak70+GbsTkz0t70DLqxhfkusviC5RJ3OdP0zTK+EA50MFHEvFbUNhINR/zbLCTYxmbOLXqjO47sDVdoUKVbQytP4GxMWFGpl69ecLEInaDKblxYqbJM65eD3dfbyQDAFHN/RPar8K2CP9Q7e3+7s80hoQayxKh8AM/bq6uvFSTMcowo+OOU5GkQ/TH+Aelt6RW1kY6PF0lF3z9MtAkspc2HBH3AUEFCXOpd5Zn7TZPBeaS9EyG99Q1FJLyBM2oY9ErC8kI5zGBBdiQZ7RS1U2cUv9g2OaBRdXHH+147/eFm2pWP/puiqiGCoyF5U4M1NhkA0pIbKQX+laWnn8HKjePqg73vzo1FuukNV29a2WPoXifR0Wlx0s8YCep4uzeQZhI2lTh0Zoc9XZ/BvJkXtHH1eRsARuqJyqpmp3yFR7bYlF3E8+SBCE8Cggu53C2je/0VKihEyP/kpCk6Q7vnJxHay6sw0iJGSOhfraL4esYNowTYsgDBgnJArnKOFaHszGPaZROTxHaRXWp3Z+XR8aT/CFWiq4n15WjfLmCKl61Tq02/FSOolQthoSEuWH1t6/C+Ki+MG0k2MekgOZm0jfcTHaGS+N5Db6rV8EG/jN6Co6vjlsSbXrJLYRPlzvxazcu79DD0YlZuxtVq8PAvwC6SMgSnugqCr5fayU29SLp/gIM1VTFdXBj1pn/niLkIb0GmwllbwC0c88TDfyhTOrMQY5CjgZprxO8Sn3GoJqeYVK12/ZjnrSSl2VM99s67+DMdQVJRzOHE+Pi7vur4S5rd7kzgkxRUJVVKuTodfXSlrxHlgEP/e9uMtp8wkoZHnl7qK7oWttVxr37sDG5nOY3Ddcv1m0U/1sD/BHTfbrgEQXn9xgGe6jtU7ub6owej9pO2QHuvb1oz84K9hqcrzyzvMJjJFswX8DkYeJ/T07q0Gbjul2YpD2ZH6nCqsEEteZfvpBAuOHDAYTtn/TzyTMilj7JiEZNbiNsSi0qdeX1Tpfv/7MLtbHl8yV5Q+aBjuz7/qN35sZJtMWz2HaxnNUy7fBCzsRUUEoHSrfVIry3FRMBYlG51t9LrjgGebD37QDizTS2WCdKf3wlGXYuU/AWXJIKs3R77lXd7RiDzB8qfqbqLOav9kcJoti4o6KIIH+m+htdlQxhjPzKHaHBUrbWqDErioNZl5KMzhTgv3uFn95IZeSqT7EjxRnAp0WcdWUjYaWCPRtDKSBsBzH+U4sh6i+2/7inLVFLn3hvcD1GeKP9pfxR4EoJ4Xx41ycV+D4IxG1zaeSW9oZwuaUOMZj3Eou7RDysV9lQToIw27NkEcDkgBIGQHoeG6LwjhLrvLaiwK58zshPO3dKBZrpSQja8Fn/xaXdmA5Kv1pPFFcGd4CoYdWmQMcV7N3t8u3wqt3FQY3qAITLJRvkz1BY93Qvzi0yZrrz3a/Jz/2kZ+oiqK7o/O2PRqzs7DlFcY+4DHgp0b1ZWmnOrG/wz+43XbKkh0YjaV63c0wtx4+Cs7YdNVGU9emEIuS2j7/XX6pFWY2qLL7+QNEYoGEsu7wInoDVuMYaW5O386USmdSePlvUREARwh3HDaYyxMN1QqzlRgTSr8dHN7kZ9DxopwrPktHhy8BwKRKry0kP1x355tVEKRo4lonK5tyVwthbVwaE1zXf6nYuF1QP0UovFU+bJsUZR58U3NM9fUxIdsfGbHGHpgcfAr9TXNCgG9ghy3tP5P+/RuSWPL90zvviXt/M5RghQASwbC+9GjtGKVfILyBv/GJ9htjNsXrBqMPZar9eV78c3+LjqbqqEBPe6JcpqCbt6NZj9ayr5bzpKwDTa4Lw9t80frBoWgZVcEO0YELS7TgKFYst9n9dTTma7huJQEDammpLiR0WFOZ2zlaaMuaLsesI2YRysfHAgGVriwRSb+HH2tF8ULDXwKa+HtWfuPA/zRz0HiX+6tNppcvCJqfkuUA03sh4JuJPSUtS4Rb6aaR+DmhEX7p2f1il1m+zkfvJVA2c+2apCCFD9VCTRMfx+REK63Cxx5SlttOhlltjZKlO65DHV+VD3mt9v5Bvo93gmTZFqNqrH8W8mczlKpAkoZi2o8Pl/Xiz8sSqDNQk+ECtY0g7VSdrsY8BXxYfwMFMf2N4cu7tUJ+ZvPRboY20Yxp5Buit8QmHFFPzlLJqi/t6FvpmJB3ggQ+2Y6gPsAHBCyv8e9aFS4P9lr5rtMEuwiyOpTGq5xckbhpg6ZCKhuNBrx504teVYkmY/asdnqKxNYlQWKMka2XFS0MEg8jja0c4CIhPR4srpApbHfm5dYWhasGdzxa21a/AyF2oa/LSmy0j9aimhlIYrw2L0I1vJUmdnHOzA11HRDPjuZODglFgoyAK45kDLbPOQcEZPQrGPpqtmKl+ytB1Rt2VqN6VvWXIsBQa9qWNHpETSF8ddGF1mbKvKHp76KBuPCCL1cv/hOeGdttdl357lgUJbi7S3PUAn7hkHjoxNNjGNiN9c986NEsGfTv0XzBPga56lV9aLZp1xKtNVETQK7horYzZKBBgA80SLE1Uf/GKg1DB/zygFnKsb9rwvtP6hDjpqlTM4yrd/b+eZ7sRhNLKj+LAXplc7OPOI4eqVhQv3OA3X73ssas3X0QnkOvP0QwqYH6g5KGFifoaJzr+ZkC1kIBaQitbX9Y2z6gy34t0D599aXpyi/p6SeAPBqvjxtGH89GD/wvOzbMTkafkrKW+zOkD2zZmBpGjfoKYc7y+CIA4Vshh5cPoqlfYVDrxO4F6hqNDOreAONDrqVRHNXpyPgWnk+dlNVKfdDV5xsErl21qMrZX6F4naCPnR5Q64yaRdQNMK7guFUQ6ehvNfUxq217LWHxq+ECTABr9vBfwpG97WelOXswRiGma+iH0+BUNFbVmK1xoZE1HIddu8mDLZsETJbTGlU0bYVRYkbGFLi5DEhwYro8z/VtIw686cKstrMXSmZRp1DLNtczGfHV7xnFlX1RuxDltT8iHf3aKDaYafaBaUFX0eWJ/u4YV5ifvI4hYn8z5cqENgzqpZE4NuOx5GWOmiWux1Djnspk6Bwb/TcfgYBemouKVQmDrmb+YE7xSZJn80bgo5jHU+E2O0iVJLo6MUD2X6vp388w2DDLc3/FmZkCFoAcSJqHG/xX+f3uzlnh0MrKyqlRhYyGWfOaGeAI1EBkASXRmVyXbpzt7GSdAiNx1NZ8YOZvVhGAeu0+BDlJt4uOKvBkXwqF9+qWvSgX6g+RAQXCG9uR2rX81IuiTj7WX/eXVGSriJKltBLkhjQnS0zHnv16s+8CX5XVtX6yDnzE1p7scnj8w6WKrLH0a1DuK3MM1SGp5j34l4xNToOswcs0mk1ZvnuxxJgbgW4887VHK7mCO89bTVeJNUi6+wrOw81I3KdZ2f7Ae/8NOIFIfA4SXHJpYQonv2osPHk4kB019R0J19IbDe4u2iDMEgroK9D/AhoPKULBQSYzyyS5U2aKtZnT3qDECLv1wU0B/5e39q3du5bQHjTq5zm9ibx0xC+7sPm7u/gR1J+EMK/kP+geu6GDDSnoO9rV1gXbhuR7IIXnyFgYFbLbX6LDSjh7guABmMZ+0QJ/PpchDcC1r/7KO3lmFlkupAibqvrVPcb08kQaVVCYRSocvyFHgl4H+fi/7CfnJBsKxCeuw1HsjIgGd5vpEBMC694IvmkFzdWgnqBNaz+E0sg3Q4gZoR6lU4ul6mvSKLjgLbZp+VXsfbC/9j1WhILOUitVZHYpKlpWnW5f/JWpd6nK93voZgJ4jcyt+IfU+eLV+neqWyUrDdyRUbUBoRv70AMA8fICWLFCGLj3Q2tHlOy6GliF8bu17kQ1FtH20TmwxLy3pMo/lRPfBPy3kRy2Bc5jh5eZKUb0JYqo0TrhgD1y+s7x/tqHHqMobMuwvA0fIavfj7D+XOz3kdVU0Uo5B/XcSlAgl8Mt2MuUGf5c9s8gURZLJWyqmh355D62Jtr1+5Fid375EkyJ5JNXpRpeO/cjuforMThik/xc7XquXBWSV35vQiunn/uHNcivWTKsNpDbi9dDf2Pz36hSUYeJjd40xlLS+ZsFX/hXF5kB6hUFkM0dceTc4a6QsJmOdhoDgu6NoYyj1D+7COnjc480eR7j5WuCO+9SpvwEiIwei+S8k6fGh2unX/U17PYpxqcdiOVJC4eFNFJE6H42Io4wgNhL6cDn/sa1mIO3zULgdCslEN1XvIT0Ome//KkNK3xfDW4kjwcmQNBFNBvCyfUDkMO6++cScVgbnQWjvyJQSu/7/a3IthYj6mHBCzXnqz2WH8dL2gF6tM1DnHBNaAr1CAqkGY7l7vig5QbChIZ1feLY+EQISSYhdOAKQRV/Dr9L9vIziKBLEg1qKlckiekO1/bGYyzIhVTjl/XMlCpYeYqx4qRDnVMAKfz3V2MHSbAwqUbvuflra4urcVQnmCVAfv6rVl+S3UoiOr+SBhggnkTjuEdIeXUhACrLDA9AGWASfmldgGVGW4hNlvwKqEiEvgLTRHhdEFwNRdV79fPrE7sdt4KelyWxOXe9rEe8NWvVHetaDfaiixoLi1b5cRaUn9Bvrvh4/mQHb7xCEEC8TV4QvzK9Kn3pXbtNf2qrYcLDaNawfb1+cgKUECMauLnLiVbZETkEn9ez8uVy0YVs/GdUOkTBRvyH/LS/ZeSYl0WHEf/QE1Opa288VQl3waPz+4BUXUqnk+DU0LcLSATeKfwKWprJzDaCM/4D7G0qRrdKoX+Tl/lNVHLh7xZamJijIlWgFJnwiwqcCgEIZhThCYWZRB2xvKEPhWhYjgptHaDauv3y3aZnLQoanqAgdPzwY0wpJhbxEv1l5rqcNZnfwC5awPn7b4X+iUed9Lo+Ie0+2njMH0aF7aEFlKW1MsDIvlm5kDFkzbmKLuj1BYiBZ0AsYVa+Gxv6wAJpSVBrLvl4mvOL2ynqeNzpIL6rCvznhOq+Wglr2FZwhi8NqJVsGXgbRhR5GRIZ9xAZ30nXHQq0iLCm2V5jEqr5RHh2X3WlJPL0kR2/5UMM08LAO4NCurMkn2NBsnncAsZ88nLQGGCrCN3l2dYEFQ7YpuQPf4sOqLrbmfCTpKCd6FW7l9MB6RJ4uaDq06DuwotvRWs6suI15hIyZ23JzMKapprZhbrWJsgQXG/A/oYxhzQ38mA1CZgC9p1Idb0+JqfFQh6DKzwaw7VKFYfWQ/t5S/UFZGeZV7Mj436XyKMMGjvqCzRygqQgI+48RlMeO/8NW6+/y//+Te2O0Hiist4VhfDl5PMB6bHvmlMnzLPyK3GvOcS4r/uuW/fpqw3U5fF7YR3KEyfciRjJYF4BW+Fp/5K8sv55K5nlrtjPfJyS+Zd+oPCN1r+2NFro8nb6qvhhpKm0fx0i7y8TATkCx5i5WnnAI4kd3hRvSCMZkP/4o8YOTSu0t+HsC+yrcJrgsfOeo+jqanobvYMZ9610dtednq8RdPvTSNl82PYFWrLdTkS7vKBmAaPvXe/4ondOTc+HSBSqDSePhM2UJ2Hyr2nuUunXLfq/Yj5Em2KPuxk8KseBpcJ6hIOas/1f6w5OVZBPVNUH02DkFldZJiq1olD951AcW++u9HMsBdqZxkgk+Q00t1geryPyKO74IH2dSr/ju21x6Dm6HzzH9Uxb3vz+DuevLDNz8oSpN2aAIxxgefn5HTQrZ+A9ag25HD7Ol2aSN7SdiTkgSw4xS1CsNAXXjhc/CCRjP7BI9jOy1SFafpxYU7nP9ySvy+/5k3qvRa62qasFwygnXNwsn7htVjfMYDotiBmxc+z76hk/GiV0Sqi907hzfF2eJP3u01zQh4UDOk7vvsp5dXV9pV9HvzHBa9YWXKB7IaYEnssxAp2ZnKDTweBPajcRNi7fIIP6EEdmuDZgrgb5K1Ry/QNsW49bIgkTMBefV3SAE0NcF8Gnwf+4nFZkH0W36Pl92733KCny2/t8c5CUaqmTEw23BOakQCVJRZrtB0VhfxHzgJK7E2TF7k6ve8MhmjNGNruRyi14OOJGsTV0sq/xlQS7lPiX64KRmsH413cObIJFvtMoC+hG5mH4/QbL4+n6EJkeqpaRLn5rTeGwJaQz0R8SkFiilpYxtAC/OkwAjUfj4RmhBENZ5wSOwVR7Vl6922suu0wE3E2K6/Q3R1vzBjNk7gwP4Sxs1AGr1s75SZ1WeAXk74RBvifuvLmdn+CuNgLXpo7mM+J3+GlJuI5SIL6dVFNtidVCob/WRYwP1HhwDprgU/M1e4cUhYBH4p2pEDGnYNrfqXUWTEu05lF5fTITtRvpX5WB9BsyQ++//uYo6mdaJ+101/5Pue/5IoMAWZYR1e/z1t3zEmAjaHvuFIy/JRnfon9fpU9eaMfleJ2sVxdrAnru/uz+KJCuoqSOUiOBkmk2ZAURY0TXxqBvUyadGz01Ciaa0SIygKPH1ZEUTdtmn7X8HIsD836nwJBcFyPpguz5GOkCgsv+6wcKbkafTlbGXw3SZS1xQWNb0BPeoDGvfRpr3WC9/8tDdvMzelgcfT6fXro5gf6bMut7rhVsEUxaIIUhUVNgs1Vbzr38CNDaBGHNN15n0XW00Z32G9e6CXC4bDs2TNCh72zGKnJopErVwo/c7MYI/Nhz9+iJb3SSx8LsQJModuVEYW8n5At5Dl+BkdeXbLMAhb6jsK0iitQ9F8QipoJeM3bMm9CZMw8FJc947Z9ZrM1XzAgMMjwQgfWWQeB1xbUNyPqBjNd6IvlCmByvuSgJ/lBOHzbByhBX7j3vjskvoRU1KeRvmEL8gnV/QooIzQR7eX+bZ+sjzFZ8+LOAivEnX3YXtsk/nJmwukg4TqIzVnRfc9pzEzaoLxBmJOhlrww9PPM1OeVdl4ttevwIob6sPGetXZk/yPv+qp8IIwKYW0ST9NGaR3ROsqPNf0DoBkloB5wo/FRTQo4mUIrXRQdK6ECjRcqLW6nDRRA1CbTdZ98RNzT83TIzB2WkeqqbUHuexEWw9i7yzlTzlIjFMF608LQrHDconSxqCpd7FAf/Qe8ZnAOmBm70NUmcvZPu51bRLKM5qPzS3ktHxSLNynEJBgcFHCfksT9FeY+Ao3Z7Sv+wX2KLLiUx7dZKwr1yD3gTZHcr2abbQfzT2xR/5WWs4I0FwO0z4c+oP0uJRIMYLH3Sja0G3kdH+/3LWkP8EFTFg6ceELqeEmdx+jj58U2UwN+T3/Xx7Wh6HBl/A697JV0vGYFfVuqC2HjmTX/V3pFj9FF8ItzZv9JH8SilRkxy8PXn4t9wTjTI5RLx4DAw4wwQp/XEnWiBTPUbsH6HhJrMy0X0MqPAh7mFAEsZeCtje2XJ2Q5LaI+VyMG2kGakLDlpQwaXKSNpOMWQ42P6mCaW8uzn1qBg1x/oeObqnj42ZKNe/Yv7yl2Bn/+afptqtMN7zlDsnzNO913VZzI+/PR4H1l+kSrUTpT9Y6rocC0i+oF7W36N1Rz3myIae5ZLE7V0aw1ipLRhjg76Dooda/psyRpTsX9EQivGm1g20lKkdvTkZfHUXPZMJkYx9N9kU1XbQSWOCw2j+Nqi8A00n6OoDYTv4nywPsJnYQA9AbYGS7crKlE8jTRnJz6LPjqkF/aDZZTVVi2d2TnsBVCK5U7fq8SAim10ZbjmhgsUwwokNt+FJN0mU9dNcJEE62kN5JEMbDHhOCMio8htR1W9RuZlUIxW/e+XNQRC9+PJsAd1FTYeKt43Eaxb5vkGqMTJej2bSRCoyCKQLdMtfG3C60NS7RFGa3+XL/Ept6V2TiiPiGz0nWF0L2id/aUkRYpiDIvsA8pivpZoQ38LUM5aUmgNvAG/dnKb3PZS4QBrjYcty64XrNU2Zn9P5C945fxGQpZfct52PbxSC7RU/i/vR1WbhROJXwx3nDzEyAlPun/liOw7sShiNsigkJqaujLFiSz79ItyjW6knbDsc1/o+9rwriBwRAqUjdjR5WLuGy8VsG+F7jwmlhETIY4Q0wwS5lZNq/WQpCYRp36R43AVN5L7HVE3ut2MJd4oRfna4k8r/ssko3xPEap/wzhIp5cdZ5yUUdfhaZQ80gnc6MHcdaZwukMxF8oMVWmkesj371317jCeKRejoxL1QyAeIEU6WnwjN7UIRKXcu0RLC/MUUMrAiwmPZ4CHkWCqG/0daddKnFsnPsHSzlvLCG+Q5UGTGP161/Adw/gs1IzVJkETmWhmHq52RA0ucFG+PR0bAUPRLXQ2dNzaao+yNFR93PysRmZ7KK36ujdD17oS1H+X5v9JAF6ME9ptqxeeRNRu2sRzhDLKeS/whH6elAAR/uZMv2F8UJhxTiKoeyLiTzPCsLA5JWDhMFNfhxGCM2wwzdMbl2ZhGtJ+98quC5zzBWyJ46//kW/8kZK1lHkpq+NOoX9sfNTt4U1LJU/wHrVa1g+0w4922JRdXVRN9Hxv/EgbyxxZ4T618QHvsoYRxxjQ428fffaXAfeixoqypSZFx8d7ZxO+m1GMXQob4X4FUhvl7MhTjuLKVYD4dCEU9W7Dm4eV92tE8nOSOCSGJ1sL14R4X5v3Nq+QC659fWe+NVixgJn2Vfar2J9wkbH2s91UuGIot+4YR+u/ZlhoEZ7PwylU092Lc3cJQnZ9jj0+Cjt4PyxCEpYAxJu4qniNaFAmikNDOzzJyURHtThK5gkD0anH14sbom+x0nrtkbmXtB0+ibFJ9lMc3UIc04HBIhHR5wPPIh9CHvUfFBHd1ppwzjBQltHt9rcCncvNNvyXCokNN+jBc34EywM1ZP6NG9CT5TaktFEtP9DyevAOutc+MOy6zECsRYtlC6shwtZU5skkE5Y8598wj7box5ZypHYcCXE45lf/49IDYByW6I6Pl3RvCylIeVsd1bF2G1xDFWoN5RkUwWb1hLM9BfjhLJ8Py8FW32Eq8ejDEzHBJ5iExEcFM/gKqtjTDaLkhy7hK5nXp6h/kV9oYCpb3hl22maorXoBB9DnpB8EV+VlPfK/K+5CdzTZSOFZCX5t0fFufsiAqnKNSGxSsWqcpEm4bIm2F+u0tXNu3jHtUee49luA7B6gWSS4NAjaM2h7k+DOkZS4u/onGKfj+MBTacAgRxhGtK0rW8evf9s5Y88UhiO8rhpN+FxpFsjn5M/G2i56SciCM6Dfn3auwNa8SV9tuYkEOBYr5VVMSaa2C6y1NBUk41JJYlQi2pE8XLwqez8INhw4NqZHtIKyA0rVQpYa44/38qYaJJ0iqQ+B7/U3M9ov8U1G5WpLVILih+SDVH5dbZswXqMpS3/k2aD7hlCZRpbxk7MlLFZZz42Ke9S6H0Au3qI77m/z04WdBHVtrtuSLx08kTI0lyqcItWLcdeKI36/zjbwOOjQzsCnmBm8sU+wV+laO0NatZRkmsI5QxzF2DSlH7ieAVRtDvUfHFHYy7SqLEhHe9r89ZKFvMvtKF/8uYAAWh5oo6slFQ0f3h78MfZIw6R/P+rmMPPM+C0O7DZaA29rn77e4Vh3D2roXnHzm5VsstZbbzyR51rtLL6IYOpjAxLTGmdlFNZN+VSDE+eD7dWG7/lHTwS3m0Rmh/oYvn1c0vaQcrqEEEEtZpQR2xQOxGfipinoTLcU3ZqJUHTMRuVb159qKoYyQeG+1D/TsbE44oH6Z1I9wVzfDrAuOhZJ4TlECjddo/6b8R53KUumdNoYL2n6VnwLqCcVTZO5s/xXh4+fyMEkujD0UsEURaHPMnbNG/fIeLZgEgFlrY4kaQJQf32sEZ2rcAwkwZEjnp1QuA9lZvU7pyBpTeJN7aI0cNdg9dNtsWn0qSFyPC3kulXW9OfLs9P7I3ItTBe9/GmdqKJ73W0VZs5JycPwwEXdx/9EH3tO216rtDPS8OJe8VHqsjCuySVEphb9kJjprv3YLVu9QLRl8zAmjvBXLg4S6OwSDSwQwkH3/KWgnGWvsZ8qaqrB8aR9Kb4yrQDUeWx5Jsmn3LS2bp/p70o+kqaS02728Ut8bnOhgQLpByPfd0dollKKQjGpGs65tqqf2rNtM8HaruIvYumf0fxY9q8LX3Eut+FWRVSr2OtX3gkaYly8+Zxfs7ABgF6HlA9OnUkfqZwfhH1PLSrfN6bPKb7P0Je8YCnY3yAWKZ5ebVhvD73STTUUwCEzh8b1Zd0cwx2ZGGbd1q+XvKw3yq23oKAgiClsxv7jQpPCa0oyGTpI6rJ3HAWfRcyEbvkJHbwuJuiyjJeBNj0Vo6sPN7Lh945Z9NIjcH00L/V2gUCkJEpLyHa95nu3N41m3zc0uu3yT+ll4M44T0oZ+JIegqhfCVKZHp1UEsxQCvjM1G1RmAxrXaW02026Kg11riiuup5goNDnj+UMotVtzVZnXmMbMTGDtFBOzUM/bNCdvDGcTAJsEziLqdv3S6n9VlT5LgRUIZImHlaTb75H5xmC4KvIPgjOXtKhj6RCRZ9VzbHfPfp2slRJmvvM7PWwevAvg0mlqEfVNWie25UJJZ59f50X6bLj1Wr2TxQS0zgImkIJIW6nbZxc75Nv/t+4//z8i4nPXc0p2KBCa7U6/PTP80GYMebsP1q2jDauaU4rtjIjfwWN2BmUu/76SOAQiyDuJ/Gy6O9woEFhAbpZ8Qo7iGtLOao+P6/++Cja0PdDPaXSKGMnX39sDWS9JiGzDai16A2vbebm/7pYdlta/Hb4a8N8zgbM3qcpe/wRvzodExjt6mScn1rYOCo1dFEYuzd4xMax5WLfyC0a7dolu7j4/OWNb5MYhevj0ah4Z4b9tbjiD21LHOPRv37M9CtKT/EttKAtL7D5ss8kkwraWxSZ4NpfdDGwFQ5mPcH/5cAMAoHZDw4St9Rn6OyuGGF1WEQZZlowq+qv94N5URz0XRCmiCdLgSaDwrJtLz0eNCMSs/vChHvG9gnoStMU0TMaXgjNz1qsqc882cskOpHrGKrB3coxrL6YyIX8xvHOgoM2bVlANKnUhyZQ0XpX1hXsfKHVZLYGqUpTpMHpM2MJ47rsruysD7inUP4tlsDX5mTil+0IsXzOS68KLy8WQhbZj7937Vmxy+u+MgZnI9g9wS32Nkuh1kqZtEjImYlPolS7xYMP8wfXLUIUN8onhcQfKAaA56Ce4K7DyQiIXvCTUlfUeq6LmB1m2fqpnPuVoFaajKL6kvTLMuzLzWOaZwd1X4QUMK9ss04s/jRnV8kszag6wQ7bP3v34VYsv+5TqmJ9JCumRPbM6SA/GI8gTGg7OX6cyPVeOb71vp0PkqP69uOniTUllmCLSOF9kdw3cE2Pr4TWwitt45dEp9Ye+LH2N3FUph1sv0z9z4vbH4ecDA8zM/MbR4rqeM4aJ7SJ/VujhtTndc2z/gbLC/pE2Hdfw9iaReaBWMcr+zJkx2tF+UgoWFNyBiqNvIg3RlnRYxNda7eoL5Mrez84kCbiVziY8rg0pvzlyS9893wlHUzTIutvJ3GgcNX98kER/JFFG2aGoe/fzkdwuY88QP6Yj3fi93N+CgNe2FyDfBeLWmr116fZOm09wRNoOj6bwgBcnMZ3xGKtqBL424XlUTMC86FO2nbWEMeHAgVLbvvvPRri19EBcpD8+aOBOUl4+VW1BlMNFvW5X5FnvGYm+O3WVIV/T2YQ2l0fsS5oLuteksxjqC7UG3uYPjsvk9RlcPsW/KtvNf59weSzNiOak1w/MHeW7C8+KGVaaXjeafZ2FlzWfRfd7XB/KXKNUdIwKPEEbr5/M2jk+Wyb7vebch5fgm53XKdtf0Iw6W68YqT98ezYsg1gLguNyLaanwr7D8ti1B7rsZV+o76vSLpiJXedFwJyPCPoPyn8ZVu66GaQdvKnEb21DZIY03eHcFviMYcwbMC+FFoliDVxZ0Uk1SqbsRD3mxQJt/zcsZNzo7vqKt4HzT60hjd2ZL2fDTD5X6WV/CkoYCUWrS3HtPLbSDgvP8ujL2G1HFw8JbXTNreLpuqlA+pvVcDsjHGkqyyK4LwMja+vM0K2x+Qso2AtcIMc9ZHsn+s0NUi13amDolaMUNTLst9/JOUKNHxemI4PA/Sq+bkfkBfCxivsMX7wQCKBSXziEDHtjdV1+aFAccrSG4FG8irPXI+L7+ZVbUEsNmp8IZFYWft/t9qtqriiPPCwHxghunVix5A/jTvDxGQl4iuXxEtigbzGut/fxQkJAGuqUL1HdrSQFVJEFCnRlM5WCE1444a1uksJQX9DUcOvFYseEqYhr+m1Py00XxkVQ9UWN3u2/o9ILFMu+yCqh7Rmdlaqxb9cnBArRGnz9UHHER/O82D6qF6TwxWq2/Ef1EUzYx3VVSkQbKxMGyCqgMheJQxuW30LHPfX+ohxC09OgUIsS+edj/iNvJgKnLTS3epPLWmV2H0prYHd2H3p4TrIGU2p0qmlPrYokSRoZQ9o6RtOlhX7XZmYxZBLlgd7oiWZ+qtPCfaKw+QmFiH1o7INxTaV+BAQ2M1NG1JCVQyLpO/RnGyLuNFOyW+o687x97cmhMDzSiwebyh60/nMEncArSknsBfItSzgRFmlCKF0L6+tCvXjqbD86emEs55mum5kbw7K/1Ijoj0CXSc2t6UmjKIQitto0HdWTFGVw68m68cwRJ0/LUUo/5lFJY0/8aXRj2cb/dOvrJnVXGw5IKGv8DoH5Se484l5EnmlIaTOOaU3EGJJWypJro+MvZ1uum9ufD/ONeJdyDnZ5kfjYuinp4+kTczviRqZy/hA8jGvgJsFVtD5TqJnBbeTLxoNDUiV1EtvZlrS14UO7wiXx/ap0I9aClp7d5fftn0i6D2QC3NeRN75ykHIo4cQIWDbCjBbCDVmkluXML2QdbTnkqnmLAbgVpmpNJAe8j/6i63uP2Mfz9+DjpUOd5Ivk/IdnyTm+pAMqPg0FT/62gnZmCb+jcmgI85ltPtqoBMDYDwPsVN184bbSjy1AcvCBDYJBHIUNS4mosts9P6PXVjtax+EU1SGooSUFyJVhDrOk9TMS/MqLyp9MGaQLyYxku+86k8IzLzrdS20bnlOxg0YAboh0b+nLpLeHeKYZTvYdvpkhfVu8e7pDC6MOVRBv7NflA+ZPqYGQdexsfXy7zx76cKtJVFEg7h/v//zdB3LjiM58Gv2Tm+O9N6TornRG0n0/uuXpde7ETMTE2q1JFahgMwECrC/K/z877FMs5n47PKVVtX7eqdmp8ZEk7K9rfyXjzNhEYB/cz44WZQaZ7vbDQWc6yGEVEPDyI+riD1oxEQKm4036iSy78RheTAHw8r2O7KqebsxQZD2AH6Ut/EwJlEAKD/fgzPRv4Trf/5WCZVZPWn6yG2ypKIWfaFZBzRAZjM4u8bzpmBHdmA+Uaj57hCg8Vyzcj746HA83DKVeX1whMxneTlp1r2ZIAaQLFI6/V5pMmrWUoFsg5VX/aT5pYi82Vc7/RpeEQTn9UccIxlZhHDBRSylo66+0xSYXi26j8sGbK8zNFhEffryvSNs+l8tGdnOdEOh5sGAq2rS9dIttzaY4csxEDg9bPEL85pyW1xeS1XGS7xmRxhWFdki62RHCCFC5m2stTo3ay0xQMQwis8qVgwjcF6Q7OhZA2i8Yl0qoetAK5xDBfl9BVC9nc4SNIpHWyqB11TjfouYa4SGPMmcU6h2jttDoDrLE9MwFowgeqn8O9y/R6BQ1hCjVGg5vvYcMzvEtcIJSEvRAU9hd82EwwWJ3ThbQLQVIeQJ0QZj4q/RdIk4VBWuulhIf7+UTtEC1e4/y9sQ2XeDvJYJv/oqwDzOgPuT7u443F9wt0rxm06/HZp+vu1UoUqQKo1xOCWnbDgNJbhYO9D6PRhAK840G0J5/gsIreo9cWP4jb2hNyp84zJRnXrCcuTgXswtU/SfCalUzR3JkaKXZD1hI+h4aCwlmgfiWYkyfB1Zi6Fi6yhdfSTzXvLmI6eFwThDkZo4kPR6aJ1nrHAco7uFhag833rTmA/SaCD1oVjBLFP+sa1V1sHMupbWlHB4m1GwGiqmzELKvRTZNk3CdfTvrF11pzOOmAcn1+DserNo4nAUibdVPxLrDoFPxxHMXuWlX2/uu3vvfufC7+s4OlBuMEF/mXzwz0f1bNwD21NZJ43/+kIhtelFI17n1+PEaCdMWvM9qMZoDo9b8D5ZKOD5B+aU+WIVKSmirins51+5H6r5ppmkRZcXm2KS1Z5f4dkouDBi8pWViRcWds2DYjjSqqvxbwRsCHNwt/jmsKmcvF0QbNYvmd4KaQjBId7lo+UYJmYjNJvR70iNIzT+a6EnxstJ8N6i0yMPcsq2emKYsSjh965hSDiusTc4oQcy/YkWEXhq93YwJlKPBelyjuEYIsIeRPmnvi1jVTXWdQXjYOQ407oq+JKUxmkHxsgMQt+KYU5p/6UjTSljg4GJQwYR9lv9RhsPGFI1ebU2yxMMmFpEPL2IxIcvLF/ehAgxzYFJxTR7Suo3H3GQu/sNO6j7hctIh0JyWM2+UNZ+0PCByezB3WqD8sPdVfbCmLF0DqC6JK3HTC5WIwyoT2f4o3Z9GmH/UU3XKdUXYjaCk5R0/hlpNUzfAZjS8NXuT9vdDgmEoiKEFoqqFGlM2Obok8/dUJ9fN46Ugr/N6z3S4u2JEqzoqQFQU/4SzK60oVHwx3kAFXl+B7UuhVSllq/aOF7GrHtR/qaMsKO/D7lCiwctFFxMt0sHozeEkGJ7sDqf8ritY5T90I86APs4aVyrMPa+nZxs7wN8MUlX7qAK3VuYQ3Edw24L6Nl9kt4DhgNdEDLdqljonmWHdRacGrTdptWT+sg99WM3JrgpArPeezDsvaJekzZMLlc74cwl3e2/4ESIw8QNP6Ko/wY7mm79u1x5craNQuWO3phUdTTjDTYhgSPjtKroKAX6Uhx4g6s7Gl+nqrzzl/zEAe2FrU6zTSWAsAQD3m8VZ3OLKJiIw75Zx840voxwWLxgIYg1Kwm/NBaGUaP6hq/KM6m85fUsu/1aYJsbUFJ9Z7DTo/cCtfFWOKSH8sw26WfsLK9X2o6Au0TSIjmfz9y3c4W+GcV5EBrBPlhUufXpZAW1qHmubt029BQmQ82oyZyC4YlmqhyEF5k64Bfr29eXNCDQu+nDQAj92FaC3sF9Zf9BUm+xnDmGE//9rn1oTTC/GOmFxPGNXOU1x2ayTCmd3KbCsvvShwzk+VFPzLQ1KYAUdgDVlNL6D7KUSGYd5jG++1a0tjsDcZjVmPa4u8+JqcBR8tSOMZx776Jz5/S5fc5irzZ7Lc1I77CUHOXQbLmbz/i4wUCFbBtJRq93GRxLc9o4Npn2/Oz72/4sTaW7XAzU8qiW/pw5G+EHgTqTfOBxLXuja4GfRqCvl6YFOIiI0dq98zQ/oWLJ8Yo8iTYEnIXVf4cjeTuY5By15KwuGWdp/FaTqaHFxz2KagjGd9acWmUt/O0blAxjnndyBOTP1bqj3oPAMldHrnjYbF0jFwk558TIQVCEXXSvf2ytNyXqcJWEymBY3WxxDwtcL7yOF6BfN4rQnqexxKG4Tdqhjaek8S5vgctUMKCrSx5813N4Sbg383CT6+gu593OXOp7vu+7hqNEkm9FpCMlW1qU5pJe9Befz5OAPOAAL8n97i3ycp3avldTLW6PxcZFhtFNziPxXuRD5sksIfNiehdpuyEpjAfsm7aiso9ejGUjQ7RIC1RtpwLWveooIXRS9N66wUoN9zEtEF7KFqx0V+xpzTqllqhR87LzOXzQPVBg7+8bINdcJw/jnSeCSRXGkVfQS7Q7EnkXZT3wzBwXvm1PGZsqfIbn5i7fX/bPzKosPl36S93aDPJrEMz+m22Z7vuxkGsSbl4yVV9ZM1hw0tPqlnWc5kD7PsXplCtk6kGc3flDAC0JEeSyn8P62CL26gqfOCtQe3BscKufDscqjrKtQJB5yetDtpyr17xec/3m0NkWCho9lpMqJg+Xl85v5kD3pnUb5hZyYA9E0XdmIhs2FyihVMe+xNRsr7yU/PUKAhznWihZRjg4yLaMR6K/70JqozJHmsyK59tAPpP7whBFK6AWAJlF4tyP5UPyBIX+FMXz5NHfKbp3EAFCarWXvE9ukSMQmrK/PBd8OlPl2iQPkIc6MGT8YTTBezvmHP8QOpEmFKlSFksmoZgwGwTlksgwrwymuiHCZcgsLes4UD0G5yi2/fsijWC2AO9nzTgaLYAnZIGgCQm72UMdQIsnLuAYVsP6v0twPerjdczgF6kSaMA5liJ+9Wex7eo3cKe2jIksdegb91VUF+mf/iqGNHVciFN/dc5k41sZlTcGHK/d55Jd7Uo8umL3b6pgDOSbPlDBuEKMgmbpI8S+l1G6WdmKJP8GgBrBx1y3AbcmLC+Rl3SSIrpLDY0HVO5M+ZIU+qQLUEUINUgmC8JrgF+huNlhiBYtuZ2a2MSD8r4uDcvJxtXLdQfytWnsMvSDEWDmQ/c8lxUnAOWXZ2/bBWV0VBnl2nHBH3Ewy+snPN7Bp4wbu3hQvAJDr8M62KX4eqynBjvhuSpK/2DaNAwPkAqGSBoEhzl7Y4hP8/bAx4vCGNwFjpZw0HQpL43jEy6O48bNh4thWtiPkrA7Hweim2X+1c/GCb9uUpLIwf3pUIjzALeCvoeWqlkELx3dfNCHD88dOMJuSvkIUi/r/nI1SB0hBpLcRp1iOfuEmOFvC1q8hW++0R7YlN/l4sfu85PSZhQz1GKGR7jJqA9zcOAPy4rnyzCTGS7PcKGZfxP/vvl9YoLByjbSd5RBPYAZhL1gawQtxkEB52+oBFjcZqxWXagcUegGv6a+DPxT0Gy7BUGFko+L9UxWiFPn1/LD057nB/mhvAagiXri2ZFTZxq1N1dVnvqbdW/tN8S1dLj8OnnEoshm33fFWfn+eliPtSz7PN5z+Sk8Ytiwj2boO3cBUUxhLRTNlAw+6dURRZdvUAJ3JxQ2Brd/7K25v9DYYBS9OHjPo5ETuPpHlDDlzbqHwJ5RnY1oY9v1e3K39TlMZ0+IFwkLa7CF1K4N33uZPDgxl4QX32/ZabXUwsoxo8y6BSrieiHTt1khvr0yneu06PMqaish/d4obQp5rEeHN09ktBi0u9Tt3Kb5NAJoVE/dkjGghd8/hWoRw2p/4DTSk+8GTeQYM2GhS3LxWb8mhbx/Db/doflQMZovG4t81xK8lBFA7/Lxc/KLe9WhvbYze6hMg0//0ttsMXbRUtkRo7CLUdHUJKc3g3H3HekYq0EZTtEs1tV5Vda6Z30pSahmoIM7ssI6j78zqvJGEXx8cNB0zowZndM/RkC0v9L+X1hXOKNT46kELAIuj99g9yjKi0NHUZmLetEZBJhCOkOdzIcFjwdR/X3IVDXGO1rOF09lcTWJ/APY0q4x0gKaJ9iGfDhmLXhWWK49l5D/jeA+qHmxA2yu4oePgFVtP8NsP/iVY7BcABDC0hsA9uVlgaDji6HBC+U+HrZ4BDVfgWil7mV6ExXeh4qDXlZirny7QW6z3zUv5mwiyBjbKl9+43bU9PlCGUfBRaYMOEVDkZyoSdWqJAiqPye5sQOk0mH84l7Lwjvb4xqRfX8tw4/23R/ysy4T9+7gnLMYxs6MKhbqTxhB0I54FwgYdi3jvOvjH2MsL7cZ9uPPn1YbBO6+QS8irdMo49eXP5VFenCJWt57wXKYZzrm/a5bxfH9gr7fxgOa8xz7GUyYM6xF9ge9oOXxqeUqZljl+Mp2a5MXTyIP3w5fFoBbAywSeAirVQepNMGYrCUPGvhmFEp9wbF4UBffaSDIU2n9uJ3FfC/BS65R9ZDdx9NjGrzvLCyq/tthKBEOffp9TxdZHk5VWnyFyWxchBsizoZbPAyxWgPJP3fXWvew+Sk93gtmv2YzaOfDNNoURXVNd1tE91CjYQNLF5oieU75eD9sUyrkxIg7YETAznuqOG8/cVxD3IwgZu3n0SxI403gu4vvzHXwbGR+NRO0fEKLshW4sZARmY5C9UnSj/VrbuBj8trt0vhBMJh/Cc3nux/gpJI3c2MvtXo3XG7B+/aBqy4rocpFusXr37d7YedLlGL/zpwH4wWlfexZ0qB0TPj4TZ53pgO9UVOm9wa+CJ0H56sT2Peiqr+ksvjW5cShn12uyMxK3929zNyXGPtXXxLNnijDUDeWychm23Ip5vfo5eTnB18U53mbMRtnSDwH1qYpD+cidRExxjHYt7uCOYbicINCGrQ+/CWN3PIS2VePEU1VTWtiJAU8HCFzYcDJ2/ubH3PUNLqkZjl9L+q/QV/s7W3HA0TgU2MV/7vQ0XEIq+4jgK/+msi2ohd8z5PFoGS3kFfjOo1uPtjKGu5/IvR7oKMvCHRgT3I9UF2CxhYb+wa7FXsttp3wAzPUCJJrL+lG3Hr/Zhr0Jzji3SYNmZlN8uUYSpKTofBO9lGR7HkRv6Ht6uHGmlZkHS2o7BSRfntbGvZwpVZd6XDF7Qvmkl+6dKpSfLeNnGKl6FhnGToes3pJyNA0ijZWd4x/GkqwLTF86A/EPVaUxAjy1ui1z6Pugd/WaW1SlSwUqfta1G90hgIyUgbfACwhQivZuGzWRR0xa046DOnEqeL6X8Vl4pbQe1A8Q23AASdkIqXwdq3iYcahIbZYzonrrwHh1LPth7UqzayGxw0HXC2yIBNt9cP+hXekPJjDWPX3N6B7llUFFailxsDwETlVhX10doJ8gWpFrv/Upy4VSRvzme7C7sAkSdq679vnTC9u1S+OHbpJExhYIbBfNPKLUe6cO5rJu8+Z9e8He8k0Tuk0NB52nJD9ffFZPJfx56FoyFsFpQ0QRMInFoAv5prZeKjhDhwmJf/Kss7opdwo6wC/MhIsKHDTysn2aLN/2bHe+7Nf0z8JZNSoVextTnPilUHdo7y/Wv4s/pjSHIbMu/yK0w1TGjf97nN72clEn10FanrWb3JnDW2ZCwRm3IpQugVna9zhZ6s8vFKEGjUxJsgSUIcSiFrvVAFdeJjEHJT/APCM95nkE4ALny/T1xnP0aWk+7JFq+fxvAaZ+4vqIBtUq3cmZkIiTTWEkgRdiJdVh3XWOT1ICn+KsYWh0VNvdnpYikpba3p2tPqRKq8Fd84g59dYLCPWE1lGJPOn6/04o6PA4Zlss7wJYsoBjJZne1ziofPvLAr9ZZk2357N7L+UlnFWqkKzakWTJWVYN994X28ztZuBdCTPrNagdvfhKQMyTvlF5X/m8JuziVHF47VdGufK5vimUppn8/nrQA20hDiVR6ImK5M4vNsFPWwnwUaVhczWXRf7hA+Gd9o5GXXcf7+MGR8UmkevLXk3+oLapyJJyzsg9W9S+12FBpaHBZZTcYyvthHayDQhbPPgtYbsNM0XDeYnhHfnBlw6SRZUdRI0Rf2G/wStO0hJ9ILeQqTeTFO1l9LlOP+w9blANtht5DB26BFkf3L+AUfF9wG+XYA5Mdwdtf62KrvnRClpOoaUiTyRJLKElg6Y4rGgajg+lCtTbs0fs+s3YJnQ84LRg/f0kO90dn8tiYzyQRlfASUUji/i1Pyu5MOviuF9rh0kqxBaXUzwYAR4ycj8Ft2J+Qa1d2r73kkv0rEiGKMeKy8npN4ny8HL8HFz7ieBd1FW4+Zt/M4pT5NzPa2ECP0RMFyVAqaU+i73nYPhOA2bC4nuXCoWDb7WFD1LrEsyxjw9Hrevfs/f4NQduIsqQSHrcoUSe5/mswTITFI6koUhwUeU0TgYKC5gBUA0xJrNCj4Gw3VEM1ni6M8KSFPjUN0tmOoUUrrV1vocUNLeW+dYg/J30BMMMM0TKpHjq9E5FD67TnzuVkgXn4mbITePO/ocaBRllE2iu3Jq6rm9ciFowlt7KL3xsMhj6Bp4H3uoUdX6TI4Y12/j1aBFGHAeBUutqseoLlGsP8Krmo4h/foQYY0P8Ad63L4GVzmdH3A1+J6H7JrERyI7v56gT82i13JSYt2Scs3cHDEd0JF1dqCr9BOlqaqzjAb5BjP1cA9izl/Pnj4EFs+6anIZeDJEK1UMCbiracvE9MkUisJGSmlEXfdeTMsEIE80sdpri26V9m/xPi2fJXAOyBUUWN89VHPZHn49hb6uAyAB8nkXNCqbD5Ee9OezWXDZVNSjN9Mw4m9WMhGS/ONpH27mQSgBGFb+eLfy7wwPGVWAVGflblw00xvOU4T5Nq138ELe7u6bZSVing2XYMIkWxIfa00k7IC/kI6rQl459EIBLX/xKgtLf/I6Sz4AeM3FiXlC80JcTQHUYeuhs2RLldYkwoPzBsTnpO0QYEJ0t7YNlVE0zh2qztHG4XS442O0SxkSJX9EwEX4vDQbEBZYH3mcDH1mcpnS0TUPEDvfogZijm2ZtLfDcMM1lB8vOQTfI8harGGCiyvoyFJ8nDDoPtOaxhrCG8nj6sUlTyw0xKFiT24DNo9fWE1f2bvXy6Hqx1uc0myjdEvt7Dl5eH6Iv4BP7bYUE15yyQmLoF/SfKwEb8e22xa/xCApR36wne8NPAEcnUhcg/QVOI48hEQP77Jn3/EVAT5uKZvfUQKTqN40/BG9VEo3AoZlvBH4WpOcSLJBQ/d6XaMI4TR6gJc58ZMGd8S8Db67RPAKb36WlpjJBWnvnT9A8Zi4syUadAWaDedV9FNglYTkvnCKssSffpeh/S1HnW3BOIfloxTXTcdud7+jQ4q83taoRcmAH5JxJB/V/4DYOn91yl75IE4s4UGqAzEMJSSFtpQu/f18ke8xRHXmgHF5VvCvSJr9yjyaLpp0q/3kTqfjEUVflhXAZqNW9pjhja9POL7jNWOR7LDqSvJiFZ0q6OVo7pvYFUx0fi0M32mfZymwZHwqbFRoJ3uIjphTHJFtMueQnNmRV5B6WAiEdUaV6liCGx0jZeQANRI6W8ewku5etdxl1M6Wdkt6bNaLQRX8BRHki2xrwNtAb17aHwI89XbNkFxnyeeA9iuEx5n31T5IbPsNEgMHKMBC7RjvA2eu/WpDco+w6cJgw1LenPW4Em/Otuvzpuu8Jcsh/h7wx1EawyvZGd/w8RNbFGl1oT5SBDRFh2h9AUAY3vUTMJG9BEDE2ivVNJiKwvgU60vOcN88EG43n7jnsoyAkoQuiILbY9879JEmfcirjYX87QB4a1xxhCIOBRQn/sfmz468JNOQO+V+ix+fPNqIjhQCxoGYR59/1sqiPTrwfENMPlQ7q0l0sGk+7BQgjw6/Bf7v41eH5CbbulP8tpJO7ooUIbkVoETCe4eakT/meWC0eZti+gV6jwfBwDZkkv7dWJjhEtjjPHs7r+4otMpcH/plRA2ESgaFNADnsnxYUrzdG9Se0cW7gjCXn1nGJz34gDkwLVQssQn7rFWLCzj+uXjFlX8+zxwIdyzfu9FDgv27X3qpv8a1H75VSOuAPqffKxcE8+4Q88jrO3qapHKpmcYflWH2tZjvgoHWyAKtstiynGJwj6eCO843eqkBWhLMh8KRiaNsjmW1vrpMWAuIOKCHzix9t1dOmH+oHYsnC4olrRoBbdY4ko1Ulu2Creo6kdwGuKGTvv4AF5OGOOg97AlMoypzgz/7vbGf4zuSd8N9UQk0HRcR/vun34fQt9BZ8Y7DBzuY2Bz2m4SNIINO8msoS4T3RK408UFVthdMuWxxRmPvhSc9rrGTwPpCkzrMGnyHJErrq2GmJgW3uBaz911CAJ6z0+KklnQr52woXAbY/Od7SJzV7mSfdDCGCdZ+LA+aTR1FNePBIHaE+lfNJtnwD5xLkiVnDCph1b7zaEZpW58EyoNbvtStpKx/4wVn/QWo/fWr7o5Q2YiqHbxYahzPB6EzYfMvM87HD2FugrkLePJ35Rm+bWeYjd4nq9HcofM3FOVw2D8sxE5iNIXnEULm9nE/NqgMYplf+4t7puUYYQNxmpRLgSS3F9Yy2aZpfPDYb+DbV1Insmmsid4+H/nTBeRuE0R0Bq8zbIZWsT9lhGvcbAwgKI5plBykHgb6TA1v7YPK+DmJ+cvhwN5i1LFjF3Db1aBS8BeDVAMitFiBhd/QWTip3o4ayeIbQw3E/xRRd9JjC20CwbCNmxeTaAZ8Up3J7EmnkekMKoBQvtzuCNcrq8RX8EXfXz5QE05d0o3qj+enkb82nSY3EvgTE7d9s0gwWm47IilPy68JeptKfHstuI7pSSLIjdf4Jih9MiaIxJdCVfoONZUMFMR813y3KhiTm+YBZwRYxdrQ2VhgBpk899+oNIr43jXHgF3MQuJtibwQA25kvWzMCBzVMinbXwJhFufiem3bV5R72+obRAgoC8xlFh2xiDw7uJRQWYHB9Vf1iTKx62cUuCjOqRAYZkGt92+kvaWEEYoioDsGQ0ENdEowogNn1aOv0op3kTAIpP9aJ9UJs0QE13JrRBmX1WtuYqaDaS/CEKWoxBiL1wf7dbxJNv9ald3gP11C0hT3rx3JjBSwl+SxAcAP69RFWWhcOSztS2+XM7n5mW7EgjK9xdmm+22bUS8TyrA/9sev2xa9I+I1Bq/S8SdNgvbXYMVghsQwRLbqJ1BrpTgZ9UNs/qZuMX8/A0eBbtrcYFGLIaIfyMRs7aZgecuacmzF38qss4c6dgzwtG5CFTmajJqb5atGhT6JMoeEG9vDmHN79Dr8WbIz9qf14c93Ar9vy5YVuEtPRWk919xT5KplbBvNzGUwwenmh0VVjBvSq6CgYc7VikcmcvkOPhc8iq7V77ZcvETN6PVQ1L9NCkcK63OLsxYfpXeKCmdtSEdM3vt7HmBd8ZSePiujwp1G0mVgKo98vntxP54lYCwLBzDqXLDcCme09gacLU8CXSumrxvtQf1BdfRyX69anolua0rDIcs+mdXu72pfZ2XUmi84RqW9vWNE++VhxzClDTkR7iJg+vMbKbN6BndXxa6uE4yL0hEON0bL8j1yUF66XZIDMetApC739JXKhGHp16Hepq7s55ZkgRRHLyEKVXbB//o7vVC5JCJ7R1+rye/2mJxDc8rukhKuCsKIccMPWObMS3Mr1HAyFwIJFvadH1SScMHlM07sZk0efdo1WW9keDwsga/bSe9lwZSbRWsMqh6Qt+W3w5UqNCuqosuRMW6GJXM1C5KfeA58XJ4jGGRWwfdm0sNTC4LB4u87wScXn81dU5/wm3mgrmfbabJp01baf0VLUU871VhmU9ZJ876JAdCFx59UIxxmN7Jm7xjuYFjct/KDQadOr59hQpzKPHs8Ft2pOVd6sXKBs2koDo9Qc7OoW7e0APyJ7/zb812JsCjQ3YR3N3Je5KpIGExEm4Dl2HnwbXkK5ix3cCmY5F5n5SC1+c//6998VxueidFKh6sChsS0AYezLqtqX5FT3gg5IiWeWYlbrJeBqzUtpEje4itRkT+Cyk3erxf2pK2sGVc7zVR37WJUe/pSZSRt3huBTKLUZ55BcQSLpWBurGass+IA+s/OisWjMM68jPPbLAZiehFsPrF/jRkqtnLwlIfKN0fgpOcWmQqo8wLMx/ZDmb1xTD8PCyFSJyChcDxliEVlLOCRn95H/1g4wSNWxU6l6TxMn9TM7SKZsnh+YmNgxQVun8mAoFWOfUluwi6kN8wAG/D4TWLJ7+LqbwKoWIBhuuw5LEUtMLUyE16go3ZN8lBM/doMAnmV+Amyd/69JNXoEPQ59NIwdP09Z128lsewhUe9pLJV8lzPe1rD2CzBtC+rfDBHI7TOPKclws8PHl7jq8gw2ZWNU7s4/aitA0blAgr9/QOQCVWN1CYn+QkJ1+DMgRiRXrHVhnEokv1eVCIpRs5JpPQ1Dx+NTIcpwu3D1EBAYgWiSgk+gpbB49Jztsi4OEnDrFyjadU4Cu/wMDr9OFYYBwOx2YCpOZcANNsJ9JHbdLAw3XHMXUT5H+BS55P4FKV7ihLKxTAnd9Tk7G9KD787/k5EmsTpeT1wGJI9zh0W+F4oi0yjN/7QGvF7/LmyX78RT/gqcfumv8L3l7v0YqxOZ/uQWfEhBTY75Wmbo1zeifoS1BXlFOd5olhum5oTiO44wZDhmtlLtKwKBMHDKjucYKT1yJTEm+H8woQVtJcVBaqXfm1V+rsVxvNGB2eMs6yUB8o2aQT7zRn9jU6qugumKQnNdl0rH3wBRIDvPtPIB38fUm/P9QxOGgrgWqvD5PT+fi1Y2Ov8XQ8aTEpAb63IF+9h3jpUdDdnxDIohtaGPNN7t5puULlNbNmkOPtis1v2P1dbVvvBqvrU2BVIwQ/3e/woLd9WRb+bUWD+YutzMClIldQqir8t3XwBtg7t3fTR/gAbZkUxmf4GsvQTtF5eUbQa6HQkruB0peW7Ou+HlFCcy93Kg2v57fEnQBMA4/iAr/p+ZMTzP0fzCWKHVPQBtR2G2XhpyRtVFpljyLI3E6LFLVpGZcDnB48+w8RtKjgV5IhCEfWb7leOtIPCQvbQ8nsT5/sL+9WY187y0ZMuNtTBNLnqE+PCXBPxlXeq/zY3v5sL4p/6WrLTE6+7yV4eJMj9gdL4pil0YDeaN07smOz0fsG8EeCiBrukCe/bsTTZxQPlxzNZh+eEdhipyiLu7HJ7sd/OyVrz9PmUo+QukoU1HMIIqvRGcRgM2lOOJiaTGjLkdJf4DKWYf/WypgazwVeAzGGY/V8I7NWPomKiVZX8P8RcAPCkNNrmGq7rfvWtN6yHq7z8njIvZvpgAsd4e2QuVyzQFEWw3EuJImNIl/l1ltWJUb0z975hHlLre3S1zx8tkF0LqCQvaMwJXzHObXAxifY/POjp6dP3OMIal8hTtQYfq2j8kLzeXxANAMj8S5N85G5YP0NrG0LPtu8vAGC4jx7+HYEtD5r+BD70+Djf0AB/hRM2wAV85mAs1HqwOCrHdNuTB/HLYnY7SESDsL1L0wu++vJs8DoogJ86WKmZygDkSLrrl3HCDvBZI519bn9KYO71OHKyqt9/61qGQBYFoEIsD8L0fRgKsAR2pqm1PYu/AgZOIxBJb06AAUF1JdfbDcV1XglhUQqI21saU/RAfBLz/Br7Koxo3AEcLcodCZJBtHC9LQRPf/3HKFw4ea5rHqs314xsffiya/hdf+Io3e/4nWC5dKQ3cANDZy0kxJFfX4ElABBFHGf3A3KK0qH1MAPCwxM4E5MZwdcwIA1W//KA1EC+k/Dl7auB4vJF25bdhzMLGb4xTHE2b0TpmINopuNj8CCNbmtdoOw8h80PtxlcVgWoXDo7FoNILH0Vn+ehG4pY5ktYkvxWhQhHorprDXPUIQCQ/HnZvbLIeiC0PBEfbJCs8q9IsILg2OEnivx6QPbffvHStj+qmuKIn04l/qbKH2KDk9kcUSId7lTd1YIlq11OMyr3hsxzambK0y8O3Y89/FptZWWWwf+Q1clE1RPWLP5NRIlHoxZZZV239s/Z0icBNNUT3cTwRbwhKBIzbCxcwRodwIBokuopDfjtigFuS9xOmzIx2x4cXST3MaFT03Bg1ZKv+rPqTjIs7gkRiDDR9CwHZDm17AB0pUFTbB5XLu8IRzelepe4sEa9SlOaG+4IeQ+5co6psDrq0Z1O213HED2Z+seN1lx+Pga59bl2/9TjURyJeQ4HUvzAxO13nilLO1cAkfowXfArHzibkDMUH4Lgjp4DQBENlnw/vLWqRk/Y2Hji3IuEhDd4zCTgmBDI0zCefYPVyTcs/fUhN4UM3T5m6KNmdzen+Jivad8dCg4FCVg8KqdXH1awQETlZ4eMlgtZiyWcA6coYPuCnV3Yh1Q6tXNKin7AEWS7FVtoNbBgC+7ujnrr/sBfK6+18V4QP08FtRDMuVif9Mmv+RWg17/0zzGm7iqxfYU7k564/ay1kTYjHDIgC8w44MaBeMIkMytjuWYpEJKijgY6Ed26D4TpfC0v60rWCoU0k6zYbJpE32NWdSI74VJ7fP/FflOtLOW2CdWoaXirH2LsuAntCQgsTE3tWI+l/StwfMINM+VPhFdeujgV1oobXIS7jXAFEMzmvqc0dNQQFRALoC7Ava3jX5s851WG9o8HmKOK1l2drC3HFezK4l6/Pk9tH2vMD57zv06AmMhLVZ6cC7Eqr+Ur97GUSU318Ic/uMLSb2sfdp9wt9kXzADk/MS9rJh/KQL9iwpCa7LpnJ6e7owL1mDdGHMuz9BdQz7+mQifcHbEjE46hu73kRPWsBGo8Xd5Y9vbr72OXmzz4sXPWG/fd0zYPkZV0wS+/F22L4YfC8TPN72Q1gdrskJG/xqwlu/CqOuUVfqKIFNscSPJf893sNk9gvQ5a1SidfCSuAmD+Zzf93MCvuhpb4eQv6h5BevGnL+yRe+H/G5woBI2iR80XZrcuX56hV2NIE4tOdWyjG7VgOfPpFmy/C/QsH9oGCwSjXaoqojbF/eEGb0qEyN0EC7G6+J/E6uE6iM8RxsYeEWJGblTENRpN5Z8C1ceJW7+TQ//zQ5o08o6iX6dXwEBJ+LXhV4fu/keVCUkndUYDO0GYv6Co96Yaqlz23SKYw8b71x2azRQmkQzK2xT5d2+gI8To5l8j3ztHatbpwWsTkX7SRg2wfnhtDDo1HeEJD4pNDPIB4CoXM8oVp1yE4nD+ky9kLfn90DRFk8G4pX3GfM+2LP41TUdYb/Kr8MRPG5G7X6rLqFS4ZHBcf+LVk3CpLhCQvSo7y6dnzp2SAeCYSjDz1zaZZ9UOvSdGQh4F4Ufi5sCF82709oywYDNtbRF2MJqxHLT3GAKesig6gHZxLDWFXM7aShkFDOs/1onuPj1ieliV7zkg/ztj5Qvxt9JF5t8B+vPOimIuwnxVXT2/k1Q9U/m5eAJHR4KCOGCd0rEKmTefDL1mEmPf7ghEPfEzBowhcxoSp186oLvVs9/7pwO5nsZ43wUszWOqRpg+rJTTJ2k1rgcs264jZiL9p0sSpv/6XuvLEIPO9rFKKNU9PfSje4y81NCc0lw061B/x5rQ6s4ciF1kIczHLNA4qOunoA8rfYNYREm+Rm+b5XdVHiOEKni9t4lGVSUOv97wjROMnSYvenfd0y4oyguGaFEwMv6kt/p2L3XGJfxJkg6n0yDpLqRVoxD0HDul7LDt6HtS172jSPklElJ/+kYgZPs8PaxOEZLrTqJRvZmXgs0gM7tl1ZKWFuTSsRD/SzOl0X0VKXUZt6LwXZ9kQkNJgP4kB5rkBLkXHyZbfFcHmAA41o2xh68v0fO2xWX0FOQD+ABXyOyhqYI5HXvAhI7O/R1csgRWoW9te87vxx35vZBAXWKvgkyoEYXQZE8qGMNCO8Ek4j5vcFWsh3Y7fuhM7CX7sEwyg9wevfdYl2PRwd8qTLLtM2i1wHuNnyrbE0lpDR66c0WWGVL8w6p31c7flWsOiCPBusLLA6QLsI8AXmsLhA4sF81N033mPzQV+97/L++8XO9pR+qvAJVLF94wh0GWwt0VZqn094L9nJfBA4A1ymgkGHw9WLpzD4OJsiQIuRrlBY0JNmQpikXPoQJcdjkWy8ZL6qMFS8PCkeIWpESB/7YdmOTkIh7Icl9ANkfpXqi9E+vlzMC/ZScP2v7z+9WJE0/Gy+ihCPiTP3gXft11Qfrvt+1/Bzhyx3tieJvwp1SRRGaDCSgsnLEVxIyY1LYot/Ungm4nwuS0z5gh/JBcGQFB7S/0/vccYoOMhF0zliT1+sDQbRGYwwPg/LH10fBYJkAs2uQjMzLSTBLeFNgreh2+sDt9+jsN/Xy+cjyOIPcz/FjV0fFr8Vac997bT2mBbK1Zjrw8phn5yC63VYOk7OQzfBQBQoYsu67kzflq26Eiz/VA/0CG7NW6Y0J2K+fOCBlLI9atkQUKJWv9I0HI7r8qtYswvtSpBnuLpqUnItaI44RPzYIjCDdUuGFlwzHdRvX6kFPGghzGJ1R6A5WG6rl7FfLgJoPQaIINaZVLsxJ+6ziJq41S7OAS6ucUkqWbazWNyaBoSxWyUv90Vw8fJYGjp6RC/pes3i6785HZJPVx2D0el8rCEr9eKXHsffWyudQU/ZGRpcd1/Ye8mBb9neDDptnSv3Ye4Nvnzs/OauOR6tBt0BgShJNl89dULQCT+qJZwlMvBjgkTyrqL4jrYLNJZR3VHxlPFeKNA2oPLOO+DushQCcRKXcENZFKCLdpwnxvv/LwD3GmMs9Fce8XLWkmExZ5aFmLCKQblbS/oYgmphRYRZth6fF4RjAiAbR4fwmDY7HyXGvx0B21KDRQtPEyf10zdJZz8ujlb0tzTO7l2Z6mfaA4UAiyZOWL2bfD/kq958iDlL+VWO8JkdIQrUgIbIGrdDE4CS981+HAQ15MA/aeo73nJBuSYNOv6YKJO3M0vbUHO3k8O+dgii2ZOAtoR4mX8vocIpRZdthPR1sAaJzRmeHL7cvZYMxYZw4CUXxPjikUS1HcTkJ7bs9eL9U/HnGC6ffjv5QWufr7gLlQpO5BJgoJIfxr1xGs2Ap/vM0InVl77gru1EC9ySAp8H8kd5AT4gnOL7pz8uT/t65vdBD4msYeqkN8djrjQaFVgxFxJ5JttU21Pr8n1diAdfb/oR3lvl+Ri2SrTaIipv5ZHcefzh6zLWEGzTjlID/puNQdeXrUCEeItohu5gXYyc93P06/pjI6r3OJ14CDHRSCra7H/eJPKnFdbWyDcuOFaiIW4yiDd/ohXFe+o2ibnOGcjq7qiKAMFd90onpmqYd4mCry0Ix5+mu4Au9Eh+9F/W1HdSzjNYDGy509WdapGeG6dp5iVD9ATrvH+666ROt81f0671cvh4S5SX556XP6nJBr2vGm5bBEkZoOahhFc+rWwHM+XJ+vcpq3/GE75A2ik8pztspOyPGaCLTje3SX2tIwuNIEL8ODRSyNWJKTkgCo7nV+3CXoSuOkjh050hlEQJMy47sWAx2Fq9XEw3CW0vawvblfFKhLT5VPDPPq2W18buNWl4VdtXYoCeVltn2EhcvHFGHQeExvmaTz5L7bEzuEhZFqa2qolaVcvQbw+MQ2Fdfcdu9OelavH4V7fOBPwWpkuCRnDAhJKo0wdSQ7xMa1PjGlqhj1k5nPSSNfP7f/ZwaMRi5+N0mFl2hIzOVyEW6gBGtdU3/wIBwNagU1cFF+ZtAkZH9AY5HXOiTmAtAkmGP06yH9Ypqz8Ep0WDj371OanVfUVSteiRXHUjh8DFoGvYcqapE3wghOwmZdgPMMk283obG2+Th8dXLFNJyN1Ialfk5GX2NJt8BywqDkkkP+AseE8vkO+cfMs8tulER3vBR7p6+lsAyd0nqOxxuVNMn43xoFIULtLXr5w/ivAQo7rHq/CCVQ76/rW94hTadA5dED4fvx3aXf+2MRDF++xZMVwiR7CdOcu/BcWVAqJQy4uttKz/etrlaVj3g9qWbcChHF4FB9Ae231Mj+EzJiJjn9WlvQ3YbJLakTBch34P3QqZpbH159GTM66znr1Udbag14f8uNjOr+r3Dkxr1C5Xmk8CE+DtKFU9Gegdt69p0WM9LF3grB3/DluWcXsJfCjLP37bB05Sf5gFujOQtnkYWX2WC2TLW+q90lkJkDcRyu7cdBo5S0UXIjL6VvBuJ/Coez+ov3vsq34N2rKvxtY04rXTgThGOjhL0NLuaP1+vw7Lh5vgA/xcdHjyCDWWULbqZ2l99rUTobjgAKhdL7EVQRiUg7WfZv/NLls8P6SmEPExqAfEnH12vWpIgUAuA9s1snJqPLW0tL8N+rjgoQRFlCoiktGuV7CeCF6tbzQzGTZYZ4wcjZDwPI6CZlHhfZu6LSG4Yeymr2QpCUW/NS/kH2dh0ec42BPxUELxiWtMo6+MFHqcAKbR33/5J+t6Cp1KXvGC+HT8gwsnqEajOA4YPdhR1PLPvlPs6R3J2dGeZeCOUOzuE6dZn66m5r+CsqFhou1mJIlcI9xkjb/hqRwA7miLCiHPE1tOW53mpHrZi1Xm0u0BmkSMsJ0QmnO6mNCVGYdwUFmWchtP7kEs4GGk5gUcsk9+CY2bRiCq3gtyp4n8qDpdQWeSzUrRt0tABoBHhRGCKJ4KSAPJRrHf8mBz64E7p+lRdmaSL871bgs7evf9JXVmjkLceHgxA1FjJLwlP2cBErgD3GJazXxxtUNbrMzSJcRXgykmg5DqFdb1x4R8OixrWUPK4wwdJpFUPqwXY1CwSKZHO93GPAIA8XoA+gZP1y0HaEPu+D85b9ikHdgQQQKM6TeOI3vvXtlAR5neCFE4BNV21TPylJAn09drUSWH620aITCHWr8Wztb/oooU3cB9cZl52gf+gHdLKdufwwDgFHcVeCiOvMAmus1elcr2AzjFlyLHfrxrKDYG2vFzTWvPtm0uISLskYioR9wFkNJb7nCtY4FGxiL0nxvqMUuXgIoJZIgqoSNveSZrUp5ZB8NLwbsgsFN2k+VB33IdKfoOAUIeyZ/+9WUS2X/wpfUi0/gavrhRuP7IQwwg4ByR62CbSlm9qaxJ4tKyp7Ymd3ypQ/vuMZ4vXSKcJhAXtyvdXiOWPGcC/i5Zi5iZa/lGW2ZSFCh1Bp1pwk5RlQB3+7/YN1mj+Enb4tGbyE4Grw8bXnn7A5ffkuDHdwxUlELM7pVeDkNNDmAUuXC9Gm7RmklhHfXVJNrDhMNWZ8BKUTStqbtNs/SMFD9u+YtddrAP5eqGrVlJDu1UmEYp/BVnUHs0uaSrb57js/0qmAXqvcdIvaih2dzPwHHTbNtpmXsLjvJ8/NHDZ+oGm4B4hqr5Bcc+vhg2H5An1CL9r4RFQK+cDaQhMZ75ymCq2zblJ8xVhdbOeuUBysMnHzRH/S3ig7yggL95gThXruWtXEGlGtyEbpIFQB9js3sPAawpV9U4rLqQx2BQ6L8I+pV6xI6Qch/OYnhkPThSi5lG6hP7lu/I02BATVA7bPXztYdin0I5FyjEVySin1W6btHK0DJKv9o3GwTAhXyO1D/ihoxuIwQTTfvtEIXdjhgH6eZHAwFJDZvSajmobiOFd9u7ywnpfZuGJcJpnrScQlgPirM9TeA1Xm86IqIQs28bS+ddA97l5082aV676he9TyV5XHklq5pXdsXAAJEzpuYjHKgxam6KlWzS47BSoimoQGEfCCk1l3AqaJIlVf5mChXwDi/lkN1EO01GbDcaECRcqq9JmzF8G+dr3JH03zhL5is+dicEhKKbs3JAhndSdYGPboB0WWnoQFA3zG4Fjp9LmioVXplG7Zbl+XdAcQxytwFDVtzmuTgQAnVGCYEBp+JQeUA9UnHAH91NYFOcuS2Pdj5QBiOwG71aD2lWH5galsmeTDE+gKeV4INTQsPXy7YLxzTqbRjgyZcmhg/uC5oxcqPmMYek6DpHS1H8T5hUxmrSZwy8Rwbs7IjPQSN2WsUfCOp2F5YKe+Vv4/o0dJzr+I2fxp0BnJhcg0h+4xStWwKdUdpQ7RAL4e4QFRuOabGXP94xT+QNSybKfOYbrCZfP/ZdjQnLMSh+czU76cVAatwu2+DgHVGx/k2UAwTkddEUd/jnw1plGEcYhJInNBtK+SBkGCdGiNj5fjsNW+ZUFPkNjUyAUs/zWAK/9LEp7hURUEUP2Cs08yxLdaHNjiP/1YQq9x78oOQ4W2UH33f+Ir2lftzh+HgKn0V25QUgu0eq4Hnt/6DkzNeuLvcxfqwD0QrsBSgqVWOeEMOIHSf6wLXp3vPkAwWP6fLGs6XGK/LyGl49fjrfrRQY4SUbJXPbtLvCohBDVtLSYW/VdBi3Lutr+Jp/uuB2HGZJYffV5NKxvIufeufdphJlI59U8a4ra41/X62AGgcD6XTGgH3KcsdsTpS4ors7ePxGdwosvEEYkm3qRLjnaZ7kZv9va0OBI1j3Sr3hF6odAgLDYpaASI/yAwXmi45XhO89OwQ0/PLtyAf5iVEpHTvDF4hvpXLDSu7wTtOtwmBDEoHRLWME4MxZc5I1wytZVmB08iOXkWtjLTyPhm7pQX/47DpIlKf7bF3FiLLP20h6E6EV8FORL/sGwd45i8WrblHeC39RjVwGhB0h33zWzbLmCczNNsDdBhg+q69VE7jPsinr/tiFdXFPpsMSjAXJ4gaXictuNyQJnq/hN8QDfjrrGmLKo3MNjPmdkAFD4+XtfQxZ58tu3+YNEtwdLvgPzxSlzVzf7w+hJxPo2Q69W+++8Cg/GmfIytqdep+e3w4j/pekqliVHkuAvieEoZmbdSgxVYv76Vb6ePYzZ2FhPtZSK9HAP5CoB/C32rIIICkuaMKR8qdU3vonEfdojo9IOHC7AVmJY0m9Tp+D9ZKdKS98daNJBxuQxVD8MPK8crfg/9etCLLiD1Xnx+xEyQkFmBcuKuB+/1xPpxNQlx0+d/PBZWYg7991AjCb3rOCtiwt7PPsq41dhuF8n70xKSNTSmVlEgNl/3cjXZ9KY+kXssQbJMlZhQADzHqiFNA+0TV8YDqbhIqmJwuRhlL45NNbNCwIKSmvR4OGLJ7j6RHAmab26kHiFF+CmYkPMD76KoAo4ZOFA9IG2YhN8qT44K0w4YEF8lhc5jGv8jYF3TRFtDMsO89htmonpK1cFYvB+cJ2nRDyjihiS0wuua2xGl3WD/vV5uwyWii0PrvDVuZQFh+d73sMr3xd5zvC8QkF6e/WAT7at24vsDQ35De9OZF+zNT7swFXX/TDkmLJv/D0WuMQJQ5GMCQQQfr2NV8Oy85e25GRLFXxyiI1iyP7ObRtZeAeGHTGRN/ZGsMjfTmhOw38Z5l0w8a0cqWw9HCae57bHRf3cLTy+vpNv07WUsXhgfTKnuAowAbbNq0F84QUZVsoVkhMLvnTh/tJX9XuwuuPamvTkz+LGI9Rnl5SKHNxtopuKhli5tj3BDTxoiiGLTkHs6gQChRcvQXQBrdA3jeR79/Isi1FY3kupEuBf/RmCCO5I+jryk+n0cslQl+1jVBmW9aRNE+Ij6b7ORG2FjYtFhXBVA7ctZ9Qcs1R/5mp9jd+ESyf08VWidC6ihFqWjUI6FMm/cQi0MKdDN6KfObkb4CSK/lzaYFXQ6scbe4fR8+5uRTyPevZS+EoE4LjlLRz91UIoH/CcPAT6h/610kYQLZDzuDyU9jErS15swVY+IJj1czr7b4gy/f2Oi/kMxVKdsLkSFg7jGI9G8LMxTxEObL8Gz/vNZ0RnLnP9zLumWNUyDLvDS1Wl6kaNlRphji7v8hKSYcJ1/o0wxKBFvWMvgxhRj59u9iTqT9YGCVjrznpFI65hvVy/ojWIe08t14xF6PvYdOdLbTYQJIfKUCjGvX0Qpdr5/TDKJgjXU9xv629qWfDhk7CQ41wZ5biMQNZ/63G/27Ltqxzs1Ojvj8uQQwnHJqgyRcyIHHXnOh324Fjk6CwoCaIg13L70MvU8IlQ/6FX3Z4gBvOKUQ3QaZ1Dyk5FRm/UxWaa86/giC2hhaTwdavOdHXUy6ev7RKE3OCR8SEof16e5Ao1pBQLnexdByy3FPee/hlQuy66hH0WPez1y6MsxZC+UDyQpgkOh43jInf6nNNLZ9A3wkrfr9FhxIgAZ04CplT0XTCqQa1xC7yCKsQp1KtrX54OD5x9F8h9Je5ZUqG8IY043SUDFc7+6n5ErIjv19TEyW+ggzt7CDDhv8YrI/LXMvkq7l5U5pLZWdsjKP7lKWdN/gsrvlrgc5MyvR8HC/JRi5Cee1MTjaYyIZ5zinJynxkInPU+awZslBBfkqqOKkSHJzLbTpRFj8rLT5ToUFsYZMaBQh0RhoOOZE4rZ+r3j98Gal72k2uekDU/GZqRPVdZI39C0/RAGRJ3R3jzr62UpXn1MN9HpC6WEQzyLnRwf4UHBE2rkFJjHZ60rcS3A1wGkAn5FIopFGEojAasvEKcpBWOpo4/gfvgu/YXgMaiLgku+4Tsyj4Qpnk0CxI0MLXyPQZvuvM5BZUJh38locT8tdWCG9Viw02GFfbiT6B/c5dCgpmRgifRn+/P7ZD0UcGod/YqyN4sQfmkQVJJP5bZX6lTO49e7TcDVLmMjjE7ML1Rh7ULUI4P1tHqpEAZEdkHSR5DYj7aeBvfAx9mEl0ouTDkSrqVlfmowHgPBDErQOux1RXyj0dgjDwN4zi3NfO3UCuBbJu+xd1wgCq0UdAcwVoPD2yCvUtTxqpAQEa6rKwY7vVnn/mNR6igmEXt5IczrC0B9phl3+2wvfhmVYpiJBk7NBPIcX1pxp+wcX6TAitcU9wKoztg78B/c0vtQ7k/8+ul9c6MPjoXHfEDUBnkz1r4d9j38SDc9dPFGQ6vvtClC4I50XHUOyeqdk3pHHDxaII14X7+1pYDTEdA6+Wu3SOQg8V8v2jPOPGd2TIG7dlTwPuvxoV72Gnr3FGxSi44uvjfq1EyirM+rGS4P4wkulR33/8n4gvJ6yeEDbM2SelXJog3rhRHEn8nps+3PTp/0BHKbISVTrghwHbcsGJ1YavBHlVWllsPQlYR+0RqVUHcQ2yIo6sLma12XwrMyUTwjCjo+wjdlpNQlKIsXtk8rn7m7N9B3TRI3lj/htSKED+3k/9XuHoTspAYKHZnCoERGv63OMPAi/61C7Vmrt/HtVrALxfltjH0F4S0K/NMibgMZ8CPwEH2eh5CKZ/nHN9XfjdwaKqQVBuAmVV/222zL4EzlaAYr39uSsaNIvhnD/6qa5p6Ukmke6V47AaID4mLdRtQLRCfv/G1Z6R6mYBF31Yb4t1KflIWbrFE3pE+bQYmnR9sqQS/VLCARF5pNosIiGPz0+li/DVAnsnDmNpQ9UeCOmZNB9j/EPs3ID5oRz8uVpk7RbIz1W/3n9NrA9EKh86IYO7z/nDlXCsq50YYhyaWAvmqf2DqCCCssFdfUwnyfS5ySD/Z4+5gq2B9/EU3QQicrw55EPd2+kvuPRnut6PZF0lkLIko/616abYem64wR+UjTgtA25bLMA4Qmokc92hr40BMNomxCmAOWiDSjx778iWbDNpdc2JIu5FCkV1brlNqWF5TX4HTr6XFrscAENH6Wo6y6RIKHUMI38SemEDF0X3CJ9G+1KR3Vkn05Z/akqYfZtTcYyVKb3fOtldlnwaExi+1QxMsAYE2Eib23GVGJaeDUVEtTyq3OSwCtHgPU7luOeqa8JnAFqr2T5h/LD+BfviCoFazCIBNHOdtOvp893lyvcT56fu5jvF0rPK2CFjxcofHWDaAkAoal8Pp2dtEtPykuK2OHZbHZ4C+s9//Nt03pp5S+HDZNApn5nl07UvZXIExOka7h9Z6j2u4Rr0bDV846SeAjGnQJOmYFjnu1NdbGdcnhxMLnu+ShUKPsBHd/2ZNKQoiUc3xly49f+QrlbYfhlKIytrt9TOAtIMilmQ6z0k+cJ/EpACKpE70TTLTHbRVsJBiRHhJng5/KbBpQllKcuIHXAdpUn0MMkBYdVjtQTpxB9q5z4qaEsoyH4BscrjcyHA0SLY1D0eV9heR95PTv4+6FBLju+FU+v+l9y5UwQNra8Pan8oYno9GjotBegQlGibNNj4/dXEl1rm7z8UBsd99CMb40KZzXYNhREVKz39RT8WGRbZNqV//N98HkNnEnKlIyK/02y9qFf5luKlMlqKW2GfD3k+i4hql+EwYGrNFCcnWTVExMYrtBvaQsvuXzs8pOAhFdfh64zQLf4+1SgV83o+zm3oB6Pvvpait6uMIeu8vJRVmQ7EXd8Uo9WutNaI3XSGNXJpzR8R/fPO7FUEpX57UIGXxqh9BNRfXDM2AyK1vaFp7DL8KJ5B3Hwjv7jXb1jV+fPt52lMGhtPLofiVyrDftnA2D1OcPKFVEkdiju+cKBv1zd7f8MKGJF8FVxFIZoGCoGbaZnVsLLYbHF2Ebl2xDeACNRlTzQu54cLJNUcoqtgEjaciP6Q1WkCLIqvYrXloayQMeyLcoFKlXET82q5WPX+kowXc6BpzcnqNgFReUx0wF2tutN2v3ZboEsJz9vqaARbhNs20KsuF9alg9p7DJYhxJM8geW4D4BxTv6IDuzwZQWigyG0xh19COFLAd7/wOcHrjG7IGaBJXChPzo3WKjKg3rdCrup43M2wflAefB1xX3vvXH1duybFWDw0DfDsOeIP0BNQFo9++Jm1hyoH/6qD13oEOVSgAOmuf123869vDVeNQie4Est3CDRy84dN/RBkzuUkK983AJB7MAUhQs3kK2eYI69EZxvspI3FkjXtMCp3vCx1eR/gZs11TFzNydm/TGKJkhhI9T5peV4h7CAbtb1UY9tapZOWqtotB7ke8YclT5L8eDwGa8/zicYYw0886TfiBRdwmBpwGb+vd2ifxz8GJlI58xGZkODh2PeKc8ghZXZdiYxaPCHQl4FRQmvtswKirXx2H5xX7jsDiTOdcZNRrxBJxUdMm+/HAffNN1B+zXQCp6Js1s6RoGYmbgqHthKq5IMAQOvHEE+Zn9BxZe9Bfz65IxpFl2vGOl+OZupY5n90T7nQXgLuZFY9jJ8DmkUfi2ruecZhtzw5aOU434sYmZHR3vh1rKprVmLcy/qio+oZUdDJTP4096ICybYNOr6wAt2PL47tM5RnHSUhoDOc3f7wlQXfB0VYAeUHs0+TCmJLaUZg9gpZbX/FrWWgiYpo23aWFdwETuG9ynIdz4Sq+t/v80RUZZnojYIa9o8TQnXAaLZQcu7fIghd42nNVMub671RW7HcyI8g5n+vkUzPX46YvBSwejN9wSfW6cwdz7EiUKQXKXiS9kVOA+Zp4rkTx13x6WwDVW415WSA+9oIvfVSZUn4VkVBUBTJrWezYjA/tuXI60i7IB1WEknSSm/pcHiSYG6wYvbxe5w468SD7eeIprT9XtczFdedx3rPYz8uieW7viWR1dVfN052cjKBFdkwTS+Ngf+MJHI4B8vLqsglzNwQZf7TqeV0AQLx1UEKwQuTS+uMEPeQj+fLbPwcuwmTM10NN0X6jNYSW228vsy7RrG77HJThoTaa302rn6JH/Ka+hufptLSFLXmVnR9rySdguyMwDg3oo2D2pC751u8n0hcjD2DVklD3eipxSf+/pp4bjcKV0RdQXJDEw0uwHsPwS+mqH39vbnqmX1pK206ipgmCbHgyFd7ZFv31jAMZGj0NToGU6NoKQ1D3pOB39I86neNzncn0544ileR7vzNfiWEcynJFx50oxYt+hsK6f/VdQt4aC6g5ETkwvRMH2MNX7DGI4qmsBx4eJ2YP1/XDALn5d4T1ozI62+b2Um/OEr9ucAH1MeLDJm0wqybvwMQ3tST3z9jArdjsGUuhHTojCPBjnM4XXLnLR+lzaLuCDK+Z16Vkbg7Wj6zrv2NwDJ8vrXL1XY3XmP8BU7dVXqmTmmkk2d+xcsFd7tt1eAexhzGlnC8ONx4oF6XGkN/+Pb4kpgPk1vO7mnZqs8jszlZIegpbUUfltYCPpMxIJUwaqbQwnOd9FWYH79eLhgz2GkmcEqVsHg+W/XVY0cihuxJsii5U1k6XK/trzurEWR9wT7p9/tbF5z2HQb6FLuinrzz5S+L0MC6uFB0iySfrczpm/mJckeW/7Lvc9vyTvxyTL4tZXX+2B6ovouKSJW7JdZozHX+us4oaEr2oSFcWahOusYl+C8qw4brR1eIj1WkN/QPNV1G4NQ4jGNU+HXBexl/691m/CuBRPUHJEj010PzsVL8wDH4oVkapratkPgOwfx2Jl3q0Rpy0Kev2iZW8XVmVLPH8NUbLiBchKT2VEB5REHbd5auj3z1ZenmSO+z2U1iNP4yxfHnV2jhJixmNxtmLz/saYrmyRG5hx4RS/rXTpIVQzIeisJNCkcS+p6WPK+DfvvdkdB/2yEMPS+T0KXxRVoCys+pMDe7wU+/vuGdhgplHUQKX+7bsV8kuDqOUxdTCZ3DfEIcs3nilcnJiNMe7Z0q9GMAoF7qCip8/xo1p9kXEJ27Mm4IiQvlmEc3soQCulXXuCzuB0/aAfSOyCgtryvxVdJoBuRpXdpzRyziZjAaT6R61xHal+3dv3opycC/0qyoqRuync5KI+ffd6mA1GBfUQoZIt0e8FccqL9Nd7ndEdl+wZ0X6IApmRePLmOeUpoPrxt8nAtKH6cxfZEP6g3MWWSiw3UkSnaUuZJdx/qXI4RiQEaR982gM1STRgTHJ7wiAHu48CObwaUrGztHa/ahrsSY1+8uiTYf0Mg9nky1FDt/vCff2+ctfR6sYdsc8HH9QGejHbewmhxF2G1sggBrag5N2TvLJSpZO0h3+1m1jlHDOQRakuTKbJDkBT0/z/CS4chnrcJK+EDl0tAa1IDC0b5RYgy7486G1fum1nBSfw2PB5FRf50kf6s7zM9IejFwP/3Afy6PKWI4LAvGCUMnHKJM7mT/3HR2vufPyg7xdTMfSXVrAUM9AZX7XLu7xmC6SojK9MQiMIOcJe3XJYAoG8v+ys5ClntunSlSuUyqvlR8Uj2n4bk52E4LPg5Eq/8NCNDBVb+qFxBfH8n1YOqi4H/uwJW/34gqsgxQd4m2cJyNXm0+odahIw2b/FSjZJwakdphQ/g9Ab/aXH0ICm+I4krV10+UFzTNq4QPjIvyXChHo8UiZxd+xS6hqfaajW+ngNgaNRw/3pgLxGB0z2pQbvRPSHoIEyPTuLYDqKmeBPmlIMawOqT5uHgkFJwKMs7p+sIPtmuLn/sKQhQw2Y2SyIBFHK8Mrr9sFbz3RuSxcV4TcwlWV3wM261h6TB+lssGEanlF6iAZpnXqdY/smgij1Oz/oNlFQGpRsaoWJ1UfU4b6SGIq3ws6o060GlXw2FqLK6BRfZyz2h0NS02AZHE4TKcYIgxK46xWxbG0GsXWKMMBhqyk30PROvPf3lPhQuCQFkh1//G7H5tzl+kdFmrAXEJ6bN4T29fzNmmDHjZEJYYcfztt34Q8yY4kka1o97uztHceS1N99z8iMmelk/tLDQpQob8x74bQnDCwkdXkn+iUr/3Z/xpLymW5mEOGYZ/1TkxUwmN8ND5Ez6mj93Gp7RGOCMLiXYWEj2HlqrM6hgzD2rmxGdvaqmzPLdKkFuyNxLAjzz8xVNFENmj9yv7/hlXQi190n49Av3CY8Ihnyu+DYaKZYR72US36suEACvoin0k/PHzg+xjrgS0v7t2EILHkL6rN1blcXfnzGwRB3uOKfN9lTzo7smiisg2iKgTSn0VpeD6SJL/CIaUnCKhZDo5hkwyBIVpVOjJEDr79WvU6scxbpRFSv1L/JZXHkdkHoN/ICbTBEleXUbMisv9D+Xa8yL9M9f1Il1jJE7ioEyIhr4VBs5iq5auVRWtL4G16hZKZkV6sCnWyi//rieoaCpRxeHfW8cIsi9c7V/Es6L3U4jiOEESRK9rx2VPCobsZNWCTGtnZj7CTgEpNSRBvX6arPnLElD1LLUZtt3e0B6uEtHxN/W2YCeKxDsmitnvL32xVKO3fH/+5O/priiSEe/rIqKtBIGuTNVANNyHgLkHq9W1snl0Zvf4BKOvWAaZ0jKq8+9TjToJ/IBNgLZUcb7QIhxgWoqT6JUvgzq7fzvrcrHke0jw5l2uyWFYCL3wQ1A+FS2uzTqGf5ldhK9IOtnCQjXd3d9SZw+s5PIIxa5GKW8FWaHRtn2bmdES9FvA9lfdD/3iD0WhA0Boh8tNr3E9KueWUefObAe895+6NzsBqwDFk3GTRmnAE5v8WTnKGPdgvaxNjIOKZRhHQnY9YP5aYjWv/FsB4vBg3njHXfR6B6Y53gc0vEaYNFCk1LK7Hoh53W6qJFJ97so8t5oisbFn2tzONjegDQO57vscMdK37uJTKg4sw8VXqXQKB8nEzm33dfSv9KDWmx8e4AAw5tegjaq7CCIfQ6rBbP9No3MrkNVXAHuiXMLWUTcvFPRT72lRuq+O9+TuE9foloUWaU5PxjDehn0OHGaxhbvmc56aSdW8pP9tecbbE1mVCIT/LZWHTPHVLW1ZfI4KiyqbvEqym9YtwmjlTEvro8qU4Q70pTF8utkj8FTJlBCFvRiEKapLX0kH+RK6fUPlVj6OA3E7W3T53wEZoFyko9MQ9v9qFwiPVYW+yGL8nB2slA6HorqPqT+EDLL9waNXD7MyrPBSYSplIhCURPhm5+hKViLzb6A/p/qU/ldkSHSr2I2VK43azv9xDfNopRQRTZbruL8N78Dv0OsVRh+irjnwc2SQHVDMe5bXFz79KuKp35Dmiv6aW3PdnVOC+8xtYG+fIDlLJmiVAmHOBkbGIfNarApQE7P4+mUHxymmSAwi1iovze0IhBYloRQ+SxlUXmttfmcXgkXOhVhFbPEATiMQn92hA8OesRnaWi73gj6wL5ygyVo2Pe39TbFjsUanj0o5t8fejt4BkuXF5dw7/hrkvOnjtP5BQ5zmUzIiAzn9ag/Jj3IQO6Oke4RscFaw8ded2uUdI0lG+Ocg7Sc5gjQ9C1NIP8ZVsCF5N931W1MQaBfL6r8Jd2egzht+GXTslNzfBE1Ghc0EO58iR/401E0RrJumzrqPnNepoP6CnrlX5a12zOWuI/6NahuGQES7lzCkceE/GWWWiHRHWbN/qwMqQYUm213U4eKyy/f+gz07ijb+f02lgoliB1fqSWPLmznT6l9g3o0MuTPcdaPl6rFUZcRyvURQW+FLRDU4kFcLWL3CNxbbPsDuMENBKLgluY//gdHlE4q5eWSS1WULovvHz8DyRQOfpi6E3+lx+yWZOV500GnaZ2PFl+5T7BWwiv03fKuVQIBZ9FEWzn7+QPOhG1YV7tlRBZJ7pP40L7fwg9WgYLxOJJgk6fMDuN6a/D5zTqx7vjQgJyJuV49Ed52Of+tCUDEcpVccjO3d2byRDeOdKm4QcWqaHkd3W/UErz3PEC5z1UmboKN48YiPR2eGTq1+vsYO7EP9mwsgP8K1Rqi9ZA2IgJQXXRhokuxqQ+ALJAdhhT7fqniemzUxYqePZFtWNmSCX+Fw1LlpntrOA34j74WhSRMTwTdZTQxF64/0W2QQXZo/kU+B0Hr6YxdQLCTmq+KoDw8P88F7agFA8NUJ4Lg+7jcPrIJm/hZslgjm+CFNWAu4NvQXWcwyQQoaSOC/xlb/zsm26F4waOplBAM0Nt5RMut8feH+LwBPRRGXbp/j8YPWOAswBVQExSxiObwS0K87I7m20kJJuvm7GDB7Rcqvx7SJh+JffHK+vrikNVfG3l61aMm/VpjoLZFUfTpNE0hLXt07yJ0f/iOMLCsjg1uXEMnN3CJMn6oKyWWgkT3tALG5sdz54o307/6koML9BzdUY3kRTPuDv+4FF3oOwXTfk1Kca6+mIYA1uBUkZmnuoT01O4Bj+Gs9MTx0ejsuYDAg2/7VZBrHlw6+bAjJrPVw6ipOQstTGWH83ZUHzSuu8reOVFCpJ4yrlknygDbu+9maIGyPapVV3IhH52hZ7GRce3NWJOBhiNIRhJeX0Qlk/YXqi8/pSFiQlmEXkkCYAJgtjZDAN736PpnXvEGKslRo7YYUmnfj3ctM9+dLG5wbjsgEr8kQXIqoUcT6TGmj+kLc/EwPOlWLBG6jjyWmQR3ZsvbMW6AJdzig59OWA4ShLfabsSEIk9e/r/D6N07uyM1SR/YE5F6gWl43NgAYqfsNqL/hGAEK/sp9YGMAhjNtDa2QlHWSB1u2tWR84aJ1C0INW2Cf739GvLr5+dDCzmjn/G0urrDJTD2bWrpWD/3LyGsOhLNfruWXVKWPUUWy+vteGacits0Ec2f9XmIE6FO2N85kfvd4svB2NxjBY3XjOOhv2I1KbcSp6Jper7M+AC5uFUOZzPK/ZTGAecs+bXcPRBjHWdpn57YVk750e4hT04/sJPKOKgbv3X3RM5uGdGgxbceo7mvCRrIeqJRhT0Egr83mv9AOf8o4sqhNNwA7XwcntSMNKUUDyZg99Obl+TFVgQsi+SGkufi/kRJQddTRYmcZjMLpyw7Ii7RMUHIIOgPFQAzddnu/eC3i+1OwImO4gk71gPrGUEfI3V+eepBTrSDjflgBLpybHVExrobcjhyRr/BI+gI4M6pmlrjFpkByaXoCLBYo5Lo1Og8qsigDlQXI+aANd71UjkJzLv8BFQTO2ckQpyw7h8MyA3dIoaEzY6gIdzra7B8Y2AzCubrFi4eW4FUORIaAH6xSm/DuhKLLRgSNwX1+8xt7J/L7RAb8C4LcGuo70KptVdI9U2Zh2gY98JVOJe7yJ4IqC9JyDpQzC3ecOGQN66Msf/lyr9v8y3HOxOBfFUH+9sNIWctezTLMikTymXzH0KEQjpifFT3MOl1QoRdastnHWrOILw9LQcqZNwCvJmU/LfP6/zOmvSGRZamZTGWhq57ADVYUS921iM5FSW4QuDDkI4/iP+aOqrlo3FVLMmHqMiqY/37KSqXtcxZlMtltCWrKAziO/a8CdiescaTZoMWX70uunhgQOj5ZYiBAfy9iMgKToH/3vU8lhp/Z6SccMJAaZSf+vEBvWj4B+8rE21DIMTmFH5O4ke3aR6f8OR7oL9rEk3gpp/6rlJhAg3wSu4UNYk29vEqnByQDFOBQKXS+3hEeWCdl3aLiAxJpAtefrWjyctd/SKwg7NiiR2N33p+qdJDEjAEAdoaa1h4g+swTnWWIzT/fKRy4UxFQYyMuPbjaENvH+NElCq06wkVx84P8eO6IrdVeGysg/ipm/wbD1VBw9rc2a3ufxEMJ/+Evy1SRPu0vk3R7pVa3NjH6mWj3DMFg2khoXSNGHsuoEQsJU2nN74tKaIWCC57DmRKNH9Vhbl5OpFMQ/BdcU6Zs1kF42ulq33NJU2q3E2ixHBqz5bMaisTVUFjFIHSUbfN6mAIgB9sGcMY3wiVxWLlUAv35zcUOahzYA5aDkURZIXtM7c94dN/rOyKP7utVEJ+M/fmj9P2AS8oU5MB5Rsw6AYgXgLxi9rsli7JmbdmhWJavkvtbzLAn3z7X/1aLUeNC1OXB14z88iY0qJ1a9aPtb5IKDm5exqS6iBIEDNEiLSd/I318OjhJj6qjmmAi13EIkvfy/Pai1DzdH/aTHQvqqL+tWdA3rEK71jS+QAnDGst7IEnyT/KDGDChjHBooEDmTbXNl9Zx9xNwVBNx1SzBeKFaxDZfeL3ZNwySnDlpd9cK8+a344pXCMHMev0EUB8EbHijMdoqj1H31m6sGRvyM5RM+w5dKft4ehwICwe/RIQEagGH+AvWDwUQDiYVvWfMztOIK9yWQFEE8ulyX0BsDgXkSBSgj/ldKtSSPqqYoN0MGmD450bEfpGRVk2U1vIKOxzDcGPOz4ef1KzcP/XsIEiFaLqck1/pb1jIk7RuPFc9yUnutxBG+kbA3CZxA6+w59l6J18F08P14RDTEdrmkqwWjE7B5Xbeia1+HzzRFiaH073oKEAjCteKg99jIZ7K9tJaQTErc+0qQ2Nc2QyZY/GiXW74gcfRXtSW1AT8K06I3Z0r0uZb6kRyq435GgzX3x7MnPnuR2mAhKxFAPJ6vz5pskVT8mb2U/vjRslfalepcfqgduW0RLt8znFRSyPgKiIorIcmFyQgzGqa5mP5GkgBBIrJJ9Gss43mJ4ekxyQkcfN1H8F8E8Use5AvSdb0H04arxSwPyUmhx6cyrrer/pMcCvdP5psBjxnvwd3QJf3RDpq6rqYZLKzQTiKeKBdSm3aLIZbnKpvaElKa3ESX5JD+IZSW7J9COmB96HHLM3Foa2Og5BVtPzRxtqijsNZ8QC40baRtyJ5ZH3HJO8HCjMzo/y8+t4Kw+VH1mzHdb+wCD9S8mwbjMk3RGmsgmnpdf2gk3ufaEXS2SaBw8V1m6Q2wIH/1sx+HYXRpLPk6iQv94TslgPE07pjwXAdwCdL8U5YXteLOWCflqiQiT5RMDfOHDT1+CxOtJu2jdFdYUoYWYUeeILVUU5Q6WsA7irLuuKCvy7oFBA8dwfJ3fCsqOrfirU2vMNsGoR0yfsSgnnclhZ8sXqiEJoh9PLMVXadvuXm/Cn3jUbuHz92EkocM3UcIzF+pl9pkjitsGeO/Z6r6ZdDgh8uuxQwZjpFYX6ZUOBJQDlK0FsI52lD3NH0p62KM0HsahxkkL2h2kFdERkEy4K/MtS5uuwsBZ9RBGcthGO898cqjjK9kJN1xIa9kJj/CjWaCSC8Pe1Sd9yDjR7E8TI0GW2bnkWQzv7LHt0KZ/qh9eOen+NWszne/KtRpTn9Gx5izxOYaSZTw0M1ffpKYJD08zXxfnVdaQfAbAXyl6TRDVgRDR6Tjo5G7pxVj4r3sY7nID9ostti/YLpxo0a4z/VeoHiVcDVagqYkgmaLIjsgNWzP6+VHIBu49mL3MmvhlZSKp+7DGPkGKOY0l4vr++kvSya5BNAUWdErhw1hTibkfxsq01Pk/76MTqIRcWntBRnQlCPx8a8ZEW7a0TQTjvySG33/TsmkyQ46lCwgDfXrIxW2/vO5lZWiDf53wX16q7/gvfKZqBvLIpudjRGzyHFf3WIRsorLFmc4IG0u/UN8/evs+8n68fSEU8fUo0BM8Cij4Uhimytg6gKx3lRfxNRTxRVJMDyuN9/9Wv8qthdDs23bZykCBYn8OCrT5hJ43h6bTcOvVoZBCVuFtQlfY22/8zjy+bA1JRIxspQZk7HBLZj/fvG5s+2h5/9ivLcFSAC4i0b/JXpfJEkNkBpTUy/vHFhZFiJv0AOYGV/6x5pSibHTvFcVvdIq9v+fo8lh6BXc6e2npWnfj8ofCLq7MIBL/98IgJ9wauhVvkvQYzSj3GnhvSytOBYIoIsk5zrDlUm8X8D7VknHGjqYGz9S511QpddE+URla9DTJTRvUMmaxXPWHZURvE/mhAReLj76hCiUDW+jdK990MQ2tkR7BJn1+mnMK8fhdUjUHyQV9AcxEYrowCdOK/1mEBRa2Nh+IL6eWntfpAjKf1Fi5YY+mv3Ac4Zy/VzjBNej0rlKZAZvfjYYRej3E8qDBSX2/q/kCrtF1+dkndwagSbyobTR0ly1gIT+F4+AYh5xXNg75qqPAaCAs6riEk11Z60tIWKzmhzeBdh9nASnUTztNTy4bDTH+um2cVXVfWvRpmv/SI3i450EDQA0dAf20Gz5cJXqzPwXiGPZUt03p4dq1osuIwtAGl2KA/sb/l01jrdJ+6a3lViK7eTGybX8mAnWEGqSlyGLv9gplea9Rr+9cHp7FLwJjxknSrpRYquVwknErRp3qj19EGXf6EgNjQd8jiWsrD/OhaODBGdNe7U2eVIzPh8SvvhBfuE1mOhjGJA52T0+vt1XwD0D6ZYnvXHng/joDFsz1dfVXrtZyQBq+WzgmtXkEG2rbozxYsXekQ/Kc76IcxTZmFt0hLsmlZqf7Gm93s/IYxo3sqFPtRWs18KXXhUYejuAK+vTw5ho5vXzT7Db8XB5Phb6UawOgxnvLx6jfIjLBWcYRm7LSlp/DfOetndW2wywI5R7bliL/GYle5QXPCI3Yn9IgKRi5OsQXyOtbICkW0cfKPhYT/Z176fOpwdgJN/W3ErJugYE2EwKs1KE59319OMoW2tdJRil3WGDrLxz7rXGwTmBbPZeOzM/OqNJFEJpebuBfk3zkz8qykEXtjtyeIKPY43cCNNDDEefsRrVXo5EurBBzmNYnF4ZI1BoXLu+QdK8Rl5w02uB+a04vkacU6sUH7FGT9M2/jn/OvpVcZWh0561H5jPN3iV7XFVfwEV8r9hKaNXGZMOnXBe6+f7gr9MU7StCJccEqKeXBGXUKMUzwJLPXu87Qthr+CK5/OS2krlnERQsqIfjb/UAhPnxp4K+z9vvie//g0fzXlej7pkJMs5fZGChpuxe2ojoTASqEkS8JkBQhNABW3eMaFxDipqnQtx7VRkxmaRyrfY7RL7a/3XR6wGdPFcnb0fQYfJS/4vuRWdGDfgXb0Octf5B1iGHTZi94RJzx2fd0CaCsQnRmaKKI0zkjPibIUf/JaaJBuK++huiqeEQfnCUicsanxQN1iqmNMWNrwF+cox7s9PslF2IK/lZfpr2VVxoQJwd0DZOn0ja6OYR7I9unUhCkE3KCP0Yh6nfFucuK54fBuJpjXd3r9mVH/blhZuI7d5KJyS62kE3aso89daLU/z+NHitQCwgU8854fDpHipn07I35NZdc7kpEn/9Ed3UU52gTVOlm2Tnkv8e578Bckx6be5HbjxaEkH7VrJ9sCMROEh8Hjc58uFHXYDopy2RbUZua/1cNdbg7x2frzkc7rJFvUw+Ko2BE3uTbl93XTrt2VaE9X7/tCZqw8KaQhoP7omRWz8cCNILC+wDAKNUNtaOiqPrUuYGH4os0ms27MD9PxsxM/qdVZKK0iozPPm8sTNCsa8r/N0wfKft0A5gzk6rMuu+ovhqUA++gR3i2eakK238Kj3Kh+TQ5EYiA/eAnJirmjQfZJflRpKXIQ3pPjAXWn5wpPcpz2fzX76jPbGdZbeTYcn8enEa0tTZ/FICKxdgqvlW+dJ5+PtNyjKU/j2AZUaYP4nCknhKaMzWZP7mxfID7ettHwSmKh9epB7P7a+YWHwIaXrMeCQ/ie5+PE8Jck7nChoYS/mLDlYyS7PrsbuRmtCExUerWCkg9tfR6sVZwbeeGy2JqMXJba3X9N2w4kz6lPygsxn8PhM9GoxgmMbYA6XPGs1nZ9PZArSszfON33L5UUZdJ1D+evKykU9MBesdLbTzTJKQqZnQKz32+srP01J9BXXd22+I6Nq4HB62nENHJNZrEnXdeLwZBlnNAfJWeDq/Ez+dVBhMwhsyN+Jz1Y9E9fU9dPgXuyckf/wyZV/ZXvIIO91/XnGoxCrH//2sSah+dvaxQmAUBqtmbWh6XmZaPgcSw4hZ1F/iNNdHST3P7Znh2BBQS3mAK4l6wDOpqmHfu2uOFBdmjwUdSt23uNecyHHl0+fxgFD38Bfaw8GMXqSx8cuqgi6zl/NhBelcVrtKEM/VvEajin9yUOGSnXQYGy5fvjJ8tPsj3PJQPcqhp7ARfkXwe0zV4tgpHShjdViaWILW/7XU+bAz1jJeAYLmWKAbhSr/e/+xwjHZ8tkLslKFz3lVCld0O8a153QykFAPlkwYnQCT7XiXkQ6hiwjrhmIlIu7ZEjP8OqfOVfh44o9J7xVZzf3WxSZcaBYazl4C6nhC71AaliFSoSh7VzwvL4fIO60GdH+ZbotvPFzPnHemJRMVlT+4xCHLq0ZyZijIrDUOdfQKUhKPgHYTsB3pnT49pTaqa30gf6kdssAHX8SrRFmvhlSdkvFOyP14eCKighs6msq0mIfGY3hLqPRbGYNAStyuJGX0vJZ4eu6TSOqzTWGtZAJIa73XAJVArpxkuP8TyZP9oftZtuXe6oYuDar6fwpIeZTw7vutj/lXVVZe70qt72tG3lsHmAfNty0rZ9BC1YRBlyGBHN2jDc10INPtuKtO30k+YLAincf+tGMXa1GHhLkPHjrWxbWKGqyJ+GUWvYSTFkd0wQeo8bvH9UXG8eHKT15HleC3MdpaLC7mykS+vFkBsEck4X8ESODdJ8OG+zQ0/1lZRIsypI5HMdqNGursT5S7bpptjO8Qft4r+tNrje+oJ1JiuVOegd6JlZKBUAiwr2sg5pNtopAKRkqher2oqXPyEGDHxHjittqueXHFtDEa+/QHVk7GSaHOrJc9rRQG3M/1Dul3aXMWUbDtpQ4Ytn7xvVj2r6RaL/fGNYzd3RDVvMv/pQWgonc1pdf/H9YX8IughycXqUWRtGyxi+IojTPff4h1KVebF315Uf8nyZYidUM7WFfeAnv0Nbq2bYTl0qbb75EX71ehgBI4QqHmXjef0zuHyaPS+EMNolDBwHfVV2p8r78wmDRpjbaBZaNo+avaXLeDXPOsSjXOgQMwg/9mL8TTT6SK9SiCtjmEIWp9kZdGqyrebI3mfV/uoyH+FvJxIQs3SMaU8iFxtJzhCTpqMnGJK1/7Kz/NsyMOmO9LeI5PMqk9Kn7OjQ8xJdummBhASkrAzQUw9E7d9opuWLns3USrlPavc2ADxx5I8o2/ZrJFJdWt+vjtmkHuuw8XWpCAc5Rbl2w8LdFA7cZKU8aIKekYlacOhv5Jb7ujDobx5Oax9UwRyS8ciMKUkBGwgzo9gNabAJiAHv595swy3/Xs43wl03pi+xjHuvKnwyoBPeBS0/UPAZ3EEP+O86mO+/GKCmy4DdwA6NxhWDWIC/bgP9VSqPChyIlAYzzMV/0PaC4T+xOBV05f9EjA5KdUDUuudgKkdNuTmpglWzrUF+jb7/1C4vvljbClynfxYorstWV7kv9Nd6lUiKma6Pdha2COEGUdSz31vGxJDUAEsg+P3V2hnETPID6F/JL+J2lnl1xdfawfW6J4G3wERbllTB9Mccpu4fLpPXRl3lAWcQdyOr1Hkx0C3kCD6RfWN5wbzkjbJRAGRjRzT7x0ked4aflej9hdyKi/QO1w+lx7agYa4aia6+0OCGYpzW5L2QWFhExo9V8AfhofR4EfMzxhmOo5kFOhsQJkgYv7Y6YDZhpSTLD6TYgmaR4a2U5pcKqYeLGMW1o1VbepScMbGcaNPsfj5XIH9jyzGpU6EX+BBOnkf2Oh214WcNlsemy6fV4nUifvnmGJGxR43m19HxgX3LYjtLGtEr57tltbipQaR9ObEr/qFNcaxnmo8JKmdPk9ZQ6lyBV+KXp+9fb1Qe8wenbvwVZeCvv6IX1mWMQMdAY/McOGriLCcf3OXwV33emzzYfLmWKid+eOpjT8q4pc2TVBcA7/JQ4btBFNjyZUUCOVtcdte9omNIcB/3r5kHN7PHZk9zC7v7byccs96lYFvZ32KonsutIQa2Wk7yqp71yj8aR2xn9ddPzjxquJTiojXJz1/D8VLbj4MlqUYQRz1k/ETUU4dzl65/amUaoSJER5lPrcKjO6T6VrmR2SBB9BLRdc7n1H79spB2bOGbS+mgAbVzjsFcg7H9XKbn3vuw5fVe1UmyTq+axC9u+nzy8T3tuilDiGWBT3f9PeAm8aIxOQ5m/pqomVegz3lO5DF4KsWllrPBPfFplAjKQ68XOh2LShur6ztRBBnwRVx/bl4M3VkhhdbwBrX47ni+pLv9mdeHIWz3N0/qFXl0zVzI7Jt/k20TNHTSJP7k3XVKHWM0HxLcL6tYhn+pj/+kLcQZmYy04tA0hxT3fdrdKIjIcJ3XEbuO9hD80+XlKqAuOa4vISFC3dbaKB0k+KwsckbruXIbE5I0/Ur3v/UkIHDXyw8W9sc1dCrM8gZdWZ3hbI56UmKddlMauBAJcUiGmRBdtTYaow4eO/UymQ9nMH/9q5mS/G4Qszt+Kk2/+pc5AoHhGxL/En+joNmBzxzxkl9q0ZXG1IaMDg+13ta3DkMQTOt366hO/9PPPRJhU5Dh0lqsOHNwpahzisu/53UXjPkF2CNWbf2ybbk/S5lpJc91rdscEOMvxwa8QrakchfazqQESq0tz9Sa2o0XrdP77mQJizZa/k9VLMfjasXEAZ3iZZzGmrLskL8pI5s/2n19019ez17L/0vsVskuoVQbLDcz6Yr/s6vXizL5GApMQp77jjjispDtYppedL1EgQnY5tz03Kb4ioi0jmW8+bcow3Rbv9SWR7A5noxpu/+pmh2/BAQcZWJEuMEboPAQF0WGPS9N3Uvj431lJgsvrzpsVEO+JQhfwEFhbO31qsvftmEKX+QUoB4cOi6hngw4lnJWb+clbRnf3Ylfrfo3f5Ri9o+k/eqCVto4dcL9e9L6dyUZPC1XR8x+lq07oRhkokjsp23XrkuKVxH/te3vVbDgbKae21wzPonarBkFVjGQh/OpfzlZGhV9XKDaeuEwrCYPdic+DHnFbJhef6O+QaoJ84S/rjQ3YKtsaKULMKjMzmmfDdq+6ZdBpzYpioOObry8nZbRJQCBpLwhAS5lsyGfJXQ9ltOmYUcuBRfBJO9Bu/COeWG9cWvgdIRWPsbe4BI60wSV+Yb2Iw+0+3H/CZH2A/0MTwkdbYun33ru8Z2sRP7ys/Fco8S4d36uyzET1S1IgcJ83z8R7u79MFEwaIukETTdjVBGQd1mpOOLNdvX3b8ifNsHqlEyqzZEKXod5Sie1u8XZv/jLSyFuJKtJehN+y/OIiSMGmPNugmvZZSZkH/V6uiBUvlPMLcLzmKwSMS/gh7PQuunGSAruvRZkFVQZd040SQ3yz18dTreS5PC9e9Db88Ixgy834Sxndbd/0Yaa9DIDdVSrZ+xeUyoG83e+Jk5Foh/U8PDKZI2bQynrkYjuP9Is9bdZW1c17M96g/49ax1vykGNc4djXppWB9kzvIXd1g80VpH+cCjzgRg6j4LcdeK4qE0HoRlNfoTs0AZJp7fgFhNiIwSfrgaTrSpntZUCBCpxRw+gtSbcblAq8gJTIl82d/4Kc12/Bw9lnW4LcJZB7KMQC+yWMaKj059jaYHybLXg+hspFKDpC3LWDTBYfO/Zl7lWxeijrbKlhvFlx9H2If60t9VRN6bvbO5843yRxrm7lyGCyDvKLhqnNvB6/wAN5sAOT/3e50iVz4co3WMnGlrADkq+yIaME/7TH2VADNnXdk31qKBVje4jISc1YK6UZH1idQ7re75mg06z44kPUlxGanZwGwaBbT57UaA1UoWP+QxBy/eS/Mo4igYgAE/l5ggz8rHIQhXrS16EgrDHkNTOrhto72O7UZeQ0lbWjxKCnx22dv/g4/g4ZtYJJeQFfzLAjvR2aw57RgVCHOB2/fHXi5zf/z1Q5Zzfqe5L3SMjDBGvNoA2FUsOwYHOt2asHooFM7WF7FD3dG85YW/+vgy2jt1pJeXOp0pxynSDIUSO85NwD/0EtZJpFML91i9b8hTYBvnNsXtx6//Otzq/QVVnLLX+R5NldW/tRbJ9ywtByIbkeTHPVZmmuYc128lho7tOcJtxRYks7vJkvXv82kTq7MdYoni+3P124///wJe7qfQlZrrwUs5VqWFNgFwx/8R915bjyvLmeDT6FK9AMLyEt57gCBwB++9x9MPklVHR1KP1DNrZnXX3n/VTxAmkRnmC5MR2GnBXNCWH4sBDZd4LPmapP3HA6p1ExV2SS2jOmy0/ImDYI38/UKnvN5H/XF0YiXkADVzbG8VIjBDAkgd6qO+sPRdaCHwhBul0xR2i7PXq23etOey18Gd/a9Xt+mTne6932gaJp5vVRTvHCUogg1z8oGapf+QUVrG4nDJyYCJfDh9vidVBx++DdklguzWsb9XjdnFG8PrSVnnYcn1sOarpa0CNE8gCx3Nc2nVHV1cEotttFRxvL8DgXkQMtaZCFd/AMlCtdwD06TmOhfa2fkkSNYOZTFlWT1kv+WAvl8A4cm5zAldpRnzlNEp+6X1Dk1ZwLK0UtZiHUQKyTRRZ9eOliaBKmefe7kCjX2nc8PIa46g6XwJsmwZxi1VlFKMlkGj7q9kA32aaW5Bdxlv1cswHj2CZRW1QbLl7Tvtfe1JcaxHprGwHSaLPjIsEd2PqgSmvD/D06uxcoaQ6r4AIU4eEkT+DdFdg1Xc/KutwYWh0HrkRwR4Z5P9iWrHIHgT+xdXiv0+9morfqEnO4pHjS912QKkhZNwYv2tCT68dgTSNeWrF72oEg19fjX/UJzB1R1xSL/E5/6u5LNavkVe1MqIgor0pGhUTlzy4QFBdm17VkA95lS1zAvDiSFZR3nS+cymsNySKpJWLw5IaiFwx5duQ7GEOsCukO7fCgwwlBk0tTY6legsHxAj6j6zHHOiXxYk8HWF39wKYJX57gmWyxXxbdREW3UeUuyW5mFRoAD8Jy6ygd29x6gAejWsmhJ7kSnS+RbmmDTkzdplUkyamzwOaAKCjz5e5Ndod1cznTH3gS5p4e2tCKGW+IcBtvudNK3mLCNRX09frBMWROcNHDpSw5w0RNDc/pk8Rlg7qvMm2t1kYs4jY1cylvx2A02EF0wrX/Uz6dfVuxPVdEKOMtr0si5Ao7neYr8o5/q2iFwsZYUFJYF40/yiNKhPCBxoVBNilUNFO7TK2518ggqQgmnlvQ0CoWCCduRhZvjdeWhmih+IOzuGActcKeJ7p2UdEt11QKdEyBJzJLwrksFTgdKkVLYt5HORwtc6V3zz4Z01djnjeVtO5hVbUeuKKvMzXWQvGR9rGvGj1tjIxKZaSIV3lhikpYt22VlyK/kIsV5WyUuclKlgUtjqEcq0DZ4n2ANzySLJ8x5S7nLwceSF5nyFxzUBWmA+T/XUAv58yK9Lk+fRmjvR397Pc/Gr5y5VhAFlwoq9gDNGyI8aIVDzTPvbF5ocHHOwoA5ZdO9gPW1d82VxAwAk1m7m1geH/npo39CK7bo5BAZD8SJnKR+eLDz5bXseUCk8pulE4CzrUIXcrpGfRmjgS6IKelS7Yod8qwCG4GKQzMeJOVmj4NQNJ6zCnrV2x4g7jTKqxo+GIi/p4OpA2IJcvOvLfOGSILlhSppFW5+XCzT8vOwHUZ5KYmxJSBWkaaV/BDubGO/qAgFi+lxsCAhdqt/surr5X3WgFSA2Sr9AuXXAL9JLkOIFk5QOh5I5eaWGAEdNONN5ANyC4MVFObPvUQWpaEv9TFmxkDzPNfs3upFjJZLTrJ6BFzOIvNN7NGhjF/S0RKrPie5D/5aUeO0xkqQw4Sf35lC9MB+is6VPv83jTQbC9QK6+I2TqmCeIoIy2bs98dINxWGqvOFLuek7rY7d8A86hT0wAXlT7kF5iJBzVbNq/enJw2czDNMf2GZO6mXO5AE52AvjSDfdzkh81ktblB/cRGpJI+nr9idhwIG7c8I6992UApU37oT86oVrYavXxCNkpdhdaJYOmyMRrcxa37cTQ5z5KcyhyJKwr2Gzsej1Yu0oakjG4HsG+uV6Ufz6cRJbKkzhgwgPMmhO6czTbaeoe7mN4komyziDYnCiuzJCgQcNUtk0y32IvenALkrm/chS5BHyYk0qdHe+mTtwZVcz8sHbiZKMbzf0MBAAUUTkk8RMmYW/vrBH+04gBdaEUGAHoWgtyvmVA61Fjs9wPgLvDNzx7M2i6pqand27pP1pgV+J3r5GT2mCdAjMxleMibxUZgqNT2YgsJtWkBGQQD3y3TJOtZwPAvjUj948NyAMAZKhaNbr39qvQVpZEbpOmKFPA0sHOCfySqspSqcpM8jzVizKbA0dh2KtM5UpLafjO6xRALAyQMYzDuOiMDa2XvKWx39wE2gfU6gTtW3xx5yymBixxtSLeECwE1VRV2SX0O6Q2p2bS9TVRM5iM8vI0mynt5yOCKp5zYCz7RvRr6lzlZT7pZNomggaedJHbg7weT5zztDxI92pg4rFRTkr+/LLTrDcNaEHMMBSfy9HaKcdxmJvmGSLqmvKL01E8iDUZGJW744DkeBg+TpdWMDJSymgUDH0Hvq8RSIfn5eHJPE0jy9oBqL/2quCLYcW1kVk5uzjsRSh/lZMZFqxX9fvNh3VMwtATi9v9XHTdxM94fv22VxtF+U4j/EywY3qV3UWgJSiZ7a7t2iBrnA6bRu7O1M9++1ooR5rfV3E257jKNZa+yOW5KPgfO0VmM7CGl+tCzI8ubxSdSM7Y64XV9jWpwDrPcZJpqKHqne8zY8OIJwYwQ8GeFMXBF2CEvUj5XnNOYqY9x0/d1tCcsJRxkPkyaXdt0xh7HGnvZT4DAF00Y2pOP01EO2QjoKuAPGQhaKR6QrzzVctIiFuNLVtWuM1L5/mTaACdOlwSi11QcQqruvf4xN+hsIapUuxX3Hna1dm2HwIkWH5VgrOMA6/YxCaRJINeixRhhx26f3Q363XMXEdYxDljrQtg6JcVLoiYiL23/PmKlv7VKIYBKyMwLTO0H8k421xmGZn/GeF5fQR8T61+TlL3Wnp0BRGdCn/NTr1nECTreHr39FemTHNYfmtDpxWsxArjYPUC/WtIbNmxkCwkWziUV+dX3fgLCzGlQm4RvnYRHzWnRf1p0O/HtHjQDZ3JY00nwGUg3QgeFvDofhld9LJoasz8Q1yN8qRqA8x7/NNq0cyhUmtmmhXn+RoA0LfuJevCKejvLhBH1ABBF8GrhqMbXFEc1b4D6vzU9ydCI2fq7gVhgt/eeoGLpWFtQtKsE67cN9oG44mloaBdmn0YUxbVlxXJHHAaefqF9hI/8fIL94LliSD1FAYtcKe65/jurpR27Y2Hud8MKlbRggPZnD5V8fcvlM19Yua6peI9h68wboHH+9+KKZjGPlfKWS65optZ9KccViMIqdHjVABDrJVkK83Aquw/+6wdv8ynVL1l/znpWWY4dosCG/eumAGIvIO7HISk8/x8lnJpEh95I5Hy2lFNJ7Qfuj1imMCFJww6gdyt+SZxEqWQxFWx8Zxw6gOfKH8nQ+AD7bVxrRALee7vzlQF4THLgkaJ4un5TX4tcwpPwTItjMu59fGZYoyV8/Cwi40+tFBHvA6nMFOmgjAJyCkRheBUayRbsNewPIoFEdLogXpl+q5PSWpWMBAlFLsblY1wx0mv68O8zuv1jpFxLyZQJNfJwNM0GB5eNFOKMbG/aYJVGyHQ+sVRO7cskZsIRi+fRz8one1maR6wEsMPXxTuho60hJegfH50F7rYXaJqPYbKJH4S3lioA5RIG9LQ31dEcmrscE1ZfYWKXPznDEuyG2vYrW7WZPmwX7QrBM6eXG9Vcxe9aYAvhlphd3e/b6oXxQKlkQWxhoegtbHOK7oISBlK6YfEUNxw2aLVu/x3oDmjzyTOfvmS5dCYJ6VReB+kArPZorTkqSFRpZjq4Ps0IMXGuSaUX8uZ+DrjUTxqIpXVPkIuW4vL5SlqsiuEfxlA5PIgt+AJu7R+socob10rpFfZbbxY3agH7K+8ofCqYfb3yuMDCaXcZahY9o6VaomLiSp2jBFE7kDV1Mz2MBDJw2x8eU5eYkPc3kXYazE7gmbvnszX/uCtdO/qK5G6QahEyjsnsFpD1KCRRslXL55i60eZjnBAr86f+Cw77BEHrvfO7wRltEepLl5TKeByAzOP2iW8DSW0mVmLovfPtfCf0U6enGD1thysX7ncun7Iu1rKiKFa1cGzJXeH5wS3BRHaa58SLAlXJsZmVAvSbZTmJo2e1zIH1ArjDdsN+5ak0sZ7p4W/FrxtPxBJ/ywMM+DNXMrgDgB9VDdzBWduDF1wfwWnPVMWXGwEXWs/FKBhVeRKXONXLz6/NdtW+GvtwzNL/JOwk4tTDbQTwsYozmec4v0RZEjQGq8WyLNJA3QHRR4UKehdAwZFFzjMaqliEhJ0tM345vkCg8uHg4S6CHKms/HLql4XULxtecdCGcWX7ICpgTR6z0T1s+iHhprAZVE5qeA/7prrfcCHIylLVBSEAml28WfD/CJtawRvji7e+7+8GiVvYG36iD5dU2BR48jSNQsLxLUBQitHH+4b2DG9vcI6mRvs1LlqVK2GdvOyTUFiwuGqcx7+bUnObDEApij/o5984HX3chaF/1kCN64w68iGotJo9czL8L/pF91KCjtd3cZiXJMC6cJAHNopsAEvh1JDMzyLXydd+mRUlhz4q3IAQVGHwn1DKRSbZvNtd4jNnIka4K2pXStfIwU4LeRGy/hW2XulwLvK5RL/UJeEz9WIN8e2IZ9rZnHMjoUecq9HkvB/RsO51xcSnlTXOJIdT6TGMc8PiVALyD7NJ0bCjm2+7X+3LVld+TlMKMwIovMBXXJUZSAofOll14YWNefN7TOvZ6nedrUOidUN4gOzKJ0I294ZYreXfqA+D/zXCg0OzFThDAF/CUXITgL90GgD2bSd6/0sb7yHn0MTmWsUP3VgX3jd8BxvIDCpfvMaCT9AlCvDF7mXQP31GSZRnKiGZYJ8/PsPh68Tis0buVAAPFAMEKV0/y5r3QWBExaA1Uz7LuUaoZiAoWZflEVrJvwUWUECyxyYFlRTmhhNYEvv/VKvPp10bkgoBIGw4gBZt13bx/c780kEsHJNwWtcTOFZNjGjwhjtTdwEETyCPjCvGmZMOHS5x91KJSFINAN66SuRXui9SZEjjgvgbYzKHDXnHplL8vUsUd/3GY+dLpdl0LNTwag82xbhT6y3EGpVLYEj3+riD2/bcjSNetXXDOIXL5u0pegfIL7NWGSWN6vLL7R2NPcs6iWasjKx2rJQ0marpC8UJKsKMkFOPXYMp76luqK7CGSZUXE5Z6PW3EJo1Ll0TDu6NY8f6Ct70hExvQItBNUAuiq29YSPZolicoyFlUZDqs6YhpD+jaYEsE52P7RqerN/2mLTIssxSYpwVRUEYnmYCmDV3+T5qzE8ooMO1LGDOLFeE3p0eTJGlM+Xe+l3GHR+pULhWWIKTYCn71e/8p5ioRmHeXximAr/bb+tfa/5vVGbQU1pcDSC3Qu51/aPANHQpH/48mS0VJzRJQV2deL+wBBL9/JzKxBMI+Jzypo0rd+6zeQAbpMsEV4hRIw2XMfBMW4zvNax3j0cKEIE8zdXcSAbRsTIrf9soqDwOgbm1p7p7Yplvd5Qge/LsGL9mgQbj8wCeK508B38VzfVNW0nHsQS9RauotCORAsMdJ6PXBaiJ/Jq6hv4XOBX1tZN01El/M9zVDT1MsdoqSKVxKz8JA6ZK4WlbqjMBBGNzzLAyQCjCIVOZZ4nSH2Nr+AqZNz96O0lZXvpySwS2BRvIJbl3/ZGJ3GDbQGTSF26x7idh5lUZC2c6/zc2bY2GA0Lixgj836cArwQyffxkz29FGWQsHY6Ml9y4y6ofud1DHJbxUQyV/3s2MAU4eRaOnSO+Df1Rl4tEAAufY2oIxHnT7DXtwJsBbfCkskNqt8qhfacXi+EfjGhSSkb0IBo2TnRj4nU2hCdh4Ph5MQyrg4oAnX2z3WZLHCrJQ3xkOVkuH35T8m5ErB7vVBMKBxHun7SVS51n9daJxTCFiAOS3Ks05EQFNSDdjxJxpjTv7eQPMwsx+xtWketgBELSCqtGUrIJ0BewrxDIK0okbfpLBH+rstUllD6tyiJ+0K93b8rFbs6w+SQl1LXqO4hkKrebHDgWxvD37xMmr1h8ilgjl868DQUJ+tANQoykWhOxxoeHLlpwqIt9Z+E8sJx6wVUBmIxGC4Mn4Qg6HkwASSWYNZ6Zumh++PDmwk3sZe6KYWWhRJUtsTKsTsApDwatMSD8n2iNYKxaMWxInvvnskVtPwPsM2jIBfACILPKGm6vjWcSrE8QrDS+mergS8zNliHmkKEXdrfBv1N9M9HWl01UxVQbBcLIPxmG5VovgRZG9XTO8tJiMRpJbRRmM+ek8z132BRH/1d/ob4RfkUxyq2vbrw5xO60EyuYiBwA5iaafn6xLVbK5zoFanMnitChpn2o75jyG1vy8S3tL+R2YmzXDsjirXnVPDnkuLTLPckdLag4xRNk1g0/wQHKh2TeeD+9yC7MF8vxsCcF8tTfUM/LUkjr3fkT/KurHNOgxdWzq3Dt5AE06l3Ul5OS2kLj3iVcFyAOQvV1HfGNdT98H3xbEp0lDXGGdz9GkfApf0N8rbITDHKl1SkRCBuhUnvNF7t5HzykgSI4kG4f2ibD0UnxwEA0CEfl6AWUeBeBMzAl7w3sHff5MxLLmW+1B4jSAw4gidRYXjNws00xyZ7xa4aP6oWAtCKKqCX+FIvp2oxDaTkxvJl0+H8cUHMlEa2j7In5pT1XshU+hgu6lhjnQygqII4l1xoLTPx2qOUXvelVV/rTZZjio2bXes46L4poVV9ZVU4ifxK0iZO6sxOkZ7VGCK35ALiR/0FWeP9SGRSugIzwJZ2vHbcbwmnEAj6QRqtxBeR8PD0i8ONxk1j5RqTiVhgF0vtkIp7wo+V1kl3CyIasBdd0u9zm/bQu+usBuhS1FuRpNjtCbtxgqg3DY0j+TMDxhQVTiDSVVN47MY+4k/S8ro1qn5Yx5jqZxDpBc4669jYYDrPw/aGPISCZ165WAop+ld8yrsvHXYbGfyJC5qYI7zTSOYRTV5nCAxidav6etUtxa4wU2N+I6JHGqNw+PsTtrv6trPy/Fhd/Ix09FORNxI5iaDOgJ4zEIiTIxdJq2YD9Pw7IknfmsoJNkqC9mPVmQXAxR7SNGVilKcIT9PjpIdWZ0J34LUUb0MxQ8zeeMAC3qhuLF34gRVjTwd3nWxvOPxnsmXEp7oly8LDsEw+oF9noM4jUufurzTkxogpmJzxxS+q7cyrJ3FFsCwqaxJSBKpFXRLHm7I6mLoogbFcTg7HA0uxYIMLV+zIrtnN2Ia07D7JrTEC007fbv83DulkiNKPEPmfaCc3z5UwGOix161ipSDQZtXAodtGzcb68HC+WtKPm9kf/pn9lhOJRp4xfxtx29xBNEnvvffdoDFQFOx9D5O8tigMZGPwIHJE4Ztf1FP9D7wGGom3tySA8CBxlaajMw4qSODQoezqOof2Zc5xXNam2O/iae3tIPvKpeyVZoAjwGvlx+qhGrNPWKDpIhyk6Si46MF6+GUjwtVGwFS0OabZPk2+0iM1AWcVMpMLvjS91UW9Ce7URuj1MA2zkp17xIPeHKuXR41aHKWoosaiZEgk9a6cZegrGMuXT8qgbb5qI3sVtb7PQ5d0DA5n6PePOELwFQ6wi6W/aqHAyT+86vPaqFFIBw2nDHSo3WRXUbt3VJty3EhStPYRHgTHb3h21prJQWFSd8otoHL4XPryQcvm+IBRGn2zh3hzLdTWn95x8eKLE25sBuFlyb2GjRl+0I1kJ7zvCi683oYbnkb7JKfNi3FRoGiLHq8oBOACzln7aWUya1Dz3tqA83Qbwea5PyPJ287gITjtp5nnNU+zU0TXzdvWxPTj7agftVmr3egctmSYNsjEjx2pRs1gL+aFaZzALllzYWYMT8mZT16rtNZaAWKrQgKO3pyhE7CyqJi0jj0A0i85tATZS2cjY++H2tM2GRYpVcaf3a0ku6snwOfbQUrYyeueuPguTcqkvscXgj6um54M0V06UfO3cbWXXfJMbql/L5XwhtI6LGwPgPYYFyH652r5UsLcIjeWR2vRsi3VFmIoiXlOh6YpOtMluqDyHF8L0fHnU64xhx0JdAOJ8UcW4nWualBdl/ApgiAg9IsyeUiLruoybCgpJPK0U6KgC71y87yD9161LQg++FOM0dJhuTcbr+qQa/RC512+H7OkNgtcjwXI9SnBFGnQ5RfK+Ufg0ruIIKQNjQZhe3brci94GGhNxy5y6pc7TLAjhuJbKbAtK8803zRWwv08EtL7x7REiCfXklFLc5pkGyzsVUFfGlIDJfIN5VZV6Joz3Sg6oHN3LctekPsGad33oVocMawWeL+/VwhnGvHmKhHCrJfRX3AGBSomtWkTGTmcb+5OwT7oGZaBOymEB6OPrRP4W1tiNiRn8Zmo7406VkeRZ8GeRTM0UU6JfxcCuz+bqiGu0lzdUGAslutgoJpixVuiCCswNweTCUP35tdPuWAlwFFw8TWr5j8aND91VUh4b2YderdqYhpXJOq/tMC5RGN9Kh+1iX2JaDQfMI6sPdAYr/uoDccLIkoDeOyMOfOWqjK3Fol5aYoVc0s0NNGJel56FhmkUSNkaLgvpKzJlk746IYzeICThx/bixg4juo6QzIo8KMBAF3H0bF4+Vi/SVyaNaHKwbkiJQ857SsPY2IlsCOFJ7TfHVAeMiXo/sEfKvrvqMF0atDv9dOn5nfZkxC1sqi5gIzQnCmyCJ3xgvBfT6TM/avdKdNKxN/zbBCzyWECpN5lJpIg/VN1aEuormaRLsQrv2qgG2AcWDr0QlvBHOeUnZf0G5rEvxhQYFLSdnZJur7BN7yX8u6IRMfXL5aGMLrfjGyVsRzIIuVVcjHbB77r3IuEUhH6AryR1h8hgCPO2oyzXTQjLfxzooKuwqc60xoowhd3arm7PX3Vw+pJx5EwJdcUzVJXf0qJkg5RWzFfWjJ88UjUQGiuS4yw0nAwuyAGRl9C+rxGKlSZOFih62mZHoSpVukbi+/FHqc2MVkFqSTm3SK9WjFqU/IlA4CvNNuy6eb4O6vc2W74u01vbEVMjZNn2K9RlweEWlyVDTmapbzWniJJpgJ4ICA3Huew5cIYfxWfinA+AwMVMU3sT77DuoFVOA0pHcYCWw04XXctxw6WZblvRbkTiHXazofmA1/dKWcQZDcvZJMDLq6n/Wx+9XwV4CPMPWKSkaw69dmlV6mvo6HsLAUvY/M2bO144Maicko8YEMtou7d3jmiForSnn7VUzuflLtcYEHJHjFr8gLqUVfVRSH7esYFN99ZjfqhsuSHjOr0kD1z/u1Zj8PvON6DLOQ1M4GUM0seH2PIXr7iYvsWD9+shjHW4x0enJrpj5kgijtqys78joz8bsyg567yqbXehJA9vClOCqq3I7OXldONdtU/dLkbA7sMaIBBYiqtWQcWU8yNDEJyMe853e6J6fcineMS5s16XWGWjbcD5rgRJGqwE0MQ/Cv4Wae5nE9RVLtv89EcWdujmsl3FV7to5f7xMjdbJRLsLGjeAWWr2p+b5Ytx+kfiEHY4vHC35jlOsZowk8NkX9dWKXbQd1bPnxPnCuvl6j2X2P/FaTAo1BulhWHGqUCYWfRVb6S/xv8ZcV9+hBi+RNonUDXiNP7gsOC08gZOKM8sCT+y1V+vpW8kPHB0Q7L3RwzuWXpU6C+LOs0Qo3M0cS6letIFCG02Ui1MPP05tWl2a0iBpFyMxF34MKSRF+f7+u+ScrgQbAVkg0AHJzcme1JKjDRcJ/qSnaSPF0vr6M1WO0m3NGUqyBI2JNrJnkOJs+fyFk+mrKXxR3Y91TjUaKVviROTwmbnrQoaPgMFnP38cv1XDTbPmqu6VJxzc+1d6mztMVm19kRm3KQkpkS7/7e+lb8o29y6rr6nixk/qz/bptEgIezc/6nr0qP+ySUwAdQq0cjHuP7P6LJjAuCilGQY5fjzF64xcKc9eEfcbbdYnelQFey9YhxOr0PWrM+9zlTLrCkQma1LvoFTav0zrsynDQFlmBbMrfIpx/CcGb1sX+wXrAHUDpLy+jShsbjGGFCl0r6OdmzBDRspOyQqq3s6a7SW4agzZlCRAx7PqOov6AMbDbkveSFRhi5PsRTR/fDfq1aNHTtc/slXe2MQteSW4kUaWqODGX2B9vpCi4pXnAH6yvqrcLAYWdS4z7chVa1vfDoLBmKRia+ZRknpfs+j32Eg7XO4Zc3b8VCzZed7bOxmyJ3XoFqt3A9n2D+tgwm0l9YtRC+r5/neSn90NZkigBWK7DWyfE05fU19rNbGCXYlWwqumrRGPUn1LLIuzNEl7oG968ZUwW0YLmsptmrc2JqITyk4jEMgI+D7o/GlUP8Jg9ETMTTzeMW9dFqYWSRbX/dWjABDM9ft0H5PsrUdq5qnNR8AZKLJ+Bll4WsyzbC6fKzJ2ibg/tTe8pZuIlNMUUMF4fsLnQyFi9vOT2FkljJKgiFWQc1r3NuhIcUWmZSimMUdGMeB2u8Vhy5gT3FwVfFKuxF07MAnn0u61OAndRaV//OoeylLabs3/QPpjL4TG6S5oEhgVEOveACB6mGFWvEGUXAsdGLQGzaJgPb3ct4fkP8Li5kguvyuelg1dE/IYuOyrzviRiMLKU9G9uKK6r6CVmJkOx4dt+mqVTdhnbxfZz0+Ph0rHXbU+cr6NZ9Su8OHxpr3xTmUVznwF44pjo5NCZlzSOfWb8psxHocU59oBhB7YbFgLsMVGvLMNsUVb0fLqPqPglo2ogbEdAL9MYlw92mCwoCoVHN5fKXC80MX9LlM1HLHizBymkAcXmm0uFdd+DBzsiiD1tJqxTaSukhd7V6svDfyEWTLmM+cH8Mp3U3+DoPAxfaPdX5yoT70p09hnxfzV5HZm+zekXOvwyNZNktWoJcWU9q0NZlBRsuwgiMbzJsAr+cWjygYIUYZsG/M1vuQWOABvZZVWEgvYUTurLoeSvqaaTAQ3ayMGECfkWLG/blGH+oQHQHIRvC6ukICa5Kc2U1RXQq7LxHv2CUpLVFmssGH7LZSnIGlfdWbitll+rRFa2BEc7M5JNtO6xU8z0jjJqka0ioi7u4cuUbFzzmYD9IQLWXB4kmJcCKGWOVS9KrKOcoM+HtJmgDTfNuN7qNW44eEF0f4ydhpJKtwuYZ5DGp/1OWRZDjw3N3brWQCff4Wxt7CGeWsPK1273E1HMLsiMs1Wxi4oXTYN58y7X4+Y6cRk/l2ndmlkaGl0FSHElJOH3+bUUJHbDTL4oPKKXNjhaN+CFcX5emZMsnXJlcwsMKlIwH79vjtYZO/7I1jUwRnaLBekbMtwWc0cdO6U0pmgNht+XcrioOE5Z/cd8lREbv9YaFLlxpEeOVM8rcN1N0tcCYBZ63YXN9BcsAQhH18/J5llree59UGUyz2V3t4JnsSm/MfoW4yMllnJkHwOiiCauy1YTipSG4oqwGxh9uomwmRRAIYozpF/6DvJsfbfD59Dqb0lm0eV1mSuNgpjcBn+ay/eTyvxj8oMZFJeHT5WxJR8bMwVc2lBBGY+xhZBf5swQxqWl1poHvoxsKfzKV4dlcArLjnSt+IOPBiqqIx2mAojiVB3p7+Wdkyup66qfstwf2qCkdwhbOUPl79VSg3u9IFdlLT+hgnwF2KJ7bNT068HuLBdqk9Db8KWKV+5Bix9Sykgk7GQH3RX2V/8hMsE/t1Xm2IdD5UDW8ff6SVWcxVOfYjyRrw5gWxEHIZzLYrme8mupZzwAqbG5uDqpD9ORIL6yA2yAL18o8ctfL90OV+efg8bWlYjY4g4v9JyAVJrBpNVdcow8gNgguQP6yoX1VTINeY/EcscEQqDxOOXqtTxAZYHfOEJJx4BKndDmJt41Z7d+7lcCuSzFG8QKQ5uPr7WEhQHyNl5E2ToldVBNttmFwLomNvr4oDmOzm/F8JH1LENXPMmDUfplP77T7Zj3xOiH+yOnFNUVD7YJli7ApvYGMpsNG4/udjt81azbUWBjxzwoKN/+ClMLxY61UIDMRtPYj0bn+q0c4++IO6bw8ZahrqmJ4OnrpbrLQqvYPTN7nR23QMpbR49ni1O1nrRKgWSa7h/S3quNFmeflrzXw8j5PHAeoKitSn+nKxdUMu27RtxjvrSLuiPsBVcqHEvRNqW5AFgowgYvlKJcv4Q69/R0pbYmQ0t06IMvLoQ3O/f+DkiN+X/3+3fhYw3ITMCGyyGVTkU9NsGdLSfjf/ivVUK91cotHvnRb+0FRmZxQ1E1Ng7A5826Fd+ZIkdEJ0jvcYEHvl3aCprfcPhEVvJEGB+RiFEP4iDha4m+NVI8BujihOZPuxkHNJOx+ayb8oHIQjbk0wkpTmbQ7voWX/b2x9P1vOCLRnZGKLjtazHbw2+nwitr1E25mfuU0xhaCsMItn9ppHLhAfnDM3XZLhHhqpQ6+gp7K/RaWo+x9YbqHGzy2Sg8JQns+2mDQqCI6p0UAl6KTkdHsmIELcmdDWPAd5inYzaGN/Ulm6BvYR9EfXJ/hnwdoCGcbggBOGPCLLu5LowfoxrYqaPOe4qfSI/Y2NLu+qlzqLZ/gUTinNFGLK58YMch97gw5FLTXfawpaj9Z7f9gnLfhsTwtYHGnpAfHWP4Jpq6Cg8Siu/xVMAUXsHWtlkTES0Os2yy6vmvpfgJmFK3ax6gbU0m6LxS08wxLOTR6CaLOV67qcokGskOSoTykSM9htn+nKep7IN9VJngamNQO1bzLXNeSeYTEqZZkoqkvOsVk3xUcFdD0t5fPTDweOZ4xneZz477WPw+YDzbsvB9aTJWxB//Bq4cx8/RXda+kub8mhJhr/nOul6FHFRuC/+9vweEg1bjLe4fDB1DcxX3V8+PblX4zXaFuvIrGf/YwC9BjfwkcAcEg8tBkLAHEwHILKJnlV8XDTKtt1l+odR33UeMIbSV4R592Xw/ICzJ2uFn16/PKX0EbLe0T218oIsVyoIJ2QPaATJp+jxEc5JKdQZWpQfuNJpPR43QbMBncghqjrATmGO2w8ebPTJG2xedBbgvBrSFr99t4CPPphRZwn1URfgJzsF77+yjUSH93rIugQqtmN/hXb/enG2bKYT26/yBhDj4xXQ0NOTVRcfuwMnTPS+nwRRJ3VFGt8f2bYDq12fnKRb9piqHwki6qdGWrXL2kF1pAEiJJJvIJK0VGjvaIYae3MSvTFf6QO0E8VjSpA8uH5zfxkWPIihRkXZcJEuEHoOhuueRvPvuNmaar+wOZfeatM8jwN8M87lCctwI92ImFE8o0dJlt5/2MAQsQR/fEK4ln8c0S0aEJEVOhh2FX5IXAWTD+clj7y5d6ev92XnleZDqoO7kLyAGS+sNZQTF4LP1JwIADRS/pb1x6DCPLYYwSIzPfHeDjN6E6i2q3mEs9H69fRdeiEBjM1OL7ReavqjgHc5dqiJqUkvvoWXSILGZ4Dg/JksKhwqw/1T7BqpjrEEDqWhaZoeWNwgZsJfEIyByXqfIDBa1KUazfE/ad8ffg5VJcuQMknbQVyu/a8jhM/V7pkrcfTO0hPQFaBBr4y1UY+ez31djCRNg1wpjdo53hrNf8bhrBOy9HGRMZxfP2QQMke94MzNnkVVaP5GCE2V42cLVWy4rP1AelTbT3tue6I2uHk/tbonvfoCcAWGYclgngtaNK+VNt1VAfEnpgVkvUqUPtfeMUgbzKqM+XRRQ1RJV3UEB7dw7o3MYJ9dvGJEGibddCYnQzMwGuN4v/0hSsvyqcs0ihA+B8PkZrK9RUz7Xp8DI0dCoz4ZEJcBB5hnCzSWbBBos6Hkh4nEa/pkDaTbpu5yMpM53S9LffVSmFvdMTm7qcQuzuxRiU8dA1EPuZU+8xCTbKOABb+D7mt9zRNzuKlBeABg9x1HfSh4axh4cszr8wRK+dPJ7LAtStggscJutQFt797UK39EaH6t/82nJWa/XQl+53w5qT0ZnejuGipESirXB0FXuckoKriasXBjbu2+P6NOM8JjjnpG+gkDYDMTs0eMBldRG952liJF+7jSVnsNbQNFSfqbJ6GlIBmoISKE2kbr12JH5UymjjSLS0ngiyKGFUAUw6PDp9bh8W8z2kIMOhFNQtX7GIkOtfc/5Q3H1cYtMNalm5wLnJayW5Sw5vzZhlyisas6hzGCivvbbayNFpGbvQnZcope0obc735hgljFwQR5OjORWfm1wQVkhRUkwSGqJ09O5z8xD6nJWNbSWaDyI481fPp7GhbmoFVDKGbootJoAYhEVpo56TobIUJ1vv21j5h7c2xvO7PqDKdDwxYznDQshvaG2gwNn7SiM33P5aqutwWKrKUfukWP77RSXEmAB8ESGAm4BFNTCm8Nt0HkaY30AFxLgSsiIdk8CEegO1+hHV5jvX//0GJtVQ1mC6sjG7MvWU3W8Z7V35rXamnBgWce6WapoVevzinGXz48HdYO3Z6Qa/qwIkvwaEtq4nqBvZQTFmqta+YxXC5QoiBHzPUlGH5RI+fi2bIWXqMeykbpozoDHHyaQ7I2XHDUykGWNNM9BLNHLLNPtnLdNfr/GVIFbkmVFVMdG6nddsTHCgBmIKxhielMXUW/aqqpG56DxMVWqKstYsEm04mnmweandooUEQO7txrEeIuSiBwgx2YYRSn87t5o5FbcDP865vt2XzOT2owoUTRz23u/p9WxKF9gfVe/wrBbrCxG7GMcx3nho+nOUSVV0QiiN7Bo2fpL7JBSccJjaynXGfG7yFvxw/d0+sW/Gm94IscBS1ToerYQEI5OH2v7cN8wxlU0MD273IsFkPPUBmI+lC1exnpUmBLLMCoRyaMLaYlDTp/mmM+RlE94xrThd2X7cJD5Pk5gQRTdFLtLIzAXrSTGu39L4zdRgV+Aht9I9H5MeKbhhEd7Er8KvxCMBeFr0CLgX14CQZrW7hkPL2CzqYbmfjtrJTww463+sFEkWyNFS9RkQNOLeHQ69gYWZPTbvm+E76aQrcE5tYPJySFqvTvjj0bYG+bPzEhKj69+KsAwsELdPUmAnPrwTlBbNA32kvLHxuPWYy73h6W4AE/AjVOCvBraybm8ZutmTR8pr37Z5HWoSjILPvMAFLBdjFKvISP8QwHGpfSYhiKk6vMDd36tc+gE+rpL/IzflqTyNOh3DHVppCXi4TMuyL1Xkz9zI5eEdKhrH25aaMrb3O294e7sq4motiAd8o6P7aIdbGKdl2DY/P7uHdphWIo6f+XkeKAb47t9lDcN3fc3PY6O17NOOijtQzL7Y+8G0ZepysW00+gH8JQ/82oE6Qi8IUtUHSvoVk2zLZ/O28a/auqZp8rtpHkwxq6zvD5UlJi0GfeHU9GWAfZ4ZTqCdc6ijNWbv1kerygfCnLtFbc0l80Svv1Qe3VqQBXg2dp/zYalnqVhmE5icDVe9w8N3N3zt3vjGC1ZRa9wF6PUpnXsp8s0Y4heg0NuxHtFfNQpK7AytDuIZ5y9WfWRNxrBTRuOXR2NVgxA7vAsbPTLIsPKRl3msSHibkObwCqAs+SkXmOGIBronUtPddcJS0k7UDZQw6wDOR94dNR2tLN3VHi2GQ7fCWmT0k8MB2U0gupoNAmrD3n8Csld9yd936VgccP1vPkGCG3D2X3bGkjKvHl+FWSNl/UqCYBGOLEp9HHNblSca0IY/bHbauFF2bzO2WzImlOEdbeooNan9EGQy5G7ESTF7WbU49n5kxg0J051sMxmOGYNElG/qn/fz0ongcrKXCgJQYmzKyBvdaCjsBnHWNShEugdvMPmbKg5KSmoR0LQi1/HxLDA04N5TmKBkI8nSknqcyEVSkoQvymIhWqn9dai6sQJH/HvSbmUL6PcI+Upzstoj3ifbyM2m/PsHwzqppOq65QEaZzcsXsRdrgYWx9orvQoOkNi/RV/XofYhprityq2PfLuhouVKLxneFFAXAg9iZ2sn3G0FIPmN09sn5udy6TRugcZGXEZqmZaJPEOiT9eZwRpI8dfC85oNjejfNWxqXbM1wg6HPGQj8V4Qk0hgDVbsL8rdwzbDWf5a313+2MoI9KxhX1Z1rPUgDPCisigL/4GgqokhIQmvivzeQwtSi98SLhoystJtu923fUXiWvurfGA478bze9waHHXO5bzuxdViVebvfTtobwjQUkONQTfib6NLwx6wZh9CghPVy6V+bZW8g0VaFa+iI/oOxRXnvWH9prZI9aRaEd12b1LYQ4qRKX5o41JzNGR10hMJUoKSec7BNR0h5O/giah4wUBmnmNvrVD/qpeJBByNfuZm8ryf+OTihBEKBkT7FLlXQcadkwIJw4zps41ksN1U4aMRVfjUa8SwqJtt8atKQj9oIaUKvIkUhnmAIhizRGozfoslsmCnBHRfJOBRHBSURyU5Tv5r86rtea1pr4IRzfhBeeHuZmawjAG9NEYhqdaXowNTFfjQbKAmGVO13k8TF4XKJwosEn6oc819IB7/jMw0a5Tmk4bRcn95IryQLDnGXN9cw3y1WtVgG/TgG/GCL1bkhPsWBM1gCXB8wxTCFRcF/SCzuvbGpEdWYuQqSi7f5Wi/KK49tbhBLAhbSp7qKSGgbvzNDM/TuY81dte6y/m72kvZ4k/mg6HpxIG47h+jhpZSuF6lz0biGlDe+hbk9EjKiHpxSCBgXQHyEeEB+dIFRqhdgxN45fNebXgFuwiKDbsSlJaqvaLnb4ctjnYHOSjWv3hOy36xQmfV329dN+P8kloSVfDUMJLoqmvZl6WYZbR21tUlS7NEGyTf32FSFHIj7J50MLt/bqhzrt59Jl8ezRh2OwvvTLXMJwr77zHXogrVcd8hE30saV3Nem5bxX00qY0ERsA9vPwiiMY88GGn4ClRS76CEi6YPy4NKA4+mMbSfK0BQ23tJ4mFV/rJAtfuURpJI7QdTEsVDaLCwrWQjsG/UDYRd0Ld+rlUvZSmineMkYud3iI4Je81BlqhjaNNPCdl3hO8SIkGy+LEtkCdF5kKxQOWCYViBWwzYBWr3xWqwA/FaNwzWFxlk1xAOU9fMlI1E/+0NzzA/TZ33O1n2h8fgDnFdXv59E71O9aQGW/a8Ey/L2WHqg/1z44kP5d+4gk5++1gPbt37U3SpAMRbH/8o8aXe+uxb1WpyaJAn+4lncbZ7M6hvkX5FnbZ72g8QGw/fovIJUC7AQHh/ZsXrPz3x1CuH9BmO4UsqHLVtCQEPr77b8iJPTnmuvvAfSF/zlwVOla/r0Hhv4PDPtzuMyqovzH41Diz8Fo+XOg+LcH/Eql/B4LnJMnk7XtP0bx+/0FVemfaygN+dc+/NeuLHb3PVyBOwnqv4Lrf28StY/G/J3358CyPqLyz4F52Po0A3eBnskY5rUciqGPWnUYxucg/Byss3W9nOoGV0TbOjyHyrVr/367rPPQZP7f93xmic7Oav3+/Rb8HoCb/w/s7yf2/Pus34frHx/SInP+juqfg+D+eZROtnn/jfN32/6Zn+8/7ws+Bv/+u38+5ffpH4/5Lxd7GbY5yf67ufy7cms0F9n6/+BE8Eb/LfHMWRut1Z79h3H839HA30vNoXoG/W9Eh0D/meZg7D/e489Q/172T1Ki5hl0Nfy300ZwwvJfPwj9zw8icOTf3+9/fQH6lz3+qwter//+AmAF/Ab9T1b4t3n8/8Ad6P8Z7vgn6RL/gXah/wXt/r9lkb9cCP0HLiT+ezb838Ef0P8m/iCg/4E+pPhvf7D/TMX/6Zb/Bbv8TzeGif/MDxj0H+/0Z7L+pzv9/0a42P+acJfHiAK/Vl0Eppn+/UstY5asf5c6+seHvDoB0dBA31XJQ9tRnLXmsFRrNfTP9/GwrkP3706g2qoAX6yAB/5SODO0w/x7NIIg73ee/0+0jz9H8qpt/3FmP/RgYPnQr3+5Byb+MdDnQxqt0b8g1J+PL37si395MdWHNuwDUoRiAIpcd7yS84rnN/BDiT1DBc+/rAFln/TR85HStJz1sdHXdr9p7Gx92OtrYMsj5iZ019zNG77e9w4xjw1h2YfPCa6o4XRhpXk1o0Y1dJQhdDm3lytnp441DJTm8GJB2prkijv9K5hH9CJPIDhcYDAK9YT+7mfENObPhhDTa5snLMHJrc/0D0zUcN8j39cr2wi4xehi4QuNLrRzHFjuoISCfD7LBQWO0YNJB5RKBSz4P2IZi2EPnzoE6uio55//o+dK96jxDcdqsXJM+Y546+vFejEa6bzQRzpZw65SisjwkQGsL5qypgtSfXEB1DLWMcs6MJJBZCRp7y/oYsQjX9lDaFFmKvrQ2iB6/mImlOm/fIwG5K5g+5gukUqwTRccnzrhKpSlfX4gkXs9D5bV2aJctE73K8xU0+Qzc8UrK/CQUee3zISPfbc2ztIGcmBtLav4XmAX3ciIPSlM89W/P8xI9EeekLRn4NzOPNgdOKx7OGQyKSzf7XiSGP5Nr5CUM0yKcuuTjB0xdn2mEHWt3ncGrzHrYv+YMZrp5WEgTD1NkTxxhVvmEZWQAhguVJLeu/lVfGY0XxVI5YpRChwKbOXcIQr/SIqzfBTh1cK89ZlXl1GpkWlNtPYZF8ohEA4t5l8ISMXEAPHe980Vy0gFjcyymmHpwP77bueo/JJi24Vo54ys8PeqRF37StiC5LhC44PUBYmOvfhqlQRh943M3uM3khzRZ65PlCsFfhGhFHynX0Pzzxxm+GNkQ+vEyvkq+ZwjGVQjDwdoy0J/oIEuHbXanMOmGZar+Hh8Z2zGBZ8XXcIGzGT7SFjt2b0S8x8jePf1C189fCCI24QnP/FsWnzmdi264rhZvVMjzfxlrT5a6fxswIb/Eu6vrI/NgJAG+m6oqOFFKrHfBi+BWP/4glSsvVheAJ1N7r7voDELxr5rNLrUKLrTZfJ7/9rJ5EpnfnSo1MVWszbFc+K1w2E+exPm3NPSa8u0IOUdbpRc69AKsqLLbcVoj3lPtGscvGkaCLF9YcKB+yoak0+hUYCnA859k0kUXTOoLnBZxX4izIbZSgCtZdczKYFUeZ0/wkfzkc4oqkDwNOpi14ZU3IqrS7bopRC7e/1yayL2zaRzgpoJqIcTE1KR0ezoFFCciOdHZn/3CKK5ix+9I2E3FtzOtmz5nlLh/l80Xdeyo0gM/SVyeCRnAybzBphocoavX9p3tmq3tvbOHRu6paNz1FLLNqSXxiTxrsSoyQhb/o1DnP9l4Hr+zkCSCOva83F+5vFrbdPiGMH6b7+mBeXClDhUlFKNkrnBY/7F01XmYuIo7ZppR0nmXgzkcUPAUasnqgUHKRmopZYDQsnOg/3tsS4FWXh8aQo+dvejQmgNU7qcrhLULBYbY+AWINbueMYQUAINrY/ihALYNj676fFVg9OORcD0WXze3GWYaILJkSfSIhWaxkTR0k/ialWYCdO7L8W3UYpBS9rJVClJEfW6uPHirV9hI1n2+rl4RfKl6ZJzpd9aTofTV+hrp4lfhgESqHRsO6feScf83gi/Pt8YdO/UiypWQ6tH/c/jb5bY8mrmFJyrnZVa//ZcjKJ7OIgUNf0/2Ylq1EnAZFyvoov5CCLM/S3wL/Z76NHe1fmCo8+zFyUBAhyN8+A2/6W3f2/ZSIH2pUzxdVg5f22iT9dI6q7mAooTLGekzvhWy2BgvMyycJMg7B7poN+tOB8vKM5XKf3QevnYGNr8nod1xia4j6ndayOor/xjj1gCk7wTU0fosyW66F5VZkkprwJWchRwedarTi4IBMVQGcU8HbmZCpF+heE+faDfFK4Nj+7XFF2nP7i/UVZWB5J/mLOUcdks6bdh2ioIzxyUXYuVzPTHUG3a910eX6a85d1fGyxX1uKL3mn3lnfY0wb/7CQmeR5dAHWH56CAZBNut0uSVcZnEanCYNA6B+cfPKs3f1iCBUaww3LG5fzGpIdXhtSJSUbzzRXlEvs0Z6IyFfYlEVAzTlDo9zwpY2KeLcVR93se4E9lDZL3H8bC9PLB4AL9UFVs8tK7pOzzqzfK6+X4zTU4caI8oShXMxv7Lov327ik2VZ1ZwpeZnRMtwUEvmGouBBjICH90CVvYcSrXTRja6YkMkH3Tb/belkc3wpdKCeuT9RhIzY6BWBFZDaPLqaDywBFhqeQX1I3DEbkxDAFvMBZ6sbbRXH3N3utBbUEILG1e9AMawkAw+YKXyAiBeYGMt2kr01vA/zVMmQ+1I3kt8grkpe4jBBRB25poesjzIrXfsTsROJpdd2iC+TEN8NG2gGfADddZ+pTRsWETLI3sO2VElgMTkbUedOgLR5VuWGNbnFTD9HWKVkaac/p0K9KsSWzsNENKqbE2ZtG8+AgPLWqKCN9kTdKU3REAxyX3AJSx8VvzH1qgfsTcs8baef5lEZ9OAr4FPf3KeTzKcEBQI1beCKPu9KeeOk7H69DrWQeY7kSgR7s5XPG5BJvG9Tx8w6K+jHqkjPa5MYKKHp+ZD22KUWf89OXD/P+HEIEv33CM27Zw0B9F3FZlFa0KVHYb0GoowvRx/xGGBFS6uV3I6YqmeeR4v75fYFFYEBBTOZIXlH3zgyOue/j/Eb97xLOTtMhMUNUaVmZ3gneD5ZO2LW2i87hVssf4kfbwJGExFVHH4IqEDryNlXJUAVH16RnissRTKW2EEESDl7K2hAk3E/5sX0G26omhLAduTxhW41EwcsHEQ59FqokASXqyVIncgMMwu1D5f6G+fnwjDtErl97BL9UoEFBbFUK/lvpASQxXbghaEsW9z2NB/u+9k1RNypZiwGFnfc+f9eY74NMvDkjPCzKtvcjsNDv7/k84FYPEIYPym6cRhh4w39zly7h1AplXhE/dCgLDe/5W2NL2EMXarBOj8/wxusula8McqY/65O1rf6DNCxpZeFjcCUmhIalWYivTis4MT1HEHffFcLs8Evr/Ze2PwSAwR4ck4xTtGhL00KCsW4uq+onPvRIiO/3s6uJ0C+p0K0f0fohlrhGrla78QHYL9sobzDRSoGmMT54CiW5TIsfrqzGS/nWGXKIMA5RnPR4H/bVw+x1aIBxnP0BEIykOIq/mPYi0MX+vkWZrWYWwZxOJMP4di2i4NmdMcePo0deZP7iJy/2Z0HzztRujER/5s40HK+HESLzCgaUV4kG70V6+HKcgYkT3Xy1ZQSngvo83ji6RNUK02WypZWK9cm9EgibKJ7iAbCKKNE2POiYY2P7hWltWdANLBaAWDbYPf+udh+pSlWOD8T7zQBd7h61nOuceOmfpUR5R3O+/3/HHQHBH6Vkiqd5UJkGrlGN+OHwQLx0O9lRNIGRN8snUri/lZbu+RKF70NW+JBzBClg7Z9HZgvom2MyXm4JLboRrca4PeuWqgEXFYhPEFItWvX48AHSeUCaV0BchhlJPMUUXA3MovLCHwuvVbBb5iFTeDsROOnOGDUO7TZz2aWzTi08lEfu1FcIx/fhpSWLjW++1oeN8+yOOtLoibUUMn5cfE1SSkgnp7mdXlMbA4dG7vr1dzIz941M8OrWQyswBXeIgY5FoKak31WnBVWXysLQE96a+TU1hDHGowpfPuiDcUBs1siVyfSAOKjxN/xVPR2irYG2zD6laIg7sv1ucFmo5cTBic+YOS3k3BJ34/tjY/xbspfY+0JImU46Y5weqhvnolWMyaPVG2QZDkIFzDTaf4h52wAtRYGhHcSYLCbIpGyy1+yvpua7FJPndZsRmKMRAL8fVmvGvpVVBmJp8jDF++Cn998uocvlTBHhll74YenG3llbdB31K61rKAaWsOt29OtHKW1sNBaujphGDYDVyRnAGjFjOvib0FQhfp2z5SxGX/RG269XNRC7FHfCAqDoagVT6wPmnAXoZn2suIp7YxzipmKwUrATbzdTxSCihoV/ig2ugwekRlGfWGVjgffGDnh795AxAyUCHosIBGNXx8SqRWi0ArKc97fgoucb3fN8O02fBUr5IIbVahb+cqZvyrxIBhrLtOsOl2TeUWCVWqK/Xx8tjSjgjp1hWJINdlEaOoY7xCa/APC9GBH/tn/EkB8+GLI9EGnX1MGsVXD+HMc/+VyeIsvAAaIkFdVTe2nlt8piueabF4swLACLxunS5CIWSz897RPsrTzANaaA58RBDazYM+klqi7AS0MLSniRjCoSKvY1YuivpGh4Yr/3YNiHZAYg+ts2PVM6MWSDaXQFxovyN9P+opSCCSCgIdWCL0r+QSb52YFHQlU6CGe/CSYUoyvOqzfEVNoVnzPI4g4dFNPPIzwYh1mKX5hBsyc8OgUEqclUHcLCMkflOWD+pois6vy8iRuUrTKd2i/3EIP2sZAyDwBGzwpE5gwQtFzOMjisQfbqdJ+TB9AYS3l4Tn499g3ilPkj3+LCoYtjB2J7aPnj4frJNgRKwRjUtTAg+ocEvsWMQyqgw8eX+OU3GmkpmEhPm/KcDf2JoY0GZ2TxzR7WkS0ssFurBZJroBiT8UsJWkE/SEZT7BMDbqm3pPvBaev3qzyTu7/bSSSFbRhRWZosZDfTqIDKN4JD0H/9g7y2RvrufYdPfAuPBuONsADxqwLdOyK2qea2sGLJRkowHWJ/9Iy0qO/xY8eP4nAZI+G1AoR9DgcI9LDGM9h5haH4CXiyZGcu+LRoQKtSVCeH6+D6wUZWYX+BFOgX/ODze1SjUGp42hqxZgLZg6qwGeZ52HrOWLjCEiY+fWeCG899WKD4Bn9xA0ifBKepN1dbBkhTtXT6g4Tz0BS0s9hURjxmL5dWn3zn0AC2ExnwwAt8p6pGhXg83FhaMFkGOSDKjwLlzbh0XNnJ+RF2do/e7Q2iy3vor58VLXUWIO9BmIf7+1AIdPKMT5RL5eiSZf588ZKGjy0V/kdX0t9VS9glfgu8OEJGg79OfWxrJWLR7b88swUgBC4YYqnvkAroEgkc8lqxm1KVgnifGSOU6m9dYcmm0Ttz3CW6+f3bbIm6N+7RGnrXVU2y8eERLAX6RDFQh9g9xjVzSAvU2F6cPrcrymNbvO3veHRkLwZdREwJGXqhAazLTHlczGa/5F7OIrSxjqWyFvUxIu1QexZ/WX9kBwS2mbxzwzVTTaAFdXAq6TtovaoSnFdOGwD6c64yX0RQ0WRfQaBReqmIbZpAbdSO2JYn4ldqJ3CgXHO97Gd9PzykU/P3r0fFttHJAm1av1b+uMBQUq4RQynovHiIfl/vWd5nkRUnbBtFQwkJlxgWkLYDVQL8opjd9MF741CYg+WZfYm+qRR/kxMtFMzFTIDCGTedu0ULAIceZgUUiuIi777Gx09M9UZcNAl6ddT3d19l/WZA/pwmpYeFLNynnPMXWb2ZIgLe4te+yRMgCVTAgmVJcxc6SZWJZasyTI8YnMQImDow6mHOfJ3LS0rXYMHdDU/VRDwC1iu1zEu+B98h4/JEdJ5NVYRyh+qUPjFIyfqlbzY38GPem+kpX7BVLl4Rz3Z0GQ/KygWLyO4SyWBWx71cW4HLbAs/2EDn5BhL4BMaxwTRFgQm+IFy0KH3wWfriBsm+zKtopzam3n00sF7MjvXMyQO9bHUPHefI+fpzHKUrB1CcJdDbg+Sf2IYEk7pgaXzC56OaEUtUZPDrCFTGCjn1SHS2vENhRadyhjFSQVUzj9Anam8ClGv5y1mlTMLMpJhpCG24YFBFCl2ExSEYe/qm/u3Kykcg0m1xZXi6EfRtWlj7lrNQ3KqzjySD4MLZfW28vImQHzGIcOfT5EGg+FEnfvVmz3WV5z9iGHpd1HO6qa5GwYb/81X/ngJ0Ax+9705fgVKJDkPbOmVExD4P7+aa8d+j8V12GxnmjHDtJLTGXjxyk5G+ayIgRSvD4131U1t3pI4wF9IC2Ux4qtVar+zNECyN1h414j45x/lHVuHFWQqRkEniMQW1FZZxpPmAZF5t9loof3GuPz9fvY16DaK+4CAsMcwn58HYyYIkRGWsMxvfck5+/JZ3fNrPE+4XVJwL48ClO8v/t0X+iHibmkCpiMqn2vEAo+47zAOhEN6osWBa4zLJB24H7mOZAOM9akP02TikndBcP32o/Mw/pFqeouWogdjW60kZEcYk1eYV0nFTVYr7gYC86rLqu74bD69SBQJVYD4WAZ90U9M1PAjuy3Cj1L9im8uEEbcSZQpflAtPSmCYNW5iZdwYUIqjt+u0ZoBSIPQtvPChechVWBnaxpOrtBK3dkgvh2O9EO/hhxUvprvMN2THcN7k5X4GLMqm64+BgKmXQEB2RhqZAZTvydeVSYwYogyzd8CS6LcSTCXQkHjI3jbkYS8jpmJBCWO2ps9sUmGIfWgnwZcg4AZ1xlQEucOzBdcRMGjFsVlxpa95lQ7u97l/qUtoxVDH825+F6elf8RbHDQ0IZXDlBieXHWrwYr4rnAs1oU/tTDap+AWaBn+1KKW4HbTF7gNxaPCuW9fTZ4fb1kFddCIb9nOs28Ngdbd7KSeeGhPe+kzdywiql5TPIQV7zAmCDhjkEGE1wTgJOT7fVJJhnYB4N7i5XYsQXR83nKDQSL4acCyEaTMfP8vBhib/zt5wB9dRuPsGuUrcd90LckRjOxjOwr6VNOGeHYwYP6JaQ+ytnfwhk2wsrIvrdrwUJIJTdvCVhyCGol2cez2g67ECo6Fc95kahJyNjrva9cqN0FvULjRkmz5vEI4787f6dgwuq/DM4p4p3BSK/hzsbXzpkHbwdsgxCqoI7Vm00XjyzNM1D3E+ScEnEp4R5bgyTvrEBJv5yWV41rPHbLwH14vCzsnaJ36bGV90OuAQU6wnRtcrJZrGdfeA1ETRahlOuQe9QGtIatbRiceXD2Bo9fQVs+GkdiAUzEpvQ2Jm/keAiFbZ99y97X/4g1TaIlOV8fFXPwtzDMG3d/6V9F3p78rqCVWgy6BOzBO5Yo471zxFdx6w/lx1lNB60jfohcFnr8pP5ZkpGLVEZw2hXmB854wv4OqvML+LNV8+szdJd/+qYoggr4KYnqMKw+wcuBGjPjTsEWDBGpfjMenjd6dewb4vHIPmNMHLOqX272RQFQzKUFpiWShqljNRn8YFXsAxz5WvLmJ3DyQQfFho0R4dHU340y1NqBEgifUSkHv9p4SURTww5r1CKwXijLRyH74mcXgcWQNylUnIeBe5fN2lpARhL398KMcWpINPRPpaJ8RfmrdqO7itRuh01enemvU9TVSucbvtc0Jhar57vhxJ9yFtH5FjmwGDNI2RlR9w0+D14uC5HLQu0ebmOLhhMdnjVgeJjldAZoXJMEGhklTFbZbu/0nO4AoWUlZg13Bo2q15eKqfxaGdpak2T4pQpj+u1HhAFbLfzrsrXCc1e/V0YhsK9ZCfT5yM9efe+jJt7Kbr+qnXuvzquSFQp408d9F/YXDbpLjDXO+VX/a2hOW+DDF31tfw34QI9GsqpMtYEfqbzT26X/WgZ5IfIfW3vgPHHWpHxYNqt4Qe8FTARSc48VNWKEua7Ag4cbuDgnCa4ytzc1lA6IQP5oNntsT9iz5J4Y8Y2G6ijoPSve8MMVtXL3FIc+Av8lW8fz7SL64DxLZR3kE397Mj3B+37nZtIU2r4471YaWWfoZUK0fhGL80RxI4YGt669kd78PVjvL7zbYzdAMvzQS75/tANyXr2uCMHUJkMOQQ1bXvXvfuVxKVEMYntlMyZc9vv3jWouc2Kvl+Mowfd6eDkf9rB5K4BCAVXB33ZL8GPC/A5zQ0458vl4rd8e3TXqLPPHq+27Q/3OZUS5RYaIoZusrTEJHKzO/YATRTsNnaP2UmKKR3J5MAIo85vulNjfiK7Bt5XYrBsxbDEy1HvtgwYdrZZt9qo3pOSsjYq5fCp69AvXYqUXPcKwZCXiIH2niUHu546+VG7jGuz5t8M8zOBOr75WxIn1GXh/nUnDflads9bprRgxFt9D9FWCup2Af8xiEfjL+4KpG32s1NUNqrQ7ETkVYVYN6A22L1hnWiSlLlbVQenFZh4EpFKCbCRAffAFQmlKykxT8kR24fRY+1s0rm3E8T3E979xzA524VAXlF0hUUMN9OFaTTBafuaJSUPvzBZ6n7HN9QNIMYJid/N+V5bTgEx3JdraXGum9nplDr0nqmo6dGowbASOVDiKDJinQaDrZisPE5nPD/biYrllKnXgIhKcGYwNbR51uXkE3/2wz8JdQAcB5/2QZUXvKIUvEF8H2duX2QLJgKfbUSkAh6mqZx/khDDwpbtfHivPoA1pUkNEeacE8nJ9+QfxP33wOm20DsVxltoHrbyGU6GQ2g8gN8BvLTXopRG/dwIxGus1QVfw4IpTsbuWAvK3R5BsjWDmtdGSXEhKXEOafFkA706wZEPrtXsk2XvrI4fCfSLZH4TQv2jUSKniXjAgqVZ+Y+OJ2/hO+SQtd3tjpsOHsL5SPvdInOlr/YVeC2KjLpPe0CgftaqHp3n/+ttvffRXtM2wF360DcTxqYfroCU0axiDuqmCBHwWC0Jy7hjeMCHMxBt66ax44/exMaRITt+CeedM0pS6TxvgNiUY4Ae5uXWrlrclK2yCvgM2jdaqZB16cObZeC3fCYYDWPN8qUqwMP5evb2TfmuBY3AxP8/Ce5+M6ndaJi4l9dWNNzY2H7VLBSyYjDW1JpAwn6FXCf3Otq5u2nslGs6Nl/IKlit1Mteh3aucf1cal2VVeJj2Amio+YnuBcT8wnoUKYXF7kF2xat7dCoqxeEjY+ixOgqlIXH52x41hwnB56oJxgGXns/rSwmiNR3GRIMht7nl5U6nBoxfzaPk5gvWS7lO+cDKITCJGEY38f7MKJl/61pShid2v9gq2NWBrZ3pfK9YVMYWlzRYGjiie3ycur+Gxa3FPdMiRjmsX3SZP+Lq/LrV8N/0dnazWEwNF8n/uLNqsaEJuJk6df3CsRr5FRZs3z/CzeD33gvpC52nRwv4/m3EKCZFQ0x2YvhiG18ttqgLTjkHKUeUNh+HePnfdnnJtyU0fCB/asNahB0TDYbFksmfyQxSlO+108a7uj189JtjtRf25Dh6K+wPOsDAwqVQH1stp8QMZIzHIDpuZ0PvNp573h7O8aFP961ZUqglYwzD5V9EkEb9BOeZiQMRODpY+bfaA4vk2k6VMy2dQkwcRBcOYX3zbzOzXeqTFzIIdGy10pxasve5PPJJyYyLJrusA0kYtnuxnKEJGC1zaGXf0zWlmoJsuzcARY5eswcOw75Yi0JIcvWexVJIT5Rl0/v1WjFe1hdjnK+cns3M3OK+nQDOTeNx2xu74nIZTycUnfQTSoSQ+1m+zCijoirgRmRk4NhM3Ro27l50rKyLIX6g2HL5xC39P3Ns7w/1Qaj44ZAmx2bp8ZccQZ/YYYkFtd5VQz3RQmpaiIUkUJXjimwZ+O+3fg3FbIXsvI/YmXEkC4fEN1jIuGyIPb29WSdyf6itVbMHk5A89nWaAJusDD7r74QHlvfLRcQwkumvryz5zpfSnZYc84wihCBjJA2asqm4Bbe3OzIQoF36tUloqNNGS0gwU5B3BZEBNHlNiwAvzhJorU37jhMKFag1dG7hw2+fnkiplXalTLdbp9Cy2n4rilPPdvfirzq6JiN/UDGZGjqQ9WiGe7VVb2EYJSWqbs2ltZDU5eqc95JgDEtkU4CSesRdxyMqmQ3pZGgtIKwnvmbU5Y3w+g3wGXy1RPhpUUWuzzczx94yTw3rgw1wDcGvIXo+U3q85zf4dLHLpdFS7EqJ7Am4wotvZOMiE6PGaevD316rBR8n0HBPeDTU2XpAjW/4eAlVQCuGBAMFYQTQgFvknnPeEjL73HIazesD3E1hyp3nN3ZRzRmKAFmBm6xLDa9KEPur5aBdfRk4KdHG17rhhZbRiYApG2M0SQ8kuerpXUkvj1ggYjPWKDQ6HdQRKZ7ouPWbdME7vPCp3o7dIcx2rvTORjaZfheJsqWi6IxZKGAOBYyLxqx5GxpRuFeL78KiIOyRSixqbQS2Y32T/1Ga9Nvwn/Xg7sOWIgL75mOqBfWug7POq/Z37t7FT8BNATzMj6XYnzWVRxLcjeFc+4o6lOv6W/rLz8vdqA/Oax4cBniEdeeZ+lbtuX/rzA/IP/DDei55RgLntdb5knwWp6v47GG5SgeLryUax1H4MZR1CxYTeHYlWbQAhZohmkwfN9eGV5fuQZtenv3kqdU1vqEH4gZjcvY9XSojC1eHXvK05ZBc+Hx67abvICzgRMPNGOtdyMAN54FWmG/tqakXp8G4dZ+j7Y1DoPuixcnjar+W6afmZo6NhQd8DI/PGkpJbr6mX5kPCExtaqHtBKs9O4MTvkd8B2Ad3At5RFIDKp7cAKnKB4+/lC17GJCIhwyxx10jLWp+CtkzhFddtqqd48PsNE4co99V0eNOV8ax4zDMQLPYw9i2EiNSEG4554M3/3D8ayvhMfZwD0/nRy2QM6RNByzdUbZoH8ExsCVY9bh8ea26nZjI5Pjuiuovc7w3VaCLBxjRMoOTeuv9aOdcDjahD6Hi8yuPMe15OzUkt99S3Ff7bHPkNJ2Tf4JObh+nHoaRHqgOam32K/mYzf0O971b2YFpnCgjigGHyu6TJt/qKwaaRp96nlMOSYfekj78ea7V/mvm34dw2N6rGzVK9oU3uLKOyBcUuSL6QcaiCrlfCAfAjdN8670HSTFdBsriSAOaMnYZbsLW5rPEcIc+QNmqSkZndHmzcOJ4QyX3ZeVlPymSjAMUR6BBszE/L1jJsVLaFWpYUtaTxtHPVWSKHtpFUVU8GJsaShpwvBFOn82at+sePUfLI6JqAuKd3aKQaNAOa9dpWfRMbr8Cp4vC8zy7Y3Ap4yaUOgpMhRWo+lt4t2N2aXxPln+07vcAMxQk8QmQhTqA7/nN15JTs2yc9Yw3BNsfLwukg+qyQ650SVYByaxrn1SpnWn3AgYZ2Y6iSsmrtMqu5wIKHpG5GsETIwzGHA2Qgk5XqudG0Bb93vDHXa+7epA5N1wD5DBGWj6sHJw3oF+0BwUxtpen0tXS95aib2jUBmqUlty4kVx6Fj4ULRV3bTX/IMrxf5LSxL+Ma1jnkQHZj2Iz+CHZezpmsWDUrj6gX8do91kXRvErBOvhcGS0PfbW5sxgt6lrbthMTtfR4AWDc5dOl052mYpGxdZrC63OjrHR+XV190R8M1H75uD5hX6+2hNymOZD9jhIU3g5Zo5T88CcfPtVefPUgmDpnkPUJ/FOyoA2nrP5ChearRf56DxApg/kNJ//eDXj7+PLeBurEe2gCAN6RxNKYQ89TEMKsdmlivLpPXlL1WYdqP6Dxvshyqkk1dmjNz/XDlTI/Zc+7GVK+g2SLd8SO00+BashZy3R5Xc4c7iSVWeCubRTcFZlz1y5vMR1lVEoWkQvYDOgmnlm0OhTXjl+w+fIgd1mx1T6aNdbEfhTcIRWQylQ9xzZV+lRDbsSD843kpufLzUqD7fEZX4bkYOkmwqoiF92OE1K2BgYHiisnvPfh/vOSfeDpbVTUChIBY39VfzGMPCuyqvyhxtxF+/z874hq/rqLsV7Tsy/spc+m+T9nZa4n03ivk8MMc25eBNARuLrkjfgPkRxyyh4Qt9+HzxSUFlRPjUeMtwI6MncfWOMb43wJnmg+m/D0EBF0tvsWsVV75pU/+o9wNGBRoKs4NpQ+c/iACuAIiTBHHvUBO+zIKrzLXeBP160y8rQVEQepgAlDYMkFNExi8hQ4mnd16bikC+W8WkjUjJ4tl6MTJoqQTsnmhm/xNM/wnA1FE71BBkgD5d4JRjMxmIvLRmcbIYMO0Uxa2RfDnbLklHNbNVV+ycaDIpfH1ZWbQVtMwTS8m+QdEi3oDqhgOG+g2LCuKa4q0u03sQ7dqu+Ghckd9lsEx9N9gmVK6TfLzRfZtyC73TsBsFInX7YWxs7pwqv2UwKXt/a1EgZV5zLv6bl+J3SnOWhf626IkWpMsfxXchxBZPchZH6EuTNfqE6Gij9cj0U+e0blBhffD2OnmkS8jIEiS7HzjY3ZGcenoihz7703dgFXzw5LFUJvqaTncGnki6k4WikXYI97gJ/SqSTDG+SlYqbptBKglDLzyDpZRFyVvevV56/rn57KzbB2TRC9Y2QizeYow0YSv/hLX26RqDoA8zbbVeg5Fv3My5/LL+rj1yBLLzG1V404N862Xg9qj3+cKRlmfLsejf5UnDYrTmZW/ZCEhJVnqHOiOsvcAZfF+fSkrCLU8Ex4Gf1sWm+aCwup83q2r/0tSiegcKr2F/rB2SH3v3KnC9OcvxjbHbF8b5eow4ufskyecr2yvdvFLEKIMT844nWLSDXu8OfVHPCBZQiiTLViE/wqO8IVWxV+XvP8l1cwSsapODguK/q69DbzWZ/mET7jl//HgtabxusmHqYslQafKZtJeNhVTtItLe5lxZNnm/SXefDQUjr9Kx8JbN3exx9749z10OaGzmMf31l68orzF/okw2Pob07DuRFVnPJxwK3nRLPXdGHehDm5/wM5ShBcLutMW2KdESjeKCz2HeyXfVx7ZR6tuYF0rKSH/QijYQGfDSfzrAzGFyx0ni13xjGLKnGZIIcKXNnw3R14h29P9Vr2m1MT+32xCDtfmHzN14WCVkhfqKXRn2wFJnRIOu4qY+yOH+FfnK4i83LJFJ8alU/2wzr/UDaJM8Xg/XLC0b/1lDQgeFhlYYUhd5D/UgX0HRSu6VTA/XW9b+RIGJxZ1M5kcSV2Fn62MDR5nFq0cgcfOpQ//Cp3DgzOhgz+DjSRU4u7NsrlBt7vDQYJq+R/t2KLfpYPN15HA0bkSyUdnpV4aniJ5WpPdjphy4axFnAH7zZkdO8dP3OgxCVQ1R6CXD05T7g7hyWJioRHKBoT7SQnWwP1ZgjFben/Ff361n52ksz6POuAb1BscSDB1VSHnzqtij2ykxzKdT+13Uz2ejh9ExpnZ92sSe1ZCTKWCYQIcz+QkCfywnqQnjM8O4Y6XkAgXKVSlptP99dWAVkJ3EJYaCtV2RBlwzuE4xdyo4hUwu93aMOzjQrOejTHqkVF7MWD/zuAfpspOFiGI9lnSSaQeiMDu3oRuJ3KEqIX1Zd/sUaJmmmCT/Cdw2ehtvS34Qukp69hUVNGBUacrL3IsYbyYHBodxNmumSRuELlpwvqPJgg/HruBCCCPW8JBQldLMgYVJ2LQzaoDaHzL4Xz+oAHc+j6jAU7uvA9RfIpkjB3pLv8Kgie6c92i6MzyHH9QZYFAVzZUiE4ZBrYUHUu8KDU3uQzAOxxqNsYcLaAX7MJ8FjZjLlxGxtR8d60WbNgK0PsYJnmKm55N9JHUj25QcK2HXH9A/Qh3MdWvsiWacFM49hnP3MSO18BkiUKQv+Ei8a8iA0rbUHSHdFlvd3XDfMzVzYyW3MX/5VpEiJ/LAY3x/f2RtWDZc/K9TF/EQarwTnHq7yvNGQQmDtvgmrZVHa28XkjorOBrps42I3DDEJzA+fYkY77Y96ERbn+PBRbzRFR6PPimNM8w/dDCFZviTXAStRTQsUYsOBKY/SYhJPUE14yT8Q5nm53ZVievfMiPEc892D6EMhR/IFqIEKmEHS0qum5LcFuy5LN/dh8Eai+ZF2mjyYxiUislXG6A5OvY+VRahFKQI9pfkZSCcpWd59JzPfI0rYfjqI+AE2S3vpFJOQZbBIT3BQt4AymVOwPed7GCgPwEmLEBnyVpi8+7QmARFlQZm+hdtghouYeFJT9yagvE/QkLgMHLnPbnveVe6ku3urQZMGux6Q/KBEq3QR+ZElu6X0j9HlgFvG72BXeV2ROxkrP9n1OgSo3RGsGNLiZFG6ujjCTlPmuKW61YQfbyMvFPgEn8tIxi2Cbhe7ToV4BziBnwDFKOUPmyzexvXOIEj/RCz18CjCuoHEZKuwsZ5YdQZ14+7gt+fpqziyLFb+OyV1SlsYVqYTnMyLWuE6VzJI9Jv5NAmB07AmfhYX8TD+YWy9xO0K0Goqa95ITQgEjX8URj2MCbD4Fq1yOwYntdjlA5OGH0MFCf5+34kLKd50BEjte0XxmOAXVbU/Fa5KL9vI9l0H8YmSsxcE8dxn49J4VU7uiqrBHhuPV8Efw6ScfXi0l97a3SgifGFztUGTn0wOiwuIoAa10T481XhAjfyePC/xM+b8GgkanpD25NwXeyR49v2dT2YtwUOboHyM/ty5QVmgQpRewQtUKQW7xUv2NZkx8PxUKvAi656+CjZN+x26Xj05MTf/G6vBuOvb9XwK5eBjLvxjUjvlAaZ+gl50Xg1OwnYSyn45dRg0mkNg+lV8Htd1Ka4Chwm/uaqzDN7wWSdwXZJu4I1Afg3+euynwtiBnIW3ZHsVM4IWP6yZbCIQjH3XXqYF0SkmuC6sEMpgnqgOMc4XAKTKYUjcF/u+ZEKk9zVt9p7ZBRYHDi76HQQRqR5ecxDVVoPIvXgTKv2bWt6K9/0YjP9KWEtD9pDCJQOhGw+K7BcZIcMEfYUTPcV5J+TChoWObxm4oTgCpFngIjof/iuQLyUhi+wl8LzzFQTfOrzLrt63r5INBbp8RemOTOvoH6VdRkc7Q+EhSxSIMd4HLUz60eQBSB/+tROxm150rfqb6XSjN1yxPiDVXsYcvEmSp6jHQExx9NDQFyoQdeekYQ2fMeahkLbdH3jQzLavzxPFaQgBKUjKe7831q5KZXz9w2wGy8tUNviSKMM7r1xbGFXXF0+tjYTDiHLGR7HeBZG1eN4X0aEjii960RXFTuYHgvtHjc8jpR6yzn+BZQWf0PVoYWfibSsWXy87J3KiC6hmEdQEiBLphT2Wh79po+BfPeA+j0K6i7/jQJYWCOi1Bu6v9HKnHTJ7NgiYjVo6OY68vsqwp3tmYvBjAm/DOxptzJm4txSwz175foFwgiDGiyzOtmSbIFzY1vz8yA37Ih/BAVIOrMTFQ255j+WfvGXJ+W+AweKr9iK9vN0s7A2ldAOc1vBLp/hziKy+7klkUG8HRWyf2YbgkdPR3021bzEXFlv6IaXY3RezgSP+Z5M22rYlra08H+MXUNzQWyOoEHrgPfTXKls/5tnoCPPEH8TZYoY77plf8cRThgfzAgsxcRvq1LJrK5o2f7djJSBM/zqE3C+h/woNAHOI0kTVky2FxP4RHvVaeaAgWENKM8AV5vMJ8+clGQ7PyCu6orNxYSBaqxz6zbbBzT+r47KIfA0c+J6wtqO7JezwEF9kmqF839wGGzzLZqHh2EA3OLsA7DwDEREexoLTddBXJ07k8jchGfzjhpN+Hw+7630cbOlKIvSQUtrm5vAwC7ANyYT9nT8XSaP2aHzgrEgT63NhCICa1HioB8PSx7I3BEhrsnwqovx3jZ69kPKR1d/8VT6itSsfJk7ciDY6O7ZNmegg0ltVihKRrnlhHxVpgPUAsEa0uk3pqIQZnQquo24tau6+VV+Rcr/QFCvBVvG+tHqvDDdfbws89dkpQQjmdrIcPES/E+1P8o5GrpI9NS8LmXr5LnxNncDMVhWb0wFbYJ+5NepKZjl3RG5im0pivm5nTNZAzojvs6V+uZpwGmUZbWazvPRmMT5yFESdYr1iZE7eZx5rnt9+ZeqhFVngUqwy5sbXgyoo/Urw8k6Tivv0ibm9nTVkayrh7gFk60RWEdQjmuKTFB5qWJAYTfI/s8y5jHW0v91pRBT3zw0tcBxURjRmiElzF3CJaZ8Ethd2OoOcxlFWKstWMjXaDjejVI7wDoStstHsISAND14oVfX2QI6f1W8YkTC0d7lyHXF3pAjTx/C6yMsQQP96cYIrSuhKsOuyfb3FJMQaKenPoY0dnNkECb0yAmw+IdElIsg063x5/uDpF3HVPi9dXuQiBdL8QDXWKPZmPkS7ZGja6MO9m4NTpeFyLNI3NXrv9ZHxF6u5l1JcwKzfapOHA8vzcl4E1S6/TUb1UnnyNnQsFDQzQB7p7DUuxkd7H6jPjR5qGNMY/0jG3wVcQTdF0GVOqOteyOc2fBKVszUWtBH/XT0ehufbgzhZzhOVYiKZcmAATZQzz+jySvlOSblQl29eNR4Nx1a8TdEf2HLRwZQn0wWslIZChbC+UVxxlvKOES5xIpbO2Y+sgD9+vpiMtzc6DxYGr0iEunw8XPAn4Kgh7r/RXG75UQcdphm/abpfMPdHzD+o+wsnwIs1qJ9J7CBjsIg4qsV9KxljJK6aeurMlrrnwxCBIqREUv5zdJFxPh/D49qe7gHqfMIAjVYyuDsDuHs7sarNxeFh5lZonw6XZBrHK2jTAcClCGq0QtakbNZz145Rr3dCruBKDHZ7FeClrDg+H95+ok5Gtf5MnFFkSrrqk9f6Cvf8zW5KVmsej2mpj4KzwSQNsUcczZQq2PArMpTvq8+9M0+AAtrCcIdM/AWi6rWDX88KyWv5UtBrngskZVByNnqNE5EilnXShL7tlvW7mT/wkzueCRTE13UQ5BW56+ThjRm8N5lmC3smvXfe6PLtuLXugpbxiDbApgsLXvfjkSz5wHvPwkHY+61p/HzL1lBY0+3o8ME+kHFM+VZnoLS3fgMA5WFZOfhs4fW7XtMG+Q0SE3LRmHXl4/aPwH+E7VmQ/E6Be63F+FGOiAH8FTVDmZeEDCNBGvxsNT7Zs5Bam3vz6BIFd3GzpshcY4M+gYYPfghGft4wkiKTPJPcVO53x62mbcoPp1g9wzDGRKoWF1zNDK7xpVwapFNNCadh0HHIti3IKL6vZ9UyP7WapUKkFy1Y4rdzH6abYW8GsT0ziCeQexWn9BzqrUHz/Dom0TkLS1+BN2CD3pM3gcjnujBXBHJOXTHlBXF3F44KHaOsL0bAgenodoWH2aRXMxfw2myhPPrBYVpulr+EhZof77IoT1zx0axPCJCXWNxcfxfx+u6x0R1w2HYjR7O/J009kcfy5VuuvKlcAwyX1tNMCAtwk57Y7DFmbNLpHixJQFr+vquOZhS5vhGdBHfKiPJW0xRvpgJnnZibc1Pv9b9p0sCP7MJsHllSU8t4Isctq9HwagjH+3LalLcl8q4EyPvoFmy9KLMFlAPT9HJj3d5mocACh39dySWM9wkFqCT05fK4dxd5wH7XvN4Ai8Eq2K4WaW3jPNvj0T76C/ojN48chQ5GrKRP/EiFxVJ7XZN9GaeigxIAa7i9pLkp0aI9exujGbAVzuQ/4Gx0iIKlmCBlFXQFt1d2JOGMiQLOOeXTdoF17QkOFIr7a+wUUuZ1IlBUp4uBxEl1HN+L9T1F4wXrYd5husn4HqlDaGuJnOWgd04sH3zOYXaOWKd/l4H/GX+1EDs4qtoAbyhVKS5Z2rrXiqmjAWxvSJSUuTr0ywp8YU+NKPIW6qB8ERgyKKgN9fjbhaBNk12wnMP0XsUBkWkzeTNj66XFFhNLasPl10c8XdcxgvAdtoqX9a9BejwW6IImz3L1VA9RmZxl/93P3zd7VKAFQ7ml4AOUIr76Q9ESsQKVrZFzsTDg1HZNBSeZ2Tdieeh+bNESDW0hrMWoEEG+j3TMX3qoiQw8Ec+qBerb+U57GFQgVIhJe9NdgNSDm3C4NQRB1c5E+DbE32YZzu5pydkZI2phxyzpEKKwjWUP83z/Yh9fQ/7HSEtDKSO1HRDiEA/w7DQg2UUp7eosTQOwncXAfKKba0R3Nu69J1IruTXr0UyKAy4jxJpSnSiUP25TlVrz3YRILqnms9AgZJZFP7AVIZkXsSZsxiLY5I9PzPrNsdXx8eGGpTdPD1+6M5NU+tN+l9+U32Awa48dijNwd7jRzUSf69+N8suBfYDwOSP3TVOJdSTt2okulAfmCM6w5wqb1HxuWXaoktSaEjRscuSVSNukEAuSfbtZfMfNwKHljZKrJbeKswST/iFysKyU3FvEJhMRY75ySFOAOqRPENcbGymKX0fYCVMVXqvHdW3XNH+gtWPv+H6+6tT0cWeKluHH/E6+oxRr8B/OQBhE7oalmc4tOG16cEFNjnq4nyvwWzx9V5krNWPfQhS/FjC6tmTy+uzGIUaR7nw2i+X9BRAIhgErZ7wjvQU5pq0wb5zY8pMKfuOoAImIPF0WetbHYi0CcxTFXE/708dARM5JQyi22Jy3KOExQfv1SDNXOknjXsZr4/pDmpxi6qo5an5ychtZjZx7fT2+Z1Kotiyc2a5EhdGh2Fr+SqhsA6bk7gvCRfSxst6rglCDoPrbS2lXf1aiu6TYK2vm2lhBCF+t9Y4ApUhe3q+nk/IOMx6CrW4UssxFOcSs5vfBLhO15YCUY6eBCLnC5GLNKsqdJsN6aX9Fa1G9nZ3hMz1yjqPg+qqrIyS865cOeDctUW8ipVQLFlOFsLOHoYQX9XWCvP4aNq6JMf512k60reLEbk4VxfT1ynpEp+TLxC8z1WEm3AGUs9c7sC93hfqsgmWxe+GBqUpliu9f1S3eamrCu418YgYmXue0hTOTNrEV2W6LYG/JKbf3RPDfyNgPPAyLmgKEyIILRQ5XUT89G/LR31UJuKmrpELOSQdVElqOQd1ohpM2voE5N7Ak3CmJajTeGv6wOlReuPPHsMEfqamoSswoavGqwNSwS3L5LeBfnkYxnMIkt/NK1Y0Nwd8gTbzWmat+Yvv8MCmh/7u6lAWmKjwcPHwREQUUdoB+IGHQFzg5kFX3uPrcApJZfrnSYS10n2vwC4cIcLZUE11LdI6IspM/cAhmmWaN2khDSOeLiWXofDNtvWG4qQVDrx+dmCj5cBPbuJAB0bBflgyz1tHy/Wiw80vRxd0uTcm8M594vlo/jlKjYVCdCZTAVXz9ikyqIcBh2C/s7Nd7+idEs31X6vxIxTF7K1cP+QQE4SDDADIitD41GhNQsm6S2ctBsV/PuVMEDZrgX0ZXXRjsxVz3x84X6jJlFlNEiODtiBU7D7OOFrq3sAkZb5zFA1kHJ1iJp63pKtHbATKqC5H0yt3GZjAWfIV9LOytl7BivCJmQstqya7tBfhR8yLZzDu/2ddvvexL2ookexvP0HsY7ZP4EK1eoMMSHop+gSn85aGs+7dP66s4E5M4ZdZMz1s5pRr+VLrilUWrV8QGgPfxWaqbYQlXEX74tcmzL5qklmN3bEw4Kco6x/D9yDqQbOgYZhH40Qznci49DNhnZxmv++ZHA19y4K4QywMjoQ5BUXM2RB4C4QmA1DDgx0JbnNT9u9Ulaj8gFVXv5ntd2K4dv9mBSSetD9I58jTFDJr/cq+6J4monIRP9J3MtUwdYG4zd7Z69rF+HafQi/+gr7TO3/j3De6qv7ehu3TEJfTWvt2vaYOl2PVTI531Zfiu8UqsGjAvOicrGOOlSJ8+qk7MTmOEpZnjO33eMQVhHWOZBvxIA7RzUmr7yTYjj4DCqRyvGWOY8J6ojFQWZuAwa3hfrSunyPKsSOTN+1OCU7cpMKRPcQfX6Xj9b+DgdOPvNLXbWvH86de9u3nunnJ9iNUM9Lt2JXGuWlKEZruTWkPngHpI0Pssu5/dANPdfh+1xYMlktYebwTlmE0Az/OYc/5+oTmWnqT+TRcObrRlWUZL/kV8bFCZ9846RzdqIlg+YUcfyZ4cD7/tV+AV1KI/srr7pVXkQ0bMfJVCdll5D8YgWkahnfj+Dqf8x9J+iq29ltQimmuswl+/OjpfLkRDJ0aQ2UcvvQsphQXNadb+XjxKSxqJYMb0GfWaXfBih3I2Nr0xtkPZhh/1eZKf7e49GZnUR1XaZt/X0AaPgc23mXhZFkftFln7dW2usI9pP70AGMnHhpV+aqnnxWTll8NvFqAmCOBD1Vk9b31vloB9vxne8gOVoUDEJVzGPDBofUfX8drDvoMl5o/mZWTjtygspXjM2iWxvEi1mrj/Vv+JPGNC9CyGH7E2Fa8u4+X57QO6MSOPihc/G90qGkwsBer5D1H7OI4v6sHDSa1bM3GKLEZmEsYEb9ICnr+7+nqTsftZ1PpX7z0vQXiz7qMYq6Ez9ocg2FwLmQY0coyz9nBDYKNKwKvSYFSXqTSJ97f4WaYexHzRgvS1VbGjTdSkqzdj5ILo1eMhg//GypNja8cev5ck8muRxg4BY3F7KWLtpnfH+m6fnbrlbnxzyy59ZvwQwh18MJQfVb/xq45pdUvNHTl6svJB0vqYKGHWHiVwbwQuPyTeD1DIu3widDYerAqLC0YX7R1WkdLbDhRuJ3+BGuDzFxvoUI3aQQwtUnOhSgkPHMrFlBzlqhFfFJkPuqxy7bJ7Tv8JfXh9xCtekmSTvU97MMtXnWxo3ssdsUIjqOXOcHEOgYL83ZKgkZHcbztPYXPO0xGYAQ2i0LR1E/MfTdexJimurJ/m7kk8S0i8926HT7y3T39Q9dzVfNNdXUkixW8iQqGbL76RPmNfFk0bKEh40lA51NT2b/z5nQTzoyx8ripeBKzBKHIVJ7yUFX9lO3iKTukMT/eF7LDhdmkEbvvTPbqXZM6ZsiWSwyYis7h4wUb5+zy2AeT84YJxOfzCWH7L9lcJoNqvR+zB7y75WOyUSkKInxvESJUIrs7mJ0Kl/m798/EGC+pf0vrXChfWQQvdVhTsak0kCRuEJxua6lQyJI1Lj+Qnvk4bYkC5VErX8jLcxzPPWRwc8PEr3FJHSfnN5WOzGiLPzjUq0JiMbVYyV1yTB9Y8CLPCuH3NKCP9Dk5MDb/sWUMJVktxroyC3X0Jjz4I7UTRVmMd0y2qWFRP7jY9lOuB+6T31PM1ManbZQSZIlUSwcvs4QhWJRUHv5h1EKnbUheHjS/BoRMve2TX7x4nVV7jn9CuH/hwtCdLXnT4kz4yhB4PxIeMCGaOggIcYzCc6s5PIIYdhtEYDB6+KlNUocL98/5GhUyogMpt2K8povPft8QRpXPnMphPwi/5G2nafbC6egOm4zNL6RmkUUx7KxGNne0rN4ZllYFJLaQi62wJo7Zym4oaNDAW6akzH15kNxe0bRoHxejBI/jSgeYf3MS/VY6kCn2/8vu+al/nyCUlIGI2O8HRmwZDYBmLlZ+/xfbYl5HRFPoy4ZjughWByrK+DJW4+3/oI2grFNOn7L/KWDQvpYj8L4b51V4jP5xeRMgwZpzAWQwRjfEIT8Ueoq/HY1bDgV3kUfYeeI49i45x1+0R+d2K4LoB0IvJ5kWZnj+ixjE2Xmm42U/8B3IC1K9R7U+7lXTOIwrEt9e9aRCVL9deoE3xfuA3Y9joQOKbIEUJYKsOHDd5ihUwXInyWc2y1VNzQBHU2YpO8+ej4okD3/DPfqXUisNKHEMplO3JOqxDlx268L5HhH3/corBLzGDsJmq5DOGHMcnqlvfhpebInILQzDWl3Mtl72xyLOgyD4q2YdUtNeG4JL/yV6hA9dHicxgfWGbv8JPaqCd1EdJiyNA3vjwXWPiWBAhTBxt2cp+ftYfzw3zooEl1lnbORE1u5F7RNV1qwfx5j1CO5t66UsTJmeZiwsldgVG2GZEPw0VE4tRpgOKJ5a0Ifut8AuL40zO/HZnNKGcjdPzYEeOLTKTOONQuj3JJiELKmTIyWlT0atWcKNfiFA0q+VbhlcYz6Lxvzk3Xzy7hkQTxUpOQaoO0otmgauYbaHQBFIOK2hHrAQo2TGK3RnGoH+QO8zRHo4qrZoLAEgh1SMLdYBUz0KB6I6IOgEvNPm4gHjlBvzeYXzzg6mLlD99tY1Tcn6ypRXGpB12PAz3+yfXkATuTeS/ijnpODaOQ4TOdIKesqd0WQObjsaMRreTKgR/jWlZlFz9M4PIjhmEyYcsjv8gpVGeAOHiMoN1gjuxlbidbsmyR5fn/h5LNrp6J3fs0rcnS24XXO7s9PDGhOba361udo8udkYVrQWQno31c2dqFlfwO27r6AGpAVRHd6cLIN6ef0o3g5eAUABESAbDtvYqaQ1CaJJgL4RuIUS9ld9gmueFoxzRrFQBJfGP9ub7NZq7Ezrf14nOdkrr5xfqDgQt8AVBv+nvwodkYt9oJQVQFuLQUgPvFdQZS2b4k66v9si80iTh/PPphsAGySr0iWcTvXO4fGD1eGmBeVaKlHiWfDc8MRIgj50G4Q5vV+HTuL98J4owVT38O+Yc/NpItv71pBbXER+vpBIg4u/eL+rWmP64SDSvOoyeDM1mTIW6QzG/v1QC69k54zUR+cv2rFfkpUgi1PPv+aVYK+Ru6DU4P0SMhV03BFvKEgArT/+a7HMq0Jen7Is27cmp7YYXGvdvvlw96L9gAAh5vLrtxhQY2X9/A3zZ0vhEKoi6848lij5JGgmBvl+cotzyOl/ReSK0AotPbwQPdpmrhROkHjvlJCbD/neJoa4qGbxBCuBzPIB632t/apfZYbUXrnkT8XYSIcgJiGxmmBGGmgu1wvdXPWUTTZ+PSC046DjJoJCY0eWFM5AwwoD+ibvJF3ODG7KtvjAg/MGpAt8saebrmPUVawHvdAcFLzIVG2Qvv0BlFyuMrzE8IVkkA+lf/nYD/0j99hGNasmSUKRu4WIVhRyoiJu/f20UQKJmMHeLIzPflVlSi0L5wtVvyzhnR1kSNgnaGIrwdtvX/99+bYOvFK4fzfGecZnnnuKV5NRcLpZe9gI2y6kNWijhGiF89Z6WpDUEak1mXankkZ3p7EfbAg2tR/u9/I8WmzpEbKkQ1FJKQt7X5XRHPH2qyVXi3bbFkVD434BBXc2b86LzKq7LCjSOjXOxhruM7aCthUnK9eanHQWPm8qmMJpkgLxwD+qASvFKTIMJXeczIgPk9jX2EbYG8vmiHIvJrQ1RHLvnIGgXNurV7gP8Bq8wtbj5DXkScBJPZU5EZ6SDa1i2f0FQbRG48ZMfLzmDcJueij9Tfit/3c091cqYm8Xka4Ijih+JHffkwXmgzdr9xZ4dlGlLF+PGXHxjVCwXDqOV8SF05H5Ghe+uV1w1SbY8voa+m/qDt6/Zl+aL2on+hf38k1PoZ+v/Jk7dRCmloDQZY60eE3OYs4lefBvZM01gc6e6Z98FhX+2g5ESfIKig9w6Mx/oPyMWuVR41xNXxuUEbiV3zUkUHZ56ymBl9cVCxvlFzL3CPD8gQWMFyDdWuOZi8nII9tlJM2iM50gn23imP5Grf7ngwsOBiI6y0kF2mZYrPinKCY62BJDDz40tioPvdAmKxfU+H5qZZvaFU1EJbDvusIlpPSfW9y6zpBQSmTbqMwY1fqW0c+qEj9PzOB/aYAewBAYgOREH1h4C6l5MP4u6HO7lbKgQiNK3fa0P2Otr0l/Hnv2ddxyvu4iyV/Ebsfui3XJRJFsQVRjOyo6uUrAHGJIGizisUB7zn6DeibAbd/jRsOKIjrTMjJbmVp/+9KxmrQcC7DEfasai2x9w2SMjTi2Vj55D6KzZwMAnwgteBRL/B5YI1i85IgygQhX14ZySecP/pqWFf/usCDir9WhY3Kuiad5VV4PFIdPBurr0NVsxzr4cV7pBdGtx0vXVHYmMNe49/tSKsmthRC44oMtzxXddaxMljVP2CCtAAttOnYT70VzFngpZ4Qr+DEEMwcYrXwutIAsUh8jlIucrC8OlQhYvo+O9s7/pmVrOrcmdr1PecPeqh7ymGf4KMPjuyjKWPmeiITd3M2oM5GEFdIIUaHwugAI4pCkaZ74kJV7PdddPvNtAGTwEJKjAaFY4nfZ8VpiImUmAvhM4Pe9lyKFtLk0XnAANz0ECaAUbKAcll8ZQawFK4YWgbhbc+sGTLtqmH3wW9u9gdTkVwNrGFD8/QcYGfJhDKUSJZ2bXuwj3U9P8tWCfZfq75uxw6g2blqdYRZ2/TpIs3Tjaya+aZc1O7OeUGtgsV81rbWWPnVMZP55fflBXAbuwMxG+rWTT2Qtli+q610O6blvnrWwk18ed5y7oLQITpkQcG6XKFXrU3LanOIwy02Y96A4bk9i0wd0AfNQt8JdTVD9gh+mI62tVVOpVbQ7klMvjRm0wJjuDwbkavtoH17+EvE2Ai8Qj4mxlHqH+hsOEvxL0j4Vas/9hOWGHXMEpWHDi/sdKGoXPas53Gxv5jrhanEazixhSSTJ9/wGcLzCdxiqNwL6qYnqwKqa4acGuZ/BM7Oi66qdEOIvfCxbJJ23UKPl0LaUiLGJeIhRbG0UJ1odEqIFY/4aVgOxnHlQm8RqBtDfC7u5axUX3Z07qau6Rr+rV5JwfP262J+3YdV2BjOXppYQB6fJ8lbZyCPYK+j10RZDrsEwqYVQzHVBU0GDlikDT/TB+RtGwORLAFf9MfAG5pXs0V2VdwvU3q/ZtwfC34zA3jEbP3d2vj67Zz6bisVbWbq59yq/wGsMZLZUVmZB4Z+zXHDTMM9Qky68tavD8hlQ61zUzLP+2x7HAV+W/Hh7Z0Vw3B6z3UUCISjgEsL6xwW9AAUr/CI/rc7oWQL2f2ar5r2sF+S7F3UTrJtxLGVEykAPMyQCsRrv8AsqNbG5cMqux4VGxAVO4+CyHj42RK9qsTC9c7t2/U/YGH2Kem6wiZvFQZ10C9xK3Jh3F/J41Pi9dxf7VFySZHQgnQzJ9GvToVf0e6oN/d3dDFJ8c42oinu4l9BMPU0J1iZvoCAtUqXQ8lATwau1PFKX5mU/FzSYLBk1GQzmyjJYdLwaf4UN4F4ydr29vqLYvfXKDMIdQOEZvsfS3uvaegy1hUxO7FJu2pkwgnHJpjHgdXnr0Mj7QPVtsvHaR8ExdNaUuP0nTe1SYXodkx382Z78e7f160rMiUfdZQBxzRQdr5F/50u+1pp0LtX8KjJH2LdgcqMd8w0KVv7Zspwf8UUaIl0R0N33hL7n/XQipuchrDko3cyu+r4myiUOCrWvgMUAPxd9lvV8SaUkk0nCcyic2odYUJUjDmOaK14qON+QXIMe/bNPqxJ6xaSEHacqLS0R5cIAFEyQLPWiKfAYtDTJIhd8xi3XxsVDgBA7Wl7jy71tA2cKQl5f378PbctQFZT0Z8jLTA3DHDgLpQXwka2f8Gr5wwq/OTwqCEAdmD2oHx2SEtlIFiTmOcdbncyfjR97jZsyjlggAd35lcGsXf8s5iuA5r4Z/1X6VCvAp2D83TP6y77JTTSRWGQ8TEPftl23zfy59N9TjH+f4N3f34aNl2aA0FrqfpK7EA9wb1C2W2Lc1Rv+OLnmOTzmeDdjWWX9F9l8+nqA48jHKHziuNINZka+4K5tL0X4HQcUfWO0niaxFtdsjCQG9ONt4GyFxy1XOQ1o2U6XIr2lyd8aBXvdnJvIi91f6RbifODN5BcVLpG7qtd64s3VIsFG2S/r7Ykzw707h4FZihf/WL3ZFrSYC71ZC3uKIOKVfHV/2dxMIfiWN8yutS3NIO6ZzxvUFKhAEx86gnc4zsFV6Gyx3d065VnnJBaLlnyX0UOpUKRzUg+zbM4eudGvz1UWXdAjX8F3TkhTZcA0mtbKAQRRX6QmmPdtL47AikBvgTwang4R9MM2DOvB0W0llkakbXM7cMwxW7uf9y8+w608Dx1r5RDTGMdCCT24uylBfaepUI5EdeQnyGglREncMHTNmschCbabtCeSWk/2nY6qQeb9j/5E6YbY1BTrMeA28MuZnFypTf0Zrv+3Z4T4nkEMN1aYwE6/yRXH3tkf+CCwzK2ar/CX6RvrfPc4OvOXqAnKqa6kd/JI3xJGTeY/sp4xuYrL1BnSokcC01RJRJkf5lPhIITPHbnpchA4YjMNQP8CUgl5frZAO+47sr3P/THLFIla+x+AkKAO5xu4q09Y16S9gNfwERpEXyLs48fNvJNKg14yIk9M8E+WjSun5uqTGD329tv80P1+cGgq6DDxjT8hl8tIpI70VFKaXv7npKXRUuVN8Jek4lu/q7e0uJPe9OKTRLXMWuuEbdNdyp1hnJ5wxIB27w1dH7NmRfvtTw/e8Xagbdh+wZkk8THvymh0SZSPok2SFUTHWXTM9ujpd8LHLU8XMwfIHvraxKdYeNdeHp8NEhGLQy4EYtHArXXV/O5BFhBm4s/ADMdKozM/iOmwsj2oZPhpt4GzzYhACk0DtBWlnKGf8QRs/hb3HMBubwkUMB7xDz3Dfl6k+f9vsSOQj1AvzXEbI0/w3vrQ/SBLdDg41BGJJkl3XW4HeD0qwj1r6aPpXm9PtcswfagEL8HxeiXQJne+oPBKOxHNKxApzlVB5a71cwupc0ag1gX/ih7zdYzSqcNMCzlOBigW3CyiO8Yv6yM1g2e1w0nV6e8qs6dXQ+LMTEczGg5ycG7LhP829CKIN/k7r3qCm0fHWgIBuoOCZUr9D7HkezC9sWaGK5A7ijUoPyV/wvRiv+f9WYVCks0CA90RKKz/Py0dZdcRHr4eiZQNkj2nt4kAP1/FNfLhKSjUV/fTwQoKp4LCpZaQA350nr8yYf44+at14hcaIrDDk4hSGDG6oPd4fAyEK6NNBEXpDlAge22495/3uebIA+UkVhDcfaLXSTTgSeZ98hxD8SnzbKHJDwUgWpnwrpLXvLHkYs0g/5heJYCXVNtdr3IljkWLy1xZC2sbuWOR+8AZqGTuN+OLIAxF8iTYoJp4OxSlhadFHNv7bYBKDxowXKkS+bMc+2qQWLoXu+OTsidtGE69RsBfsbyBYDY4YXy/QeTPywPph/h2PESLOB0J99RFxI/1sS152rLXyhx3Aa0LIBiEY+dBJ/b2k1h0sqC7bbLxS8I/8h88gVPGBl1JpEyGtxQNvlBvw+uinnHwl49+44D6PHUGuN48cj2xsObv7EQkJUj/toSjQohFB8ZehmvEArC6p//lmW9tGtulg3BEoD2qEbseKCiP0xbPs8JxfqPbnU1P4lDujcDHT35WiXy+tO6xRz0LPWpTtsVKkHH4WZB8nOmSDSe7BAP5qiRfS3w6c2/yJ9tbbllLD1cRGcJsZCMniwf1Dl3MvNvNLWD/9WZI9/XQ2Yj3DE2n+1Nj/1RWVtqVcH1NOu9lOIkD+4nM/jiuN7CyYKCZ79+p4x2hTgAqBySjfhAK7CaRCWjK+D1J1DyqdwBkevvvWkbxEACCeCZydN+HSckew4VVR8UHjRat3SPlp68Cvx6LH43P8OSMyc9cLaxBIVV5T+nEz7oRAb9XXmcyWeSwSToAitVkt9ZTCEA1w0Gb/rK1HaatzQFnw287EyAR4z0v3tfh/CdjEFMbY/vrj9HIsCVL2xIb4Dq9CFrpgBYvDqPxIWGkiGhUyhTthLke5ItWjPQvNHyL6Sp0Mvkb5V/jXxoQs/86NuMWerH+T/Ip5PlC5IBV8ji73G3PAqyYYQngAEYRl+mAgwUT+0CuA1oiwyvifTyF2hDk+qMjjDlfJsc0B8Vf8jmKPaN5qaCHbTZgbqXDhlei7sCPWSY+H/yQNiDaAWyOoemd6oXS4xTtw/ELC+weUmIe+BU2Tcn/+Rs+gSIsCHQcVvFVPnx4AEb7PMPizFSd0IvPS8EN8ZrbbctRoS75Vnc7/dmhdX8rXmDNYD4tdWeA2MZJ/D1/iNri+jvmFuyd1C0h2RjVcL/GWKCEWfbJJZUNXL91hQACrZR0jr5Kd8fYJ7t/l1ddiuJ6spNp1NyepL5u5Tyod8bHPVxx3CFWMqs0uu4VNrfFvdg94ihmkxnSkV+Cf36Apud51/SHl9qDmLvyVNJyow4B2IC5LkEPa462xxuEeRfPQ0xL9aGVu1DGdSTD0dwbB0xrmxLPIkq3a+mbGRm5d0VT/9Dfvx3/Jg9KIptQ8KhEy48ypn6wGYLnWJ39LAVZ6euo4xT1HU6niM/Ja7DnozzErM0TH3fM20P11hV+uxGzzbxJUzsZbaE/ClLrSLDCaaRM5eH09w+fUvZ2onoDrhYCsUPVHSJ6Vm/8mf3ZInW9UXvCl43qiEWwgr3F4ZLa1UJzBASvrNICvax0Q19G7h4cphy2f+HPo4hSmUq/P6YFYhCV56vGNQPvnRsxeNXiyoHP74ZsPK7NWh+ym/4sxjlWD9bhusjha5R5M+witV3SIMbAcU5gz3n4svxTCuOhgPeqvsySiPLSnBX80RAKOeCmZPgO2XDjajWGLs7RRlqzpJQXgvWkR7h9CWPUGku1oHBD6EQR/1iBwgCJ7ivpAjLlR+82bKBKalyh1M3uD+y1HqOSca6MST8fFAuXwcbjm98AziRf5P4E2Uxb8OpXyeIHa/cCY6wAZOzM2KPvk1zRzSAW5cKrErDFE5cr8eJxeMJT4Kv7qQLBUSl+CVO3Q3u38+WSHjuv3kxMK9a4vw9xh585RIvQki9h1o9krWnmfr1u/v5evv+/O3k0ov0V+bCyprynzQ4iP+pHm+PZZK4gGyKT0hSwvAiz0Uq0Zy4gcjc99If3V64q8I75MSVSDCHBsiH5uUoHb0wGSHt/5KNeF3faEqsFtCGAn0CzTkMKcKq8TKlZ+W4CP/OEyps8cyRfZbJN/bfxjNoW8/APieh1zqVq93oEW5aYoQhbyHlWNPBAKozuoFjwcMxAU2lIP3gVLbOCYWtqOQexovQfrUojeWLZcS+qqLtaj9EtZPTN6qiAwcQ4JG9GFEpRIeEKGt9IZHhctgFDiVS2cM72a5GtHaFjxwJBSXoPYsMeH3dpCcDiKB382h0k2C4TOhLB5fa1JJXoI/ILZY5oW6uu+5w1QHpR7CQ5+k2PTWPpSwPSNgQi1Xz+qqvqwUAVnvQ5+FPOzkYlpM9jAppHkOB4TVsFncflTlIjN7Gc1P6emvXTCL0yvlplgtKD3lYmgV7ptYAVCqjtVZv1IqIaSZnnYBxIvpLAkyTRSviHqulGB0/nU5Ee4z/Obx88nK0tsuN9FUszOY82OqmOha3axcZivpgfhoAdp/jlnUTSQFHfqauyT88OGJ5bivmcksd7EDZGoy4Z3kZbes8hQf6Neixym9pe4rETapjG57LQQyiOMkbu7+7NL7mLKCD6MrUZN2XGydS5FBy+Lv4njf22X06x4K90AI9X/SH3NjeL7fo6Z1I2UT0otOJ6zuhbSOS8kx7gy2Jya2GrQnMdL5VW+k1ZhSfNSi8oPSzUdIQAO3UwYEEUlRzpvSuJ66X998nGW7RnNjbJFD+oE2TdAajMgmYV7UGhkFse4AIKDdDaI1eSvpzdr5rBclaY9eCeb/TxwJna8NE8jHtT4cTuXbcqNeZYrBJL6d2eCgu5re5N6rtw5Ef8VjIGB4+IbLWQjmXlLkTddyQKdkN5FwvviEoZ/xwFva0w5KI4yrYGzmTnA/AAmfWXbs/mkv67QXJenXRihZ+E9QR0+ECbf6hXCfWXBWxM4zoNKGGtO5ahUZ/uVGTxT4IoX2xCxhwUqi0aDOPsyMajax0uP8eXvsAf4umI38JwkqU/td4SOS8HftSxlQZGMSHYxcsBlX8CLDwqLFHhJO7UbmsJURs+Yvy/x1BG20fLXCaCfeApRzdA4bWWOitGaFuawSszJs0w+h1M53HdEuj5A4iJFDMdBAMcfaILx/LWg5t63SN5hAKBy7L+81N+gnPZxwYmJy9v6Tfruul2nk7hFjPqC7AZrodF3tWOQWaZxtCn4n8LAXj9fXiMTpj0cPZfMovmtEzjgw4ih2Ze+e3LO+BWbVkw2APmCVIXka3Jv6/ltpimFfs3UqHHXXz7H4mOIOtpIp2iLXH/V59V4j6c5/H1q3lq0nwnyiHJsM7MxwXvKu2kTBBcpked5PceIAMPPu/Z325jMRA/cuSt2fQapFSpc/tpOp0OUzghIQ1jCgT2ab9yHAgkKYslT530bfnntawE1SSHwvrlClao8+4/xr7T7afV38x7qnkCf/FWxtMZ37vNUtEmfoLGhxGi6ztf0lB/RRsjIoGtL+lgPK60cM02zrK6R8K2Ou40vt5mZbgnXKxAQ0fqbrmGaF8NrDKFfaNyI6NqUDbrBR/WGujb749N/aca8pkIsKQeTaEv93E/3wQeyA+ivpit1Fxt1JoBVXkv2d0gUFN2q8o9FYFvasEb7Jlqsqo6HMG1xhn54G40amXQE23e321qFS9GPoj8jnn1A+S3ujeX77FOdbKoLJabssDPCSDozwpBUa8G5aZ25XYyDBFltYYtXyhuPFg6zG63VkIMY8Y5UMr8pbM/9i8Sg0sJp6uQQH87FYAmSytx16chqedjo/a4W85/sm86aMfRj45xM94xFV8VwUY8IlHEfPxCkNstCjQnTTdG2M1IUgfdHad8s970wnri1WnIhxsK+uMfBBE09VfXxppxmW6Lp9H9C93wxHp2CuHEsDGspAdBtsd0Q8rVYD1T7KRKHSN00i17+d/MIfy0DGVfbzSzzhd6Zeb7vrve/pI2YiYHvas3xRv37idA+h+org/Dr4yseMZRYFpt9NPaMAAXtCWGfO+sxtCXboYr03xN3oCxc31gzwH5LqXuzV5WOlX0aN/PXyPXX9OjVvfdGfxL2KbFBisr+VFM98UAXkanXtWLnASrzB1SxoBN8jO7XLy2Hi+GB9VIRRHf7zF8yFo6mitJljC6c7U8DPRo9GuKqtwQRA3uHo1a4q28+kzr3w9xjBVrD7Dce3v8YF7sqj+XTzLAG8llcQQhi3A+/4IccaaiftarbTj0qof4TZcxpEKWJ3jXAGshqGyhfe0YSTH+Ytu5KXmAg1Hh0B3mTDQS9Ehztg/z2FYW719dW/PVvBx7yI4WK0u1ik88MNXfkWyfRXryM5YXkKOscCkQPl/PxLnAQP8W0/qGAp52FLTYpdHXy0ncPsk0gpni2ok09e7G4ifeToODtbCAEaAl1gChfv0otRcxzsNt4mQPpV/2hWK/6Jv6g/MjpCdegvQRCGTer7wpsHr9imB6s6pJo2as5gAK80qBPRaJI+avyD1DBsNRpzKAHSgW2H7IdAgUDEfbuc5r+1n8m7sson96lSuRTWokm44nOwiZyhKj+WRmjVl3vB/cAQ+GPnlrT065WnmnR4hq66YyUFUnw/GO/dvRwUYZ0PbPolucHVbc7FDQihP9l8w87cXSTcxYB01BFbdZrs/+OTu68iOGMqJZmg2mvnukT8FDI/nk5kaKC0FO7Zos3VndybA736OT6mnTRjd6wNIRkVzUkASeBALboUuKsa477b/nV3AyxSQj3hJyVpkF6xNwR103pmu5nfTZVtwzDUlvOiunWAf0SL7cClrfp3gkKj0PHi6t3kSxhP3wpEDJpVTX0jeo1eyy7T+4ODY+prjhjt3ibBrjuia9NkKQ5tN/NJ2K6OYdhr8Fd3IuKIYMsl0NjhQRODUE4WKbSWuvqJcTYfNNkoiT98HMoavRH7M6x7i+Hvp6iTx7ayxF/klyEuWzzpiV6GAUZqsnaKJoRej/u10Shv3/rWV11p/U1j06/0vJkmyVfXeZDNZeyPRX65Qr/XSMt6L0qnfIQ8n9DQmHWnVy1Is1jKqTeN7aGSOdzkglh/9DfkZUHCj3GNqH/XBQfyYuwIkh59iyVvlGnrSTv0FLLNj9Ca3b4QJbPgq9wL9iZo6FqP9gfRPSGJ7rqCfoHZ1bsqEABib2s7DUgjHgMfKHhQorVPbGqxRnfSuEn1zZ1zGynxD1bH6BI6OpnJvwIw+421R9dX5LspdCw3uqjFEndWbzjDO1ABP7mySVaZRqa2n69rJ27B3vGF9pLyCtHh3G3Ydw1VGkLDYpKfAgoh+0XVHaAbvKEHR4u//6mjrvkq49GhTzE5jUBXfQbUZw0B/91PLUjT6xuhfGllneSq4NlNLouKb+iSbPvMqJnaJzx5P/dQPO0U46CHIG/G+OrzsBSbB6JfG8m9j7sjYHcj70hkPkzgt3lVsfAzA9+Idry6tOSILr8VEXkNsS2gmn8Pl5UZWrQd8w8JaTGKYGKXy3hxJJIsJRJZ91MxPOc3Oaq+alzxbS2RppQWEm0oy+jpo0U14KoJU1sQxibz43FwOvdr0yCn83cN22wWWdyd6G0rzBaOplpPOLwmcOL77IJ3J3A971Uhja0D7l1mR6akXScWC5kLwvbON7YVEhEIg4uXFSvaARMzQeMWvEbYEQ5lC+PyPm+qfgaNSlhbdaczE0J3YgLfVO2qcnxevgi7lrnE3mrVt3iMOzR+tcPp3GfptTK4mKb2A+rjgBSsnGNFC15LIegEhb9KzkNOsPKpEjemLa+UNqCcgj/jZ7smIm1/AwI6nyOC34/QORjYfhy7iKGuSe0tTm6gpeTNt/r3ClUudLPYaz2ykpeQfJSLRaDnM2SxhJDz4QAUxH+e/3rqe6wXtNA3DYBK79SDLnCqgVm4eXSKx+CY5bObwKfGEVhx736jLSNM6Ax1sUfrGWNrQyaGK7qKLeO2dEvoFvhZbIHbMniYPZAonS90NJtyg9uxnui93Yae/wbRPGuyLFIp9Hzo4cT/7seyKfHWu2T0P3BCJxX3yoJROuvG0zjzKYgSmr8fITgXuHmuEquB6lrhKJCRWXoL+qdEwGQzcoU8WinTzl3jAQ473sN3+ThXjGbgLS4uEOZ/ZU/yNaevw4mX9OrQYzVyh8D4+hJr7aOM2xdY0NaVNgUxUuM9PrBofJqrRvn1Z3AJjaIf0/3x2hwDI0hHpg141bGQyVzEWf8Gdrf19vT9qd7X1aw3gGAW5N6RQihxVClnMYHAbgSRLMphbfIw+HINLhKSb4AfpRK2DxtMhPCpxJFhEDdNLvSPng5kFIxdsHcDVcZm56H4nuVDSNXqhK9CrC4pJEp4JI96Tf+IjqLuoAxPvkC7x+loisUeRYqkqpAVoH1gTfCcW9z1ASJXF9UMgk/h78ZaCxvF34XI+2rv+/3dURdY09s9OMDd0zYcdWhHwT9frHwUx6BLGew9mwPURLc7MIMEqXXeaCKrIpU4WMfQzgyi1+pF8T8ci9Amnjheg51RctggqCDRf0q3K6vtnJ5HQewCllhYleL8TAkm900S6Xn1KKO2X7yuymAQFv4i+qvLDNuSPJwMnjQIy7US9YmkfaIp7mt4Mgd04V05b7upm399kffq6M12RuTsTt9s6flHWh5HeMrcv0F1iX6/F7nD9P0SjjA+tABx9Jx29AYoHrSLCvcxJivTfxadQbHHbjarlChitYvo7+EuLhQI9Ov/3wBIlGbwbS8WHGTxDkXr2c6xAupEGh0Q0dS+3XAHuUfur3d5PJIa0CssSgdwnfo19XVjVdiOkYRIjrmOBlNPd/+AOew9I7aysJAj6ej7ZpnXgWSVObChjvsLoBQlDiXemd90mbzXGguRctsfENRSy2hzo8JIRKxvpAMc9o3uGAL8oxeqrKJW+rfJ2ZYcHDF8Vc7/utt0ZaK9Z+uqyL6S0fmohJnZipfeINLiCrk17qWVh4/B6q3D+qONz869ZYrVLVdfaulT6F4pMPisoMlHvDzTHE2zyBsFGPq0AhtrjqbfyM5cu/o4yoSlsANlVPVVO0Ov9VeW2IR95MPEgThZxTg3U4/GpnfaCnRQqZHfyWhSdIdX7m47qO6sw0YEjLHQn33LwevYNoww4iABowTEoVzlHAtD2bjHtOoHJ6jtAoLqZ1f14fGE5BQSwf308uqUd4cIVWvW4d2O15KJxHKVoNDokQY7f23gBdFBNNO7vtQHMjcRPqOi9H+9dJIbqPfSsLYwG9GVzHxzWFLql0nsY2fw/VezMq9mxx6MCoxY2+zemMQ4BdIHwFT2gNHVcnvY6Xcpl48zUeYqZqqqA5+1Doz4i1CGjJrsJV08hpEP/Mw3cgXzqzGGOQoPskw5XWKT7nXEHTLK3S6kmU76kkrdVX+JbOtJwdjqCtaOJw5nh4Xd91f+eG2epM7J8QUCVVRrU6GXl8racV7sCP4ue/FXU4bJKDh5bW7i+6GrrVdady7Axubz2FyZLiSWbTT/WwPH0RN9uuARBef3GAZ7qO1Tu5vqjB6P2k7ZAe69vWjPzgr2GpyvPbO8wnsK9mC/xKRh4n9PTurwZiO6XZikPZUfqcKqwQS15l+isCB8YMH4xu2f9PPJMyKWPumIBk1uO1r0WhTr6+qdUi/RmYX6+NL5sryB02Dndl3/fL3ZobJtMVzmLbxHNXybfDCTkR0EEqHyjeVojw3HdNBojH5Vp8LLniG+fA343xEBu5scJ0p8/C0Zdi5T3yy5JBUhmPeda/uaMUesPGl6m+izmr+ZnOYLPobd3QECczfRG+zo4sxnL+PYndYoLStBUrsqtJg5qU0gzMl2O9u8VcXcimV6kP8SHEmMGkRV03ZaGgJQ2RjwA2E5TjOd2I5RPXd9hfnrC186QvvBa7/Jf5kfxkjCkQ76Qc/zsV5DY4/ElHbIJXcMs4QNqfEfb2vW8mlHUI+9qssSAc07NZf6mhAEgAuIwAdz23RGGfJVV57USB3fieEp70bxWKttGRkLfjZv4tLO3A5KoM0niiuX94C1MOo3wMsV7N3t8u3wut3lS9u0AUmWCjfJnuCxrqhkzi0yZrrz3a/Jz/2kZ+oiqK7Y/K2PRqzs7DlNcY+0DHgt0b1ZWmnOrG/wz+43XbKkh2+Gkv3upthbj0gCv61maqNpq5NIRaltH3+O/xSK9/ZoMvu5w8QhQUSyrrDi+gNuBrHSHNzIjtTKp1J4eW/i4oAOEK447DBXJ5oqFaYrcSYUPrt4PAmP4OOF+Vc8Vk6OnwJACZVem3B+eG6O9+sgiBFE9c6WdmUu1oIa+MymOC6/ivFxu2C+ilCP1vlf2yTps2Db2ruW5E+JmT7I8PW2IMt9zmF/mY4IcA3cIct4z+TTv4NSSz5/umdd8W9vxlKH4UAFsGwSGa0dozWL0Bv4F8g0X5j303xusHoQ5luf54X/9zfoqOpOirQ054olyno5u1o1qO17KvlPCnrwFDr8uVtvmj94FC0jK4+DtGBA0uM4ChWLLfZTTralzRct5LABmoZqS4kdFhTmds5RmjLminHrCNmEcrHxwKEylSWCKfk4cdaUbzQ8JeAJv6eld848D/NHDTe5f5q89vLFwTNT8lyQOm9EEBmYk9LyxLhVrppFH5OaIRfena/2GWWb/AxewmczVy7JiVI4UO3UNPExzE50cqon2MPacttJ8OstkbJ0h3XoY6vyoe6VntHYPLxTpAs22JUjXVkoX72l1YVSNJQTPvx4bJe/HlZAn0GajJcoLYRpJ2qM9WYp0AP6y9RENPfGL68Wyv4ZzaIdn8ZE82+jXxDzJbYhCPqyVkqWUUyjr6Vjgl5J0jgU+0I6gN8QHzkNf5dq8LlwV4r5DpNHxeGV4fWeJWTKwo3bdBUSGej0YA3b3rR6yrRZMye1U5Pkdi6JEiMMbL1smKEQeI/cGMrBzhIyIwHiytU+rE783Jri8ZVgzserW2r3wETu9DXZSU2GqK1qGYGkhivzYtQDW+liV2c8/eGug6YZ0dzJ4eCIkGGwRFHKga3zzkHBGz0axj6arZipfsrQdUbdlajelY1SRNgqDVjSxozoqYQvr7oQmsz/b7U9HfQQFx4wZerF/8Jz4ztNruufHcsmhbc3WU4egFfOCQeJvH0+AP2jfVv+zCjWX7Rv0fzBfsY5KpX9aHZpl1LtNZETQC5hovazpCBBgE+0CDF1kSdjFXmAx2fVwfMUo79HRPef1IHGzdDn5phXL2z98/zZDeaWFKJLAbolc3NPuI4ZqRjQSXnALv97lWNWbv7gJ5Drz9EMKmB/oOShhEn6Gic6/mZAtZCAWUIrW0jrG0f0GW/O9A+ffWV6cov6ZkngDwGr4/7gz6ejR74X3ZsmJ2MOSVlLfdnSJ+PZWNqGDXqa4Q5y+OLANC3Qg2vHkRTv8Kg1ondC9Q1GhnUvQHGh1zLoDiqM5FBFp5Pn7TVSn3Q1edLAteuWnTl7K9RvE7Qx84MqHVGzSLqBhhXcNwqYDqGrKY+ZrVtryUsfj1cgAngmj38l3BUb/tZac7eB6Nh0zX0w2lwOhqraszWuNCo+hOHXbvJgy2bxIcqpzWqGMYKo8SMjClwcxmI4MR0eZ7r20YceNP9sNrOXiiVRZ1CL9tcz1TEV79nFFf2Re1ClNf+iHSUtFFsMNMM+TCCrqLLE/2dMa4wP3kDQ8T+ZsqVCWMY9CsjcWzGY8nLHDVLXO9Lj3sqU6FzbMzffAQCemUuKlblB3Q188f3FJ8keTZvBDGKeTwdbrMDV0miq9MX2P5rNf37GQb7w/LcX3FmpsEOgA84zcPt81f5JXfnrHBoZWXl1OhCRsOsebcZ0Aj0QGQBLTGZXJdunO3sZJ3CV+LorUE+379ZRQDqtfsQ5CTdLiaqwJORNPrZq1v2olyoESIDDoQ3tiO1a/mpF0WdfKy/bpJSZKuIkqW0EviGNCdLTMee/Xqz7wJfldW1frIOYsTWnuxyeBzhUkXWWObdUO5rcwzVoejmXfhXjE1OA6/BqzSaTVnIvVhizI1AN5757kcruYI7z1tNV4k1SLr7Cs7DzSjcZ1jZRj57TwacQCQ+BwkutbQfGqdINRaePByojp76jvrU0ssGdxdtEGYJBUQKzD9C4yFFKDjI/M4yRe20mWJt5rQ3KDF+3PrgpoD/y1v71u5dS2gPGv3znN6EXznil13Y/J1dRAT1J8Hf1/IfTI/d0MGGNESOdrV1wbbB+R5I4Tl+LYwO2e0v0WElnD1BnwFsjf1iBP58LkMaQGhf/ZV38vxdZLmQImyq6tf3GNOrExlUQT8sXOH4DTkS9AYI4v+yn5xTbCgQSFyHo9gZEQPONjMhJgTWvRF80wqaq0E9wZjWfgilkW+GEH+FepROLZYq0mRQcMBbbNOSVOx9sEn7HitCQWeplSoquxQVLatOty/+ytS7VOX7XXQzAbpG5lYcoXS+eL3+nepWyUoDd2R0bUDoxj7MAEC8vABWrBAGzv0w2hElu64GVmH8bq07UY2FtX10Diwx7y2pcqRy4pv4/N1I/rEFzmOHV5kpRkQSRdRonXB8PGoi53h/94ceo+jHlj/yNiBCVr8/wvpzsd9HVtNFK+Uc1HMrQYNcDrdgr1L+8ueyI4NEWyydsKlqdtST+9iaaNfvR4ndSfIlmBLJJ69LNbx27kdq9VdicMQm+bna9Vy5KiSv/d6EVk6R+4c18K9ZMqw24NuL10N/uflvVKmof4iN2bSvpaQzmQXk51cXmQHqFQWwzR1x5NzhrpCwmY52GgOM7o2hjKPUP7sI6eNzjwx1HuPla4I771Km/ASIih6L4ryTp8eHa6dfRRp2+xTj0w7E8qSFw0IaJcJMPxsRRxjA7KVM4HN/41rMwdtmIXC6lRaIjhQvIb3O2S9/asMK5OvBjeTxwAQIpohmQzi5fgB2WHf/QiIOa6OzPuivCJTSI8nfCm9rMaIeFrxQc77eY/lxvKQdoEfbPMQJ14SmUI+ggJvhWO6OD1puIExoWNcnjg0kgikqCaEDVwi6+Av4XbKXn0EEXZJoUFO5IkVMd7i2Nx5jQS6kGr+sZ6ZUwcrTXytOOtQ5BZDCf78aO0iChUk1es/NX1tbXI2jOn1YAuTnSbUiKXYriegkJQ0oQDyJxnGP4PLqQgBUlhkeQDJ8qM8r6wIsM9pCbLLkV0BFIvQVmCbC64Lgaiiq3qufX0jsdtwKel6WxObc9bIe8dasVXesazXYiylqLCxa5cdZUH5Cv7ni4xnJDt54jSCAeJu6IH799qpEMrt2m/7UVsOEh9GsYft6/eQEOKGvaODmLidaZUfUECBvZOXL5aIL1fjPqHSwgo34D/5pf5eRY14WHUb8Q09Mpa+98VQl3AWPye8DUnUpnU6CU0PfLiARRKfwKxhppjLbCM74D7C3qRjdKoX+TV7mN1HJhb9TaGFijopUgVJkwi8qCCoYIJhRhCcUZhJ9xPKGPjSsYTkqtHWAauv2y3ebmbUoaHiChtAR4ceYVkws4iWG/M51OWsyv4G7aIHm78kK/TOPOuV1fULZfbTxmD+MCttDCyhLa2WAUX2zcuHXkDXnKrqg1xdgBp4BtoRZITc29MEOZCRBrbkE8TTnF7dT1PG400F8VxX4zwnVfbUS1rCt4AxfGVAr2TLw9gdW5GVIZNyDZXynXHco0CLCmmZ7N5NQzSfMs/uqKyWRp4/s+C0fYpgWBt4ZFNKdJfkcC5LN4xbYzCcvB40BbhVhujzbmqDCgdqU/OHvogO67vZviCQpaCd63e7ldMC6BF4uqPo0qLvw4lvRmo6seI25hN+ztuTvwpqmmtmFutYmyBBcL2GTYczBzQ0/WE0BpYCRE6WuF2JyWizkMTjCo325Vqni0HoYP2/pvoDsLPNqdvy65BJ5tMFgR32BRk6QFPyKO48xtMfOf8PWa3L5v39TuyM0nuisd0UhfDX5fEB67Lvm1AnzrPxK3GsOMe7rvuvWfSK1gb48fi+sQ3nihDthIxnMK2ArPO1fkVfWP2+lstwV+5mPUyonmQcKX7b+taXRQpe3M1fFDyNDp/0bEHl/mTDIETjGzNXKAx5J7PCmeCmN+oL8x580dhhGYbwNZ19gX4XTBI+d9xzNVFfT2+gdzLhvpbO77sx8jaDbn4HLBmHbF2ipdjth7fKCmgWKvne9g0TvnJ4ehEgUug0nj/qYKU99qL+muUtl3rDo/4r5EGOKPe5mn1E5DiwV1iMc1Jzt/1p3cLqCfKKqEEz7wLe4yjJRqRWN6j+H5th6d6WfYynQ/m2MRJJfornF8ngDkUdxxwfp61T6HeS2xaHn6H7wHNczbXnz+1ucv7LMzMkTpt6YAZZw+MjLz++gWTkD71FryOXwcb40k7qh7UzMAV5y6LsExcrQn9rx4geGZOwHLpJFRrY6RMuPE2sq9/me5HX5PX9W792Rq23qavH9KuenuFk+cdusbr6D6bSAM2Ln2PfVM34MSui0UHuncef4ujxJSu7TXDCHhQM5zuy+ynl1dZHSr2NeTvCatQUH6F6IKUHkcl+ByUxO0Jlg8Ce1mwgbl2+QQX2IIzNcGyhXg/oVKrX+Abatxy2RhAmYi88rOsCJIa6LAGnwPy2nFRmi6BYzv2+79x4lhX97n28OnNItfXKi4ZZgOylQSdGRZvtBUdgkbB5QcneCrNjd6XUvHaL518hmN1K5BQ9H3Ci2hkn2Nb6SYJcS/3JdMFIzGP/6zsGeYGFyGmUB3ag8DEkyWB5P14fI9FC1jHTxfyxd17KjSAz9JXJ4JJhoMpjwRs4ZTPj6oX2namdqy+s1dndLOudILVG1pr7wJWBT6TPEQFhilpYzNJ84O1wEhUfj1zMCGYbSzvEdgyv3tDh7t9dcfplIuJtU1+mvF2vNG8zRmTPcpLPwYQdOtXbMd+K04kMgqwMGek/UeXM73/5VbiShTR0rpGT1/SQF6TpqjnzsuJxqSyoXBqtaFzFQqPHgDBTBJ+Zz7B1aEn0BiXauRKSMgdnPzqyzaFqSNY/qY5OppF1I/7AMpN+QGXr++vVR1I+kjtvpqj+Vel3zSfs+siwjpl/HpbvHKcKG3/avQzR+QzKovH+W8kOfWPfKiDpezy7SROzYP7NbMUCuYuSOeYcio7B8wIsSzkmujUNUU8SdG941BjqasRIygK0n3iMtmbbN32uBfhcHFj6dCkNKnY+0C9TzMdRFhFA+jx2ohBminaKOVQ3kspY8obHN2YlosEjQ0MZaN1jvfzpkN9+jh0chiqK9fL1E9tdl9uO5lr+FMG0BCUNmJt/my7aYe+UWobXxg1povM5i63BjO60a17rxCbhoOz6IsaHvbM7KM2hkirdFfDOzG0PwscfusZPQ6LSAB+kXi8PIVWKVv5xAyJU5eAhGVp+KzQMvRI3itkoYUvcvP5LwN6g148esCZw59TCQ8ozW/p6l2nzIjMghww2ReG8ZJFGXr7YBqh9QvNYDyRbSfAmai9HgRSW5+RgvAkK99qz7Tp8CWjGTQX7NFKLHWfcHpErQTPKn95s8W3+DdCWG3wkouc/0UdyFbzL0Zca8LpGOE+ucFV7nnPSvCR/ejyNMaVDLXhp6cGRrfCi7ruTbdH9ChKF4ZU5buzQryKOqspJgRAR9i1ia/bbmNzwmxdHmnyN0/Tiw/ZckVm9TRr6Nr+aPiQ6M2IFCi0SQtkKHcz9sYmi7jr4jL2j43TIzB3VkeqabMHuepEW09i70jkTz1JDHcUGyiKTLHdcvnDRscvd1qw5RQc8eHAP+9t2QMmmCv+KucstwVrAMZH5Z74GjUp6kxbj4g4MBjBMI+I9RXiPAqN2esBVexbbkMjLXbq0sXuvrBitBd5eabrrtR5XeJsRDLyOVaCzg3NCJuEf9XgokHCR/6eNubDWwHh376R/UGhBf8a168NQDQNcz0ixNqJN9qVj1P3tcnejHDqfbUYgn8Lpn3PWy4X+K8r0gNpF60y/bO744fZTukHB2SkZVj1FrxKSHj36fwhPOyQY5XTIaHA7mnAF6aT15xZqvMP0GTr9DQ03qZRJ2mqH/gV8LCYYyCFbK9+qS8R0esx4v04NtIc3IWErcBhyhMEbcvFRDib4Tapp4IvDopUH+rt/Q957La0JtyMa8uor60l2BnVdNv0011hH9y1Dtyhmn6yrrIx5vYbo9VFEeTxVoB8b/kCo2nIuEoXBvK89hNcf9YsjGnpXCxCzdWv0IKWz4xfp9B0WONaFbvEZM9PmGYitFm1Q08pIndnhnRX7Xr/SeTJzh2F9n07e2g0zcKzCMnLIlkfK1z4thNhC+8+PG+5CYxQHUBNgaLF+upIbRNLIvO/7w2K1Dem7feMpYbdmyqZ3Bng8leOb0/ZscMKkNz5TQ3GCBIliFpIZaaNpt0ve6iS4S4z2rAR3JwBcTjlIyNPLMdt5vr1EEBSSj3x+K4g2R1D/RZNjDexU3AcqflfDXLfU+BsjEKXo9m7Hvv5FFpFuuW4TahNebZtpvEK79VTzAp9iW2jmgLCTT8eMMo3tC6/xZClpiGM6w6N5nLI6yJBv6DUA5alllNbACn9rJbHrbC/UFUGs0bGl6Ps763UZ8dYefhSidaiQV5QHnbdfDK7NAS/mpsn58a7N4INHjwx0nC3B6IuLuLx2RUhOPIWaDDCqtvRNXYV4Sz9/9op6jG2oHrHxeFEQ9Z3hXETikRcZG7PB0cfeJlyqYt8J2HhcpiImQ3wDXDBp+rZpW6wHNTBLBVKHjvMrXSO9XyFzYdnGWdGEM89GWz7QSVSobxbOLUF0N4yAfXvo9orgIu45IwviWD2BGN+6uM0OwHYa7UGq81Uauh2ynrrJ3h/FIXhE2qmE/+NIXSKSjJTRKU0uAVCq9S7asOE8+ow6CFAtECnAYDbq6sdTIulbsXDqJ+ks7by0nPkFWAEVi7GNdAzUG0ZG/U16bRE3iwpm7X7UjaXBOSMrl6fgIEIpuYbOhE9ZWe4ylYe/bzY5y5Lab0UrKvTi21lW//l2a/6MAghTFrNuWj3uSMLtpY88RizATPN8T+3lSfUT89Zl+wvigcuMch1B3h+SPM8KwuDkFQOEwV59fIwB72OCapzcuy8MspFXXyq8LnAk5bElj1Vf0E3/keC0UQU7rqFPZio++dfv1pqVUJngPWy3tB9YRRjtoiq7Oyya8qYv4Jo1ljrx4Hdp4g7WsYcQxBuz7m0ef/hRwL2ysMF1qWnI+RO9sIrUZ+dglsBHspy+3YcaPAuOorlL6yIfNxbzebXjz8OJ6DpFyHzQBScHB1+I5IR5lC97m5Ur+ah/bmS8NVi1wTPsyrUq+Ek860irbTcQzgjLrigis/hXDQot4o/dLZZruWpyrixG66zP8/mCwQ/TDIsZBAZx4E5WloJENxoVRYGhfT3ZSyXlbL0YRSAPRmfuj5xfEXlKp9dqtvB7Q9v3QOB+nleroFuKYDgwGiUgfwfcs+iaVUa+gkOy21oQzjoPSlG233wj0V2a2wZ8UEhmu34PviYqWB3LIwhM3oDvNbEhtw1q5oeWx4B1Ur6Ew7LrcQK55i6cLryHi1pTmwcUTHt/Hr5lHm/djyzhyO46kNHznh/8T8g3cOCyznRAtyd7msp8I9nt8j7XbEBqmMmugzCAJNr8PON0T4D+cBUX5F2z1Hf4mb324Q85H/UmMP5ho+pT4lnq2QdTsq8vESmf1IekUUgUG/uaLK8UP2wy0Vc9hH0IP9kaIt7Ks3+x3xV3svk06MkRagJ8tOd4lDClgVa4RSk0ilY0TN/ErXcLt8XXa2jmX4Jj2qL9eLZUDdQ/ALBpcGgTlGay9yXDnyGrUnf3tj9P3i8JTYcBAIwxC1tbVrRPW33TOyDPF4RucZ43FQqY2C/R5KejG2y52yshCcKDen3VO39a8SV9tpQlFOJJK9WFMcfpuF1hrWcaPx6WUpbBAtG98v6JV3ftBtGHfsXE9ZFWMHzCmFtPEGCvBy5pykHWGZlCS2GsqNVqKpOJRPdsCk6HopgU/UR5T2yZc0FjG0m9lNti+Id9coyjE8bJlPHrznhvm16h1FfBcgsV2r9/kpxM/SObcXLelHzh4IEVgLmUwhW8vIlwrCoX9PFrfe0Ff7fA/DDeDFUP9vUzW2hmSsmVk0xSPGXoxnM0y+pfQU+BVm+/7zx0x+IO0ytRPRnvaPuupiFmX2WG2fOYcAt7yizX6u2DCARXs4TNGHm2YbFUx1ws3j1TYIt9uw9X3tvbu6x2OdPfLDN1DbqpZTSdrvfTGk4RXqx05hYimPjZAmNZeVsrg3ZRNNdhxwd8ebvjsf3iHcLvJdPp938bH/p7ydtNKsgQQySxmmJLbFAwkOr2mye9Mt5DcmgsxbExHlaprtJzyoYgxuC90dPpuPIF4IP8Zl7c/15cDThcbSbR4fyWGMF2j/vV4j7qEpxM2aYwHNFGl0ALoyYTTZO68QOnwt5JeMI0tHLuUMMMw2L2MXfPEPTqaLZhGQFqro2mWBNBfH2tEf5UEDkRw5BvNTiBeX3Xm9StjIHmNo+3dhYnvrv76SbbFZrG7hujxsJDzevPmZz49O7lQ6dXCbN4raOuEJdvrbqtyc0bLHk74LubeHzRE7Tlpe63UjlAj8mslRrlLg6imlwCZWgylcdNd+7FbtnqBWMsWYFwaYUrJvzTg2QXmWyCEg+r5U8Velr1Gn0R9JxocTRrFCKVp+SDPYyszTd/FprV1e0+/K/lIkshOu9nfKv68NhcaGCA/GNm+O2KzFHIYSHHZvJxzK/upPdo2Fa3tzH8RS0dHE7Xsqgsecq60wVaGTKvaK6XsJAtxLtGgB2XmNnCg51fOBqdOZVQu5hvhn10Li2fF9Dkh9hmi6BOW/f0JYqHq6eWG9/bQq91UQz4ccHNgnBTvZjjhKOQw67Z+PuBlvbDXeokqBoKYyqf8HxaaVEFT48nQgajLX1Hoo4uUit1SiR28LiaosoyWgTW9N86W6GvkA+qKeOzUQ3B9NCv0doFApCQLS0x3vRZ6tzeNZt83LLzs4sf0UnBnXKDlFDykhyCmKoCU6bFxKcMco4L3TN0WBvGw1mXCut2kv+WhzlTVfa8HaCiE/lDOIFnd1mx16jW2EZEzkIUyZh76YYOu+InhdOzjm/iyuLp9Hsrsl/pWrlzEVDJuomE1hYb6dp4hip83sg+isxds8EESsWSPsn7x1B5SnSKXsubeM3/evO7/KZhMgnlMXYPiuV2dMPLe98d4kS79PlzNriQxNo0vyTIYKUbttI2T66HZ9vm1+8+OX0y8r3JOwAQVVqvfQ6WjN4pwY/Syf7BsGW1C05xWahVOoUSN3Dnsdf7qSOAAD6FXJQuK9NlhX4OCHFSzEiX+Jc8teTH1gT78A1W1oe+HekrkUcEPoUZtDahek5jaBtRa7EbUNncJvyqW3ZaXTzv8yjDvowG991nGHiuyqpMxhrGujsf5rsXtxSSGLolj9wSPyPhumdQ3SouFu3YqLiHdP934MslRPFGPxaQrNWzKeuU/b1sQuID96jETSpLvnMo1vy1OMPmyT2WT8dtLkjj/3B/vYuAr7M96TPxpYAaJwDxKAOGWQYfO7vIRfg+LpMBcC3pV/Wo/uMeLg7oL0pSIeMmxeFB5vu3l24NmROb2jzgRnrGhPltqmip5RiOIgYmu+Zp8uDt9kEQnvTqOaQi3dAyrzyd6oako2nmw0aatiIgmF/rQ+G+s3tV1BTNf2Hc8W4NcJgnSEOyR8qRxnnZXdBYK7ikUv8ESxNocXPSgHTFSjnnp3+KDi8WAR/bvb609K3IF/aOO/tGIdk++Fnub5UBr5VReZORIpTtWy90SwJuF76tbxDBqVDSBpAokA8D3YG7/qoPJ8Mle/MSFrr7ruc4jfpgVq3q/XEqGWnky8pKi2Qdl2KebRazAD+99EROAvNLNOvAIbY6uVHiWe+skP2x/5/0Dt1JBuXfxlupvvOJqaM8vHeiD0QjChLbTI+qErvfQ8a3/2NkgO++PfX+S2JpiS7QlJPcoJPsYhKZHZ8xqwZm00QOiE2v3P5H26ziqsA6+n6b+s+K2eiEHJ8DczFXjyDCd8LLGCWuiz6UxQ/IRdM2zfo3lRX0i7auvYXxNQ/OLWN+H9qXITtSqisoYGFNy+G8WeTzeGKZ5j09srV2Svkyu4lWwL09klTu4erssrv508pPYvY+aDKZp0TXVyS+QuOqqbFDFz8hjDTfDEPWb+Qgu99FfoB8L0U5WlVOpHFiwuQZ6F49Z7/JXp9k6bT3BEyg6PprcAFicJXbE4q2wFIXLhZVRM3zzZg7WdtaAIIYcA0Nue+oaDYlydOA5aOGoWHCcZKKg3lqDvw0e+7yqPEsFzYyJy62ZkqAObhDbXR/xzm9O61ri1OOYLtAbe5jQXVBo5jRe++b/5beaz3XC9L02I5bRr37grjTeH/+gFkmpEVmn2duRv9KOWnS3Iz5LnmmcmgR+QcRwQ/160Cjz0TZdVU2ZQCx+tzuu07aV6E+6G604baOeHVm2AY7LwiKK/c4Olf/zZRFmj/XYytWo7yuSrHjxOo8TARrPCOpP8s+yLV14cUg7faYRu7QNkjnz4w7BtkRjBuH4gFMMVsaINb2OkozLVTEjMeo3ORQv5b4iJ3uN7qq/id5v9qE1vLGj6/1owJGvSq0QDlEFI7FYbflOq7CNpPPgsyykSKt9wfldMDtrv3bJfHvJgH220ud2zvgmqyJJYL8MTajPI0S223xZRs5b4AY59kHSP9NpaiC1XYmDYVaEMMyDsp8/snr6GjEvXCcEPnbWwtwPyOPCxjPocWHwgJDAxR/yK+HaE6vrAmVAcsrSG5FFsjJLXe8VXc3D2vxIat7RiYRSae3/b7VbZX6Gme/hFWghunVSx9GV9jqC2ORlklIK8gGxgF7jXfW7OCEDx5qoTO/RHSumuRySeUI2hbPlYhNchGGt7lJA0K8pakBZkeQhQRIIml5/poUVSqPkmNp6zZ6t/wGJZcqUD4jqAauZnZVoUZVJE2IFGGs+Nug40v3yPJj9ls+RI1Sm24kK6sKZs77lWaoQbKxc6yNvEVG8UhzctqRygvis9TciLCI+RAaxLF1wUIkKvYjxnaTU3fLHlrRS6ihGa2A3ch94uA5KyjJv+dCSD76ooSxqRQ9g6RNOlhWvzlRKI8iliy9/YAWdfNYPI9orAdOblAdMxaQbhm9vEiUhMJubNeSYKTkeSZ6tOfgWccOdUZ5Q1x1j9RsTQhJZKeW3N+S96aCz/PoCrqnEsOcrtSISZFEmCKl2D64t8zfqvWEF7dn4Zd3NdF7I3nyZD8WMiHaLbB3bry0xYQyDMMLG/L6zIoYpHWE1+U8EQ8xRaQnCfO5ZUpMIjU6NvT3b6O9+5c20fkWWAwR9VdBfUHaAO5+4J9NnEkDvOWP0BkIseUtk2f0gY28nm/4xN6Ef5xrxTuSYbBPVXhFU6cktaxNX3WGjvFLBlz+4l8PNAqvYfMXhvYLbySeLBQb0lt+n3sysrK8LG1whoYztXWLouxC19urOT9v2saj3gC7MWR56x0MHIY8dAoSEbcvHbTHQuElpXdL0At7R7lNhmiMfgFmlptpAeiBULIWv7t9hH4/qxsZShzv5o9DKFR007n4gBUDxacor9txJxZgm4YnJoCLO5bTrbKADB854HiKn7OaNsNVoan2eh0l8Eknkm9eEFEsut7H7H7qw2ud8kE5eGqoaMF6AlCHmOHdccw/MK72w+IA2g0I+SaFyZWV/QKDnXa9rgXUpczxu4BBgGxL+fes87t0hini+g22nj1dY7xbvmg7/xLnvW9SvtAqzIdXHxCDZOjK2XqmOo5dPwlpiVTLIq6rMfoOffz3WedEjl117cVOc3jlVM9ZmmpTMfeN7Pkxe6wv4N6vDySxXOdPeb8jjbAchxBIaJ37aBOxBIzqSmWy4UyeR9DOHpd7ijRs7fJFNSeud8bx4APAjv7WHMQkvgPLTr3dG756w3e5vlVCJfUfVENhVEhXU+l5p1gINkNkETq7pvCnYkiyYj2RquRsEaDzXIp8PPjosBzd0edkeHCHxSZrPqnHvOogBJIvk1vAtVAnVSzFD9tFIi2FW3VxAWvZTz7+GVwTBOcMRhkhCZj6ccQFLvVH7/aUpML1asB+XDdheo6mwgLr05TqHXw2/WjKyXuiKQvWDAVfVxOvzNuxSY8aeYyBgPWz2C/OqfBtcWopFwou8agYYVmTJKr3Jhnj5CJnWoVq/uUWtiREixkl4VrFgmBfneNEXPUsAjTesiUV0G2mZsygvvS8PKvfTWr1KdmhDIfCSquw+C7nqVZEnmXIyVS9hfbyoxnCE2A9fmhd8FL71v/3hyZQxhijlG5arPmZm+riaWR5pyG/AU9ivqsP+ioR2mKwg2goQ8oRojdHxz6TbROgrMldcLPRuP3Ijq55iDt3aagLbVshnnfFrKDzM4TR4OOnmDv3vB242MWzpuG/QuOvruUBlL5Yr7bByTt5xGopwobSgrT8YQCvOOBl9afkLCLXiPHFj/I29oXfKb3GJKM53xHLkaF/MLVH03xFSqJI7oiNGL9F4wobX8NCUizQPxLMcZfgyMFZNwbZJvIZA4p2o5QOrhsE4Q4GaOZD0emido21wGKJfA/NRabnfVaU/SKOClIdieYtEuce+FUkDM9uWG3PE4XVCwYov6xILyfeaJfs8v65jaJN6e1uNdoQ8sFyNM8vdoInDkkXeVNxAKBsEPi3rpQ8KL/56c99N+3UbG26v42hAucEM/WXywT+d4pi4A7anME4a//WFQkrdCSa8TK/HidGWH9V6OyrapI+PW3C6xH/haQdz8nKxshhlQVNl5vNHGsZiuWkmqtH1w8aYaNRn/3o2Cs60kPwkeeT4mVnyoBiONMpi+hsB68Mc3KyuPu4KJ+0XBOvlR6L3TBx9YMRf6ag5hgnZAE0WtJ+oaYKm/y30hHA9Cd5Z3/TEg5yyqZwYpq2y398lDL2Oaxo07jUAmf5EswD8avu2MCZQjhVpUo7hGCLAHkT5p76tU1FUxnV506ilOFPbCnhITOO0BWNkAqGtrOlzPPR0oMp5qDEwcUggwvbFb7TxiCFFlRZbtT7BgCkFxHlngfDwhbXndYgQ4hQcqZBmT1Hp0wkHubvfsINyWLmEtCgkhZWkh5K6Q/0HJrMHdysVyo93U5gro4fiOYLqkricEinbNN+jukZzJ/Xqqtf3RzVtK1c+iF69rCin026iFT9uPTCloVfvrm5uiwRCUeZDK0UVsjhFbHUMUXdXVPfrxhFTcF992okWbkcQYfkdawA1pZ+X3uQmNL3caRlBRZ7bQLVNIUWupps6TZe2vJ0gbSnNb+j+IVdo9qCFjAvpem1g9IYQUqgP9s3HPG6+Mcp86EfpgX2cVa6WGfO7n5xkfkf4YqIm/4IqdGdlDtm2NLPOoGf3SfrrMRzogpC8jYKF7kWyWGvFqVH9mrRyUp00UD92o4ObIjDrtKNmfgvqM6vjbHOl5S9c1NzuB45eoR/ZficI799gR90uf5crT840USj/ojcmFg3NOKNJiMBkrFoRLDlDP7IF73BxB9PnVOQ2/UhPHFA/2GZV+5wDCEsw4P1Gdla3gIKJOGzLWmai8nmAw8IFv7xQNSK/pzHfDyrF1VxFWki5lbYzb77XCpvciJJKm8DWgN4rVId7ZpEOyjP7/D5Da/184noC3CUQV9HqumWolwJtGdl6EBrBPlhUvt/zyb6UrOS5srZr35GZBNWDKrEyhiequbAQXmBKj1+NfigvcUSgthp87+W7oSl7g4W78vcHSZ3VsJYQjty2LV1oizA3m+iVxPGd3KQtxRYyj6k3uc+ZYQ65C2nI86WemGmqogfJ7AiqKcXtP2TJkcQ49GNqh1ow9jsBcZhVmfq4m+7EFOAoeeqLMZx9fwXrTulz787sW+zmluvBu8FicpJ8veZuPuHDCgMVsnUgasO7SeBQXOLKMsl44BfX3b/P0hRvmwuBWh6U4p8zZwP8IFBrlg48LCVnsg3w1Qj081FVDwcRMdiaNo3TE8rWFC/Ik6h9wFnY9884otbCROsoRWuzyTCJw1aJ5ooWHvcoKD4Y31lySpHUcD9UKOmHPG+lCMifK2VDteOLZa6G3HC/2ptKyiJySYmJg6AAu+jh3ZnqoIvUYcsRlcCwspvC18/wd+Y0/Av6daPwzWWechwK66ge63COKudyVjiPXxp0NdGD7wYOzwn7Zh5uch3NZbX1wsWu47qurVlyILpGQFpitMdZrq/xRff4cp4E5AAHeIl2/62Rj22V5r3pSnY7LDatEozuUhoI9yodEk8mEZlmc5vF9Y7EMO6xLW0E+RB8GMNExmAVV6jYTxmse9FQL9+K0XtvRiPW7OdogfCS12Clm+wbl6yVq5ESVB8zXfwH3QMF9u5bgFzTN3lobRq9dCrTjrSAPoLZkEib5eXIM0uYuaY5J2ws8wme6l/p7tm/Y1Yk4WnTPXWrC8ivQTD7f7Zl/P0eK7lF/u5Ec9FLqsYCS4+LW3rjNAfa98lWI18+U47CYi8dAbQk5CXlw+KXxx6wV5O5xFmA2oNjh+v3aXGsbMn7BgSZj7Q9ZMu6BtUZVNutjjdbQ171DqWoCMnD5sWzTyzo3tVmx+xM8syRyIZGjyTN5DzZF8vQFZmSHeSPnH4+nofjXA1F6wR7B1nn4UQM952JdZCnSJUY4XJrSDfbHwyR1QyqAZBZRc7uDBeSZsh35yBcZofu5+D+ggjgU5u5pkN0CxyB0JTZ85zXNbrC1VHqIQ91YMiwY9SX01r6Ev4QOhFHFKlQBktGvhAxOwSlosAwnwSmmjHAJUjPDeM40HcI7Cg03fsiNW8xAO9n9TCYDIAnpBdBEyJ2s4cyghZPnMcxrIoNf5fgBtTFy5DBL1IhUI+zDFno389im8Vv4E5paDOZv6E+HIqgzOI//VXwaeq4EKvs35zOhrc8yS0GHK85pKJZfOVwsoXm/1TBEMg3g6eAcYUYBS1i9wpdJ6HeemHKovQbAKp5nb7tI27MWJojH/EkBfQrVjTuUak1p2uUvef3CyqIVwmSya/XZ4Q/vrCbvo9mNbmfqlCFo9xel4qlZGW/8+0L5Gtd+0rQD0aAmQ/N87uMMAIoPz8H08woraHyIFWPC+6EUc+vn/B4e10eVmb2oHgZhj6HcbBr1juso3hfwrEVlP7BtHkcHyDljYE4vizmHLQxPPXbAR8vvCbvznA0h72qiXlxmp5wcRw3rj9cDFP9YRJfX6uzILpal1/9bBjx2y5GkeTdXYNCnAO4FdQfaqwkAbw2dNWhDx9eGmDCdky5CFKu2/djq5AyQQwk2pUyh1LS+Zjm7iuata8+3WkHbMrvcvFz7tOTUhcU05RsgSe4SqiOOTjwH/OC53M/kRguTfBXtfwm/vXpfWIvjZVMZGgojXoAMwh73l691BAHBZy/oRJgcaup2N6vwhJezeiWVM/APwXNNGsQVCjpuFhHZ19hbP1afjjq8/tBfigtAWiinnh2pNQZB/XNFYWj/GbdG98b4mraX3+dPEJBYJO+LTgj/X4e1mOs63eZ7iXvMocYd6xTtfeXu4AoJrMGiiZyAp/0ZgmCzVcogdszCmujPTznrbp7aKowil4tfODRwPLsdyeImNyy9vFiz6BMJrQyzbKd7X17jOkcCOEi4dfm7T71Vcf+XmcHjvQ14oW2laxajQ0snxJKL2ugIm4XMvfVBvH1lby5Rg26T1YaEekOWm5SyHN63vDuCIwagnaXbzM1aT4OABp9x3bOaNDKf7tMMYhxMzs4Dt5Rv0MzOYWMn71FKeu2XqeQ9tfw2x6rjgrRdN1ZpN9y8FJCAL3Lxc/Zze7tDX1LMzHHQtf4+C+9zWZTE6yFGTAyu2oFTc1SfDMYd9/BG2NVKMEpmsWaMi3y8u0YPSW+igXo4JYks9bj77Qiv1EEnx4cNJ8Lowfn/J8REPWvtP8X1mVOa5RwzgGLgPPjN9g9CNLseKOoxAWDYI0vmEIaTZn1hwVPB1H8fchcVFobrOeHp5KwmAX+AWxxU2lxBi0zbEIuHLIGvMgsV5+rz/9GcB/UspoethThw0fAqtbduJgPfuUYLH0BCGG8KwD2pXWFoKPHUO+Dcp2DrQ5BLZcnGLF96c5M+feh4KCXlZDKfTNKdfK75sWcVQBpU12k62/cjhI/D5RwFFxkSoBT1GTRCqpYKXKCoIZzlirTQ4o3jF/cZ115a39cI/L9ftbxR/vujuy2debaBk45g2HMRCvCV9n5AQR9EecCAcMsJZy3XbzTpvyyq/F7/PnTYofA3TfoQ8RlHCT89nHnPIsPLlLy+5uxHOboln63ZS1brpvRd6s9oDlNsd+B8VOGNcjhoFc0P7pSKkKGlY9eMmuTvHgSefi2/zEA3BphgcB9WCkaSKEJRmcNaVTBk1Eodl+WwYO6+EYFQZ6Ky8ftrHq7eh+pRJVDsh9Pj6nw98vCguK2FkMJsO/S7T1fZH5YRW7wBSaxYebviLBodvYwxGLzRPf82sb29auf0uN8YLbXq1E9H6ZRxyj6Vt92jbwdVKtYz3i/qix6rHy6H7YpZlKkhQ04ROCcD1R23m5k2Zqwa17Ims9PMyCV14HvzvqFa+BFS9xiIWjphFZ5z3BtJQMynl5FF8Wd8Wtu4GLS1nzFqUMwmP+8qq7/HsBSyZu5sY9StBWXGvB37+CiSXKosJFmdYb2ti/s/Ahi6N6J9WA8LzePbxJVKB0SLn6T5528gd6oynO7gwehy2j1bwLrL6r4SyoL7VuKLPrZ5YJMjLht7nXhemIaPkNOVN9IHseyMnRG0uuaizF3QC8rPTt8la3nbdqinT7xGKxJUw7OBcoqYIylsa29gTmGwniDQhq0PNw1Duz8EtjPgBFVUcxbpEUZPB4+c2HAyZvflp9SVNeaqGS59zcr/wZ9sbezHw8QgU+Vld1+pYPjeG1vFwF89ddEthYcrz9PFoOir4F8Ktuq3vqDrYzx/i9CtyMd9CDQgT1J355iEzS2mljvfY3QqbH9hB+YoQSQVDpRM+FG+5tpMJzAxJtdHBM9maXL0uQoJf1XG30nWTSXVeh90377O6sbgXHUoLJTQIa9NVTs4Uq1stH+hpsXzEW/dOlcxPjX1FKKFYNjWyToeI7VR0TGqpLVqbhDvKuol2kI/kN/IO45RVGIIK1Kb0MaNA/8Nk5jF4topci3qwbDTicoICO513tgCRFaTqZ1Ny7qCFl9fsPQmzgV/P1XcRnZOdSOsqMpFTBwQiJiCq+3IhwXHBpDg+WssOw1CKeebT+MTa4WxT9u2ONKgQWZaGMYvz38RfKDObTt3fYePbCs8lKAWqqNDB+Qc5GZR2NGSA9UK3L7rz41sUCamMs0F3Z7OknSxn3fLqc7Ya30OHa8dZrAwAqB/aKRX4yyl9RSdd5+bNa9H+wl0Tj1pqHpMMOIHO6LT8IlD7uHoiGtAkobIIiET8wDD+aqRXuo4Rc4TEr6lWWdwUe+UdYCfmUiWFDgpuaz6dD68DHD9+Aubkn/JJBJpTZhMDnVCjcGtY/87tX0WfwppjkMWb7SJ4x3TK7suP8u9WVGM302Bajp2froTira0FcIzLgVoHj3zlq7/W4vHLyQXyWqY4yXRKAOxRPUwSo8OnMwkTko9wHgCe8yUeeBC58f3X0zjvUWo6Zns/qdhsvmJfYvqoNsUKnciZC8InEuIZQk6Ey4jNIvk8YaQFK4y6YahiZHudn5YSkKbWzx2dBKJxZODe6cQdavsVhCbCeyTkjizlf7OKMjw+GFrJO08kLKAoyWZwdc5KHzzxZfw2XoJl+f1eJ+5JqxNqpAk2JDozVmWDvdefddJ0qzAOlIWli1Qs2m4ykN0k7pQ6V/x+E3ZxOjssdr2zTO5dXRx2KcJsv560ANtIQwliaiJAudOJzbBj1s55eJyiuZbN+3MES8N7ZxYyXUcf99M2Z6UGgafPaord4rap6yKK6tR777qHSbAvUMB/MMq+AYV6kDtJJo4rUvo1NrklVVPeotTwhvzh24dJLMqOIkaIr6Df/xansUo+ADta9AuZmqqC+5SXH+YetLhuywXUl+aNETyP6k/AOOsv4Bvo2HWSHcHOW7NQpz4AQxqhqGlIg0EkUyh9YGHMVjRRV/eihXIt+qOyXXb8Ay8U4z5u2180O+48X+tSTS8gdl9C+UkDk+C2O938iHX2Vje24NJCkQWlyM92AEeE3I9Bbsmem90jnV77cRP6RlBDBGPac8n5HyOxsWnvuPm7O7CP4KkhJWrfazU54ml3LeCAH6I2C4InpMLg5N6loHw3EqtmQi3dhUKGh8qcrvJDIuUZvS+HjcvtKfv8GpX+AuigiFjMt+5Vh76s8SIAtJvZHE9wk+oLTKwkBxAfsCREMo2STjQzBcR9CjNQz+TgGpqxz6tjOmOF8xXatbeY4oaX5r69i8/GfoEQaY5gnlyNGrdAr5z64T3V2/4tVlwmpM9eMOugMNgoQySfQrn6py7p/05VX+rT6UXntY5DE2FfydBqhSlPKMjhB/39qnQjPf4xwKFmvlHaJvkWLdCd6UePLpT0f4JT7CHfS4fRUuUjo94GJ0HQf5qiIfCOzyeYI+tQhOzYmRcYvytXBLwDRAR36zI13EXRDHyptlVMjVmHmABxBz/nr2DD448aytRJeGR2OwUdkYgbuapkTMXSJTFDZRciW8386HqRkP5IlmVv3swa3Q7i3cp+GyBM4BuYIC6/v1lVQyx19Pod62ACRAujajUUl/iPT4fj6bBZdNhXfQMhUj/GYlEz7JP5724WYOhBKAYaWPd8v/bHhMqAykOgt754KF3nGeIvRWN1rvg7T219XzQsAcE87BhEk2Jzpji0TsgHvojSuvtLDolQJa/uoUBhb/5HWWfADwlgoz84TmlbiqDKjDxkNnyZrKjVmAR6sFxOekTR9gQvRr7DsqoWiYWlSZopXFveGGD9EmZkiU/BEBG+HTXK9AWGBd5HEy9JlIeUwH1zJC7HILKog5pqHTzheGK66i3HBNIfieQNZi8yNc2EBHlqyzfK/p5i0OVYTXosfVC2saGaiPQ9k3ujVYP35hNf4k7fDOx2KYbmGOk516G0pjLtHD8338A3xqs8fY6yPl3Gt9vS9xOTaCN0PTrrNfYpCUAtfbz3YHvwAOTiQsQfoKmCMPIcHDu8zFtVz5BR+3mCxtEMEk+q4q/gg+CvXWPIZlnAn4Wp2cSbJCffv6XJMA4TR6gJc5oYu9O2BajW8uAbzC691aEwu5IvX95Q9QPCZ82Rz1mgxNxvPKhtkzckK0PzhFGcJPv0vQ4ZaCxjRgnMPSSQzLqmH3e/iiY4x8WmNSg2jED1E7ok5xOxBbl/5NmRvvhZHxepDqSIxjDom+KcbrcD8Pch2GKM4UMC7H8P4XSbO9xKPxqoq3Msz2fFoOkQ15XgBsNqn5gGnO9On8qQ23hEWSwygL0QkVdC6gj6XaLfGVMcH6tTBs4yFNYnCS8Tkz0Vc9m2NwhJxsCWyVWIdoLZa0gdTDSiCsNSlUwxLcZGkxI3moFtHJNvmFeA+KYa+Teta0ndNTtV0MKuMfiCA/ZF0C3gZ689Lu6OGx81U10bbWdPFot0B4nGmv+kFi+2+QGDAgD/PVY7oPnLm+V+2T3wCbLwzWDLnljMeVOEuyX11Ll2lN5mPYH3BnyZXm5OyC7/jUhQZFGo3/nigCmoNDMHoAEMa2fAIm8s0BEDG+haJrTEFhfIwNOafZLQ+E290l7iXPA6AkoSsi4+Y0DBZ9xNHg80plIH87AN4aFhwhC2MGhZHbmfzZkJeoa1Ij363QueRRB3QgEzAOxDz6/DutLDqgI89XxOxCpbXpRAPr+sNOAfJo8PvF/338ZpHcbBp3jN9G1EhNFiMktwGUSDitr2rpczwPjNZvXYh7oPc4EAzOhkTSvxsLC5yD87gszpdXvii0Sdzgu3lAjYRCepk4Aueydiwp3PYNas/orC0gzOYXlnFJBz5gDkwLFXJsxrqtqPEXjncXL9vSz+fpI2FPefvVBuhl/u6XXsqvcW3H1zJpHFB3uoN8QTBvjyGPfPrJUUWFi/U47BSG+W7ZcmcMtAUGaJXF5vkcgns8BdxwrjaIFdCSYN5/HYkwSfqUF9unSV5bBhEH9NCZdWi+heWnHfXFwtmAQlEtJkCbVY5kA4VlG28vmkYg9xGu6GgoO+BiYh8HvYedF1Mp8lLhz37vbHf0E3lXXI+KoOm4gPD9n37vQ332ZoU79B/soGOLP+wiNoEMOslvviQSzhO54sgFVdmON6eSwWmV+c0c8XGNjQjWF5qVcVHh2ydR+r1peqxTcI2rIXvfOQTgOTuvVmyIt3wumswlgM13/SFyRv0lh6iBMexlfI/1QbOxJSt6OGrEF6H+V7OJJvwD56JoSAmDiljx/fJoQqn7EHnyg1t66pZj1r3xjDP+AtT386vuDlBJC4oveDFXOZ73fGvGll9mnA8fwlx5S+Px5O/KM3yb1rhog0sWk/6Fzt9QlMNi/7AQOwvB7J+HD+l7Z3cmqAximV/7i3uhpRBhPWGe5UuGRHt4bXm0z/P04LHfwLdeVGayqoyZ3rtO6hqP/JoEEZze5/SrsZbNLg9wlVu0EQTFKQ6ig3z73nuhxlbtUAk/ZyH9WBzYW4w6vtgF3HYxKhTcY5CiQYQayvDrN3QWjorWUgJJaDFUQ9wuC5qTnmpofxEMW9lpNgu6x0fFGS2OeGrJm0FfIJSvtz3B5cbK4eX1aNvznhJxyhrv1HA8X438tenUuYnAn5i4f3eDBKPl9iMQ0zjvddDbVOTra8Xf2DuKXlLlVK4OSp+0GSLxNVPkoUF1OQEFMf2Wfo0CxqSqesAZAVax1N5s+GJGiTy/v1FpFNHfJceAXUx8ojUE/hUCbmR8TEzzLMXQKdNdvdciLNn12fdekAbTGCrk5VEGmMssWEIWOKZ3yb68gQM3XEUXJEIzLChwUZxVIDDMglrv30h7Q/YDFEVAdwyGgiroFGHkDZzVgH5yI/wKhEYgQ2+cVPNaRMK71lsl8jAvPksVMg1MOwGGyFkhhFi4Pdiv4XWy+t+q7AZ/NRFJU9z/diQLksFOlIYaAD+sVWZ5pnL5uNafd72e0c0vdCVklO6s1j7frakHg0TI4/c5f/y270EbEJ/J++SWO6si9P2MRghmSIxjYCpuBNVGjJPBMIb6b+oW8/c1cBToptUNFjUbA/qBTMxe7zKW1qwuhUbYF3qZPNSxYYCntSMqS9FoUu0k3VTKd0mUOURc2x/GnJqT0+DPkp2hO28Pf74juL0NU5LhJj5luXZs/RsjVylh+6QnNoO9rGZ5WFTB2D69vWTUT7lSdshIyluvu+BJsI3ha0rZR1C14e0L776K4UBmXW61tqyTBysrcNaE3ojOO3+/B5yucI5Pl5XQ1x0H4qVhCo90/Te7H8/iMYaBAxh1rlhq+AtaOiPO5ieBbgUzlJX6oH6vOAZpKDc1TQS71sXxkCSXTEr7d7WvMRJqS1cco+LB/GJE3fOwpenijpwIdxEw3f1GymyOxt1F9lW2GcYF8fDHG6Ml6Z44KM3tJkqBmHUgYpM6741KXuM6bGO5z00+LDXJAimOXn0UKsyM//V3+qBSTgTmF/1sOv81p+gcq1Oy15iwFRBGtBt+wDKnX6pdoJqV2BBIsLBtelBRxHmXy1ihnVRp0NVbtN3I+HhYAt/2k/7mGZPvBq0yqHJAzp7eFpcr0CIr8lsKtGnXDIkrWZD8xFPg49IUwSC98PqbiQ9HyQgGC/s2wmcbX/SvqjzhN3FAXc/+pcmqjmvx+ytaCgbaKqY8mZNGXL674AFdePpJNa9DbyZWHyzNHjWD6wvXG9/U6QwLTAhzniaPx6IbJeVyJ5QvYJuabPEItVSrsjdrDcCf0Kb9wDc5wqJAd3u1zcQ5ga0gvjcTdQSW48uDp6UxmLPcwPlLJ79lko9inf78/7tPv0rFMyFavOEigyEhroBxlnlRfDfklHZCCkiRZzbiFsp15EpV9SmSN/hCkKXupXCz8+uFPasbq4fFl2aKu7Qxqj5dsdCiOh00TyJRqlsWUBzBYjGYG6tq2yJbgP6zi2zwKIwzH+3sq1VDdCeA9Sf2byFDhUYKfuWh8NXhWfG5B7oM6rwA8zFdX2JvHHufh4EQseWRkD+dEsSiEubxyE/vo38snOARo2DnXLcepk+q+n6RTJ49X7HSsOwCt88kQNAKy7xEO2JX0hkXgA14/Cax6Hdx9TcBVMjAMF32HNesfDGlvBCO90bNkuShkPq1GQTyKvETZO+0v0RFaxD0MXpxHJvhXpIm3PJj3P2jXGPJyHlu4B21YkyWYOqPkT+Yo3rV1rLEOcIvDx7ewitLMMmWtFO9uPdRGgeMShnku98OIBOqmKhditITel2jtXhCQDrZXmraIYtmuypElE2cFYnxZxk7lYzHOcDNQ1dBQGJfRBETfACto8PF52KQYXaSml7YWlUrYeDf/qE17+PYYBwMxGY9puRsAtBsy3tP3P4GC9Mcx9IElNsBl7qcRJfl9imIKBfCnNRQs/Vtqbfff/E2EmgSp5ftwGFIcjh7XOF7pQwyDlr8oTVCf/y5sl+/EefVy2Hd0v2r/+UunRAr48U8JFZ4SIHJzmlcpyiXNsJ79cqCsrLzPFEsNXXV8gR7mmFIs/XkIxhGAYLgYeQNTjDidiRy5CxwemGvDbSXFV7UIP7aqgx3/ZrOGx2tKUySXBopU6cR7Ddn9Dc6qWgumKZENPm+1fzBF0AE6L8LjXR4e4iDuZQLsDQUwLX6DZNz2/cG/PqWaVuOKkyKQG8tyA/vYM42FnSzJMQ6yppa+zwzOLcS71C+z2xexTj7YZNbcrurzovvwSrvuTILkIIf73bq5Jqvi2z46oGn/2LrY5gUpIhKEYR9TVc9wNa++dVddDjAhhlBSMa/gSzDDG2Xk2W1CjodCRuwrjhvi/N+SAnF2dwtP7iW3x9/AjQBMI4P+Kq+kxDH7Y6q80KLlN8jaloMs/PimlaKJDDHmCQt46PZLRhaocFnhwfdOHO7AqyCnFAooH7T/fKJtlD4lTy0/N6F5e5ht5jS0lq7d9SEmjLqOld0If5aSiK80kZxW313myUj/quvOTs/8bqZzfVBgtwfKA1vmkJHdqd57cSO2YzvD8xrHi6osE3q8Hc/1iq5eKD8ODpr8dyrHieqMIg7uexBGPZzNrY0fj7lyLmLZGEVhzCCyp1JGEeNduSjCsmohDQp/op8glLM/3pZXYVZr39B+jgu7i8EDkonK5hgFDn/HzFnADzJlbrbmm3b/XsfNOPhKh93oPSLmTvsxTHON9DXK3zRFEWw3EcOAm2M1+Vz5sWJUYO1DK6mH2LtOnTxXTrVk2wDqCQfaEoJV9bOfbQxkXY7HvT0dOl7mmCVi6S52LzOyCrXJ6+2B9EAgMy/NEknNePWjbWpvQa2bnsAwHAXPdw7AFvuVcMJfOjRWb2vgf+Fe+2AC7jMwRio8WBxVArpeiAP4pfFbL4gEQ3C9lecP/A15GeFl14G/NTBitWceyBH0ly/jBN2gM+a6KT7x9NVLLqqZNGv6TkuQ9zdmaEBkuD+9U3l3O7Bm5yXm0DVlrW23v6Uwlz4GHKyfr3/zrWKQFgUgAqxOgjT92EowFLYmabW9iz+Chg4i4EnvTkBBgTVlVxvNxTXCVPCohTgt7csoeiB+KTm+TX2VRjRpAM4WpQ7EiSDaOF6Wwie/eaPUbhw8lzXPFJvrjnZ+vBlv+D365PE2X4n7xQrpCO7gRkYOmshIY78+gosAYAo4ji7H5BTVg6tRzkIPDyOMzWZEfwMA9Jgr18ekBrIdxqF3r4aKC5ftG3ZfTSzkOEbw5Tk80ZUjjmIZjY+Ag/S6LbWBcrOc9j8cJvBZVWAyqWzYzGIxLKw/Dwv3VDEMl/Ckha3KsQ4Er+61jBHHQIAyZ+X3avKvAeBlsfjgwuSVT6MBSsIjh1+vMhvBmT/7Rcva/ujflEc8YtTib+t8ofY4GQ+x5RIRzv16l6CJatdQTMq94bMc2pmytMvDt2PPfpabW3llsH/kNXJxPXj1iz+TcSpR6MWWeddt/aPbumTAIbqiW5q+CLeEBSJGTYWreCMDiBANEn1lAbsds0AsyVup02ZmG0Pji6S+5jSmWk4sGrJ1+uz6k46LO4JEYgw0fQsB2Q1tewA4kqDptg8rlzeEY1uRvUucWGNelWmNDfcEfEechUcU2OvuEd3Omt3HUP0dOofM/riivMRyK0vtPsXPR7FkZjnaCDFD0zcfueZsrRzJQhSH6YLnvKBsyk5Q8khCO7oOQAU0eDI98Nb63r0hI1NJs69SEh4g9dMA46JQHgaxvNvsDrFhmW/OeSmkKPbx4x81Ozu5hQf8TXtu0OBUpCAxaNydvVRDQtEXH12yGi5iLVYwjlwigKyL9j5hX1IpVM7p6LoBxxBtluzpfYCEmzB3d1Rb90f+GvltTbZS+JnqaAWgjkX69M+/Q2/AvT6l/45xsxdJbavcWfSU7eftTbWZoRDBmSBGQd0HIgnTDKzMlZrnoFAUtzRIE5Et+4DYTpfK6pXLWulQpppXm42TaLvMa87kZ1wqT2+/3y/qdaWctuEarxoeHs9xNhxU9oTEFiYmpdjPZL2r8DxcTfMVDweXgl1cSqtFTe4GHcb4QogmC18T2nouCFqECyAugD3to4PN3ku6hztHwswxzWtuzr5shxXsGuLC39znto+0ZgfPOd/kwAxkZfqIj0XYlXC5Sv3iZRLTf3whz+4wtJvax92n3C32RfMAOT8xL2qmX8pAv2LCkJrstmcnZ7ujAvWYN2YcC7P0F1DPvaZiB53diSMTjqG7vexE71gI1CT7/LGtrf/8jp6sc2LFz/ja/u+E8L2MaqeJvDj76oNGX4sEb/Y9FJaH6zJCjn9G8BavUvj9cpYpa8JMsMWN5b893wHm90jSF+wRi1aBy+JmzCYj/6+Hw34oqe9HUIRUvMKzo05f2WL3g/53UChUjZNHjRdmdy5fnqFXY0gySw50/KcbtWA58+0WfLiz9Gwf2gYHBKNdqiqiNsX94QZvWoTI3TgLsbr4n8bq4T6IzyqDQS8psSc3CkI6rQbS7+lK48SN/+2h/92B7RZbZ1Ev85hQMCp+HWh8GM334OqhbSzGoOh3UAsQjjujekldW6bTUniYeNdyO4LDZQm1cwa21R5ty9g48R4Jt8j//KO1X1lJaxOZftJGTbF+eG0MOjUd4QkPhk0M8gHgKhCzylWnQoTSaLXmXkRb8/vgaItngzEq+hz5n2wZ/mrazqifpXDwxE8bkbtfqsvoVbhkcFx/4vWTcpkuEJC9KjvLl2cOnZIB4JhKMPPXNbln0w69J0ZCHgXhR+LmwIXLbrT2nLBgM21skXYwl6I5WaFwZT0kEP1A7KJYX3VzO1kkZBTzLD+G53g4tcnoctd8dIP8nc/UrEYf5ouNsUOzp91MuB3U+Kr6Oz926Dqn0zo4CkdHQpw4YJ3SsQq5N58Mq8xlx77cEPA74m5NWAKmdOUOvnUBd+tXvzMOR3M9zImxSjma5JQL4Dpq04xdZJak2rMu+E2Ei7ed7KsbP4X3wvzGD3seBfjnFLR359udJeZXyS0kAQ32xr077U2tE5iF1IHeTijMQ8kPu5eEwhPq31DWIRJfobvW2U3FZ5jRKq5vXdJBhWlzv+eMI2TDB3lb/r3GxPuKIpLxigR8LK+FHc2du81wWW8CdLOJ7MgrW+kFZMIDJz7pezwbWj7ipd944g4ZVKyf3GMwEl3ePtYHKNl1iuNR/ZmwgUawOT2S6skrH2RSsxD/SzOl0X0VK28zKIXg+36IhMaTAawIT3WIBXIufgy2+KFPMAAxrVsgj14f4+dtysukacgH8ADvkZsDU0ZyOveBSR2dmh4csgRWaW9te+7uBx35vZBAXWKvgkyoEYXQ7E8qOMLEN4JJhHze4OrZDtw2/dDZ2Av24NhlB/g9O67xboeiw74Um1WWZvH4QF6G751vmYSUhm99GZLrLaleYfUb9iOXxWrD8ijwfkCiQOkizBPQB7rCzgO7FfNTdM9Jj/01fse/69v/Fxv6Ycqr0AVqxBPucNgXwJdV+bptPeChW5I4ABwnQIKGQb/Wiyd2cfBBBlShAxHaUEjko1omnLhQ5gQh02/ryXnRZWxkuVB4QjxUqTUgT+23dgkJOJeRHIfQPZH6TVR+qfXqxmBfpGcP2n7z68rkqafixdRwhFx5vXgXTu8Xgfrvt8v+VHhyx3tieJvwp0yRRGaHCSg8mrEVxIyE1LY4t/WngmYnwuSsz5gh+pBcGQNB7S/0/vccYoOMhF0wViT1+sDQbRGYwwPg/LH8KNgsEyA3TVIThbVJJgVvCmwVnY7feD2e3T2mwp9PrY8ziD3c/zY9VHza7m+uO+9th7TgrC1Zjrw8ohn5yC63dYOU7CQzfBQDQoY8u67kzflq26Mi7+oB/oFMmat0hsTsN88cUDKWB61bIkoUapY6RsPRnT5Va1ZhPelSDPaXTStOBe1RhwjfmwQCEG2ZUKIVwzHdRvX6kFPGghzGJ1R6g72MlTL2a+WATUfgkQRakKrXFSQ9lknTfLSLM0CJq12KildtrFe35gElrJYFS/1R3Px8FkZOHrGLph7zeLZvjsfkU1XH4PR632twCn145Udx95bK19ATdUbOV11XNt7yINt2V8HHTbPlPqx9wbfPndxctYrGa0G3QKBqUg0Wz53SdEKPKknnqcwETLAInlWWX9HWgWXSyjvuPzKeKGUWRZQRW4dyXdYSwEYiVq5IayLUUS6TxPiff+XgXuEsZB7Kkl4uW5JMZ3y2kPNREQg3ayl/Q1BNDGjwizaDk+LwzGAFQ2iw/lNFhyPkePCR0B21KDRUtPEyf10zdJZz59HK39bmmd2oWZ6ufaA4UAiyZOWL2bfD/mq9l9EHKT868YIJ0dII7UkIfIFRqGJwUl6578JAxryYB609Rzv0ZBuyYJOv6YaJO3MyvbUAu3k6O+Tgii2ZOAtkR6lX8vocIpRZdthPR1cAaJzRmdHodtXssGYME6chKJ4HxzSqJajuIKE9t0evF8q/jyThdNvR38orfN1d4FyoclcAkwU0sP4Vy6jWbCU/Fkakbryd9JV3SiBPglgaTB/pDcwE+Jxjm/6E3rS3ye3ED0k/gVDodoQj7zeaFBq5VDG7Jnm28uGWp//s0os4HrbX+CdZb6fUYtlqw3i8mY++V0kH44eCy3lBs04JWC/6SRSXfk6VIiHiHbILyZk7LSHu9/EHxNZvfB8/CXAQCelYLv7cR/Pk1lc91K2YdmxEhVxi1G04RuHGOdl3zjuNmeoprOrawIE5upPNjFd07RDEmyvqlTMebpr+EKv1EfvRQ23g3qO0Xpgw4Wu/kyL9MwwXTsvMao/QOf9w103faKvIox/s5er8CFRXlp8Qn1WlwsKrxlvWgZLGaHloIZVPO/VCmDPl/ObVfbyHU/4Dlmj+JTivJ2qMxKMJnLd2C49XCMSHkeC+E1ooJCtETNyQlIYLazeh7scXXGUxKG7QGqLEGBadmTHYrCzDMMmHoS3lral7cvFpEJbcqp4bp5Xy2rjdxu1oi7turHBTCott+0lKUMcUYdB4TH+xaafpfDZhNwlLI4zW1VFra7k+LeGxyGwr77itntz0rV4/Sra5wN/SlIlwSs5UUpIVGWCrSHfxzWoyY0tccesnc56SBb7/L/+nBdiMHL56yYWXaEjc5UoRLqEEa11Tf/AQOBqUCmqg8vqt4EiJ/sDqEdS6pNYCCAkwx6n+RrWK355Dk6JBpv8+jqp1Q3juF71WK47kMLhEzA07FGpukLfCCE7KZl1A8wyTbLehsbb5OHxdWgKWbUbGY3K/JyOvkaT74BlhUHJpQf8BY+I5fJd8A+Z5xbdqAlv+Ch3T19LYJm7JPUdDjeq6ZNJMTSKwgXa2vXzB3FCAUp6rD4/SO2Q72/rG16pTefApfHD4fux3eXfOCNRTN6+BdM1QqT7iZPce3BcGRAqpYr517ZVH2/bXC2vH3Ab6iYcyfFFYBD9ge331Ag+UzEi5nl91tuQ3QapLSnTRcj34IXINI2tL4+ejHmd9fyzuqMN9UX4v8ZmZlW/d3RSo36h0nwSmJB8R6nmyVjvoG1dmw7reekCH+Xgb9SynNNLeKgg8/xtGzzL+Gke4MZI3+Jp5MlVpZgtY60fZrMUIWsgVtu97TAwlIouQmb8reXdSOWwfCyrv3jvq3oP2rGuxtc2kqzWgTlFODpO0dPsXvwZhodlw83xAfYvPjx4BBfKKFt8My9/9bUKobvhAKhcrLCQoIxaQNrPsn/nUJbPD+kphDxMagnxJx9f4UuSIFALgPbNbJyajy3tS16G/VxxUIIiyhQIktKuVbGfGF6sbjVzGDdZZkwejJDzPIyAYVLifZmFLyKFYeyVrOYrcEW9NS/VH2Rjs+XRbQjYqSAIE1rTKOvjBR6ngFBo7779k/S9Bc+kLg1hvh0/wMPJ6hGozgOGD3YUdTy374z7Okd6dnRnmXgjVDs7RNnW5+upuWFw1lQitN2sxLErRPuMkTd8tSOAHU0ZY8Q5Yutpy/O81A9bsV5FvLsgzCLHWEGITDTdTWVKjMK4GSzKOA1n9yFXcDDScgqPWC6/BcfM4xFVbgW5M8X/1BwuobLI55Vo26ShA0AjwqnAlI8HJQHko1jv+DE59MGd0vWpuyrNFud7twSdv3v/k7myRiFvPToYgKixil9SnrKBiFwB7jEsZ4ccbVBW+Bma1LhK0HISKIVOYV1vXPiHw+KGNZQi6fBBEmnVw14CbGoWiVRI5/u4RwBAniwgPoGTr9BB2gj7vg/OW/apAHIEEECjOk3jiN77N7ZQEeZ3ipROCTVdvUz8paQp9PXazMlg+tvGiEwh1m/Es7WHdNnCG+gHl5nQLvEftENa2e4cHginoKNYqDDyCpOgnb2ulCsEcY4pR479Dl9QYQi05RWa1ppv31wiRNolEVOJpA8go7HcR69ggUfFMvEeH+szSl2ARgSzQhRQkba90yx9nVoOwUvDuxGzUHSTFcOr4z5U+lsEhDqUPfvvzSLy/eJP6UOir28QdpVw+7GFGEbAOSDRwzaxtnwzW5PAq+XNy57Y+a2CyH+f82wZjnSWQljQrnx/RVjxiAH8a7QUczfVio+yzKYs1OgIJtWCTlKWAXX4v+4brNH8Jerwac3lxwPXh42vPf2Ay+/JcWO2RytKIGZ3SmGDkNNDmAUuWi9Gm7RmklhHDbs0H9homF65EArKppUvbtNs/SMFD9u+EtddrAP5epGr1lJDu3UuEYp/BXncHs0uaSrbF7js/0qmAXp/4aRfvqDE3c3Ac9Bt22ibCYXHeD//08Bl6weagnuEqNcNint+NWw4JE+oR/hdC4+AWjkfSENgOveVw1SxbS5Mmq8Jq5v13AUhB5t8zBzxv4QH+o4D8uIN5lSxnrt2BZFmdBvyQRoIdYDN7j0MvKZQde+04kIag02h8yLsU+aVO0LKSTSP2ZnzQKMQtYizJfIv35WnwYaYoHbY7uFrD8M+hXYsM46pSUY5rXbbpJWjZZB8tW80CYYJ+RqZfcAPHd2ADyaY9tunCrkbMwzQT0gCAcsMmdFfdPyyQTC8y99dUVrvyyw9Ec6KvPUEwnKAn/V5Cn/B9aYzIiohy7axdPE10H1u3nSzFrWrfuH7VPLwKmJJzb2qOxYOgIQpOxfxWIVBazO0cssGl50SVVENAutIWKGpjVtB0zS1Xl+mZCHfwBI+3U2Uw3TUZoMxZaKFyuusGYvQIMN9T7N34yyxr/jcmRocgmLKzg050kndCS62DdphoaUHQdEwvxE4diptoVh4bRovt6rWrwuGY4ijFRiq+jbH1YkBoDMq4AwoDZ+yA+pBFCfaQX8Ki+LcZWms+5FyAJHd4N1qULvq0NygVP5ckuEJNKUcD4QaGva1fLtgfLPOphGOTFly5OC+oDkjF2k+Y1i6jkOkNPXflAljRpM2c/glInh3R2QGGqnbMvZYWKeztFwwM3+L3r+140THf+Q8+ZTozBQCRPoDt3jlCviUyo5yh0gAf4+wwGhck6/s+Z5xqnhAKln1M8dwPeHyhR86JiQnrPTB2fykHwOlcbtgi49xQMX2t1kGEJzTQVfU4R+Ft84sjjEOIUlsNpA2JGUYJETLl/H5chy2ymEe+AyNTYFQzvJbA7z2syjtFRFxTQx5GJlFnqe60RbGkPybwxR5j31RChwcsoPuu/8Rw2lftyR5XgKn0V25gUuu0Pq4Hnl/6DkzNWvIXuZvVAB6od0ApaVKrHNKGMmDJH/YFr073nyA4DF9vlje9DhFfsIh9PHL8Xa9zAEnySmZy7/dBV6VEOIXLS3mVn+XQcvz7mV/00933I7DDGmihn0RD+ubKLh34X0aYSayeTXPF0XtyW/qdTADR2D9Wgzohxzn7PZ4qQtK6rP3T0Sn8PILAiOSTYWkS472WW3Gr1sbGhzJukc6TFbk9RAI4Ba7DFRiRB+wOE90vCp6F/kpuNGHZ1cuwENGpXTkBD8svpHOBSe9yztBuw6HCUECSreEFawzY0Ejb4xTtq7C7OBBLCe/hL36NBK+qQv15b/jIFmS4r99ESfGKm8v7UGIXszHQbEUHwx7FyiWrLZNeSd4ph67Sgg9QLr7fjHLVig4N9MEexNk9KC6Xk3lPseuuPdvG9LFNZMOSzwaEA4vsUxcbrsxWWBsFb8pH+DbUdeYUBZVeHjCF4wMAAo/f+9ryGNPfvs2f5Do9mDJd2CGnDJ3r2Z/GD2JWN9m6NV6/+mr8GCcqagSe+p1en47jMjVAvgVe1JBBIUlTRhSPtTiG59E4rJ2z6m0A4cLbCvRz+mneaXg/WSnTivf7WnSQYbkNlQ/DDyvGqz4H/t1IRboYH2c/LaHjFCSecmyIu7Hj3oinZi65JC9ki8+KTNxFb4biNHoHjW8dnFpD8e7zvlF6K/HyTujEhIv6cgtIsDsXzfymY0a83os9vACyTJWYUAA8+qpmTR3tE0fMxyM/UlSI4XJ/SB9Cmh4NY8RUFBai3oPnz3B1UeCM0nr4YXEQ7wANhUbYrrxRQRVwCELB6IPuBWb4HOd4aww4gAF8XlRFjCu8RcG3jVFtCGsOsxj13Eixo9cl4jB+8F5HBJxDypiSM5bcF1jNbq86/WPz9tVMNdstXOlr06VLDg8/+Y9vPZ9kecMzysV5G0vHvDJtnV5kb2iIb/i3YFsS77Eux246rLthhxT9oU/xwJXOGEokjGCAML3beN1P2/8qc0F2VIln+xioxiyv3HrSpbejmF7TBSNvRIs8tsJzWn4N8e8EyY+tSNVrYfDxH1f9jCr2dXCw+M7+TZdKhmLe9YnC4qrARJg26Luxce8IP1CuUJyYMGHLt1v+rB+D1Y3XFuSN/m1uGEP9cklpbIAuk10Y9kQC9e2B9DAnaYYsuwUxK4PQFB48RREF8AKfdVI/u2enmUxCst7KVUB+/fK+iCCO5I+9+JgOr2ac9Rl3zGq9PNy0KYJ8ZF0nUeitsLKxaJCuKqB25YzaI5ZqV9zsT7Gd8SlA8p8laick6iglmWjkA5F8jcOgRamtO8GNJuSqwFOonwfcxssClp/eWPrMHra3LWMp0HPHwhfi8A4rkULR79aCCUDz8lDoH/or5U2gmiBnIb5prTMrC15tgVbyUAw6+t09m+IMv35DLN59+VcH7C5EBYO4xiPRvC9MncZ9ux7Ce7nzidEZ05zyaZNU6x67vvN4aW6VnXjhVUaYQ4u7/ISkmPCefxGGGLQrF6xl0OMqMd3N3kS9aO1QQLWurNe2YhL+JrPb9kaxLWllmvGIvS5bbrzpTbvCZJDZSgU47e9E5Xa+e9+kE0Qrqe47/q+qHnG+yxhIcc5c8pxGYF8/dbjftZ53RY52KjB326XIfsKjk1QZYqYETnoznk47M6xyN5ZUBJEQaEV9q5XqeETof5Fz1d7gBjMQ0Y1AKd1Dqk6FRm8QRebcSo+giO2hBaSwsetO9PVUa8YP7ZLEHKDR0ZGUP4038kZakglljr5dh2w3FLc3vTXgNpl1iUsm/XwrZ8eZSmG9IHinjRNcDhsHJeF8y44vXJ6fSWs9LmNDiMGBDhzEiCl8t0Fgxq8NG6GF1CFOIZ6fW7z3eGBs20CuS3ENUkqVDSkEaebZKDC8T67LxEr4nObmjj6DbRzxxsCSPjXeGVE/lIlH8Xdytqccztv3wiKf3jKWZJ/YcWHC2QXKdPbvrMgHzUL6bE1L6LRVCbEC05RDi6bAMFZruPFgI0S4gNS1UGF6PBAJtuJ8uhWefmOEh1qS4PMOVCoI8Jw0JHMYRXM6/n4ZaDmad+F5gl585WhCdkKlTWKOzRND5QhcVeEN39tpSzNq7v5PCJ1soxgkFepA/0VbhA0rUNKjXV41NYKX3egDCATkpWKKZRhKAwGrDxEnKQVjqb2H8G98U37BaCxqEuC0z4gu7Z3hGluzYIEDUytfI7BG69iSkFlwu6fSSgxv7ZaoFEt1l9kWGOP/Qn0T+FSSDAxUnAn+v35uh2S3ioY9c6eJfk2K1A+aZBU8h6q/Ffq1E6D9/KbHqpdRseYDYjeoMPaCSBHhnW0OipQTkT2TpJ7n5i3NlzGZ8f7iURnSi4NuZYuZWEyFQjvjiBmDWA9trhCkXkExshjPwxT+2J+C7USyLbpS9wMB7BCGwXNEax180Am2KsyZawOBGSgq9qK4bd+bxO/8ggVlJOoHXx/hC9LgD1m3jY7bE++WZSyHEjGDs0EclxfmvA7bJzvqMAK15SXwugO2Dvwb26pvStXNj1eWu/MKNO5aI9vYJVB/qyFv7t97TfCnV9dnODwfJe6dEIwJzqOehVE3S4pXQAsHo2wJlz3b205sOkIaL3ctGsAdLCcrsfaM0585baMQVt+l/D2feHC1W+0dWyoWCcnHJ389+EoOcVZGSsZ7hcjiS7V3effRHwpee8RYcO8TVL6oQnihSvlnsSfkXkX6xYdX2gPZTbCKidcESA7blizurC+wB5VVpZbD0IWEcsita4h7iZWxNHVmcwX+10JzMFE8IQo6PMI3VqQUJSiLF7bPK5mU/53UBcNkjfW35BaEeKndvR/hasXIQuJgWJXrhAYoeG/xRkGXr4fuVBfzPnNXKsF+HJWLhtDv0FIuzLPVIjLcAZ8CxxkL8cuVPJxTPF1FlcDh6YKSS8DILP6t902/xA4UwuK8fjnpmLcKIK/du8vuqapB5VEuleJ+2aA+JA4W5cBvQQi+42vPSLVywUs+rRaH29W8pXycI0l8or0cTUw6ciwuRb8SsECEnmo2SQiII7Nj4eL8WcPeSYPY2pDvTIJ6pgl7WE/I7ZPQGRoR98uVpsbRbIT9V6vn9NrA9EK+86IYC57vrh2zgWVCyOMQxNLAX3VM5jaAwgr7cXXVIJ8novs0yy/3Q1sFXztv+gmCIHz9S734taOv+TeneN+O5jvMomMORHl36qXZn1j4xkWqLzHaQlg23waxg5CM5Hj7u3L2BGTTWKsBjYHLRHpSw/v6gGbDNqdU2JIm5FCkf2yXKfSsOJFfQROP+cWO28DmIjW1wqUTedQ6BhC+CT2yAQqjm4jPor2qSZvZ5FEX/6qLWn6YU5Nb6xC6fUq2Pas7cOA0PiBdmiCJSDQRsLEVrjMoBR0MCiq5UnVOoVlgJbPYSrnJUddE94j2ELV/oh5ZvkJ9MVnBLWaWQBoYj8u09Gn610k5wOc7/d7esV4OtRFWwaseLr9bcwrsJAKGlf94dnrSLT8qLitju2Wx+cAvrOff5vuG1NPKbw/bRqFc/PYu/aBbK7AGB2jXX1rPcfVn4PeDYYvHPQdQMbYa5K0j7Mcd+rjrYwzK+DEgqerYqHQI2xE9z95U4mCSNRT/KErzx/4WqXtm6EUorY2e8l6kHZQxIpMpykpei5LTApYkdSJPkluur22CBZSDggvyePuzyU2jihLSU58A3WQRtXHIAOEVfvF7qUDd6CNyxbUlFCWyYBlk8P5Qvq9QfK1uTmqsj+IvB2c/rnVuZQY3w3Hyv+X3jtRBQ+stQ1f/ljF8LQ3clz20i0oUT9qtpF91dmVWOfqspMDZL/LCMbIaNM5z94wojKlp1/UU7FhkW1T6vv+zfcBYDYxJyoSijP9vGe1Dn8ZbiqXpagltsmwt4OouUYpsxFDY7asINm6KComBrFdwR5SdvvQxTEGO6GoDv9aOc3Cn2OtUwGftv3oxrcA+P3nVNRW9XEEvbYHkgqTodizu2CU+rGWF6I3XSkNXFpwe8RnvvlZy6CST09qkKp82I+gmrNrhmZAFNYnNK0thh+GE8ibD4h394ht6xpfvs3u9pCB4LzlUPxIVfhe13Ayd1McPaFVEkdi9s+UKCv1yZ/v8MKGJB8GVxNIboGCoGZcJ3VoLLbrHV2ELl2xDeACNRlTzRO54NIpNEco69gEjaci36cvtIRmRVaxS/PQ1kgY9kC4XqUquYz4pV2s15RJewuw0TkU5PgIAak8otpjLtZcaLudmy3RFYQX7PkxAyzCbZppVZYLX4eC2VsBVyDGkdy95LkNMOeY+hEd2OXJCEIDRW7LKfwQwp4CvPuBjxFeJnRFjgBN4lK5C26wFpEB9b41ctb77a6G9YWK4OOI2/L2jsXXtXNUjNlD0wDP7z3OAJ+A8njww2zSbqrq/fMVPNIjyKECBUh3/nXdTt93a7hqFDrBmVi+Q6CRW9xs6ocgcy4nefW8ATC5O1MSItSMvnKEBfJQdLbBDtqYLVnTdqN2h9NS5+cBLtZchsTVnIL9ZRIrlMRAqvdOq+MMYQdZqfWBGuvaKp001/VmOch5i18suZPky+MxWHtejDTGGH7iSd8BL7mAw9SAy/ltuUL72P8QmEgVTCYyIcHDse+VR19AyuS6Ehm1eEKgDwKjhNbaJgVEW/n82jmv2jYGEic650bjtUAkFe8xbT6XA/TNN1B+yXUCp6J80o6BoCYmbkqHthKq4oMAmNbMEA+ZH9FhYa9ev7PCEY2yKzRjmU5HM3Us9zPdU070LQF3Mqkexk8BzaK3RTXXNOGwWx0ctHCc70WMzMjo2/h2rKprVmJc8/JYR9UzoqCTmeJurlkFlG3tdXxmBfo9PHZsm6Ai7ygJAZ3h7Pqzryy4HxRhBZTvzXea1BBbSRMCs2fIattDbi0DTVREW9ejquEmcErvYZbLcCRU/f5+szuiastELxTUsGdOCL0CRrOFinN/iyB0jac1U60u7u0N2oIVRrEHMf99hGS8fzli8lTA6s30MT6xTufucAw1gSJvkYJHaZvlNGDuJp46cdgUn85XUOX2opwcYF8bode3VFsSvtZREJRlcun5pBjMl2058tzTLkj7hUSStNZbOuzvJJgarJx8/BpGzjrwYP06oimt38f1jOV5FbH+5rEvl8Ty9bokkdXVbzeMdnIwgRXZME3PjYF/jSRyOAcrqrosJMxcEWX68dRqPAGA+OggheCFyal1Roh7SOb5Mhvf+2bC5ETX/UWRPqO1xPoyHl/mnYPYnXa1Kn1CbS99Ms73HN/kOb4vfBwrS1PUF7egy6OSdAqyMwLjXIg29GpDbp5v8X4icTF291ot9a9GTy0+8bdHxAu7Ubgy6kqS65uod4G99xD8ZMqXrz+aqx75h7bSpqOIcZQQC4589Y2sy9YahoH0jb5Ee29qFC2lYch7MvBbmkd9z8H5bGT6JvbyYaQbf7EfCeFcSvKFG12pWYt+QyH9X123gIfmDEpORC5Mj/Q2lvAx1nhE0RRWAA+vE1P2cc0gcB7sPWLNgDz+tpmc9IOj1M8F3qA+XmTIpBUm3fzuAPCmnvx8xgRux2CrQgjp0BkGgh2mcDzlzpszpc2jbg9y/s08LCNxN7S6J137jcAyfL61q8V2V15j/BlO3UW6x05ppINnvuWDBTe7bdXg6ocCxuZwODncuKG3LjWGfvPt/iExHybXgt3SqlXvW2YLskbQQ1rLd1hZM7gmo0dqYdBMoYWnV/Kuw2L/vuWSMYONZgKnUgmL5/NFXzx2IGLIHiWLkjuVpcPlXH/dWY0g6zOWpZ/Pd5lx2ncYKCs3RT1458OfFqGBdXGh6JZJMVm5826mOyocWf5l36e25Z34wZh8W8nqlNkeqL6LykiVuznWaMx1fl1nFDQmW98QrizUB/3CJfgXlWHDJdMVIrPK9IL+rKbLCJwah3GMCt8ueJTxu1xtzj8USFS/gIJEvx6azErxHcfgm2ZpmFrXUuI7BPPbiXSpW2vIXh8/aptY5ceZUM0ewodvuABwEZL6pgLKI0ravvJ0ueXzXVVugbx9Nr9IjMYfpDh8/Rot3YTF7GbF7PmL3U3Z3AUiv6FbxJL3IyfJgiE5D0XhKoUDCX0OS56WXr/8bk/o33YIQy+qJHRpfJbmgPILKizMrvfTj294h6FCeQeRwof7dOwHCc6O49TZVEJnN+8Qx2yeeGhyMuC0R3uHCn0ZYFBPdQEVvr9GzXHyBUTnzpzrQ+JEOebWjTyhAG/VNS6P370nbcD0DsggzY8r8VXSaHrkbl3acwcs4iYwGk+k3q4jtA/au76vuSID/0zz8kVdkO10Vho5f/dSA6rBPqQUMkS63eGP2FO/TXeF3RH5dsKdF+gAKZknj85DkVKaDy8rvB8zSu+HMX6QDPV65ihz0eE6EiU7ylzIrmP90xFCMSCjyPvk0BGqSSOC4xMeEoDdXJjJZnDqyspO0ZJn1JkY0/LZJNHmAxq5hoOp53Lj9+fk3/ZxSdmNNWxbADyu7+hktMMa1qOjCJuNjRBATc2uKVtnuUQtazvprl/rpWNUf/SBliSFMhkkeUL31zO8pN+LSauxCt5RuTK0BjWgcLAvlBjCbr/yfvE+qdUf1K/hcSdy6tdJ8lvdYWYD6cXA/bx7Pjs9pozhsCoZJwydsI9yuZP9Y9XZ6Zqyhe3j82IySXVfAoZ6Aiq/C+3qGoPpaiGq0gOLwAxylrQflwCibCz7rToLma+pdcZI5XKp/lDxQb05DS/M3nZacDkQrf4bEKADVT/rxyA+PpJ7g6mLgp9dgSt/PhFV5jmA7hJt4TgbPdx8RK1dRxo2+apGxTgvRGr7FeG3BHxrc75DUHhDlGeqPn6iOqFxWiS8Z1yU50I5GiwWObrwI3YJTbXnZHw6BcTWqH7/8sZUIgaje1aDcoN/QNJNmBiZxi87gJr6TpBvCmIMi0Oat4tHQsmpIOOcLo/5wTZt9gtfQYgSJrtBEhmwiOOhwa8PWweP3og8NkxLYs7B4oq3YbsvWNqNr+WyQURqxQkqoFnmcaqvL1k2kcep+TvD8pqAVCNnVOyV1O+CNtJdEBd5n9ULdaDDrvvd1FhcA4vs5Tej0fU42wREErvLcIIhxqw4xG5VGv1bO8EaZTDQkB3tqydaf/rlPRUuCAJlgVz/E7PbuTq/SOm81D3iElI2e/fbPpmjTRnwsiEsMeLw3S59J6ZVcCSNage93Zy9uYqXNF5T8yVGe5yzlzPTpAgZ8g99N4TghKWPLiR/R5V+bffw1R5QLE39FDIM/7BzYqISGuGh4ytkpo9dRlZZA5yTpUQ7M4kefUvVZr0PuQc1U+KzFzW/8qKwKpBbslcSmB+5/8VTRRDZo7cz//yEK6Hmd9J+PAL9wEPCIdkZXwZDxTLCPWiiW/R5RIAUdOU2EP6QfSF7n2oBfV9d2wvBbUifxRvqar+6Y2LWiIM9x5T5d53c6ObJoorINoioE8rrLCvB9ZGk+BIMKTllQsl0sve5ZAgK06jQnSN0/n0vUavv+7BSFim9H+A3P/Q4IosY/AcxuSZI8uIyYl6e7j8r1x4n6R+FrpfpEiNxEgdVQjT0pTBwHlsv6VxU0foQWKuuoWTWpAeb4kv5Fp/lABVNFao4/KN1jCD7wtn+Ip41vR1CFMcJkiD66+W47EHBkJ0sWpBr7cRMe9gpIKWGJKj3Hkdr+rAEVN/zywzbbmtoD1eJaP9NvS3ZkSLxjoli9vNNH1uq0Wux3T/6e7gLiuTE87qIaCtBoCtj3RMNlxEwd2MvdaltHp3YLT7A6CuWQca0il7F564HnQR+wCZAW6o4nWgZ9jAtxUn00JdendzfzrpCrPg3JHjTJr/Ivp8JvfRDUD4Vza7NOoZ/ml2EL0g62sJMNd31vqTO7lnJ5RGKXYxKXkuyRqN1/TQToyXop4Ttj7rt+snvikIHAND2p5uew7LXziWjzpXbDnjvH7s3OwGrAcSTcZNGaYATm+JeOMoYtmA5rVWMg5plGEdCNj1gfi2xmlf9VoA4PJg33nEnvVyBaQ7XDvWPECYNFCkv2V12xDwvN1US6XVsyjS1miKxsWfa3MY2F4ANPbls2xQx0ufVxYdU7liOiw9T6RQOkomNW69zfz/Ug1ouvr+BA8CYb4M2qu4iiLz3qQaz708aHWuJLL4C0BPlEraOukWpoNlrS8vKfXi8J3dZ/ELXPLRIc7xzhvFWLNtxmMVm7pyOaWxGVfOS93ctct4eybpCIPy3VB4yxYe3tFWZ7TUW1TZ5VmQ3LmuE0cqRVlamypTh9vSpMXy62gPwVMmYEKU9G4QpqvO7lnbyAXTbisqtvO874na26PLfHTJAuUhHpyHs/2oXCI9VhXeZx/gxOVgl7Q5FdZmp34QMsv3Brdc3szCs8EBhKmUiEJRE+Gbj6FpWIvM30J9TfUr/FRkS3SJ2Q+1Kg7bxP6xh7q2UIqLJch332/AO/A69nGGUEa8XB76ODPIdinnP8t6lTz+MeHyvSHNGv+bWQnenlOCyqQ3sNQuSo2KCVikR5mhgZOhzr8XqADUxi3896GA/xBSJQcRa5aWpHQDRoiSUwicph6pzeZmfyYVgkXMhVhFbPIDTCMRnN2jHsHto+vYlV1tJ79gHTtBkqZo37f2m2LFYo9N7rRzrba/72wGU5bHLhbf/GuS8MXNaf6chTvMpGZEBnX64h+RHBYidUdI1QDY4K9j4dad2RcdIkhH+HKR9J3uQpkdpCmlmnCUbklfTnd8lBYF2sar/Tbg7AnVa8dOgY6fifhM0GRU2E+y4ywL5caiLIlg3TZ1lGzivU0H9BT1xD8tb7JgrXEf8jWrr+0BEuwcwpHHp3zllVoh0RXmzfeodqkCFJtud1O7issu//Ru7NxRt/H9NpYKJYjtX6Uljy6s50eovMO9GhtwZ7rLScn1bqjJghV4hqK3wFaIaHMirBaxe4yuLrRmQO8xQEApuSS7zMxids1AszD2XrC6fEd3fvwZWzBq4mlcpfA+P207JLPCygw7TPhorPnWfYs+AVezf8K1WAgFm0UdZOP/6Pc2HbljXuGdHNUjukfrdPNjCDxaDgvFXIsEkSR8ZwHpL8s2mgli2Ym5ATkRczzcSXa90+K0LQcVwkB5yMLRXZ/NG3g9XqrhBxKlpuu/dZb1GeHnzDOEy5ytpE3QQTx7x8ejI0bHVj0fYgXyov7kA8i2cS4Tac96ACEh10qWBJsmmNgQ+Q3IQ1uj9qcv7vlgTIzZ6T9Z5YUMm+JYORx2r5qnt1OMX8igMTZqYCO5kMTEUfWXSd5ZBdGnKIp8CofX0y86gWEgsFsVRbx7up5331BIYwYcngOPK3E8RWCXN/BZsVgjm+CFNWDNQG/qDzGaVICUNKPCvsdW/CrItu8cYNK95AAM0Vt5Rcut4fOH2F4CnoohL12y//aA1jhJMARVBMYtY9Q8F9F+dkZxrZaEk3fwUA2bPSPm+MW3kofgbH5yvzy5pTbWxtedLtORvK4z0mkiqPh6mCaglr24d5E43nwkDy8pI774qiOQmbhbGrK5Dcu5pZEs7AGwurHA+eCP96U8KKty/cEM1lhfBtN/7y1ZyoecQTPc5KMU5t3rsA1iDW0Fi5ubq20OzAziGP9Ydw32nt8MMBgOy7a8m09g/dPBhQ0hmrZtTF3EUWp7KCeOnKzda1Fztrx2poNKbMM6XTJI7tHKfbG2CsN3rRVZxIx6cvWWxg3Ht1VmQgIchSkcQXp4HJ5D1x1SffEFHwoy0DDuTBMIEQGxphAS+6eH3ybQUDVJWlUJrF6TQvBtvXm66X19a4cJwRCZ4RIbgUkSNItZnKhvVZ+LiJ7rXqZdI4DZ6W2IavCJb1u5pDTThCnv0uNuqhzC0xb4T1gdh8vj3BV5+4+T2wqx0ZEtA7gV6ycvKBsBG6n4D6m84RoCCX7kPbPRAcMa1oRWSsg5yZ6v2JRkfuGzdklDDFsjn82fEezVfH5rZCe2c3+biGhvN1LOpuWv10D+N4sWBcPaDtfyKqvUhqklWf94r51TEtplg6qzvA4wAfMq3xhnNzxaPFt5uBiN4rG7sO/0Ju0F5GXEquqb31lkfGC5uEUOZzIvfshiAvGWftrsbIoz9qOyjc9uaSR+43cep6Ud2Enl7HYP37j7okY992reYtmFU9zFhI1l2VMqxuySQR2aLb2iHX2UYWNSmG2A7HwcntQMNKWUDyZjdv83T82OqBgoi+SGkufjfSAmo3l/RbOc5jMLpgw7Ik7RMUHIIOgPFQAzddn1u/CXi212yImO4gk69AfSNoY6Qu1+eupdTrSTjd78Au3CsdkTFuBpyG7JHvsIj6WPAmUE188QtVwWSK9MTYLFEIdd9oVOvIrPSU3mAHDfacOcD5Si04IovYEHgnJ0ccaqqczgsN3CHFBo6N/qacMe9zf+Mgc0gnKtbvLhrCV4XgGQI+M4qLxPenFB02YigMfhdXPzKXon8PJEBf4OgsPrXFWj1uijpliuTMK69HvhKpxJX9RVBlQVpOTvKmaU7jByyhK+9qr7FfC3r9C1wzsTgbx1B/vrFSFnLH87ST4pE8rl8xdCuEI5YHDXdTzpdUqEXWrL5jrVmFh8cloKUM28AXE3KfloVr//NmPb6RJalZjSVma7fBG6woljprkV0LkpyvcCFIR95FJ+ZG6oWonHVLcmEqcuoYP77ISu1tk15lMtktyaoKffgOLZfBexGWMNAs0GLz58HXN0xAHR8MseAgH4fi8kITIL+9P2dSgw/seNX2GFANapO/HqB3rR8AvaViZehkENyCF8mcSPbtfdO+Tke6Bdt4km8klP/YUpMoEE+iV3CCrGmXp2V8wYgAxTgUCl0PN4R7lknZd2y5gMSaQLXn6xo9ArXv0msJOzYogdjc56vqnWQxIyBAewMNX15AOgzd3RUITZ9fad04E5FQI2NOL+BakPsO8b3LlFo1RFOiptu5Mtze2wt9tJYAfGrmP0NhntBwfG+tEnb3kncV/DP/rJMHenj9iBJ96281LVNjPdEtFuOYDBtJLSuEQOP5dSAhYSptObnsUpojQIFL+BciYZMdZiLlxPpEAT/Ma4pUzVLL9zteLbPuaQptdkJNFsOjdnyUfdl4moorGIQOsi2ed5MCSwH2wZwzjfCKXFYNdcCnX2ncgM1DuwOy8FAoqyQ36b2Ex7d994dUUTX+TCILGe//iB9MqCkTEn2nGfErBOAeAHIK+bfS7Ioa9LmDYpl+ay432KGLfm8C/23WowaZuJV7fyLkR/chAYv56X60fqbpIIDzcuZVBdRgoAhWqTl5DfSx6eDg/SoV/QimMh1HILkvaK4vCg1D/eLfWXHgjrqtzUL+oR1aL80jS9RwrCG6upJkvxRfhADJpQBDg0U0LzxZfOVtV/vETiqkThfLMF4oVrGNl96b/PdMEhyFKTdnQvMm5+OKx8iBDPL+RVAfRCQ4ZXGaKvaB91buuHF2JCfo2T67tCFsvf7jQNi4eCniJCALeAQf8L6rgDAwaSidw/5cRhxjdsSKIpAsq7wBcTmUACORAHKzM9co5aUqWKCdhNogOHvCxHfs4y0aqK0llfa4RCGK3NkGT+qebVlr8lBkBrRdLkgP9JvWMidtG481W+Sk9xPKQz0hYC5TeIKXmEr8uVKPgqmh8vNIaYjtM0pWS0YnYLL7bQR6+t58ESbmQJOt7KjAIwoXSsOvreFeCr7lpYailmZaxcZGuLaZsgCi2ftdMMMHgZ7VltSE/CPOCJ2dyxIW6ypE8mtNhRL0J+/PZgF89n2ygAJWYsA4PV6fNJoi6bkTWz28oeVkj/UplLDmKF27bREO2fHMKuVEXA1EZTWTZMzEhBmPY7TPn8MpAQExeSTaNLZRvOTXdJjEpK46bz2YLqIcpI9yJcka/xnJ42HCthZhcmhB6eyrr8XfSK4hX7fmmwGPGc/B7dDp3dHOmrqupjksrNCOIp4oF1Kbdo8hlucel3QnFTW7CS+JIfwBaW2ZPsQ8gbehx7ytBD7tt53QlbR6ksbS4s6DmfFPcBG60peiuSRrysmeT9QmIkZ5Pvh91YYzl/yxXZc9w3LMJOSe11hTL4gSmMVTEvP8wsd3PNEC5JONgkcLq7bJLUCDPxbM/txFEaTjop7JcV/abquZUmRHfhL0PhHvPfQDbzhvfd8/aXO7I2N2FgzQVOFSspMqaR8D4lmOYCe1hwLimnAfTIkZ3/z63p9DpinJchEqE0kzI4zC00tNgsT5UR1pTfXN8L1pEAOLETLX4qT0WsAzipJmuyAn/MbGYjnziA6G5ZkRdmvaG3BO8xE/pfKOU/8oC67RRmXra7Af40v9OLMVXLstmbn9Mn3jfrcPTc2IoIfM3kcIz7GU58bBEbJzJmi/XNV7XKI8MMmlwzaTEcIzC0TAiIJKEfxW/PDuuoQNBQV10V2hh+rGAcJZG/IelDWjwTEMv+vDHUuLiuJwGcUwF7z3zHY22MVRolaiMk8At1aCNR7iRpF+xBWn1au2c7B/J6P7SZIOFoWNQsgnf2XPbpl1vC+Zs8+ve0UszHe3MtRxTn6ax5izRPoaSaRw0NWbfRSYJD081ThfnldbvnAbHmiD6PfDVARBV6T+h2V1Nir9sve1zqeg4iRcLeE8nWmGzuqtPcU6wWKVwFWK0lgSga4ZIEnB6yc7XmtxAB4G8dcxE50KlKIkXTuEowSY4Cgcn29uL4R9zyrwtiHfo3+c6RflQmz8ZOebbWoadLeOEb5gSB7pBph9BfU4zEBJ5q/3dF/0E7Z0khu990fk0HgLHnIqM8Za5L/VsvtZmPLi487ed2CuGXTdmBdyQz4jUlS1Y4EyDlEWF9+kZ/8EksGwzlA7W5tQ729s/f9ZLxAPIIpJpQAIAP0F5voR5bMdRAU/jgv8q8j6okgsghQHtv/V7/GrbLVpNB8W/pJCGBwAge++oQaFIZF13Zj0MuVgShxM6AuqdPrNp7HF82Brik/Cc2/En3aBrAd8983NnrLGnrrJeWpw0M4xJkW+MlovggCHaCoxKc+rRz4M6z4n5ADUNnfuEeKlIixkV2H0VzCbLa/5zHE4LdKapfms3Jk30Pf50eezXfA8r+Y+IE6sDTEzP8liBHq0e9IF1+U5h/LDyfyMGWbQ5EI7F9De8b+DhR50JbWkWcZUnlT/dIfma5DgOe/e4cMxsyeMW/IhOR6Chc+8HC3xcH/voreVXLzng+er2ebt3KMWadept84CiuHL3sgr6DaHwsp9AzcxHmtxwCMWh0z3eOV+IW1+0GMhPinFi0B9HfdBwRnNNXOMQg57ZfLT/aZkYsLbGbR8/0kv77ssFv7J6lSXtZppLSDXcOZSNLt9heGZ8nTvuemE3AxL3n2rV1V5Ef/IADzykJYTKUrLnWmIDNSHe6FGy0c/k68empyiVn09MayqnbhZVXty1Hma7+IzaR+GhANgBraMw00mw581RoN78XnMS2RSuuzYRSTAYexBk6aGfID/Rs+ndR2EwdN1TpyYKZWeMPEmh/MBMufohCWoUlj1HBzo1y/f/fgNGbJOAMekkYRtSxC1iuHQxHaVHdUW+qg8j8piPkaNnEcS55ZfzcWjuQj2GvQKLPDEqgex7n1cLx1QuuxkHo2IHM4uu39hi/g9A86W561Z86HtpEAtuarLQqt9BICh5X8WcGxywg/2VbNnoLF/bp4O8n2GuPGKTGwOqkhek0rub++pvVaL8T137zlC3UotWq9EDpzyUzXnAFe35j8hfVmXjfr/HYFCxNjv1IVbzYoRrtp8RplzC8FnKAJsy0Rof/XznrZnVuoEoCOEfW5Ajd06ZVqEIx38d0OvOwHlIuTKIE+x5hJ9pEsDHyj4WHipLPup/zONvCTf1NxC9pvaONDo2SU5AY2746r6kNdm9EoBg5jDw1kYfG6lxsE+gUzyXjs9PzyjTBUcLlk7+Xzr52Z8FdTCKKw0xLZ9XVZTsf0KNSFYOjx16q0fMSVg/NTCkGD75FUOolIqesdCMklxA1XqeYb04ql64+1A5n0ClbvUXXjnvPvTq881hp0UqPaj8F0C51iCasQ+1fE9nxV/xx6DBtlwVq3ne4C6Wk7rGoBzlg5Ql04IS8+wEiOAJZ6t2lUZ8NfwZVHpbm4Zcu48F9S//UW95AfjjpVsCr0/b7YnvZclL6ccj2faEgJhnRaPQIXboXtKI4QR3M+J3LcYHgICQEUNznagYQgLIpozce1UsIZmkcy3QOkiazO7ZYHTMZ00JQZPY/GRtH1uxfcCjbs2dCOPGfe/9xD+PpN8nrvH8s/VnndPLhWINgzNJF4rp8/LcXzXOilNVMhzZLfTXUULMEP1uU/QcJE+gM1i6GMAW6qw5/OkY93fcThhVu8t+WX4a15kQe4AcHNA2jp1P2uhqYfyPKoyIDJDzhBsV4JWplwTnhiqW5zTsIbVze98Uwv+xuWF7ZhNikrnFzNqZAZy198Z2rpzfMYiz8lgzAeS9ynxyBC2NSu0YPXVHatIWhp8h7N1hyEpQxQrZMk65S2Iue8G39BUmBoVWpVbvAVpaN0rHBbIHqCsK//eGzcfAUNtvwsX7YFsej5b/RwkxpDcNbefETzOkkm+TAYIjT4TaxV3r1h2rGaHGmp4l0vZATyE0HqB9QfPbNsVC44ETjaZihKIsZXHSqqKE+18RkYviijSswb9b7RGO94L9YaA0XFT2+M82bTEEmyivhv8vSBMJ3jw6z+udqkSa6yQ9EI+D5qhHeTI6sv027fI9/Idg2Pj0hDnv8CkhV1Rp1ow/QoolxgIawlxgNqTtfhn/A4rf9q9pVnthK0NdNkOOLHoz5qnRseg0J4aO4kVspdmYZxLC73aEjTONY+mVtAnzOkEFflsdqsyZmtC+jjdf0bXkrM1245CM3fdX7+wdHhBesBb+Oe63oYPvwliRuMr0j+TxM2PZRg1md3fk5CyTz9y91SRoiHMuMHrWX7/rzuMtuqhFiW0tn7qq4HgmOVJ+L4gEvh7zNRiMrytKWDOlzhLNZ6fSOQI4j0Xzvd90dFWZ40zcW46wozGTnQl6y01vObpAiBjEaGma4L5LW95hDqlNWps26sHBU0Xo9+dCWVRBK44nW9Phgy9RP6g+SMf1VeIr08CJfYz2wL3aT5ixa3JXn1MtwShTN6MRMWZSfdfgK7b+hPVRiBGO/u69Cch+dvahQqAodUbdWsDUvJSXrGYah/8jvz+Q80Ub+bYPd4e/YPzH8wk85AeEkawKMpyrZukx2ezw4NHoI4ZX2vAYd60KNJZ4+S8PAn6KP5Qctmm3tg0wXls55zvAF5VRKu0YIS5G8Qq26fbocf0idfBxlKlq7nJtMLkz1NRR2cqhJ9HS7Ivw5InbxcBCXEDauKHI0+lrTtdzltNvSMBY+hmJjIOsBKrdb29zn+NGw2Qe4WJzHNk78KtevCXXKa8xUj4CCfxD8/VIjNZWgcuDL6jC2sifDJl/pIP71uFp7874aOwLeu3sl2f1ebWBiBr+trPjjLKSJLeUCKUHxlkUXrOWQ4bL5BXeizI1yNN9v5+sy5Z1whK+ikKj1axg9N3BPjo4+yTZPnn6BS4STcQ+iOgzWzWlC6ckm3ZvRAPbHNPGDHL0VbxIlblojpIH9/3PbLK7z8pTeFcVTxI53JDSHOY5IMKg5+rTCY3pZiGO/QNZ36ceX6WsIqUGLY2/kuvkJ+mvHSAiwN51j9g3bTrUkNmQ1s3bkyR7io8aTwrgntX1lXkad2q2h1S1lmChsHyLctJ2VZh1+DQZRfFsV/szoM97WQg8fUAmXZ7aR6PE/w99+4UZRZTRrews8YuytTZ+ZXkaW4opUStiP0s9sGkN6DCmsfBdOqBwNpPWme18xYRzEr0DsZqdx8fcgNhJzTATiRZfwoHc7baJBTeSnlp1rlz89jG1CjXVyh/Zds0wyhnoMYaYK/qTaYVnu8eYYrmdjI7WuJkckFcBYF7CbNp9ooOwMuJVHcQFFXLO/5ACDw/XNcUVU8fXhsFYm/8QLRPmMjUcRQTq5djzpioV5MOh3lLGPEVCy0IXyHJe+KykcxvCzUek8fVmO3Nd0S0k4bclNmJVYtyw7bH6b/IAsvZadLGqWu17Tuybww3XOLxaQiz4u1O470EOeLFBu+mMnt2/pe2B/qWlTDdmpibnFVj3vFG2F4FOeLYJT0543P4PCp1rzg/GjlMAgc1FVYjSLtT/z1K36ufzNfM+mv2msqD1bjLL/YL+Wbj+F/Y2vR/zoaxeLLFIJCH6Yvg1HMDG5qMrVqS268qn91mQ//NxMJkFkqQNUnlLKNIGaIjqLR5XXR3PvkzP+mDEyaLf4NIolfZpJ7pPU7tDRHlmZaID4EKSsd3KkHpPavNdPSIWc11WLqEeq9DcCf2FIsSJb1GolY5mbXaahFaIEG651D/jCQU5RK55s5m8yCkyznB4VT82ciFwz6a7nlvCEM+uuHU1sHmdGHqD8SbYiiz/j8TMtWRehMCDTg/dyrbbil/sV8I9w0Y/QCy6B1i8wjfCrkHHDlB/LjwRk0n+vWwXj/QQc1XTrs+NZXrxzBD3i4cyror1J5lGFfIFWYpi8uRuoLhv/I4pRRhdcLKOXnyvBRypaFyRQxpOokM0ZJturTV9reK02adWhd82yjxQsUlHmtKWwH/V29CkXZiNZHPTNLgDAdz8rZa019oglygEUgfndqPQPNJD0A/xW9LKhniVNWbC1tTCtbAkQLVLAkUeENb0xh8u4xibg28soPOIHY+7OKjRsA3kKM4BNZN5pm9AveSAsBjmxs8GqP7fBxZvhZ8dZbiC27CPdwvK/4WCY0zEUlUkUHDc5XCKKSuBcC/WY/vWdk7PlwUHS8HjMegwTDkMQENxs+tB/SXmk2wGy+hRwuPUix+dUiwVsuzi8UUg7no2fXjhR17pJSQgdSqE6zE8eXL3WBaRvkKVMLfPAnx332MhrVoTcH02WiJa7VYJ3wPt1s/afvv0r1yt8Rw55pMo0pjsiVcs2ymuxUfcR9OdEr6JEqO9YzSscQkZKnikoosi/fzbHL1fbOHeXH6OHICTpBAvG6E9xvmQcf6BgodJ59Wwnt5eT8Ox/+qs9bgwOTL9dcYYWYI2Nrksctqp6wuIDzzg8FvquPDJueJIsgZ4tJzroXVADxzuP8XebBjOSxmNPYvs39NxOOXu+ct8zkbzBUy6bmEABbzSdpVc5y5R6Vxbez+LtPTj/Kd8mFRa3C3lu/46XUsY2GkYrjRzkk3ISXU4Oxl6bFpTyNUPZFRomLzMylmk/RFameWCBB9ALRdU7nyHrjMh81TOYZS24jPrmztk5fg771Dt2y73nY0nIvyjBcp5dNYhc7xXE6vrtdVvkXYhgQ0x1v99lJuChUCvyZuyZy5mQoPs+JOAZXIdnItDe4xeNK/kHp1235RkN/uYWW5R3KvATwIqY9Nyd8nVkm+Fp3ByXrdixdot2K5/Whccvp50m5fi5V0tdn9oy/zrYh8rWjMIjT5jrFhtarmADny8yW4V/q4z9qC7F6In1qYaiqQwzaNmpuBCgybOM2+K4hLQT3mrRcGdSEx9Xh4ocv61IdxYMAn5X5nL/1XNmN/hIU9VL3v/EkQLhrpQf9tsc1NArMcDpVmI1ub7ZykkIZNVPkOxABsZ8ENSCqqC0kQGwssMtlMh5Wp//uryZy2N9Aszt6haJe/ksfPk9zFYF1+F8raGbgElu4pBdaNLk+1V9ag4dSq8tbgyEIprS7thW77bVz/wmwwUtwbi5mkNiYnJUpyabded0ZbXTA9whFXb5oW2rPXKJr0XUc8zaGj/6XYwNRIVkiqfla9iT7cqkuz1Qb6o1ltd16zmTyizqaXq/Ipu2ypWxgAE5xEkahVZ43n78uI5s3Wm15Ux2nJa/l/yV2i3AXEbL2l5ueNNnrreKNonQ6fnk6JM59/9jCshD1Yhju73qBAu0z1blpqUVyBf5TG4Z2536Rh+k2+8iSRjA5nggoq+0V1QpeAAK2MtR/mM7poPAQEwSaOS9V2XM9djuJTr6XWxwWon66HMgXsJ/pW3297LLfNlTmspQE0INFxuWrhQOGRqzZWmlOmXq328HLVf/6j5L0HotqX2aUXAeR/d27k9K6laCxKF9tIelNS7O/gp8IAr6fllU6DiFcWfB3bX8v/AVjEuXc5pL2CMRijJ9vZgNx2HHZp0SuF9RxgWrrhUXRkjiYHY9p4gqYb3T9tfoGqSbU5f9upTk+UyRDLV4AQSVWSnmMX7dVuwwauYm/wG+oyk3raRkdHABI0h1CEFI2C/IYXNMCKaoqZmQjcBAM4h7UC2vo161XTgmCDl9Lx9jqbEglKq/Q3dd6pIFyYucfEaljqNdd+WurWzD167kHd7ji6YvPxnP9hfq9c3OZj4mgbH4EGOa7/pC/m/fD/PxBXUQVp6hmhBISajY9Gl9fs3XO3gnwbR2ISkqMUuG54DakLbtqu1+o9Q+3MOTHES01RG7Ke/3sh4ARfSwZJ+TUhDRC4q9aHTkQMu15Y7vgJACDRLzLb7Hka/aqDrKiS5v4SQEV5o3hVXgz7MMVp+2+MOm7/n3o7RlBm4H3m9CWXTv7X0tjFRrZoViKNR6rx4Ca0Wj13khRX/jrGv6dfuKmjt+pKZEf3MbirDZ3XurX9WyP0oO4ntROF6FQZd+/Uct1M/7MSfr6HQYL1dqWY3jUaB903Wcg9loR7CuOB26alfYEDGCGoetVQKv5fkYROxwVw+tIi0ryCzxSjdrcD1Ju2mF9tSAm0CXyRX9jnBv1GB8tmjSYJcBJA7KMgC8yaMIIj0Z2etWCZNkbQTTmp5CDqC7LmFX+YXF9Na/SrfG/hjLzmh2FFx//0JjsqG4VPu/J3pnU7n7pIw5zcy7DBTzvyDtKkFr+G/wANpsAOD/3e51+jnTYem3rKV2XwOUozOvRgHlaZ+QpOOg560ievmYVtDr+pYfErGTkjQiMh0fuaTZPZ1TIPNui+ITZpUdGBTPRz6eMrhmBr5aT4CGO2X/9vTiPAoaABhjwcwnh51m54AvkqrVGTlymmWOochuzLKTV0F1PSyisc5NDCJ5LLmv7v/gIXr4KBGL5Mrx3mWAmOpNUpxUgPG4scP0+7MUyd+ytMZHP6R2lHt/Q0ofWg9UCjl1Bk2OwodMpcbOFvvxZewJ6KDuS1hz/Vx+f//ZGGanlhU5nxLKyOENfkRnnyuceavmW4U8jF/YxW0+XJt/Sz20K6tgr/264lfvrVDHSWud7NBRG60r1J92zuBwfSf+JXtCieaKq9nH1Kz40TMviTi3UIJndTKakdU9ch2Zj2fjyC+74aree+/8AXraXqUJJNf+FHKtcQxsPsCN22TAfdtXXZsHAJQFLA4t0/imgej/TUZ82CmrAZidcOEjWKEEAXcr2nM3XNYiNUELUKrCjU4nQigjgdeiv9sEyqtQjoISblduWTodz96drKcb3uPvkr+FvVrf1I3vDpyg0i1L/Z9e04J4VaIIN88qJWtXvNaOsSqTxVtIRk4Ro/gYX3YRfoYu4NYacznWCu8GcksLwZla3ZVwLI2qEeu3qEC1SyEYn61o77UBXj8QSB600HB+eUGRfhIz1FsI3X2CyUKMMgJo0fO9BB7dcBMk5kSJlHGdEXFCNKPUBCE8pFF7sa91c5pzJuIAxejTjwJFl1KqRmjBWSbaNe6dx9SwNNSX/Pusd6hyVLS2rbAWCZsstKoptmo9c02o52SaDen8tG5jLygobeqpkrz+m+cYRLK/pHVJs/zgYP3Bm1bVfn8bBTpSuxsRyRPy8oRJQ+d8Cz5/WLlhCboYSpDgFSJQECmL6Fqv55a+3Bh9FYueTXwngnV35zXQ3hSFFHAGulsdzHvVe/qWenDiZdKEyFBuYFk7Cqf1fT/DxcyCQoauBUQ6SRrTMFei/U3VHz3ClMQuI7xNs5Pu1fjZ50xsriRoykJJZu0klRCcEOY3j2yH90ql6XVaWlyKyiYu0/7G7yvFrpsp6s7qgqIXA3Z/8mKotNiF2R8xAqTDAUFbYNvrk1pK7fkGOqP8uSsJLv6okgdYVBYUdwhobHClWKDURtFqqb4YAqU7HCLAk0gD+EzfZwt4xYHQIfVpOy4ijzFX5osQlIU1ltw+FlNL2Ic8TmoHjY84PGZjd4emWOxU/EEs6eKdUMdLT32mC634Xw2gFx8p04BurfcGi5FJA0JFb9mIgguGP7+yz4tbTvT8z3q4QSxGbh5pzZNCPDBHdMKMG2nc27nvwZrrtxQJl9flj38BGC6PD/rKcG2UThVQpKgdaAgmWFaAM6E8IBDS6jbDapeMD2pT9Sb9hDUzBsovBAYlQsEEH8h5mmOp9NLekL8RfPcuCz1yrEnUwigFJ3jaicyrmqTUR/h0r4FdB0KQ1riuVa5Wjz7bUQvsV3C3xePNdLa8IqqNqTU1XxZWtip9OL5tGfnFn7mTq0B2kwQdHjPLax4firoWdfsXEqOr0I83qXLIZbA8IbTmmIBDciXlkmRbFAKlPNf5w5IMWQo0nDQFGYL6/6msl/P2SgceQ19lZBzE8/p9y8dfPXa4JE8rFDfsAMUYszgYhUOvKhucntgX4by4WNhGHHj1sZJ1nfWx+BIDEPqzC/uLQfwotBW3YYVhjaLK0IPG2+hXI0lcox/dBSBEw3SBCd93GOuIPnfy2YgvfMl0yk9aXB/SzS0AEV5Nkv27CKzoNZ140YzX2fmtvivnLrOJ6+uoo8pFPvgnFPSykp7mtDy6LshdlpFV2zXV7IMIv63ES1aWm5p5GdEladvbPsXOpSdU3SBAz1+pAwOnSw+409SP8dQfaAGKjjRu0WwfnRf6IcrJistrjULqkn8wU4biNFqYIgSwIFi4pufNMGihFW5t3y8qVFAS+PYL4Qc6NSC+rfl+8XEDmnTniUZ/6cGBkUnv/oPfavy2nfndOJCnO+MVTPGqU1mt0jvwd9mV6yFC8PyAWUzipidYlISibU92FV14kjXPtjwHtZVRWn4f5O5kM9sEGFG11hNUpQe5dL5r9byaPkC8wzHxhh73oj7WQJ+RiH4wnvWy/Yun9Xvqq/sFNpJF1krmf3yyOOJA7Z6z3qLYS6aL1ZuSvX7gedUZDvE5WTryV4ZioPVPJzu2NetwE4q1vaY1lnkZDA1utzWw358RxS7KmMLDQX60XLWxfN3Xk0hK/iPgig/aSryLbD5p+1scs73S2zSssRzd+ajMSBTAglcvy4gdxDxM6ZcVSry9FXicvNaTK9BfFPqGneLpZjP5BVGTyeJGPgQSIKiHfNGGrPPqbC3t2VAqpsC5GIjeKZWfT7l870EbihRwXYrBmIMdzD4dqW2b1zuCRzrcDuhKzB+ZA66J8iuwu1KyFfDR2jsxvbiKwl9WQGZIgPAr9Os2NUowi+Ldh8pelBWkIUAzFcP5A6X8D0qqaMAzCin4MYDpAnChqvaFpg6GtsCg6qazyLXJdmrOvTKH1gkmeqEEBwMqBGS84jEvi1DpGJdi+8MUtEH0ssUm1rsNfOmWzCWJPmR8LwGBnuqbv2KmgwyX1p7DWuG+IgsMWjlXkxckeJZsQVPfbEec6CjHuuffUjP8rJ9F1CQzyZM7CGuHrevecZZLXu9MnnUiretXO/at60fa2lBnBC1YGtZ6Rk/UYh1EwyZV131YBQ8TKKDZkatVUz4NMcLgGbh+VcPpRSyhSTWOAvpREFNO7eEiWLusMwDAQ42+8KrhyaGN9TObuMZ1rGRmUaiHzhv1N/e6ySbvyENT0CvaQtEM/MzN+7N/d0w9JSYoEr1LcrP+6zgKQUg7s/gw2IzI1zmRd6/RXZuR/N1rol61vq/Q4SxIneud8pYp8A9xP/4SWu3JmoPdhjqe3X2le7OTs/eFLx/6W4HtPSZpr6KkZveAIkwsMJ0HwkwVq6oqga1ihv1h9l7nEMUs9yfu0NSJnHGV9RJk9xqMUGuPOJxvk9McSIBY9mIYzgYnop3yWTA2MhyxVncw2WGgDrYzFpNW1ru3Mz7J+W4pAReg24Ixem5JINNwwgvMbfcfSnuRbdT5J/9Pv3HSECCKjilJL3jTPX88iDImkO/QyUZYcD5l67e8xmoS4zymMC1fe11FVbzrbECmVhuB6+NrRv7UkhSGnIDBjsMw/z/jYPKY7ufDdYCV7XfyP3n8FRz9Z5TI0RvSZEJi9ds1gyNYY/J74qK2E4bHi0UZebziIk6dRHsTm0ZFFtxLg2Egu9enAELYDiIXltLEh36pfh0iupvfj4XKZz+t6XMjh77SVlyuEClAOBO9bNJZ/1Z1MehraQgRh4cUFEg8R5n+DrH49U5Q2moX2zUVODjD0nf/8VPFy1Q8/GiMqguTLyNejua+uZC2q8OUMYU76C2Hwa5P20vTgQKAfIKmsnFPSon05pUehXTRZWBaF+q0zpznveXnfscwD0c4zbnCR/h/JL6kVS9NRbmmM3mDf+13Ttnlx13UOnhRCOGt7TogvZvCET88+P7dumw89Nx8JHXx4hw0fPqlhLOdznIS/VshMw5f7wWYF63IYTc5vGKFDHFSrIIE/AVY4BAesP3+VTpn2V/znZ1WU4/oiipRg3zALEUUPbjlJ6ff8/DjZoklj4s83yullPF3QcRrNhmMiFF4w+guVfi1ymZNtlybsnkuSltVc+EaFpxjBOdg3B9NDrVqe4eFBXxABu2Vomm2BUbbwb2RO9SVAtZ15u39jXOY494w8Kp1SZ94Y5APV4QoP0kIAPgEpNaYMzXKLDQf2Q05AoSReUz3MAnrgj4ykExEDWUqpfzjNig6YDD499uv9Ru9VCfMXAk3/Jhlgog4r44dxIykxH4ohUKkbT31QEaX3qgZxxHAMhiT8y941VpoZoSCzzBhkTD32pC1+QvP7ZfzOx5wK0RwKBJEkoH0p1MY4VPa1pQNPQop6anFdXfxVzr2iYM0b8rq73Jx+0eVldF4060ZuUd6Uhjmb0ZZAm5E32Bu84EP/ZaFgWeJgrBUgaHvJcc2MIanYCfO6GJofd0eyB1/wR7R4/ZnCO49QeTQCC5wiAflBLn2HLS9bllcGWc+9CfPTCD9oWOhm873dUWh2EsXjOtlQ9SsWhrN+UI6uY6dB8I8DKJENU8AmnskOFJ7QPwbfKp8q34UpP9Ev2dzFa+H0e9qpDUZGi8952zQwfZtrTZdWktQcmGaIwoXruR0doNDJY2IGAq+syWmtVBklauJdsPXzHjZwbli/fjfdNyjTIkwKRf37cvqLlGDJQQlPaCmpM6K8IDigqwsnDv9cjigSL3iiB+FY/UWau8/2OsjM4MKLZglf52hDYZeq/LvnWv4+sYHe/Ki3jlJuwVKtw1BmQ0PHpHgf6oh5MvXFadHLcJThq9cEO8Jz2ImNjIrkepVtGGvAxeIFteL0wE7rbQ25VtHh6+HfKJ5OOJlUGFf2/WHd2kvgTkA/VC/3JDdpLUO0gpK33y0rTy6mz01Ya/DhNWTOPbOQ7qH4m7atCjelQMuHfNKo10qLC43LBmS0wAt+lQMUOUOkwfs11i3SBNNBgYI6j5VrKqDhmoDRHU3EappdPyt5SL704fI9QSIzxnn7/ToVnWxrJH2OogfpzDIga0AliMEY2Kh5P+qpczYISWRxifjfdK3tWYHAWDkiLYexWHl98v0CTazjzOjDO/379PeM1jkF1KqTFLYtA4oeT5CoVd0k6AsQ2QX+nr6Rnbq/n6Av7rFqTZlrdV+w/Zo9S7T5cJyrYlA+R1oAJhbCPP3fu+8/oLqbeeeh3xzBW2/864jGYfLkD+yH+H2zQBtLWv97uoLEBaZH8wyAObTQYAMpV5ZCq6LEwKUqn5SjhpceVQlp8Pax2CzAKzWO1d7bM2ETT3IWGFvKNOrXzAB+m/jpFoM69wIarFes1uaDfGZhqkG9PeCGQ6Nb5zq5NHkpg5HI4fP3Orx78xntz0mFI/X1bmKSCPicgriAHPN87SjkOl5g/3tqxx3Ix2UncUJWhQ+biqdpEUOX26j8KLTvfyu0r6NZ5mXetaYgNC+MT8ymDbNoBXWOqT57Qfy/fS5VhpvZOUbYEg7IVQyv0nsR6IuZjMOvfthQ+288Bn+UtSPtrw8shT8hzwsiClfeu6Ox/JeA+uTwuhw6eKauKAxSEO24ztivyJ/zxeuMyuB2ARyQABwjVLvtv+fKV0nApD3SDctRldywNBuq7PyXVcH6GZ80VrTBRw5tOy4IPapn8D+DZiM+w7YafBjSKYthxAhzHjU4J/+3MplEcJKioS1p54iMuuR1YZxOAYEgViZwLqyHUQgLrn7CGw7FqhRFpuXczLMZX7IpQuKJ6xYZJ4dCbyvoT/6xLQN748djFWNvOE0lNsJsAjvP900cYtsb1VrjKvDzlIY4C+VAtqHbf801w9gTmjb7iOo3fD4zJkvV88mTB0183bvKeq3HvHpZSxHJ8nxH5I2SZE3LHsCp554LdFBpG3JESJ6XMV/4P9xOKhiVa5+Bcdewl+UL7UNPIgpmxGCcoBpCd9N1tuQzHEnUtrlq6nja9ZkwGDJ04ZyK7skNb0zVHuHfWGRG4mguzQi2pstYskZbHf0mSNurlqo7Np1YnXJIkJItYyZLIBtM/faDn/GnzRh3IZa2KWXYBDR7o/lr5ykRun1W5yeG7Szofvc2/A2vNxs7bGgVlj9gcrnw0ZcFCAll8f9fls2OXmKiqsmhWb0XCPrFQeZWA5J5bHLVYZtRxmM8wAcYCsGV0R3JgLIXP5AU43vf71zzjcOlKs4w//QxC65tzIjSDesmjSJr7FxmH73WZVgxFCkT/k0JXvU3gvDHicmQwF8mfkjXRtF12/HeSaxxZxseChXAsSRI5w9AtJC+s1/TQfnjw19j5/08E30hDAxLz/Og9IiaqX5FLOJr6pC12XTmTeJImP34fh7gEWAUqcmpwpsccfblA6hOwT9v0FY3YZjT0KkAo/iEj6H8VWP0Oj8yOjRH2GP4iNf7tE1D+sF/ru+VY1OLMbi4gjs223tSgA6dBq2VHtkbLMWSddCLD6qcfqCHSpuEFPYauOTA+x4YwNRRLNmGTIUCVV+hz4gE8GuUCeUC6g459uEvgLWETlxjqd2US7vRnseLncB3PiIhYxdLGCV7L/7xCo2mZO8LcDSLkYJLI5rygzNgbZ6o7Eb7UzLWGRkFn99LITca9u4vgoGI83rfb6opjfE3hca9xJADmNOmfftCRDQjtZCb/lxjwivBAyIPu/xirrGs0xGBqwVGlXVcDbwzOJ5isoAkraQzDykesUF1ZaboSFPYzKzf0dFN381OfsaLpFDPVrY4aaDIbj/ceCI75cMfQUHt4ZT4TLTGoAlNHf1xNYAaZbWqTI+DCE9uwlwD99Y5FLFecMLZIZ2DTAyGq9MXMVlaCS3gmXWYk4MsO3+/yYXN1N+5G921Uo9jWe4GQoPYQwQeXms74jXZAdE7sXzDgjQLfXDEUj2P1BV1UQx0AYgs8ZSe6zNokkxMkg2G18q7PBmozPlqnVkGEU9nBq32t9MDE+tM3c51SXB8ooD3sby6QvEzzClPyp49IWMJlJYxZmu9cU+3tmOFpN/2O5ggxm/oR/Oo5jifL3u5nQ8p5CqFIjdKlZNdn1vS8qUpQFidq/CzqWiS6wf2e4nUQd0kvGfDn5lZDMtzB6reT0GPRyGvCsPxZ8boLzJGuSyFLetL8KDbNVOM3vsIcgD7TbUEOH2NPDcL0GtJHKOo+DcphrkvBgzde7Z0Lt5CM05n/UX7BSNmHjPhdcnxAOSvd9k8GD/QzykM5bmr8tg0GO/wzOWcIp8ODyo4EaBjtSFrSIRA/YYT/uRTXex+cpLESKJFhF9ZdT6Kzy6CASDCvAtgt0kkKGJBwAKfA/z9v2IMW2mUIRI/E0iMuGJv09EU5KFuWRMb7KGHFm+ItSGEpmv4E00k5cYVtlu80so/5XLZn/RCJlpHuxf500um+R9kjlzssHTMlS9WVFVRemoetPb52u056e9aOe1v1CbH0+WuH6593rTQdrCmfdJa+qa/GlKX3m7NntXfEJjhD+RB0hf9JPnLPmRSjVzx/UC2fv7dON5SXmSQbAa9Wwi/Z+BxHVaXn81GQCqtoNMoxO4PV6O0f4ffu6pTfhElLeTvp6M/V9B1ENWXTiv2GcovaHpO9qw/WAmC244WsZL/QhZ0Fc5hUtOy5CqnYRauijb7bW7/0WMsUwqI9EN3+5tYGOLGn4I2RYJMQpdRuxjK60bffkqn6FwuP9giTcoG0HGhbUWrrGefF2U21Yct+1za3gEZ3NKJYEqVSG9dAecO0qHq+7hu9wd78w+zXP1CpJ1kHzJsYoDHbCTGpMRjs5r9sq3AXXj660yVJDt1JYfJjp1yhBIfKftKVcsrEpbZVfMzb3IxKEkDNapI+rKzP42waJSql/gXTtD1JDDR05QrlUzPQn7U6EIDoSp5BMOYF/b5LuK2HnMZysHMWohYqsOfc0TVlDpuvc2VgNjU9iymqdyJhq2MD2T3CXTTo+q6vBNNJp9hYY5Wn0VVvKufMJ1tuWMXO+KDZr2x37/Cv+SKJyo8R5ZjpN2/e6jgjEk+dzcaUo0mY90pHHVd0u6cD4vX31DyZSeH63flL3Oq0NAvl6CbgvIM42/yHH/XAVYTzaTK/7rpy0ETopiAgCkQpuMEqC/5X3iKdAtvH9kF4EDnal1BFpw0kFFlokXSjK/yU3jVdzuH54LUNzrGxQ+Nz7g6S4FiIBjVl66gRvfOxCRpotplueyFeMUGOBOSUtMngBT05SE5ocu/Miv3IS9XCluIPzn4VCXzzR/UwWgtdMyr1rynwkOBXBpPQE2GXOT4pidiIsi0sx/cI2j7XCrvF1cg2ny1VvFqm6KmsQ9bthAK1F9mfAWYykC41XY+zXiCwn9h+3F6ZBMIj41XggxoU+a32fiP3DhKUkryPLUx3sbnYP4cvbPTksbkIE4cIDl8HyP94lVbvoAoy6nCFa9iv+Ttr+743JC1rVZup/HKwj6jru4B1ADvuSyrarif98CtlMmtxeUwcmKWKMqh5we6ALhQCs5ZK4Xce/R65i7UTeNxoVkp/il5+wk8HL8PAutuzmXtuvR5BMee2WFyRC3Q2qM5QMjlKoLrzlj0uY1ptRAOdDvKlhDyqoaPMHN5KWUz+Z7b22gNmq2IKjf5SozO4sahUtq6zAtI/PY0UnUr3V2Ig689pVw6bvInS74HWstPPizhj+tEO+dmvqZw8LsPKpHHEt0I+rkfeLckdB0m3tunztsO2TX7tQqojfBHEnoZ1ncEF4ybaHsKrfroIQ4xB2fg9QT9bE0R43jN+F4AlHRbyEp7ETmOH9XkevMFN5iLbgTa46RUYBvRuQ89Kt4HcIoQCJRWRa43cTtlQ0YlLV90gfZyDGLpr+rt32nYb5gWlV90MOxZkRG5dPtf16DP5EduNwbfKyIOm5yu1YyMOUW0+ZSUz0b/zlEjD5BByFqGjKOO8mryKAVYHExX6fO60PocHMedRHZLZLtPkes/yd9K9PxVttG/riVEvoOaSXpSMKDYZufqGmhpSAJXSJApnCfTjG+5UP3CZj7oysGUBtYdXKqUTN4cd1s6gu8dwYV+Tql2ZqD6VTJGjEVBqNks2kIWAf+1T49gX9TKypDbVcLH0df2abxrTAk7i8vcHfQnz0ZexPG3Rd4Ac/axQYt/kgJ3UC3d8g9pbR5IUPabXdIwY3PiAxGEHVr7i6mUMXi49VuNeBXSDEzsw4YpbwQ9Pn0dEf6H3ebBm8uEwXW5Hr4dCB7xxEzad1uTnwwC2o+wT4waSexvOugDh2sqyeO0rux1cDaqsY9ey4UlyXW7iMy802l2nQaW2yTRYKQkep/0akjOyfk4QfOkhFP3t7Q2oPguarkj8oYwM0XA08dJ9QWl3P4KOXT7y5cjcsZqUfB63l1mzMjgRorA6z9tRATop8TPBc6tYfxcPYw/PRrcB3Plvy5nU7JRV60Q2QmCc1WR+CtZCf77nd1p+GQHY9m59DcMK/I9QqwxRUDpmTS5n6W59E20d5vqN8J3gQaODSAHjhFf8E6w1yXnzw0dji7DXw40uJTVg2vjYUjhvfgbWTfm0ovLNxtDBONXTpwdCzyoYuVU8qXN0xCo1xqDcoS+JP8MS8gRoLijFtvOJ8P6u+BuqHhoQFxnIwdFmPrRdPdogr9+SAPxIgKh4tu6TZv6r2OCXNDEXj6nnr7/4/WoANHcN5njJDjC3IiZOfOI2vmSVDm2canHNku2fJk2bNJw1r8Sepw4pHQR5YufDZrzGdVtLsiSTwKs6XCUy0tx729yZbfh3T1T2AaZu27MidEgnoBIDDmpOnu363WvgswQ7AxwQEgeg8Dja4ywv075qIB8hiaq4bvUXEMPDSIq8joyuKwMLpoIBv6zXSZd15XaSvKgkfszXy/Mhr+GWi0gSe7daS6FfTMsxtT/9fBXgUaY+WWtINj9N2aVWeehScaotFVjiK3Fd/Tzi5qpxarJiYyOh3tPdBWI1qhq9fzqhDx+aX0kJR6SYImBJIiZzdx1nETd5xzVn/fubtyPty2/NKvWQffP57Plfwq86/ksu5L0wYVQw65480wR+vxSDzmwYfrmCY53GOkO5N7OQ8SGcTbUd34WTW7hT22FA39X7aAPJIDs0Ud1NVR9XIO774Ju97n+K5NzeHDHiAEWIGn2mvNkMyvQzKagHvNZqOxIL6WTngSXd3s2mhy1HXgYddGNY02F2wSG4L+Bm0VWJM0cy82PulLVW/gladTo0JzFPv9mn5iZm09KGbVeDHfQ5s9t8OG8YZSHlRzNPZlumMJozzcnCyg2ZRO4icd1ozZ1wvScON/cn8nqg7N4tLREE1AulpenFudi+ctjO/sr/O/wj50M6MlI5EOiTQuWUaTPDUelLxIKccVF6CvDnqlD86jFaeAjol83OrrX+lelToL8s6IzKr+wZxoZd6MiUI4zVSo245/Sm9W3bnaIFsfIwsfBSUekBFNB4Fn/qhIYAGzFVAcgtyAPTk/DJlpl/K80RZ9ogSm2j7n5rP7w7kRKDRAittReSJ53mOsvhczcbfWXxd0579LiiWZUYWJPn03aAUzoKHlMMQrq/Cs13HVHuZt+bbOJwufG37VlvhMrQBbUoW2kQvYsOKh16EgKo6q675tkddLmu/9N2yREPF7e73sNmvIel4IG6BDqlHA6BuT4fRgC4+OIZlXk/JsxxuzCSmPelnLv+/Z9avRViDeKfYqJNgdng/nfp1pITzxzUZcHD72j9nPZp1ObLtohG/BNBSXBRUCI/rytzh+sB6cDBP31Y9ZZ64B32KDS0EvmfRg7xoziZpyYGd2iG15aWOaoz3kKXAy3UXE8nDAGblsKfroBIkZSr2v6/rxw2MoOvTznyj9F75iL6FfkThJ1pkkze0vDSSFlya/tC/5gY9P8Qwxp7FoT/KfUkW0HXxaFdVvF0PxHy9Z1K95vwD7i6fnnWGhHUHPg4nXvGFzCVdhj1KDbDew8D+iPDXO5PKRmI2bU8zdJfqZey5IlGcByA957MZkD0tgaL3cAL8XqcNOyT4Um6G/ObJtwdlv8oBS8++uUrpINLVU/L3pXEHEFFRcRS1UMNA9mOFvNCPGEuxArly4vSjrPQ+mVViRt+JvQgIlWdv5NH1CeQKb1a9OWshRMlFi/IyN/bHZd9w9OV7k3x/0RObsx0OwsyGiGqeB9f+CYi62CNetH6R6JNCeCLjNRwWHD3+07xRGNUeiMxlgNzYnP6Zkvk7NmeLhp+KY5nbtxYhHJczgcbRb5m86G5m9yKEfrh7X8TuYH9nJ8SXfFkIBYQKT7jIjoY6pZDypR9REQNhoZ0KJxOf3Ds8X3L3DGrY1cBU25bgMsEfm1TNXTuR+QiMkqcjpQ/FjedznI7EJGUit0w7zIl+Kxjocd124k421gn8eZ+Z+B5vVf48UxYPyKonOb4b8jUOLY+OLRRZB1nnt3/KGtN6AlBfaCYRd2Wg4Cx2OmP3mOOZKiGsX8nHH5V4yqg7QdAX0sc1q/2GlxoCkUHj98pvCD2CbCI9OOEHNgZS9SyEKaK3aPjpphAD/sSiD3tFuwQWedmJVG32gfH/9LsWDqbS4v5leYtAnCs/cxfGW8vz5XufTUknssyO+vJ6+rMI81/6UOA7Zh07zRbDGp7ffr0DYth/shgUyMYLGcin9dhnyhIE04lgkHxaN0QAhwkEPRJCjsLvGiAx4l/4ZqujmIoK0SzphY7OFKOZYCC68NgOEgQlfaFQ2x6UPrlqJtwF7VXfCZD5SRnL7aU8kKe6HIYd562sHBXb3+jUrkFFt09SsnuVTvX55iZU+c06tilzF98++5zMjWs94NOF4j4Kz1RYJFJYJW5lj9oaUmLgjmek2bDbto182b0u5px8EC0eMlOy0tV14fsu9Lmt8umPM8gV4OzT+G3kKX0ONcYx4RntnjJjRe/+ei2ENUWHevEw+VboYB++bfns8vTeqxv0JhDHvhGGjyVODF1YiEqSuwVSTxoly5aTxm1i48Oy8UxGl5l8zLtkF7irWHJh2r2A9/Hp4xWCf5KvY9smb+SCX5MxW4K5eePg9abS3JHs3fUCnRquE4bQ9f61PFXPLZGtDkxpVfP1K/S+D7h2TuFcAs9H5Khx1uWAYQjmneP2xdjV4U/hdVZ+taD28vBQ6biwdjHik5M2KtJu4lEGU8832+WVCstjRfRv3IGvNDRO2sAgtR3TELmCcs8o3qxu+pN0FF5vHt97knT6KUPqZwWWvwzRThpfxgB6X1Pafq1JEvx8zAKW3psEqmxEbIgL1yhPUYubOXUahiR44C5e6xHM5gxZXvDX/x0UjHTWzAdAjRvGYgw7NSBbmRhqH9Mo7/Zxu0TEWwXbB0QW22Fj7bDXkaZ/9SOiw2gC36l6NmgQ97i1JqbcrsY0CXn8KH/lfel3U5iiNt/5o+38xF1WFfLtmMMRiMMcbmjs2AzWZ28+tfCWdWZTnd1VndVfP1TPucXMCSQIpHEU9ECNE4LqdWdCBet8f85ha3Yk9HsjN27UoSwQxdHVc6xbb7UKNEKnQ4wV4u0gH6VvRAy2PTmDtbnV+pZwCCdNlKfjpyeyFnYH6lh9yAag5I4CTzu3RzSqvnAM1WVz2683Mq1k80ovECqbS75kQyA1QbjDQgh1VsHtRojbMV3Uw+jdOEX11P2q0BRKVBWQrnlKEklFzOThsqv4x5u5+wANmJ3MKgWxTpHKo9K6R7xFkDo5PMSriBu0TdNpbF3YasHKpcW5a+6OJyv9KjiGip4HSslHn1Ixt2Q90HRlFO+1XIcXkMuM2xyY/kNZugzhbdi83n/dbFzuIu5+CDHXWpEots3phajnsyQ454bVwuW2DRpaJLKv9QUdZG3ttNeT5zV3rB3zBt1zS8Rk610J+jYZKZVZfz1ZhR3FkPMjXGo7XuDEpfaJe1H+0zZmoH47Q4HS1AFNetWkxhKx3TFe/sDL8gHaVf6pbcx1KiSiLHb7n1DhILVe7QhlPV27ygbjfauno2r8Y60JE91ewQ6tJL7KHEz6Tz8rx/7gJvYCUcRbcZlMRKOeATTFEzCs5+cTATpDCzVUZ5jjfLXhZWImWo2lr0j/C4MyfVsa6etSSuiF5Q8gLGdnnzeJlvZxGs1FMgV0AlkhxgHAx6a7zDGY+BA9pY7ma2bsaA1Iy/AXJT9wgTr4zVaLmctBKI/HaID+LkVOPOto8HwttGtEptnbUvFihrpVRqVvpmdamLUFoLvOK6Hrqdl5GuYhvqn4VwTrLGo3cap1WOKk4q3yYmcLZY5HyCD/l0HBUyNHnYZ8dY5uiUDWKZSpZWznsr1ThmjDReBAOd3FNYRZU7cQfmciwy1IFZn5NTI44O2RDFX2gZBmPcKJqk3PWBUw391Epf2KoTKEBtdGF+m805ct7OiUR6rInLMr6dSrEqT7bkulK42TW9m3FcP/ttc1LucGFIqr0gVUGvgI0xnA0R7tQFXFA8VaMKh/B27LIsunh0RqGiGLT6aX6l+Agnpb49LyDbXq9o/pRqYWQZJg4s+kYkLTvrNPW6NIIebhG68CwFOGY9KLfWRMB9tBUtnY1Sy8W1Y27qlhH2Lr3ZJIyqqOy5JRWHkHetoazZg340KL+WFoKzE/Y95ZA+O6BU1EUue1uvyNjfOxMM5VjOiehX64OytuaXEpFYPUV5oSEWscpih+3ZEpeQ1mCX/Z4kKnfTLnusWFS7NHYu3c3V1XnLeOADY7LmOcFxV+IkmpSyQgJOBCnzkhjT0+3Gw5XWXb3CCO7Q9hUp0OtWkIC9vBz2MC0pbt19r9/2o7KXyd5c78/GHrmJchILrjggPWQml+LkEieGC3UB1RRAdy5rh/cu8qWDMZNB1k64eIVjLOZUNYlDJKz7Rhch7/Mhtqj20JULz95y6kqhHELDF1f0BPvdi8CiIvrURXmAxOu4Zt3pjLHSdrsJEaJo6z0i+8c5p7Mm3IXW6OR0tE5hf0qu5WbJ6JZa7Qqy70rkjO37BScSh1CTCBQPO83ronYVAdglBqSUeNAthSAzXaMnctzQg4met+kKAdUOcFtkNvwgnUprfnDR5mhuqSo9tWQSnK+OZTrVFTMV+WTU/CLd5oTYn5ntOBwpVhD2N5epOnp3E64EFXBLU1/timvvunBK8MPBRc+KsyDX5gqXgxAfBbGS50VeNNQN4/7k21OyUw72/ckr20Y0i9hdnQbmYHn9whnHuHTE896DBA1ufsvbVZmTthiX7jEw9vWUlytiojW70fScFBEWY50d2tDHtRht1v4WI0KMO7JunYcargVnhS0zITwGW+E4jPuNyMiDBrn/9ewYhE6KBg+14sbc5EQywZSBeFMWOMycn0O8hkK9xNUmYa/rQ0+xpRkpK88qlfXA37IVe0asRaQdxlD180NEJIjeQAtidguTWIv1WPSt0bgB9GvlKhqrKaLEw3KYzjh89rJckbrY2FYnk/hq8rtNZDUrjddHPJaWK7Tp3NZubuZpIBaE0m22fVbQhZGfq3E9ZfShH+CaAbm8nlCdPmY7P1VZPkuP9IFRAM3CGI0ftMI2khUc1xXh8HGMpBmdnnPkyFtTL+gSKa3OLIorpbLY7hTcI6JNVKLn/uYMQcgkB211FnHaQWD6fDy2WLVW97d9TDKVseb2He4lkAdtRhe93FYbmjg2xHjDl8NoOOMJarOr3q+CitEXeRMUU+EloSmBwTltdD9DxV5xyWsuIByAe1LQ2DKIOg5GwC/odKvZ2qOnXStz9hFO9BNFOGYAMEwCHtNai0GkHWVc9P5KVqJGFmHYrIXW2p5urXyozAp4/Z3DK1Z7wxr+dnKyUisYbwwny9BIRiHI7Fjm6a4ZFZXSAnEVGx1bZIO3v1RodaJsI8SOR7kz8E1BDIBUch1f5Ka69PSx57lwLFmZIJIVGCaj4JEVNENQC2WBkrdDj9f7VK22BK40F3sJ19AihAonaLkvdD9hTaEDcNChcjqmmROJeHleH8Z6z0nnYVoK6VXb5DsYvES1JKkVa35N2G0pt9pJIoRyQzjr+VkbxWPW216OhtvSDjLX7q2DTwtNddzBdTg+fjJPtw6NOdPlOAWFi1r8cLSmMbLxc1Jra+Ks8NTR9zun2dtryT0t1zESSoa+lLO1DHMRKalV+olx8TIdWSfLfGEqd5NdjtHtzikIFxOqcUJll++IrUXBYG0lV4exOazb7RpdZmt1ONlMlR1ydcfJqAznRETA2QIRlKGdJXXIOBrVeYAhJDgrEcPrbQVmoHNqzQNbsWHn96f7ZK0ZanNMh6iKDuL5mg5srRVW3abdxS1F0TInkYszzdxjPrVbnAbAumHvBeWM7lscD+YXEm4pPSBYtYKbNadndV/dMmhEYY54UTCMtyfocOFP5lZdKBzwbJTcqyMY8UdpPGKpROIqATHNil9IiEgXK1HIe8nurk7R+lxMmYppelwuetqhbcnKI6EbSKkkvrGvucexvJmmF11CKuCqpGkUifAh0XTBC4Cbj+txydE+9HvTcul3XuAxJWJtBUFVYyefOh6f1F1EHawNO+2wWgi3wlLheGHa9kUfpkOjHqD3nc4bw3a+2hi+Q0qSZLvA0o2VxmhL4+ix0KMVzwe6R9RUkoGvpd5Gb9EvF6YP5j0fHqjDemHYS0mCnqicF2Is4xIfAm972LEoKaU8dD3zk+3LcM1TdlyeyiSjEl/34o0iCoJGe6tqh6wDi7nuL0M9VsxqRGtyXc41MzCDNuwwQg8izq/+rrnIwo1XA4MtWKU6BBqMC/Aoi3sscOGFiyQD60nPO/wiKHl0sXLtwfhyc5SVa5uD+1nIZL3R3E0/WW0qA5rBajM38lZmxfEKdzWQK0YDm06y0IP05sf3DZe9xCuztMb1IJyY0svsKVoMF7m/CPeRUdSCap1QRlHohe76IIB6ar+wjmeT5+GzpIuhW1AmcJeLwVR3kE+gFyuB62p46ySdzuL50oZAy2sHMcAGTQ1q2REAQYGPi3HarYxoZ1Chc6kA13CJaHoN6M786hw+QA67xgf3v1WUZDR41kfy0FsHy8ERdnDtvRbcx2aV0MqgtYXbrd3NqqvzvjB2vYhdPC6LGYuZ/KG78RZ5FS1MNraLni0s3hJEjhvn7eQW0Db6UwaMN49M0yEchnyhR7kycOs9I/TA3z16ByFNms029GaCp97H1TiGFYyGNF46tPBt1byYLcK66xbYmQPjlO5ypS6NKs9Nu3BV1We2wm7mqUQmQH883ViyOdbLFXnunM60F6q655DdtqXM9U6MgkW25/p0XENTQEVtcdhcRA6IRhByRaA0v+33PAx314ecpUheMeNClW6Cet6YQz/uhEvlErfSYjqabXGHsJIUSobflcvRj1hRA/pmTUvXjiJvOU+kAmTuaC13PGYybroldgLwIfy8Iy5HM4bBkpHDqgjH1/Ddufz1nOdyk/AWEpVcWetQzx9t3sty3upzzh2ziEKngNkyyqyGj4lXwd3ReAbVADzmjeRu0z5kp0Q2pfIGet5BoHWU2HfdBVEiu66xmDlTyblVZIgRaXmJ9aqNJmJZn2m5cqq8O8sYt13o0lZ0xc3VI/NpqRLmPnFgksta5RVcFNdvvIKKxllj8NLyej429catogvucfOuf4d9ywdHTVxJriIfE0psIby1kvfcS1X5Sx1JoN2hcrKOyrOkBDEHNATfOGefLhv0CjjPSDcIvreXShA6ksu5inr0WQ4RkbOV2W2c5ssrVVGHkdtxzoqQgJbnJDvibZodWcPfXMaxABx0F141XecUZC2tcrGP3Zxa+uYeqVPd80aXbufNn9vS3yKXeJbKdlstdh21TJcyW6ONCvNCxEj3zBncR8YJxGla0N1+EuskuKxzwIwMP3G1TRgHfo8s57kuyErHVPMrOL160xkJdvY3Wi4cjGNO4Ta+NwVbPnM4nJoZfL7rZBnbnVuvDuah3+4NtcJzMd7eTBOIGs4MN6Uj5ECxUFEltBzw9KEV9sDR4vTYQeQbz9knRizyXt85jSJdpu5iw8B/Xm0O5bD288IyrbktLl3esgjTO4C8ISAYiTBkx/IOF0cu9VjYFCEEnq7eNOGQmcHBVZFaPeAOrveIn9rmHXuX2qbbis4qrentmyoMnEso9X5dBb7Ee/ZFEdKlojL8qUegmc4pZt7QxLXs45GI7IveZeUJSzEGKrmzuK8vqenM96fELsxQChv4lOpiZyFlT8ruVSKNa74zgmG3CwXGX+7WC8JOZTfOsu6yO3MIsScMJVRX1yUXkRakKGbtwb1ZgbA2IlwzstywzFGhJSWOB850rNO8z6vZns5rDaMtfYM21KKsL9dLbBglASyGYWum7ZOlkJ+pY9DAnOWJP5/88mrnR1VaymIQ7vmxdW0Ynt+Xgtfr3FrnjTiRZr2iAgoGrlGfJ+mCH/SzJqPTxkAnwXDtSVkF5NAG2hFVZNs2NvJRo3RZj/nTeTIrvMfb2BVSbltgyXKFcVI26WgApyG/UXtXDQ2D2tXXWphnsmRrdoe1c87fXmNW4+/XOuqOqnusqnY/nPEmkW9sUojHZXjhbYJdr4jBSxAFE/CjgecDXI+IltYQqjzO9SQR+thWss/yLhYbWd2iO0UJE22LideDRHYWWR9PlZbe593am/OEoKsYpjuOd7rKGbNbkwRtB961SOvFaoWKgp5NS03Nwwgnu9X8XiFmKZ+G5ALYwmTPb0Ot+81QRKvJ5mljK87LK09rkpKS6VSQGL5T0qEe3Iu33ypsetVPjhnzTRbytG9A2r9AWwonhT1ZzgqWX0reXsbDhlxUzQVujg58I2V17Y4XqcnstRIfzJGJHfW2VCp6cHc7knTVzpSOsWgSuUDsEfLGTY006kmTFEoYqXZTeTtpsHHZSRZKbmgRcbko5SK3A9uKMVrZUkmc4N2RqJuV6coDumJURJTJzkBaOwHSiuFPKqjSZTAlc8tJkOWBeSko3Kx/eAn8QHv2UnY9q0bwA2denM4/wO5wc12IsrkuFMNLXb7k7nUBD+TnukAlWS91Ifa3c92JoBmB48TfXvfoYvOMsjOduyoc/EjZYnexOjMXhN9wIFsgL6QCBLZof4NLKeCT4PBUH9VtNL45hUu/4UI+ylGZRy18ISHy+i2NfIabfsFat/upTwyK3E8Madgm95Ms+xkn72eTKI2Tl+uxxOeXs15zPxN/ucS8Wcp8YRieHIUoy17vY/4fQ9LwXodb458K91OexP2OLW/H3VXWPmHUS1+8DNjMudxvGJWBC/NhCjgKFcN/0c+giFIEwOODr5BGaq+II/g3unZR077WgMskvlZ60k6StkDFIVEbhOBPvN2Am0aaqAYj+Vrarx/rP7YKb7dpgSq/3y4Qwdx0m4OOiyj418vSuAD/B0BgoGGch4JKAy/jXr7I0zCE1fk6atLJ8+emEHBclWnRzkNM8r+RImyra0tQJnpp+lQWrfVySPwcbHyiUfoBGdR7ZKAM8h4XBPKLUIGjH0EFDlGxvUMAirWrPrXlp9CDmf0PAqKOvFApQjBSGHIq6/eQ+ufBgUUe0IA8QQP1H0UD8QQND8MeFSFX1+UAxznzmiYNvhVC09blJXJeugD6z0dj2h7gQH8mUPTl+AhLf8bIl0NxfBHEfHB7c7CJ6hR0Dorzfq4AHX1p7V4dnoDNfUI+Iyj+euZri/PR7e3RY5u/K8ym7Oog+t6AvSjV1qvjqP1eQeZeMArj6LvgeCN78onoX8/VUea1aR99c7vP8PByhQ0E+BvsoTj5gD4Ce0DVvfMvFb8C631bOPPQFE4/NHUfnndNzRj90ve/AFvyj2H7BqGh1yRR+CL7AViqCPCdWcxD7VVwsqdZJpRZWc9VcYyhSQn7Au0335zmzzv9wQMkwmCwgHymcPgHqhRMmL9A59PYw1n26dm5iceS7O80TM+1wfdPGkEfzmHMQ9lZ5TXA24ajkI8xGIfk86Xzo7oA86T5nAZlAbtZK/M/4t2u/3UVSNL4Z4z9Fj0Uib2SoDeTAfAqEn8/HyjslxEm/D1hgt3+9Mpi/tfMEfk4jUn2CW99opMeFcefkUDLVMx2ssXyZsecf5QWJFN9Qtk/ntd12QFW8Tqby7pNyrgsvEwry+plsM5R295eRguO4x/Zq8fBfWNySJJ6a3LQP29tHgzBU+H9sVWhnov0r1oHEn/U6I+NfNg4kK/K+YudQf7DxoF5B6L1Hrh7j0BqhjTPvCJ6RIdXv4IBg9JLyjqdAEC81xJBkmah5t3KDkqqraPo9eBN2R04/SL+x7kewAJpEdW726x927noE1i+sz7wM1ufB3uFIAH4gPNAjYcpANUzi/VTNAZLPsLkCYPFnqgMlPyOzni54DYKWuB9ZtHvU2YUeaKi6A+oKC8D87EA/gsP1UfzS3D3AeU1kw/rW/LxBnwPcg29iDkFz5BABUzkP5fq9+fEh2VNMJ/pb4VNvB957Isx/tnm4WkvXu/gJ44wGTEh8WyEGczHKepXjjD2AO0nQaNnM+nXje6zyMCjLwiMmPVyGGV+OUhfT/DzCfDFK+v5duS/tds/OKp/7JexH7Sgrxj6Q7/swyb1rw069r8EaebvBmn8HwFp7G8FafIZpN+FBzEYYRTqCJhkGGD0wE9XpNfuRwKMczhRARd4CTC2tRdc7mFsUMjL4US5//5wo35WBvDdzQjwemEw/NTO49ZUZdFE/9zIJUGyn5lHt51B33O/Z9FL6ldNb/LZ9H4nUgYijWtmqd4dgTtcPirLdw22CWSol+j2ab5w8+GKQVnfoRTeQQo80x+75kNy5vM/BX4U/eCeEiz1Hnv4E+w9BiZ/HvaeRc7fZ9eQeyLlVX/cM2LhP0VuJPogN4Z84i8+C3v/MkpAfiB0/AtCTF9TIi8RpZeUCMxhvH7/AzmRN5zl6+29JS5BN+Psfju/K8o/5CCvYvhDDoKRP5uDPA8voSzymabYrx/mW4DRFPaZQmgCRRCaZDGW/vYCvxPF4urau70p9jJZfj9ehrHE43XRB2je2/ypEQaS/r7OeYNh6tqV7cus/nSf5RwogBLV+PXLR3sDC//5Vgio6rTIC2GsHCi6OcwCJrVXtz6ged81uPcrf9gO///tKDTbgEJkWTlENezjv+YzQVmc0jr/b+3Vff1GNsuvSdIK4gxutoeAcnWX//u/tV9eET6IC0LzGTL/23r21S15xd6dVqZ/fa79jxER7JGHENR73wVDnvAQ9pfxkPdZiif8kYQ61XpRpW9W1HzQa/jX3DACfFovm1GS52nbRj/YSnmCDGB2QLJX9V6UYfTv78LsfxlPn2iY8H/zoR7YAPoso/2fdY6pDwTLQTNp1cBR/aMY45sxhPT1Z8Qcf3yp5UPE/BP7zJt4lrBmftkgv4+Zf2FAOpgifx/ks6/HL/fy+0noHxAI+ZAOZLEnqH/mluO/TCAfWdD2k1B/Op2w4Gl6LqR8ivxJqP+Eo8hnFP12oJ+uIyX+o+rlvR+9eOFYfzvsU78A+5/giu7HkCjzJEf6FP+/LLxBfygkSn2NSs3MYkjB3f5gKPLN8t47xfb+Qkg1SLyiiLKZZ0BCjrzG4T9YvyvaNPvKUl74DiQp/6/5C7f1perXCO4Lg3rThP+1NALX/6Q/EhH22gCMPNJVX6TwhWP9U7kV8bjIhPyosvuSb/z5C9M+lNOi4bRyvLsnfU9JeXPbD7D4MzmHdG7o7az7geyBD2fFOzB7VZWlvwvlOaZxz5d8CHdfrerrstIgK7vwx6ws8YRbfnQRzDuIPQHidxg99UAuqScu4lNyif6MTOpT1GHPKPw74bKfvwgKenhV1/4lpMHoAfKvF11016dFOXxcE795UgQBf7x/f7jq3YzUUdvVRTPHMT5/vB9/Py341/D4bkUm8WQxxVMt+CdSXvAZxRLGlr5Gq+GC8DWwn7DE/wE= ================================================ FILE: Documentation/etcd-internals/diagrams/etcd_internal_parts.drawio ================================================ 3LxXk+xIlib2a9qMfJg2AAH5CK1FQAMva9Baa/x6Anmruqu6i7vD5YwZjXlvZiA83B3uR5/vOOJvH7o7+TkaS3VIs/ZvEJCef/swf4MgGIXR5+VtuX61QBAO/Gop5ir91Qb+s8Gq7uy3xt+7bVWaLX/quA5Du1bjnxuToe+zZP1TWzTPw/HnbvnQ/vmuY1Rk/9ZgJVH7761ela7lr1YcAf7ZLmRVUf5+ZxD47ZMu+r3zbw1LGaXD8YemD/u3Dz0Pw/rrqjvprH2p9ztdfo3j/m8+/cfC5qxf/zMDKG7j8nsfWwsOP6BgOiU1/QdE/Jpmj9rttx3/ttr1+p0Ew7a2VZ/R/6Aw8LcPlQ/9Sg/tMP/0+Tz/uPe2VDFHaZX987N+6LO3e9W2f+jO4CgKfp72ZZ2HJvuXzmm0lFn62432bF6rhx9KFGetMSzVWg3981k8rOvQ/aED2VbF+8E6jE9r9Nu75FlL9sxNlWvXPu/B39b+m6SB0O/vf9vve8toGX9tNK/Odx3UOFTvLOz+TLb8NsnDzvEd0J3FK/p/j44F/nvVL2vUJ9k767/z5zeWvQvOzj80/cYvPhu6bJ2vp8tvn/4Hhv4mPNfvDTj6d+RX0/FPccR+F7ryD6L4j8boNxUo/jH/P6XkufhNUP4fCM0H+F8Lze/kSa5HdtIfDhxltWbWGCVv+/EQ7c9ciYft6Zgq8T8aoqQp5rdV/yWBv1P+R2Z+18a/EKNHwPIcBwDgv4gLKIz/HUL+zAjwdwP1BzbAxF+w4YP9t7EB/Dc2ePND4qeJLLMofV6VoXj+/h8eqfyf/8ahZ/frnznwbzrzr6rVVWn6DqfmbKnuKP6HxvwoyM/+EOpvCPPOta2Prt6/8+yPKof9F7EFBqG/Y58/sQUk0H9w6g+MAf+SMf9tfIH+1+rxR6r/q7X5S5v4D+8B/GcY9csG/ovR/VA4w/2V6SqSEfr7XmXH/2irZf2vYM0HQ//+Oxl+Zw30F4YLhMG/Y8i/8wb8vfG/njmf/7ztSqM1WtZhzv7XxuvfDNDnQxB5/m/WCv0XtvzuHv8b1APB8b8Df1aPv2AA9FfUx//Bq/96+sP/efpX3U9wRv28kr+ryV966P/NKOEv/EaMYMBfO5l/USfg5+cvmPfbsn/E528f8tdbiBv7xxbTlUvp5gHIfDGQz49mOSXrFM+V+v5hRJoMnld69WQ0ezvQPiV6vvpcYezzRz9J3pUPOH5HVE7Lfl0ThrZPmkOli5wfAbfTcROtL/VtrGKoRItuLatqLZP9Ai1HsVbVVM4wwLILhKfpmXbzlZ1BZFmrt55Oo1hJXFV+TdpxUNrD5qp0z950RrsRhy74fiWlWC5MfPbTb/nSr9ejEBCFI8R99/3DfSrnzlNW7Q7JQuy+/Z+mEv+y5p//NamE2I33pcMPSW+UfzDXMdCiWZgZecOCcEiHVlc5GXr0XDW5jJHkQV+3t5nhgnOm43IiVEIHqYgRrBRsKtfiWAFC3HOTmuiguzhNOJNiJxaxIPZMnm4mR08L/eGLUlfuZ1UXu2jYCjA7OS+UEpC5NW5XuNfKNxIe30kB+qYnEUwdh/PsVgdvmJztQJIlxsgO8i7PovtCPSayDOMiwpaR+eo+HSWpRW/q5uHlnF/BpjIIZvshxve+LxTXY0KO9XE+EQ6bZQhtxIiCNcQP9gyl+KjzK9O260WsGxcTbuYWnzkpMHsJ/nFNyJL8RTTOPkCFQZ0V+2NQdW28RJ6t6jtrBWMzg1Qza7lgWB+m8xjvaUK4wdp8v8QBgoKNTjQRPgTgDlmPCGwaJ4kH0kNohcMXws+E25s+F/k755hV5ordjVwXqRPiuqErc7+hQ2al6qxxT25HCXsUrXlCq8+EyVkim54PT/NkabNZUgMJro9DoQofD5+XkjyQjmmufVWMg1b1o9IzjxamBrVWaoCjHLcycK2Qm32k+JkwKIlGGVihCIeNSlYsGw9Z1bsxYr45brQQRD/55jR6cLpq+CUEPC83UqvlC3/5siwZPsPMncyUm6NBJiR6oaKcrfEs5PmfPPO/ohvk3jJ6tHX5jhmHVOHIrAEN3g3kqR5spr5PdryLTNFh0mK8A4juixOfQk9dxFQ8UgX96w5R+OVQnA/Dc2Pc3mlkJv06BhErPlWAhaEd6rdF3IF2FrhqnsbksOsSdkSVDr/5ISS777nu8sFdDnjJN+xe8sGFD4ud4M8y6bbysUSh/JPCp0MBSVSN9+eW975Hz0ul6/kagtO63jfBRNjirFlhNU2Ap1ST1V+DW9LU3jP9woh7A4zNxJ2XTySoGfP00IP7Khz5NAzxKK+nfk9ZC90CKjx/H5UL9wP+RDclSgQr0IfJdo6OAolMMHvx+gHu+dXuT2rWjyWZTO2ZaH08CCVHdFE5RhxLnB5jM1Hv6Jfc9L3A3xHOYElMKn9NIY7IZMDGb2L5mnP4BEZuyweVuTqUv8w4Gqjiz91spTFD3+g78lWKWyua5xpC3Q9/e4JoYa+u+KS0Hgvge2DskDn/fXuAE10iwMt4DQaY80PcAohytC2W34MrX4cGUZQQyzkG1dr+AaXUMVwQfnfwqRvAsgqKKg/QRmQYW4eCl2lUeud91Q4hLkgTjC0woJotUupWI0HDWsI2eIsPGdC2O/WVt3EI7nIszueyz5/+pA+/lKOZ+Du/GqI2Tg4L8Nv1/qFOoeGSQbwsxlR5i1eiy/F3RBGHVY3Yr7jNo2elOFYTpZw8pKCOGAEFzONenOMr4Y+r5epMCAt8b/BXSiS0jhR3RxNby/s6flomJxVT7Kqb2LdUsBf6EPiAq8MBS0bfnazBD0VCky/zGttfRZ7R7MotmnrJKvXpayelLsj0BoOgo4JK5qPkJrOYFqRfRRY7/fBu6KBntQIb346QeGIWD9ye3b55EdfgzLXG39tevkML8/NOAIYaLLxLBtwlP56zV4RPMjpjXvI3VqwRBNItFQAtPn3qcSVVnOqVEjh9CEcOkKWQr+CHJbqqeKS/NjzZjYpJgJkPvSNfciv7YJ2qv7RaD4elq562kzH7vkTvzzsJgyEnbt+HV+Rmpk8oz2YGd67CEKuwBWj4cI8jd9gta/Jyhzx19nXr+9ubP7CFmgcY5vtlbG0nE9/vtHWvCVsVsCcl+SWDblEidbMKZeTRNzC2bno6IIDG9bpuY7uHY9srVqhI+n4V7k7fCq9qj3gJ1YKm1OKSNx5ukphWzlk78Kh1r9OUOFc2M0yldNo7mFxF9wOFOjRmbaKOQLuR+mubpM51JZWlOtXKq6DJPV/+Xt3CLlnIWhAroPLHe9frN5GoEPOjdZQRYKiQjl/XltLzCXm46NZcFoYDtMcEV6H3Pre0ft6HFGop99v24SufMguLFy/fFLOD7Mfz+37qCAjUX48Tvirls1srBbY1O/krUqpO3nvTw1Qw2tqzBKqu0DoXbZ6wPiHE4JiThcYy5h1+f03ldSM4Rr90CneXf1eM4lnjyzPADdBxWbEHcHHA4aQyc08wlDVCKMzMp8DOFXq8S0wwBX8ztKZEPvFac8Opo92tyg4X4lTv0wapOC+sFmmFL6hzLpb3dOnHdxnC682HDDwLGP4o0s68OmUkDME7rylghKI/0StHbh0x3zCZiPnPncXLvTyf8iXp4FNMnu2IZ69FHx7TXMyXvcS56LFkPlDEXS4tXOlEbOffkHhdqoshFKrX5BxjBsDtwhUYbtlOuVoaTyxOiSkuelpZwhux7KhhEx+Na2e/H2/RkSiiDlerxpr8E36w3HymU9P6Ry/mNwCidTuMJI1+Sa6AHDzZEG5iKBHHiMYLuUxuwRY658122GdU4FfX311hhFSLIhR5bXKB5WsAFiKcu5eWnwmi+eJJUNLH3lJe1A9n9n21wAksWmVMDhfa1FfeWVAMOkLhjOEzq6lbXlTw8WbU9gqQPZnlCjfeqYvzILV8rTe+h1eJqbTxGH3v8rt8VWVzQBbzAOd7ba/13pF8SU/EXp3F4YFXwW2X9O7riWQzsLACpWYfqyuEvKU13d5WFGm0nGuhO3olsH7S1uYlp0yOvMdkSher5OJ3qbu7vjF9ywhfCmjFAh8zclSDAhBHrMnTlelkwQigzxQBrRVexQmuUu5SZ5CjPl85mwxQgfTpGlo5SG+hvGuUIRMrfvIADu90nYQsY2ppa6UF4Y7OOWKhTNIvpSdjYX0il2Y2QURQzQGUZavdEmkk9hpeJ6/pLWp4SU/0rQeM4wSyg5XeV//6cTgNLVpCLy3eYoPuhw12QKfHfLIpXyk1RHSAoiDN8DHRUUaRnUpak4rqD58zel8a24ciNbVDXaXAKz2Zpn4b3hOaMa4XruzGgQ2xWNUy72wdYypiJsK12nMHOEd66FdOJa8XTpOu+whe1fmyvb23VV8BedRsArR91KErHUJ099bAtdxxY+F3SBFNzoU8EQwrxDAuuCpecnw3X2dj1GVfhK5njzzLwAnu2+hzy+FomnFaJEgNHNOVvDWyMO+rvmFN9eWB1mxH9TXY4oGMwLfh3FY/d9LJ2McQ8r0c1TJPJTciWPJ4+eJVtvokNG3s1jr4Aene1xTve0e2al+PqnGWbLq8c/KJuuDyNJmr7J3P3Wj8jZpcB0RXY7qjl7WhfsxFjVOGHJ9vZFUIuPNukM3o+RVM+HF1RKqtHOhH22kFxrvIUT79GjYcrX4Npfshj6jJT1MKYT8RKO3rCZTaKuoQNDO6kusJc3Qo+yF5Sr6bHAexVEWEujpTbPJdNY2byq9tiuBSC0HL1PK2/sJKVcCj1GgHd0jFyxNlFUb6W5f+sZHw9JqEr6r9oqL1pH0cdVjglLQ0jBhqrhJiA2gBI5pt5cB2NcCSOJds4QuVKDHDUl7v6uqG+yZs0ZTAYglq4giU5DI3ZeIqFZAiOk2T4wrIt9BJdO6e68dbJ2vj6PWgT2HXu/sgCufBuckTRxvzHpHe8TrBcbZeAdpaSv8J17jXFlOf46UvwD9OLA/rooPULGMJThE6sNhMq1Nh5g0Ufbbpklw6i0TfETKZgLxj4ULXI3IFPbzpk559MpEQpzYu+jQfc1S1PvaPnPeqNpJh/qCfa75qPRkm/QTr9Zzyo8F78mafD6XsO6iTU7UyEz5ehEopNVFVuXiGKgP5BIyMiK5yK45NB8FXSsKMd5qJM5ufEqnfXI1eqwYerQrKmkKX0MKBrrZgJkq/eSY4ZdvktibRUJ03XzEkX3M987InbgMpqIH0bPv7ceCuQXcgOGxevyx4d5qCICpul+wiI/F3BC37otENp07BFAq6uNOZRMcL4aQKJ8tVPTzqj+NB6IS5yZ5tKqer5kpAgKP/qFI1mce+LEHPJCQYzZD8VcHSWCElwogBqJ1i8MXCFgTn7FiEszKxPzqUfzkERtMlm+o8fNV6w48DJ6sx5sNE2sr43TZ/oAW1lMWmdlKZ7ZbcqOW30mOYy7jQc+woX537K9cdfjxmlrO61H9ViIbnjf4u00AadyDW06MRj2Rf3Zs0Fj+fF2uryZwkf8KS4AHhsUo9JXquj4E3ZWHxKLlSRCSwu8tMwZLvnG/yTuKnjU8eao1kGOXyp/QOi6tACZ4zNn7Gy0uYdUgxJa5AS++AXuVsiPXkYBClRS9TFCvoR1cvTsqDPcQYeoFNKb7u0sHHjfx60cBsZJC4Pi0qSqI1kOabpDIYLGn+K0LCmg/HDfiV/rjg33RVilT9RYJI0nJc3ZQROhDFF4z6LwIMsX8DDPG/BG2Bv8IMgf82xBD5N8SQGtqVof7/V9FAUODvyH+qooH9NxU0/gdg+DxD3o+/TW1NNrTjP+z/+E/UM7K0yH4voQ7zWg7F0Ect+89Wav5V3vuNvP/sowwvEvtD0jpb1+s3mr7U/jMvk23ef8b/ew3wn+XWdx1/ov5fMGQZtjnJ/mcSh/3quEZzka3/k444/tf8fLKzaK32P6/kr9jy21DjFbY/VrY+wN8hjPjHD/YvxUcUgf9OEPA/O/z5Dr/W/duk/8L7f6zy/4VCYv+fhvD/31Ve/vfAe2d5oXr3F3jPmIRWgs+FPPMt8wUp6QuohSNIe9i1S/h9zLfYue2ROi+M1ViFQWAAoNHWCwOtBB2KEkw7AKv7TB3SbEU7PClSpyhzp+km9DVUA11dLun6Y1JGS8RUw0STucN6pOemilZ+MzOBKSHPf/LMVLvfvKB7t6H3PvHmYy9ymOheQpNSL7NFZQVeQzp7PR4dubvg2LjMV9OeKHEgC7IhPUgRoqigWAcRE7zweyL99DmFkmGepo20Rr9N+dxtFxejHGeWKkTxW29es7jq1xigxYUuSqUA0qXd6smfVA6/jzSwxu9kkiX1k0gbR2Lj54HRVw3E6psiJ9xG02p/SFqg8CxO0t/0ZDElUJM6ceqNlvZTt48XYVqKO2FHyUtAilcbhaHInEzEG8+684N7L1wA7qoncCpXrdktaH1nerh+EVqjnhH+IbLz1/q9lrfNnISJUGKtUsOoSgoFbHthO6AbczXXavJQ+i7JQiJ8hrSUxH0g5roB4qoQ5ImQ9sR92Knj0t7rEtw759H3ZoPexAHUCIVwE6xzWyaz2HAiY9P6ajFbxWBDpw4/IRSOifa3EnodFh3hWWfkDPVevDlNDNGwQwX7jLwgC+FT+ZUNs9GPCvTxW8Q2hIwQze+L6gmh993BN2To1DIsxyZTIp07P/Xh5arygiaiBro9u30F2oaQK8SqDBx5S3hsVi9SWHdoQImXcLP0KIVQG0aSb2BVGucgkkhM0mPBeDCT2MzwiV4MPaY8dKa8jd2Nik2pgUFtcauTtu8utPLCMvRI8QCwRxJxO4YwnaDpnlfW0hoK+rgc4U2bRFm7v3oBmuXYBqqi1cAEDFhJSvccBBvL+zrU+QHNsyTrPIF1K5IoQAb8QZoBSe0FyV3fqgdJkSw4xyPbQFTZhpRZkdIXEllJoSrI7SdZc0hFPlzuS8Io06Ak15OefzD+l85Q0t9JkRGpEqcktTDSgh7EQmAtXpbIJyYUxu8SfSnHEnqaNC4Sl8tJLni6yOBGrSIKAihrIumJtVCydF7rCtB98JDZzop4KyiNrJ5M7lXz+yZlaXkX+vGKM2dSqXgRwmdCnBxgoUG5FzPikoxME4bLyVf+qDcDopSEznP65QelZJTIOZQ4ZOG3uwqTM7GuJFvBnJjKYRWHagN8+rZewbfUOpGY46dfMipvkA5AziFIYCX7iMwgjWn0G4j6gcnqFaUk4LG5Yf51Ce6GxL55lfNAyHMqy7pmIL4Kevbk1Uv1u0BsDncs2ptLdYVPIvyE5BNvS2KmPx5/bpydWNvpYYKwfKGgIqoN4xiIbRJmylgVDxC4kE6FO1ClID6kkKjpIfjFufN5Yiow4x9cXxo5Y2U18ZOv4ooPJz+o9hus12+k39HaIxsBi4uSQIrRUWB06LHaIq1HTZVVVTIgnaBso0tVYmonL1Uv7dgJt4VzDiy2lCK2T0yLDjHWHNqqMlULvBxeslEzZFteYnIrGWVxFTXHjkcxayjUuiKbSIoN4ctL7xthC0O8y6+3Tsjxj3Fvpiu0G0sPxbYZIOvFBMR9FCtLb2U1Mj2rcxXO6dJmHlbNQrRGi+x01DrrlY2vMr5+QSSA3bBaprF5nw5NqUmbzbnS+wuFDldl84tJPjk3ImitDUwZ2EKVKFfRY0Zl/pa5ZmgRa2w7cWrRVg9B1pweFwGvwYKKVjKHl+MrY9K61/ixO+TRDm+X3kTRsZE4l33C2ptfBnGB5YhsdGrCKG1w+NJ8sUxmGKaquiRffcGe04k5H49Vy05aRnbacQVlBSi1dlVkE2+V1tZd5gkN3BP0BrmAp1hy8TZUJwZ0S9cy3EDzwKlJ3fhZ6jR1cxY1Ekiig+8GsYukfgpyhvsWlea8RQhb1Cs4simAqObEEWrKgfRzIlsAHkcXKuxumqLPE7/flh754Wh7+xgF7hSv5gT2XWGvWR8l6Yit6QZKny+QGicIbr2D3rcjZbWVmhrwitsKoQL2BYhMAbRtAlHxM70YyXATtgG+0L2fQyep9SRkMd6gKmVfMV7DX0Eo1BYeHN/YM5dHgJzoY0d87UFqlgRCHS9+YMahkon71NZODIjPiLLpNCtmpFl+3fMUxnW4qj4Eenx6ErqSVvfi5bd+a80CTbeJhTQyY90L5ULx/eL+IsQ1X/+Jleyqr/wQWNXrxBQXnz13MX0fHRk39zZ5fd0D6G7P3lAcWp/4ojuiWHWXFDrrWP0p1uqjk5kaoe/P/hdfBy+MxV4Pn0ufJXhLYjn6yoaL61KBSEzv6AOwA+2H9l0aDpPVvEKmyRLzPIa653QKhPie8VI+QNz7NfQJwUFwXDpbcBwM1Uk1r6suinRzo/jSmhjxHm4oAWzbZCR7et+feMFD5kMb7ID3/ThlSYiB/apnXIvNuYc+jDmJ3c+w9GO4vH51sa/SJhH3vxUK3c8p7dULR0gxZry0uvFSQJ6tcvNro+sXjn0NCp5QWyHghRSSrFqoH8qAJfZdscz4A4j6wehwioUkL9Z/B4Zf5gnriJJnutYLSYkv1sS4wFuiqGpVVA/HLbHHBsL4zkQ4iZtYyXPsVzery/DNT8jLTn4O7VQnqVACrm82+IK3Gxhr8/1krH2Gz/CrX9V2uesZTPFxZZmHuK48e/Kqnx5ViZ0yBTN6kG3jzmobhN0ZYqQd1zGUv2ZHqY+1V/hMRV9zJX54PQnc8wknOF7H/b06Tzlh4cE6+3bbH4eCAvZ+GgrLAB1lpa4E4I15BcD4Es8KWkbDPR9+DRo+TkTqhhO2cE9QgB8OPeXZqx4g2ZUzRk6MM8ZNGkV24/EDuFQkwT5BPauO8G3VL6lCOOz6pbPeOgbXfcCyWqE47CRtuX9V6DMQVfxlzT6zpqWrV2dwP7+w30WGw1uxu3iivNdeuxI4ADMeWiD2MUBxPq4L0XfH8tnXxlyiZHUvAgpe3NBiwOOV/yyUkOOnBjJ/kfj6EOy7oxpbvC1G4oCBMKz9AVx2bixcjnJxKMGyfFd7/KeqnSMvHpTVEZprd6Nji7LD25slQPqvpW9y+B1vGk6g7b3v+Gn22PtlT5/AGXM3CHMesXEVNN/eRiBHmWDGarBQUcr/vmF7S1sOIUVoW0feFIbri+euRyPoB4IwyhCE15rLNP6GCUIW5PuLE7Eo0F6lC3IAJyzAiznX97Uzcv3iYqNBpJ7sJq4rQCvSJgZI2r4vd+AcQauDfmLvLer61UsVoa7e6my6ZfDPhjjBo0adF+HPVX+d/iB7szJflTmN20oJ5fNW6N5zCx0PYFN/IcNg573mGHqEDJ33+FgKnGoPq9+oPiTmWNCmKK99Ys2nBVl7K/kofTrCfb/ghPEB3toGrjRmInC4s58tHEcu9rrXrTxfDqSoO7oZOu9jtvuYhhPz4yhjocw2HXy8GJctCnEiVUyEzdbbj+VUsjW245+InYidMDXcXNuhdFvS7V7Xj7Oh74mYBCbAC0U/2AeqP93+8LzQsG1/rQT1iw4f5U028u21AahhND8LNcbyXPTRPJc8OLHeQF6WOrvpEF7PYfBrO7UDCxjpiNOfKI+w7gaK1JJYGcUglF+zM/I7KXLjO1+yAWpCIK8AmFT3kAT4Fwir+2nhPTW/xxhAo7sxyacm4hccTj0dUEi6xL4EnxgJeCsu0A666Ct0Hx1tP1j4RHkvT5MRQ39uJBCvcWzru8rREf4YLILwj8y+BQoFd3MmfufIc4Mg+gIN/TE6KPkKm4TiFKXYmIL8tF/my4YdvQwpKdYK0sWTFwivAytyld1Lw2DSzw0PGql4Yld2BFkHtKNH9E0zYtPE3s1VbM12xxi58J38ovCCGWjNFvkbqTMH9soLkGK/dvk4osN+bgO/ze+pCQSPKvkliJ3jSrE67GvPl3Z84lySpLzQLaWwATWS/snXRZLkhehLsj8Y6k+Xn/b3XJ1VFqTwXrFvPs9axZP4/rOfsxXk8Yf3kvPr0tUYy2neiezgI7Whh9wRTwAQE5zR2y8NR5crAY8FVb3T9thCypAnqtBC4Nj/7dbMCet9uSY82KY8W2Q8uMS9imYMUAWeuQedg77vY88FAguvRKFAnz5Hyi+E2JaaQ1NM/NxbZIJNpeFDrGFZpMnCEKQmrEfLZIN/zJl0ZmdY0pAK5qFX+J5+0kfvklvpiCu88CeXbhDlJi/lFq/HFzdhBd6PwwECv1if8fXvc/9hfjb0tTrp2mc97R5X1BXyARp40p76X0J8QtLf+//++/taQr8cwxqoIsEEEmbYFQh51sE1MSS1yg+tiDbj20652WdfeGUUwx/nwX+NH7+RX7Yx17KRB7Zxr40x9Jos1f4iBv1rBr2lxox3F88xx4eKWwI9K/3DTn52w7tw5AWow7dw6J2BybfrDzc9YjNq9VbtY//zHct/uSNd/WGfzPnbnd079CUmhsB3NsBoX1q576gh/nxxBfp1V9fX2qR59vwz4/ePszHH/nJNb8w2gbQr8ingZ02/qIErn39w+R1VH3vGjI9sUWvom6VRTHLYkeEjn49M/zos+uR7KZWQrYn4e3TdUsiLdnrlG9ZYJKyjx6bfdDDvIvyr3E+Vnl8euUy2CX0pygRT3EJbQSVctM0aoaZ+OKDn/V9Fuc8bGFJ5tntY/JppA9md9wSTv2KjZL857OxM2TyhXW/kb71fcGFr5LloeYS4O9mPTcTm66vo4fts9VrFb2Wd3a7o1nRxbYfmbrUVo9rd8Gpree6SiBgIPf+utbMO3zaNaTpyoZtHubYRBpqsJWHmpVazJ3EH9HEyqgnA94jU6EyRJl2XYfOYkJCNG1YxQe9ROAeRMunLF8XId+soU24st9qjAmFF+kI3ZU3cvnEl5ijMFa22QvAZz4KH5qPEKBqkPc6UREqiVQAN08Noly4GJlRt2KS2HRCC1BIfIBnHACLuBY1I+TZxyXYtYQm75Uht/T32QUVJQzQ2B9ZeTJeBZdYnTV1nmbPJAn9lQ4tHu2XU8OoKVxjnINkYsuv1xdtf7028qWGnf3l/vIRNwOtERnyeX3VDPr/hcC4+i5UR9kXCM3Iun6tRcq/G69HF0zH5Z/9ruO5HeF/lrvjsMYA3zfIY1EWP4M/UYCG++0le0eCpAOghut2Zmpc+UBlEZln8hFLRZBq1vEuSRpuNouUaIXOiGPFn2Sa1eo4SaBoruzuhg83v+TubjECzuNgdbuS1G7aI5lLNSM1mfIzJ1TLfTpjZ5vgJ58ero6svfptJHAJ0pMkW555VeEwKqpVuKIbgk/gbrh6PoGYpOyV/aV0/mSdWVrfiOt3rcVQfwY1ElbBWQAz977Uvyas+1EaOO99iRfglwjv4JKY4Qswx1ZOavWADVwvmDSB2ikD4NvpKp44DLD+Sh7RLxYH38HioXvdOHIHMTvxu/ZNc8iagWjc9nS6qxcrWRcFhy4KkTCBkmuUuYqPKqRCgTFZUt0j9ppVNfVWvt6OLRG2/BOvU4tmpupfYngGAw5vW3C7BQB8BAloZcMZW4T5BkiDLpg7szZyOat+G8/Fzl0ZGTHYolwOdJypsubjk/XpSRt6hoIWez7QDt2/NxJ3iiFlt09M3/T5ZH+qCXsaVfEckH/MNhL+e7Ahaj2NUcQTFNfLy3EwYIUcW6ZYy4p220usObuPwy6Hos6wgaJJfMaxX9Ot+S+xqWQdgPfPd30mFrhbCE6Bb984jLxwTnfA0U1PZmaIaFkgPx53vBJXXFIm2E7E8FiL7rcZh0p+4AXBC14KPfTknPnRQLqfmXN3ca6wrXwBGM/b2aYBBJ0veYN+qUBUewFNCjfCaGbo+ASmvTf9UHDZWustzEu+3NeAAn6/f+0nds96O5QAoCAq3PkihFf4kDNKnM/dyKRBmma94KXMbUZwervpR4lFH+MaLYPl4M5KmT1SFROgu1wIuU23uEth9FGkt1RvfzVmiaMGCWsdrrnfAN5+oRd+jrFDlmNEX2pUzKek9u9nKH9SbUwp1Ej40OojH42yyuU+iKreCVi6T2q6+58U0n8AEOFdMlg392E21PKh5GAQS1lTVlgAFoWXizs83oguOtlfZo6J7plO2BV9/6vSh+ZkpWzE0W3ysXtPmnhxFyDffQxwXZKSmQ/kkhCkprdRqkS9x1ipQp5ov+Q0wraPu5ueOKWj7WHmZFdUejc4AF8pmpswb3KvmSYRb0NWhYgW0NtVSYeJlX20i1LnbkU7VIjLoH2ylP+EASY5Q4DkOaTD+Q9SKVE4WOz6J853x8p7ySghfkvAN2+rjq5/L/7ryRiasxbgDU5qlj4EOlPSRdE627u4FyX8/ookMhEivqkQ1alxkXHex2DynfH1TA8pd5VH0qYx6p6nO9kgKOk+NRH4QmKDK8NW3ifwi8Jr5oifoeLPhC0YBCZB1rvrkdp/mdMgPycSVoeNdsGqEcIYtZRNflJbfwgLAiFwRMgerSqpe6pV+t7Yzjr7t8aH1PbkBU0C0ve1DUxkLHzilqxluxjj7nLfBuTxwNDnXVo6tAYf3uNvoUnRQyBwXwUByHnKuafB35bW41df3c1avDQbiZuqCp+FeyC4TEZ35jJ7X6dLCaTGIrh8opxT+HA7O9Iav+9ld9AmQvJrqhcBouiqItxWdL8gLVJP0FHhpPUT9wT6gqXMQy50aO/Bs2feUsMAhjspgukFRXU5oEBiiYYYh4NhXrCk1M3TGyiOwJnblEJhjtWkrMNDfkGGILzqlXZhSjLdQ+jbdardGo4n0g7Cp0RdtJyJQOn16LJOvTKlzaXYUqRtxe7Ck7FPabie3E4llgE7dC/gd+6dzZvxo9U28zTOobO2ID5FG1h/mBbnsUCk4J7y9NIgSBO7M53rK5zUGG8oo+cLG2wCkrYvZfnKglVJcI74HMg1eIRDwUnQj3bSEJ4TgbtpXhikuuEggey7xGLiBGnpkshXUuydFDeOFvlELTESKv2PU27xHXkpI8rppr5xKXT+ZYIU2WJalSagvcoTJIg/WJe9eoPWmq2NNNUf+LQu9pdlCLcTvQtZo112UuFG57mUS6ZRvPlTFNe1KTLB/RZRapouHvalOzmUqCXVCV4GeX7y8YPhEOj6ZNMNXAhRzzKd9HYZ7DX/dvYcfd6GnHJ/zrnAIhf0xWnqSC3Hj3UyoHqumTiOdDYrw9aVREbRkgo8+y8/xJdRIdD0KOIT4CglpIdo5LaT9/Q41BIqHW3dF0uVVSvJ4b3al/cHVz8oUVQ/egCk+sszHlllK4nvsdaJ4FfCWgnd5jX1dihpJY7qom2nzhF2relJ81EfiGNMx9zb2SJmmgegkPNB6D8ZP5iVXaBgFSG+jgGGAxqriuPXGRZCkgEDP3u61OxTpfJA1syrDbzNo0c6P+o0wh8DaD/2iZ/bqfVK1yuRBUgt6aiYPccpIfgQqnhWPYwFuOQhDXNei6hLVMJnHe8j2a0/vC/MHAYNSNXyhZG5eFNsqIReyZUSV8MFpnuA01ntV2OrGk3VxzhH26LZ1I4uKMJCXU+PNHB9O5GZSFDiT3B+Pctpn82a4sItSETIZIV1eMba8Qd+tmf6TUa4mo1wIlbQ9nyqf2ZyjyvYTdjnseLYog+6/PHhZPIy+iIPBTNdKM9kO4MRqrUnrpDYS6QkVyC5oy64vcDq6zZ0Xgs3ALJfzJFKEjbLJGR4W8yIPXiXNx2SDK5kecyDaMgW0rM1lvjUjD1kwpb1g6QeNeI3gT2AvNzvkfL93TZcLmZ7DnJbNgPB1kXjfVCGphPbwb0nIvSkQuILn2EcGZnVRZ64Boq8tDsXX3Vb4NqaIoZILKm1+0NcRGeDoZX1KL0+61Xr9Juu3OZuT7rqgw8h5cDlSIGq0ilU/GN6KcPcdWiAfBP41cVJGDtC3Y180sqarQ6bMltUQnSb2VI0gD07DF+s5/LY6M0EKVt8cu4/vGgdcQHKJjZz79VquIWYZlH1U/lzySR1SdkLDuI6qaZA6kw0U4Jp1KUraqyI2hKM1uChcH2e6YPNxO4uEatXlJ3+2NAw7x8aQXWSW5bJmeFdctip0VkAZlzhLbqGlNuNLar3qMUHV6n5yl1VxdFpeACyw7TS5v6qrElyiIfO2PEmKvEQUdD9S0wuf8OsZc3OUVCNLSZJ1VLAaHB8knSXFct11a9kcDtc86aJwBshNEW7DaIMZh5XuKdbE2C9MhyDT6DMV4/cSXq93R4hrA+UEu0XUPmirHWweVJqT6A7OUU/TMqtB83iOtbkBCcsCfFnnDHZ6QlEhuFDD+clRM9PXkI+bC86NWMowgD1kLbdDKcD3iDVK5jMGZcumEQNmZHFCvOaLo7d8r6kgFWwbyFNnPS0mcXKDr/JFHxALV34eUnzBV7+lLQIVeeGSFp4WYbMkXeZiScnxXG/K6oR8S3CR6Td5Es0207zyIPivrINfD4tI4z1GP6/RVoBKHHDy5Te3XsSJ2XJlX5UH6rmuth2Vw7vrIBPgBKBVN6gG8wRmNd9l+ucyaOnoY3FpQzcSdkCd4tYFa94U4XRQpF0ySTd411vxAPotj7grfeFJqD1nYBC5zNUJf2twoh81r+JvIxZgcaxDnXBWMK5NIHFlYQ2ilMrzJB9+DOfOCN89SkfhBpvDa5jo1MFd5gRGwmWLsXxs0JWwtF7VKLcXI2CyoDnR19dcxtMXT0CfIt1tvsrqpLnj1DKlO3U9voGqkqEG4M7t1iFhRyXHx6xlR62P3OUDUhshF1yGhIM0dLihErob3tmucX6460mO41J1PA+5MV6SO5HOudQ0YtO4xZr3ZcZOH3H7nDKf7hsABl84vPBUV25jZ7/gBqdc95s3foOGOH0xwOu86+kFGQxYGDtGRt/YOkncCMbEILYWTXlUfDItFlA0ISk4sfQADuTA/PTn19HmsS63XBspVW0D0Qq8CGTfc1VtmPr3ibmfjKWeBrHkGCDB+Cd9VX6ynV//0So1Un4QuzBp8PAdihlQY/+Uj9CvgISBK4MDPuE9y2q0O3ivm4NmOhyUjP4wO0iORiMIvVPzh9IfGxMYBrOVsP0E7DLWcpWsZebgn9H3lzFt38PADkKW/bdMNq98VCqE7ZpYmycQnyFoD8hLmLZhcgqU9yfMr2QC1237APNfUZa7d2i7JHiGSx//rZVR3+RNLSWucPUFZpOYlkFJcc719sRog2TQTdToPI/w52hCLiBxkgjaQriuNVU/eesShRxoXJamXst4WnUWmMVu8Dwm5MZH9SjUHkt/iuNh8z+C0mwLliDB+dYSv/vdGy41Zq+P+NYsItsaN70kilvE941v2BmslMVBy2Uub3Ceq3xKH4HDBYM6rVI08HuP/GPWdvYIXGFLDxVx1cAqgLcs0nt5nqhbGbHLB2JXVUjfY+GZkfen/bLuwPtu5QE6e5KCajaAUV9DuLKecOcetAMQmppqx4HkEBCxaQA2oe16cXVzmmA3vG8YsDXdOcabIArQmKw1SzVDc8wJMbWd8pLeCvjP66LrUvF3WsN5yfrMmQmxM79a46W6uPx83AgpT3anIG8Ea22OQXIChHvlEjHXQZNwL8RrvYw9pFa6DHTH9JYN3prLt3oRNhHhODd8q5uDfsOgjvZrsL0QWz4S7cHR50zlr8/hgSHO0B7zAOp+H/GtZadCUDlB7M7J5pGAhqt9xSP7eVTrwMHZHJCvHfixxMe1g7RRV76il+ecAd9935cF0E6QYxkCI3knyPbzxCRDa5lW4KfjJLxwXjuCoDz66jqu9Q0qx7UGsvaF+SW+CcW9W3hV3sfL8IahiI/vAz/P6Eqq8OlmhYGOfBNOxO7RcN3q3nWsyKFvCK2twd/Whn43aUp4ZGH6DmJUKvl5x/Y6vkhAMWSc/UgPHdMm8MZU3konAJOl1lY0MWdTCM7QkQ5vr/gqvEtbM7vTUWS7q2yhABqBYeAn2JMPNgiIXvzw/RQHc62Ijyta/IFKbUDA68NkjfY4k66P+2JuXNJDn2CEfSVPcm8v53vN/ZYNzLbYSbgyjkts9JZQ1eFMSDC0ik9uBCXg0rKVaq9FsUTtcLNctw0cBxGsRD6etyxqd7byE0h4rBRMGHCEy1sMxRIfg1Hd80ufOfJOQE4345jXfWCfw35zQU/ZnhDgCNX3+6G487hkfwD6CHS6131gdkF5fDNdIvRVh51uWkz9DJTZ01N9A6CcjZATIDiSStf5ChBnBaI5Il+oCp80If0esFpfc9/cear2FgP5sNimLHJqu2VRDs6LgTnGfMQRonnDMXkOL2tbI4KYGK+FyDBYxgMg55RD5i0KO42LUxxYc1IwHl1piuudxqIBHnsUx2YEffzrXgyn4+FMA0Hgc3PgXMcMj2ltyWUtim4NxgU7ABuOhOYo8+zBexL9tyQVM8U97zQR56lQPrbLSV/Y2GB7W4ky7kSjmWU7JqO6HDC7yHmRbADL2bAyJRIkojmn+EaOFSscodN4TaD781glqXmIiXUeo/s8P8Ab2Bq7ZIuXa0QNUnL9NZaEP4TX+Yl4pzlgbIgROND8J2qZCEwsyAwEANO5OhIPZja8+S8VcgkULuzN77RzTbCiBwoEgseQ5ys4z3Kayg2BnPM4l9fKIne2b9W0TATij6s09JF3TUBxuqv3xsNkbvRmlm1621kObR1PHMrr0bY5cZDv6lryaDOsj0EkgyBTmaoWsyN7GLkIjAkQwVd4bQQMMJewk+Fs8NiCF9E3hJDbhrV2rgpolAozXaAmNDPKHjOyxAwFbBcfZPNWmRUgD2QbpcIeEQfKk0skrY8uEhM/85laN1Fex3MOQzX9UIvguBoQe+5h8CjouoG0obsmzHc8d/weQoC/rnL0GFwhCL59/xnQW3nruUTPlFqacTT6fiGBtJ7XCr6slR15BfKvKiTY8K3X2JGY3BERqPP2iR+7c1RQ8MxNrR6tkpN0b1oSGrZyF+VD6X3InBhvDJS/UDP9PFenI63w2oka996j6gqgI9NggZUYC809sBJErBAZPdF1JE4XhsFPVJw6AgznroWtVJoWaJ4TeVxUZ9nli7ASR7L3cUp4HGFkVijZgqVH1DVBdRS4fvuqbr7guFJrBLEI/RtSUKBKoz5NFSt71ET5nmDgmDHF0PWNCrBSirUGPXqJez1nafc9kpGY/+nHeTMESMelWq2+j4Q35McsFW0Iis+FxPHpvccWiMXIZqc2bB3IuTMDDt0wqHljdeDzEbh+01HPgwFhFNoDymsCklACAq11lR7TZLjNZ17aF+oy+zjmkEz7eZId9hmCOOpYUbKtAdpbItFellO4qoC30OyLIv14kLG9UB10+Y5/nzHdMuVYO9834QxGHadMBHicUOJ46yKkZOMz34MLu+4xktLa6G8OCr0OyU9LVaj8WRZa8kBoNw2t+dJKItoT4ihQycS7L1jo1SsZSPxQ3zF6BkG8J8nzCavcg9F4lOCGBaoVM6S7b5XxC9BZwjtjeXgaBvQtEfQ0rB6PszQRIdi4G0b3vO3fKTdeKEHHLd8zdJSYf6Q5zt5SCPZmjXlb/hy78hs2aYukqrTXKw7BzeyGXif4UtCM7HSyUTkxnn2wi4zanyfG5Vwvsu199JbLtTy2v5wmfsGv+B5PO+biIrSZzPNMkRLd1TSkv27ske3doCdly6mBzDoNFY3NpAYDCc+vymBTrniaOKTEOFn34zYktQetdKMOgHcAJuKDGhN6hL2IdmP4NM8aD0aVeT5KcS0tFQAQOFKdTTf17spq84u8cMSgAANgD0ggorGly7E0znnVvOcwXAGl39ypdQWhhHVQ68fiRbYHLs/GwtlLQtjXsnzdVSbUw5LkjICjMfdBCPDJJG06z/D8ixshYRvB+7wVFx9T5iBPgjZmCHytxFuWGYBP23zesw2FTMdGGwkPSWvmKGWPOl3Vv8y64HJwsqGIiZywlyUar8X3EFSu4CiuCgq2ADNFMT3xjY3v9TgB32KGWMC7q6mzF9Cp7USc7s/lcWXqDxgY0sOvIwlc+3O+E63gUKFIAITpBTt6bPCunEmzxaTg+G4tXHOZGw7qdYSbmxLK8UL6++N35t5F2ufhVZ7PuA+pqAtH9UUfmpbVuoHCq1bJyWgzN8SmC4m+aqocKgjPP4dt5Ka5Wxs7xlZ9tHg2TZ2P8zwoJi68AMc8vcpH1fsD80nguyDdkTSBSS/y4LWz8sTlmwD9fHmPYkPz8KT05RmeP8+LBEK1cUwCFxqvDKJRnLl/Lbrf38PXcZNYf8hhr3emrevaNXD2nsQgDsz4QM76SuaTqIe1hGufi3uDAwaYlJByw1TDt+yGCqj8qhxTFS/H9mVhDfjHbGcCgaHFlnG3nSeHQKvMx4rSqPA6SplQQUBcyQW3kOL5Q+UZeIccboU/g4B1iPHzTRphSTWo+d2svNMMVgCWCJfTIhH4iw1RXF8wpLAuHmu7bYPVh2KqUQHazn9trYJjTp/VBNWJVlbSXL2xJ0MmpM6dC/7XMZPusULzYSP7+/U2XFj3J6fVEQUL+E4JX95TCePLVoGzxhi83pJygrJbNDUSkFTGdvTE0GGLvpJSn/DQrFrRKyCIf2Z56Is3jpB/viij1UP4sffFIA9K8TXyJ42wiXsQ44O4wgkq5G/PXl0qMD0f3p2SpTnwFr85YkZYrxMjSj02wxIBvc0uSKhQ6LENCWGFQb7SsUmNljMiYYsxy/0+rpFfgIobhl0vhKL54tFzm+weZuElIQ9+6yZzmX221deCcfD7XNDjqMVU9adW15zpPXU6TY0fSCefPlkQa5TUlRNqeLyJuSmLJYKeTMt9ho8QRRqEwC6w5z4Lh284Fmz2263uVTdugpj++htV4FxMqaqyN3a+mm1KoMjdXiOQRr9S1rc2ACeKxteujwY7Cn0ZLUrDewqMxPKx+OwahVHDsKKFal9xAwqCnWHWePsG21JC5FvTZ4LwBx577fTR2ofG0AxhZuNj0OAjji/n47PBWXft+1RJmucf7PFdvKAReIL8XzRd1dKsSLN9mv8el8sGGm3cucPdnac/1LfnTMTESOy2InPlWmm1jY71QZ3LEXMdph9yu04UDoKqO3RjKzBYOk4hn5/pwxskAHjQIBficQi3cw6GNr55YoyWE+EfigEJc00xCmR6ctjVxs8IN4KfcsNXJVjuC6L/j3GWQkgrjjjyiIm5fycwzwfnvSECwCtXjsB3tD3lVa6wf9yDhpzuxSkco03pP4GftkGAw9dI6CTR8TWl1k+m7m2kfn4M9n3s3ud10GeR5aJ5Dpkvt376A8rlXyPjnxJ8z1ydeR4t2t4ajLH9nQRzuzMMvyS8r1LXT0Q3NMvMgHkBbY4gRpmFTqnHXXp+tc3ypij2PODq44VI7/FUCC/XzBffiZWMPxgD+AEJUsFw6nnRKm968LMnuZarQRqj9X5osMzoUNJ2WZJPEGN3lcu3c+n1IG3RFmPi25hTII0iMR/TKIFyddUOfQjF6L+pl3+92a9oH9oOO1cITN9w+Dn24Lg8xv58rspNhZ59oy+PYh3BJy3BQjNtzR/AAmVp9OqsyuSKEl/VMG4n+hqDIv7XpPpQSkEuVN3OhmSjRhZZn78GjjPPO2iJY/MM7eACcoEKxRAUFYzlJ9NIvaSkqesoAGCxBwTm5xtFbVHEdBSr7UKzKf5cFj5KU6SfzA5uG57gr1+K0pdsPiS2NPDgHM1JwJQzLOhL8iJBZGCf958y5SviE67e5/NKAxU0Oievwj5+a55zH2qSMtJRqPNJWDeb05uTt2G+kXWnXzddnOJVC9uJkwbSJCelNr6qbAh0IdmMliI5nvEJhNpAAQvH6idVnjGp9jtPES0qp23KrEi10MJuAmNC0UDMO69VgkEqdmtyql5dLZtbWCZPC34J6WMwWEeFClJTbeg5xbLDWtCETscuj7207Dz8YUlCuNIT7SMq0ms2FwrajZwhVXYBI7a5OVT5Md2dP2Sjxr5rm7sFWzP1viv+4mDko4nxkkHH4bwqaZF8t5waRwbO+TWPl09MpSA2BMqX8Fd0BWEbV0NkC0o2XRzWSZJRcgCWQp+GyHgfskn8VXRHfVaqi9IFpNI5xK/6mJpNFCgFOMsFnb5Mr1QLe79p7nn5KmA4vP65+u57kdQvOQMMnruKKB8r2fMPBDGTln3WU611/OmdCUpIE/owOOCQmYYe4DP1XXfsrUQ/dx0pBPkBjdjMVRGOo5WhzuEe/7XijwVeVooALiSM/pqwxKkgi60/fMfd8MLCiV+XQQvzPdNY2s9shgj8QNH0BWTJyOhfIybzI08qBuHc8C7yRiANaSETZLUc40duGkY9IdLiqJAbyPE86e+voUvdwkLVtOZziNbQafxHCGYh6T7qnxg9j0NP4lTFu7+29gpN0eqb+JfTC/jwUahHzfNC+71HCYIueEDjsu6eINATZ+vRs9Ed9MX25tc3EmWHfxttWBGDUNGo83UvllfXgRL9Syqe8uEwB+laFoljxGOzzP9i1yQpl84Om3zVp9sPVQ8IEE+fp3nOSCJ/kbv7UHJZHQSWoighV6+ePZA48UJOqe7paaNGgV65ADqlwWYjS9+/mTHF/qNJN/yFkaAM9mMpp+hIRIOZcv7izDb/XQa5Y9UatnAFFZULrCSXEpufc8M3xe1HPd6/Q1euScHS2EnO5y/eklA7sLywj4JaWarpcozSKC4HX3SQNGQg9i6kWYIpoAsZMZJ0As8QcZyyj2bT8xzGP2EQEu0+H4jqf2w+aF3hi+kMTHLy6+ZLkGd9mGXbdmJkkvTdEqjtdPfY03lCeyZlsAbFG7Q1LHkZGOw2r/gENif21G1sIfgdSM1BLwn0O/jint51twIpgzFCGmnQBRQkmwsy67oFIj58SbWcKDSF86G5YQl4vv+iUEJwBpvYrA7Fx7Xu+nQYx1Fk2L+gz6BwBnzg1js3rrdDpb4Y6Y0XYdA7mdOIM2Iw5s6tjAs+pS3G9ws0pPCGPf9xmAkrRLvUQEuy4+kAflUI4fCaLb9Wn939a9hokChxNKNGraabvu6Y92oVWzDSZ12sYfaoIiEEHUSCv21T3NeNKlUSF2syEEysnj7MK6NExFPxiM9BhPBEbJM7oEQhs4jRpiXWVz7vatGGcdyf1T/YfNL0BY+Y5DmsaxRO8gzM9ps9opz+YHU39p2S2bzajbyH6nC+JFW9yTlxrOl7AAunEK9qKmLXRYM0iK3GTI6bPyv2fDrkA/J80fkylsKzsPvBSeQy2cfPq+5l7upkB0Yhn76Ye14MNT37zR8tIUmMzosAwSxo7WaBMk2o46yui3TxUY9hD3JJNUrHuQOx2TJvFPPolcZ10nVrwBAG++otCHCG2LD6m7cLXV8ZBLaRMQA5E4ZwOzT5GUUuAMwqiu8X5GqwvzzFJbaAQx27na9bMafrT/Rx6nsIIPfknbYJmm2YGuS2gyRWi3+Rlxe7SvXUVxih7UZ/wzqxRb+eAbfE5SyQt7X7EcwwAjMiZJZbnOsSx89P3PKxJfHQJutxw8BHPHTTc58zThSHVgs9/G0J4ZvPSVhZ8d1x3DXkLCeQc5gSVg+giSuJWDdqtiNnbqpBBUHRCCa2jy2YwZTGeh0KoCAgqINnVb5nZatsw0iQjIn46XXUD9hTSsly34Gkqp009zUd7saE0fuwov6c7e25blNraU3s8nbmMipZ6PeM1KyK24Q4PJAPOsbKXA5LOTCrGbCDNHzoFX+p0WvseR8vEm8SId7Z3AhQ1T8Nl2YRxwWnwNVh+grxeWOkz2E/eMFM0rACzvlS9roOX3Wru7grlmTPcDpy+WNyz88OQ3zPqrcqM38LU2Y5NfzVBj/l9INKa+9eVSWl0LI6nr8dHfF6hQX2A0K9bVW1Bqocka03SDljvqQVdOqIjkMT0Q9KBIgG5RMYK8iHNhw3cmEiTj85+X5s1+gQ+in+8i/4bIOJY5LeUildpE+0L9xsfweJ467yafEQyvkenHqYYbxX5selIAc3i3rIK/7GuXDxunvOaUaCSmHw0rZv8x1PVlInDw71Jr32j8Bv164seCC4JdhuyRBzkjRRExduWJoEMBnUSHs4gEDLmt5/bI2s5iWIW5Eg6eBCbSS7yqYufSWdB0eM/cLSsCPGwZ4GxrqPDY0Nt91U7fuea4vQeEsEHKjp/h6MwbLcwHUJ/zePzqskunEklO+rm7qEmQnLSR0KjQ57UpaFRMERvLMF4NGoFnq2vgdJESFH1aHNMPkmI+JtgCJ5F2Fuog25MvX1/RQKcFUxmS9G+DbNVEwspuJOiLBbJIXO5/2PoTVQgRZScJTfkJd8a5psO25RWNmZHKyP4juKT2/xbIfdWw87eHBeA2kutA+qrZ/DywReyvUEm1xzhX1BTE+V37n3QcAkOmyIBQ9j4VKxOUG15cKzR63wqWMm7e3vO3v+AQUhWrpSVwyXOYIq7jjf+UK7xe5fv670wm+RK7LwPvk3HuvCDQ13X++Gn2o5sfdPz7F+uOyzIK6zlWVhTx+aof455dpPTe98EM8q8follQtrTaMhKv6VqLJmiQcn05jGC9Q1FMkASiUoTdgQZMGZuD8sukp6/4NIE/6ABRHM82H3kUmFv3oXMcFH4xLSh9ELBmW46F9wD86CXGkUH09EzxNGG7ZPr1MOduiSRbrXwJSgrKgmMA6RRdAvwCROgiL7XDeFFt1e9EodOIhDcWfkPd1Klq1gzpfaa/rFbmx1ah8Zc52F4HDWray2FyB1ERfy5Hx1PTdxjMxi3k9/6XXkJrcPjNv9JM+EFqk4kf5np8reOLT9tlejll/Hmki2ObVJEdMMwAONOQncHXYIimk9TWduDpvP7E1H4oq4Z4Tw37KpBccSHuepQPjBs/jNP4HdaP61SHpElQx32XH1d6LHN+jZIyPgI34fg5GNjGCCbAMI+NBpdSbh2itWhvc4xl6SSCE3CvKThzMUTF6qnbApGO+brP15da9YzmNhACvuDVBSFkXwhSPGmnyRdR4FUodcBucLWU3rdqL7woMHNaISPqz2n7QGtCx5XdLV5lyUysTeMNC7tAKiWah6/qwa+Wmk6g1Okfse90zoI0uPUdvocJqCTG3g4V6Nn4GVYo9SB1hyDGh9U1FzY9e37qDIVClZrL8HROn63kKUcT/ua7Rx0KP193QYatkBoP5mnvaSVUFCPS1HQWjW5W4lC66XbTL2epUwSu9vOv6WGEYTynm1Mr0iuXRapsff/nacBabwG6ECMfoE3+i5o2Goqo6oj8elnfx7PYJbceRSS75IY0dJmTWpJ/PWxV8b10Tz+BsAlzHH+S0EpCkfnOMyFphkt/E4dkMPR3CxVr8UNuQ8DFmYIOuMIaVRzASztUwvOREBSfEenhR1fj9GXJfMuZiCcsJharJU9qKfJw4fwBGPI3gK47u8ZPxp9aSpyTFATMP4iKLOBRdGkg/1WVbUFh9etR+Sex2Pwp89lmakjCcdz2+MVH0wbmSynIxpOAkA1Gj5AtsUVMSVXdaNZfWSfq/EPTUq8UrQhYdMqC8PqdLP8va0jn6mpWoMSGeZVUYPmVWCtdtqkPI7WznKOcRa7xdaCdS4mzUdCuT3aqoJ/IILlbFXfHcl6ZqKEkuR7VW/DrCcqPxeXMJK5i0KiGTmCeIsBQighAteSJ25yDMi+j6CZkwfHVgc3NmcfXIDjPb9dBksl+oY3H/z1twbjgeJUyhEjlP427r9/bUWOgzLr/DOuIJ/NFnlBOpgIvhzD8hfQoyAes1kaB9DjUHV2Av8ffmx1e/39U5vrH1c8/LpxK5rMNT/NqHoEer5dIp9iB0khIgAu8ZDPZzwN+a/yD/RE0si7kOx33BU8VIfBob3CQ7e+YVpXOGjAACnYfylO3iY4Fh6BPFT94cWfzo/1vyjQNyghicdhU+9WYCWuLFuJtbuLkP3N+L9JtbcqocaIrk6Ri+/7H0+n5neb+KDWmnouhjaLkhSfiKWK3XuDVGGHO1We55r8Cc3v8xwf3R7mHC4ahwJFb4crMuxBQlRQBgQ7AuBmBuyuhvFT0ZNMaT8Je8LaaXCEi+mNrWVUKbd0PmCFC3t9aL01FQfGBK57DGWqiW+fjXcHm5F7J3amKaWAqOKBqhLFjVuXixp8X+hFnayeQOe3RSV8g9rGDkHz1/99TKO6qqr99Ms+88WtSJbxsusQv2akbT2UlFUihwpadfJ7m3D2gQVVP4yynjpEp3nEIIRCy+C8/bidHgfXA2wzz1UAmQJ/ReCKd1Pnx/dM8dLwa9t2C3BO2Fl2G+Q7/+mRoKH0/GpCdKqONlrvz/SPpWLI0z2WxgQdJV4OaKCO7l25PtJN6DopxXkz+7QvI9+rgVam6Xf00ODuVt7pTpxaveHiFKEcJYkooJQHoWoCs6y5yoh+6q67sJ5Oq5I4pTHD/2OH5ljYvIvLd39SA+pmOcuGYGl8aJYWYaiTDCIonUDs+JPsLzelZBB0GfptYjbWcVkMQxgyTCjwlFxL3g3pMhJzIf3lA94BLvMf1BdWyn3gstaNS04Tl1ussKm21d4Dv3Xfr3eKFLpr/fuMdvNyPWSM+k429wF665XfLL0M4bPwC4QvOo7VSNwMFp2qUDuTnGWgTquEpUgE+zOTzz8dCq5mmoajgtvXkifua0tJyyFx57B3zcmW+7bMCe9WyCbyF/6JJTRGUfxvTaKCYeQ21xDXNO/mXgZDZx5+6qA7LWTRogPdI3Yf7/aK2Ku486f9xvpRmUR8sid5z8Xnr0fmcM9Eo0R4zgmN1LRNNjRC6YoHhd/VRvpeb/LJzmho0lixktxtB4o17tJtGSTvc5sVSfqXH0iNSwkYzgpCPDfmcOG6tiL4hBq8TDmRf/dSFrez+A1+qT3V1NjVy9trfrir3lRFKMHZZgS8fco1587Ff81zmXkzMOdAJRSV+YhId2tyAWRi0bZ9kAB7owR4tkTRc5iFv6TVAz2fUEQoZ4fdwqsm+K32pRDzFRpS/VCqdm/tD5flP10zCz9fD7cg79JeG95bqv9xNpXR5Su+rjucStnL6NgX2R3W5PtKscEFkUBfYmgMJUYOV29zwA3YmfeDZ27wPITnpK5565+GD/5R817B7spR601kxJFcgb9EkGfnS0nniXp8kJI17+dQzd4pSM3u2sKHHBOJ4vk3RQ7+zVvKOYIaHOlSZWuMgw4jfhI4RSJZrlhoS8mMVIkpvXvxFQlor+zoE9DRRqw9HuSlPTKLtByb8h5IUKa6MI9k2Z3ASddAP5iKcrwvKhcrU1tXBaGqBUvBKst5pIbDrdewMwPHYZOjnztxqD5aI+EEbJ/taG0a0gFv92Yf0hFGUo3FNuZhMOkO/ZwzUr5yrb9DSE8OhN456onKCzzKbfs+0EzNdTb4UiwHzk0aWtM4WYDNrf+4nFnuU+BwNWiB0RnqMFXTU6/XOxmzq8YxeohRD7DIJ7IB1HlEgP7F+JQDdA5TVxeh7AKP8BtBfzrwNhI3OzBkqYtfxVnbW/njC4QQ5uu2rdIdaa9g0kZesSrDrNmHT4Hq1SVYtZimfMgqdBfY61BlReBJr2P9ynkIxYIWiRrMUlzZAC4A7s3pm7YmGdcc+ZvNMISBYd/mxSRtc3lOJ9pDKU8nSs5cGxykzqzRmYWJO/aGbjKgvkNBvKNOXMiErk3IJq1mwIdGtyJsfc9KQtldywjl+JxjqRnKVwhO49lHzb6HA0CYgpeinWAs2L33Pu+Y/2u53eaTshEwFR8i6ahtuzIXZupzdVhKe9fUPtq8qndXxWvwLU2P4zc30K100qGQx5LaJ3rio6pQHzwrH04QlFY/dX6OJFTWOlMAsfXlp59kYfTXShmvn7i5QhSs8XLzrv7wuYcud2Gmv08o7EOxaGwyzhvihvfHtbpJWtFmAyDBhVZwvoiLlaTwlN9Jx2pSaCH4F2P9dpjlyCppiK6Qtck/zWY1vepy5zblBMXuKhJyWcGXeD0OFUp7K9z3vaeQToivpLgb41ZRcU+R2f+tc5FBIPhofF+Lb49SVvsVDd2AaInSFdZhU04XLx9nR9+Z7eVkq/MrI4/hiS37uKB5JRMKyCBNv/tZIAw5SJUeRrVOkb6Omi1n2b4vRtQkyAtNvcrpfNWj0GBq/0CV3hAbRpfEd5F5i1wqxF7qPOEBdrppurYi60o3nKm1WR/55WXi9gkA6u+GagSaFUtQ1T/Rum9XPku6LlMqTwsAW3ixYojRhL79UUfo1whxl6zeHz4wiYifgPceeIfReKsydGVz02pP5QxbU3dxpUqiLEEzUGned7htu0HSXSWiQVXNFuqIIhMqKBd6s9SudJxWf9AjlSh3sAZAa/++kIlg/Rb2B925QBDzrS9oPmVyJ1z5BJ0oEguYw4zGIK7Cna0RbbIwI9zxgzc9Ykf/pWhuzLMade1xCNBJ4tJPw5A5EMqluwe6gNdsVvF+RwMJQzYpazbhSqDn/Pq98INhawTjuijcGElaJvmGSIE+2CUft5LjSE7sZ0rX6m0T51JJ+rbfh3ia2jJwBy4k+QqW459/NCW//KGqOKgblsk0NCn8zDBegDlEYTUmv+TcfN3gOoW/eoTNPitZEyDV8H80AxdgU/vA84fXDNSsBCJ2QYRD5tQvCoxyYuMRiAIbk5rBNw+Os/TWS5umXnKN7xT50liXL+W1ZOKSIx/6U/7cH4GtGcdm3upnKVymvHZaj+egcg62UEgxDMjMY+8umSVEtK9bPHqrcawu40KgyfI9w1CA6U98ednV6+edEcu082t+0lP47nW4st/9SPZ3C4i6LfTipRbwE4EIZDeh6HVaAUCy3FkvT3OG6JT9oX7PEjSkKkFdzauh7gtZ/ed0rSMhrwD6dLz89lVBalg2M7htgXjEvzvnuZE9aRbsJD9UO1iXVfUtypMGO0lTY+c5kv4gN/Ti/w28U4ETiIxvtgpPHzpO0RO5FRbsqqvoFKrKcNBFxWfKTvmIlUIB9LuiJlJ7Ty8gURX773fErXsVGzQnBR6zWHTpL40+TI+m5LppwNHD5YN+oDE2qJFtqsV1yI1y21KX8i7tN41R/rLe7iiq7JZ9StRt5/FifGD4UUmsQA/JwPd3j2mUs6aTBLG12HlryoF7HaSkoOBWIZk/7wl53EaulHfxWkS917p/5k/IPCqbrdlWlAcx6G8/x4Bmw/X5sUAEl+Nu4D1Y6inhu3NJXiAazWXuCLHz63CEz0WuD7Pby0YJLISF7QYu0vgR+ATjnJfbk1T+of72bugdIgKxv9cPCAoFIKvYAe2udjn1FyTEwhl94IJ0u9i8bcs5tUGTM7g3jemixVMSMSIjoyL9DU1aGI1iRA8SGDzxdAKJb0IbU0fjMdeOK1S1hwYJDJ6Yndcp9ZM5UeaoL9TW+vLEzaXikiaPus9L0IApBZ5bV+69K9AaOvuntxSMJoHqUbmVHv3L/+v/M77MYy1ghNFlmIjJqdvkWRpWAAx0+t6PC9BQsd5SnwjG+EG6BTyiFSK4bUx+XAIbO43yg4rjsMbl++f1x6Q5dbbAxRcb2erfjR7GnKXRvYRHjqfFGcY9tj2K7ef9u9xRnOSkELrsdSWwVG/e5j9SfE8Xxgu1oUBa6vhforcvjOTO/OzhyRhv19fegXwPSavXKBd+8KcEG8fzhXIVUJxlvcjdRL7y14ONcJG1lyYNP/sOlagh2HxeMcTn2DaiVotLHirWhz7uk+TgdgWbfXfKhwaqTZN3AA7BU7MPEfyQH9dK/qG/KheWmGC018d/kxBu5SgWvjkpsJCPhwA6KNEsTX+yQAwIkPXgzSIYu0yvc1vqI8bPX9uJHE8wdGG7RxHmZztENGYEY1R17YWFfhUWBP1mAB1KynxXBE7f39raj83+jTTNBJY/YHewFg51ISWQcLLz+c73dqsRraPHJCRxTn2N1JFyM1LGpNmL130HnSt/vF63oeyLLE4xbMe+I42fIxO3fa21H20y6Ts5Pvvu98HZ58nccGa9cHXZb68L1vi7Sj3Q8lwOJJtpzK5fERxSUFTc4C0Dvl3wdSWcAnl5MfHN2p6wMTfliCiT7E5qYZMv0kftPiIKKgZVXuahz9lVi1v6Tskz5PXVK69CKDDkVPQq8ZAoGWQKdy7ELKOrMjmSm3w0oH524Hq2udTB87p2AfIAevlox8y5o2kO/M/OoYqvIoCXj7c6aVX4BWzvfxGmWkKrgmShg5cLiSWVgiypncM22IrRhv7+6s8Yp8K7vm9Llg8brB6ijdiXXPw2qZlCvHufGNGs2/7R2ynEa1xY7iq7uMTW0fvCMoW6GuUoBa1/1wKMHCLMLSr+XhlwB32VLVrwphPblMf8GhfVdf9zv4AUjocMcj3RP1jUU6RFn368t55SVgc7aTwpsNbuP14EIO/LiTBWVCSvLKclwqgUpM4fZYSKk4C4jdVkJOdbbmNK6asI8Yd0BomuxsFmr9GLz5F//wtvtFsMtg0kBy+vuby2ByAqA6JPFjQdOE1q4xpQ0IVMoDZxQjhc1GkFEm+MDiEsn1tOkUVCo6QB3hhNpFZwWiRTHwpmpWqS1wIe8CrY/R1wAVY6jgnjhllq9JD4wsSZhNoyeGNlyABX3Pu3RtsffQILcfd9Hic0aQUPdgHLNCP4ideQTSyiDuQzd/uDuTVh6gBCsXbixr0b2nmzxnT4Jf9UJpVXVsuafUTIYAk8D7w2uBlD72yHVYyRpr41+UqrV9SKPV5gJbNBXNCThHdKok3qGef49fIi2Hpgr+NzjIZtHO/BHKV+8n3hjnOaAJMz+6lcKCpqU2ek4pWoKwCeizRX2l3mj+Hd6/V59l0EQOIQuH8j5z0KOMeEqjsb4TJvZs7/Q0SVsMSI75Lvq6FssYxEU6cv4Gev52gSui/2l+zNaoqDMGrEBqTQBQ6QoU91qypReiO+AgJxg+OhZwkggXQdswGBw0A3wUyMZr1vNbBvqSIwYBNHM/9ALjhgS0mxagSY+4SK0/exGMetLUcnuen3jyL3SYeoingSO7QbUPDf3v0RIQLxBrZxD0OTocQC7ulL9NdsHkUtPIoBIKAx3vjTR7dxio39vGeDEFyf1rcjW58L00V1N2fd0rrEX+lBcL6RPDSkz88PDyJ8OzYMbHJ+HewIC26H0BcQNWY3pNFlCw4T+9vpM15JnjgEEf5Jbri/Fom9Y1Ptbgo6AJtseLQh5TSfX828LpPC7HD/8UjhdcAaStK8xrms+duP6vFmsytWqF+L3a08xT7t2bMoegrAWm1qzVguCO75WkS31/kIHBZ9OLh89MWWaRCetW+rlQYM1uEe7f7U5e6OsQf85jC8d0ITW1AE/UbddcsAlkGcI8Ak4EpYrdakr4uElXTFsZAl7+BF5w+oxedUknzLfJYpPawjHk7aLBHkxH3enhtc3kxFIYDQdOig6663zH9NAR1I/Qw8RFrkeJ1JYu4joMizLvbkqT2jSWeGqoAFkUuh8x5sKsQ64TjLoRWwDFB8bTg43EQFspamE+ZdVsabyBWjtv0+V2HbjW1aKQPSkXFcPh6Np207yPNzeC2qeeA1n3+ag3ahsVTrZuVIRuaOSsyv2Oy/bfTlXY+sRhHA6BU4rcrVtSyzqV4ARd1tcQVXZYypbOFToTCydsXewIHmhkyMTb4u+PNt5Cs1eQINWQg26+fo6dkIQ9XsRNZAmOgiS7up9dS1AMPqGH8af3DO6KiLfAv4HoJ7kryYemfOPsZWed4XwRWHAFPPLAEPUmUAPfX8Rt3FMbfICcS/K1i5OE3ylC7kJg/h221uTG2Fwcmm4GML/g1gUHnqKHR7h7R6dXJqUWBUsBDH/l3S4KvWnsq4wuCy/CjBhqc9Df4UT8P5Khfdm/BrX7shhZdbIt7icu9UNyZsdqrDHzo5qSt8wcqnhYmqSgYhhbLRW5BERhvyUZ4kF1htmmZdA4Tv5Z5fJdcPTo8q2zBdZljrmslW8Go88gwOJWa53Iy9EFoTGj7mSUUg9TQAu01/0oR//uvHZhREu1rbbuEkJEDjNhRrIdjv5ixKl9Sw/FJTLGBau728f9kXYweVb0GqOB0g0iDMD0kgaBO0cuxNUy0LseC//f/ja9c41w/D/pbPAikGH51vZESXC2MwW7x06mTz4cPLM+6YFNpqSgcp1LCLU8Uhei4aPPlnEo5gLeyP3hRDCKiC+6itOBzPz9pTZQqb2va0eU+8CbXZpSBqD/H1BrH3+2rTo+CKIyTkvjsP5qe0GdCptxT04+PgGXgt2ioO5RPxN/8s5oYI850Ay9ttBMADNPUGY3iU5aXZhlRdzlT2hMOwTLAgCUvUiylBAU6UdipuZMqLqrZ0NBOQyXxc5IK5hjypo2+Fq92hf/A2QCvu+U8cr67jSPms8BjJzweBudy8pGbac9qshyGcSIjeBbqii6FAuf7an1q0DLw0mUZ8l+GGRr3g57LV24Hd+JYX/mY8zn/ThqZIrZErc4OOtBH84PKy3katOBVNdqH1GXZE+XZHs+0RYDYpBKMbJI7XSnxlJJBL06hwzCWa4DnQAn218YNgu8NKMpCfZcq27FumObGzIsm9P2crTyYUvWTwX4nbsY2l86G6GB02Q7Vou57ehKS+LeA3USSm0vOa/xPOajIy6hxDKoHAAGcoav7YzvLqu2TJ89I1SQfheiukcJm1jjVgmJFN8bVxFLrv1XhRgphvC4PNmwcwpLKp0sG/E+YlnrFLAubk59/2133E5pPH/VjGEihjYKKd/xyfZCjWhQRH+/5gPaXKb5aNf+bEitxQTArfPL4IgC1WwMktX+hOvz4l8UJBh4FFLjDgyH/CESueNE+KntlOdtvEE1GcAdIK6oRqtw8rKNWSdGvXlOhcghQbYI9t6SPUMbxNBAo1/Df34IOn/D982OkakWdPalOWcJ0IRNOwZ3Y2Af9ao/3B2WVW7/PEjhTxm0JpkRDWfniyG3UWDWzHqadUo7wdMuG6owTzHOH+juz9Fdg7NYMvgDnBts2zTkOIJOfSgcob7NEb5dCpkqJDad31yKUzazOyF44yN3MAlVKGxgQWXYSoiAT6tIzoNtQFd8gIZJmSufkRR0H26xLUTC++6vT8I3UbbQ0bGsqfzfIMmZwP85g42ISKL8WXoc4WhsT/fAdBBGQtoZrSM82hUc4SX8SmcsFSiEo3aNpKr31I1WpYEjTHlHL+66PJoox1fDCUEDo9WKczniVvW9x+otOq/1sIujMgOkjVo0dPboCw/NnoSgxIHvS/mkhFg5N3lzIzvxotE7dy0s1PODLvUu9NNyxuayodgtATiHl6gMLTu2wEDPAjkVTc0GVQ5Tr/CThvH7G1VCZFJ0AztNJLQVJX+0kJl9GrSLQ56lm4L86dbAxTo9pMLLlbnd94GyUflOCJEAURZwXJ1Y84YyNjPOI2VJaXUKKPvd5SsaA+rG/aMCFUBRFvHaKq2tVgpKRPkSpMvjqxgcMJ1szp+jL2xZgVv8m9wGRev8Or8pfAeK9ZwMQkf+UQQyZBpRFXNnWKJBnxG1Jj3lrrv3ZYFKXxjL5rJqWlEp6kiFRf585NeBAu9AHxr2H4fYUco5ijDDRzj4JaF3SZ80gl47lzI8Hxmpuy8h9RV4TsSLIj0gKAXQqJjTK0oKPwLjyaNrzvDQ0NJ3yIxGg5o0nrPwlKPv+645dyhimoiMu1QcJw8cxoQn1UIuE10+apzKUeali0IB1IW6m8TBMDz27dzvieZ3P5SKPGuofZXn5iqlLFPxDhYtuCrFlbwx6Pb0/zYNzgt4Uo6ZDdPd21NWg6JF8g/Kx13GabSIf96ymYf6KBekE9tdD2B7EfPFM52+RQbTzAt1unOeJwrK8vtpHKM4QbldMdCVQuKqvROS8Zg0OxhFX+SC7rFgKEefA6gFjd9cbupX3u1tcEXTlPdezWN7hZefy+bAlxrAiRSkdbyFL54CU2Y9NZScizav6deDxiiE13p05DFzlfyFnWHCMojDocoQODJqBVBDz7YYurUhsaCZrBEZQZCgO5MvLU/6SsZ8g5B9MHftBcg0eGcyfP3UpaP2lJq4jjiWmJT9T2aswDpgQSEActyyJmfz3RSKjYf5iW1bEEjKZPqXG4QoduHiE/9hABzIRVN0KDEMkH31OX5wlxMaQqNxvUOyrhwV9bC9ub7W8oUqjigOXwe8zygg2EtG2d7S2OsmGv+/DLXaNkQxeZZeP2hUUWbwwknxYvkyfKufKxSYzv+i28c8y9FSAuROcwhdOjwxlXKc9vIs6fc5GP/ODOt4nFX/qCAjufXpOAH3jFAo9yv8J2gF6AV6SeChzdBTDkx6vePb3A+v1lOrkVZi+ZjFHQ4X0DWZbAZcS8wzrEfb9UaMj01t/TfZA7111xE49soENnZ5mWeDM7wuOyi79JAuvpRjEQrnQ/mVI8fzbXCUOWEnMPVpsf49rEMnHSX+/Z4FCfiuD0dfOXwp9DtnDY7SBbu76DakPJ/qeNS7tuuQkcwhNomYW5Q7VPIrA0bjFSfISeEOQuyH4YbMnNISuQlQJLzBlQ0tlUdzOJ5frGxSaRfIC5FYF2i3C/SZ1XyEJjxAxxv2BAYMhJuVfd8OjPh3NKTTDUF3DNc7w5wxNpKvLqtEdOvlKsyxXH2FoQ65SKNdJJ5Dk//3VRpYyi4i/zYe8lu3ETnxHe6XvVbbyOc/Yebl1/gnmlrzqJpteyeioZ2bMVVenfZRzIn5XfYYR98NeWIklOOQTrLEHNo2Hr3xVkDJ3AOO1qdxWoTDc4/NOa0SXqw20uqMpFOVNl8J9m43gojDG7Cgaj9SB0euJXW5FtxImc4TDJi5rVl+OQqOFEUHkUP+IOhA380ATb7UhT8nF6MTLs3rV2hNRJ2K/rA02vmaHJ8vyJA3CNxl9muGwpfoEieXG2YG3O3pSZNOGR5mTzN53sq92tuat8zeEF1yIEdKIWpjN5fTAPsdybSuRnis8/QZyPHB4DjTZ8+uAWlKhSrsjyaI+mg5wLyh3wf2BS4iZRw/8Rg++e2sYap6gf946SlHyOxcR3kjL7tsY5FPQN6aBfx5QP4y0edG7m8ZHSmGflsv39Hn05dNtZT4eySJnvHxjYhEf0f0cOvPdg5Td6GscocW4qqCnqhXoA8y4oppG3tjE1N66Oko/0FbcJOWuC7p+n5WXLf1giHqcOzoRD4qG1CnEjhSaOEI+N2W5Ct8uP9Hbk/4yLDvnhIZo5P9J0vS3UwcR8MtV+L06QuSvySIoIVCMmBNRQAM3/JHUvL/kj+8JavvHPwlYRqD5ZpCm1Ye5ld0y44kODuDpZVuAQDiJd9uUfULzoUwMGlw19SlIqm4OmVSun+JUQe0q2He21D0i8kCebHl/FLiPjXValn/1Dr7MZC8IB3+1rZr/q2yA/MAn+9GmanXyEUSI/gzCP0Um+7RoSmmCVXj/36oiuvNsc0z63F2ewkNxvLub/V7z0fE8SOmcmBqppklRfoLPZpT+xoKDIGi29ei/6Sx9dvf8fNRTk60v79ZdDnfJk45GlicnMZTn0iJpAaElMuaEcbLOMsff3lLo+9KEqQ7+7lviguZpmUXfr5evy4p8wijyMfj94K59UfE/a2I+MOxVlzqWQjNG/SMfr2gXeNzXOPxXfzXMZXY9EteRA1RJrMSiyBYIlp/lFbo/FsvIIXv5gABfOD194EFqIajPov6DQoCmepvTiZBqTl6h+nPOQa++fz4UbKcluP+Yb8RjqOjAyEe8KWTQwYOO5+UdgIAyc4q0rI9RWJS+0hsAFyrv57UpXpnC3WvbtsncjOZTX1xWldklzDdcfg2MfEaIQrTGOfffS8xs2SNaEge1Fw4SgISgBS7V1b91gN5o0xbI/cTVDoxSN/iW/Mtwr8aX1vBGKpcaBdyswUNTw4EijErrlCvCUo7KjWPnqwhyh8S/fA1lQo2aoH1AN6HcWSkyYegBKNGPxMllh8HA0+g4xWyHVCy86NxyWLdPCj+WNQxb/BoV8GrLsPDNnFI+Nj2IWyBkXiQZ/waotslaTXSbJ8Dt7Z7eiEs9mK9yeYbRZB6D6CEVko+0wwmhis3fJ88isx0m6Q7tnHJeVVlaP2BIDZJO8QnB8JWNsoDDS4JSmLDlUZZqcWl94ow/070T2ttNIL/u0/moH5Jr5mrE9V9lCUNEnxgjnig8Q7nSA/vI+1SD8VB3pf4RDN6GtbTFTLX5DjRsVG63knGjFLKtxdYqVw39Hb6KB599IxbqrE7UsdpI/J6hpVv0/GnTLhQuTgRke1JPQr2GYZDl/GsfxfG3/YOCFuolezmdYyZDN1vzm8SuVSqD84M6I3YL4Ck0Gq/G+yo6OVsV5MyRoRBBlPVDAWtBb6ORrlMUqxNjsa/RTtlzFFobpYgKlc5h3x5PtxsZNDBz08olmuKT/EofdbpGeqgZXIJkkELDJ4Rz3/83/k6lt/YAfjUbKFBmNOk05PvRgoen2RvLu3moGtzqoNdTweAaKAowVLrdxfpMmVWDciWU5f+AZfbRzKQdGorQXAWGICtd4wZ50GdZ74o1sOlHsnQdXI3KVNiT6UuPNi9HI/UC0ETS7CHMvPYREIKMOp+qsS+ZdEdFXau9Y9R9tobZ0sC4rO2Ls8ZKbt5tCNVRnflLjYWWjFy89dg8WNZ8cPJEtzS5yfZjeyce9eT7e3peGT75J8MM6dccajI3OfpXrsGSQp7VQzDP1jO1YdcyqkQaIFual+TeJCefCHYwoDOMJy0Bo2ln1UmPu6haym6CHP8KIlhyYoNYF8KrewD+0yeZ6UNQqVh5UXxRps3zp5bacGSA+FsDUZ4K7dAYCzQY9qBHYtsQBF1ZfN1BhleoLaJSiYlbeJpWYvU5o8coosbiVVM5P6c+21zoq18t5rTtGVWjhQzEuCQfU9zbvfYjLVQDfuLyrz+NQYvv3Xk+OG6BIrQB98ZLtFAmx3Xmdvt+Jqq/ez1Q0gLPOrnobn708GXZPwqnh9+lfc77hP+kE3tzEa6+JPfvzh46Swk32qud277jp9I4UkqjvwtMquDrFCJRfI57gRgCUqCBa0T+VfUE9ENr/G/C82qIRVKNBNeLSvcaGrpH7TK2ui65LlTa65QqSX8B8nM+Iqt8icf1Y5Iv85i/h4+uko1J0hLiMxj17BROEtfxsqWK2ynoIepd/NsTrO2AwuRPBn7al/4krrrlRrAfKPnoj546i/F0UsShCmioz8WAyA5eB6rWmlxX/CczJIyKn2X/AnKqT0+BnS+FAw+bp7grYLP6Ndfy+R0zcUgOnzi89qvQPGr6X9ChA2qiaw/eMfSwRLzXj7XeMRUfktYLjbtFbEisjteXpgm3z8CWEDtRqVMhhlSUD/ba1ux/HS4OplbinPxvgI3IzxVf1K/y6ZT2Ex2eeNsdZwWT8Y3LUU/AUFcM0pSTVJ6BAPU3+OrS8UqnyIiamrji3noE0GRVDil7HHngyxtO/qZClJLjEErS0eicfsEwGBwhm3PxSyHPGMSasdxH8e2o0Y1ByjSQgXsQ601ix3Swlmf9gkFQzjfalMo3yMoXjFSbVwtuKP7t++QV95rjSYzZFurHUiG/kQNlbKX1L4P0+fxOzEidpNCuUCeLIc3yi3FgSGcLV65+CW4nM/g0x5yUrPqhUhgHj1SaUr+/FRI7I6b/oEngSklxxvJ5YCpN2KfbRBJXBLi/bJX4NIcYcDsYPWHGstL/qlzK+KJ0rJdIkIlfTuNQMJK0fFd1na4L7N1sjSRY0J43QucHPUc2/qlPKdMwVbaNCt2rpD3J01HoziAuCnu0gpiaC23dvIWY5P3MlbD7QtdeukMyJBlcarc6ebNJaSqb6zx1KA4j7PfFeTzLWtAv6Y4mQBld5Y7bqmdco2sfcWyL3zy4TGR1NgiTz5owKJupmBVrNNhxuhCex8csx4XlJUYefWEtFPRcZiAx7Cj2idGwpiycWHJIXx8B3+tIXxzuKGalEcXry1bH7LC3k7QJJiAwo7fkwrl/RezgHRwSHmI7C65tjKj4MI4tPEQ2ZyKaPN63D7+5uY54nhNx3tqClnv0W0s/1OKJlgPpHNDQyUvDhF/98JrEb7ww3AY1jw/xCiRfL+57YRFUmTfKcUVlmyF4z8AaOj6X1CDdZLoi2vPQJRZcL82Ud8CdtREUAQ85Rvq3D8rSaZH+lk1psRHolxhpjVqk7/G7FWczkJfl8ktrg3W7vN0RuBMcemQMe7Z4oO8+uV0cpYW5tGOZCl5uCNJ+VTYiZlS53Bm/tIAwA6TWErlWGBUqdhX8JER7ev4mj99l4x/6dZtDdX1haV6PJSDxPSaW1Kt82yj+10FHvCja1enQ9iHNy1LT5WyOOtWgTcXQhnFuvkEfII9NUiNyzf3KqZBLFIXPBB4dTdjTeAkTRl9YBlWBuHuuE2hp5ls8SNqDKLPM6U2rg/+upf1Vg0eoeXEWCDo6BweKdnL52CJHT7UvaLKrEzzr5RLod+o+fyv4BhIkLqRKnQgFh2YmlWsVv1QxQY+KPa2Llo3IM0/xdzo/hloRf8j3V57qynZjqiKaumz38y5fjaTlq6CO97eoUY7tlde8qUWIKKmAVik3N+ULEVB9VGEyNQNyNtO6f54HPeGYsbU0vBvVfc0yluO6it53xmsCJDCgAOaHybXllfP+sA4lHo/NWXUZsbmUtdfpab+jJ+R3Jg5CkTuCQIFxWPGS9d4X+1mVTFysg0VfZHu6iH/g1vLrENAMUalVwtjYFztBeD4Lvv67aKCZJuJGFjXod7L4W8cSJ2hPF8eDoZIsohNp7Bxzy2lWtE2YputW9cooiiH6Ru9wnmgU+TfSxoxVqm5mGtzUVC7n7meX0ZyVFb5MAfCln7YF8DcewX6SSQ/Dv1uQn0h9OeUXB4gFiNtNUwxnxNwgcgxyehmSDEEQqaVUNx03qBVrfFP/gzTXGLdsxyKqNRjFsywYMSgLSFFEqJFnEIjDGBYcFIc16C0JsCeRYQ9nXtCKQouYmRvuYqGQR0yhBntLJ/ZHCo8RuLKXhv360LNFDv0olVTeS41NK4UNw10lWMH3ZqC8/KS8Gj12zioIh7nmFEkiu0B/mVya0CyTAd28aWo3HxlQfgJ/7f9foGgtQyOfJp/VdOSMN2BXmS9kWq6SezywHC8fyybJSKh/xGCKxJ2jCQq8hW+ugZ+yDU2iM+6/+n/evSbOkVMqnXEX1PJe0eFvT9/rpIkmWMleTuPEdCaUt0ImOzlwKe86/2gnIKbD1dnqYV0XPTEwTdXwAVWZlJGqnsSmADjb5yiKZsYXCQSNwEuBdn2jzRkiQGaGmjHEgAIv+V+R7y+RHei61sBsByXRUR1+ZWJ+Eg4MrWoISJUIIc6d1KuVGGSwo1yG4g2UmLn8O59sQAoi07zk6Q6onindBdObwSIeAUJR26LMQ31qPBkf28gQnJ2Es4WxzKGboogT+sNiioGiwdraUzlf5usLETn+2Fz7Sj6g7GC33NUGAZxMWFBymKI9JM0MCs8gWB0pH0YTVPb2L6REmAxFASXJZvIx9KJJPCjPNcFEPNw6uuEvTS5C4WhOCaYpDNeqAQ5OhMZztEHQYY+XRlmdwl1+jiFpsawhfi38TYpp/i/wMR2dnBSkhbXnMgJU77i/ZnMK03U1Aht2/upl4lMVOaUJFJ693yp9uW3Z55L8s/6PuPfadh1p0gOfpi/VC95cwoOEd4S5A0B4783TC8lzStPdI/0XktZMraram9gkkMyM/OILkxGk5muqousv10hz/Lrt7qvnBZaacPlIwu4oYDA5lrvFYJrmNPBZoGPQaWleoh53jwwmunIOn/GtdedSsKVY31kS/A0+SodC65Gmv1pkKssfIDxMU51Xa5FbgOCj+fVU5E2TjHx6Co1pbQBCNu2cp3S/fNCe/4Y6Xm2GKd+a2Oz89sF43nmPMhymA5SmQPEkTvHrz6pKhiyjZcvfodybqybd5sqICJWYmI5l2x9g6SDcxbsaG0LRCO6p9yl1vucJceNlDTH1jQSvIpji7K1FlHoQPUzTiC3JZZBNFoT0HHnWb9BvOwgVoNLIvL95+eIHFXZN7aVL3pGD9qvfNKzg5GOXaoj3PnpzKZUJ1tHjaYLhvxPlYE2qF3SO8TAYwmOg1JyToHemZeIO7lwQBAoXijYJWOaWXnDnHaCsVByDjADkoShzsn/1hGOqjB8YIA3IuKbonRr2Wcta8xq7S2Fku9ROoJ/FOAXP5R+AyfvgmrIQxiMsoa7fOVZzvwn4OFOSKJhBTld08qbXicVQpSOkDBhCAByLjrjqSQRvyqvd8u8IjLYUsSbSUfl02e+avk0+WSgKVrHLhDKPRZn+he9ILOw91tF0sZ22jV8VdXP1Xb0eRfw7D8b7WXEIDUa95q58jO19E91Tyx3qm66wgaKonW8PdpRXg+E3SMUnvyH+1TDX8bl3ZZMHDrKuDp0iq8fA+fLQSelkeqpq7YsHz/pKaEUuDxgU7KVfISGvIwfVhp1rgA5BwydjEwd8RiYCptvtepHW7lxH996aL0rsZeEN/e5XZcDRWtDAx10lQD108avGPRq0o1CiXtKjo8KSNTfrUYkz2UA2f3kUEMCKnPyumyCe+BVqHdlXC2ioPJhuFHid33T9+2W28qN9C6IW34sb9XNCnVg1fT+PzVZgb/hEKxlocdGxxONj/HxZzlrEbs5WDX7SkGVWJnAVwToCNlvfRxPlGkVoF7ziqh+XRwW+lUv5EYeBfsWyYdP07ywDn8EkgnXFIALjoA5N4INmhSzP+F9f7XbPeQ/8QicOL38HPIFTnC9g5WMEgh7FZ4RmYF+BmnBnFE4aRpUFgmpohA/3GToXiLcAjQOkTCHPEH2Up/fGGaPJ37zUlSq8XbXdAecgKwxgHwvCmTuE7+X4DLrwsbt8XQR/3u+APJY05/zomyW7F/LqAoRHy2fGVgY3dCwMEpIFQrj6z17XhyhFY04tjVyiqiVVmKYYcZ9r30tkki2KKM06VBOlUXRkg1AFR/vX6N6n9mUZ4AvY1O/3YCzNLN/mDuaHRAsKyuFz15Q+4d+BskCm97fEH7Fnj0EPVJe0GD1BPFv5XdezJESHYsQoYcUYwTBSBm68rgedw199I+4T5mEc19TBqwITmnya1ukbpTitrflz/50rLdo6mwOZ7hNVzouINhSMO/Y31z87O5oRdsulMzq3XxvcABcNwUnY+ELpdpbO9/3mGLvFtn4WgocdWWScRNlBGa1YhBDeAfNDk76i8MvcOUPrXbtuMOHndcxX712332tdjsRFmICU7o3NdcAKMBJ8U+O+yiTzh/quZQ6ti6+OrCsW3KstcQp2BSTqGGnTYypcQNbGXamVq2n1JqW+o9w61nAX3YJtX8q3Zs7fN7W/Rgwh0QfDkUdHR8vwyyyvrZyDDkOXsk/f0AP8wSn3Xo3gdUD7TfO89KmtW87ioD9DkuV+YstDAdkXUb8wsLXTBD8dhlVyfv5ejxdxojmGAIovssLntuRtLucHmwxmwOOXvz9aaDP87zcbm6zUDp0UDcZzlGY3RfIq5fDL6HIhym6QTxIsmYQujFoVxp9liNqjpWbE+NVQj9MUt2GZnHPPOSbtlh+LY0VguLbSPUaalw5bRbGuv6POQdB+iVECHpD8V3eI+5ZGfJAAshRdgDaN6C5H5j7v9+atyoFV3fGp/Jat6gtShhdwThm5kXevv7II51wL3O3nJ9NDv8N3lhZgVHYMrb0N3sz/tEd+KMaavEoMHT9BlfVBjKBFkMd6cuJKpvBar3f3QUVx8CIqErC2+y6njmT5zpkYQYTxSTH6czKfKaMwBT92xAD2/vVaFrVfNC3jdzXRqYGzC1b3b1hT9pEbwxFwN6RBYoN9ZTGoolipzV8euQnosOrLUOCWTz3f8884dY4/fokOBk1nHWVjh769RYNFEYvQAEYMFkT5e3G9QAInTyqPiaQKLH0MGbk21K6EM5bnPZBZpu7oFWQRYVuWEOyithzlYqZaJu6vAfr9AgsQFCWW+V9JAfTM7KvUbHQdmd/opECVCNUq/KlgCOP1C2djNLeGQ3f54I45ds8Xvn+Y4VD9gpqalRGfsn23lfANzErnsxBL3wY4UMaKe7sCHm9UAf/QV2PzYmaaRyGeqbKGVlasSKHhHkqXyxbFX0cbDr2IMN+yazmJeRfqYw6RBcBACniYQhMlZjgkN79OIcF3uLc2cKcOAS/7nG/xL9sdKi75mVcgdPJaFh3wVXcvBszzizR9qTc5f1rQ/Cr0whLm0keUQY8FYebll7ToVB4TAgKgMBAA3gguUbZkWj0j+/XDHAzFs1IVLbgaIWicfmRCT6RHn+M2vl2fG2bKN0WRZq+q6IGzrl7Oq5B2wTiTj7UyYhP+PbT7K7gij1qwtJChI3VDlY1oEAcq9l3AemgCkpgEJjfWRWZSljcEL10r1N69MJCWVw+mXo79o8Drn6ZEjFQFfrVxGZ23v5ofb7JCpkjbvV2AAdl1YUx9+zJAkFyM9O1GVd0Wd79LqNo9RtEUD8qvd7P+1WkHLieibsQon8vmm3/IpHd8UQkx23FvqWGGrCxfq3JyCDlc5dR+DI472P7KtwuEpM4+o7+vgMADObxgtfUUzNJuxkkr78zWkrRp+WMxRpEo4xvqUg7XdDvAT2TsSwUAL12mYnLL+yZHzovaIYnXX1dX2jAh0x56znv5EOyEmSvPv6fFN7idBmtNOU6yt2XuXUFnOrVRPMC7y4GYjmIeUwtIcdKPKr6ow93SjoRf2OuZgRDDqImZ60kInW7ql+usmV9aAjUOSUP97Y9lBqfGw/ID2a1s0SisE2mtR+P05hPxsC2TPeT6zBWaRYNj9HpEn32TbZUAWc0FnEus3d33keVZmPPrQ+ayRnNcz8BHSZkK4ZX3ozQJJ9+BbsnN/JfIh7kRXY09ibJEi3vRydtkNoUsOidSk/Y2l0UzhwBvj6xZs+6YtSdYFqYB7kj0p76Ym8bPKCEmcWNCVSEQPvZI2c/qCh6a4CmyZbnq+5DbwxcqUyWQ8/6O+YSaKl9k7UoBVKobWds4mk/TL7tX5sQS+wskkwcWrTXMY9AfHHyfwKOB0SCvU/gRezKA7F3CPkpR7vKXRCHbTWR8pHHbCd13am2DItRf4Ewd0cfEcUlx6YqzYzBdqlNcN+51P935Mor8q+hYW/8w7R/Yp7Sxl08yo7JHxorNSo/LGu+sOKsFjfRfr+W+sUo0Vw5rKVAr1F4yZ33pX9o4k4zJeoEn400q6ANUjuWFic2nZ75GgpMyREGcgstkMV6OjTYR2Yk9dJWxEKP+i+siafQaf8R12R/HeRPTg+K6F/d2QDKry51ycX7kZyOhFE6pLlVPLzRttPcCw4PinkWTpibZm1grf0rPooFfDnXxNhO+v7Bp/9CFtIBSNJFkvOTgQOk4BHqn1e8wYGzUaKOaKuZFUOK6sh8+jDF5FG/p4IdpBFyFHCEgomxaYliAZ3wiTbsJL6AdTUuvcHGBo4Nkv78lLd1cS008Sh+kfgSOLdFd3ntQpN29J9SHxYEyAhKwwcgB8NdMzNwTSHGE56wpM6Zvf9WnkY3yNLyW7Ml8CyD54bt8H57Ag2cO7KckXyckxFlM/LlblBxFQOHqfsszitfl+Mag3WAOzKa1cRinwxy+W9G72DeXzROXgoxI15p5SVSG9JP/4WFaWqxf4reO3K+klmJjkhgc4RisS6VkVvYoioz6eFVQgnQGmXiQpdhBQ5x2euj4+OCeoAMNHmCY/m2pK0+oIPIeJiPP7rUSvuJVsgibdTyWjhm8lVSJPgPBxc32iYa8bImEhZOhEI0BkLCy7T+oB4m63BqiC6CC2WAU+1IoIABr2pvIpEOBqIA3a7SsAsePuf3l0729bbLKumksT0XstJscQymLMWdOeDtc4NesAs7aWXaeHAy0u0SInfSF3QEv7nvTiu1qQwaEvM6bPjkZdpV61ifql2G99Oaq7wSSifB5Ht++MjONpF8F0IaHEHXdrtrgMIvN9tCGFSlF5988JA7tA+NETJK/ZgBmHkCp+RJzMqxGtdW8s/ZIP3AyhZyx/VGVD0ymDnr50qaIcXzhhWgOcCkMwnieyAu721OX1/l9mZIAuXeJT4LKgtwGladbWr/RD7qK2dWlj/WKb0ehO7OxbhsNYDejFKdHlzxbK2Vz3vZ4tO+KkV1eVD6kX32cwITH5bQHy7cA6/F9MpCaIyuiNgGPcGgk1rpn220G8KG3x3EETAbJfHOmuwTB9r3LcoYqjMxcWZqiODYFsTQVk4ETeUYnybEuNIXWMTKnU3BSYvfA1XhjxY5VoD3FgyquM9GCYC7vWBESM1YyrRCDcPGJT0GHASNTxhvGNNbqkkFlYgXATvijeJ1JDy5hfIfgu5owDOk2z1YEKSHKz4S8affzGQn7aN4+KqLqPaZz3gdbNnsoUt5Zg+PQJL0Ef2WOHeYDhagHInvHBRtpIU6mXFCNn95tPVzUkInH3uvVvjAdOq6z00O16pZApVYaWwN0N2RzX351SUUdxnfzswbrPHRj0FV1wvhxkU+A9oBehqJP0FufkXCxXL2W1bTEcfAAzU2ak1IYXOdKQYGGYgH1MctD/n5Z8xKBsS62ylpxD5SjDVOhcv9H+nMFeDKJ89mpG+KYsMVSRv83hwmDCHVKzYRCo/PepY4nrVd4jPrPWAwF7HjUU5NklMbprm0gqFzyPjQKGWhJNtDzSZkxGLDeQIDI8XYeUl+NTd6Bw7WUbpKodk3utqvUL1df2qfjoWb8knzwqdjI6d5b/8U2RCNkzPn9QuWfETf7XQB8fTaFU2ZmlGX05AJwjDr6I364EgdpKapO8l5DqWVKzQ8pEgssv/R4zBAes6bw9VFlCYmQJle9jQthoE2XGJdtYWNltdtN3UAs8DVPxrZFs+7Sa2PEyW1FXtweSz5Aspn+kh4ua75ABh5mb7X0ydpjW/1LRn2a0EHjPZbGug8BDJoDd33gm54zWUlNOBWAohaHNne8NzGtPxf8Nu6bZ5/vZQxl50G65WFCgoRZNo2/SH7B1uy3q3uOKYi9+kQOIdpU1Oee9cHknI5jkkVgouvjCu54eh4q3SzO4vjqX8gJg7oGxxvyZF2P63diH0F8y81u1bDXjWZ1ctiarNasu3nU38a983Gz1GBucu/OjwlMOPGiGeCNb9Sz4+8X46iaUV8YdWHx50ZdcSofGFa8eyGcto+EQYM/+cHa7+NUaBDyD1IlZPjoa159Oll3CThxqVrGeOSmGJodOn1b+Fc7vSD80t/4mQgiX37M0oFXGZESbybU51EXnPt1RoE4dap+J48IDGM6Eqai8kac4Uq/fkf4+ugYPLPnTqT07S47nNlwB2dL/5hqeyvVb2PUDw6aT50GAQBWpTtncGLgv9Q19dAxLgC4XA65Zr/QTJfn6BB0Zg+Wu0icztiC2X0w4Ii2VZ9FD3p1rFJJn3sWgcFFak2/WglB5RwDd8WVpB9ngK8NfukPYYN3WJKaT+tAHLFSNk9/fnGI7CMGD2pKzevZFtkxDeQdTFq+H1lbUdd8rtstDa9JCQzGblz9jSOakLx1ir5CTcUioe+XyiV10nVDSZWp/IPpcTmhtdPB6XAaIkoaMbWVdc+ndr3dcO/O2Ieg/fOyV219lMCfKPeiNHl9bfl6/bLujWXz0oehuym8WiF1sj1F6zw0CLq+uXGTK64+RPkgQsl6eGm+YHrwnTZudW6oSd3qJNFQFqOYp/u3DsHwrIPSuKJweNOJOCJbHKIYF5w1ftZOBEKC+r7i2PMtVI91b4f5un4+H+l4T1MrbeY3lfjgw3dezDrKIT4ma8a+35qghSLSvUoik2hvUPUBBMYQMwJ1pVkdq7wiXiOei0TlrKt7TNhfS5EYmn07yV8nx6Pc5IEcjc15Af9AnFARLNXVRS/n5gm13nnfcbV8AeRNiN6EHMri8DbtFi7I21iLdc4Z8TsLMXNzC9Lztf+RvngcaNDyrudl5AMd1AFmpZ0uRRQZh+Xt+x7cxyFoQSeGSxHWYolzk6aewStE5l/qJ3511vYojN/AACbv94y5oGiVmMzZSRMTkFftlk5FJt6Z/jr7j1HEDRcCXau3r4fkXVKvvX91TK+OFQ7N+ywDxm3hLgic2WtpbbfgCMAL1bULk5QughWRaE+VodkPAYlyHVyvHKQ+LXZN9zbAsAU96OfrSXBpq351zGWKITGsce5zv0FPQgk4094t2FaIPhz76/o6Eqw7/ZzDgQji+sO3irfwHq9PxXEMN0/T0O7TNW3Hjt6rmjaE+B1KZoDOJKa66UGw+IiJjM0eqZ963AydayZM99FL9zcDLFfkgttVI4TkvvL8EbB3SgLvihwr1acdgdXpGZUVyCs75e+GybLPgYqAOcPfV5lKbNJAcbDXkVPJb3CgIwY1GURpkZU6zEqiPEl1apFRYb4sPx1cg1ZOcxz9BFWSFAxQO50jdE63VA0PEFd82VnTq6KGbivLpoYpnytOeYk2BIHngxcOs20M26qhoILlcDed0Pq6lugzR2ofn4VEJXFf32TEUVuf48vXE8taih57gD1NBBPvASVZu5/dg+pzsY+4MhtTSTQelV8NUxhVr56LaIFp48Rpv3NEfGsnMQj/7pjEcjFSHr+htmsPtUbZh+R3YVrnNfK5K9bLtvnXlmpRpR7RKLgw5kE/dDFMgeNH0FeF2OL5YZqo4ItCOLoy770FRG7lLbeRCyGwFU4Wlro8VxITE8XfeMFUvEITqNNEjOs4YsDNAnAj0vQLbq1KhaFyuATowzbx1DAt60fxnEJjCRq4svSDvRNnu4WH19QUdrG0xMflNmnr9PGIopP+2CQ7AcdjffKubC+bTdzQV9J1Q/Ppm/OzSP2Ge7IzObJ/xNwu1ZeiHZBlns6OPLvTknBX/vbQQ/riylAaIN+QdbUYadh6u38zLF9ry9I+J06liQVPbFRR9E2j7rLtUervRWHs8gs3eA6ft1CNXzdPlQYCd1cL1bqJJpoOVFqbaqVaul+z4+CWUMl0/MplOoIEPfp4b6/M7miKHburyTbsSlQ6mj+obA3qtZWOMIcOV5RiX8ON4vI66OrSmpoiLnMRf9xQbngdz/FQYjd1PSKFFrIhzjmHloNBlzQ4eE1bxOGegi0x8zmTSO1GL2iHO9nQ/XiJGcn5sowMSTe/lqG8Htsbn02tU2fAS/YG/J+bUEsDLOtUgCsRP9r11i41V4ZWtLL0HNJ2jHEAUs3Y5iG6lyPrOrOMw6/BQx5udYlj4gpvk5+mD25JdV0McVzUwdetUKitCW7tPvPzUWtGJu3yaV1cTtqJxdCOIwgTtICX1LHu2V0f0yGTT6z9WAOy1dd3hWoM2+Tq9HbemwlUzcxRuqgGcZYddbimCvFKWnP9m+5LGTFIgR/eWdUsa0Yy/vCqXyyFwvHGnpZXrbiJ46A8CEWqws+Ps0PdQ2QcEmJ07L4TcqlRWUVNL3ItnKntGBiyGFdkNpExi9FF8UF4co8o6P3YaEPrFO7C9ns90kWk11MU4rtqNMfDNp/Pve+qES0peDZ7Hai1/zXbaJ6w/XxbO3TMzfQd02xhuiHWHHModzENl9jVXMpdPn5ZNKT6GR/Ua16Ojemq1WED6tRnsn8Dc3cD+lfXJMA0MpppmswQhnKVMFn+tEpiW3+H2UWf4LW+AkmeWdPskdwbwxB6Jdnu4M5EAAJ/YQRtqDX80UqrqO3h7NVrXII6gmYWoHwO9Av2JYYwu3aKI++DKeuFZVuC5zUDoK9hs2hT8nMSSJck2A/OLV8FGMiUqcwa0Fbz/BnB5rN9YpnqXIJue1zSqM2czzvzj0IV99kkxYVjejOKXpm7KrP6KfyaA+Ei44UswOmUhQbo98VehOVJTUMrjfn1cuBKFPh3qvChaQeP7QRSvb0bl8mMyfdxefbjepbAb8J8jV0Nah8YJG1yEdCvV1L/9T6Bcj0CL6LZC9HYFC06nmrziNTAItqDiAnt2TYImnhvraZOFbotC6caT2L4zj9LTu6Viag5crCQh1KtwI+29GKD4EOiDQ+0DRn9UIfjSrBtlmTD5GCn+4LSAuzMNEo6STfb/0ZCP9C6owgWhn4vhhDnvVz8XaxjR8sD3DowPG3ygR9yirzljvsbE03oEicIVFPePglftTuCi/g686XRm3wNUTgcTpiQm9dwl6Qc7DNyAa1KXPCuzUTrKl+DkkKMMgzXJU0zwPnM6hYZN5l7WRN38FwTyfZSCx/zE+NfRScMo/6y3FhilqC/Y2zsgM/lUwCuMdqi/1LO/muF20DdvdFJKyPBdQG+YWpk4gtm21KKdlEvFEqAs+4jKZPwkZCXmUQWUjNcKM4fT5Y2pSOGISknI+SdrOWQvOxM4OR5/zpnTAw9fN6T23DKqngtceVKP7qKJVz6LK3wTVRHBMFilLJf715BCi1YbXS/guVFGR7IBOPd47AMmoTTnXPbdZdyEo5AGrMldpvpI432SfJ7vtqVjMmv81LlIdiX5IG22nIZ4NJzWIavspWGyZ4kiUyJ26+vDRcVOqRHkR9uBnBanT7nrJIzFgrExns4ARe8PU6q+U5sD2aN5dWEyOezMOE0F6WrdtF7QpZU0qgBgRLYUBRZt96DPFfSw1RZdeHjj5gVj/EaEc16VNGFrBiHMrrOZlSZNtBu/9oSiq3fmEunbKUuXZZTfjjlhbTiBYCcZgtWHWw7ZI1GAUZsn/CkXHaCqErSG37lXWnJkkeUW9zZOvTNVsyOzxQxsDMMk7ald5eBnDGB7BC/DHjNh8cGpHeUJA16h2A0UcSpikAqlpZVKxQqCN8V7gyKiccVX6CdtZIk2N353JDWsiqk02dss3PzL0HFBJ2CgvhIP+R93shiUHdSYHr+tkHiF8tU7jFGXmub1dZoQ4XDjLcLoPf5JAATRJI/dnTSUaeuS8a9bOaXVtx/e5PBYoBEa+EvVDFmzPaFqEBYn6lrke5e36hd6mRxr2WF/pquohZm66cVcvLOfZRyXLHH3t4MleaGMrmN7faXAvFVpn1bF/BhzKAkjljYeYRubm1I5Nq1Wcrl2XFH0j53wNYWSdIWDKSDAj5AQML2C1jLJluUD+9Mo6MZU1598Sbc8v7C+uWFl/5ptkCW1pr5HDXSb6JLomWSGb9Os1AcstZ5UAyTuaxJtXdMRTHhbO080NrNG+VyNjY+ECKAUfTjmTs5q4jTBIGcfbXRUkFO1OtE4oYxqJUJZLHSGNbH7WGFGanDMqIOUmHmUwyV4qAKTtIhVc/z7embX6TPmu1spwmGmIppWcXqdIJlXfP2XWzZJ7o+OTKayIut8var//JvtrpNwyuy4iMIoSDpOMarhV1EA1SU2y+vL8+M/HYzGt/YUnqATNaKs4HtjyIdsvKFLoJ4v8/7tD/lYcNo5z3mCmiZxeqrZBBFvu/Tuu7AUkmVWNryEnt2JIDl1MRELDAB9GJy6egEW2yLwpvRvEHnthTTgjnNH0/UjhSMp+H6EA8a+8DWHnwOqw5iycVyTQwxC8nX5GoOfcfrNH94uFb0Y3qjXRHnzFDV0T7Bd+nfn5Pp63IKt5gp/PSSZwYM1P+GpoJu5ps775ccMwTgkTUXHZpEPutJQ0k4ZhJ/O1p4hK8/PcqWi0RPCUH6qjY4pwveirkz9h5LGFQl+9r3+5o95sPVbD+1QJKkvsu5/KUqlwjtqBo3L8s2Vcvf7iInSR6NnSyrydd7bh+HvXnhWhwajHrCTJdlbwPMUgoCKBKESPa9gDNvrBUwPjCfPond+t85Y2rSbpIIJ06lTozRsU9oZ5CBUoboMZfKT+mh3vgVYpc6qX3UCHTRpiCJXhC4sYZmfAmlw8XLGX6E2YR/8mWs8Zf1yhjSEo/JR4GBLCf9YI+qUry+dANDIe1vj1aFYrwILYHWeE7cNJiMu34kcUTfszciHfsaqWssURPymUYmXuIRwvb75nvvUAFxmpJcYQSE++URdLWJ479O12Jz4rpnCD2yjim0DS6x9h3puk6km06DIO9270RjGb/AzCHxA6P7QZAX7p36lxS4Y2v3djklMKuE0y+poLnqrUtm9GHTtEyr9ISL9oeEHXW40Zaop+GLt0Sb8+pYftQXrX3Zezf8IJTHugsySMOPFMGnjRoKXDxBFN4Mek4MH8aB1lEsaBdBmvpNH4xewgiCrBttMAfUWUCPBS58ThhzSsy8ah/iQbfg0MwwrzGedba3C/ODhKNCmuXQO7/27dxn8S7LscyxxKHYtsIOLzsCp7Ip//V2/lZUrO7p9YqoutLeLdaS+/dEJ+Bd2sIrqttE3ZQMAchOWaG5fXhH6exaxFFZuM8vZmZv4Bd+Vq7JX1zYMwW1JwGHy6/DMp0HobpgSG6Ph9WDyo5qmUEwgy2E84oq1ObaS7Zv3gdmPIqXWa7CMFyce5YTJ5Z9ZFkEqfXU6mhj3xio046/CsKEXGcYWQgu6J0h/QpKy+hi2cmReA51Cyn5euVfYltktf4IaOZ8FXlNVwqiHTa7hEF3RdOu3eXGCLUm7O5cNXPskymW7h151HN/9z7X8ZImU0k6gLx5nn95/A2r5n4qE+BF56fiQoWqtOl9XYUrv7vFbRBMaa6R8Vsez0Q1E7mxaBoM8n0hK4u+0ums24hfkbqiJLsrodkBx/nDM+D9dwy3kP33yg+TtiQB/KM3axD8ylhnX5CBwvpyxlu+HJaKlE2MrHFtBFXOo2vvDt2u42ISIxJf6IHWw/VhPJpCMw31ZOEiGnVj8je1GaHrnIuOphHt3JNU+Mw+v4zDMfmFhSFjK7MXZYm46VNAL9vLXoO45HtiJG0Mpfch+4HuMV9YtcaoP2NeCsmUqzHmc+mmNemFYL7nAQV+uTAGgH2RJ5SdeK+SSYa+s4/32RwZMjWulhfTfeHmbX5nBgDcNc9F0wcBy27m1PnXIb6OKU5axyVZcxLKk6Qc7OQ3SDswq5qrbxO9C2UPx0d7tMfN3Rn1mIPGyjUGF++4J5nQtD+EQQggePH4xsKUmyfv1TWu8i7cnChOEpxZDPbOyUbfi6xqe6/r3cnJry7QmhurrSdJi8a+dzhch3ijDfGVMGnS1Ctw+wanDgZJnaCg63prncqP9DHNBORzFROZJCuMyA+BShXDR45MX+mjgNhUB7kX7FgRWWsarPYluS0jfZ/SPtwZCwTYeTj62NEFpiWfOveUD+bS9px9u/QYsShAIihF5ZqT7trzKbbAJlQwyWhTsKSS4vPz2NHJNX24qXzT1JUDg9meRHTmQLqL2s1tKcMm68olZ1/HO27WKQzU4g789wR/d/NjaWYnozUHhw6UPTarS1Cz7mybRATTjBZfDfkwXanx0UhkxDLf93uALXgbZ7v3Yfd7wWt4RuSjXaeZ6PhbY24wBHJ/hnzm8S96EjEvjQak32SEV08Zwa2+73HQWQ2GGGFN8H6PtQ9C9NNnTtxO+0bvDr3vVpM3yFuh84REHY8nrlvHs1R1/9vJj9ncKp0ZETCfBs/EF/yMGocl29o+P8ZKelaBpPICMzdny9aaRaeaJL44NJKuQQDHS1+QIw3GHhCfsqA1abcCrXUGzLNBzkn+5Z0vJsx0cyW/0+yQYSKS4GEWFJKbM/B34nlNiDZKjyLta2pWzzE/QlVLWMngM1x87jWlnaDPj9F70ZUmW91L/nJTl/pQZphtlLvCK2PZgC1Eky2NEMU7m8EPZXDk/mqZRz1H86IftFtFnR69SmF3pWZa5Hc7NeUkHb49l5k4+yc96Zpv9UalWUhLT+GKSa3ZlKPxy2n4de8idKcXbm10SvEtjM0slZouLvfdLEauSkZxMZEiGRHcIWmC6s5jzr6TJHCkr4o1Q4hFWBEUaBzuY5WVGjzS7qj2hcs7YfQ6j/tkh9GN9FZfEciYB2bl342Q7FHI6TSkPSt8unTZyMRrg3s/o2x6WK0k6Zo2fgNgjczUH0xW6C+v7hhlBkGsVIIYfMBG8r2sm4ke/sD3wuaUu5wmA2v075CtEiKXyPDhgliMaTj07MpBB76EKXxGZsQEfBttFEHKJ/E+JKqF7jBcLz/bYtkI42OhqVheFlGaNiEfSNh4AZk4tKvYuxtV3dkzXzYiaJ9inJ1SyUAMUoirF8XEUDL336DEzpWqHeHXiteNQlDIGDhT0jpX2VX7Hkxnc/hdrndgOcOVaXH8SaQm8f0KGP6vQX4sH2jAC5iWw+KzVGrV6q/aU19X/MhFhy5kNmWT1Fu3Svh8SApExII4pgGRFttSkl4losbF/qeeA8tz6ymxu604yzGwUpjhkEojjDGtj55Jwu9uydO2RHYUust7bhrRDnRzaHF8fgePIuDZurdjObKlQfRYfoLgF+eXrUbTy1AB06tc9PEL0XyZ6t7hysSSFp6M604iLXP0EZeR5c+7L64JitLSPlvsNGxkO9HeMFbdLi02A4RCWR1FT3P91SESIvjMZzUNZWq1/rR4Qf44l7wGSj5c5KB5gdfeiUbI4aXK0ww9vMSbvjY/D8sYYZqxZiDQokbvVdkh33IiwmOgEn5Mui3mGtp9DZdT1dRmS9ASBys243IVo1/RlOj5fjUGqV9JbHK5yxzK3BMRftv45KwUEYrCyWNHN6zCqfVbQeDdUqAyw4sdiS49A985ta2zDaFqy+LjQ420+J7azpRaebORh1WswpFoBy8TGioWPtlmUrYtK236mzrBBzs0Z0hAuY5yvCTQyQsOKU1597ZY07eqadZC3ZGTKiz+cw59XzX9IkW4w7SqJpNGo+iRnxSF3eHWvm2/4IOV7b74OOWOPQ4G2bgeki4TWiC8NspJdd3rCc+2FZScrIW2/vGhzuCIKClFVZy0EaP9Zws1QpkVxoA+/MA9az27HjvOq6G7k5SQ+0UXYFjD6KrpYtgo4Hmkn/sH+aohyVE7omtUTdFeC6FJbvHcWxWW0X8zI2L083foCK2WH9wjIJiEGo0fHPs9ABukj2Emht3eXNT3u8o/iAGLVRwKyENjJCll5QcSEalD8XTyoOgdn2lhoMdK4zYbIJ+m374h/bG7h1sm8ch03OH6rPWRQGyYUl6Xd7ttN8kPL4lAZmvoXFRvcBg8No+RrLcvaccPqXLuYH2sPBgBNjx+QUpe3OtkS82WlHkcV7X2mHh8o9bBG2/hqmqmVWFGHWJ17Cy+ouCLrh8QTRMnQvBsRLL/cMOzdRgbWeGj43q7p2o5e9vco7HVdami90ey7CYqEVTxFQ2cOLBV47p78yF6KxJ2y5K+0QTC3jS79x703MCxzfZdb9Y9vi2xxoD3mxZcThrqVgZ9SUWCM7iGV2LgmHqVxgZOsbOzr1sqjSuJa0SfD9Yv2omJMxcDkATR4xM1t6QwLsLF0RXm8iv63owKH3Xau9vco8su0jzbuHBpTj0tqMb6WV/KRs8EwWXq7mcgPnIMwfT9/tJR8M1R+PdseZr6qiX/U0k1JJfM4WLrXSCDxNCfN9VP0RUMyJhoGfOi449R9XFd+b+ivdkbGk2KE8nxsX5vHSUcQm+wwECLBAcyWNnfTKfW2tZK+R5OV4ZQ7GrOAX1t5PyiieWdyddRT3ULwKZoK68mIZ1MvghHAHdyMU1feRUGb2E76VcB634EPNxepOYb4X7oe8dEXV60/MgHUIjp9n0f6kVFzhu71WWhpQak4hf18AsaUZlD1DZBUJnNNnarHVm7Zp9Xgr1U3GULmD3ZIya41X1TSLdLnuThH98bM5E8kzr/RpNIqEXanOeDLXwIAfotGUIhtX1g3a5iBPAj6W56aEFKMvFKoFF5uvcvN61kSkMSf/nb2sE8/7Bv28OFuXkXRfFvKP/7l/035OGW0BjPWb+CKwgC/7m0Z/Oanf/hEir8G8p1p5QNXbbO1/OWv3/9bwTx9zPXPxco8t9B/U1w7ai+a/n3Njjy7yTy53KZVUX594kE9e8gAx5cjpc/l4r/8RQQbf7zbGDCn1zWtv8M5fc7AlXfP59hxU3M731sHSxCYdn2Snb6byj19+vE7Zb9ed+fC8t6tX8vzMPWfzNwF+iZkWFey6EY+rhVh2F8LsLPxTpb10dZ3OAT8bYOz6Vy7dq/f82Hfv37Rxh7Xi/rPDSZ//ebP1PHPrM7XwF4wr9jKPrPhfB3AYLofy7w599B/Hl1/cdXZjZXz5xk89+L/8uVW4ZtTrN/MSfY3zeu8Vxk67+avL9zlX2L7F9Kwpy18Vrt2X8ax/9sLf9+1ByqZ9D/jwThCP6fJYiG/vM9/gz178f+i0j8j3H870vJP1L5r6QETILz92U/9M8P9v9YcP7fgvJfRen/g1Um4f/bq/x/tGP/Gfe/WouljEfwa9XFYMzs7yezjFm6/l2M+J8XeXWCFWIBnlXpszZx8tjYw1Kt1dA/f0+GdR26//AGpq0K8IcVrOHfFeKGdph/j0ZRlKYf+/m/rh0B1q5q23/e+VdC/uNykv8M9Hnxjdf431Dmz0tEHPtHG3HVhzXsA1KkYgB4rTteKXjF8xv4j5F7jgmfn7wBZZ8vwwix0rSC9bExZLtpFj9bH/b6Gvg3H6UtddfczRux3vcOcY4VWfbhC5IrawRbWN+8mjGjGjrGkLpc2MtVsL+ONQyM5ohyQdnay5V3EBoWP2QviyRKwAUOY1BP6nQ/o6YxfzaUnJBtnvCUoLY+0z8wWcN9jwbPVtlIuMXZYhELjS20cxx44WCkgnpevwsGXGMfcy5kVCbkwb8xD5raHj5zSMzRMc+P/1/f+7pHTWwEXkuUY8p31FsRhPcSLNZFqY91qoZdpZTR4fMGcayiKWu2oFRECKGWs475/SsuD47ope0dQICio8HbQ1n5zVWP/dg+tqXWchPG9YGYYCG1K/g+fpdYJfmmC49PnQoVxrO+OFDoQ80Pntf5oly0Tvcr3FS/6WcWCiR7WDSnzvSbizzLWhtnacN3aG0tr/heaBfdyMk9JU3z1dMfbiT7I08p1jMIYefKRgTHbXs44rJXVNLteFI4EXyviHpn+CvOrU86duTY9ZlC1vVDgjJ4TXgX/2fGWK5/DwNp6t8vmqeudL9FVCVfIQwXKsXu3YwUnxnLVwVShQIUIGKAR3aHGOLzUpzlo0hIC4vWZ15dTmVGrjWx2udcKIdAFL+YRTCzKi6HqEfft1AsIxM2b5CNYOngbEywnaNygihfu5DtnFEVQa9K3LVIyheUIBSaGH5dwPF6GWmVFOX3jcroMYhfjuxz1yfOlYK4yOgVBlMGrO3PHGWEfLXQOvHvfH35gvMymOY9HKBuLvuBBrZ01GpzDpvleKESk5HO+EwIP4/pCxswl+0jabVnh6TmPyOg+xohVo8YSPI24clPPZuVn7ldi644bl7v1FgzgUdUfiD+/IDG2GxAugVISLE5wJ0xumHiRpSZ1KYN8QXqs4wIpOLtxYsgTU+8+76Dxiwc+67R2FJj2E5/UwHwr7FmrnTmR4dKXW41a1M8J1k7AhYzmjTnnn0hW6aFX9ERxpdrHVpBVWy5rTjrcfTEusYhmqaBklsAkw7cV/GYfgqNAXs6FFyaSuP4+tWevaxiP1Fuw20lhNay67kviVZ5nT/go/loZxRVKHkac/FrQyluJdQlX/Svx57u9ct9LFbfTDsnrLmQeXZiSilvLDs6JQMBAHHkdrpHUc1d/JiOpd1YCDvbsiU4X4VraZKuMHG0vyLUYIQta6IAB+fDQ6/n7xRE+bCuPZ/Nzzz7WtmUKEKwvunXJKdcmBKHknqVo2Rs8Jg1eLLKXEQchVUx7SjJnM5AHjf4HLV64jvnoFcKcmlkn3il58H+1liV/DQ4GpqCj939viG0gilVTlYJqheTjbBfRzWr4xlNQAk0ML8vJxDAsvHpTY96Bc4xLAKmzuLzzV2GCSeYHHkiyROhrg0ULT5xVK4vZsLUrqH4NkwwaEk6mSokKaT0ixsv3lR+hQ6KXj0XL48bmi44V/rN5XQ4fYnqO02AdAAHEqhkbDun2knHaG6EX58n+p2deGHJKmgptdQ88iACtOj1nIAc3LN8V781F8PwHg4iQY3Pn9gpqlAnAZNRtYou9kEQYe5vgdfZ5lDDvauyBUefsecF8StPh/Nu2O1Lb/2+5WP6KQ1liPphZvy1iR+6QhJ3NRbg2DOdkTqj+134A+OlpokbBGH1SAfdIKPl6/n5qRfSD62Xr4Whf8/wO2Pt38fU7pXmV1f2tUYshkneiagj+LAFuqheWaRx8Vh3WMFRYMuzXnlyvi+8tDfzMk5HrqdcpPUg2KcvhALf6IaHtz6F1/kZXGAvf0xwDpbFnKWIinpJmpppSz84swUcISxlpj+GclMauzgaprjl/bPWWPZa8wa9k86Wd9hThs/ZSUz8DP2XRnkOv3PEuNUucVpq30Wkco1BqwyY8Dyr1n+wBPM1f4fllMv4jUkOrwioE5O0usler0vsk4wJi0TYl1hAjShGod94EsbAPEuKwu43HrCfigokPH8ZE1OLB4Nz9EuVkcFLdkFZZ6PWL113PvU1OFH8elRR9k4trFkW77dwcb2t753JeZlRMdUSEPiGofxCtIGE1EOVvIURr3ZRtK2e4tAAfuV+t9QiP5oSXSgnqk7UYUM2PAUgRWQ6jy6mDr8DXzyF/EryBP6InBgGTqqLZ6FqtovirgRmvQVpA5Jb8LsHzbASAzCsr0AHGsk3tl9Tw48y2Rr4aBEwX+pGslvkX5IXu4wQUgduKoH7QZgVrz4hsxOxp1RViy6QE90MGyoHfALcdJ2pT5g3JqSStYFlL1++yeBkSJ03DU7CoG9uWMNb3N6HaKmULI2053Ro86bYglnY8A7BCd7Zm0bj4CA8McswJT8irxWG6Ii/LMxbQKoo/x0iSUxWYtTM80baee5Svx+OAu7i/u5CPnfxDwBq3MITWdQV1sRLzXzox7uUeYzlCgR6sJfPGIOLvW14j1/bz6tHqAtOa+Mby6HwuWQ+simF3/PbFwjCfA8hhO0P4Wm37GEgkYi4TErJ24TILVsQqvBC1DG7EUaEXhXIQmTzt2ScR4J/zkYHk8CArJTUkby86p1f0s99nE0IOnSIr05RITFF3tKyMr3j2w+WTti1tovK4WbLH6D5JUilkbjy6AMQ0KVDb3u/UvSFo2vcM/nlCMarMhFBEg5eStsAeDJO+ZF9BtvKOoCwHbk8YVu1+IUXDyIc6iyUcQyyB+OliuUaCITbB6+7CbLz4Rl3gFzLrzrnUp4g7a99U/CfmR5A7pgL1wRtyuK+J9Fg3de+vd4bFa/5gMKOvc/NGvG9n4o3pwWHSVnWfvgm2vzG54Ft9QBh8KDsximEhtd8k7l0ASdmIPMv8UsHslDz3merLQl76EIF5unZM7ym38WrAaVzxJ/0ycpW/YE0LG5l4atxBSYEmqmYyOc9rb9A8/hz0JUIs8O60n90EPLsGezBMUk7RZM2FSUgGPPm0rJ69EOPBPh+P6saC/2SCN36Fc0fYolr6CqVGx2A/bL1ywbVgV7QNEYHT6EklyrRw5Xf0VLYKkMOIcYhLyc57MO6epi9DgUwjrM/AIKRFEfxF9NeBLpYjS3KbDmzCOZ0IhlEt2sSOc/ujDF+HTX0QuOnP3mxP3Oad6Z2YyT6O3eG5ng9jBCplzMgaUPUeC9UA91xBiaKVUNvixBOhPczvHEEJQWE6TLYwkzE6uT0GMImiqd4AKwiSrQ13wO0iywdU9oip2tYzAGxrLEbtPpiqZEq36/jC/GfeoAudw9bznVOvPichUR5R33a/3zHHQHKH6Vkiqd54MxUpoIL+eHwgL50O9l5KQIjb+aHSOD+frV0zxcofB/yiw84R5B81vrtyHQBbm0m5eWWUMIbUSqM29NuKWsNHCp9lNDbpN8eHzxAOg9IrfvEpRmhxFNMzlVALEov+LHwCrQDEo1DpvB2InDSnTFqHNpt5tJLZZ1KeCiP3L31AI7uw0sKFhttvlKHjfOsjjqS8NG1FDJ+XXyNE0pIJqe+nV551xoOjdwF4uAbM3NNaICvbj60AnvhDjHQkQisKelXkiKnquK1MPSEt0Z2TTWhjdH4hq8PyFD45WMp5Mqkqk8c1EgD6/R9OkRbAdsy/RaiJu7INoIpWajlT0ObMXVayLkl7sb3R8Z4W7KWyGsgpEgmldFOD1W1c1FKxuDR0gZehoN4A2Ya7j/EvK1ffz2BoR1Em0zGT6V0sta0A0V32GbJJ8/rNs03Rs0H+35YzRlrSrPwxcLgYYr/gKv3n1VCl8uZQsItvODL0rW1s5boOu9GWtdA9E1hV60Q6K93YWGjtnBVyNRvH0idnAKsEVOmg5uYpnKxcc6WMxl1UWtlv/RyIHYp6oQFQNHVCobS+8w5C9DNfrD8yu+NcYibAl11RdiJtpspI6BRg/xzijWuggFSo6hO7Gtjwe6NfqXM3UPGNJTweSwkEIxdHQMrF6FWcsh07CbnwueJ7nnaTt2n/qt4EMNsFRPXnalJGJ1koLFIuu5wScYOfbNQYtXWv0oSUr8cKu2/83Rdy7IiO/Zr5h1vHvHeF/YNKGzhPXz9kPv0TMTtuBG9q4siU1paS6mUDFtywC5KY89wp9gWNwA+kxHxX/ePGPLjF0P2FyKdhjqZrQ6vP8cJLr6Q59g2cIAoaU0N1FHZxaOyWKEF1s0iDAvAovX6LL2J1dYvX/uGRyePcIMp4HfiGvC1gcluUf0AvDS0sIJXyahjoWbNCfsbk9ONb+z3Xwz7kswIRH/XZVdGp4ZsMK2uwHhZgRpJ8aaUkgkhoCHVki8r/kUm+d2BV0LVOghnoBUfSzG64pmDIWbSoQScQZZP5KGYfp3RyXjMWv6FGTR/w6NXQpCazvUprCxz1r7HgT8hm7q8b/IJq06ZL+0v95CAsouIsk4ARu8KxNYCELRaryo87VH2m+xY0hfQGFt5eU5xv/YN4pT1R77FlUNXzwnF7tSK18P1i20JlIIxqO9gQPRPCTzFSiIqpKPXl3jAq9luLZlYz9rqWgz9jaGtBudk+ctf1pGvLLBbuwOSa6QYiwkqCdra/MPlNMW+MeCRBlt6Xpy2/z7KM8Xnr9e9pLAtIyprm0fsbhn134yL8BT0v3sFvLbF+uH/xm/yCK8G442oBPGrBoWBIrar1r6yYsXGSjif4nAOjLSq7vR1kldxfBgj5bW/zuYcDhDoZY1XePAKQ/Ez8GTJyT/g2+IRrStRnT2uh5sXG1mF/QukQL/gJ188kxpHUsvT9oS1M8ge1KXDMO+PbZachWssZZIr8Ga49T8vCxTdv1btAOnT8LL09u6qEGnrjs7+IOE6NQXtbTaTEZ85qrXT58A7NYDtRA488AbPVNW4FM+XG0srJssgB0QFcai4zIdOaie9vsLBHrHbPSC6uONw/1nR2uQh4o7CMj6/l0Kgs29840Kqpg9ZFe+D1yx6bakMvrqSAXdLsFv8lXh5RowG/7zm3LdaxOInMH2rAyDUgFMz6jdmArrGAoeYG/ZQqlIS7pUzQqX+rSssOTT65N5njR/++LV7qh7t5+wMve/rNt356AzXEn2jGCje6l/jWjikA2rsKK+AOxTltS3eCQ48PnOTQVcRUyKGXmkA6zJTnTezO6Y8yHmMtva51vaqvkaknerA4uZ/FYcgsC3kUxgfK9MEWlBHr5Z+ozaoKsH51bwDoL+WOg9EBBUt1gxDjdIrReyyFOribsL2IhV/UjeDys+GG+QgH4bxJZ1acPx8KnGMXhZoywZPSpMSQ0m5QQylpIvyJfpDc+TFkMd2krJdHI8VJNxiVELaAVQJ8Ity+WQv3hunwpwszxxr/Muk5JdeaKlgH8wCKJxz83XYNJiz+7JKDSgU5YO4Q4NP34QajKRsU/Tuqd/fQMvGZUCynCall4Ws3LdaCpOsXaaMgbcETWDx4PoOW8KCbUtLH3lpnYtVpzLMgBicxAiYOjLqaS18U8hrRjdgwT87nqmpeIasX2m5n/5Ovkem9Y3oPJupCPUZ60v6JiAlG1SB1T7Aj3l/oedixTa5NGOe7ekqGZWNC1eRPSSSweyeMz+OAlf5Hn2xkS7IKZHAN7SeBaItCEzwC+XgBsgXX+wzaZn8x3SKcmku8+qlk/dldmkWSBybc2147rkmzteZ9axYJ4LgvoA+A0j+iVFEeJUPli4oeTqmFbVCLQ6zx1xhoIJXx1jrJheKbDqTMYqTSqha/gB1oYo6Qv2Bt5lNzm3ISMeJhtj2r+F1mWEPQUEY5ta/Ing+ksIxmNTYXCVOQRzfuzYVH7t9SU7dW2f6ZXChql27qB4CxGccMoLlEukIIJfOjX+Pe4XtNUwYlv1W5aofmntgsPG/YuNPU4AW8Fl394IalA5yPtjSuyCgv3FV9dJ4jjuV9+mAMfAJw3SS1xt4aeYXo3w3xEBK80vjff1Qu7+mHvAX0kZZjPhptTocLA2QzAUL/zFi/v2f4ib2aYe5ilHQBSKxDXV1nvOkdUJk0e8OWmpB/P+fz38G3cXJEBIQ9homqCSdckGIjaiCZX4fKs471u/2uX7G+wv3Wwqf9VWA8vPDf8dKv0T8U1mA6YjK956w0CeeJ0pC4ZTeaHHiGvNh0j4t0bSJZSN8OVBzWhaTVPwHBNffMHkv45+odrBpKX4xttMqQvaEKTWjok5rbrY78TAQmFc/rPqZ3s2nV4kioRoQH9ugb/qNiRp+5o9NBHGm38nDhcKEe6kyJy+qZRdFEKy6tMkarUxEJYn7MTorBGkQ2vFMXHh/pArsbMui+SN0Un+1SOBEE/3Sr7EAddiWG2VHemD4YLESn2B27dD110DQcQGRuGsNNbbCeThSv65SGDFEmeYfgSVR7iKYW6Gg6RW83URCfs8sRIoSZ+Mvvtim45j50J8G3MKQmbYFUBLvCS0TLuPwVYviumDr0XCqk99udfxo2+jEKEALLnnWd+X/CDY4aOii+6/MYjU5G5CZV5VwoW93KPxtxs25ALNAr85UykeBu1xeYRdLJoXy3YANzZ+fbuJWKuTvyuaF15Zw7y9Wsm48cpaDdJgHVjG1SEge4krz3etMeBKQwQSlxzg5O/6Q5pKBfTF4sFmJnToQPd9fuf91i/9TAWSryZh1fU2GONpg/3OAoX6MV9i1yj7ggQgia7wQ68Sa6ZBxygQnHh42ppAFKOf8Sm/cCTsnh8FpBBshlcJ6JGDJ0Q4c//WsrsduhIovxfdMErUIGTPdY+Mi7SnpDZp2Slo0n0eYwO2Dg4IJe/gxOKeITw4jg4Z7O994VxG6HtgGIfobZuAv1gePbc030M83LDgl5jLic+4tkrp5iZJBNa9mg2s89sjAfXi8Kp2Dog/ptRX3JdeAAp1RtrUF2a72uy+8BqImi1DKfcoD6gBawzYODM48OGeHp5+grV+NI7EQJhJLco3ZnzgeQmEnYF3Z/wVfsaFJtCKX+6tiHu4K47Jzz4+m/qbXpuDAg5Y6DLoF7MU7lqiSo/dEs3z0l/LjrKaD+sMgQm4bXLp4pf5VkfEHqY3wcmosCL3pgoMD1M2V8HevF/M79ndwBZYogiuVcxo3UVR/Q9ODWivnLsERDBGpCQCI7xuZPetCPB47V4KJU14P68OaFADFQlphWiJpmDo3i8FPVsW+wJHvtfhrR+sVow56i7VGjMfz8LTK2GgnSiB8TmUcbHbJmoqWhp32pIHW8izK8nHEmvzyQWAx4i0KFZcRNJBst84GMpJ4fjdmTHNLolFwKTUVKMr//JvX1dek9nhsavZWsM1x3yh9YAR+21pYol5uy4l/yllEl0fkwGIsf/1j4/4Xfl+8XFeikIXmc35aRzS8+PTtEcOjvKBzQOPaNNTIOGXy2vkM3sDpHhBadmo1cG/QqHr/qIQq7o2h7S1Nx79UYUK7QUwYsN3BoNj7sKPrUH93TiFwoNkp9P3K7179nrMhXOVwzPrg3M0za1kBHR3F78ctnR8a9reYaJwn/42ORgva/htupW/dAOwD6NFYVpW5MfAzkw96v3XgeiwvxMFray+cp96WVi/LZhU/HPyQiUFq7rWiVoyxz0fgwY8buaQgCa62dpcaKw9EoGCy2iNxZuxdcl+M+VZDdRQUkJUu/HJFrTp8xaPPMDBl+3yfLqIvzrNU3kPBv5Zm4vwG78ctrLQttWP13E6aWG8cZEK0/yIW54viTowtbt9HK7n8M9ruDz6cqR8hGX7pJT+82gG57kFXhHDu0rGAoJat7uYCt4umtUIxiB2U3ZhxORjcB9U+zIWZpucp4e9+eTkfDbD1KIBCAVXBP05H8FPK/B3mRpxyFstpbr8BPTTqqorXq52nR4P+w4hyh4wxQ7d512ASOFhdhhEnym4ee08dpNQSz/T2YQRQZpfulSTYib7F943Y7QcxHDE21Gcbwhad7I5tj3owpPRqjJq5Ayp+9QvXYZUfv8KwYiXiJAOvTUDu54l/VOHgGuwHj8e8zODJ7qFRxJkNGPgwr7Rlv5vO2dvsKkaCJc8Y/5Sw6cDtVHYRyzBY3RumHvS10o9uUJXTi8ilCItqQC7YvnBbaJGU+kRVR2UQ22UUkFoJ84kA1ZL338RrUmbaiifyG6enJtjjaetijh8gfoDBwZSH3TjUh1VfStTYAH241TOMVt9lZrLIv/KVPhZs/wQhpBhheXyK4VDWy4Csz0Z0jbU1TOMPyhL5b1TVdOjSYNgIPan0FBkwT4NAt91RXiayXF/M5BK5Y2p15GLyb2ZPS1tnU+0+wfd/2GfjH0AHAef9klVNHyiFrxDfhLkbyGyJ5MDTnbgSgMPU9bsPckoY+No/ps/KCyjum9UIUdyMQMxPIP9B/J8+MC8HbSJxWqTuRSu/5VQooo4TyA3wqbX564P6e1KI0Vi/DfuSL0FlZ3+vJRTsryDZW8EqGqMjwUgCriUtvvprYpli6Y42W/9KMncfYo/CAyI9XoTQf2jcSpnyuWFAUu3iwaYLd/CDCkha7o/WysYvYf+kYhmQJNe35geZK+KgHyZ7oEk+G1WPLusBTdGlR5+CDe1yzMTProU4PvNxfaKeIW8Zg3qoEhTpiVgYkUvP8IYFYRbe0mtvJzt/TK0hxXLmCtZTMGlb6QFtaOW3hAF+kPun6dTqsWWFTVE3ZLN4qyvWo0dvWQxz/c0wHMKaH0h1ikXJ7x6cgww6GxyDi8V1lb57MWrQa7m4VtRPN1xsar9qnwlYOBtbZs8gYb5AZvVXI8rc/XwMSjxeOy8VNSzX6mxtY3fUBe/WGpfndXRazgpoqPWNH9B4QCztV5FSWPI5yb40+1enolISvTKGnuqzVFoSl3/d2XCYEH7vhmA8nqm0ZTOVMN6ycUo1GPq0j7w+2dzquU0VcfrwJetnXK98YeUUmFSM4odwvwtKFr+mkZTxjd0mW4eHOrKNN1/uhsVVYnNpi2WhJ37Or9cM97h+GvEAnbeU0/6LLstX3LwZyFI8AkvE7jaLqdEqBd/Potps9NepT537YeVYjfwJK3YcX+Fh8OcYhMxEl/nVAkHwGAmKSfEIunNGJtsGarnHfXjJBUg5orT1OoQZ/LrVlB9baPlQ/jaGvQoHJhoMi6VzsJA5pCi/+6ANt358fArac3NW9uI4ei+dLzrCwMKlSJ86raDEHGSMpzA+H29Hny5ZBt4Zr+mlT8+j2VKkpVMCw9W/iCBN+gXOM1MPInB0tItffYQ2yXW9KudaNkeYOIofOIL1PXis3PlQ36KUQaBj643m1Ip9rvWVT0pu3DTZ5/1fH+/eZDlDEzBa5tDaeeZ7zjQF2Q9/BIocvRcfHIb9sA6FkPQefJulkIGoqnYImq1m/Hwop6TYOD1fmKXDA+fvqjmNJ91gHMqHy3k6peh0mFEigj7f9cdMMiqqAm7ERg6OzdS9ZZPepBNlWw3xCyX2h08/VfDPHLvnS30RKnk5pMWxeXb+S46gb+ywxZLanrql3mghtR3EQhKoyvmIbBUGrqvfY7nYEbscE3blHMnCEfELVzKpWuLIHn/RiSIYG3vTnNEiJJ81Lwtgk53DV/Ob8dD2/3IRCYzkuvmTpcD7UbrXkVORU4QQ5oykQXM+l4/wGZyeDAXokIC2WDTU6+I1Ipg5LPqSyAGamPMqwKu3hlrn0IHnRUIN6j69R/jy+3cgMmqjP1KuO51XannjuIriNYvTm/zdxPdsFC8qpnNLh7IeL/CgduojjJOkxPWjfWgtInW5vpajIhjDFtkMoKQec/f5ikpmR3oZ2koIG4ifFfdFK5gAn+ExUCuEn1dV5IZitwrMlXlq3F5sgBsINsf4/U7p9R4Q2uDVqdZWy7A7I/I34Aom38rGTaZGg9P2l3/8Tgu/XqjhvvBqqKvzgRrf8ekW6pBWDAkGCsIIoRG3yaPg/DVijqXjNJrXR7ifo4y7rl/yQTVvLENkA26yrQ28KWESbLaH9s1t4KREGz/7gVdaRmcCphyM0SQ9lOR6oA8lu31ihYjd2OLI6HVQR6T4ovdpXPID3sHE52Y/D4+wuqXWewfZZdotU2XPRNGb8kjAPAoYF43Zyz62ovBsNt9HZUk4E5Xa1NYKbM8GFv9HabJfy3+3k3tOR4oJ7FdMmRY2hw7OOu8mOLjnEL8hN4fwuLyW4ny3TJ5I0HjVu48N9ajPJwAX/9lulPtJHz1zGT0GeIT9FLnqqs4yuDrzB+Rf+GU9t7wgoWduTbGm39Xraz5/Wa7Sw6K5xtM0CX8MZdvD1QKeXUs2LUCRZogWMyTtveP1rfvQrlfXMPtqfU8u9ELcaMzecWRrbeTR5tFrkXUcUgjf76A99BNGJZxquJVgwwcycMN7oRXmO2dum9VrMW47lnh3cQj03+lw8ry7n20FmbVbU2vjIZ/A07uGUlpY5vxX5gMCU5fZaDfD6sAu4ITvFd8hWIfPjbwiqQUVT58QqasXj3+UI/sYkIinDLHn0yAdan1L2TcEs6k61SnwcfFaL0nQ36boSa8r09RzGGageeJjbFeLMSkIj1zwocu/HP/eK3hKfNzHs+VVC+QCafMJS0+cr9pX8AxsDTc9qUy/U/cLE5kCPz6i+pc5Pto61MUTYalyASf1tvtq50IOd2GIoPL7Vx5jOct+aUjhuFIy1MficOQ8X3NwgdZfAU69DCM7UR3U2hx3+rXax42Oo9/YkWm9OCfKEYeq/pulv/onhppGX3pRUB5JR/6avfx5adThZxW/l3A4vtlPGiUHgjuW+kEUK4rcMf0iY1lH3F8IB8CN03znu6OkWB8GypNYA5oy+TDcjG3td03gHn2BslOVnM7p6mHh1PPHWh6q2s//pEg6jVACBq+zrfU1YaXAKulQqHHNWF+apqBQkTl+aRdF1clo7GokacDxJjh7N2vZ72fyPa2IiboNCTd/RCHVoAPW7su26YXc/wqcbgovivxJXpUE7UKlo8BUWIFqQK9Oz+qz5Jnt4Ow+v1Okm0sS3wBZqiN4jg4CvpxZVettV7Ij2PF6WSidVJ+fcq1LsgpIZtMEpEodTHeUMMjI9hRVSX6t1U6zlFD4iszNCN8YYTDWZPzN6duogQPtm1h3x193vZ/6RebC+BgghzHR8mkXf6PHf+jwN4HdLzLp7uhnz1AXmrSRmqS1MB6kkN6Fj0RbxT+OWnwR5fy/JKWF/5iPYV9nDmQ/ii3gX5KDr2M2O2ueo4/ozzO6Y9GFSfwJ4XZ6HBnvr711BTM6Xfaxdmwh5/ts8ZLBuVunKy+/LUWjEtvcI7t3EmzywNuYA5E8TNy5HLyY6PenvSGHab/kgIM0hV9g1jS3L8zJT1BXD0+tCJYdBUR9U/+iDGjnOYevcaHdB5GPrxNk+kBO8/0/v2GCYzIN19iM+ABFGJAbzyiFvfQwiyjEYdc6LmZ39te6y3tQ/QdNz0uUM0lq8ldvfu8DqJDnX/pwkCnpb5BN5UrsPAcUrEacvcZ30OPM+ZHsJhestZvDq64G5i7kNWnqnELRMjaBzYBq5oVB4291F/gDXxMHdpudMumr3a4i8JfgCZ2GUqDuOXbuyqdadiNenG+lT3GZalydnwqX+X1CTpJua6Ai/rLDWVrBxsjwQGENXOCeH7cgP18sa7ySQkEqaBruEqiohP+ovCp/uQn/4ENxPQ9k1z/9Q/G+l/BmbuqLRT6/eU2GxSKe58IQy1pKlwAyEt/Wok1A8NtzCp5RNxjCVwoqG8pnxkuGWwG9mGdojcnVCH+WR2r4tQxoQi/S+/Kxy7s5NKn5q/cARwcaCbKCW0sVfxYHWAEUIynmOZMm+N8VUb1fdQj8adIfVobmMvYxBShp0EBYJHpmFRlKvOzn3lUcCsQquRxESkff0cuJyTIl7JZUsxJTvIIzijZD4VRfkAHycKlffWS0ZDFTS0cvXyDDyVDMnljTwx5ZMuqFrfv6+MajQfHby8rqvaQdhkA63gVJh2wP6wsKGe43KhaMa8pn+xCdP/Oe06lm+wHJXTbfxVeTfSPljmjXRIt1wW34yaZ+FIzMG8ajc7BrrvGGzaXQ/DWWRsq44t3BPa/n3ynNVZ36z25qUpRqa5rcUk5qmORujNTXsGiPG9XRUBnW+6XIbmBQYnLzzTT5lkXI6ximupx4+9KSvXX6Ioa++zL0Ux/+8PS0VSX8WV5+hd9aupGWo5FuDY+kD4M5lS4yekhWKh+aQmsJQu0ghyTTJuS8GUyzKMx72F3FITiHRqihFQrxKdV/9Hf48rY+3xNQ9CHmH85HoORHD3KueC2/b85CgWy8wdVBNOC/dXLwZlIH/OVI6zoX+e22xVpy2KN5+acahDQi6iJHvQnXweVItimvtSPhD06F54hf9deh+bK1uYK26vv40feq+AYKb+Jwb1+QHXKHjblMTvKCc2oPxfN/fquOH/yWZfKSnY0fXBSxwYgeNjjfaN0Bcn14/EW1F1xCGZIqc4MEBI8GnlAndl24R14c4gZe0SAFD8cDVd/GwWl358uk2m/6Bc9U0nrXYuU8wJSt0uA7HTudTrs+QKK9K/ysbItil56mGE9C2uZ35WuZfbrzHIZgWvoB0j6xxwT3T7bvosaClb7Y6By7p+dAXmSz1mIqccer8OIjBtAAwvxSXJEcpwjudA2mzbGOaBQPdBbrpvvdnPdBqVdn3SAtKwXhINJIZMBn++0NJ4ebVgtavwlaw1gk1ZgtkCNlnnyc7158Yvdbm/PhYHrmdBcGaY+JLb9kXSVkg/iZXlv1xVJkQcO85+YhzpPCjIL0/KwOL5NI+W1U/epybAhCaZf8QAy3Hy8Yg6uh4AaGj9UaUpb6AA0TXULzRR22To2Uq+u/fzKzfPK5mkniTp08e23g7Ioks2lkCb9NpH/5TG69BR2NBXwd+UEuLhq6O5JbZ7o1GCbvif5rZi4GWDI/RRKPO5GulHb5demr4jeTqSM86JcuGsRVwl+8PZDLunX9KcIIlSNUMgU4/nFfAmAtTdQiOEDR3mghe/kRqQlHKp+BCsz+787Kz1nbUV+Ov6GCFEu8eFCn1clnnw7FzNyy1lId/m7dzA56egNT2de3W51ZrRiJMlbQAIm1hhsB91z+JjHzmOE/Ceh5DKhFnUla47zPLu0SctKkgjAc4CgLbsngAcE4lewZMrXS+zPp4EyzlsMhG5BG+WD26oPPnuCejTTeDOOzrJfGCwid8amd/UT8HYoS4o9V1/9iDZO284yfkfvXRYDbsxaUX5H04q8sasGo0JKzc5QJ3koeDA7lHtLK1iyOTFjyfn9j4MPp530gBBGaZU0pSugXQcKk/F4ZtEUdDlkCP1nUETrfn6rDUHRsIzfcIJsihUdHutFZx85B+7RTGt9TTpq/EWkUzFUREUVjoUUl0RwKD07t2f/mmPuUI8xYN8Kv+aR4wsyWnFqd4+nYIDqsFbLNKdbwAjMNl/53UgeSfcWJAnbdM8ML9NHSRPaxSvZlw8xrGNewMFK3XCES58qKm+JNQz6EZo32AumhyPLhJk3LPMyNXdzO/Mu/ihQpkV8W44fzt/jjpuHyd4P6hJ9Jw0xx7uUq7xuN2d94p1/KanmcDU45fyZFZ0NddnCxH8eEBOaHzwmjXc5XvQmb8wL4bHaaouMpYMUpofmXbkaQLN/SxwMrUc8rFGHjiSmv0mJSX1AteC2+EOb7hdNXYvYMzITxHPM7wvhLIWf6++sQLmAGSUtmQ8muDX8+LN0+p8EbqRbE2mXxz9/UL9muEvQAp97nxiLUqpShntH88ncHOl3doZeZ3xmn7DCfRPICm62ZOsWkZBWu0hsc1D2kLOYSHN/7nQbKA3DSYkSG/A0mnyFrSEBEWVCmb+NOKYWKmPpS2wwWoLxv0JC4HBy5L5/ueurCy47Po4ZtFh56SPKjEm/STRRnnh62MrxGVwBumbjhofK6IvcyVn3z2zwFqDsQrByz8mJRur45wsky5nykptOEP95G3ijwCb6QkZxbBd0pD52K8P6vAxsYPC9KxcsmS9e43RyC9G/MUi+PIuwHSEy2jlr7jVVX2LSfA3x6mX+KJ8tiHbgZqVPayrAyneJkUTYK138kg0R/eUCT0F87uuRdXMTH+JexDRJ3KECrqaz1IA0hEDT+VRj1NGbA4ju0LpwEnNRidwBMGn4NFST4h+MgbqR06RiQWndD8YTgV1V1vjWuSqZj5Mehg/hEybkJQTz33bks2ZSLu+N6dKbW59W/Vt6knH95dJBc7WkVEb6xpd6hOUhnj8UFRFDDxuhenmq8oEb+Lp6X+AXz/i4StDwhHel1rM5E8Kz7Wy5mq8CPtkD5GP19CoOyQYUovYEXqDMK/pSmHGgyY+DFpdTgRbYjM0s2y4YDus2BnJmH/4DvYD6b+/EDCuXgcymDc1Z75QWmYYZMuqhHL2V7CWV/nDqOGs0hMG2W39d1PxRXg8OEfw0wZfCG7zpZ8XfSDbwVyJ/B36/91Bg7kovgSo5fMxO44oe1s0OEgnEcmmnZEJ1hwucDK4QyWheqQ4z3AwCpchiSDOVxrLkQ60NDW4Nv9aHNgYOL4QBBRGpGcwnjxm4ReRAfQqUbEKo68XlegwnMlLU15IgoXDIQuvWh2DHJGBln6Cdc6CUuByGXDiz0fMfALcURIM0Cl/H18l+BNJWULHNT4HnvJwiBffq3U7tPoJIt1f3NXH5iyz6HV2lX8dktUHTKEgVijP9FS4t+NXkI0of/rhOxu172nQo2h3zQB67ZAJBqP2dO3iLJS9QTIKY4emzpGxWIpveyqIGvBPNRSNufLzxqVjc014XiNISAFCTlu+7OOnWlTOZ/mM1gRZXJBl8RVfQU9ccRJvUTiJfWxcJpxAUToNjwAZG1fN8X0aEzTm561RXFSZcXgodXjS8TpZ6yzv+AZYXf6OPTwsEk+16ugV71XuzFN1DNIqgJECXSjwasiGbABME/esh9X4X0lP+OA1laICBzCz9/pZcH7ZH5u0HAbNTKK3DE/CnjkR25hcGvCbiGf7baVDDJYCtgn/3KBcNcXhc2TLK8uoptw2hlO+v7P//GIZKv4AApB1bikrGw/dfyL9625QLg9/uLVWeVTP+wSmdHKd0ApzX82ivBEiFboPsSGTb7SRH7d3EgeOJ0FAxF6V2xEFZH+kNKsX9uZgdH/O8m7bTjSFpX+wHGr6C4YbAnUCH0wnsUbHW+fa2r1RHmjT+ItycMdz4Lv+Gpr4wv5oU2YuEO1KtV39U0bZXg5DYFYfrvhtDnR+h/hQaAOcRZqurpnkHi8AqPZqt9UBCsIZUV4grz/UbF+5IMh+fkHd/x1X5gIFrrAlyMYivc+md1XB6T5siB50SNEz8d4USnaJJZjvJD+xhs+C6bjUZTCz1/AxiBBAMRER6nktN1cK9OnMm1+GdfgNFEs/6cL7sbAhxs6UYi9JhR2v4p4HERYAeSCee3fG+SRp3J+MJ5maX298YQADWZ8VIPhqXP9WgJkNZk+UxE+d8Wv3shFROru/xdvaK1r14mTjyINnkHts+56CGSqyplhUj3srKvijTAegBYIzrdoXRUwoxefeMO1dnU0v/qoSblYaUpVoLt0r215qiNT7E9NvjVV6+EEQmCNQeP8d+J9jd144mrZV8tqlKmzOAD33MvMItdJ9Z8wjbYZ26L+4pZrwOR28Sh0oRvugWTNZAz4od8bcyPJlxGVcW71a6m3q7GV47DuFdsM0GW1L2KRPOD7idTL63Iww/FKlNh/HyohrKfBK9ultbcd0it3fW2iG2olHtGkK0TWUVQz3hOLlJ4qWFJYjTJ/5llweWsp/3bnVZE8eDa0RLHQWVEa0WYtPQhl1rORWBH6WQLyGmcVa2ybC1Tk+NxC0oVCO9B2CYb7REB0vDihVLXrg9y/Kz+wIiEocOHq7YJ/0wUYQUY3pRFFQHo325O+IgSuhHstu4/f7UIsUEq+ntqUw/nDkFCZk6AzSckukIEmWa9H8+fPG0SdxPw0u3HH6RE2j9QTTSKfZgv0a05mrX6+BzW6NVZtJ6r9MuMwTe/Mm6y2udWyhuYtau2RTSyPC8XZVgfsmsxqp/Js7+jU6mguQHySNegcQk+OcdIfR/0VKOExvhXMvLgZcN+jqHbmtHP50a+jxGQqJxviaBNePB3+h1drg9xslykKsXEMuXBAJoob1nQ1cz4Xsm4SJcfXjVeDcfWvEPRX9j+oKMlz9YHsFIaihTC/sVJzdmKmyBc6sUsXbBfWQF/fh9MJruLLqONwRsSox8+GW/4G3LUmAy/eKn24mzCHtMMkFyTfzlIEBVf9PMXToAXa9CwkNhJJmARcVRLhg40rxQ3Tb10Zs8+18sQgSKkxL9ZugAfGO/7NXyuG+gBoM43CtF4I8OnN4C7dzOrOlwSnVZhR87lcWmucbyCtj0AXIqgJjtiLcph/c/WM+rtpuQGWmKwu1n+tXhNkuvl7Rfq5VQXLMQVx5akqwF5b2Z0FC67K3mj+TymZQH611I7i7BXHC2UKjiwGRvKzxwK/ypSoID2KDogCzdBVL0P8PG8lPyOrwS94blQUkalYGNzmokMse2LJvT9sO0THCKHQfokCwE65bDbKMgb8jTpyxtz+GhzzRGOXHIP3uiL/Xy0/obW6Yx3wKZLG96O85Usxcj778JBmOtqGr88sj2W9vx4OnyyL2Scc7E3OSjtbVwAoDwsKyefr7z+NFvWIjpAKaEQjUVXvp/hFfivsL1Kkj8oC6QGk1c5IgbwV9SKZF4Scgz0LBGvTuPTI4+orX12n67QDACmJTL31KJvoOHDPwQjvy6MZMgsLyQ3V8fTc5vlWPLLKTbfMIwpler1cz4tCppPUh8apFMtCafh6G+seQcyiu79rloeZHa71ohk0oIt/vrPy3RzzGUQx7fCZAa5V3HOrrHZW7Qo7nMWvau09e1vrueoD+RDIPK1rcwdg5xTX85FSTz9jaNCzyibyQg4MB3dqfEon/V64UJeW2yUR784TMvt+i9hoRanW5XVhSsBmg8pAfIS66fQ3TLZ3AGbPiMOO5/Y05zfRVNv5LED+ZFrf662EMOl7bJSAnTHFgdid6aEcUivf7HkrzHY89Q9zShy8yA6+de+Wd4bmuKtTODsC/sU3Dz4Q038Q9NXNlntK0saap0u5HxkNR7NlvD8H6fNRVchbi1A/le3YdukrA5QDkzTq539DA4LhTY4/OsrLmX8byRAFaGvt8+5fewD+92KZv9rGlbDTr1KW5cU+ZFMzjnc0D9y88pR6GTEWvomr1RYbXXQNTmQcSo+KQGwhsdP24cSbdp39ikGs1BYzuK/4Gx0jMO1nCFlE3QFdzZ2IuGciUPOu+TL+QDrOlIcKJTP38VOIWPMC4HiJlsNJEnr8/zdbOArGi/YL/OOsl3Gj1gdI0dL5bwAd+fE6sXnAmaXmPUGtwqD7/RXC3GAo6od8IZKlZKKpe1nq5kmHsH2RkRFWZtHm3YYCEdmxLG/UicV/E3WAgW1kZ78+ghc02RXrOAwfVDxv0kgubxbiW1qic0kktpyxf0Vr8/HM8LIjTrFzwdzlF6PBbqgLfJCvdRTVGZvPTygIof2iEu0ZKhPJQQApYif/lK0VKxBZWvs3SwMOLXTUOFF5s6D2D56nHu8xmNXCls5KURYHBOd8LceaSIDz8S7aqHqer/5iMIahAox7R66D5Fm/KQcbo9hWHcLEbmG+LdZhnf4Wnr1xoTa2LlIOoQobGs747I8f7GPb6Dga2SVoVSx2o0IcYon+O00INllJR3qIs0jsJ3VwAKiXxpE93YOdE/vpE/D+jST4YDLCImm1BcKFa/b1JXW/nYhliuq/a40CJlVOYxsTUjWTWwpm7MINgfTG7P+xszo+PRyw8pf5pcvPblFKsPluNUv43dYA4x3LK/wc8CtbqX60qBg49YT+wLhc8Ufl6ZS+0y7rRc/UBFaEzjDXmpsVoulY9mxTjN7TtGoLRAzlfZZIVYk//WL6CbtyKHVg5KbLXeKt4az/iUKsKyUPNjELhMxY5kFpClAHdIXiOutg5Tl342wC6ZqvFHP+97veflCW88+yfM+6tL06WDKjuGn4kl/k5Ro8D+cgTCIPAxbs7xH8Lrs5MKGnPTouDbgt3jm1vlHaqehgyh+K2F068jU/B7GKcax7n13m+WDFRAIhgErZ7ixDibNs3tpPTixFxcVAmTkAImIfV0WBjbAEi2OgMws9Gy4AgxE5II0hHJPrGWPUx4T/qbOyMydzdJ0VMnWfoIxSy8x+6gFan0Lcp9YjVwGfTt/V1qqjixc+aHEpdGj2Fb9lVA5BkzJ/Q+Ei/hr54Nfh5EGQc1vkLK++W5Ef0uJXzXMvbOCEJmd7caAUqSm/3enk/JPKxnDvWkVsipEOcLs9u+LP0zcVSNSTb0GIuQGk6u9qCh3WQzrZ8Mdb2XtegfD53rsnWfJDXXfxEj0NKYOeDctUS6RUaoNi5lCOPnLUKKb+nlh0fwMB9fEBP95XS86dnlhD6eKYmaa+YDolHxb+G1lOsxEB4By9nZD5/5s0JDXsCz2Jh5aqlRl+PFTP6WrZhZ8OMg3YWDCvOY9WpisTezY+XQI5kpetbszwf9i4zjxKCobChAiGy4VOdpE/fIdKED/WiXglq6SCrmkPVRLaDWFTasZXtYGBuY9wJJwryLqyXA1/GV1qLxy1x/DBn9SM1GVmEnUkk2BqfGQ5OpXwn95GsXwSovcrztTdzYC/wVp4Y3O3M0b25eXSQl/nAzwbwDPLwePTCKm/vqho19IGPUVTk9k032uufaQZNa/XOm4lXrAtfiNQwQ4W2qIviN6T0TZORg5BLMtq0EdpCWky2QSGbpcpmt2DLe0cBz0sxdTpRgfYp9WMiRa9seSUd55WnGcLXb9KLp8urWtGDcPiPfR+nlWGg2D6kygBO7yF9RkWo8hDsNB6eR/d0//CdH8OJSmODNxyl3lHqCAgCAcZBhARoTW51ZjQkrWLTI3PRT7u3PulWGLpviP0dUP/DdoqBnOgy/Vdc5tpowRwT8QO/FeZh2v9GBjMzI9OIuHsg5OsFJf27JNovcTZFRXIh2Up0uscCr5GvvamKtXsGKYMTOjVb3m924CftSaJJv71y//BZ2f/0hHkWR/5xn6iOJjFl+iNQh0VMFjOawwhZs+yn7+7dNmlldqEZfMWtn1KJfUwN9aV/yq7PSa2AHwvj5L9Qss4SrCj3/X5FmTJqn1PDwHEy6Ksq8pcl9ZB5INPcOsAj9Z0VItlY8B++xtw3wefjLwtQDuCrE8MBLqFBS1YCPkJRC+AEjN31wfoSsv6vnr6hJ3X5CKag7L3Va276ZffmLSReujdE08TTGjFpifuxlIIq5m4Rv/ZmurMu9v7hV3dXr+/Zs3EkEm/0XNrClc/OeCftDPPva3jnwIvXOez89ywFIc+qWR3mYawccwU7sBzIsuyBrGeCnW56+qE4vXGlFlFfhBX09CQVjP2JYBv9IA7b2M2v9km1HEQOHUnt9OCUz4b1RGahszcJg1/J/WV3Ns+3Ys8tbzrcCp2xwa0rd8wvvy/AFUxrwsHXezzOkaxQ/mv9u7u/85Mm6IsIaB/tqupN7dSIrQ7k/aaOgSUi8Jcq+q/7MbYLr731ftyWiLpH0kO0F5VhvCyzIVXHDcaIFlF6n/spWDW21d18mW/yI+NqqMe7De2U+a+N/Y+1eyp+fLb4cNeAW16q+s7v/SKvIpI1axSRG7brwPYxAto9BB/P4Op4LX0v4UW3evmU2091RHf/fV0eX+QDR0YQSZf/XKv5FKWNGCZp3fzaO0pJEIZszfSW/YFS8PqGATy58SJ5Id+FWfF/ndn8GXkVl9VaVjDUMD7fAUOnyXi7dtc9Rhk03QNNYGB5j2pxcAI/k6sDLMHfW+mKz85fDbFagJAvhQfdXvWz+7LWC/X453/EjlKBBxKZczLwzav+nj+d3pPOGa8GdrGvn0K0tbKV+z/pBYUWZaQzz/Vv+NPFNKDCyGn4k2l2af8/LiBoBuLMir4sXvTneKBhNrifrBS9S+nheIevhyUvvRLJwiy4mZhSnF26yEl9+hmi6ZfL6r2vzVey9rGD3s51WM9dgbx0sQHK6DLAOaOMbbBrglsEkl4E1pMarPVZrEh0f8rvMAYr5oQ/rWqdjZpWraN7sxcWFsDnjEgAQcu5BT5yQ+f1Qk8ndFGjsFjMWdtUy0hz48+7d/D+qR+8nl1kP6LvgpRAf4Yqg462HnNx3Tmo5aenLyZeWLZM05U8KivUrg2Qlcfkl8EKKQfwdE5O08WBUWF4w+PnqsJiXXCRXuIP8CNcDnHzbSkRp3oxjZpPaBaiU6cagQM3KS61Y0KbIYdVnluvXwveEbBfD2ile8Isk2dy9ntCqzSXe0GOSe2KAJ1HLnuLhEQEH+dUnQyFge9oOnsKXg6RjMngJRaN77mbnFgovNGed4LGuhMBUpSxcw29i5BK5Pkq1ph5irSpRB1GA1tUpSUcmLv2M7ZIpP5YzOzwvZUSvsygjUNtw9pp/m3pnxJfpFbFTlCflCrLKGH9cCdP74gHY54sI6wY/vrxJAddCM+EPcXQo7/JQpUkScG8QqlUyAaDWROg38sQ+IFg+bOv0F14oUzsFI3VYU/OpMFIVYpK9ahu5VKqSMS49+T2KdNtSCvkqpXMsb4WDfPmd58MDjV+RHHyUdtFeAz3qEPrvQ6oBjsq5dqUJxTf7fZPooL6w7MKwyNu/wxPWI488GSvFGSb7aKLkdR/rMQRonhv0M3rM/RZXI+incto8JPVCfzJ75gSGnzW8ZQaZIV2SwmD0SI7qiE+CLeQ9Vui37EIjFkQI2iapPdf3uC0rlt8EJ7eZBDMfv5KmLiWoFVsHEIUiMWBn0HAUHcKzFCvpnfkI56nCcwRHw46sywzQ62uH3GzUqpUP66yJBQ5Nd8K6SQJbe/VVBfxJx+b6eZtwHb+o3iHRi7mg9i7aa7W4lavCze32tYVlVIFILpcg7V8HprdymogEFjEV2miwsyvz2AWWb1kGzZvhIgXJgX5iwCa76opnG3C/9vq8mMAVqyUiInO1O8sy2xVFExROtDrbEHfsyttrCXCYCNz9gR6CybC5LJ+/+H/pIxgolzKkGLzOW7Usr4oDD8aDaG7QmmEWGLGsmSILHUdkaj+jU3CHmfBF3WgFYkU+7e+h77ix71t38jjjoVpQwLYBebD4v2vT8BWoC55OVQdr9JGqQE6DrVnfh31YyXxHVIPF33ZsB0d/l2gusLd4HcjnLxwea3CQl/41tMIHipk65AoIr1eDVLn9mZg8Yinlb0RnBfFQieRAbAe9XRq8EoiUJlEH5nq7DOnT5YUrvOqL8+8cpAV9ih1E7VSk8RoIgpvqnuS3/a8voLQ3h2FzetVzuxqPPgqH7qOUwpRmvDCGUAM5fooM0R4nOYH8RV7wiOLOwTunj9EeggN4EyN3g8liQEUIev/KnBt+zgf1P9C1aROG99TensuG2ao/qpun0wN/8R/rNtlkGyoSref4hpBK/Qiv65WQ/DRWbyHFughBPLllL9VsRFI4g2ILNdWc8YYJLMPPgxp4rs5M8E1C2PemmoAsm5egpGFPR6054YxxEaobzE3+sqLG+wxB/fW44Ir+G1JDlSs1Aqg4yi3ZBqoT/QZENqBxeMJ5cSVC64zS/s6zF1NBnmOM9GnVGtxcAkFJmxg7mAaqeRxLZHTF9grjQfscF+KswEPeOEFsQTl2s/fGrbZzSE86XnzSmv2Enomi/a7WBFBFQI06zJ5PAx3GIsZlJsVP1tS5vEdsz2NHqdkqHEM6alkX76n9iEN1xi7TFiCeIGtJa7QlRISlzxCSFE1/J2+uWPH9Mde7vseTjq/e+nlsG7uSov4VQOzc7/DFlhF9965vbY4ub08XPAUjPJ+a5sw1PaMSd/Jr4AakBzMR2rwsh0Z1rrZvBIqA0ABGKxfHtd5WMAaEMRfIXyvwgVL+1erDt8yIwgWxXuoDSpGb8+X6F5u5FHvcq0dnNGPPkoO5AsYJYUIzL6osY0ol/vZX6m3wgYCWYWwZ6wIolO/xR15d75H5pU8gXhrshdEGyCnuS2cbuL1I+iH68YYF9VppSRJ56DZ4cSZDHzsJoR7arCBgiWLiJJm3djP6uOYf1L1adfzWpxXUkx0upJIgUwaEMfRtsf1wU9q06nJksw2Vtjb4j+XtzdIqY+TkTDRkHy/asV+xnaCo1c/3UGf6Tvp/IbwlxiFkHv24IcbQlBFKeqdscPjWIE2n3Ymx38hq3FaX289dfrhnMOhwAQh4vb7txDUH3+q+BL19acKwDrzv/okTRp2mroBDHETT9Ka/zJZ0nymiI/PRW+OCXvToESZmJV05yOuwr+HpT13JkgzQQz4kQ6gP/V+td7kbVXnzsm0y2k4xATkDmc8uOccxe6BW5Of1UbSx7YJleCFBxkkMROWPLC2cgYYQD/pN0UyB/LWHIt+bCAfEHtwoCu2RYzrObKzFC0esOGllUOrGoXn2Byi1WhFgTZELzWAXUv6x3i4CVfoNlq1ryNJLpW7p4TaMGOhZm7q+MAlDUHBFueWTnu7JLetHoQLr6bRnn/ChL0qX+Bo9F9+f36v87aFzwStEKG57/jMs897SopafxERLljV5/41Mbi5FKpEHJQL+nJf1ZEr2ms6lV6sjPTF4zrsRA6/HjrgA2EtuEyC2TwkbJKMjnPoLpyWdAt1+dfM22OFKa+GswaOrf9ryYb5U0ZQUKx8a5WKNdxXdQ1sKm5XqL046Bn5uptjTaVIi+cA/OAbXipZgWG308eEQH6NM3OCxtLRSIRTkW06exZHnsnoNkPojVrG4fEjdYwswR5tflKRCTRDr3YianPMLA850DTrXFJdib8VJziHCZqfgT5bf2V93c0z8V/+QJ9YrgmBZHcid8dfAeaHP2YHFnD2N/5QcXxq/8+qhcLgLOaONDmuj9jJrYXS+5atN8eQIDe40aJn6v2Ffmi97J/oX9L/ylMXjr/zpO3WSpZOBoMsF/ZkLO0ZdPzYJrVd+2gcydmp5/NxSpXQ+nFOQEhw7qz5vF0KytRBYy6d1PQhsXMBxV/H7sSZY9kX7KcOXNxUHH+UXMvcL9IKRAYQXIN1aE8cHV5ZDcs1Pmv3nNlJdvItuf6NW/seAiooGMj7IyQXaZUSsxLcoJibcUBIf6kzi08L80XcWW5Miy/Jq3F8NSSjEz7ZSiFDN+/VVUz9tMn6lToFS4m5s5BXJ/l7BYPB+GGXaauRdOJTV0nKTDJ7b13cTYu8yWv5DEtnGfsZj5K+Wd1yZinJ7HhRmTG8ARmCDISQSQ9hBg99IXXrTl8C53w8RQkj/tK32Ara9pfx179jfvOF53EWcv4zcT70W75aIpriCrKJrVHVvlcA9x9Bsu0rBCeSLAYb2TUTfuyKPjxREf3zIzW4ZfAwbuOd0GF64BM4l0czEc2AADMdLU0vnou6TBWQ0CdCKyEFUoC39gieL9kqPiACpUcR/NXypvhN+0tMhvn1WR4PQei4p7VXXdv+pqsHl0OjjPkD9WKyXZh+dLL4xvPUm7vrpjibXHvSeeWlV3PYqphQDh8lyJ3dDbVP0mX+6IKhAEtp0+SQ/WPdWZCkXlC+GMgA8h5ktfC72gCoyAqOWi5iuLoqVCFz9jkr1zPt/za7u3rnSBQfvD3Ws++opm5CMi4LOry1gGvIVF/NzNmDlQhx0yKVpgybmAEMCjTdG48yWryXquu3ES3QbK4BEIgiqCZYXb6Q+8ImTCTiL0mcD0vJ+hh755DFPwIjQ8BwWgFRhQDkoujanVIvRFFpK+uQ3oFcrD2i9MzOL+Gewup0NE39jiF6To2IA/5tIqWRKZ1fUeyv+0b/5KMHiZfqBd83DrDZ+Wp1glQ7hOiiq9JN6pj5ZlzU7u5/Q18VmpmlfaKj43fxXieH75QV8F4iHuRAaOmk1nL5YtZhh+DxmGY5+3ulF8n3S+t2C3BESYGvNc/FWvyKfntj2lYVTYNutBd9iYJpYD7gYQ4m5BPryqBSE3TEdSX6uq0S9rcyG3XB4vbsMx3VkcybXo5T6E8SGVbQKxSDpi3lHnEepvJEqFK8X+olBr9T88J52IL3gVD08igO20UYWs5gOvcdDPSGjFaTa7hKOVrDD3H8AFItvpnNqI3MsqpgevEpqfFvx6Bt/Cj66rfmpMcMS94LFyMmaNUU/X0hrKodYlQYm90bRowxRKD+T6t6wEZD/zsLLIVwh8ezPq7q5VPWx/5rSu5h79aH5Nzfnx42dn0o/dMFTIXJ5eTlmQLs9XeSuHcK+g38NUJLUOy6SRZjUzIU2HDV6uKDTdDxtkNINYIwlU8c8iFpBbukdrVdclWn+z5tw2gnw6HveiePS93fsE2Jr9HDoZa3Xt5jqgg4qocYLVv4qqkLLgjv2ag4Z5lp4U5ZVFDZHfkMbkhm5F5Z95HAtyVcGr4dEdyw1rwPsAAwFRjYYQMTYu/A0YQOkf6fN9ztQiqPezWzX/da2gn6W4m3jdxHspY1oBdIA9WYDVWJdfgLlRzU3IVjU2AiY1YAuXkOXIsbFKxViV5UfLvQf3l7vBH7HOTdFQq3josy6Beklai4kTYc+aQJCvYv8YC5rOLkRQEfV9GuzoNeMe6kN4rbshCzjH+ZpMpnuJgtTH1UhbkiY+ogJTK4OIZBG8WgeO429+5lNxc+mCQ5PZ0K6iYGUnSCE8wKR/Ifj56vaGbvsyoDYId0mVZ40W//5Wz9lzYBIOPXFLsenrlw3FUynNkaijy4jfiA94z5aYr1wkfcvQLLnLT8ryHw1h1iHdiZ/DO69Gez+e/Kxo3MEL8GO+6BCd+itfBr3etHOh9U+Bs/K+hZsL9Xhg2pj615bt9n83O8aon8ZMN32QD7V/QAjRPfQVB6WXeZXQ12TZJBHJ1TXQGKCHAkR6+EOhLYXGOkHQ+cSl9PrFSMo0p7kS9KITTOUFyPEv27S6iW9uesRDuvriElkePIiCKZpFPjTFAYuVJhV+xd8xS3UB2xhQAgcXyHz59ymgbGGpy8/79+EdJe7Csp5MZZmZAahjF4WMMDnStTN/jVC40ccQJhVFyQN3Bq1DEirGWrmCpJzAeRuG73SElT1pxjxuyRDEzo+CgX9uJcdQIhe06K/ar9EhMYU7fCPUL/ssO93EUpUJCAnxn37ZtuDnMXdDP8Fxjn97dx8hXpYN+iZi95O1lXyAeoO6xZb6tsaZ39GlzwGX49kAs876K3b+8vEkzVOPWf7AuNIMdkW+5K5sLlX/HSSdwIjWTzJVS1q3xzIKenG28TYj8laqXID0bKZLSVi/6d2ZB3bd8EzmRR6szItwP2lm8wpKlljbtGu9CXfr0HCjHY8K9sWckN/9RcJbTVThU7/YFbe6BLRbCfmLKxG0cXVC2d9NKAaVPM4vtS6t4duxnTuuL1ABJzh2FusMgUXs0t8Qpbtz2rPLSylQPYeXyMfoU6MJUA9ybt8autKrrZcXXfIhXsNn/ZaUxEVrOGmVDQSitMpPOO3ZXpqHHYPcgHCyBBOm3IPrPtSBp9tKOostw+Rz9p4RcHI//19+hlt/OhhrFVLJHMdQD+HcWtShvr5ftxrJ7MhLkNdIyZK8E+iYcZtDF3qzHF+ktpzq4Y6tIvb9jD0sd+Ls6Cp0WMka+mUizB5UfoMZq4O254b7nEAONdKawkr9KpCk3d8eBRY5dlatVv1L9I0MB5qrvi6y5doCcqprqR/CkjfkkVN5j+6ngm1SuvUmdGixyLbVEtMWTwe09MgROyfe97hIA0QwHseCEFcLZn25wnfYd3R/lTs8KRWH2vmegElQFvLM3VOnrWu+v5DTiRMIRUGk7uIkzr+VSINRsxJBTfNMlo8mf89XJTVBFBi188f5heLUMdBl4Jt7Si2T/50yyl9BYXr525v+hY4qd4uPLB/H8ln9vd3F9L4XlzK7Zc4iL3qd7lruL945KW8OaMftyNWRe3Z8P/2pE3veLvSNeA84szQZpj19xQ6FcTEEp1lhVqx912yPrW4Xwk55arg12MEg1A4+Jfqj5cbwdLiE0ix2uRCLFV5laN5vB7SItEJvFn/ARxqN/dl8h4/lUS0DrDMmwTUvBqEIBdhe+O1M9UxgrAm+iP+YVuPQhIQTIO4wM9L35deYP212pMoRGYV1LiPk68HrX/ofJEleh0Q6CnEUxa3rrULvH0pxWCsD7PtXmzOccswfegEH8MAvRbrELnA1AY1G8jllckX4Sqz8tV4ucXWveNSbMDiJQ9nuMR41pGlBzNMAiwW3C6iu+Yv72MsQxesIynN7Z8rs6eXQxLOTMcIlg5KeG7oRP927SLIN/6Z1wa2HQifYAwq6gcJn+gYd6szzYH0Q2440NHdRf1R7SPmAz8X6zf+3CoMinQ0cvCe/jPrz/XxUNFd6jHooWi5E94TRLx70cB2fNECqtNS+UvA9/IhkKyRqagUtwGcXqCsz559rjHo3XpE5oisCeQSNo4MX6Y//F4FQFfTpYCizoWqMjG23nvN+9wJVgPykBtxbCPVa7SYCjX043yGUuNLAMYvcVHGKQ+jAjhj9M8s+zi7yj/3FEjhJrc2NmnCTRKLZ/JWFkL5xOx57MNFALet8Y6E48lACH6INi0lgImlKOUYK0E34NLjMYgnrRyqZL9uxjw6lR0thuAE1+9K2MeQrFJwF/1sIVoMR4+sFOn9GH8Q4rL/xGDHmA0DU1wCVNirItvSNjrVe/vADaE0I3SAUpx4mrT+X3HqDDdVlm43XF/xQ8AgZhKkB0FIaY6GUvfjgjfIDUR/9lFMvZfxbF9zniSsq9eZT45GNLe90PzKlQOqnPVQVWnQyLP4yVDMRgtOljD/d7OjbyDUdQrgi7UON2O14UeGksfi2E53zC9XBfOqq8OXPOFqs7+/6Yh//W3d4o52FkbUY1+OlRLvCLCoBQXbohlD8gwP81VM/Yj4dmNv8Sc7WO7ZaI9XExUibmSjFEeH9w5ZzLzbrQ9o/41nS/Qt3Dmo/wxPrwdQ4/9UV1balvQBXT6fZTjJE//xzP47rGztZONFs9trqeCdYU4AKgcWqn5QG1gRSIS2V3AeleQf9ncAMj9B96lhZYgAQzwRm5y2ktL0RGLwmqQFovGiNDi3htg6Deix6IjnHnzuiM3+9sAaBVOU1fWEv408I9FZ93Mlq2cemkBQwUofTv75amJIJBm12eG19Wl/dA8rC33amZiYie156r8T/S8CmljgmzicYpzfGUiBlT25o4AoaZGMLXnAEgimPjJcWqtMRW3gT7vG0J9E91nPQDJPxR+4U8DHKv8K/PqZU+Tc34hV7uv5t8ivm+cCUglKJOb68T8IDrZriKOkDRBCXCcZBgon6YVcIrTFpl8k/nULuKHvAmCQQLl8picMD8lf8jmKPGcFuGDHbLYQf6WgR1PizcCPeyY9P/GQdkDaAWyOoemdGoXaELbhI8kLC+wVayqPAhqZJveG/1TMY2mKAx0GFYNcT3AMgIvYZAV9bCdIgM/8bwSQ8c92WY2ZbCq3mdsGnw+r6Uj/mnCFGVOzqgrSpmf57+JJwwPV17C/afblbQLIzrpF6SbZUjfAYziaNizyj9IYBBVEt61hllZ1McM4IPKT2SgzPV9Svft3NSRnLZu2TxsRCEggVzx9ilWBasyte4dBr8pu9A5kSFq1xA+1V5Bc02Jda77qGKaU96LmLfiWDpNowYB3wyxLkkPZka+xxuEfJOoxvicF6mZt1wmQyAv3NIPh6w55EFtuKXdufzNyorSua6h//FoLkL3lQmvH0tY5Kgqwkc+snqwFYrvUp3HKIl77xdd3inuOp1IgZfSX2HPbnmJUZahDeeZvY/qrCD1/ijvW3CSrnki1yJnH6evIssrrlkDl4fT0r5PS9nZiRguuFAK3QjEdMn5Wf/zZ/dmidb3ReCKXr+ZIZbiCvcfhUtrVQkiEhpxgMgK9rHVDPNbpHQGiXK58EPgxpir5yb8zfA7VJW/a14xOD9s+NnP1q8BXR4PcjsB5O4ewO3a3gl+A8p4Xrcd1UcbTqPVjOEdkv6ZASIDmmKGf9/Vh+Xwjn44Pz6b/Okpj2sZ4Rg9GUSCQW5HSCB3y5CKwbo5bgGLMsOctPCxD3pkW8fyhp1xtItmNJSBpHGP5Jg9AFjOwp6gM150brN3+iKWhe4q+XORvSbzlKp+dcm5V0uh4eqkdAILWwh75FvsgPh/pM28irVMrjBWoPRnDPBTR2Zh1Q9smvaebRCvKQr5pw5hCXK/sTCGbBMfKjBqsLIXIpf0hKcyJnd/IHzg6DMO4nJ1X6PV+WvaPOm+NU7CkOdepGd1as8uGPV7+/V6g/r2XvFpTfkjA2ttzXtAWT0qPB8pzcAWeH8QBZtLFQ5UWCg16qNeNYiWeIuS/kv3pdkXfkhy3JapAAjg3xz0srivxHHY7PfJTrwm17StfgNgRgCQzHNpQ4f9VXCRWrsC1AR/4IBTdmnhKKbHaovzb+MZsiQfkBcr2OuVytfu9Ci3rTNKmIeY9pZh6KhdkddAsejh1IGmvph+jCJTEJXCsd1yR3rN7DdSkkfyxbvqUMzZDqUf59OSMze7ogcWmOSAc1xBKUSARSQbbSHR4PKwBREjQ9mjOjmpRrRxlE9cGSUkGHuKgnht3eIjAcJYCvzVGazSJpsBFiXR970sgeAr9g9tmmhfq67wUTlAeVXkbC3+Q6DP59Q8D0SQAJdV49qmnGsNAFb78KfpTys1HIaTO50GHQ9DgeC9HA3+LzpyhRh93Pan5OXX/DibCwvVZmotmC3lc2hl7qtoETiOju1NgVljEdo6zycA40WShxSdNppANTMgyzAtP59BTERCAImy/MJ6fIXLTfRVrM7mPPrmbgkWd1iXlYL6cH7mCE3xw+Z0ky0S/h1tXYpyfMRSf+JQLfTBOjSRoy1ZaN6GL9e88SS/+tei1yhN7fwGWn8jaN6eV8C7E8ogS9u7s/u/QupowUosRutC83To7Bf7HBz5JP6gYfx+N1O9lKL8Qp7b+gvuZm8Xn/jpXWjZxPai26vrt6Ntq5LyQnhDo4vJY6WticxxvKq3yn7MKW56WW1B/+1Q2UBDh0s1FIFpUSG4IlS+tl/PXJJ1m2Zww/KjYzaBPk3ACprZBiF/7BoJFdXPMCCA7S2cBX07+e3qyZo3JVm/YQ3GwO8tCduPHSfZ18MPPH73y2qTfu254YytrfnQkqtq/tTRm5eudk8lcwBgKOT26sUMx0FmxV2Qw1Cw1Sfg+J6ItLHP6NA972+OWhJM70Bslm9gD7A9jvS9ueLaCCdYXmujydwox8m+hJ+ggAMflULxHuKxvZmtB1H0zGOWsqR7U624/CEpmKVILURqgzLFBZNDrEO5eFQ9U+XkZCLH/DHuDjSt0g8LKsPXXQkQYhh3/XspQFTbES1SXogZR9gSwBKCzS4CXt9G7qKluZPWv9PuRTx/jGKB83hH7SKcY1yxCMnbkazuh6lCMaOafPMgU8QedI35Hf9QEUFy0SJAlDJIGhCSHyV4Jae9+ieYcDgMrx//JSf4ty2scDExOXv/Wb/NkNp/5O0haz2guyG6JHZt/Vrkllmc4zlhjAhYm/er68Rjb69kj8XAqH5bdBEiAexizDveG7p+ZMWPFpxRUTBF+QqpADXekdI7+t75fGPtbXrAkvWOBjCXBUGx20U/VFqT/a83K8x9dd4T51fy1aeIJ8shzbzGos8J7ybtpE0UNL9HlezTGiQPALnvPZNjazsINw74pbn0FuxYpQPo7bGRBtsCLakLZ44I8emPehQqKK2srU+Z9GWF75WkBNWohCYK1QpanP/mOD69v99Pqz+Q99T6BP/qo4Rhc673kqxmJO0NhQ4gxT5+v3VB7JQanYZGpbhu2Hk1eenaZZ0dZY/FTH3SaX18xst0TrFYqoZP9t17CsixV0ljQuLGkkbG3KBtuQo3pdXZ+D8ek/DGtdUyGVtIvLjK3B99PBxEB1AP2170rfxUafKYgqryT7GxIFRbeq/IsiiCNveKN/Uj3RNNdH2bY4oyC6zUaLLSZGnLvbHb0i5PhHM/BIZDAovyW9uXyefarTTfOg1FJcbkZZ2WBHBJJrPTw3vbO2i3XRMKttfPFLZROwwmV3s7UbapBiwZVL9jdF7bl/0ARUWnhdm1wS5j0ckSG5zD2Pie1WQMw+6Gop/ymB5a4ZyzwOwStMz9pMVQwX/UiAGffJA0Fasyz0mLLdFG87K8cxeH+0/snywI+SiV+rJRcTPOqLexws0NRTVbA/5QzXkk1n/CO654vx2BQmjWvjeEuLINwW2w2hH5vzQbWfpgiIMiyr6JV/N48I1zJQSbXd7DJf2J1Z5/vu+uBDOaiVmsSu1bxg1r+fBO1zpL00iLjgQPXJocSzxOrjsWdFKGxPCIfvrMexlmqHKjZ+T9KBsnB9482ABC2t7c1eVQZe9t+kmT9mbryix6juvTf7k3ROmQu/mBJMNd2TD3SRmXZdK34eoDJ/QBUHOsHH+H710nJ4OBHabyiCmG6fhUvBo9HSMKZMsIV3gmlgRrPHIkLzlzBmEf9wtYrwjC1gv+79sPdYgdYw5/WH9x/z4lb1sQOGHdZQOYsrjICPB9EHfJMrD/WzVnXbaUcl1n+kjD1NsrSwuwZYA9ltA+Vrz8qiFQzT1l3pCwyklozeoGyKiWJXSmB9mN+BqvL3+sqKv/7t0Ed/lFjRhlNsyplh1o5+6jTeizdi+RE1KgYPrgJn+VxIdpGHhClhDJgGmnYWt8SisdXNy8A7qDaF2OLZivbrO4vNT0KQhoXgZAMpQktkAET5BNXXVqU8B9YmKDxIvxoPzfnVJw0G9UdNT7SG7SWS6rjZfVfg8/iRou/BaR6Flb2WAyggKh2CKwpDy1+Vw4AFI3Kns4MRqhUwP3Q7RBoBJOy1c4b51H8i7sOqcO/RJQqXdqorRGpwiIUeEWbAK2vWmuf/kB5gKAIbX3t62tXOMz1ePNOw3JG2YxmZf9zHiR8+ztCuZxfD9oOw6naXhkaUDD5cDnMTzzQ5b5MIA1X0Zr8y+290chcknGAlrbQaXH/5TJ+Ch0J3+I2JNB1GvtY1W7Jxhpvjc7THJ9/XlIdtzIZ/I0jxNFMWCQoQYJspZd6+5qT/lB/dy1CHgghfzDl5GuRHyl1p3dSu6X42vGmGbZq21vJ2wrQu6Jd4YyuI8g7Tu2Hh89h48fUuUSUSRG8IhCxG00xjo3vdGcsOzr2hEXDNk2b8lm7LBNc9CbUFkjSH/ruFVPpu7mE6a3gX96Lh6KAo5dDYEUnQQxgNtqW29rr6KTk2n2860bJxBDkUN8YjdedY95fLXE/Rpw/j52gwyR7KXo51MzIzjKIC1VRtFs0IvX/u18RRsH/qWVsNtw10n/l+5OXJNlu5uiyAav7L9XQUlCvyd420aPSafCpDJPwtCUU4b/K0irKOqZD7wNwa8jufk0KKO8x8Rk4ZaOwY25T5U1FCrCziiqLl2XP09/U6faUEl5FbrvmRerMjB7rAC7Eivehkro5p/eDAqOQPT3zVE/QPzuzE1QADknpF3WsQMJIxDMSGj2jO8KWqlmZiK8WfUjv0MXOdmvRcfYAioWecmfgjTafbtGD0All2lkLHe7uPv+jXm6U7ybAOeOBvnjyyVaehqZ1Xyzq5d3BncmG9jL50dBh3ByE8U5O3yKTpNIAAc9h+YeWE2KZM+OETyu9v67hHvfxoVKlDal4R0MW/ESMoawhexVO7ysQZdpRcWnmnuTbYZmMYsvormm/2WUbsjMwzmYK/G2iedsoxkCMIdnN82Rk4is2n0M/NJj7M3TjI/TgbClk/M9w9fnVN3IKJC9WXl5+WJNnlpyahtym1FcIQ9/GiKluDvmP2KSEt+ZKY9NFTXirJFP+y39mwUuk8J6+5amHqPOlb2yNDqpwsOfGH1b6NnNSipKdN4kA4l8+NzSLr3a9sSpzN3DdtuNlneneRvK8IVrqZZT7SAM/RJXTZBO5OEPpeLiMH2ofcviwfyygmSW0PcpaFa1x/bCo0plCXEC+6V3USoecDwezkdTCyHMo3jij5vmnEGjdf0t7sOZ2bErpRD/p8uaamxusRiqRrXTj2V726pWHY4/WvH07n4abUy+LimiSIqo4EVLLxzC9WCngOQSUiBVd6mkyGl2mRvj5tf6BvC8ohwid+smMm1xIeUMyFjwt5/4AkJOLw4b1FinJfbGtr9EQ/pxyhN/hTrHK1n6NE69WVusL0DbV4AnI2yzeRWWYmRYSOid+rX09tR4yaAeS2CTnlpWLoFVUtEAtvLL3yITxm+fykyInTNH7ca8DK2ziDMMZ5xIO3nLmVYZMgVR3n9jG7xgV4K7JMzoAvWRLOPkiUrhdWek0JE1ayp0bvfBNfeJ0o2VUlkZhv/PyY4ST+rgcKmLHW+jTyfgiK5NWnSkPJ/usG03mrKciSHmFYDO8VaY6r5HuQukZpOlI1lvlg/jmRANnsTJWOdoLLuWNlEPM+1/BJH/4lsylIi0s7lDkfBUa39vx1CPWKXh1i7VaBTZxnJqPaOt50DJ2LGEnlvhhR4pTfDy6dV2vduC/vBDKxQYN7umGzIXAsgQQg1sxbHQ+NyiWCDWZof19vzzhw976scL1DALcW/ZIQUk+gSj1NGAW4EsazJUe3JCDRyDaERsuBCL6VTrn822QWREwlhoqhtulOpcNEOVByMXbh3A1XmVi+jxF7lQ0jX2oys4qItHxjSyRkZzJu4kV0DvNAxIDzBdlhtWIqDH0WOparUNGA9EE20vVua9RFmVpfVLLIIEc+GWgsbxdhl2L9Y7zv91VEXeNMXPwTQm9MuXE1oB8E/X6J+FMfkSpncPZcD9Ey0uziDBKl13lgqqJJdBHgsCkemS2s9AtiQbkXIE288D2PeZJtsmHYIZJxFV7XV1u5vIoDSIWssPCrxQUEUqxumuXSd2vJwJ0g/d00QKAt+sX1R1FYL6IEJB186JEW+g3WFvntUV/3WtFVOraLmMp71U3bBu2PuVdXb7LXJxNv+mRPK7jQ8irGl+QGC2LIzPm5zh+uG5V4gPNhQp5jkrZhcBDqKausCAtnPw75a7UZjDvwtVNhYhWvH9Z4A+LiQY3CvPrzBYhUawbL9hPVS1P3XPye7VA/oiPA0U0D/TqvAvbp4DCc7aaWR15Dck0k+RA/Q7+unmG+FNM1iwg1cNfNGPr59AeYwzI6eisLEzuejnFqgX0ZSFpZCxftiLeAgKImudy76/NtNt+D5lKyrSYwVa3UU/qELQiVyfWFZITXP+GF2JBv9nKVTfxS/+CE5cDgihusTvLX26IvFRc8XVfFzIeJrUUjz8xSP8iGlBBdKK90Le08eQ7MaB/MG29hdOstV+lqu/pW/z6F6lMuRygunvpAz7PF2TyDuNGsZUAjtHnabP2t5Mj9o0+qWFxCL1JPTdf0O/pUe21LRdJPAUgQRPAoIrvzhXUqv7FSZsTMiP9KQpNsuIF68R2sebMDIiRkjYX22i+PrGDbMMtKIAyYJySJ5ygTeh7O5j1+43J4jtIubLR2f10fmU9IQS0T3k+vaGZ586RcvWod2p1kKd1ULFsdicgSZfX3Z0FclFBcP/nPQ/MgcxMbOyHF+8f/xkob/1YKwQdhM7uKTW4eX776dZLbCB+e/2JW7t/U0INViRl3W9XrgwC/QPoIiNIeKKpKeR/ry2/aJTBCjFuapUnaEMStO6P+In4jdg23kklfgRhkPm6Y+cJb1ZiAHAWcDlNef4kp9xuSaQWV+a5U2Y5G2spdlX+obOupwRzqihEPd06mxyM871fC/FZvSudGuCpjGqbX6dAbayWvRA8sQpj7XtqVb4OGDLK8cncxvMizt+ub9N7AJdZzWDwVrVQW70w/OwOMaul+HZDkEZMXLsN9tPbJ/20Vxu7n2w7Zga19/RgPwYmOlh6vvPMDEv/Ijhi8gcjHpf6e3dVkLdfyOin89nR+f1VODWW+s4IvioTmDxnMT9T+bT+TcTvmnJuGFMzkt4/NYE29vqzWpYIanT28Ty6FL8sfNA1O5tz1G783K0qnLZmjb5vMca3cpiDuZMyEkXxoQlOp6nMzCROmOptv9bkQom9aj3CzLiyxSOeA60zZR2Bs08kDEs7SQ9ZYnn3PvbrjFX+A4cvV30ad1frN1jDZzCfpmBgS2b+N3lbHFGM0fx7V6fBQbVsblNg1tcGtS20Gd0rx390SLy/kv/TXGJJHTjKR/RZJ1ZSNjpUIRDUm0kB4ThBCJ5VDXN9tf/Hu2iKXsQh+6AUf8o/2lwmqQoz7hYnjXNxX4AQjGbcNWikt6w5Rc8r8x/94lVI6ERTgv8qGDBCGvfpDHw1IAiBlDKDjuW0G522lyms/DpUu6MTodHazWOyVkc2sBd/7d3FpBy5HZdHGl6T1I9gg9LDa5wDH1ezd7Qmt+Opd9UOYTIGLNia06Z5iiWEaFAFtiu4Fs9Pv6Y97lCeu4vju2Lxtj8bqbHx5hXEAeAz4rXF92fqpTdzvCA5+d9yy5IaPzjG94WW4Vw+oSnwctmrjqWu/EIfR+j7/Db/U6mc2mbL7BQNE46GMcd7wInoDrsYxv7k1UZ0ll+6kCsrfRUUAHCHCdblwLk8s0ircURNcLIN2cAVLmEHHi3quxCwfHbGEAJMqo7aR/PC8XWhWUZTjiW/drGzKXSvEtfFYXPS84KVi43ZB/RRj8FYFsGMxjHUITc1/KirAxWx/FMQee2By8Cn2N8uLIbGBO2zZ4JkM6m9JYin0T+++J+7/7VCCVRJIBNOm2NHecca4QHgDP4HG+41/NtXvBrOPFKb9+X7y836LgX21UYWe9sT4TMU2f8eyHquVQCvnSV0Hll6Xj+AIRRuEh6pnTAW7ZAcGlljRVe1EabObcvUPZXpeJQMDalm5LmRsWL8Kv/Os2JY1W45ZR84SlI+PDQIqW9kS8qWOINGL4oWGvwQ0+feswsaD/2nmsPEv71dbn165IGh+So4HTO+FACqTekZelpiwv5tOE+eExcRlZPeLXVb5Oh+7l0DZzLVn0aIcPUwLNU1yHJMbr6wGH3vE2F47mVa1NWr23QkD6oSqfOhrdXYUoR7/BMmyLcG0xEAX+ud8GE2FZB3D9Z8QLeslnJctMmeopcMFahvht9MMthrzL+DDxhsoyOlvDV/erRXysxpUvz+shWWfRrkhdksd0pWM9CzVrKJY19hK14L8EyTw6XYE9QEhJGFlTX7XqvJ5uNcqtU4T7CHI6jK6oPFKRROWA5oKmWw0G/DmLT9+VSWWjtmzOt9TIrcuDVNzjB2jrFhxkAUYaRz1AIOE7HhwhEp/YaezLq+2GUIz+ePR27b6HQi5i31dVlKjo3qL6VYoS8navAjVCPY3dYpz/txQ1wHx7Ore5NJQLCoIGHGkE3D7nHtAQEa/gqGvZjtRu78SVL3hZzVqZ1VTDAmWWrOOrLMjZonRq4surLa+nzc0/Q0aSIsgBkr14j/pW4nTZteV767NMKK3eyzPLOADR+TDpr6RwMBu7H/mw45W+cH+Hi0QnWNQql4zhmabdj3VWwuzAOSaHua4QwYaBIRQh1RHlwwq0VgYOuCXB8xyjv+NCe8/uUPMm2VO3TSv3t3753myG0ttuUQXE/TK5lYf8zw7MomoUXOI30H3ssas3QMQniO/PySwqYH5g5KGlSboaNzr+Vki3kIhbYqt46Cc4xzQ5bwW6JyB9tJ09Zf27BNCPkvUxw1jj+9gB/GXHRtmN2NPWV3L/Rm+D2w7uBbFjfYKYd72hSIE4Vulh5cPYt+gwqHWTbwL1DUaBdS9AcZHfMtiBGawsUkVfsCcjN3KfdjV5xsErl2zmcrdX6F4naCPnR0w+4ybRTJMsK7guDUQ6ViqmvqE07e9lvHk1XAhLoJr9ohfytO9E2SlNfswziCWZxqH2xBMPFbVmK1JodM1nERduymDo1gkTJfTGlcsa0dxasXmFHq5AkhwanmCwPdtIw2C5cGcvnMXRmdxpzLLNtczHQvV7xmllXtRu5CUtT9iA6McDB+sb4bCrGho2PLEfzPGFR6kr2NI+N9OuTJlTZN5aSSBz0Qi+5mrZannf5hx/yp05B4b+7cfgYRemotJVQmDrmbh+JzSk6bP5o/AR3FfYKJtdpEqTQ1t+gDZf61WcD/D4MCcwP8VZ2YGWAByIN882uC/yi+1u2dFQCunqKfOFAoWZc1rZoAjMAOZhYzMZkpdekm2c5N9ih+ZZ7YGhT9/u4oA1Ov3ISrpd7vYuAJPRjEYvFe34se5WKNkBhSIYG7H16mVp15UbQrw/ropWlXsIk6X0k6RG9LdLLVcZw7qzbkLYlVXz/4pBvARR3+yyxUIlP+qis6xr0F5r8wxNZdmmvfgXzI2uQ2yhi/TaDZ1ofZiSXAvBt141muPdnqFd563uqGRa5h29xWeh5fRRMByioPCe0+FvEimAQ+JHr20MEPQlJaITx4NdMdMfUfDtfxGg7uLNwi3xQKiRPZfQBMgVSx4yPrMCk3vjPXF28xtb1BihL364KdQ+MtbB/buX0vkDDrz893eQl46EpRd1PzNLqKi9pORzyv5D7bHb+jgIgaiRqfaunDbkHwP5egcPzbORNz2l+iwU96ZIHgAprFfrCicz2XKA3Dtq7/yTpk/i6IUcoxPVf3qHnN6eSKLqRjMIRVB3JArQ6+DoMEv+yk5zUUiiSZ1NEqdGbNgtpmNcDG0740UmlbUPR3qSday90MszXwzxeQj1qN86olcURaLgQFvqf2WlOrsg0M591iRKjbLrVzR2aVqWFl1hnMJV6bdpabc76FbKeA1Cr8SKG0Ixav1769hl5w88EfG1CaEbdzDDgDEywtgxQrhYO6H1Y843Q0ttAvzd+vdiekcou+je+CpdW9plaOVm9wk/HcjOeyIvM8NLzNTzZgii7jRO/GAfXqi5mR/7cNIMAx2FFjZBlTM6vdbuGAu9vvIaqZo5ZyHen4lGZDL4Rf8Zcof4Vx2dJAZm2NS7qtZHf3kAb6m+vX70VJ3UkIJtkQK6atSTb+d+5Feg5UcXKlJf55+PVeuiekrvzexVb7o/cMb5NcsGV6byO0n62G8sflvValkwOTGbvrHVr8zlYUU/KuLzAT1igLI5o48cv7wVkjcLFc/zQHB9sZUx1Hun12CjPG5R5Y+j/EKdNGbdzlTfyJEx49N8/4pMOPDt9OvokynfYrxaQdyeb6Fy0E6LSFsP5sxT5pA7H3ZMOD/1rVYg7/NYuh2KyOSHSVd4vc656D8aQ0nUq8GN9PHBxsg2CKeTfHk+wHIYcP7c4kkqs3OhrFfEaqlT1G/FdnWYsR8PHyh5ny1x/LjBVk/QI+2dUgToYtNoR1hgTTDsdydELb8QFrQsK5PkphojNB0GkEHoZJM8efwu+wsP5MMuzTVoabyJJqc7mhtbyLBw1z86sKynplahavAfOwk7TD3FEEK//1o3CCLNi7X2D03f21tSTWO2gRzJMjPU1pF0dxWkvFJyTpggEQaj+MeI+XVRQCobCs6AGWAafildSGemW0hNVn6K6AiFfsKbBMRDFH0dAzT7jXILzTxOn4FPS9L6vDeetmPdOv2arj2tZrcxRY1HhWt+uNtKD+h31wJyYxmh2C+QhBAvENfkLB+ek2m2F2/rWBqq2EionjW8X29fkoKlNBHMglrV1K9cmJ6CNHXs/Ll8rCFboJnVDtExUfih/z0v8vIcT+LDzP5YSeuMdfe+Joa7aLP5vcBaYb8nU6S16LAKSAJeKf4K1h5pjPHDM/kD7C3qRi96gv927wsbJKai39TaFFqjapcgVJkKiwacCoEIJhZRCcUZTJzJMqGPQyi4zkmtnWI6ev2y3eHnfU4bASSgbARFcaEUS08FmSW+sx1OeuKsIG7aAHn76kK+xOPBu13fUo7fbwJeDCMKtdDCyhL62WI032z8tHHVHT3KrqwNxYgBp4BscVZpTYuCoAFsrKo1XyK+rr7S9op7gTC7SChqwri50bavtopZzp2eEYvDajVbBkEB0ZUZRlShfARhdhpzxsKrIjxptleYxKr+UQEbl8NtSTz76O4QStEOK5HoX+GhXxnaT4nouwIhA2M+RSUsDHBrSJsl2dbE1YEYJtyMPxddMDU3f6J0PQL2oletXu5HZAuoZ+LmjEN2i6++Fa0lquofmMt0eesbeWzcJalZU6hrbUFMgTXG7CpKOGR5kYevKYBU8CpidbWC7V4PRHzBIzw6B++Vasksh82yFumLyAny/yaGz8etcQ+Y7L4UV+gkRMkBT/SLuAs43Pz37L1mlr+79/W7hhLJibrPUmMXk4+H5CRBJ41deI8q7+S8JtDSvq677p1nyh9YC5f2Av7UJ8k5U/ETAfrCrmK+PYvySvrn7/SWe5J/SwkXzqn2AeK3mj9a0uzhS5/Z69KGEaW+favQ+T9ZSEgR+CaM1+rD3gkqSOa4g1p9AfkP/6oscuyKutvBPcC+yqeFnjsvOcZtrqa3sHucCYC+zt7687O1wi6/VmkbFCufYGWbrcT0S8/rDnA6HvPPyjszpnpQclUZdpo8mnY+go0TP81zV0a+7pF/1fMh1hL6gkvg0f1OPCvuB7RoOVc/9e6QzAVFJBVheI6jNzSqihkpVUMZvxchufq3ZN/rq1C+6cxU1l5A80tlcfriAJGuAFIX3/l30FtWxL5rhGEz3E905Y3v7/D+SvLzLwy4dqNm+AIB1hZfkEHzeoZ+o9WQx5PjPOlW/QNbWdqDciSQ58lLFaWgWvXTx4EUvAfuEgWHbnqkOwgSe2p3Od7Utbl9/xJvdciV8cytOLzUU+4uDkh9dqsbj6D5bYgZiTuse+rb/5YjDQYsfZP886JdXnSL7VPc8EeNgHoOLsHGu/X1UXJv459Y4LfrC0YoHshpgSey39ENrN40WDDIZi0biIdQrlBBvUhj8z0HMBcTfpXaPT6B9iOkbRkGqVgL76gGgAnhqQuQrQh/ricXmSoatjs/L7t3n/UL/Lb+3xzkS/TMicvmV4JzEmFSpqJdScIi8KhEOuA0rsTFdXpTr97wyGWf8xs9mKNX4hoJMxia9h0X5MrDXc5DS7PAys1w/Gv7xzYBIdQ06iI2EbnUURR4fL4hjHElo9pZWxIVK2rPL5EbCYFQwoSS8zSfkw9JK4OF0Hj0Xj4ZiTDUNa5oWt+qj0rr97rdY9bJhLuJtVz+5tn7XmDP3TuDg/pLlzcAavWz/n5uq34CsjfCYN8T9L5czs/4V1tJKFPHStk5O8IviXpuWqBBE5aTbUtVQuD/VoPMVGo8eEcNMF/rdfsXVoSQwFJ9k+FSDkDs8HOrLNo2ZI9j+rrk5mk30j/qgyk35AZev/zt0fROL912k53HfzU+54vOgyRZRkx4z5vwzsvETbDtudP0fy7JIMq+vdVBvSFdXxO1Ol6dYkuYucezN6PAekqRu4YLRYZheUiTpTwj+Q5OEQ1Zdp58VNjYKMZKyEDOHpCG2nJchzuWUv0WFxYCDoVhpS6GGkPZM/H2BARQgleP1AJK0Y7RR1/NUiXteQFjW3BTkSDJYKONva6wUb/l4fs5mf08SRGUbSXb15k/7bMBr5nh1sM0zZIYcjMFDpc1ZZzrzwitDZhVAuN39lsHW9sp//GtW5CAi7bjotSbOg752MXOTQypWYTR251Ywx+7bn77CQ0Bi3gUXZgaZx4SqpytxsJhTJHr8DI60txOIBC1Chuq4Qhdc+HiYRroNeMG/MmcufMx0DJM1n7Z5Zq6xUz4gcZHojEe9skibri2wZk/UDGaz2RfCEtXtA9jAZfVL4Pl+JlRKj3nnfHFJTQilkM8rdMIXnBuj8hVYJmkrv8v5tn6yPKVmL4s4DqE0yB4i1ck6O8lXKGRLpuanzs+L7mb89P+KC9QJjRoJe9Mo3ozNf0VHZDKbbpCWKEoThlzlqnsn6QT/2qnwQjIthbxNLs0VpHfE6Kq89/QOiFaeSEvCT+NEtGjiZUi9dFB0bsQKPFV5C20oCLMG5SaLvPviNvaPibMrMGdWR6ppswZ56kRbT3LvbPr+6rMYfjgmQT365wvbB0s7gpPP5RXeIHvWdwDrgWejFl0QR3p93Pq+JZwXJQ+WX9l45KxTcrxyUcXAxwnEjA/xTlPQKO2u1f9of/UkfyGPnTbq0s3iv/gDdBd7eabYYTJj+j/RKvvExUorEBuKET8YzGs5RIPEjh0qfd2OrgfXRs0L+sNSIOUVN9eOoBoesZaZYm1M0PKlXDYE9/Fxo48fS4CvEGXu9Ku142w6CstAVxiMyf/qq9I/8xRumJCXenZFT1GbVGLHoIjOcS3nBONsjlkcngfuCPO0C83pN3qocK02/A+l0aajI/l7DLisMA5hcSXMog2BnXq0vOdXjK+pxMD46NNCNjK2kbfQiFMdOGV00lOSbUsvCvwKG3DoW78UDHM1f3hDqQg/n1L+krbwV+/mv6baqxjuh5U3V+7jjdd1Wf6fgI0+OjivIiVaSfGPfHVLHhWiQMhXtHeY3VGvebIRtnVkoLsw17DROkdGCeDfsOSlx7Qrd0TZgkOGKxlZJNKht5Kb5O/ORl8dR89kwWznzYv82mmr6DShwfmWZBOZJIhXrAM8wGwndxPngfE7M4gJ4AR4fl25PUOJlGlnfSgMMeAzIK58Ezxm6rls2cHPZD6Ivnbt9r5IBJbXxlhO5FC5TAKiQ11ELTXpNp6yZ6SIr3rA7ySCa+WHCSkbFZ5I6raX6jCAooRmsBRXGmSBpBMpnOoK3iJkDF+ybCdcv8wASVOMWoZysNQw1ZRLr9dItQW/D60Ex7RPHa3+VLfMptqd0TymMyGwN3GL0LWudgKWmJYT6mTfchY38oW3KgvwtQzlpWWR28gaB2c4fe9lLlAWtNhi3LrhestTbhfk8cLETl/kZSUV5y3nY9vDILtFTBL+9HTZ/FE0leDHfdPMLpiUi7f+WIjJo4DLEaZFBpXft6CsNLHPf0i3qNXqyfsBLwFES9NryrCBzTIuMgTnx5uPfGSxXct8J2/idREAshjwjXTRrmV12vjYhmJolgfrHr8hU/0vsdMze23R9bujGGCfQlmFbil8lm+Z4iVP+GcZBPPzvOJC3jriO+cfrIJ3CjB/fWmSHYDsM9KDM1tZHrId+pu+q9YTy/fIKNatwPoXSAFOloC43S1BIQlUrvkS0rzlPIqIMgpQKRAR5Gg61uLDWynp26t0Gi4dLOW/sR3yArgCYx9vWugRqj5Cy0jNMnUZc+8fx5+NqVdLggJOX2DXwEDMWwsdk0CHurfcbWMe3x8rMaP9vD6BXl3R+2NtSw/hua/ycBBClJWa+tXniSMKdpU98VyzgX/NAX+3lSQ0T82zP9hvFB/YxzGkPdE5N/mhGGxc0tAQuHP/V1mBE4wwbXfaPxWA5mIf13r9y6wLlQwLY0/vof/cYfOV1LRZCzOulU9sclR90e/rRUygTvcatn/cC6wuhETdnVRdXED3UTx7exrZET71MfH/AuaxhxzQE7/u6jz/4y4H7c2HG21LTkBkTvbiK1mcXYfWEz2q9QbuOcGwXGVT2lCpGALcSi3h148/Hyfo1IeU6agKTo5GrxmhCfcgR/8wul4NvXd+Zbh1UbmGlfZb+K+4kXneg/x/uKVwLl9p0QWP3XDAst4oM+vMo03b24d5cidNfn+BNgsEv0wyKmUQlAvEmqStDJBvvESWTqhy+7meRqNs8oAmkiBvMERnFD7C1Veq8/Cv+StiOgcS7Nfqpr2IhruTC4SEQKhNC36YdURuMHxWS3tRacfz5QlrHt9ncFOp9bbfQvFZKYXtiD50RF2wc1ZOGNG9CT5Q6ktv/j6TqWHEd24Ne8O7050hvRG9Hc6K3onfj1y1JPvIidjY3eHlGsQgGZCRQQNeoNLc8J3kH1GgrDnscN5Fp0eLbwOiJtbWWdXDLhyX3+mnl0xWfsGFfpxpGUh2N++D+h3MCNwwrbi/GS7l2hBKnoaKM2Nl5L6NiLWUN1BkmwWTvhbE+B/3AXFOUF2P70uEbexnBHXIAGk5S8MckKKEmTP2yLvPLDUIiVzptTNiikDk1c48tvhp+OFeqrUcABhJ7sjRCauqxH/rviLvVHm40MkZXgtWXX/4pDBliVZ0Zym8pV6yZtImRLtD2+Tl979yu6ljMagtBRBVD3AMyiwaVBUJ7BOpsC967yivvrcwfjdBwoPJUmDDTCMGId47X14vqbzhn7ljQc4XU1WCLmr3aB3oKKbrzjYZeCLAQH6v1Z9woc3Z+M1VHbSIJjuXo9jCnJtG6B9Y5lgmRcKkWOSkQ/kluI19f+GSQHDlwHNyL2hfEDxjRSlppjLfp5Ww2KwdAMShJ7Q2VmR5FUMr6ursQUKL5pMUjV56htEy7qLGMbtzqb7KclNa5VVeIUHAWPNd73ouI76n0NPJdos73wm/x04SfJXJvndfQDB0+kDK2lCqdI82PCs+NI3K+zC3wBOvQzeDPcDFYMDfYqXRt3SKuOUSxLOmdIYDiHZYyDMDLgVdtD+3NHDP4grSoL0tGZtvd6qVLe506UL++5gIC3PLDW0EomGlDRGd5j7NOmxdY18xVw68zELQ6cLloDf+vuT7PDseEdzNA/5KaeX9lkr1+j9WVR6PSzoBDJMsYWCNO6YGcM3k/51IAdF4Pt4YbP/kd3BHebQmeHdptv57iU7abVdAkhklmsKCO3KRxIdBKmKegtr5S9hoswbMzGF9U0aDUVQ5lg8Kc00OnYeALxQf4zqe5gbr4usC42lmnpPmSGsDyz+fV4j/uUp1M2bc0HNFGV2AHoyUTTZO28SBnwUcsCTGMLxy4VzDAMdi9j3z5xj45nG6YRkNbqaZolAfQ3xgYxhIrAgQiOHPHshtL3eM288c0ZSFmTeNP6KA28NVjf6bY4LHY3ED2eNnJ9Nd56z5fvpF9UFjqYLT4q2rlRxX4Mr3txc04rPk4EHubdbzRCnTntPnqln5FOFN+VGJU+C+OGXkJk6jCUxi1v/Yz9sjULxNqOCOPyCFNqcdCAZ5dYYIMQDqrnrxcm2M4av9OXlupwPOkUI1aWHYA8j6PONH2Xm9413T39ruQjaaq43eYcdfIWNg8aGCA/mPm+u1K7lEoUyknVCu61VZ+pO7suk+ztKn4Ry0BHC7Wdug8fcq524VZFTPdyVkrdSRbiPKJFT8oqHOBAr0PJB7fJFFQp5xvhn12LymfFjDkl9hmi6AtWgv0JYtHLN6oN/zjD59VPDRTAITeH5kXxXo4TrkoOs+EY1wNe1i8mrF/phYEg9uIz/g8LTS9RfyWTaQBRl//GUYAucib1Sy318LpYoMoyXgbW8jWcrVBh5EPqG/PYZUTg+mheGt0CgUhJlraU7UYjfryPZbb7vmHR1yl/TC8Dd8ZFWsnAQz4QxNQlkDJ9NqkUmGNe4HemfovCZFibKmW9fjI0ZWjy18vT1hM0FEJ/KGeQ7X5rtybzW8eMyRnIQjkzD59hg77JE8PpJMA3SbC5pnseyuzfl6Z+Cwl7kUkbD6slttTR+6YkvTVkHyR3L9nwjaRSxZ5VI/DUHlG9qlSK7t0zf928EfwpmEyK+UzTgOK5/TVh5L3vz+FF+ux4uJpTy1JimQfJMhgpxd20jZPno/n2/rX7z89fTLy/1ZyCCSqs3mhDbaA3inBjLDg/WLaMDqHrbid3KqdSkk7uHCZcvzoSOMQjSKgVUZXfOxzoUFiAalaiwg/y2lKBaU704R/oSx8+n6GZUmVU8VNsUEcHqtckZY4JdTa7EY3DfcVfFcvuKMu7G35lmPfZgt77LOOMNVk36ZjAWN8k43w30iYwqWnI0tg/wSM2jy2XP63aYdGuX6pHyPdPN/5a5ChdqM9i8jczHcoWip+3LQlcxH71mCklK3dBFXrQlReYfPnJFIsJuq8sc8G1P97FxFc4mI2E+NPATBKBeZQAwi2DDr3TFyOsDYuswlwHelX9aj+4x4uDugvSkolkKbBkePF891FuH5oRhdvf0kT45oYGbKXrL9k3W1EKLXQt1vTN3dmDJHpZ6DmmJbzKNe1PMdELTcXxzoONthxVQnSlNIY20LBmf60rmPnCaslsD0qVpkhLsGfGk+Z1OX3Z2yi4p1D+BksQa3ty8YN2pFg95+WjSQ8ulkIe2Y/fWvt27InG+zUGZys5H1JYnG1WQr1TMmVRkDOT7+RV7bYIflk8hH6Rorh9oSkk1yAZAL4HcwffJpzMgPxI76Q0XlozN0XMD7Nq15rgUQrUKZNZVBTNPijDubw8ZkV+0PZFSgHyyjb7xGO0PftK5VlOM0h+2P7s/Q13ckl5d6nJzZGs+CtyZsEA+mA8gjCh7/SIupHnP3R8+7ydfFBc7e3c7zSxp8SWHBkpfArJ3yahG/GVsHp4pV38gOjU3oN3rP86jqqsi++XZfxOcVcLyMmJMDdz9TgyTC8K9jhhbfz+6syQvkVD9+1fY3nJmEjn+2lgfM0i60Ds46F9GbITzeuFKhgYU3IGGos8Hm+MsuKDT2yjf2VjmTzVr+FAmci6cPHX7bH466eTX8Tuv1/pYFk23VC9IoDEVV/nw0t6jzzWcjMMUb+Zj+ByH30A/ViMd7Ku3frFgQWbG6B38ZitVb86zc7tmgmeQNHx2RYmwOIssSM2b0eVJH49WB11M7Bu5mQddw0JYigwMOT2Q31HU6ZcA3gOWjxrFpiTQpSUpre4ZvLYW6iLPBN1KyG+XsNUBHVyg9Ttxoj3QXvZ3yXJfI7pQ6N1hgndRZVmLlPYt+Avv9W+vxdM32s7YjktfAbumyX74x9eZVrpRN7rznYWQtZTi+H1xHspcp17pWFQEgncUr8eNOp8dm1f11MuEkvQ767ndl0tBZPhxStOO6jvxLZjAnNZWER1tPx88X++LMacsRk7pR6NfUXSFS+F67wQoPGMoP6keC/b0kdfDumm9zRiX32DFM56e0O4LfGYQzg+4BSDVQliT8JZkUm1qlYsxZ9NiaSven9jNxdGbzU04hO0+9CZ/tjTzX62wOTrSi/FU3qBkVisvhzTKm4j6T74LI8o0u4EuLhLZmcdYZctzU8H7L1VAbdz5pGuqiyD/TJ1sbnOCNluS7DNgrfBDXLsjWR/R6dtgNT2TV0Ms2OEYR6U/fxRXlegE/PC9WIYYFcjzp8BeVzYeIUfXBx8ICRwyZs8ZFx/YnVTogxITtlGK7FIXuWZ5wvxt31YWxDLrRZfSCRX9v7vVrtdFVeUBz5egxaiWy/3HF3rwhkmFq+QlFqSD4gF9Brv69/FCQU41vTFfHy6Z6WsUCKySMm2dLdCasMvYdqrt5QQ9GuKGlJ2LPtImIaibjTvaWHFyqw4prGF2XeMPyCxTLn6BlE9ZHWrt1M9rnN5QuwQY63nDLqufAu+D7NH9Zgc8WL6naihPpo5+6iu6gXB5sp1AaJJiOpX0uB1FVUQxHttjpiwieSUGMS2DdFFZSryYyZw08rwqh9b0iu5pxi9hb3Ye+DhOqgZy2jKqadvfHlFiqSXHwBLn3CyrHh9ZXIWQx5dHvyJlXT6Xt+M5KwETG9yETI1k20YvmkkSkJgNjdrKglTcTySPltz8h3iRTujPqGuP8f6NyaEJPJKLm5/KD6Wi86KcACuqSawH6iNKhFkWaUI+eofXFsVGuprsIp+2ESw73a6vsjeHsybYkZEvyW2SRxhSy0YwyCMcLDg09sxw1SuuFr8O4Yh5qz1FGHe9yy/0hiNL529fcf83J+Vt7JGiG0XCPov0RCg/AR3PnFfoa80hLQ5Z4wWQmxlSxXFeyPjx0k3421t4mecG8S/kHNyLFQXYqg20lvRJ66+o1YVMjFQ3rhfwO0Cv7D5m0T3Cm4nXywWmpCmaJfRzqxirAsbfiNCHbu7wlCtlPTu21/vrvskkvEBdGHOi8g/HzoI+ewQIiTs2AHuSKHOTWrnkZYf8q5+XyrTnsUAjlVmvVrICMWapfDV+zP28axvbKwMuFfeKq1+45PGvTekAig+TUXNXjupmtMkPjEZVMR5nP69WujEgTOeh9it+nkjnFc8dQHPwyQ+SSRyFA0hJ7LHbez+hy7s7rEP0i0q8/UKGT9Eqghz3TtpuAfmVX5UvkGbQbGY5Ej95tXnhEDPu4+hh/ZXnZNxA0aAbUj0962L5OMNcczzPey4n2SFjX7xv9MZXDh3aJLxzeooHzJjTE2SbWJz+6j1eX6Ui7CX5CWb5Leurc8GP/95rvNixB67fqRNdT/u9bISfaZJ2do3/sNHqbAKwL/ZPU7mxYuznP2GfM5xEUKqoHHip03EHjRiILnFRjt1Eeln5rDMX/xxY4cD2dSs2RnfTwYAP4pbfxiTKACUnx3+FWsfwvH6v1VCZVaL6yF06jQuqVVbadYGDZDZFE6/03VTsC3bMB8r1HK3CNB4votyPfjotF3cNJRle3CEzKdZMb/MezdADCBZpLCHo3zJqFFJObKPZlYO88srRKRj3838a3hFEJw7nFGEpGQewDkXspSGOtpBU2B6teg8LhuwvVZ/wSLq0V/PPYN6+NWSkc1C1xRqnAy4qiZ935rpVDozfjgGAqeHzX9h/qXcJpdVUpnyEv+yQgwr83SVNbIlhAAhsyZ6NRq3vBpihIhxEp9VLBlG4Fw/PtCrAtB4w9pEQreRVjib8rP760PVftmrXysubaoEXlG188kjrhZq8iIzTqGaJWpOgWpNV0yCSND98K3yXXB8Tl+hzDFCqcC0vddzzKwAf+W2T5qKBngKe7wMOFiRyInSFURbEUKeEK0zBv6eDIeIAlXhyi8Lad1baZWXr1pDv3a6yHY18l5n/DuUPuZyOjxcdHtHwfGG202KOjr5tGjSf5q5RBU/UWr9tAtO2XEainGxsqHtczKAVlxJOgby8hcQGtV94sb4G3tD71TQ4TJRXlrMcuTofJlbpug/E1KpijvjM0G/kvmEDb/loamQaB6IZwXK8FVorrqKbZP0HUKZd+OOD+0GBuMMRWrmQNLroXWuvsFRhB4mFqDycmt1bTxIo4bUh2L5i0x5576VaQsz21aYc8zhTUrBaqAYMgsp95qn+zwL33Po0mbT7FY/Ix6cXJ2zqt2kidNWJN5SvVCsWgS+bFswBpWXfr2577Y7vNaBu+95tqDcYIb+Mvngn151LdwF21OaF43/+kIhleGGE15l38eJ0XYQN0Y3qvpkjI9bcPs0EPCshzll+bKKFOdhW+fW80cexnK5aSZu0PXNJphkNtdHeDYKzvWIfKdF7Aa5VfGgGI40q3L6GwEbwBzcrp4x7ion718INqq3TO+5NAbgEB/y2XAME7Ehmi7oZ6KmCZr+tdATo/UieHfV6IkHOWVLvTBMX5Xgc1cwJJzfadA5YQAy/YXmIXhr57YxJlTPFWkzjuEYIsQeRPmnvq1TWdbm9+tPo57hTOOo4CEJjdM2jJEphHaKbszJ8KHDl1JEOgMTpwwi7Kf8jTYeMaSss3Kr1ycYMJWIuFoeig9fWD+8ARFikgGTimj2ktRPNuEgd/cbdlANK5eSNoVksJp+oLTp0eCByezJ3WqN8uPdltbKGJF0jaC6JKmmVM43PfCpvtW96fXta+H4UU3HLtQ3YtSCHRd01k+0GiSdD6Y0fF5337S3TQKhKA+glaJKRZpitj6HuL9rqv9140go+FO/u4kWb1eUYEVLdICasrdgtIUFTYI3LSOoyPNaqHEopCxe2faapq++aG6YdZQetPTnIVdo/qCFnIvoZm1h9IYQUmxOVuMTHrc0jLIe+lH5YB/nF9cojHXsFydbxwh/mbgtDlCF7q7MqTi2bjU59Ow+SR8+w4EuCKlmlix0L7LN2itOja/DotWL6uWB+rEbA9wUgVm3G3XrKKn3/Bpnh6vsYOHi9vbecCxEQewEvShqv8GOhlP9LldenGWhUHGgNyaVLc24o0VI4MjYjSraSo6+FRve4fIOp/elKl32lp848Hpjm13vcwEgLMGA3zfzq75FFEzEYTvWttIXX4Q4LH5hwY9eZhx8aCwIwlr1dE+VF1Lp5O0q2uO7whY3oqTapbA9oPcKNdGe26SL8sw+a1dkr+930kyAu4TSKtl9vwzNUqIdo9gPQiPYB4sqtzZfrKDmFc9VjdMErsKkqBHWqZ0zPFHPpY3wIlP5/Gp+huorjQjU1UPgC4EXWYo/2LinHD9I6q6mvURw7HVd5UFbjHn5RK8kju/kJm8ZtpBFQmnkPuemNRQepCPPl3pipvWSfEhhR1BNKW3/IEuBpOZpnFM3NKK53ymIw+yLac677S9MBY6Spw6M4Zz7EO07o6+9v/Kj3K2tMEKtxRJykgOj4W4+5aMaAxWyTSjpg9amcCQtSW1bZDLwi+ftx7M0peZwEVDLw0r6c+ZsiJ8Eas/yiUeV7E6OCb4agb7fr5ePg4gYbm2XJdkF5WuGl+RFNAHgLKz2OxxxZ2OSfVaSvTlklCZRp8ZzTYuPexTVAIzvrDi1TBv4M9QoGUQ8b2cIyJ+rVUt1o8Ay35bc8KDe21rOY3LJiImDoBD70oPWW6/BkKjTUWIqhWF1t8QjyHEtd1tegH7dKAJrmacCh6ImbsYmmuPa/borXCSCDn3b+MF3A4cXhHMzDzf5nu3X7pqFSzzX8zxHt5VQ8syQtKV4T/LCWJMv/cGX6yIgFzjAr+R8jgZ5O3Zl3Zuh5rfLYtMqw+guZ6F4r/Ip82Qak1k+d3nS7EgC4z7b0WZYDOGbMS1kDFdphcr9UsC6ly0lBHaC3ns7monuPKYFwkvRgJVu8yOpWLt4xWpYv61sCR50DxTY+9MB5Jpp5Kl3WSwYVK6fWQm9RaslkS4vqpFnlij3LGtO2UThUzwzDvn+sH9mVqbR5dAf6n4tIL8Gwey/2ZbJcZwrucXB7sZz+ZFfOgtOelLesobTHGjfp9it8g2YahQXZ+kJoCUhglwMS1Cde8h+29wjrhLUHpw73GiXzbGKrewbEGTe8vaQLfs7vNzh5Xj1qbEN5NdaJMdlRJ4OL12f1Ibu/dXumJPLvjUS+dAasaxbnK8EUhV5ElOxg/JWsvfb93Gca6B4nWD/JJsimojhvnOpCYsMqVMzWm4d6WfnjSHKK4caAGRWiXN604PkGQq8OYyW2aU/c3gfIAIE1Gat2RDfIkcgNGV9eM7vW0PlmjjzkYc6MGTUMy/B7WxjiX4InUhiilQpkyXjQIyZHYIySWSYdwpT7RjiMmQUpnmeqBaBcxRZ3v0ldX8xAe9njSicTIAnZIGgCQm72VMdQYsnzucY9oUNf5fgBtTDq4jBv6RKoD5nm4r40Z7FtsrfwJ3K1Gey0KBPNJRhlSd/+qsY0NT5Rezqo3EGG93KpHQYcLzWkElWeSjR5Ijtv6mCEZBvBl8F4woxClqkXog8N6U0o7QUSf4NANX93tj2ETdnLCuQt3SRInpINY37VGbP2Rrn2qwJUEkIFUgmC8J7hN+BuFtBgOYNuV8vsY5Gpft+X1hG1o5WbAeQrw39kKEfjAAzH9rnvcwoBii/uAbLyim9pYowe51fuBdHo/j+hMfb74uotvIHxSsw9D7Nk13zj8u6qn8QrqOi9A+mzeP4ACl/DKVRsJlr0MfoMm4XfLwoTP6d42gB+3Wb8NI0PeHiPG/ceLgY9gqGSRIOu7chul6XX/1sFPPbLsWx7N99i0KcC7gV9DlfiZqG8NrSdY8+fHhpwRF2EspDkGrdjrfzgtQJYiDJqdU5ktM+wHRvX9G8Ez7ZTrtgU36Xix+7zy7qtaCYruYLPMF1SvXMyYH/WZQ8XwSpzHBZigv18pv498nuCxN0VraQoaV06gHMIOz5ey28IhwUcP6GSoDFrady04TSFoV29Crqw8A/Bc2yGhBUKPn8sq7BClFi/1p+uK/n/UF+KKsAaKKeeHZm1JWEzc2Vpav+Zt2bxw1xDR2sv04ekSiy6acrOTM73g/rMdf1WKZ7KfrcJcYd61+6dnBfIIoprImiqZLCF73ZoujwNUrgzozC+ugMj73V9weaaoyiVxsfeDS0fUfrRQlTOtY5BfYKq3RCa8uqutnZt+cwXQMhfklY2Pw9oI7X+LnX2YVjY415setku3klJlZMKWVUDVARty8yf+oN4ptvqnHtK+zfeWXGpDfohUUhj/Vo8O6KzCsC7S41K7NoPgkBGtUSp2B0aOWPPldNYtysHk5CLf7s0ExOERPkmiTn/fYxKKT7Nfx2xrqnIjRbdxb5bAX4UUoAvcvDr9nL702DjspKrbE0dD75S2+z+dSGa2mFjMKueklTs5zcDMbdd6hh7AtKcYpmsbbKyqLSXPNDSUK5AB3clhXWfvydXhY3iuDTg4Pma2GM8Jr/MQKi+ZX2/8K6wumtGs0FYBFwcf4Gu4dhlp8aispcOIj2KMAU0urqbDwseDqJ8u9D5rLWu3C93jyVRuUs8g9gS9paT3JomWEL8uCINeFFYbnmWgP+N4L7pJbV8rGljB4+Ala16cfFevArx2CZACCEqdUA7MvrCkHnB0P9N8r1Lra6BLV8fdFMnK/hzlRwnyoOelmJmfJpR7lJf9e8mKsOIX1qymz9jdtRk+eBMo6Ci0wpcIq6ItlhnahlQRDUcM1ybflIqcH4l3uvK2/vj2tEjuO9jj/ad/dkv60z17VwxpkMY6V6GQlVH4QQdCDuFwQMq5Jx3vHwXp+Kr1OPx/nnT8sdAnffoDeRVEmY8tvbm4s8OblYLe4jZznMNWzj7qpGsT0vp+9Of0BzlmE/gwkyhjXJ4aRXtDj7Si4jhlXOj2w1FvnlSeTh28HbBHBrhEUCD2C1bCGVJhiDNeXxBZ6MQokn2CYP6uLbFwjyVFI9bmc1utV/yxWqnrLzeHrsBR8HC4uq19kMJcKBR3f3/CWL0y4Lky8xmY3yYEfERXfyhyGWmy951+GY2xHUP6XHfcPsx6jH1/UwjSZBUe2lOQ2iuahes76pCXUeP6d8uh+2KeVyrEctMCJg5wOVX7cX244u7rofsdbzaib04g3gu/PPwrXwoqdeuRC0fEGrsue4vpIhmUxC2cdJb/6aG3iYvLWHNPUIBvNvoe4/xwlOKnkzN/ZWy67mMhM+9h4u27SASgdpV3fobueLXW9Rirw7tR+M5xfWeaRxjdIR4eE3ed2pBvTGlzJ3O3gQuoz2RyOwz5cq/5LKYqfJsU0/u1ySqZl07b0u3IeYhvdQEPURK+NY1abByEbTcAnmDejXzq4eXxX7+TV90a+AeA6sRVMuzoXqKmKMrbOds4E5huJ4g0IatDq9NQmd4iuy7wEj6rKct1iPc3g8A+aLASdvHR0/Zaiht3HFctqRV3+Dvtjb3c8HiMDXi1W8z0qH5ylsmocAvvprItuIrv+5LhaD4sNE3rVj15rxYCtzvP+J0N1Ihx8Q6MCeZJqvOgSNrRb28Q8zchtsv+AHZqghJFdu3E642f1mGgwXOOLtLo2pkc7y19aVOCMDoYuPSZGsZRU/geVowc4aZmieDajsFJFh78wX9nClRt3oYMOtL8zFv3TpXCb4YekZxUrhuS0ydD5m9ZaQsa6V11TeEd7XlGCZYvDQH4h7rCiOEKR70duQhe0Dv83L3KUyXilS817hsNMpCshI4X98sIQIraTTuptf6oxYY9ZgSCMuFdf+Ki5jp4C6UXF1tQYHnJCJhMKbrYzGBYfGyGQ5O6o+OoRTz7af5qbUixqcN+xzlciCTLQ5jMcHPpDiZE5907qPTw8sqwoqUEv1keFDci5z62ytGPkA1Yrc/qlPbSKSFuYx7Re7fYMkafO+b48z3KhRPzh2agZNYGCFwH7RyC9GOUtmvwzeec6sdz/YS6ZxSqOh6bSimBzuL59GSxH1D0VDOhWUNkAQCV+YDx7M1Yv+UMMDOExK/pVlXeFbuVHWBn5lIlhQ4PYqZsuljeFtRdrgLV5F/ySQ6UVt4mBxLzvaGNQ5i/vzyp7FnxKaw5DlkN9RsmNK7SSfY2m+VjzTV1uCmp7tE99pTZvGCoEZtyKU7P7V6HfQ76WLl4pQoQbG+GkM6lB88TXYpU/nLiYxJ+U9ADzlPSbufXDh8214GuPamhS3HzZvtCxaNj91flEdZIMq9U7FVIiluYJQkqBz8WtWQZW29gCSwn0+NTA0uerNzg9LUWlzS66WVnupdBtw5wyyf43FUmK7kHVCUm/+do8zOnMcXsgmzWo/omzAaHl2wCUeuv7OojB8TcPim6tevLfSMPZGlWhabmi8JgzrZDvvaU2qtguQjuSFfdWo1fY8pUP6Jb+p7M8cfnM2MSp/vLZD41xRn59ESrJ0uX4dqIGWECXyRFRkaRCnezugh+0sWKiykul2aOIQ8/7YJa2dUuf9982Y6UGhWfje467WVtS6FElaO5/UPnHltSXqmy7mm3bJMZ7ahGgt04SwL6Pb6LJd1x/UX54Q3l47cOkkmVPlRdAU9Rv+4zfOKMXhG+qEUL2Zumy+Spvh/MPWlxzZYaeWg8imJ5D9yfgHHOWfB/i2PmZHcHtWWmeW1sCJUly3DCkTWSxJZAGtLTDFc0XVYHooV6rcL29Kv78By4SW5Yzmd/NDvpPF+bUk0osHZXwElFA4Po8S47ORD7/Kx+7aWkhWIbT8Mv6DEeA1JbNbdGbm41fu9TqOVnqTthnCGPVYeTEj1TGbNl4Ej5tz+hg+RFmN6k7/nVOeJpdq3ggR+iNguCr5TCENbebZJ8NxL2zJJbp1qEjU+eqlaGlsfiV9ypLzcfvq5/oNTj2AuyhjFDK/jlBg3WU8S4AsJKUhaRAQfEjptY2B4gJWAERDrNg05yMwXEc04jUK/6yANF4cqjk5U15CQjevrbpGlLSOxj43v/gd9BgDTPOCCuT8vOgMCp5dJ/q7EZLVY6J6zIzzDvsTDcOUskj0UK6Xeu3vTPDr4H49lF5/WOQ5tjV8TANUq2p1xWeEa7f+rtE88DmXgqVG1SJUkyjWm+BNTaaAfvdEUOEj3EOP23/BZUZnJ1yOnusix0viQ5Fd3k/QpxbRbTgpNm9J+S7cEjIt0JE1dqTLpA+TRNVY5gV5OjMP8ABizl/PniEAFs86avzV8XgMNyofY3BX05KJuU8VisImSqlFTXPfTMP4IE80s6/3Ht4q7d3ifZkeS+AckCsosL5HoGayNf56Cn0cG0ACpO9yGpWNh0iP2vPZLLhsKmphx9SM+JuVTAQk/3jah5u5EEoAhpU93q34O8NjSuUg1Vk6Oxcu9I7zFGF0htn5b6RzDs8oShFzLbgAEybZgujNLZawE/5AGq4KWWnTKwW0/NUtTSz5yess+QDgLRNn5gnNK/Gtc6AOmw+dJRuqMGcRHu0OEJ+LtgKACdHD3HdURtEos6kqQ2ub0+CWj9A2YUiU/BEBB+GzwqhBWGA95HEy9JXKRUKH32WE2OUWXyDmWKZBuwcM11xNedGaQfA9gazFFsS4uIGOLHlvB37bz1sSvRBejx9XL65ZbKIBDuVHfOuwcf7CavJOu0ErxnKYbnFO0p3STLW1lvjh+QH+Bj613RNMeMsFJ6yC9pWWcyN4K7KcJv8lBkk59Pz96nbwBnB4IVEF0lfgOPIQEj68y1o821ME+LyldOnCGCZRra75M3yrlKb7DMu4E/C1BjmTZI0Gzvf9nUQIp9ET/JgT+8S/Q6bT+fYrgp/wRr82xEKuSHMf/AmKx8SDLVC/zdF0vL75MPtmQUjOG6coU/zpdyk63HLYWiaMc1g2SVFVt+x+Dwc6Jsi7M6dXGI/4Keln3KteD2Lr8tEoa+P9KDaFB6mOxDgWkBRYUrIO9/Mgz2WI8soA43JN/1+RNPuReTRZX9KtDrMzX7ZL5ENRlACbTa9iwHR3evfB1EVbyiLpaVal5EYqOpfQ2345HXEomGj/Whh2yZClCbBkfM4tVGhmawzPiFNska1T+5TsxZY3kHpYCYS1J5VqWYKbbD1hZB/VYzrdpqCU7kE1nXV6XQ3tFPRUb18GVfA3RJBvsqkAbwO9eWlv9PHEPV665Nhrtvi0VyI8znTf5kFi+2+QGDhAPha8zuk+ceZ7fJuAPEJs/mKwbiodZz6uxF3S/dt3dJU1ZDFGnxPubaXW3YJd8B2f+sikSLMNtIkioDk8RfMDAMLYVU/ARI4CABHzKFVDZ0oK4xNsKDjd6Xgg3O4ecS9FEQIlCV0RBbemYbDpM4mHgFdrE/nbAfCrUckRijjmUBR7vcVfLfmVDF1ulbsTe488m5AOFQLGgZhHX3/WyqIDOvJ8TcweVNmbQbSwYTzsFCCPFr8F/u/jN5vkZsu8E/w241Zu8wQhuQ2gRMLtgpeePeZ5YrRxG2LyAXqPC8HANmSS/t1YWOAC2OOyuAevHii0ydwQeEVIjYRK+rk0Auey9iwp3s4Nas/ovCshzOEXlvFIFz5hDkwLFQtsxvqtbHABx/svrzjyz+cZI+FMRXfoAyRYv/ulX/XXuLbnG4U0T6i/vEH5QjDvjBGPvD+T+5JULjGSqFcZ5tjy5c4ZaAtN0CqLLYo5Avd4SrjlPH2QaqAlwXwgnKk4ycZUlNu7TYUth4gTeujMOrRHaQdZTx1YNJtQJL3KCdDmF0eyocqyrb+XbSuS+wjXdDxUPXAxSYCD3sOuwNSqstT4s98725+fibxr7oNKoOm4iPCfP/0+gD65xop3FDzYwcCWYNglbAIZdJLfAlki3CdyJbEHqrJdf85kk9Nr68hd6XGNrQTWF5rVcXnBd0CitLbpRmJQcIO/Iva+CwjAc3Ze7cSUbuVadIVLAZvvP6fEmc1BDnELY5hgHuf6oNnEVlQjGnXiQKh/1WySBf/AuSSZcsqgElYeB4+m1GsfYl95cMuHupWE9W4858y/AHW8f9XdISrrYXmAHxYvjuf9wJ6x5ZcZ56OHMNf+0vo8+bvyDN+WPS764JHlZBzQ9RuKctrsHxZiZzGcg+sMIGPvnd4ClUEs82t/cS+0HCGsL86z8lUgyRmErYj3eZ4ePPYb+PaR1Jmsa3Om976X+9YnD4sgwst/X0E9NorVFyH+4hZ9BEFxSsL4JLXA1xZq7F49KuPXLGZvmwN7i1HngX2B2y5HlYI/GKTqEPGKFFj4DZ2F47Kz1VAWOwzVEa/Pw/aipwbaBYJhayfLZ9Hw+bi84sWVLj3VGFQAoXy9nQmuNlaJvv4H7T68r8acuiY7NZzPVyN/bToNbiLwJybux26SYLTcfoZSlhQfA/Q2lfjmu+IapsWxINdu7Rmg9EmfIRJfc1UZWtRQUlAQ89mywyxhTK7rB5wRYBUrXWMjgRll8jp+o9Io4nNXHAN2MQ2IzhR5IQLcyHxbmO7bqmlQlrf6wiIu+fe97x9RHixzqBHBp0wwl1m0xTx0Lf+rBMoGDG74ln2Yiu2woMBFcXaJwDALar1/I+1NJQhRFAHdMRgKqqFLghENOKsBfRdmdIiETiDDx7yoVlgkwv+u94sooqJ8L3XEtDDthhii5KUYYdH2YL+WN8j6X6uyG/yrjUma4v61I1mQHHbjLNIB+GHtKi/yF1eMa/PWmvWKb36hazGnDHe19/nuLCMcZEIZj8f++G3fwy4k3pP/LmxvfknQ8R7NCMyQGMfQUr0YaswEJ8NhjIzf1C3m72vgKNBN6xssaj6G9AOZmL3ZFSxrWEOOzOhTGlX6UMeWAZ7Wiak8Q+Pp5aTZ9qICj0SZU8L1/WHMmTW5Lf4s2RV58/bw5zuGu9u0ZAVuk0tRGtcxjgT5VjK2T0bqMJhgt8vDokrGCehNUNAg4yrFJWO56Pz+C0+iYw6HJedv8aUPWiBqnzqBQ4X1uNXe8l4Z7LzEWQvSEIN3/94HWFc0J5fHyqhwJ6H01TGVR/rPkd+PZ/EZ08QBjLpWLDODBa3cEWeLi0C3khmq+vWgfr88B3motleWik5jSOMpyx6ZVs7val9rptSWrThGJYN1YETz4WFbN6QduRDuS8B0/xsps7k6d5f5oW4zjIvSGYw3RsvyPXFQVjhtnAEx60SkNnO1jUqFcR22sdrnthiWhmSBFEevAQqVVs7/+ju9UbkgQutA35vBH9YUX2N9yc6aEI4Kwoh+ww9Y5ozvyylR3U4dCCRY2C47qTjm/K/H2JGT1lnYN1u83cj4eFgC3/aLPoqcKXaTfjGoekLunt02V6jQoqiKJof6tOumzFUsSH7iGfBxWYZgkFH6n5tJTlfNCQaLPl2Mzw6+GMdLfcJv6oK6nv2gybpJGun4FS2FA22XU5HOaSstxy76QBeeflKNcBrtxBqDrTujbnKf0vNHjbrcYYEJcS6y9PFYdKtmXOFGyhecTV2xeYRa6lXd27UB4E/sss/AtwXCokB3E7p24tzQUZHAn4kmBstx8OBpWQLmLLdwIRjkUaXFKDXZz/9rn+xQa56J0FKDyxyGxKQGh7MqyvLYkEveCTkkJZ7ZiFus1pGrXq+AInmTL0VF7gWVm91fL+z5tbFGVB40U96Vg1HN5UmlHjfZoPsyiVL9soDiCBZLwNzYl74tig3oP7soJo/COPPWr0+96ojhhrDxxP4tYqjIzMBbnipfn76dXHtoKKDOCzAfywtk9sYx7TpNhEhsn4SC6ZIhFpUxn0d+eh/9Y+EEj5glOxeG/TB98mXsX5Ip8ucr1jqWf8HtMxkQtNK2vpITsyvpjgvABjx+k1j8u7j6mwAq5mCYLnuNa14JTKUshOtrqFWRPBRRvzaDQF4lfoLsnX2+kqq3CPocemkc2+Fe0jbainPcg7NaE9kseG7g3VfNWCzBNG+zeDBHLTT2siQFwi8PHt6ib55isiPr1+vLaWdlnjAq51DgHT1AJlQ5UbscZxckfEd78cWQdPO90vVTkaxuVYk4nzg7lpL3MvYvMhnnELdO4wUCEisQZULwIbSOLpdci0lG+UXqRunodaNGYXAHp95q57nBOBiIzfpMxTkEoNm2r03croGFac9zaUPK64FLXS6izwvnEiWUi2BObqnZPjpKCz4H3sUiTeL0sp04DMku54wrfK+USSZhhz+0Rvycf67s12/EFT5K1HT0R/j8cpduhFXJYp0yKz6kwGLnLGkylMtaUVv9qqTs/LouFMss42X7ojPNMKQ7RvoWTbMEQfA0ixYnGGk7UyV2Fzj7YsIG2suKAjVIv7Yqw90I03Wjoz1FaVrII2UZNIL95oz+RieV7RemKQlND+1VPPgCiACfY6GRHu9OabCWagEnDQVwrdFgcu4+HxMWjirrqvEFkxLQW0vyzbuYu40l3S4psY6K/moCnhncW012qNhntqgTnH2z6S17/bcpyuNkVW2urRKk4Me7m3ql4ZsyHw4j9I1fbH0OJgWpklqG0aeh6w/A1oF1GB46nGDDzDAik99AlmGGtq+b580LdDoSN3C6kqIrr/shJRTncLfy4Fp+f/wJ0ATAOD7gqz69jLhef9a9H9mkoo2oZTPMzktrVquyyJxjmnZMgOa3aOqlDl89HvbjzO0qOBXkhEIh9ZvuV0y0jcJC+tDyexeX+wN75ZRV9tprcRvp6mgYXNlHuLBURPTNWtXrjN1rl5z4p74W7PzE63a21gcJcn+gNLppCh3Zneb1CztnK7nfMK/7uPiCHdKAj/1c6/TLA+XHNVib54RmnKjSJO706wzisF+zuWXJ8ylnwX1JFn7hEEZQhTuJ46jTrnLWERlXkC4nh8SnKMX8q5c1XjDrfwTIGMfF+4XAQe0VFRPNsuD/IeYcgCelfu2O7jjOR9sH3Xy4ytsbKOPLzD0mcIx7hMb6jQSaogiWeythqI/JuryvorwwarCXwdONU2o8ly6PpX/5smMCleQNTRnhKfq1jw4m0V7Pg56eHn1PE/ziYnkuN78389oLyG/3AdEAgMy/NEkvt+PWj42lCwPbdB8AwHAPPb07BFvu18MFfOjZ259AB3+FE3bABTzmZEzUfLA4Kkd0M5An8ctitgdIRIOwfUjzG/4OxVXjlZ8DP3WyUj0XPsiRtN9fxgk7wWdNdNrf3hzD3Ptx5GRZdX/rWgRAFgWgQixOwvA8GPKxGLbnubFck//6DJyEIJLenAADgupIjnvoimO/Y8KkFBC39ySi6JHoY+P66McmTGjUAhwtyi0JkkG08O1MBE9+/ccoXLh4rq0fqze2lGw8+GtVcFf1UZgcd9TFWCadyQ3cwNiaKwlx5MdTYAkARBHH2eOE7LywaS1IgfDwBM7YYCbwGAakwapfHpAayS4O3u6x6Sguf2nLtIZgYSHd08c5SpedKGxjFI1kegwepNGtV+srB89hy8NtRodVASqXrpbFIBJL3nn/vHRNEevyFdY4u1UhxJGwahvdmDQIACRvWQ+3yNMBCC1PxAcbJKv8OxRM3z8P+Ikivx6Qw2dY3aQZzrKiOOKnU4m/qfKnWONkuoSUSAcHVbWVYMpqm9GMynWQcc31Qrnal0OP8wg+ZlOaqanzP2R1MWH5hDWT74gwdmnUJMu0bbfhOVvaLICmeqIT656I1wRFYrqFBRtYoxMYEE1SA/UCfrtkgNsS98uiDMyyRlsTyWOK6cTQbVg15W/Vb5odj6tzQQQizDS9yD5ZzA07Al1pfCkWjytf9wwmJ6EGh/hitfotDGmpuTPgXeSbcUyJVeGAHnTSHBqGaPE8PG604rLrMch9yF73Tz2exIlYlmAkxR4mbq91DVk6uByI1KfhgG/5wNmYXKDoFARncm0Aimiw5MfpbmU5ucLORjPnfElI6MBrxj7HBECehvH04292tmPJrw+5IaTo3huBhxrtXV/iY76GdbcoOBQkYPGonHyHoIQFIiz6A9IbLmBNlrBPnKKA7QtW+sV6UmnV1i4o+gFHkOWUbP6qgAWbcHu3VKd5I//d+FcTHTnx81RQA8Gcgw3xEP+aXwF6/Uv/nFPibBI7lLg9a7EzLK8mfC0Ih4zICjM2uHEgXjDJLMpUbGkChKSwpYFORDfOA2Fa75UVVSm/coU04jTfLZpEuyktW5Gdcak5P/9iv6GWpnJbhKpXNLxXDzG2nZh2BQQW5rqyzcfS/hU4PuGGmbMnwitvTZxzc8N1LsSdWvj6EMxmnqvUdFgTJRALoNbH3b3l37u8ZGWKDo8HWMKS1hyNrEzbEazS5N6/Pk/NEL2YHzznf50AMZGXyiy+VmJT3utHHiIplery4Q9/cIWlO/MYD49w9sUTDB/k/MSjKJl/KQLtgwpCY7DJklyuZk8rVmPtFHEOz9BtTT7+mQiecHZGjEbauuYNoR1UsO6r0WftsL3zKrelV8v48mI/VfuniwjLw6hynsHDu6J5M/yUI162a7m0PViTFVL614C16HK9qhJWGUqCTLDVCSWvW25/twYEGTJWL0Xz5CVxF0bjOb/dcwI+6GXtp5C9qWUD68Zcv7JF94f8bnCgYjaOHjRdGNy19YPCbrofJaacvNKUblSf56+4XtPsL9Cwf2gYLBKNtqiqiPsHd4UF/ZYGRmggXEzfL/+bWCWUvfAcbWDgJSWm5EFBUPu6sfiTO/IkcctvevhvdkCTlOZFDNvy9gk4Fj8O9O6t+nNSpRC3Zq0ztOOL2RsOB32upNZpkjmKXGy6M9mpUF+p45dRYrsqH9YX+DgxXMhu4iv33JwqyWF1zps+ZtgY58fLxKBLOxCS6BNoYZAegKhMSylWnTMDiYLqStyAt5ZupGiTJ33xmw0p053slf/qms5g2OT3aQsut6DWsJdfoVThicFx74OWdcwkuEJC9KQdDp1dGnZKJ4JhKMMvXNKmfSKd2sGMBHyIwo/Fzb6DZu1l7qmgw8ZWWCJsYhViOkmmMzk9plD5gGxi3KqSue0kEFKKGbd/rRMc/NtHdH4obtwjf/sjZav+d9LFOjvA+rN2AuJuTHwUjb1/E1S9i3nbeEwHpwJCuOBeErEJqbtcTDWl0uMfbgjEPTE1R0whU5pSZ4/6wnejZT93TvvLvU5RNonpFkVUBTB90SqGRlJbVExpO956xIXHQeaFxf/0vXcaoqcVHmKYUir6+9GNHjLzU0IzSXCSvUb/XmtHyyh0IHWUxyuYUl/iw7aagTytDjVhEgbZj59OZXcVXkJEKrljcEgGFaXW+1wwjZMMHaQd/XvGjNuK4pAhSvi8rK3ZnUxtt0W4jNd+3Hpk4sfljTRiFICGc7+UHb6PzVDwsqefAafMSvJPx/Dt+ID33uSYV2JWcTixN/NeoRF0bv++CglrKlIJeWhYxOVrEgNVKpWRDaK/fz/IjPqzDnzIgNVIAXIunsw2eCaPMIBxDRthD94/QrtzxDVwFaQHPOCjh+ZY5768Ha1PYleLvi8OOQMzt/amu7Ov7SzcMSqgTtEzQAZUb0MolEd1qgDhnWESMT432Eq2Bbt9P3QGdpPDHyf5AU7d0K7m9/HogC+VRpE0afg+wd2GT5luiYQU+iB1bI6VlrQckPp5N9NHxcoTcmmwvsDiAOkijAuQx/ILAgf2q+am6QGTH/rqfs7/1zf23076ocqvr4rFG4+5U2crgS4L47Kbe8XezpvAAeC6BBTSdb5aTY05ptEAGVKEfE/SigYkG9A05cCnMCM2G3+qNeVFlTGj9UHhCFEpUmzDvWXVFgmJuBuQXA/I/iRVM6X1g1YsCPRTcv6s7X+/W5E0/Wy8iBK2iDPVg3et97c6WafrKvk5wl9nsmaKvwlnThRFqFOQgEqLCd9IyIhIYQ9/U3tm4H6+kJwMPjsWD4IjS9invYM+lpZTNJCJoDPGnN1BGwmi0Wt9fBiUN717BYNlAsyuQVIyK2bBKOBdgV95e9AnbnWTfdzU2+ND0+V08rim3irPkt/yreI+99a4TANk65dhw+tjnq2NaP/RdBULjipR9JdwWeLuzi5YkAT3r39Uet5iNj3pNFRdOeeq3VQOk7OQzfBQBQoYsva7kzflq26Mi7+oB/oFMmatUocJ2G+eOCBlLI9atkQUKJWv9I0HI7r8qtYswvtSpBntLpqWnItaI44RPzYIhOC1vYQQLxmOazeu0YOeNBDmMFqj0B3sbaiWs18NA2o+BIki1IRWuSgn7bNK6uStWZoFTFrllFK6bGO1dpgElrJYJS/1R33x8FkaOHrGLph7zeKvfXc+IpuuPgajV3etwCn14/U6jr23Vj6H6rI3Mrpsuab3kAfbsr8OOmyeKfVj7zW+fe785Kx3Mlo1ugUCU5Loa/ncBUUr8KSeeJbCRMgAi+RZRfUdaRVcLqF0cfGV8VwpXq+AyjPrSL7DWgjASFTKDWFtjCLSfZoQ7/u/DNwjjLncU0nCy1VDiumUVR5qJiIC6WYl7R0E0cSMCrNoOzwtDscAVjSIDufXr+B4jBwXPgKyowaNFpomTu6nrZfWen48WllnaZ7ZhprpZdoDhgOJJE9avph9P+Sr3H8RcZDyr2ojnBwhjdSChMg3GIUmBifpnf8mDGjIg3nQxnO8R0Pa5RW0+jVVIGlnlran5mgrR3+fFESxIQNvifQo/VpGi1OMKtsO6+ngChCdM1o7Ct2+lA3GhHHiJBTF++CQRjUcxeUktO/24P1S8eeZLJx+O/pDaZ2vuwuUC03mEmCikB7Gv3IZzYKl5M/SiNSVdUlbtqME+iSApcH8kd7ATIjHOXb0J/Skv09uIXpI/BuGQrUmHnm90aDQiqGI2TPNtrcNNT7/Z5VYwPW2v8A7y3w/oxbLVhPExc18sjtPPhw95lrKDZpxSsB+00mkuvJ1qBAPEc2QXUzI2GkPt7+JPyayeuH5+EuAgU5KwXb34z6e52Vx7VvZhmXHClTELUbRhm8cYpz3+sZxuzlDOZ1tVREgMFd9XhPT1nUzJMH2LgvFnKe7gi/0Sn30XtRwO6jnGK0HNlzo6s+0SM8M0zbzEqP6A3S6H+666RN952H8m71chg+J8tL8E+qzulxQeM143TBYyggNB9Ws4nnvRgB7vpzfrLK373jCd3jVik8pTueUrZFgNJHpxnbp4RqR8DgSxG9CA4VstfgiJySF0dzqfbjN0BVHSRy6c6SyCAGmZUd2LAY7izCs40HotLQpbF/OJxXaklPFM/O8GlYbv9uo5VVhV7UNZlJpmW0vSRHiiDoMCo/xbzb9LLnPJuQuYXH8slVV1KpSjn9reBwC++orbrs3J12L16+ifT7wpyBVErySE6WERJUm2BryfVyDmtzYErfM2uqsh7xin//Xn/NGDEYuft3Eoiu0ZKYSuUgXMKI1rukfGAhcDSpFtXBR/jZQZGR/APVICn0ScwGEZNjjNN/DesVvz8Ep0WCTX18ntbphHFerHstVC1I4fAKGhj0qVZVohxCyk5KvdoBZpk7W29B4mzw8vgpN4VXuxotGZX5OR1+jyS5gWWFQMukBf8EjYpl85/xD5rlFNyrCGz7K3dPXEljmLkl9i8O1avpkkg+1onCBtrb9/EGcUICSHqvOD1I5ZPdtfMMrtOkcuDR+OHw/Nrv8G2ckiknnWzBdIUS6nzjJdYPjyoBQKWXMv7et/Hjb5mpZ9YDbUDfhSI4vAoPoD2x3Uy34TMmImOf1r96G7CZIbUmZLkK+By9EpmlsfHn0ZMxrrefXqpY21Dfh/xqbmVX93tFJjfqFSvNJYELyHaWKJ2O9hbZ1rVus56ULfJSDv1HDck4v4aGCzPO3qfHXi5/mAa6NtBNPI0uuMsVsGWv88DVLEbIGYrnd2w4DQ6noImTG30rejVQOi8ey+ovXXWU3aMe6Gl/bSF6VDswpwtFxip5m++bPMDwsG66PD7B/8eHBI7hQRtnim3n7q6+VCN0OB0DlYomFBGVUAtJ8lv07h7J8fkhPIeRhUguIP/n4Ct+SBIFaALSvZ+PUfGxp3vIy7OeKgxIUUaZAkJR2rZL9xPBitauZwbjJMmPyYISM52EEDJMS78vMfRHJDWMvZTVbgSvqrXkp/yAb+1oe3YaAnQqCMKE1jbI+XuBxCgiF9m7nn6TvLfhLatMQ5pvxAzycrB6B6jxg+GBHUccz+35xX+dIz5ZuLROvhXJnh+i19dl6am4YnBWVCE07K3HsCtE+Y+QNX80IYEddxBhxjth62vI8L9XDVqx3Hu8uCLPIMZYTIhNNd12aEqMw7gsWZZyGX/chl3Aw0nIKj1gmd4JjZvGIKreC3C/F/1QcLqGyyGelaNukoQNAI8KpwBSPByUB5KNY7/gxOfTBndL1qdoyfS3O924IOut6//NyZY1COj06GICosZJfUp6ygYhcAe4xLGeHHG1QVvgZ6tS4CtByEii5TmFtb1z4h8PimjWUPGnxQRJp1cPeAmxqFomUSOv7uEcAQJ4sID6Bk+/QQZoI+3YH5y37lAM5AgigVp26dkSv+40tVIS5S5HCKaC6rZaJv5Q0hb5e83JeMP1tYkSmEOs34tnaQ7po4A30g8tMaBf4D9ohjWy3Dg+EU9BRLFQYeYVJ0M5elcoVgjjHlCHHfodvKDcE2vJyTWvMzjeXCJF2ScRUIukDyKgt99ErWOBRsUi8x8f6jFLloBHBLBEFVKRtXfpK36eWQfBS827ELBRdv/Lh3XIfKv0tAkIdyp79brOIbL/4U/qQ6PsbhG0p3H5sIYYRcA5I9LB1rC3fl61J4NWy+m1P7NypIPLfZzxbhCP9SiEsaFa+vyIsf8QA/jVaipmbavlHWWZTFip0BJNqQScpy4A6/F/3DVZr/hK1+LRm8uOBq8PG155+wOX35LjxtUcrSiBme0phjZDTQ5gFLlovRpu0epJYRw3bNBvYaJjemRAKyqYVb27TbP0jBQ/bvhLXXawD+XqRq1ZSTbtVJhGKfwVZ3Bz1Lmkq2+e47P9KpgF6f+OkX7yhxN3NwHPQbdtomwmFx3g//2ngsvUDTcE9QtT7BsU9vxo2HJIn1CP8toFHQK2cD6QhMJ35ymGq2DbnJs1XhNXOeuaCkINNPmaO+D/hgXZxQF68wZwq1nPXriDSjG5DNkgDoQ6w2XbDwGsKVfVOIy6kMdgUOi/CPr28YkdIOYnm8XVmPNAoRM3j1xL5l+/K02BDTFA5bPvwtYdhn0IzFi+OqUhGOa1m26SVo2WQfLVvNAmGCfkaL/uAHzq6AR9MMM23TxVyN2YYoJ+QBAL2MmRGf9Px2wbB8Dbr2rywusssPBF+5VnjCYTlAD/r8xT+hqtNZ0RUQpZtY+n8a6D7XHd0veaVq37h+1Sy8MpjSc28sj0WDoCE6XUu4rEKg9a80NItalx2ClRFNQisI2GFujJuBU3T1Hp/mYKFfANL+HQ3UQ7TUZsNxpSJFiqrXvWYhwYZ7nv66mpniX3F587U4BAUU3ZuyJBWak9wsU3QDAstPQiKhvmNwLFTaXLFwivTeLtluX5dMBxDHK3AUNXOHFcnBoDOKIEzoDR8eh1QD6I40Q76U1gU5y5LY92PlAGI7AZdo0HNqkNzjVLZc0mGJ9CUcjwQaqjZ9/Jtg7FjnU0jHJmy5MjBfUFzRi7SfMawdB2HSGnqvykTxowmbebwS0Tw7o7IDDRSt2XssbBOZ2G5YGb+FnW/teNEy3/kLPkU6MzkAkT6A7d4xQr4lMqOcotIAH+PsMBoXJ2t7NnNOJU/IJUs+5ljuJ5w+dwPHROSE1b64Gx20o+B0rhdsMXHOKBi89ssAwjO6aAr6vCPwlvnK44xDiFJbDaQJiRlGCREi7fx+XIctsphFvgMjU2BUMxypwFe+1mU5oqIuCKGLIzMPMtS3WhyY0j+zWGKvMe+KDkODtlB993/iOG0r1uSPC+B0+iu3MAll2h1XI+8P/Scmeo1ZC/zNyoAvdB2gNJCJdY5JYzkQZI/bIveLW8+QPCYPl8sq3ucIj/hEPr45Xi7XmSAk2SUzGXf9gKvSgjxm5YWc6u+y6BlWfu2v+mnPW7HYYY0UcM+j4e1I3Kuy71PLczEa17N801Re/Kbeh3MwBFYvxYD+iHHGbs9XuqCkurs/RPRKbz4gsCIZFMh6ZKjfZab8evWhgZHsu6RDpMVeT8EArjF9gUqMaIPWJwnOl4ZdXl2Cm704dmVC/CQUSkdOcEfFjukdcFJ7/JO0K7DYUKQgNItYQXrzFjQyBvjlK2rMDt4EMvJb2EvP7WEb+pCffnvOEiWpPidL+LEWGbNpT0I0Yv5OMiX/INhXY5iyWrblHeCZ+qxq4DQA6S77zezbLmCczNNsDdBRg+q69VU7jPsinv/tiFdXF/SYYlHDcLhBfYSl9uuTRYYW8Wviwf4ttQ1JpRF5R6e8DkjA4DCz9/7GrLYkzvf5g8S3R4s2QVmyClz+673h9GTiPWth16t9p++Cg/GmfIysadep+fOYUSuEsBfsScVRFBY0oQh5UMtvvFJJO7V7BmVtuBwgW0l+jn91O8UvJ/sVGnpuz1NOsiQ3Ibqh4HnlYMV/2O/LsQCHayOk9/2kBEKMitYVsT9+FFPpBVTlxxe7+SLT8pMXLnvBmI0ukcFr21c2MPRVRm/CP31OHlnVELiLR2ZRQSY/etGPl+jxrwfiz28QbKMVRgQwLx6aibNHW3SxwwHY3+S1Ehhcj9Inxwa3vVjBBSU1qLew2dPcPWR4EzSengh8RAvgE3FmphufBFBFXDIwoHoA27FJvhcvXBWGHGAgvgsL3IY1/gLA++aItoQli3mses4EeNHrgrE4P3gPA6JuAcVMSSnE1zXWI02a3v94/N2GcwVW+5c4atTKQsOz3e8h1e+L/Kc4XmFgnT24gGfbFuXF9krGvIr3h7ItmRLvNuBqy7bbsgxZV/4cyxwiROGIhkjCCB8Oxuv+nnjT23OyYYq+GQXa8WQ/Y1bV7LwdgzbYyKv7ZVgkd9OaE7DvxnmnTDxqRypbDwcJu77sodZfV0NPDy+k2/SpZSxuGd9Mqe4CiABtsmrXnzMC9IvlCskBxZ86ML9pg/r92B1w7Ul6civxQ17qE8uKRU50G2iHYuaWLimOYAG7jTFkEWrIHZ1AILCi6cgugBW6KtG8p17epbFKCzvpVQJ7N/71QcR3JL0uecH0+rlnKEu28Wo0s/LQZsmxEfSdR6J2ggrF4sK4aoGblvOoDlmqX7NxfoY3xGXDujlq0TpnEQJNSwbhXQokr9xCLQwpX07oK8puWrgJIrumJtgUdDqyxtbi9HT5q5FPA169kD4SgTGcc0bOPrVQigv8Jw8BPqH/lppI4gWyGmYb0p7mZUlz7ZgKy8QzPo6rf0bokx/PsNs3n0xVwdsLoSFwzjGoxF8r8xdhD3bLcH93PmE6MxpLq9p0xSrmvt+c3ipqlTdeGOlRpiDy7u8hGSYcB6/EYYYNKtX7GUQI+rx3U6eRP1obZCAte6sV9TiEr7n81s0BnFtqeWasQh9bptufanJeoLkUBkKxbizd6JUW7/rB9kE4XqK+67dRc0z3r8SFnKcM6MclxHI92897med122Rg40a/O12GbIv4dgEVaaIGZGD7pyHw+4ci+ytBSVBFORabu96mRo+Eepf9Hw3B4jBPGRUA3Ba55CyVZHBG3SxHqf8IzhiQ2ghKXzcqjVdHfXy8WO7BCHXeGS8CMqf5js5Qw0pxUInO9cByy3FraO/BtQssy5hr1kPO/30KEsxpA8U96RpgsNh47jInS7n9NLp9ZWw0uc2WowYEODMSYCUiq4NBjV4a9wML6AKcQz16tzmu8UDZ9sEcluIa5JUKK9JI043yUCFozvbLxEr4nObmjj6NbRzRwcBJPxrvDIifymTj+JuRWXOmZ01HYLiH55yluRfWPHhAq+LlOlt31mQj5qF9NjqN1FrKhPiOacoB/eaAMFZruPNgI0S4gNS1UGF6PBAJtuJsuhWefmOEh1qCoPMOFCoI8Jw0JLMYeXM+/n4ZaDmad+55glZ/ZWhCdlylTXyOzRND5QhcVeE139tpSzNq7v5PCJ1soxgkFehA/0VbhA0rUJKjXV41NYSX3egDCAT8ioUUyjCUBgMWHmIOEkrHE3tP4J745v2C0BjUZsEp31AdmXvCFPfmgUJGpha+RyDN175lILKhN0/k1Bifm21QKMarL/IsMIe+xPon9ylkGBipOBO9PvzdVskvVUw6p09C7IzS1A+aZBU0g1l9it1aqbBe/t1D1Uuo2PMBkRv0GHtBJDjhbW0OipQRkT2TpJ7n5i3NlzGZ8f7iURnSi4MuZIuZWFeKhDeHUHMCsB6bHGF/OURGCOP/TBMzZv5LdRKINumL3EzHMAKbRQ0R7DWzQOZYK/SlLEqEJCBLisrhjv93iZ+5REqKCZRO/j+CN+WAHvMvG122Jx8vShFMZCMHZoJ5Li+NOF3WDvfUYEVri4uhdEdsHfg39xSe1eu1/R4ab01o5fORXt8A6sM8mcN/N3ta78R7vzq4gSHZ1fo0gnBnOg46pUTVbOkdA6weDTCmnDdv7XlwKYjoPVy064B0MFiuh5rzzjxldkyBm3ZXcDb940LV7/R1rGhYpWccHTy34ejZBRnvVjJcL8YSbSp7j6/E/GF5HUjwoZZk6T0QxPEC1eKPYk/I9Pl6xYdX2gPZTbCSidcESA7blixurC+wR5VVpYbD0IWEXtFalVB3E2siKOrM5ktdlcKzMFE8IQo6PMI7ZqTUJSiLF7ZPK6+puzvoC4aJG+svyG1IsRPzej/ClcvQhYSA8WuTCEwQsN/izMMvOgeuVDfzPl9uVYD8OWsXDaGfoOQdmWeKRGX4Qz4FjjIXo5dKOXjmOLrzK8aDk0Vkt4GQGbVb7tt9iFwphIU4/HPdcm4UQR/7d5fdE1TDyqJdK8U980A8SFxti4DegvE6ze+9ohULxOw6NNofbxZyVfKwjWWyCvSx9XApOOFzZXglwoWkMhDzSYRAXFsfjxcjD97yDN5GFNr6v2SoJZZ0h72X8T2CYgX2tK3i1XmRpHsRHXr9XN6TSBaYd8aEcy9ni+unHNB5dwI49DEUkBf9RdM7QGEFfbiaypBPs9F9ukru90NbBV877/oJgiB89Uu9+LWjL/k3p3hfjOYXZFExpyI8m/VS7122HiGOSrvcVoA2DafhrGD0EzkuHvzNnbEZJMYq4DNQQtE+tJDVz5gk0Hbc0oMaTNSKLLfluuUGpa/qY/A6efcYOdtABPR+FqOsukcCi1DCJ/EHplAxdFtxEfRPtWkcxZJ9OWv2pCmH2bU1GElSq9XzjZnZR8GhMYPtEMTLAGBNhImttxlBiWng0FRLU8q1yksArR4DlM5Lzlq6/AewRaq5kfMX5afQF98RlCrngWAJvbjMh19uro8OR/gfHfd9I7xdKjypghY8XT725hXYCEVNC77w7PXkWj4UXEbHdstj88AfGc//zbd16aeUnh/2jQKZ+axt80D2VyBMVpGu/rGeo6rPwe9HQxfOOg7gIyx1yRpH2c5btXHWxnnK4cTC56ukoVCj7AR3f9kdSkKIlFN8YcuPX/gK5W2b4ZSiMra7OXVg7SDIpZkOk1J3nOvxKSAFUmd6JNkpttri2AhxYDwkjzu/lxg44iylOTEN1AHaVR9DDJAWLVf7F46cAfauNeCmhLKMi9g2eRwvpB+r5FsrW+OKu0PIm8Hp39udS4kxnfDsfT/pfdOVMEDa23Ctz+WMTzttRwXvXQLStSPmm28vursSqxzta+TA2S/fRGM8aJN5zx7w4iKlJ5+UU/FhkW2Salv95vvA8BsYk5UJORn+ulmtQp/GW4qk6WoIbbJsLeDqLhaKV4jhsZsUUKydVFUTAxis4I9pOz2ofNjDHZCUR3+vXKahT/HWqUCPm370Y6dAPj951TURvVxBL22B5IKk6HYs7tglPqxljei120hDVyac3vEv3zzsxZBKZ+eVCNl8bAfQTVn1wzNgMitT2haWww/DCeQNx8Q7/YR28Y1vnzzuptDBoLTyaH4kcqwW9dwMndTHD2hURJHYvbPlCgr9cme7/DCmiQfBlcRSGaBgqB6XCd1qC227R1dhC5dsQ3gAjUZU80TueDCyTVHKKrYBI2nIt+nb7SAZkVWsUvz0MZIGPZAuF6lSrmI+KVZrPf0kvYGYKNzyMnxEQJSeUS1x1ysvtBmOzdboksIz9nzYwZYhNs006gsF74PBbO3HC5BjCO5e8lza2DOMfUjOrDLkxGEBorcFFP4IYQ9BXj3Ax8jvEzoihwBmsSFcufcYC0iA+p9K+Ss9ttdDesL5cHHEbel847F17VzVIzZQ9MAz+49fgE+AWXx4IevSbupsvfPd/BIjyCHChQg7fnXdTt9u8Zw1Sh0gjOxfIdAIze/2dQPQeZcTrLyeQNgcnemIESoHn3lCHPkoehsjR20MVuypu1G5Q6npc7PA1ysuQyJqzk5+8skliiJgVTvnZbHGcIOslLrAzXWtVFaaa6qzXKQ8xa/WHInyZfHY7D2PB9pjDH8xJO+A15wAYepAZfx23KF9rH/ITCRypmXyIQED8e+Vxx9DimT60pk1OAJgT4IjBIaa5sUEG3ls2vnvHLbGEic6IwbjfcCkVS8x7T5XA7QN99A+SXTCZyKskk7BoKamLguHNpKqJIPAmBaX4Z4yPyIDgt79fr9yh3RKNpcM5bpdDRTxzL/pXvKiXYScCeT6mH8FNAseltUfU0TDrvlwUELx/lexMiMjHbGt2VVXbMS45qXxzqqnhEFrczkd33NKqBsa6/jMyvQ3fDYsW2C8qylJAR0hrPrz76y4H5QhBVQvje7NKkgtpQmBGbPkNW2h9xaBpqoiLauR1nBdeAU3sMsl+FIqKr7fl93RFWWiV4oqGF/OSH0DhjNFkrO/S2C0DWe1ky1vLjOG7QFy418D2L++wjJeP9yxOSpgNWb6WN8Yp3O3OEYKgJFOpGCR2mb5TRg7jqeWnHYFJ/OVlDl9qacDGBfG6HXTqosCV+rKAiKIrn0bFIM5ss2HHnuaRuk/UIiSVrpDR32dxJMNVZMPn4NI2cdeLB+HdGU1u/jesbivPJY73jsyyWxfL0vSWR19dsOo50cTGBFNkzTc23gXyOJHM7B8rIqcgkzV0SZfjy1HE8AID46SCF4YXJqrRHiHvLyfJmN730zYXKiq/6iSJ/RGmJ9G48v885BbE+7XJU+oba3PhlnN8c3eY7dhY9jaWmK+uYWdHlUkk5BdkZgnAvRhl6tyc3zLd5PJC7G7l6rpP5d66nFJ/72iHhu1wpXRG1Bcn0d9S6w9x6Cn0zx9vVHc9Uj+9BWWrcUMY4SYsGRr3bIumyNYRhIX+tLtPemRtFSGoa8JwO/pXnU9xycz0amHbEXDyPd+Iv9SAjnUpIv3OhKzVr0Gwrp/+q6BTw0Z1ByInJheqS3sYSPscYjiqawHHh4nZheH9cMAufB3iNWD8jjb+vJST84Sv1c4A3q40WGTBph0s3vDgBv6snPZ0zgdgy2zIWQDp1hINhhCsdTbr35pTRZ1O5BxnfMwzISd0PLe9K13wgsw+cbu1xsd+U1xp/h1F2ke2yVWjp45ls8WHCzm0YNrn7IYWwOh5PDjRvqdKk29Jtv9g+J+TC55uyWlo163zKbkxWCHtJadGFpzeCajB6phEEzhQae3klXhfn+7eSCMYONZgKnVAmL57NFXzx2IGLIHiWLkluVpcPlXH/dWbUg6zP2Sj+f7zLjtO8w0KvYFPXgnQ9/WoQG1sWFolsk+WRlTldPd5Q7svzLvk9NwzvxgzH5ppTV6WV7oPouKiJVbudYozHX+XWdUdCYbH1NuLJQHfQbl+BfVIYNl5euEC+rSC/oz2q6jMCpcRjHqPBtg0cZv8vVZPxDgUT1CyhI9OuheVkpvuMYfNMsDVPrWkh8i2B+M5EudWs12evjR20Sq/g4E6rZQ/jwDRcALkJSOyqgPKKg7StLl1s+u7J0c6Tz2ewiMRp/kOLw9Su0cBMWs+sVs+cvdtdFfeeI3EG3iCXdIyfJgiEZD0XhKoUDCX0OS56WXr/8dk/o33YIQ8/LJHRpfJbmgPJzKszNtvfTj294h6FCWQuRwof7tOwHCc6W49TZVEJnN+8Qx2yeeGhyMuC0R3uHCn0ZYFBPdQEVvr9GzXHyBUTnzozrQ+JEOebWjSyhAG/VNS6Lu96TNmB6B2SQ5seV+Cpp1D1yNy7tuQMWcRMYjSdSnesIzYP2ru97LsnAP9OseFMXZDutlUbO371UgGqwDymFDJFudvgj9tRv011ut0S2nXDrBTpASubJo/OQp5Tmw8sK78eM0vthjB/khXo9cxSZ6HAtiZItZS5k27L+6QihGJBR5H0y6AjVpBbB8QkPCcBuLnzJZnDqyspO0ZK9qDMxpuWzSaLNBzRyDQdTzcXG78/Jd/ZxSa8bq9kmB3hc39HJaIY1rEZHETYbGyGAmupdU7bWcolK1nbSXb/WW8eo/ugDLUlyZTJI8oTur2d4Sb/nk1ZhJbyjcmloNWpA4WBfKDGE7X5l/eJ9Uqs/qF/D405k1K+T5Le6w3wNpBcD99P1/Ov0mCKGw7JgnDB0wj7K5Fb2j1Vnp2t6LWwfnxfzklT3LWCoJ6Byl2tXWxtMWwlRmR5YBGaQs6T9uAQQZWPZb9layHxNjTNGKpdJ1YeKD6rjNDw3e9tpwOVAtPpvQIAOVP2sHoP4+EiuA1MXBf91Ba78+URUkWUAuku0heNs9HDzEbV2HanZ5KsaJeO8EanpV4TfEvCt9dmFoPCGKM5UffxEeULjtEh4z7goz4VyNFgscrThR2wTmmrOyfi0CoitUf3+5Y2pQAxG96wa5Qb/gKSbMDEyjd92ANXVnSDfFMQYFoc0bxePhIJTQcY5XR7zg23a7Oe+ghAFTLaDJDJgEcdDg98ftgoevRF5bJiWxJyDxRVvw3bfsLQbX8tlg4jU8hNUQLPM41TfX7KoI49Ts+6FZRUBqUbGqNg7qbqcNtJdEBd5n9ULdaDDrvrd1FhcA4vs5Y7R6GqcbQIiid1lOMEQY1YcYrcsjL7TTrBGGQw0ZEf76onGn355T4ULgkBZINf/xOx2rs4vUjovVY+4hPSavbuzT+ZoUga8bAhLjDh8t0vfiWkVHEmjmkFvNmevr/wtjddUf4nRHufX25lpUoQM+Ye+a0JwwsJHF5K/o1K/tnv4ag8olqZ+ChmGf9g5MVEJjfDQ8RVepo9dxqu0BjgjC4l2ZhI9+oaqzGofMg+qp8RnL2p+Z3lulSC3ZK8kMD9y/4uniiCyR29n9vkJV0LNXdJ8PAL9wEPCIa8zvgyGimWEe9BEu+jziAApaIttIPzh9YXsfaoEtLvapheC25A+izdU5X61x8SsEQd7jinzXZXc6ObJoorINoioE8r7LErB9ZEk/xIMKTlFQsl0sveZZAgKU6vQnSF09u2WqNH3fVgpi5S6B/jNDz2OyDwG/yAm0wRJXlxGzIrT/WflmuMk/SPX9SJdYiRO4qBMiJq+FAbOYustnYsqWh8Ca9Q1lMyK9GBTfCvf/LMcoKKpRBWHf7SOEWRfOJtfxLOit0OI4jhBEkR/vx2XPSgYspNFCzKtmZhpD1sFpNSQBPW6cbSmD0tA1T2/zbBpt5r2cJWI9t/U24IdKRJvmShmP9/0saUavebb/aO/h7ugSEY8r4uIthIEujJWPVFzLwLmbuytLpXNoxO7xQcYfcUyyJiW0Tv/3NWgk8AP2ARoSxWnEy3CHqalOIke+tKrk/vbWZeLJd9Bgjdt8pvs+5nQCz8E5VPR7NqsY/in2Ub4gqSjLcxU3V7dJbV2z0ouj1DsYpTyWpAVGq3rp54YLUE/BWx/1G3XT35XFDoAgLY/3fQclr1yLhl1rsx2wHv/2L3ZClgFIJ6MmzRKA5xY5/fCUcawBctprWIcVCzDOBKy6QHza4nVvPK3AsThwbzxljvp5QpMc7h2qH+EMKmhSHnL7rIj5nm5qZJI72NTpqnRFImNPdPmNra+AGzoyWXbpoiRPu82PqRixzJcfJhKq3CQTGzcep1791AParn4/gYOAGO+NVqruosg8t6nGsx2nzQ61gJZfAWgJ8olbB1180JBX+8tLUr34fGe3L7iN7pmoUWa450xjLdirx2HWWzmzumYxnpUNS/pvmue8fZIViUC4b+l8pApPrylKYvXXmFRZZNnSbbjskYYrRxpab1UmTLcnj41hk9XewCeKhkTorBngzBFde4qaScfQLetqNzI+74jbmuLLv/dIQOUi7R0GsL+r3aB8FhV6Iosxo/JwUppdyiqfZn6Tcgg2x/cenUzC8MKDxSmUiYCQUmErzeOrmQlMn8D/TnVp/RfkSHRLmI7VK40aBv/wxrm3kgpIpos13K/De/A79DLGUYv4v3mwNeRQbZDMe9ZXlf49MOIx25F6jP6NbfmujulBPeamsBeX0FylEzQKAXCHDWMDH3mNVgVoCZm8e8HHeyHmCIxiFirvDQ1AyBalIRS+CRlUHkub/MzuRAsci7EKmKDB3AagfjsBu0Ydg9137zlcivoHfvACZosZd3R3m+KHYvVOr1XyrHe9rp3DqAsj13Ovf3XIOeNL6fxdxriNJ+SERnQ6Yd7SH6Ug9gZJV0DZIOzgo1fd2qbt4wkGeHPQdp3sgdpehSmkL6Ms2BD8qrb87ukINAultW/CXdHoE4rfhp07JTcb4Imo8Jmgh13kSM/DnVRBOumqbNsA+e1Kqi/oCfuYXmLHXO564i/UW19H4ho+wCGNC78O6PMEpGuKKu3T7VDJajQZNuT2l1cdvnOv7F7Q9Ha/9dUKpgotnOlntS2vJoTrf4C825kyK3hListV7elKgOW6yWC2gpfIqrBgbxawOoVvrLY+gJyhxkKQsENyb38F4zOr1DMzT2TrDabEd3fvwaWzxq4mnchfA+P207JzPGihQ7TPmorPnWfYs+AVezf8K1GAgFm0UdZOPv6Pc2HblhVuGdHFUjukfpdP9jCDxaDgvF3IsEkSR8vgPWW5PuacmLZ8rkGORFxPTskut7p8FsXgorhID3kYGiu1uaNrB+uVHGDiFPTdN/by3qP8NLxDOEy5ztpEnQQTx7x8ejI0LHRj0fYgXyov7kA8i2cS4Tac1aDCEh50oWBJsmm1gQ+Q3IQVuj9qYr7vlgTIzZ6T9Z5YUMm+BYORx2r5qnN1OMX8igMTZqYCO5kMTEUfb+k7yyD6NL0inwKhNbTLzuDYiExXxRHvXm4n3beUwtgBB+eAI7r5X7ywCpo5rdgs0Qwxw9pwpqB2tAfZDbLBCloQIF/ja3+lZNN0T7GoH7PAxigsfKOklnH4wu3vwA8FUVcur722w8a4yjAFFARFLOIZf9QQP/dGsm5lhZK0vVPMWD2jJRvh2kjD8Xf+OB8fXZJa6qMrTnfoiV/G2Gk10RS9fEwTUAteXVrIXe6+ZcwsKyM9O67hEhu4mZhfFVVSM49jWxpC4DNheXOB6+lP/1JQYX7F66p2vIimPZ7f9kKLvQcgmk/B6U451aNfQBrcCNIzFxffXNodgDH8Me6Y7hv9WaYwWBAtvnVZBr7hw4+bAjJrHVz6iKOQsNTGWH8dOVG84qr/LUlFVTqCON8yyS5Qyv3ea11EDZ7tcgqbsSDszcsdjCuvToLEvAwROkIwsvz4ASy/pjqk8/pSJiRhmFnkkCYAIgtjZDANz38PpmWvEaKslRo7YIUmnfjzctM9+tLK5wbjsgEj8gQXIqoUcT6TGmj+kxc/ET3OvUWCdxGb0tMg3dky9o9rYEmXGGPHndT9hCGNth3wvogTB7/vsDLb5zcnpuljmwJyL1Ab3lZ2QDYSN2vQf0NxwhQ8Cv3gY0eCM641rRCUtZB7mzZvCXjAxeNWxBq2AD5fH6MeO/660MzO6Gt89tcXGGjmXo2NbeNHvqnkb85EM5+sJZfUpU+RBXJ6s97ZZyK2DYTTK31fYARgE/ZVjuj+dni0cKbzWAEj9WNfac/YTsobyNORdf0Op31geHiFjGUySz/LYsByFv2abu9IcLYj9I+WrepmPSB232cmn5kJ5G3VzF47/aDHtnYp32DaRtGtR8TNpJlR6UMuwsCeWQ2/4Z2+FWGgUVtuga283FwUjPQkFLUkIzZfWeenh9TFVAQyQ8hzcX/RkpA1f6OZjvLYBROH3RAnqRlgpJD0BkoBmLoNutz428R3+6CFRnDFXSqA9A3hlpCbn956l5OtYKMu34BduFY7YiKcTXkNmSPfIVH0seAM4NqZolbrAokl6YnwGKBQq77RqdeRWalp7IAOW605s4HylFozuVfwILAOTsZ4pRl63BYZuAOKdR0ZvQV4Y57k/0ZA5tBOFe3eHHXErzKAckQ8J1V3ia8OaHoshFBY3CXX/zKXon8PJEBf4Mgt/r3FWjVuijplimTMK69HvhKqxJX+RVBlQVpOTvKmYU7jByyhO+9LL/5fC3r9M1xzsTgbxVB/vrFSFnLHs7ST4pE8pl8xdCuEI6YHxXdTzpdUKEXWrLZxVo9iw8OS0HKmTcAriZlPy3z9/8zpr0+kWWpHk1lpquOwA1WFEvdtYjWRUmuF7gw5COP4l/mhqq5aFxVQzJh6jIqmP9+yEqlbVMWZTLZrglqyj04ju1XAbsR1jDQbNDg8+cBV3cMAB2fzDEgoN/HYjICk6A/fe9SieEndvwKOwyoRtmKXy/Q64ZPwL4y8TIUckgO4cskbmS79t4qP8cD/aJNPImXcuo/TIkJNMgnsUtYIdbUy7N0OgAyQAEOlULH4x3hnnVS1i0qPiCROnD9yYpGL3f9m8QKwo4tejA25/mqSgdJzBgYwNZQ07cHgD5zR0cZYtPXdwoHblUE1NiIcwdUG2K7GN/bRKFVRzgpbrqRL8/tsbXYS20FxK9i9jcY7g0FR3dpk7Z1SdyX8M/+skwV6eP2IEm3U97q2iRGNxHNliEYTBsJrWvEwGMZNWAhYSqN+XmsElqhQMFzOFOi4aU6zMXLiXQIgv8Y15Qp66UX7mY8m+dc0pTa7ASaLYfGbPmo+iJxNRRWMQgdZNs8b6YAloNtAjjja+GUOKycK4F+fadiAzUO7A7LwUCirJDdpvYTHt33upbIo+t8GMQrY7/+IH1eQEmZguw5z4hZJwDxApBXzL6XZFHWpM0bFMvyWXK/xQxb8uly/bdajBpm4l3u/JuRH9yEBm/nrfrR+pukggPNy5hUF1GCgCFapOXkN9LHp4OD9Kh39CaYyHUcguS9PL+8KDUP94t9ZceCWuq3NQv6hFVovzWNL1DCsIby6kmS/FF+EAMmlAEODRTQvPFt86W1X90IHNVInG+WYLxQLWKbL7zO7GoGSY6ctNtzgXnz03LFQ4RgZjm/AqgPAjK80hhtlfuge0s7vBkb8jOUTLsWXSh7vzscEAsHP0WEBGwBh/gT1ncFAA4mFb17yI7DiCvclkBRBPJqc19AbA4F4EgUoJf5mSvUkl6qmKDtBBpg+PtCxG6WkUZNlMbyCjscwnBljteLH9Ws3F7vyUGQCtF0OSc/0m9YyJ00bjxVHclJ7qcQBvpCwNwmcQWvsOXZciUfBdPD5eYQ0xGa+pSsBoxOweVm2oj1/Tx4os1MDqdb0VIARhSuFQff20I8le2kpYJiVuaaRYaGuLIZMsfiWTvd8AUPgz2rDakJ+EccEbs9FqTJ19SJ5EYb8iXoz98ezJz5bHtpgISsRQDwej0+abRFU/Im9vX2h5WSP9SmUsP4Qu3KaYhmfh3DrJZGwFVEUFg3Tc5IQJjVOE77/DGQAhAUk0+iSWdrzU92SY9JSOKm89qD6SKKSfYgX5Ks8Z+dNB4qYL9KTA49OJV1vVv0ieAWurs12Qx4zn4ObodO74501NR1MclkZ4VwFPFAu5RaN1kMNzj1vqA5Ka3ZSXxJDuELSm3J9iGkA96HHrI0F/um2ndCVtHySxtLgzoOZ8U9wEbrSl6K5JHvKyZ5P1CYiRnk++H3VhjOX/LNtlz7DYvwJSX3usKYfEGUxiqYlp7nFzq454kWJJ1sEjhcXLdJagUY+Ldm9uMojCYdJfdO8nJLyHbeQTyt3WcM14H5ZCneCcvzfGwO2KclKmSijxTMDRMHjR0+iSPtpk1ttGeYEkZWoTueYO8oJ6j0EQB3kWVdccGfC1oFBM/dXnJXPCuq93fBGhveYDYNQrrkfQnBPG5NC75YPFEIzRB6cOYiu07XcFN+l9tKI9eXH1oJJfaJ2veBGF7jtzRJnFbYI8e+91l38y7BN5edChgznaIwP48o8CSgHCXoLITztD5uafrVVMWRIHY19DLI3lBNry6IDIJlwa8MdapOO0vBNYrgrIVwiLduX8RBpmdytPbYsGcS8x+iRjMBhDeHXeqOu7PRjThehiaDbdOTCNLZv+zRpXCmH1pf7v46bjWZw8U/HFWa0t/wEHsawUwzmepvqu7ShwKDpJ+vidfD60o7AGIrkN8kjS6AimjwmHS013LrLHpUPI+13zv5QpPNFt+PMV25QWP8u1pOULwKsNqbAqJkgiYLItth9eiOcyF7wNt49iQ38qOhlZTKxybDGDnEKKY054PrW2krizp5BVDUGpErR3UhTmYk3+ti0+OoP36MDmJR8SktxZkQ1OOxMS9Z0eYaEbTRjjxQ63V999EkCY7aFSzgzSUro8X2PpO5lhXijf5nRr13233Ae2UT4DcWRdcbGqNHn+Lfd4hGykMsWZzgAbW79BXzt4+zbQfrx9Iejy9SjQEywKKXhSGKbC29qAr7cVK/iagHiioSQHnc91/9Gr8odptD02UbBymCxQk8uPURM2kcT8/1wqGHK4OgxMWCuqSP0XSvaXjQHJiaEslYGcrM4ZhAdqy/Oza/tt1/7YeU564AERBv2eBPptNJklgPpW9i/Oa1CyP9QvwCOQCV/dY90pRMDq3iuazukVa7/r6PJfugU3Pnbd0LT32/UHhH1NGGPV7+fCICfcCroVb5lyBG6du4UkN6UFqwzxFBlknOtbsqk/jfQHvWCXua2hlb/1DHO6HLto7yiMqXPibK6Nogk7WKeyhbKqP4L02ICNxfXbULUagan1ppH/0QhGZyBLvE2WX8KszjR2F1DxQf5BU0B7HRyihAJ84jPSZg1NpQGL6gvh5Yu+3kQEq/aNEcQ792H+CcsVw/hjjh9ahU7gKZ0JOPHXY2yu2gwkBxubX7hVRpv/jolLyBUyPYVDacLkqS4y0wge/lIzAxD3kO7E1TldtAUIB5FTGpxrcnzU2hohNa795JmB2cRAdR3w01vzjs8Id3XW/iw6q6h6NM53aSq0VHOggagGjol22hyXLhs9EZeKuQ27IlOm+OllUtFihjA4w025c79ls+nTVO+4rbunOV2Mrt5ILJpdzZEVaQqhLnvs1fmOmV5nsJf31wOjsXvAn3WatKepGiy1nCiQStmjdoHb3T5S8UxIamQ+77XBb2r2NhzxDRWeJWnVyOxIzXq7RvXrAPaNlnyih6dEoGr7se9wWM/s4U87182eNmHDSG7ensqkp/+xlJwGp5L0DtCjLI1kV3xnj2Qo/oRsVZXoR5yCysjVqCneNCbY+t6fzOTwgjmtZypne10ewHQhceVRi628PL45ND2GinZbWP8FNxMDl8F7oWrBbDGS+vHqF8CXMFZ1jGrnNKGv/GWc+be4l1BtAxqt1n7CUes9AtigsesTmxX0QgcnGQbxCfY62sQGQbB3fU3+wr+9jX/Q4nB9jJ31bciglaxkQYjEqz0sSnzfU0o28aKx2k2GWdvoVs/LVs7xUC84LZbNg3Znr4RpKohPLmrhn5G2cm/moKgRd2O7I4Q4/jDdxIE0OM+y/xSJVeDoS680FOo1gc7lltUKice/6OUnxGXnCd64E5Lni+RJwTK5RfccYX01b+Pn49vcrQ6NBBD9p3iMdL/Ki2uIiv4Ey5r1A3kcsMSavOeOd141WhX8ZJ6kaEC05JMQ/OqFOIcYongaReXZ42Rf8ruPLpvJTWYh5mIaSM6GvzN4Xw9KGBt8Ke+8W3/Mun+cMpl+NO+5xkKbczUtBwK657tScEVgolWRImK0BoAqC4xTMuJMZJVaVLOSy1mkzQNFD5FqNtan+8z3yDzZgulrOD7zP4IHnB5wG3ogP7DrSh91F+I28Xw6DNHusdccJtv89LAG0FojNBI0WUxhHpOVGW4ldeCg3SbeU5VFfFM2LnPAGJMzY1bqidTXWICUvrf3GOcria/ZWchC34a3ma/lJWZUyYENzegJaOn+hsGeaGbJ9OTZhCgAa9jFrU3xnvJgeeGw7vZoJ5fsbHnxnv7wUrM9eyq1xUbqmVdMIO7+h1Fdrbn6bhJUVqAeECnnn3F4dIcdU+rRE/orLpLcnIo3/rju6iHG2Cap0sW8a8k3j3OfgTkmNTr3O79uJQkve3ayfrDDEjhIfB7XOvNhR12A6Kcl5n1Gam3+rhNjf7+Gj8aU+nZZQt6mZxVGyJi1zq8vO4adduS7Sjq+d9ITNW7hTSEFB/dE+KWXtAIwisKzCMQs1Q62u6eh9aG7AwfNJmnVkX5ofp8NqIr9ToLJRWkdGax8XlCZoVNflv8/SOsh83gDkDObuszc73B8NSYPvoAd4snqpDtlvDvVypbkl2RGIgP3gAyYK5g0F2Sb5XaSlyEN6Rww61h+cKd7If9r+affWe7AzrrDzr99ft04jWlKbPYhCRWBuFv5XPO09eL2m+BlMeh6EJqNIG8TlTTghNGerVHt3JPkF8vGmi/qHEQuO9e7H9tfMLN4H1D1iPBYfwPc/Hif6XJG5xoaaEX0zY8jGSXe7NjdyMVgQmKr23gpI3bb1urFGcC3nMZbHWGfkfTde1LKmSA38JGv+I9x66gTe8956vX+rM3ZiIiTEnuqlCJWWmVNKylM7eV3U9EByrPBHHB1wKf5+JQlSWpy0d1OEKZ7HW6xuBHEGk/9rpvl8qyvKkaS7GXVeYyciBvmSltZ7fJEUIZDQyzHRdIK/tNYdQp6xOnXVj5aig8Xr0oyupJJLAFa/r9cGQqZ/QHyRn/KvyEunlQbjEfmZb6CbNX7S4Lcmrl+GWKJzRi5mwKDvp9hPYfUN/qsIIxHh3X4fmPDx/U6NQETikaqtmbVhKTtIzDkP9k9+Zz3+gifrdBLvH27N/YP6DmXQGwkvSAB5NUbZ1m+zwfHZo8BDEKet7DTjUgx5NOnuUhIc/QR/ND1o229wDmy4on/Wc4w3Iq5JwjRaUIH+DWHX7dDv8kD75OshQsnQ9N5lemOxpKurgVJXo63BB/nVA6uTlIighblhV5Gj0saRtv8tps6FnLHgMxcRE1gFWarW2v8/xp2GzCXK3OIlpnvxVqF0X7pLTnK8YAQf5JP75oUJsLkPjwJXRZ2xhTYRPvtRH+ul1s/Dkfzd0BL519U62+7vaxMIIfF1f88FZThFZygNShOIriyxazyHDYfMN6kKfHeFqvNnO12fOPeMKWUEnVenRMn5o4p4YH32UbZo8/wSVCifhHkJ3HKyZ1YLSlUu6NaMH6olt5gE7finaIk7cskRMB/n747ZfXuHlL70pjKOKH+lMbghxHpNkUHHwa4XB9LYUw3iHrunUjyvX1xJWgRLD3s538RXy04yXFmBpOMfqH7Sbbk1qyGxg686VOcJFjSeFd01o/8q6ijy1W0WrW8oyU9g4QL5tOSnLOvwaDKL8sij+m9VhuK+FHDymFijLbifV43mCv//GjaLMatLwFn7G2F2ZOjO/iizFFa2UsB2hn902gPQeVFj7KJhWPRhI60nzvGbGOopZgd7JSOXm60NuIOScDsCJLONH6XDeRoOcykspP9Uqf34e24Aa7eIK7b9km2YI9RzESBP8TbXBtNrjzTNcycRGbl9LjEwugLMoYDdpPtVG2RlwKYniBoq6YnnPBwCB75/jiqri6cNjq0j8jReI9hkbiSKGcnLtetQRC/Vi0ukoZxkjpmKhDeE7LHlXVD6K4WWh1nv6sBq7remWkHbakJsyK7FqWXbY/jD9B1l4KTtd0ih1vaZ1T+aF6Z5bLCYVeV6s3XGkhzhfpNjwxUxu39b3wv5Q16IatlMTc4uretwr3gjDozhfBKOkP298BodPteYF50crh0HgoK7CahRpf+KvX/Fz/Zv5mkl/1V5TebAaZ/nFfinffAz/G1uL/tfRKBZfphAU+jB9GYxiZnBTk6lVW3LjVf2ry3z4v5lIgMxSAao+oZRtBDFDdBSNLq+L5t4nZ/43ZWDSbPFvEEn8MpPcI63foaU5sjTTAvEhSFnp4E49ILV/rZmWDjmrqRZTj1DvbQD+xJZiQbKs10jEMje7TkMtQgs0WO8c8oeBnKJUOt/M2WQWnGQ5Pyicmj8TuWDQX8st5w1h0F8/nNo6yIw+RP2RaEMUfcbnZ1q2KkJnQqAB7+debcMt9S/mG+GmGaMXWAatW2Qe4VMh54ArP5AfD86g+Vy3Dsb7Bx3UdOmw41tfvXIEP+Dhzqmgv0rlUYZ9gVRhmr64GKkvGP4ji1NGFV4voJSfK8NHKVsWJlPEkKqTzBgl2apPX2l7rzRp1qF1zbONFi9QUOa1prAd9Hf1KhRlI1of9cwsAcJ0PCtnrzX1iSbIARaB+N2p9Qw0k/QA/Ff0sqCeJU5ZsbW0Ma1sCRAtUMGSRIU3vDGFybvHJOLayCs/4ARi788qNm4AeAsxgldk3Wia0S94Iy0EOLKxwas9tsPHmeFnxVtvIbbsItzD8b7iY5nQMBeVSBUdNDhfIYhK4l4I9Jv99J6RsefDQdHxesx4DBIMQxIT3Gz40H5Ie6XZALP5FnK49CDF5leLBG+5OL9QSDmcj55dO1LUuUtKCR1IoTrNThxfvtQFpm2Qp0wt8MGfHPfZy2hUh94cTJeJlrhWg3XC+3Sz9Z++/yrVK39HDHumyTSmOCJXyjXLarJT9RH35USvoEeq7FjPKB1DREqeKiqhyL58N8cuV9s7d5Qfo4cjJ+gECcTrTnC/ZR58oGOg0Hn2bSW0l5Pz73z4qz5vDQ5MvlxzhRVijoytSR63qHrC4gLOOz8U+K4+Mmx6kiyCnC0mOeteUAHEO4/zd5kHM5LHYk5j+zb330w4er1z3jKTv8FQLZuaQwBsNZ+kVTnLlXtUFt/O4u8+Of0o3yUXFrUKe2/9jpdSxzYaRiqOH+WQcBNeTg3GXpoWl/I0QtkXGSUuMjOXaj5FV6R6YoEE0QtE1zmdI+uNy3zUMJlnLLmN+OTO2jp9DfrWO3TLvudhS8u9KMNwnV42iV3sFMfp+O52WeVfiGFATHe83Wcn4aJQKfBn7prImZOh+Dwn4hhchWQj097gFo8r+QelX7flGw395RZalnco8xLAi5j23JzwdWaZ4GvdHZSs27F0iXYrnteHxi2nnyfl+rlUSV+f2TP+OtuGyNeOwiBOm+sUG1qvYgKcLzNbhn+pj/+oLcTqifSphaGqDjFo26i5EaDIsI3b4LuGtBDca9JyZVATHleHix++rEt1FA8CvFbmc/7Wc2U3+ktQ1Evd/8aTAOGulR702x7X0Cgww+lUYTa6vdnKSQpl1EyR70AExH4S1ICooraQALGxwC6XyXhYnf67v5rIYX8Dze7oFYp6+S99+DzNVQTW4X+toJmBS2zhkl5o0eT6VH9pDR5KrS5vDYYgmNLu2lbsttfO/SfABi/BubmYQWJjclamJJt253VntNEB3yMUdfmibak9c4muRddxzNsYPvpfjg1EhWSJpOZr2ZPsy6W6PFNtqDeW1XbrOZPJL+poer0im7bLlrKBATjFSRiFVnnefP66jGzeaLXlTXWclryW/5fYLcJdRMjaX2560mSvt4o3itLp+OXpkDj3/WMLy0LUi2G4v+sFCrTPVOempRbJFfhPbRjanftFHqbb7CNLGsHkeCKgrLZXVCt4AQjYylD/YTqng8JDTBBo5rxUZc/12O0kOvlebnFYiPrpciBfwH6mb/X1sst+21CZy1ISQA8WGZevFg4YGrFma6U5ZerdbgcvV/3rP0rSeyyqfZlRch1E9nfvTkrrVoLGony1haQ3Lc3+Cn4iCPh+WlbpOIRwZcHftf298BeMSZRzm0vaIxCLMX6+mQ3EYcdlnxK5XlDHBaqtFxZFS+JgdjymiStgvtH11+obpJpQl/+7leb4TJEMtXgBBJVYKeUxft1W7TJo5Cb+Ar+hKjetp2V0cAAgSXcIQUjZLMhjcE0LpKiqmJGNwEEwiHtQL6yhX7deOSUIOnwtHWOrsyGVqLxCd1/rkQbKiZ1/RKSOoV535a+tbsHUr+ce3OGKpy8+G8/1F+r3zs1lPiaCsvkRYJjv+kP+bt4X8/MHdRFVnKKaEUpIqNn0aHx9zdY5eyfAt3UgKikxSoXngtuQtuyq7X6h1j/cwpAfR7TUELkp7/WzHwJG9LFknJBTE9IIib9qdeRAyLTnje2CkwAMEvEuv8WSr9mrOsiKLm3iJwVUmDeGV+HNsA9XnLb7wqTv+veit2cEbQbed0Jbdu3sfy2NVWhkh2Ip1nisHgNqRqPVeyNFfeGva/h3+ombOn6npkR+cBuLs9rcealf17M9Sg/ielI7XYRClX3/Ri3XzfgzJ+nrdxgsVGtbjuFRo33QdZ+B2GtFsK84HrhpVtoTMIAZhq5XAa3m+xlF7HBUDK8jLSrJL/BINWpzP0i5aYf11YKYQJfIF/2NcW7UY3y0aNJglgAnDcgyAr7IoAkjPBrZ6VULkmVvBNGYn0IOorosY1b5h8X11bxKt8b/GsrMa3YUXnz8Q2Oyo7pV+Lwne2dSu/uljzjMzbkMF/C8I+8oQWr5b/AD2GwC4Pzc73X6OdJh67Wtp3RdApejMK9HA+ZpnZGn4KDnrCN5+ppV0Or4lx4Ss5KRNyIwHh65p9k8nVEh82yL4hNmlx4ZFcxEP58yumYEvlpOgoc4Zv/19+I8ChgCGmDAzyWEn2flgi+Qq9YaOXGZZo6hym3MspBWQ3c9LaGwzk0OIXguuazt/+IjePgqEIjly/DeZYKZ6ExSnVaA8LixwPX7YS+WuWNvjYl8Tu8o9fiGlj60HqwWcOwKmhyDDZ1OiZst9OXP2hPQQ9mRtOb4v/r4/Lc3ykgtL3Q6I5aVxRn6isw4Vz73UMu3DH8aubCP2Xq6NPmWfm5TUMde+XfDrdxfp4qR1jrfo6EwWleqP+mexeX4SPpP9IIWzRNVtY+rX/GhYVoWd2qhBsnsZjIlrXviOjQby8aXX3DHV7v13P8H8LK9TBVKqvkv5FjlGtp4gB2xy4b5sKu+NgsGLglYGlik808B1fuZjvq0UVADNjvhwkGyRgkC6FK252y+rkFshBKiVoEdnUqEVkQAr0N/tQ+WUaUeASXcrNy2dDqcuz9dSzG+x90nfw1/s7qtH9kbPkWhWZT6P7umBfesQBNsmFdO1Kp+rxllVSKNt5KOmCRE8ze46Cb8Cl3ErTHkdK4T3A3mlBSGN7O6LeNaGFEj1GtXh2iRQjY6WdfaaQe6eiSWOGil4fjwhCL7ImSstxC++QKThRplANSk4XsPOrjlIkjOiRQp4zgj4oJqRKkPQHhKofBiX+vmMudMxgWM0aMZB44so1aN1ISxSrJt3DuNq2dpqCn591nvUOeobGlZZSsQNFtuUVFs03zkmlbLyTYZ1Ptr2cBcVlbY0FMle/0xzTeOYHlN75Bi+8fB+IEzq679+jQOdqJ0NSaWI+LnDZWAyv8WeP60dsEScjOUIMUpQKIkUBDTt1jNL3+9NfgoEjuf/EoA7+zKb6a7KQwp4ghwtTye86j38i/15MTJpAuVodjAtHASTu3/eoKPnwOBDF0NjHKQNKJlrkD/nao7eoYrjVlAfJ9gI9+39bPJm95YSdSQgZTM2k0qITohyGkc3w7pl07V67KyvBSRTVyk/Y/dVY5fM1XWm9UFRS0E7v7kx1RtsQmxO2IGSoUBhrLCttEnt5bc9QtyRP13URJe+lUlCbSuKCjsENbY4EixQqmJoNVSfTMESHU6RoAlkQbwn7jJFvaOAaND6NNyWkYcZa7KFyUuCWkqu30opJS2D3me0AwcH3N+yMDsDk+33Kn4gVjSwTulipGe/k4TXPe7GEYrOFamA99Y7QsWJZcCgo7cshcDEQx/fGefFbee7v2Z8XaFWIrYPNScI4N+ZIjohhk10L6zcd+DN9NtLxYoq88f+wY2Whgd9pfl3CibKKRKUTnQEkiwrABlQH9CIKDRbYTVLh0f0KbsT/oNa2AKll0MDkiEgg06kPcww1Tvo7klfSH+6lkWvOZalaiDUQxI8rYRnVMxT62J8O9YAd8KgiatcV2pXKscfbalFtqv4G6Jx5vvanlFUB1Va2q6Kq5sVfx0etk08os7cydTh+4gDT44YpTXPj4Udy3s9CsmRlWnH2lW55LNYHtAaMsxBYHgTswjy7QoBkh9qvGHIx+0EGo8aQgwAvP9Vl8r4e+XDDyGvM7OOojh8f+Ui79+7nJNmFAubtgHiDFicTYIgVpXNjw/sS3Av7lY2EQcevSwkXWe9bH5EQAS+7AK+4tD/ym0FLRhh2GNocnSgsTb6lcgS1+hHN8HIUXAdIMI3XUb64g/dPLbii18y3TJTFpfHtDPLgERXE2S/boJr+g0nHnRjNXY+669KeYvs4rr6aujyEc++SYU97CQnua2Prgsyl6UkVbZNdftgQi/rMdJVJeamnsa0SVp2dk/x86lJlXfIEHMXKsDAadLD7vT1I/w1x1oA4iNNm7Qbh2cF/kjysmKyWqPQ+mSfjJThOM2WpgiBLIgWLik5M4zaaAUbW3eLStXUhD49gjiBzk3Ir2s+n3wcgGZd+aIR33qw4GRSe39Qe+1f1tO/e6cSFKc8YuneNQordfoHPk77Mv0kKF4f0AspnBSE61LQlA2p7oLr7xIGufaHwPay6isPg/zdzIZ7IMNKNrqCKtTgty7XjT730weIV9gmPnCDnvRH2shT8jFPhhPetl+xdL7vvRV/YObSCPrJHM/v1kccSB3zljvUW0l0kXrzchfv3A96oyGeJ2snHgrwzFRe6aSndsb9bgJxFvf0hrLPI2GBrZam9luzonjlmRNYWChv1ovWti+burIpSV+EfFFBu0lX0W2HzT9rI9Z3ulsm1dYjm781GYkCmBAKpflxQ/iHiZ0yoqlXl+KvE5eakiV6S+KfUJP8XSzGP2DqMjk8SIfAwkQVUK+acJWefQ3F/bsqBRSYV2MRG4Uy86m3b92oI3ECzkuxGDNQI7nHg7VtszqncEjnW8HdCVmD8yB1kX5FNldqFkL+WjsHJnf3ERgL6shMyRBeBT6dZobpRhF8Ldh8pelBWkIUAzFcP5A6X8D0qqaMAzCin4MYDpAnChqvaFpg6GtsCg6qazyLXJdmrOvTKH1gkmeqEEBwMqBGS84jEvi1DpGJdi+8MUtEH0ssUm1rsNfOmWzCWJPmR8LwGBnuqbv2KmgwyX1p7DWuG+IgsMWjlXkxckeJZsQVPfbEec6CjHuuffUjP8rJ9F1CQzyZM7CGuHrevecZZLXu9MnnUiretXO/at60fa2lBnBA1YGtZ6Rk/UYh1EwyZV131YBQ8TKKDZkatVUz4NMcLgGbh+VcPpRSyhSTWOAvpREFNO7eEiWLusMwDAQ42+8KrhyaGN9TObuMZ1rGRmUaiHzhv1N/e6ySbvyENT0CvaQtEM/MzN+7N/d0w9JSYoEr1LcrP+6zgKQUg7s/gw2IzI1zmRd6/RXZuR/N1rol61vq/Q4SxIneud8pYp8A9xP/4SWu3JmoPdhjqe3X2le7OTs/eFLx/6W4H1PSZpr6KkZveAIkwsMJ0HwkwVq6oqga1ihv1h9l7nEMUs9yftpa0TOOMr6iDJ7jEcpNMadTzbI6Y8lQCx6MA1nAhPRT/ksmRoYD1mqOpltsNAGWhmLSatrXduZn2X9thSBitBtwBm9NiWRaLhhBOc3+o6lPcm36nyS/qffuekIEURGFaWWvGmev55FGBJJd+hloiw5HjL12t9jNAlxn1MYF668r6Oq3nS2IVIqDcH18LWjf2tJCkNOQWDGYJl/nvGxeUx3cuG7wUr2uvgfvf8Kjn6yymVojOgzITB77ZrBkK0x+D3xUVsJw2PFo4283nAQJ0+jPIjNoyOLbiXAsZFc6tOBIWwHEAvLaWNDvlW/DpFcTe/Hw+Uyn9f1uJDD32krL1cIFaAcCN63aCz/qjuZ9DS0hQjCwosLJB4izP8GWf16pihtNAvtm4ucHGDoO//5qeLlqh9+NEZUBMmXka9Hc19dyVpU4csZwpz0F8Lg1ybtpenBgUA/QFJZOaekRftySo9Cu2iysCwK9VtnTnPe8/K+Y5kHop1n3OAi/T+SX1Irlqaj3NIYvcG+97umbfPiruscPCmEcNb2nBBfzOAJn559fm7dNh96bj4SOvjwDhs+fFLDWM7nOAl/rZCZhi/3g80K1uUwmpzfMEKHOKhWQQJ/AqxwCA5Yf/4qnTLtr/jPz6oox/VFFCnBvmEWIooe3HKS0u/5+XGyRZPGxJ9vlNPLeLqg4zSaDcdEKLxg9Bcq/VrkMifbLk3YPZckLau58I0KTzGCc7BvDqaHWrU8w8ODviACdsvQNNsCo2zh38ic6kuAajvzdv/GuMxx7hl5VDqlzrwxyAeqwxUepIUAfAJSakwZmuUWGw7sh5yAQkm8pnqYBfTAHxlJJyIGspRS/3CaFR0wGXx67Nf7jd6rEuYvBJr+TTLARB1Wxg/jRlJiPhRDoFI3nvqgIkrvVQ3iiOEYDEn4l71rrDQzQkFmmTHImHrsSVv8hOb3y/idjzkVojkUCCJJQPtSqI1xqOxrSweehBT11OK6uvirnHtFwZo35HV3uTn9osvL6Lxo1o3corwpDXM2oy2BNiNvsDd4wYf+y0LBssTBWCtA0PaS45oZQ1KxE+Z1MTQ/7o5kD77gj2jx+jOFdx6h8mgEFjhFAvKDXPoOW162LK8Msp57E+anEX7QsNDN5nu7o9DsJIrHdbKh6lcsDGf9oBxdx06D4B8HUCIbpoBNPJMdKDyhfwy+VT5VvgtTfqJfsrmL18Lp97RTG4yMFp/ztmlg+jbXmi6tJKk5MM0QhQvXczs6QKGTx8QMBF5Zk9NaqTJK1MS7YOvnPWzg3LB+/W66b1CmRZgUivr34fQXKcGSgxKe0FJSZ0R5QXBAVxdOHP65HFEkXvBED8Kx+os0d5/tdZCZwYUXzRK+ztGGwi5V+XfPtfx9YgO9+VFvHaXcgqVah6HMhoaOSfE+1BHzZOqL06KX4SjDV68JdoTnsBMbGRXJ9SrbMNaAi8ULasXpgZ3W2xpyraLD18O/UTydcDKpMK7s+8W6tZfAnYB+qF7uSW7SWoZoBSVvv1tWnlxMn5uw1uDFa8ice2Yh3UPxN21bFW5KgZYP+aRRr5UWFxqXDchogRf8KgcocoZIg/drrFukCaaDAgV1HivXVEDDNQGjO5qI1TS7flbykHzpw+V7gkRmjPP2+3UqOtnWSPocRQ/SmWVA1oBKEIMxsFHzvtRT52wQksjiEvG/6VrbswKBsXJEWg5jsfL65PsFmljHmdGHd/r3098zWucUUKtOUti2DCh6PEGiVnWToC9AZBf4e/pGdur+voK+uMeqNWWu1X3B9mv2LNHmw3GuikH5HGkBmFgI8/R/z77/gOpu5p2HfnMEb73xryMah8mTP7Af4vfNAm0saf3v0xUkLjA9mmcAzKGFBhtIubIUWhUlBi5V+aQcNbz0qEpIg6ePxWYBXqlxrPbengmbeJKzwNhSplG/Zgbw28RPtxjUuRfQYL1itTYf5DMLUw3q7QE3HBrdOtfJpclLGYxEDp+/x+Hdm89of04qHKmvdxOTRMDnFMQF5Jjna0ch1/EC+9+ndtyBfFx2EidkVfiwqXiaFjF0uY3Kj0L7/rdC+zqaZV7mXWsKQvPC+MRs2jCLVlDnmOqzF8T/2+dSZbiZnWOELeGAXMXwKr0Xgb6YyTj86ocNtf/GY/CjrB1pf31gKfwJeV4QUbjy3h2N5b8E1CeH1+XQwWfqisIgBdGO64z9ivw5X7zOqAxuF8ABCcAxQrXb/vtc+SoJmLRHumE5qpIblmZDlZ3/sipYP+OTxoo2eMmhbccFoUf1DP4zaDbiM2yrwYchnbIYRoww51GDc/J/K5NJBCcpGtqSdo7IqEteF8bpFBAIYmUC58J6GIWw4OonvOFQrEpRZFrOzTyb8SWbIiSeuG6RcXIo9LaC/uQf2zKwN348VjH2htNUYiPMJrDzfN/EIba9Ua01rgJfT2mIs1AOZBu6/ddcM4w9oWmzj6h+w+czY7JUPZ88edDE172rrNd6zKuXtRSRLM93RN4oSda07AGceu65QAeVtiFHhOR5GfOF/8PtpIJRufYZGHcNe1m+0D70JKJgRgzGCaohdDddZ0s+w5FEbZurpo6nXZ8JgyFDF86p6J7c8MZU7RH+jUVmJI7m0oxga7qMJWu01dFvgrS9aqm6Y9OJ1SmHBCnZMmayBLLB1G8/+Bl/2oxxF2Jpm1KGTUCzN5q/dp4SodtndX5i2M6C7ndvw9/werOxw4ZWYfkDJpcLH31ZgJBQFv//Ztns6CUmqpocmtV7gaBfHGRuNSCZxyZXHbYZZTzGA3yAoRBcGd2RDCh78QNJMb73/c413zhcquIM808fs+Daxowo3bBu0iiyxs5l9tFrXYYVQ5Ey4d+U4FV/Iwh/nJgMCfxl4od0bRRdtx3vncQad7bhoVABHEuCdP4ARAvpO/s1HZQ/Pvw1dt7PM9EXwsCw9DwPSo+omepXxCK+pg5Zm01n3iSOhNmP7+sBHgFGkZqcKrzJEWdfPoDqFPzzBm11E4Y5DZ0KMIpP+BjKXzVGr/Mjo0NzhD2Gj3i9T9s0pB/85/peOTa1GIOLK7hjs70nBejQadBa6ZG9wVIsWQe9+KDK6Qd6qLRJSGGvgUsOvO+BAUwdxZJtyFQoUPUV+oxIAL9GmVAuoO6QYx/+AlhL6MQ1ltpNubQb7Xm82Al85yMSMnaxhFGy9+Ifr9BoSva+AEezGCm4NKIpPzgD1uaJym60PyVjnZFR8Pm9FHKjYe/+IhiIOK/3/aaa0hh/U2jcSww5gDlt2rcvREQzUgu56c81JrwSPCDysMsv5hrLOh0RuFpgVFnH1cA7g+MpJgtI0ko685DiERtUV2aKjjSFzcz6HR3d9N3s5Ge8SAr1bGWLkwaK7PbDjSeyUz78ERTUHk6Jz0RrDJrQ1NEfVwOoUVaryvQ4iPDkJsw1cG+dQxHrBSecHdI5yMRguDp9EZOlldACnlmHOTnIsvP3m1zYTP2du9FdK/U4luVuIDSIPUTg4bW2I16THRC9E8s3LEiz0AdHLNXzSF1RF8VAF4DIEk/puT6DJsnEJNlgeK28y5OBypyv1pllEPF0ZtBqfzs9MLHO1O1clwTHJwp4HsurKxQ/w5zypOzZEzKWQGkZY7bWG/d0aztWSPptv4MJYvyGfjSPao7z+bKX2/mQQq5SKHKjVDnZ9bklLV+aAoTVuQo/m4omuX5gv5dIHdRNwns2/JmZxbA8d6Dq/RT0eBTyqjAcf2aM/iJjlMtS2LK+BA+6XTPF6L0fQQ5gv6mWAKevkedmAXotiWMUFf8mxTD3xYChe8+WzsVbaMbprL9ov2DEzGMmvC45HoD89S6bB+MH+jmFoTx3VR6bBuMdnrmcU+TT4UEFJwJ0rDZkDYkQqN9wwp98qovdT06SGEm0iPArq85H8dlFMABEmHcB7DaJBEUsCFjgc4Df/yvGsJVGGSLxM4HEiCv2Nh1NQR7qljWxwR56aPGGWBtCaLqGP9FEUm5cYbvFK638Uy6X/UkvZKJ1tHuRP71kmv9B5sjFDkvHXPliRVUVpafmQWufr92ek/6uldP+Rm1yPF3u+uHa500LbQdr2ietpW/6qyF16e3W7Fn9DYEZ/kAeJH3RT5K/7EMm1cgV3xdk6+ffjeMt5UUGyWbQu4XwewYe12F1+dlsBKTSCjqNQuz+cDVK+3f4vas65RdR0kL+fjr6cwVdB1F96bRin6H8gqbnZM/6g5UguO1oESv5L2RBV+EcJjUtS65yGmbhqmiz3+b2Hz3GMqWASD90t7+JhSFu/CloUyTIJHQZtYuhvG707ad0is7l8oMt0qRsAB0X2la0ynr2eVFmU33Yss+l7R2QwS2dCKZUifTWFXDuIB2qvo/rdn+wN/8wy9UvRNpJ9iHDJgZ4zEZiTEo8NqvZL9sK3IWnv85USbJTV3KY7NgpRyjxkbKvVLW8ImGZXTU/8yYXg5I0UKOKpC87+9MIi0apeol/4QRdTwITPU25Usn0LORHjS40EKqSRzCMeWGf7yJu6zGXoRzMrIWIpTr8OUdUTanj1ttcCYhNbc9imsqdaNjK+EB2n0A3PaquyzvRZPIZFuZo9VlUxbv6CdPZljt2sSM+aNYb+/0r/EuueKLCc2Q5Rtr9u4cKzpjkc3ejIdVoMtadwlHXJe3O+bB4/Q0lX3ZyuH5X/jKnCg39cgm6KSjPMP4mz/F3HWA10Uyq/K+bvhw0IYoJCJgCYTpOgPqS/4WnSLfw9pFdAA50rtYVZMFJAxlVJlokzfgqP4VXfbdzeC5IfaNjXPzQ+IyrsxQoBoJRfekKanTvTEySJqpdlsteiFdsgDMhKTV9AkhBXx6SE7r8K7NyH/JypbCF+JODT1Uy3/xBHYzWQse8as17KjwUyKXxBNRkyEWOb3oiJoJMO/vBPYK2z6XyfnEFos1XaxWvtilqGvuwZQuhQP1lxleAqQyEW23n04wnKPwXth+nRzaB8Nh4JciANmV+m43/yI2jJKUkz1Mb4218DubP0Ts7LWlMDuLEAZLD9zHSL1615QuIspwqXPEq9kve/uqOzw1Z22rldhqvLOwz6uoeQA3wnsuyqob7eQ/cSpncWlwOIydmiaIcen6gC4ALpeCctVLIvUevZ+5C3TQeF5qV4p+St5/Aw/H7ILDu5lzWrkufR3DsmR0mR9QCrT2aA4RcriK47oxFn9uYVgvhQLejbAkhr2r4CDOXl1I2k++5vY3WoNmKqHKTr8ToLG4cKqWty7yAxG9PI1W30t2FOPjaU8ql4yZ/suR7oLX85MMS/rhOtHNu5msKB9/7oBJ5LNGNoJ/7gXdLQtdh4r196rztkF2zX6uA2gh/JKGXYX1HcMG4iban0KqPHuIQc3AGXk/Qz9YUMY7XjO8FQEm3hay0F5Hj+FFNrjdfcIO56EagPU5KBbYRnfvQo+J9AKcIgUBpVeR6E7dTNmRU0vJFF2gvxyCW/qre/p2G/YZpUflFB8OeFRmRS7f/dQ36TH7kdmPwvSLisMnpWs3ImFNEm09J+Wz07xw18gAZhKxlyDjqKK8mj1KAxcF0lT6vC63PwXHcSWS3RLb7FLn+k/ytRM9fZRv961pC5DuomaQnBQOKbXauroGWhiRwhQSZwnkyzfiWC9UvbOaDrhxMaWDdwaVKyeTNcbelI/jeEVzo55RqZwaqXyVjxFgUhJrNoi1kEfBf+/QI9kWtrAy5XSV8HH1tn8a7xpSws7jM3UF/8mzkRRx/W+QNMGcfG7T4JylwB9XSLf+Q1uaBBGW/2SUNMzYnPhBB2KG1v5hKGYOHW7/ViFchzcDEPmyY8kbQ49PXEeF/2G0evLlMGFyX6+HbgeART8ykfbc1+ckgoP0I+8SokcT+poM+cLimkjxO68peB2ejGvvotVxYkly3i8jMO51m12lguU0SDUZKovdJr4bknJyPEzRPSjh1f0trA4rvopY7Im8IM1MEfPo4qb6glNtfIYduf/lyRM5YLQpez7vLjBkZ3EgReP2njYgA/ZT4ucC5NYyfq4fxp0eD+2Cu/NflbEo26qoVIjtBcK4qEn8lK8F/v7M7DZ/sYCw7l/6GYUW+R4g1pggoPZMm97M0l76J9m5T/Ub4LtDAsQHkwDHiC94J9rrk/Lmhw9Fl+MuBBpeyenBtPAwpvBd/I+vGXHpx+WZjiGD8yomzY4EHVaycSr60eRoC9VpjUI7Ql+SfYQk5AhR31GLb+WRYfxfcDRUPDYjrbOSgCFM/mu4eTfDXD2kgXkQgVHxbt2lT/3VMkAua2Mvn1NP3P16PChDNfZM5ToIjzI2YmTOPqJ0vSZVjG5d6bLNky5dpwyYNZ/0roceJQ0oXUb742aA5n1Hd5oIs+STAmg5HubwU9/4mV3Yb3t0zhW2QuevGnBgN4gmIxJCTqrN3u173KsgMwc4AB4TkMQg8vsYI++uUjwrIZ2iiGr5LzTX00CCiIq8jg8vK4KKJYOA/22XSdV2prSQPGrk/8/XCbPhrqNUCkuTeneZS2DfDYkz9Xw9/FWiEmV/WCoLdf2NWmXUemmSMSls1hthafEc/v6iZWqyanMjoeLj3RFeBaI2qVs+vTsjjl9ZHUuIhCZYYSIKY2cxdx0nUfc5R/Xnv7sb9eNvyS7NqHXT/fD5b/qfAu57PsitJH1wINeyKN88Uoc8v9ZADG6ZvnuB4h5HuQO7tPERsGGdDfedn0eQW/tRWOPB31Q76QALIHn1UV0PVxzW4+y7odp/rvzI5hwd3jBhgAZJmrzlPNrMCzWwK6jGfhcqO9FI66Ulwebdno8lR24GHURfdONZUuE1gCP4buFlkRdLMsdz8qCtVvYVfkkaNDs1Z7PNv9omZufmklFHrxXAHbf7cBh/OG0Z5WMnR3JPphimM9nxzsoBiUzaBm3hcN2pTJ0zPifPN/ZmsPjiLR0tLNAHlYnl5anEulr88trO/wv8O/9jJgJ6MRD4k2rRgGUX63HBU+iKhEFdchL4y7Jk6NI9anAY+Ivp1o6N7rX9V6iTIPys6o/ILe6aRcTcqAuU4U6ViM/4pvVl962aHaHGMLHwcnHRESjAVBJ71ryqBAcBWTHUAcgvy4PQ0bKJVxv9KU/SJFphi+5ibz+oP706k1AAhYkvtheR5h7n+UsjM3VZ/Wdyd8y4tnmhGFSb29NmkHcCEjpLHFKOgzr9Sw113lLvp1zabKHxu/F1b5juxAmRBHdpGKmTPgoNah46kMKqq+75JVidtvvvftE1CxOPlfb/XoCnvcSlogA6hTgmnY0CO34chMD6OaFZFzr8ZY8wurDTmbSn3Pm/fp0ZfhXij2KeYaHNwNpj/faqF9MQzF3V58NA7aj+XfTq16aIdsgHfVFASXASE6M/b6vzBenA6QNBfP2adtQ54hg0qDb1k3g9jx5hR3IwTM6NbdMNLC8sc9TlPgYvhNiqOhxPGwG1LwU83QMRI6nVN358XDlvZoZfnXPmn6B1zEf2K3EmizjRpZm9pOCmkLPm1fcEfbGyaf4ghjV1rgv+UOrLt4MuisG6rGJr/aNm6bsX7DdhHPD3/HAvtCGoOXLzuHYNLuAp7jBp0u4Gd5wH9sWEul4fUbMSMev4myc/Ua1myJANYbsB7LyZzQBpb4+UO4KVYHW5a9qnQBP3NmW0Tzm6LH5SCd3+d0lWyoaXq50XvCiKuoOIiYqmKgebBDGerGSGecBdi5dLlRUnneSi90oqkDX8TGjDRys6/6QPKE8i0fm3aUpaCiRLrd2Tkj82u6/7B6Sr35rg/Imc3BpqdBRnNMBU87w8cc7FVsGb9KN0jkeZE0GUmKjhs+Lt9pziiMQqd0RiroTnxOT3zZXLWDA83Dd80p3M3TiwieQ6Ho80if9PZ0PxNDuVo/bCW38n8wF6OL+muGBIQC4h0nxERfUw160Elqj4CwkYjA1o0Lqd/eLb4/gJn3NrIVdCU6zbAEpFfy1Q9nfsBiZisIqcDxY/lfZeDzC5kJLVCN8yLfCke63jYce1GMt4G9nmcmf8ZaF7/NV4cA8avKDq3Gf47AiWOjS8eXQRZ57l3xx/aegNaUmAvGHZhp+UgcDxm+pPnmCMpqlHMzxmXf8WoOkjbEdDHMqf1i50WB5pC4fHDZwo/iG0iPDLtCDEHVvYihSykuWL36KgZBvDFrgRyT7sFG3TWiVlp9I328fG/FAum3ubyYn6FSZsgPHsfw1fG++tzlUtPLbnHgvz+evK6CvNY81/qMGAbNs0bzRaT2n7fDm3TcrgfEsjECBbLqfjXZcgXCtKEY5lwUDxKB4QABzkUTYLC7hIvOuBR8m+oppuDCNoq4YyJxR6ulGMpsPDaABgOInSlXdEQmz60binaBuxV3QWf+UAZyemrPZWssBeKHOatpx0c3NXr36hETrFFV79ykkv1/uUpVvbEOb0qdhnTN/+ey4xsPevdgOM1As5aXyRYVCJoZY7VH1pq4oJgrte02bCLdt28Ke2edhwsED1estPScuX1Ifs+pPntgjnPE+jl0Pxj6C10CT3ONeYR4Zk9bkLj9X8uij1EhXX3OvFQ6WYYsG/+7fn80qQe+ysUxrAXjoEmTwVeXI1ImLoCW0USL8qVm8ZjZu3Cs/NCQZyWd8m8bBu0p1h7aNKxiv3w5+EZg3WSr2LfI2vmj1SSP1OBu3Lp6fOg1daS7NH8DZUSrRqO0/bwtT5VzCWfrQFNblz59SP1uwS+f0jmXgHMQu+ndNjhhmUA4Zjm/WHravSi8L+oOlvXenh7KXDYXDwY80jJmRFrNXEvgSjjme/zzYJitaX5MupH1pgfImpnFViI6o5ZwDxhkW9UN35PvQkqMo9vv889eRKl9DGFy1qDb6YIL+UHOyit7zlVp458OWYGTmlLh1UyJTZCBuyVI6zHyJ29jEIVO3IUKHeP5XAGK658b/iLj0Y6bmIDpkOI5jUDGZ6VKsiNNAztl3H8P9ugZSqC7YKlC2qztfDZbsjTOPuX0mGxAWzRvxw1C3zYW5RSa1NmHwO6/BQ+tP4iWp2IlJudsL+j4R6+RC7+rn1TeO49oUqoGDi1fTMN5/DsR7O+JNQn4FbESYjXutqer/6N1DNfgNQ6fFJf9JftSZBfOQA2wNcASn/V3yzdHteWP4HGMdSY2JMeL42CgDSGxeTNWwuMPIHbIPkTCpTSDtRcR6iJWJ+EQAg0meZCu9cXqKwwhSO0fI6o3ItdYeF9e/Xb9/mkkMfRgklsMLT/8K2RsShEKPNDVJ1b0Sfd5rtTipxnYdMPH3XXNYS9HL+KkefohqdFOMl/1Y9Utp/LkZrD+HyVjKb78sU24dqH2Nw9wGdzUesz/eFEn4bzehpc7FhGFRW6v8bUYnlgHRQii9m2zhvR+WGvpiSYcNcSv/46Ng09EwJzfzRvXRkNexb2aPLzEUll75np6nC6MdJOLZFcN36nfAxaqyf5tyOf7TQLoQjdFyjqmzo82caHtcL8PDMZsJ98SIYrHiVfqTxHMw6tewBYqOIOr7Sq3n8Fdd7lG2pjz6aeGtAXXz0Ibw+eCkakwX7/3ffvo5cNKGzIRespV25Nv5zgydeL/X2FwK6gwe6UDo9/8d+7F1mFw01V07kkBH/f7Uf9uXPsSugMGQMuCkDbZeyw/XscIVXUIhWn1yVi9Is4SPhe46BBypeArm5k/UU384QWMrHe96Z+IbJUTOVyI5pXWLS/gzLgnt90eb4fBmjs5ISKOz894QaYcmu8tifDUtplyHidZeQoimHnr4xUKX3gfwS2qbo1JjyN1qafyj0qs1X2S7YoqCnAJZ+dxjOSwIJvF5YiTdRUWop4Jbk9EyuqGXYkf7WsCT9RkU35FD10QLbh0ME/kPUpfgv0MwAawpmWEIEYE+X5w/dR8pJqwFMnQ/DVXyq/bmPP+vsvnEON85dIJK4FbaXyLkZuGgufjyI+s7z1iDqaPv54219SLmhJDN9aaBoI5Y0x5s9CM08VQEHxM10q2MI73Lsub2Oiw2GOSzej+BspfoFDaTiNANC2rhBMUWtZ7po28kZ0i8Ncv9s1dZbM9AAtQoXYlV9idrw/p2vci300heAbc9R6Tv/Z1rKR7DciLKsiVVmlmg2Tf6jobaasU4ERmniy8AL789jvgf+whDphPN/ziLp1BSuT7+8BUo77K9BD0QNZd/+GEmGf5cn7QYNcVOnKH3VQI8JDm0lJxxdDp8japOMzCJNXl792vyND/WsZ/3Lgj6jFvzT0RgSDq1GUsRcTAcgsoVdd3DcDKq33RfmgdLAdE8YS+sbyb7xsgy9IS3JO9D2M+3vJXxE7bP3bmF/o5sSqZCPuhA6ATNqhiNCCpDODhTX5hTut/mPiVmx3oJmcolYg3Az2mOvx6eHOnNWP1eAA7kuAbeFbsI9C7Du0qsj4D9UQYYYLsO6DeyMqZDx73qdQqZcLFT3Nh+Idx8ogdNiWLyQm4V9OR0cjQVsN7AndIjuKah4tiTRcdfIG7NhHqPl8D4Hm0CDTeBRGsl2L93xT8tfsKhNASiTdJTbt7Mg80B4xjfQh/tp0ZS/UThGfIy3m5IvR/bu46NMELanygUtkhTBTONbPMpHP0D/mwgi106Pc0ZDOdYY4xbLfOyKnnfBudkbxlJZsQ/GG+YgicCSYM4jgRv4JmG4riJhmyMVyk/hX5EUA33B9i8R/Kk8O/H83r3wf0lzUm38ryMEyRkubYTn+uOYbA4AGmt8y/jT2mM+VYxSm5nd5+lFBH0LzV83oMQ6iPtTPg1ci1Lnc0hPng2YfOqSipc80REsbmRo7NgtThw3P62txpHhqAPvPzc9EDYwzGeAVLdvq0eoBKQPulgUEZM6bDFnAS23LyaqoWQ8OnBrtXFZid5T1k7k7hWogV8i14MrUpA9ytIKMFUQQexdsVOeWazg2c41SwGvFKb+mJ8e5QDqfBgF3L0cFM7jVd3cRQ5Qn2a3cXRWNMS6k5CUFXvdo89fbLk5UQOXdco5uIAazb6ZLfzoiOE5QMyCOcwEbRNh5Sa1STFeHREDKL8z6kBpzaoNvVgrYVwX9MWUJ1R1RNz0UMu5zsAaP8UpDwYg8yoLjyUiM5lY+ws1x/840I6tAUxoOIX4QSJ9f4faZdPV7f0uMnEyd/u5IXAEcZF0R3N6KRaDhil43Ip2X+bsK4M1m41DSiTSEfk2HZ4irzObfzSksI+lg7pAjbO5ZiH7NvRqIj5TmOw0U8BZ+7oVaYuLxNpH2Q3DQCxz92elrw9iLYzZXODniJ1/CkSiinK8iB2SzDURr/7k3MZjs6WX9+4+R3e3+rMxd/LpRG8j4yh7X1DBSRrEuHPvaWy9ZxbWUU0pzp4bujL/tBE8F7pvZJwzF3USsAT1fUEnvzNDbqhQb18HQ2TVSIopWyrtN5sBACghDwAt1qdxv54Es31qdHBSR19aXQA0thKrggI7fwUgqymb31xwM4JzCuvvlHDI2enAtX5pvzkdi61mzeg+Il7BWVYvs/o0JuyVx0woeZUcL/el/d23kmNSdQ8zPW/LTLvIPN0gIdp1CD9ThJEhhF/cOl7Qd0bQMg6KWJLvc58p9pKkWTUcbmcHDJNl/69fX+aiQ9BLKeNOQxE4XQS6ixrTJKMgIGeuL+nVdwj6j9/jjld//MAUafdjpemAxYnbUcXEg1k7iFFxroG+ODkudrp6FT05d0KseLcIiOBM5Ck4LsKAO3l1+h67LnJoTSEjgVEJmfPgyyED3uM68scKi/uanJ9iimeoa1mc+5QHXzPVJLdrgLlu9t9H/2ruu5laVLf1rdtXMw3GRwyNJCIECAhR4IwkQUURJv366kby3Fa7tc7a9a+6dUVllddOJtb4VgaYQRUM/i1yYavoKcylztOuB1w3PXlD26KrBcW94IeGSmnkEq5Zws+Z4r67KUwqNKLxGPMoZxlkRtD9yz/pSHSkciGyUzKkCmPFHaTxgqUjiSgHR9ZIfSYhI5xNRyDrJag/rvHG5kNIVXXe4THS0TdOQpUPCMJBSSXxhHTKHY3k9jpOZhJQgVInjIBDhQ6LxiBeAb36cHscc7cK4Ny7Gbut4DlMgxlIQVDVcZ+eWx8+qGVAbY8GeTawS/KUwVjheOC+7vPPjvlY3MPqOh41hW1et5+6alCTJsoGlO5Yao43nW4eFEa2439AdosaSDGIt9XR0Rt14pLtA7nl/Q22mo7k1liQYicpZLoYyLvE+iLZ7k0VJKeZh6JntLFeG9zyl2/GuiFIqcmdOuFBEQdBoZ1KayNQzmMMq6atjyUyOaEVOi6FnCiRowfZHGEGE2cE160QWTrzqzdmcVcqNp8G8AI+yuMOCEF5IJBlYT3rY4RdBya2NFVMH5pfrrawcmgysZyST1UKzF93ZaGIZuBmsNvhGzkQvOV7hDnPkgNHAppMsjCCd4fH9uc0m4UQvjOO0F3ZM4aTWORj1idwlwoUyippTzdqXURRGoWbneVBPrUbGdq/zPHyWdNS3I0oH4XLe66oJ/Qk0MSJ4Xw1v7KTdXtwnjQ+0vLYRPazXVK+S1wJwUODjYpx2KgJ63aswuFRAaDhGtFkF3J3h1Tm8h2zM2gXrXypKdJzzrItkvjP1xv1aMOG995p3oc0kopVea3K7ndqLSVtlXT43OxFLHC4NGYM5u3174g3yIBqYPF+OOjY3eEMQOe44bCc3grbRPafAePPI+bzx+z4bzYJM6bnpihE6EO9unY0QR/Vi6TuDg6de6Drf+iXMhtRO3DfwbdW8mI78qm1H2J4DdIrNTKmKeZllupXbquoyS8Ec/FQiFWA8Hi8MWT9W4wm5b9etbo1UdcUh5rKh9KkpBt4oXXFdfJxCU0AFTb5ZJCIHWCMImSJQmtt0Kx6mu6tNxlIkr+hhrkonQd0v9L47mkJS2sSpMJiWZht8TRhRDDnDm8X46AasqAF9M6WlQ0uRp4wnYgF67mgltzymM3a8JEwBxBBu1hLJVg9hsuTIYWWA41P47lz+sM8yuY54AwkKrqhmUM9vLd5JM97oMs4+pgGFnj1mySiDGt5GTgl3R+MZVAPwGDaSO51XPnuOZF0qTuDMWwi0lhK7tk0QJbCqCguZPRXtG0WGGJHGSTgrm+BMjKs9LZfrMmv3MsYtRzNpKdri4uCQ2XmsEvoqWsOLXMYkK+FNcd3CyangOGgMXhof9tu6WthlkOAON+z6t1k1vLfVxIlkK/I2osQGwlsreMdOytIdz5AI2h0qI6ug2EuKF3JAQ/D1eu/SRY0egM9zpGsEX1ljxfPXks3Zirp1WQ4Rkb2RWk0YZ+MDVVKbI2dy6wkhAS3PSVbAWzR7ZOfuIjkec+CDmv5Bm804BZlKk0zsQjujxq6+Qqp45jhHm26GzZ+bwl0iSThwZbksR2ZLjeOxzFZorcLrQsSR7pg9WEfKCcTuPKLb1VmsIi+ZZsAzmruRrS380HM7ZDzIuiArLVMOr+B0qkU7j7C9u9AyYTPfZhRu4StdsOQ9h0PRTOHzXTtjvjTtarLRN91yNVdLPBPD5UnXAauhZNgxHSAbioWKKqJlj6c3jbACgRY3C9eIfOI5a8eIedbNzHWtSMm5TSyY+M/Kxabop26WG7oxjMXF41MaYLMWIK/3CEYi5vLacDbJWi5mobDIfQi8mXrShE2qextbRSp1g6/xWYe4saVfsJdUFt2UdFpqdWedVKHnbEKpVtPScyXesRJFiMeKyvC7DoFmOqOYYUMT27C2WyKwklmbFjssxhio5PbiqkpifT2sTwlteIVSWMCnVEemgRQdKdsHiZwfMnPu9abpC4w7NqcjwoplO0zTNjH3HEKsiLniq5PDmAtIA7ooeuXAvVkBsxYivGdkvGCZrUJLShj2nL42dsM+r3qz2081jDZmC7SmRkWVHJJwPi8IYDHmlqZbLlkI2Z7aejW8Zrnj9zu3OFjZVpXGsuj5K/7Y2BZMz68Kwelm3HTGz8NIGvSKClwwMEe1P0sJvpntNRk9L+boWZjb1lmZeGTfeNoWVWTLmi/krUbN5FnI7/ZnvcQ7vAltIeaWORaNJxgnpecZ6kEx5BdqZ6v+fE6Z1aESBkmWLM1qsWa45m9NMaN2V9MZah9Ve1uWzarf43Ukn9goF7djP+Etgp1OiN6JEAUT8O0cz3p4PyJaGL2v8jjXkYTvYkvJ2stmKNayukRNRfEjbYmJh41EtgZZbXelFl/kbuoM1wnBqWLYbL12dgc5ZcwpSdCW5xzyuBpNJqgozNLzWFMzP8DJdjK8V4gZy7s+SoC3cLaGt6FW3aLPg8nZ4un5Uhxur9xNSUqKzrucxHBTifuqtxNntVTY+DDbrfWQr1Ofp905dPtHaEPhpLAii0HB8mPJWcm4X5Ojsk7g5uggNlImh3abSHVqTZVwox+ZcK2exkpJ97ZpkqSttrq0DUWdyARihZAn7lxLx1lUR7niB6pVl44p9RYur6ORks21gEgSpRhllmcZIUYrSyoKI7zdElU90W25RyeMiogy2c6RxooAt0L4jQVVSnpd0pecBL08IJeCwg36h5fAF9qza9vpoBrBF0peGA9fYHe4oS9E2dAXsuHaly+4S1/gB/JDX6CSjGtfiP3l0PdM0IzAceKP1z262CylrHTGHRQOfqR0ZCZGq2eC8AMHvAX8QkrgwObND3grBXwSHFZ1QdUExzdVuPQDF7KjHBRZ0MAXEiLXo3/hLPmCXLudrnUEgVwq+thvouswJPFCkpfqKIjD6HVGgr5UOvWlIvw5x7BbyjAzzE8ehSBNXxcy/MaQ2L/04UftaHfuytQgbBwdL62IP/xFXGfrnBQYzaHdpaJugLa8VAS5z1VV0YOSlzp1HXuALFGTgYlEFPysm6pIgvX1NLD3aFYXbeUF76wHvqgANmycKgya9xbOXhoGfhi8y4M35CWRR9K+1lVB6jRxF9ws9xm5rzMsihic2y8WExjygrO/PhRxy24SQW7HvJDiOswvLj6OjDPI7VA4fTfUhVgPQw2A+EmJf44Riv0YI1XR5n4AR0EA/4uqiYqwyJ1UK4ryCpN90DQnIz7DHk7bFP8YRB9jA32KjU/D7xFEX4COr4IDRiMvGH03GvVnEUEwn9AagPjGtZgXOfjHfwNIdkXeXJujBCgDrFSnDZzhhXwtbq8TDgXxeFM6XUv/XGO9KqIPUYnjn9RYvws24C68INgvVXQHFoz4w2B5VB/TFTCt94Cp+zhLnQEoNyx3qlcOY5BTUVHFZ8B157WFF8WprzmnooXUb6ogeC28aWuC6iurq6COz46bvpY92CDOg8o8lcF1hKdYu9QIRVpUw4px+NntIArjNH1TjyAe+ID6sHL8GCDqzbHd8Pkiv4JkAKfxO+vAPvErsCeWDyWRD6G1DLzGyUNAq1+TsreqDEWeTEg/mQ+7m85Jm6DKnSbgoVqovwN6r2f4np7qo7gJjNIZhLyvnPIWf3es9Z2A2XnPwEB5TOA+Z+z7YvFpdhPMC03e0P6JC4lhL08cyHvaf5n/SKJfTmEyYHziGYUZzMUp6jspfGelUfQJfT+B7K+jLvb37GyQukUv/arghwpwAJIg9qDKfEv5W3v8N6n6oWF8Fb4PDSP53F37DcP4e0TH/5MgzfxvgzTxfwLSX+7r/RbR6U9E+X/WOUcI7MY9fyFA6PW+iz6UFkEVA5oE1W/77fRno0nms5mG3/Xb0fs4n2Jux/hmR51+ZsupFDrSftyBnyH8uXR2cNFj01y8HgWzvWnwpM9/lVj53/+q9bs4/EDRfoy5r1DFf98xx+55SRD4g94l0Z/+3Kf98t9SA8wndO8XqAEQsD3mC4fqUZy+NnuTVLxmB0DN9TjyOW1yjJvNq24Avy96hLyWfikRWLjRIZufS/ilff6s7sHpz+oe7A/pHgIlX8i3OYPbQOMvFv2zSYPXE38PqiWcfFgOyYM/ZAiFhdd/P0jQRhjqyaF8WwezIo9t0aGSeN6WJsgnA6N3dddFPDS+HRn84fyrC6M5bpAuijpu4iIH5+MWTVNkt2L12pZL4xC2aaA48k5dgggdFHfxEQot71wPewCTEKj8QKSgkrrgQqtBRCNnyHRkxxBo1OjFi2uvQNmXvqgSIKeXVTwmNUiExh8051V43won/UVKFEXus6Y08qhF0WdZffTb/Ffm0ZUSUpjjeYAnOM3mlocPzLlnahb7/uDk3ueobqEOsQMVb30l+DfR/0EJkMzP5MIbBlBP6I9/G/mfZZ4f/A2p8XzIjKACp/tpLyVz4hz0SqGlux51q/v2H3gv96Lllln+Mox4g4SibdI4B0KU5xf5/ZzJ+wquEizQW3c5QwphX1j2UbQQ4olo3VuCr+Pts+tMD1xaBJClyJXFM7i70j/mFlhUXNbBG5XopUXrf4Xf+SVe5L0TST6Tv6fJVpTGvolL7Cdyqm8o5Tt1dOvN17dUvTMzGEOT0tNs+8/M+aPdRwZb+0LdG/6LOcfuzfnT2qdOAvsvBqaH3oPMfOgQYMxd20F/3yuKpHWDKgfeZf0Se4P9LStl+CEGEOrfgjAUuQ85ycf00DP7SjDfha5P5JP/H13/Hui6v2+BwL8NXPAGqqJo3sYg8LynwDrAFv8D ================================================ FILE: Documentation/etcd-internals/diagrams/write_workflow_follower.drawio ================================================ 7LzXruw4ljb4NA3MXHRBUsheynsT8tJNQ957Rcg8/Yj7ZFZlVeXf3fNPNQYYzDF7SwySItf6lqfi317scIprMtf6lBf9vyFQfv7bi/s3BHnhEPr8Ai3XrxYEeyG/Wqq1yX+1wX9rcJq7+K0R+q310+TF9ncd92nq92b++8ZsGsci2/+uLVnX6fj7buXU//1T56Qq/qnByZL+n1uDJt/rX60kBv2tXSqaqv79yTD02ydD8nvn3xq2Osmn4w9NL/7fXuw6Tfuvq+Fkix5Q73e6/Bon/C8+/evC1mLc/zsDQuc/qD3UbwTheigT/kPz1v7ff5vlm/Sf3zb822L363cKTJ+9b8aC/SuBoX97MeU07uzUT+tPn9fzVwBPZao1yZvib5+N01iA7k3f/6E7R+I4/Hrat32duuIfOufJVhf5bw/6FuvePOzQkrTorWlr9mYan8/Sad+n4Q8d6L6pwAf7ND+tyW932bOW4pmbqfehf+7h39b+G9Bg5Pf73/YLHpls86+Nls0J1sHMUwNm4b/PZNtvkzzcnMGA4awA8v+SHBv6l2bc9mTMCjDrP7Pnd1o/Cy7OPzT9xi6xmIZiX6+ny2+f/juB/4ad6/cGEv8L9qvp+Bsaid8xV/8BiX9tTH6TgOqv8/8NJM/Fbzj5v4EZ5L/GzO/Uya4HOvkPA4662QtnTjLQfjw0+3umpNPn6Zhr6V8bkqyrVtBq/gLg74T/gczvsvgnKHrwVZYkBEH/IibgKPkXBPt7PsC/q6c/cAGl/oQLL+J/iguv/19y/4WSm6f/MY3/8S+WX+ofpPevVuMPqEGIP0EN8j+GGvSfUBOsj1w+TXRdJPnzW5uq5+f/EdDa//lPgHr2vf89R/6JW//I1KHJczCcWYutuZP0r7z6Yc3P9jDm3zAOzPXZH5TcxZ8wm/gXyTIKI38hXn/HFZjC/yref+AL/KfS/D/FFuy/FuY/Ev0fYf6nwvhXfwP67/Dpl/D9g7S/GJIT/kxmqmxG/vJtiuM/+mbb/xWceRH4X353DX/nDPIntg5G4b8Q2D+zBv698V/OG/y/b+7yZE+2fVqL/9re/ZPNer0oqiz/ycDh/8CV39Xy/4BwYCT5F+jvheNP6I/8GfHJv7LqX05+4r9P/mb4ceaZn9/070Lyp4bhf9M4/YmnkWIE9OduyT8IE/Tz509499uyf9Dzby/61y0izOOjiNnGZ0z7gFSxmujnj+F4Ne9Vz5UOfnAyS0fPb3YPVLwAHdiQkYNQf64I/vlhnrToqweaghGN1/Nv30aRzysvkdrHzpdEuvn8kZ038+6campkh+0dp+kdm39DvcDwTtM13jShqg/Fpx3YbvdWvUnmeWd0nk6z3ChCU79t1vNwNiDWpvbP0fZmt5OnIXq/Fa3aLkJ+9jN+ym3cr0ceEIbEqPsex4f5TCmcp6q7A1bExH2HP001+ebtv//b5QrmdsGbjV80+2HCg7uOiZXtyi7oG5WkQzmMtinpOGDXpitVgqYP9rqDjx1vpGB7viAjNXLQmpygWsXnaivPDSSlo7DomQn7m9fFKy0PcpVK8siV+ccW2GVjX2JVm9r9rOriN4PYIe5LrxujRXTpzJ8r/rbaO5Eew8lA5sfMEpQ5Du/ZrQnfKL26kaIqnFUc9F2f1fBGRkLmOc7HpE9Bl7v/dFSUHr+ZW0S3cwXAZgoE5ccpJb/jWGl+wMUCH5JiJh0uz1HGTFAVb8mvRywERkyGsLFdt93ktvMJ6eZu+ZmTgQtA8JdvI44SbrJ1jhEuTfqquS+LaVsLEHl1mvdqVJzLTUrL7fVGEGOcr3P6zTPKj/bu/aYOGJZcfGGp+CGAcKhmQhHLvCgilB9SLx2hFL8W0v2Ya1WCOeeisXfi7tS2yr2YNC1TW8cPPhVOrq+G8GhSRvomyV5mrP5MmJ019jHL6WleHGO1a2ai4f0xJ0wVkvHzq6YPbOC667tr1sHq5tGYRcBKS4c7OzOhSUk6Bbw32M0/KH4mjGqq0yZequLpw2Q7UcyHqpvDnHDvkrR6BGHzol/mAM13g7ykSBTVTumNchOvUFUVK+S4dVC5+uMZiI3IQaxpZ289C3n+Zc/8ALpRGWxzwDpX6NlpzFSeylvIFNxQmZvRxza/i5t+Za4aCGWzwABqeJPUqzJzH7O1gNbh8LpjHAUcSstpeh5Mul8WW+mwTWHMSU8d4lHki4yfTf5C/SoJzbrM2eG2NerJOhu/y0PKvmHg+9uL9AUIkG/6BtmLlF48ccI/y2T7JiQyjQlPhlwODaZxPf0+j7y/3+T51Zhmucfwsu/3TXEJsXl7UTldF5E50xXt2xK2PHe/hXkR1P2BrI9NeoBPNGxY6/LQQ3hrAv00TOms7qd5L0WP3BIuPT8fkYu/B/pKbkZWKF5iD5sfPBOHMpXivhUwA8Lz37hfud0+mmSxjWei/TEgjJqwVeNZaaoIZkqsVPvF3/TH/FYkGOFNjsLl6tuW0oTOJmJ+Z05oeEdIEfRne+Gq0Mbqm5tnC9fCdVidPOXYGwcjgVDcRtU91wjuv8Q7kGSHALIS0sp+bFAYwKlHl+Ib9IAXtsYgwHgDhbjzRd0SjAusK9fvQ6iBPUMYRkrVkkBa4/uCldyzfBgFO3i1HeQ4FcPUB+xiKkrsUyWqLK6AeYHYYdSFGJL1iSyk5aucufVEMoieci3REWMOdt1BB3ibp+iu5+p8Lsfy6U+HKKAcy6XvFUiI3nklKqGg6/1DncogFYsCLCZ09ZPu1FCSYESVxk2LuQBu6xw4OUm0VK1mDymYI8VgiQgEkBd7K+QTHAhtIcUV+e1IgBIFbxPN/+KZa5Rjmz4ti5fLOXG1XRo6OjxKYwy94N0ToK1g70E10IcisS3WZUt8gSCveHGVDssAsipjDvSkMkSF2REIcjRIzb200uY220HMqypSb5zAhg521Ru4C90ESxduC+DPs1sQEwkdyV17+r7d7T31qLh+KcjSo0306Ui41Mdyjpr0ymZvLmvxJqo9QWC2ZyKoJ5dXO++0TjKjVkNniJDYAfMM9pbCuMZ3nUxMoMOzr9VwGbSKcXCUW+kUL2LQTUCr/fB4thlZN5uLNyD6eN5ZHE0ldYchumM3t7xidbULdPA1jtqlT4THD/cE+ov6dUtf/lTm3nf/jOMdrC/Uwe0DjsvvZX36QaXe7+UzABW2a/BIKyogg+kwMnPzGmOVyTuyPsPydMAgQxhN0yW+AUl8AKxwmQ7DJv56Yy8B0Z7JGmklQ2vlrewC0qYJo16LfhJx596XJfOuYuW4RhsMMJjeZf+FxCYyF32mz1D/oU2gm5TB9xWdZwbdKZuoK4NQfV/Dxm9FzDsIL+HqKwDrDbtE1qj1kTrGighcyue37yr5+bg8QnIbPo+iET4Skq+x37F0jHH9TjnSM/67H2OAT5VH5UtUb4b7wvwrCMdxGSgENoHFiYFIhfynVyLXWb0SQEo36fvbjSgTza7xLIFpG7wtZVeknFeMcCThFbG1zeVA3m9bA2aEJFhAp/jri2DFOFl0obpCwoQcl5MGkJBGAklrq/A4Q0UnxdLKvSri3JHHuqQUV4k3xxpaElJAm1tem3z9ph5IKc3NMe+wRgjiZlN29EIG7+LFwFR+bJclAWs+FfBZoehLU74ckCkr4yjRA6qAk6rxxK8Su03MBl4ylYqvu0i3e3s+FWvaI5eUPvuZLIBGnx7VXK2Xu6WlHPB0OTHUXW892phU6pbvmAIm1ScwBjdbek0JCxK+0hVZft0vpV5bjyvOyDkpB0Zdox9q++KWS70MoV/Dcb5lT2GoNt6dlujKV/wiSvuZTs/bH7lYgQPEmm6cKAYLSK7BArq4CGkTOJWmmCFKpUp/ok/snTc/EK9ZQ4Gsg10RlNLKMpIEfXbBNVAAGxWvA6Dla0FYsXrik/zRt0yQjNNZvIEUeJHD6pwtkFKfhxqYBSeQI5bOFD2LlrnVTYcfa8Z8AIDcxa53tAtOU14npRdbswsDsslsrU/n5H3X7+2tax8P5okA8t7XB2jvL1Zu+Ym5u7d5IgQE3PXp4L4eT7aAKyfSWv7RulIsOkY3fPuGoa1e8B38i18Zap6s8wmyU6VnMeAKbUh1eguH3P/6obW864TcKmQnopCwStxAIpjEnCUwteXk4QRizxyDnR3d5QVtcuHSV1hgXm+1WCxYQ8zlmno1ym+pvlucozMnfeIAgRxMk0Yca+lZZ2cl6U7ONeGRQjEvbaRTaX88l261YUzS7QlWVaf/ZMpMfVt0X4JudJgJkJ4a+wCa5wXmJye/rxHYcTSPHVbBLyP9pBY7Th/Ug72RCOmuBii1ZHxCkigvyDkzcU5TvUbZs4YZj1CwxlCZ+4ciLfNFhkZDd3axbfO2gsc14/wg3vmPAHfU5jTb+uXblNAxO5Ou3V0HyDvyw7xKJgNWOM+G4SUFzRCq7gc8VgcAecRsgYzvbCJXPsX4N9gj3/HnD4+CIVWyeBf2eDC8lKKk5OtkLYjDep2d1dZjFfuBO4s8h2Zk6OLPI6ej6+ZlUxA98mxfCfbEIYK3Dtya5i1Cvd3POlDY8oHN0LsT/N48v7RX8I8iFEc1aVWRyW5MctT5CuWr7s1F6vrUb034BbNjaGjB+05c3b0eURMc1fZF7xQzfSPVZbF3NTifp7Ek8Jp8D8Z3a7kTwNrYPNaqJRlLTU/gWVUS6YEN8gW7AmCij6mjcmMX4DD5nE5kgUXO6hm2qOUZLVCU/os+kq48bSVGw0xijHcgMXqv6VPUrfhO7ycqsLEaxvSphH52HNTWVAnum1z1Ue+m6/xcBbopQWsjhh3bKPv2jWpNhc5KZxzCoVSAJ9ouzey7rcPjQ6MLUAlv3fhFRecJ+wTmcOAl61kUs/RSp+QOMiJOtvvGQ91mQhV5rfkqlBpZ4aatvsDq2k54Z3zV1dDmSHrmSYziczdjkzoT0TK+LIvnS9i7Mml8HZ7rx1pne+eZ7WQu8TD630mWzkPws8ePttZvQgcHMILz6gAAfXrG/HHXBKCLmdcB6AuJjxEr47YaEL0oeErQpAGuPrYz6CgHHMWQ74asVM4qM78YnS1QOfBoZZoJvcMB2Y3ZyD+RSEwyHyF5dS971o0xDY9SDJo+UVHxYJ9rsekDFaXDjBjNkgmTKXji5lCMleI96YvX9CoXP1aEyRk903W1eoZqE/04jJyM72ovz92AoFdOo1xw2pm32q8aa0Gsxu5Nh85OgxRdZSp45SFXX3ELY94iF52qawufLjNwU7QBDGmgrldRDeTPREt6pDzbfr88dOjwLxQdrmheDvr1uoqiGuGruFVBk2AEq4ayNUynyaAMDvukN9jUIErxoksnLzQjOpuP4cHYjLvpke8ab2jWRsKgY3zpSrPYx3fbopHLaDhZEfWtw7W1I1pCUBPUetUUypUrSd458JjgFPJ4DLgIOAQny6Xa+jq99fZDHgdJN3MqxpnyqVOwbfHAK2arq48+KHXxddROr9+NmaJCIcSB5ybl7t1vtR3I41GzgjPkIRAhFl0/7HtbJtq6I7ldHol4kH0NIGisfj6v9t5QBUV9xTUlQtKjlUZGDvyQgG/GIdJZ8ZWEylD/q3IVT4M5QfBOk6dLLgHuzHSclOqrDg5HaGAFXQs+fcarW1wMWLVkvsQqYMCoCy7CB2o0ycpm1jlOVOwjq5eglNE3Jjh2Q20lve7aI+cP/Q6SifvQUeaHrKxpmdEhRmjT2mTxtP2PGRLefjhuoQD9aSWCcFVJdBNkgmja8XzTVjE2kmWQjPoX5QuJf8oXkn+asoX+LGUI/U8lDMl/ShgyU79zzP/3qhkYDv0F+29VM/6syPQ/Vsyg/uuMbZFXxe+Fu2nd66maxqTn/9bKZJ/1+1NKBMRbfxWHf6P13wZoE8jK/nRpi32/fiMwIP2fZdt/z84if8KPhw3rFYIn/AVD0d8bot8e+XPDnX93d/31LqfBQROAmz7Ztib71Sg0/e+P/xlgFWvzEBjgivtPa9Pb9Fmz4j8h8Os3Bu/JWhX7f9IR/o0VgNz/KaKe8DDZm+/fn3n5M2D8DH22m1x/6PAb+P82swUa/ljcgeG/Q+m/4789428w+zXn30D318X97+Pw9xr9fwbE/2Fk/QDpbPZfwMJx7Ld7gKt/h/4C/X7/N2iBm+sPN/+Imj/gFIfIP+IU/gv0NyT/L7H6LwXi72z9r4H4+h8B4j8jDYNf/4A06h/U3K9N/Tbuj8el/nmqfyjfU/94oubXtv9pqn8Zfv8bh7P+Xyx9/T8rWP7vFb28DZS4/F9FL86mjBp+LtRV7Lk3zChvSK88SfnGQ7/F78ftkQe/P3IPpH87p7IoAoIM1gHp051iY1lBWQ/izZBrY5ZvWE+kZeaUVeG0/Yy9pmZim8un/XDO6mRLuGZaWLr0+IAO/Fwz6ndhZygjleVPfiY3bhBPD2Ab5hhSII8BMu6ZGWQsrYwqXzVOFHS0923nY6C/Pjx3Pvc2jCe6muiK7ugA0aQkqRjew+SMrMKRyl9jyeB0XOZ5p+zJb1M+T/vKm1XPK89UsvxuP0G3+frbmpDNRy5GZyDaZ/0GJjFdIO8jj5z5vdh0zfwkoKwjc8nzINirhVIdpJYy4cOy+ngoRqSJPEmz7/zkCS3Sszbz2g+rfE/TPUBmdqvujJ+VIIMZUe80jqFLOpNvshjOFxmANBv81QNJ0IVmL27JGAc7IM2LMjr9TMgXVZy/1h/0omuXNErFCu/UBsE0SiwRH5Duhoa51EujpQ9tHLIipuJnSM8owgvhrhuirgbDnsjim/kPO01S+Y6mgo7eeYyj3eE3dUAtxmDCgprCp1B5YjqxuetDvVqdanKR00Sf0IMkZPfdSKOJyp70rDPxpvZbgVxAirCox0TfFQPJSSpkyquYVmucNeQV9phrSQUl22+QDZfi4P2Fgas96HVcz12hJaZwvtojKHUNJBtlA/ZH/vOWWBfBrphoCngWHenRcKPMEMNhQDVZo9024gzGfAiaBgFJbZ2TTGMpzc4VF6Bc5nLTKwG1p5QJ8JUJPvzXavicmTjclT9t1o/DhTdBXMcBLR8Q8SCRdFOEMCmWHUVtr52pYo/Lk0C6QVaN+21WsF3PfaRrRgst0ETUtHKvUfThxdBEhjBiRZ7mvScg7WUah+hIPGg7oplvRQvXuxlhWqYrwQvoPpJ1vqNVXmbMjcZ2Wmoq+vOT5PBoTT184U2jONfhtDDSQXhw4ZstcDr80jInMzXJKHpl5RU7yZXEO6Kq0E8sJc3vLXkzniONLG1dNKnWi1qJbFWgnd4kDAIxzkKzC+/gdO0B7QqxY/SQ2S2q9FMxBt0Yb5Aaru6bVpUNLPQVVGfJ5UoFMuvPhCQ9oVKHCyDXKmQFnWecUNIAfwzIHDBaxpYlC/jBaAUjCx4jT0X8Hq7KFmxiqOlesheu8XjNY/qIXN59UIk9sy804YX5m07qG2YjWPAoGtrpMaELxOA684aSceKKdscZBXp0bly+fUq4EXnsgHAeGH0udd22HCI20cifon7p4RDJ3eHPVX8LuamJWUKeiHqSfU2t7CsQz4/gZs7nDAhJ2t5I1FDNhxA4hO8ybil4nYwwtFJOTThwraJetJTp+SGF1fkVy8zWUC48hLG2Ss4pWuonz0NqIZr9VIOAa9aCCHlgjQcbEU/KikTLyVERbBzwxqbsR8vUTVNzMJvhfGcqTWYbp6g0gHb8QrrSuUYOXysJP2a2w8YEb09909i6A1+eqLi4HfO9qHClk82qvMuG56azXHQM7lyJS2XVBxPryxw76RPH5FBeoL4uiI9y75YrdjvHjOW+mxAH5NLk7yw3jtmremIHzuBrgjfk3TrthoMZnZG4+WwMDsDGW5uBXZAp6Gs5Pde5YsjGttLl3ce78vuNxJ7QFCvI5W/1hUlG70JLAfdII6tN8qhRVbxVoZt6zJn7QV56vDdjmLeXx0Sge7ThspOt8eWF2pz1/jW/3AF7pCP4KiDB4rlYWqoh5Xy7XwpxQ9WE7kxmIRhj8sTaBjUAbpqWprmUUAdJ0tNLhZBMdcfNek71+nmHVQ2qjX7XVJvstd41fe5xDfwTDia1QpdU8ck+1hcO9mvfsfzICOCly/30WeqyDGuRdApM41PoR6mP5WEOC5YPirFr2WOUK5sNmrgMRDVr5kkt4yHmudA9hM6zj1TusCzJ6/H9b8dMwnh2g++cRP6S7vYCj0Pl7sWYZPlM7PkHVl5vKLdOGP6MHn7fnlK0Tm4bEIDbjuAS8YaoQoOMzwLj8msBucXpplwLBiWvsERO2hhpxOGCSdfqseGCTryiWGodMjreaWBvD4C85OUmYhsgepFFUptuYWSnsVbI36VvvRSSnxF1NxhOyimrCszzEqdtvOshAgdiflKmljf3FpS3eRvdhiy3TcQsthIDKIEg6Q3qZTIidO/w8ZXcZmzCGNr16yQ0n1wDf7PDEJ85vww+6g7MA+x/nr3hJLI//sVwJKnubzlytqn+c8jBnL3CNijz++x/C034IngCWPhSeW0RKCWXOMCGT5pKhSnc6JkT9IX6Fxv6LBpnu33FXFdk9nlM7SiYDIyIIxfkYoT5N1D0GSUgaFp7n+g4OGZQWtHUfRwb1k4LlT2z0m/8edzjz2exsm9+3690I2PuxVr8RI7jvBRZTMDjbhZCT6xlgD+MOalvWBD5y/JF8xrSUGdtKh1/K7D7r1P5NiCNp6SEBWh1k7WEPVsVVqCjW1DGAAqFzJhPJZGVEtO8XukvxkIVHqxY5cIJxsNo9gTNwTJQI7sjK6zLjPdkJbB9B6RyZZCj5XwIlPaaVpf1w/Nr4tGBKPnlEpImbaIWBf5t2s1lhfYrFlWvPKd+abNcqiE/tDtyI/sPnBrrnRXjWJArCuSr+Vz+fkZLelxFEWC+r66BuptnwDTyoC3Rih903/mr3kfxcMYE7aZtipRA7WjtsY+aWOg4UFfySzSzyD8fd0IQTTL8NuepZjw6OefYf76PQcEh93taGs9BA+PkvgKRnX1F0AyI50Q9Z5BBiAKFRs4LlfvxQmzC4xSQh8cuZQHEA6aHeiXohfPmtMuTxO0CcYK3hqb4x6nn9Rm9nRaQKkbjYdwGB9T/hOEF182OpPGgGNv962RLAeNauO3FazWMfA/aAh1XkC6/6HgCle5LpOp7H40rQyO4EJEN4R8FlJbzvlHjcGyv797ZW5Lt/kUhEci3Oxx8APwXsYIdP7XD9Y2l14viwY5aYgs+KZZGHEIQ/U+i8ivMlS8wPolkRFF+9ZH8OQ1SYiCPWrQJXhp3ZxKb9kU/IEpAzF9L/6jxe75ZNEM+4Lnzq/umwS99+jjOhP9BCO+Bja/h5Qc0QiXORSvRwpWOM+EbuO0963iUkuB9mwRLHO+gDrIfnWQeGMZpUxRfe6myJHATpCIqvyC/yuNQf9U+LECCtIFXnJj2vr6c2oJ88mxReaD6me9LyI71mQXTbhiqA7wmyO7hrzQAhyHCBlBFahtwqiH/FOjPhgQpYGZTlNHX1b698aBHu7GByJzW7eSU9gKVbXDeZxAhYhkvbJrccjQ8y0ywaQgeG8vASxsQLfDqY2pNJWNJyjak9nLZsH10spc25jM6jhtJWS8I1ARJrbMzSSC979mjaeITwLx+6hNwIMf92S/w9TsX35AwSGp9DGUq1cXHhB8rJhSbRp1Yk1Jx9xndR3NqxZ666Y/HTqVenFt+aXyR/LPln3vfX94HByfJMpSCLxx/ES+kfQ3fh+eVQXy+QEswv+jw0kCwUX6ADsAtq/tZqDXX52bO9rmV0UmMFgZY6n1tjwpGgUCB7jQOIuKUI81/vDzKuTsk0Wtq5zSL0n7NzqlgUuwmv2LNR7iNwKIGEUo7IgoUXjCqf0+HHJkVHP+BreEmlJBZqF9lJObpgCPKJY81/PhIEKhUIl/YxwHoXibev4j48fIAT7OZwH8eJFFAOfbt3ZT4jL4sHsPEB7OgsKeRfsmlYI6ytChqrPA4nJODUa+4yxhB06oPV9Gv/s29+Xhgtymn5VbDhnQJIgkYsKrU+W9tWVz+utHJoLVAHuqBotuI9cyEvVlO7ro0uIWGb/nhmBMfvbNfFN4IC2/5qgSeOncQAC9QTvza5WOIDvd5DAqawWkjjEwaFRDELUmt2j0e6POtnx8/l6aZIPZrJe5gg2Z/4nWZpkUpedP8T+3hp8tPOziP6tQVLYErHsTzvFM9ge/f+nmfij7+cK94vy59g3O8DkzkRi+ljwPsTkQKQrjoTEC/PJ59oYYCHtbNwfimDlbHItXEDoam4W+P5k7UHOs9E+E+F/mqEOEtHXW84KAmCuxvNHg4uE8DH4ocspGlCn/6HLm4UXJfGx7LcOnzbJmLPjqLHnKLqjJLV5akdHE7OzYf/XXObLAHy1GmXLIPsyG/+St/5C67tYG64ot8YukO02760m75emxxFzfw/RgcKAqr/Rnf/j73H+bn49Bos6F/1tN/04a5YjHCo0D55uGbkh+X9Pf+v///fS1xWM9xCzWJZEMZN301BHvWIXQpovTaD62ovhD7Qbv5Z19kY1XTH+chf42f30lY96nQ80kA9+lozCkCVJbuvjGL/TWD2TNzIfpb4NnzQ8VPhjwr/cNOfnYj+mgSRLgn9mgcnJEt9vsPNwPqY7X6rbvH9++fWP/DE9nmD/vkzt+e7N9xqHApAoPZIKsHtPLBqCl9vUkN+fVUPzT6rHv2/DPj+4+zcccXcM3s7D5DjCsJGehnTb+oQWqvv3IZjGqPb8HND7aYPQ7t2qoWNR7o+MHng+lfh6yfeC9nMrq3sfCbXLcSi7KbX+WH6BwaNfHjY95stH5l9NcxGaYOwvooVbrP2EvTFpQRNtaJGuliXd6KDf0lQKMY/ipmv4BjyJTFNyBSoKYt7OuBk3/hTsyKC2LY1VuKdcGH0SrBORnJR51ZFJLtAfFw8i+XSm1gq9jp/Wz12uV345zDVzOd5RL6AS/95lPN+nCju2uUpU9jciSNIljr4Byha1vLcpTSsM5q62Icsjhbxq1bqxdP4A6Z82I1C0R+E9pgC01ZTFNF7WPBYj7teM2Gg0fgPEwplLdYVbM47LPK+KnaG48IxA0dSsNSdGkP/ErC07gr2V2NEguRhw8jxKlZtmh3XhmFVmSngjpuRPEh3yxCavq4y103oiSlp15QNs8RQt0bntDqbZOK6zvSFg/bkbsmOC7FJFlHda4At0HK1pFjtyfLXGdd8tmGvlXLSGe35/T4Gipfmtco+3D0MJpb8AXWmwKh4WC+xXC+pI9EtpmKhaK4m5Z6vuPp3EKeqBPijcVn4l2h0OL0t5mvRxZPzxaf/e/x/j3i+6q/WsgfE3yzvEggQ/IAf2UmBwv9VwagITIRNCJs/+VaUXkhdZTYdfXjSiWLbbXqV1EM1u40ozQoVZDlRDzrPmv1c1Zg29r5rxd7xArOrbp0AtvVxX/RTt2H6ZOwQm5Yud3NjzK5eu49SCvfHT/u/HwNbPMmbztLY4hNDNUR/LOJj0XDjdqP5Rh+An/LN9MZNhzty6hv1jRP7vGV9U91nf71GKqX5CeyTjk7JMfh+/puGRAf5kPPX7EnqvhNxXf0ymx5RrhjaRe9AMkGoZXsG8LcHEPIzxxqgz5PqPogD+u3RoDv6bFQoxmcJIbYg/z+jE9wKdqQ7tzscvq4kWqfIYkOV5UUbYER266/MjHrgo5A2uIkbY+1IKzs2qsB1o6tMr1/U7zXyuegm0HmBhYETyCsuX2KQ14SAvUq5M29JryiLMO2jz7xN3d6untb3issfRabCdVjfAH2Hq+wF9JaDNtFm0WPQTZ2PfMB/rxbLh00Ty5al13e+fuJ+nAfDgqhFgcqe9nAEX4HqicZI0kw1RFV1yyqa7cQlJo4tF+rWHC62mh6pEuigEPJa9th2Kbfctzu+Nt/18TV8x7EBzbY38nEvhGjC2Q691fEQDomOdFlZZZ6sGU9rrARTYfQi5qgqzLjS6XqXMn8u5mnxXz8BsiLfQc9vtu5iLGHCyWzlvrHv+a2CSVottPgu0wo7BUZcPadBtfRCT4V3IqvlWPbE1LK1g5PzeNTbbgCLwt+WwMJieX+vp/QvRjdVI2gimJI54VVRhUu0qS8BvtbbxXGbeuVbnXpYpo3os04KyLuSe90k5yQ7GbaDqmmUijTF3rI55qPv0XumCRGz4zW++NtSbIRUWuSrTB6MIgnWjkMGCfWBW4OpX4XbEYBZ5579YUHa87gXibG1oCIZFosrvDKdO3W8Mbnctc3v2W1rCe0QN6V0nXHPnpTrw9mnSaJRg1ddxVIw1iVussTeHTR0Y86fzTsyA3aZyP3n/Mtsf1aGVezDFd+tF7Xl4GaJNi7/MYkKalYy8bqSUlLVju502Nv6mx1qM2NUAk7aNln0y/PL6Hh/aPlVV7WRzw5I1Kqu5Wxb/jbdE8g3MO+iVQ7ZPS5kUuLqIZ6l+De3c9srleJxf7kVsYTjbDsiCVRELCOEF9Uqyn14vDzEzjfhah+c1GL0UuR3nHfvEL9dYVvX/3QGe9w/sTVdh0SsIdkY6Kci2v634oW3y/ZxiZKZnddYTo9rQphuHhiXXOxvZkJF676qMZcxYPT1ld3piVTZGaqPChC0lX0GvtMBRl4wwbZE3y++Rgko6AMKgZff2K7V3d69Ivm0sYyySHaDUo6455xqTfOqqCwAHGyUMXcweuKbtZmY969681z6AZi7LxPYSI0GO9v9zB0ziEnQRtaTlgJwT3Xz+RdATzbgu9qx6eDJ3BMdPYZNqpUQUhQKDsPtTQM9L2LRtqbO/icN1uLQ4SVudBlujd6KGTM5F5zEAymsglGCuP7CykZTTynQ7CD6e2/vj7+OEhBy4xSZHVDE6WfHV8vJIh0mw40dOsDTP/JfSDL4GGOv3RuFLhqGGhxRSICU6Bsh+OmmrEwNCXTiiLQ8d2Jrjbs2JubgCK61FdjaE31rm/gyAQuw5RebM76KKNZoKwKmm592JPZxsZJ+ujJG+8XKtIGc3k0U6gtuXcZbpLoH+oOUEX7Lnn/OYUvlTkW7LWjRN5peHpnIc7O2KWfdYW1Tz+TU2LQ7YsDSS431irBi+8gj5IMQwf7uV7KdU/hjrFqsXLJPoJZ5+I+PzHQzmi+ld4TnUcABBJZy35i2o70uBDCzYbatKSVkEj0KGQBh3ZIx85cscPm8ISocbqxN+7AmcyId4oHn+DBS40owbB8G6/R91chObEL13VtUzrIHBGqLMJtLfoX7IBwdW6Z7ijfdWX2LF/plfze6BYfhouRP0xpBoVCezWIh5q0ZX2Fi75vGWe25RLRYGmzc1tqSl/wXWJXkC+vODFTjlehrOiVQdWaivnYxvG3Rd/+d0Qfc2HmgliKvnRIlfuyenZRK/kj+oXUPFpNX2a2mDTpHSqzJhnZgh5jUZ4zINRMDSMOeZQMQEI7mHEuG+2+31OLwPLht0OVDWWT0yI52kPtvkj9tXNVM8I3ZMsPlsXUsWtFBsfFF0bUoWCrRF80eGBS9ESZ803/2K5Iua1uZtVLfxDH2Z797dOAVlkWSk4qgB3wQsliX2qDx0mEjS4OWRZs7TpJOsAvQhQNhkb+9q+vx9DeC9sLp7HCvkA243zp74TwKKJ/sSB75u7BK9ebQp0UvWKXbgkwr07UB1DpqgUCDwnbQVnyvlfNkOmWzT3WQ3WBPr0vIpwkAsn1GKSShXXTXKdGfMRVMV0hJ697nNPUHHXp03aBasprifHH8Nk/dNVQFgY4Nd/c8RJkYaVlSbDp72NRTvfsQISL+jiTYIsVs/WVEhtw+m7DDp+Icrc57cKYrB/FXHut9po0bpjx2+Gmq8NY7PgW4csRURxkHCxuuXaWK74QSe3OnvVe7mKJmTGR6sOu6oeSYOKfdQhiuJu47fKeQIpycT4748PhQOYhaJT1WFx4p/NjjWRXZaCed4UidFbsIQuh9Req/GQjgBL8cezV7ot47/fdsvVG5+e05nU3YWJbZcE712gmYwPyXVPqaEsUqZEl8VKhVd/0Veig5O3KU/X2Pzt6W0vCMdmF1K44mfuMTWgCWJ+z2xNu9cH4Uc3bXu3F9H3Y49Qyujwlkg1WJ5qfHN6OCfcdO7AYReG1CEpBT8h74EE2smWbQ2Xsnjcwk6W+uZ4gAZrHINdzhH1zFpIS7aE9D6/Qtw60QtSamAX/HfRCR60qrIa4+rrUkzmU4kSmeZ9126JNrpgYyLfbWlYMICIuQuItvGnCmBam5Ippv8qU7rT1q3y2NE1fgU8Rtyocx+ft+G6EYtfYokIKIfO20sFrYyW33AHisSDN7r9Kn9dJfNlAAixy3Ty737qvU0JmYOtne4IUdUsY5H5QM0qv+B1Ya3fUTKcqWVYMTLRbghhlg6OkajsMe90dntA94aJ0RtjNUH7HGZOdxo0ZaM7CuSBNh2HLHHINF44K2e73QMl7h5QU/0mY72TsbvQJkNpeZH/yjnZZtlWPusdy7N0NKUQRkdu+Fqg3UpqOoJUer0+MWtihgb38UvJuzNGmCR4RZ7s9RoPeR2owqlhwOF93nRxxM09S8rVeAvspvy0T5ZLrQmXu7afDZV5piU25mRPmkNrPy70g+Rr2rEPhsihdyiayMmrXtM9dPK14gR8sRZvRoASX2GFXZsnqch3AgxQCrMPvgEhoC7x+su7Jp4K1NBLUK+xus0ozuxfqsakPPPB943M0nujvk0rBC4Q3w6Rb3OOYteJQmK/LYpVjTOWtj/1E+kL6kvY+3Iq2jOaTpnwVm/YjsN5GhPB3faRDHUpPQB14E4epdakvJKjByWHSAcH/zEREpKmJDNLZoKSxwNRVxC2MM7oo0mL8sry7oEL/qD1NmFyBbFFq0Cd/WzMUi7dPSpRzh++UY4y6wfijnECLg6yZuQN1mS5vMoNDhvY/61U3Jyscp1Fow2ma6Q01jYp0kHB+bhORvrjihYSzfXHnpQ7lhLVWLESXpZAwixx+rMX+hxxc3zpfwvUEx2mte0GA3YSoqIPMlkJuW6lt3XIrhirn5g/cXqcq5t8PBEdvNL7I3NRu68u/4Q+aC8Nv1hg4DWkOcoDXebcLSDJYqDQPnIoD3zrL/AQl5Ch1NkN7RHyxHR7SDCmrBLkOIAEW4PIMV2Boy9RUe6FPtKZ1oWSHQAZyHIWmtWzz/fjcT8TSLpNcCxyUEeITvmo/0c6vf3iTW7k4yUOcdWQMhhIW0rk/5SP8LWFx5KvwRC7kyPMG608BMHPIysaTVrAv7gvTs9VJ0ui14qGNx4eLLIv71Kj7OOwq0QuNahT2FJ7J+5cy7cEheg+j6/FdZ5+gfkQqRt2W2rvHEV8R5BvRl7R8psWrcDFciLBRKdJ03QMuf3lZ/nfA+y0jC1J5haBWxrwzEFoqQuWbG8pnKavCiuad+x3IyQdRYT/Tk/M84p+jCaWEpVkmGRvl+87S/MStWxILsHU5hn5t8+m0RWRXX0sUCam0XnrA4O5ch0uaTp/wJWndZyMyLDpBLfH9vUfLZ+YC2Ih3y2OqawgLIFHaY2FovePB4pUijXqh8EVLCHztVYcYGm8EMhiNZsDvexYftfblj8iXPvmhY74eORUEyiJjUJaZ/qkTfnsh/K5LOXidorDK8XQB6w5yHHYRYosnKGhWC5rNPUYb53F37sk4IKlrmX6eaAGDMZeFUBv5XCCvbi8L6sf3jUKuYXrHfFNUBVuLsxe5YRmevWC28WWCbHQi8QVMdFtr4Zc1SFFxXmthI/wq7s586T6pPh93Ui7SwympH4p3Pp5FCxJCBvWWcNfB0ugopXu7zSOiN6YKDccCygag5vJuQIZNxgTBj0F1czJvFDbxcY8+IMVWzlR/COy5MiWwOSI0pQU+EgHE3ODV+Fb1GgxXM8wdvGKdKWS6egCP4ucVx4OEV3vC3m4UpoqYth7WJ0MNoFeWgoXe4zjWFdQviOdYEqcEJ8yP68JlU+/YThTm8yKBdF4/w7A6h/o+7+0Na8e1R6rxRsUtvSnNv3t018BrmWTHMdQrDKGfd9sVXXoNq8YhR/mRTswd8Xj/tKPvOYnH3gjeOlP42TsWbNJWyMQhzC9MMLkSlgM/muSmQNVUCO6DHjZlbQj4VMHOZhBX5M6n6lLBZTCSYxMT/QD4aqLPOiv/ZZPE9XfVwSE8geMozIgnHuwwGL/E6f2qDu7asZDUjPSF1MaEwdeLKzrjMSbDmI7V2vl0gD/OCA+Qp/h3UIqj4b/rDuV74qR8lSQVPgElVH06MxqOnepVWlEN+azq5AbQKI5sHH5Rmq5FkjBG1NgrCLZNH85efRyJgFeihYCOeAPFUCILCRQ3g7AOuaMcJOz0C4ED5oN4HS6IBQPt87gAR6yD7+ETzuNSwwkaE9gbgPkg3IoJxG65ZOStT1+26wn9NTH2yC7tDcFqMSNehJFYrlwnAJDgRLI9Y2+kiZ8wIX8fqN5e69jdZa6PDoeEqNznPHYaX8dhPFKUI3tOxUSgZPtGU/qcAGt7K0G4lGylxLJ4LoAQ71RjDhSFvc4nGQFuBSWaj6G25f3OU9mCj2+SpnaCvMLr3ixvENHCgGHodQvw2qacSBh9LRQ9jn86Qoi+EGp5Cl7i3LOH4An0QUkq5ap7/bJUWuZS/eguLwdpY4sfXS0phBNPVp4fuIIZSsgeEg9ksiGi5OPGVmiYStaSETs11Zx4Rk4LqED/53Vk2ggwmxgCzgxFcUI/cG99FVe+fCvpsFoYr7mmwim+zlciet2BElOKoZERPl7LQhFyRRcwBNneNdBktPLxLb6ZWMiQeONv8ct614JqZqQhMHxMZbnD66rmudpR2LnOa33tPHYX30+zbAuFhfOuTGMSXAtUnf4eAH+YLq3RLoqP2Q+OxzrH44eKZvL5eGlUfvW9FvFu2h+FSEdRoXNNKxdH8TBykzgboqK3BHQECnGX9KXj1RKJjaySd4xgt4sa/dpUyKxUdr4hXWwXjDsXdE1YGtxvIcyXvbZqUBmpLs7EIyZPTKDWWN4eQyJnYRFyrWnjokmWAoEb5qFX0XF1MPE8wxJx2Pcj5YN/DWm903UQvzEChfuuJo/ClaLoPY6vCb81UM+lRq428kJgcfBFHsp+XjsMWKt66g6Vb13KiOnd7qmncKUnY8gQfBdxHs5Zw+GztI12dmpBMYNly1jUKX1cjBXw5QzUfBOw+ka65ed9VBPrJaAnWjIAh+81yMSWyYEbOZW6e+IVhNoROnm860ReLoJAH6849yQULX2H2Jk8r/CypMq0as56KDdpp47sO6Y5FQiUVTix4kqOmTDXgrRJ5Ic9EN1yI0mtNShqk0bgUjCwzuIhy1Q7f7RUDU4wCNycE/gOvAKiVlKjw49REYDlrN1xxAqaCF/jvH4sCTFJpdWb94Pwjn7ZtWZMUfW6sDQ9A3BsgdqsYvVayzWhUjgL6DAti1k/vAm9XpIwfkw8CFBImqX+QMqWQhScQmBn35VHNVl+91q3HqS67DFNBawwfr4BAg05ijraVNOKTwf1t0Ljo6rmaNNAoNAcyjL7WJC5v3AT9sVBBO9mfwrt2IcwtNECxT2vziR0XnDqAHURWnHJVRzhjd+/KZazxhx+PBwBBinMa11qwlWVevrAWD+PnfUyair5ZtRR4YpNDm+4MhuADCx9qO9ZI4dhwRPkhZRTf6PZeoTgRiWmlwtsuG+dCyvY2+K74EV0mSYclAhGFtWPx1jamBR9hBvFv2U/gik/olTDnl+DM3SMXL6UNS1AKYQAUWPZ1z/HrsKOz/oqaxoDWMUpurmvZbYZuVUsp3qDajVeShYv4qKT/uebFtTSrIoPeGVdKI0ydd+CIb/htwyOpx1rdVHGSpdloSmZ6RsGNl438WD7a7GL9imZiS4GA5etj81MFhafb50jllILDHnKqXlx7sdsKPoIO/mHOSDRg7hEjFpCGjH+ovoPJ+Zl0QUorq3rUct77egQhKGJ7n1M2xyuorXfGEhHTBo0Qe6ERTKeOqaaKvNaNh04h+FLOAtip96XpBo1YWOcK5DZnoSymCvvW1PSd69rYK4KqZ22rOQkEk+FF0bBTyTpsmVBlm/SiinXisB7ikJ6LIWHPQHaXGDotVOgLDNBr757gbMNlcqmVp9ID0lb7qjVgDl9PbzsthJKeHGRhEu8eFQVlmxlcAiq1Eic1CWN2KCVYbiReqfW+3qMQOhwUyqRw9W1BUjotG4mL/frCoQ6DycCjtnp15EEof8534k3aKwxNASj7EYcIzEFV8nlxWYzaHr3Dmn43I1G7T6j3c1I9Xxh4/0KB/s7JMbr4VVZrmSI6LiPJu3FHoZRtKaFo7vRqNnscjfC5xuNAzHVDh1G15/DNmrX3b1LHHOvP1K82rYppmUZVYsQX5Bnn0ET4vr9QsUsCn2YHWiWIhSQeQj6VXv88o+E/HzpleYi6/SE9PUZnz/vi0RS8xG4DK0MUZtkqzrL8NrMcLynt+dnqfmQw93vwtj3fejQApzEoA7CeiHeDpD5BOpxq5DG6xKAc8BBixYzfpwb5Ke4kQqp37rANRXg2HfbeAv9UduFRBF49SmE2y2zQ2J17uUkeVIFA6MtuCRhvuLDn5gRxUMXOfSLeMKOviaJGDDr5xto4prpcPv9ccrBsHgJ2hJSzatMEi8+xklzI7DKuUSiHz4fVH8oplsNZHzFt2s0aCqYq57hJtWrWl7qN/FEyJQy+Gsl/jpmMjxaaD1c7Au+FkqI2/EUjDZhUIn8MtJbDHTKevNN5O0pge63op2w6lddi0U0U/ADu3Bs3OMAKe2JTt1uVKMGw+RrVaexAn6E+vMFM70Zo4++ryZ10qq3VT5hhEvdk5we1BUvSKW+R/4acokbxfgetCIvIVD8FqgV44NBThj9+FiODJl9cSFSgyOPbsgoJ47KnU1tZna8GYt7gttu8LpGeUE6aVluu1GaEcrHKHxU/7CrIItF+N12hc99V1cHGkxAwXtBj6GWcz1cetPwFnDqdFm6MFJOMX+iIN6qmauk9PgAgbmtyjWGn1wvvKaXlCQGgqE+9C1DHo2BOxZ9XNCtHXU/7aKUfYcfpiKFlNF17du55W73OYVjd3/NUJ78CllBbQDNNENs/RD/v2i6qqVZkWb7NP89LpcNNNq4c4e7O09/qG/PmYiJkdhtRebKtdIqPAjE5LQ4i545NFI7IJOrb3+cGkU1K9bHRhlIGB4ctyW7Ge5rhXxATZ8Lo7/0GMDps3NOjWM52sqnF9CwM0luFw2+4dX0HZgqyYoCJd/YJYgaTaX4NjrWB3UuR8x1mH7I7TpROAiq7tCNrcBg6TiFfH6mD2+QAOBBg1yIxyHczjkY2vjmiTFaToR/KAYkzDXFKJDpyWFXGz8j3Ah+yg1flWC5L4j+P8ZZCiGtOOLIIybm/p3APB+c94YIAK9cOQLf0faUV7nC/nEPGnK6F6dwjDal/wR+2gYBDl8joZNEx9eUWj+ZureR+vkx2Pexe5/XQZ9FlovmOWS+3PrpDyiXf42Mf0rwPXN15nm0aHtrMMb2dxLM7c4w/JLwvkpdPxHd0CwzA+YFtDmCGGUWOqUed+n51TbLm6LY84CrjxcivcdTIbxcM198J1Yy/mAM4AckSAXDqedFq7zpwc+e5FquBmmM1vuhwTKjQ0nbZUk+QYzdVS7fzqXXg7RFW4yJb2NOgTSKxHxMowTK1VU79CEUo/+mXv71Zr+ifWg77FwhMH3D4efYg+PyGPvzuSo3FXr2jb48inUEn7QEC820NX8AC5Sl0auzKpMrSnxVw7id6GsMivhfk+pDKQW5UHU7G5KNGllkff4aOM4876Aljs0ztIMLyAUqFENQVDCWn0wj9ZKSpq6jAIDFHhCYn28UtUUR01GstgvNpvhzWfgoTZF+Mju4bXiCv34pSl+y+ZDY0sCDczQnAVPOsKAvyYsEkYF93n/KlK+IT7h6n88rDVTQ6Jy8Cvv4rXnOfahJykhHoc4nYd1sTm9O3ob5Rtadft10cYpXLWwnThpIk5yU2viqsiHQhWQzWorkeMYnEGoDBSwcq59Uecak2u88RbSonLYpsyLVQgu7CYwJRQMx77xWCQap2K3JqXp1tWxuYZk8LfglpI/BYB0VKkhNtaHnFMsOa0ETOh27PPbSsvPwhyUJ4UpPtI+oSK/ZXChoN3KGVNkFjNjm5lDlx3R3/pCNGvuube4WbM3U+674i4ORjybGSwYdh/OqpEXy3XJqHBk459c8Xj4xlYLYEChfwl/RFYRtXA2RLSjZdHFYJ0lGyQFYCn0aIuN9yCbxV9Ed9VmpLkoXkErnEL/qY2o2UaAU4CwXdPoyvVIt7P2mueflq4Dh8Prn6rvvRVK/5AwweO4qonysZM8/EMRMWvZZT7XW8ad3JighTejD4IBDZhp6gM/Ud92xtxL93HWkEOQHNGIzV0U4jlaGOod7/NeKPxZ4WSkCuJAw+mvCEqeCLLb+8B13wwsLJ35dBi3M90xjaT+zGSLwA0XTF5AlI6N/jZjMjzypGIRzw7vIG4E0pIVMkNVyjB+5aRj1hEiLo0JuIMfzpL+/hi51CwtV05rPIVpDp/EfIZiFpPuof2L0PA49iVMV7/7a2is0Ratv4l9OL+DDR6EeNc8L7fceJQi64AGNy7p7gkBPnK1Hz0Z30Bfbm1/fSJQd/m2CYkUMQkWjzte9WF5dB0r0L6l4yofDHKRrWSSOEY/NMv+LXZOkXDo7bPJVn24/VD0gQDx9nuY5I4n8Re7uQ8lldRBYiqKEXL169kDixAs5pbqnp40aBXrlAuiUBhvBLH3/ZsYU+48m3fAXRoIy2I+lnKIjEQ1myvmLM9v8dxnkjlVr2MIVVFQusJJcSmx+zg3fFLcf9Xj/Dl25JgVLYyc5n794S0LtwPLCPgpqZammyzFKo7gcfNFB0pCB2LuQZgmmgC5kxEjSCTxDxHHKPppNz3MY/4RBSLT7fCCq/7H5oHWFL6YzMMnJr5svQZ71YZZt24mRSdJ3S6C2091jT+cJ7ZmUwRoUb9DWsORlYLDbvOIT2JzYU7exheB3IDUHvSTQ7+CLe3rX3QqkDMYIaaRBF1CQbC7IrOsWiPjwJdVyotAUzofmhiXg+f6LQgnBGWxiszoUH9e669NhHEeRYf+CPoPCGfCBW+/cuN4OlfpipDdehEHvZE4jzojBmDu3Mi74lLYY3y/QkMIb9vzHYSasEO1SAy3JjqcD+FUhhMNrtvxafXb3r2GjQaLE0YwatZpu+rpj3qtVbMFIn3WxhtmjioQQdBAJ/ra0cV83qlRJXKzJQDCxevowr4wSEU/FIz4HEcITsU3ugBKFzCJGm5ZYX/m8q0UbxnF/Vv9g80nTFzxikuewrlE4yTMw22/2iHL6g9Xd2HdKZvNqN/IeqsP5klT1JufEsabvASycQryqqYhdFw3SILYaMzlu/qzY8+mQD8jzRefLWArPwu4HJ5HLZB8/r7qXuauTHRiFfPpi7nkx1PTsN3+0hCQxOi8CBLOgtZsFyjShjrO6LtLFRz2GPcgl1Sgd5w7EZsu8UcyjVxrXSdetAUMY7Ku3IMAZYsPqb94udH1lENhGxgDkTBjC7dDkZxS5ADCrKL5fkKvB/vIUl9gCDnXsdr5uxZyuP9HHqe8hgNyTd9omaLZhapDbDpJYLf5FXl7sKtVTX2GEthv9DevEFv16BtwSl7NA3tbuRzDDCMyIkFluca5LHD8/ccvHlsRDm6zHDQMf8dBNz33OOFEcWi308LclhG8+J2FlxXfHcdeQs5xAzmFKWD2AJq4kYt2o2Y6cuakGFQRFI5jYPrZgBlMa63UogIKAoA6eVfmela2yDSNBMibip9dRP2BPKSXLfQeSqnbS3Nd0uBsTRu/Divpztrfnuk2tpTWxy9uZy6hkod8zUrMqbhPi8EA+6Bgrczks5cCsZsAO0vChV/ylRq+x5328SLxJhHhncyNAVf80XJpFHBecAleH6SvE542RPof94AUzScMKOOdL2es6fNWt7uKuWJI9w+nI5Y/JPT87DPE9q96qzPwtGprl1PBXG/yU0w8qrb17VZWUQsvqeP52dMTrFRbYDwj1tlXVGqhyRLbeIOWM+ZJW0KkjOg5NRD8oESAalE9grCAf2nDcyIWJOP3k5PuxXaND6Kf4y7/gsw0mjkl6S6V0kT7RvnCz/R0kjrvKp8VDKOd7cOphhvFemR+XghzcLOohr/gb58LF6+45pxkJKoXBS9u+zXc8WUmdPDjUm/TaPwK/Xbuy4IHglmArLEPMSdJETVy4YWkSwGRQI+3hAAIta3r/sTWympcgbkWCpIMLtZHsKpu69JV0Hhwx9gtLw44Yp0E0te5jQ2PDbTdV+77n2iI03hIBB2q6vwdjsCw3cF3C/82j8yqJbhwJ5fvqpi5hZsJyUodCo8OelGUhUXAE72wBeDSqhZ6t70FSRMhRdWgzTL7JiHgboEjeRZibaEOuTH19P4UCXFVM5osRvk0zFROLqbgTIuwWSaHzef9jaA1UoIUUHOU35CXfmibbjlsUVnYmB2vX+I7i01s822H31sMOHpzXQJoL7YNq6+fwMoGXcj3BJtdcYV8Q01Pld+59EDCJDhtiwcNYuFRsTlBtufDsUSt86phJe/v7zp5/QEGIlq7UFcNljqCKO853vtBusfvXryu98Fvkiiy8T/6Nx7pwQ8Pd17vhp1pO7P3Tc6wfLvssiOtsZVnY04dmqH9OufZT0zsfxLNKvH5J5cJa02iIin8lqqxZ4sHJNKbxAnUNRTKAUglKEzYEWXAm7g+LrpLe/yDShD9gQQTzfNh9ZFLhr95FTPDRuIT0YfSCQRku+hfcg7MgVxrFxxPR84TRhu3T65SDHbpkke41MCUoK6oJjENkEfQLMImToMg+102hRbcXvVIHDuJQ3Bl5T7eSZSuY86X2mn6xG1ud2kfGXGchOJx1K6vtBUhdxIU8OV9dz00cI7OY99Nfeh25ye0D43Y/yTOhRSpOpP/ZqbI3Dm2/7dWo5dexJpJtTm1SxDQD8EBjTgJ3hx2CYlpP05mbw+Yze9ORuCLuGSH8t6RtwbGEx3kqEH7wLH7zT2A3mn8tkh5RJcNddlz9nejxDXr2yAj4iN/HYGQjI5gg2wACPnRanUm49oqV4T2OsZckUsiNgvzk4QwFk5dqJ2wKxvsma39e3SuW81gYwMUQBigpiyL4whFjTb7IOo8CqUMug/OFrKZ1O9F94cGDGlEJH1b7T1oDWpa8Lulqcy5KZWJvGOhdWgHRLFQ9f1aN/DRS9QanyH2Peyb0kaXHqG10OE1BpjbwcK/Gz8BKsUepAyw5BrS+qai5setbd1BkqpQs1t8DonR9byHKuB/3Ndo46NH6ezoMtewAUH8zT3vJqiChnpajIDTrcreSBdfLNhl7vUoYpfc3HX9LDKMJ5bxamV6RXDot0+NvfzvOAlP4jVCBGH2Cb/Tc0TBUVUfUx+PSTv69HsGtOHKpJV+ksaOkzJrUk3nr4q+Na6J5/A2Ay5jj/BYC0pQPznEZC0yy23gcu6GHI7hYq18KG3IehixMkHXGkNIoZoLZWqaXnIiApHgPT4o6vx8jrkvmXExBOeEwNVkqe9HPE4cP4IjHETyF8V1eMv60etLU5BggpmF8RFHnggsjyYf6LCtqiw+v2g/JvY5H4c8eSzNSxpOO5zdGqj4YNzJZTsY0nAQAarR8gW0KKuLKLuvGsnpJv1finhqVeCXowkMm1JeHVOlneXtaRz/TUjUGpLPMKqOHzCrB2m01SPmdrRzlHGKt9wutBGrczZoOBfJ7NdUEfsGFytgrvruSdE1FiaXI9qpfB1hOVH4vLmEl8xYFRDLzBHGWAgRQwgUvpM5c5BkRfR9BM6aPDiwO7mzOPrkBRvt+ugyWS3UM7r95a+4Nx4PEKRQixyn8bav//loLHYblV3hnXME/mqxyAnUwEfy5B+QvIUZAvWYytI+hxqBq7AX+vvzY6vf7eqc31j6uefl0Ytc1GOp/m1D0CPV8OsU+xA4SQkSAXeOhHk74G/Nf5J/oiSUR96HYbziqeKkPA8P7BAfv/MI0rvBRAIDTMP7SHTxMcCw9gvip+0OLP50fa/5RIG5Qw5OOwqfeLEBL3Fg3E2t3l6H7G/F+E2tu1UMNkVwdo5df9j6fz0zvN/FBrTR0XQxtFyQpPxHLlTr3hihDjnarPc81+JObX2a4P7o9TDhcNY6ECl8O1uXYgoQoIAwI9oVAzA1Z3Y3iJ6OmGFL+kveFtFJhiRdTm9pKKNNu6HxBipb2elF6aqoPDIlc9hhL1RJfvxpuD7ci9k5tTFNLgVFFA9Qlixo3L5a0+L9QCzvZvAHPbopK+Yc1jJyD56/+ehlHddXV+2mW/WeLWpEt42VWoX7NSFp7qSgqRY6UtOtk97ZhbYIKKn8ZZbx0ic5zCMGIhRfBeXtxOrwPrgbY5x4qAbKE/gvBlO6nz4/umeOl4Nc27JbgnbAy7DfI939TI8HD6fjUBGlVnOy13x9pn8rFESb7LQwIukq8HFHBnVw78v2kG1D00wryZ3do3kc/1wKtzdLv6aHB3K29Up04tftDRClCOEsSUUEoj0JUBWfZc5WQfVVdd+E8HVckccrjh37Hj8wxMfmXlu5+pIdUzHOXjMDSeFGsLENRJhhE0bqBWfEnWF7vSsgg6LP0WsTtrGKyGAawnJtR4ai4F7wbUuQk5sN7ygc8gl3mP6iurZR7wWWtmhYcpy43WWHT7Ss8h/5rv15vFKn013v3mO1m5HrJmXScbe6CddcrPln6GcNnYBcIXvWdqhE4GC27VCB3pzjLQB1XiUqQCXbnJx5+OpVcTTUNx4U3L6TP3NaWE5bCY8/g7xuTLfdtmJPeLZBN5C99EsrojKP4XhvFhEPIba4hrunfTLyMBs68fVVA9tpJI8QHukbsv1/tFTHXcefP+410o7IIeeTO858Lz96PzOEeicaIcRyTG6loGuzoBVMUj4u/qo30vN/lk5zQ0SQx46U4Wg+U690kWrLJXme2qhN1rj6RGhaSMZwUBPjvzGFDdexFcQi1eBjzov9uJC3vZ/AafdL7q6mxq5e2Vn3x17woitGDMkyJ+HuU68+div8a5zJy5uFOAEqpK/OQkO5W5ILIRaNse6AAd8YI8eyJImcxC/9JKgb7viCIUM+POwXWTfFbbcohZqq0pXqh1OxfWp8vyn46ZpZ+Ph/uwd8kvLc8t9V+Yu2rI0pXfVz3uJWzl1GwZ7W7rcl2lWMCi6KAvkRQmEqMnK7eZ4AbsTPvhs5dYPkJT8ncc1c/jJ/8o+a9g92Uo9aaSYkiOYN+iaDPzpYTz5J0eSGk69/OoRu80pGb3TUFDjink0Xyboqd/Zo3FHMEtLnSpEpXGQacRnykcIpEs9yw0BeTGCkS0/p3YqoS0d9Z0KehIg1Y+j1JSnplF2i5N+S8ECFNdOGeSbO7gJMuAH+xFGV4XlSu1qY2LgtD1IoXgtUWc8kNh1svYOaHDkMnR752Y9B8tEfCCNm/2lDaNaSC327MP6SiDKUbiu1MwmHSHXu4ZqV8Zdv+hhAenQm8c9UTFJb5lFv2/aCZGurtcCTYjxyatDWmcLMBm1t/8biz3KdA4GrRA6Iz1OCrJqdfLnYz51eMYvUQIp9hEE/kg6hyiYH9C3GoBuicJi6vQ1iFH+CWD/51YGwkbvZgSdOWv4qztrdzRheIoU1X7VukOtPewaQMPeJVh1mzDp+DVapKMWuxzHmQVOivsdagyotAk97H+xTyEQsELZK1mKQ5MgDcgd0bUzdszDOuOfM3GmGJgsO/TYrI2uZynM80hlKezpUcODa5SZ1ZIzMLknftDFwBw/wGA/nGnDkRidwbEM3aTYEODe7E2PuelIWyO5aRS/E4R9KzFK6QnceyDxt9jgYBMQUvxTrAWbF77n3fsX7X8ztNJ2QiYCq+RdNQW3bkrs3U5uqwlPcvqH01+dTur4pX4FqbH0bub6HaaSXDIY8ltM51RcdUID541j4coSis/mp9nMgprHQmgeNrS8++yMPpLhQzXz/xcgSp2eJl5919YXOO3G5DzX6e0ViH4lDYZZw3xY1vD+v0krUiTIZBg4osYX0RF6tJ4am+k47UJNBD8K7Heu2xS5BUUxFdoWuS/xpM6/vUZc5tyokLXNSk5DODLnB6nKoU9tc5b3vPIB0RX0nwt8asomKfozP/WucigsHw0Hi/Ft+epC12qhu7ANETpKuswiYcLt6+zg+/s9tKyVdmVscfQ5Jbd/FAckqmFZBAm/92MkCYchGqPI1qHSN9HbTaTzP83g2oSZAWm/uV0nmrx6DA1X6Bq2+gNo2vCO8i8xa41Yg91HnCAu10U3XsxVYUbznTarK/88rLRWySgVXfDFQJtKqWIap/o/RernwX9FymVB6WgDbxYsURI4n9+qKPUa4QY69ZPD58YRMRvwHuPPGPInHW5OjK56bUH8qYtqZu40oVxFiC5qDTPO9w2/aDJDrLxIIrmi1VEEQmVNAu9WepXOm4rH8gR6pQb+CMgFd/faGSQfot7A+7coAhZ9pe0PxK5M45cgk6UCSXMYcZDMFdBTvaIltk4Mc5Ywbu+sQP/8rQXRnmtOta4pGgk8WkHwcg8iEVS3YP9YGu2K3ifA6GEgbsUtbtQpXBz3n1e+GGQtYJR/RRuLAStE3zDBGCfTBKP++lxpCd2M6Vr1Tap86kE/Vtvw7xNbRkYA7cSXKVLcc+fmjLf3lDVHFQty0SaOjTeZhgPYDyCEJqzf/JuPk7QHWLfvUJGvxWMqbBq2B+aIauwKf3AecPrhkpWIjEbIOIh00oXpWY5EVGIxAEN6c1Am4fnefpLBe3zDzlG96p8yQxrl/L6klFJMa/9Kd9OD8D2rOOzb1UzlI5zfhstR/PQGSd7CAQ4pmRmEdeXbJKCeletnj1VmPY3UaFwRPk+wahgdKe+POzq1dPuiOX6ebW/aSn8Vxr8eW/+pFsbhcR9NtpRcotYCeCEEjvw9BqtAKB5Tiy3h7nDdEp+8J9HiRpyNSCOxvXQ9yWs/tOaVpGQ96BdOn5+eyqglQwbOdw24JxCf53T3OietItWMh+qHaxrivqWxUmjPaSpkdO8yV8wO/pRX6beCcCJ5EYX+wUHr70HSIncqotWdVXUKnVlOGgi4rPlB1zkSqEA2l3xMykdh7eQKKr995viVp2KjZoTgq95rBpUl+afBmfTcn004GjB8sGfUBibdEi29WKa5Ga5TalL+RdWu+aI/3lPVzRVdms+pWo28/ixPjB8CKTWICfk4Fu7x5TKWdNJgnj67DyV5UCdjtJycFALEOyf96S8zgN3ajv4jSJe6/0/8wfEHhVt9syLSiO41Def4+AzYdr82IAia/GXcD6MdRTw/bmEjzAtZpLXJHj51bhiR4LXJ/ntxYMElmJC1qM3SXwI/AJR7kvt6Yp/cP97F1QOkQF438uHhAUCsFXsAPbXOxzaq7JCYSye8EE6Xex+FsW82oDJmdw7xvTxQomJGJER8ZF+poaNLGaRAgeJLD5YmiFkl6EtqYPxmMvnFYpaw4MEhk9sTuuU2um8iNN0N+prfXlCZtLRSRNn/WeFyEAUou8ti9d+lcgtHV3T24pGM2DVCNzqr37l/9Xfuf9GMZawYkiS7ERk9O3SLI0LICY6XU9npcgoeM8Jb6RjXADdAp5RCrF8NqYfDgENvcbZYcVx+GNy/fPaw/IcuvtAQqut7NVP5o9DblLI/sID51PijMMe2z7ldtP+/c4ozlJSKH1WGrL4KjfPcz+pHieLwwX68KAtdVwP0Vu35nJnfnZQ5Kw368vvQL4HpNXLtCufWFOiLcP5wrkKqE4y/uROon9ZS+HGmEjay5Mmn92HSvQw7B4vOOJTzDtRK0WFrxVLY593afJQGyLtvpvFQ6NVJsmboCdAidmniN5oL+uFX1DflQvrTDB6a8Of6agXUpQLXxyU2EhHw4A9FGi2Br/ZAAYkaHrQRpEsXaZ3uY31MeNnj83kjie4GjDdo6jTM52iGjMiMaoa1uLCnwqrIl6TIC6lZR4roidv781tZ8bfZppGgms/kBvYKwcakLLIOHl5/Odbm1WI9tHDsjI4hz7G6ki5OYljUmzly56D7pW/3g970NZllic4lkPfEcbPkanbntb6j7aZVJ28v333e+Ds8+TuGDN+uDrMl/ely3xdpT7oWQ4HMm2U5lcPqK4pKCpOUBah/y7mG1LuIRy8uPjGzU9YOJvSxDRp9icVEOm36QPWnxEFNSMqj3Nw58yq5a39B2S58lrKtdeBNDhyCnoVWMg0DLIFO5dCFlHVmRzpTZ46cD87UB17fOpA+d07APkgPXy0Q8Z80bSnfkfHUMVXkUBLx/u9NIr8IrZXn6jzDQF1wRJQwcuFxJLKwRZ0zuGbbEVo439/VUesU8F9/xeFyweN1g9xRuxrjl4bdMyhXh3vjGj2bf9I7bTiNa4MVxV9/GJraN3BGUL9DVKUIvafy4FGLhFGNrVfLwy4A57qto1Ycwnt6kPeLSvqut+Z38AKR2OGOR7ov6xKKdIiz59ee+8JCyOdlJ40+Et3H48iMFfF5LgLChJXlnOSwVQqUmcPksJFScB8ZsqyMnOttzGFVPWEeMOaA2T3Y0CzV+jF5+if/4W32g2GWwaSA5fX3N5bA5AVIdEHixouvCaVca0IaEKGcDsYoTwuShSiiRfGBxC2b42naIKBUfIA7wwm8isYLRIJr4UzUrVJS6EPeDVMfo64AIsdZwTx4yyVemh8QUJswm05PDGS5CArzn37g22PnqEluNuejzOaFKKHuwDFuhH8ROvIBpZxB3I5m93B/LqQ9QAheLtRQ36tzTz54xp8Mt+KM2qri2XtPqJEEASeB94bfCyh17ZDisZI03863KV1i8plPo8QMvmgjkhp4hulcQb1LPP8WvkxbB0wd9GZ5kM2rlfArnK/eR7wxxnNAGmZ/dSONDU1CbPSUUrUFYBPZbor7Q7zZ/Du9fq82y6iAFEoXD+R056lHEPCVT2N8Lk3s2d/gYJq2GJEd8lX9dCWeOYCCfO30DP305QJfRf7a/ZGlUVhuBVCI1JIAodocIea9bUInRHfIQE4wfHQk4SwQJoO2aDgwaA7wKZGM16XutgX1LEYMAmjud+ANzwwBaTYlSJMXeJlSdv4jEP2loOz/NTb57FbhMP0RRwJHfotqHhvz16IsIFYo1s4h4Hp0OIhd3Sl+ku2DwKWnkUAkHA473xJo9uY5Ub+3hPhiC5Py3uRje+l6YK6u7PO6X1iL/SAmF9InjpyR8eHp5EeHbsmNhk/DtYkBbdDyAuoGpM78kiShacp/c30uY8EzxwiKP8El1xfi2T+sanWlwUdIG2WHHoQ0rpvj8beN2nhdjh/+KRwmuAtBWleQ3z2XO3n9ViTeZWrVC/FzvaeYr9WzPmUPSVgLTa1Row3JHd8jSJ7y9yELgsevHw+WmLLFIhvWpfVyqMmS3Cvdv9qUtdHeKPeUzh+G6EpjagifqNumsWgSwDuICDycAUsVstSV8XiappC2Ogy9/AC06f0YtOqaT5FnksUntYxrwdNNijyYh7Pby2ubwYCsOBoGnRQVfd75h+GoK6EXqY+Ii1SPG6kkVcx0ER5t1tSVL7xhJPDVUAiyKXQ+Y82FWIdcJxF0Ir4JigeFrw8TgIC2UtzKfMui2NNxArx236/K5Dt5paNNIHpaJiOHw9m07a95HmZnDb1HNA6z5/tQZtw+Kp1s3KkA3NnBWZ3zHZ/tvpSjufWIyjAVAq8dsVK2pZ51K8gIu6WuKKLkuZ0tlCJ0Lh5O2LPYEDzQyZGBv83Y3oW0jWanKEGjKQ7dfP0VOykIer2IksgTHQRBf302sp6oEH1DD+tP7hHVHRFvgXcL0EdyX5sPRPnP2MrHO8LwIrjoAnHliCniRKgHsf+Y07CuNvkBMJ/lYx8vAbZahdSMyfw7ba3BjbiwOTzUDGF/yawKBz1NBod4/o9Ork1KJAKeChj/y7JcFXrT2V8QXBZfhRAw1O+hv8qJ8HctQvu7fgVj92Q4sutsW9xOVeKO7MWO1VBj50c9LW+QMVTwuTVBQMQ4vlIregCIy3ZCM8yK4w27RMOoeJX8s8vkuuHh2eVbbguswx17WSrWDUeWQYnErNczkZ+iA0JrT9zBKKQWpogfaaf6WI//3XDswoifa1tl1CyMgBRuwo1sOxX8xYlS+p4fgkpthANXf7+H+yLkaPql4DVHC6QaRBmB6SQFCn6OXYGiZal2PB//v/xleuca6fB/0tHgRSDL+63kgJrhbGYLf46dTJ58MHlmddsKm0VBSOUynhlieKQnRctPlyTqUcwFvZH7woBhHRBXdRWvC5n5+0JkqVtzXt6HIfeJNrM8pA1J9jao3j79Zip0dBFMZJSXz2H01P6DMhU+6p6cdHwDLwWzTUHcon4m/+WU2MEWe6gZc22gkAhmnqjEbxKctLs4you5wp7QmHYBlgwJIXKZZSggKdKOzU3EkVF9VsaGinoZL4OUkFcwx500Zfi1e7wn/gbIDX3XIeOd/dxhHzWeCxEx4Pg3M5+cjNtGc1WQ7DOJERPAt1RZdCgfN9tT41aBl46bIM+S/DDI37Qc/lK7eDO3Gsr3zM+Zx/J41MEVuiVmcHHeij+UHl5TwNWvCqGu1D6rLsifJsj2faIkBsUglGNsmdrpR4SsmgF6fQYRjLNcBzoAT7a+MGwfcGFGWhvkuV7Vg3THNj5kUT+n7OVh5MqfrJYL8TN2ObS2dDdDC6bIdqUfc9PQlJ/FvAbiLJzSXnNf6nHFTkZdQ4BtUDgADO0NX9sZ1l1fbJk2ekapKPQnTXSGEza5xqQbGiG+NqYqn136pwI4UwXpcHGzYOYUnl0yUD/idMS71iloXNyc+/7a77Cc2nj/oxDKTQRkHFO365PshRLYqIj/d8QPvLFF+tmv9NiZW4IJgVPnl8EYDarQGS2r9QHX78y+IEA48CCtzTzZB/BCJXvGgflb2ynO03iCYjuAOkFdUIVW4e1lGrpOhXr6lQOQSoNsGeW9JHKON4GgiUa/jvb0GHT/j++TFStaLOnlSnLGG6kAmn4E5s7IN+tcf7g7LKrd9nCZwp47YEU6KhrHxx5DZqrJpZD9NOKUd4umVDdcYJ5rlD/Z1Z+iswdmsGX4Bzg22b5hwHkMlPpQOUt1mit0shU6XEhtO7axHKZlZnZC8c5G5mgSqlDQyILDsJUZAJdekZ0G2oim+QEEkzpXPyoo6DbdalKBjf/dVp+EbqNloatjWVv5uXGTO4H2ewcTEJlF8Lr0McrY2JfvgOgghIW8M1pGebwiOcpD+JzOUCpRCU7tE0ld76kapUMKRpj6jlfddHE8WYanhhKCD0ejFOZ7zK3rc4/UWn1X42EXRmwPQRq8aOHl2B4fmzUJQYkD1p/7QQC4cmby5kZ340Wqfu5aUaHvDl3qVeGu7YXFZUuwUgp5By9YEFp3ZYiBlgx6KpuaDKIcp1fpJwXj/jaqhMik4A5+mkloKkr3YSky+jVhHo81Qz8F+dOtgYp8c0GNlyt7s+cDZKvylBEiCKIs6LEyuecMZGxnnEbCmtLiFFn/s8JWNA/dhfNOBCKIoiXjvF1bUqQclIH6JUGXx14wOGk62ZU/TlbQswq3+T+4BIvX+HV+WvAPHeswGIyH/KIIZMA8oirmxrFMgz4rakx7w11/5sMKlLY5l8Vk1LSiU9yZCov8+cGnCgXegD497DcHsKOUcxRphoZ58EtC7ps2aQS8dy5scDYzW3ZeS+Iq+JWBHkRySFADoVExplacFHYFx5NO15Xhoamk75kQhQ88YTVv4SlH3/dccuZQxT0RGX6oOE4eOY0IR6qEXC6yfNUxnKvFQxaMC6EDfTeBimh57dux3xvM7ncpFHDfWPsrx8xdQlCv6hwkU3hdiyNwa9nt6f5sE5QW+KUdMhuns76mpQ9Ei+QfnY6zjNNpGPe1bTMH/FgnQC++shbA9ivnim87fIINp5gW43zvNEYVleX+0jFGcItysmuhIoXNVXInJeswYH44irfJBdViyFiHNg9YCxu+sN3cr73S2uCLrynutZLO/wsnP5fNgSY1iRopSOt5Clc0DK7MemshOR5lX9OvB4xZAa784cBq7yv5AzLDhGURh0OUIHBs1AKoj5dkOXViQ2NJM1AiMoMhQH8uXlKX/J2E8Q8g+mjv0guQaPDObPn7oUtP5SE9cRxxLTkp+p7FUYB0wIJCCOW5bETP77IpHRMH+xLStiCZlMn1LjcIUOXDzCf2ygA5kIqm4FhiGSjz6nL84SYmNIVO43KPbVw4I+the3t1reUKVRxYHL4PcZZQQbiWjbO1pbnWTD3/fhFrvGSAavsstH7QqKLF4YST4sX6ZPlXPlYpOZX3Tb+GcZeirA3AlO4QunR4YyrtMe3kWdPmejn/lBHe+Tij91BAT3Pj0ngL5xCoUe5f8E7QC9AC9JPJQ5OorhSY9XPPv7gfV6SnXyKkxfs5ijoUL6BrOtgEuJeYb1CPv+qNGR6a2/Jnug9646Yqce2cCGTk+zLHDm9wVHZZd+koXXUgxioVxo/zKkeP5trhIHrCTmHi22v8c1iOTjpL/fs0Ahv5XB6GvnL4U+h+zhMdpAN3f9htSHE33PGpd2XXKSOYQmUTOLcodqHkXgaNziJHkJvCHI3RD8sNkTGkJXIaqEF5iyoaWyKG7nk8v1DQrNInkBcqsC7RbhfpO6r5CER4gY4/7AgMEQk/Kvu+FRn47mFJphqK7hGmf4c4Ym0tVl1egOnXylWZarjzC0IVcplOukE0jy//5qI0uZRcTf5kNey3ZiJ76jvdL3Klv5nGfsvNw6/wRzS151k02vZHTUMzPmqqvTPso5Eb+rPsOI+2EvrMQSHPIJ1tgDm8bDV74qSJk7gPHaVG6rUBju8XmnNaLL1QZa3dEUivKmS+G+zUYSaEizo2g8UgdGryd2uRXdSpjMEQ6buKxZfTkKjRZGBJFD/SPqQNzMA0281YY8JRejEy/P6lVrT0SdiP2yNtj4mh2eLMuTNAjfZPRphsOW6hMklhtnB97u6EmRTRseZU4ye9/JvtrZmrfO3xBecCFGSCNqYTaX0wP7HMu1rUR6rvD0G8jxwOE50GTPrwNqSYUq7Y4ki/poOsC9oNwF9wcu72YePfAbPfjurWGoeYL+eesoRcnvXER4Iy27b2OQT0HfmAb+eUD9MNLmRe9uGh8phX1aLt/T59GXT7eV+Xgki5zx8o2JRXxE93PozHcPUnajr3GEFuOqgp6qVqAPMOOKahp5YxNTe+vqKP1AW3GTlLsu6Pp9Vl629IMh6nHu6EQ8KBpSpxA7UmjiCPnclOUqfLv8RG9P+suw7JwTGqKR/ydJ099OHUTAL1fh9+oIkb8mi6CEQDFiTkQBDdzwR1Lz/pI/vieo7R//JGAZgeabQZpWH+ZWdsuMJzo4gKeXbQEC4STebVP2Cc2HMjFocNXUpyCpujlkUrl+ilMFtatg39lS94jIA3my5f1R4D421mlZ/tU7+DKTvSAc/NW2av6vsgHyA5/sR5uq1clHECH6Mwj/FJns06IppQlW4f1/q4rozrPNMelzd3kKD8Xx7m72e81Hx/MgpXNiaqSaJkX5CT6bUfobCw6CoNnWo/+ms/TZ3fPzUU9Ntr68W3c53CVPOhpZnpzEUJ5Li6QFhJbImBPGyTrLHH97S6HvSxOmOvi7b4kLmqdlFn2/Xr4uK/IJo8jH4PeDu/ZFxf+sifnAsFdd6lgKzRj1j3y8ol3gcV/j8F/91TCX2fVIXEcOUCWxEosiWyBYfpZX6P5YLCOH7OUDAnzh9PSBB6mFoD6L+g8KAZrqbU4nQqo5eYXqzzkHvfr++VCwnZbg3nC+EY+hogMjH/GmkEEHDzqel3cAAsrMKdKyPkZhUfpKbwBcqLyf16Z4ZQp3r23bJnMzmk99cVlVZpcw33D5NTDyGSEK0Rrn3H8vMbNlj2hJHNReOEgAEoIWuFRX/9YBeqNNWyD3E1c7MErd4FvyL8O9Gl9awxupXGoUcLMGD00NB4owKq1TrghLOSo3jp2vIsgdEv/yNZQJNWqC9gHdhHJnpciEoQegRD8SJ5cdBgNPo+MUsx1SsfCic8th3T4p/FjWMGzxa1TAqy3DwjdzSvnY9CBugZB5kWT8G6DaJms10W2eALe3e3ojLvVgvsrlGUaTeQyih1RIPtIOJ4QqNn+fPIvMdpikO7RzynlVZWn9gCE1SDrFJwTDVzbKAg4vCUphwpZHWarFpfWJM/5M905obzeB/LpP56N+SK6ZqxHXf5UlDBF9Yox4ovAM5UoP7CPvUw3GQ92V+kcweBvW0hYz1eY70LBRud1Kxo1SyLYWW6tcNfR3+CoefPaNWKizOlHHaiPxe4aWbtHzp027ULg4EZDtST0J9RqGQZbzr30Ux9/2DwpaqJfs5XSOmQzdbM1vErtWqQzOD+qM2C2Ap9BovBrvq+jkbFWQM0eGQgRR1g8FrAW9jUa6TlGsTozFvkY7Zc9RaG2UIipUOod9ezzdbmTQwMxNK5doik/yK33U6RrpoWZwCZJBCg2fEM59/9/4O5Xe2gP41WygQJnRpNOQ70cLHp5mbyzv5qFqcKuDXk8Fg2ugKMBQ6XYX6zNlVg3KlVCW/wOW2Uczk3ZoKEJzFRiCrHSNG+RBn2W9K9bAph/J0nVwpzBTYU+mLz3avByN1AtAE0mzhzDz2kdACDLqfKrGvmTSHRV1rfaOUffZGmZLA+Oyti/OGiu5ebchVEd15i81FloycvHWY/NgWfPByRPd0uQm24/tnXjUk+/v6Xll+OSfDDOkX3Ooydzk6F+5BksKeVYPwTxbz9SGXcuoEmmAbGleknuTnHwi2MGAzjCetASMpp1VJz3uomopuwly/CuIYMmJDWJdCK/uAfhPn2SmD0GlYuVF8UWZNs+fWmrDkQHibw1EeSq0Q2Ms0GDYgx6JbUMQdGXxdQcZXqG2iEolJm7haVqJ1eeMHqOIGotXTeX8nPpsc6GvfrWY07ZnVI0WMhDjknxMcW/32o+0UA34icu//jQGLb535/nguAWK0AbcGy/RQpkc15nb7fuZqP7u9UBJCzzr5KK7+dHDl2X/KJwefpf2Oe8T/pNO7M1FuPqS3L87e+gsJdxor3Zu+46fSuNIKY3+LjCpgq9TiETxOe4FYghIgQauEflX1RPQD63xvwnPqyEWSTUSXC8q3Wto6B61y9jquuS6UGmvU6ok/QXIz/mIrPIlHtePSb7MY/4ePrpKNiZJS4jPYNSzUzhJXMfLlipup6CHqHfxb0+wtgMKkz8Z+Glf+pO46pYbwX6g5KM/euosxtNJEYcqoKE+FwMiO3gdqFprcl3xn8yQMCp+lv0LyKk+PQV2vhQOPGye4q6AzerXXMvnd8zEITl84vDar0LzqOl/QYcOqImuPXjH0MMS8V4/1nrHVHxIWi807haxIbE6Xl+aJtw+A1tC7ESlToUYUlE+2Gtbs/91uDiYWolz8r8BNiI/V3xRv8qnU9pPdHjibXecFUzGNy5HPQFDXTFIU05SeQYC1N/gq0vHK50iI2pq4op76xFAk1U5pOxx5IEvbzj5mwpRSo5DKElHo3P6BcNgcIRszsUvhTxjEGvGch/Ft6NGNwYp00AG7kGsN4kd08FanvULBkE532hTKt8gK18wUm1eLbih+Lfvk1fca44nMWZbqB9LhfxGDpSxlda/DNLn8zsxI3WSQrtCnSyGNMsvxoEhnS1cufoluJ3M4NMcc1Ky6odKYRw8UmlK/f5WSOyMmP6DJoErJcUZy+eBqTRhn24TSVwR4P6yVeLTHGLA7WD0hBnLSv+rcinji9KxXiJBJn45jUPBSNLyXdV1ui6wd7M1kmBBe94InR/0HNn4pz6lTMNU2TYqdK+S9iRPR6E7g7go7NEKYmoutHXzFmKS9zNXwu4LXXvpDsmQZHCp3erkzSalqWyu89ShOIyw3xfn8SxrQb+kO5oAZXSVO26rnnGNrn3EsS1+8+AykdXZIEw+a8KgbKZiVqzRYMfpQngeH7McF5aXGHn0hbVQ0HOZgcSwo9gnRsOasnBiySF9fQR8ryN9cbijmJVGFK8vWx2zw95O0iaYgMCM3pIL5/4VsYN3cEh4iO0suLYxouLDOLbwENmciWjyeN8+/ObmOuJ5TsR5awta7tFvLf1QiydaDqRzQEMnLw0TfvXDaxK/8cJwG9Q8PsQrkHy9uO+FRVBl3ijHFZVthuA9A2vo+FxSg3ST6Ypoz0OXWHAZNVPeAXfWRlAEPOQY6d8+KEunRfpbNqXFRqBfYqQ1apG+x+9WnM1AXpbLL60N1u3ydkfgTnDokTHs2eKBvvvkdnGUFubSjmUqeLkhSPtV2YiYUeVyZ/zSAsIMkFpL5FphVKjYVfCTEO3p+Zs8fpeNf+jXbQ7V9YWleT2WgMT3mFhSr/Jto/hfBx3xomhXp0PbhzQvS02XsznqVIM2FUMbxrn5Bn2APDZJjcg19yunQi5RFD4TeHQ0YU/jJUwYfWEZVAXi7rlOoKWZb/EgaQ+izDKnN60O/ruW9lcNHqHmxVkg6OgcHCjayeVjixw91b6gya5O8KyXS6Dfqfv8reAbSJC4kCp1IhQcmplUrlX8UsUEPSr2tC5aNiLPPMXf6fwYakX8Id9feaor242pimjqst3Pu3w1kpavgjre36JGObZXXvOmFiGipAJapdzclC9EQPVRhcnUDMjZTOv+eR70hGPG1tLwblT3NctYjusqet8ZrwmQwIACmB8m15ZXzvvDOpR4PDZn1WXE5lLWXqen/Y6ekN+ZOAhF7ggCBcZhxUvWe1/sZ1UycbEOFn2R7eki/oFby69DQDNEpVYJY2Nf7ATh+Sz4+u+igWaaiBtZ1KDfyeJvHUucoD1dHA+GSrKITqSxc8wtp1nRNmGarlvVK6MohugbvcN5olHk30gbM1apuplpcFNTuZy7n11Gc1ZW+DIFwJd+2hbA33gE+0kmPQz/bkF+IvXllF8cIBYgbjdNMZwRc4PIMcjpZUgyBEGkllLddNygVqzxTf0P0lxj3LIdi6jWYBTPsmDEoCwgRRGhRp5BIA5jWHBQHNagtyTAnkSGPZx5QSsKLWJmbriLhUIeMYUa7C2d2B8pPEbgyl4a9utDzxY59KNUUnkvNTatFDYMd5VgBd+bgfLyk/Jq9Ng5qyAc5ppTJInsAv1lcmlCs0wGdPOmqd18ZED5Cfy1/3+BorUMjXyafFbTkTPegF1lvpBpuUru8cByvHwsmyQjof4RgykSd44mKPAWvrkGfso2NInOuP/q/3n3mjhHTql0xl1Qy3tFh789fa+TJppgJXs5jRPTmVDeCpns5MClvOv8o52AmA5XZ6uHdV30xMA0VcMHVGVSRqp6EpsC4GyfoyiaGV8kEDQCLwXa9Y02Z4gAmRlqxhADCrzkf0W+v0R2oOtaA7MdlERHdfiViflJODC0qiEgVSKEOHdSr1ZikMGOchmKN1Bi5vLvfLIBKYhM85KnO6B6pnQXTG8Gi3gECEVtizIP9anxZHxsI0NwdhLOFsYyh26KIk7oD4spBooGa2tP5XyZry9E5Phjc+0r+YCyg91yVxsEcDJhQclhivaQNDMoPINgdaR8GE1Q2du/kBJhMhQFlCSbycfQiybxoDzXBBPxcOvohr80ufh/5P3XsuxIkiWIfk09dgk4eQSHOzhzkJcRAA7OOfn6gXlEdGVl5lT1zHTVHZEbcuLsA2y4ucFMbelSNTVVKBysMcF0heUbLcDBiNB4htYI2m/xXCuLm3uz6povPX7riF9Jv5Niuq8GPmago5sCt7B+n2aAGi3/CzanMMPQIpBh57dfJt9lnlG6ROHfp1fpw22LLnu9VZvUfE1VdP3lGmmOX7fdffW8wFITLh9J2B0FdCbHcrcYTNOcBj4LdAw6Lc1L1OPukcFEV87hM7617lwKthTrO0uCv8FH6VBoPdL0l4tMZfkDbA/TVOfVWuQWYPPR/Hoq8qZJRj49hca0NgBbNu2cp3S/fNCe/4Y6Xm2GKd+a2Oz89sF43nmPMhymA5SmQPEkTvGrz6pKhiyjZcvfodybqybd5sqICJWYmI5l2x/A0kG4i3c1NoSiEdxT71PqfM8T4sbLGmLqGwleRTDF2VuLKPUgepimEVuSyyCbLAjpOfKs36DedhAqQKWReX/z8sUPKuya2kuXvCMH5Ve/aVjByccu1RDvffTmUioTrKPH0wTDfyfKwZxUL+gc42EwhMdAqTknQe9My8QdtFwQBAoXijYJWOaWXnDnHaCsVByDiADkoShzsn/1hGOqjB8YIA3IuKbonRr2Wcta8xq7S2Fku9ROoJ/FOAXfyz8Ak/fBNWUhjEdYQl2/c6zmfhPwcaYkUTCDnK7o5E2vE4uhSkdIGTCEADgWHXHVkwjelFe75d8RGG0pYk2ko/Lpst81fZt8slAUrGKXCWUeizL9C9+RWNh7rKPpYjttG78q6ubqu3o9ivh3Hoz3s+IQGox6zV35GNv7JrqnljvUN11hA0VRO98e7CivBsNvEIpPfkP8q2Gu43PvyiYPHERdHTpFVo+B8+Whk9LJ9FTV2hcPnvWV0IpcHjAo2Eu/QkJeRw6yDTvXAB2Chk/GJg74jEwETLfb9SKt3bmO7r01X5TYy8Ib+t2vyoCjtaCBj7tKgHro4leNezQoR6FEvaRHR4Ula27WoxJnsoFs/vIoIIAVOfldN0E88SvUOrKvFlBQeTDdKPA6v+n698ts5Uf7FkQtvhc36ueEOrFq+n4em63A3vCJVjLQ4qJjicfH+PmynLWI3ZytGvykIcusTOAqgnUELLa+jybKNYrQLnjFVT8ujwp8K5fyIw4D/Yplw6bp31kGPoNJBOuKQQTGQR2awAfNClme8b+62u2e8x74B504vPwd8AROcb6AlY8RCHoUnxGagXUFcsKdUThpGFUWCKqhET7cZ+hcYL8FaBwgZQp5huijPL03zhhN/ualrlTh7artDjgHWWEA61gQztwhfC/HZ1CFj93l6yL4834H5LGkOedH3yzZvZBXFyA8Wj4ztjK4oWNhkJAsEMLVf6x1fYhSNObU0sglqlpShWmKEfe59r1EJtmiiNKsQzVRGkVHNtiq4Gj/Gt371L4sA3wBm/r9HoylmeXb3MH4kGhBQTl87prSJ/w7UBbI9P5M8Ufs2WPQA9UlLUZPEM9Sftf1LAnRoRgxSlgxRjCMlIGG1/Wgc/irb8R9wjyM45o6eFVgQpNP0zp9oxSntTV/7r9zpUVbZ3Mg032iynkR0YaCccf+5vpnZUczwm65dEbn9iuDG+CiITgJG18o3c7S+b7fHGO32NbPQvCwI4uMkyg7KKMVixDCO2B+aNJXFH6RO2dovWvXDSb8vI756r3r9nuty5G4CBMQ0r2xuQ5YAUaCNzXuq0wyf6jvWubQuvjqyLpiwb3aEqdgV0CijpE2PabCBWRt3JVauZpWb1LqO8qtYw130S3Y9qV8a+b8fVP7a8QQEn0wHHl0dLQMv8jy2so56DB0Kfv0DT3AH5xy79UIXge03zTPS5/auuUsDvozJFnuJ7Y8FJB9EfULA1s7TfDTYVgl5+fv9XgRJ5pjCKD4Iit8bkve5nJ+sMlgBjx++fujhTbD/36zsclK7dBJ0WA8R2l2UySvUg6/jC4XouwG+STBkknowqhVYfxZhqg9WmpGjF8O9ThNcRuWyTn3nGPSbvmxOFYEhmsr3WOkeemwVRTr+jvqHATtlxgl4AHJf3mHuG9pxAcJIEvRBWjTiO5yZO7zfm/eqhxY1R2fym/Zqr4gZXgB55SRG3n3+lMW4Zxrgbv9/GR66Hf4ztICjMqOobW3wZv5H+WRH4qxJq8SQ8dPUGV9ECNoEeSxnpy4kim81uvdfVBRHLyIigSs7b7LqSNZvnMmRhBhfFKM/pzMZ8goTMGPHTGAvX+9lkXtF03L+F1NdGrg7ILV/RvWlH3kxnAE3A1pkNhgX1kMsihWavMnj9wEdFj1ZShwy6ee9/yjnzrHH79AB4Oms46ysUPf3qLBoohFaAAjBgui/L24XiCAkyeVx0RSBZY+hoxcG2pXwhnL8x7ILFN39AqiiLAtSwh2UVuOcjFTLRP3VwD9foEJCIoSy/yvpAB6ZvZVaja6jsxvdFKgSoRqFf5UMITx+oWzMZpbw6G7fHDHHLvnC98/zHCofpuampURn7J9t5XwDcxK57MQS98GOFDGinu7Ah5vVAH/0Fdj82JmmkchnqmyhlZWrEih4R5Kl8sWxV9HGw69iDDfsms5iXkX6mMOkQXAQAp4mEITJWY4JDe/TiHBd7i3NnCnDgEv+5xv8S/aHSou+RlXIHTyWhYd8FV3LwaM84s0fak3OX9a0Pwq9MIS5tJHlEGPBWHm5Ze06FQeEwICoDAQAN4ILlG2ZFo9PfvVwxwMxbNSFS24GiFonH5kQk+kR5/jNr5dnxtmyjdFkWavquiBs65ezquQdsE4k4+1MmIT/j20+yu4Io9asLSQoSN1Q5WNaBAHKvZdwHxoApKYBCY31kVmUpY3BC9dK9TevTCQllcPpl6O/aPA65+mRIxUBX61cRmdt7+aH2+yQqZI271dgAHZdWFMffsyQJBcjPTtRlXdFne/S6jaPUbRFA/Kr3ez/uVpBy4nom7EKJ/L5pt/yKR3fFEJMdtxb6lhhqwsX6tycgg5XOXUfgyOO9j+yrcLbEmdfUZ/XwGBB3J4wWrrKZil3YyTVt6ZrSVp0/LHYowiUcY31KUcrul2gJ/I2JcKAF66TMXklvdNjpwXtUMSr7+urrRhQqY99Jz38iHYCTNXnn9Pi29wOw3mmnKcZG/L3LuCznRqo3iAd5cDMR3FPKYWEOKkH1V8UYe7pR0Jv7DXMwIhhlETM9eTEDrd1C/XWTO/sARqHJKG+rM+lhmcGg/LD2S3skWjsE6ktR6N05tPxMO2TPaQ6zNXaBYNjtHrEX32TbZVAmQ1F3AusXZ330eWZ2LOrw+ZyxrNcT0DHyVlKoRX3o/SJJx8B7olN/NfIB/mRnQ19iTKEi3uRSdvk9kUsuicSE3a21wWzRwCvD2yZs26Y9aeYFmYBrgj0Z/6Ym4aP6OEmMSNCVWFQPjYI2U/qyt4aIKnyJblqu9Dbg9fqEyVQM77O+YTaqp8kbUrBVCpbmRt42g+Tb/sXpkTS+wvEEweWLTWMI9Bf3DwfQKPBkaDuE7hR+zJALJ3CfsoRbnLXxKFbDeR8ZHGbSd036m1DYpQf4EzdUQfE8clxaUrzo7BdKlOcd241/1058so8q+iY239w7S/YJ/Sxl4+yYzKHhkrNis9Lmu8s+KsFjTSf7WW+8Yq0Vw5rKVArVB7yZz1pX9h40wyJusFvhlvUkEfoHIsL0xsPj3zNRKclCEK4hRcJovxcmy0ichO7KGrjIUY9V9cF0mj1/gjrsv+OM6bmB4U1724twOCWV3ulIvzIz8LCaVwSnWpenqhaaO9FxgeFPcsmjQ1yd7EWvlTehYN/HKoi7eZ8P1tm/YPXUgLKEUTScZLDg6UjkOgd1r9DgPGRo02qqliXgQlriv74cMYk0fxlg5+mEbAVcgRAiLKpiWGBXjGJ9K0m/ACytG09AoXFzg6SPb7W9LSzbXUxKP0QepH4NgS3eW9B0Xa3XtCfVgcKCMgARuMHAB/zcTMPYEUR3jOmjJj+vaXfRrZKE/Da8mezLcAgh++y/fhCTz4zoH9lOTrhIQ4i4k/WouSowgoXN1veUbxuhzfGLQbzIHZtDYO43SYw3crehf75rJ54lKQEelaMy+JypB+8j88TEuL9Qv81pH7ldRSbEwSgyMcg3WplMzKHkWRUR+vCkqQziATD7IUO2iI004PHR8f3BN0oMEDDNO/LXXlCRVE3sNk5Nm9VsJXvEoWYbOOx9Ixg7eSKtFnILi42T7RkJctkbBwMhSiMQASVrb9B/UgUZdbQ3QBVDAbjGJfCgUEYE17E5l0KBAV8LBGyypw/Jjbn3y6t7dNVlk3jeWpiJ12k2MoZTHmzAlvhwv8mlXAWTvLzpODgXaXCLGTvrA74MV9b1qxXW3IgJDXedMnJ8OuUs/6RP0irJfeXPWdQDIRPs/j21dmppH0qwDa8BCirttVGxxmsdke2rAipej8m4fEoX1gnIhJ8lcMwMwDKDVfYk6G1ai2mnfWHukHTqaQM7Y/qvKBydRBL1/aFDGOL7wQzQEuhUEYzxN5YXd76vI6vy9TEiD3LvFJUFkQ26DydEvrN/pBVzG7uvSxXvHtKHRnNtZtowHsZpTi9OiSZ2ulbM7bHo/2XTGyy4vKh/SrjxOY8Lic9mD5FmA9vk8GUnNkRdQm4CscGom17ll2mwF86O1xHAGTQTLfnOkuQbB977KcoQojM1eWpiiOTUEsTcVk4ESe0UlyrAtNoXWMzOkUnJTYPXA13lixYxUoT/GgiutMtCCYyztWhMSMlUwrxCBcfOJT0GHAyJTxhjGNtbpkUJlYAbAT/iheZ9KDSxjfIfiuJgxDus2zFUFKiPIzIW/a/XxGwj6at4+KqHqP6Zz3wZbNHoqUd9bgODRJL8FfmWOH+UAh6oHI3nHBRlqIkykXVOOnd1sPFzVk4rH3erUvTIeO6+z0UK26JVCplcbWAN0N2dyXX15SUYfx3fyswToP3Rh0VZ0wflzkE6A9oJah6BP01mckXCxXr2U1LXEcPEBzk+akFAbXuVJQoKFYQH3M8pC/X9a8RGCsi62yVtwD5WjDVKjc/yH9uQI8mcT5rNQNcUzYYimj/zOGCYMIdUrNhEKj896ljietV3iM+s9YDAXseNRTk2SUxumubSCoXPI+NAoZKEk20PNJmTHosN5AgMjxdh5SX41N3oHDtZRukqh2Te62q9QvVl/ap+OhZvySfPCp2Mjp3lv/xTZEI2TM+f1C5R89bva7APj6LAqnzMwoy+jJBeAYdfRH/HAlDsJSVJ3kvYZSy5SaH1IkFlh+6fGYITxmTeHro8oSEiFNrnobF8JAmy4xLtvCxspqt5u6gVjgNU/GtkWz7tJrY8TJbUVe3B5LPkCymf6SHi5rvkAGHmZvtfTJ2mNb/UtGfZrQQeE9lsa6DwEMmgN3feCbnjNZSU04FYCiFoc2d7w3Ma0/F/w27ptnn+9lDGXnQbrlYUKChFk2jb9IfsHW7Leqe44piL36RA4h2lTU5571weScjmOSRWCi6+MK7nh6HirdLM7i+OpfyAmDugbHG/JkXY/rd2IfQXzLzW7VsNeNZnVy2Jqs1qy7edTfxr3zcbPUYG5y786PCQw48aIZ4I1v1LPj7xfjqJpRXxh1YfHnRl1xKh8YVrx7IZy2j4RBgz/5wdrv41RosOUfpErI8NHXvPp0su4ScOJStYzxyE0xNDt0+rbwL3d6Qfilv/EzEUS+/JilA68yIiXeTKjPoy449+uMAnHqVP1OHhEYxnQkTEXljTjDlX79jvD10TF4Zs+dSOnbXXY4s+EOzpb+MdX2VqrfxqgfHDSfOg02AFiV7pzBiYH/UtfUQ8e4AOByOeSa/UIzXZ6jQ9CZPVjuInE6Ywtm98GAI9pWfRY96NWxSiV97lkEBhepNf1qJQSVcwzcFVeSfpwBvjb4pT+EDd5hSWo+rQNxxErZPP357UNkHzF4UFNqXs+yyI5pIO9g0vL9yNqKuuZz3W5peE1KYDB24+pvHNGE5K1T9BVqKhYJfb9ULqmTrhtKqkzlH0yPywmtnQ5Oh9MQUdKIqa2sez616+2Ge3fGPgTtn5e9auujBP7Y5V6UJq+vLV+vX9S9sWxe+jB0N4VXK6ROtqdonYcGQdc3N25yxdWHKB9EKFkPL80XTA++08atzg01qVudJBrKYhTzdP/WIRiedZAaVxQObzoRR2SLQxTjgrPGz9qJQEhQ31cce76F6rHu7TBf18/nIx3vaWqlzfymEh98+M6LWUc5xMdkzdj3WxO0UES6V0lkEu0Nqj6AjTHEjEBeaVbHKq+I14jnIlE56+oeE/ZXUiSGZt9O8tfJ8Sg3eSBGY3NewD8QJ1QES3V10cu5eUKtd953XC1fAHETojchh7I4vE27hQviNtZinXNG/M5CzNzcgvR87X+kLx4HGrS863kZ+UAHeYBZaadLEUXGYXn7vgf3cQhK0InhUoS1WOLcpKln8AqR+Rf6iV+dtT0K49cxgMn7PWMuSFolJnN20sQE5FW7pVORiXemv87+YxRxw4VA1+rt6yF5l9Rr718e06tjhUPzPsuAcVu4CwJn9lpa2y04AvBCde3CJKWLYEUk2lNlaPZDQKJcB9crB6FPi13TvQ0wbEEP+nk9CS5t1a+OuUwxJIY1zn3aG/QklIAz7d2CZYXow7G/rq8jwbrTzzkciGBff/hW8Rbe4/WpOI7h5mka2n26pu3Y0XtV04YQv0PJDNCZxFQ3PQgWHzGRsdkj9VOPm6FzzYTpPnrp/maA5YpccLtqhJDcV54/AvZOSeBdkWOl+rQjsDo9o7ICeWWn/N0wWfY5UBEwZ/j7KlOJTRooDvY6cir5DQ50xCAngygtslKHWUmUJ6lOLTIqzJflp4Nr0MppjqOfoEqSggFqp3OEzumWquEB4oovO2t6VdTQbWXZ1DDlc8UpL9GGIPB88MJhto1hWzUUVLAc7qYTWl/XEn3mSO3js5CoJO7rm4w4autzfPl6YllL0WMPsKeJYOI9oCRr97N7UH0u9hFXZmMqicaj8qthCqPq1XMRLTBtnDjtd46Ib+0kBuHfHZNYLkbK4zfUdu2h1ij7kPwuTOu8Rj53xXrZNv/KUi2q1CMaBRfGPOiHLoYpcPwI+qoQWzw/TBMVfFEIR1fmvbeAyK285TZyIQS2wsnCUpfnSmJiovgbL5iKV2gCdZqIcR1HDLhZAG5Emn7BrVWpMFQOlwB92CaeGqZl/SieU2gsQQFXln6wd+Jst/DwmprCLpaW+LjcJm2dPh5RdNIfm2Qn4HisT96V7WWziRv6SrpuaD59c34Wqd9wT3YmR/aPmNul+lK0A7LM09mRZ3VaEu7K3x56SF9cGUoD5BuyrhYjDVtv92+G5WttWdrnxKk0seCJjSqKvmnUXbY9Sv29KIxdfuEGz+HzFqrx6+ap0kDg7mqhWjfRRNOBSmtTrVRL92t2HNwSKpmOX7lMRxCgRx/v7ZXZHU2xY3c12YZdiUpH8weVrUG9ttIR5tDhilLsa7hRXF4HVV1aU1PEZS7ijxvKDa/jOR5K7KauR6TQQjbEOefQcjDokgYHr2mLONxTsCVmPmcSqd3oBe1wJxu6Hy8xIzlflpEh6ebXMpTXY3vjs6l16gx4yd6Av7kJtTTAsk4FuBLxo11v7VJzZWhFK0vPIW3HGAcg1YxtHqJ7ObKuM8s4/Bo85OFWlzgmrvA2+Wn64JZU18UQx0UdfN0Khdqa4NbuMz8ftWZk0i6f1sXlpJ1YDO04gjBBC3hJHeue3fUxHTL5xNqPNSBbfX1XqMawTa5Ob+e9mUDVzByli2oQZ9lRh2uqEK+kNde/6b6UEYMU+OGdVc2yZiTjD6/67aVQON7Y0/KqFTdxHJQHW5Gq8PPj7FD3EBmHhBgdu++EXGpUVlHTi1wLZ2o7BoYsxhWZTWTMYnRRfBCe3CMKej822tA6hbuw/V6PdBHp9RSF+K4azfGwzedz77tqREsKnsVeB2rtf802midsP9/WDh1zM33HNFuYbog1xxzKXUzDJXY1l3KXj18WDal+xgf1mpdjY7pqddiAOvWZ7N/A3N2A/uU1CTCNjGaaJjOEoVwlTJY/SiWxrb/D7KJP8FpfgSTPrGn2SO6NYQi9kmx3cGciAIG/MII21Br+aKVV1PZw9uo1LkEdQTMLUD4H+gX7EkOYXTvFkffBlPXCsi3B85oB0NewWbQp+TkJpEsS7Afnlq8CDGTKVGYNaKt5/oxg8dk+sUx1LkG3PS5p1GbO5535R6GK+2yS4sIxvRlFr8xdlVn9FH7Nge0i44UswOmUhQao98VehOVJTUMrjfn1cuBKFPh3qvChaQeP7QRCvb0bl8mMyfdxedbjepbAb8J8jV0Nah8YJG1yEdCvVlL/9T6Bcj0CL6LZC9HYFC06nmrziNTAJNqDiAnt2TYImnhvraZOFbotC6caT2L4zj9LTu6Viag5crCQh1KtwI+29GKD4EOiDQ+0DRn9UIfjSrBtlmTD5GCn+4LUAuzMNEo6STfb/3pCP9C6owgWhn4vhhDnvVz8XaxjR8sD3DowPG3ygR9yirzljvtzTzShS5wgUE15+yR81e4IbuLrzJdGb/I1ROFwOGFCbl7DXZJysM/IBbQqccG7NhOtq3wNSgoxyjBclzTNAOczq1tk3GTuZU3cwXNNJNtLLXzMT4x/FZ0wjPrLcmOJWYL+jrGxAz6XTwG4xmiL/ks5+68VbgN190YnrYwE1wV4w9TIxBfMtqUU7aJeKJQAZ91HUibhIyEvM4kspGa4UJw/nixtSkcMQ1JORsg7WcshedmZwMnz/lXOmBh6+Lwnt+GUVfFa4sqVfnQVS7j0WVrhm6iOCILFKGW/3r2CEFow2+h+BcuLMjwQCca7x2EZNAmnO+e26y7lJByBMGZL7DbTRxrtk+T3fLUrGZNf56XKQ7AvyQNtteUywKXnsAxfZSsNkz1JEpkSt19fGy4qdEiPIj/cDOC0On3OWSVnLBSIjfdwAi54e5xU853YHsway6sJkc9nYcJpLkpX7aL3hCyppFEDAiWwoSiybr0Hea6kh6my6sLHHzErHuM1Ipr1qKILWTEOZXSdzagybaDd/pUlFFu/MZdO2Upduiyn/HDKC2nFCwA5zRasOth2yBqNAozYPuFJuewEUZWkN/zKu9KSJY8ot7izdeibrZgdnyliYGcYJm1L7y4DOWMC2SF+GfCaD48NSO8oSRr0DsFooohTFYFQLC2rVihUEL4r3BkkE48rvkA7ayVJsLrzuSGtZVVIp8/YZufmX4CKCSoFBfGRfsj7vJHFoO6kwPT8bYPAL5ap3GOMvNY2q63RhgqHGW8XQO3zSQAmiCR/7Oiko05dl4x72cwvrLj/9iaDxQCJ1sJfqGLMmO0LUYGwPkPXIt29vlG71MniXssK/RVdRS3M1k8r5OSd+yjluGKPvb0ZKs0NZXIb2+0vBeKrTPu2LuDDmEFKHLGw8wjd3NqQyLVrs5TLs+OOpH3ugK0tkqQtGEgHBXyAgIDtF7CWTbYoH96ZRkczprz64k245f2F9csLL/3TbIEsrTXzOWqk30SXRMskM36VZqE4ZK3zoBgmc1mTau+YimLC2dp5oLWbN8rlbGx8IEQAo+jHM3dyVhGnCQI5+2qjpYKYqNeJxA1jUCsTyGKlMayP28MKM1KHZUQdpMLMpxgqxUEVnKRDqp7n29M3v0ifNdvZThMMMRXTsorV6QTLuubtu9iyT3R9cmQ0kRdb5e1X/+XfbHWbhldkxUcQQkHScYxXC7uIBqgot19cX54Z+e1mNL6xpfQAmawVZwPbH0U6ZOULXQTxfp/3aX/Kw4bRznvMFVAyi9VXySCKfN+ndd2BpZIqsbTlJfasSADLqYmJWGAC6MXk0tEJttgWhTejeYPObSmmBXOaPzxRO1IwnobrQzxo7ANbe/A5rDqIJRfLNTHELCRfk6s59B2v0/zh4VrRj+mNdkWcM0NVR/sE36V/f06mr8sp3GKm8NNLnhnQUf8bmgq6mW/uvF9yzBCAR9ZcdGgS+cwnDSXhmEn87WjhEb7+qFG2XCR6SgjSV7XBOV3wVsydsfdYwqAq2de+39fsMR+uZvupBZIk9V3O5S9VuURoR9W4eVm2qVr+dhc5SfJo7GRZTb7e03wc9uaFa3FoMOoJM12WvQ0wSinYQJEgRLLvBZx5Y62A8YH59Ens1v/OGVOTdpNEOHEqdWKMjn1CO4MMlDJEj7lUfkoP9cavELvUSe2jRqCLNgVJ9IJAwxqa8SWUDhcvZ/gRZhP+yZexxl/WK2NISzwmHwUGspz0gz2qSvH60g0MhbS/PVoVivEitARa4zlx02Ay7vqRxBF9z96IdOxrpK6xRE3IZxqZeIlHCNvvm++9QwXEaUpyhREQ7hdH0NUmjv8qXYvNieueIfTIOqbQNrjE2nek6zqRbjoNgrzbvRONZfwCM4fED4zuB0FeuHfqX1Lgjq3d2+WUwKwSTr+gguaqty6Z0YdN0zKt0hMu2h8SdtThRluinoYv3hJtzqtj+VFftPZl793wg1Ae6y7IIA0/UgSfNmoocPEEu/Bm0HNi+DAOtI5iQbsI0tRv+mD0EkYQZN1ogzmgzgJ6LHDhc8KYU2LmVfsQD7oFh2aGeY3xrLO9XZgfJBwV0iyH3vm1b+c+i3dZjmWOJQ7FthV2eNkROJVN+a+382dGxeqeXq+Iqivt3WItuX9PdALepS28orpN1E3JEIDslBWa24d3lM6uRRyVhfv8Ymb2Bn7hZ+aa/MWFPVNQexJwuPw6LNN5EKoLhuT2eFg9qOyolhlsZrCFcF5Rhdpce8n2zfvAjEfxMstVGIaLc89y4sSyjyyLILSeWh1t7BsDddrxl0GYkOsMIwvBBbUzpF9CaRldLDs5Es+hbiElX6/8S2yLrNYfAc2cryKv6UpBtMNmlzDormjatbvcGKHWhN2dq2aOfTLF0r0jj3ru797nOl7SZCpJBxA3z/Mvj79h1dxPZQK86PxUXKhQlTa9r6tw5Xe3uA2CKc01Mn7L45moZiI3Fk2DQb4vZGXRVzqddRvxS1JXlGR3JTQ74Dh/eAa8/47hFrL/Xvlh0pYkgH/0Zg2CXxrr7AsiUFhfznjLl8NSkbKJkTWujaDKeXTt3aHbdVxMYkTiCz3Qerg+jEdTaKahnixcRKNuTP6mNiN0nXPR0TSinXuSCp/Z55dxOCa/sDBkbGX2oiwRN30K6GV72WuwL/meGEkbQ+l9yH6ge8wXVq0x6s+Yl0Iy5WqM+Vy6aU16IZjveUCBXy6MAWBf5AllJ96rZJKh7+zjfTZHhkyNq+XFdF+4eZvfmQEAd81z0fRBwLKbOXX+dYivY4qT1nFJ1pyE8iQpBzv5DdIOzKrm6ttE70LZw/HRHu1xc3dGPeagsXKNwcU77kkmNO0PYRACCF48vrEw5ebJe3WNq7wLNyeKkwRnFoO9c7LR9yKr2t7rendy8ssLtObGautJ0qKx7x0O1yHeaEN8JUyaNPUK3L7BqYNBUico6LreWqfyI31MMwHxXMVEJskKI/JDoFLF8JEj01f6KCA21UHsBTtWRNaaBqt9SW7LSN+ntA93xgIBVh6OPnZ0gWnJp8495YO5tD1n3y49RiwKkAhKUbnmpLv2fIotsAkVTDLaFCyppPj8PHZ0ck0fbirfNHXlwGC2JxGdORDuonZzW8qwybpyydnX8Y6bdQoDtbgD/z3B3938WJrZyWjNwaEDZY/N6hLUrDvbJhHBNKPFV0M+TFdqfDQSGbHM9/0eYAvextnufdj9XvAanhH5aNdpJjr+1pgbdIHcny6fefzbPYmYl0YD0m8ywqunjOBW3/c46KwGQ4ywJni/x9oHIfrpMydup32jd4fed6vJG+St0HlCoo7HE9et41mquv/t5MdsbpXOjAiYT4Nn4At+Ro3Dkm1tnx9jJT2rQFJ5gZmbs2VrzaJTTRJfHBpJ1yCA46UvyJEGYw+IT1nQmrRbgdY6A+bZIOYk//LOFxNmurmS32l2yDARSfAwCwrJzRn4O/G8JkQbpUeR9jU1q+eYH6GqJaxk8BkuPvea0k7Q58fovehKk63uJX+5qUt9KDPMNspd4ZWxbMAWosmWRojinc3ghzI4cn+1zKOeo3nRD9qtok6PXqWwu1IzLfK7nZpykg7fnstMnP2TnnTNt3qj0iykpadwxaTWbMrR+MU0/Kp3EbrTC7c2OqX4FsZmlkpNF5f7bhYjVyWjuJhIkYwI7pA0QXXnMWffSRI40lfFmiHEIqwICjQO97HKSg0eaXdU+8LlnTB6ncd9ssPoRnqrrwhkzAOz8u9GSPYo5HQa0p4ZPl26bGTitcG9n1E2PaxWknRNG78BsEZm6g8mK/SXV3eMMoNNrFSCGHzARvK9rJuJHv7A98LmlLucJgNr9O+QrRIil8jw4YJYjGk49KzKQQe+hCl8embEBHwbbRRByifxPiSqhe4wXC8/22LZCONjoalYXhZRmjYhH0jYeAGZOLSr2LsbVd3ZM182ImifYpydUsnAHqQQVy+KiaFk7r9BiZ0rVTvCrxSvG4UgkTFwpqR1rrKr9j2Yzubwu1zvwHKGK9Pi+JNITeL7FTD8X4P8WD7QgBcwLYfFZ6nUqtVftae+rviRiw5dyGzKJqm3bpXw+ZAUiIgF+5gGRFpsS0l6lYgaF/ufeg4sz62nxO624izHwEphhkMqjTDGtD56Jgm/uyVP2xLZUegu77lpRDvQzaHF8fkdPIqAZ+vejuXIlgbRY/kJgl+cX7YaTS9DBUyvctHHL0TzZap7hysTS1p4Mq47ibTM0UdcRpY/7764JihKS/tssdOwke1Ee8NYdbu02AwQCmV1FD3N9VeHSIjgM5/VNJSp1frT4gX541zyGij5cJGD5gVeeycaIYeXKk8z9PASb/ra/DwsY4RpxpqBjRY1eq/KDvmWExEeA5XwY9JtMdfQ7mu4nKqmNluCljhYsRmXqxj9iqZEz/erMUj9SmKTy13mUOaeiPDbxidnpYhQFE4eO7phFU6t3woC75YClRle7Eh06Rn4zqltnW0IVVsWHx9qpMX31Ham1MqbjTysYhWORDt4mdBQsfDJNpOybVlp09/UCT7YoTlDAsp1lOMlgU5ecEhpyru3xZq+VU2zFuqOnFRh8Z9z6Puq6Rcpwh2mVTWZNBpFj/ykKOwOt/Zt+wUfrGz3xccpd+xxMMjG9ZB0mdAC4bVRTqrrXk94tq2g5GQttPWPD3UGR0RJKaripI0Y7T9LqBHKrDAG9OEH7lnr2fXYcV4N3Z2khNxvdwGGNYyumi6GjQKeR/ppP8hXDUmO2hFdo2qK9loITXKLp21VWEb/zYyI0c/foSO0Wn5wj4BgEmo0fnDs9wBskD6GmRh2e3NR3+8q/yAGLFZxKCAPjZGklJUfSESkDsXTyYOid3ymhYEeK43bbIB8mn77hvTH7h5umcQj03GH67PWRwJ7w5TyurzbbbtJfnhJBCJbQ+eieoPD4LF5jGS9fUk7fkiVcwfrY+XBCLDh8QtS8uJeJ1tqtqTM47iqtcfE4xu1Dt54C1dVM60KM+oQq2Nn8RUFX3T9gGiaOBGCZyGS/YcbnqXD2MgKHx3X2z1Vy9nb5h6Nra5LFb0/kmU3UYmgiq9o4MSBrRrX3ZsP0VuRsFuW9I0mEPam2b33oKcBxzbbd71Z9/i2xBoD3m9acDlpqFsZ1CUVCc7gGl6JgWPqVRobOMXOzr5uqTSuJK4RfT5Yv2gnJs5cDEAS7B6fqLklhXERLo6uMJdf0fdmVPio097d5h5ddpHm2caFS3PqaUE11s/6UjZ6JgguU3c/A/sjxxBM3+8vHAXfHIV/z5anqa9a8j+VVENyyRwutt4FMkgM/XlT/RRdwYCMiZYxLzr+GFUf15X/S9qbvaHRpDiRHB/r99ZRwiH0BgsMtEhwIIOV/c10aq1trZTv4XRlCMWu5hzQ10bOL5pY3pl8HfVUtwBsirbyahLSyeSLcARwJxfT9JVXYfAWtpN+GbDuR8DD7UVqvhHuh753TNTlRcuPfACFmG7f96FeVOS8sVtdFlpqQCh+UQ+/TSMqc4jaJggqs9nGbrUja9fs80qwl4q7bAGzJ3vEBLe6bwrpdsmTPPzje2MmkmdS599oEgm1SJvzfLCFDyFAvyVDKKS2D6zbVYwAfiTdTQ8tSEkmXgk0Kk/3/sWmlUxpSOIvfls7mOc/9m17uDA376Io/gXlf3/Yf0EebgmN8Zz1K7iDIPAft/ZsXrPzb26hwr+gXHdK2dBl63w9j/z52/+Bk/i/4n9+7PrzHkFR/wrC2sG9o/qu5R/3afpfaeKPu2VWFeWf34n9+WS8/HFd/M8vAZvNf3w1sOBPLmvbv3ry+zcCVd8/PhM4/we9BtqNIHwLpeL/oXpz+z9g5M+3idst++O5P24s69X+eSP7Fpnz5+Uwr+VQDH3cCv92l52Hrf9m4Hug5+rfnlGHYXxuws/NOlvXR5vcoJF4W4fnVrl27Z+/zYd+/fOXMPlcL+s8NJn/57g8Y8tmZ7UG4Av+FUP+ug5/1zT91zV//tmD38X1NxdmNlfPgGXzX/f6Z/D+aA6l8L9ugPb+B/SvEEL9deffWvxdXX979fdt/l9KyjJsc5r9R5OA/vHgGs9Ftv5HD/7ZIpiR/1Dy5qyN12rP/l0//pnw/PlRc6ieTv+bxJIQ9HfiSv+dBP7R1z8/93dC+D878v9CLtH/XC5H8OW/7uDs8+eZuOdD3F8//gV/nuF+9/Hf9b+/B+K0//FZ+HcT++fPkhj+TxqG/+7en534h4f/fcs4wBYAIVX6LJM4eczaYanWauif90mGdR26f79C/nqWaasCPLOClcXGy5ilQGLy6gTrj43//HX6CCGQTPY3SNks7NkfYwXaWsp4BGPYncUcj+W/ptWSDjD9r8cwN8sa/9ELNq/alhvaYf6NNgpBOPSY/n+tzb9+0w999k8W8P8O2CQg6F9B4bO/lUMYgqF/AE0Yh/4RMmEE+q/CTOwfZJNrq+zPhfC3Evq86frvp/Ef5ufv57Wrvt8/MDVbqjtOfk1Bf03k/5R2ID4ARpc/x/y/aApwHP8nU4D94xQQ/2QG0P+yCSD+c3D436qSsH+ukv5NiWAo+rdK5EEBiP5v1iH0/6IOwf+bVAiO4P9ebui/E4f/ag1C/jcIyb+xEgIh/oaV/PW7/3VG8h+KF05if89RIPT/3+ULIv6Oovw3yxf1/wnm/I9y8/fA9TdyRGN/B1Mkhv0/EaN0m/dfp+H/SKb+U1FB/7tkBUX/js7i8N818of8/4Ow/JOmYPJfSQSH/voP/TspxP8XYY6Z5/j6m8f+VO//N96B+HMB/Jtc/9Hm/1YpJ6H/j0r530g18u8MOPg/E+g/IRv+S0f/T0MShv7vG5L/ayvhfx+6/jX8/+VLBiYeVvNvYg5Bf6fNMZT41/9mk5D+z0XxL8um6mIwPuzvJ/OXkQT9M4vpPzXC/rnl9e+toMc+QlGazvN/kFXi7yyp/wt76c8O8994jf8FZf64RMSxLx6Lsfqwhn1AilQMwE+lO14peMXzL/A/I/ccEz4/eQPKPl+GEWKlaQXrY2PIdtMsfrY+7PU1iOtAzU3qrrmbN2K97x3iHCuy7MMXJFfWCLawvnk1Y0Y1dIwhdbmwl6tgfx1rGBjNEeWCsrWXK+8gJFb8kL0skigBFziMQT2p0/2Mmsb82VByQrZ5wlOC2vpM/8BkDfc9GjxIsZFwi7PFIhYaW2jnOPDCwUgF9Vy/CwbcYweTDRmVCXnwJ+Y5i+MPnzkk5uiY58f/T5993aMmNgKvJcox5TvqrY+EegkW66LUxzpVw65SyujweYP4vaIpa7agVEQIoZazjvn9K6oFUpOk7R1AYGsCDd4eyspvrmIPrQ3j5y9uwrg+EBMspHYF38fvEqsk33Th8alTocJ41hcHCr3X8+B5nS/KRet0v8JN9Zt+ZqFAsoKIOHWm31zkWdbaOEsbvkNra3nF90K76EZO7ilpmq+e/nAj2R95SrGeQQg7VzYiSDPUwxGXvaKSbseTwonge0XUO8NfcW590rEjx67PFLKu1fvO4DXhXfyvEWO5/j0MpKl/v2ieutL9FlGVfIUwXKgUu3czUnxmLF8VSBUKkHiVAZEoO8QQn5fiLB9FQlpYtD7z6nIqM3KtidU+50I5BKKXi1kEI6vicoh69H0LxTIyYfMGUdiWDnICBNs5KieIbmwXsp0zqiLoVYm7Fkn5ghKEQhPDrwt8272MtEqK8vtGZfQYxC9H9rnrE+dKQVxk9AqDKQO7jJ85ygj5aqF14t/5+vIF52UwzXs4QL0Q9gMNbOmo1eYcNsvxQiUmI53xmRB+ELaEDZjL9pG02rNDUvOvHtB9jRCrRwwkeZvw5KeezcrP2K5FVxw3r3dqrJkgEkR+1Mn52cDJqoB0CxCIb3NgzwCjGyZuRJlJbdoQXyAv5YhAKt5evAiOJ4l333fQmIVj3zUaW2oM2+lvKgBxBayZK5350aFSl1vN2hTPSdaOgMWMJs25Z1/IlmnhV3SE8eVah1ZQFVtuK856HD2xrnGIpmmg5BbApAP3VTymn0JjwJoOBZem0ji+fjU3LqvYT5TbcFsJobXseu5LolVe5w/4aD7aGUUVSp7GXPzaUIpbCXXJF/0rwu9ev9yaTHwz7Zyw5kLmWYkppbyx7OiUDAQ+iSO30z2Kau7ix3Qs7cZC2NmWLcH5KlxLk3SFiaP9FaEGI2xZEwXAE8eGXs/fKYhuxLr2fBY/86xrZVOiCMH6pl+TnHJhShxK6lWOkrHBY9bgySpzEXEUVsW0oyRzOgN53OBz1OqJ75yDXik4QyD7xCs9D/Y3x6rkp8HR0BR87O73DaEVTKlyskpQvZhshP0qSVsdz2gCSqCB+X05gQCmjU9vetQrcH57ETB1Fp83dxkmnGBy5IkkT4S6NlC0+MRRub6YCVO7huLbMMGgJelkqpCkkNIvbrx4U/kleCt69Vy8PG5ouuBc6TeW0+H0JarvNAHCoB1IoJKx7ZxqJx2juRF+fb7R7+zEC0tWQUuppeaRB5Fvi17PCTh7eJbv6jfnYhjew0EkqPH5I2YUVaiTgMmoWkUX+yCIMPe3wOtsc6jh3lXZgqNP3/OC+KXlxnk37Palt35vWUu+0lCGqB9mxl+b+KErJHFXYwEBDaYzUmd0vwt/YLzUNHGDIKwe6aAbRPJ/PT8/9UL6ofXytTD0z9xlzlj79zG1e6X51ZV9rRGLYZJ3IuoIPmyBLqpXFmlcyKuAFRwFljzrlSfn+8JLezMv43TkespFWg+CffpCKIgJ2fDw1qfwOj+DC/YJPybI/8NizlJERb0kTc20pR+c2QJSp5Qy0x9DuSmNXRwNU9zy/llrLHuteYPeSWfLO+wpw+fsJCZ+uv47PnYOv/xJuNUucVpq30Wkco1BqwxsXfKsWv+BJZiv+Tssp1zGb0xyeEVAnZik1U32el1in2RMWCTCvsQCakQxCv36kzAG5llSFHa//oD1VFTgoOeXMTG1eDA4R79UGRm8ZBeUdTZq/dJ151NfgxPFr0cVZe/Uwppl8X4TF9fb+t6ZnJcZFVMtAYFvGMovRBtISD1UyVsY8WoXRdvqKQ4NEE/T75Za5EdTogvlRNWJOmzIhqcApIhM59HF1OGX6IKnkF8q0sAfkRPDQIYu8SxUzXZR3JXAqLcgXFpyC373oBlWYgCG9RXoQCP5xvYr5v5RJlsDHy0C5kvdSHaL/EvyYpcRQurATSVwPwiz4tUnZHYi9pSqatEFcqKbYUPlgE+Am64z9QnzxoRUsjYw7eXLNxmcDKnzpkEGAPTNDWt4i9v7EC2VkqWR9pwObd4UWzALG94hyFw0e9NoHByEJ2YZpuRH5LXCEB3xd/rsFpAqyn+H5xOTlRg187yRdp5W6vfDUUAr7q8V8mnFPwCocQtPZFFXWBMvNfOhH+9S5jGWKxDowV4+Ywwu9rbhPX5tP68eoS44rY1vLIfC55b5yKYUfs9vXyAI8z2EELY/hKfdsoeBAxTEZVJK3iZEbtmCUIUXoo7ZjTAi9KrA6Ss2f0vGeST452x0MAgMiMZPHcnLq975HXa4j7MJQWVC8dUpKiSmyFtaVqZ3fPvB0gm71nZROdxs+UP8Khs4QiBx5dEHIJCVDr3t/UrRF46ucc/klyMYr8pEBEk4eCltA7CDe8qP7DPYVtYBhO3I5QnbqsUvvHgQ4VBnoYxjcGoqXqpYroFAuH3wupsgOx+ecQfItfyqEizlCY47tW8K/mOkB3BmxoVrgjZlcd+TaLDua99e742K13xAYcfe52aN+N5PxZvTgsOkLGs/fBNtfv3zwLJ6gDB4UHbjFELDa77JXLqAEzOQ+Zf4pQNZqHnvs9WWhD10oQLj9KwZXtPv4tWAlKHiT/pkZav+gDQsbmXhq3EFJgSaqZjI5z2tvwDb8ReYUCLMDutK/9FBqGfPYA+OSdopmrSpKAHBmDeXltWjH3okwPf7mdVY6JdE6NavaP4QS1xDV6nc6ADsl61fNsiK+oKmMTp4CiW5VIkervyOlsJWGXIIMQ55OclhH9bVw+x1KIBxnP0BEIykOIq/mPYi0MVqbFFmy5lFMKcTySC6XZPIeXZnjPHrqKEXGj/9yYv9mdO8M7UbI9HfuTM0x+thhEi9nAHB6qLGe6Ea6I4zMFGsGnpbhHAivJ/ujSNIpSZMl8EWZiJWJ6fHEDZRPMUDYBVRoq35HqBdZOmY0hY5XcNiDohljd2gxDFLjVT5fh1fiP/UA3S5e9hyrnPixecsJMo76tP+6x13BCh/lJIpnuZBEIcyFVzID4cH9KXbyc5LERh5Mz9EAvf3q6V7vkDh+5BffMA5guSz1m9FpgsI52FSXm4JJbwRpcK4Pe2WstZAMp1HCb1N+u3xwQOk84DUuk9cmhFKPMXkXAXEovSCHwuvQBlU0ThkCm8nAifdGaPGod1mLr1U1qmEh/LI3VsP4Og+vKRgsdHmK3XYOM/qqCMJH11LIePXxdc4oYRkcurb6ZV3reHQyF0g/ndjZq4JDfDq5v/J03UtS4rE2K/Zd7x5xHtf2DegsIX38PVL3p7diJmYiO6aosiUjs5RSqmXVmAK7hEjnYhATUl/V/GVVFMpK0PPeGcV99wSxpRMKnwHoDL7rw9FIzcm10PipCYaqFP18oiuAdoy/1aiIR7IPoElWan13yDPKfc6yHsk7sGP18Z4V3LWxP9BSJXNOmNcPqob16rVjMWjtQuyDCehAmYaH3+I+Th/c8UFhvYQY7aZMJfy2dnyHlw2yv7Wcvb9fjdCazJC4PfjZi/Yr7arUKwsHqb4APzp82+X0PX25pj4VH70ZenWOVhH/HjqT9q2SAxt4dCdGMQvtXKwyVi5JmZaNQRWJ+cAa8Sc6eFfSlOl+POujrMZfdVb7bjNeiQOKemFFUDR3QmWNoTMtQjQwwZYeZfPznjEQyVgpWAv2R+mTkBEjcrgEltcBz+QmkR9ZpWdBd6b/I1w+pwyZqBEyGMxgWDs5llYvQqtVkK25/5KLn6f+Lku12uHPFSqFzHsTrNx05t/GWOSDDRVWd+fH5Jx49CutFR3za+WxdRf74hh2JIDdlEae4Y7xba4AfCZjIj/un/EkB+/GLK/EOk01MlsdXj9OU5w8YU8x7aBA0RJa2qgjsouHpXFCi2wbhZhWAAWrddn6U2stn752jc8OnmEG0wBvxPXgK8NTHaL6gfgpaGFFbxKRh0LNWtO2N940G58Y7//YtiXZEYg+rsuuzI6NWSDaXUFxssK9IaJN6WUTAgBDamWfFnxLzLJ7w68EqrWQTgDV5CzFKMrnjkYYiYdSsAZZPlEHorp1xmdjMes5V+YQfM3PHolBKnpXJ/CyjJn7Xsc+CtkU5f3TT5h1Snzpf3lHhJQbh5R1gnA6F2B2FoAglbrVYWnPcp+kx1L+gIaYysvzynu175BnLL+yLe4cujqOaHYnVrxerh+sS2BUjAG9R0MiP4pgadYSUSFdPT6Eg94NdutJRPrWVtdi6G/MbTV4Jwsf/nLOvKVBXZrd0ByjRRjMUElQVubf7icptg3BjzSYEvPi9P230d5pvj8zfiSFLZlRGVt84jdLaP+m+0XnoL+10/Na1usH/5v/CaP8Gow3ohKEL9q0BAlYrtq7SsrVmyshPMpDufASKvqTl8neRXHhzFSXvub6MThAIFe1niFB68wFD8DT5ac/AO+LR7RuhLV2eN6uHmxkVXYv0AK9At+8sUzqXEktTxtT1g7g+xBXToM8/7YZslZuMZSJrkCb4Zb//OyQNH9G1EFkD4NL0tv764Kkbbu6OwPEq5TU9DeZjMZ8ZmjWjt9DrxTA9hO5MADb/BMVY1L8Xy5sbRisgxyQFQQh4rLfOikdtLrKxzsEbvdA6KLOw73nxWtTR4i7igs4/N7KQQ6+8Y3LqRq+pBV8T54zaLXlsrgqysZcLcEu8VfiZdnxGjwz2vOfatFLH4C07c6AEINqBakfmMmoGsscIi5YQ+lKiXhXjkjVOrfusKSQ6NP7n3W+OGPX7un6tF+zs7Q+75u052PznAt0TeKgaaV/jWuhUM6oMaO8gq4Q1Fe2+Kd4MDjMzcZdBUxJWLolQawLjPVeTO7Y8qDnMdoa59rba/qa0TaqQ4sbv7XaQUC20I+hfGxMk2gBXX0auk3aoOqEpxfzTsA+mup80BEUNFizTDUKL1SxC5LoS7uJmwvUvEndTPoeGu4QQ7yYRhf0qkFx8+nEsfoZYG2bPCkNCkxlJQbxFBKuihfoj80R14MeWwnKdvF8VhBwi1GJaQdQJUAvyiXT/bivXEqzMnyzLHGv0xKfumFlgr2wSyAwjk3X4dNCwCHXmYFFIryQdyhwadvQg1GUrYpevfUz/27zd5lQP6cJqWXhazct1oKk6xdpoyBtwRNYPHg2gK2hAXblpY+8tI6F6tOZZgBMTiJETB1ZNTTWvimkNeMbsCCf3Y8U1PxDFm/0nI//Z18j0zrG9F5NlMR6jPWl/RNQEo2qAKrfYAf8/5Cz8WKbXJpxjzb01UyKhsXriJ7SCSD2T1nfhwFrvI9+mIjXZBTIoFvaD0LRFsQmOAXykHn+xdf7DNpmfzHdIpyaS7z6qWT92V2aRZIHJtzbXjuuSbO15n1rFgnguC+gD4DSP6JUUR4lQ+WLih5OqYVtUItDrPHXGGgglfHWOsmF4psOpMxipNKqFr+AHWhijpC/YG3mU3ObchIx4mG2PZv0E+ZYQ9BQRjm1r8ieD6SwjGY1NhcJU5BHN+7NhUfu31JTt1bZ/plcKGqXbuoHgLEZxwyguUS6Qggl86Nf497he01TBiW/Vblqh+ae0A9G/srNv40BWgBn3V3L6hByxTngy29CwL6G9NbL43nuFN5nw7bW1bCMJ3k9QZemvnFKN8NMZDS/NJ4Xz/U7q+pB/yFtFEWI35arQ4HSwMkc8HCf4yYf/9R3MQ+7TBXMQq6QCS2oa7Oc560Togs+t1BSy2I///z+c+guzgZQgLCXsMEHXRTLgixEVWwzO9DxXnH+t0+1894f+F+S+GzvgpQfn7471jpl4h/KgswHVH53hMW+sTzREkonNIbLU5cYz5M2qclmjaxbIQvB2pOy2KSiv+A4PobJu9l/BPVDjYtxS/GdlpFyJ4wpWZU1GnNzXYnHgYC8+qHVT/Tu/n0KlEkVAPiYxv0Tb8xUcPP/LGJIM70O3m4UJhwL1Xm5EW17KIIglWXNlmjlYmoJHE/RmeFIA1CO56JC++PVIGdbVk0f4RO6q8WCZxool/6NRag/9Ryo+xIDwwfLFbiE8yuHbr+Ggg6LiASd62hxlY4D0fq11UKI4Yo0/wjsCTKXQRzKxQ0vYK3m0jI75mFSFHibPzFF9t0HDMf+tOAWxgy07YASuI9oWXCZRy+alFcF2w9Gk518tutjh9tG50YBWjBJc/6rvwfwQYHDV10/5WXryZnAzLzqhIu9O0Ohb/NuDkXYBbo1ZlK+Shwl8sr7GLJpFC+G7Ch+fPTTdxKhfxd2bzw2hLu/cVK1o1HznKQDvPAKqYWCclDXGm+e50JTwIymKDlEidnxx/SXDKwLwYPNiuxUwei5/sr978pWX8qgGw1GbOur8kQRxvsfw4w1I/xCrtW2Qc8EEFkjRdinVgzHTJOmeDEw8PGFLIA5Zxf6Y07YefkMDiNYCOkUliPBCw52oHjv57V9diNUPGl+J5JohYhY6Z7bFykPSW9QdNOSYvm8wgTuH1wUDBhDz8G5xTxyWFk0HBv5xvvKkLXA9sgRH9D3PzF+uCxrfkG+vmGBafEXEZ8zr1FUjcvUTKo5tVscI3HHhm4D49XpXNQ9CG9tuK+5BpQoDPKtrYg29V+94XXQNRkEUq5T3lAHUBr2MaBwZkH5+zw9BO09atxJBbCRGJJrjH7E8dDKOwErCv7v+ArNjSJVuRyf1XMw11hXHbu+dEU6P09UnDgQUsdBt0C9uIdS1TJ0XuiWT76S/lxVtNB31UQIbcNms1fqX9VZPxBaiO8nBoLQm+64OAA/UIl/N3rxfyO/R1cgSWK4CqZOY2bKKq/oelBrZVzl+AIhojUBADE943MnnUhHo+dK8HEKa+H9WFNCoBiIa0wLZE0TJ2bxeAnq2Jf4Mj3WvyN4fCKUQd3KrdGjMfz8LTK2GgnSiB8TmUcbHbJmoqWhp32pIGRWizK8nHEmvzyQWAx4i0KFZcRXJzfbp0NZCTx/G7MmOaWRKPgUmoqUJT/+TenuK9J7fHY1OytYJvjvlH6wAj8trWwRL3clhP/lLOILo/IgcVY/uZmxP0v/L54ua5EIQvN5/y0jmh48enbI4ZHeUHngMa1aaiRccrktfMZvIHTPSC07NRq4N6gUfX+UQlV3BtD21uajn+pwoR2g5gwYLsDpfrsYUfXof7unELgQLNT6PuV3736PWdDuMrhmPXBuZtn1rICbrIXvx+3dH5o2N9ionGeDAiVhha0/TfUV9+6AdgH0KOxrCpzY+BnJh/0fuvA9VheiIPX1l44T70trV6WzSp+OPghE4PU3GtFrRhjn4/Agx83cklBElxt7S41Vh6IQMFktUfizNi75L4Y862G6ihonCld+OWKWnX4ikefYWDK9vk+XURfnGepvIeCf1c5i/MbvB+3sNK21I7VcztpYr1xkAnR/otYnC+KOzG2uH0freTyz2i7P/hwpn6EZPill/zwagfkugddEcK5S8cCglq2upsL3KowrRWKQeyg7MaMy8HgPqj2YS7MND1PCX/3y8v5aICtRwEUCqgK/nE6gp9S5u8wN+KUs1hOc/sN6KFRV1W8Xu08PRr0H0aUO2SMGbrNuwaTwMHqMow4UXbz2HvqIKWWeKa3DyOAMrt0ryTBTvQtvm/Ebj+I4YixoT7bELboZHdse9SDIaVXY9TMHVDxq1+4Dqv8+BWGFSsRJxl4bQJyP0/8owoH12A/eDzmZQZPdg+NIs5swMCHeaUt+910zt5mVzESLHnG+KeETQdu5WEXsQyD1b1h6kFfK/3oBlU5vYhcirCoBuSC7Qu3hRZJqU9UdVQGsV1GAamVMJ8I0CV2g1CakTLTVjyR3zg9NcEeT1sXc/wA8QMMDqY87MahPqz6UqLGBujDrZ5htPouM5NF/pWv9LFg+ycIIcUIy+NTDIeyXgZkfTaia6ytYRp/UJbIf6OqpkOXBsNG6Emlp8iAeRoEuu2O8jKR5fpiJpfIHVOrIxeTf7NKW9o6m2r3Cb7/wz4b/wA6CDjvl6xq+kApfIX4JszdQGZLJAee7sSVABymrt99kFPCwNf+MX1WXkBT06xGiOJmBGJ+AvkP4v/0gXk5aBOJ0yJ1L1r5LadCEXWcQG6AT63N3/yH35NCjMb6bdiXfAk62vp7LaFgfwXJ3gpW0RgdCUaxcS1p8dXf5f0plu5os/WvJHP3IfYoPCDS40UI/YfGrZQpnxsGJNUuHmy6cAc/qICk5f5orWz8EvZPKpYBSXJ9a36QuSIO+mGyB5rks1H16LIeMAxKevQp2NAux0z87FqI4zMf1yfqGfKWMaiHKkEfk4iFEbn0DG9YEGbhLb32drLzx9QaUixnrmA9BZO2lR7QhlZ+SxjgB7l/mk6tHltW2BR1QzaLt7piPXr0lsUw198MwyGs+YFUp1iU/O7BOcigs8ExuFhcV+m7F6MGvZaLa0X9dMPFpvar9pmAhbOxZfYMEuYLZFZ/vXHM3c/HoMTjtfNSUcNyrc7WNnZHXfBurXF5Xken5ayAhlrf+AEXroml/SpSCks+J9mXZv/qVFRKolfG0FN9lkpL4vKvOxsOE8Lv3RCMxzOVtmymEsZbNk6pBkOf9pHXJ5tbPbepIk4fvmT9jOuVL6ycApOKUfwQ7ndByeLXNJIyvrHbZOvwUEe28ebL3bC4SmwubbEs9MTP+fWa4R7XTyMe4MZh5bT/osvyFTdvBrIUj8ASsbvNYmq0SsH3s6g2G/3dUK7O/bByrEb+hBU7jq/wMPhzDEJmosv8aoEgeIwExaR4BFMJIpNtA7Xc4z685AKkHFHaeh3CDH7dasqPLbR8KH8bw16FAxMNhsXSOVjIHFKU333Qhls/Pj4F7bk5K3txHL2XzhcdQQ2/KEX61GkFJeYgYzyF8fl4O/p0yTLwznhNL316Hs2WIi2dEhiu/kUEadIvcJ6ZehCBo6Nd/OojtEmu61U517I5wsRR/MARrO/BY+XOh/oWpQwCHVtvNKdW7HOtr3xScuOmyT7v/+YX9SbLGZqA0TKH1s4z33OmKch++CNQ5Oi9+OAw7Id1KISk9+DbLIUMRFW1Q9BsNePnQzklxcbp+cIsHR44f1ds0XjSDcahfLicp1OKTocZJSLo811/zCSjoirgRmzk4NhM3Vs26U06UbbVEL9QYn/49FMF/8yxe77UF6GSl0NaHJtn57/kCPrGDlssqe2pW+qNFlLbQSwkgaqcj8hWYeC6+j2Wix2xyzFhV86RLBwRv3Alk6oljuzxF50ogrGxN80ZLULyWfOyADbZOXw1vxkPbf8vF5HASK6bP1kKvB+lex05FTlFCGHOSBo053P5CJ/B6clQgA4JaItFQ70uXiOCmcOiL4kcoIk5rwK8emuodQ4deF4k1KD60HuEL79/ByKjNvoj5brTeaWWN46rKF6zOL3J3018z0bxomI6t3Qo6/ECD2qnPsI4SUpcP9qH1iJSl+trOSqCMWyRzQBK6jF3n6+oZHakl6GthLCB+FlxX7SCCfAZHgO1Qvh5VUVuKHarwFyZp8btxQa4gWBzjN/vlF7vAaENXp1qbbUMuzMifwOuYPKtbNxkajQ4bX/5x++08OuFGu4Lr4a6Oh+o8R2fbqEOacWQYKAgjBAacZs8Cs5fI+ZYOk6jeX2E+znKuOv6JR9U88YyRDbgJtvawJsSJsFme2jf3AZOSrTxsx94pWV0JmDKwRhN0kNJrgf6ULLbJ1aI2I0tjoxeB3VEii96n8YlP+AdTHxu9vPwCKtbar13kF2m3TJV9kwUvSmPBMyjgHHRmL3sYysKz2bzfVSWhDNRqU1trcD2bGDxf5Qm+7X8dzu553SkmMB+xZRpYXPo4KzzboKDew7xG3JzCI/LaynOd8vkiQQDJ7z72FCP+nwCcOEZ241yP+mjZy6jxwCPsJ8iV13VWQZXZ/6A/Au/rOeWFyT0zK0p1vS7en3N5y/LVXpYNNd4mibhj6Fse7hawLNryaYFKNIM0WKGpL13vL51H9r16hpmX63vyYVeiBuN2TuObK2NPNo8ei2yjkMK4fsdtId+wqiEUw23Emz4QAZueC+0wnznzG2zei3GbccS7y4OgXtHO5w87+5nW0Fm7dbU2njIJ/D0rqGUFpY5/5X5gMDUZTbazbA6sAs44XvFdwjW4XMjr0hqQcXTJ0Tq6sXjH+XIPgYk4ilD7Pk0SIda31L2DcFsqk51CnxcvNZLEvS3KXrS68o09RyGGWie+Bjb1WJMCsIjF3zo8i/Hv/cKnhIf9/FsedUCuUDafMLSE+er9hU8A1vDTU8q0+/U/cJEpsCPj6j+ZY6Ptg518URYqlzASb3tvtq5kMNdGCKo/P6Vx1jOsl8aUjiulAz1sTgcOc/XHFzgyuMAp16GkZ2oDmptjjv9Wu3jRsfRb+zItF6cE+WIQ1X/zdJf/RNDTaMvvSgoj6Qjf81e/rw06vCzit9LOBzf7CeNkgPBHUv9IIoVRe6YfpGxrCPuL4QD4MZpvvPdUVKsDwPlSawBTZl8GG7Gtva7JnCPvkDZqUpO53T1sHDq+WMtD1Xt539SJJ1GKIlBC31rfU1YKbBKOhRqXDPWl6YpKFRkjl/aRVF1Mhq7GkkacLwJzt7NWvb7mXxPK2KibkPCzR9RSDXogLX7sm16Ife/Aqebwosif5JXJUG7UOkoMBVWoBowo8Cz+ix5Zjs4u8/vFOnmksQ3QJbqCJ6jg4AvZ1bVetuV7Ah2vF4WSifV56dc65KsApLZNAGpUgfTHSUMMrI9RVWSX2u10ywlFL4iczPCN0YYjDUZIAWdbdTAgWtrWXfHX3e9n/pF5sL4GCCHMdHyaRfgvAH9oQMoiHH8IpPujn72DHWhSRupSVoL40EK6V34SLRV/OOoxRdRzv9LUlr4j/kY9nXmQPaj2AL+kBx8HbPZWfMcfUR/ntEdiy5M4k8It9PjyHh/7a0rmNHpso+1Yws532eLlwzO3TpdefltKRqV2OYe2b2TYJMH3sYciORh4s7l4MVEvz/tDTlM+yUHHKQp/AKzprl9YU5+grp6eGpFsOwoIOqb+hdlQDvPOXyNC+0+iHx8nSDTB3Ka73/8hgmOyTRcYzPiAxRhQG48oxT20sMsohCHXeu4mN3ZX+su70H1HzQ9L1HOJKnJX735vQ+gQp5/6cNBpqS/AZ6VK7HzHFCwGnH2Gt9BjzPnR7KbXLDWbg6vuhqYu5DXpKlzCkXL2AQ2A6qZFwaNv9Vd4A98TRzYbXbKpK92u4rAX4IndBpKgbrn2Lkrn2rZjXhxvpU+xWWqcXV+Klzm9wk5SbqtgYr4yw5naQUbI8MDhTVwgXt+3IL8fLGs8UoKtKiI03CXQEUl/EflVfnLTfgHH4rreSC7/ukfive9hDdzU18s8vnNazIsFvE8F4ZY1lK6BJCR+LYWbQKC355T8Iy6wRC+UlDZUD4zXjLcCujFPENrTK5G+LM8UsOvZUCjtkjvy8cu7+bQpOav3gMcHWgkyApuLVX8WRxgBVCMpJjnTJrgf1dE9X7VIfCnSX9YGZrL2McUoKTB4BSR6JlVZCjxsp97V3EoEKvkchApHX1HLycmy5SwW1LNSkzxCs4o2gyFU31BBsjDpX71kdGSxUwtHb18gQwnQzF7Yk0Pe2TJqBe27uvjG48GxW8vK6v3knYYAul4FyQdsj2sLyhkuN+oWDCuKZ/tQ3T+zHtOp5rtByR32XwXX032jZQ7ol0TLdYFt+Enm/pRMDJvGI/Owa65xhs2l0Lz11gaKeOKdwf3vJ5/pzRXdeo/u6lJUaqtaXJLOalhkrsxUl/Doj1uVEdDZVjvlyK7gUGJyc030+RbFiGvY5jqcuLtS0v21umLGPruy9BPffjD09NWlfBnefkVfmvpRlqORro1PJI+DOZUusjoIVmpfGgKrSUItYMckkybkPNmMM2iMO9hdxWH4BwaoYZWKMSnVP/R3+HL2/p8T0DRh5h/OB+Bkh89yLnitfy+OQsFsvEGVwfRgP/WycGbSR3wlyOt61zkt9sWa8lhj+bln2oQ0oioixz1JlwHl8KwTXmtHQl/cCo8R/yqvw7Nl63NFbRV38ePvlfFN1B4E4d7+4LskDtszGVykhecU3sonv/zW3X84Lcsk5fsbPzgoogNRpOywflG6w6Q68PjL6q94BLKkFSZGyQgeDTwhDqx68I98uIQN/CKBil4OB6o+jYOTrs7XybVftMveKaS1rsWK+cBpmyVBt/p2Ol02vUBEu1d4WdlWxS79DTFeBLSNr8rX8vs053nMATT0g+Q9ok9Jrh/sn0XNRas9MVG59g9PQfyIpu1FlOJO16FFx8xgAYQ5pfiiuQ4RXCnazBtjnVEo3igs1g33e/mvA9KvTrrBmlZKQgHkUYiAz7bb284Ody0WtD6TdAaxiKpxmyBHCnz5ON89+ITu9/anA8H0zOnuzBIe0xs+SXrKiEbxM/02qovliILGuY9Nw9xnhRmFKTnZ3V4mUTKb6PqV5djQxBKu+QHYrj9eMEYXA0FHRg+VmtIWeoDNEx0Cc0Xddg6NVKurv/+yczyyedqJok7dfLstYGzK5LMppEl/DaR/uUzufUWdDQW8HXkB7m4aOjuSG6d6dZgmLwn+m+IkxhgyfwUSTzuRLpS2uXXpa+K30ymjvCgX7poEFcJf/H2QC7r1vWnCCNUjlDJFOD4x30JgLU0UYvgAEV7o4Xs5UekJhypfAYqMPu/npWfs7ajvhx/w9QplnjxoE6rk88+HYqZuWWtpTr8dd3MDnp6A1PZ17dbnVmtGIkyVnDxK2sNNwL6XC5QF8Jjhv8kYNYLoBZ1JmmN8z67tEvISZMKwnCAoyzoksEDgnEq2TNkaqX3Z9LBmWYth0M2II3ywezVB589QZ+NNN4M47Osl8YLCJ3xqZ39RPwdihLij1XX/2INk7bzjJ+R+3d7GrdnLSi/IunFX1nUglGhJWfnKBO8lTwYHMo9pJWtWRyZsOT9QJUHG04/7wMhiNAsa0pRQr8IEibl98qgLepwyBL4yaKO0Pn+VB2GomMbueEG2RQpPDrSjc46dg7ap53S+J5y0vyNhqZgroqIKBoLLSqJ5lB4cGoPknkg1viUI8xYN8Kv+aR4wsyWnFqd4+nYIDqsFbLNKdbwAjMNl/53UgeSfcWJAnbdM8ML9NHSRPaxSvZlw8xrGNewMFK3XCES58qKm+JNQz6EZo32AumhyPLhJk3LPMyNXdzO/Mu/ihQpkV8W44fzt/jjpuHyd4P6hJ9Jw0xx7uUq7xuN2d9Y21/KanmcDU45fyZFZ0NddnCxH8eEBOaHzwmjXc5XvQmb8wL4bHaaouMpYMUpofmXbkaQLN/SxwMrUc8rFGHjiSmv0mJSX1AteC2+EOb7hdNXYvYMzITxHPM7wvhLIWf6+5uMJGAGSUtmQ8muDX8+LN0+p8EbqRbE2mXxz9+0Y9muEvQAp97nxiLUqpShntH88nf3U7q6Qy8zvzNO2WE+ieQFNlszdYpJySpcpTc4qHtIWcwlOL73Ow2UB+CkxYgM+RtMPkPWkICIsqBM38adUgoVMfWlthksQHnfoCFxOThyXz7d9dSFlx2fRw3bLDz0kORHJd6kmyjOPD1sZXiNrgDcMnHDQ+V1Re5lrPrmt3kKUHcgWDlm5cWidH1zhJNlzPlITacJf7yNvEGDtMgXMpJzq6A75aFTEd7/3TydAsUoFS+bLF3jdnMI0r8xS708irAfIDHZOmrtN1ZdYdN+DvDpZf4pniyLdeBmpE5pK8PKdIqTRdkoXP+RDBL95QFNQn/XcCfv4iI+xr+MbZC4QwFaTWWtB2kIgaDxr8KopzEDFt+hdeEk4KQWuwNg0vBrqCDBPxwHcSOlS8eA1LobiicEv6qq861xVTIdIz8OHcQnSs5NCOK5785lyaZc3B3XozO1Pq/+jTAi5fzLo4Pkak+riPCNLfUOzUE6eywuIIIaNkb38lTjBTXyd/G8xC+Y99dI0PKEdKTXsToTwbPub7mYrQI/2gLlY/T3KQzKBhWi9AZeoM4o+FOacqDJjIEXl1KDF9mOzCzZLBsO6DYHcmYe/gO+g/ls7scPKJSDz6UMzlntlReYhhky6aIevZTtJZT9ceo4ajSHwLRZfl/X/VBcDQ4T/l38L4M3fNfJir+TbuCtQP4M/n7tp8bYkVwEV3L8mplAix/Wzg4RCsZxaKZlQ3SGCZ8PrBDKaF2oDjHeDwCkymFIMpTHseZCrA8NbQ2+1Yc2Bw4uhgMEEakZzSWMG7tF5EF8CJVuQKjqxOd5DSYwU9bWkCOicMlA6NaHYsckY2ScoZ9woZe4HIRcOrDQ8x0DtxRHgDQLXMbXy38F0lRSssxNgee9nyAE9unfTu0+gUq2FOjyFaUntuxzeJV2FZ/dAkWnLFEgxvhftLToV5OHIH34r52I3fWy71SwOeSDPnDNBoBU+zlz8hZJXqKeADHF0WNL36hANL2XRQ18JZiPQtr+fOFRs7qhuS4UpyEEpCAp33V31qkrZTL/w2wGK6pMNviKqKKnqD+OMKmfQLy0LhZOIy6YAMWGD4is5fu+iA6dcXLTq64oTrq8EDy8anyZKPWUdf4HLCv8Rh+fFg4m2fdyDfSq92IvvoFqFkFNgCiRfjRgRTQDJgj+1UPu+yqkp/x3HMjSAgGZW/j5K708aI/M3w0CZqNWXoEj5k8Zj+zILQx+TcA1/LPVpoJJBlsB++xXLhhi+bqwYZLl1VVsG0Yr21nf//k3Bp58BQdIObASl4yF7b+Wf/G2LRcAv99frDqrZPqHVTo7SukGOK3h114JlgjZAt2XyLDZT4rYv4sDwROno2AYZO+KhbA60h9Siv1zMzs44n83aacdR9K62g8wfgXFDYM9gQqhF96jYKvz7WtdrY4wb/xBvD1huPNZ+A1PfWV8MS+0EQt3oF6t+q6maasEJ7cpCNN/HUKfH6H/FRoA5hBnqaqnewaJwys8mq32QUGwhlRWiCvM9xsV70syHJ6Td3zHV/sBF0qJdQEao9gKt/5ZHZfHpDly4DlR48RPRzjRKZpklqP80D4GG77LZqPR1ELP3+B5IMFARITHqeR0HfTViTO5Fv/sCzCaaNaf82V3Q4CDLd1IhB4zSts/BTwuAuxAMuH8lu9N0qgzGV84L7PU/t4YAqAmM17qwbD0uR4tAdKaLJ+JKP/b4ncvpGJidZe/q1e09tXLxIkH0SbvwPY5Fz1EclWlrBDpXlb2VZEGWA8Aa0SnO5SOSpjRq2/coTqbWvpfPdSkPKw0xUqwXbq31hy18Sm2xwa/+uqVMCJBsObgMf470f6mbjxxteyrRVXKlBl84HvuBWax68SaT9gG+8xtcV8x63Ugcps4VJrwTbdgsgZyRvyQr4350YTLqKp4t9rV1NvV+MpxGPeKbSbIkrpXkWh+0P1k6qUVefihWGUqjJ8P1VD2k+DVzdKa+w6ptbveFrENlXLPCLJ1IqsI6hnPyUUKLzUsSYwm+T+zLLic9bR/u9OKKB5cO1riOKiMaK0Ik5Y+5FLLuQjsKJ1sATmNs6pVlq1lanI8bkGpAuE9CNtkoz0iQBpevFDq2vVBjp/VHxiRMHT4cNU24Z+JIqwAw5uyqCIA/dvNCR9RQjeC3db9568WITZIRX9Pberh3CFIyMwJsPmERFeIINOs9+P5k6dN4m4CXrr9+IOUSPsHqolGsQ/zJbo1R7NWH5/DGr06i9ZzlX6ZMfjmV8ZNVvvcSnkDs3bVtohGluflogzrQ3YtRvUzefZ3dCoVNDdAHukaNC7BJ+cYqe+DnmqU0Bj/SkYevGzYzzF0WzP6+dzI9zECEpXzLRG0CQ/+Tr+jy/UhTpaLVKWYWKY8GEAT5S0LupoZ3ysZF+nyw6vGq+HYmnco+gvbH3S05Nn6AFZKQ5FC2L84qTlbcROES72YpQv2Kyvgr98Hk8nuostoY/CGxOiHT8Yb/oYcNSbDL16qvTibsMc0AyTX5F8OEkTFF/38hRPgxRo0LCR2kglYRBzVkqEDl/aLm6ZeOrNnn+tliEARUiIp/3N0kfG+X8PnuoEeAOp8oxCNNzJ8egO4ezezqsMl0WkVduRcHpfmGscraNsDwKUIarIj1qIc1v9sPaPebkpu4EoMdjfLv9EWSXK9vP1CvZzqgoW44tiSdDUg782MjsJldyVvNJ/HtCxA/0YJZRH2iqOFUgUHNmND+ZlD4V9FChTQHkUHZOEmiKr3AT6el5Lf8ZWgNzwXSsqoFGxsTjORIbZ90YS+H7Z9gkPkMEifZCFQEF+3UZA35GnSlzfm8NHmmiMcueQevNEX+/lo/Q2t0xnvgE2XNrwd5ytZipH334WDMNfVNH55ZHss7fnxdPhkX8g452JvclDa27gAQHlYVk4+X3n9abasRXSAUkIhGouufD/DK/BfYXuVJH9QFkgNJq9yRAzgr6gVybwk5BgJ0uBXp/HpkUfU1j67T1doBgDTEpl7atE30PDhH4KRXxdGMmSWF5Kbq+Ppuc1yLPnlFJtvGMaUSvX6OZ8WBZfuUx8apFMtCadh0HHIdh3IKLr3u2p5kNntWiOSSQu2+Os/L9PNMZdBHN8KkxnkXsU5u8Zmb9GiuM9Z9K7S1jfgDdioD+RDIPK1rcwdg5xTX85FSTz9jaNCzyibyQg4MB3dqfEon/V64UJeW2yUR784TMvt+i9hoRanW5XVhSsBmg8pAfIS66fQ3TLZ3AGbPiMOO5/Y05zfRVNv5LED+ZFrf662EMOl7bJSAkwFEgdid6aEcUivf7Hk70Lk56l7mlHk5kF08m9sjbw3NMVbmcDZF/YpuHnwh5r4h6avbLLaV5Y01DpdyPnIajyaLeH5P06bi65C3FqA/K9uw7ZJWR2gHJimVzv7GRwWCm1w+NdXXMr430iAKkJfb59z+9gH9rsVzf53WXINO/UqbV1S5EcyOedwQ//IzStHoZMRa+mbvFJhtdVB1+RAxqn4pATAGh4/bR9KtGnf2acYzIBkOYv/grPRMQ7XcoaUTdAV3NnYiYRzJg4575Iv5wOs60hxoFA+f42dQsaYFwLFTbYaSJLW5/m72cBXNF6wX+YdZbuMH7E6Ro6WynkBeufE6sXnAmaXmPUGtwqD7/RXC3GAo6od8IZKlZKKpe1nq5kmHsH2RkRFWZtHm3YYCEdmxLG/UicV/E0UBgW1kZ78+gi0abIrVnCYPqj43wTEXN6txDa1xGYSSW254v6K1+fjGWHkRp3i54M5Sq/HAl3QFnmhXuopKrO3Hh5QkUN7xCVaMtSnEgKAUsRPfylaKtagsjX2bhYGnNppqPAic+dBbB89zj1e47Erha2cFCIsjolO+FuPNJGBZ+JdtVB1vd98RGENQoWYdg/dh0gzflIOt8cwrLuFiFxD/Nsswzt8Lb16Y0Jt7FwkHUIUtrWdcVmev9jHN1DwNbLKUKpY7UaEOMUT/HYakOyykg51keYR2M5qYAHRLw2iezsHpkZ10qdhfZrJcMBlhERT6guFitdt6kprf7sQyxXVflcahMyqHEa2JiTrJraUzVkEm4PpjVl/4zV1fHq5YeUv88uXntwileFy3OqX8TusAcY7llf4OeBWt1J9aVCwceuJfYHwueKPS1Opfabd1osfqAitCZxhLzU2q8XSsexYp5k9p2jUFoiZSvusECuS//pFdJN25NDqQcnNljvFW8NZ/xIFWFZKHmxil4mYscwC0hSgDukLxPXWQcryryPsgqkab9Tzvvd7Xr7Q1rNP8ryPujR9OpiyY/ipeNLfJCUa/A9nIAwiD8PWLO8RvC47ubAhJz06rg34LZ65df6R2mnoIIrfShjdOjI1v4dxinGse9/dZvlgBQSCYcDKGW6sdyDHtJfWgxN7cVEhQEYOkIjY12VhYAMs0eIIyMxCz4YrwEBELkhDKPfEWvY45THhb9qmzNzZLE1HlWztJxiz9BKzj1qg1rcg94nVyGXQt/N3paXqyMKVH0pcGj2KbdVfCZVjwJTc/0C4iL92Pvh1GGkQ1PwGKeub70b0t5T4VcPcOysIkdnZbgwoRWr6fz2dlH9ayRjuTauQVSHKEWa3f1/8YeKuGpFq6jUQITeYXO1FRbnLYlg/G+54K2vXOxg+12PvPEtuqPsmRqKnMXXAu2mJcomMUm1YzBTCyV+GEt3UzwuL5mc4uCYm+M/retGxywt7OFUUM9PMB0Sn5NvCbyvTYSY6AJSztxs692eDhryGZbE38dBSpSrDj5/6KV01s+DDQb4JAxPmNe/RwmRtYsfOp0MwV/Kq3Z0J/hcbx4lHUdlQgBDZcKnI0Sbql+9AAfp3VQJu6SqpkEvaQ7WEVlPYtJrhZW1gYN4DLAn3KqKeDFfDX1aHyit3/TFs8FdqJqoSM4lasikwNR6SXP1K+C9PoxheaZH7dWfqzkbg/yAtvNGZu3lj+/IyKeGPkwH+DeD55eCRScTU3xwo9AsJo77C6Ylsus811x6SzPqXKx23Ug+4Fr9xiABnSw3Rd0TviSg7ByOHYLZlNaiDtIR0mUwiQ5fLdM2O4ZYWjoN+9mKqFOND7NNKhkTL/lgyyjtPK46zxa4fRZdPt7YV4+YB8T5aP89Ko2FQnQmUwF3+gppM6zHEYTgonfyv9/SfEM2PQ2mKMxOn3FXuAQoICMJBhgFkRGh9bjUmpGTdInPTQ7G/nnOvDFs0xX+Mrn7gvwGrzXAefKmuc24zZYwI/oHYifcy63ilBxubkenBWTyUdXCClfralm0SvZ8go7oS6aA8XWKFU8nX2NfGXL2CFcOMmRmt6jW/dxPwo9Yk2dy/fvkv6Pz8RzqKJPs7z9BHFB+z+BKtQaCjCh7LYYUp3PRR9vNvnzazvFKLuGTWyq5HuaQG/ta64ldlp9fEDoD39VmqX2AJVxF+/GuTZ02apNbz8BxMuCjKvqbIfWUdSDb0DLMK/GRFS7VUPgbss7cN83n4ycDXArgrxPLASKhTUNSCjZCXQPgCIDV/80yFrryo5+9Wl7j7glRUc1jutrJ9N/3yE5MuWh+la+Jpihm1wPzczUAScTUL3/g3W1uVeX/zfrmr0/Pv35zFCDL5L2pmTeHiPxfMwXn2sb915EPonfN8fpYDluLQL430NtMIPoaZ2g1gXnRB1jDGS7E+f1WdWLzWiCqrwA/6ehIKwnrGtgz4lQZo72XU/ifbjCIGCqf2/HZKYMJ/ozJS25iBw6zh/7S+mmPbt2ORt55vBU7d5tCQvuUT3pfnD6Ay5mXpuJtlTtcofjD/de/u/ufIuCHCGgb6u3Yl9e5GUoR2f9JGQ5eQekmQe1X9n90A093/vmpPRlsk7SPZCcqz2hBelqngguNGCyy7SP2XrRzcauu6Trb8F/GxUWXcg/XOftJEsHzCgb6SPT1ffjtswCuoVX9ldf+XVpFPGbGKTYrYdeN9GINoGYUO4vd3OBW8lvan2Lp7zWyivac6+utXR5f7A9HQhRFk/tUr/0YqYUULmnV+N4/SkkYimDF/J71hV7w8oIJNLH9KnEh24Fd9XuR3fwZfRmb1VZWONQwNtMNT6PBdLt62zVGHTTZB01gbHGDan14AjOTrwMowd9T7YrLyl8NvV6AmCOBD9VW/b/3stoD9fjne8SOVo0DEpVzOvDBo/6aP53en84Rrwp+taeTTryxtpXzN+kNiRZlpDfH8W/038kwpMbAYfibaXJp9zsuLGwC6sSCvihe/O90pGkysJeoHL1H7el4g6uHLSe1Hs3CKLCdmFqYUb7MSXn6Harpk8vmuavNX772sYfSwn1cx1mNvHC9BcLgOsgxo4hhvG+CWwCaVgDelxag+V2kSHx7xu84DiPmiDelbp2Jnl6pp3+zGxIWxOeARAxJw7EJOnZP4/FGRyF+LNHYKGIs7a5loD3149m//HtQj95PLrYf0XfBTiA7wxVBx1sPObzqmNR219OTky8oXyZpzpoRFe5XAsxO4/JL4IEQh/w6IyNt5sCosLhh9fPRYTUquEyrcQf4FaoDPP2ykIzXuRjGySe0D1Up04lAhZuQk161oUmQx6rLKdevhe8M3CuDtFa94RZJt7l7OaFVmk+5oMcg9sUETqOXOcXGJgIL8uyVBI2N52A+ewpaCp2MwcxdEoXnvZ+YWCy42Z5zjsayFwlSkLF3AbGPnErg+SbamHWKuKlEGUYPV1CpJRSUv/o7tkCk+lTM6Py9kR62wKyNQ23D3mH6ae2fGl+gXsVGVJ+QLscoaflwL0PnjA67LERfWCX58f5UAqoNmxB/i7lLY4adMkSLi3CBWqWQCRKuJ1Gngj31AtHjY1OkvuFakcA5G6rai4FdnoijEIn3VMnSvUiFlXHr0exLrtKEW9FVK5VreCAf79jnLgwcevyI/+ijpoL0CfNYj9NmFVgcck3XtShWKa/LBnodRXlh3YFhlbN7hiesRx58NlOKNkny1UXI7jvSZgzRODPsZvGd/iiqR9VO4bR8TeqA+mT3zA0NOm98ygkyRrshgMXskRnRFJ8AX8x6qdFv2IRCLIwVsElWf6vrdF5TKb4MT2s2DGI7fyVMXE9UKrIJJq5AYsTK4cxQcwLEWK+if+QnlqMNxBkfAj6/KDNPoaIffb9SolA7pr4sEDU12wbtKAll691cF95OIy/f1NOM+eFO/QaQTc0frWbTVbHcrUYOf3etrDcuqApFaKEXeuQpOb+U2FQ0oYCyy02RhUea3DyjbtA6aNcNHCpQD+8KETXDVF8005n7p9301gSlQS0ZC5Gx3kme2LY4iKp5odbAl7tiXsdUW5jIRuPkBOwKVZXNZOnn3/9BHMlYoYU41eJmxbF9aEQccjgfV3qA1wSwyZFkzQRI8jsrWeESn5g4x54u40wrAinza3UPfc2fZs+7md8RBt6KEaQH0YvN50abnL1ATOJ+sDNLuJ1GDnABdt7oL/7aS+YqoBom/694MiP4u115gbfE+kMtZPj7Q5CYp+W9cnQkUN3XKFRBcqQavdvkzM3vAUMzbis4I5qMSyYPYCHi/MnolEC1JoAzK93Qd1qHLD1N61xHl37+cEvAldhi1U5XCYyQIYqp/mtvyv7aM3tIQjs3lXcvlbjz6LBi6j1oOU5rxyhBCCeD8JTpIc5ToDPYXccUrgjML65Q+Tn8ECuhNgNwNLo8FGSHk8St/avA9G9j/RN+iRRTeW39zKhtuq/aobppOD/zNf6TfbJtloEy4mucfQirxK7SiX07201CxiRznJgjx5JK1VL8VQeEIgi3YXHfGEya4BDMPbuy5MjvJMwFl25NuCrpgUo6egjEVve6EN8ZBpGY4P/HHihrrOwzxd88NR+TXkBqyXKkZSNVBZtEuSJXwPyiyAZXDC8aTKwlKd5zmd5a1mBr6DHO8R6PO6PYCAFLKzNjBPEDV80giuyOmTxAX2u+4AH8VBuLeEWILwqmLtT9+tY1TesL58pPG9DfsRBTtd602kCICasRp9mQS+DgOMTYzKXaqvtblLWJ7Bjta3U7pEMJZ07JoX/1PDKI7bpG2GPEEUUNaqz0hKiRljpikcOIreXvdkuePqc79PZZ8fPXe13PLwJ0c9bcQaudmhz+mjPCrb31ze2xxc7r4OQDp+cQ8d7bhCY24k18TPyA1gJnY7nUhJLpzrXUzWASUBiBCsTi+/a6SMSCUoUj+QpkfhOq3Vg+2fV4EJpDtShdQmtSMP9+v0Ny9yONeJTq7GWOeHNQdKFYQC4pxWX0RQzrxr7dSfxPfBKwE85rBHbBiyQ5/1PXlHrlf2hTyheFuCF2QrMKeZLax+4uUD6Ifb1hgn5WmFJGnXoMnRxLksbMw2pHtKgKGCBZuoklbN6O/Nuew/sWq868mtbiO5HgplQSRIjiUoW+D7Y+Lwr5VhzOTZbisrdF3JH9vjk4RMz9noiHjYNme9Yr9DE2lZq6fOsN/0vcT+S0hDjHr4NcNIY62hEDKM3Wbw6cGcSLtXoztTl7jtqLUfv7ul2sGsw4HgJDHy9tuXEPQvf67wJcvLTjWgdedf1Gi6NO0VVCI4wia/pTX+ZLOE2U0RH56K3zwy14dgqTMxCsnOR32FXy9qWs5skEaiOdECPWB/6v1Lnejai8+9k0m20lGICcg87llxzhmL/SK3Jx+qjaWPbBMLwSoOMmhiJyx5YUzkDDCAf9JuimQv5Yw5Ftz4YD4g66CwC4ZlvPs5kqMUPS6g0YWlU4sqldfoHKLFSHWBJnQPFYB9S/r3SJgpd9g2aqWPI1k+pYuXtOogY6FmfsrowAUNUeEWx7Z+a7skl40OpCuflvGOT/KknSpv4HL0f35vfr/DhoXvFK0wobnP+Myzz0taulpfIREeaMXkFleYzFSiTQoGej3tKT/S9NVbEmOLMuveXulWEspxcy0E6eY8euvonreZvpMnQKlwt3czClaQ6DWZNaVSh7Zmc5+tC3Q0Hq038v/aLGpQ8SWCkEtpSTkfV1Od8TTp5pcJV6zLY6Ewv8WDOpq3pwXnVdxXVagcWycizXcZWwHbS1MUq43P+0oeNxUNoXRJAPkhXtQB1SKl2IaTOg6nxEZILevsY+wNZDPF+VYTG5tiOLYPQdBu7BRr3Yf4Dd4hanFza/LkyAm8VTmRHRGOriGZfsXONUWleBsxkvOINymp+JPlN/KX3dzT7Uy5mYx+YrgiOJHYsc9eXAeaLN2f7FnB2Xa0sW4MRdfHxXLhcNoZXwIHbmfUeG76yVXTZItj6+hr1F/8PYV+9J8UTvRv7Cff3IK/Wz938apmyilFJQmY6zVY2IOczbRi28je6YJZO5U9+x7oPDPdjBSgk9QdJBbZ+YD/WfEIpcK73niyricQK3krjmJosNTTxmsrL5YyDi/iLlXmOcHJGisAPnGCtdcTF4OwT47aQaN8RzpZBvP9Cdy9W8suPBwIKKjrHSQXablik+KcoKjLQHB4efGFsXBd7oExeJ6nw/NTDP7wqmoBLYdd9jEtJ4T63uXWVIKiUwb9RmDGr9S2jl1wsfpeZwPbbADOAIDBDkRB9IeAuxeTD+Luhzu5WyoEIjSt32lD7D1NemvY8/+5h3H6y6i7GX8Ruy+aLdcFMkWRBWGs7KjqxTsAYakwSIOK5TH/CeodyLsxh1+NKw4oiMtM6OludWnPz2rWeCiaWAmoWYsuv3RwUCMOLVUPnoOobNmAwOdCC94FUj8H1giWL/kiDCAClXUh3NK5g3/m5YW/u2zIuCs1qNhca+KpnlXXQ0Wh0wH6+rS12zFOPtyXOkG0a3FSddXdyQy1rj3+FMryq6FEbngIFyeK77rWpsoaZyyR1iBILDt1Em4H81V7KmQFa7gzxD4EGy89LXQCrJAcYhcLnK+sjBcKmTxMjreO/ubnqnl3Jrc+TrlDXevesgrmuGvAIPPrixj6XMmGnJzN6PGQB5WQCdIgcbnAkIAhzRF48yXpMTrue76iXcbKIOHIAgqMJoVTqc9nxUmYmYSoO8Epue9DDm0zaXpghOg4TlIAK3AgHJQcmkMtRagFF4I6mbBNRs86aJt+sFnYf8OVpdTAaxtTPHzE2RswB9zKIUo8czsehfhfmqavxLss0w/0K55OPWGTctTrKLOXydJlm4c7eRXzbJmJ/ZzSg1slqvmlbayx86pjB/PLz+oq4Bd2JkI31ay6eyFskV13eshXbet81Y2kuvjznMX9BaBCFMijo1S5Qo9am7bUxxGmWmzHnSHjUls2uBuAD7qFvjLKaofsMN0xPW1Kir1sjYHcsrlcaM2GJOdweBcDV/ug+tfQt4mEIvEI+JsZR6h/obDhL8S9C8KtWb/w3LCDrmCU7DgxP2PlTQKn9Wc7zY28h1xtTiNZhcxpJJk+v4DOF9gOo1VGoF9WcX0YFVMcdOCXc/gmdjRddVPiXAWvxcskk/aqFHy6VpKRVjEvEQotjaKEqwPiVADsf4tKwHZzzyoTOIVAmlvhN3dtYqL7s+c1NXcI1/Vq8k5P37cbE/aseu6AhnL00sJA9Ll+Spt5RDsFfR76Iog12GZVMKoZjqgqKDByhWBpvth/IyiYXMkgCr+mfgCckv3aK7KuoTrb1bt2wKXu3GYG0aj5+7u10fX7GdT8VgrazfXPuVXeI3hjJbKikxIvDP2aw4a5hlqkuVXFjV4fkMqneuaGZZ/5nEs8FX5r4ZHdjTXzQHrfRQERCUcAljf2OA3oAClf4TH9TldC6Dez2zV/Ne1gnyX4m6idRPupYwoGdAB5mQAVqNdfgHmRjY3LpnV2PCo2IAtXHyWw8fGyBVtVqYXLvfu3yl7gz9inpusImbxUGddAvUStyYdxfyeNT4vXcX+1RckmR0IJ0MyfRr06FX9HuqDf627IYpPjnE1EU/3EvqJhymhusRNdIQFqlQ6HkoCeLX2J4rS/Myn4maTBYMmo6EcWUbLjheDz/AhvAvGzle3N1Tblz65QZhDKByjt1j6W117z4FJ2NTELsWmrSkTCKdcGiNeh5cevREf8J4tNl65SHimrppSl5+k6T0qTK9DsuM/m7NfjfZ+POlZkaj7LMCPuaKDNfKvfOn3WtPOhdo/BcZI+xZsDtRjvmGhyl9bttP/3WgfIV4S0d30hb/k/gUhRHORVxyUbuZWfF8TZROHBFvXQGOAHgoQ6T9fEmlJJNJwnMonNqHWFCVIw5jmiteKjjfkFyDHv2zT6sSesWkhB2nKi0tEeXAgCiZIFnrQFPkMWhpkkAq/Yxbr4mOhQAkcrC9x5d+ngLKFIS8v79+Ht+WoC8p6MuRlpgegjh0E0oP4SNbO+DV84YRfnZ8UBCEOzB7UDo7JCG2lChJzHOOsz+dOxo+8x82YRy0RgNj5lVHwzy3nKILnvBr+VftVKsCnYP/cMPnLvstONZFYZTxMQNy3X7bN/7n03VCPf5zj397dh4+WZYPSWOh+kroSD1BvULdYYt/WGP07uuQ5PuV4NsCss/6K7L98PEFx5GOUPzCuNINdkS+5K5tL0X4HQcUfWO0niaxFtdsjCQG9ONt4GyFxy1XOQ1o2U6XIr2lyd8aBXvdnJvIi91f6RbifODN5BcVLpG7qtd64s3VIsFG2S/r7Ykzw707h4FZihf/WL3ZFrSYC7VZC3uKIOKVfHV/2dxMIfiWN80utS3NIO6ZzxvUFKuAEx86gnc4zsFV6Gyx3d065VnnJBaLlnyX0UOpUKRzUg+zbM4eudGvz5UWXdAjX8F3TkhTZcA0mtbKAQBRX6QmmPdtL47AikBvgTwang4R9MM2DOvB0W0llkakbXM7cM7jHkf95//Iz7PrTwFgrn4jGOAZa8MnNRRnqK02daiSyIy9BXiMhSuKOoWPGLBZZqM20PYHccrL/dEwVMu9n7D9SJ8y2pkCHGa+BV8b87EJl6s9o7bc9O9znBHKoodoUZuJVviju3vbIH4FlZsVslb9E30izoLkqdeAtVxeQU11L7eCXvCGOnMx7ZD9ldBOTrTegQ40Epq2WiDI5yqfERwqZOXbT4yJ0EME4DPUDTCno9eUK6bDvyP4q988kVyxi5XsMJkEZyDV2V5m2rkl/AavhJxCKvEDexYmffyuRBr1mRJyc5pkoH1VKz1clNX7o67X9x/n54tRQ0GXgGXtCLpOXThnpraAwvfztTU+ho8qd4itJx7F8V29vdyG578UhjW6Zs9ANX6e7ljvFOjvhjAHp2B2+OmLPjvTbnxq+5+1C3bD7gDNL4mHak1fskCgbQZ8kK4yKse6a6dHV6YKPXZ4qZg6WP/C1jU2x9qi5PjwdJiIUg14OxKCFW+mq+9sBLSLMwJ2FH/CRRmV+FtdhY3lUy/DRaANnmxeDEJgEbC9IO0M54w/a+CnsPYbZ2BQuYuC2PZ6e4b4vU33+ttmRyEeoF+a5jJCn+a9/aX+QJLodHGoIxJIku663Ar1/KME+aumj6V9tTrfLMX+oBRzA83kp0iV0vqPySDgSzykRK8xVQuWt9XIJq3NFo9YE/okf8naP0ajCTQtingpYLLhdQHGMX9RHbgbLboeTrtPbU2ZNL4fGn52IYDYe5OTckA3/ae5FEG3wN60LbnvnO94aENANFDxT6neIPc+D+YUtK1SR3EG8Uekh+Qs+F+M1/98qDIp0FnDwnkhp5ed5+Sirjvjo9VC0bIDsMa1dHOjhOr6JD1dJqaainx5eSDAVHDa1jBTgs/PklRnzz9FHrRuv0BiRFYZcnMKQwQ21x/uLQIgC+nRQhN4QJYLHtlvPeb97nixAflIF7s0HWq10E45E3iffIQS/Et82itxQMJKFKd8Kae07Sx7GLNKP+UUiOEm1zfUad+JYpJj8lYWQtrE7FrkfvIFaxk4jvjjyQAQfog2KiadDcUpYWvSRjf82mMSgMeOFCpEv27GPNqmFS6E7Pjl74rbRxCsU7AX7WwhWgxHj6wU6b0YeWD/Mv/EYIeJ8QNRXHxE30s+25I2OtVb+sANoTQjZIAQjHzqpv5fUuoMF1WWbjVcKfsh/+AxCFR9oKZU2EdJaPPBGuQGvj37KyZcy/q0L7vPYEeR688jxyMaWs7sfkZAg9dMeigItGhEUfxmqGQ/A6ZL6n262tW1kmw7GHYHyoEbodqyoMEJfPMsOz/mFan8+NYVPuTMKFzP9XSn69dK6wxr1LPSsRdkeK0XK4WdB9nGiQzaY5B4M4K+WeCH97cDc5k+0t962lBquJjaC28xASBYP7h+6nHuxmV/C+unPkuzpp7MR6xmeSPOnxv6vrqi0LeX6mHLazXYSAfLnn/txXGlkZ8FEMdlrq+Mdo00BKgQmo3wTClgTSIW0ZHwfpOoeVDqBGR6++9aRvEQAIJ4JzM6bcGm5IzB4VVR80HjR6h1Sfto68Oux6PH4HH/OiMzc9cIaBFKV15R+3Iw7IdBb9XUms2Uei4QTwEhtVks9pTBEAwza7J+19ShtdQ4oC37bmRiZAO956b4S/y8Bm5jCGNtff5zeGEuClD2xIb7Dq5CFLljB4jAqPxJWmohGhUzhTpjLUa5I9WjPQvOHiL5SJ4OPUf4V/rUxIcu/uRG32JP1b5NfMc8HKhekgs/R5X5jDmjVBEMIDyCCsEwfDCSYyB96BdAaEVYZ/9MpxI4wxwcVedzhKjm2OUD+it9R7BHNWw0tZLsJcyMVLrwSfRd2xDrp8fCfpAHSBnBrBFXvTC+UDrd4B45fSHi/QIl56FvQNCn352/1DIq0KOBxUMFb9fTpARDh+wyDr604oROZl4Yf4jOz3ZajRlvyrep0/rdD6/pSvsacwXpY7MoCt4mR/Hv4ErfB9XXML9w9qVtAsjOq4XqJt0QJseiTTSobunrpDgMColrWMfIq2RlvnyF4SPWVGK4nK6l23c1J6stm7pNKR3zs8xXHHUIVo2qzy25hU2v8m90DnmIGqTEd6RX45zdoSq53XX9IuT2ouQt/JQ0n6jCgHfDLEuSQ9nhrrHG4R9E89LREP1qZG3VMZxIM/c0geFrDnHgWWbJVW9/M2MitK5rqH//m/fgveVAa0ZSaRyVCZpw59ZPVACzX+uRvKcBKT08dp7jnaCpVfEZeiT0H/TlmZYbouHveBrq/qvDLlZht/m2Cytl4C+1JmFJXmgVGM20iB6+vZ/icurcT1RNwvRCgFar+CMmzcvPf5s8OqfONygu+dFxPNIIN5DUOj8y2FoozOGBlnQbwda0D4jp69/Aw5bDlE38OXZzCVOr1OT0Qi7AkTz2+EWj/3IjZqwZPFnRuP3zzYWXW6pDd9H8xxrFqsB7XTRZHq9yDaR+h9ZIOMQaSYwpzxtuP5ZdCGBcdrEf9dZZElIf2tOCPhkjAES8l02fAlgtHuzFscZY2ypI1vaQAcW9ahPuHEFa9gWQ7GgeEfgTBnzQIHMDInqI+EGNu1H7zJoqE5iVK3cze4H7LESo559qoxNNxsUA5fByu+T3wTOJF/k+gzZQFv0qlPF6gdj8w5jqAxs6MDco++TXNHFJBLpwqMWsMUbkyPx6nFwwlvoq/OhAsldKXIFU7tHc7fz7ZoeP6/eSEQr3nyzB32LlzlAg9ySJ23Wj2ilbe5+vW7+/l6+9r2bsJ5bfIj40l9TVlfgjxUT/SHN8+awXRAJmUvpDlRYCDXqo1YxmRo/G5L6S/el2Rd8SXKYlqEAGODdHPTSqS+Ecdju98lOvCbntC1eA2BGAJNMs0pDCnyquEipXfFqAjf7iM6TNH8kU22+RfG/+YTSEv/wC5Xsdcqlavd6BFuSmKkIW8R1UjD4TC6A6qBQ/HDASFttSDd8ESGzimlrZjEDta78G6FKI3li3Xkrqqi/Uo/VJWz4yeKghMnEPCRnShBCUSnpDhrXSGx0ULQJR4VQvnTK8m+doRGlY8sKSU1yA27PFht7YQDEfx4GtzmGSzQOhMCJvX15pUoofAL5g9pmmhvu573gDlQbmX4OA3OTaNpW8ImL4xIKH2q0dVVR8WquCsV8GPYn42MjFtBhvYNJIcx2PCKvhbXP4UJWIz+1nNz6lpbzjhF6ZXy0wwWtD7ykTQS902cAIh1Z0qs34kVENJszzsA4kXUliSZBop3xB13ajAdD41+RHu8/zm8fPJyhIb7neRFLPzWLOj6ljoml1sHObL6YE76EGaf85ZFA0kxZ26Gvvk/LDhiaW47xlJrDdxQyTqsuFdpKX3LDLU36rXIoep/Q1cViJt05hcdloI5RHGyN3d/dkldzFlBB/GVqOm7DjZOpeig5fF38Txv7bLaVa8lW6Akep/QX3NjeL7/h0zqRspn5RacDxndS2kc15IjnFlsDk1sdWgOY83lFf5TlqFJc1LLSo/LNV0hAA4dDNhQBSVHOm8KYnrpf/1ycdZtmc0N8oWPagTZN8Aqc2AZBbuQaGRWRzjAggO0tnAV5O/nt6smcNyVZr24J1s9vPAmdjx0jyNeFDjx+1ctik35lmuEEjq350JCrqv7U3quXLnRPxXMAYCjotvtJCNZOYtRd50JQt0QnoPCe+LSxj+jQPe1phyUBxlWgNnM3OA/QFM+tK2Z/NJf12huS5PuzBCz8J7gjp8QEy+1UuE+8qCtyZwnAeVMNacylGpzvYrM3imwBUvtiFiDwtUFo0GcfZlYlC1j5ce48vfsAf4uGI38JwkqU/td4SOS8HftSxlQZGMSHYxcsBlX8CLDwqLFHhJO7UbmsJURs+Yvy/x1BG20fLXCaCfeApRzdA4bWWOitGaFuawSszJs0w+h1M53HdEuj6A4iJFDMdBAMcfaILx/JWg5t63SN5hAKBy7L+81N+inPZxwcTE5W39Jn133a7TSdwiRn1BdoO10Oi72jHILNM42hT8T2Fgr54vr5EJ0x6Onktm0fzWCRzEw4ih2Td89+Sc8Ss2rZhsgOALUhWSr8m9ree3maYU+jVTo8Zdf/kci48h6mgjnaItcv1Vn5fjPZ7m8PepeWvRfibII8qxzczGBO8p76ZNEFykRJ7n1RwjAgQ/79rfbWMyEz1w567Y9RmkVqhw+Ws7nQ5ROiMgDWEJB/ZovnEfCiQoiCVPnfdt+OWVrwXUJIXA++YKVary7D/Gv9Lup9XfzXuoewJ98lfF0hrfuc9T0SZ9gsaGEqPpOl/TU35EGyEjg64t6WM9rLRyzDTNsrpGwrc67ja+3GZmuiVcr0BAROtvu4ZpXgyvMYR+oXEjomtTNugGH9Xr6trsj0//pRnzmgqxpBxMoi31cz/dBx/IDqC/mq7UXWzUmYCo8kqyvyFRUHSryr8oAtvShjXaN9FiVXU8hGmLM/TD22jUyKQj2L673dYqXIp+FP0Z8ewDym9xbyzfZ5/qZFNdKDFlh50RRtKZEYakWgvOTevM7WIcJMhqC1u8Ut54tHCY3WithhzEiHekkvlNYXvuXyQGlRZOUyeH+HAuBkuQVOauS0dWy8NG73e1mP9k33TWjKEfG+dkumcsuiqGi3pEwIz7+IEgtVkWakyYboq2nZGiCLw/Svtmue+F8cSt1ZILMRb2xT0OJmjqqaqPN+U02xJNp/8juueL8egUxI1jYVhLCSDcFtsNIV+L9UC1nyJxiNRNs+jlfzeP8NcykHG13cwyX+idmef77nr/S9qImRj4rtYcb9S/nwjtc6i+NAi/Pr7iEUOJZbHZR2PPCFDQnhD2ubMeQ1uyHapI/z1xB8rC9Y01A+y3lLo3e1XpWNmncTN/jVx/RY9e3Xtv9CdhnxIbpKjsTzXVEw90EZl6XSt2HqAyf0AVCzrBx+h+9dJyuBgeWG8oguhun/lLxsLRVFG6jNGFs/1poEejR0Nc9ZYgYmDvcNQKd/XNZ1Lnfph7rEBrmP36w/uPcbGr8lg+zQxrIJ/FFYTAx/3wC77JkYb6Wau67dSjEuo/UsacBlGa6F0DrIGstoHytWckwfSHaeuu5AUGQo1Hd5A32UDQK8HRPshvX1G4e31lxV//duAhP1KoKN0uNvnMUHNHvnUS7cUbsbyQHGWdQwHp4XI+3gUO4qeY1j8U0LSzsMUmha5OXvruQbYJxBTPVrSpZy8WN/F+EhS8nQ2EAC2hDhDl61eppYh5DqyNlzmQftUfivWqb+IPyo+cnnAN2ksglHGz+q7A5vErhunBqi6Jlr2aAyjAKw36VCSKlL8qB/eUM7DUacygB0oFzA/ZDoGCAQl77Zymv/WfiPsyyqd3qRL5lFaiyXiis7CJHCGqf1bGqFXX+8E9wFD4o6fW9LSrlWdatLiGbjojZUUSPP/Yrx09XJQhXc8suuX5QdXtDgWNCOF/2fzDThzd5JxFwDRUUZv1yuy/0cmdFzGcEdXSbDDt5TN9Ah4K2T9vTKSoIPTUrtnijdWdHJvDPTq5viZddKM3LA0h2VUNScBJQIAtupQ465rj/lt+NTdDbBLCPSFnpWmQHjF3xHVTuqb7WZ9N1S3DsNSWs2K6dUC/xBtbQZS36d4JCo9Dx4urd5EsYT98QyBk0qpq6BvVa/ZYdp/cHRoeU11xxm7xNg1w3RNfmyBJc2i/m0/EdHMOw16Du7gXFUMGWS6HxgoJnBqCcLBMpbXW1UuIsfmmyURJ+uHnUNToj9idY91fDn09RZ88tJcj/iS5CHPZ5k1L9DAKMlSTtVE0I/T+uV8Thf7+rWd11Z3W1zw6/UrLk22WfHWZD9VcyvZU6Jcr/HeNtKD3qnTKQ8j/LQmFWXdy1Yo0j6mQet/YGiKdz0kmhP1Df0dWHij0GNuE/lNRfCQvwoog5dmzVPp6nbaSvENLLdv8CK3Z4QNZPgu+wr1gZ46Gqv1gfxDRG57oqifoH5xZsaMCBiT2srLXIGDEY+ALDRdSrO6JVS3O+FYKP7m2qWNmOyXu2foARUJXPzPhRxh2t6n+6PqSZC+FhvVWH6VI6s7iHWdoBzzwN08u0SrT0NT2q2Xt3D3YM77QXkJeOjqMuw3jrqFKW2hQVOJDgDlsv6CyA3STJ+zwcPn3t3XcJV9+NCrkITavCOii34jipDn4r+KpHXlidSuML7W8k1wdLKPRdUn5FU2afZcRPUPjjCf/7waap51yFOQI/N0YX3YGjmLzSOR7M7H3YW8M5H7sDYHMnxHsLrc6BmZ+8AvRlpeflgTR5acqIrchthVM4/fxoipTg75j5ikhNU4JVPxqCSeWRIKlTDrrZiKe5+Q2V81PnSumtTXShMJKoh19GTVtpLgWRC1pYhvC2HxuLAZe735lEvxs5r5pg806k7sLpX2F0dLJTOMRh88cXnyXTeDuBL7vpTK0oX3Ircv00Iyk48RyIXtZ2MbxxqZCIhJxcOGiekUjYGo+YNSKXwcjyqF844ic75uKr1GTEtZmzcnclNCNuNA3ZZuaHK+HL+KudT6Rt2rVLQ7DHq1//XAa92lKrSwuton9sOoIQCUb10jRksdyCCph0b+S06AzrEyK5PVp6wulLSiH8N/oyY6ZWMvPgKDO57jg9w+IfCwMX85dxDD3hLY2R1fwctLme507hSpX+jmM1V5ZyStI3lCLxSBns6SxxNAzIcBUhP9e/XqqO6zXNCC3TcDKLxVDrrBqgVh4Y+mVD8ExS+c3gU+MorDjXn1G2sYZhDHWxR+sZY2tDJoYruoot47Z0S/AW+FlsgdsyeJg9kCidL3Q0m3KD27Ge6L3dhp7/OtE8a7IsUin0fOjhxP/ux7Ip8da7ZPQ/cEInFffKglE668bTOPMpiBKavx8hOBe4ea4Sq4HqWuEokJFZegv6p0TAZDNyhTxaKdPOXeMBGLe9xq+ycO9ZDYBaXFxhzL7K3+QrT1/HUy+oleDGKuVPwbG0ZNebR1n2LrGhrSosCmKlxjp9YND5dVaN87LO4FMbBD/nu6P0eAYGkM8EGvGrYyHSuYizvgztL+vt6ftT/e+rGC9AwC3JvWSEEKLoUo5jQ8CcCWIZlMKb5GHw5FpcJWSfAF8K5WwedpkJoRPJYoIgbppdqV98HIgpWLsgrkbrjI2PQ/F9yobRq5UJXoVYHFJI1PAJXvSb/xFdBZ1QcT45Au8f5SKrlDkWahIqgJZBdIH3gjHvc1REyRyfVHJJPwc/magsbxd+F2MtK/+vt9XEXWNPbHRjw/cMWHHVYd+EPT7xcJPeQSynMHZsz1ESXCzCzNIlF7ngSqyKlKFj30M4cgsfqVeEPPLvQBp4oXrOdQVLYMJgg4W9atwu77ayuVVHEAqZIWJXS3Gw5BsdtMslZ5Tizpm+8nvpgACbeEvqr+yzLghycPJ4EGPuFBvsDaJtEc8zW0FR+6YLqQr91U3beu3P/peHa3JXp+M3embPS3vQMurGF+S6y+wLtHn9zp/mKZXwgHOhw44lo7bhsZAqCfNssJNjPnaxK9VZzDuwNV2hQpVtH4Z/Q2Iiws1Mv3qzxcgErUZTMuLFTdJnHPxeqZDvJAKAUc3dCS1XwXsUf6h29tNLo+0BsQai9IhfId+XV3deCmmYxQhomOOk9HU8+0PMIeld9RWFgZ6PB1t1zzzMpCkMhc23GF3AQFFiXOpd9YnbTbPheZStMzGNxS11BLq/JgQIhHrC8kwp32DC7Ygz+ilKpu4pf59YoYFgyuOv9rxX2+LtlSs/3RdFdFfOjIXlTgzU/nCG1xCVCG/0rW08vg5UL19UHe8+dGpt1yhqu3qWy19CsUjHRaXHSzxgJ5nirN5BmGjGFOHRmhz1dn8W8mRe0cfV5GwBG6onKqmanf4rfbaEou4n3yQIAg/owDvdvrRyPxGS4kWMj36KwlNku74ysV1H9WdbRAhIXMs1Nd+OXgF24YZRgRhwDghUThHCdfyYDbuMY3K4TlKq7CQ2vl1fWg8AQm1dHA/vawa5c0RUvWqdWi346V0EqFsNTgkSoTR3p8FcVFEMO3kvg/FgcxNpO+4GO1fL43kNvqtJIwN/GZ0FRPfHLak2nUS2/g5XO/FrNy7yaEHqxIz9jar1wcBfoH0ERClPVBUlfw+Vspt6sXTfISZqqmK6uBHrTMj3iKkIbMGW0knr0D0Mw/TjXzhzGqMQY7ikwxTXqf4lHsNQbe8QqcrWbajnrRSV+VfMtt6cjCGuqKFw5nj6XFx1/2VH26rN7lzQkyRUBXV6mTo9bWSVrwHFsHPfS/uctogAQ0vr9xddDd0re1K494d2Nh8DpMjw5XMop3uZ3v4IGqyXwckuvjkBstwH611cn9bhdH7SdshO9C1rx/9wVnBVpPjlXeeT2BfyRb8NxB5mNjfs7MajOmYbicGaU/ld6qwSiBxnemnCBwYP3gwvmH7t/1MwqyItW8KklGD274WjTb1+rJah/RrZHaxPr5krix/0DTYmX3Xb/zezDCZtngO0zaeo1q+DV7YiYgOQulQ+aZSlOemYzpINCbf6nPBBc8wH/5mnI/IwJ0NrjNlHp62DDv3iU+WHJLKcMx77tUdrdgDDF+q/jbqrOZvNofJor9xR0eQwPxt9DY7uhjD+fsodocFSttaoMSuKg1mXkozOFOC/e4Wf3khl1KpPsSPFGcCkxZx1ZSNhpYwRDYG3EBYjuN8J5ZDVN9tf3HO2sKXvvBe4Ppf4o/2lzGiQLSTfvDjXJxX4PgjEbUNUskt4wxhc0rc1/u6lVzaIeRjv8qCdBCG3fpLHQ1IAsBlBKDjuS0a4yy5ymsvCuTO74TwtHejWKyVloysBd/7d3FpBy5HZZDGE8X1y1sg9DDq9wDH1ezd7fKt8Opd5YsbdIEJFsq3yZ6gsW7oJA5tsub6s93vyY995CeqoujumLxtj8bsLGx5hbEPeAz4rVF9WdqpTuzv8A9ut52yZIevxtK97maYWw+Ign9tpmqjqWtTiEUpbZ//hl9q5TsbdNn9/AGisEBCWXd4Eb0BV+MYaW5OZGdKpTMpvPx3UREARwh3HDaYyxMN1QqzlRgTSr8dHN7kZ9DxopwrPktHhy8BwKRKry04P1x355tVEKRo4lonK5tyVwthbVwGE1zXf6nYuF1QP0XoZ6v8j23StHnwTc19K9LHhGx/ZNgae2Byn1Pob4YTAnwDd9gy/jPp5N+SxJLvn955T9z726H0UQggEQyLZEZrx2j9AuEN/AQS7Tf23RSvG4w+lOn253nxz/0tOpqqowI97YlymYJu3o5mPVrLvlrOk7IODLUuX97mi9YPDkXL6OrjEB0YWGIER7Fiuc1u0tG+pOG6lQQMqGWkupDQYU1lbucYoS1rphyzjphFKB8fCwRUprJEOCUPP9aK4oWGvwQ08fes/MaB/2nmoPEu91eb316+IGh+SpYDTO+FADITe1palgi30k2j8HNCI/zSs/vFLrN8nY/ZS6Bs5to1KUEKH7qFmiY+jsmJVkb9HHtIW247GWa1NUqW7rgOdXxVPtS12jsCk493gmTZFqNqrCML9bO/tKpAkoZi2o8Pl/Xiz8sS6DNQk+ECtY0g7VSdqcY8BXxYfwMFMf2t4cu7tYJ/ZoNo95cx0ezbyDfEbIlNOKKenKWSVSTj6FvpmJB3ggQ+1Y6gPsAHxEde49+1Klwe7LVCrtP0cWF4dWiNVzm5onDTBk2FdDYaDXjzphe9qhJNxuxZ7fQUia1LgsQYI1svK0YYJP4DN7ZygEFCZjxYXKHSj92Zl1tbNK4a3PFobVv9DpjYhb4uK7HREK1FNTOQxHhtXoRqeCtN7OKcvzfUdUA8O5o7ORQUCTIMRhypGNw+5xwQkNGvYOir2YqV7q8EVW/YWY3qWdUkTYCl1owtacyImkL46qILrc30+4amv0EDceEFX65e/Cc8M7bb7Lry3bFoWnB3l+HoBXzgkHiYxNPjD7Ab65/5MKNZftG/R/MF+xjkqlf1odmmXUu01kRNALmGi9rOkIEGAT7QIMXWRJ2MVeYDHZ+XB8xSjv2NCe8/qYONm6FPzTCu3tn753myG00sqUQWA/TK5mYfcRwz0rGgknOA3X73ssas3X0QnkOvP0SwqYH+g5KGESfoaJzr+ZkC1kIBZQitbSOsbR/QZb8WaJ+++tJ05Zf0zBNAHoPXx/1BH89GD/wvOzbMTsackrKW+zOkz8eyMTWMGvUVwpzl8UUAwrdCDS8fRFO/wqDWid0L1DUaGdS9AcaHXMugOKozkUEWnk+ftNVKfdDV5xsErl216MrZX6F4naCPnRlQ64yaRdQNsK7guFUQ6RiymvqY1ba9lrD41XABJoBr9vBfwlG97WelOXsfjIZN19APp8HpaKyqMVvjQqPqTxx27SYPtmwSH6qc1qhiGCuMEjMypsDNZUCCE9Plea5vG3HgTffDajt7oVQWdQq9bHM9UxFf/Z5RXNkXtQtRXvsj0lHSRrHBTDPkwwi6ii5P9DdjXGF+8jqGiP3tlCsTxjDol0bi2IzHkpc5apa43pce91SmQufYmL/9CAT00lxUrMoP6Grmj+8pPknybN4IfBTzeDrcZgeukkRXpy+Q/ddq+vczDPaH5bm/4sxMAwuADzjNw+3zV/kld+escGhlZeXU6EJGw6x5zQxwBHogsoCWmEyuSzfOdnayTuErcfTWIJ/v364iAPXafQhykm4XE1XgyUga/ezVLXtRLtQIkQEFwhvbkdq1/NSLok4+1l83SSmyVUTJUloJfEOakyWmY89+vdl3ga/K6lo/WQc+YmtPdjk8jnCpImss8xqU+8ocQ3UounkP/iVjk9PAa/AyjWZTFnIvlhhzI9CNZ772aCVXcOd5q+kqsQZJd1/BebgZhfsMK9vIZ+/JgBOIxOcgwaWW9kPjFKnGwpOHA9XRU99Rn1p6o8HdRRuEWUIBkQLzL6DxkCIUHGR+Z5midtpMsTZz2huUGD9ufXBTwP/lrX1r964ltAeN/nlOb8IvHfHLLmz+ZhcRQf1J8PeV/AfTYzd0sCENkaNdbV2wbXC+B1J4jl8Lo0N2+0t0WAlnT9BnAKaxX4zAn89lSANw7au/8k6ev4ssF1KETVX96h5jenkigyroh4UrHL8hR4JeB0H8X/aTc4oNBQKJ63AUOyNiwGwzE2JCYN0bwTetoLka1BOMae2HUBr5ZgjxV6hH6dRiqSJNBgUD3mKblqRi74NN2vdYEQo6S61UUdmlqGhZdbp98Vem3qUq3++hmwngNTK34gil88Wr9e9Ut0pWGrgjo2sDQjf2YQYA4uUFsGKFMDD3w2hHlOy6GliF8bu17kQ1Ftb20TmwxLy3pMqRyolv4vN3I/nHFjiPHV5mphgRSRRRo3XC8fGoiZzj/bUPPUbRjy1/5G1AhKx+v4X152K/j6ymi1bKOajnVoIGuRxuwV6m/OXPZUcGibZYOmFT1eyoJ/exNdGu348Su5PkS7Alkk9elWp47dyP1OqvxOCITfJzteu5clVIXvm9Ca2cIvcPa+Bfs2RYbcC3F6+H/sbmv1Wlov4hNmbTvpaSzmQWkJ9fXWQGqFcUQDZ3xJFzh7tCwmY62mkMMLo3hjKOUv/sIqSPzz0y1HmMl68J7rxLmfITICp6LIrzTp4eH66dfhVp2O1TjE87EMuTFg4LaZQIM/1sRBxhALGXMoHP/a1rMQdvm4XA6VZaIDpSvIT0Ome//KkNK5CvBjeSxwMbIJgimg3h5PoByGHd/XOJOKyNzvqgvyJQSo8kfyu8rcWIeljwQs35ao/lx/GSdoAebfMQJ1wTmkI9ggJuhmO5Oz5ouYEwoWFdnzg2kAimqCSEDlwh6OLP4XfJXn4GEXRJokFN5YoUMd3h2t54jAW5kGr8sp6ZUgUrT3+tOOlQ5xRACv/9aOwgCRYm1eg9N39tbXE1jur0YQmQnyfViqTYrSSik5Q0wADxJBrHPYLLqwsBUFlmeADK8KE+L60LsMxoC7HJkl8BFYnQV2CbCK8LgquhqHqvfn4hsdtxK+h5WRKbc9fLesRbs1bdsa7VYC+mqLGwaJUfZ0H5Cf3mio9nJDt44xWCAOJt6oL49durEsns2m36U1sNEx5Gs4bt6/WTE6CEvqKBm7ucaJUdUUOAvJ6VL5eLLlTjP6PSwQo24j/4p/1dRo55WXQY8Q89MZW+9sZTlXAXPCa/D0jVpXQ6CU4NfbuAROCdwq9gpJnKbCM44z/A3qZidKsU+rd5md9EJRf+ptDCxBwVqQKlyIRfVOBUMEAwowhPKMwk+ojlDX1oWMNyVGjrANXW7ZfvNjNrUdDwBA2hI8KPMa2YWMRLDPmd63LWZH4Dd9ECzt+TFfonHnXK6/qEsvto4zF/GBW2hxZQltbKAKP6ZuXCryFrzlV0Qa8vQAw8A2wJs0JubOgDC2QkQa25BPE05xe3U9TxuNNBfFcV+M8J1X21EtawreAMXxpQK9ky8PYHVuRlSGTcg2V8p1x3KNAiwppme41JqOYT5tl91ZWSyNNHdvyWDzFMCwPvDArpzpJ8jgXJ5nELGPPJy0FjgFtFmC7PtiaocMA2JX/4u+iArrv9GyJJCtqJXrV7OR2QLoGXC6o+DeouvPhWtKYjK15jLuH3rC35u7CmqWZ2oa61CTIE1xuwyTDm4OaGH6ymAFPAyIlS1wsxOS0W8hiM8GhfrlWqOLQexs9bui8gO8u8mh2/LrlEHm0w2FFfoJETJAW/4s5jDO2x89+y9Zpc/u/f1u4IjSc6611RCF9OPh+QHvuuOXXCPCu/EveaQ4z7uu+6dZ9IbaAvj98L61CeOOFO2EgG8wrYCk/7l+SV9c9bqSx3xX7m45TKSeaBwjda/9rSaKHL25mr4oeRodP+dYi8v0wY5AgcY+Zq5QGPJHZ4U7whjfqC/McfNXYYRmG8DWdfYF+F0wSPnfcczVRX09voHcy4b6Wzu+7MfI2g25+BywZh2xdoqXY7Ye3ygpoFjL53vYNE75yeHoRIFLoNJ4/6mClPfai/prlLZV636P+K+RBjij3uZp9ROQ4sFdYjHNSc7f9ad3C6gnyiqhBM+8C3uMoyUakVjeo/h+bYeneln2Mp0P5tjESS30Bzi+XxOiKP4o4P0tep9DvIbYtDz9H94DmuZ9ry5vd3OH9lmZmTJ0y9MQMc4fCRl5/fQbNyBt6j1pDL4eN8aSZ1Q9uZmAO85NB3CYqVoT+148UPDMnYD1wki4xsdYiWHyfWVO7zPcnr8nv+pN5rkatt6mrx/Srnp7hZPnHbrG6+g+m0IGbEzrHvq2f8GJTQaaH2TuPO8XV5kpTcp7lgDgsHdJzZfZXz6uoipV/HvDHBa9YWDNC9EFMCz+W+ApOZnKAzweBPajcRNi7fIIP6EEdmuDZgrgb1K1Rq/QNsW49bIgkTsBefV3SAE0NcFwHS4H9cTisyRNEtZn7fdu89Sgr/9j7fHDilW/rkRMMtgTkpUEnRkWb7QVHYJGweUHJ3gqzY3el1bzhE86+RzW6kcgsejrhRbA2T7Gt8JcEuJf7lumClZjD+9Z0Dm2BhchplAd2oPAxJMlgeT9eHyPRQtYx0kaw1hcOWkMlEf0hAYole2q+hBfjVYQJoPBoPzwilD5R1TuAY32rPyqt3e81ll4n4dJPiOv3NMda8fb5U7gwP4Sxs1AGr1s75SZ1WeAXk7/yAfE/ceXM7P8FdbQSuTR3DZ8Tv8NOScB2lgH07qabaEquFRn+tCxsI1HifHDTBp+Zr9g4lCgEPx/u3gsWc/jD+Tq+zYFqiNY/K65OZqN1w/6oMuN/gGXr/87dHUT/TOmmnu/Z/yn3PFxUE8LKMqH6ft+6el/AxgrbnTsH4uySDLPr3VfrUhXZcjtfJenWxJqDn7s/ujwbpKlrqaDUSaJlhQ1YQsa/o2hhENmXSudFTo2CjGSPCAzh6XB0p0bRt9llL5FicD+93ygeS62KkXJA9HyNdgHHZf/1Awc0I6WRl/NUgXdYSFzS2BTPhDRrzGtJY6/bR+788ZDc/o4fFEYIgvXRzAvO3Zdb3XCvYog9lgRSGRE+BzVZtOffyI0BrE4Q133idxdTRxnTab1zrJsA/ZduxYYIOfWd/rSKHRrpULfzIzW6MwK89d4+Z+EaneCzMDjSJYldOFPZ2Qr6Q5/AVGHl9yTYLUIgchW0VUbjuuSAWMRX0mrFj3oTOnHkoKHnGa//MYm2+Ykb4wsMDEVhvGQReV1zbgKwfyHitJ5wvhMnxmotS4Ity+rAJVoa4cu95d0x+Ca2oScN/yxTiF6z7E1JEaCbYy/u7ebY+wmzFhz8LqL7+5MvuwjY5wpkJq4uE4yT614rua057bsIG9QXCjAK97JWhh2e+Jqe863KxTY8fwTTJynPW2pX5gzzyV/3EDyyAvUUMxRyteUTnJDva/AeEbpCEdsCJwk81JfhoAqV4XXSghQ40WqS8uJX6pwiiJoG2++w74oaGvykzc1BGuqe7CbXnSVwEa+8i70w1T4lYDONFC0+7wnGD0smipnC5R3HwH/SewTlgauBGpEnh7J10P7eKZhnNQeWX8V46KhZpVo5LMDgo4Dghj/0pynsEHLXbU+aH/RJbdGnp226tJNwr94A3QXW3km26HcQ/vU3xV17GCt5YANyQCX9G/VlKOBrEYOmTbmw18D46xu9f1hrih6Aq3mfqAaHraXEWJ8TJDzJRAn9Pfhfi29H0ODL+Bl73SrpeMgK/rNQFtvHMm/6qvSP31UfxiXBnJyVE8Wilhk1q8PXn4t9wTjTw5RLx4Hw/X2eAOK0n7kQLZLrfgPU7FNRkXi6ilxkF/odbCHApA29lbK8sOdthCeOxEjXYFtyMtCUnbfjFZdpIGk4x5PiYENPEUp5Fbg0Kdv2Bjmeu7gmxIRv16l/cV+4K/PzX9NtUox3ec4Zi/5xxuu+qPpPx4afHQ2T5RapQO1H2j6miw7WIKPLpbfk1VnPcb5po7FkuTdTSrTWI4dL+cEzQd1DsWBOyJWtMx/4RCa0Yb2LZSEuR2tGTl8VTc9kzmRj9Zf42m6raDipxXGgYBWmLAhloPkfTGwjfxflgfYTPwgB6AmztI92uqETxNDKcnfgs+uiQXtgPltFWW7VMZucfL4BSLHf6XiUGVGyjK8M1N1yg+KNAYkMuFOU2mbpuggsnWM9oII9kYIv5iTMiMorcdlTVa2ReBsVo1SdJ1hAI3Y8nwx7UVdh4qHjfRLBumecboBIn6/VsJkGgwotAtd9u4Wvzsz4U3R5htPZ3+RKfcltq54TyiMhG3xlG94LW2V9KSqTpr2FRfUBbX9ISbejvApSzlhRGA2/Ar53cpra9VDjAWuNhy7LrBWu1jdnfE/kLXjm/kZDll5y3Xf9Z6QVaKv+X96OqzcIJxy+GO04eYtSEJ92/ckRGTiwKmw08KJSmpq5McyLLPv2iXKMbaedH9jkSIl8b3hX4E1ECbcN2dLmY+8ZLBdy3wnTeN5ZhEyaOENMM6sOtmlbrIUVPIk7/IsfhKm6k9juib3S7v5Z4ozTta4s/rfgvk4zyPUWo/g3jIJ1edpxxUkZdh6dR8kgncKMHc9eZxpkOxVwoM1Slkeoh38m76t1hPFMuRkcl6odAPECKdLT4Rm5qEYhKuXeJlhHmKaCVgRcTHs8AD6PAVjeGHBnXSpxbJ5Bgaeet/QpvkOVBkxjzetdAjmF8FmrGapOgid9o/j5c7Yjap8BF+fZ0bAQMRbfQ2dBxa6s92tJQ9XHzsxq/20NrFeneX6bWlaD+G5r/JwF4MU4Yt61eeBJRu2kTz/kfS9e17CgSQ38JTH4k5wwmvJGjycl8/dC+U7WztXX3jjHdaumcI7UkllEu+IEvfuZJDV7ir8/0E8YHlR3nJIL6OyJ+nBGGxc0tAQqH2eY6zBDsYYvpvtF6DAczkF5/V25d4FwoYFsa609NPfFHTtZSEeSsiXuVqbn4aLrDn5ZKmeA96vTsMzCuMDphW/ZNUbXRTX7xI21ta+TE76mPN1jLBn655oAev3n02U8B96PWjrKloST3jX/cTSQ3sxj7FDbD/QrkLsq5UaBd1VOq4PVmCrFodgfefKz8Pkak3CeFQ1J4co14TS+fdAR/8wul4Lvn7MxfHVZtYKafKqsrrhYvKtZrx0vFK4Zy+xvjaPMrhoUW8UZuXqXb/ru43z55Uf0nx+43Crv4Z1jEJCyBE2/jqhJ0okXZKA5N/fBlN5NczeZpRSDMl0Hfb6P4QsxXqvSPfiv8A9qON4VxSVarrmG/XMuFwSAR6S0Evk3dhDIaNRQR/dZZcM6yUJYx3fYbgc7nVhf+SSGx6QUf8D0R0fZBDll44gZ0Z7kDqV3UKDe0PCd4B9VrCAx7HjsQa9Fh2cLpL3FrK+tkkwlL7vPXzKMrPmNHu3I3joQ0HPPD/3H5Bm4clpleiJd07wo5SAVHG7Wx8VpcR1V6DZUZJMFm7YSzPQX+w10QhONh+9NjGnEbwx2xARJMYvJGRSsgRU36MO1LzQ9Dxlcqb07JIF91aGIaV34z7HSsUF+NAg4g5GTuF64py3rkvyvuYn+02UjjWQleW3L9rzBkgFV5ZiS1qVS1btImfLZE2+Pr9LV3v4JrOaPB8x1ZAHUPwCwKXBoE5RmMs8lw78pq3F+fOxin40DgqTRhoBGGEeMY6tYL6286Z+xb4nCE19WgiZCr7QK9eQXZOMdDL/m14Cyo92fcK3B0fzJWR2kjEY6lSn0YU5Jp3QLrHUMHybhUshSVL/1Ibj5e1f0ziA4cuA5mRIyKcgNKN2KWmmMt+HlbDbJBUzRC4HtDZmZHEmQyqldXojIU35QQpMpz1LYJE3SGto1bmU3m0xIa2yoKfvKOjMUa53tR8R31vgaeS7CZnv9Nfrqwk6CvzfM66oGD56sMraUKp0jzY9yz40jYr7MLfB469DN40+wMVgwJ9ipdG3dIq46WLUs8Z4inWYehjQM3MuBV20P7c0c09iCtKgvS0Zm293opYt7nTpQv77mAgLc80NbQSjoaEMEZ3mPsU6bF1DX95THrzIQtDpwuWgN/6+5Ps8Ox4R300D/kpp7VbLLXr9H6ksB3+lmQL9EyxhYI0zpvZzTWT/nUgB0Xgu3hhs/+R3cEd5tMZYd2m2/nuOTtppR0CSGCXqwoI7YpHAhk4qcp6C2vlLyGjVB0zEaVbBqkmoqhTFD4UxrIdGwc/vJB/jOp7mBuvi6wLiaWKPE+JBq3PLP59XiP+5SjUiZtzQc0kZXQAehJR9Nk7ZxAGvBRSzxMoQvLLBVM0zR6L2PfPnGPimcbpl4grdVTFEMA6G+MzcvgKxwDIvjriGc3FL+HOnPGN6cheU3iTeujNPDWYH2n2+Iw6N1A1Hjar+urcdZ7vnwn/SIS38FM8VGQzo0q5mN4ncrOOSX7GB54qHe/kQhx5rT76JV+RjpefFd8lPssjBtqCV9ThyIUZnnrZ+yXrVkgxnYEGJNGmFSKgwI8u0QDG4RwUD1/qShvO2v8TlUt1eF40klaqCw7AHkeR5kp6i43vWu6e/pdyX+lqex2m3PUyZvfPGiggfxg5vvuiu1SylEoJVXLu9dWfabu7LpMtLer+EUsAxktxHbqPnzIudKFWxXRneqspLITDMR6eIucpFU4wIFeh5wPbpPJiFzO94t7di0qnxUz5hTfZ4ikLlgO9ieIRapvVBv2cYaP2k8NFMAhO4fmRXJejuGuQgyz4RjXA17WL8qvX1FFQRBTuYz7w0KTKuhqMpkGEHW5bxwFyCJlYr/UYg+viwWqLONlYCxfw5gK4UcuJL8xh15GBK6P5qXRLRCIlERpi9luNMLH+1hmu+8bGn2d8sf0MnBnXKDkDDzkA0F0XQIp02eSSoZZWgW/M/VbFCbD2lQp4/WToclDk6uqp60naCiE/FDOINn91m5N5reOGRMzkIVyeh4+wwZ9kyeGU0mAbSJvs033PJTev6qmfAsRVYmkjYfVElry6H1TFN/aax9Edy+Z8P1KxYo5q4bnyD0ie0WuZN27Z+66OSP4UzDpFPXppgHFc7s6ocS978/hffXZ8XA1p5bExDIPgqFRQoy7aRsnz0fy7f1r95+fv5h4f6s5BRNUGL3RhtpAbuTFjjHv/GDZMjq4rrud1CmsQoo6sbMof/3qSOAQiyC+lgVFeu9woENhAapZ8Qo7iGtLebo5kYd/IKo+fD5DM6XyqGCn0CCODlSvScwcE+psZsMbh/0KvyqW3ZGXdzf8yjDvswW99xnaGWuibtIxgdG+Scb5bsSNp1PTkMSxf4JHbB5bLn1apUOjXb8UD5fun278tYhRvBCfQaVvZjqkzRc/b1vimID+6jFTUpLvgiz0oCsvMPnyk8kWHXRfSWKDa3+8i4mtcDAbCf6ngZnEC+YQHAi3NDL0Tl+MsDYskgKzHehV9av9YB8vDuouCEvCk6VAk0HluO4j3z40v2R2f4sT7psbEjCVrquSb7aCGFrIWqzpm72zB0n0Et+zdIt7lWvan2KiFoqM450DG205ivjS5dIY2kBDm11dVzDzhdGS2R7kKk1fLc6cGUeY1+X0ZW8j4J5C+Rssga/tycYP2hFj5ZyXjyY+uFgMudd+/Nbat2NPMN7qGJyt6HwIfnG2WQ71Ts7kRX6dmXQnarXbAvhl4eD7RYziVkVSSKpBMgB8D/oOvk04mQHxEd9JaahaMzdFzA2zYtca75Ey1MmTWVQkxTwow7m8PGYEbtD2RUwB8so2+8RipD37SuEYVjMIbtj+7P0Nd1JJenepSc2RrJgaOTNvAH0wHkGY0HdqRNzI8x86vn3eTj7IrvZ27nea2FNii470Knzylb9NXDfiK2H08Eq7+AHRqb0H71j/dRxVGBfbL8v4neKu5l8nK8DszNbjSNO9wNvjhLbx+6vTQ/oWDN23f43lRWMinO+ngbE1i6zjZR8P7cteO96oKiKjYEzJGWjM6/F4Y5QVH2xiGv0rGcvkKX4NB/JE1IWLqbfHYOpPJ7/w3X+r6WBZNtWQvcyDxFVf54MqvkcObdkZhsjfzEdwuY86gH4sxDtR126tsmDB5gboXRxqa9WvTrNzu2aCJ1B0fLaFCbA4g+8vm7OjShS+HqyMuhlYN30yjruGOD4UKBhy+yG/oymRrgE8ByWcNQPMScZLUtNbTDM59M3XRZ4JupXgX6+hK5w82UHsdmPE+qC97O+SZD5L96HROsOE7IJC0ZfJ71vwl99q398Lpu61HdGc4j8D+82S/fEPaplWOp73urOdBZ/15GJ4Pf5eilxn1TQMSjyBW/LXg0aZz67t63rKBXwJ+t313K6rxWAyvHjFKAfxndh2TGAuC/NSHC0/Ve7Pl8WoMzZjJ9ejsa+vdMVK/jqvF9B4RlB/UryXbemjL/vqpvc0ol99g2TWentDuC3xmEMYNmAkjVbJy574syKSalWsWIw/mxyJX+X+xm7Oj95qaPgnaPehM/2xp5r9bIHJ15VeCqeogpFYjL4c0ypsI+E++CyPSMLueLi4S3pnHH6XLM1PB/S9VQG7s+aRrookgf0ydaG5zui13RZvmwVngxvk6PuV/R2dtgFS2zd1UdSOXzT9oOznj6xegY7PC9sLYYBejTB/htfjwsYr/GDC4AMhgU3exCFh+hOrmxKhQXLKNlqReeVVnnk+H3/bh7UFsdRq8fWKpMre/99qt6viivLAx2rQQnTrpZ6lap0/w8TiZIJUSuIBsYBeY339uzghA8eaqvTHp3pGzAo5IoqUaEt3K8Q2/OKmvXpLCUG/pqghaceS/wrTUNCN5j0tjFCZFUs3Nj/7jvEHJJYpV94gqoeMbvV2qsd1Lk0vO0QZ6zmDrivdvO/DzFE9JoerdL/jNdRHM2sf1VWpEGyubBe8NPGl+JU4eF1FFjj+Xpsjxm08OUX6ZduG4CISGfkxHbhpZXjVjy3pldSTtN7CXuw98HAdlIyhNfnU0ze2qJEs6uUHwNInnCwrVl+ZlMWQR5UHd6Illb7XNy06Kw5Tm1SEdE1nG4ptGoEQEJjNzZhyQlcs90qfrTm57uVFO608oa4/x/o3JoTA80oqbn8oPpaLzDJ/AK6pJLAfKI0i4kRZpS9C7R9cWxUa4muwgnyYhLfvdrq+r7096DdJjy/9FpkmcfgttWAUhVDcQYNPb8c0XbnCanHvGIbos9bTF/2+Z0lNYyS+dOb2HfNzf1bOyho+tl0g6KuCwUP5Ce58Yr5MXWkIaXNOGy30suUtlWXv/Ro/TroZb2sTPuPcvPzrdU6Oheh8DNVGesv6xNZ31Cp8JgTyG/MLuF1gFZ2/SXSv4HbyxaChCWmydhntzMjGujDhN8KVsbsrFNFKUe++/fXuuk8iGh9AF+a8iPzzoYOQzwzhi4AdO8AcMdTZSek8wvJDztXvS6HbsxjAscostYWMUKgZElu9P2Mfz/pGx8qAe/mtUMo3PinMe0MKgOLTVNTMtROKOU3CE5NBRZzH6t+rhU4MOON5iN2qnzfcUeOpCzgOJrBJJF5H0eBSInnsxux/6MLuHvsg3KIyVTWk/fBVRajr3knDPjCv8qPyDdoMCsUkRco3rz4nBHrefQw9tL/KnIwbMAJ0e0V/37pIPt4QxxzXw477SVbY6Bf/O53BhbGHJhrfrI7yITPG1CSYJja3j1Kf50e+cHtJVMkkvnVtfTb4+c9znRcj9pj1I26K+3Ev1Ur0mSIka9+4Dxel/MoD/2b3GJEXKms5+w35rOO+cLGCxombNgF90Ijxyi0m2skLTz8zi2b+4o8bMxyvTcmanfb9ZADwo7j1hzEJPED52eFfsfbBHa//WyVEYrS4HkKnTuOSXLWVYmzQAJlJ4fQ7XTcJ25INc7FMLnf7AhrPd5GvBx+dtouZhrxsD46QuDQrZtW8dwPEAIJ5FfZwlKqEGJWYv/bRzMphVr1CeHXMu5l/Da9wnHWHM4peKZEHcM6GDKkhjnZQJJheLTiPywZsr9VVWEA86uu5Z1APv1oyolmomkSMkwZX1cTvWzOdSqfHD0tD4PQw+S/Mq/JtslkllikncqoVomiZp6ukES3OBy8iayK10dhFbfARwsdJeFaxpGmedf34QK4KQOMNbRMR2UZKZm3Sz+6vD1X7Za9+LbuUqeBYRdbOJ4/Ymq+Ji8hYmWyWqDl5sjVdIQkiXvfDt8J1wfE5fZk0xwghA9P21OeYWQGm5rZPmLIGeApzqAYcrK/IidIVRFsBej0hWqcN7D0ZDh4FisyWXwbSurfcyqqvWEO/drrAdPXrvc7Ydyh91GV1eLio9o6C4w23mxh1VPJpkaT/NHOJyH4i1/ppF6y8YxQUY0JlQ9vnpAGtuJJ0DKTlLyA0ivvEjfE39obayaDDJLy8tJhhidH50rdEUn8mpJAVe8ZngnxF8wkbfstBUyFSHBDPCoTmqtBcdQXdJvE7hBLnxh0X2g0MxhkK5MyCpNdD61x9g6MIOUw0QKTl1uraeJBGDSkPxfIXifTOfSvTFqa3rTDnmMWalISVQDYkBpLvNU/3eea/59ClzabZrX5GHDi5OmtVu0nhpy2LnKV4oVC1L/iybd4YFE789ea+2+7wWgfuvufZgnKDGfrL5IN/esW1MBdsT2leFPbrC/WqDDecsCr7Pk6MsoO4MbpR0SdjfNyC26cBj2U9zMrLl5HFOA/bOreeP9IwlstN0XGDrG8mQUWzuT78s1FwrkfEOy1iN8itigPFcIRZldPfCNgAZuF29YxxV1hp/0KwUb0las/FMQCH+JDOhqXpiAmRdEE+EzlN0PS/hZ4QrRfOuatGTRzIKVvKhaL6Kgefu4Ih/vxOg87yA5DpLyQPwVs7t43SoXKurzZjaZbGQ/RBlH/q2zqVZW1+v/406hlGN44CHpJQGGXDKJFCSCfrxpwMHypU5SLSaRg/JRBhP+VvtPGIvso6K7d6fYIBXQkvV8tD4eEL64czIFxIMmBSEcVcovLJJgzk7n7DDqphZVPCJl8ZrKQfKG16JHhgMnOyt1Ij3Hi3pbXSRiReI6guSaoplfJND3yyb3VvUr99zR8/qunYhfJ+GTVvxwWV9ROlBEnngykNH/Xum/a2CSAU5QG0kmQpi1PM1OcQ93dN9r9uHAkJf+p3N1HC7QoiLGuJDlBT9uaNtrCgifemZQQVeV4LNQ75Kgs129Rp+uqL5oZZR+pBS30ecoXkD1rI2Yhq1hZGbuhFCM3JaFzCYZaGktZDPyof7OOsso1MW8d+sZJ1jPCXjtviAFXo7kqfsmPrVpNDz+4T1OHTLOiCkGpmyUD3ItmMvWLkqB4WpVxkLw3kj90Y4KYIzLjdqFtHSb5ndZwdtrKDhY3b23vDMR8FsRP0gqD9BjsaTvW7XHmxloVAxYHcqFi2FO2OFi6CI2M3imDLOfKWbXiHyzuc3pcid9lbeuKA+kY3u97nAkBYnAa/b+ZXfQsImIjDdIxtpSpXhBgsfGHej1QzDj4UGgRhrXi6p0gLIXfSdhXt8V1hix0RQulS2B6Qe4WaaM9twkU4ep+1K7LX9ztpJsBdQnEV7b5fhmYpkY6W7Qeh4cyDReVbmy+GV/KKY6vGaQJXplPECOvUzmkOr+fSfnECXfncan6G6iuOL6irh8DnAy+yZH+wMU8+fpDUXU17ieDY67rKg7YY9fKJWgkM24lN2jJ0IYqE1Ih9zk1rKDxIfz1f6omZlir6kMyMoJpS3P5DluKVmqdxTt3QCOZ+pyAOMyrdnHfbX6gCHCVHHijNOvch2HdGXXt/5Ue5W1thhFqLJsQkBUbD3lzKRTUKKmSbUNQHrU3hSFyS2raIZOAWz9uPZ2lKzWEjoJaHlfjnzJkQO3HEnqUTiyrJnRwTfDUceb9V1cdARAy3tsuS7ILyNcNK4sKbAHAWRvsdjrizUdE+K9HeHCJKk6hT4rmmhMc9CkoAxndWrFKmDfwZaoQIIo6zsxfInytVS3Yjz9DfltiwoN7bWspjYsnwiYWgEP1Sg9Zb6mCI5OnIMZnCsLJbwhHkmJa7LcdDv24UgbXMU4FBURM3YxPNce1+3RUuEl6Hvm384LuBxQrcuemHm3zP9mt3zcImnut5nqPbcih6ZkjYYrwneWGsyZf6YMt14ZALHOBXdD5H83o7dmXdm6Hkt8ug0yrByC5loXCv0ilxRBoTWT53edLsrwTGfKajzLAYwjdtWq8xXMUVKvdLButetiQf2Aly7+1oJrrzmBYIL0UDVrrNj6Ri7EKNlbB+W9kSPOgeKLD3pwPINdOIU++ymDfIXD+zEnoLVku8uryoRo5eotyzrDllEplLscw4pPvD/JlZmUaXQ33IW11Afg2Cmf+zLZPjOFdii4PdjefyI6k6A056Ut6ShlEsaN8n2638DehqFBZn6XGgJb14qRiWoDr3kPm2uYdfJag9OHe40S6bZWRb3jcgyLyl7SFb9ndQ3UF1vPrUmAbyay2S4jIiTocTr09qQ/eutjvq5JJvjXg+tEYs6Rbry4FYRZ5IV8wgv+Xs/fZ9DGMbKF4n2D+JpogmfLjvXGzCInvVqRktt/7qZ+eNvmQ1hxoAZFaRdXrTg6QZCrw5jJbZpT5zeB8gAgTkZq3ZEN8Ci78o0vpwrN+3hsI2cea/HupAE1FPq7zb2cYS/RA6nsQkoZAmQ8SBENM7BGWiQNPvFCbbMcQkyChM8zwRLQLnKLK8+0vo/mIC3s8YUTiZAE9IPE7hInozpzKCFk+sz9KMig5/l+AGxMOqiMa+hIIjPmubsvDRnsW2yt/AncrUZ6LQoE80lGGVJ3/6qxBQ5Pl92dVHYw0muuVJ7lDgeK0hE63ykKPJEdr/UwUjIN8MvgLGFaIktIg9H3luSmpGacmi9BsAqvu9se0jZs5oVrze4kUIyCHWFOaTmT1na5xrs8ZDJc5XIJnM8+8RfgfCbgUBkjfEfqlCHY1y9/2qaEbUjlZsB5CvDf2QoB+MADMf2ue9zCgGKL+4BsvKSb0lizBTzy/cC6NRfH/C4+33RVRb+YPiZRh6n+bJrPnHZVzFP3DXURDqB9PmcXyAlD+G4sjb9DXoY3QZtws+XuAn/84xpID9uk04cZqecHGeN2Y8XAxVg2ES+cPubYiq1+VXPxvF3LaLcSz5d98iEOsCbgV9TjVR0hBeW6rukYcPLy04wk5Ceq9XtW7H21EhZYJoSHRqZY6ktA9Q3dtXJO/4T7ZTLtiU3+Xix+6zi1QXBNWVfIEnuE7Jnj5Z8D+LkuOKIJVoNksxvl5+E/8+2X2hvM5I1mtoSZ18ADMIe/5e82qEgQLO31AJsLj1VG4aX9oC345eRX5o+KegWVYDggopnV/GNRg+Suxfyw9Xfd4f5IeyCoAm8olnZ0ZeSdjcbFm6ym/WvXncENtQwfrr5BEJApN+upI1s+P9sB5zXY9lupeiz1183NFe1bWD/QJRTGZMBEnlFL6ozRYEh6sRHHNmBNZHZ3jsrb4/0FSjJLXa2MAhoe07Wi+IqNwxzskzV1ilE1JbVtXNzr49h+kacOFLwPzm7wF5qOPnXmcXjo015oSuk+xGTUy0mFLSqBqgIm7f1/ypN4hrvqnGtmrYv/PKjAlv0AuLfD3Wo8G7K9BqBNpdalZmUVwSAjSqJU5B69DKHX2umPi4WT2chFr82aGZmCI6yDVRyvvtY5Cv7tfw2xnrnoyQbN2Z12crwI9SHOhdHnbNXn5vGnRUVmqNpaFzyV96m8mnNlxLK6RlZtVLipyl5KZR9r5DDWVUKMVIikHbKiuLSnPNDyny5QJ0cFuSGfvxd3pZ3MgLmx4cNF8LbYTX/J8R4M2vtP8X1mVWb5VoLgCLgIvzN9g9DLP81BBEYsNBsEceJl+trszGw4KnEy//PmQua70L1+vNkWlUzgL3ALakrfUkh5YZtiAPjhgTXmSGba414H4juE9yWS0fXcro4SNgVZt+XKwHv7I0mvEAQphaDcC+tK4QdH5QxH8jbO+iq4uTy9cXzMT5Gu5MBvepYKCXlZDJn3aUmvR3zYu+6hDSp6bM1t+4HSV5HihhCLjIlAKnqMuiHdaJUhY4Tg7XLNWW/yo1GPuy73Xl7P1xja/jeK/jj/bdPdFv68x2LZyxJk1bqV5GfNUHIQQdL/cLAoZVSRjneFivT8XXqcfj/POn5Q6Bu2/QG0+qJEy57e3NRZ6cbKwU95EzLOoatnF3VSPbnpdTd6c/oDnL0J/BBBnNmMRwUitSnH0llRHNyOdHshqL+HLE6+HbwdsEcGuEBRwLYKVsIYXCaYMxpVEFT0agxONtkwN18a0KgjyZVI/bWY1u9d9ShSin5DyeHlXh42BgQfE6myYFOPCo7p6/RHHaZWFyJSoxUR7sL2HRnfxhiOXmi951OOZ2BPVP6XHfMPMx6lG9HqbRJAiiqZrTvDQX0WvGNzW+zuPnlE/3wzbFXIr1qAVGBOx8IPPr9mLb0YVd9yPGel7NhFTOAL47/yxsCy966pULTkkXtMp7jukrERLJxJd9nPTmr7mBh0pbe4hT/0Jh7s3X/ec4wUklbvpG30rZ1Wxmwsfew2WbFlDpvNrVHbrb+aLXWxAj707tB+P5hXUeaVwjVIR72E1cd6oBvVGV524HD0KW0f5oOPr5kuVfUlnoNCm2qWeXSyI1k66914X94NPwHgq8PmJ5HKvaNGjJaBo2Qb0B+drZ1WOrbD+/pi/6FeDPgbUo0sXYUFkFlLZ1pnM2MMdQGG9QSINUp7cmoVN8BeY9oHhdlvMW63EOj2dAf1Hg5K2j46YMMfQ2rhhWO/Lqb9AXc7v7+QAR+FIZ2fusVHie/KZ5L8BXf01kG8H1P9fFoFB8mK937di1ZjzYyhzv/yJ0N1LhBwQ6sCeZ5isOTqGrhX78w4zcBt0v+IEZSghJlRu3E2Z2v5kGwwWOeLuLY2qks/S1dTnOiIDv4mOSRWtZhU9gOVqwM4YZmmcDKjuF17B3poo+XKlRNirYMOsLs/EvXTqXCXZYekYyYnhuiwSdj1m9xddY17I6lXeE9TXJW6YQPPQHYh8riqPXq1OpbcjC9oHf5mXuYhmvJKF5ajjsVIoAMlL4Hx8s4YuS02ndzS95RowxazCk4ZeCaX8Vl7FTQN0ou7pSgwOOS3hCYs1WRuOCQWNkMqwdVR8dwshn209zk+tFCc4b9tlKYEAm2hzG4wMfr+KkT33Tuo9PDQyj8ApQS/WR5kJiLnPrbK349QGqFbH9V5/aRCAs1KPbL3r7BkFQ5n3fHmu4UaN8MPTUDApHwQqB/aJevxjlLJmtGpzznFnvfrCXRGGkRkHTaUUxMdxfLo2WIuofivbqFFDaAEEEfKE+eDBbL/pDDQ/gMEnpV5Z1hW/5Rhgb+JUJZ0CBm1rMlksZw9uKtMFbvIr6SSCTSm7CYLGqHW004pzF/VGzZ/GnhGLR13JI7yjZUbl2ks+xNF8rnqmrLUFNz/aJ77SmTGOFwIxbAUp2/2r0O+j30sVKma8QA6X9NAZ1KL6gDnbpU7mLivRJeg8ATzmPjnsfXPh8G55Gu7Ymxu2HyRsti5bNT51fVAfZoEq5UyHlY3GuIITAqVz4mlVQpa09gKRwn08NDE2ucjPzw1IUytySq6WUXizdBtw5g+xfY7EU367XOr1Sb/52jzM6cwxeiCbNaj8ibcBoOWbARA66/s4iP3xNw+Kaq168t9zQ9kaWSFpuSLwmNONkO+dpTaq0C5COpIVRa8Rqe47UIf2S3mT2Zw6/OZsomT9e26EwtqjPTyImWbpcvw7UQEuIEmnCK6I08NO9HdDDduYtRF6JdDs0YYg5f+yS1k7J8/77ZvT0oNAsfO9xV2srYl2yKK6dT2ifuPLaEvFNF/VNu2RpT2lCpJYonN+X0W10ya7rD+IvTwhvrx24dILIyfLCKZL8Df/xG2cU4/ANdXyo3HRdNl+5zTDuYetL/tphp5aCyKYmkP3JuAcc5Z8H+LY+akdwe1ZaZ5bWwApiXLc0IeFZLIpEAa0tMMVzRZRgeihXKt+qN6Xf34BlXMtyWvO7+SHfyeL8WhLpxYMyPjyCyyyXR4nx2YiHX+Vjd20tJCkQUn5p/8EI8JoS2S04M/3xK/dSj6MV34RthjBKPlZezK/qmE0bK4LHzTl9DB+CpER1p//OKUcRSzVvuAD9ETBMEX26EIc28+yTZlkVXXKRah0yEnSuUmUtjc2vqE9Zcj5uX/lcv8GpB3AXZYxA5tfhC7S7jGcJXgtBaq80CHAuJPXaRkFxAcMDoiFUTJpzERiuIxjxGoV/VkAYKotoTk6XF59QjbpV14gQ1tHY5+YXv4Meo4BpXlDxOj8qlUHBs+t4fzd8snp0VI+Zcd5hfyJhmJIWgRzypSrX/s54vw5u9aH0+sMiz7Gt4WMaoFpRqis+I0y79XeN5IHPuiQsNooWIZpIMt4Eb0oyBdS7x4MKG+Eeety+CpcZlZ1wOXqu+zpUkQsFZnk/QZ9cBLdhxdi8Rfm7sEtIt0BH1piRKpM+TBJFY2gV8nR6HuABxJy/nj1DACyecZT4q2PxGG5kPsbgrqYl4XOfyiSJTqRcC5rmvumG9kGeaGbU9x7eCuXdwn2ZHoNjLJArSLC+R6BkkjX+egp9HBtAglff5RQiGQ+RHrXnsxlw2VTQwo6uaeE3KxkPCO7xtA83cyEEBwwre7xb8XeGx5TMQaqzdHY2XKgd40jc6Ayz89+vzjk8oygF1LXgAkyYZAq8N7dYRE/4A2mYwmelTa0k0PJXtzTR5CevM8QDgLdMmOknNK/4t86BOmw+dJZoyMKcBXi0O0B8LsoKACZEDnPfEQlBoswmqwypbVaDWy5C2oQmEOJHBJwXlxVGDcIC470eJ0NdqVQkVPhdRohZbkEFMccyDco9YLhma9KL1gyC7wlkLbYgxoQNdGTJezvw237ekkh9cXr8uHphzWITCTAoP+Jbh43zF1aTd9oNWjGWw3QLc5LupGYqrbXED88PsDfwqe2eoPxbKlh+5bWvuJwbzlmR5TT5LzFISKHn71e3gzeAw+sVVSB9BY4jB73Ch3dZi2d7Mg+ft5guXRjDBKLVNXeGb4XUdJ9maHcCvtYgZoKokcD5vr+TAGEUcoIfs0Kf+HdIdzrXfgXwE87o1wZfiPXV3Ad3guIx4WAKxG9zJB2vbz7MvlngovPGSNIUfvpdigy3FLaWCWMsmk1iVNUts9/DgYzJ692ZkxrGI3aK+hn3iteD2Lp8NNLaOD+KTf5BqiM+jgUkBpaYrMP9PMhzaby8MsC4XNP/XyTNfCQOSVZVvJVhdubLdvF8KIoSYLNJLQZUd6d3H0xdtKXMKz3NqhTdSEHmEnrbqtPhh4wK9q+FYZcMWZoAS8bm3EL4ZrbG8IxY2RaYOrVP0V5saQOphxV/MfakkC2Ds5OtJ7TkI3pMpdsUlOI9KKazTurVUE5BTfX2pREZe0M48SaaCvA20JuX8kYfS9xD1UXHXrPFp7zyxWF0920eJLb/BomBA+SjgXpO94nR3+PbBMQRovMXhXVT7ljzcSXuku7fvqOqrCGKMfqccG/Lte4WzILt2NRHJkmYbaBNJA7N4SmYHwAQxq56AubrKAAQMY9SMXS6JFEuQYeC1Z2OA8Lt7uH3UhQhUJKQ9SVj1jQMNnUm8RBwSm2+/nYA/GpUsrgsjDkUxV5vcVdLfEVDl1r57oTeI84mpEIZhzEg5lHXn7UyyICMHFfjswdV9mbgLWwYDzsFyKPFbp77+/jNJtjZMu8Eu824ldo8eRHsBlAi7naBqmePeZ4oZdyGkHyA3uNCMLANiaB+NxYWuAD2uCzuwSkHAm0SOwReEZIjrhB+Lo7Auaw9Qwi3c4PaMyrvSgh1uIWhPcKFT5gF00KFAp3RfisbjMew/svJjvTzecaIO1PRHfoA8dbvfulX+TWu7blGJswT6i9vkL8QzDljxL3en8lVRYVNjCTqFZo+tny5cxraQhO0ymKKYo7APZ4SbllPH8QaaEkwF/BnKkySMRXl9m5Tfssh/IQeOrMO7VHaQdaTBxrNJhSJajkB2qyyBBMqDNP6e9m2ArGPcE3FQ9UDF5MEGOg97PJ0rchLjT37vTP9+ZmIu2Y/iAiajgsv7vOn3wfQJ9cY4Y6CBzsY6BIMu4hOIINOcFsgibj7RK4k9kBVtuvPmWSyem0duSs+rrEVwfpCszIuKnwHBEJpm24kBgk3mBox911AAJ4z82onpnjL16LLbArYfP85RdZsDmKIWxhFefM41wfNJrasGNGo48eL/F/NJlrwD5yLoimlNCKi5XFwSEqq+xD78oNbPuQtJ4x3Yzlr/gWo4/2r7g4RSQ/LA/ywUFmO8wN7RpdfZpyLHsJc+0vrc8TvyjN8W/a46INHlJNxQNdvKMppM39YiJmFcA6uM4CMvXd6C1QGMfSv/cW9UFL0YnxhnuWvDInOwG9FvM/z9OCx38C3j6jMRF2bM7X3vdS3PnFYOB5e/vsK6rGRrb4IMZVd9BEExSkJ45PQAl9byLFTe0TCrlnI3jYL9hYlzwP9ArddjgoJf1BI0SFcjWSY/w2dheOys5VQEjoU0V9en4ftRU0NtPM4zdROls+C4XNxecWLK156qtEID0L5ejsTXG2MHH39D9J9OF+JWWVNdnI4n69G/Np0GuyEY09M3I/dJMBouf0MxSwpPgbobSpyzXfFNFSLY16q3dozQOmTPkMEtuaKPLSIIaegIOazZYdZwqhU1w84w8EqVrrGRDw9SsR1/Ealkfjnrlga7GIa4J0pcHwEuJH5tlDdtxXTIC1v9flFWPLve98/gjRY5lC/eJ80wVxmwRby0LX8rxzIGzC44Vv2YSq0w4IAF8Xa5QuGGVDr/Rtpb8pBiCAv0B2DJqEaukT4pQFnNSDvwowOAdfx1/AxL7LlFxH3v+ut4kVUlO+ljugWptwQfcl5KURotD3Yr+UMov7fquwG/2pjgiLZ/+1IllcOu3EW6QD8MHaVF7nKFuPavLVmveKbW6hayEnDXe19vjvLCAcJl8fjsT9u2/ewC/H35L8L25tVETreoxmBGRLjGFqKF0ONmWBEOIyR8Zu6Rf99DQwBuml9g0XNx5B6IBO9N7uMZg1jSJEZfUqjSh/q2NLA0zoxmWdIPKlOmm0qGXgEQp8ipu8PY86syW2xZ8muyJu3hz/fMdzdpiXJcJtcsty4jnEkr28loftkpA6N8na7PCyqpJ2A2ngZCTK2kl0ilorO77/wJDjmcFhS/hZUfdACQfvUCRzKjMeu9pb38mDnJcZYkPYyOPfvfYB1RXNyeYyE8HcSil8dVbhX/zny+/EsPm2aGIBR14pmZrAglTtiTHHhyFbSQ1WrD+r3y3OQhmpTs1RwGkMcT0nyiLRyflf7WjMlt2zFUDIZrAPFmw8H27oh7q/rxX5xmOp/I2U2V2fvMj+UbYYxQTyD8UYpSbonFsoKp40zIGadL7HNXG0jU35ch22s9rkthqUhGCDFUWuAQKWVc7/+Tm9EKvDQOpD3ZnCHNcXXWF+Ssya4o4Awot/wA5ZZ46s6JaLbqQOBBAvTZScZx6z/9Wg7ctI6C/tmi7f7NT4eFse2/aKOIqeL3aRUGlFOyN2z22YLBVpkRdakUJ923ZTYigHJTywDPi7LXihklP7nppPTVXKcRqNPF2Ozgy3GoSpP+E1dUNezHxRRN0kjHr+ipXCg7HIq0jltxeXYBR/owtNPquFPo50YY7B1Z9RN9lN6/qiRlzssMC7MRZY+HotqlYwt3Ej+grOpyzb3Ipd6VfZ2bQD4E7rsM3Bt8WIQoLvxXTuxbugor8Cf8SYGy3Fw4GlZAuYst3DBG8RRpcUoNtnP/2uf7FBqjo6QUoPLHIaEpAaHsyrK8thel7TjUkiIHL3ht1CtI1upakASnMmVgiz1vMLO7q8X9qxujBGVB0WXd+WgZHN5YqnHTTbovkQgZL8soDiCQRMwN1bVt0W2Af1nFtnkEBij3/r1qVf9ZbghbDyxf4toMjIz8JanwtWnbyfXHhoyqPMCzMfyAom5MVS7TvOFJ7ZPQMF0SRCDSKjPvX56H/Vj4Tj3MktmLgz7YfqEauxfgi7y5yvWOpp/we0zCRC00ra+ohMzK+GOC8AGHHYTaPy7uPqbACrkYJguc41rXvF0JS+462uIVREcFJG/NoNAXsV/guydfb6iorcv5Dn04ji2w72kbbQV57gHZ7Umkllw7MC5ak1bDE43b7N4MEfNN/ayJMWLWx48vEXfPEUlR9Iv9ctqZ2WeMCLlUOAdPUAmZDmRuxRnF8R/R3vxhZBw873S9VMWrW5V8DifWDsWk/cy9iqRjHOIWaehgoDE8HiZ4FwIraPLJtdiElF+EbpROnrdKFEY3MGpt9p5bjAGBmIzPl2xDg5otu1rE7trYGHa81zakPR64FKXC+/zwrkEEWEjmJVacraPjtSCz4F1sUARGLVsJwZDkss64wrfK2kSSdhhD60RPuefK/v1G3H5jxw1HfXhP7/cpRuhVbJYp8QIDymwmDlLmgxhs1bQVr8qSTu/rgtBM8tQbV9wphmGdMdI34JpliAInmbRYjgtbmcqx+4CZ1+U30B7WYEnB/HXVmW4G366bmS0pyhNC2kkLYN6ob85o7/RSWX7hSlSRNJDU4sHXwAR4HMs1KvHulMcrKVawElDAFxrNJiYu8/HhPmjyrpqVGFCBHprSbw5F3W3saTaJcXXUdbVJuDowb2VZIeKfWaKOsGYN5Pektd/m6I8TkbR5toqQQp+vLuplxuuKfPhMELf+MXW52CSkCIqZRh9Gqr+AGwdWIfhIcMJNswMIyL5DWQZZmj7unneqKDTkbCB05UUXXndDykhWYe95QfXcvvjT4AmAMbxAV/16aWX6/Vn3fuRTcjaiFg2Te+cuGa1Ign0OaZpRwdIfgumXurw1WNhP87sroBTQUwIFJK/6X7FRNkIzKcPLb93Ybk/sFdOWWWvvRa3ka6MhsGWfYTxS4VH36xVvM7YvXbJ8f/qa8HMT7xuZ2t9kCD7B0qjmyKRkdkpTr/Qc7aS+w1zuo8JKuwQBnzs51qnXw4oP67B2BzLN+NEliZ+p19nEIb9ms0tS55POQv2SzCwikEoThbuJIyjTrnyWUdEXEG6lBwilyIk/b9e1lBhxv/wkDGOi/cLgYPSywoqmGXB/UfMOQBPcq3uju44zkfbB918uMrbG0jjS889yrO0e4TG+o14iiRxhn3LYaiPybq8r6K8UHKwl8HTjVNsPJcqj6VXfckxgUryhqYM92T92kcHFSmv50BPT4+6pwlW2Viay83vzbz2AuLbfUA0ACDzL03SS+249WNj6fzANN0HADDMQ07vDsGW+/VwAR969vYn0MFfYfkdcAGPPmkTMR8sjkgR1QzEif+ymO0BEtEgbB/i/Ia/Q3HVWOXnwE+djFjPhQ9yJO33l3FCT/BZE5X2tzfHMPt+HDlRVt3fuhYBkEUBqBCKEzc8D4Z8NIbteW4s1+S+Pg0nIYikN8vDgKA6ouMeuuzY7xg3SRnE7T2JSGrE+9i4Pvqx8RMStQBHC1JLgGQQxX8784Ulv/5jJMZfHNvWj9UbW0o0Hvy1Krir+ihMjjvqYjQTz+QGbmBszZWAWOLjybAIAKKAYcxxQnZe2JQWpEB4eAJnbNATeAwN0mDVLw9IjkQXB2/32HQEk76UZVpDsDCQ7unjHKXLjhe2MQpGMj0GD9Loltr68sGx6PJwm9FhFIDKxatlUIhAk3fePy9dk/i6fPk1zm6FD7FXWLWNbkwaBACSt6yHW+TpAISWJ+KDDZIU7h3ypu+fB/xEkV8PyOEzrG7SDGdZkSz+06mE31T5U6gxIl1CUqCCg6zaijclpc0oWmE7yLjmeiFd7csix3kEH7MpzdTUuR+yuuiwfMKayXV4GLsUYhJl2rbb8JwtbeZBUz3BiXVPwGqcJFDdQoMNrNEJDIgiyIFUgd8uaeC2hP2ySAO1rNHWBOKYYioxdBtWTOlb9Ztmx+PqXBD+4meKWiSfKOaGGYGuNKqyxWHy1z2DyUnIwcG/aK18C0NcavYMOPf1zVi6RKtwQA4qaQ4NfWnxPDxutGKz6zHIfcjU+6ceT8KEL0swEkIP47fXuoYkHmwOROrTcMC3fOBsTCxQdPK8M7k2AEUUWPLjdLeynFx+Z6KZdb4ExHfgNWOfpQMgT8NY+vE3O9vR5NeH3OBTZO+NwEOM9q4v4TFfw7pbBBwKArB4REq+Q1DCPB4W/QHpDRswJoPbJ0aSwPZ5K/2iPSG3SmsXJPWAI8hySiZXK2DBJtzeLdlp3sh9N05toiPHf54KaiCYddAhHuJf8ytAr3/pn3NKnE1khhKzZy12hkVtQnV5sa/xtcK0DW4cCBdM0Is8FVuaACEpbCmgE1GN80CY1lOzoiolNZcJI07z3aIIpJvSshWYGROb8/M/9htKacq3hSt6RcF79RBj24kpl3/B/FxXtvlY2v8Cxyfc0HP2RHj5rQlzbm6YzoaYU/NfH4KZzHPlmgprvARiAdT6mLu33HuXlqxMkeHxAEtYUpqjEZVpO7xVmuz71+epGSKV/sFz7tcJEBU4sczia8U3+b1+pCESU7EuH/7wB1cYqjOP8fBwZ1883vBBzk84ipL+nyLQPgjPNwaTLMnlava0ojXaThHrcDTV1sTjn/HgCWdnRGuErWveENpBBeu+En3WDt07r3JbarWMLyf0U7V/ugi3PJQs5xk8vCuaN81N+cvLdi0XtwdrMnxK/RqwFl2uV1XCyEOJEwm6OqHodcvt79bweg0Zo5eCeXKisPOj8Zzf7jkBH+Sy9pPP3uSygXWjr1/ZovtDfjc4UDETRw+aLgz22vpBZjbdjxJTStQ0pRrF57grrtc0+ws0zB8aBotEIS2iyML+wVx+Qb6lgeIaCBfT98v9JlbxZc8/RxsYeEkKKXGQENSqNxp/ckeaRHb5TQ//zQ5oktK88GFb3j4Ox8LHgd69VX9OsuTj1qx1mnJ8IXvD4aDPldg6TTJHkYtOdyY5FeLLdawaJbor0mF9gY8TwoXoJq5yz82pkhxW5rzpY5qJMW68TBS6tONF4H0CLfSrByAq01KSUebMeEVBdSVuwFlLN5KUyRG+8M2GlO5O5sp/dU1nMGzS+7R5l10Qa9jLL18q8ERjmPdByjqmE0wmIGrSDofKLg09xfOFogjNLWzSpn0intpBjzh8CPyPxc2+g2TtZe4pr8PGVlgCbKLVy3SSTKdzakyh8gHZ+LhVJX3bScCnJD1u/1snONi3j6j8kN24f/3tj5it+t9JF+rsAOvP2AmIuzH+kTXm/k1Q9S76bWMxFZwyCOG8e4n4xqfuctHVlIqPf7ghEPeE1BxRmUgpUpk98gvfjZb93DnlL/c6RdkkpFsUkRXA9EUrGxpBblExpe146xEbHgeRFxb30/feaYicVngIYUoqyO9HN3JI9E8JzUTeSfYa+XutHSmj0IGUURqvYEp9kQvbagbytDLUuIkbRD9+OoXZFXgJX2LJHoND0Iggtt7ngimMoKkg7ajfM2bMlmWHCBHc5yRtze5karstwiSs9uPWIxI/Lu9XI0QBaDj3S9lh+9gMBSd5+hmw8iwn/3UM344PeO9NllYTs4rDibnp9wqNoHP7Vy1EtKkIOeSgYRGWr4kPZClXRjYI/v79vGbEn3XgQwa0fhUg5+JJTINl0ggDGNcwEfrg/SO0O0dYA1d+9YAHfPTQHOvcl7aj9Qn0H09XjeA4kEVPs7kYQjEzK7PIAlvMp1+Ve3aDSXrcbqnqw3sfzxYNTw45Iquwt6a788txZ24fFFCn6JsgA2q0MRTLgzq+AeGdYBIxvze4SrYFt30/dAb2XnswjPIDnLq+XazrseiAL1Vm+WqyODxAb8O3ytaXhJRGL3VsgVW2NO+Q+g2b8ati1QF5NDhfIHGAdBHmCchjdQHHgf2quWm6x+SHvnrf4//1jZ+rk36o8gpUsQzxlDsM9i3QVWmeTnMvWOiGBA4A1ymgkGHw78XSmX0cTJAhRchwlBY0ItmIpikXPoQJcdj0+14yXlQZK1keFI4Qb0VKHfhj27VNQiLuRST3AWR/lN4TpX96vZwR6BfJ+ZO2//y6Imn6uXgRJRwRZ94P3rXD632wbte95UeFL3e0J4q/CXd6KYpQZyABlZUjvpKQmZDCFv+29kzA/FyQ/OoDdigfBEdWcED7O73PLafoIBNB54w1eb0+EERj1MbwMCh/DD8KBssE2F2DZGReToJZwpsCa0W70wdud6Oz31To87HlcQa5n+PHro6KX4v1zX3vtfGYBoStNdOBl0c8WwfR7aZymJyFbIaHKlDAkLXfnbwpX3VjXPxFPdAvkDFrlTpMwH7zxAEpY3nUsiWiQKl8pW88GNHlV7VmEd6XIs1od9G05FzUGnGM+LFBIASv7SWEeMlwXLtxjR70pIEwh9Eahe5gb0O1nP1qGFDzIUgUoSa0ykU5aZ9VUidvzdIsYNIqp5TSZRurtcMksJTFKnmpP+qLh8/SwNEzdsHcaxZ/7bvzEdl09TEYvbprBU6pH6/Xcey9tfI5VJe9kdFlyzW9hzzYlv110GHzTKkfe6/x7XPnJ2e9k9Gq0S0QmJJEX8vnLihagSf1xLMUJkIGWCTPKqrvSKvgcgmli4uvjOdK8XoFVJ5ZR/Id1kIARqJSbghrYxSR7tOEeN//ZeAeYczlnkoSXq4aUkynrPJQMxERSDcrae8giCZmVJhF2+FpcTgGsKJBdDi/fgXHY+S48BGQHTVotNA0cXI/bb201vPj0co6S/PMNtRML9MeMBxIJHnS8sXs+yFf5f6LiIOUf1Ub4eQIaaQWJES+wSg0MThJ7/w3YUBDHsyDNp7jPRrSLq+g1a+pAkk7s7Q9NUdbOfr7pCCKDRl4S6RH6dcyWpxiVNl2WE8HV4DonNHaUej2pWwwJowTJ6Eo3geHNKrhKC4noX23B++Xij/PZOH029EfSut83V2gXGgylwAThfQw/pXLaBYsJX+WRqSurEvash0l0CcBLA3mj/QGZkI8zrGjP6En/X1yC9FD4t8wFKo18cjrjQaFVgxFzJ5ptr1tqPH5P6vEAq63/QXeWeb7GbVYtpogLm7mk9158uHoMddSbtCMUwL2m04i1ZWvQ4V4iGiG7GJCxk57uP1N/DGR1QvPx18CDHRSCra7H/fxPC+La9/KNiw7VqAibjGKNnzjEOO81zeO280Zyulsq4oAgbnq85qYtq6bIQm2d1ko5jzdFXyhV+qj96KG20E9x2g9sOFCV3+mRXpmmLaZlxjVH6DT/XDXTZ/oOw/j3+zlMnxIlJfmn1Cf1eWCwmvG64bBUkZoOKhmFc97NwLY8+X8ZpW9fccTvsOrVnxKcTqnbI0Eo4lMN7ZLD9eIhMeRIH4TGihkq8UXOSEpjOZW78Nthq44SuLQnSOVRQgwLTuyYzHYWYRhHQ9Cp6VNYftyPqnQlpwqnpnn1bDa+N1GLa8Ku6ptMJNKy2x7SYoQR9RhUHiMf7PpZ8l9NiF3CYvjl62qolaVcvxbw+MQ2Fdfcdu9OelavH4V7fOBPwWpkuCVnCglJKo0wdaQ7+Ma1OTGlrhl1lZnPeQV+/y//pw3YjBy8esmFl2hJTOVyEW6gBGtcU3/wEDgalApqoWL8reBIiP7A6hHUuiTmAsgJMMep/ke1it+ew5OiQab/Po6qdUN47ha9ViuWpDC4RMwNOxRqapEO4SQnZR8tQPMMnWy3obG2+Th8VVoCq9yN140KvNzOvoaTXYBywqDkkkP+AseEcvkO+cfMs8tulER3vBR7p6+lsAyd0nqWxyuVdMnk3yoFYULtLXt5w/ihAKU9Fh1fpDKIbtv4xteoU3nwKXxw+H7sdnl3zgjUUw634LpCiHS/cRJrhscVwaESilj/r1t5cfbNlfLqgfchroJR3J8ERhEf2C7m2rBZ0pGxDyvf/U2ZDdBakvKdBHyPXghMk1j48ujJ2Neaz2/VrW0ob4J/9fYzKzq945OatQvVJpPAhOS7yhVPBnrLbSta91iPS9d4KMc/I0alnN6CQ8VZJ6/TY2/Xvw0D3BtpJ14GllylSlmy1jjh69ZipA1EMvt3nYYGEpFFyEz/lbybqRyWDyW1V+87iq7QTvW1fjaRvKqdGBOEY6OU/Q02zd/huFh2XB9fID9iw8PHsGFMsoW38zbX32tROh2OAAqF0ssJCijEpDms+zfOZTl80N6CiEPk1pA/MnHV/iWJAjUAqB9PRun5mNL85aXYT9XHJSgiDIFgqS0a5XsJ4YXq13NDMZNlhmTByNkPA8jYJiUeF9m7otIbhh7KavZClxRb81L+QfZ2Nfy6DYE7FQQhAmtaZT18QKPU0AotHc7/yR9b8FfUpuGMN+MH+DhZPUIVOcBwwc7ijqe2feL+zpHerZ0a5l4LZQ7O0Svrc/WU3PD4KyoRGjaWYljV4j2GSNv+GpGADvqIsaIc8TW05bneaketmK983h3QZhFjrGcEJlouuvSlBiFcV+wKOM0/LoPuYSDkZZTeMQyuRMcM4tHVLkV5H4p/qficAmVRT4rRdsmDR0AGhFOBaZ4PCgJIB/FesePyaEP7pSuT9WW6WtxvndD0FnX+5+XK2sU0unRwQBEjZX8kvKUDUTkCnCPYTk75GiDssLPUKfGVYCWk0DJdQpre+PCPxwW16yh5EmLD5JIqx72FmBTs0ikRFrfxz0CAPJkAfEJnHyHDtJE2Lc7OG/ZpxzIEUAAterUtSN63W9soSLMXYoUTgHVbbVM/KWkKfT1mpfzgulvEyMyhVi/Ec/WHtJFA2+gH1xmQrvAf9AOaWS7dXggnIKOYqHCyCtMgnb2qlSuEMQ5pgw59jt8Q7kh0JaXa1pjdr65RIi0SyKmEkkfQEZtuY9ewQKPikXiPT7WZ5QqB40IZokooCJt69JX+j61DIKXmncjZqHo+pUP75b7UOlvERDqUPbsd5tFZPvFn9KHRN/fIGxL4fZjCzGMgHNAooetY235vmxNAq+W1W97YudOBZH/PuPZIhzpVwphQbPy/RVh+SMG8K/RUszcVMs/yjKbslChI5hUCzpJWQbU4f+6b7Ba85eoxac1kx8PXB02vvb0Ay6/J8eNrz1aUQIx21MKa4ScHsIscNF6Mdqk1ZPEOmrYptnARsP0zoRQUDateHObZusfKXjY9pW47mIdyNeLXLWSatqtMolQ/CvI4uaod0lT2T7HZf9XMg3Q+xsn/eINJe5uBp6DbttG20woPMb7+U8Dl60faAruEaLeNyju+dWw4ZA8oR7htw08AmrlfCANgenMVw5TxbY5N2m+Iqx21jMXhBxs8jFzxP8SHmgXB+TFG8ypYj137Qoizeg2ZIM0EOoAm203DLymUFXvNOJCGoNNofMi7NPLK3aElJNoHl9nxgONQtQ8fi2Rf/muPA02xASVw7YPX3sY9ik0Y/HimIpklNNqtk1aOVoGyVf7RpNgmJCv8bIP+KGjG/DBBNN8+1Qhd2OGAfoJSSBgL0Nm9Dcdv20QDG+zrs0Lq7vMwhPhV541nkBYDvCzPk/hb7jadEZEJWTZNpbOvwa6z3VH12teueoXvk8lC688ltTMK9tj4QBImF7nIh6rMGjNCy3dosZlp0BVVIPAOhJWqCvjVtA0Ta33lylYyDewhE93E+UwHbXZYEyZaKGy6lWPeWiQ4b6nr652lthXfO5MDQ5BMWXnhgxppfYEF9sEzbDQ0oOgaJjfCBw7lSZXLLwyjbdbluvXBcMxxNEKDFXtzHF1YgDojBI4A0rDp9cB9SCKE+2gP4VFce6yNNb9SBmAyG7QNRrUrDo01yiVPZdkeAJNKccDoYaafS/fNhg71tk0wpEpS44c3Bc0Z+QizWcMS9dxiJSm/psyYcxo0mYOv0QE7+6IzEAjdVvGHgvrdBaWC2bmb1H3WztOtPxHzpJPgc5MLkCkP3CLV6yAT6nsKLeIBPD3CAuMxtXZyp7djFP5A1LJsp85husJl8/90DEhOWGlD85mJ/0YKI3bBVt8jAMqNr/NMoDgnA66og7/KLx1vuIY4xCSxGYDaUJShkFCtHgbny/HYascZoHP0NgUCMUsdxrgtZ9Faa6IiCtiyMLIzLMs1Y0mN4bk3xymyHvsi5Lj4JAddN/9jxhO+7olyfMSOI3uyg1ccolWx/XI+0PPmaleQ/Yyf6MC0AttBygtVGKdU8JIHiT5w7bo3fLmAwSP6fPFsrrHKfITDqGPX46360UGOElGyVz2bS/wqoQQv2lpMbfquwxalrVv+5t+2uN2HGZIEzXs83hYOyLnutz71MJMvObVPN8UtSe/qdfBDByB9WsxoB9ynLHb46UuKKnO3j8RncKLLwiMSDYVki452me5Gb9ubWhwJOse6TBZkfdDIIBbbF+gEiP6gMV5ouOVUZdnp+BGH55duQAPGZXSkRP8YbFDWhec9C7vBO06HCYECSjdElawzowFjbwxTtm6CrODB7Gc/Bb28lNL+KYu1Jf/joNkSYrf+SJOjGXWXNqDEL2Yj4N8yT8Y1uUolqy2TXkneKYeuwoIPUC6+34zy5YrODfTBHsTZPSgul5N5T7Drrj3bxvSxfUlHZZ41CAcXmAvcbnt2mSBsVX8uniAb0tdY0JZVO7hCZ8zMgAo/Py9ryGLPbnzbf4g0e3Bkl1ghpwyt+96fxg9iVjfeujVav/pq/BgnCkvE3vqdXruHEbkKgH8FXtSQQSFJU0YUj7U4hufROJezZ5RaQsOF9hWop/TT/1OwfvJTpWWvtvTpIMMyW2ofhh4XjlY8T/260Is0MHqOPltDxmhILOCZUXcjx/1RFoxdcnh9U6++KTMxJX7biBGo3tU8NrGhT0cXZXxi9Bfj5N3RiUk3tKRWUSA2b9u5PM1asz7sdjDGyTLWIUBAcyrp2bS3NEmfcxwMPYnSY0UJveD9Mmh4V0/RkBBaS3qPXz2BFcfCc4krYcXEg/xAthUrInpxhcRVAGHLByIPuBWbILP1QtnhREHKIjP8iKHcY2/MPCuKaINYdliHruOEzF+5KpADN4PzuOQiHtQEUNyOsF1jdVos7bXPz5vl8FcseXOFb46lbLg8HzHe3jl+yLPGZ5XKEhnLx7wybZ1eZG9oiG/4u2BbEu2xLsduOqy7YYcU/aFP8cClzhhKJIxggDCt7Pxqp83/tTmnGyogk92sVYM2d+4dSULb8ewPSby2l4JFvnthOY0/Jth3gkTn8qRysbDYeK+L3uY1dfVwMPjO/kmXUoZi3vWJ3OKqwASYJu86sXHvCD9QrlCcmDBhy7cb/qwfg9WN1xbko78Wtywh/rkklKRA90m2rGoiYVrmgNo4E5TDFm0CmJXByAovHgKogtghb5qJN+5p2dZjMLyXkqVwP69X30QwS1Jn3t+MK1ezhnqsl2MKv28HLRpQnwkXeeRqI2wcrGoEK5q4LblDJpjlurXXKyP8R1x6YBevkqUzkmUUMOyUUiHIvkbh0ALU9q3A/qakqsGTqLojrkJFgWtvryxtRg9be5axNOgZw+Er0RgHNe8gaNfLYTyAs/JQ6B/6K+VNoJogZyG+aa0l1lZ8mwLtvICwayv09q/Icr05zPM5t0Xc3XA5kJYOIxjPBrB98rcRdiz3RLcz51PiM6c5vKaNk2xqrnvN4eXqkrVjTdWaoQ5uLzLS0iGCefxG2GIQbN6xV4GMaIe3+3kSdSP1gYJWOvOekUtLuF7Pr9FYxDXllquGYvQ57bp1pearCdIDpWhUIw7eydKtfW7fpBNEK6nuO/aXdQ84/0rYSHHOTPKcRmBfP/W437Wed0WOdiowd9ulyH7Eo5NUGWKmBE56M55OOzOscjeWlASREGu5faul6nhE6H+Rc93c4AYzENGNQCndQ4pWxUZvEEX63HKP4IjNoQWksLHrVrT1VEvHz+2SxByjUfGi6D8ab6TM9SQUix0snMdsNxS3Dr6a0DNMusS9pr1sNNPj7IUQ/pAcU+aJjgcNo6L3OlyTi+dXl8JK31uo8WIAQHOnARIqejaYFCDt8bN8AKqEMdQr85tvls8cLZNILeFuCZJhfKaNOJ0kwxUOLqz/RKxIj63qYmjX0M7d3QQQMK/xisj8pcy+SjuVlTmnNlZ0yEo/uEpZ0n+hRUfLvC6SJne9p0F+ahZSI+tfhO1pjIhnnOKcnCvCRCc5TreDNgoIT4gVR1UiA4PZLKdKItulZfvKNGhpjDIjAOFOiIMBy3JHFbOvJ+PXwZqnvada56Q1V8ZmpAtV1kjv0PT9EAZEndFeP3XVsrSvLqbzyNSJ8sIBnkVOtBf4QZB0yqk1FiHR20t8XUHygAyIa9CMYUiDIXBgJWHiJO0wtHU/iO4N75pvwA0FrVJcNoHZFf2jjD1rVmQoIGplc8xeOOVTymoTNj9Mwkl5tdWCzSqwfqLDCvssT+B/sldCgkmRgruRL8/X7dF0lsFo97ZsyA7swTlkwZJJd1QZr9Sp2YavLdf91DlMjrGbED0Bh3WTgA5XlhLq6MCZURk7yS594l5a8NlfHa8n0h0puTCkCvpUhbmpQLh3RHErACsxxZXyF8egTHy2A/D1LyZ30KtBLJt+hI3wwGs0EZBcwRr3TyQCfYqTRmrAgEZ6LKyYrjT723iVx6hgmIStYPvj/BtCbDHzNtmh83J14tSFAPJ2KGZQI7rSxN+h7XzHRVY4eriUhjdAXsH/s0ttXflek2Pl9ZbM3rpXLTHN7DKIH/WwN/dvvYb4c6vLk5weHaFLp0QzImOo145UTVLSucAi0cjrAnX/VtbDmw6AlovN+0aAB0spuux9owTX5ktY9CW3QW8fd+4cPUbbR0bKlbJCUcn/304SkZx1ouVDPeLkUSb6u7zOxFfSF43ImyYNUlKPzRBvHCl2JP4MzJdvm7R8YX2UGYjrHTCFQGy44YVqwvrG+xRZWW58SBkEbFXpFYVxN3Eiji6OpPZYnelwBxMBE+Igj6P0K45CUUpyuKVzePqa8r+DuqiQfLG+htSK0L81Iz+r3D1ImQhMVDsyhQCIzT8tzjDwIvukQv1zZzfl2s1AF/OymVj6DcIaVfmmRJxGc6Ab4GD7OXYhVI+jim+zvyq4dBUIeltAGRW/bbbZh8CZypBMR7/XJeMG0Xw1+79Rdc09aCSSPdKcd8MEB8SZ+syoLdAvH7ja49I9TIBiz6N1seblXylLFxjibwifVwNTDpe2FwJfqlgAYk81GwSERDH5sfDxfizhzyThzG1pt4vCWqZJe1h/0Vsn4B4oS19u1hlbhTJTlS3Xj+n1wSiFfatEcHc6/niyjkXVM6NMA5NLAX0VX/B1B5AWGEvvqYS5PNcZJ++stvdwFbB9/6LboIQOF/tci9uzfhL7t0Z7jeD2RVJZMyJKP9WvdRrh41nmKPyHqcFgG3zaRg7CM1Ejrs3b2NHTDaJsQrYHLRApC89dOUDNhm0PafEkDYjhSL7bblOqWH5m/oInH7ODXbeBjARja/lKJvOodAyhPBJ7JEJVBzdRnwU7VNNOmeRRF/+qg1p+mFGTR1WovR65WxzVvZhQGj8QDs0wRIQaCNhYstdZlByOhgU1fKkcp3CIkCL5zCV85Kjtg7vEWyhan7E/GX5CfTFZwS16lkAaGI/LtPRp6vLk/MBznfXTe8YT4cqb4qAFU+3v415BRZSQeOyPzx7HYmGHxW30bHd8vgMwHf282/TfW3qKYX3p02jcGYee9s8kM0VGKNltKtvrOe4+nPQ28HwhYO+A8gYe02S9nGW41Z9vJVxvnI4seDpKlko9Agb0f1PVpeiIBLVFH/o0vMHvlJp+2YohaiszV5ePUg7KGJJptOU5D33SkwKWJHUiT5JZrq9tggWUgwIL8nj7s8FNo4oS0lOfAN1kEbVxyADhFX7xe6lA3egjXstqCmhLPMClk0O5wvp9xrJ1vrmqNL+IPJ2cPrnVudCYnw3HEv/X3rvRBU8sNYmfPtjGcPTXstx0Uu3oET9qNnG66vOrsQ6V/s6OUD22xfBGC/adM6zN4yoSOnpF/VUbFhkm5T6dr/5PgDMJuZERUJ+pp9uVqvwl+GmMlmKGmKbDHs7iIqrleI1YmjMFiUkWxdFxcQgNivYQ8puHzo/xmAnFNXh3yunWfhzrFUq4NO2H+3YCYDff05FbVQfR9BreyCpMBmKPbsLRqkfa3kjet0W0sClObdH/Ms3P2sRlPLpSTVSFg/7EVRzds3QDIjc+oSmtcXww3ACefMB8W4fsW1c48s3r7s5ZCA4nRyKH6kMu3UNJ3M3xdETGiVxJGb/TImyUp/s+Q4vrEnyYXAVgWQWKAiqx3VSh9pi297RRejSFdsALlCTMdU8kQsunFxzhKKKTdB4KvJ9+kYLaFZkFbs0D22MhGEPhOtVqpSLiF+axXpPL2lvADY6h5wcHyEglUdUe8zF6gtttnOzJbqE8Jw9P2aARbhNM43KcuH7UDB7y+ESxDiSu5c8twbmHFM/ogO7PBlBaKDITTGFH0LYU4B3P/AxwsuErsgRoElcKHfODdYiMqDet0LOar/d1bC+UB58HHFbOu9YfF07R8WYPTQN8Oze4xfgE1AWD374mrSbKnv/fAeP9AhyqEAB0p5/XbfTt2sMV41CJzgTy3cINHLzm039EGTO5SQrnzcAJndnCkKE6tFXjjBHHorO1thBG7Mla9puVO5wWur8PMDFmsuQuJqTs79MYomSGEj13ml5nCHsICu1PlBjXRulleaq2iwHOW/xiyV3knx5PAZrz/ORxhjDTzzpO+AFF3CYGnAZvy1XaB/7HwITqZx5iUxI8HDse8XR55Ayua5ERg2eEOiDwCihsbZJAdFWPrt2ziu3jYHEic640XgvEEnFe0ybz+UAffMNlF8yncCpKJu0YyCoiYnrwqGthCr5IACm9WWIh8yP6LCwV6/fr9wRjaLNNWOZTkczdSzzX7qnnGgnAXcyqR7GTwHNordF1dc04bBbHhy0cJzvRYzMyGhnfFtW1TUrMa55eayj6hlR0MpMftfXrALKtvY6PrMC3Q2PHdsmKM9aSkJAZzi7/uwrC+4HRVgB5XuzS5MKYktpQmD2DFlte8itZaCJimjrepQVXAdO4T3MchmOhKq67/d1R1RlmeiFghr2lxNC74DRbKHk3N8iCF3jac1Uy4vrvEFbsNzI9yDmv4+QjPcvR0yeCli9mT7GJ9bpzB2OoSJQpBMpeJS2WU4D5q7jqRWHTfHpbAVVbm/KyQD2tRF67aTKkvC1ioKgKJJLzybFYL5sw5HnnrZB2i8kkqSV3tBhfyfBVGPF5OPXMHLWgQfr1xFNaf0+rmcsziuP9Y7HvlwSy9f7kkRWV7/tMNrJwQRWZMM0PdcG/jWSyOEcLC+rIpcwc0WU6cdTy/EEAOKjgxSCFyan1hoh7iEvz5fZ+N43EyYnuuovivQZrSHWt/H4Mu8cxPa0y1XpE2p765NxdnN8k+fYXfg4lpamqG9uQZdHJekUZGcExrkQbejVmtw83+L9ROJi7O61SurftZ5afOJvj4jndq1wRdQWJNfXUe8Ce+8h+MkUb19/NFc9sg9tpXVLEeMoIRYc+WqHrMvWGIaB9LW+RHtvahQtpWHIezLwW5pHfc/B+Wxk2hF78TDSjb/Yj4RwLiX5wo2u1KxFv6GQ/q+uW8BDcwYlJyIXpkd6G0v4GGs8omgKy4GH14np9XHNIHAe7D1i9YA8/raenPSDo9TPBd6gPl5kyKQRJt387gDwpp78fMYEbsdgy1wI6dAZBoIdpnA85dabX0qTRe0eZHzHPCwjcTe0vCdd+43AMny+scvFdldeY/wZTt1FusdWqaWDZ77FgwU3u2nU4OqHHMbmcDg53LihTpdqQ7/5Zv+QmA+Ta85uadmo9y2zOVkh6CGtRReW1gyuyeiRShg0U2jg6Z10VZjv304uGDPYaCZwSpWweD5b9MVjByKG7FGyKLlVWTpczvXXnVULsj5jr/Tz+S4zTvsOA72KTVEP3vnwp0VoYF1cKLpFkk9W5nT1dEe5I8u/7PvUNLwTPxiTb0pZnV62B6rvoiJS5XaONRpznV/XGQWNydbXhCsL1UG/cQn+RWXYcHnpCvGyivSC/qymywicGodxjArfNniU8btcTcY/FEhUv4CCRL8empeV4juOwTfN0jC1roXEtwjmNxPpUrdWk70+ftQmsYqPM6GaPYQP33AB4CIktaMCyiMK2r6ydLnlsytLN0c6n80uEqPxBykOX79CCzdhMbteMXv+Yndd1HeOyB10i1jSPXKSLBiS8VAUrlI4kNDnsORp6fXLb/eE/m2HMPS8TEKXxmdpDig/p8LcbHs//fiGdxgqlLUQKXy4T8t+kOBsOU6dTSV0dvMOcczmiYcmJwNOe7R3qNCXAQb1VBdQ4ftr1BwnX0B07sy4PiROlGNu3cgSCvBWXeOyuOs9aQOmd0AGaX5cia+SRt0jd+PSnjtgETeB0Xgi1bmO0Dxo7/q+55IM/DPNijd1QbbTWmnk/N1LBagG+5BSyBDpZoc/Yk/9Nt3ldktk2wm3XqADpGSePDoPeUppPrys8H7MKL0fxvhBXqjXM0eRiQ7XkijZUuZCti3rn44QigEZRd4ng45QTWoRHJ/wkADs5sKXbAanrqzsFC3ZizoTY1o+myTafEAj13Aw1Vxs/P6cfGcfl/S6sZptcoDH9R2djGZYw2p0FGGzsRECqKneNWVrLZeoZG0n3fVrvXWM6o8+0JIkVyaDJE/o/nqGl/R7PmkVVsI7KpeGVqMGFA72hRJD2O5X1i/eJ7X6g/o1PO5ERv06SX6rO8zXQHoxcD9dz79OjyliOCwLxglDJ+yjTG5l/1h1drqm18L28XkxL0l13wKGegIqd7l2tbXBtJUQlemBRWAGOUvaj0sAUTaW/ZathczX1DhjpHKZVH2o+KA6TsNzs7edBlwORKv/BgToQNXP6jGIj4/kOjB1UfBfV+DKn09EFVkGoLtEWzjORg83H1Fr15GaTb6qUTLOG5GafkX4LQHfWp9dCApviOJM1cdPlCc0TouE94yL8lwoR4PFIkcbfsQ2oanmnIxPq4DYGtXvX96YCsRgdM+qUW7wD0i6CRMj0/htB1Bd3QnyTUGMYXFI83bxSCg4FWSc0+UxP9imzX7uKwhRwGQ7SCIDFnE8NPj9Yavg0RuRx4ZpScw5WFzxNmz3DUu78bVcNohILT9BBTTLPE71/SWLOvI4NeteWFYRkGpkjIq9k6rLaSPdBXGR91m9UAc67KrfTY3FNbDIXu4Yja7G2SYgkthdhhMMMWbFIXbLwug77QRrlMFAQ3a0r55o/OmX91S4IAiUBXL9T8xu5+r8IqXzUvWIS0iv2bs7+2SOJmXAy4awxIjDd7v0nZhWwZE0qhn0ZnP2+srf0nhN9ZcY7XF+vZ2ZJkXIkH/ouyYEJyx8dCH5Oyr1a7uHr/aAYmnqp5Bh+IedExOV0AgPHV/hZfrYZbxKa4AzspBoZybRo2+oyqz2IfOgekp89qLmd5bnVglyS/ZKAvMj9794qggie/R2Zp+fcCXU3CXNxyPQDzwkHPI648tgqFhGuAdNtIs+jwiQgrbYBsIfXl/I3qdKQLurbXohuA3ps3hDVe5Xe0zMGnGw55gy31XJjW6eLKqIbIOIOqG8z6IUXB9J8i/BkJJTJJRMJ3ufSYagMLUK3RlCZ99uiRp934eVskipe4Df/NDjiMxj8A9iMk2Q5MVlxKw43X9WrjlO0j9yXS/SJUbiJA7KhKjpS2HgLLbe0rmoovUhsEZdQ8msSA82xbfyzT/LASqaSlRx+EfrGEH2hbP5RTwrejuEKI4TJEH099tx2YOCITtZtCDTmomZ9rBVQEoNSVCvG0dr+rAEVN3z2wybdqtpD1eJaP9NvS3YkSLxloli9vNNH1uq0Wu+3T/6e7gLimTE87qIaCtBoCtj1RM19yJg7sbe6lLZPDqxW3yA0Vcsg4xpGb3zz10NOgn8gE2AtlRxOtEi7GFaipPooS+9Orm/nXW5WPIdJHjTJr/Jvp8JvfBDUD4Vza7NOoZ/mm2EL0g62sJM1e3VXVJr96zk8gjFLkYprwVZodG6fuqJ0RL0U8D2R912/eR3RaEDAGj7003PYdkr55JR58psB7z3j92brYBVAOLJuEmjNMCJdX4vHGUMW7Cc1irGQcUyjCMhmx4wv5ZYzSt/K0AcHswbb7mTXq7ANIdrh/pHCJMaipS37C47Yp6XmyqJ9D42ZZoaTZHY2DNtbmPrC8CGnly2bYoY6fNu40MqdizDxYeptAoHycTGrde5dw/1oJaL72/gADDmW6O1qrsIIu99qsFs90mjYy2QxVcAeqJcwtZRNy8U9PXe0qJ0Hx7vye0rfqNrFlqkOd4Zw3gr9tpxmMVm7pyOaaxHVfOS7rvmGW+PZFUiEP5bKg+Z4sNbmrJ47RUWVTZ5lmQ7LmuE0cqRltZLlSnD7elTY/h0tQfgqZIxIQp7NghTVOeuknbyAXTbisqNvO874ra26PLfHTJAuUhLpyHs/2oXCI9Vha7IYvyYHKyUdoei2pep34QMsv3BrVc3szCs8EBhKmUiEJRE+Hrj6EpWIvM30J9TfUr/FRkS7SK2Q+VKg7bxP6xh7o2UIqLJci332/AO/A69nGH0It5vDnwdGWQ7FPOe5XWFTz+MeOxWpD6jX3NrrrtTSnCvqQns9RUkR8kEjVIgzFHDyNBnXoNVAWpiFv9+0MF+iCkSg4i1yktTMwCiRUkohU9SBpXn8jY/kwvBIudCrCI2eACnEYjPbtCOYfdQ981bLreC3rEPnKDJUtYd7f2m2LFYrdN7pRzrba975wDK8tjl3Nt/DXLe+HIaf6chTvMpGZEBnX64h+RHOYidUdI1QDY4K9j4dae2ectIkhH+HKR9J3uQpkdhCunLOAs2JK+6Pb9LCgLtYln9m3B3BOq04qdBx07J/SZoMipsJthxFzny41AXRbBumjrLNnBeq4L6C3riHpa32DGXu474G9XW94GItg9gSOPCvzPKLBHpirJ6+1Q7VIIKTbY9qd3FZZfv/Bu7NxSt/X9NpYKJYjtX6klty6s50eovMO9Ghtwa7rLScnVbqjJguV4iqK3wJaIaHMirBaxe4SuLrS8gd5ihIBTckNzLf8Ho/ArF3NwzyWqzGdH9/Wtg+ayBq3kXwvfwuO2UzBwvWugw7aO24lP3KfYMWMX+Dd9qJBBgFn2UhbOv39N86IZVhXt2VIHkHqnf9YMt/GAxKBh/JxJMkvTxAlhvSb6vKSeWLZ9rkBMR17NDouudDr91IagYDtJDDobmam3eyPrhShU3iDg1Tfe9vaz3CC8dzxAuc76TJkEH8eQRH4+ODB0b/XiEHciH+psLIN/CuUSoPWc1iICUJ10YaJJsak3gMyQHYYXen6q474s1MWKj92SdFzZkgm/hcNSxap7aTD1+IY/C0KSJieBOFhND0fdL+s4yiC5Nr8inQGg9/bIzKBYS80Vx1JuH+2nnPbUARvDhCeC4Xu4nD6yCZn4LNksEc/yQJqwZqA39QWazTJCCBhT419jqXznZFO1jDOr3PIABGivvKJl1PL5w+wvAU1HEpetrv/2gMY4CTAEVQTGLWPYPBfTfrZGca2mhJF3/FANmz0j5dpg28lD8jQ/O12eXtKbK2JrzLVrytxFGek0kVR8P0wTUkle3FnKnm38JA8vKSO++S4jkJm4WxldVheTc08iWtgDYXFjufPBa+tOfFFS4f+Gaqi0vgmm/95et4ELPIZj2c1CKc27V2AewBjeCxMz11TeHZgdwDH+sO4b7Vm+GGQwGZJtfTaaxf+jgw4aQzFo3py7iKDQ8lRHGT1duNK+4yl9bUkGljjDOt0ySO7Ryn9daB2GzV4us4kY8OHvDYgfj2quzIAEPQ5SOILw8D04g64+pPvmcjoQZaRh2JgmECYDY0ggJfNPD75NpyWukKEuF1i5IoXk33rzMdL++tMK54YhM8IgMwaWIGkWsz5Q2qs/ExU90r1NvkcBt9LbENHhHtqzd0xpowhX26HE3ZQ9haIN9J6wPwuTx7wu8/MbJ7blZ6siWgNwL9JaXlQ2AjdT9GtTfcIwABb9yH9jogeCMa00rJGUd5M6WzVsyPnDRuAWhhg2Qz+fHiPeuvz40sxPaOr/NxRU2mqlnU3Pb6KF/GvmbA+HsB2v5JVXpQ1SRrP68V8apiG0zwdRa3wcYAfiUbbUzmp8tHi282QxG8Fjd2Hf6E7aD8jbiVHRNr9NZHxgubhFDmczy37IYgLxln7bbGyKM/Sjto3WbikkfuN3HqelHdhJ5exWD924/6JGNfdo3mLZhVPsxYSNZdlTKsLsgkEdm829oh19lGFjUpmtgOx8HJzUDDSlFDcmY3Xfm6fkxVQEFkfwQ0lz8b6QEVO3vaLazDEbh9EEH5ElaJig5BJ2BYiCGbrM+N/4W8e0uWJExXEGnOgB9Y6gl5PaXp+7lVCvIuOsXYBeO1Y6oGFdDbkP2yFd4JH0MODOoZpa4xapAcml6AiwWKOS6b3TqVWRWeioLkONGa+58oByF5lz+BSwInLOTIU5Ztg6HZQbukEJNZ0ZfEe64N9mfMbAZhHN1ixd3LcGrHJAMAd9Z5W3CmxOKLhsRNAZ3+cWv7JXIzxMZ8DcIcqt/X4FWrYuSbpkyCePa64GvtCpxlV8RVFmQlrOjnFm4w8ghS/jey/Kbz9eyTt8c50wM/lYR5K9fjJS17OEs/aRIJJ/JVwztCuGI+VHR/aTTBRV6oSWbXazVs/jgsBSknHkD4GpS9tMyf/9vxrTXJ7Is1aOpzHTVEbjBimKpuxbRuijJ9QIXhnzkUfzL3FA1F42rakgmTF1GBfPfD1mptG3Kokwm2zVBTbkHx7H9KmA3whoGmg0afP484OqOAaDjkzkGBPT7WExGYBL0p+9dKjH8xI5fYYcB1Shb8esFet3wCdhXJl6GQg7JIXyZxI1s195b5ed4oF+0iSfxUk79hykxgQb5JHYJK8SaenmWTgdABijAoVLoeLwj3LNOyrpFxQckUgeuP1nR6OWuf5NYQdixRQ/G5jxfVekgiRkDA9gaavr2ANBn7ugoQ2z6+k7hwK2KgBobce6AakNsF+N7myi06ggnxU038uW5PbYWe6mtgPhVzP4Gw72h4OgubdK2Lon7Ev7ZX5apIn3cHiTpdspbXZvE6Cai2TIEg2kjoXWNGHgsowYsJEylMT+PVUIrFCh4DmdKNLxUh7l4OZEOQfAf45oyZb30wt2MZ/OcS5pSm51As+XQmC0fVV8krobCKgahg2yb580UwHKwTQBnfC2cEoeVcyXQr+9UbKDGgd1hORhIlBWy29R+wqP7XtcSeXSdD4N4ZezXH6TPCygpU5A95xkx6wQgXgDyitn3kizKmrR5g2JZPkvut5hhSz5drv9Wi1HDTLzLnX8z8oOb0ODtvFU/Wn+TVHCgeRmT6iJKEDBEi7Sc/Eb6+HRwkB71jt4EE7mOQ5C8l+eXF6Xm4X6xr+xYUEv9tmZBn7AK7bem8QVKGNZQXj1Jkj/KD2LAhDLAoYECmje+bb609qsbgaMaifPNEowXqkVs84XXmV3NIMmRk3Z7LjBvflqueIgQzCznVwD1QUCGVxqjrXIfdG9phzdjQ36GkmnXogtl73eHA2Lh4KeIkIAt4BB/wvquAMDBpKJ3D9lxGHGF2xIoikBebe4LiM2hAByJAvQyP3OFWtJLFRO0nUADDH9fiNjNMtKoidJYXmGHQxiuzPF68aOaldvrPTkIUiGaLufkR/oNC7mTxo2nqiM5yf0UwkBfCJjbJK7gFbY8W67ko2B6uNwcYjpCU5+S1YDRKbjcTBuxvp8HT7SZyeF0K1oKwIjCteLge1uIp7KdtFRQzMpcs8jQEFc2Q+ZYPGunG77gYbBntSE1Af+II2K3x4I0+Zo6kdxoQ74E/fnbg5kzn20vDZCQtQgAXq/HJ422aErexL7e/rBS8ofaVGoYX6hdOQ3RzK9jmNXSCLiKCArrpskZCQizGsdpnz8GUgCCYvJJNOlsrfnJLukxCUncdF57MF1EMcke5EuSNf6zk8ZDBexXicmhB6eyrneLPhHcQne3JpsBz9nPwe3Q6d2Rjpq6LiaZ7KwQjiIeaJdS6yaL4Qan3hc0J6U1O4kvySF8Qakt2T6EdMD70EOW5mLfVPtOyCpafmljaVDH4ay4B9hoXclLkTzyfcUk7wcKMzGDfD/83grD+Uu+2ZZrv2ERvqTkXlcYky+I0lgF09Lz/EIH9zzRgqSTTQKHi+s2Sa0AA//WzH4chdGko+TeSV5uCdnOO4intfuM4TownyzFO2F5no/NAfu0RIVM9JGCuWHioLHDJ3Gk3bSpjfYMU8LIKnTHE+wd5QSVPgLgLrKsKy74c0GrgOC520vuimdF9f4uWGPDG8ymQUiXvC8hmMetacEXiycKoRlCD85cZNfpGm7K73JbaeT68kMrocQ+Ufs+EMNr/JYmidMKe+TY9z7rbt4l+OayUwFjplMU5ucRBZ4ElKMEnYVwntbHLU2/mqo4EsSuhl4G2Ruq6dUFkUGwLPiVoU7VaWcpuEYRnLUQDvHW7Ys4yPRMjtYeG/ZMYv5D1GgmgPDmsEvdcXc2uhHHy9BksG16EkE6+5c9uhTO9EPry91fx60mc7j4h6NKU/obHmJPI5hpJlP9TdVd+lBgkPTzNfF6eF1pB0BsBfKbpNEFUBENHpOO9lpunUWPiuex9nsnX2iy2eL7MaYrN2iMf1fLCYpXAVZ7U0CUTNBkQWQ7rB7dcS5kD3gbz57kRn40tJJS+dhkGCOHGMWU5nxwfSttZVEnrwCKWiNy5aguxMmM5HtdbHoc9ceP0UEsKj6lpTgTgno8NuYlK9pcI4I22pEHar2u7z6aJMFRu4IFvLlkZbTY3mcy17JCvNH/zKj3brsPeK9sAvzGouh6Q2P06FP8+w7RSHmIJYsTPKB2l75i/vZxtu1g/Vja4/FFqjFABlj0sjBEka2lF1VhP07qNxH1QFFFAiiP+/6rX+MXxW5zaLps4yBFsDiBB7c+YiaN4+m5Xjj0cGUQlLhYUJf0MZruNQ0PmgNTUyIZK0OZORwTyI71d8fm17b7r/2Q8twVIALiLRv8yXQ6SRLrofRNjN+8dmGkX4hfIAegst+6R5qSyaFVPJfVPdJq19/3sWQfdGruvK174anvFwrviDrasMfLn09EoA94NdQq/xLEKH0bV2pID0oL9jkiyDLJuXZXZRL/G2jPOmFPUztj6x/qeCd02dZRHlH50sdEGV0bZLJWcQ9lS2UU/6UJEYH7q6t2IQpV41Mr7aMfgtBMjmCXOLuMX4V5/Cis7oHig7yC5iA2WhkF6MR5pMcEjFobCsMX1NcDa7edHEjpFy2aY+jX7gOcM5brxxAnvB6Vyl0gE3ryscPORrkdVBgoLrd2v5Aq7RcfnZI3cGoEm8qG00VJcrwFJvC9fAQm5iHPgb1pqnIbCAowryIm1fj2pLkpVHRC6907CbODk+gg6ruh5heHHf7wrutNfFhV93CU6dxOcrXoSAdBAxAN/bItNFkufDY6A28Vclu2ROfN0bKqxQJlbICRZvtyx37Lp7PGaV9xW3euElu5nVwwuZQ7O8IKUlXi3Lf5CzO90nwv4a8PTmfngjfhPmtVSS9SdDlLOJGgVfMGraN3uvyFgtjQdMh9n8vC/nUs7BkiOkvcqpPLkZjxepX2zQv2AS37TBlFj07J4HXX476A0d+ZYr6XL3vcjIPGsD2dXVXpbz8jCVgt7wWoXUEG2brozhjPXugR3ag4y4swD5mFtVFLsHNcqO2xNZ3f+QlhRNNazvSuNpr9QOjCowpDd3t4eXxyCBvttKz2EX4qDiaH70LXgtViOOPl1SOUL2Gu4AzL2HVOSePfOOt5cy+xzgA6RrX7jL3EYxa6RXHBIzYn9osIRC4O8g3ic6yVFYhs4+CO+pt9ZR/7ut/h5AA7+duKWzFBy5gIg1FpVpr4tLmeZvRNY6WDFLus07eQjb+W7b1CYF4wmw37xkwP30gSlVDe3DUjf+PMxF9NIfDCbkcWZ+hxvIEbaWKIcf8lHqnSy4FQdz7IaRSLwz2rDQqVc8/fUYrPyAuucz0wxwXPl4hzYoXyK874YtrK38evp1cZGh066EH7DvF4iR/VFhfxFZwp9xXqJnKZIWnVGe+8brwq9Ms4Sd2IcMEpKebBGXUKMU7xJJDUq8vTpuh/BVc+nZfSWszDLISUEX1t/qYQnj408FbYc7/4ln/5NH845XLcaZ+TLOV2RgoabsV1r/aEwEqhJEvCZAUITQAUt3jGhcQ4qap0KYelVpMJmgYq32K0Te2P95lvsBnTxXJ28H0GHyQv+DzgVnRg34E29D7Kb+TtYhi02WO9I0647fd5CaCtQHQmaKSI0jgiPSfKUvzKS6FBuq08h+qqeEbsnCcgccamxg21s6kOMWFp/S/OUQ5Xs7+Sk7AFfy1P01/KqowJE4LbG9DS8ROdLcPckO3TqQlTCNCgl1GL+jvj3eTAc8Ph3Uwwz8/4+DPj/b1gZeZadpWLyi21kk7Y4R29rkJ7+9M0vKRILSBcwLP/0nRdy5IqOfCXoPGPeO+hG3jDe+/5+qXO3I2JmBhzopsqVFJmSiW5T49BhLCpXaMHr6nsWkPQ0uQ9mq05CEsZoFonSdYpbUXOeTf+gqTA0KrUqtzgK0pH6VjhtkD0BGFf//HYuPkKGmz5Wb5sC2LR89/o4SY1huCsvfmI5nWSTPJhMERo8JtYq7x7w7RjNTnSUsW7XsgI5CeC1A+oP3pm2ahccCJwtM1QlESMrzpUVFGeauMzMHxRRpWYN+p9ozHe8V6sNQaKip/eGOfNpiGSZBXx3+TpA2E6x4dZ/XO1SZNcZYeiEfB91AjvJkdWX6bdvke+ke0aHh+Rhjz/BSQr6ow60YbpUUS5wEJYS4wH1Jyuwz/hcVr/1ewrz2wlaGumyXDEj0d91Do3PAaF8NDcSayUuzIN41hc7tGQpnGsfTK3gD5nSCGuymO1WZMzWxfQx+v6N7yUmK/dchCav+v8/IOjwwvWA97GPdf1MHz4SxI3GF+R/J8mbHoowazP7vychJJ5+pe7pYwQD2XGD1rL9v153WW2VQmxLKWz91VdDwTHKk/E8QGXwt9nohCV5WlLB3W4wlms9fpGIEcQ6b92uu+XirI8aZqLcdcVZjJyoC9Zaa3nN0kRAhmNDDNdF8hre80h1CmrU2fdWDkqaLwe/ehKKokkcMXren0wZOon9AfJGf+qvER6eRAusZ/ZFrpJ8xctbkvy6mW4JQpn9GImLMpOuv0Edt/Qn6owAjHe3dehOQ/P39QoVAQOqdqqWRuWkpP0jMNQ/+R35vMfaKJ+N8Hu8fbsH5j/YCadgfCSNIBHU5Rt3SY7PJ8dGjwEccr6XgMO9aBHk84eJeHhT9BH84OWzTb3wKYLymc953gD8qokXKMFJcjfIFbdPt0OP6RPvg4ylCxdz02mFyZ7moo6OFUl+jpckH8dkDp5uQhKiBtWFTkafSxp2+9y2mzoGQseQzExkXWAlVqt7e9z/GnYbILcLU5imid/FWrXhbvkNOcrRsBBPol/fqgQm8vQOHBl9BlbWBPhky/1kX563Sw8+d8NHYFvXb2T7f6uNrEwAl/X13xwllNElvKAFKH4yiKL1nPIcNh8g7rQZ0e4Gm+28/WZc8+4QlbQSVV6tIwfmrgnxkcfZZsmzz9BpcJJuIfQHQdrZrWgdOWSbs3ogXpim3nAjl+KtogTtywR00H+/rjtl1d4+UtvCuOo4kc6kxtCnMckGVQc/FphML0txTDeoWs69ePK9bWEVaDEsLfzXXyF/DTjpQVYGs6x+gftpluTGjIb2LpzZY5wUeNJ4V0T2r+yriJP7VbR6payzBQ2DpBvW07Ksg6/BoMovyyK/2Z1GO5rIQePqQXKsttJ9Xie4O+/caMos5o0vIWfMXZXps7MryJLcUUrJWxH6Ge3DSC9BxXWPgqmVQ8G0nrSPK+ZsY5iVqB3MlK5+fqQGwg5pwNwIsv4UTqct9Egp/JSyk+1yp+fxzagRru4Qvsv2aYZQj0HMdIEf1NtMK32ePMMVzKxkdvXEiOTC+AsCthNmk+1UXYGXEqiuIGirlje8wFA4PvnuKKqePrw2CoSf+MFon3GRqKIoZxcux51xEK9mHQ6ylnGiKlYaEP4DkveFZWPYnhZqPWePqzGbmu6JaSdNuSmzEqsWpYdtj9M/0EWXspOlzRKXa9p3ZN5YbrnFotJRZ4Xa3cc6SHOFyk2fDGT27f1vbA/1LWohu3UxNziqh73ijfC8CjOF8Eo6c8bn8HhU615wfnRymEQOKirsBpF2p/461f8XP9mvmbSX7XXVB6sxll+sV/KNx/D/8bWov91NIrFlykEhT5MXwajmBnc1GRq1ZbceFX/6jIf/m8mEiCzVICqTyhlG0HMEB1Fo8vrorn3yZn/TRmYNFv8G0QSv8wk90jrd2hpjizNtEB8CFJWOrhTD0jtX2umpUPOaqrF1CPUexuAP7GlWJAs6zUSsczNrtNQi9ACDdY7h/xhIKcolc43czaZBSdZzg8Kp+bPRC4Y9Ndyy3lDGPTXD6e2DjKjD1F/JNoQRZ/x+ZmWrYrQmRBowPu5V9twS/2L+Ua4acboBZZB6xaZR/hUyDngyg/kx4MzaD7XrYPx/kEHNV067PjWV68cwQ94uHMq6K9SeZRhXyBVmKYvLkbqC4b/yOKUUYXXCyjl58rwUcqWhckUMaTqJDNGSbbq01fa3itNmnVoXfNso8ULFJR5rSlsB/1dvQpF2YjWRz0zS4AwHc/K2WtNfaIJcoBFIH53aj0DzSQ9AP8VvSyoZ4lTVmwtbUwrWwJEC1SwJFHhDW9MYfLuMYm4NvLKDziB2Puzio0bAN5CjOAVWTeaZvQL3kgLAY5sbPBqj+3wcWb4WfHWW4gtuwj3cLyv+FgmNMxFJVJFBw3OVwiikrgXAv1mP71nZOz5cFB0vB4zHoMEw5DEBDcbPrQf0l5pNsBsvoUcLj1IsfnVIsFbLs4vFFIO56Nn144Ude6SUkIHUqhOsxPHly91gWkb5ClTC3zwJ8d99jIa1aE3B9NloiWu1WCd8D7dbP2n779K9crfEcOeaTKNKY7IlXLNsprsVH3EfTnRK+iRKjvWM0rHEJGSp4pKKLIv382xy9X2zh3lx+jhyAk6QQLxuhPcb5kHH+gYKHSefVsJ7eXk/Dsf/qrPW4MDky/XXGGFmCNja5LHLaqesLiA884PBb6rjwybniSLIGeLSc66F1QA8c7j/F3mwYzksZjT2L7N/TcTjl7vnLfM5G8wVMum5hAAW80naVXOcuUelcW3s/i7T04/ynfJhUWtwt5bv+Ol1LGNhpGK40c5JNyEl1ODsZemxaU8jVD2RUaJi8zMpZpP0RWpnlggQfQC0XVO58h64zIfNUzmGUtuIz65s7ZOX4O+9Q7dsu952NJyL8owXKeXTWIXO8VxOr67XVb5F2IYENMdb/fZSbgoVAr8mbsmcuZkKD7PiTgGVyHZyLQ3uMXjSv5B6ddt+UZDf7mFluUdyrwE8CKmPTcnfJ1ZJvhadwcl63YsXaLdiuf1oXHL6edJuX4uVdLXZ/aMv862IfK1ozCI0+Y6xYbWq5gA58vMluFf6uM/aguxeiJ9amGoqkMM2jZqbgQoMmzjNviuIS0E95q0XBnUhMfV4eKHL+tSHcWDAK+V+Zy/9VzZjf4SFPVS97/xJEC4a6UH/bbHNTQKzHA6VZiNbm+2cpJCGTVT5DsQAbGfBDUgqqgtJEBsLLDLZTIeVqf/7q8mctjfQLM7eoWiXv5LHz5PcxWBdfhfK2hm4BJbuKQXWjS5PtVfWoOHUqvLW4MhCKa0u7YVu+21c/8JsMFLcG4uZpDYmJyVKcmm3XndGW10wPcIRV2+aFtqz1yia9F1HPM2ho/+l2MDUSFZIqn5WvYk+3KpLs9UG+qNZbXdes5k8os6ml6vyKbtsqVsYABOcRJGoVWeN5+/LiObN1pteVMdpyWv5f8ldotwFxGy9pebnjTZ663ijaJ0On55OiTOff/YwrIQ9WIY7u96gQLtM9W5aalFcgX+UxuGdud+kYfpNvvIkkYwOZ4IKKvtFdUKXgACtjLUf5jO6aDwEBMEmjkvVdlzPXY7iU6+l1scFqJ+uhzIF7Cf6Vt9veyy3zZU5rKUBNCDRcblq4UDhkas2VppTpl6t9vBy1X/+o+S9B6Lal9mlFwHkf3du5PSupWgsShfbSHpTUuzv4KfCAK+n5ZVOg4hXFnwd21/L/wFYxLl3OaS9gjEYoyfb2YDcdhx2adErhfUcYFq64VF0ZI4mB2PaeIKmG90/bX6Bqkm1OX/bqU5PlMkQy1eAEElVkp5jF+3VbsMGrmJv8BvqMpN62kZHRwASNIdQhBSNgvyGFzTAimqKmZkI3AQDOIe1Atr6NetV04Jgg5fS8fY6mxIJSqv0N3XeqSBcmLnHxGpY6jXXflrq1sw9eu5B3e44umLz8Zz/YX6vXNzmY+JoGx+BBjmu/6Qv5v3xfz8QV1EFaeoZoQSEmo2PRpfX7N1zt4J8G0diEpKjFLhueA2pC27artfqPUPtzDkxxEtNURuynv97IeAEX0sGSfk1IQ0QuKvWh05EDLteWO74CQAg0S8y2+x5Gv2qg6yokub+EkBFeaN4VV4M+zDFaftvjDpu/696O0ZQZuB953Qll07+19LYxUa2aFYijUeq8eAmtFo9d5IUV/46xr+nX7ipo7fqSmRH9zG4qw2d17q1/Vsj9KDuJ7UThehUGXfv1HLdTP+zEn6+h0GC9XalmN41GgfdN1nIPZaEewrjgdumpX2BAxghqHrVUCr+X5GETscFcPrSItK8gs8Uo3a3A9SbtphfbUgJtAl8kV/Y5wb9RgfLZo0mCXASQOyjIAvMmjCCI9GdnrVgmTZG0E05qeQg6guy5hV/mFxfTWv0q3xv4Yy85odhRcf/9CY7KhuFT7vyd6Z1O5+6SMOc3MuwwU878g7SpBa/hv8ADabADg/93udfo502Hpt6yldl8DlKMzr0YB5WmfkKTjoOetInr5mFbQ6/qWHxKxk5I0IjIdH7mk2T2dUyDzboviE2aVHRgUz0c+njK4Zga+Wk+Ahjtl//b04jwKGgAYY8HMJ4edZueAL5Kq1Rk5cppljqHIbsyyk1dBdT0sorHOTQwieSy5r+7/4CB6+CgRi+TK8d5lgJjqTVKcVIDxuLHD9ftiLZe7YW2Min9M7Sj2+oaUPrQerBRy7gibHYEOnU+JmC335s/YE9FB2JK05/q8+Pv/tjTJSywudzohlZXGGviIzzpXPPdTyLcOfRi7sY7aeLk2+pZ/bFNSxV/7dcCv316lipLXO92gojNaV6k+6Z3E5PpL+E72gRfNEVe3j6ld8aJiWxZ1aqEEyu5lMSeueuA7NxrLx5Rfc8dVuPff/AbxsL1OFkmr+CzlWuYY2HmBH7LJhPuyqr82CgUsClgYW6fxTQPV+pqM+bRTUgM1OuHCQrFGCALqU7Tmbr2sQG6GEqFVgR6cSoRURwOvQX+2DZVSpR0AJNyu3LZ0O5+5P11KM73H3yV/D36xu60f2hk9RaBal/s+uacE9K9AEG+aVE7Wq32tGWZVI462kIyYJ0fwNLroJv0IXcWsMOZ3rBHeDOSWF4c2sbsu4FkbUCPXa1SFapJCNTta1dtqBrh6JJQ5aaTg+PKHIvggZ6y2Eb77AZKFGGQA1afjegw5uuQiScyJFyjjOiLigGlHqAxCeUii82Ne6ucw5k3EBY/RoxoEjy6hVIzVhrJJsG/dO4+pZGmpK/n3WO9Q5KltaVtkKBM2WW1QU2zQfuabVcrJNBvX+WjYwl5UVNvRUyV5/TPONI1he0zuk2P5xMH7gzKprvz6Ng50oXY2J5Yj4eUMloPK/BZ4/rV2whNwMJUhxCpAoCRTE9C1W88tfbw0+isTOJ78SwDu78pvpbgpDijgCXC2P5zzqvfxLPTlxMulCZSg2MC2chFP7v57g4+dAIENXA6McJI1omSvQf6fqjp7hSmMWEN8n2Mj3bf1s8qY3VhI1ZCAls3aTSohOCHIax7dD+qVT9bqsLC9FZBMXaf9jd5Xj10yV9WZ1QVELgbs/+TFVW2xC7I6YgVJhgKGssG30ya0ld/2CHFH/XZSEl35VSQKtKwoKO4Q1NjhSrFBqImi1VN8MAVKdjhFgSaQB/CdusoW9Y8DoEPq0nJYRR5mr8kWJS0Kaym4fCiml7UOeJzQDx8ecHzIwu8PTLXcqfiCWdPBOqWKkp7/TBNf9LobRCo6V6cA3VvuCRcmlgKAjt+zFQATDH9/ZZ8Wtp3t/ZrxdIZYiNg8158igHxkiumFGDbTvbNz34M1024sFyurzx76BjRZGh/1lOTfKJgqpUlQOtAQSLCtAGdCfEAhodBthtUvHB7Qp+5N+wxqYgmUXgwMSoWCDDuQ9zDDV+2huSV+Iv3qWBa+5ViXqYBQDkrxtROdUzFNrIvw7VsC3gqBJa1xXKtcqR59tqYX2K7hb4vHmu1peEVRH1ZqaroorWxU/nV42jfziztzJ1KE7SIMPjhjltY8PxV0LO/2KiVHV6Uea1blkM9geENpyTEEguBPzyDItigFSn2r84cgHLYQaTxoCjMB8v9XXSvj7JQOPIa+zsw5iePw/5eKvn7tcEyaUixv2AWKMWJwNQqDWlQ3PT2wL8G8uFjYRhx49bGSdZ31sfgSAxD6swv7i0H8KLQVt2GFYY2iytCDxtvoVyNJXKMf3QUgRMN0gQnfdxjriD538tmIL3zJdMpPWlwf0s0tABFeTZL9uwis6DWdeNGM19r5rb4r5y6zievrqKPKRT74JxT0spKe5rQ8ui7IXZaRVds11eyDCL+txEtWlpuaeRnRJWnb2z7FzqUnVN0gQM9fqQMDp0sPuNPUj/HUH2gBio40btFsH50X+iHKyYrLa41C6pJ/MFOG4jRamCIEsCBYuKbnzTBooRVubd8vKlRQEvj2C+EHOjUgvq34fvFxA5p054lGf+nBgZFJ7f9B77d+WU787J5IUZ/ziKR41Sus1Okf+DvsyPWQo3h8Qiymc1ETrkhCUzanuwisvksa59seA9jIqq8/D/J1MBvtgA4q2OsLqlCD3rhfN/jeTR8gXGGa+sMNe9MdayBNysQ/Gk162X7H0vi99Vf/gJtLIOsncz28WRxzInTPWe1RbiXTRejPy1y9cjzqjIV4nKyfeynBM1J6pZOf2Rj1uAvHWt7TGMk+joYGt1ma2m3PiuCVZUxhY6K/Wixa2r5s6cmmJX0R8kUF7yVeR7QdNP+tjlnc62+YVlqMbP7UZiQIYkMplefGDuIcJnbJiqdeXIq+TlxpSZfqLYp/QUzzdLEb/ICoyebzIx0ACRJWQb5qwVR79zYU9OyqFVFgXI5EbxbKzafevHWgj8UKOCzFYM5DjuYdDtS2zemfwSOfbAV2J2QNzoHVRPkV2F2rWQj4aO0fmNzcR2MtqyAxJEB6Ffp3mRilGEfxtmPxlaUEaAhRDMZw/UPrfgLSqJgyDsKIfA5gOECeKWm9o2mBoKyyKTiqrfItcl+bsK1NovWCSJ2pQALByYMYLDuOSOLWOUQm2L3xxC0QfS2xSrevwl07ZbILYU+bHAjDYma7pO3Yq6HBJ/SmsNe4bouCwhWMVeXGyR8kmBNX9dsS5jkKMe+49NeP/ykl0XQKDPJmzsEb4ut49Z5nk9e70SSfSql61c/+qXrS9LWVG8ICVQa1n5GQ9xmEUTHJl3bdVwBCxMooNmVo11fMgExyugdtHJZx+1BKKVNMYoC8lEcX0Lh6Spcs6AzAMxPgbrwquHNpYH5O5e0znWkYGpVrIvGF/U7+7bNKuPAQ1vYI9JO3Qz8yMH/t39/RDUpIiwasUN+u/rrMApJQDuz+DzYhMjTNZ1zr9lRn5340W+mXr2yo9zpLEid45X6ki3wD30z+h5a6cGeh9mOPp7VeaFzs5e3/40rG/JXjfU5LmGnpqRi84wuQCw0kQ/GSBmroi6BpW6C9W32UuccxST/J+2hqRM46yPqLMHuNRCo1x55MNcvpjCRCLHkzDmcBE9FM+S6YGxkOWqk5mGyy0gVbGYtLqWtd25mdZvy1FoCJ0G3BGr01JJBpuGMH5jb5jaU/yrTqfpP/pd246QgSRUUWpJW+a569nEYZE0h16mShLjodMvfb3GE1C3OcUxoUr7+uoqjedbYiUSkNwPXzt6N9aksKQUxCYMVjmn2d8bB7TnVz4brCSvS7+R++/gqOfrHIZGiP6TAjMXrtmMGRrDH5PfNRWwvBY8WgjrzccxMnTKA9i8+jIolsJcGwkl/p0YAjbAcTCctrYkG/Vr0MkV9P78XC5zOd1PS7k8HfayssVQgUoB4L3LRrLv+pOJj0NbSGCsPDiAomHCPO/QVa/nilKG81C++YiJwcY+s5/fqp4ueqHH40RFUHyZeTr0dxXV7IWVfhyhjAn/YUw+LVJe2l6cCDQD5BUVs4padG+nNKj0C6aLCyLQv3WmdOc97y871jmgWjnGTe4SP+P5JfUiqXpKLc0Rm+w7/2uadu8uOs6B08KIZy1PSfEFzN4wqdnn59bt82HnpuPhA4+vMOGD5/UMJbzOU7CXytkpuHL/WCzgnU5jCbnN4zQIQ6qVZDAnwArHIID1p+/SqdM+yv+87MqynF9EUVKsG+YhYiiB7ecpPR7fn6cbNGkMfHnG+X0Mp4u6DiNZsMxEQovGP2FSr8WuczJtksTds8lSctqLnyjwlOM4Bzsm4PpoVYtz/DwoC+IgN0yNM22wChb+Dcyp/oSoNrOvN2/MS5znHtGHpVOqTNvDPKB6nCFB2khAJ+AlBpThma5xYYD+yEnoFASr6keZgE98EdG0omIgSyl1D+cZkUHTAafHvv1fqP3qoT5C4Gmf5MMMFGHlfHDuJGUmA/FEKjUjac+qIjSe1WDOGI4BkMS/mXvGivNjFCQWWYMMqYee9IWP6H5/TJ+52NOhWgOBYJIEtC+FGpjHCr72tKBJyFFPbW4ri7+KudeUbDmDXndXW5Ov+jyMjovmnUjtyhvSsOczWhLoM3IG+wNXvCh/7JQsCxxMNYKELS95LhmxpBU7IR5XQzNj7sj2YMv+CNavP5M4Z1HqDwagQVOkYD8IJe+w5aXLcsrg6zn3oT5aYQfNCx0s/ne7ig0O4nicZ1sqPoVC8NZPyhH17HTIPjHAZTIhilgE89kBwpP6B+Db5VPle/ClJ/ol2zu4rVw+j3t1AYjo8XnvG0amL7NtaZLK0lqDkwzROHC9dyODlDo5DExA4FX1uS0VqqMEjXxLtj6eQ8bODesX7+b7huUaREmhaL+fTj9RUqw5KCEJ7SU1BlRXhAc0NWFE4d/LkcUiRc80YNwrP4izd1nex1kZnDhRbOEr3O0obBLVf7dcy1/n9hAb37UW0cpt2Cp1mEos6GhY1K8D3XEPJn64rToZTjK8NVrgh3hOezERkZFcr3KNow14GLxglpxemCn9baGXKvo8PXwbxRPJ5xMKowr+36xbu0lcCegH6qXe5KbtJYhWkHJ2++WlScX0+cmrDV48Roy555ZSPdQ/E3bVoWbUqDlQz5p1GulxYXGZQMyWuAFv8oBipwh0uD9GusWaYLpoEBBncfKNRXQcE3A6I4mYjXNrp+VPCRf+nD5niCRGeO8/X6dik62NZI+R9GDdGYZkDWgEsRgDGzUvC/11DkbhCSyuET8b7rW9qxAYKwckZbDWKy8Pvl+gSbWcWb04Z3+/fT3jNY5BdSqkxS2LQOKHk+QqFXdJOgLENkF/p6+kZ26v6+gL+6xak2Za3VfsP2aPUu0+XCcq2JQPkdaACYWwjz937PvP6C6m3nnod8cwVtv/OuIxmHy5A/sh/h9s0AbS1r/+3QFiQtMj+YZAHNoocEGUq4shVZFiYFLVT4pRw0vPaoS0uDpY7FZgFdqHKu9t2fCJp7kLDC2lGnUr5kB/Dbx0y0Gde4FNFivWK3NB/nMwlSDenvADYdGt851cmnyUgYjkcPn73F49+Yz2p+TCkfq693EJBHwOQVxATnm+dpRyHW8wP73qR13IB+XncQJWRU+bCqepkUMXW6j8qPQvv+t0L6OZpmXedeagtC8MD4xmzbMohXUOab67AXx//a5VBluZucYYUs4IFcxvErvRaAvZjIOv/phQ+2/8Rj8KGtH2l8fWAp/Qp4XRBSuvHdHY/kvAfXJ4XU5dPCZuqIwSEG04zpjvyJ/zhevMyqD2wVwQAJwjFDttv8+V75KAibtkW5YjqrkhqXZUGXnv6wK1s/4pLGiDV5yaNtxQehRPYP/DJqN+AzbavBhSKcshhEjzHnU4Jz838pkEsFJioa2pJ0jMuqS14VxOgUEgliZwLmwHkYhLLj6CW84FKtSFJmWczPPZnzJpgiJJ65bZJwcCr2toD/5x7YM7I0fj1WMveE0ldgIswnsPN83cYhtb1RrjavA11Ma4iyUA9mGbv811wxjT2ja7COq3/D5zJgsVc8nTx408XXvKuu1HvPqZS1FJMvzHZE3SpI1LXsAp557LtBBpW3IESF5XsZ84f9wO6lgVK59BsZdw16WL7QPPYkomBGDcYJqCN1N19mSz3AkUdvmqqnjaddnwmDI0IVzKronN7wxVXuEf2ORGYmjuTQj2JouY8kabXX0myBtr1qq7th0YnXKIUFKtoyZLIFsMPXbD37GnzZj3IVY2qaUYRPQ7I3mr52nROj2WZ2fGLazoPvd2/A3vN5s7LChVVj+gMnlwkdfFiAklMX/v1k2O3qJiaomh2b1XiDoFweZWw1I5rHJVYdtRhmP8QAfYCgEV0Z3JAPKXvxAUozvfb9zzTcOl6o4w/zTxyy4tjEjSjesmzSKrLFzmX30WpdhxVCkTPg3JXjV3wjCHycmQwJ/mfghXRtF123Heyexxp1teChUAMeSIJ0/ANFC+s5+TQfljw9/jZ3380z0hTAwLD3Pg9Ijaqb6FbGIr6lD1mbTmTeJI2H24/t6gEeAUaQmpwpvcsTZlw+gOgX/vEFb3YRhTkOnAoziEz6G8leN0ev8yOjQHGGP4SNe79M2DekH/7m+V45NLcbg4gru2GzvSQE6dBq0Vnpkb7AUS9ZBLz6ocvqBHiptElLYa+CSA+97YABTR7FkGzIVClR9hT4jEsCvUSaUC6g75NiHvwDWEjpxjaV2Uy7tRnseL3YC3/mIhIxdLGGU7L34xys0mpK9L8DRLEYKLo1oyg/OgLV5orIb7U/JWGdkFHx+L4XcaNi7vwgGIs7rfb+ppjTG3xQa9xJDDmBOm/btCxHRjNRCbvpzjQmvBA+IPOzyi7nGsk5HBK4WGFXWcTXwzuB4iskCkrSSzjykeMQG1ZWZoiNNYTOzfkdHN303O/kZL5JCPVvZ4qSBIrv9cOOJ7JQPfwQFtYdT4jPRGoMmNHX0x9UAapTVqjI9DiI8uQlzDdxb51DEesEJZ4d0DjIxGK5OX8RkaSW0gGfWYU4Osuz8/SYXNlN/525010o9jmW5GwgNYg8ReHit7YjXZAdE78TyDQvSLPTBEUv1PFJX1EUx0AUgssRTeq7PoEkyMUk2GF4r7/JkoDLnq3VmGUQ8nRm02t9OD0ysM3U71yXB8YkCnsfy6grFzzCnPCl79oSMJVBaxpit9cY93dqOFZJ+2+9gghi/oR/No5rjfL7s5XY+pJCrFIrcKFVOdn1uScuXpgBhda7Cz6aiSa4f2O8lUgd1k/CeDX9mZjEszx2oej8FPR6FvCoMx58Zo7/IGOWyFLasL8GDbtdMMXrvR5AD2G+qJcDpa+S5WYBeS+IYRcW/STHMfTFg6N6zpXPxFppxOusv2i8YMfOYCa9Ljgcgf73L5sH4gX5OYSjPXZXHpsF4h2cu5xT5dHhQwYkAHasNWUMiBOo3nPAnn+pi95OTJEYSLSL8yqrzUXx2EQwAEeZdALtNIkERCwIW+Bzg9/+KMWylUYZI/EwgMeKKvU1HU5CHumVNbLCHHlq8IdaGEJqu4U80kZQbV9hu8Uor/5TLZX/SC5loHe1e5E8vmeZ/kDlyscPSMVe+WFFVRempedDa52u356S/a+W0v1GbHE+Xu3649nnTQtvBmvZJa+mb/mpIXXq7NXtWf0Nghj+QB0lf9JPkL/uQSTVyxfcF2fr5d+N4S3mRQbIZ9G4h/J6Bx3VYXX42GwGptIJOoxC7P1yN0v4dfu+qTvlFlLSQv5+O/lxB10FUXzqt2Gcov6DpOdmz/mAlCG47WsRK/gtZ0FU4h0lNy5KrnIZZuCra7Le5/UePsUwpINIP3e1vYmGIG38K2hQJMgldRu1iKK8bffspnaJzufxgizQpG0DHhbYVrbKefV6U2VQftuxzaXsHZHBLJ4IpVSK9dQWcO0iHqu/jut0f7M0/zHL1C5F2kn3IsIkBHrORGJMSj81q9su2Anfh6a8zVZLs1JUcJjt2yhFKfKTsK1Utr0hYZlfNz7zJxaAkDdSoIunLzv40wqJRql7iXzhB15PARE9TrlQyPQv5UaMLDYSq5BEMY17Y57uI23rMZSgHM2shYqkOf84RVVPquPU2VwJiU9uzmKZyJxq2Mj6Q3SfQTY+q6/JONJl8hoU5Wn0WVfGufsJ0tuWOXeyID5r1xn7/Cv+SK56o8BxZjpF2/+6hgjMm+dzdaEg1mox1p3DUdUm7cz4sXn9DyZedHK7flb/MqUJDv1yCbgrKM4y/yXP8XQdYTTSTKv/rpi8HTYhiAgKmQJiOE6C+5H/hKdItvH1kF4ADnat1BVlw0kBGlYkWSTO+yk/hVd/tHJ4LUt/oGBc/ND7j6iwFioFgVF+6ghrdOxOTpIlql+WyF+IVG+BMSEpNnwBS0JeH5IQu/8qs3Ie8XClsIf7k4FOVzDd/UAejtdAxr1rzngoPBXJpPAE1GXKR45ueiIkg085+cI+g7XOpvF9cgWjz1VrFq22KmsY+bNlCKFB/mfEVYCoD4Vbb+TTjCQr/he3H6ZFNIDw2XgkyoE2Z32bjP3LjKEkpyfPUxngbn4P5c/TOTksak4M4cYDk8H2M9ItXbfkCoiynCle8iv2St7+643ND1rZauZ3GKwv7jLq6B1ADvOeyrKrhft4Dt1ImtxaXw8iJWaIoh54f6ALgQik4Z60Ucu/R65m7UDeNx4Vmpfin5O0n8HD8PgisuzmXtevS5xEce2aHyRG1QGuP5gAhl6sIrjtj0ec2ptVCONDtKFtCyKsaPsLM5aWUzeR7bm+jNWi2Iqrc5CsxOosbh0pp6zIvIPHb00jVrXR3IQ6+9pRy6bjJnyz5HmgtP/mwhD+uE+2cm/mawsH3PqhEHkt0I+jnfuDdktB1mHhvnzpvO2TX7NcqoDbCH0noZVjfEVwwbqLtKbTqo4c4xBycgdcT9LM1RYzjNeN7AVDSbSEr7UXkOH5Uk+vNF9xgLroRaI+TUoFtROc+9Kh4H8ApQiBQWhW53sTtlA0ZlbR80QXayzGIpb+qt3+nYb9hWlR+0cGwZ0VG5NLtf12DPpMfud0YfK+IOGxyulYzMuYU0eZTUj4b/TtHjTxABiFrGTKOOsqryaMUYHEwXaXP60Lrc3AcdxLZLZHtPkWu/yR/K9HzV9lG/7qWEPkOaibpScGAYpudq2ugpSEJXCFBpnCeTDO+5UL1C5v5oCsHUxpYd3CpUjJ5c9xt6Qi+dwQX+jml2pmB6lfJGDEWBaFms2gLWQT81z49gn1RKytDblcJH0df26fxrjEl7Cwuc3fQnzwbeRHH3xZ5A8zZxwYt/kkK3EG1dMs/pLV5IEHZb3ZJw4zNiQ9EEHZo7S+mUsbg4dZvNeJVSDMwsQ8bprwR9Pj0dUT4H3abB28uEwbX5Xr4diB4xBMzad9tTX4yCGg/wj4xaiSxv+mgDxyuqSSP07qy18HZqMY+ei0XliTX7SIy806n2XUaWG6TRIORkuh90qshOSfn4wTNkxJO3d/S2oDiu6jljsgbwswUAZ8+TqovKOX2V8ih21++HJEzVouC1/PuMmNGBjdSBF7/aSMiQD8lfi5wbg3j5+ph/OnR4D6YK/91OZuSjbpqhchOEJyrisRfyUrw3+/sTsMnOxjLzqW/YViR7xFijSkCSs+kyf0szaVvor3bVL8Rvgs0cGwAOXCM+IJ3gr0uOX9u6HB0Gf5yoMGlrB5cGw9DCu/F38i6MZdeXL7ZGCIYv3Li7FjgQRUrp5IvbZ6GQL3WGJQj9CX5Z1hCjgDFHbXYdj4Z1t8Fd0PFQwPiOhs5KMLUj6a7RxP89UMaiBcRCBXf1m3a1H8dE+SCJvbyOfX0/Y/XowJEc99kjpPgCHMjZubMI2rnS1Ll2MalHtss2fJl2rBJw1n/Suhx4pDSRZQvfjZozmdUt7kgSz4JsKbDUS4vxb2/yZXdhnf3TGEbZO66MSdGg3gCIjHkpOrs3a7XvQoyQ7AzwAEheQwCj68xwv465aMC8hmaqIbvUnMNPTSIqMjryOCyMrhoIhj4z3aZdF1XaivJg0buz3y9MBv+Gmq1gCS5d6e5FPbNsBhT/9fDXwUaYeaXtYJg99+YVWadhyYZo9JWjSG2Ft/Rzy9qpharJicyOh7uPdFVIFqjqtXzqxPy+KX1kZR4SIIlBpIgZjZz13ESdZ9zVH/eu7txP962/NKsWgfdP5/Plv8p8K7ns+xK0gcXQg274s0zRejzSz3kwIbpmyc43mGkO5B7Ow8RG8bZUN/5WTS5hT+1FQ78XbWDPpAAskcf1dVQ9XEN7r4Lut3n+q9MzuHBHSMGWICk2WvOk82sQDObgnrMZ6GyI72UTnoSXN7t2Why1HbgYdRFN441FW4TGIL/Bm4WWZE0cyw3P+pKVW/hl6RRo0NzFvv8m31iZm4+KWXUejHcQZs/t8GH84ZRHlZyNPdkumEKoz3fnCyg2JRN4CYe143a1AnTc+J8c38mqw/O4tHSEk1AuVhenlqci+Uvj+3sr/C/wz92MqAnI5EPiTYtWEaRPjcclb5IKMQVF6GvDHumDs2jFqeBj4h+3ejoXutflToJ8s+Kzqj8wp5pZNyNikA5zlSp2Ix/Sm9W37rZIVocIwsfBycdkRJMBYFn/atKYACwFVMdgNyCPDg9DZtolfG/0hR9ogWm2D7m5rP6w7sTKTVAiNhSeyF53mGuvxQyc7fVXxZ357xLiyeaUYWJPX02aQcwoaPkMcUoqPOv1HDXHeVu+rXNJgqfG3/XlvlOrABZUIe2kQrZs+Cg1qEjKYyq6r5vktVJm+/+N22TEPF4ed/vNWjKe1wKGqBDqFPC6RiQ4/dhCIyPI5pVkfNvxhizCyuNeVvKvc/b96nRVyHeKPYpJtocnA3mf59qIT3xzEVdHjz0jtrPZZ9Obbpoh2zANxWUBBcBIfrztjp/sB6cDhD0149ZZ60DnmGDSkMvmffD2DFmFDfjxMzoFt3w0sIyR33OU+BiuI2K4+GEMXDbUvDTDRAxknpd0/fnhcNWdujlOVf+KXrHXES/IneSqDNNmtlbGk4KKUt+bV/wBxub5h9iSGPXmuA/pY5sO/iyKKzbKobmP1q2rlvxfgP2EU/PP8dCO4KaAxeve8fgEq7CHqMG3W5g53lAf2yYy+UhNRsxo56/SfIz9VqWLMkAlhvw3ovJHJDG1ni5A3gpVoebln0qNEF/c2bbhLPb4gel4N1fp3SVbGip+nnRu4KIK6i4iFiqYqB5MMPZakaIJ9yFWLl0eVHSeR5Kr7QiacPfhAZMtLLzb/qA8gQyrV+btpSlYKLE+h0Z+WOz67p/cLrKvTnuj8jZjYFmZ0FGM0wFz/sDx1xsFaxZP0r3SKQ5EXSZiQoOG/5u3ymOaIxCZzTGamhOfE7PfJmcNcPDTcM3zencjROLSJ7D4WizyN90NjR/k0M5Wj+s5XcyP7CX40u6K4YExAIi3WdERB9TzXpQiaqPgLDRyIAWjcvpH54tvr/AGbc2chU05boNsETk1zJVT+d+QCImq8jpQPFjed/lILMLGUmt0A3zIl+Kxzoedly7kYy3gX0eZ+Z/BprXf40Xx4DxK4rObYb/jkCJY+OLRxdB1nnu3fGHtt6AlhTYC4Zd2Gk5CByPmf7kOeZIimoU83PG5V8xqg7SdgT0scxp/WKnxYGmUHj88JnCD2KbCI9MO0LMgZW9SCELaa7YPTpqhgF8sSuB3NNuwQaddWJWGn2jfXz8L8WCqbe5vJhfYdImCM/ex/CV8f76XOXSU0vusSC/v568rsI81vyXOgzYhk3zRrPFpLbft0PbtBzuhwQyMYLFcir+dRnyhYI04VgmHBSP0gEhwEEORZOgsLvEiw54lPwbqunmIIK2SjhjYrGHK+VYCiy8NgCGgwhdaVc0xKYPrVuKtgF7VXfBZz5QRnL6ak8lK+yFIod562kHB3f1+jcqkVNs0dWvnORSvX95ipU9cU6vil3G9M2/5zIjW896N+B4jYCz1hcJFpUIWplj9YeWmrggmOs1bTbsol03b0q7px0HC0SPl+y0tFx5fci+D2l+u2DO8wR6OTT/GHoLXUKPc415RHhmj5vQeP2fi2IPUWHdvU48VLoZBuybf3s+vzSpx/4KhTHshWOgyVOBF1cjEqauwFaRxIty5abxmFm78Oy8UBCn5V0yL9sG7SnWHpp0rGI//Hl4xmCd5KvY98ia+SOV5M9U4K5cevo8aLW1JHs0f0OlRKuG47Q9fK1PFXPJZ2tAkxtXfv1I/S6B7x+SuVcAs9D7KR12uGEZQDimeX/Yuhq9KPwvqs7WtR7eXgocNhcPxjxScmbEWk3cSyDKeOb7fLOgWG1pvoz6kTXmh4jaWQUWorpjFjBPWOQb1Y3fU2+Ciszj2+9zT55EKX1M4bLW4Jspwkv5wQ5K63tO1akjX46ZgVPa0mGVTImNkAF75QjrMXJnL6NQxY4cBcrdYzmcwYor3xv+4qORjpvYgOkQonnNQIZnpQpyIw1D+2Uc/882aJmKYLtg6YLabC18thvyNM7+pXRYbABb9C9HzQIf9hal1NqU2ceALj+FD62/iFYnIuVmJ+zvaLiHL5GLv2vfFJ57T6gSKgZObd9Mwzk8+9GsLwn1CbgVcRLita6256t/I/XMFyC1Dp/UF/1lexLkVw6ADfA1gNJf9TdLt8e15U+gcQw1Jvakx0ujICCNYTF589YCI0/gNkj+hAKltAM11xFqItYnIRACTaa50O71BSorTOEILZ8jKvdiV1h431799n0+KeRxtGASGwztP3xrZCwKEcr8EFXnVvRJt/nulCLnWdj0w0fddQ1hL8evYuQ5uuFpEU7yX/Ujle3ncqTmMD5fJaPpvnyxTbj2ITZ3D/DZXNT6TH840afhvJ4GFzuWUUWF7q8xtVgeWAeFyGK2rfNGdH7YqykJJty1xK+/jk1Dz4TA3B/NW1dGw56FPZr8fERS2Xtmujqcboy0U0sk143fKR+D1upJ/u3IZzvNQihC9wWK+qYOT7bxYa0wP89MBuwnH5LhikfJVyrP0YxD6x4AFqq4wyutqvdfQZ13+Yba2LOppwb0xVcPwtuDp4IRabDff/f9++hlAwobctF6ypVb0y8nePL1Yn9fIbAraLA7pcPjX/z37kVW4XBT1XQuCcHfd/tRf+4cuxI6Q8aAiwLQdhk7bP8eR0gVtUjF6XWJGP0iDhK+1zhokPIloKsbWX/RzTyhhUys972pX4gsFVO53IjmFRbt76AMuOc3XZ7vhwEaOzmh4s5PT7gBptwar+3JsJR2GTJeZxk5imLY+SsjVUof+B+BbapujQlPo7Xpp3KPymyV/ZItCmoKcMlnp/GMJLDg24WlSBM1lZYiXkluz8SKaoYdyV8ta8JPVGRTPkUPHZBtOHTwD2R9it8C/QyAhnCmJUQgxkR5/vB9lLykGvDUyRB89ZfKr9vYs/7+C+dQ4/wlEolrQVupvIuRm8bC56OIzyxvPaKOpo8/3vaXlAtaEsO3FpoGQnljjPmz0MxTBVBQ/EyXCrbwDveuy9uY6HCY49LNKP5Gil/gUBpOIwC0rSsEU9RalrumjbwR3eIw1+92TZ0lMz1Ai1AhduWXmB3vz+ka92IfTSH4xhy1ntN/trVsJPuNCMuqSFVWqWbD5B8qepsp61RghCaeLLzA/jz2e+A/LKFOGM/3PKJuXcHK5Pt7gJTj/gr0UPRA1t2/oUTYZ3nyftAgF1W68kcd1Ijw0GZS0vHF0CmyNun4DMLk1eWv3e/IUP9axr8c+CNq8S8NvRHB4GoUZezFRAAyS+hVF/fNgErrfVE+KB1sx4SxhL6x/Bsv2+AL0pKcE30P4/5e8lfEDlv/NuYXujmxKtmIO6EDIJN2KCK0IOnMYGFNfuFOq/+YuBXbHWgmp6gVCDeDPeZ6fHq4M2f1YzU4gPsSYFv4FuyjEPsOrSoy/kM1RJjhAqz74N6IChnPnvcpVOrlQkVP86F4x7EyCB225QuJSfiX09HRSNBWA3tCt8iOoppHSyINV528ATv2EWo+30OgOTTINB6FkWzX4j3flPw1u8oEkBJJd4lNOzsyD7RHTCN9iL82XdkLtVPE50iLOflidP8uLvo0QUuqfOASWSHMFI71s0zkM/SPuTBC7fQodzSkc50hTrHs947IaSe8m51RPKUl21C8YT6iCBwJ5gwiuJF/AqbbCiKmGXKx3CT+FXkRwDdc3yLxn8qTA//fzSvfhzQX9ebfCnKwjNHSZliOP675xgCggea3jD+NPeZz5RiFqfldnn5U0IfQ/FUzeoyDqA/18+CVCHUut/TE+aDZhw6paOkzDdHSRqbGjs3C1GHD8/paHCmeGsD+c/MzUQPjTAZ4Rcu2erR6QMqAu2UBAZnzJkMW8FLbcrIqataDA6dGO5eV2B1l/WTuTqEayBVyLbgyNemDHK0gYwURxN4FG9W55RqOzVyjFPBaccqv6clxLpDOp0HA3ctRwQxu9d1dxBDlSXYrd1dFY4wLKXlJgdc92vz1tosTFVB5t5yjG4jB7Jvp0p+OCI4T1AyI41zABhF2XlKrFNPVIRGQ8guzPqTGnNrgm5UC9lVBf0xZQnVH1E0PhYz7HKzBY7zSUDAij7LgeDISo7mVj3Bz3L8zzcgq0JSGQ4gfBNLnV7h9Jl393t8SIydTp787ElcAB1lXBLe3YhFouKLXjUjnZf6uAniz2TiUdCINoV/T4RniKrP5d3MKy0g6mDvkCJt7FqJfc68G4iOl+U4DBbyFn3uhlph4vE2k/RAc9AJHf3b62jD24pjNFU6O+MmXcCSKKOf/4+69miQFuizBX9OP04YWj2gIdKB5GQMCLQMNv37xqPpazO72zK7N9NhuWWZlBolw3K+fe65wv4vEA7fZCrS1d1+rFE729Fj9W8AqznohC3sVQTdqA5Wcn9sxNZxSMLyLxr52l1NRCS3jX6W50UN3JH47wVNBeOYHiSJpM1FrwI6HVDIbO/S2KifGubPM5xxpCcOq19NN5sBCL6CGAAp1mdKvx47Ofq1ObwxVltaTQQ4thKlggo7+YKQVbXPbIw4GAKeo7oKcR8dGD8/ZZ4TmuGWu/mpW7wLnJaxV1aw4vzJhlyytWiFg3Ghhgf5ba6MklP7epfy4ZC/rYm93wpTklilyQR5OihZ2cW1wydgxwygwSGpJP6dzn7mHNtWs6VijsESUpluw+J4uxIWsl9BHMA1Z6nQJxCJqXJuMgorRsT7poOtS7h7d2xvP/PrDKbAY4abzhqWY3bC3QwBn7SRN4bmE+vrWYbnT1aPwqKkLe9VlJFgCcyLHwGwBEtTBmyNs0HmaU3MAFxKYlZCZ7J4CItA9obOPrrDoX/30FJ81U12i+sinPOSbb33QszY481pvbTzyvGPfPFN2mu0jKeGKxfGwbvD2nNLA/oqi2a8g4ZswMoxWJ7BZc92o/nR1QImCGLE4UFTiY+RHTG/7rYoK81g2Sp/MOfD4wySa00QlMBMH2fbEigLEk8OL5/pd8LZvMKwpUxK2YtsJ0/OJFq4rPiU4MAMJFUct79snDM3add0aAjQ9pkpd5zkPFonWIss93PzUT5khU2D31qOcbkmWUCPkvDlOVcugvzcWvVU3J0LHom8XmbnPm5MVhuXu9z7sn/pY1BBY3/VvY9gtVRczDXBBELz40XTnpFGabEYJDSxavgnJHVJrQXpsLfU6E3GXRTt95j37CYlQF01PFgRgiUr9wJcSKrCfx9o+XBrGhZoFpmdfeKkEcp66SC7GqiOq1EhKS+E5TiOT1+RCeuZQX7895nOiXic84/r4u7J7ZpBFHyewIMr+m7pLK3EXq2YmPdDKFGYa8AuwMI0m9GPCc60gPdqT/O3wC8F4FCOjngD/8hJJynftn/aIEj5bWmztt7PW0kMzaO3HjZKXPTGswnxN6IuQj07HaWBBJr/l+2ZMt+XLHp1TP7iCGpPOu3PxaKW95f70jKIOxBp8JBgGVqi7ZxnAKV90osZmWbCWVDw2kbAfc3k4bNUFfAJunQrk1bBOIRQN37Tr50F5LeQz5NDUbJYC7iEoYLkYo11jTgaHCoxL5TENZUgz5ofu/ErnsBkUukv6tP+tKNVpsnQK9Z9Ez+Qj4FyQe69lf/rmVZHKoa1DvOmx9drmfh9Md+eRNmG6knKoOz22i3XwL+8gkvkWd3pwWIfjGeb8bScnAt2Y3t2jvFnovsPPcfSikffKweg+xe2PvRslIVdXi/X+JD+Cp/7pVzP6TMAbsiT1sYJq1SzfiZ9520SkYZ5+qt1emUdz6nvbG2JVTak35/54KtZxwB6vLUeyz1l+4c0WbLYnqqrPQO57JWzd5fNM7Hxmr08dqAIiX4fQannmGRqO6xWO0NJ191ng7p7DniZwVrHLQRUuTm0s+9hPl2unGLtGh9pIekUDzKlqMDKsO8pnmtO89uCNTgrfjcCvnsVqDjB3eJY2FrGpuH5jLvfYEGm/YW1kl8BZcjLIlKOoDmrnst+m76WlYh0oH5lxNgDORx6bdD3r7D0Tn11OwHdGvSnlB8NRlUxgdzSWgrVHPH4byV23/6HvSrKF8XrefAOCthH8vm0tpOTePCMl1RBVsyoSkBFBbktjWvMbk+eGlKZg6rdGQpi3aAhvPuatb4L3t6xitl8FIMjlvPoJJMXtVjIQ+flDDFaQv020zFY85S2aML9d/0J/ZbNI419CrEhRRfArEG9tZJO4naZUNqAK6B2ix+d8bAQlK5kHIdglaFJyXODvw3lOcoFQ35OV7BMIMRMrapTSDMRDjdN5a1n38peYiPBkXCZ4YcKD8ozg5axH0idtplZ7nsPDQd3PVzMMRoF04dXzexn3hJzaPjTXRpKcMbn+Nn9ex/QNteVvVN7vSXQ3Qq5liZ7hRQVxIewkd6p52tExHFbcIrn5Nz9XWav3DzMy0yrWrE+ZpTsk/+Y6JykbNf1KcCaztZkV0qSW1nOhGfUE6qG+zXlSw6BganZgfVfhmG83nl+hHe5v31QntOfL92Xbz1CDmRHXZA6FBA2AqiKljCXDlfMfQ4sxygCSLpbxCoof+t1wg0UR2ntrPeD47ycrHA897QfHdn73Ymr56nLE2B7JOzKMEjBTCpwkbANpNErOGj5A8Az10riws7MwVqFZDdEANXYorT37j+y1s0euE9lN2rJ7l8odTIwps69PWSqwidcqXC0rKsUWOwTUdE9Qvw1NYseLIiz3WmPrxgKpEQqAXMP7c1vbwa99ShmDCCVngVWqoutA445L8VfAzW/vmtnhuh+OSmVXFzGvluKy67bWbRgI8zFT+aivr8zkuAMoij0nYG/WZ7AsHuSMyBZNRQopKGV5MHbgFL99Xu21aHQNIR3DghdCHOf225amOWKPxjA9zfZSfOT6hoiyBcQsC7Yp0vHr9ZEqyBKffXz2XGMPuOf9kUt2g9EN1iwr4Ycr6kPBnmfMzS20aGg0mgTflgnfnBl7t/LK8GPNtAhWJM8zLSnSCEMySrZobntCd3QtY65m3gNSyS+EEbrbgDMwDVlL3WP1Y5qEO39n7jeTBU/zNmT9xfw9HXGW1NcNOD7VOJqm1T8adKmki64GPpI/LethtP7CjqSCFIRDIxPtD5CPCI/O8VFZlNlx7JMib8FrJLfkF0l9w66ifCrtjfDfUMA3B5+jYtLqP/NOT35xwudVEcQIgqT4Sh3l6jhGelnyHepZfL1gnjO6W9bU/pOj+Pb61RWiZKk4qvZhC7f3q4Y679Yx5K/bY0nzzf/SKwsdJ4TqLgYcQV2lPuYjbhP/rdD11ygCu2SX7sOSqQlovwivBIpzPj7+AJaVhcSX0M+Ci9PSgs3RH9tIeX23qBWWztOVMrRPqgzUS1Ym8ohdF8djdbOFqORtrOcwH8Iv5l6E06iWalA+ueotU+IKh4dKQSUqvanlWNsqo9h7meeUCKm8iaqs0C3C5uVlx9IBvygV4iV8M6HVq57RKsF3zalCe9iC/WYEwPKeeckpzA9/WOH5Bvrs77n6DxqfbzDzyvr3/egd5nctkLLftWAY/l7Ljsyfax8eyP6ufSDJ+XstkP3379obIymOYfh/+sceXXTfEV5nMF+FAf+ETnRbZ7N7jvsn9BnbZ7yg6SGww/pPIJUCrAQHh/Z8XvPz3xxChX9Cuf6U8rHPV1CQEPr71/+CgmALuOb6ewCHoT8HjvqzVn/vgWP/jP89r8rrsvrH4zDyz8Fk+XOg/JcH/LZK+T0WOCdPLu+6f7Ti9zsC1Z8/14TOf6XXUL8RhO+gTPyvmjd3/wX524g96R6N+Tvvz4FlfaDyz4H8U+bO34/DODw/2Hnchk8Obgw9n8Z5rcZyHJJOG8fpOQg/B5t8XS+nvsFVybaOz6Fq7bu/f13WeWzz4O+rPx3HFuOw/j0dxp7PT1/PVwie8M8QSv7jQPQc+C/PEeJfjvDn31b8+XT9209WPtdPP+Xz34P/twO5jNuc5f9RP/3tljWZy3z9D07E/g4f6LT/UDDmvEvWes//XTv+r8b376XWWD+N/heBgkn8nxGY/pd/1L+XLgJB//0d/zT8703+G6H5l1b9v5cjAv/vy9H/AqHJz3r9IyP430/R33PB7/8qGuDDv0jGvxHnf22E8K9H2Wyb91874f8cmYHp/xyZQfD/WGYw8p/x/yGpYeYZVL38l9MmcMLyHz743z8Ko6D/Rgr/3PJ/rkyS/8+w7X9AGP4XiPB/BiZR/x+Tr/9ZEkAi/3tQ6V9VF4n/W80F/XeU1v+PsAYloX/GMOhf//03CEAS8H+qKCD/A6KwVMkEfq37BHQO+/vJLFOerX8HKPnHh6I+gcywgAHW2SMtSZp31rjUaz0Oz9/TcV3H/t+cwHR1Cf6wAqn6KzPc2I3z79EoitJ0UfyfpIkAxKjuun+c+Zd+/VuuRP6joc+HT7Im/4Qyfz4i4jSU/4Rwtc+a7wNSpXIE1NZwvErwyuc38M3IA8dEz0/ehHL/8zDfRG07wfbfGLLdNIufXQB7QwO8W6i1Sf019/NGrPe9Q9xjVdvvIxAkV9YJtrQ/RT1jZj32jCn1hbBXq/D+OPY4MrojyiX11hVX3tnfFpLkIIskSsAlDmPQQBr0MKOWOfsbSn6Rbf7iGUFtQ274MNnAw4CGD1ZvJNzhbLmIpc6W+jmNvHAwUkk9n18lA46xo8VGjMZEPPhKeM7m+CNgDok5eub58b/1XOWedLEVeD1Vj2+xo976SKiXYokhSkNiUA3sqpWMjv4LGLplWzVsSWmIEEEdZx/zywBuIxArzLo7BHW9RDR8eSgrv7iaPfQuSp7/uC/GDaGYYhG1q/g+fZZEI/m2jw6/yYQa49lAHCn0Xs+D5w2+rBa9N4Iat7RP5s9CieQlEXPaTL+42LPttXWWLnpF9tbxauBF77KfOHmgpO98DbTPTeRwFBnFeiYh7NxjzYIQzgDHXK7EFd1NJ4UT4eeKqVeOK0lh+9nUk1M/5CrZNNp95/Ca8i7+jx5jueE1jqRlfD5okbnS/RJRjVQiGC41it37GSn9GStWFdKEclIihwGLm3eIIXxFdRZflZAOFm1/Xl1OYyaus7Am4FyogECCQDn/gqIaLkeoR9+3UC4TE7UvntdN2wAekXA7J/WXJt4tZDfnVE3Qq5r0HZLxJSUIpS5GHxek/g4y0qkZyu8bldNTmCiOHHCXnxRqSVxkrEThN/+tA5zjnJCvDlq//KtYlUBwFJNpX+MBChWxPjSylaPVm3O8WY4XajGd6JzPhchH2Ao2YS7fJ9Luzh7JrH+0gB4ahFg9YiTJ24K/Qea9Wfnp27Xsy+PmjV5LdOuXx/3oktPfgFcrJN3fRldvDgT5MLplklaUmexNm6ICsl8mBNLw7uJFCdT6uYehh6Y8moa+1dlKZ9jeeFHh/SuwVKi95RtQZcidbm+q56RrT8BiTpPWPLAKsuV69BEdYVJc+9BLqmarbcVZj6O/rGseomWZKLmFMOnAQ51MmV/qDJjTkeDSVJYk1wz227jscj9RbsPfagStVT9wHxKti6Z4wEcP0N4s60jydObi15ZS3VpoKr4clBi/B+NyGzINrKx3ooaLmGcmZpT6wvKjV8F2XaI4cTs9oKjuLkFCJ9JuLsQ73/IlPJXStXXJUJkk3pUYNRlhy9s4xPmfT3rg7wy4TbG+O5/JzzzzWt3UOEawoR3WtKBcmBLHilKqSTI3eMpbPF1lLiaO0q6ZbpJkzmAgjxsDjlo98VVwkJKB1QVyQCjZebC/MdakIAuPlqbgY3c/LwitYUqT01WCmsViYwzsi8XaPc/oAkqgofVRnFAAw8ZnNz0ZNYj/LQKmzeLz5i7DRF+YnHgiLVKhaUwULf0krlaF+WJa31J8F6UYtKS9TJWSFFHGxU0Xb/1Sfcly0M7FK5KWpkvOlX59+T2coUKNnSZ+PjdIoNKp6516Jx2zvRF+fZ4Y9O/UiypWRSupo+bpV11vMZo5BZHms3rVvzEXo+geDyJFTf+PIwZVqZOAybheRRfzEUSYh1vgDbY9tGjv63zB0aftRUkABUfjPKhvsQz27y0bKVBbyhSNw8r5axN9ukZSdzUXkK5jORN1xverDEbGyywLNwnCHpAe+u0T9fGC4jRK6YfWy8fG0ObXHtaZmuA+vt1e60F95R97whKY5J2YOkKfLdFF86oyS0p5FbCSo8CUZ73q5IJAUPQXo5inIzffQqSNMNy/H+hXl27Do9v4Rtfpj+6vuJvVA3c45ixlXDZL2jZMVwXhmYOFCGIlM8MxVpvavsujZcpb3v21wXJlLVr0Tvu3vMOeOvpnLzHJ03QBZOKeowLcr7jdLUlW6Z9FpAqdQescRAR5Vmv+YAkW6MEOyxmX8xuTHl4ZUicm6U2bK8olDmnORGUq7EsioGacoNCvPSljYp4txVH/aw+YT2UNwlkfxsK08sHgAv1QVWzy0ruk7LPVGsUwHL+5RidOlEcV5a/Mxtpl8X4DlzTb+tqZgpcZDdNsAYFvGCouRB9JSDs0yVsY8eoWVd+abxKZYD3asNtaWRxthS6UE9cn6rARG50CkCIymycX08D2mCLDU8gvzBEGE3JimAJe4Cw1/e2iuPurRtiB7Brg6t09aIbVBIBhc4UG0EiBuYHYD+mr37cOLi1D5kPdSH6LvCJ5icsIEXXglhq6PsKseO1HzE4knlrXHbpATnwzbKQe8Alw03W+Q8q8MCGT7A0Me6UEFoOTEXXeNNgoAn1x4xrd4vY6RFujZGmiPadH2xfFlszCRjfIIRRn7zuZBwfhqVVFGemLvF6aoiPqIIB4C0gdFzQIX6cW2FEk97yJdp67NK+Ho4C7uL+7kM9dggOAGrfwRB73pf3lpXY+jONVyTzGciUCPdjL54zJJd42vqbPOyjqR6hLTu+SGyug6DlkPbIpRZ/zM5QIwnwOIYLfPuHpt+xhIOORuCxKLbqUKOy3INTRhWhTfiOMCCn18tsj9iWZ55Hi/tkaoBMYkCKWOZJX1IMzg8SP+zjbaPhtS9urGiRmyEtaVmZwgveDpV/sWrtF43Cr4w/xo24gSCdx1TGEIC+KjrztpWSogqNrMjDF5QimUluIIAkHL2VdCEJQp/zIPoNtVRNC2I5cnrCteqLg5YMIhzYLVZKARRvJUidyAwTCHULlbsP8fHjGHSLXb8EQv1RgyY7YvSj4T0+PwK3vwg1BW7K472k82ve1b8pro5K1GFHYee9zu8b8EGTizenhYVG2vR+Bhba/9nlgWj1AGD4ou3EqoeMN3+YuXcKpFcq8In7oUBYa3vO3xpawhy7UoJ+eOcPrxl0qrQyiCD/pk9Wt/gNpWNLJwkfnSkwIdUu1EP/1XUEOwTkBvfuuEGaHDXXwDXV/CACDPTgm6ado0ZaqhgRj3VxW1Y9+GJAQ3+9nVBNhWFKhXz+i9UMscY1ctXbjA7BftlHeoMabAn2n+OAplOQyNX648iteyrfGkGOEcYjipMf7sK8BZq9DBYzjHA6AYCTFUfzFdBeBLnb7FmW2mlkEc3qRDOPbtYiCZ3fGnD6OFnmR+dOfvDicBc07325jJPoz96bueAOMEJlXMCDhUNR5L9JCw3FGJk400+jKCE6F19O8aXKJqhO+l8mWVirWJ2ckEPaleIoHwCqiRNfwYA0pG9sGpnZlQTewWABi2WD3/Ct2MFHVSzk+EO83I3S5e9RxrnPipX+WEuUdzfn+xzvuCFD+KCVTPM2DXE2wsXDEj4cH9KXby46iCoy8WT6RwsOtdPTAlyh8H7LCh5wjSAFr/2ZktoCVpEzGyx2hRjei1hi3Z/1SNWDrDvFRQi+Lfnl8+ADpPCKNERCXbkYSTzEFVwOxqLzwx8LrFxgt85ApvPsSOOnOGDWN3TZz2aWxTi08lEfuX0YIx/fhpSWLTW++1saN8+yeOtLo0bUUMn1cfE1SSki/TnM7g/pqdByauOu34pmZuTYywatbD63AFNwhRjoWgTUl/Tb/Lai6VBaG/uKdmV/fhtCneHrBlw9WhjlAN6vkymRaQBzU9CuH/DodoquBbZl9SlEXd2T77Wm0UMuJgxjolDkd5NwSd+P7I2P8W7KX2GshpEy/GqOfHqrp56JWjMmj1Rt4GQ7iBZhptP8Q87YBWooCQzuI/rWYIJOyr71mf7LM2qX4el6/6YE56QGY9+NqzVhbWWUgliYPU7wPjt5/RgldLucbEW7phR+WbuydtUXXebXSuoZiYAm7Zke/FVqljU36wtUR07wCIHVyBrBGzJgebhOaKsTWOTvOYrRFa9T9MqqR2KW4FxYARVcnmOoQMOcsQDfrY8VV3BvjEDcVg56CnXi7mSoGGjUs/FNscA00kJpE7csqGwtmb+yAt3cPGdNRIuCxiEAwdnVMrFqERi0gy3m3BRc9T3TP8+00QxYo5YMYVqdauOF825QxSAaayrTvD5dk3lFglWqivY2PmkYUmI69rluSDUZRGnuGO8QmvwDwGYyIt90fYsiPHwzZHoi0a+pg1io4fxPHP/lc/kaWjgNESSpqoPbSyu8Xi+Wqb14swrAALBqnT5OLWCzt9NRPsHfyCNeYAtqJg6xwcWDSS3y5AC91NSjhRdKrSKhYY8LQX5Ld+Oh+78GwD8mMwOjvuvRM6USXdabRFBgvSlDPWLwopWACCNiQr4IvSv5BJvkZgceEqjSgzn41fShGUxxj0MVU2hWf08niDh0U084jPBiHWYqfmkGzRz06BQS9km91CAvLHJXngIq0IrK+5udN3KDslO+p/nwPMVhQGVLmAcDo6YHInAGClstZBoc1yl6d7nPyABpjKQ/Pya9HvoGeMn/kW1w4dHHsQOwONX9muHayDYFSMAb1HQyI/iGBp5hxSAV0+MwlfvkVC1sKJtLSpjxnXXt0aKPCGVm02cM6soUFcmt1wOQaKcZk/FKCVrBCKqMp9tEBtzRY0v3gtPU7lWdy97dfj6SwDSMqS5OF7GbqFbDy9eAQtN+KWl5dI2332vET38Jjg/F6WAD9VYH1bCK2vcxtYcWSjZTge4jDMTDS8npPHzt+LA6X0RNeLYDa53CAQA9rPIOdVxiK/4KZLNmZC+4WjWhViq+vw/Vw/WAjq7A/RQrsF/zg83t6RaHU8LQ1Yc0XeA+qwmaYp7H1nLFwhSVMfPrOF24892GB4htcuAGkT4LT1JqrKwOkqTo6/UHCeagK2ltsKiMes5dLp31951ABthMZmIEXeObrFRXi8XBjacFkGfiAKD8KlDfj0nFlJ+dH2Nk9enc30C7vcbh+UrTUWYC8R2Ee7/ahEOjX0z9RLpWTS5b58+AlDR9ZKvyPpqS/zcewS2wLvDhCRoVbpz62tRKx6PYNz+wACIEtt1iqHVMBXSKBQ4wVu6mXUhDvM2OE8vXrV1iyafTOHHeJbn5vmy157Y17dLrW91WTbHx4BEuBPloMZOb2j3DNHNIBa2wvTp/bFeWRLd72dzw6MoNBFxFTQoZeaADrMlMeF7PZhjzIWYQ21rFU1vJ6hEg9XgOLG9YfsgMU20zeue6aqSrQwmt0Kqkd1eH1Ijiv/G4A6M+5ynwRQUWTNYJApbRSEbs0gbqom7AtT8RW6r4gxaLmBtnPhmF8SKfq761HxbbeywJtWr/NLeICQ0m5RnSloPPiIfpDvWf5kEVWnLBdFI0lJFxiWEDqDqwSMC+K2U0fvNcPhTlYntmXqE2luE1OtFAwFzMBCmfc99wtWgA49DArYKEoLvIeanz6xNSgx0WToFdPtb8dXOs3A/znNCk9LGThPuWcG2T1ZooIzBa/9k2eAE6gAhYsS5r70EmqTCy7F8MMiM5JjIC9RuZ1mDNf5/KS0jXocHfD01ciHgHrlWrmJe3B98i0PBqdZ9MXQrljdUqfGLhk/dI3mxvMY96b6W++YKtcGBHP9nQZj8rKBYvI7hLJYFbPGa6twGW2hR9spHNyiiVwh8YxgbYFigl+oBysWf3gs3XEDZO1TKcop/pmHnvp4D2ZnesZEsf6WGqeu8+J8zRmOUrWDiG4zyF3AM4/MQwJp/RA1/kFT0e08ipRk8OsMVMYKOdfY6R20xsKLTqVMYqTCqicf4A6U3kVot7AW8wqZxakJ+NEQ2zDA4EoUuwmKAjD3lWb+7crKRyDSbXFleLkR9G1qVPuWs1DcqrePJIPgwtl9bby8iaAfsYh3Z9PkQalEkWN+2VgPtJXnMOEYWm7KGd109wNg4Fv85U/DAGawbnvzfErkDTMeWBIr5yAwCe/mmvHfk/Fddhsb5oxw3SS0+t4YWQno3xWREcK40PjfXVTm7ckDpgvpIWyGNGq1WvYWRog2Rt0vKtH/POlvGPrsILshVHQCTSxBXVVlvGkeUBk3m82Wqi/wkZ/zs9ane6ieAgICHsE8zkeTJkgRHpYwjK/DSXn7Mtndc9Wf1q4XVJwL48FKN8t3u4L/RBxtzQB0xGVzzVhgUfcdxgHwiE92uLAVcZlkh7sGF5Hsg4KXdWHaTJxybtAubbD5DyMf6KawaKl6MHYTi0J2RGmxAjzKqm4r9WJu47A/MtlX+70DD69SBQJVYD4WDp90Y9OVPEjuy3Cj1Ltim8uECbcSZRv/KBaelIEwb7mJl7ChQmpOH67emcGwA1C246BC08jX0DO1jT8ukIn9WeD+HY40Q/9GnOQC26+w3RPdgwfTFbiY8yqbLr66Aio/wYMyEZ/RWbwHfbEq8oERnRRpvlbYEmUOwnmUihoegzebiIhr2dmIkGJo/ZmT2yScUw96GcDrkHATOsMKIlzB6YBF1HwWIviMmPLXnMvO7ve5d7Slt6JoY/mXHwvT8//CDYINHThlQOUWAzO+mUlRjwXeFaHwp96XO0TMAv07AyluBW4y+QFfmPxpFDe22cDo/WSVVwLhWzP9Dvz6hxs/clK5oWH9ryTNnPDL+yVxyQPcYUBCmcJdww8mGDjDJz82t6QZJKOfTB4sFiJnTqgPZ9WbkBZjD8rgGxUGTPPj8EQe+NvvwkwVLf+GHaNsg24D1byidFMLBNrJEPKKRMcO3hQG0Lqo5zdFs64EVZGDoNdCxZCKrl5S0CSQ5A9zD4zq+uxC6GiU/Ecg0RNQsaM975yoXoX9ApNGyXNqscjjP/u/Z2CCWtoGZxTxDuDkUHFnY2vnTMP3g4YBiF8gcxubzZdPLJUT0fdT5BzSsSlhHtsDZK8swIl/fK7GDWu8tgtg+nD42Vh7xS9S4+svB9yDSjQEaZrk5PNYj3jwqtAa7IIpVyHPKA2oDVsbcMg5sHZGzy1grp8VI7EApiITemtf72J4yEUtn32LXut/xFrmkRLcr4+L8zB38I4b9zd0r8c1T35bcosdRh0CdiDdyxRxnvviEZxaw/lx1lVA4up/BC5LPT4mfpnSUYuUunBaVeYHzjTCfs7WK9SwJ+tmo3P2F/+6ZuiCNaEfJOoDsPqExgO1JgZdwq2oItI9at68ryR0bNviMcj+4wxccqqYblZgwKgmEsLTEskDVPHajL4wb6wD5jI15I3PwMnHzWQftvoER59h7tRxlo9UALhMyrlYKOLl0Q0VeywJjUC/YWyfBSyBj+7CCyGvEmh4jyO3Lts1s4CZiRxtxemT9+GREP/VCrKV5Q/+Z90X5Hq7bCJ0Zv++o36Wul93feaxsTi1/luOPFnOYvofIsc6IwZuOz0qG+Dz4OXy0LkslC7h9vYou5Eh2eNGB5mOZ0BGtckgUpGCZNVtjs4A6c5wNCyErOGe51GX1dLxVR+rQxtrUky/lyFMf32I0KHrQ7+rTu3wnN/tVdGIbCvWgn0+cjPWLX3URNvZbeNaufeq2NUskKB2fRx34XdokF/ibHKOb/1MCqa0xa4+aKt3W9LCmCPRvJL+dY6fqTyTm+X9ltEywuR/8jaA+eJsyblw7JZxQsGL2Ai4Jp7pKgRI8x1BR40buTinCS4ytze1Fg6QAP5k9nssf3Fni73xIhvVFRDwWrM4g0/XFEtd09x6CPwDdk6nqeL6IPzLJX1kE/8GZPvo7zvd24mTaHui/PupIl1xkEmROunsThPFDdibHDr2hvpzd+j9W7h3Z76EZLhh17yw2M7IOc1aIoQfLtkzCGoYcur/u04Pi0likHsoGz6F5f94X2jqsucmGE4jhK018PL+XCAzVsBFApYFfxtdwQ/JcwvmBtyypHPh7G2A7qr1Fnmz6y27x71e5cR5Q4ZI4Zusq7GJBBYnYcRJ4ruO/bOa5ASUzySy4MRQJnfdK/E/kb0Db6txGbdiG6Lkf661yFo0Mnq2GavBl1KzlqvmMunosd+4Tqs9KLHMCxZiThI32li4Pu5o5bKbVyFPf92mIcZ3Ok11Ir4ZX0G3o0zadjPqnHW+n0reozF9xi1SlB3XzA/ZrEI/OV9wdSNPlLqajpV2r2InIowv3ToDYYvWGdaJKU+fr1GZRCbeRSQSgmyiQAZ8xdQpSkpM03JE9mF01Ptb9G0dhHHDxA//AqUO9iFQ31Q9oVEjTWwD9fqC6PlZ/4yaeid2ULvM7a5fgApelDsbj7synLqkOmuRFeba83U3qDMofdoVVWDThWG9cCRCkeRAfPUCXTdbOVhIvP5wQwuljumeo1cRIKYwdTQ5lGXm0fw/Q/7LNwFdBBw3g9ZVvSOUvgC8XWQvX2ZLZAMzHQ7KgUwYarqGQc5IXR86W/DY+UZLMz7vkJEeacEYri+/IP4n31gnDZah+I0S92DVl7DvaCQ2g9gboCzlhqsLhPbO4EYlfWaoC94sOmv2F9LAfnbY5BsjWDmtd6RXEhKXEOafFmA2Z1gyYbWa/+YZO9tiBwK94lkfxBCa9GokVLFvWBAUq38xqYTt/Gd8kla7vfGTMcPYbVSPg9InGlr3ULGgtioy6Q3NMlH/dLC07x/Oz7c2uSvaJdhBn50DcTxqYdrYJF01jA6dVMFCfgsFoTk3DO8bkKYiTf00lvxxu9To0uRnL4F886ZpCk1n9bB/mIwwA9yc+vuVd6WrLAJ+g7YNFqrknXo0Zln3VjaLwwHsOr5UpVgYdxeg72TfmeBMLiYn2fhvU/m5fdqJi4l1Wr6G5uaz6tPBSz46mtqfYHDfIaMEvrFtq7+uw9KNJ4bL+UVLFevr7mO3V7l/LtSuSyrwsO0F0BDzU90L0DnF9ZjkVJY7B5kXxj9Y6eiUhw+Zgw9VUehNCQut91Rc5gQfK6aYBxQBmBeDSWI1nScEhWG3OaWlzv9NqAgcR4lN1+wXsr1ygdWDoFJxDC6ifdnRsm8rWtJGR/dbbBVsL9Gtna+53vFojK2uKTB0sAR3ePj1MM1Lm4t7pkaMcph/bTL/BFX57d+EwebTovsZrHYK1wk/+POL4sNTcDNXt9+WDhWJVthwfb9I9wMfu+DkBro/H1sAd+/9RjFpGiMyV4MDbbxX8UW9cEp58DliNLmMyEMv+0WQ74toeED+VPr1iLsmKgzLJZ8/ZnMIEVpr53W39Xt4ZPfHKu9sCfH0Vthf9ARBhIuhdrUqTklZsBjPAXRcTsbenfxPPD2eE4Pfbpv1ZJCNZliGC7/aARp0k4Qz0wciMDR0crbag8skuv6l5yp6TfExFF04RDWNv82M9ulPnkhA0XHVivNvUr2PpfHfFIy/aLJPuuBE4btDZbTVQGjZQ6t7Pt7fVNVQbbdG4FFjl6zB4JhLdahEJJcg2exFDIQZdkMfr1WjJcNxRTnK6dlMzN3uG8ngHPTeNwN+q64XMbTCUUnwxclQsj9LC0zyaj4EnA90jMQNnttDRv3Bh0r66KLHyi2XD5xS/+POHb3h/ogVPxwSJNjs/T44xxBH91hiQW13lVDPdpCajqIhSSQleOKbBn477d2jcVshey8T9iZcSQLh0QbLGRcNsSe3t6sEbk/1taq2qNJSB5rnCbAJiuDz7r94oHl/XwRMYxkmtHKku+0lOZ05JRnFCEEGSOp0Df7FrfgDnZPBgK0S7+FQyrqdNESEsw3yPuCyACaGN9FgBdnCdTOpn3HCYUKpB46t/Dht89ApNRKu1Km2Z1TqFltvxXFqWe7N/irjq6vnj+omHwbOpC1aIaHV/e6hXGSlKi6VZdWQ1KTq3PeS4LRLZFNAUpqEXcdj1HJbEgvQ2sBYQPRmlGfN4LxK2k1+q8S4b/LS+SGfDNz7C3z1Lg+2ADXEGyM0XNP6Zk9v1LAi10ujZpiV0pkj8IVDL6R9YtM9BqnrQ9/e50afJxAxT3hsaHOzgPW+IZPl1AFtKJLMLAg9AAacYvcc85bQmafO06leW2E+2+YcufZxi6qOmMRICuYJutSw6sSxP5qOWhfXzpOSrTeWje80DL6JWDKxhhV0gJJrgZ6V9LLIxaI2PQ1CvVeA3lEiic6bv0mXfAOBv6tt2N3CLObK623kU2m30WibKkoOlMWCphDAeGiMWvexkYU7tXi+7AoCHuiEotaG4HtWd/kf5QmbRv+sx7cfdhSRGBtPqVqUO8aiHVetb9z9y5+Au4bwOP8SIr9WVN5IsFuMc61r6hDua6/pT//vNxP2ugY8+gwYEZYd5693i97Ht4a8wPyD/ywnkuekcAx1jpfks/i9BWfPSxX6WHRWKJpmoQfQ1m3YDHBzK4kixagUNVFkxni5trw6tI8aNPKc/h6r+qa3tADcaP+dfY9XSo9C1eHXvK045Bc+HwG9abvICzgRMXNGBtcSMd154FWmO/sb1MvToNx6z5H2xuHwHqkDiePq2st00/NzZwaCw/4GJ6ePpSS3DS+vzQfoJi61EK7L/wa2BlE+B7jOwD94F7IYyQ1IOPJDZCqfPC4pWzZw4CJeMgQe9w10qHmp5A9XTDqsnvZOT7OTuPEMdquihb3mjJNPYdhOprFHsZ2lRiRgnDLOR+8+YfjX1sJT7GHe3g6P9YCOUPq94ClO8oW9SM4OrYEqxaXhte9thMTmRzfXfH18xzvTRVo4gGKFs0gUm+9H9s5l4NNGEKo+PzSY0x73k4Vye23FA/VPtsc+f2eX/8Eexv4OPUwjPRANZBrs1/Jx2zud7jv/cqOTONEGVGMOFT2nzRpq1YMVJU+tTynHJIOvSV9+PNcv4bWzNuHcNie0U8qJfvCG2ziSOQLilwR/SBjUYXcT4UD4MZpvvPeo6SYLgNlcaQCmzJ2Ge6Lrc1nieEefYCyeykZndHlzcKJ442VPJSVl/1MkWQaoTgCS5Yb82PASo6V0q5Q45KynjRNfv5CvtFDuyiqikd9e4WSCibeBKfPYM3bdU+eo+YRUTUB8c5uUUhUaIfV67Qseia3X4LTReF5nt0x2KZ0E0oNBaLCClTdFt7tmH0a31/LPzq3PUBVEUl8FGTxGsFzfhXn5NQsG2c94w3B9meWBdJB9dkhV5okvwDJrGuffFE70+0FDDyyPUWVkleplV3PBRQ8RuaqB4+O0Blz0oELOl2pgZvARgHvDX+m63VXDzLnuqsDH8ZEy4eVg3gD2qIDSIixvTyVro6+txR9Q5M6UpO05PqN5NLT8aFovXDXfuUfRDn+4aQ08ZZxdes8MmD2o9gMDpKDp2EWC4pPayPaOnq3z5owia0QrIfDkdH2yFuXM6Pdpa65YTP5vY4GLxicuzS6dLLLVFQqtowttHo7xibnt8/BQMQ3E3VvDp4N9NOqj8phmg854MBN4eWYOX2bB+bk26/Km6cWBEv3HKI+iXdSOrTxnM1XuNBsg8hH5wE8fcCn+fzwasbfJ0N/66se7SAJA3pHX5TCHnqYhhRis0sV5d/311uqLutB9h803Q9RTiWpzh5783PtwAq5/7gPB5mSfqWVy7fEfr8+Bb9Czlqiy+9x5nAlq84Ec+m+wVmVA3Pl8hLXVUahaBEZQGZANvPMoNGnvHL8hs+JA6PNTqn0Ua+3IvCn4AidilIg7zmyr9KjGnYlHpxvJDc/jVdUHm6Jy/w2IQdJNxWwIn7e4TQpYX1keGBhDZz/Ptx3TrofLK2dgkKBK2garuJXmIR3X/xL/nAT7uJDft43ZFWt5lK858S8kRnabJJ3+13iYTaJ+z4xxDTn4k0AMxJfl7wBO4SKW0bBX/TtD8FjCioryqf6Q4YbAT2Ze2j06a0S3lceqaFtGBpYkfQ2u1Zx1bsq1b98DxA6UEngFVwbKv9JHGAFUIQkmGNPquB9FuTltOUu8IdBu6wMfYvIwxRgScPACUX0zCIylHha97W9cMgXy/i0ESkZPVsrJiZNlaCbE9WMDfH0jzBcdYV7eYIMkIdLvBKUKmQxQ01GJ5sh3U5RzJpYw8FuWdKrma36av9Eo07x68PKqq2gbYZAOv4NnA7pFlQnFDBcOyomjKuKu7pE5315x+5eRuMC5y6bbeJjk31C5Qrpt4Hmy4xb8J1O/SjoqTOMe2dj57fCazaTAqOtTZWUccW5/Ou7HL8ozVkeWmvVFSlKlTlN70KOK5jkLozUliBv9gvV0EAZluuhyG9fp8T44utp8kyTkJcxSDQ5dra5IXvz8EQMfcZl6Kc+aPHksF5K0JpOdgafSrqQhqORbgn2uA/8byKdZHiTrFTcNIVWEoRafgZJhkXIWT0YRp4b17C9FZvgbBqhhkbIxRtUlgcMZfjwlva9JmDRB5i3265AybfmZ1z+SH5fH7kCWXiNvwZRh3/9ZOP19BrwhyMtyzfPrneTLwWH3aqTueUgJCFR5RnqTLhmgBh8XZxLR8IuTgXHiJ/Vx6b5orG4nDara2/pa1E8HYVXcbjWD/AOvYeVOQ1OcvxjanbF8VqveY0ufskyecr2yg9vFLEKYIj5x6OtO0Cud4c/qeaECyhFEuVbIz7Bo74jVLFV5e89y3dxBa+ok4KD4/5LW8fBbjb7wyRqO7X+PRW01jVY8R1gynrR4J62lUyHVe3A0d7lXlo0eb5Jd52PByGt36fnK5m9u+MYBn+a+wFS3chh/KuVrSuvMH+hTzY8xu7uOeAXWc0lnwrcdko8d0UfGoCan/MzlKMEwe2uxtRvpCEqxQM7i30n21Uf1069zs68gFtW8oNBpJFQh4/m0+t2BoNNhxqv9htdn6WX/jWBj5S5s/F79eIdvT+V8d1tTEvt7sQg9TawuY2XRUJWiP/SS/N6sBSZ0SDrue8QZXFuhH5yuIvNyyRSfOqXdnYZNviBtEmeLwZrywv68FZRsALDwyoVKQptgIaJLqDvSe2WRo3UW9P+FMkRizv7ll+SuBI7Sx8ZOLo8Ti0amYNPHWofPpUbZ0ZHfQa3I13k5MKhu0K5sadLhWHymujfPvGij8XfO4+jcSOShVJPryq8l/hJZWoPdvqhizpxFvAHb3bkNC9Nu/MgROUQlQwBjlruA3aTYmmiEkEARX20hexke/iKOVJxB8o3+t+aldZemlGbdxXYGxRLPHhQJeXBp26HYkZmmkvxGn6rbr42ejgDU1rnp1vs76tkJEpfvkBDmMOFgHUuJ8gL4THdu2Nk4AEEylUqqbX9PLuwCshO4hLCwJpQkQWrZHCfYOxSdnSZWujtnjQQ06zkYEgHpFZczFo8cO4B1tlI48UwHss6STQD1Rkd6tFPxC8oSogt+1r+6homab5f/AjfNWgNt6W/mnUkPXsLi5owKjTk196LGG8kBwZBuZs00yWNQgOWnBZkebDB1DouhCBCPS8JRQn9LEiYlF0LgzaozSGz78Xza4SOp6kaDIX7OnLDBbwpUrB35Ds8qsjeaY+2C/1zyHG9ARZFwVwZEmE45mpYEPWu8CBqD5x5QNd4lC18sW6EH/FJ8Jj5mnJidrajYYNos2bA1odYwTPM1FzyN1IHnH35gQJ23TPDA/ThXIfWvkjWacHMIxjnMDNSN58BEmXKghviRUMehKa1+gDprsjy/o7rhrmZCzu5jfnjfxUpUiI/LMYPRzt746ri8meF+pj/krqR4NzDVZ43GlMI9F2bsGoWpYNdfN1J0dhAk21c7McxJoH44d+YUU/787oIi3N8+Kg3mqKjyWfFKab5h26GkCxfkuuAnqi+CxRi44Epj6XFJJ7wMuEl/0CY5+V2X4rpPTATxnNMuwfRh0KOpAWogQqYTtKSUVPy24Jdl6Wb+9B5PVH9SD1NHtSnExHZKmN0B1HvY2URalGKQEtpfgamk5Qs76GXmfaIEnb4HkT8AJulGhrFJGQZLNKjHF5bQJnMKdie0x46ygNwUiNEhrwVJu8hrUlARFmQpm/hNqhqJCae1NSDCSjvozQkLgMh99ntzrvKnXR371fQpMGuBSQ/KtEqXUR+ZMluKcMjdDnglvE72F+8psi9jJWf7DIOAep2BCvGtDhZlK4ujrDTlDluqe5U4cfbyAsFc4LPZSTjFkGzi12jQrwHnMBPgMUo5Q+bLN769c4gSPtELPXwKMK6gYnJVmFjPbrqDOrG3cHZ87dVHFkWK/+dkhqlLgwr0wlO5kWtcL0r6STaZj5NQiAa1sRP5yIexj+MbZC4XQG22os1b6QmBILGPwrzOvQvYPEdWuV2DCK12OUDkYYfQQUO/mHfiQsp3nQESO17RfGY4JfXy/5U+EsybD3bdw3oJ0rODAjiuc/GpfGqnNwVVaM9NR7/An+GSTn78OggvdW7UUT4wuZqg75+8nVYXECEV1Dr3cNT9QfUyPbkeYmfMee3kKDhCWlPzn2xJ4Jn3+18MmsJGm2C9DH6c+c6ZYEMUXoFL1ClFOwWhuyrMqPj+alU4EXWPTUKNk2HHbqMgfwyN/8rNMO469v1fArl4GMu/OP76pUHmIYvZNB5NToJ20so23KvcVRpDoFpo/g8U9eluAoEE36VhmcZvOHTT2ADMU3HG4Fsdf565KfC2JGchbdkexUzgSV+WPO1iUDQ9101TAuiU0xwXVghlNE8UQ1inBYA5IvDkHgo9n3JhEgbatocPLMPLA4ELoYdKBGpHo05iGqrQeRBvIkXXf+qtoj3/QiMbySspSJ7SOGSjtCNB0W2QUbI+IVa4URPcd4JubBhoec7Bm4ojgBuFriIzof/CqShJGSRGQLPO60g+NbhXXb1vv0X2VBgla8o3ZFpHcNjaZfR0c1QeMgSBXSM90ELk35s8gC4D/8sJ2I3rei716/K2Y3ecMX6gFR7GXPwJkmeohYDY4qjx4a+UIGoeycNa/iMMQ+F1O3+wKNqdkN9nihOQwhwQVLe+72xdlUqk/EXsxksL1NZ50uiDO+8cm1herm+eKpdJBx6lDM+ig0u0KzF876IBh1RfNGLpih2Mj8QPDzW+DxRr0PW+BZIVvAJXY8WdibetmLxtbJ3Iie6gNUsgpwAUSK9cMDy8Fd/F3xrAfd5LKS7+BMOZGmBgIw1cH+plzvtkNkzQEBsXqWT44jRKuOe7pmJwY8IvHXvaNQpZ+LBUsA4e+XbAOoEQXSDLM6uZJsgXNjO/PzIDWuQj8EBXA6sxMVjbnmP5J+8Zcn5r6TH4r/sRTK83SzsDaU0HURr+KVX/DlEVl/zJDKot4Mits9sQ/DEaehv7+a3mAuLLf2QUuzvi9lAiP8ZpI22bUntKs/H+AUkNwzWBDKEHngP/bXK1o95NhrCPPoHcbaY4Y575lc88ZTxwbzAQkzchvpX2XcVTZu//eISoKZ/K4TcltB+iQaAOURp8tKSLYXE4TE86rXyQEKwipRmgCvM5xPmz0syHJ6RV3RFZ+OCXXzEKod+1Z5w84/UcVlEGiMHnhPWdnR3hB0eokGmGcoPza2zwdNtFhpODXSD2AVg5xnQiPA4FZymgXV14pdc/tQMB19u+NXu42F3g4+DIV1JhB5TSt3cHB5nAbYhmbDb+XORNGr/HzRdx5bjug78JeWwVLRk5Rx2ylbO8euf2HPfas7p6bYskihUASAwaTmclWli5jeGAKhJtZd6MCx9rkdDgLAmy6ciyrdb9O7Fp5hY1ebv6hWtffUyceJBlMk5sH3ORAf52F+5rJDPvazsqyI1sB4A1ohOtSgV/WBa/wUN2juTWvr2N/xIaVhpiv3AZmnfSn38NLfYHhN866uXgxBMsmU5eIz+Mtp5YkcT95O8b1GVEqX7LnzPvcAs5i825hM2wT5zW9RXzHodiNTEFpXEfN0tmKSAmBE/ZGutu4pwaVUV7Uaz6mqzarkUBVEvm3qMLIl9FbHi+V0rUS+tyAKXYuWp0FoP+kFp+4FXO01+XD4kxm47W8jWVMI9I4jWiawsfM9oji9SeKlhSWI0yf8dy4LLWEf5tzuNiOL+taMljoPKiMYIsc/SB1xiWBeBHaWVLiCmcVa/L8v+JGqyHG5BqQLhHQjbJK05QkAaXryQfz/bAzF+Vn1g5IOhg8tV24S7E0UYPobXZVGFAPq3mxNc8YNuBLute+utBiHWSEXnpzL1cGYRJKRnBNh84kNXiCDRrNPy/MnTOnHXPv+5vchFSqT5A9VYodiHyYluzdC0UcfnMEbnl4bruX7aVBs8PZdwnVXcWy5vcKztb1OEI8vzUlEGv0OyDebrpdLs7ehUymimgTjSNShcjE/WMVL5g57fMKYx/pWMfy3pgn6OoNuYUde9kfzRfBKVsi0WlAn/a8YfhpftQZwkFcmXYiKJcmAATZSzLOiqp3wvp1yoSg//1V4Nx/54i6Jz2HTR0ZBmwwWslIZCmTDbKP5xpmzHCJc4EUsXbC7J4L/fB5PxbqPLaGLwhkSoy8fjDecBR43x0EZLtRdnHfSYov3Nl27BJCyxyFH3z50AK1agYSGxk4zBIuKoEg/dR5sicVO+l8rsqXu9DBEoQkokpX+GLjJOnmse1w30AFAnDwM02sjg6TVg7t3Mfi0uDk+jMEPrcrgkUzheRpseAC5FUJMZsgZlsZ679cz3thNyAy0x2F0vwUuZcXy9vP1CnYzq/IW4osj4qF+fvDc9PAqb3eWsVjweU1IfBbnBJA2xVxwt1FewYD3S5FYfCu8qEqCA9jA8IAPXgVe9D/DrWfnxOr4S1Jrngo88ygUb6dNMpIhpXjSh7odp/s2qCPzkiRcCBf51GwVpQ546eXljBh9NpljCkX3sg9f6Yj8fpb+hdTqjHbDp0oS343wlSzHy3rtwEGbbisIvj2SOpTk/jgqf7AsZ51zsdQZKe2sbACgPS/LJZyuvPvWWNsjfaD2hELVFlXN3eAX+K2yvkuQPCnR6F+NXOSIasFfUCCX+I2QYCcLgV6fwyZGF1NY8u0dXKOhOzxoic08N+joaPvhDMDK3YSRFZmkhubk6np7bDMuQXk6xeZqmTcnnt7qgWTlobE25NAinGh+chsGNQ7brQETRvt9Vy/zUbNYf8tFpwRTb3n2ZbobZDGJ5RhDPIPYqzuk11nuDFsV9zqJzlaa6AWvARnUgHwKRrm1l7gjEnPpyLkri6W8cFXpG3nRGwMHRUa0fHmaz+lu4gFcWE+XRHIdpqVn/BSy+xWlXZXXhso9mQ0KAuMTqFqpdxps9YJM74rDlRo5itRdNvZ7H9KVH+nlztQUY/tkuIyFMwE0GYremmLFIp3+xJAFh+ef59TQjS/WDqCToBCNKe01TvJEKnHlhbsHNgzf8zVcHdmSVRvPKkppapws5H+kbjXpDOF7LKXPRVYj9EyAvV03Y1CmjA5QDU9RqZ93BYqHABMm/vuISxstDAaoIdb09zu4jD5zfrah3wGKwH2z91s/WxUV2xJN1Djf0j9y8chQ6GfH3yeNXKqzmd1AVyZdwKjopAbCGx0uahxJN2rP2KVoAW+EMPge50TEK1nKG5E1QZdza2ImEMyYKOOeSLssFp+tIcKBQ3L+LnULK6BcCRXW6akic/M6zvVnfkxVeMF/mHaa7hB/RdwwtJZGyAtydE6sXnwuYXSLWGewq8PPprxbiAKmqHfCG6vuJK5Y2n+3H1NEItjckKsrYHFo3A184Ui2KvJU6KV8EBxkU1IZq3PYhuKbJrljBYerwxQGR6TJpN2JTV2KTiT/fhivuXLxc19GC0A472csGffy8Fgt0QVNkxff6nqI8O+vxN7FiaI6oREuGcivBByhFtOpL0RLxBypbI+dmYcCprZoKLjKzHsT00OPcozUau1LYykkmguKY6Ji/1VARGXgm3lULvrbTzkcY/ICrEJPuofsAqUc34XBzDIJftxChrYl/m6U5h6ckV69NqImdy0eFEJltTGtclufP9/E15OdaWmlyFX27ESFO8QTfnQYku6w+x3f5zCM4O6uG+US/1Ijq7Jx9JJ/u49asRzMpDriMECvy70Kh4jWbX6U07S5EUkU1+UoDl1mVw8j+iI9xE1vCZiyCzf70+qy/yc4qPr3csPKW+eVLT2aQ8nBZdtWm/A6D6ZPsWF6Be8CNaiTqUv/NWFhPLAfC54pcm6YS80y6rRddqAiMCeSwlx82f4ulY9nxl6TmnKBhUyB68tlnmViRrO0X0Y6bkUOrByU3U+pkZw1mNScKsKyUNJjELhERY+gFpMhAHdIX8OuNhZTl342wC6Z+eP0973u/5yWHtp594ud91KWo08GUHcNPxZO00ydW4H84A2EQeWimYjiP4HTpyQU1OanhcW3AbvHU/mXup5mGDqL4rYTRrSMTPT+0U4wi1cl3k+X9FRAIhgErp9mR2oEY014aD07sxUUFfwPaAImIPFUSBtbHYiUCk0XFQk2Hy8eARy5ITSj32Fj2KOExQfm7I83c6fyZjireGtcf0+QSU/dboEZekPvEKuQyqNvZXkn5tSThyg45KrUexbbqr4TK0mBK6lvgLqLczAbvF4QKBNXt8En7Ot+I/v7EXlUz984KQqh3ph0BSpHo3t+dTso7jXgM9rqRyaoQpRAzm78Pdpmoq0akmnoFeMgNJldz+aLcZTCslw53tJU/2zkYPlMj5zxLbvj1dYSET62rgHfTH8omUuprwmIqE1b2MpTwplonKOpWs3BFjPHW6XrRMssLe7ivKKa6ng2ISkm3gd9GqsJMeAAoZ287sG53g4bsB0tir+OB8f1UKX60X7e0v6kBHxaSxwxM6Ne8hwuTNrEZWW6HYPbHqXZ7Jvg20o4TD8OypgAhMuFSlsJNVC/Pgnz0r1UCbqhfUiaXpId+H7SagrpRNCdtfA1zHnCScKcifpNmK/jL6lBp5a4/hg3+65uK3w8ziUq8yTA1Hh+pakv4L04ja05pkPt1p9+dDcFfkAZeq8xdv759eZmUMPxr5suCoyq8HDzUiYgCCjtAc0gY1RVOTmRTPa6+9oBk1r9Y6biVqs81+I1DBMgt1UTfEb0jouzsjxyCmYZRoxbSEJ9LZ2IJumymq3cMN5RgHNSzFxO5GB9in1YyIBq2Zckw6xylOM4Gu1qKLp9ubSrGznzifbR6npVCw6A6EyiBu2z9H5n8xgCHYb+0sr+7p/+EaHYccl2cqThltnwPkE9AEA4iDCAiQqtzozABJakGmekOiv3dOXfKoEETvGXUrwuDvVjq4Tz48rvOmcmUESJ4B2LGzsuso5UeTGxGpgdn8UBSQQYr8ZQt3T70foKI6kokg/x0sRFMJf/DchOz1QqWNT1iZrT6rdm964AfNTrJZt7VZq3feVlLWvJH8naeoY8wOmbxJVqDQIcVPJbDClO47qGs+2+fNr28EoO4JNZIr0e+PjWc/1TZq8pO/RE7AN7XZql+gT/4F+HHv2vyrE6T1HoejoUJF0WZ1xTar6wDwYaeYVaBn4xwqZbKw8D57E1Nfx5+0vC1AOYKsTw4JNQpyN+CDZGXQHgCIDUM+LHQlRf1/HV1ibochKLqw7C3le27qc1O7HPR6vi5Jp6mmFHxdfeuB5KIqlnIo3Y2tip1wHFbuKtTs9z8u3EK6XyO6mld2Hhrg+kNzz72t4q4hNpZj9saFliKQ70U0tl0zXc1PTFrwLzogvzBGP+J1Dn/qsTiNFpYGQV+0NcTUxDWM6ahwa80QHsnpfY/2aYVEVA4P8drphgmvNcrIz8T03CY1bxW6as5Mj0zEnnjySuQdZsD7ZOXT3Bfjjf8jeCcH9xOU6urZc+f/27v7p57pNwQYjUD/bVdSZy7/shCsz9JraBLQL0kyL6q/u/cgKO7/33UHo+mSJpHvBOUYzQBvCxTwfnHjRZYepFqm64c3Cjruk6m9OfxsfHL2AfrnP2kiGD5hAN9JXtyvvx22IBVUKv6yur+L6winRJiFNsnZNeN92AMoiUUOoj2LznlvyftT7F195qaRHNPv/Dvvjq63C5EQxdGkFmuVt6NVMKKFjRrtTeP0h+FRDBtzie1Zle8PKCCjQ1viq1QsuBXfV5kvj+DJyHz91WVljEMNbTDU2DxXSbepslRh0nWfl0bG+xjyp9eAIwkt2B5mDvqfTFJ/ovhNytQEwSwod/1e9/62U0Ba9sM7/iRylAg4hIuY14YNNvJdbzutJ5gjfmz0bVsasvSlMv3WLskVpSpUhPPv9V/Pc+UEAOL4WeszKXeZ7y02D6gGwvyqngx3+lOVmBiLVHPf4la7ji+qAYvJzUfxcApspyYWZgSvElLeGmPr26TsZuv3/qv3ntZg/Bh3Vcx/sZeO16CYHEdZGjQxDHONsANgU1fAt7kBqP67EuT+PCI+ToPwOeLJqRu3Rc7u+Sb9PWuTVwQ6QMeMiAAxy7k1Fmxxx8VifxdkcZOAWNxay1j5aEPx2z3/KAeqZ9sbj0++YKfQniAD4aK8zfs/KZiSt1RS09OniTnSFqfMyUsyqsEnp3ApZfE+wEKebdPhM7Og1VhcUHro6PHfuTHtgKZO8g/Rw3wucVGOvxG3SiGJqm40E8OTxwqxJScpF8j6hRZjKr05br18JwhD314e8UrXpFkk9mXNRqVXic7WgxST2zQBGq5M1xcQqAg/7okKGQkDfvBU9hS8HQEpqIDLzTv/czcYsFF+oxzPJY2UJCIlKEKmKntXAz/TpL90RYxV5UoAa/BKt8qTkQ5K/7SdsgUnfIZnu4L2WEj7PII1DbcPbqXZM6Z8iWaIyb65QnpQozyBz+2Aej84YJ2OeLCWn7L91cJoNqvR/wh7i6BLX5K5U9InBvEypUEhsmLE6nSf3MwfaLBg/qXtP61IoV1MJ9uKwp+tSaKQgzS+xqa6lRfSB6XHs1PYp021IByuZSv5fVwsGeeszQ44PEr0tJHSfvN5eOzGqLPLjQq4JisbVZfobgmD+x5EGaFcfuaUUb6HZy4GnL8WUMJXstxrowfu+NIjzlI7cSwVuMd0y2qWFJP4TY9TOiB+mT21PM1KanbZQSRIlWWwGL2SISoskqAD+YdVO621CUQgyMFbBK/HtX1uyfIldf4J7TrBzEc7clTFxP+ZPgLYccDiSErgZ6jIAHHGqyguvMTSGGH4wyOgC9flSmm0OEOv5+oUAkd0LmN+DVNdv67SgJZOnf+Bf1JxCV/LU27D15Xb+DpxMxSehZtFNPeSlTjZ/vKjWFZv0CkFnKRdbaM01u5TUUNChiL9NRZWJT4zQVlm8ZBs3rwfHz5wHKYMAmuytFUYe6Xft9X7esCtaQkRM5m93H0psFR5IvHys/fYnvsy8hoCn2ZCFx3wY5AZVlfhkre/T/0+WgrFDPn13+ZsWReShH5HI771V6jP4JZJMgwZoIkeByVjPEIT8UeIs4TcasRwCnyaHsPPMeeJce46/aI/G5FCd0A6MVm86JMz5+jJnA+Xhmk2U/iB2IC9K9RbbjdSiYXUQUS2+veNIjOl2svsKZ4H8hlLB8daHyTlCQDbNWB4qZOqQKCK1Hg1SxbPTUHDMWcreg0fz4qkTyIjYD3K6VXAlHiGEqhbE/WYR267NA/7zqi/PufUww+xAzCZqoSeAwFQUxUt74NLzcl9P4MwVhfzrVc9sajz4Kh+6hkMKVorwwhZB/OXqKD1EeJzmB/EVu8Qjg1sE7uo6QlUEBvfOSucWksyBAhj7Zsv35+1rDnhnnRIDLvrO2cSJrdfHtU1XWrB/bmPZ92NvXSlyf8m2Uu8SnxKzDCNiP7aajYWIoyHbh4ckkbqt8Kv7AEwRRMrjujCRNsgpkHO3JsiZ2kmYDS7Uk2GV2wT4aegjYVvWoFN8ZBpKJZrdiyosJ6FkP89bnhiOwaEk2Sqm8KQnWQXjQLUsV8C4UmoHJ4wThS9YGSHaf5nWUN5ge5wxzt4agyqrkAgPykemRhDqDqWfghuyOiT+AXmnxcgL0KA3HvCLH5wdRFyh+/2sYpOeFsaT9j0g47EYb7/fvWkAwmiYqcYk46gY/jEGEzk2Dn11O6rEFMR2NHo9spFUI4Y1oWJVf/xCC64wZpiiFPED9IaZQnQIW4zBCdFE58JW+nW7Ls0b9zf48lH129kzt26duT9W0X4tvZ6eGNCSO0v1vd7B5b7IwuWgsgPR/r587WPKEQd9zW0QNCA5iO7U4XQKI9/5RuBouA0gBEKBbHt/YqGQ1CGYrkL5RpIVS9ld9gmudFYALZrHQBJfGP8eb7FZq7Ezrcq0RnO2X0k4O6A8UKYkExLv1dxJBM/Gut1AekhQSs1MC6gjxjyQ5/1PXlHplXmhSSw3A3BDYIVmFPPJvYnSPlg6jH6xbYZ6UpWeSp98CTIwni2GkQ7sh2FT5D+As30aSp6uHfNefg10Zf619NanEd8fFSqg9E/k3Co2+N7Y+LwvKqw5nJ0GzWVOg7lPKboxNEz86ZqMnIX7ZnvSIvRZNPPf+eX4q3n9wNvYYQh4i18OuGEEtZAiDlmV+TwacCcSJtX4xpT05tN+Kncf/6y9WD/gsGgJDHy9tuXEHQ/ffXwJcvDThSgdWdf16i6JOkkVGI4wiadsvrfEnniTIKIj29ETz4Za4WQVJ67JSTlAz731hPXVUyZIMU4M+JAOp9r/2pXWaH1V645k3G20mGICYg8ZlhRjhmLvSK3Jx6fk0sfWCJXghQcZJBITljywtnIGCEA/4Td5Mv5YYwZFt94YD4g1sFvlkyLOeY9RVrgeh0B40sXzo2qP77ApVdrAixxsiEZtEXUP/ytxsELPcbLBnVkiWhRN+fi1cUaqAjYeb+yigARc0Q4ZZGdr4rs6QXhfY/V78t45wdZUnaFChjKMLbbV/9f/u1DV4pXGHN8Z5xmeeeFpXk1Fwhll/vBWSWUxvMp0RqlPTVe1qS1vjQazLrSvUd+ZnJfoz9YaD1aLnLh7XY1CFySz9BLacU5HGuoDvS6dNNrpLvsS2OhCb+Ggzqat6cF5NXcV1WoHBsnIs13L/4Dspa2KRcb3HaMfB106/5GU0qQF+4B3lApXgppsGGrgOP6AC5fY3Dn62BfLEox2Jya0OSxu45SMZFjHq1+4C4wRKmljC/Jk8BnyTSmRMxGeUQGp7tHDCqLQIzcMXx+mYQYTNT8SfKb+Wvurmn2y/uZjH1iuCIFkdyJ7zv4DzQZu3+Ys8OxraliwtjLr02KpWLgDPK+JA6ej+jInbXS66aJFseX8PeQw0T7Sv25fmid7J/YT+HcxqDt/6v49RNlnIKUpMx3uoxOYc5n+gF13w90wQyd6p7/t1Q5Gc7OCUjJ0g6fFtnFgP9Z8SSkH7e/SSUcTmBWsldc5IkR6SfMlh5fbHQcX4Rc69wzw8oUFgB4o0Vobn4dzk+9tnJMyiMFygn20S2P9Grf33BRYQDGR1lpYPoMvOtxKQoJyTaEuAcfm5s0QJyp0tQLK4Hwww7zfwLp5IS2Hbc4RPbek6s711mySkksW3UZyxm/Ep5F9SJGKfncWDG4AewBQZwchKYdMJCgN1LKbyoy+FezoZ9Aknm2lf6gLO+Jv117Nnffcfxuosoexm/Ebsv2i0XTfEFWYXhrOzYKgd7gKNpsEjDCuWxCAf1TobduCOPhhdHdKRlZrSMsPoM3POatR4okMdiqBmLbsNg/CkrTS2dj55D6rzZIEAnIgtRBbL4B5Yo3i85+hlAhirqwzml8kb8TUuL/PZZ+RC81mNhca+KpnlXXQ2WgE4H7+oyZ7ZSnHGCULpBdGtx0vXVHUmsNe498dSKsmthRC0EcJfnSuy61iZKGqf8EVbACWw7fZIurLmKPRVfRSjEMwQ2hBgvfS20giowAqKWi5qvLAyXCl28jIn3zubSM7WcW/t2vk57w92rHvqKZoT7IODdlWUsfcHEQmHuZswYqMMKmAQtsPhcgAsQ0KZonPmSlXg9110/iW4DafAQOEEFwbLC6bQHXhEyZqcPxE3g9ryXoYe2uQxTCB9oeA4KQCs4QDlIuTSGWn+gFFlI+ubBnBORcrE2hYn5s3OD1eV0gGgbW/z8BB0b8DCHVsiSyMyud1Hhp6b5K8HgZfob/Hc49YZPy1Oski5eJ0WVbhztFKdmWbOT+zmlBj5/q+aVtl+Pn9MvcTy//KCvAnERZyJ9W8mms/+ULabrXg/pum2dt7JRQh93nrtgtwREmBIJfJQqV+jRc9ue0jB+2TbrQXXYmMSmDWYDiFG3IJygqH7AD9MR19eqqPTL2hzIKZfHjdpgTHYWR3I1fLkPoXPkd5uAL5KOSLCVeYT6GwkT8UqwPy/Umv0Pz0k7FApBwYOT8GEraRQxqwXfbWyUGwm1OI1ml3C0kr/M/Qdw/oftNF5pPvzLKqYHr2JamBb8egbPxI+uq35KRPDEveDR92SMGqOerqVVlEfNS4Jia6PpjwVTKD2Q61+zEhD9zIPKJF8hkPZG2N1dq7jY/sxJXc09yqleTc358RNme9KOXdcVyFieXk5YEC7PV3krh2CvoN/DVCS1DsukkkY1MwFNBw1erig03Q/rZzSDmCMJVPHPJBYQW7pHc1XWJVx/s2rfFoJwnYC7YTR67u5yPrZmP5uOx1pZu7n2ab8iapxgtfSrfElZdMZ+zUHBPEtP3+8rixoivyGVyXXNDMu/43EsyFX5r4ZHdyzXzQHvfQw4RCUcAkTf+OA3YAClf6Qn9DlTf0C+n92q+a9qBeWW4m6idfvcSxnRX0AH2JMFWI11+QWYG9XchGxWYyNiUgO6cIlZjhwb+60YszK9cLl3/075GzzEPLeviprFQ591CdRL3JpMFIt71viifBU7py9oMjsQQYVU+jTY0av6PdSH+J7uhizgHBdqMp7uJfQTD1dCdYmb6AgLTKl0IpQ/YGltOIrS/Myn4uaTBYcmo6Gd7xcrO1EK4AEmvQvBz1e3N3Tblz61QbhDKgKrt3j6W117z8GRsOmJX4pNW1M2+Jzf0hiJOrz06PX4gPdssfHKRdIzddWUu/ykTO9REWYdkp342YL9arT39eRnRaMOXoAdC0WHaNRf+tLvtaadC7V/CpyV9y3YHKjHfcPClL+ybKcH/qOMUC+JmG7iEI7a/0akai76ioPSzdxK7GuybOKQ5OsaaAxQQ/E3vpqj0JZCI40g6HziE3pNMZIyjGmuRK3oROP7AuT4F21andgzNi0UIE15cYksDwF4wQTNQg+aIp/FSoMK0s/vmKW6gC0MKIGD92Wh/HsLKFtY6vLy/v3y9jfqgrKejO8yMwNQxw4K6UF8JGtn/BqxcEJOFycFRckDtwe1Q2Iqwlq5gqScwAULhu9khL973Ix51JIB8J3cF8yxE+9vjqFELqrhX7ZfpQNiCnb4Rqhfxi073URSlYkICQlcv2yb/3OZu6Ef/zjHv767jxgtywal8af7yepKPkC9Qd1iSX1b48zv6JLngMvxbMCxzvorsv/i8SQtUI9R/sB1pRn0inzJXdlcivY7SDqGEbWfZKqW1G6PZBTU4mzjbYTk/a1yEdKymS4lcU2TuzMO7LrhmcyL3F+ZF+F+0szmFRQvkbqp13oTztahwUbbLuXvizEhvztFgluJFZGrX+yKWk0C2q2EvMWRCFq/OrHs7yb4+JU8zi+1Ls0h7djOGdcXqIARHDuLdbrIIlbpbci3u3PatcrrW6BaDi+hh9GnShMgH2Tfnjl0pVubLy+65ONzDdyalpTEh2swqZUFBKK0yk8w7dleGocVgdiAeLIEEyT8g2se1IFvt5V0Fpm6IeTsPSNg537ev/gMv/40cK1VTCRjHAMtgHNzUYb6SlOnGsnsyEsQ10jIkrxj6Jhxi0cXejNt70NtOdXDHVuF7PuOPSx3n9nWFOgw4zXwylicXahM/Rmr/bbnh/ucQAw1VJvCTLzKl6Td254v/OHZWTFb5S/QNzL/Jps7yJarC4iprqV2iEvekEdO5T26n19sk5KtN6BDjT5sWy0RbQq0T0uPHLJz7KbHRerAgwk45ge4UjDryxXSYd/R/VXu8PSteNTK9xjcBGUh19hdZdq6Jv0FvEacQCiKH+ouTuL8a4k06DUrEdQ0z2T5qHJ6viqp8UNfr+0/zi8Wp4aBKgPP2BNqmbx0yihvBYnp5a9vegodVe4UnCwfx8Kt3t7un+S+F4cyumXOQjd8je5a7hTv7EQwBrTjd+TqyD07Uq4/NWLP24W+EfcBe5bEw7Qnr9ihMD6C4CQrjIq17prtsdXpAtguTxU3B8sfxNrGp1h71Fwfng6XUJrFLgdiscKtdNX97YAWkWbgzp8fsJFGZX+W0OFjeVTLAGuMQfDNi0EoQgG2F6SdoZwxjDV+iniPYTY2TUg4AfwOMyN9X6b6zLXZkXyPUC/McxkhT/Nf+9L+IElyOyTUUIinKH5dbwV6H5TgsFr6WPqXm9PtcswfegEb8MAvRbo+ne+oIhqO5HPK5IoI1afy1nq5PqtzRaPWBP5JHN/tHqNRRZoW+DwVsFgwXUBxjF/UR26GfN2OoFynt6fMml4OTTw7GSF8PHyTc0M34qe5F0m2wd9t3RvkNDrRGlBQDRQ8U+p3qD3Pg8khlhWqaO6g3qj00JcD78V6zf9LhUGSzgIG3pMpo/w8Lx+/qiM9ej0ULR+ge8xolwBquA4u8ZEqKdVU8tPDC0m2QsKm/qIFeHeRujJj/jn6qHXjFRojuiKQS9A4Orih9nh/HghVQJ0OhjIbqkTI2HbrOe93L1IFiE+qwLzFQKuVbiLQyIPzHUKJK/Fto8gNBad4hPatkNG4WfZwdpF/7C+SwE6qba7XhBPHEs3mryyEtI3f8ciFiQZqWTuNxOLIAwm8RBsUk8iE0pTwjOSjm8g1uMxiMeuFCpkv27GPNqWFS6E7PjV70rYx5CsU7AX/awhWgyvG1wt03ow+iH6Yf9djPpHgA6K++qi0UX62Ja93rLXyhx9Aa0LoBqE49TBJzV1y6w4WVJdtNl4p+CP/ETMIU3ygpVTGRClr8cCKCgNRH/2UUy9l/GsX3Oex8/nWm0eNRza2gt39yIQCoZ/2UBRo0cig+ItQzUQAdpfS/3SzrW0j33QI4XxoD2o+3Y4XFU7qi2fZ4Tm/UO3Pp6aIqXBG4WKmvyvFOC+tO7xRz0LPWozv8VKiHXH+fH2C7NANoYQHB/irJV7IcB24t/mT7K23LaVGqomPkDYzUIongvuHLedebCZHWj/9WZI9hTsbtZ7hiTR/auz/8opK29Kujyun3WwnGaB/9rkfx5VGdhZMNJu9Z3W8Y6wpQIbAZBUuocFpAqGQlorvg1Ldg04ncIdH7Lg6+i4RAIhnAnfnTaS03BEceFVSfFB40eodWsJtHfj1WPREfI4/Z0Rn4XphDQKhymtKYTcTTgjUVnHOZLbsY1FIAhipzWuppxSGZICLNju8th6trc4BZcFvOxMj+yB7XrqvxP8LwCbmZ4xtzh+n18dSIGRPbqjviCpkYQte8ASCfR8ZL01Uo0O2cCfcFWhXonus56EZJiNO7r7gNcq/xL82JlT5d2/ELfZk/evkV8zzgX0LSiHm6HK5WABaNcFR0gOI8FkmGAcBJuqHXQG0RqRVxv90Crmj7AFjkkg4QvWNbQGQv+J3FHvEiFbDfLLdRISRDhdRibiFH/FOfjziJ2uAtAHcGkHWO9MLpSMs0UHiFxLeH9BSHvoWNE3KDf+1nsHQFgM8DipEq57gHgARsc8I+NlKkDqZeWkIk/DMd1uOGW0ptqrT+VyH1fWlcMacIXpY7MqCtImR/PvyJWGD8XXsL9w9uVtAsDOqkXqJt0QJ8QjOJpUPXb10hwEFXi3r2O8q25lon2Aitai+EsP1vkqqXXdzUvqymfukMpEY+2IlCMenijG12b9uYdNr/JvdA5liFq1xHe0V5Oc3WEqtd13D1Lc96LkLfyWDJOowYB2wyxLEkPZ4a6xxuEfJPPS0xGCtzI06ZjIZgf7uIHhaw55EFllfq7a4zNiorSua6h//Fv34L3hQGtGUmkclQWacOfWT1QAs1/oUbznAS09PHae452gqVWJGX4k9B/05ZmWG6oR73ga2v6qQE0rcNv86QeV8vIX29JlSV54/rGbaZA6Wr2fFnL63E9MTMF4I0ApVfz7JswrzX+fPDq3zjc4LsXRcTzKCDcQ1Do/KthaKMyTgvzoD4OtaB9R19O4REdrhyyeGD12awlTu9Tk9UIu0ZE89uAiUf27k7FWD9/3own745sN/eatDd9P/xbjAq8F6XDdVHK1yD6Z9hNZLOqQYSI4pzFlvP5ZfCuFCdPAe/VdZEtEe1jMffzQkEolEOZngAV8uAuvGsCV4xihL3vSSAvi9afncP5S06g0E27E4IPUjCP6kQeAARvYU9YEac6P2mzfRFDQvUepm9ob0W47SyTnXRiWdjosHyuETSC3ugWeSL/LDgTbTFvIqlfJ4gdqFEdx1AI2dWRukffJrmgW0glwkVWLeGKJyZX8iwSw4RnKKvzoQIpcyR1KqHdq7nT9wduiEfj85qdDv/rLsHXbuHCWfnuJRu240e8UqD+bc+v1csebek72bUH5L4thYcl/TJkxKjwrLc3z7vBVEA2TS+kKVFwk2eqnWjGclgSHmvpD/8nVF3pEcW5LVIAEcG6Kfm1QU+Y86HNx8lOvCb3tC12AaAjgJDM821GdOlVcJFau4LUBH/ogvrs8CJRbZbFN/ZfxjNoXi9wfI9TrmcrV6vQMtyk3T5PeT95hq5MGnMLqDbsGXYweSxlr6IbpgiQ0CV0vbMcgdq/dgXQrJG8tWaCld1aV6lH8pr2dGTxckLs0haaP6pwQpEpH8IlvpDI+LFYAoiaoWzpleTd9rRxlE8UCTUlGD+LAnht3aQnA5SgQ/m8Mkmz+kzoaIeXHWpJI9BD5g9timhfq670UDpAe/vYwEv8mxGTx9XcDExYCE2q8eVVV9WOhCsF4FP0r52XzJaTP4wGbQ5DgeE1HBs4T8KUrUZvezmp9T0153Ii5sr5bZx2hB7SsbQS9128AOhHR3quwKy5iGUWZ52AcaL9RnSZJppH1D0nWjArfz6cmPCF8UN0+cT/4r8+F+F0kxO481O6qOh67ZxcZhvpwemIMepDl8zpJkoCnh1NXYJyfMhyeeEr5nJLHexA2ZqMtGdJGW3rPE0n+tXoscoffXcVmJvE1jctlp8SmPMEbv7u7PLrmLKSPFMLYaNeXHydaFFBu8LOYSx+dsV9CseCvdAKfU/5z6mhsF9z7HTOpGziel/jies7oW2jkvJMeEMtiCmthq0JzH68qrfKeswpLnpZaUH55qOkoCHLrZMCCL6hvpoilL66X/1cnHWbZnjDB+LWZQJ8i+AVKbAcUuwoNBI7s4xgUQHISzga0mfzW9WTOH5ao07SE62ezngTPx46V5Gvlgxk/YhWxTbtyz3E8gq38zExRsX9ub0nPlzsn4L2EMBJwQ31jxNZJZtJTvpitZoJPyu0lEX1yf4d91wNsaUwGKo0xrkGxmD9A/gE1f2vZsPuWvKzTX5WkXRuhZRE/Shw+ICVe9RLivLGRrAsd5MBnnzakclepsuS9LZApSiVIbovawQGXRaJBgXyYOVft46TGx/F32AK8rdYMoyLL61H5H6oQc/I1lKQuaYiWqi9EDKfsCWXyQWKTBIu30bmgKWxk9a/448qkjfGO+nBNAP+n8RDXLEIyVOSrOaFqYIyo5J88y+QJB50jfken6AIqLFjESBwESw9CEEPkrQc29b9G8wwFA5fh/cam/Rjnt44IbE5e39ZvM7bpdp5O0Raz6guyGaKHRd7VjUFmmCYz58eHCwF89X14jG6Y9Ej3Xl8fyWycJ4A8jluFf991Tcyau+LTiXwM4XxCqkH3t29t6fptpSmOcmRo14foLfCw+jqqjjXaKtnxrTn1ejvd4miPep+atRQtPkEeWY5uZjQnWKe+m7fNx0RJ9nldzjCgQ/KJrc9vGZiZ2EM5d8eszyO2nIr6c7XQ6ROvsB21I63Pgj+Yb96FAHwW1vlPncY24vPK1gJqk+Ii+uUKVqjz7j/WvtPtpNbd5D31PoE7+qnhGEzv3eSrGZE5Q2FDiDFPna3p+H8lGqchgakuGrYeXV4GdpvmrrtGHq467jS+3mdluCdcr+KCS9dddwzQvVtRYUr+wuJGwtSkbbEOO6jV1bfbHp+cY1rymQippB5cZS4Xvp4OJgeoA+qvpSt/FRp8J8CqvJPu7JAqSblX550UQW97wRuMSLVZVx0PZtjhDP7yNRo1MJkLsu9ttrSLk6Ecz8EhkMEi/xb2xcM8+1cmmulBifh1+RllZZ0cEkmstODetM7eLddAgqy188crvJmKFw+5GazXUIEWiI5fsbwrbc+fQGGRaBE2dHBIWXByRIbnMXZeJrFZEjN7vain/fX3TWTOWeWxC+DI9azFVMVz0IwFm3McPBKnNstBjwnZTtO2sHEVg/WiNy3LfC+NJWKsl/8R42Bf3OJigqKeqYG/KGb4lm07/R3TPF+OxKYgbx8Lxlv4Ad1tsN4RyFu+BbD9NERClm2bRf/9NHhGvZaDiarvZZb6wOzPPd+16n6Ns1EwMYldrQTTq30+C9jlUXxpEXLCveORQ4lls9tHYsx8oaE8Ih++sx7GWaocq0n9P3IG0cH3jzYD4La3uzV5VOl72adzMnJHrr+jRq3vvjf4k7VPmgxT7+lNN9+QDXWSmXteKnwfIzB9QxYNK8DG6X720HC5OBNbriiCm22fx+uLhaKoYU8bYItj+NDCj0WMhoXpLELGIdzhqRbj65rOpcz/sPVagNMx+7eH9x7j4VXksn2GHNfiexRWEwMb9kAO/5MhD/axV3XbqUX3qP1LGngZZmthdA6yBrLaB8rVn5Y/pD9PWXckLDKQaj+7w3b4Gil0JgfVBfvuKItzrKyv+6rcDD/1Rn4rW7WL7nhlm7ihXJ9FevB7LC6nxqwsYID1CLsb7R4DEKWZ0mAaadv5ssUljq5OXvntQbQKxxbMVberZiyVMop8EhWhnA/mBllAHiML5VWopUp6D0yZ+BRB+1R+a9you8QflR01PuAbt9SGVcbP6rsDnkZPC9OBVl8LKXs0BFBCVBsEVhaHlr8rBfHYWkTuNHfRAqcDxQ7fjQyOAhL3nnGG4+k/EcawC9y5donBpJdqXSHQeMdEjxHR4ZY1adb0f0gMMRWA9taanXa0806LFNXTTGWkrkpH5x3N29AhRhnY9u+iW5wdVtzs0NKKkz/E5zE8C0+SCRSIMVNGb9crsv6uTuyjhBCuppdng2stn+gR8KXSHX59I00HoqV2zxRuvOzk+h3t0Cn1NudjGbHgaQl9XNeQPQQECbDGlLFjXHPdcyWluhtoURHifnJenQX6k3JHWTema7mfBm6pbhmGprWDFTOuAeonXtwIvbzO9ExSegI2XUO8SVSJ++LpAyGRU1dA3utfssezg3B0aEVddacZv6TYNMO5JrE0QpDm03y0mUro5h2GvwV3ci4qjw/dbDo0VkgQ9BOFgmUprrauXkGPDpclEy/rh51DU6I/UnWPdXw5zPUWfPIyXo/4kuyh72ebNyMwwfr5QTdVG0YzQ+7hfE4X+ztWzuupO62sek3Ly8mSb9b26zIdqIeV7OvTLFfkbI/3Re1U+v0Mo/jUJRXh3ctWKMo+pkHvf2Boync/pS352mOFG/jvQ2DG2CfOnosTou3xWFC3PnqfT1+q0lRIdRm755kdqzY4c6AIvxIr0HztzNEztBxtGJW94oqueoH9wZsWOChiQ1H+VvQYOIx4D/9MIIc3rnlTV0kxs5ef3rW36mPlOiXu+PkCS0NXP7PMjDbvbVH90fVm2l0LDe6uPUjR1Z+mOM6wDFvibJ5dslWloavvVsnbuHvwZX1gvoy8dHcbdRgjXUOUtNGg68SHAHLZfUNkBtn0n/PCI7++v67hLvfxoVKhDal4R0EW/ESMoc/BfxVM734nXrTC+1PJOcnWwjEbXZeVXNGnGLSN2hsYZT/7fBJqnnXIMxAj83Rhfdga2YvMolLvZ2IP5GwexH3tDIfNnBLsrrI6BmzBxodry8tOSJLv8VCX0NqS2QhjiPl5UZWtQd8w+JaTGKYlJnJYIUkkmeMqms24m0nlObnPV4tS5UlpbI0MqvCzZEceqaSPH9UfSkia2IZzP58ZikfXuVzYhzmbumzbYrDO5u1DeVwQrncw0HmmA5/ASu2wCsxPEvpfL0Ib2Ibcu08MyiokTy4XsZeEbxxubCo0o1CE+F90rGonQ84FgVvwaGFkO5etHvvm+qcQaNSlpbdaczE0J3agLcSnf1NR4PWIRd60DR96qVbc0DHu0/tXDaQLclFpZXHwT+2HVkYBKNq6RYqWI5xBUIpJ/JafBZHiZFMlr0xYHpS1Ih4hc9GTHTK4lPKCYAx8X8j5AEuPPwAnuIoW592lrc3Q/Xk7ZYq8L56fKlX4OY7VXVuoKktfV4jGI2SxpLLPMTH4QOiJ+r3491R3RawaQ2ybgvy8VQ6+waoFYeH3plQ/BMcsnlyAnTtP4ca8+K2/jDNwY7xIP3vLGVgZNjFR1lFvH7OgX4K3IMtkDvmRxMHsgULpeWOk2JUyY8Z7ovZ3GnvgaUbwr31hi0uj5McNJ/I0H8pmxVvskdH8IiuQVVyWBZP1Vg2mC2RRkSY8w/AnuFWmOqxR6ELpGaTpUVJbhMO+cSIBsVqZIRzvB5dyxMvB53DVwySO8ZDYBYXFphzKb+8Lo1p6/DqFe0atBrNV+YQMXmEmvtk4wbF3jQ0ZS+BQjSpzy+sGh82qtG+flnUAmNqh/TzdsNASOxZAIxJpxK+OhUrlEsP4M7e/y9owNd+9iBesdALg16ZeEkFoMVcppwCjAlSCaTTm8JREJR7YhVFr2P+BX6YTP0yYzIWIqMfQTqJtmVxpMlAMlF2MXzN1wlbHpeRixV9kwCqUqM+sHkZY0Mj+EbE/6TbyIzmMu8BhwviA7rFRMhaHPQkdyFXxVIH2QjXTc2xy1j0ytLyqZpJ8jXAYKy9tF3KVI4/R3fV9F1DX2xEc/MXDHhB9XHfpB0O8Xf37K86HKGew930O0jDT7ZwaB0us8MOWrSnTh47DxOTJLXOkXxPxyL0CYeBF6AXMly2CDoEMk/Srcrq+2cnkVB5AKWWHiV4uLCPQ1u2mWS8+pJR23/eR30wCBtvAX1dz3y7ohJSLJ4EGPtNCvszbJtEc9zW0/zrdju5Cp3FfdtK3f/ph7dbQme20ydicue1rRgZZXMb4k118QXWZO7jp/uKZXnwPsDxMIPBO3DYMDV0+ZZUWYOMvZ5K9VZ3DdQajtCvtU0cqx+usQFxdqvsyrP1+ASNRmMC0vVtwkcc7F69kO9UI6BBzd0NHUfhWwR/uHbm83tTzyGpBrLMnHhxv6dXV146WYjlGEqI47TsbQD9cf4B6W3tFbWRjY8XSMXYvsy0CSylz4cEfcBTgUJc7l3lmftNk8F5pLyTIb31DUUkvoEzYhVCbXF5IRQeOCC7Egz+jlKpuEpf7BMcuDiyuOv9rxX22LtlS8/3RdFTEcE5mLSp6ZqXDIhpQQXXxf6VpaefwcmN4+mDve4ujUW67Q1Xb1rZY+heJRDk98HTzxgJ5ni7N5hs9Gs6YOjdDmqrP515Ij944+rqLPErihcqqaqt0hV+21JRVxP/kgQBDC4wfZ7RTWqPzGSpn5ZHr0lxKaZN3xlUvoYNWdbeAhIXMs1Pf8CsgKug2zrATcgHFC0uccZULLg9m4xzQqh+corcJCa+fX9aHxBBTUMsH99F/VKG+BlKtXrUO7HS+lk3zKVkNCskRZ7f1b4BclFNdOgXtoAURuIn0npGjnvDT6ttFvpRB8EDejq9j4FvAl1a6T3Eb4cL0Xs3LvpoYetErM+NusXhsE+AXCR0CU9kBRVd/3a6XCpl4iI0a4qZqqpA5+1Doz6i2fNGTXYCuZ5BWIfubhupEvglmNMYhRwMkw5XVKTLnXkEwrKky6UmU76kkrd1XOUdnWU4Mx1BXzOZw5nh6XcN1fCQtbvX07J8QVGVMxrU6GXl8reSV6cCLEue+l/Zs2aMAgyyt3F90NXWu70rh3Bz42n8MUqHClsmhn+tkeYFRN9uuAJJeY3GAZ7qO1TuGvqzB2P2k7ZAe29vWjPwT/sdXkeOWd55M4J9sf/3VEHi719+ysBms6pttJQdrT+Z0qvBLIQmf6KYoExg8ZDC5s/7qfybgV8fZNQ1/MEDbOYrCmXl9W61B+jc4u3sfXVyjLHzQNdmbf9eu/NzNMpi2ew7SN56j+3ob42cmICUL5UMWmUpTnZmImSDQ23+pzIT6eYT7izTqwxCKdDcaZso/IWIad+yScJYessgL77nt1Ryv+gIMvV38ddVbzN5vDZDFc3DER9GH/OnqbHVOM4cw9it39j6Wr2pJT26Jfc9+BQh9xd+cNd3e+/kLnjJHOSDqVLmrvJXMuRXy5bc0vxa7IDWJccjPYU4xUd4u+uJBNiEQbokeMUp5K8qhsikaFCwjAGx1qACRDUa4TiiGs77a/WHttoUtbONd3PBr7g/1F9JMB0k5A9DgX+yU43oiFbfMrpZayh6A5RZZ2aaeUCisAPKQqTUD73LBT08TRfEEAqAg/0/HcJomwplRmtRv6Uud1fHBau54v5kqKetp+r/1bXNp9y1GpX+MKwkpz5ud6KIU+vutq9u52uJZ/+a5MozqZI7wJc228x3Ck6RqOApukOt5s9XtcMY/0hGUY3h2Vte3RGJ2JLC8x9j4c8/3UsL5M9VQmpjq8g90tuyiYgVYZstecFHHq4SejtEWVbTh1bQIwMKHu81/zSy3Ts04WXeUNAIH4Isw4w2vRm281jp5kxoR3hljYk8xJf4uKPuMIoLbN+HNxwoFSIpYcIXzhtYPNGdz8VbzI54rO4tGhi//ZpFKrTSg7HGfnmpXnxXBiWzstmmJXcn5tHArhHcd7odi4XUA/hTC4lR5oGSRpHFxTs3SJewif7o8EmWP/iRx48v1NsbyPbt8OW8p7Jg3/G5JYcP3T2++Nu38zlEAZ+yiCbuLUaO4IqV2fe/v+xy/cb4TeZLcb9D6QyLZy3ahyqkWDE2WUgac9YTaV4c3d4bSHa8lTinmS14Ei1oXmLC5vPf+Q1ZQsQRvrvoYlirdlM5La9MZtlcZ1xynFT4BaSqxzER7WRGJ3luLboqaKMe2wWQCy8TE/h0qVpgAl+OFFap6/puEvAI39PSu3sd9fmtlv3MupaoPupQsA5qdg2A/pvSYAT4WeFJclRM1kUwn0nOAQvbT0fm2XUbzKR+3Fx2zm2jEIXgwesgWaJjqOyQ5XSgGPPSBNp510o9waOU12VAM6riwe4lqt/Qfhj3t+wbItgpVI+y1EZdGkIgOiCiNqxQXLenHnZfLk6SvxcH25DT/pFI0qxyz58LD2Ogps+hvDl3VrCVVG81NvmjLglG6kG6C22MJsQYvPQk5LnLK1rbANwD2/AD7Rjl9+gPMxUFqj6lplNvP3WsbXaQIdCFptUuUUVioJ1LC+okIyHfXmO3nDDV9WCcdj+qxWcgrY1sV+rI+hpRUlxQ8iB0KNJR9fIyE1HgwqEwlodcbl1CaJKjp7PGrbltUBYTvf10UpNOpPbWHV8EUhWpvXQjWcmcRWfs70DXTdR55t1ZlsAgh5CfpaHIno2z5nH8BHo1/C0JezGcndXwqq3pCzHJWzrHES+4ZaU5aoUiNs8MHLiy64NhL6dU1/jQbCwvGeVL72H3ONyGrT68p22yRJ3tkdiiWX7wMH2EPFrhaBn9yY/8SHGo2Chv8ezeOtY5DKXtGGZpt2NVZbAzY+k6s7sGUP6VcgwPkqIFuqoOGRQoHAAb44YBYz5K9NeK/EDtJvijxVXb96e++f50lvODbF4rfoX61sZvQhy1IjGfEKPvvI7XUvakzb3fvcc+D2h/BNaiD/TElDCRNwNPb1VAaPtIBP6HxrWT/Gsg7gsl4JtE5PeWG6XMU99fiAS6H1cYPw41rwgf5Fx4bZTqlTlNdif4bkAU0LUYKwUV4izJoul/uf+5aJ4cWDcOKVCNDakXN9eY1G+vLen40P2JaCUVijQh3PXY88SbMVe7+rz9cJXLtikqW9v0TxOr86dmqAzTNsFkHTv3EFx618no7Cy6mPGHXbaxGJXg7nI/y3Zg+tYpboLS8tjNkFERIyHF077AYlw7Esx3SNcpWowSjo2k0aLMnAQKKY1rCkKDMIYyPUJ9/JpA8Ex4bDcWzfNsLAGQ7IqDtzwUQadjK5bHM9EyFXVs8orMxrtXNBWvsj1GDcgpHBSNIfSPGaAi9P+NdjXCJe/CqGgPzNlCtiStfJF0aiyIxGopvaSho7Lk2OeyIRgX1s1N98BAx4YS4slAX4VTVzB30KTxw/mzt+Ooq4HBlssw2VcawpE/3R/ms1vPsZBgtkOPYvOTOTnwRAB5RkwQb+ZX7x3T5LFFgZST5VMpfgIG1eMfswAjlgqU+KVCrVhROlOzOZJ0+LLLk1P5D+m1X0mXr1PngpTraLCsvvyXASBvfyltww4+sfln4MhNO3I7Fq6akXWZk8pL9unJAlMw/jpTBj6AZUO40N25q9erPuHF3l1TErSft0xFKf9LI59McmsqQy1CtQzktzdMUmyOa9+BeMTXYDrf6LNJpNXvA9XyLECb9qPOOVRzO+/DvLWlVTsNWPu/vyz8NJCdSjGMn6gXuP+yyPxR4L8A6xtCCJErgS8U8WDERHTn1HgLX4eoO7CzcAMfkcwHnqn0PjAJnPWcCgZ4kgdtJIkDa12/tLMYJOfbCTz/3FrT1zd68lsAaVrFy7N6AXjnhFFzR/vYs/XqlEiH4p/0H1yA0cTEAC+GiVW+dvG5TtvhicI20iZMBsf4EOM2atCQCHTzT2i+K587l0cfhU++qvrJNmepGkXAyRqaxf3qNPL06kYBkGGahE0RuwReBVkJ9XpZWUEUzAY7+oDkah00Pq622mAoT3zXvDuKblVUcFeowyzP3gCz3bdD6i+XoUTzUSS9yg4K/BW2iTApetfbBw6x5LTIZnsRVLIr1kBS7KTrMu7kqVu1Ck+710I/5wjcSu6I/QuPzl+neimQUjDuyRkrUOwBvzUMNnxIvrsxUrgHx9P5R6hPGuKb6Z69WtdiesMpC6j/aBxMa9xWX2K+3oxsC/jeSgxbMuM7zITNZDHMvDRu34A3SJCZ+j/ZUPLYJh0JJAaRt+fFq/L2G8Od/vI63JvBUzFujZFSO/WA67IC9Sprlz2X+DSJoMGTOJYnTEk3nIGqtXVRFCd+Jc8U2J5OKXpepuO/cjsXorNthCE1eOej1XpvDxS783vpWS310hDVQ1S4rUOnS70Xpor2/+G1UqaCC2UZtKm3Iy46mPg1Wdp/qXr8g/2txhR8Yezgrwm2Grpz5A8N7o8jiK/bMLgDY+90gR5zFenso78y6mcsUDRPiYBOueHDk+bDtVJa5b7ZOPTztgy5PkNgOohABR/ayHLKZ/ZC+hfI/9G9diDO42877drSSPdbhw8cl1zl5RKQ3D4y8H1+PH/SZAUHk46/zJ9sNHhzXnTyWioNY7E4Sr3JcLF8erFdrWfIRdxH9Nzflyj6ViOVE9vhpt4xAmVOWbXDn8HGqGY7k7zm/ZATOAYV2fKNJ/IUQQcQAcqIyR+Z/C76K1VDrmd3GsAk3pCAQ23cHa3miE+BmfqNyynqlc+itH0mYUd7B98l8I//1ozCDyJiLW8D03f2VtUTmOygQy2Befx5USJ5itwMITF9UPAaJxOI57CBVXF3yGyjSC44MMIAG+sM5HUr3NhSaNqxzIY74vv2kinMbzjgrDyr162fWLnI5dv5qXJbZYZ73MR7hVc9Vs81p15qLyGgnyVq5YE8hOoJpLLpp/6cHpLxH8TLxFXAC30r0i4tSu3oY3teUwoUE4q8i+XpUUf0yIFnTU2KVYLa2QGPzfq1nZcjnwQjTeM8odJCMjWkGV+reMHHHT8NCjCj4Rhbz2xlXkYOddKrsPQNHEZDoxVgk8KweETzv5KqfEmUgt3T+jP4O9TfnolAnwb/Iytwlyxv91oQWxMcpi+aUiY25RPqWCPgum58EJBKlIHpG0wQ8JqUgG823tw+q6VdluUbMa+g2HkQA8/rgxImUDCTmRwum5LmZV4rZvF+2H+Xu8hP/Io0a4XR8TVh9uHOINo8z0wPKlpdXCR4i+WdmA1iXVvvLO77XlIwPPAJn8LOMbE3ifBFIir9Rs/HNVu4raKew41O4AritztLIDZV/NmNEt0z+DFwbUcroMnAVCsrQMsYS6kITuhOMMOZyHSNNsrzDx5XxCHLOvmlxgWfJIttdyAYKoge+efi7eaZzNES9aHGp+wnxykt/o31YRqsvSrfFL9EObojf8LTog626ng1+cfOVEL9u97O6jLr6b8Yo2DcrOv/Ytbw1bkt3GWAL6rE2JXhjDUFIrV9ba+CIE1+uw8SBioeaGHqQmPqSA4BOhrNfPYNWIz6KvhUel2VYuo8B8KC9ryT4HrDR1a2akHXwJXVKnkKO+vkLOLyhICzuHUKTLzH/D1mt8+d+/qd0hHE1k2jsCH7yYfD4ALfIcY+r4eZarAnWbQ4j6uu+6dZ9wdSAvl9tz85CfKGZPSI8H4/KZEk36F+QVdeWuRJo5Qj9zUUJkOPUAweutq7bQW+Byd+oquWGkyKR/FSLrLwP6YgS2PrO1/HyPJHRok78ujaC/+McfNLYpSqbcDWVew77yp/E9dtazJFVeTW/Btz+jnpnMzrpT8zV+1f4UVDQ/pn0NLdFuJ6Rerl8zH6LvHffA4Tsjp+eHxTLZBpNLgEbCESDxVzR3KdSrFv1fMh+gDKFHnRQc5eNAEn49gkHJmP6vdAclS8DDyvKHqCB0C6skYaVSkrBW2STL1LsjVrYpAzvd6LEovY7mForjVUQORm3vC18nYnXg2xYFrq15/nNcz7RlTfV3OX9pmZmVJkS5Ef27wgGUlsrrgFk+ffdRasBh0XG+VIO4ge2MjQFaMoBe/HylSLC23eiBAAmpvkWyv5EpD8H0oticin2+J2ldqueP6r0SuVqGpuQ0LZ9gfjNc7LRp3dCDYbefz4jsY99XV68oGNNIvnZP/c7QdXniBN+nOacOE/3gOLV7CuvW5YWLVUe9PsFt1vZroHtNTPFpLkvzVGqwvEb5gzcp3YRZqHR/EdQHO1LdsT7kqhNVrhDrn8G2tKjF4iD+5uJzsvbZiSGqc//XoH9YTs3Tn6yZ1Pyedu8+cgJVe59tNpSQLXmygu4UnzjJQEGQoWp5fp5bOGQcQHx3vCRb3el2rzuEM1pPZydU2AUNRlTPt4aK9zW6Yn8XY+9ynG+kpj/+1Z1/MsFA+DRKPLwRWRDguL88rqYNoeHCShFqAl6rMossAZUK3hB/gSVyaWld9dGrQ/iv8Gg8XD0QQSDtbN/W6XJPi6t3etVhlgkDu0l27P5mKXPeQJrI7OHB7IUJu0+q1XN+ErvlXwJZneAX74k6d27nx7/LDUPVqaO4FKsOLykwx5ZzyLPicqpNoVxIuGodSP8BjQtmXxF8YrxibxMC73NQtNMlJGQkSHk7uc68YQrmPMqvTqaCekP9yzKgfoNm4P3tb46idiZ13E537VXyfc8X4fvQsoywdp+35pwXD+p+27Mnr/8tycDz/j1Kj7jgjs3QOl6vLlJ5+Ny92anIL1xFih2phDwpUUzA8AJCC46FAHhTxJ0TPjX8TTSjBGj4rh5VRkIwLIt51uJ3LDbIeZ0MAlKdj4TzRc/HUOMhVPJePZBRI/x1kjxW9Rcua7ELGNucmtAGjjj115jrBmr9Xxyym5/RRaLw9/v14s3y1N+UWc91TH8LQcL8QhgiOfkWU7bF3EsPD6yNH9Rc43YmVYcb1anVuNaNj4JF2zFBDA99Z9FmngEjWSgmemRGN4bfjz13l5q4RiM4JEgPOA4jR4pl5rYDLpfm4CUYWX1JFvNZIXzkt1WAobpn/UhAlK/WjBmzJrDn1IW/lGe09s8s1MZLZngaGh4AQ3pTx9C6ZNvmi/p9Ea/1hLIFM1hOdWDi+6aUPEyMFAEq33vWHZNXACtskNDfMIXoNdb9CcgCMGPM5f5tnq2PIF3R4U8CStqbPMlZmCb7sUbMaAJm27FGm+F9zUnPTsigvIYwJb5a9lLXgjNb41PaNSnfpscLIRJnpDltrdKoABevykoAIf6bW0QR1NEaR3hOkq3Of4bQ8ePA8lmBrxRDhI7Gl/NXRQeS775Ci4QTtkIDcz9sYmC7z77DbmD46zIzBnkke7KbYGuehIU39y50z0R15ZBBEE4w0aTLbccv7DRscod9ZButgPcOzgFRfCfEDQJl7rirnDKcJTj7Mr+U+8JRIU/SYlz8wYY/jBNwyB+jvMcPo3Z7QlVIFVuCQ4p0u7Uif6/s850E0d1yummWH1Vam6AvvYxktDE/4/ab0GfUnqWAwkHwlz7uxlb9zqOjvP5FrQF68IrsglP/AbqeFGZh+tnZgcey7+1xdf08K5weW0Jfx+tccdeLuu8VpbJAFpq601+2d2RpbRSeELV3XPzJLinXkEEMnvZc3OvOsQa6HCwabBqk7QFg1R67Y9WXyH77pN8mgCZ1MwG+jND3QHbBvqUMnJkyvbxkTIfElMuIxGCZUDOSphS3AY1KpB43rKxL0TH9DANJOOZ3q4C/aw9wPHN5Tz8LsGC3rqK+dNZPz6um36Ya7tCe1WWrssfpvsv6jMeHmx73J0mvpQrUE2b+kCo8XIsA/8Dekl5hNcb9JrHGmqXCgE3NXP0IKiyQpfy+AyLbnH5bvEZk5B0h3wrRJhSNuOSJFT5ZkT81mz6TgZA09TfZVFH3LxPHBrqe45bA477qsSS5fe47Px+kD9GZH76aAEsFxdsR5DCaRoq1Yo+BHw3QcutBUtJsy5ZKrQx0fSBBMrvvFWyAhTa8UlR1ggWIQBkQGnwhCKdJlXXjHShGekr94kg6shhglGKhnmeWrShuI3HSl4xWPBxndB7TvGjSrUFZ+Y0D8vck/HVLXU//MnGSVs9G7PsKtPBES3cLVxvg+hBkewTh2t/FC3yKbantE8hCLB09exidC1hnbykIgSRp3SR6nzRp3BQs4G8BylmLMqV+J+DVdmYR217I7Idao2FL0+s11kobMdUTegta2tWISdILztuuB1dyAZbSq7J+VNSZP6HoteG2nQUIMaFx9y8dkeITA0NGAw0yoSqJI5GswDBPv8jX6ITqCUoeiwP4K8O7DIEhwZMWZIWXgzivv5S/fStU59KRBBkQdgSIqhMgu6pqrQUEOQkoWYW2zZbsSOx3SN7wdtOmcMMk6amLN61olYp68d4iUFfDOIinmx5nFBdh16FJGD/i+anRgzjrTKJUByMOkOqK3Ij1kO34XfbOMJ4JG8GjHPaDLxxfiHQ0uUZqauEjlVLvYC3Fz5NPygMnxByafjiM+Ka6UfhIOWZs3xr285d23lqaf50s9xWJUa92DfgYRGeupIw68apAhzP9sLUtqGCOCtLtasj4IRTNhGddQ82tdklThZXHyc5ypLeHVEvcuWmq1mS//mua/0cBOCGKKactX/MkwFbTxq7NF2HGub7L9/Mk+xD/N2f6deODTI9zHALdE2J/nBEE+c0uPhQO0vV16MF3hw2iulrjUAxIAWp1r8y6gBmXg6YwVn1FvP5HjNdC4sS0jjqZqpjoqNvDnZZSmsA9bNW0HyibG62gKbo6L5vwwW/0SBrTGBn+PtXx+c6yBiFbH+Djbx99+hcBd8PGDNOlJgTbQ3t74/FNz8cuAfVgv3yxDTNm5EhbdqTShzwq5/N6t8DNRYr7FSLpOQkUEIKTqflrglzc4tzNzaWcbV/dmW8VlM1PTPsyrUqm4i8iUivLSfgrAjLzjlC4/iuGBRb++T2sTDbdvdh3F0NE12fI48GgjfbDwsdB8RnxJipLTsUamA6jQFcPV7RTwVZMlpQ4TIc08vG0/AaoWyjVXn0k9gVth0cgTJxWsq2ZkG3Y4LdIRPA43zWJB5NGrQJCrNtaA8xoGkhTqt3+VqCzmdEG/0Ihke74/fecP950vxwy9/oN4EkzC5DbsJYeYHk1eP+q134g6Dj0gK15i6QLo0L81pTGSccTEj/n3zCPNu/HlrTFdhwxYTjml/+j4vOZcVCkOi5akr3NRT/hLGVUxtppUBWWyTWQ5i8JNisnmO7JZz/s5fdjWNDsO0TBHm14Qtr/+RMfezBv+DivCD3VQHJ2aCK6Ell9ChoOVYGOKExxp8hpGYG6ajnoA7+TeiBUkZb1yP5a3PnuaNKRRNPi+9iC7d7ckH6sytFDoUmEsrHjJmbTJdxeW6eunX1ztmGNGsu2eP5F9z6YRXxNg195BmVtItjZohx1V//443QcP3AqdPCLEQYhZWny1nHr33bOyDX44Qiuq4ZjLpObBfBY6bcxlgNfIrSg9FfvT9mXb6nupK2W1IQ8GAml/DKmOFXaBVRbivTjcSlFISwg9YgfNlrlvR94C/RtC9FCSoaZASZrPk30seLcrCkHUSMJ8oehe42neotjeDzKV1vAIhA9BOcn0qtq24RwKkWa2iPNOtU3mEI3koSerCUikcK4Tpjfo9pVn+XiTKpj/zY/XciJkdfmOC3xwsETKgJjKYMpVNwIdcwo5PbrbH2XBQ719D2Snr8T+/l7may1PSRlS4qGwZ8zwJK0RZHagWrpZ1WbQ/lnjkjkRVpl6iejNW3eekl81mVWmC3enAOftTzgRlMKMhx+nDV4Y+QSukFVFXmziHGm3Bb5Vhuuvru1T1/vYKQ5Bzl0L7mpZjmdzPXWGlfg2FY9cxziDW1svsC0ypopiXRTNtXfjXP+9nLD9/7DJwTbTSTSQ3l0zzoucXsIKVkCACMXI0yxbQoG7Dex0+R3hlMITk2HMDymo4zX9a+c8qGIYbAvtN90bAwKuV/+My4ff65v+5MuKhII/jkEEjUcvf6b8R51CUMkVNLoL2jCS679oCcZTpOxMxyugUclsCABLzS1lCBJkvCzjF3z+j0imk2QgL60VkcQFPZBf22sIY0tUeQLgkNHNNsBfx/yzGh3RgLiGkeb0oWJ76z+6iXbYlHwUwPEeJrQdSuM4c2XayX3T2BbkMp76dfaYUn1mtPK9JwRoougvgM7j/cLf9actL1aqmeoovm9oqPYpUFUE0sATS38IxDDWfuxW7Z6ASjT4kBEGEFcyg/i49kF7JufC/+q5y8ZZk1rjbxEVhIVjCYVJ7nSMP0vz2NJM0E8xaa2dftMfy35UJKIdrtZRxV77OYAA/mFH/Rs322+WQoxDIS4bFj72sp+as+2TXlzu/I/j6X9RuNnWlUXvORcaoOtDMlWtlZc2jEKoB20+Z24kVufAb0OMRvsOhV/YjE/EPPeWli8J6bNCbrPAE5coOjvrxMLZVcrN6S3hl7uphrwwYCeA/3CGSdDUFvChlmztOsFL+sNs+vNy/DnxGQmZf5hoUnmVDmedO0L6jJ3FPq/RUj5bqn4DlwX46uyjJaBMlwFocofOzIBfkcMfGnh1z6aFVq7AJ+nxAqTT3et5nqnN/Rm3zc4vK3ij+mlX884R4jp9yY9AJBV8YUyXSouRZAm5e81U7eFQTysdZlQTjdpijjUmSw7ynp+A4V+fyhnEMxua7Y6dRtLj7D5Cwtl5Dz0wwbc8evDidhHNp416bp935Tcb1mR7pyHZSxuomE1uAY/OlfneU+B9oG394IKPCjhS+osa5bB9xDvJLEUVeeZmethNP9fBJNMYJes6694bpcnGHv2/VVeqEuPl6tZlcDHhn5gFAljfNRO2zg57i/bvL9x/9n55xOfu5yTb4MKpdbKUGm/5wfRY8Raf7BsGS1UVe1WaCVawnkV22mYvf7qSMAACQG2EjlJ8HbQV4Eg/6pZ0RI5sGtLWLI+fy//+Mnq0PdDPSXiKCEnV/8s9Yt6TXxq6UBrUhtaW/TN/VWx7Ja4eO3wV4b5nM03e58irbHCqjoZYxDu6nicn5rfWDLRNYEfu9d5RPqxZULfSC0c7uolOajw/MWNbwMb+evnUrBwp7qFm2z+Z20LFOHgv3rMBBfEJ8dz1W+L69t82aeiQfrtLQi0f+2vddGRFfRnLUb/xcB0DAKZH/oFbsnf0FldPoLKsAgSSLffrKq/2g/6teJf3QVmCGi85HA8yAzT9uLjAjMk0rvHT6irbz+fKlVVFly94fjA+K35mnj0k75IohPYjiYb1Clt3ezziVgIPIp25rtow5J4SBULbWh8Ba53eV2/nS+UEs/mIJZJAjUodaYMpl+X1RWd+fv6FIq/xRLo2px09KIdPpLOeekV/sXFfMBA+/F31q4ZOZzmyaN/NrzVY+xibbMYqK2YiosInanwxHK5m9z3Yu5gu4UPo0b+JYBQfcmA7znIx7/rYNJ9rOe9uNBkpZ7rPGKGWTIrhXVwEWjFSc9LnKBelGFdThZRHDMo+8InH/JKN/NEol9zdqXEULSiYcyw/ZN3D2yFAneeQhHqI14RObRmVvvig9H4uQl1J8afHTruS8e33rOyQbQVz3q8JDan2OQtAcpdHMo8HVW16IopNbiSNnpBdGLuvhepfxNHJcpG9svQ/rS4rVjopDmQnulqHEmy41hznOAm8m6VHBKP01TX/Bssz2sTZt19DSJrGhoHZB4v7UuhHa1l+SfC35qS01co6LV4Y5jmPTJRtXoL2jI5kluBvjhhVW4j8uNQiPwXJ7/Q3fXkZDAMk6jxTmS/xFVXZYPMeyMDN/QMAvjfzsevuY84vvgxF+1YVdmVTH8HNtdfvIuBTaX8q9Ns7baewOkrOj6bXP+wOIXukMmYYclztwNKo6r7xkOelGWvAYoOOfwtue3xe9QF3NY+y0FwZ0V94iSiBa6oDaLoDOyxVZ6lnGrE6O3UZIniJz3w7a6NSOc3l3kvcerSZBdojTVMv52TCPLS2X3z/+W3Gu++QOJZmxHOCLYf6DuN99c+yEVSqmjWqdZ25mza4YvmdKi35JlKy0ngF2gMNvjfDBppPtumq6op49DF73bbsdu24v1Jc6IVIayfa0WmpX/islCQZCnZKTP/bFkEW2M9tmI1avsKJStSsNd5QV+MZ/zqT3Jv2ZYuvGmonbxphG91A0Ta8Jwh2JZozAAEGRCchMsYMif2LLG4XCUj4qN+E0P+lp47sjN2dFZNQXu/2YdWd8eOqPez+US+KtWCO3n5W4lFqcsxrdw2YvaLz7IQx8yWBfOnIHfKYnfBUNxkgL2t9Omd1o9klQThuy9d5errDKHtMVhTzxnz6yCHPSj9pzpN/YXa7sSGYTOCSPJF2e+XKF++is4L3XGBD181N/cD9Jqw8Qp6hBvcL5BAxx52CIj6+uq6+JFfcsrUGp6CsjJLHZeN7uZlbX4kNEp0QaFQmvt/Xe1mmV9h5rtI9Y0Q3Tqho4lKZc8gNhgRw6UCe0HsR6+RrvprnBA/w5rIZO8SHcWnuRhieYI1hb3lfBPcqG6uzlIAwN9Q1AA3I8GFgiTgVK32poXiSr2kydpkZ9fS/gGJZcok7/PqAaUanZmoUZUJE2QGMGW8OmjbwsO6Lkgd5StyqEx2O1oBXTjT5lFepQyA+kq3PqTwkOSW/OC0JZ6jqLfWR4SaaHzyJGSaGmf/BDx0I9K3k1Jzyj+2pJZCh5NqAzqR88LDdZBSilTEU008ZJFDkVeL/oOlrztZVqS6UiGNAIcoDuaECyLxVo/krRUFiU3IA7Ii0w1GNgX7YcC3m5vSxZgsaQZK3qs5mRZywp2UXlfXnWP1tyYEQ7NSyB93yHvD/s0ie3xcU4pB15dqiUexokwgTO5eXFvmys9VQOnXUzFrPs103dDeHKSHkyOkPjxVxxa7JQYIwwCMWrDfd2ZEkqXNrQbjRSBAnpWaQKT3zIKcRL/oUqnHtfT+6VfGSGs2Mu0voC9zGgtk59fzibgicSUBoMwZqTUAZIpbIoqOB429lWyaZ2xcP8415F7QOVnGT2UjoNKSR1QnunrCRmJTzhc9xM3BZgFleL7j8Fm/7uSLggMdUETl0pqZErV1oYI7RKWxfUr4pxS82t7d5bVtH/Na/9GFOctD93zpIOBSQwBhoGX6iMUHKj1JrYMZbsDY6nNJZHPmw6dWqSE3gBZwFYUjq/NP2MezeuCx1MBO9CRCuqOTQBwPkD4oPk15RV07JunTxL0++auIc2j1vhrgRD5jPA+RXXbzhlpyNLU+w4AYMvEYdOQ1KsSCQ2/U/g9dmO0rH5idl7osB6QbQGUI2/YT1/QL80o3LLxvzCCXT0Io3VnZn8A3867X1MC8pTket08I4A0K/z11HvfOEEUM04GW3ccrqHWLe0+nfyH0ofDanVZhNqTamOgYVUf61kvVefbihZpLLAs6dleV0W/g+8dznRctcqi15zfJ7u1LNmJ1JjDB2DemZ8KEXdnPvpkdgmW5TBvW/gAubdkQypfAODHTxsEvGtGgzKDCHb/QpJ9pOHUXd9yo4YA2Ka130nXj4YMf+aO+jIljP5SfHu4VKT1qOd2/U/oJlBJVQ2BVSVTgq7ISlPkNQKYSMLmn68FBUzBBJhLx5WmgL8ZzL+L14qPTtBFdE5ftxRECk6T5LOvPrn0+AKOg3ByOQhZ+Wsln0D7qaTHMspNzUEt59fw38ApFaXs4wxBKsMwHMzqgcOVnKQeBf9urOes12R/ba1QZ5H4OcTv26VfDXy0ZVi9Ehf+0k/xa1fjbU3SrVMmxp0ng0x4q+3PzsvjodFryRcLwjGwEMFxkySooWIOyPoSldSjXCr3INToC6Dhx7ykWJMnSthsdv6v8oPEGNzH/20ZCpE3cTZ/bBcr9Mle3Em1Cl1CkxCurz0K6YivswlJaxOslrE8Wb3Sbi/2QVd3Ak5jWP/rTFXF9DH+4r5uO/KqZ4SNyZrqYLiofT6EOWQP9FQqtMFk/b8sB0OuiVVJDvEmz0NCXRLq4KUBpPbERZVcyhm5tVY5qK8hbZ+QeChe2aRUcLqJ5Qv/wwGbjw5aI++YXd309Fz/RjcVKPc2cFneEACKEK01g60/yoxVXnIy+sPxzCLVkv35j/Ft7Q+y43yICWlxKRNHYaN3kI+DEPxGS8JI+ozP+3bz+ug23YYAp5wnmC57lP5IpA31VJXib+HsIBMaOWiYwa/BbZ8jhM/0lvV5aZ6sbGIa/Q4f9n7A8SlVpL9KoAOmlWO4i4M65b0XSgOS25foc0Uid4KDki5pAAeKzZsk+z+x9Dm1Sb4rZqGfIfJqr0ka56wR6miLPGJITcGUDgZdpstogMfzfbO6naQ+nscD2Ps/mKzeYgX+Z/O9XJ9kGYn/XU+gXgfzNhYJKzQ4mpEzv14gRph/VWjtK6qSNr1mwu8RnkbQDaXG5KZGPsqCpMuP9EoaxWB6CjOrf6lExzOv11bPvRYGZGmJekke2nxkl8xXDYXpZTP9WwPogDTaro427RAv7DYBa6QnEnvGj/ynxIZw1TZIhFfyS5ddP+DQB038j9LhwvVDGXhViYr6csiFdMKyuot8/JQiw5z0NKs0OX5j++mXB96mtx4TJQDpXqElpkibRAH4R5b/o2zoVRaXftzuNaoqQtSV9bxITCGGCMJYAv1ZUtTkeeiKQxTxUSRA9hc/D9sXfauMRhooqLbZqfZ0BWXKQrWQB9/KFtWc0AOXi9BOpkKAuXurTCflyd3/LDsphpRPMxKEUlJIeSOru578wmTrpR6p+zPg0hbGSWshf41ddEpdTImSb6rt416jOJN9dxR5/VNMyc8mDtIo1o5xIu4mQ/Lh1vy0Nvfx0dfOY2BcoynxgxfFC5KeIqs4h6p4K7/6mccQ42FdeOxHcY3M8KCqx+qGm1GO1JjeAiXWmZfwq8pwGqC0cKnI53eRputVFsYO0xVW/IfqXXP2yFy1kdEjUawP+HgDCuPqkFCZmEEOBceOlH6X73eMs07VIGsd+0YJxjOBNRk1+fFXo9kqeomWqRp0B7+1jxOGS9DcFIVH0ggKeRTApc0XwUT4MQrrwThjwP3ajfZ0iIGW3o2ocBe7N8jhbdGn6Cx01j+OBERv6keV3HKf8LXbUrPKvufKiDeMH5MfvgfmiIUh7NFD+UxmzljhTzH6eaII7WDzB5F2S2Kae8PoB2YM3s9rn/IOwKPm9Xs+u6uF+30YcqqVMI5GZPEBA7gZZN5T1yO8J2PeDSnJURxIWTGyF7cqb415Bgx5/mNQmoDn8nhWowz0zMfvHkPusXKG5el5cTx93CfiVN7tuGeql+LWkaL4IDaVeLCo+ynxRrJSVDF3WVu3bIpn8tKBKzIxk0GouTIjhyNJlVr0fypsfIaCtBt9lfSc0RHcwEUc8/iCpvermEoKR07alA2wR7GQTsWIIsmObsKXwguUxrmD7nOnGkDuACr0P9fpMQ+ZdQKTGr5qS3/6DLDmU6Kd2Tu1Qc/r+JJ8fpmSyPp+mu2DpM5QMfsAkbT0HZz4pce3dlR3Fbmy5FigNHGOT4Gs1/TAJE1bwVyFbB7w6KE0ChvwSV6aBxQOzOM5+vEdTKBYdftHyoOT/GXMqQE70Z87CiYSlYE+W/j0a+vM8WXaRzyMGW9OmcXoB2ZoiBXahtf9xFkr5U46oNWHePEve3CwsTOKwlaK5IrjXPHKS/63vLGmpSGqwH6of5ocMY6bQlz+XygZvR5Yi7wbbEL/am0rIImxJ0YkGgAC+iUHpDHnQePy0xAhPQFDaDe7wM0TJ7IZhgb9pFL6xzFOOAGEd1WMdzlFl3/YK5jGrAncTvfhuoJEctR7y5Sb32dxmWy907NiO41iqKQa8oweYyUd7nOXaGt9EjyzXhQL2ZwBv3uqPGvIsszSeTZOyx6bgaRXA3y6kAfeswikwWBJhaTa3WVzvUAwiLtUSepAPgUfqBjQGK78CxX6J37kXDc76Zvx79mbUY9V6RetzL3n9nXSTHXFJmbkcSUHlGeniv+j+i8A+ffsh11TBTrVNI1bDM/VMC8DjjAaD2iwvR4ZcwswxjDmhYpFJkFQ7hKen/olZkYSXRfT4Iy9ffg0Aqf92W8bHca7YFvm7Hc1FL8gq9Wl6XDyCghD0N75PNBvx9sly5BZr6dAvlgSxQj4sfnnuAXU3mYNexVd7cO5grVwmTYmmuG9fQMYTtpdsmfcg24NsOdWpUDXgVkooREWInRbDX31iAs8uNztsZYJrjGg2NFokqAbtij5fhg5PltQgemLqea6LIHQNROsEuidW5+GEDs+T8XWQp1CV6OHyqFA3Wx4MiXIG1B+QWXna6nQHEGbAd+YgXGab6OfgOT4P4OObsaZD9HA0ChG40TO02zWaRNdR6kIvdSCxsCNl1m5NbQn/EDoaRzgm4TqFRT4XkTsApDxHkl4C4s0YIAKg5bp+nj8l/PQoNJznxlR30T/eT2lhMOkfnhBYlEB5+KFOafxGPNEuTVIyPPxrght+DlKGJHJjEvpzaVMXuV55D9so/hbulLo6Y7kC9OFQBGUW/4u/cj6Bnzdklr1Ca1T4iJPYwp/hNYaUN4pDDCeLa/7bKhh+4ZvBlb51hTAOLHzHho6d4IpWGCIv/C0AVd1O2/YR0Wc4zSGPvzDud/AVgbh4as7pGmXKrLBAgbLll0xmWW8EPZ/bDd//ZTW2XzJXhaPY3rcMp1hlKfl2fOFrTT0E4A9GfDsfmvdz6WH0ofz8Ggwjw9UGz4NUPm+w40Ytv/8Cj4/b5WFlZC+KF0HAO/WTWrPepmzJPVDbkn7EH0ybx/EFUu4Y8CNrktegjuGlPfb34zl2cp8M+eWgWzUxw0/T6y7O80G0l4vBsj9MPHuYnQkQ1br81c+GEbPtfBQJ7tM1P4C2P24F9KccS0kArg1Rdb+XDy/Np8JWjDsQVK7b4VkyIE0ACfBWJc2hkHQ+rDr7+statk93wv4u5a+5+JX79MLl5QerUraAE1gleEee9PePecEwuZ8IJJ0mCFstfxv/+vS5YFalBAMaGlzFX8D8uT13r1g5RL4Czr+lEt/hVlOxKWxhcmwzOiXek+BfBM0w6s+p4MJ5U7ZGsWFs/o38sOX383/5obT8QBP++rMzxa84qB+6KGzpb9e9fjwAXRP++jfJI+Q4KunbgtbTw3tZj76uxzI9S95lNjrucCerykHfX1BMpPTfLxET8CI2k+MspvqhiDX/QHW0hlfeqqcHpgrGidVEBuYXmK6ldBwPiy1lnSx1BWUy/SrDKNvZ2rdXma4B5W4MZDd39/FDHvtnnW0w0taI4dpWMGs51uF8SnCtrL8o4nZDc19tAFPfiUI3ctB5WalHmDOouYFDr/Qo4G5zpBx+4y4VIzUIJg4+NKrEVk6qwMocXSbp6LgZHRgHStTvwIxNIelnCi9k3dZrONT+Dfy2xqrDw1+67hTUb/n3rQT94l0Ocs1O9mwKcJRGYoyFpjLxv/Q2lU1NsBZGQIrUqhYEPgvxQ8L08wQKTMlAguAEBTdlWuSlYus9zrPF8sXBTUGkzNfeqUX+/CBkenHQfC2kFlzzf4wArf9K+//cukirjRTO+cciwPz8W+weBGl2Kr+fQAcDZ44siEONKs3ay4KnEy3+/ZC5qNQ2WC+PwZOwmDnmBWxxU6lxBiwzaAAOGFI6uIgUXV+rz/yt4D7xZTVceCnCl498p1p342K8+JUm4ZT9IISuVB/YF9YVAM4e/rnej+5seLVRfLldTo+tW7Nn3H9OCflmWXGp2DejUCd/bV7kVQWAOtVFuv6t25Hi9w0F5Pc1MiWfUVRF3gyqWCpyFMWHaxYqw4UKBURu2ltXxtxf0wgdh7eOf7Tv6bBuW2e6bcCU1knSSNQiZMvODwDggOz7cxhGKSCM5SCdOuW3VY3H+c+eFjvw9b4BHhqXcZAwm+fMeRafdCTlz5FRNGxrpva0ZS2ajpMRT6u+oDlN4T+B8VOS0rHhJNZffnalUIQkJZ69YNQGdjMY9PJt39M/uDWCHIr4oFQ0gESgpEbpwih/7/wDYoc1deari2/kz8njcfmanVVrV9cTyp90CtZr6WEZPA4K5CSnNUmcA32HaJ/5xvLTLHKdKWCBCjN/h7hFtbKXIRabyzvXYenb4Vd/kR7bA6leq0b5eplGHf9+iqxYNaTYP7WiXF1hqyx6tXx6XrbJZ0Kkhs0nRJ+cD3h2PU5kWiq3q25IGe9H0wGZ0T7bnfUL3YCLmjjFghLCBaziniHqigVYPLFFF8Wd/jfcwIGFrTn4qYNgkPHYquuP89NU7CEf2JOKtqJTHTz2DiyaJAcKC2pWe2gf64Yvj+ND50nMF+O5uXEeSVT9iBB1kAe7nkT54o2yOLf790a/ZTR7BYX7Gy/+JZW5VhEik3hvucASPW6bZ13oHp0Gb8jR6ojEcSwrXSMFra7pGHaG322mV4esovm+TF3Uy0dfhTUI3EboQFo5mDRVqrW2b48hNz5fIc2vPJ01Dqz85ihvgNGqKOYtUqMMHE+fvOHPyBtHy0zpT1ObqKRo5cjKf4u+qMfezxeIgJdMiU6/EsF5spviQB9f/RsiW3O2218XBQPRoUNeZZmVor3YSh+f/4LQ7UgE/efovjtJFVeyUAJeDbh3Dz20a3i/wBdmSAEglHbUTIje/u00GK5PxZudHxMtmYXbVMUoxXy2jY5J5I1l5XrfsBR/pzQ90M/6q+zkoGFvdRl+uVItbYS/IcYN0tFfunQuYuQw1BSn+ODcFgE4X7HyeGisKlGeiidEugpnDZ3zX/oD0K8URSEEtTKxDWnQvPBbv/SdL6IVxxRHDoadSH4fGcnd3v2OECLEZFp3/cbPkNJmBQQU9JIQ5V/FZWTlQDuKtipVn4KjAhrjSL0V4bggwBjqFG2GZa8CCP5e+6lvYrVI/vmALl1y1JeJ1ofx6MEDyk/yVDel7V1ioCiJlb5oqTqSTIDNRWacjRFB/Re1wrb/ok9NzGEG7JDNDT+uhmGE/jyPQ2t2WEs9Ap+KRqDwd0LffRHQn4+yltSUNcZ6ddZ5XuwlEAiuEMB0GmGEDc/NJOGSh91L0aBW+kobAAADL9j93piuFvWlhsdnMHHhryzrCjzx+VHmZ1cmlPoK3OR8NmxCGzwjVAZncUriLwQyyfjGDQYtm+FG/qwzf3o5fQ9/igkahpZD8MJ4h8XKivtjqW8jmomrKb6anq2PnqQidG0Fvh23HBDv7lWrj9/thY0UIlv+NJh0k+irQ3E5eTALl8hsmCdP3HkBeMI4ZNS5X8OnpzkKaZsKHzU9ldVKGi6bm1h/Xv3LBpXSk3AJG/FzCfwwlMi4Wy/9MmnM4UsKd9lUg8BkSw81vyxFIvQtvhpC6vjCrr+eM8D8GyyWoNsFrROUOPPdvsbozBBwweokrdwQNz9Gy1ADwjPA9U8X2eHWNYOpr2pxPLEmzQ0vfkmx/aI1Jikr3RlHqROpWb7QkbBQcvUzmo7BVUC9BA9P/4nD355NGM9eq20RCJ1XZx/zcZos198E6i+WEMbChJZYoaGn/VjfDNuZNX7iiiXboXBDxLhjGzdmgp/PvycjpxeFpoG3R22lrD/jEnl+bV1M6aPSaYqfq9uwq5sFTTpSHfwqgUDZfRntWhXMqup/7vK68ObaP5OOYRleXCiB43/Lf9zaGvko8ICWDaSHrIr6FpsUYV62vmTQDlqV4IcmMX3Zn5R5wVHWv8C3cWEzBJuzVFq9MAaa46OqITEBTSOex3JgbT5RPNef5E8v5UrER3am5P5bsIwqaUYqbju/5DterL+RRGr+ooye/aEizWRhrPUb9vKrbGyvrQEECfgVN+m+GAFcEyx9OGsme7e0L/k4Gt7DTD0AYfyV8nyGymPWTST3XzNndRF4cIIUVq36p6cMgS3lvKEc8I+AIRLvkjk/NKljniRNy/CS8URj4SGnMqUsKkmk37w6pfH5mn2pv/4Wpx6fuSiiH6DfFpvD7aW9RwAtGK5Aie+jTICrlQl/xQUU+xENrqSSjAm/5TqcFq1h8E8KME2mf4qVkcXFxkQtb+U1/jDjqM1zc/M/RY/gj2leQA6dvUykgP/eOto9NRuvDhlWY6qdT9CdvyBIcAP7HeIlS9fupaxb+Y/8Unr1ZZHn2FTgMQ1AJUnlFZ0hojyqV/0y36VtHORrSQl/Co9TzgRuUjz5/+fpKhYdhbLg18welyXuLgF2aIAQ3L9+uHk9s+lFOi8JV+pUHaXDjni98QHuoAf2NbjK6fyAq8H3PGTXJD4S2Tl8jD41i17DSYl1S8o1c3PEtMCPrLMDXaVdlKaqzjIa5BvM1MM9sDl/PXv6FzjxrKsml4EnQ7RSxZCAWk1bJqYuUygKGymlFnXdC5mGCUCcaGK1cItulfZv8T4tnyVwDrgrKLC++0vNZXv49RT6ug6gBEj3KWhUNh8hPejPZ7Og2FTUow9TM+JvVjLxIvkHaR9t5kEoARRW/qBb+XeHh4wqQKizcjcumukN5ynC/JjWJwiRj7v7ZlmJmGfDJZgwyZZEZ62JhB3wF9JxVcgrh14o4MtfvMrC0p97nSUfArzm4sQ8pnkhrroA3mHrkbNkQ5XWJMKD8wHC56TtF+CE6G5tGyqjaJw71DtHa4fT4ZaP0TZlSJT8CQEX4fPSrIFZYH3kARn6zOQypaNrHiB2vkUN2BzbMmlvh+Gaqyk/XnIIvkcQtVhfCS6uoCNL0TmvoO2mNY01hDeSB+rFJU8s9IVDxZ7cBmweP7Oahtmn18uh6sdbnNJso3RLbe05eXT+Cw8BprZbigmhXHLCIuiXNB8rwdux7TbFLzBIypEfbOdnA08ARycSv0H4ClxHHkKiR3fZs+/4igAft5TNnyiBSVSva/6IQpXSjYBhGW8EWGuSE0nW6Mu9wmsUIZxGD/AyJ3ZpcEfMx+DbSwSv8Ga3NMRMLkhz7/wBksfEnS3RoC3QbDivop8CqyQkN8QpyhJ//rsM7W85am0LxjksH6X4Xbfsdvc7OqRI+LFGLUoG/JCMI+lUvwO2df7qlL3yQZxYwsNUB2IYSkh62VK69PfzRb7HENWZA8XlWcG/JGn2K/NoumjSrfaTO52ORxR9WVaAm41a2WOGN4bda/zEa8Yi2WG9K8mLVXSqoNDR3A+xK5jo/FoYftI+z1JwkvGpsFGhmewhOmJOcUS2zpxDcmZHXkHoYSEQ1hlVqmUJbnSMlJED1EjobB1flXT3quUuo3Y2tFvSY71eDKrgIUSQIdm8gW4DvXlpfwjw1Ns1Q3KdJZ8D2q8QHmc+V/Mwse03SAxcoAB7acd4Hzhz7VfzIvcImy4MNizlw1kPlHhztl3dh37nDVkO8feAO0epDa9kZ3zDxy62KNJqX/pIEdAUHaL1BQRh+Lwfg4nsJSAi1l6ppsFUFManWF9yhvvhgeN284l7LssIeJLQBVFwe+x7hz7SpH/xam0hfzsA3hpXHKGIQwHFid/Z/NmSl2QacqvcH7HzyaOJ6EghYBw48+jz77SyaI8OPF8Tkw+9ndUkWtg0H3UKmEeL3wL/9/GrQ3KTbd0pfltJK7dFipDcClgi4X1empE/x/PAaPM2xfQL/D0eBIOzIZP0r2JhhktwHufZ23l1R6FV5vqXX0bUQKhkUEgDAJelY0nxdm+Qe0YXnwrCXH5mGZ/04APmwLRQscQmrFurBhdwvLt4xZV/mGcOhDuWn93oIcH+1Zde6q9xbcc3CmkdUHf6vXJBMO8OMY+E39HTJJVLzTTuVIbZ12K+CwZaIwu0ymLLcopBHU8Ft5xv9FINfEkw/xKOTBxlcyyrNWwzYS0g4oAeObP07V45r7yjdiyeLCiWtGoEslnjSDZSWbYNtqptRXIb4JpO+ncHICZ94aD3sCcwtarMNf7s98Z2x3ck75r7ohJoOi4i/PfPf/+CvoXOinf8eriDic2vfpOwEUTQSX59yRLhPZYrTXyQle0FUy5bnFHbe+FJDzS2ElhfaFKHWYPvF4nS+mqYqUnBDa7F7H2XEKDn7LQ4qSXdyjkbCpcBNd99D4mzmp3skxbGMMHaj+Vhs6mjqGY8GMSOUP+y2SQb/pFzSbLkjEElrNp3Hs0obeuTQHl4y5e6lZT1b7zgrD8DtYe/7O4IlY2o2sGLpcbxfPByJmz+Rcb5+BHMdTC3AU/+Sp7h23aG2eh9shrNHTp/Q1EOh/3jQuwkRtPrPF6QuXVuZ4PMIJb5tb+4Z1qOETYQp0m5FEhye2Etk22axoeP/Qa+fSV1Iuvamuit6+SuDcjdJojoDMLzVQ+NYndlhGvcbAzAKI5plByk/gr0mRo+WofK+DmJeehwYG8x6tixC8B2NagU/MUg1YAILVZg4Td0Fk6qj6NGsvjBUAPxuyJqT3psoE0gGLZ282ISzYBPqjOZPek0Mp1BBWDKl9sd4ffKKvEVfNHPlw/UhFOXdKP64/lp5K9Np8mNBP7YxG3fLBKMltuOSMrT8muC3qYS31wLrmN6kghy7dW+CVKfjAki8aVQlb5FTSUDCTHfNd+tCsbkun7IGQFW8W3obCwwg0ye+29UGkV87zfHgF3MXsTHEnkhBtrICm3MCBzVMinbXwJhFufiCrftK8q9bfU1IgSUBeYyi45YRJ4dXMpLWcGB66+qizKx7WcUQBTnVAgMsyDX+zfS3lJeEYoioDsGQ0E1dEowogOw6tGwtOJdJAwC6b/WSbXCLBHBtdwaUcZlFc51zLQw7UUYohSVGGPx+nC/ljfJ+l+rshv80yYkTXH/2pHMSAF7SR4bgPywzrsoC40rh6UJ9WY5k5uf6VosKNNbnG26P7YZ9TKhDPtz/vh126JPRIRjEJaOP2kStIeDFYMZEsMQ2aqfQI2V4mTUD7H5m7rF/P0MHAV+0/oGi1oMEf1QJmZrNgXLG9aUYyv+VuY7e6RjywCkdROqyNFk1NwsXzXq5ZMoc0i4sT2KObdHr8WfJTtjf1of/Xwn8Oe2bFmB2/RUlMZzzT1FrreMbaOZuQwmOO38qKiKcV/0KijoK+feikcmcvkJugseRdfqd1suQlEzev0l6t86hSOF9bnFWYtO6Z2iwlkb0hGT9/6eB5yueEpPn5VR4U4j6TIwlUe6717cD7IEjGXhgEadC5Zbrxl9ewPOlieBrhXTv2vtYf1BdfRy/161PBPdxpSGQ5Z9Mnu7v9K+1sqoNV9wjEp7e8eI5svDjmFKG3Ii3EXAdPcbKbN6BndXxa6uE4yL0vEaboyW5XvkoLx02yQHzqwDkdrc01cqE4alX4f3NrVlPzckC1xx9PJCocou+F9/pxCVSyKydzRcTX63x+Qc6lN2l5RwVWBGjBt+yDJnXppboYaTuRAIsLCf/KCShAsun3FiN6vzqGvWZL2R4UFYAl+3k97Lgik3i9YYVD0gb8tvhytVaFZURZcjY9wMS+beLAh+4jnAuDxHMMisgu/NpIenFgSDxd9Pgk8uPpu7pj7mN/NAXs+202TdpI20/5KWop52qrHMpqyV5n0TA+AXHn+uGuEw25E1e8dwB8PivpUfDDp1ev0ME+JU5tmDWHSr5lzpxcoF7qahODxCzfWibu3SAPInfvJvz7clwqLA7yZ82pHzIldFXsFENAlYjp0H35anYM5yC5eCSe7vrBykJv/hv/7Nd7XmmRitdLgqYEhMa3A532VV7StyyhshR6TEMytxi+9l4N6a9qJI3uIrUZE7QeUm79cLe9JW1oyrnWaq++1iVHP6UmUkTd4bgUyiVDfPIDmCxVIwN1Yz1llxgPxnZ8XiURhnQuP81ouBmF4Em4/tX2OGiq0cPOWh8vUROOm5RaYC8ryA8rH9l8zeOKafh4UQqROQ0Gs8ZYhFZSzgkZ+/j/6pcIJHrIqdStN5lD6pmdtFMmXx/MTawIoLVJ/JQKBVjn1JbsIupDfMgBvw+E1iya9w9TcBVCzAMF32HJbiLTBvZSa8QEftN8lDMfVrMwjcq8TPIXvn30tSjRZBn0svDUPb33PWxmt5DNvreC+pbJU81/OeVjM2SzBNaJUP56iFxpnntET4+eHDa3wVGSa7snFqF6cfb+uAUbmAXv7eAWZCVSO1yUl+QsI1OHMgRqRXbG/DOBTJ/iwqkRQj5yRSGs5Dp5HpMEW4fZgaMEisQFQpwUfQMnhces4WGRcnaZiVa9SNGkev+3UYrX4cK4yDgdhswLw5lwAy2wn0kdt0sDDtccxtRPkdgNT5JLqidE9RQrkY5uSWmpz9Q+mv745/EpEmcXpeDxyGZI9zhwW+F8oi0+iDP7JG/B5/UPbrN+IJXyVuPvRX+P5il16MvdPZPmRWfESBzU552uQol7eivgTvinKK8zxRLLdNzQlEd5xgyHDNLBQtqwJG8LDKFicYaT0yJfFmOL8wYQXtZUWB6qVfW5X+boTxvNHBGeMsK+WBsk0awX5zRn+jk6r2gmlKQrNd18qHXwAnwHefaaTDP4fU2/N7BjcNBXSt0WFy+ny/Fizs7/zzHjSYlIC/tSJD3sO8dajods6IZVAMrXnxTO/darpB5TaxZZ3ibMhmt+x3V1NW+8Gq+lTbFQjBD/dn7JSGb6qi380oMH+29bmYFKRKahXF34auv4Bbv+zd9NH+ABtmRTGZ/gay9BO0Xl5RNBrodCSu4Hal5ac670eUUJzL3crDa/ntwRPgEwDj+ABWfTsZ8fzuqLsgdkhFH1DbYZiNl5a8VmWROYYs+zAvtLhFy6gM+OzwqBsmblPBrSBHFIqo33S/cqQdFBayR5bfmzjfX9ivxvztLJ2etLGhDqbJVV2MC/ObiK+8Vf2PufntXBD/vK8lOz32up3s5WGC3B8pjW+aQgd2o3njxI7JTu8Q5o0AFzXYJU14346lzi4eeH48k3V4TmiGkaos4s4utxf77ZysNU+fTzlK7iJZWMMhjKBKbxSHwaA95ahjMnlDhpzuEp+hFPMvX9bUYDb4CpA5DLP/M4G92ikqJlpVyf9jzAUgT0qtba7huu5X33rDerRK6PeUeTFThwkc4+2RuVyxQFMUwXKhEkXGkC5zeJbViVG9M/e+YR5S43t0tc+dFsiuBbwkITTmhK8Y5za4mET7HQ96evr0PY6wxiXyVK1BZxW1/yKvzxdYA0Ay/8IkndwOazc0tiH0bPP5AgKG++jh3xHY8qDuT4ChR+d8Xwb4E07YgBbwmYOxUOvh4qgc001PHsQvitnuIBANzPYuTSF89eVZ4++gADh1sFI9lQGIkbTXL+KEHeCzRjrrbn9KYC58gJys3p+/dS1fwC0KSIVYHoTp+zAUYAnsTFNjexZ/BQycRsCS3pwAA4HqSq63G4rrhAlhUQqw21saU/RAdIl5fo19FUY0bgGPFuWWBMEgWrg+FoKnv/5jFC6cPNfWz6k314xsfPiy3/Dn3cVRut/xJ8Fy6UhvAANDay0kxJFfX4ElQBBFHGf3A3KK0qH1VwYcD4/hTExmBF/DgDDY+xcHpAbyk7xCb18NFJcv2rbs/jWzkOEbwxRn80aUjjmIZjo+Bx6E0W2tDZSd57D50TaDy6qAlUtny2IQiaVh0T0PXVPEMl/CkuS3KkQ4Er3bxjBHHQIEyZ+X3SuLrAeOlsfigw2SVT6MBCsIjh1+rMivB2T/7RcvbfqjelMc8fNTib+p8odY42Q2R5RIv3bq3b4FS1bbnGZU7gOZ51TPlKdfHLof++trNZWVWQb/Y1YnE1WPWbP4DxElHo1aZJW17do/d0ufBNBUT3QTwxfxmqBIzLCx1wrW6AAHiCapntIAblcMgC1xO23KxGx7cHSR3MeETk3DgVVLvt7dqjvJsLgnRCDCRNOzHJDl1LAD8CsNmmLzuHJ5x2t0U6p3iQur1as0pbnmjhfvIVfOMRX2jnp0p9Nm1zFET6b+gdE3l5/Pgdz6XLt/3uNRHIl5fg2k2MHE7beeKUs7VwAn9WG64Fc+dDYhZyg+BMEdPQeQIhos+X54a1WNnrCx8cS5FwkJH/CYScAxL+CehvHsG6xOvmHprw+5KWTo1pkvHzXbuz7F5/ia9t2i4FKQQMWjcnr1rwoWiKjsdshouBdrsYRz4BQFzr5gZxfWkUqrtk5J0Q85gmy3YgvtDU6wBbd3S310f+CvldeaeC+IH1JBDQRzLtYnffJrfgXk9S/8c4ypu0psX+HOpCduP2tNpM0IhwzIAjMOqDgQT5hkZmUs1ywFjqSopYGfiG7ch8K0vpaX70rWCoU0k6zYbJpEP2NWtSI74VJzfP/ZflOtLOW2CdV40/D2foSx4ya0JyCwMNVvx3pO2r8Ex8fcMFP+WHgl1MWpsFbc4CLcrYUrgGA29z2lpqOaqICzAGoD3NtaPtzkOa8ytH8QYI4qWnd18m05rmBXFhf++jw1fawxP3rO/zoBYiIvVXlyLsSqhMtX7mMpk+rq0Q9/dIWlP9Y+7D7hbrMvmAGI+Yl7WTH/QgT6FxWExmTTOT093RkXrMbaMeZcnqHbmnzwmXg95uyIGZ10DN3vI+f1ho1Ajb/LB9s+/ttr6cU2L17sxvf2/cSE7WNUNU3gyz9lEzL8WCB+vumFtD5ckxUy+teAtfwUxvudskpfEWSKLW4k+Z/5Dja7R5A+Z41KtA5eEjdhMJ/7+3luwBc97e0Q8pCaV7BuzPlLW/R+zO8GFyphk/hh06XJnWvXK+xqBHFqyamWZXSjBjx/JvWS5X+Ghv1jw2CRaLRFVUXcvrgnzOhVmRihA3MxXhf/m1glVJ3wXG1wwCtKzMidgqBWu7HkW7jyKHHzb3r4b3ZAk1bWSfTrHAYEnIhfFwo7u/4eVCUkrVUbDO0GYh7CUW9Mb6l1m3SKYw8b71x232ig1IlmVtimyrt9AYwTo5n8jPzbO1b3nRawOhVNlzBsgvPDaWHQqe8ISXQpNDNIB0hUrmcUq065icSv95l6L96ePwNFWzwZiFfeZ8znYM/il9d0vPpVDg9H8LgZtfutuoRKhUcGx/0vWtUJk+IKCdGjvrt0furYIR0IhqEMP3Npm3WpdOg7MxDwLgo/FTcFLpq3p7VlggGba2mLsIW9EctNc4Mp6CGDqodkE8P6rpjbSV9CRjHD+q91gotfXUwXu+IlHfK3P1K+GH83XazzHaw/66TA7ibEV9HZ+zdB1T+Z0MET+nUowIQL3ikRq5B588m8x0x68OGGgN0TM2vAFDKjKXXyqQu+Gz3/wTkdzPcyxvkoZmscU2/A6ctWMXWSWuNyzNrhNmIu2neyKG3+598Lswg97GgXo4xS0d9LN7rLzM8TmkuCm241+vdYG1rFkQupgzycrzELJD5q3xNwT6t9TViESXbD96OymwrPESJV3N67JIOKUut/T5jGSYZ+ZR/69x0T7iiKS0YoEfCyvuR3OrafNcZlvA6S1ifTIKlupBHjF2g49wvZ4dvQ9CUv+8bx4pRJSf/5MQIn2eGtszhGS613Eo3szYQLNIDO7ZdWSljzJpWIh/pZnC+L6KlKeZt5Lwbb9UUmNJgMgCE9ViMliLn4MtvguTzAgMY1bIw9fH+PnI8rLi9PQTqgA75GZA11Ecjr3gYkdrZoeHLI8bIKe2s+d3457sztgwLyFH0TRECNNoIieVDHNxC8E0wi5vcGW8m2YLfvR87AXroHwyg/xOnTt4t1PYgO9FJllmmTReEBahu+VbamElIavfRhC6yypXmH1G/YjF8Vqw7Io8H6ghMHRBdhnkA8VhcwHNgvm5ume0x+5Kv3Pf6f39hdH+nHKq9AFcsQT7jDYN8CXZXm6TT3goVuSOCAcJ0CChkG/14sndnHwQQRUoQMR2lBXyT7omnKhQ9hQhw2+b6XjBdVxoqXh4UjxFuREgfubLu2SUjEvRfJdUDsj9J7ovSu18sZgX6enL/T9p9fVSRNPxsvooQj4sz74bt2eL0P1v183vJzhS93tCeKvwl3ShVFqDMQgMrKEV9JyIxJYYt+U3smAD8XJKd9wA7lw+DICg5of6f3ueUUHUQi6JyxJq/XB4JojNoYHgXlj2GnYLBMgNk1SEbm5SSYJbwpsFa0O33g9md09psKfT6yPM4g93Ps7Oqo+LVY39z3XhuPaYDbWjMdeHmOZ+sgut1UDpOzkM3wUAUSGLL2u5M35atuhIs/rwf6BWfMWqUPJmC/fuJAlLE8atkSUaBUvtI3Hozo8staswjvS5Hma3fRpORc1BpxjPipQXAI0i0VQrxkOK7duEYPetJAmMNojUJ3sLehWs5+NQzI+RAkilBjWuVeOWmfVVzHb83SLABplVNKybKN1frBJDCUxSp5qT/qi4fP0sDRM3JB32sWT/fd6UQ2WX0MRq/PtQKj1I9Xehx7b618DtVlb2R02XJN7yEPt2V/FXTYPFNqZ+81vnV3fnLWOx6tGt0CgSlJNF26u6BoBZ7UE88SmAgZgEieVVTfkVbB5hLKJyq+Mp4rRZoGVJ5ZR/wd1kIAIFEpN4S1EYpI92lCvO//InDPYczlnopjXq4aUkymrPJQMxYRSDcraf9AEE3MqDCLtsPT4nAMYESD6HB+nQbHA3Jc+ByQHTVotNA0cXK7tl5a63l5tLKPpXlmG2qml2kPGQ4kkjxp+WL2/ZCvcv95xEHIv6qNcHKE5KUWJES+QSs0MThJ7/zXYUBDHs6DNp7jPTekXdKg1a+pAkE7s7Q9NUdb+fX3TkEUGzLwlpf+Sr6W0eIUo8q2w3o62AJE54zWfoVuX8oGY8I4cRKK4nU4pFENR3E5Ce27PXi/UPx5xgun347+SFrn6+4C5UKTuQSYKCSH8S9dRrNgKf5DGpG6sk/clu0ogToJgDSYP9Ib6AnxGMcP3YWe9PfOLUQPiX/DUKjWxHNebzQotGIoIvZMsu1tQ43P/6ESC7Te9ud4Z5lvN2qRbDVBVNxMl9153HH0mGsJN2jGKQH8puOX6srXoUI8RDRDdjEhYyc93P46/pjI6oXnYy8BBzopBdvdzn0sT2px7VvZhmXHClTELUbRhm8UYpyXfqOo3ZyhnM62qgjgmKu6dGLaum6GONjeZaGY83RX8IVeiY/eixpuB/Uso/XQhgtd/ZkW6Zlh2mZeIlR/iM7nx7tu+kTfeRj9ei+X4SOivCTvQn1WlwsKrxmvGwZLGKHhoJpVPO/dCGDOl/PrVfb2HU/4Dmmt+JTifJyyNWKMJjLd2C49XF8kPI4E8evQQCFbLabkhCQwmlu9D7cZuuIoiUN3jlQWIcC07MiOxWBnEYZ1NAgfLWkK25fzSYW2+FTxzDyvhtXG7zZqeVXYVW2DnlRaZttLXIQ4og6DwmP8m026JffZmNwlLIpSW1VFrSrl6DeGxyGwr77itntz0rV4/Sra50N/ClIlwSM5r4SQqNIEU0O+j2lQ4xtbopZZW531kDTy+X/1OW/EYOTiV00sukJLZiqRi3QBI1rjmv6BAcfVoFJUCxflbwJFRvYHuB5xoU9iLgCXDHuc5ntYr+jtOTglGmz8q+ukVjeMomrVI7lqQQiHj0HTsOdKVSX6QQjZSci0HWCWqeP1NjTeJg+Pr0JTSMvdSGlU5udk9DWa/AQsKwxKJj3kL3iOWCbfOf+IeW7RjYrwhk65e/paAsvcJalvcbhWTZ+M86FWFC7Q1rafO8QJBSjuserskMohP9/GN7xCm86BS6JHw/djs8u/dkaiGH98C6YrhEj2Eye5z+C4MhBUShnx720rO2/bXC2rHnIb6ib8kqOLwCC6g+3PVAs+UzIi5nl92tuQ3QSJLSnTRcj34IXINI2NL4+ejHmt9fxZ1dKG+ib8X2Ezs6rf+3VSo36h0nwSmBB/R6niyUhvoW1d6xbreekCb+Xg76thOaeX8FBB5vnb1Hia8tM8wLWRfMTTyOKrTDBbxho/TGfphayBWG73tsMAKBVdhMzoW8m7kchh8SCrv3ifq/wM2rGuxtc24rTSAZwiHB0l6Gm2b/4Mw8Oy4froAP5FhwePYEMZZYtu5u2vvlYidDscgJWLJRYSlFEJSNMt+3cOZfnsSE8h5GFSC4g/+egK35IEgVwAtK9n49R8bGne8jLs54qDFBRRpoCTlHatku0ieLHa1cxg3GSZMX44QsbzMAKaSYn3Zea+iOSGsZeymq3AFPXWvJR/lI1Nl+duQwCngiCMaU2jrM4LPE4BrtDe/fgn6XsLnkptEsJ8M3bAwsnqEajOQ4YPdhR1PLPvlPs6R3K2dGuZeC2UOzu80q3P1lNzw+CsqFho2lmJIld47TNG3vDVjIB21EWEEeeIractz/NSPWrFeufR7gI3ixxhOSEyr+muS1NiFMZNYVHGaTi9D7mEg5GWE3jEMvkjOGYWjahyK8idKn5XcbiEyiKflaJtk4YOCI0IJwJTPBaUBJSPYr3jp+TQh3dKV1e1ZZIuzvduCDr79H6XurJGIR/9dTCAUWMlvyQ8ZYMjcgW4x7CcHXK0QVlhN9SJcRWg5CRQcp3C2t648I7Dopo1lDxu8UESadXD3gJsahaJlEjr+7hHAEIeL8A/gZPv0EGaF/b9HJy37FMOzhFgALXq1LUjep9f20JFmD8JUjgFVLfVMvGXkiTQ12tSJ4XpbxMhMoVYvxbP1h7SRQNvoB5cZkK7wH/UDmlku3V4cDgFHcVChZFXmATl7FWpXCHwc0wZcux3+IZyQ6AtL9e0xvz45vJCpF0SMZWI+wAyast97hUs8KhYxN5jY31GqXJQiGCWiAIy0rZPkibvU8sgeKl598UsFF2n+fBuuY5KfoOAUIeyZ/+zWUS2X/wpdST6/gZhWwq3H1mIYQScAwI9bB1pyze1NQk8Wla/7YmdPyrw/PcZzxbhSKcJhAXNyvfXC8ufYwD/Ci3FzE20vFOW2ZSFCh1Bp1pQScoyIA//V32D1Zq/vFp8WjP5scDVYeNrTz/k8nty3JjurxUlELM9pbBGyOkRzAL3Wi9Gm7R6klhHDdskG9jXML0zIRSUTSve3KbZeicFj9q+YtddrAP5ei9XraSadqtMIhT/CrKoOepd0lS2z3HZ/6VMA/b+xkm/eEOxu5uB56DbttE2EwoPeD//aeCy9SNNwT1C1PsGyT2/HDYckifUI/y2gUcgrZwO0hCYznzlMFVsm3OT5ivCamc9c4HLwSYfmCP+F/BAP1FAXrzBnCrWc9euINKMbkM2SAOhDrDZfoaB1xSq6p1GXEhjsCl0XoR9Sr1iR0g5fs1jemY8uFGImkfp8vIv35WnwYaYoHLY9tFrj8I+hWYsUo6pSEY5rWbbpJWjZRB8tW80DoYJ+RqpfcCPHN2ADSaY5tsnCrkbMwzYT0iCA5YaMqO/6ehtA2d4m33avLA+l1l4IpzmWeMJhOUAO+vzFP6Gq01nRFRClm1j6fxroPtcf+h6zStX/cL3qWThlUeSmnlleywcIAlTei7isQqD1qRo6RY1LjsFqqIaBMaRsEJdGbeCJklivb9MwUK+gcV8spsoh+mozQZjwrwWKqvSesxDgwz3PUk/tbNEvuJzZ2JwCIopOzdkSCu1J9jYJmiGhZYeBkXD/Ebg2Kk0uWLhlWm83bJcvy5ojiGOVmCo6sccVycChM4ogTGgNHxKD6gHXpzXDupTWBTnLktj3U7KAEV2g0+jQc2qQ3ONUtmzSYYn0JRyPBRqqNn38m2D8cM6m0Y4MmXJLwf3Bc0ZuZfmM4al6zhESlP/TZgwYjRpM4dfIIJ3d0RmoJG6LWOPhHU6C8sFPfO31+c3dpxo+U7O4q5AZyYXINIfuMUrVqCnVHaUW0QC/HuEBUbj6mxlz8+MU/lDUsmynzmG6wmXz/3QMSE5ZqUOZ7OTfgBK43bBFh9wQMXmN1kGCJzTQVfU4Z8Lb51pFGEcQpLYbCBNSMowCIgWb6P7chy2ymEW+AyNTYFQzPJHA7q2W5TmehFRRQxZ+DLzLEt0o8mNIf7Xh+nlPfii5DhYZAfdd78Tw2lftzh+HgKn0V25gUku0eq4nvP+yHNmqteQvcxfqwD0QtsBSgqVWOeEMOKHSf64LXq3vPkQwWPqvlhW9zhFduEQ+vjleLteZECTZJTMZd/2Ao9KCNGblhZzq77LoGVZ+7a/Sdcet+MwQxKrYZ9Hw/ohcu6Te10tzEQ6r+b5pqg9/nW9DmZgCKxfiQH9iOOM3R4rdUFxdfb+iegUXnyBY0SyqZB0ydE+y834VWtDgyNZ90iH8Yq8HwEBzGKbgkyMVwcG54mOV74+eXYK7qvj2ZUL8JBRKR05wReLH6R1wUrv8k7QrsNhQhCD1C1hBePMWFDIG+GUraswO3gQy8lvYS+7WsI3daG+/HccJEtS/I8v4sRYZs2lPQzRi/goyJe8w7BPjmLxatuUd4Lf1GNXAaEHCHffb2bZcgXnZppgb4J8PayuVxO5z7Ar6v3bhnRxTaXDEo8auMMLLBWX265NFoCt4tfFQ3xb6hpjyqJyD4/5nJEBQeHn730NWeTJH9/mDxLdHi75CcyQU+b2Xe+PoicR61sPvVrtv/sqPBxnysvYnnqdnj8OI3KVAL7FnlTgQWFJE4aUjlp8o4slLm32jEpasLgAW4l+Trr6nYDnk50qKX23p0kHGeLbUP0w8LxysKJ/6teFWHAHq+Pktz1khILMCpYVcT96rifSiolLDuk7/uKTMhNX7ruB+Brdo4LXNirs4fhUGb8I/fUYeWdUQuItHZlFBJj9q0Y+01Fj3g9iD28QLGMVBjgwr56aSXNHm+SB4WDsT5IaKUzuB6nLoeFdPyCgoLT26j189gRXHwnOJK1HFxKP8ALcVKyJ6cYXEWQBhywciD7QVmyMz1WKs8KIAxbEZ3mRw7jGXxh41gTRhrBsMY9dx4kYO7kqEIP3g/M4JOIeVMSQnI/gusZqtFnb653P22UwV2y5c4WvTqUsODz/4T288n2R5wzPKxTkYy8esMm2dXkve0VDfsXbA9mWbIl2O3DVZdsNOaLsC3+WBS5xwlAkYwQOhO/Hxqt+3vhTm3OyoQo+3sVaMWR/49aVLLwdw/aIyGt7JVjkNxOa0/BvhnknTHSVI5WNh8PEfV/2MKvp1cDDYzv5JllKGYt61idziqsAE2CbvOrFB16QfqFcIT6woKML95s8qt+D1Q3XlvhDfi1u2EN9ckmpyMHdJtqxqImFa5oD3MCdphiyaBXErg4gUHjxFEQX0Ap91Uj+456eZTEKy3sJVQL8e6d98IJbkj73/GBavZwz1GU/Ear083LQpgnxL+k6j1hthJWLRIVwVQO3LWfQHLNUv+ZidcZ3xKUDSn2VKJ2TKKGGZV8hHYrkrx0CLUxJ3w5oOsVXDYxE8TnmJlgUtPryxtZi9LS5axFNg549FL4SATiueQO/frkQSgp+Jw+B+qG/UtoXRAvkNMw3paVmZcmzLdhKCpxZX6e1f02U6a4bZvPui7k6YHMhLBzGMR59wffK3EXYs58luJ89nxCdOc0lnTZNsaq57zeHl6pK1Y03VmqEObi8y0tIhgnn8WthiEGzekVeBjGiHt3t5EnUT9YGMRjrznpFLS7hez6/RWMQ15ZYrhmJUHfbdOtLTdYTJIfKUChGH3snSrX1P/0gm8BdT3Hf9XNR84z3acxCjnNmlOMyAvn+jcft1nndFjnYqMHfbpch+xKOTJBlipgvctCd83DYnWORvbWgOHgFuZbbu14mhk+E+hc9380BfDCPGNUAndY5pGxVZPAGXazHKe8ER2wILSSFzq1a09VRLx872yUIucZfRkpQ/jTf8RlqSCkWOvlxHTDcUtw+9NeAmmXWJSyd9fCjnx5lKYbUQVFPmiZYHDaKitz55JxeOr2+Elby7EaLEQMCjDkJmFLxaYNBDd4aN8MLyEIcQ706t/lu8cDZNoHcFuKaJBXKa9KIkk0yUOH4nO2XiBTx2U1NHP0a2rnjAwEm/Cu8Ml7+Usad4m5FZc6ZnTUfBMU7nnKW+J9b8dEC6UXK9LbvLIhHzUJybPWbqDWVCfGcU5SDSycgcJbreDNgooT4kFR1UCE6PJDJdl7Z61Z5+X7FOtQUBplxIFFHhOGgJZnDypn38/bLQM3TvnPNE7L6K0MTsuUqa+R3aJoeSEPirhde/5WVsjSv7ubzE6mTZQSDvAod3F/hBk7TKqTUSIdHbS3xdQeXAURC0kIxhSIMhcGAlUeIk7TC0dT+E7g3vmk/BzT2auPgtA/IruwdYepbsyBBA10rn2XwxiufEpCZsPtnHErMr6wW3KgG6y8yrLAHfwK9y10KCSZGCu5Yv7uv2yLJrYJW7+xZkB+zBOmTBknFn6HMfqlOzTR4b7/uocpldIzZwNEbdFg7AeVIsZZWRwXKiJe9k+Tex+atDZfR7Xg/kehMyYUhV9KlLEyqgsO7I4hZAVqPLa6Qpx6BMfLYD8PUvJnfQK0Ysm36EjfDAarQRkFxBGvdPDgT7FWaMlYFAjLQZWVF8Ee/t4lfeYQKiknUDr4/wrclwB4zb5sdNidfL0pRDCRjh2YMOa4vTfgd1s53VGCFq4tLYXQHzB3417fU3pUrnR4rrbfmK9W51x7dAJVB/KyBv7t97TfCnV9dnODw/BS6dEIwJzqOeuVE1SwJnQMu/hphTbju39hygOkIKL3ctGsAcrCYrgftGSe6MlvGoC27C3j7vnHh6jfaOjZUrOITfp3899EoGcVZKSsZ7hcjiTbR3edvXnwheZ8RYcOsiRP6kQnihSvFHkfdyHzydXsdX2gPZfaFlU64IuDsuGHF6sL6BnNUWVluPAhZRCx9qVUFcTexIo6uzmS22J9SYA7mBU+Igj4/oV1zEnolKItXNo+r6ZT9LdRFg+CN9dekVoT4qRn9X+LqRchCbKDYlSkERmj4b3CGgRef51yob+b8pq7VAH45K5eNod8gpF2ZZ0rEZTgDvgUOspdjF0r5OKboOvOrhkNThaS3AZhZ9Ztum3UEzlSCYjz2uS4Z9/WCv3bvL7qmqQcVv3SvFPfNAP4hcbYuA3oLRPprX3u8VC8TsFfXaH20WfFXysI1ksjrpY+rgUlHis2V4JcKFpDII80mEQF+bH48XIw/e8gzeRhTa+qdSlDLLEkP+ymxdQGRoi19u1hlbhTJTtRnvX5GrwlEK+xb4wVz6fPBlXMuqJwbYRSaWALkq57C1B5AWGEvvqYS5PO7yD5Js9vdwFTB9/7zbgIXOF/tci9uzfgL7t0Z7jeD+SnilzHHovwb9VKvH2w8wxyV9ygpAG2bT8PYgWvm5bh78zZ2xGTjCKsA5qAFIn3p4VM+ZJNB23OKDWkzEuhlvy3XKTUsf1OdwOnn3GDnbQCIaHwtR9lkDoWWIYQutkcmUHF0G/FRtE81/jiLJPryV21I0w8zavpgJUqvV842Z2UfBoRGD7VDYywGjjYSJrbcZQYlp4NBUS1PKtcpLAK0eBZTOS/51dbhPYIpVM1PmKeWH0NffEZQq54FwCb24zIdfbo+eXw+xPn+fKZ3hCdDlTdFwIqn29/GvAKEVNCo7A/PXkei4UfFbXRstzw+A/Sd7f5Nuq9NPaHw/rRpFM7MY2+bh7K5AmO0jHb1jfUsV38OejsYvnDQdwAZY69J0j7OctSqj7UyzjSHYwuerpKFQo+wEd3vsroUBZGopqijS88f+Eql7ZuhFKKyNntJexB2UMSSTKYpznsujU0KoEjivLo4M91eWwQLKQaEl+Rx9+cCG0eUpSQnusF1kEbVxyADuFX7xe6lA3egjUsX1JRQlkkBssnhfCH9XiPZWt8cVdodIm8Hp3e3OhcS47vhWPr/wnsnquCBtTbh2x/LCJ72Wo6KXroF5dWPmm2kX3V2Jda52vTkgNhvU4IxUtp0zrM3jFeR0NPP66nYsMg2CfX9/Pr7ADIbmxP1EvIz6T6zWoW/CDeVydKrIbbJsLeDqLhaKdIRQyO2KCHZuigqIgaxWcEcUnbr6PwYg51QVId/r5xm4c+yVomAT9t+tONHAPq+OxW1UX0cQa/toaTCZCj27C4YpXbW8kb0ui2kgUtybn/xqW92axGU8ulJNVIWj/oRVHN2zdAMiNzqQtPaIvhROIG8+UB4t8+xbVzjyzfp3RwyODgfORQ7qQw/6xpO5m6Koyc0SuxIzN5NsbJSXfZ8hhfWJPkouIpAMgskBNXjOqlDbbFt7+gidOmKbQATqMmYap7IBRdOrjlCUUUmKDwV+T55owU0K7KKXZqHNkbMsAfC9SpVysWLX5rFek+ptDeAG51DTo7PISCV56j2mIvVF9ps52ZLdAnhOXt2ZoC9cJtmGpXlwvehYPaWwyXwccR3L3luDeAcUzvRgV2efEFooMhNMYUdIewJ4LsdfIzwMqErcgRoHBXKnXODtYgMyPetkLPab3c1rC+UB50jbsvHOxZf185RMWYPTQI8u/coBXoCyqLBD9NJu6my98938JweQQ4VKEDa86/qdvp+GsNVX6ETnLHlOwT6cvObTfwQRM7lOCufJwCQuzMFIUL16CtHmCOPRGdr7KCN2ZI1bTcqdzgtdX5+wMWayxC7mpOzv0hiiZIYCPXeSXmcIewgK7U+VGNdG6WV5qraLAc5b/GLxXccf3k8AmPP85HGGMOPPek74AUXcJgacBm/LVdoH/sfAxOpnElFJiR4OPK94uhzSJlcVyJfDR4T6MPAKKGxtkkB3lY+u3bOK7eNgcSJzrjReC8QSUV7RJvP5oD75hsov2Q6gVOvbNKOgaAmJqoLh7ZiquSDAEBraoiHzI/osLBXr99p7ohG0eaasUyno5k6lvmp7ikn+pGAOZlUD+OngGbR26Lqa5pw2C0PDlo4zvdejMzI6Mf4tqyqa1ZsXPPyoKPqGa+glZn8rq9ZBZJt7XV8ZgX6Mzw4tk1QnrWUhIDKcHb94SsL9gdFWAHle/OTxBXEltKEwOwZstr2iFvLQGMV0db1KCu4DpzCe5TlMhwxVX2+3/R+UZVlohcKcthTJ4TeAaPZQsm5v0EQusbTmqmWF/fxBm3BciPfg4j/PodkvH8xYvJUwOjN5AGfSKczdziGikCRj0jBo7TNchIwdx1NrThsik9nK8hye1NOBrivjdDrR6osCV+rVxAURXzp2aQYzJdtOPLckzZI+oVE4qTSGzrs7ziYaqyYfPwaRs468GD9OqIprd/H9IzFeeWR/uGxLxdH8vW+JJHV1W87jHZ8MIH1smGanmsD/xrxy+EcLC+rIpcwc0WU6adTy/EEBKLTQQjBC+NTa40Q95DU82U2uvfNhMmJrvqLIn1Ga4j1bTy2zDsHsT3tclX6mNre+mScnzm6yXP8XPg4lpamqG9uQZfnStIJiM4IjHMh2tCrNbl5vsX7scRF2N1rldS/az2x+NjfniOe27XCFa+2ILm+fvUuwHsPwU+mePv6c3PVI+toK6lbihhHCbHgl69+kHXZGsMwkL7Wl9femxpFS0kY8p4M7JbmUd9zcLqNTD7EXjyKdOMvtpMQzqUkX7jRlZq1168ppP/L6xbw0JxByonIhcmR3MYSPmCNvyiawnJg4XViSjvXDALn4d4jVg/IY2/ryUk6HKV+JvAG+fEiQ8aNMOnmdweEN/Hk5z0mMDsGW+ZCSIfOMBDsMIXjKbfenCpN9mr3IOM/zKMyYndDy3vStV8LLMPnG7tcbHflNcaf4cRdpHtslVo6eOZbPFxws5tGDa5+yGFsDoeTw40b+uhSbeg33+wdifkwuebslpSNet8ym5MVgh7SWnzC0prBNhk9UgmDZgoNPL3jTxXm+/cjF4wZbDQTOKVKWDyfLfrisQMRQfYoWZTcqiwdLuf6q86qBVmfsTTpuu8y47TvMFBabIp68E7HnxahgXFxoegWcT5ZmfOpp/uVO7L8i75PTcM70cMx+aaU1Sm1PZB99ypeqtzOkUZjrvOrOqOgMd76mnBloTroNy7BP68MGy6prhCpVSQX9IeaLiNwahRGESp82+C5jN/lajL+kUCi+gUS5PWroUmtBN9xDL5ploapdS0kvkUwv5lIl7q1muz1sVOb2Co6Z0I1ewgfveECwkVI6ocKKI8oaPvKkuWWz09Zujny8dnsIjEaf5ji8PUrtHBjFrPrFbPnL3bXRX3niPyBbhGLP885iRcMyXjoFa5SOJBQd1jytPT65bd7TP+mQxh6XsahS+OzNAeUn1Nhbra9n3S+4R2GCmUtRAod17VshwRny3HqbCqhs5t3iGM2TzwyOR5w2qO9Q4W+DADUU11Ahu+vUHOcfAHRuTPj+pA4UY65dSOLKaBbdY3Lok/vSRuA3gEZpPkxJb5KGnWP3I1Le+6AvbgJtMYTqY/rCM3D9q7vey7JwD+TrHhTF2Q7rZW8nL99qYDUYB9RChki3exwJ/bUb9JdbrdEtp1w6wU6YErmyaPzkCeU5sPLCu/HjNL7YYwdkqJezxxFJjpcS6JkS5kL2basfzpCKAbk6+V1GXSEalyLYPmERwRgNxemshmcurKy02vJUuqMjWnpNkm0+YBGruFgqrnY+P1Z+Y99XFJ6YzXb5ICP6zs6Gc2whtXoKMJmYyMEWFO9a8rWWi5RydpOuuvXeusY1R99oMVxrkwGSZ7Q/fUML+73fNIqrIR3VC4NrUYNKBzsCyWGsN2vrF+8LrH6g/oVPO5ERv0qSX6jO8x0IL0ImJ9Pz6enxxQRHJYF44ShE/avTG5l/1h1drqmdGH76LyYVFLdt4ChnoDKn1y72tpg2kp4lcmBvUAPcpa0H5MAvGws+y1bC5mvqXHGl8plUtVR0UF9OA3Pzd52GrA5EK3+axCgg6t+Vg8gPjaS+4Cui4KfXoErd92LKrIMUHeJtnCcfT3afEStXUdqNv6qRsk4b0Rq+hXhtxh8an1+QpB4QxRnoj52ojyhcVokvGdclOdC+TVYLHK0YSe2MU0152R0rQJ8a1S/f3ljKhCD0T2rRrnBPyDpJkyMTKK3HUB1dcfINwE+hsUhzdvFX0LBqSDinCwP/GCbNvu5ryBEAZPtIIkMGMTxyOB3x1bBc29EHhumJTbnYHHF27DdNyztxtdy2eBFavkJMqBZ5jGq7y9Z1C+PU7NPimUVAalGxqjYO64+OW0kuyAu8j6rF+pAh131u6mxuAYG2csfRqOrcbYJiCR2l+EEQ4xYcYjcsjD6j3aCMcqgoSE72ldPNP70i3sqXBAEygK5fhex27k6P0/pvFQ94hJSOnv3xz6Zo0kY8LD/pem6tmU1duAvkcMjOecwwBs5DTDk8PWX3sf3wV72sRdDN+pSlaSWQlhixGnYb/0g5k1wJI1qJ73dnaO581r63XMzED/7t6S1s9CkCBnyH/tuCMEJCx9dSf75lPq9P9OgvaRYmsc5ZBj+VefETMU0wkPnIKSmj91GWloTnJGFRDsLiZ5jS1VmdUyZBzVz7LM3tdRZnlslyC3ZGwngRx7/4qkiiOzR+5V9/4wrppY+br8egX7hKeaQ9Ipug6EiGeFeNtGt+vJDgBV0xT4R/pQOkH3MlYD2d9eOQvAY0nf1pqo87u6cme3DwZ5jynxfxQ+6e7KoIrINIuqEUl9FKbg+EucDwZCSU8SUTMfHmEmGoDCNCj0ZQmdDv35a/TimjbJIqX+J3/LK4w+ZR+AviMk0QZJXlxGz4nL/Q7n2vEj/zHW9SNYIieIoKGOioW+FgbPIqqVrVUXrS2CtuoWSWZEebIq1MuTf9QQVTSWqOPx76hhB9oWr/Yt4VvR+Cp8oipEY0evacdmTgiE7XrUg09qZmY+wU0BKDYlRr//9rPnLElD1LLUZtt3e0B6uEp/jr+ttwf4oEu+YT8R+h+TFUo3e8v35k7+nu6JIRrzLRURbCQJd+VUj0XApAXMPVqtrZfPozO7RCVpfsQzyS8pPnX+fatJJ4AdsAlxLFecLLcIRpqUo/rzyZVRn929mXS6WfA8J3rzLNTmOC6EXfgjKpz6La7OO4V9m98FXJPnZwkI13d3fUmePrOTyCMWuRilvBVmhn237NjOjxei3gO2vuh/6xR+KQgeA0I6Xm1zTelTOLaPOndkOWPefujc7AasAxZNxk0ZpwBOb/Fk5ypj2YL2sTYyCimUYR0J2PWD+rsRqXvk3AsThQb/xjrvo9Q5Mc7oPaHyNMG6gj1LL7nog5nW7iRJL9bkr89xqisRGnmlzO9vcgDaM5Lrv84eRvnUXnVJxYBkuvkqlUzhIJnZuu6+jf6UHtd78+AAHgDFDgzaq7iKIfIyJBrP9N/mcW4GsvgLYE+USto66eaGgab0nRem+Ot6TuzSq0S0LLdL8PRnDeBuWHjjMYgt3zef8a36q5sX9sOUZb//IqkQg/G+oPGSKr25pyyI9KuxT2eRVkt1v3T4YrZxJaaWqTBnuSF8awyebPQFPFf9iorAXgzBFdekr6SBfQrdvqNzKx3EgbmeLLj8ckAHKRTo6CWH/r3aB8FhV6Issws/ZwUrpcCiqS039IWSQ7Q8evXqYlWGFlwpTCfMBQUmEb3aOrmTlY/419OdUn9L/igyJbhW7qXKlSdv5P65hHq2UIKLJch33N+Ed+B16vcJPStQ1Bx5HBtkBRbxneX3h068i/vUb0lyfv8utue7OCcGlcxvYWxrEZ8kErVIgzNnAyDRmXotVAWpiFl+/7OA4xQSJQMRa5aW5nYDQoiSUwmcpg8prrc3v7EKwyLkQq4gtHsDJB8Rnd+jAsGdqxraWy72gD+wLx2i8lk1Pe39d7Fis0emjUs7tsbejd4BkeXE5946/C3LeL3Va/6AhTvMpGZGBnH61h+R/chA7o6R7gmywV7Dxdzu1yztGkozwz0HaT3wESXIWppCkxlWwIXk33TWsCQi0i2X1X4e7M1DnDb8MOnJK7q+DJqPCZoydT5EjfxrqpgjWTRJn3SfO61RQf0HP3KvyVjvictcR/1q1jWMgot1LGJKo8J+MMktEuj9Zs3+rAypBhSbbXdTh4rLL9/6DPTuKNv5/l0oFE8UOrtTjxpY3c6bVv8C8+zHkznDXjZarx1KVCcv1EkFthS8R1eBAXi1g9QrfWGxLgd1hhoJQcEtyqZ/C6JKGYm4emWR12YLo/jEYWL5o4NPUhTCcHrdfkpnjRQedpn02VnTpPsVeAavYf823WgkEmEUfZeFs8EeaD92wqnDP/lQguUfqT/NyCz9YDQrG61iCSZI+U8D11nhI55xY93xpQE5E3K4e+dx1Mv2NC0HFcJJecTC1d2fzRjZOd6K4wYdTk+Q4utuqf/Da8wzhMlcdtzE6iReP+PjnzNBfq5+vsQP7UP/6AsiPcK0f1F6yBkRAyosuDDSOd7Uh8AWSg7BCn29VPM/Nmhix00e8LSsbMsFQOBx1bpqntvOI38h7YGjSxETwTVYTQ9E6lYZFBtGlOf34FAitJwO7gGIhMV8VR314eJwP3lMLAIKvTgDblbrfPLAKmvkbsFkimOOHNGEt4NjQX2QxyxgpaCCB/y62+ndOtkX3gkFTLxNooLHxjpJZ5+sL938BeOrz4ZItPR4/aI2zAF1ARVDMIpbjKwH9ujPiaystlKSbv4MBs9dHGXpM+/FQNEQn5+uLS1pzZeztVYuWPLTCj95iSdV/p2kCacmrewe588OnwsSyMjK6dQmR3Mwtwi+tqpBcRhrZkw4QmxvLnS/eSP/OTwIq3Ae4oRrL+8C0P/rrXnCh5xBM9z0pxbn26jcGsAa3gsQszT22p2YHcAR/rSeCx05vpwU0BmTbv5pM4/jSwZcNIZm1Hk5dxZ/Q8lRGGH9n5UHziqv8rSMVVOoJ46plkjygjfumWxOE7VGtsoob0eQcLYudjGtvzooEPAxROoLw8jI5gay/UH3xOf0RFqRl2IUkECYAZksjJPBNr76P5zVvkKIsFVq7IYXm3Wj3MtMdfGmDc8MRmeA1GYJLEPXzYX2mtFF9IW5+pkedqkUCt9HHEpOg/tiy9sxboAl3OKLn05YjhKEtNszYGITx699XeP1rJ3fkZqkjewxyL1AtrxsbAIzU/QbU33CMAAV/5T6wMQLD+W0NrZCUdZIHW7a1ZHzhonULQg1bYJ/vHyNe3Qw+tLAz2jl/k4sr7Gcmnk0tXauH/mXkNQfC2S/X8kuq0qdPRbL6u66MUxHbZoK5s4aXGAH6lO2N8zO/e/Sz8HY3GMFjdeM46G/YTUptRInoml6vsz4ALm4VQ5nM8r9hMYB5yz5tdw9EGMdZ2mfnthWTvHR7jBLT/9jxxzuqCKy7+6Jn9huTscW0HaO6rwkb8XqgUoY9BYG8NpsPoR0OyjSxqE03ADtfBye1Ew0pRQPJmD325uX5EVWBAyL5IaS5+L+WElB11J/FzjIYhZOXHZAXaZmg5BDcDBQDMXTb7f3itYjvT8GKjOEKOtUD6htBHSF3f3nqUU60goz6cQW4cG72h4pwNeR25Pj4Co8kL4Azk2pmsVtsCiSXpifAYoFCrluj86giizJSWYCcD9pw10vlKDTn8gGoILDPToY4Zdk5HJYZuEMKDZ0ZY0W4v6PN/oGBzSCcq1u8eGgxXuVAZAj4wSq1Ce9OKLrsh6AxuM9vfmPvWH7fyICHIMitsb4DrdpWJdkzZRZ+26gHvtKpxF0OIqiyIC3nQDmzcKcfh6xhfZTlkC/3us1DjnMmBg/VB/K3ASNlLXs1yzgrEsln8h1Bh0I4Yn5W9DjrdEGFXmjJZh9pzSK+PCwBKWfeALyalP2kzOv/95j2xliWpeZnKgtd9QRusKJY6q5FdC5KcqPAhSH/8Sg+NXdUzUXjrlqSCROXUUH/91NWKm2fs08mk90Wo6Y8gu3Y/ypgd8KaJpoNWnz5vuTqiQCh4+MlAgJ0eBGTEZgY/TvvfSIx/Mz+BuGAgdQoO3HwAr1p+RjMKxNvQyGn+BQGJnY/tmsfnfLneKC/aBNP4qWc+K9SYgIN8knsFjaINfXyKp0ekAxQgEMl0Pl6R3hknYR1i4oPSKQJXH+2Pj8vd/2HxArCjix6MnbnfVSlgyRmBACwM9Sk9gDRZ57PWYbYPPhO4cCdioAaG3HpwdGG2D7Cjy5WaNURLoqbH2TguSOyVnttrID4q5j9awxXQ8HZ39qs7X0cjSX8h78sU3303/4ySbdXanVrY6OfiXbPEAymjZjWNWLisYyasJAwldb8vqiEVig44DmcKZ8pVR3m5uVYOgXBf8E1YcpmHYWn/V3tuy9JQu12DC2WQ2O2fFZjEbsaCqsYhE6ybV4PUwDkYNsAzvhGuCQOK5dKoNNhLnZQ48AesBxMJMoK2WNqf8aj+17fEfnnvl4FkWbs4E/SNwWHlCnIkfOMiHUCEC8AecVsuCWLsmZt2aFIlq+S+xvMsMffPtf/RotR00LU5cHXjPzyJjSonVr1P9tfJxUcnLyMSXQRJQgYokVajv9a+vh0cJIeVX9qgvm4jkOQvJfnt/dJzNMdsEF2LKij/qZmQd+wCu1a0/gCJQxrKu+RJMk/yQ9iwIQywaGBApn3q22+tI67/wFH9SOumiUYL1SLyOYLrzf7hkHiMyft7lph3vx2XPEKIZhZr0EA9UHAhjcao63ymHRv7aaasSE/Q8mk79CVso+nx4GwcPBLREigFnCIv2D9UADhYBLRe6bsPI2owm0JFEUgaZf7AmJzKCBHogCl5nepUEtKVTFGuxlcgOGfGxH7RUZaNVZayyvscArDjTnTlP+pWbmn9ewgSIVoupyTX+mvWcgTt240Vz3JSe63ECb6RkDfJnEDS9jzbL3jr4Lp4fpwiOkIbXNJVgtap+ByO+/EVr8vHmsLk8PJXnQUoBGFa0XB8FiIp7K9tFZQxMpcu8rQFFU2Q+ZYtGiXG6bwNNmL2pKagH/FH2J354q0+ZY4H7nVpnwNxutvDmbOfPejNEBC1iIAeb1fn/SzRVPyZjat/Wmj5C+1q9T0S1G7clqiXdJzWtTSCLiKCArrockFCQiz+v3mY/kaSAEEisnHn1lnG82PD0mPSEji5us+gvkmiln2IF+SrN9/OGm8UsBOS0wOPTiRdb1f9ZngVrp/NNkMeM5+N+6ALu/56Kip62Kcyc4G4SjigetSatNmEdziVH1DS1xaixP7khzCN5TYku1DSA+8Dz1lSS6ObXUchKyi5UAba4s6DmdFI+BG20beiuSR9R2RvB8ozMxM8vPqeysMl4Gs2Y7rhrAIUyl+tg3G5BuiNFbBtOS6Bujk3jdakWS2SeBwcd0mqQ1w4L8xs19HYTTpLLk6zss9JrvlAPG07lgwXAfwyVK8E5bX9WIOmKclKmSs/yiYm2YO+vX4LP5oN2kbo7vChDCyCj3wGKs/OUElrwG4qyzrigt+LugUEDx3R8nd8Kyo6mHFWhveYTYJQrrkfQnBPG5LCr5YPVEIzRB6eeYqu07fcnP+lPtGI/fAT52EEsdMHcdETOlvKE0SpxX2zLHhuZp+OST44bJLAW2mExTmlx8KPAkoRwl6C+E8bYw6mk7bqjhjxK6mUQbZG6od1RWRQbAs+CtDnavLzhLwGUWw10I4RXt/rOIk0wv5s47IsBcS81+hRjMBhLenXeqOe7CfB3G8DI0n26ZnEaSz/7JHt8KZfmgN3DM4bjWb082/GlWak7/mIfb8Az3NZGp8qKZPXgkMkn6+Jt6vrivtAJitQA5x8rkBK6LBa9Kfo5E7Z9U/xftax3OQKRrvtli/YLpxk8b4T7VeoHgVcLWaAqZkgksWRHbA6tmf10qOQLfx7EXu5FdDKymRz12GMXKKUExpr5fXd9JeFk2cBtCnMz6u/GkKcTY/8rOtNv376a8fo4NIVHxKS3AmBPV4bMRL1md3jQ+00448Udt9D8fPJAmOOhQs4M01Kz+r7X1ncysrxPv53wX16q7/gnVlM9A3FkU3Oxqh55jgQx2iH+UVlixO8EDa3fqG+fvX2feT9SPpiH4pqUaAGWCf1MIQRbbWUVSF47yov46oJ4oqEmB53PBf/Rq/KnaXQ/NtGycpgsEJPPjqP8ykcTy5thuHXq0MghI3C+qSvkbbp/P0sjnQNeUjY2UoM6djAtux/n1jc7DtcbBfUZ67AkRAvGWDn0zmiySxEUpq4jfkjQsj40r8BXIAK/sb90hTMjl1iueyukda3fb3PJYcg17Nndp6Vp4aBih8PtTZhSNe/vlEBPqCpaFW+S9BjNKPcSeG9LK04Fg+BFnGOdcdqkzi/xras0440tTB2PqXOuuYLrvmk3+ofB0jovzcO2SyVvFMZUdlFD/QhIjA491Xh/AJVePbKN17PgShnR3BLnF2/Q0K8/pRWD0CxQd5Bc1BbLQyCnAT57UeEyhqbSoMX1DTl9buBzmR0l+0aImgv+s+wDljuX5OUczrn1J5CmRGLz5y2MUo95MKA8Xltv4vpEr7xVen5B3sGsEmsuH0nzg+a4EJfC//AYh5xXNg75qqPAaCAs6riHH1qz1paQsVndHm8C7C7OH4cxLN01JLymGnP9VNs4uvqupfjTJf+0VuFv3RQdAAREMHtoNmy4WvVmfgvUIey5bovD07VrVYcBhbANLsWB7Y3/DprHW6NOqa3lUiK7fjGybX8mB/sIJUlbiMXZ5iplea9Rr+3YPT2aXgTXjMOlXSiwRdrxKOJWjTvEnr6YMu/0JBbGg65HEsZWH/3Vg4MkR01qhTZ5cjMSNNS/vhBfuE1mOhjGJE53jy+vt1XwD0D6ZYnnVgz4dx0Ai256uvKr32M5KA1fJZwbEryCDbVt35RYsXekT/U5w1JcxTZmHtp8XY9Vup/cWa3u/9mDA+81Yu9KG2mv1S6MKjCkN3R3h9fXIIG928bvYZfisOJqdhpRvB6jCc8fLqNcpUWCo4wzJ2WxLS+K+d9bK7t9hkgB2j2nNFXuwxK92huOARuxP5xQdELk6yBvE51soKRLZx8I3Gh02zr30/dTg7ACf/puJWTNAxJsJgVJKVJj7vrqcZY9taySRFLuuMHWTj6brXGwT6BbPZdOzM/OqNOFYJpebuBfnXzkz8qykEXtjtyeIKPY43cCOJDTEaB+K1Kr2cCPXgg5xGsSg8ssagUDn3/AOl+Iy84SbXA/O34vn64ZxIofyKMwZM2/jn/LvTq0ytDp30pA1T9LvFr2qLq5gGV8INQtN+XGaKO3XBe6//3RU6ME7ctCJccEqCeXBGXUKEUzwJLPXu86Qtxr+CK5/OS2krlmkRQsr4DDb/UAhPnxpYFfZ+X3zPBz7JX025nk8y5iRLub2RgAu34nZUR0xgpVCSJWGyAoTGgIpbPONCYhRXVbKW09qo8QzNE5XvEdol9tf7Lg+YjOliOTv5PoNPkhd8X3IrOrDvQDv6nOXw8Q4xDLrsRe8PJzx2fd0CuFYgOjP0o4jSOD96TpSlOMhroUG6rbyb6qp4RhycJyBRxibGA3WLqU4RYWnjX5yjnO72SOOLsAV/Ky/TX8uqjAgTgrsHyNLf93N1DPNAtk8nJkwh4ASlRiPqdca78YnnhsO7mWBe39/rz4x6uGFl4Tp2k4vKLbWSjtmp/qR3odX+PE+p9FELCBfwzHsGHCLFTft2RvSayq53JCP//Ed3dBflaBNU62TZ+st7iXffjb8gOTL1JrcbLwol+ahdO94WiPlBeBg8Ppd2oajDdlCUy7agNjP/jR7ucnOMztafj2Ref7JFPSyOih1xk2tTfl837dpdifZ09a4XMiPlSSANAfVHz6yYjQdOBIH1BYZRqBlqY0NX9al1AQvDF202mXVjfphM6U4MUquzUFJ9jM48by6P0axoyP8mTx8o+3UDmDOQq8+67Kq/GJYA7KMneLd4qgnZfguPcqP6NT4QiYH84CUkK+ZOBtnH+VElpchBeE9OB9Sdnis88XHa/9Xsq89sZ1hv5dl4pI9PI1pbmj6LQURs7RReK986j9NUWu7JlH/T1AZUaYP4nCnHhKZMzWb/3Nm+QHy8bT/jK4mF1qtHsfu7zi88BDa+ZD0SHML3PB8nxr8kcYcLDSX8xYQtHyPZ9dndj5vRisB8Sq9WUPKhrfTBWsW5kRcui63JyGWp3X1o2nYkeU59El6I+BwOnx+NapzA2AaowxXPam3X1wO5osT8tdN9f1RSlJ+uezh/XXGhoAf2ipXefj4/OUEhs1Ng9vuNlLW/5hj6qqvbFt+pcTXQeD35MI1ck1nkSdf1YjBkGSf0R8nZ4Gr8TH51ECFzyOyI358eLHra19Q1KHBPVu7kp2xc1V/5DjLYe11/rsEoxPr30MbWPD5/U6MwCQBSszWzPi41LxsFj2PBKews8h9poj83ye3p9uwILCC4xRTAvWQd0NE07di3xY0PskOjj6Ju3d5rxGM+9OjyOWAUPP4F9LHyYBSrL32w6aKKrOecbiC8KovXZEMZ+jeI1XBO70scMlKuowJly3fgf5YfZ3ueSwY4VTX2Ai7Iv45om71aBCOlDW+qEksQW972u/5tDvRMlYBjuJQpBuBKvd4P9zl9dHy2QO6WoHDdV0KV3g3xrnndDaUEAOSTBSdCx/hcx+ZBqFPAOuKaiUi5tEeODIZV+cq/Gzqi0HvGV3GGu9mkyowCw1jL0V1OCV3qA1LFKlQkDmvnmOXx+QZ1oc+O8i3RbeeLmfPAemJRMVlT+4xCHLq0ZyZiTIrDUOdfQKUhKHiAsJ0Aa+b0qPaUmumt5IEGcpsFoI5fibZIP35ZEvYLBfvj9aGgCkrIbCrrahIin9kNoe5jUSwmjUGrsrjR11Kc7tD1O43jKo21hjUQieFuN1wClUK66dIjPI/nVPujdr9blzuqGLn26yk86WHmk8O7LvZ/ZV1VmTu9qrc9bVs5bB4g37actG0fQQsGUYYcRnxmbRzva6FGn21F2nb6n+YLAincf+NGMXa1GHiLkSn1VrYtrFBV5LRh1Bp2EgzZHROE3qMG7x8V15sHB2k9eZ7XwlwnqaiwO5vo0nox5AaBnNMFPJFjgyQfz9vs0FN9JSXSrAry8bkO1GhXV+z8Jdt0U2znKEW76G+qDa63vmCd8UplDnoHemYWSgXAooK9rEOajXYKACmZ6kWqtuLlIESAge/IcSVN9QzxsTUU8foLVEemTqbJsf55TjsZqI35KeV+aXeZErbhoA0Vvnj2rqh+VNMvYn3wjXE1d0c3bDH/6mNpKZzMaXX9xfeHHRB0EeTi9CizNoyWMXxFEH/33OMppSrzYu+uKz/k+TLFTqhmagv7wI+HQ1urZtxOXSptvhkIv3o9jIARQhVNsvG8/hkcPs2eF0KY7BIGjoO+KrtT5f1Jw6AR5vYzCy2bf5q9pctoNc86xD+50CFmEKb2Yvx1NEqlVylElTH+Qhan2Rnc1GRbzZG9dNX+6jIf4W8mEhCzdIRpTywXG0nOEJMkkycYkrUP2Vn+TRn46Y70N4gkfZVJ6VP259DzEl263wIJMUhZGeBOPRC1f62Zli96Nr9Wyn1Su7cR4Ikjp6Js26+RSHVpfb86ZpN6pMPG16U+OMgpyrUbFu6mcOAkK+VBE/SM/KgFh/5abrmvC4P++uG09kEVzCEZj8yYkhSwgTAzit2QBhuDGPB+7s023vLwcr4J7ropeYll1HtV4ZMBHfMuuPIDBenojnrAf9fRfP/BADVdBuwGdmg0rhhEAvx1G+ivUnlS4ECkNJhhLj5F2wuG/8Tir6ArfxAxOijVEVHrnoOpHDXl5qQKVs22BhkafR/ULi++WNsKXKenCxTVZaur3Bf6u3oVS4qZrI92FrYI4QZR1LPfW8aPIakRlkDw+6u1M4iZ5AfQv5JfRO0s8+qKr7WD63VPAm+BibYsqYLpTzlM3QMuk9dGXeUBZxB3I6vUeRHQLeQEPpF9Y3nBvOSNslEAZFNHNHvqxI87w89K9P5CbsVFeofrh9JjW9A4V41EV19odEMxSmryXkgsLD7GwCr4g/BQcryImU5RhuNoZoGbDQgTxIxfWx0wm7BS4mUAKbagWWR4K6X5pULq4SJGce1o1ZYeJWdMJMfab3bT9Arkb2Q5JnUq9AIfwsnzyF4nkzYO1mh5bLKkrRatP2LIN8f4GPun0fz6c6Swb1lsZ0kTeuV8t6wW92sQaV9O7IoGtCmO9UzyKUbl7GmSGkqcK/BK/PL0/etNymMOcOJGX1EG/voremFdRgh0jDQ2z4Gjxs5y8sFdjn/V573Jg8mXa6lyYspTqf1Tpi1pnri6AHiXhwrfDaLAli8rEsjZ4rK77hUdQYL7uH+XeXAze2z2NLewu/9mwjHrXQq2lf0Nhuq53BojYKvlT17Vs175R+OI7az+7pMzjxoupbhoTTz4azhdaps6WJxoBHHUY8b/iPrX4dyl62mt/CaoCNFJ5hOr8OgOqb5VbmQ2SBC9RHSd8zmxX78sJB1b+OZSOmhA7ZxjMNdobIPL9Nx7Hra83qs6jtffqybxi/ulaT69u103ZQixLPDprr8H3E+8aEyOgpm/ftTMK1B6nj/yGD2V4hLL2eCeSBvlA+Wh1wudjn1KG6vrO1YEGfBFXH9uXgzdWSGF1vBGtfjueL4ku53O68MQtjvMP/X6eHTNXMjsm3+dbWM0dJI4SvPuOqWOMZqUBOfLKpbxX+rjP2kLcUYmI604Ns0hRX2fdDcKIjJc53XErqM9BA+6vFwF1MXH9SUkRKjbWpukgwSflUXOz3qu3MaEJE2/0v1vPAkI3PXyg4X9cY2dCrO8QVdWZzibo56UWCfdLwlciIQ4JMNMiK5aG41QB4+cevmZD2cwf/dXMyUebhCzOwaVpl/9yxyBwPANiX+Jv1bQ7MhnjnjJL7XoSuPXhowOj7Xe1rcOQxBM63frqE4/6Of+EWFTkOHSWqwoc3ClqHOKy7/ndReM+QXYI1Zt/bJtuT9LmWklz3Wt2xwR4y/HBrxCtiRyF9rOTwmUWlueX2tqN160Tu+7P0tYtMnyB1WxHI+rFRMHdIqXcRpryrJD/rqMbP5k9/VNf3k9ey3/L7FbxbuEUm2w3MxPV/zBrl4vyuRTKDAxee474ojLQraLaXqf6yUKTMA256bnNsVXxEfrWMabh0UZf7c1JLY8gcnxZETb/aBqdvQSELCVsfHBDd4AhYe4KDLseWnqXhqp95WZLLy86rBRDfmWIHwBB4WxtderLodtwxS+yClAPTh0WkI9HnEs4azezkvaMr67E71a9a//KMXsqaQNdUErbZQ44f49af27kgyelKsjZoNl604oBpkoEvtp27XrkuJVRH/X9vcqWHA2U89trhmfRG3W/ARWMZKHk9ZDTpZGRR8XqLZeOAyryYPdiZQhr4gNk+uv1TdINWGe8HcrzQ3YKhtb6QIMKrNz2meDtm/6ZdSpTfpEQUc3Xt7+lsklAIGkvDEGLmWzIZ8ldD2Sk6ZhJy4BB8Ek71G78I55Yb1xa+B0hFY+pt7gYjrTBJX5hvYjj7Sbuv+ESJtCg+EpoaNt0W9Yzz2645XIX342nesnNu6dn+tyykR1CxKgMN/1x8LdvR/mE4zaImkETXcTlFFQtxnJ9GLN9nX3rwjf9oFqlMyqDVGKXkc5iqf1+4XZ/3gLSyGuZGsxetP+i7MICaPGVLNuzGsZZcbkX7U6eqBUPgjmdsFZBAaJ+FfQ41loDZoBsqJLnwVZBVXWjRNNfLPcw1en4700KVz/PvT2TKDNwPtNGNtp3f2vpbEGTdxYLdWaTs1jQt1k9sZg5lgg/nUND38fadOm8NfV6AfuU2nWurusjet6tkcdgF/PWvebYFDj3J9JLw0rReYsf3GHxWOtdZQUnnQmAF33WYi7VhQPpekgLKvRn4gFyjD2/AbEakJkkvDD1XCiTfSkpkKASC3m8B9IvRmXC7SK/IEukS/7m9LSbKf06LGsw20RzjqQZQR6kcUyVnx06ms0PUiWvR5EZz8qNUraskxFExw2PzTzKt+68Oloq2y5SXz58QdLqS/9XUXkPdk7mzvfT/5I49ydy3gB5J0EV41yO3idH+BmP0DOz/1efx9XPhyjdYycaWsAOSr7IhowT/tMfJUAPWdd2TfWooFWN7iMmJzVgrpRkfWJxDut7vmaDTrPjiQ9cXEZidnAbPIJaPPbTQCrlSx6yGMOXryX5knEUdAAA34uMUaelY9CEK5aW/QkFIY9xqZ0cNtGex3bjbyG4ra0eJQU+Oyyt/8HH8HLN5FILiEr+JcFZqKzWXPaESoQ5gK378NeLnOn/pqS5ZzfSe4LHSMjjBGtNgB2FcuO0YFOtyasHgqFs/VF7FB3NG954a8+vvzsnTrRy0udzoTjFGmGQomd5ibgH3oJ6/ijUwv3WL1vyL/ANs7tF7WpX//dcKv3F1Rxyl7nezJVVv/W2ke+Z2k5ENn4SH7UY2Wmac5xDSsxdmzPEW4rtiCZ3f0sWf8+aRtbne0Qyye606vfBv7/A3i5QaErNdeDl3KsSgttAuCO+OXAQvxtQocDA5dEPI9syv0XATWGmUmGvFMxE7a+4kWAZI0aRdClbs/ZhZ5JbqQaY3aFH1+NjO2EBKjDhDqCF3RtJCASbjVeX7tfgr+Rb0+zgc/fp3CNf7O67Q81mAFNY0WSBx+nZUTvbEATbFhQT8xuPq8ZFU0mT7eaT7gsJnMYXUwXh+I34dcUcr+eG90d7tY0TnSzti3TWplJJ7brt42xKocc7Gdf61c/sNWn8MzFGp0gxieWuJch44ONCl0ITBbq1BFIk04YfOjgl4ukeDdR5YLnzYSPmgmjEcDw1EoVpKE1rGUu2YKPWHPACh4cWVZrOrmLU43i+nRwO88o8lhXy/BZ79jg6WLpOXWrUKxYbklVHct6lJbR6p9jsZj/17KBveyicqCnyfYWsazXj+Bly+yQ6gTHwQaRO2ue82IaD7tJvpo/jifT53WVQMp/FnhGeqfiSKUba5DiFCFJFmmIHXq8FZa/3hpCkkjfgAplwHd29TMz318c0+QREVp9POfR7vVf6slNs58hNqbqANMiKDh3/usJPiEHCpmGFpn1KOtkz16R8Tk1b/JNT56KiAyfaKPer/VxqJvZOFnS0ZGSrdbLGjE5Icjt3MCJmVdOteuycoKcUF1a5cOH2zVeWAtNMbrVA0UtJOF9lMfSHKmL8TthR1qDAYey474zfl4re2sIckRDuKiZIH+amgKxriSqnBjWuejI8UptyajXc2MzRUhzv6wIyxID6D95Uz3sHyPOxBDS83pBHnWpKRctLRllqbtzqJSc9w91ntAMgI89ESqyvodv2N6v+gBf8oV3WpMSI/+cFrjud7GsXvGcwkSBuToXLMkeDQI6Ss9dLESywhHOASdtAzMEM+vvKrlUqXVoJU9Fw8SSyQ2zWqSHs3nfoz8z/SBVGGfMiHMDG63ML/6X5dxoh6zkRtV40BJItO0IY0F/QhBAY/oEbz0mPaBN3Z88jFtgCrZTjS5IhIINOtD3MMP0EGClLYeQcA0cBz5zq8n0waomJPvbhM25VOb2jwzuVAW/Cpwmo/PfWr1WJUG2pRX7UPS2zBesd7WCKmqupnct01RXsapB/nvVNPpJv9ZO5S7zhXT44MlJWYf0UL21cvJQysymzRF51uaaK2BnRBnbtUSR5E/cp+q8qkZIe5rpQ6AIVoktkXUkGIH5/mqg13AYUpHPUtf5tQ9yfIK/yMVfP3elJS2olDYcAcEYqTo7lMTsqxifj9RX4M88PO4SHjsG2Cy+vo04wgQIiXPYlRMS0H8RWhra8MO0p9jiGFEWHC0UqTpQaTcIgEsRccMkY2/dpjYRDoMKe6mHb4Wp2Z8+1Af0cWogBFeL4kIvE1SDgQs/mfEWf7+1/0uFy2rS9hcaGIoop9DF0h5X8tPdNkIokuInBWXX3+66feDhl/U4yebScmvPE6ambKf4B+x8btHtDRLE7LW6EABdZtzdrn3Ev+5AG2BsjHmDduvgvCiIpGQrrmgDAeVLjhSWBKd9srBVDMKCYOGyWrrPTwelaGv3blm9UqIo9EeUPui5kfllt++L1wvIvLNHOhm/IR5ZhdLf/9F/7d9R8uB7/ihKmolLoAXMrO3X6FwlHPfl91CxdCPAF9MEpUv2JaMYV9Lfi2j8RJ7mNpgixi/ooj0P63OyBRyADaj65oibU4a8u110599MHrFcYJgNYZe7GMReqBPycAQXKL/Yr1R+v5exan90E+0Ug2Lv5zNLEwHCnTM++HTfSEzV+zP61y/cSL5mR74gq2T+yvJs0p+57JTORj9eBgl2WNtTXebJ2MF277Dbzbtp2lOcJY4c9FfrxYhb6OWuUttSiEovM+gv5aqK/WCYZ32s+s5nx7rievLSp7USSQQDUvmirD4Q/7CxWzcc/WIp+oK83FEaO1w098S+6htWNQUH2VDZ4ycBDhIgmoyGecY1ZfI3F/b80jmkwYaUSPwk1V+H8f7agXayIJaEmII1g3A8//CYvhX24I4+5YZfEFdi98gaGUNSTonbxZazUUTn5sQKSwuF/aKFrJgC7lEc1t/cqdUkgX8bf8Gy9CANAYqhWD4YaeNvQFrTkqZJ2smHBUoHBCeq1ugYxmQZO66qr1w35ZZ4HsM7V6EyRsVmT9JhgGCVwIwXAiZk6de7ZiM6gRgSNvA+ttTl+vdLvHLK4TLU+RVBKgKDnZmWuVO3gQ6PMp7KXtOhIyseX3hOVRa3eNTih2JG0E8E/6VR854HXyuEv3ISw5DBIE/2rOwJvq53zzk2e9GdOZlMXrWrde9PM0iOv+XsBF6wMen1TNxiwHmchim+boe+iVgyVSepo3K7pQcBZILjNfKGpIZzRKuhRLPMEQppmax+7+IhRb7sMwLDQMy/8argyqGDDylVesfvXOvEpDUbnTf8b+r3t/jpVxmDml7RGbN+HGZ2Jo493H3jkNWsyogmJ6z2r+ssICn1yO3P6LAS2xJs8e3d4SrM8u9GC/Oq9W2VH3fJ0sz4uqHcUK+D+xhIbHsrb0XGEJdEfgeN7qduyd2IULtOWIPv/cvyUsdO3RxEV/x5wHAylDg5EE1dUWyNG+yTau8ylzTl6Cd7n7Ym1ExgXICqs8/6tMrg/PkUo5J/OBL4ogfXCTayUONUzpptgfFQtWZQxQaLfaTXqZT1hv7tvxayrGFPk5gE3SZcMGtXk5lOmGZ0hkk41c5PuTUXyYaPcZeWKyYQlTS0VguWdX4GDmUpNN+hV4ly1HQo9Gt/j9ll5H3+4rTylH2dNO1mig2Vc3mMrkdoXSNsZTmOeRWFWZNj/yHj4wi44ZZiuMFq8UL8h9k/Fc88ReOxDE4OhRhZg37NYMjWFH2e9GjtjBXw6tEnweh4iFd+kzJK3WOgi2FnANgoPg+YyBS3AwQL69/GxUKvhS6ZXd0QpOPlscgLPR7kCnfeK8sVQxUoB4L3LZnqv+pONj9NfSGjuPLTCk3HBA/CqGhfZEryTrexobuonwsMfReQjyZdnoYIkzlhEki+TEI7WfvqyfaiiSFvinM2XChLXJu815YPRyLzgJDKyrs1IzmXW/s09k1+Nl4ksXEb7GnNe1nfd6oIIGjnmze4SP9P5Nf0iuf5pPQMzmxw4H+u37b56ff7dYmsEuNZ30tSejmDLyID93y8tu8QZu4QGRsDeIfNAD7pcarnc/qJf62Q2U6o94MrKs7jcYaaXzfCxASoVkGj4AdU4RgdsPH8VToV+l/xX1A0SUkYiyTRonPDHERWA7jlJOfhiXx4xWYo8yecr5cz6vR3QcdpdhuBS1B8wdgnVoe1KhVecTyGdAY+y3pO9+AbE59qAudg31zciPVmecZHAH1BRPxWoN/siKy6xX8jc5qQBNV21u39jXGZ09I3y6R2a4N9fVAAog5XfFA2CvgJSKmxdWzVW2q6cBDzIgZl6ZobcRExo3AUFJNJOMhSysPD63ZywFSEDPhnCDpj0GQ8WEgs/5tkgEsGrE4I6yVyZj00S2LydzqNUUPVwW861JXiKRqz+C9719l5YcaiwrFTVLDtNFCOhMRWGLLBN8DdBtVdGjiRLGICOdanNFb3tWciX0ar9tcThrYEq1L6VcVZN+R/73pzh8VQlsl92ayXeFV90zrubmZfg9iMssH+6EcI85eFghWZh/FehKDtFcctO8WU6mTsCzGMMO2u7IyBGExY9eKZKriP2PgMCou8KoPwg1IHLldfjqKsLLqeexeXpxkjWFwZVhfe3iR2O4URaZttmBZKlemuCMYzbep2KIG4QBI5MA1s4vk5kSqQBmIKvYo05S7+yhMLqe6uXgtn3tNObzA62UIpOJaJG9vc6oa8UpTuwgxLVh7czv3kggidMmVWJArqmp32StdJpmX+Bdsf/+Ei94aN63MzQ4exPcrmUDK8L2e8TAmWXYz0xZ6Wv2ZSViQP4uriScAfjyerzI+e5EF5zniZ5h5wgwEyM4T4slkyMHjGVLmlqf/uudYfJDWxW5iM3lXrLVqadRzrYuyYlJLuQ5twX6FDgpH8gsBYoXlN8Ev6LvfjErOh+EHjOtYeCal6Sa30e2C397eOWpvkCIz4bxTPVzzZXJxW7v1hw95rACegH6pf+rKX9bYp2VEtOO+W1SefMucmri348Do6l75VyfdY/U3b1sSbVqEFoZ48GfTa5mPzcoAYrYhKWJUIQ88Y7YhhTQ2bssB0UBBBnafGs1TQcE3EmS9DplpeXB87eyihDuD6PUESO6VlH4Zuw2TbmsjIUQ0gnVlHVAukBDmaI5d070c9Dd4BLomqLon4m661PSsIMDauxChxKjX+kIUhiIl9eStBBHd4n/6e0bakQbTqpMRtK0BETyApzG5uCvQFSJyKeE/fxP2+fz/BXPxjt7o6t9q+4Ps1+7bkCPE0N9WoIkdeASUWwwLz37vvHxB1t8qvj4UlSvT+9NcRjceVXzByCPkJi0ifasb4e7qKphVuJPMMiDm0MGADaU+RY7uhpcijm4BSkk6QH02NGfD2qdQtAJU61+7v7fnhP4HibTC2lO200CoAf/sJv1uK2tKPGLBeqVk7BEVm8deCenugDcfOsM/15zHUpY5mpsTP3+sI3i0UTDBnDYG217uJWSYScw78AnrM87VjkOf6kfPvqV/+QBGP+0k/dFWFuGsEhpFwbLnNJkhi5/63Quc6umVe5l3vKlL34/TEHca0ql7U5pQeipfE/9vnWmP5mZtTlKvhiFql+Kr9l4G+nMk8guaDj23w+mPwv3JOov/1gaWJJxYEUcLgxn93NFX+ElBICa/LYYBnGqrKohXZT+uMf6ryOV++zmos4VQAgEQAjFDr9f+eq1w1CVPOxHQcTzdKxzFcrHHzX1YFH2bip3OSAz5y7DhpRRpJO4P/GHUbiYzbagpxzOQcjpMTzPv06J7C38oUCiUomoG2rJ8TKvlmL4TxBg0CBKn6A+fCfliVtOHmI77uUGpqSWJ73it8hw1khyZlgbxuiXVLKPa3ikFKxLFN/PUfj11Ng+l2jdSJswXsvNw3aUwdf9JanW/Az9M66i60Czmm4fw114xTX+z6ApG0MH6QGVfk5kHK7MGywPCvul3bqWxe1VIlijLfCXVjFNUyig946rmXIhM1+oYeCVqWdSpUwYdwsgbGlDZgYcIznWUJoX0cKFTFzRSME9Ri6O6+X0cOWJ4iW8dadW06nfbMWBwdv/GcS97Jj69P1R/x31hkVuYZPi9IrmXqVLYnR5uCLsr7q5WbO7XcVPuVkChnW8H+bJHqcC0cxqAQToc170qqHUsu8B+I2ZvdXztPmTScszmRFHaK6Pu5t/FveL3VOXHHaLCCgMnlImIsCwgk1NX/f1mxvsySkk1Ljd3qv0QwqA6qtDuQzOOyq437gjYf8wEYYKokXyd3ogDJXn1AUkwYguDrWa8frjVphoVnSDlwbWNG1e+4bvIkcebOF84x6N8Cr8YqZ+O/KcGr8XoQ4ThxBRKFyyIO+dpopu2/gn+Sa/p1TB+DKgAsGfoNRhC0kMM5aJmo/gjxp3PKYZ7JoRJHlmPmeVQHVCu0oCEX6TV1yN4cpvB/0kRaw/R+HoAIMIa21K8huhJ19wUBUqcSntdpa5s4znnsNkBRIPFjqn/VGIMhTKwBzQn+mAHqDwHjMJBxCMgVXiX+63GWkFZwx2Z7TwqIQ+dRb+dH8TpLqeZc7BKipmQe6KHzLqPEvQWQHPnhgQNOnaSyYyp0LNLtFQesRAJcoy2oFDFvLHFEuADXEr/Smsr9pl76jQ0CUe0ksQsJBZm7VMMYNfjpR1AZLKeGQISTWUpUQp6wXBjdEe/LTOM2JvhlU1tQSYR8Xgm5MbB/hygOPM6LvmGuq535N4XGu6SYB5zTYQLnQiWsoPSY//1BYyao0QM8D7d8Ur6z7dOVANQCoyq+fAvQGRxPKVtAklY22IeSjtSkv3WhGmhXOexs3Mnx/YWbk33Ml0lhvqNuadZBidMj/HSiOx3AiKhiznjKQiHZU9TFloF9+BZQjbpZNXYggIenNnFuAbx9XZpcLzjjnZgpQSYGJ7RfiFoco8Y2QGYD5pWoKM7P5+fBVh7s/I3tem2kqaJ8R1KHuEMCCK/3X/I12RE1vlL9ugV5FofoSOV2nugr+SYpiAtAVE3kzNyeUZcVUpZtMLw2/uUrIMpcrvZZFBD5fK2o1/92emRTg237ua1JXshU8D623zYYccYl7cvFs2dUKoPSMtbq7dfvGfZ2rJD82T4HG6XEDX0YAdNdFwm5y/sGkEqtcizxk9y4xYXcsl4uXQXc6tzEyKZhWWkc+OcVUgd9U/BejH9mZrOcwB+Ydj8VMx2VsqosL5wFa7zMGOOLHLbtkBRAt2u2mvz3EdQI9pvuSXD6OmXuFhCvpQicptPPTzWtfTFh6N6L5esRPTQTTDFcTFCxUuGzP6KteQGQ/PWuuwcXRuY5xbE+d02Zug4XXIG93FMS8vHBRDcBcqw1FR1NUGjYCDL4BfQ39ZCSonCK7FHxUzffACNmD8UBEWHfBXDbTyJpckHBAp8D/P2/YgxH7dQxkZAfSIx40uAwyS8qY8O2f1y0xz5WvS7WgVCGaWEk+VG0lzb4bgtqr3zUy+M+8kuZGAP7vsyfWQo9QNA58fDDNnBPuThJ0yT5aQXQ2id0+vNnvGvl9b9Rm7zA1LtxeM55M2L/hXUdyVs5zD8tpC2D01sDZ7wusCAeyIfkEEOy8lUfCqUlnvR+oP+x917dbqJpovCv6XWuuhYZdEnOIASIcDOLnIPI4td/vLJdXWW7u+tMh5lv5mx7bwkECJ6cX0vfPx3HS8KLDJq+wOwW0u0YeJj72eZfZi2gpZbTSRjgb4SrMNp9B893WSX8JEpawL/PlkYOv22hW1c8GrFLMX7Ckn20XvqJF0C5rVgeKZkXsGCqcAZTmpbGRzH2L+EoabNbXs0X9xhPlRyi3MBePisWBoTxiaCNoSBT0GFUNo7xutE1SPHIW5vLNjZP4qIG7rjQNOK9qF4uL8psovdLihza2oIw+F0n/TFRQr2xBYLbqMetem/H2/Zg5+Xhd1s/UGml2JMK6gjYYxYa4VLssGnFPtlG4A4i8VpTpahWnal+tKJHMUCxixZdqarFEQrTy1azPasz0S8oAzPKUHqyL3ccYNEoVCd2D4Kkq1FgwrMu5ls8nhOFqOGB+UJZ8CiOM5fZ59qo3TjMYSgb89IC9K4++P0V3qqbOiydxRXAsamsl5gkcisaljKckNXF0JseVNvmH+Fo8ikeZFiJTKriHN2I62zDbavYkgiWdsb69nL3kEueLIkMnbaBtj99qIDHJJd71xpaDiZzfydw2LZxs3IuLB6fRcmnleoP78guz6nEAreY/Hb0iz2InvG5fdoBZhNLpdJ92snlg8ZkPoIApkCaj4ePuZL7hMdQvxPNKdvAONC5SlfQiaAMdFCZcJI046l4Cq+6dvvgOT9xjZaxiU3jU65KExAxEIzySZdQrTt7bFI0Wa6yXHRCNOM9nApxoekjsBT06aQ4oc2eMit3AS+XCpuLnuwjZcE8sxN74LQWPMyj0pyzJAKBmmpHwEyGmuToTY/kSFJJa52EQ9LWPpWOF5VA2zy1RnEq63Ybhy5o2FzIMXd6ETOwqQyUm60HUg87KPwXFo/TQ4tEeXw4YrTH6iJ7m7V7yvVDiQtJfo1NRDTR3pveQ2+tpKBx2Y/iBwg5PE8jeRJlU1wGUZrdcls88vWQl0/d8b6gc1PO3EoT5R1HBl1dfagG0nOaZtWwkYvh5pvJzfnxYOTYLDCMw3YEOoBxoeTcYy4Vau2w43y1gW4apw29lPxLJG/dgYTj115g7eVx3FddQk7hYb3YfnyImq81W70BlcuVJNfukehyC9NoAezrVphOAeSUNR/i5nS5lPXoOnZnYRUYtiKq3OgqEfYSFw6TksZmLoPEbXYjUZfCXoXIf1pjwiXDIiNp/NywSj6zfgo8rhWtjHvx1Y0A33tiErVN4RvFkPcJr3cJm/uRd9axdZZNts1uLv3bQroDBV0e1nMADcZ1uJy5ViJ6QEDMxhlENUKepSliFM0p3wnAJV0mqtQui5wgtnK0ndcB17iNLSTWEZSU4wvZ2ic9KA4CfIoABCjvJTW/yfejqKmwoOWDzrFOjoAu9crO8nbDutS0qHjhxrB7SYXU1K6fqUHI6IZ2O/jPIyQ3ixqP2QyNV4Jqr11SkIX29kGjNpBBSBuGisL25lTUVgiw2Ju20mVVrnUZYMeVQte7yLZInume5C4FtnulZXSXaAnQZ6+mkh7nDCi2WbmqArE0NIZL1E8VzpFpxr3bUHWZzbzfFr0p9azd27dCMnlzWC1p85/vEM71fUy0PQXVr5Ix4CwGVM1yp+/oJBBec3Yo/sTuaRFwq0q6BHbRPk20tSnhe36Y6wPz5JeR5VH0bNBLwexdZNDiJ6TAbbeGbviTui8OSFB2i1XQMGNx4gmRpBXc18umUgb/5OZnORBlQDMwufYLrlwadEO6KiRdhF1evfMqYobQ5ap/tkB5RCMzas9ljj0ZKDSPtHb8NlD4Z3XQEw7mRJKHcZ7ZY+MsTGNPvZLzuyRXzSQyr5VO0mM38MyiyBqnJNFBkqOmuEfGRzGWxQWc2N7UWMDFt7G7PaCXCjMTFFx9GFVXUIrlU8ihW0++GNA9UvOc17P2MCNGBh0pAq972oAKkKdE5wH41jA8Ww8ipMP898YcmddmbELV6qzlIjtCcKYqEn/EM8k/ny977JF0Y+5WJn0WwwpdhxQrXBEw+kWZnHfXbPpNNu8m0d8o3/oaYBvgHDyM6IBXkj0OOTvf0PbQZfjJgQGXsrpxTdT3CbzmnyXrhky67PLFwlHB8IqRsyKBB1WsnEpdbvPY++oxR6AcoSuoD2EJGQoi7tidbV47w7qrYC+YuGkguM6GDwxlqlPT7a32P/OQevKyCISSb6omqavPxAQ5p8m1OHc9uT64JCqwaN5vKiMowMLcgJsZc4rafjmpcmQRUocvd/nuyrRhUcZj/pTQE+QmJZMoH/zLoDmXUe36gO7yToJn2h7K4SSE81m5sl2I9v264Qtkrrrxio0adQRUYqhR1dl3Mx/vWZAZkn0BOyCgtl7giTlCWa9VEBU4n4GJacQq1UffQb2IibyO9jYrg0YTwSA8y2aSeZ5vS0FtNPpGXsdlZsNPQy0nkCR33kkmBV3dT8bYfWb4qyBGmLpFpaD4+7PMKjO/+joewsJSjT66T+5D35+YmdxZNd7R4eEQzhkeOarVqlqeXhVTm5dUW1wQAQUe0ZcEMbWYdxXFYYvsg+o5F3Sjbnhb8uVmVTqY/nkiS/aJwNuOy7IzRW9cANXsTNTnGGKnlzjohvfjM4sJosUpu6fW5tWHbBClffXO9rzO7sRZ3YOef5dNr/cUMNlDRLU1TD1tg3u/c7pZX9WnTO7Bgx4jBlCApFlzxlP1S4FebALqMc/plm7JobTSGRPyar2MOsOsB9wPumhHkabCTQxD8GfBzTzN4/oVybV3OxLVmfgprtVw0x6TtX/WPjFTOxuVImycCG6hxX01PsI5/SD3MzWYazy+4RtOO6453kHEpqh9O3a4dtDGVhjPneDrNzLeO3/PTy0psBiUi2XFrkWZWHhZZKWfwv+WQKy4x3ZGok4KqxvwGHlyvuGwcEVSIY8oD1ylX1O1r0813w1iQPXjjQ32MX+q1CmQf1Z0RuUndk9C412rKJQRTJmI9fCJ9KbVWzdbVIsidOIjf6dDSoJvvu/cv1QlMMCwFRMdGLk5tXF6EtThLBOf0hR9pAUmXxBzcVn95O2RkmoQiFgSa6J4/sEcnxQy827KTxZ35ZxDi0aaUYWR3V02bnqwQkfB44qR3/ZPqeGqP5R33c1NOt6IV+2u2vR6x3cfnbAHbaEluqb+dpv7lrrht7LqujqeH0n9XD+rbZIiEU0Xfo9eUy52yWlgHUKtEoxbj24ewpA4H4U0q6L7Z40xZhVmGneWhLvut+sSoysDolasXYy1l7/XuPs8y4lyxD0Tdbl3sHfYIIe1PyrTxlp0AbIpv0lw7pOi+1rmx8esB9wBlP6MmFXaPMA9LFBh6AVzXYwdIkaxU05MjXbSDSfJ7+agv7IEiBhuuUVRv8M46LYU3GQBjhh1u0TT03OCfila7HAeR4bk3cOcRLekVoqsUk16sW+p329oUfBzcxl/sLFo7iYGNH7MMeEpVWhZ/pPFYN1ScSzzaPl+vBXH63FE3B13H3Jt8ysONF53D4OLuRI/jQpMu4Ef5wnmY8NcJveJWYvp7fysJP+6XZQlSzIwyw147cT45VPGUjvZA/ileBUsWoqUWIx5r9SyyMdqiQh2g1d3HpNZsqCp7F6T3uZkVEL5QUZSGYGYB9PvjWYERMwd6D2TDieMW8fB6JlWJK3/rNCAi/d0/6w+oJy+TOvHok1FIZgYOT8HRkYsdp5XhKDLzHlF3RY+VqOn2ZcgYymugvv1AJuLjYLXM6K0p0SZI0kXqagQsOGu1jshUI1R6JTGWQ3LSGR3zMuTu7/g/k3Db5rTuTdBTiK199tDe4n8m077+rNyKEfr233ydsYDsBwup7tkKOBYQJR9Dqjo4qpZ9SpZdiEIbNQycIuGaXc3xxKvf4DH7ws1C5pyvA3wiKjXMGVHZ65PoSaryEl/44fi/S56mZ2oUGqEtn9N8qE47MPBt2M14uFt4Mj5ePGegWXVZ/Di4DNueaMzi+GfA4jEsdHBY5Mg6zx3Qfyk75dCi3P8MoZt+NFwEGCPF41kGf6QFNXIX+ceFZ9iVB2k7UgIuZvj/MT3OweGQhHRyacK34tNLJwy/RAiDjzZZSmkAc3lq0OHdd+DL7YlkHta77BBp62YFkZXa4hLfFIsuPo2p8vmV5ik9oO9c3FiZpzPnKtMOivJ3ibU+8zktRXmvL8+qUOfrdkkqzVLjCvrwg5t0XKwbhLIxAh3llOJp81QlylIk4+7Cfv5qbQgEPBAN0WToKA9xIP2eYz6LKppZ0CDNkrwwsV8Debb467AwkUDYHEQoS2skobY5KT1u6ItgF7VVXAZBEopTp+tsWCFNVfkIGscbePgtpo/SyVyiiXa+pFRXKJ3l59yT88oo2fFKiL6zV98mVKNc78AsF1EwN3nyxLMSxGMMscrhJbqKCeZ4yJtNmjDVTffN+09rgR4QGy7nJ2GlkunC9jrJs1n67+yLIYuH5o/Db2BDqEjuNrcQiK1hkWone4jothNVFh7rWIHk94MA+Dmvh2Xn+rEYb1cYQxr4hhodFQgxdWQgm+Hb6lo7ISZ8qaJiJnbYG+dQBDH6XpkXrYM2lHua2DSkYp7xHnyjME+4qdivQfWzE6poDxTgdti6uh9o9XmLlmD6fWlEs4aQdBW/7wjZcTFyFKDITe2fMmR6noEvjsp5j0DMwt7n8WD7d+wDEw4pr4Ovh+1nufuE1Nf92PenLUQOPyVnzhzSvGeknM5cpcDUUQvvsuWOxSpDc0XYTewxuskw+alAgpR7SH1mTPIs+XWDs9dr/2SyqK322WOPIpScprCcZ/9Z6oIl8sPICjNF5+qY0tdPmYKuLShgzIeYwulfPbIUNZh5NaaBqGMHnLoK+8Oz+AUVmz5vRCXfTTQUR0ZMB1ANK8ZaH/Ot5xaKMPQvJTjv9AGLd9C2MpZOr8tlhacyxtyNM7yEjrIF2BbdJePmvou7ExKoTUJsw4+XSC5C81eSKsjmXCvR9C9w/7dP8lM9I51UXju4lAlUAzitjxTjeCI1KNZVxKqHfhW5E6Kxzxbjqt+ltQzLwOpefBxddBPtqNAfmUDtgEx+1DilZ+1dDtCmz4BmoehRuQad0Rh5CSkMSwuL86c49QOxAbF75CvFJavZjp6G8n5jEmUxOLxlWvv+TJUZvhGoLS8D5jciW1+J7rm6JbniSSQw9GCSS4wtHrEUst4GKA3EyHL1i7pnW6y9VGInHPHR48YdNs2hLUYnoqRZdhCJHkwyp/qx1u67tOWmP1wPpWUprvism2CuQvwV3sCmc2Fjct02yNEas7paNDYMQ0qJrSfwdRiseEtFKCT2TSPS6Pz/VqOsT8S9l18uvNQ1/SLFJg3ojnzzGj4ObFbne2nSClrx4xHS9C1kbRqgWa64e3y1muNHmfPljqX3cyFPLAvQ1Ff1P5MFz6oFMZzzLjHPXmTDFvcCr5UeY5mHrTuAMNCFVd4plX1/Smocw7XUGvrZeqJAT2J2YGIZuNv/oDWuPe1378LL29AYQMunHe5tCv68gnObD5Y7yn4Vgn1Vqu0RORFH9yLrMIRpqrpXByA7dU6Vc9+RbaEvSCjJ0QBxHYZK2g+tyMkipon4niJRJy+LA4Kfs+RX6PF5YDOdnj/aDdzhyYqvl94U58QVSimctghzSss1r39wudObzwc1w18LHpkpEo8PD3mevhmV0RljcZdaaY+5XWWkcMwgh+fMlKlcIH8Edi6bOeIdDRaGz2VO1VmKa3L2bpBdQ6afFaaSCkS959tUIg0Wd2SQiRKye6YSFHNoKX4o2FN+AzzdMzG8KR9qgn6FvZA1if3JsgzgDVEMA0pgmBMmGUn34Xx5VQDP3U0BFf1EvkSG2vavT/qHKofn0QieUxYIxXvfODGIXf5MOTTuzNvYUvT28dv+yTl/IbCiaWBxp5ULh1jencsdVQBFBSf46ECEL6DtW2zJiJbAua4ZDHyz5LiB2BK41ELwNrWFZLJKy3NbNNCL41+53DbbVdNfUlmsoERoUJky5djtl3H6Rp32T6aQvK1OWgdp3vWfVoo9hmS93tJqbJ6qxdc9jDRWUxZv/lGYBLxxAus57DPjfDw+LbDRLZm4e2tK3gRP70ThHJsL8c2Rfdl3f4sSoQj05l1vQbZmNIW3m27DSgPLeZN2p44Nob3RdqQXhidqvCa9R0a6mdk/OUDI6IWeUngDCgOl4Mo45dNBExmCTuq/P1mQKX1OikIRvvLNuIsqS8sf+nLxn+CtCT3CJ+b8X4e8lPEN0t/1uYTenNiWbAht0MbsEyaPg+xnKJTg4U1+TJ3Gt1jokZsVhAz2UUtR7kXgDHXEePJ7Rmrb7PBAbsvBrRFLP46CJH7oFVFJjxMQ4UXnIPn3rhLo0LGuWZdAhV6Md3Cs0Zu/ONxTyGsX6YnJMbBJ6ejY6GgzQZ+Bnaebnn5Gu4SZdjq6PT4tg5QjTw3geYwP9V4DEbTVYvWbFGyi+xKE5iUaLJKbNJaoblhHWoayUl+xnSll6mdoC5H3Zmdzwf707jo0iQtqfJGSFSJMmMwVOc0UmffnebECNWjw7itph7HHhA3ln2+Q2pcSefNvjAioSXLUJz+tYUhYAlm90O4lj0B1y0FFZMUPVhuFD9FXiSQDcczj92zdGTf/dJ55bqQZmPOy5tBDpYxGtoMisHj6mcEDDQw/JZxx6HDXa4YwiAxn9PZDQp2kpo7a0aHc9ANuXkOPJOBzmV3PX4gWIrQwS2culRDtaSWb0PLpkHyYIP9eN45Stw1YPu/as/EDJwzGSAV79a9w8oTpAy4tyygIHNep+gEkNoU4728vXR/I26DlclKZA+yvjPvVrnVkC1kmn+katz5GVZCxgw0iLUKFqZz09FvizmHCfBrxTE7xjMjOF/azxoFvZeDghvc7NqriKPKGa/3zJ4VjTEOtOAlBZ7XcHHnt5XvmIDJ6/2xtT3Zm109HvrZkv62g5oBcXjlsEEGrRNX6o1pq4D0KfkysxBKY3atd81SAXBVMI8pCqhqyaruoICxz401eJxX6huMyoMsPBwZjbDsng1wvb29PUmp0teUmkNJDwLp8yNYkFFXn+9ngVOjqdPPFY1KYAfdjxBu3sqdxIIZO96otB+md+RAmr2MTUlGyhC6OenPPipTi7+Ak9+NuIW5TQ7xV8dC9EXuZU8iUpKtNIiAN/D5nm5TRJ7OItJuABg9JzDPSi4axi87ZrGFnSM9+RC2WBHlbBY5EDZbgLZ2z/ci+qM1Xl7/6jGyvbyRmXnnXjtoPRUd6WmbGk7JGN4GQ1c58yGrhJZwSmGut77do2czwmNOuGaKBIG4mui9x/bLqKRXpu8sVYqMY2Po9BhuIoaVygUms2cgBaghIIXaRO6WfUOnZ6WODwyV58aVQA0thKmAQYdnb8TlzWLXixwMIJyCqvUyDh1q3T+mJ83X+ymx1Uu7dw4IXsJaWU6y/Vkm7C2Ji5bzGDvcMU//9NrIEaU/NjHb35KbtKG72X5MsvMYOKAOJ0ZzK3+vcEFbIU3LMChqidPDPo/MRety0nSslhkiiOPVm5+uzoe5pBdQypuGJLa6CHIRFa6NRk6F6FAdN69tY/YcnNMdjuz9xabAQoQdjxMWQ2bFHjYBgrWjOPrH7OvLQ4elVlf33KXG1u9UhxZhEfBEhgFuARTUwqvNr9BxmGO9gxAS4ErIjDZXBhnojtCZS1fcb5/102N80kx1Dqo9GzOfq1/Vfpu03p6Wam3CgeNs6+TootWsJxITjpDvl9UNnp6Va/i5oGjyWZDwQRgJdlNHMKy5qtXn+G6BEgU5YqGnqOiJkakQn9ZDFWT68mzkLpoyEPGHSTS7ESVPjyxkWSMj8BBH9grHdhvvri+vX2K6ICzZsiK64yLNXxZ8jHDgBhIqjt7dVxfRN8aqqsbgofFyVaoqyzjQJFoJDHvZ5od+SDQZA7+3GqR4jZKIGiD7wbKqWnjduTLoqToZ4dv32+kgE5s+WEmmGfZ8bP2WVvus+sD7rj6DYddYnc3Yw3med8NL0x2jRmmSGUQ34NFytU9ukFrx4uVrqe8jEjZJsOKL75nUJ3xdMF2J54EnKnY9V4goz6SXt707NxjnKwa4nl3uxiKoeWoDKR/KlihjIyruMseyGhkpowPpiU29ns0+HSOlHPCE68PnzPbioPttP4AHUXSv2JkbkX0zamLe+ps8+okG4gIMfEOj2+XCsw0vXtqT/Ez4hWA8CJFBj0B8eQ5E+bV01/0IIj7dtfC+nfZSiZeZcdM+tlGkWCPNyPTLhF4Ieel0/AY8yOjTvm+Gt6ZQrME+9J3NqSFq3TMT9kbcGvYLZGS1JxYvFWEYeKHOliRATj0FO6gthgG9pMK+CoR1ucv9bqkOsCfgxi5BXQ1j53xec3WzpJeU13wuQXZNTSbRYy8DBbSL0dp7yEhvV4FzKV+uoQRpxnSZO5+lc5gE8p05vu7/IcvlYTK3GOrSSE+k3WMdUHuvJV9go5SkvGtLH656eFfWqdt609k4pInotqBs6oz39c3Y+IuzEdF8CNuttxmb5Wj6+IyTE4BujM/2Ut4MdJ5+uu+dYGSdvNP6k2K3y98NIp+tyvn+SKOPgad+gasZpCOIhsxRtS9gtWqGa4V0WlcBqekLTpXTydNgjl1nuX2oqjH1YJ2PnYq1LPDHq7stWsckKXi9eqvlCqr6pCHnsRCW7nBZIrRPeqsOHagCIlt6/95w9IUalu1kltDiZXsyINw9+d2NwBnZKnqVf7Nqfbf27XDYZgyx92BTK3lbUA+zywpghnEG6YizG6dd8kYn+ddK4O+OwSoWWO7wJK4MYlFh9cAc9vIh4m7FmsAqQLDkoJExQ1EdrJ3LvOquE+eSsaFsoIfJAHI+cJmo7Rh76+jwaDMCPhPqQckfMRyU0QimozEUrF3k8Rkk9z6f6e0sRYsf3teTr4DQVoLb1rWB5MydJqSgaqKsF1kENMJLTWGMS3Zi0lST4uiN3VqLCP0QDP7Bhdz9FeHdKamY9Sw9kOSylW4ERXHbPeqJ7PhIDIaXXnUwT/dwzBo0oj9T//znwiSBxil8KItBSXALIG9tYKKwGcdYMqAS6B2iw6dsqHk5KehLQjCzV8fkMMOvy+Y5yBlCn64kJ6nHh3Qoq0F8oyEOqu3WXYqqk17ESPgH7dCegvGXlKd5N2Nc8nbczPjeHEd/2aBO+tIMg5YhnVc6bivCjpBi6wlNlRFFR0gun+HPyxA/oKb4YOXxGAVnJaRKEm8TPKsgL4Qd5EbV1320NIvlp0Cuz5ObyqTRu8syMuMy1O5pkcQbJH14nRXllRo/S3BG0301S6SO71rH+mbQEaiLPi3WFWsaBazZgv6u3DYfTjgpvuVvj6epjmjHFY+3ZV2oBpwRVmQG+cQNCKqSFBOG9Bf2eTlatFF4kPhmaDenuL7bDMebZb4518YFgf9uvPvDrsddb1v251p0Jb3bDDHWi/L2BKN4zBQ9O/IbTxyMgr33KSA8Q31rrN9aiR+q0KT6qIcaGxRXrvWF9prJJZeRbEdt3ty3yu50iMnTUx+TmGcit5HZSpJVisk3CKjpjqA+A01C2w0CLHMbY22HHKkQCgi5mntOTWV5n/uTixBkKNk76FIVHBsaNlwMXzxuvjrHTHbHSVkqlhxdwNxKDIu2XRunpiHsiZlyqiovic5wG5go1hSB2awXsu4cqBmR7jcqkEleLoqdtjw7/8x5tZa81jWEtI07PBPCMDWvpjDNAbs0hulqlhvjA9vVRJDMIGeZM3UeDy+3C1ReErkkfTLHErogPP8c2GgzaN1gzKLkP3JFvUyw6zum+uQb1DdqTYTPuwmfrBm6p6wk+L4kWgDLouuadzHQCEM0CiavT2tEN3QpQraiHz1SSgpC8+1pwAlgQ+aubqGamibhTK+J/XAy72ruiiyfnL+rI/YcP3UDDg81DMZxee41Opfi+1b2XCClDeNiN13B9qiEZIRFAxPtdlCPCA/2nqoMSm84lsbIg3dr0Sm4WVQfsCPLaak9EO7l8/hq41OQj1r1he/06JMnvB4VQQzPi/KX2FKOjmOkm0SvvpoERYE51mhPSVO7NEPxVfmsK0RJYr6XzWUtnO5nNdRpu+99ppwuQ5oP7lNemes4wZdn3uMI6sjVPu1hEz0f8q16GblnFczcpgwZm8DsF+CFQHH2iQ8fActIfPQU0XTGhXFuwHD0yzeSldcaNPzcurpc+NZBFZ76luSR3EPHwfFQXS0+KDgL61jsCeFv+pz5wyjnspfTTHXnMXL43UVFrxTkztQyrGnkQejcxLULhJQfRFmU6Bpg06xYobjDCqVCnIivJrS45YWtAvxWrMo3u8VbD5oHVt7Fl6xMf+QPw1+/QJ99PVb/iMbrF3BeUX1+L71Df84FVPY5F6Dh67nMQH8597IDmc+5l0iyv54LaP/xOffESIqlae5P32Z03bqWcFuDfsk0+OFbwWns1epY9k/ohdsLX9B4GbD98idQSgE6wcGuLZuW7PjNLpT/E8p2h5gNXbaABQmhr58i8O2X229/yC9XeH/5+M/kDfsF/7Jrr9Kl/LKbhH7BsC97y6wqyq/fTnw9OZq/bBe/ft1ncMrnJkCo8mCztv12T5/3CFSlX87x7f+4Lb5+IgjXQonwH5o7tX9Gvt7CFrWX/vzyWL9c23KfXK4eWDsa2qcK5G2hKXut2bz8CSHa666YtLrMGaIAb8tquaQYlC1Jer0UjzsLtvp0HKr+1+Pj6dvh3/aACou/XOTLfczLJa+/3McF58/Fl+56Ig6+3kZtVfTX++TCSjZdOwA2qiRq6a8fdFWagtOZKZurM4o/l4Ku7c+dfGCHM3/COXCtdRmuY7Kvl86HfrG/bmL/HAL4M4lAv0c5RUA/IBwmoR/RjUH/KnQTP6D7J/jEAQXcs2mu5g9ilzL7azj74dxlivo5SpZq6MGZwx9H/w+Xul6jbrze9PEMXr5QmUdr/3tJCvvGr99ICkV+QlLQT0iK+JeRFPkTkvoO7vNlvYO3abRE8zJMAJ77JTUye4wS8ME+RePvETMv09Bk7NAO0+caKIrebnn+6yfe1+dFAKSrtv12ZD/02XfAJ/9JAp36jp0JHP4F+VGEIz+BPop8k/X/fPhTfxz+VRcVADqfV3oes2T5Ss3Rt428OrL0N3ygRXHW3i+6/nA0ysXDsgzdTxhlGcafoS3PY5yEfkAbjH6Ht+tQ6PPzE+R9ve0P+fwJpb9sIsII1BNbPRnzsUOqWAxAjxu2W/Jucb3TwR9OZungemUXTyUycADrM7Ln69c7kr/+mActPtUdi8EZldvy1vOBISua5kj5xA9Uopx0XGXbYqzGLoZKttnWtqvWfvAW1AoMb1dN5Q4Dpj6h8Hh4D6exVHeQed7u7eugUa4UoSqtB+u6BOuRU1U+j/7hjs5lR3WBZSlaMb9JYM31az73yzsBRVIUfjvPvgc5slw4DlV3OjwLyfP0P7tKyuIfv//XpAruNJ7FhijNroy/c+99YOVH8cgu40eSdmU3arAWpMdOVZOrJE3v7Pv01kc4U8LDfQoyUiI7rckRphV8qtbyWEFS3AsvPTHh5+w24UTLnVzEktxzebo+BPY1s6hYlKYGXI43PxvkAnEbPc2MFtC5Pa7vcKs1K5I+SSlzNZMIY/YdFFdejgFGT06gqAp3z3b6LI+is5CelHmOe+LSmtH5Aqr6FKUlTuYUsfmYAGEzGYLx/RBTW98X2tPjQoH3KTGRdofnbsZI3gr+LqOgB5IRo86vHo5Tz3LdPEnp5E4ZJH7hDAAcfT4QW/Fn+X70ASEN+qQ56J2p6/tnZqNdWZNRcA43KDW3lDNJ9mE6jfGWJrdnsDSWddthWHKIF3sLQc3QrprRjXyNL0WE0l1qpd2XQvRFOas5FTm45phVj4U8G7UuUjekzLupTf1KDJmd6pMhAGdU2qJoyRMWRMeTo7ws6fwzCcg2pkd5GbvwAjKfhU+BIu2S3vGOa97bot13Vjf3ysw8Vno1hL0wAxbllJ3BS4Wf/EXFIHBe3hpt4KUiHFYmWchs3FXd7MaIs3Lq3iIIm2bta/SwdDGotxSIotoorZHP4ttXVeXuc9zUqVy5ugbyQGQv1LSj/TooEywYAkg3yL159Fj77buPOGQKV+XvyOCdUJ6awfowt5cTbzJXdKQy38EJt86ibmhhpk/8oXm0DvvvMyQwgKE4H4briylnY/GJ9usYxu340CEeQzakX2d5g9pJEqrpNV7Ocl1irqyzoZXvUrL53vM5o9RT+FRmDZuXoJSE8uQBf26TbSufTDTGPxjqtWswTeifAPK5fRa4r0wzX0L4tSzneeMicnaXrLCbJqBSpslq6y7Maepsmfkmb+cK3dcH5QI80bBxn0DSU7A0ARRIDfGoLod5vrIWOSVCuv5eLBduO4ZGJyMrN15i9wffuSYBJeqN2z5FY5/VpU40fdSXJHk9QIJtAUklNWKLyr3HsSKYMTnd6o2w6NXcCgqc4Q62wqWq9ZDiiE4GcrQS2zfc3b+R9DqjhCrUoWpx43gnNH/qJjuNOfYEoQeQB4JPowABIoR4ouLpSbJNAl7xaWXZZ8j34Nilc1BjJSDwiy1xCCDewCDuQG+nBBMC68iltYP+DRBGZ6RYzUmkNjYUVlL3/oQx8ARo3UC2XTBMucMOrmLkMhSiyhIgwQ64e8Jvb8SQ7mtwR2q+SJlTjySDbG/OXbTFkIMdp9MBvY1DcJbjZ3pZn1/H0z4GIMdysQUKLwq9cXNMwsCh5wc6hUEpd5CiPEldXePl1uUUOKOIw6rGwaIk8TR6dkqR9a1UExCm3GMclkhPQKB8sBQKRHvrTAoLamuozzhRoo6050YkjpH3NUjavtxUTsl33cS+rcO91IcQCi+uAM0Ze3aqgV0QCR9imdfkBhh5IrJ3brOfdaWUPgVyUumCzGxIBNkrpORQLX9w88NGzHeRxW7/mQG3s5NewY3vRHj84mYPXq+nBZ6E0FDce4mt05mtocXEabtBdz2YxScdCG/10py9JqHJ6I55KZ5ksUQIzLZMALXUC63HhdYpptdK6PARCt9hnsEtyQ9LYtGpCMTvoGS7V1wCTWLo7fmc2xlKdroJYLXsLs9WPeskYwbSH2d/nEkYDPnt9H1swU/uhYbq9Miw7qlxt0VaAyIEIT56w55lTb+fQ56627L2/elNKGYTjx0O8+19X9tOvVnWawVpj2DR4J5WVAAG02Zk5uQ15p5HoEOlA/EnHDKE3jQdcvMoEky1EQiZ9v0q3Ny+BY0z1UiVSC0ZWi3PeeNRD5o0yilrB5Gwz+X1Stx3NnFcpXUgXkfQi/xEkdBExqxN9BFqV9oEsknpnk9F55lOt/MqaHLPV613N/NzFvI2wkuEioJWj8BvIlm7TRfXMfeAJKR0tJ6Okh4gFBGdxpPHsIDoSempsVuf20Y/bUOKtMzTavtPF7nKY/JbVE+G22Ae9fy+f3U35JOY3EPAUj6/tkrg2JObA5LSTfrcmh5jgtExQGahrog6lx3xZqMhwlGkm4X3ecw76rQe2mc5EZIFcAq3J0gwBgSVNb46QcKA7G879iAhDgSK1ibhMoayRgqliUML8liQS7vEN64QT441tOjTynG7u3W0Pauyo6Q4Nfu0wSvBC6tZWbA30rlvXvTMz6wh+jMxLRsy+CgwDNWUjfusjJhwN9EFooCTiv4g3jl+mvgDmMm3WETPLJ5PkIkSS9qlXjF9tCOVAYk+XKK5mN7OHOeyx9P5wNzOcm6xyrzFTm6Fn3XsnyTOEGZNTzF5h4RNegf3Z9m+cr28gzyqnFKyZ5Qltt7mjbg7N9QQ2snvx1N2FeZWh4tdk02Ohij5GW+lp/WHLyZgALGmE0aKAULRtQYL2MtBqAdJ3OIYN0QpV+k1WEP3OPmOREcNA7wOnoq8KbUsI5HXJm8Y1MErMyjIALBEXwgrFpeDkl7ylvGifjgyC3CBG9iszj0ESmpTXwNXIUhkD6Ujxo6sZk511mGQbloBATmvR7lgjXeY8jQorVibje9RVfLQ2niMrLO0ZkvXVhfmSQ9yrTfIszAbns/pgTuLO7siBBjcedLe+b4s2Qwu7ECr+UvqSqFoG023tRVD31vhaRMb8U4w82Dt1UsOlR5Fj8u0Ltbp2e/S5/b07y+rjKi5QBYy8Ml7ThhIAFO4/fJM7XXwcASxR4rD9oIt8gurUuGtT7DAoJaave6whpiv99CqQXpK5VkTHJ3YMajrpTrTpBH7/mpZe2El6YyOKeKRTDHfWk/H0nJZLs30gHFJfwywqtrtmijjbaux5eU1vc0MAPS3vvWgcXzB/GCn5xuk+CwsDW1WId5GvMZ3th9WzIXdnvTp5hPVvcvEgERBmlFjYhKcprqVsiQV0+++cO99ZWwviNQMaJHTsIV9PR7mefcu04x7euHCrwLc3Ga7mqeNr2NSxx+J9F6cqYPcPd3Nd84kQAunSdehkld1vuqs4Gt1QCAXm70gYxtN5J0OIbF5S/C0n+PKg46DtIhe7hu/LBheijFKeupUKYjd9D6ae132Rfj0nFHkOSyhfIe4vnLYm2Z8zQqiB+7jqXhLZJOepQOzprJEqH20ow4EtrzjI2Q1wrM1j412M/4ShGKvRrUqMsmJS7Y6vn35XbbmS2ra+FmbMAqzvW9onnVGju68QRmErT6eonuIiT5T6uv1WFTvuL6N/ZSVP12YWO6vMwKoDc19KmqKuasxKNkTColywQPyGTsBwsQuVXdLjUWA/Wg97OAObnJUD7/G7q4BWnGFJ0rvUZMfDyXE/ERiDMuTGL3V9CFoJmKhlwMT2FD1Q/pQ/Gey77e5KiLiaXLFqp5V0zxTkBK8R1hphLD9MPK2tjCtKrBRaYxd2BXQHaRrizSyVl36+0pjoCOItXTjCxTty+0TmN2GX0nLYvhdz/Wb3EBGwMmPtnIxpxowRZ5KvvClSla4YS7f4O7qRrASvmhKaLYlPXElRnlyJ/OgdCagZeL1erlPCbcKkyam7np/aetkaVyzHsxX2PXPbZClYxeeyWVH36ctor0dKMFxsgEBre23tUyBLGbQHcAXEi8llod10SF6lvE3QZM6uFgfdqdjYAg07PNNl+TKUSTmhtPJC8o7HitMM6IX2KOaPun5yxMJKWYVIrRBH6Nu9LG/56JXtZGKiTt7vRer1lMx2k/I3swZPxq8y2/2xVDJrEF/uVULRgeXHZMyeqLranGdqg30ZTByMrGorTw2HYK9UxrjvOORuNMDLfEa+GrsUjXYaFdI1hSmQhQu8m4L7sWYp8gFh+o8hLVJDMIUH4AMaSCuJ1H15HWgJT1Qrse2UBfrGmKDgt0RzbeNbW5T3G6VsClOkdEgeTSxqi/fu+EwGYwh4Cfldo9bJ0rhS5cOXqh6bDQvxYOzCXfSPd9UbldNlYRDe4/qSvV67Ns8Bz2X0HA0Iaqlw+V9QbSIvA1Q7RaDLxeOJLlHx+OCncn93hEg3cXA0eutPvRpsPR6pfadoqsxFsNEWcsYPLa4EwUzl8Wqd0qZbbba6KVVmTEmZELouU6UL+5pqXVH7ZeYFewu9QELsdi0stb8Guj7Gcj16+KIi7LfHXAai8/nxdIaqqCoaFjeREi6pFLPyN7TJ+GTscl4VJ5KdEuw56ZyBU+DawLnnaYOh3p5hD3SYZSraOnttlDBCjZlfHydr85h1uHFK3lKrAJO6HXBQXhPDQZZmc0yJciCvXj1LSh5sIUkx87YQ4nfZ+lS40pbXjRwKx0kT5+VNS0xGsTwH7Q23Hn68X2EhH9cGL9jgPrjQgTuqhLppv5J/9ju03yoOBvIMghG/YsChtjPA4a/Bgd/GzJE/oUhw9sPIUPs//w+5v9jXH8AV/1fH4FH0e8C8CT+YwCe+nemdFDoj6R0UIBe0DcXTemvKZ1vqb3/dIbm15TgJ9EDtVmUZtP/Wtr4M4z8vZTvT5I1+L+VWOA/QiwEIBZ26PNq6uZP5ndsL6h/yen90Tzg/5MXXzJ2t9/TAE7+l9MA+kdogAI0oEdT810C+D8hJX6fEo7mP3zi9fpaB/AuGbquWpYs/cuu/y0EhNy+syL+61O+KPb3U45Zn9LT5VECQLfRPFfJz/K7v83iZke1+ADSv2A3/Ot2AI7+Bf62yR1fMfHZeP9m47JcquvhAD4/+/4q5OdhnZLsbz3c1xKJJZqKbPlbB35NvGZpkf1NTP4GTz9j9G/7pqy9ROyW/e52f4a7r99w/1Jd8xftg+LfUQqGfEcBXx7+64l/IYIfr4V+b7qi5HeX+gKeHy71oadfn/0fILEf65J+ILHfUFMazSVIW39w/6ksmH9fWfBdqhmhSJxHfp6vBj8/MDsD/QKBrh4W+oVAwQvgf4T9fAB/diPf7b39dO/nEt8fefsrFyY/Z1+f/+Qi8Hf7QBnA7479yKdvif/uKC44lL80a5xN/cUn8y9VAvL5zDjJnzfcl+qtf1xe4cR1g+jvqYfAkZ/Xun078Hdi61/n+Hzj7d9QFXjsP8/ZtAHD9X+a7sAJ6hfoO1zgPzFCbz8RS9/Ljn8eEv5AwRCQqfbXza8VPdOw9umvPD5MSzkUQx+12gAKTz4grLNleX+FIYDu39M434P8gvT0/qKC8G+bwTclAzb+on8+W98U0K/SB/7HNA/1BzUP8QcVzz+qUXAE/QVC/qo784OJ+q/WCT9WOulPlv2BeOa96troQzS/Q380fcM2ArBWDlN1XhQQfTsiKas21aL3sALgL1OWfdv4zbHOtfsr2r9n8QQcUPXZ5Lw/Qnf5HPoTuvtJbRuKfpTODxVRyfVz7b+kd1pdBPUzRfVPERQU+oOgwH5Wrfqz4rZf3Ze/QVqPLFmivmiz33zpd4YtDP1EMv2sOvZ7yRS1lyzuoyVjgIiY/yWk92PE7Aey+zsFjd+hNo0yKk9+RgxEQmXxzxH7t9niD6Mbo34hfx/Gwn6E/F+JSv6rtAL2szDWPwZhPKNS7GcQppAYJYh/JYS/q/yG4Z/A99+pc7GfxX3+hs7N2njY+b/sYD47rg++2Tu/h/zvdfP/JVT/vl68/UG9+I2G/nmK8R8DOvI/iaSp/24k/bMw1v88kkb+e5H0/51b/kNV+e/p9qtlfzmp6QcT0B/xsL7UmX/HGChDcQL8E4e3SEbkl63K9v9oq3n5J5hKKPxdZOTnvi1C/oL+pI8LRX/VvP983Pyst+c73FzWbHtZqOzQ93+p/Qdu0G9jIygigK/9wej81lrxe9BzFEF8qvl/itvfBWb+kz0FP5DEbwXkb1y4X126r8/7076Gjx+eTfyWfXHHf0Y00T5jv6Txfwz9f1T95Tf0SYb8cwztH2K50E+k6E9t3u/jb/88svn/gzsOoeRvHfI/X3uIX/f8Fa/8s/V9XPgvQWbydzHmvxlg/quI/7tyHvujLj0O/7Pl/M99epjEf0Hgv7j01O/pkUR/f8F/sUdP/gGR9e+hvd/QBfwbuoD+Jl38LPHwG6IlcPR3UaRfbiT1n6HZfwP5fYP7v5r8fhVjv8aQvpdrfzRF8cOV8H9zhgL7791391faJYk//aF2yf9cxx34paX+S8cdZ0LZM6VpPlKbb111543Bj9aD3b4GlbXofRW799RNK7Gc5waxthVaj93jRUfSCaaw0ryaMLMaOtoUu5zfyoV/pLY1DLRuC1JBPXTZkTYwc114kr0kkCgBFziMQT1p3PoJvZvTc0XJF7JOLzwhqLXPjCdM1nDfo/4lWlYSbnGmmIVCZwr9GAeO32mxoK5tpaDBPma4MwGt0QEH/kcca7Hc7tG7SO8dfb38lx4rn6MuNDynx+r+yjfUXS4KdWMsMgSxjwyqhh21lNDhqYA5GkVT1kxBaQgfQC1r7ZNigKYTUAWatKcPgc4Y1FdclJEUtmJ2vQ2i6w/7wtjeF2IsoDYV38Z0jjSSa7pgf9YJX2Ec4wkDhZ7LsXOcwRXlrHeGV+F3LU2eE18gWUGErDbdFDZ0LWtp7LkNlMBaW0713OBRdCMr9ZT4mt797cmOZL/nCcW4JsFvbNl8JsT1cMhmclje2vGgcMJP3yGlZLgc5dYzGTty7PpMJetaO88MXmLOwb9BjGF7ZRjIu5GmaJ444qkIqEbKAQwXGsVs3YQUzwnLFxXS+GKUA5sGfYMbRBNPWbXnpyoiLSxYz2lxWI0e2faO1R7rQDn06cKbPjMXNVwKUPd2nnwxj3TQKBynm5YBBq746zGqn1Uo2plsp4yqiNuiRl2LJFxB8XyhC0HqgArLXkJaNbnc0ZXKbqMfybbkse9nlKsF8SZDOfBf2WeZsSnMCOndQsuLU/JF9nhbNulGGXYG9GM9oYEpba1a7f3BsBxfCfF4y7iMD54IU8ImzGbbSFrt0SHJ/dsd3PoaIRaXGEjyvMMvL3EfjHTBdim6Yj85o9Mi/f5ZJuJSQ8dzBX2KPukUn7FwLOgdw24NHTWCRCePmyl8eixHBNLw9s0Jn1LLs+87aMyCse8anSl1mukMhfJPMK/vnqvd/WlApSG1urWqrh0vHQEL2Y28Tz0jI2umB6lg86PsWLteUBVTrgvOuOztxTjmLtzvJkquPkzacF9FY/IsdBrwdMA7NyqJovcElvN9W8V2oOyKP9QAWsquZ1MSrfI6v4SP7qGdWVSB6Or0m1saSnUqvi65opdD/OyNt1OTsXdPOjuo2YC+ODGhVAXL9k7NHqBfb2S3W4+iujN70S0SN3MmHtmazf4hF46li4ZKR+Emh6hJ82vWhD5I3TKB23Mn6DQTsK49LuanL75WVzUMEaxv+iXOKQemhKGk5HIUzRUeswaPF4kNib2wKrodRYk1aMhlB4+lFldQchaSE9BfK3mEnBw788GxJnqJvzc3Ct43J1UgtIIpTYoXEarnOxNiEShptjqO1nmUQP17Kts+D9DGJedtNCowXnDmMW0Srid3aDp4weTIEXEe83VtomjxjMJykekXpnUNxbVBjEFz3ElUIYoBZbzZ8c3dPysJkEWvHbObR83tVrCO+IHla7f7EjW226fG2YZ4Kh7bzq420jabE+GW6xu97hG7QcmoaCm21DRyYA7cbNRTDAZZHqVSfXAuBME57ESMms8vde2oSh0ETIbVIjjYE0H4qT95zmCaXQu2rspmHL3uPS8IoOBuOOcE3Tb31ucpa9FTG8oUjP2ece9VeN4qJHYWcwbdUnd7pI7wVApvoN3kfsdNgrB6pIM+y9CnrpcfRiF+pPWcWhhaf+6HscfaO/dXu1W6V72z1BqxCCY5O6R2/8kU6Ky5ZZFEhbTwWMGCBkWGccuD9Txe1hVaNg9bql+5cDN8f3ulEAqmXK54cBqv4H08Bwd0hD3vHehKwuy5CIt6jpuabkvPPzKwzolQSnS/D+WqNo9ib+jilLbnUmOZvOQNesbdQ9pgVx2eRyfS0XXrPOjIOAb5011ntXOUlHo6C1Su02iVgYGDHKPVX2QJ5uneBksJm3ErHe9u4VMHJup1k8nyW+jjjA6KmN/miEfNMEKhz/3EtIm5lhgG3ed+AD8VoKaLSek7phWXDM7RlCpDkxMfBWUdjVbLhmE/6/dgh5F8qaJMSSysmWf3g7ioXhdlo3NOojVMs3gEPmEofyP6QELaronuTAvvdlb1tX5FgQla4PrN0op8b0p0puywOlCbCZjg4AEVkck0OpgGOi8FmqOQzxQ13xuRA8Nk8ABHoekPB8Ud8TPjDPSBgklSmwtNsBoBYVi/fQNoJM9cQR8J+VRfDx2cWvh0Sp1IdgqcLLqRQ/MBteN31XeeCL3g1TOgNyJy1apq0Rmyw5NmAnWHDyA3HfvVx7SC8YlorQDtpezdaZwMqOO8gXVoUYUdluAUVmUXLI2SxPHm2h3aKBRT0DMTnGBEuTC5r9HcWQiP72WQkE+B0wtTsAUddOOdPFKF+Q1Mx4zvYMHizHXHm31dpVYuGwVcxflchbyu4u1AqLEzR2T/H0/XsSUtjqyf5u7xZon3PrE7ILGJ9/D0g+qfuYs+fbqqOhOkiM+EpFDSV87MS7/lNE+1lnmM5SoEerGXLxiLS/19VKevG5bNG9QVZ3Tpg5VQ/P7IfmNTir/Xd6gQhPmeQgy7AeEbj+xj7/ewxG1TWtllROm4gtDEN6JPxYMwIqQ0K+gxWaqSdZ0ZHlw/EwwC83c43ZP8shm8BfSVfc7rFw9g9JVe0yExR1Rp3ZjBC90XS2fs3rpV53C740/xq+3gPLPE1ecQgbbLdOzvqpKjCo5u6cCUtydYSmMjgiScvJR3Eehwd8lv7DPYXrcRhB3I7Qv7ZqQKXr2IcOqLUKcpOMCUrk0qtyAgPkOkPL+ouF6d8UTI/XcfEb/W4EYgsVMp+N9Ij+A8xgduCdqWxePIktF57mNX1J1Kt3JEYc89lt+W8EOYiw9nRKdNOc5xhjb6+3s+H6TVC4TRi7I7pxEG3vK/4kNXcGZHMq+IXzqShZb3g711JOyVCw0YpzdneMN8KuUngyZlf9Ena3vzD9KwtJOFr8FVmBAZtmYjgTpv4JTiNQHedWuEOWBTGwJTO14BwGAvjknGJdq0rWkRwdgPl9fNyw8DEuHH885qKgxrJvTbV7T/EEvc4o/WfJITqF+2VVxwWluB5ik5eQoluVxLXq2sJmvl6gw5xhiHKF52uqdzDzB7nxpQHNdwAgQjKY7ib6a7CXR1fq4os/XCIpjXi2SUPB+bKHn2YKzp6+mxH1t//MmLw1XSvDd3OyPR36W3DM8fYITI/ZIB/cxFg/djPTI9b2SSVLfMrorhTFDfx5umD1F3wnxbbGVnYnNxZgphM8VTPABWESW6lgdX1LGJY2JaV5V0C4slEJYt9iwgpKmJqlXl/EJ80I7Q/Tnijvt4F14FVyVR/tle7v/e8UAA+aOUTPE0/wUdA+aKi/nx9AFffnrZUzSBkXc7IDJ4eJSOHvgKhZ9TVviI8wQpZJ2/jMxXcHqZyXm5I7T4QbQG4468X+sW3AwsviSk2rTq89ELpMuItGZI3IYVSzzFlFwDwqL2oz8V3qhgtqxTpvBuJnDys2DUNHb7wuW3znqN8EoeuVfNCE6e088qFptcvtHHnfOdnjqz+OVaCpm+H3xLM0rIZq99vEFTWwOHJu7+u1CRWbhfbIFXt19ZgSm4R4x0IgI3JW3gKoCSaiplZegZ76zinlvCmJJJhe8AnPr0ADdr5Mbkekic1EQDd6peHtE1wFvm30o0xAPZ/65MX6n1wsFx3yn3Osh7JO7BjzfGeFdy1sT/QUiVzTpjXD6qG9eq1YzFo7ULqgwnoQJlGh9/iPk4AC1FgaE9xJhtJsylfHa2/F8T699azr7f70ZoTUYI8n7c7AX71XYVipXFwxQfgJ8+/2YJXW9vjolP5Udflm6dg3XEj6f+pG2LxNAWDt2J/y6AqhxsMlauiZlWDUHUyTnAGjFneviX0lQp/ryr42xGX/VWO26zHolDSnphBVB0d4KlDSFzLQL0sAFW3uWzMx7xUAkYKdhL9oepE8CoURlcYouD1jAiNYn6zCo7C7I38cDbf04ZM1Ai5LGYQDB28yysXoVWKyHbc38lF7/f+Lku12uHPFSqFzHsTrNx05t/GWOSDDRVWd+fH5Jx49CutFR3za+WxRRIx94wbMkBsyiNPcOdYlvcAPhMRsR/3T9hyI9fDNlfiHQa6mS2Orz+Eie4+EKeY9vAAaKkNTVQR2UXj8pihRZYN4swLACL1uuz9CZWW7987RsenTzCDaaA58TBpRPiwGS3qH4AXhpaWMGrZNSxULPmhKF/PbzHl/v9F8O+JDMC09912ZXRqSEbTKsrMF5WItDdN6WUTAgBD6mWfFnxLzLJ7wy8FqrWAZ2BU4wsxeiKZw6GmEmHEnAGWT6Rh2L6dUYn4zFr+UczaP7So1dCkJrO9SmsLHPWvseBXyGburxv8gmrTpkv7a/2kIBj9RFlnQCM3hGIrQUgaLVeVXjao+w32bGkL6AxtvLqnOJ+4xvwlPUnvsWVQ1fPCcXu1Io3w/WLbQmUgjGo72Ag9E8JfIuVRFRIR28u8UBXs91aMrGetdW1GPrLoa0G52T5y1/Vka8siFu7A5ZrpBiLCSoJ2sAFTDlNsS8HPNJgS8+L0/bfn/JM8fm7DlxS2JYRlbXNI3a3jBq4fCM8Bf3vwj5e22L98H/jN3mE14PxRlQC/qrBdVkitqvWvrJixcZKOJ/icA6MtKru9HWS13F8GCPltRLQPocDBHpV4xUevMJQ/AwyWXLyz1/nihGtK1GdPa6HmxcbWYX9I1LgX/CTL55JjSOp5Wl7wtoZVA/q0mGY92GbJWfhGkuZ5Aq8GW79z6sCRRf8jztA+jS8LL29uypE2rqjsz9IuE5NQXubzWTEZ45q7fQ58E4NYDuRgwy8wXeqalyK56uNpRWTZVADooI4VFzmQye1k15f4WCP2O0ewC7uONx/UbQ2eYi4o7CMz++VEOjsG9+4kKrpQ1bF+8VrFr2xVAZfXclAuiXYLf5KvDwjRoN/XnPuWy1i8ROYvtUBEGpAkx7qN2YCusYCh5gb9lCqUhLulTNCpf6NKyw5NPrk3meNH/74tXuqHu3n7Ay97+s23fnoDNcSfVns/bC7f4Nr4ZAOuLGjvALuUJQ3tngnOPD4zE0GXUVMiRh6pQGsy0x13szumPIg5zHa2uda26v6BpF2qgOLm/Y/sQOIbSGfwvhYmSbQgjp6tfQbtUFVCc6v5r+OC9dS54GIoKLFmmGoUXqliF2WQl3cTdhepOJP6mbQRKThBjnIh2F8RacWHD+fShyjlwXasv/uzk1KDCXlBjGUki7KV+gPzZEXQx7bScp2cTxWkHCLUQlpB3AlIC/K5ZO9eG+cCnOyPHOs8S+Tkl96oaWCfTALoHDOzddh0wLAoVdZAYeifBB3aPDpm1CDkZRtit499XOBK25cBtTPaVJ6VcjKfaulMMnaZcoYZEvQBBZPgCJQCQu2LS195KV1LladyjADYnASI2DqyKintfBNIa8Z3YAB/+x4pqbiGbJ+peV++jv5HpnWl9F5NlMR6jPWl/RNQEk2qAKrfUAe8/5Cz8WKbXJpxjzb01UyKhsXriJ7SCSD2T1nfhwFrvI9+mIjXZBTIoFPaD0LsC0gJviFcnAl3hdf7DNpmfzHdIpyaS7z+qWT92V2aUDzk+ZcG557ronzdWY9K9aJILgvoM8Ain9iFBFe5YOhC0qejmlFrVCLw+wxVxio4NUx1rrJhSKbzmSM4qQSqpY/QF2ooo5Qf+BtZpNzGzLScaIhtuVBQJQZ9hAUhGFu/SuC5yMpHINJjc1V4hTE8b1rU/Gx21fk1L11pl8GF6ratYvqIQA/45ARLJdIRwC5dO6vwfsbfeU1TBiW/Vblqh+ae2Aw8b9i409TgBbwt+7uBTW4k4DzwZTeBQGB/wrqpfEcdyrv02F7y0oYppO83sBLM78Y5bshBlKaXxrv64fa/TX1QL6QNspixE+r1eFgaYBkLhj4jxHzBuh3kNinHeYqRkEXYGIb6uo850nrhMii3x201IL4//8+/xl0FydDSEDYG5jvz8MpF4TYiCpY5veh4rxj/W6f62e8T7jfUvisrwOUnx/+O1b6FeKfygJKR1S+94SFPvE8URIKp/SyxYlrzIdJ+7RE0yaWjfDVQM1pWUxS8R9Arr9h8l7FP1HtYNNS/GJsp1WE7AlTakZFndbcbHfiYSAwr35Y9TO9k0+vEkVCNRA+tkHf9MuJGn7mj00EcabfycOFwoR7qTInL6plF0UQrLq0yRqtTEQlifsxOisEZRDa8UxceB8SnI1ntyyaP0In9VeLBE400a/8Gv96LVlulB3pgeGDxUp8gtm1Q9dfA0HH5e9i3NZQYyuchyP16yqFEUOUaf4RWBLlLoK5FQqaXsPbTSTk98xCpChxNv7ii206jpkP/XnALQyZaVuAJPGe0DLhMg5ftyiuC7YeDac6+e1Wx4+2jU6MArTgkmd9R/5PYIOFhi66C4ASq8nZf03PY54LfbtD4W8zbs4FlAV6daZSPgrc5fIKu1gyKZTvBmxo/vx0E7dSIX9XNi+8toR7f7GSdeORsxykwzywiqlFQvIQV5rvXGfCk4AKJujCgZOz4w9pLhnYF4MHm5XYqQPs+T7lDsjir/eLR7aajFnX12SIow32vwQY6sd4jV2r7AMegIvCxHgh1ok10yHjlAlOPDxsTCELUM75ld64E3ZODoPTCDZCKoX1SCCSI3A5AftmVtdjN0LFl+J7JolahIyZ7rFxkfaU9AZNOyUtms8jTOD2wUHBhD38GJxTxCeHkUHDvZ1vvKsIXQ9MgxCpoL2Tv1gfPLY130A/37DglJjLiM+5t0jq5iVKBtW8mg2u8dgjg/Th8ap0Doo+pDdW3FdcAwl0RtnWFmS72u+88BpgTRahlPuUB9QBsoZtHHj762iyw9NP0NavxpFYCBOJJbnG7E8cD6GwE7Cu7P+Cr9jQJFqRy/1VMQ93hXHZuedH/7XAP0BXF5aWOgy6BezFO5aokqP3RLN89Ffy46ymg7ZuQYTcNnr+Wf2rIuMPUhvh5dRYEHrTBQcH6FZVwt+9Xszv2N/BFVgiaPcnzmncRFH9DU0Paq2cuwRHMESkJgAgvm9k9qwL8XjsXAkmTnk9rA9rUgAUC2mFaYmkYercLAY/WRX7gkS+16L9MzjFqIPu/q0R4/E8PK0yNtqJEgifUxkHm12ypqKlYac9aTEYL5Tl44g1+eWDwGLEWxQqLuPIuVW7dTawkcTzuzFjmlsSjYJLqalAUf50hkj3Nak9HpuavRVsc9w3Sh8Ygd+2Fpaol9ty4p9zFtHlETkwGAso2Rlx/wu/L16uK1HIQvM5P60jGl58+vaI4VFe0DmQcW0aamScMnntfAZv4HQPGC07tRq4N2hUvX9UQhX3xtD2lqbjX6kwod0gJgzY7uBXErCHHV2H+rtzCoEDzU6h71d+5+r3nA3hKodj1gfnbp5ZywoFsun7cUvnh4b9LSYa5/1dt6OhBW2DD1/1rfu78Rb40VhWlbkx8DOTD3q/9b87+nghDt5Ye+E89ba0elU2q/jh4IdMDEpzbxS1Yox9PgIPHm7kkoIkuNraXWqsPMBAwWS1R+LM2DvkvhjzrYbqqAukiwu/WlGrDl/x6DMMTNk+328X0RfnWSrvoYD4NyfzS96PW1hpW2rH6rmdNLHeOMiEaP8xFueL4k6MLW7fRyu5/DPa7g8+nKkfIRl+5SU/vN4Bue5BV4Rw7tKxgKCWre7mApeaTGuFYhA7KLsx43IwuA+qfZgLM03PU8Lf/epyPhpg61GAhAKugn+cjuCnlPlbzI045SyW09x+A3po1FUVb1Y7T48G/YcR5Q4ZY4Zu867BJLCwugwjTpTdPPaeOkipJZ7p7cMIkMwu3StJsBN9i+8bsdsPYjhibKjPNoQtOtkd2x71YEjp1Rg1cwdU/PoXrsMqP36NYcVKxEkGXpuA2s8T/6jCwTXYDx6PeZXBk91Do4gzGzDwYV5py343nbO32VWMBEueMf4pYdPNID8WsQyD1b1h6kHfKP3oBlU5vYhcirCoBuSC6Qu3hRZJqU9UdVQGsV1GAamVMJ8IcCHHDag0I2WmrXgiv3F6aoI9nrYu5vgB4gcYLEx52I1DfVj1pUSNDfCHWz3DaPVdZiaL/Ctf6WPB9k8QQooRlsenGA5lvQzI+mxE11hbwzT+oCyR/7KqpkOXBsNG6Emlp8hAeRoEuu2O8iqR5fpiJpfIHVOrIxeTYM1gamnrbKrdJ/j+D/ts/PPXExgkDVnV9IFS+ArxTZi7gcyWSA4y3YkrASRMXb/zIKeEga/9Y/qsvIBuTrMaIYqbEYj5CeQ/iP/zB+bloE0kTovUvWjlt5wKRdQBGj+x4K/WBlxeJf6eFGI01m/DvuRL0CK0v9cSCvbXkOytYBWN0ZFcREpcS1p8VYLsTrF0R5utfy2Zuw+xR+EBkR4vQug/NG6lTPncMBCpdvFg04U7+EEFJC33R2tl45ewf1KxDEiS61vzg8wVcdAPkz3QJJ+NqkeX9fxdKPvoU7ChXY6Z+Nm1EMdnPq6DOxjzljGohyrBDlsRCyNy6RnesCDMwlt67e1k54+pNaRYzlzBegombSs9oA2t/JYwwA9y/zSdWj22rLAp6oZsFm91xXr06C2LYa6/GYZDWPMDqU6xKPndg3OQQWeDZXCxuK7Sdy9GDXotF9eK+umGi03tV+0zAQtnY8vsGRTMF8isoL+1rbufj0GJx2vnpaKG5VqdrW3sjrrg3Vrj8ryOTstZgQy1vvGzAs4v7deRUljyOcm+NPvXp6JSEr02hp7qs1RaEpd/3dlwmBB+74ZgPJ6ptGUzlTDesnFKNRj6tI+8Ptnc6rlNFXH68CXrZ1yvfGHlFJhUjOKHcL8LSha/ppGU8eVuk63DQx3Zxpsvd8PiKrG5tMWy0BM/59drhntcP4145FrMKKf9xy7LV9y8v+vh8AgMEbvbLKZGqxR8P4tqs5EFtJk698PKsRr5E1bsOL7Cw+DPMQiZiS7z6wWC4DESFJPiMSF7MTLZNlDLPe7DSy5AyRGlrTchzODXrab82ELLh/K3MexVODDRYFgsnYOFzCFF+d0Hbbj14+NT0J6bs7IXx9F76XzREQYRLkX61GkFJeagYjyF8fl4O/p0yTLwznhNr3x6Hs2WIi2dEhiu/jGCNOkXWM9MPYjA0dEufvUR2iTX9aqca9kcYeIofuAI1vfgsXLnQ32LUgZEx9YbzakV+1zra5+U3Lhpss97UIRhe5PlDE3AaJlDa+eZ7znTFGQ//BE4cvRefLAY9sM6FELSe/BtlkIGoqraIWi2mvHzoZySYuP0fGGWDg8c0HdDpPGkG4xD+XA5T6cUnQ4zSkTQ57v+mElGRVXAjdjIwbKZurds0pt0omyrIX6hxP7w6acK/oVj93ypL0Ilr4a0ODbPzn/FEfTlDlssqe2pW+plC6ntIBaSwK6cj8hWYeC6+j2Wix2xyzFhV86RLBwRv3Alk6oljuzxF50ogrGxN80ZLULyWfOyADbZOXw1vxkPbf+vFpHASK6bP1kKvB+lex05FTlFCGHOSBo053P5CJ/B6clQgA7p714iDfW6eI0IZg6LviRygCbmvArw6q2h1jl04HmRUIO9ht4jfPn9OxAZtdEfKdedziu1vHFcRfGaxelN/m7iezaKFxXTuaVDWY8XeFA79RHGSVLi+tE+tBaRulxfy1ERjGGLbAZQUo+5+3xNJbMjvQxtJYQNxM+K+6IVTIDP8BioFcLPqypyQ7FbBebKPDVuLzbADQSbY/x+pvRmD6A2eHWqtdUy7M6I/CVcweRb2bjJ1Ghw2v7yj99p4dcLNdwXXg91dT5w4zs+3UId0oohwcBBGCE04jZ5FJy/RsyxdJxG8/oI93OUcdf1Sz6o5o1liGwgTba1gTclTILN9tC+uQ2clGjjZz/wSsvoTMCUgzGapIeSXA/0oWS3T6wQsRtbHBm9DvYRKb7ofRqX/IB3MPG52c/DI6xuqfXeQXaZdstU2TNR9KY8EjAP9G9macxe9rEVhWez+T4qS8KZqNSmtlZgezaw+D9Jk/1a/rud3HM6Ukxgv2LKtLA5dLDWeTfBwT2H+A25OYTH5Y0U57tl8kSC5uzefWyoR30+wZ791eflftJHz1xGjwEZYT9FrrqqswyuzvwB+Rd+Vc8tL0jomVtTrOl39fqaz1+Vq/SwaK7xNE3Cn0LZ9nC1QGbXkk0LUKQZosUMSXvveH3rPrTr1TXMvlrfkwu9EDcas3cc2VobebR59FpkHYcUwvc7aA/9hFEJpxpuJdjwgQzc8F5ohfnOmdtm9VqM244l3l0c4oHvxsnz7n62FWTWbk2tjYd8Ak/vGEppYZnz3zYfQExdZqPdDKsDu4AVvtd8h2AcPjfymqQW7Hj6hEhdvXj8oxzZx4BFPGWIPZ8G6VDrW8q+IZhN1alOgY+L13pJgv42RU96XZmmnsMwA80TH2O7WoxJQXjkgg9d/tX4917BU+LjPp4tr1sgF0ibT1h64nzVvoJnYGu46Ull+p26X5jIFPjxEdW/yvHR1qEunghLlQtYqbfd1zsXcrgLQwSV37/tMZaz7JeGFI4rJUN9LA5HzvM1Bxdo+Rvg1KswshPVwV6b406/Vvu40XH0GzsyrRfnRDniUNV/s/RX/8RQ0+hLLwrKI+nIX7NXPy+NOvys4vcKDsc3+0mj5EBwx1I/iGJFkTumX2Qs64j7o3AA3DjNd747Sor1YaA8iTXgKZMPw83Y1n7XBO7RFyg7VcnpnK4eFk49f6zloar9/M+KpNMIJTHo1NtaXxNWCqySDoUa14z1pWkKChWZ41d2UVSdjMauRpIGEm+Cs3eylv1+Jt/Tipio25Bw80cUUg06YO2+bJte/rXNtm8KL4r8SV6XBO1CpaMgVFiBan6l/3hWnyXPbAdn9/mdIt1ckvgSZKmO4Ht0QPhyZlWtt13JjmDHm2WhdFJ9fsq1LskqEJlNE5AqdTDdUcKgIttTVCX5tVY7zVJC4WsyNyN8OcJgrMkAJehsowZuAt2f3R1/0/V+6heZC+NjgBrGRMunXYD1BvSHDmBDjOMXmXR39LNnqAtN2khN0loYD1JI78BHoq3iH0ctvohy/q9IaeE/5mPY15kD249iC/ghOfg6ZrOz5jn6iP48ozsWXZjEnxBup8eR8f7GW1cwo9NlH2vHFnK+zxYvGZy7dbry8ttSNCqxzT2yeyfBJg+8jTkQycPEncvBi4l+f9pLOUz7JQcclCn8ArOmuX1hTn6Cunp4akWw7Cgg6pv6F2VAO885fI0L7T6IfHydoNIHaprvv/yGCY7JNFxjM+IDbMKA3HhGKeyVh1lEIQ671nExu7O/1l3eg91/0PS8QjmTpCZ//eb3PoALef6VDweZkoCHEipXYuc5oGA14uw1voMeZ86PZDe5YK3dHF51NTB3Ia9JU+cUipaxCWIG7GZeGDT+VneBP/A1cWC22SmTvtrtKgJ/CZ7QaSgF9j3Hzl35VMtuxIvzrfQpLlONq/NT4TK/T8hJ0m0NXMRfdThLK9gYGR44rIEL3PPjFuTni2WNV1KgNaI4DXcJXFTCf1Relb/chH/wobieB7Lrn/6heN9LeDM39cUin9+8JsNiEc9zYYhlLaVLABuJb2vRJoD89pyCZ9QNhvC1gsqG8pnxiuFWQC/mGVpjcjXCn+WRGn4tAxpmiPS+fOzybg5Nav72e4ClA40EVcGtpYq/iAOqAIqRFPOcSRP874qo3q86BP406Q8rQ3MZ+5gCnDQMilBEz6wiQ4mX/dy7ikOBWCWXg0jp6Dt6OTFZpoTdkmpWYopXcEbRZiic6gsyQB4u9auPjJYsZmrp6OULZDgZitkTa3rYI0tGvbB1Xx/feDQofntVWb2XtMMQSMe7oOiQ7WF9QSHD/UbFgnFN+WwfovNn3nM61Ww/oLjL5rv4erJvpNwR7ZposS64DT/Z1I+CkXnDeHQOds013rC5FJq/xtJIGVe8O7jn9fxbpbmqU//ZTU2KUm1Nk1vKSQ2T3I2R+hoW7XGjOhoqw3q/EtkNDEpMbr6ZJt+yCHkdw1SXE29fWrK3Tl/E0Hdehn7qwx+enraqhD/Ly6/wW0s30nI00q3hkfRhMKfSRUYPyUrlQ1NoLUGoHeSQZNqEnDeDaRaFeQ+7qzgE59AINbRCIT6l+k/+Dl/e1ud7Ao4+xPzD+QiU/OhBzhVv5PfNWSiQjTe4OogG/DdODt5M6oC/Gmld5yK/3bZYSw57NC//VIOQRkRd5Kg34boJ1uCb8lo7Ev7gVHiO+FV/HZovW5sraKu+jx99r4pvoPAmDvf2BdUhd9iYy+QkLzin9lA8/+e36vjBb1kmL9nZ+MFFEbsERiw4X7bugLg+PP6i2gsuoQxJlblBAoJHA0+oE7su3CMvDnEDr2iQgofjgapv4+C0u/NlUu03/YJnKmm9a7FyHmDKVmnwmY6dTqddH6DQ3hV+VrZFsUtPU4wnIW3zO/K1zD7deQ5DMC39AGmf2GOC+yfbd1FjwUpfbHSO3dNzoC6yWWsxlbjjVXjxEQNoADS/FFckxymCO12DaXOsIxrFA5/Fuul+N+d9UOrVWTcoy0pBOIg0Ehnw2X57w8lhcKd56zdBaxiLpBqz9dcd+snH+e7FJ3a/tTkfDqZnTndhkPaY2PJL1lVCNoif6bVVXyxFFjTMe24e4jwpzChIz8/q8DKJlN9G1a8ux4YglHbJD8Rw+/GCMbgaCk5g+FitIWWpD9Aw0SU0X9Rh69RIubr++2czyyefq5kk7tTJszcGzq5IMptGlvDbRPqXz+TWW9DR+Ls4gPwgFxcN3R3JrTPdGgyT9wR6g7GRGGDJ/BRJPO5EulLa5delr4rfTKaO8KBfuWgQVwl/8fZALuvW9acII1SOUMkU4PjHfcFl9SxN1CJYQNFetpC9/IjUhCOVz0AFZv93ZuXnrO2oL4cG/AbFEi8e1Gl18tmnQzEzt6y1VIe/Uzezg57ewFT29e1WZ1YrRqKMdQYMYQ03As65XGBfCI8Z/pMgAw8gUK4zSWuc97tLu4ScNKkgDAc4yoJTMnhAME4le4ZMrfT+TDpY06zlcMgGpFE+mL364G9PcM5GGm+G8VnWS+Pl7/aLUzv7ifhbFCXEH6uu/+UaJm3nGT8jtwFPw+1ZC7ZfkfTiryxqwajQkrNzlAneSh78dwEKaWVrFkcmLHk/sMuDDaef94EQRGiWNaUooV8ECZPye2XQFnU4ZAn8ZFFH6HwfVYeh6NhGbrhBNUUKj450o7OOnYP2aac0vqecNDtQURTMVRERRWOhRSXRHAoPVu1BMe/vTiDKEWasG+E3fFI8YWZLTq3O8XRsEB3WCtnmFGt4gZmGS/+7UgeKfcWJAnXdM8ML9NHSRPaxSvZlw8wbGNewMFK3XCES58qKm+JNQz6EZo32AumhyPLhJk3LPMyNXdzO/Ku/ihQpkV8W44fzt/jjpuHyd4P6hJ9Jw0xx7tUq7xuNGQTG7peyWh5ng1POn0nR2VCXHVzsxzEhQfjhc8Jol/NVb8LmvAA+m52m6HgKWHFKaP6VmxEky7f08cBI1PMKRdh4YsrrtJjUF1QLXosvhPl+4fSVmD0DM2E8x/yOMP5SyJn+AGqgAmaQtGQ2lOza8OfD0u1zGryRakGsXRb/AKWJyHaVoAdY9T43FqFWpQz1jOYXYJ2kdHWHXmZ+Z5yyw3wSyQtstmbqFJOSVbhKLzmoe0hZzCU4vvc7DZQH4KTFiAz5G0w+Q9aQQIiyYJu+jTulFCpi6kttM1hA8r6kIXE5WHJfPt311IWXHZ9HDdssPPSQ5Ecl3qSbKM48PWxleIOuANoyccND5XVF7mWs+ua3eQpQdyBYOWblxaJ0fXOEk2XM+UhNpwl/uo28QVNGkS9kJOdWQXfKQ6civAeaIACXoIhS8arJ0jVuN4cg/Ruz1KujCPsBFpOto9Z+ueoKm/ZzgL9e5p/iybJYB25G6pS2MqxMpzhZlI3C9R/JINFfHtAkBFbD2uQdXMTH+FexDRJ3KMCrqaz1IA0hEDT+VRj1NGag4ju0LpwErNRidwBCGn4DFRT4h+MgbqR06RiIWndD8YTgV1V1vjWuSqZj5MehA36i5NyEIJ777lyWbMrF3XE9OlPr8yr4NUzK+ZdHB8nVnlYR4Rtb6h2ag3T2WFxABDVsjO7VqcYLauTv4nmJXzDv7yBByxPSkV7H6kwEz7q/5WK2Cjy0BbaP0d+nMCgb7BClN/ACdUbBn9KUA01mDLy4lBq8yHZkZslm2XBAtzmQM/PwH/AZzGdzP35AoRx8LmVwzmqvvMA0zJBJF/XopWwvoeyPU8dRozkEps3y+6buh+JqsJgAypnlIoM3fMfJir+TbuCtQP4M/n7jp8bYkVwEV3L8mpnAET+snR0iFIzj0EzLhugMEz4fWCGU0bpQHWK8HwBIlcOQZCiPY82FWB8a2hp8qw9tDixcDAcgEakZzSWMG7tF5EF8CJVuAFV14vO8AROYKWtryBFRuGQgdOtDsWOSMTLO0E+40EtcDkIuHVjo+Y6BW4ojQJkFLuPr1b8CaSopWeamwPPeTxAC+/Rvp3afQCVbCpzyFaUntuxzeJ12FZ/dAkWnLFGAY/wvWlr068lDUD78d5yI3fWy71QwOeSDPnDNBkBU+zlz8hZJXqKeADPF0WNL36hANL2XRQ18JZiPQtr+fOFRs7qhuS4UpyEElCAp33V31qkrZTL/i9kMVlSZbPAVUUVPUX8cYVI/gXhpXSycRlwwAYoNH8Cs5fu+iA6dcXLTq64oTrq8EDy8bnyZKPWUdf4HIiv8Rh+fFg4m2fdyDfSq92IvvoFrFsGeAFEi/WjAimgGShD8o4fc93VIT/lvOZClBQIyt/Dzt/XyoD0yfyfo7wq0yitwxPwp45EduYXBbwi4hn+22lQwyWArYJ79yjUBnSCIYZLl1VVsG0Yr21nfP3HDmuRrOEDJgZW4ZCxs/438i7dt+e9ip/eJVWeVTP+wSmdHKd0AqzX82ivBEiFboPsSGTb7SRH7d3EgeOJ00Oic7V2xEFZH+kNKsX9uZgdL/O8k7bTjSFpX+wHGr2Bzw2BPYIfQC+9RsNX59rWuVkeYl38Qb08Y7nwWfsNTXxlfzAttxMIdqFervqtp2irBym0KaPrvhNDnR+h/Gw2AcoizVNXTPYPE4TUezVb7YEOwhlRWiCvM9xsV70syHJ6Td3zHV/sB/WXEugAHo9gK/3c5C8vlMWmOHPieqHHipyOc6BRNMstRfmgfgw3fYbPRaGqhB6xdAHWeA0aEx6nkdB2cqxNnci3+73/Xn3+iWX/OV90NAQ6mdCMReswobf8U8LgIsAPJhPNbvjdJo85kfOG8zFL7e2MIgJrMeKUHw9LnerQEKGuyfCai/G+L37mQionVXf6uXtPaV68SJx5Em7wD2+dc9BDJVZWyQqR7WdnXRRpgPACsEZ3uUDoqYUavvrxDdTa19L96qEl5WGmKlWC7dG+tOWrjU2yPDZ766pUwIgFZc/AY/61of1M3nrha9tWiKmXKDD7wPfcCs9h1Ys0nbIN55ra4r5j1OhC5TRwqTfimWzBZAzUjfsjXxvxowmVUVbxb7Wrq7Wp85TiMe8U2E2RJ3atIND/ofjL1yoo8/FCsMhXGz4dqKPtJ8Opmac19h9TaXW+L2IZKuWcE1TqRVQT1jOfkIoVXGpYkRpP8X1gWXM562r/ZaUUUD64dLXEc7IxorQiTlj7kUsu5COwonWwBNY2zqlWWrWVqcjxuQakC4T0I22SjPSIgGl68UOra9UGNn9UfGJEwdPhw1Tbhn4kirADDm7KoIgD9280JH1FCN4Ld1v3nrxYhNkhFf09t6uHcIUjIzAkw+YREV4gg06z34/mTp03ibgJeuv34g5RI+weqiUaxD/MlujVHs1Yfn8MavTqL1nOVfpkx+OZXxk1W+9xKeYOwdtW2iEaW5+WiDOtDdi1G9TN59nd0KhU0N0Ad6Ro0LsEn5xip74OeapTQGP9aRh68bNjPMXRbM/r53Mj3MQISlfMtEbQJD/5Wv6PL9SFOlotUpZhYpjwYQBPlLQu6mhnfKxkX6fLDq8br4diadyj6C9sfdLTk2foAVUpDkULYvzipOVtxE4RLvZilC/YrK+DX7xeTye6iy2hj8IbE6IdPxhv+hhw1JsMvXqq9OJuwxzRwjRYr/3JQICq+6OePTkAWa9CwkNhJJmAQcVRLhk4ypljcNPXSmT37XK9CBI6QEkn5X6KLjPf9Gj7XDfQAUOcbhWi8keHTGyDdu5lVHS6JTquwI+fyuDTXOF5B2x4ALkVQkx2xFuWw/mfrGfV2U3IDLTHY3SzBS9lJcr26/UK9nOqChbji2JJ0NSDvzYyOwmV3JW80n8e0LEDB2mCaRdhrjhZKFRzYjA3lZw6FfxUpcEB7FB2QhZuAVe8D/HleSn7HV4Le8FwoKaNSsLE5zUSG2PZFE/p+2PYJFpHDIH2ShQA91thtFOQNeZr01Y05fLS55ghHLrkHb/TFfj5af0PrdMY7UNOlDW/H+VqWYuT9d+AgzHU1jV8e2R5Le348HT7ZFzLOudibHGztbVwAoDwsKyefr7z+NFvWIjpAKaEQjUVXvp/hNfivsb1Kkj8oC5QGk9c5IgbIV9SKZF4ScowEZfCr0/j0yCNqa5/dpys0A4Bpicw9tehLNHz4h2Dk14WRDJnlheTm6nh6brMcS341xeYbhjGlUr1+zqdFwTWG1IcG5VRLwmkYnDhkuw5UFN37HbU8yOx2rRHJpAVb/PWfV+nmmMsgjm+FyQxqr+KcXWOzt2hR3Ocseldp6xvIBmzUB/IhEPnaVuaOQc2pL+eiJJ7+xlGhZ5TNZAQchI7u1HiUz3q9cCGvLTbKo18cpuV2/VewUIvTrcrqwpUAzYeUAHWJ9VPobpls7oBNnxGHnU/sac7voqmXeexAfuTan6stxHBpu6yUsIE2GYjdmRLGIb3+xZIUlOWfp+5pRpGbB9FJ0JddlPeGpngrEzj7wj4FNw/+UBP/0PS1TVb72pKGWqcLOR9ZjUezJTz/x2lz0VWIWwuQ/9Vt2DYpC/SoZTFNr3b2MzgsFNpg8a+vuJTxv5EAVYS+3j7n9jG48FfcimYHKgarYadepa1LivxIJuccbuifuHntKHQyYi19k9cqrLY66JocyDgVn5QAVMPjp+1DiTbtO/sUL0CtcBb/BWujYxyu5Qwpm6AruLOxEwnnTBxy3iVfzgdE15HiwKF8/g52ChljXggUN9lqIElan+fvZgNf0XjBfpV3lO0yfsTqGDlaKucFODsnVi8+FzC7xKw3uFUYfKe/vRAHWKragW6oVCmpWNp+tppp4hFMb0RUlLV5tGmHgXBkRhz7K3VSgQgCGWyojfTk10fgmCa7YgWH6YOKAyHT5fJuJbapJTaTSGrLFfdXvD4fzwgjN+oUPx/MUXozFviCtsgL9VJPUZm99fCAixzaIy7RkqE+lRAAlCJ++ivRUrEGO1tj72ZhoKmdhgovMncexPbR49zjNR67UtjKSSHC4pjohL/1SBMZeCbeUQtV1/vNRxTWgCrEtHvoPkSa8ZNyuD2GYd0tROQa4t9kGd7ha+nVGxNqY+ci6RCisK3tjMvy/HEf30DB18gqQ6litRsR4hRP8Ow0ENllJR3qIs0jiJ3VwAKiXxpE93bOPVKpkz4N69NMhgMtIySaUl8oVLxpU1da+9uFWK6o9rvSgDKrchjZmpCsm9hSNmcRbA6ml7NAzsg6Pr3asPKX+dVLT26RynA5bvXL+B3WgOIdyyv8HHCrW6m+NCiYuPXEvsD4XPHHpanUPtNu68UPVITWBNawlxqb1WLpWHas08yeUzRqC8RMpX1WiBXJf/0iukk7cmj1oORmy53ireGsf4kCDCslDzaxy0TMWGYBaQpwh/QFeL11kLL8OxF2wVSNN+p53/s9L19o69kned6vujR9OpiyY/ipeNLfJCUa/A9nIAwiD8PWLO8RvC47ubAhJz06rg3kLZ65df6R2mnoIIrfShjdOjI1v4dxinGse9/dZvlgBQKCYcDIGW6sd6DGtJfWgxN7cVEhQEYOiIjY12VhYAMs0WJw261Y6NlwBRhg5II0hHJPrGWPUx4TtL8z0sydzdJ0VMnWfoIxSy8x+6gFan0Lcp9YjVwGfTt/V1qqjixc+aHEpdGj2Fb9baFyDJiS+x+gi/hr54Nfh5EGQc1vkLK++W5Ef0uJXzXMvbOCEJmd7cZAUqSm/3emk/JPKxnDvWkVsipEOcLs9u+DP0zcVSNSTb0GGHKDydVeVJS7LIb1s+GOt7J2vYPhcz32zrPkhrpvYiR6GlMHupuWKJfIKNWGxUwhnPxVKNFN/bywaH6Gg2tigv+8rhcdu7ywh1NFMTPNfEB0Sr4t/LYyHWaiA0A5e7uhc382aMhrWBZ7Ew8tVaoy/Pipn9JVMws+HOSbMDBhXvMeLUzWJnbsfDoEcyWv2t2Z4H+xcZx4FJUNBQSRDZeKHG2ifvkOFKB/rRJwS1dJhVzSHqoltJrCptUML2sDA/MeEEm4VxH1ZIBLAFMPlVfu+lPY4FdqJqoSM4lasikwNR6SXP1K+K9OoxheaZH7dWfqzkbg/yAtvNGZu3m5fXmVlPCnyYD+BvD8avDIJGIKOOwQ/ULCqK9weiKb7nPNtYcks/7VSset1AOuxW8c+rsYvSH6jug9EWXnYOQQzLasBnWQlpAuk0lk6HKZrtkx3NLCcdDPXkyVYnyIfVrJkGjZH0tGeedpxXG22PWj6PLp1rZi3Dwg3q/Wz7PSaBjszgRO4C5/QU2m9RjiMByUTv539vSfEc2PQ2mKMxOn3FXuAQoICMJBhQFURGh9bjUmpGTdInPTQ7G/M+deGbZoiv8YXf3AYC6WZjgPvlTXObeZMkYE/0DsxHuVdbzSg43NyPTgLB7KOljBSn1tyzaJ3k9QUV2JdFCeLrHCqeRr7Gtjrl7BimHGzIxW9Zrfuwn0UWuSbO5fv/wXdH7+Ix1Fkv2dZ+gjio9ZfIXWINBRBY/lsMIUbvoo+/k3T5tZXqlFXDJrZdejXFIDf2td8auy02tiB8D75izVL7CEqwg//h2TZ02apNbz8BxMuCjKvqbIfW0dKDb0DLMK/GRFS7VUPgbis7cN83n4ycDXAqQrxPIgSKhTUNSCjZBXQPgCEDUM+LHQlRf1/HV1ibsvKEU1h+VuK9t30y8/Memi9VG6Jp6mmFELzM/dDCQRV7PwjX+ztVWZB8Jt4a5Oz7/234lTyOS/qJk1hYv/3BasVexjf+vIh9A75/n8LAcMxaFfGultphF8DDO1G6C86IKsYYyXYn3+qjqxeK0RVVaBH/T1JBSE9YxtGfBrDdDey6j9z7YZRQwcTu357ZTAhP+yMlLbmIHDrOH/tL6aY9u3Y5G3nm8FVt3m0JC+5RPel+cPYGfMq9JxN8ucrlH8YP47vbv7nyPjhghrGOiv7Urq3Y2kCO3+pI2GLiH1iiD3qvq/uAGhu/991J6MtkjaR7ITlGe1IbwsU8EFx40WWHaR+i9bObjV1nWdbPmP8bFRZdyD9c5+0kQwfMKBvpY9PV99O2wgK6hVf211/1dWkU8ZsYpNith1430Yg2gZhQ7i97c4FbyR9ufYunvNbKK9pzr6O6+OLvcHoqELI8j8q1f+jVTCihY06/xuHqUljUQwY/5OesOueHlABZtY/pQ4kezAr/u8yO/+DL6MzOrrKh1rGBpoh6fQ4btcvG2bow6bbIKmsTY4wLQ/vwAUydeBlWHuqPfFZOWvht+uwE0QIIfqq37f+tltAfv9crzjRypHgYlLuZx5YdD+TR/P707nCdeEP1vTyKdfWdpK+Yb1h8SKMtMa4vk3+i/zTCkxsBh+Jtpcmn3Oy4sbALmxIK+LF7873SkaTKwl6gevUPt6XiDq4atJ7UezcIosJ2YWphRvsxJefodqumTy+a5q87ffe1nD6GE/r2Osx944XoHgcB1kGdDEMd42wC2BTSoBb0qLUX2u0iQ+POJ3nQfA+aIN6VunYmeXqmnf7MbEhbE54BGD/13uTk6dk/j8UZHI3xFp7BQwFnfWMtEe+vDs3/49qEfuJ5dbD+m74KcQHeCDoeKsh53fdExrOmrpycmXlS+SNedMCYv2OoFnJ3D5FfFBiEL+HRCRt/NgVFhcMPr46LGalFwnVLiD/CNqgM8/bKQjNe5GMbJJ7QPVSnTiUCFm5CTXrWhSZDHqssp16+F7wzcK4O01r3hFkm3uXs5oVWaT7mgxyD2xQRPYy53j4hIBB/nXJUEjY3nYD57CloKnYwNwIEjLvZ+ZWyy42JxxjseyFgpTkbJ0AbONnUvg+iTZmnaIuapEGbAGq6lVkopKXvwt2yFTfCpndH5eyI5aYVdG4Lbh7jH9NPfOjC/RL2KjKk/IF2KVNfy4FpDzxwe0yxEX1gl+fH+VAKqDZsQf4u5S2OGnTJEi4twgVqlkArDVROo0yMc+IFo8bOr0F1wrUjgHI3VbUfCrM1EUYpG+ahm6V6mQMi49+j2JddpQC/oqpXItL8PBvn3O8uCBr1+RH32UdNBeAT7rEfrsQqsDjcm6dqUKxTX5YM7DKC+sOzCsMjbv8MT1iOPPBkrxRkm+2ii5HUf6zEEaJ4b9DN6zP0WVyPop3LaPCT1wn8ye+YEhp81vGUGlSFdkMJg9EiO6ohPgg3kPVbot+xCIxZECNomqT3X97gtK5bfBCe3mQQzH7+Spi4lqBVYh7HggMWJl0HMULMCxFivon/kJ5ajDcQZHwMNXZYZpdLTD7ydqVEqH9NdFgoYmu+AdJYEsvfurgv4k4vJ9M824D97Ub8B0Yu5oPYu2mu1uJWrws3t9rWFZVWBSC6XIO1fB6a3cpqIBGxiL7DRZWJT57QO2bVoHzZrhIwXKgX1hwia46otmGnO/8vu+msAUqCUjIXK2O8kz2xZHERVPtDrYEnfsy9hqC3OZCNz8gBmByrK5LJ28+3/oIxkrlDCnGrzKWLYvrYgDDseDam/QmmAWGbKsmSAJHkdlazyiU3OHmPNF3GkFEEU+7e6h77mz7Fl38zvioFtRwrQAerH5vGjT80fUBM4nK4O0+0nUoCZA163uwr+tZL4iqkHi77o3A6K/y7UXWFu8X8jlLB8faHKTlKwAbDWB46ZOuQKGK9Xg1S5/ZmYPGIp5W9EZwXxUInkQGwHvV0avBKIlCZRB+Z6uwzp0+WFK7zii/PvLKQEfYodRO1UpPEaCIKb6p7kt/2vL6C0N4dhc3rVc7sajz4Kh+6jlMKUZrw0hlADOX6GDNEeJzmB+EVe8IjizsE7p4/RHoEDeBMjd4PJYkBFCHr/ypwbfs4H9T/QtWkThvfU3p7LhtmqP6qbp9CDf/Ef6zbZZBsqEq3n+IaQSv0Ir+g9N17ElKbIsv+btSTRLSLTWaodOtJZff4nqeZvpM3WqMiEi3NzMVbQZ0U9DxcRilOnAxRNL2pD9VviFxXEmZ367M5pQzsbpebAjxxaZSZxxKN2eZJOQBRUy5OS0qehVK7jRL0QomtXyLcMrjGfR+N+cmy+eXUOiiWIlg6tMeUgvmgWuYraFQhNQOaygHbESoGTHKHZnGIP+Qe4wR3s4qrRqLgAghVSPLNQBVD0LBaI7IuoEfqHJxwXYKzfg9w7jmx9MXaT88attnJLzky2tMCbtsONhuN8/uYYkHlCjr2JOOo6N4xChM52gp+wpXdbApqMxo9HtpArBX2NaFiVX/8QgsmMGYfIhi+M/SGmUJ0C4uMxgneBObCVup1uy7NHlub/Hko2u3skdu/TtyZLbBZc7Oz28MaG59nerm92ji51RRWsBpGdj/dyZmsUV/I7bOnpAaADV0d3pAoi355/SzWAREAqACMlg2NZeJa1BCE0S7IXQLYSot/IbTPO8cJQjmpUqoCT+0d58v0Jzd0Ln+yrR2U5p/fxC3YGgBb4g6Df9XfiQTOxrraQA0kIcWmpgXUGesWSGP+r6co/MK00Szj+fbghsEKxCn3g20TuHywdWj9ctMM9KkRLPku+BJ0YCxLHTINzh7Sp8GveX70QRpqqHf23Owa+NZOtfTWpxHfHxUioBIniQlKFujemPi0TzqsPoydBsxlSoOxTz+0slsJ6dM14Tkb9sz3pFXookQj3/nl+KtULuhl6D80PEWNh1Q7ClLAGQ8vSvyT6nAn15yr5o056c2m54oXH/5svVg/4LBoCQx8vbbkyBkf33N8CXLY1PpAKrO/+8RNEnSSMh0PeLU5RbXudLOk+EVmDx6Y3gwS5ztXCC1GOnnMRk2Ffw8bqqZPAGKcCf4wHU+177U7vMDqu9cM2biLeTCEFMQGQzw4ww1FyoFb6/6imbaPp8RGrBQcVJBoXEjC4vnIGAEQb4T9xNvpgb3JBt9YUB4g+6CnyzpJmvY9ZXrAW80x0UvMhUbJC9/AKVXawwvsbwhGSRDKh/+dsN/CP120c0qiVLQpG6hYtVFHKgIm7+/pVRAIqawdwtjsx8V2ZJLQrlC1e/LeOcHWVJ2CQoYyjC221f/X/7tQ1eKVw/muM94zLPPcUryam5XCy93gvILKc2aKGEa4Tw1XtaktYQqDWZdaWSR3amsx9tCzS0Hu338j9abOoQsaVCUEspCXlfl9Md8fSpJleJ99gWR0LhfwMGdTVvzovOq7guK1A4Ns7FGu4ytoOyFiYp15ufdhQ8biqbwmiSAfLCPcgDKsVLMQ0mdJ3PiAyQ29fYR9gayOeLciwmtzZEceyeg6Bd2KhXuw/wGyxhanHza/Ik8Ek8lTkRnZEOrmHZ/gVGtUUl2JvxkjMIt+mp+BPlt/JX3dxTrYy5WUy+Ijii+JHYcU8enAfarN1f7NlBmbZ0MW7MxddGxXLhMFoZH0JH7mdU+O56yVWTZMvja+h7qD94+4p9ab6onehf2M8/OYV+tv5v4tRNlFIKUpMx1uoxMYc5m+jFt5E90wQyd6p79t1Q+Gc7GCnBJ0g6yK0z84H+M2KRS4V3P3FlXE6gVnLXnETR4amnDFZWXyxknF/E3CvM8wMSFFaAeGOFay4mL4dgn500g8J4jnSyjWf6E7n61xdceDgQ0VFWOogu03LFJ0U5wdGWAOfwc2OL4uA7XYJicb3Ph2ammX3hVFQC2447bGJaz4n1vcssKYVEpo36jEGNXyntnDrh4/Q8zoc22AFsgQGcnIgDaQ8Bdi+mn0VdDvdyNlQIROnbvtIHnPU16a9jz/76HcfrLqLsZfxG7L5ot1wUyRZEFYazsqOrFOwBhqTBIg4rlMf8J6h3IuzGHX40rDiiIy0zo6W51ac/PatZ64EAecyHmrHo9kcHDTHi1FL56DmEzpoNDHQivOBVIPF/YIlg/ZIjwgAyVFEfzimZN/xvWlr4t8+KgLNaj4bFvSqa5l11NVgcMh2sq0tfsxXj7MtxpRtEtxYnXV/dkchY497jT60ouxZG5IIDd3mu+K5rbaKkccoeYQWcwLZTJ+F+NFexp0JWuII/Q2BDsPHS10IryALFIXK5yPnKwnCpkMXL6Hjv7G96ppZza3Ln65Q33L3qIa9ohr8CDN5dWcbS50w05OZuRo2BPKyATpACjc8FuAAOaYrGmS9Jiddz3fUT7zaQBg+BE1RgNCucTns+K0zEzCRA3wl0z3sZcmibS9MFJ0DDc5AAWsEBykHKpTHUWoBSeCGom92AXiFdtE0/+Czs38HqciqAtY0pfn6CjA34ModSiBLPzK53Ee6npvkrwT7L9APlmodTb9i0PMUq6vx1kmTpxtFOftUsa3ZiP6fUwGa5al5pK3vsnMr48fzyg7oK2IWdifBtJZvOXihbVNe9HtJ12zpvZSO5Pu48d0FvEYgwJeLYKFWu0KPmtj3FYZSZNutBddiYxKYN7gbgo26Bv5yi+gE7TEdcX6uiUi9rcyCnXB43aoMx2RkMztXw5T64/iXkbQK+SDwizlbmEepvOEz4K0H/vFBr9j8sJ+yQKzgFC07c/1hJo/BZzfluYyPfEVeL02h2EUMqSabvP4DzBabTWKUR2JdVTA9WxRQ3Ldj1DJ6JHV1X/ZQIZ/F7wSL5pI0aJZ+upVSERcxLhGJroyjB+pAINRDr37ASEP3Mg8okXiGQ9kbY3V2ruOj+zEldzT3yVb2anPPjx832pB27riuQsTy9lDAgXJ6v0lYOwV5Bv4euCHIdlkkljGqmA4oKGqxcEWi6H8bPKBo2RwKo4p+JLyC2dI/mqqxLuP5m1b4tGP52HOaG0ei5u/v10TX72VQ81srazbVP+RVeYzijpbIiExLvjP2ag4J5hppk+ZVFDZ7fkErnumaG5d/xOBb4qvxXwyM7muvmgPU+ChyiEg4BrG9s8BtQgNI/wuP6nK4FkO9ntmr+q1pBvktxN9G6CfdSRpQM6ABzMgCr0S6/AHMjmxuXzGpseFRswBQuPsvhY2PkijYr0wuXe/fvlL3Bl5jnJquIWTzUWZdAvcStSUcxv2eNz0tXsX/1BUlmB8LJkEyfBj16Vb+H+uDf090QxSfHuJqIp3sJ/cTDlFBd4iY6wgJVKh0PJQEsrf2JojQ/86m42WTBoMloKEeW0bLjxeAzfAjvgrHz1e0N1falT24Q5hAKx+gtlv5W195zcCRsamKXYtPWlAmEUy6NEa/DS49ejw94zxYbr1wkPFNXTanLT9L0HhWm1yHZ8Z/N2a9Ge19PelYk6j4LsGOu6GCN/Etf+r3WtHOh9k+BMdK+BZsD9ZhvWKjyV5bt9MB/lBHiJRHdTV/4S+5f4EI0F3nFQelmbsX3NVE2cUiwdQ00BqihAJ7+8yWRlkQiDcepfGITak1RgjSMaa54reh4Q34BcvyLNq1O7BmbFnKQpry4RJQHB7xggmShB02Rz6ClQQap8DtmsS4+FgqUwMH6Elf+vQWULQx5eXn/PrwtR11Q1pMhLzM9AHXsIJAexEeydsav4Qsn/Or8pCAIcWD2oHZwTEZoK1WQmOMYZ30+dzJ+5D1uxjxqiQD4zq8Mbq/ibzlHETzn1fAv269SAT4F++eGyV/2XXaqicQq42EC4r79sm3+z6Xvhnr84xz/5u4+fLQsG5TGQveT1JV4gHqDusUS+7bG6N/RJc/xKcezAcc666/I/ovHExRHPkb5A+1KM5gV+ZK7srkU7XcQVPyB1X6SyFpUuz2SEFCLs423ERK3XOU8pGUzVYr8miZ3ZxzodX9mIi9yf6VfhPuJM5NXULxE6qZe6407W4cEG2W7pL8vxgT/7hQObiVW+G/9YlfUaiLQbiXkLY6IU/rV8WV/N4HgV9I4v9S6NIe0YzpnXF+gAkZw7Aza6TwDW6W3wXJ355RrlZdcIFr+WUIPpU6VwkE+yL49c+hKtzZfXnRJh3AN3zUtSZEN12BSKwsIRHGVnmDas700DisCsQH+ZHA6SNgH0zyoA0+3lVQWmbrB5cw9w2Dnft6/+Ay7/jTQ1sonojGOgRZ8cnNRhvpKU6caiezISxDXSIiSuGPomDGLRRZqM21PILec7D8dU4XM+479R+qE2dYU6DDjNfDKmJ9dqEz9Ga39tmeH+5xADDVUm8JMvMoXxd3bHvkjsMysmK3yF+gbaRYUV6UOvOXqAmKqa6kd/JI3xJGTeY/sp4xuYrL1BnSokcC01RJRJkf5lPhIITPHbnpchA48GIehfoApBb2+XCEd9h3ZX+X+meSKRax8j0EnKAO5xu4q09Y16S9gNfwEQpEXyLs48fNvJNKg14yIk9M8E+WjSun5qqTGD329tv84P1+cGgqqDDxjT8hl8tIpI70VJKaXv7npKXRUuVN8Jek4lu/q7e0uJPe9OKTRLXMWuuFrdNdyp1hnJ5wxIB27w1dH7NmRfvtTw/e8Xagbdh+wZ0k8THvyih0SZSPok2SFUTHWXTM9ujpd8LHLU8XMwfIHvraxKdYeNdeHp8NEhGLQy4EYtHArXXV/O6BFhBm4s/ADNtKozM/iOmwsj2oZPhpt4GzzYhACk4DtBWlnKGf8QRs/hb3HMBubwkUM3FXN0zPc92Wqz982OxL5CPXCPJcR8jT/tS/tD5JEt4NDDYFYkmTX9Vag94sS7KOWPpr+5eZ0uxzzh1rABjyflyJdQuc7Ko+EI/GcErHCXCVU3lovl7A6VzRqTeCf+CFv9xiNKty0wOepgMWC2wUUx/hFfeRmsOx2OOk6vT1l1vRyaPzZiQhm40FOzg3Z8J/mXgTRBn/dujfIaXS8NYBbJJngmVK/Q+x5HswvbFmhiuQO4o1KD8lf8F6M1/x/qTBI0lnAwHsipZWf5+WjrDrio9dD0bIBsse0dnGghuv4Jj5cJaWain56eCHBVHDY1DJSgHfnySsz5p+jj1o3XqExIisMuTiFIYMbao/354EQBdTpoAi9IUoEj223nvN+9zxZgPikCsybD7Ra6SYcibxPvkMIfiW+bRS5oWAkC1O+FdLad5Y8jFmkH/OLRLCTapvrNe7EsUgx+SsLIW1jdyxyP3gDtYydRnxx5IEIXqINiomnQ3FKWFr0kY3/NpjEoDHjhQqRL9uxjzaphUuhOz45e+K20cQrFOwF+xsIVoMW4+sFOm9GHlg/zL/2GCHifEDUVx8RN9LPtuT1jrVW/rADaE0I2SAEIx86qb+X1LqDBdVlm41XCv7If/gMQhUfaCmVNhHSWjywotyA10c/5eRLGf/GBfd57AhyvXnkeGRjy9ndj0hIEPppD0WBFo0Iir8I1YwHYHdJ/U8329o2sk0H445AeVAjdDtWVBihL55lh+f8QrU/n5rCp9wZhYuZ/q4U/Xpp3WGNehZ61qJsj5Ui5fCzIPs40SEbTHIPBvBXS7yQ/nagb/Mn2ltvW0oNVxMbwW1mICSLB/cPXc692MwvYf30Z0n29NPZiPUMT6T5U2P/l1dU2pZyfUw57WY7iQD5s8/9OK40srNgopjsPavjHaNNATIEJqN8EwqcJhAKacn4PkjVPah0Aj08fPetI3mJAEA8E+idN+HSckdw4FVR8UHhRat3SPlp68Cvx6LH43P8OSMyc9cLaxAIVV5T+nEz7oRAbdXXmcyWeSwSTgAjtVkt9ZTCEA3QaLN/1tajtNU5oCz4bWdiZAK856X7Svy/AGxiCmNsf/1xen0sCUL2xIb4Dq9CFrpgBYvDqPxIWGkiGhUyhTthLke5ItWjPQvNHyL6Sp0MXqP8S/xrY0KWf30jbrEn698kv2KeD1QuSAWfo8v9xhzQqgmGEB5ABGGZPhgIMJE/9AqgNSKsMv6nU4gdYY4PKvK4w1VybHOA/BW/o9gjmrcaWsh2E+ZGKlx4Jfou7Ih10uPhP0kDpA3g1giy3pleKB1u8Q4cv5Dw/oAS89C3oGlS7s/f6BkUaVHA46CCt+rp0wMgwvcZBj9bcUInMi8NP8RnZrstR4225FvV6fxvh9b1pXyNOYP1sNiVBW4TI/n38CVug+vrmF+4e1K3gGBnVMP1Em+JEmLRJ5tUNnT10h0GBHi1rGPkVbIz3j5D8JDqKzFcT1ZS7bqbk9SXzdwnlY742OcrjjuEKkbVZpfdwqbW+De7BzzFDFJjOtIr8M9v0JRc77r+kHJ7UHMX/koaTtRhQDtglyWIIe3x1ljjcI+ieehpiX60MjfqmM4kGPrrQfC0hjnxLLJkq7a+mbGRW1c01T/+zfvxX/CgNKIpNY9KhMw4c+onqwFYrvXJ31KAlZ6eOk5xz9FUqviMvBJ7DvpzzMoM0XH3vA10f1Xhlysx2/ybBJWz8RbakzClrjQLjGbaRA6Wr2f4nLq3E9UTcL0QoBWq/gjJs3Lz3+TPDqnzjcoLvnRcTzSCDcQ1Do/MthaKMzhgZZ0G8HWtA+I6evfwMOWw5RN/Dl2cwlTq9Tk9EIuwJE89vhEo/9yI2asGTxZ0bj9882Fl1uqQ3fR/McaxarAe100WR6vcg2kfofWSDjEGkmMKc8bbj+WXQhgXHaxH/VWWRJSH9rTgj4ZIwBEvJdNnwJYLR7sxbHGWNsqSNb2kAH5vWoT7hxBWvYFgOxoHhH4EwZ80CBzAyJ6iPhBjbtR+8yaKhOYlSt3M3uB+yxEqOefaqMTTcbFAOXwcrvk98EziRf5PoM2UBb9KpTxeoHY/MOY6gMbOjA3SPvk1zRxSQS6cKjFrDFG5Mj8epxcMJb6KvzoQLJXSlyBVO7R3O38+2aHj+v3khEK9+8swd9i5c5QIPckidt1o9opW3ufr1u/n8vX3Pdm7CeW3yI+NJfU1ZX4I8VE/0hzfPmsF0QCZlL6Q5QUucuWXas1YRuRofO4L6S9fV+Qd8WVKohpEgGND9HOTiiT+UYfjOx/lurDbnlA1uA0BnASaZRpSmFPlVULFym8L0JE/XMb0mSP5Iptt8q+Mf8ymkJd/gFyvYy5Vq9c70KLcFEXIQt6jqpEHQmF0B9WCh2MGgkJb6sG7YIkNHFNL2zGIHa33YF0K0RvLlmtJXdXFepR+KatnRk8VBCbOIWEjulCCFAlPyPBWOsPjogUgSryqhXOmV5N87QgNKx4YUsprEBv2+LBbWwiao3jwszlMslkgdCaEzetrTSrRQ+ADZo9pWqiv+543QHpQ7iU4+E2OTWPp6wKmbwxIqP3qUVXVh4UqOOtV8KOYn41MTJvBBjaNJMfxmLAKvovLn6JEbGY/q/k5Ne11J/zC9GqZCUYLal+ZCHqp2wZ2IKS6U2XWj4RqKGmWh30g8UIKS5JMI+Uboq4bFejOpyY/wn2e3zx+PllZYsP9LpJidh5rdlQdC12zi43DfDk9MAc9SPPPOYuigaS4U1djn5wfNjyxFPc9I4n1Jm6IRF02vIu09J5Fhvob9VrkMLW/jstKpG0ak8tOC6E8whi5u7s/u+Qupozgw9hq1JQdJ1vnUnTwsvibOP7XdjnNirfSDTBS/c+pr7lRfN/vMZO6kfJJqQXHc1bXQjrnheQYVwabUxNbDZrzeF15le+kVVjSvNSi8sNSTUcIgEM3EwZEUcmRzpuSuF76X518nGV7RnOjbNGDOkH2DZDaDEhm4R4UGpnFMS6A4CCcDWw1+avpzZo5LFelaQ/eyWY/D5yJHS/N04gHNX7czmWbcmOe5QqBpP7dmaCg+9repJ4rd07EfwljIOC4+EYL2Uhm3lLkTVeyQCekd5PwvriE4V874G2NKQfFUaY1cDYzB5gfwKQvbXs2n/TXFZrr8rQLI/QsvCeowwfE5Fu9RLivLHhrAsd5UAljzakclepsvzKDZwpc8WIbIvawQGXRaBBnXyYGVft46TG+/DV7gNcVu4HnJEl9ar8jdFwK/q5lKQuKZESyi5EDLvsCXnyQWKTAIu3UbmgKUxk9Y/6+xFNH2EbLXyeAfuIpRDVD47SVOSpGa1qYwyoxJ88y+RxO5XDfEen6AIqLFDEcBwEcf6AJxvNXgpp73yJ5hwGAyrH/4lJ/g3LaxwUdE5e39Zv03XW7Tidxixj1BdkN1kKj72rHILNM42hT8D+Fgb16vrxGJkx7OHoumUXzWydw4A8jhmZf992Tc8av2LRisgGcLwhVSL4m97ae32aaUujXTI0ad/3lcyw+hqijjXSKtsj1V31ejvd4msPfp+atRfuZII8oxzYzGxOsU95NmyC4SIk8z6s5RgQIft61v9vGZCZ64M5dseszSK1Q4fLXdjodonRGQBrCEg7s0XzjPhRIUBBLnjrv2/DLK18LqEkKgffNFapU5dl/jH+l3U+rv5v3UPcE6uSviqU1vnOfp6JN+gSFDSVG03W+pqf8iDZCRgZdW9LHelhp5ZhpmmV1jYRvddxtfLnNzHRLuF6BgIjW33QN07wYXmMI/ULjRkTXpmzQDT6q19S12R+f/ksz5jUVYkk5mERb6ud+ug8+kB1AfzVdqbvYqDMBXuWVZH9NoiDpVpV/XgS2pQ1rtG+ixarqeAjTFmfoh7fRqJFJR7B9d7utVbgU/Sj6M+LZB6Tf4t5Yvs8+1cmmulBiyg47I4ykMyMMSbUWnJvWmdvFOEiQ1Ra2eKW88WjhMLvRWg05iBHvSCXzm8L23L9IDDItnKZODvHhXAyWIKnMXZeOrJaHjd7vajH/yb7prBlDPzbOyXTPWHRVDBf1iIAZ9/EDQWqzLNSYMN0UbTsjRRFYP0r7ZrnvhfHErdWSCzEW9sU9DiYo6qmqjzflNNsSTaf/I7rni/HoFMSNY2FYSwnA3RbbDSFfi/VAtp8icYjUTbPo5X83j/DXMpBxtd3MMl/onZnnu3a9/yVtxEwMfFdrjjfq30+E9jlUXxqEXx9f8YihxLLY7KOxZwQoaE8I+9xZj6Et2Q5VpP+euANp4frGmgH2W0rdm72qdKzs07iZv0auv6JHr+69N/qTsE+JDVJU9qea6okHuohMva4VOw+QmT+gigWV4GN0v3ppOVwMD6zXFUF0t8/8JWPhaKooXcbowtn+NNCj0aMhrnpLEDGwdzhqhbv65jOpcz/MPVagNMx+7eH9x7jYVXksn2aGNZDP4gpCYON++AW/5EhD/axV3XbqUQn1HyljToMoTfSuAdZAVttA+dozkmD6w7R1V/ICA6HGozvIm2wg6JXgaB/kt68o3L2+suKvfjvwkB8pVJRuF5t8Zqi5I986ifbi9VheSI6yzqGA9HA5H+8CB/FTTOsfCmjaWdhik0JXJy999yDbBGKKZyva1LMXi5t4PwkK3s4GQoCWUAeI8vWr1FLEPAenjZc5EH7VH4r1qm/iD8qPnJ5wDdpLIJRxs/quwObxK4bpwaouiZa9mgMowCsN+lQkipS/Kv8AFgxLncYMeqBU4Pgh2yFQMCBh7zmn6W/9J+K+jPLpXapEPqWVaDKe6CxsIkeI6p+VMWrV9X5wDzAU/uipNT3tauWZFi2uoZvOSFmRBM8/9mtHDxdlSNczi255flB1u0NBI0L4Xzb/sBNHNzlnETANVdRmvTL7r3Vy50UMZ0S1NBtMe/lMn4CHQvbP6xMpKgg9tWu2eGN1J8fmcI9Orq9JF93oDUtDSHZVQxJwEhBgiy4lzrrmuP+WX83NEJuEcE/IWWkapEfMHXHdlK7pftZnU3XLMCy15ayYbh1QL/H6VuDlbbp3gsLj0PHi6l0kS9gPXxcImbSqGvpG9Zo9lt0nd4eGx1RXnLFbvE0DXPfE1yYI0hza7+YTMd2cw7DX4C7uRcWQQZbLobFCAqeGIBwsU2mtdfUSYmy+aTJRkn74ORQ1+iN251j3l0NfT9EnD+3liD9JLsJctnnTEj2MggzVZG0UzQi9X/drotDfv/WsrrrT+ppHp19pebLNkq8u86GaS9meCv1yhf+ukRb0XpVOeQj5vyGhMOtOrlqR5jEVUu8bW0Ok8znJhLB/6O/IygOFHmOb0H8qio/kRVgRpDx7lkpfq9NWkndoqWWbH6E1O3wgy2fBV7gX7MzRULUf7A8iesMTXfUE/YMzK3ZUwIDEXlb2GjiMeAx8oeFCitU9sarFGd9K4SfXNnXMbKfEPVsfIEno6mcm/AjD7jbVH11fkuyl0LDe6qMUSd1ZvOMM7YAF/ubJJVplGprafrWsnbsHe8YX2kvIS0eHcbdh3DVUaQsNikp8CDCH7RdUdoBu8oQdHi7//qaOu+TLj0aFPMTmFQFd9BtRnDQH/1U8tSNPrG6F8aWWd5Krg2U0ui4pv6JJs+8yomdonPHk/91A87RTjoIYgb8b48vOwFZsHol8byb2PuyNgdiPvSGQ+TOC3eVWx8DMD34h2vLy05IguvxUReQ2xLaCafw+XlRlalB3zDwlpMYpgYpfLeHEkkiwlEln3UzE85zc5qr5qXPFtLZGmlBYSbSjL6OmjRTXgqglTWxDGJvPjcXA692vTIKfzdw3bbBZZ3J3obSvMFo6mWk84vCZw4vvsgncncD3vVSGNrQPuXWZHpqRdJxYLmQvC9s43thUSEQiDi5cVK9oBEzNB4xa8WtgRDmUrx+R831T8TVqUsLarDmZmxK6ERf6pmxTk+P18EXctc4n8latusVh2KP1rx5O4z5NqZXFxTaxH1YdAahk4xopWvJYDkElLPpXchp0hpVJkbw2bX2htAXpEP4bPdkxE2v5GRDU+RwX/H6ByMfC8OXcRQxzT2hrc3QFLydtvte5U6hypZ/DWO2VlbyC5HW1WAxiNksaSww9EwJMRfjv1a+nusN6TQNy2wSs/FIx5AqrFoiF15de+RAcs3R+E/jEKAo77tVnpG2cgRtjXfzBWtbYyqCJ4aqOcuuYHf0CvBVeJnvAliwOZg8EStcLLd2m/OBmvCd6b6exx79GFO+KHIt0Gj0/ejjxv+uBfHqs1T4J3R+MwHn1rZJAtP6qwTTObAqipMbPRwjuFW6Oq+R6ELpGKCpUVIb+ot45EQDZrEwRj3b6lHPHSMDnfa/hmzzcS2YTEBYXdyizv/IH2drz18HkK3o1iLFa+WNgHD3p1dZxhq1rbEiLCpuieImRXj84VF6tdeO8vBPIxAbx7+n+GA2OoTHEA7Fm3Mp4qGQu4ow/Q/u7vD1tf7p3sYL1DgDcmtRLQggthirlND4IwJUgmk0pvEUeDkemwVVK8gXwq1TC5mmTmRA+lSgiBOqm2ZX2wcuBlIqxC+ZuuMrY9DwU36tsGLlSlehVgMUljUwBl+xJv/EX0VnUBR7jky/w/lEqukKRZ6EiqQpkFUgfeCMc9zZHTZDI9UUlk/Bz+JuBwvJ24Xcx0r76u76vIuoae2KjHx+4Y8KOqw79IOj3i4Wf8ghkOYO9Z3uIkuBmF2YQKL3OA1VkVaQKH/sYwpFZ/Eq9IOaXewHCxAvXc6grWgYTBB0s6lfhdn21lcurOIBUyAoTu1qMhyHZ7KZZKj2nFnXM9pPfTQEE2sJfVH9lmXFDkoeTwYMecaFeZ20SaY94mtsKjtwxXUhX7qtu2tZvf/S9OlqTvTYZu9M3e1regZZXMb4k119gXaLP73X+ME2vhAPsDx1wLB23DY0BV0+aZYWbGPO1iV+rzqDdgavtChWqaP0y+usQFxdqZPrVny9AJGozmJYXK26SOOfi9UyHeCEVAo5u6EhqvwrYo/xDt7ebXB5pDYg1FqVD+A79urq68VJMxyhCRMccJ6Op59sfoA9L76itLAz0eDrarnnmZSBJZS5suMPuAhyKEudS76xP2myeC82laJmNbyhqqSXU+TEhRCLWF5JhTvsGF2xBntFLVTZxS/37xAwLGlccf7Xjv9oWbalY/+m6KqK/dGQuKnFmpvKFN7iEqEJ+pWtp5fFzoHr7oO5486NTb7lCVdvVt1r6FIpHOiwuO1jiAT3PFGfzDMJGMaYOjdDmqrP5N5Ij944+riJhCdxQOVVN1e7wW+21JRZxP/kgQBB+RgHe7fSjkfmNlhItZHr0lxKaJN3xlYvrPqo728BDQuZYqO/55eAVTBtmGBG4AeOEROEcJVzLg9m4xzQqh+corcJCaufX9aHxBCTU0sH99LJqlDdHSNWr1qHdjpfSSYSy1eCQKBFGe/8W+EURwbST+z4UByI3kb7jYrR/vTSS2+i3kjA28JvRVUx8c9iSatdJbOPncL0Xs3LvJocejErM2NusXhsE+AXCR0CU9kBRVfL7WCm3qRdP8xFmqqYqqoMftc6MeIuQhswabCWdvALRzzxMN/KFM6sxBjGKTzJMeZ3iU+41BN3yCp2uZNmOetJKXZV/yWzrycEY6ooWDmeOp8fFXfdXfrit3uTOCTFFQlVUq5Oh19dKWvEenAh+7ntxl9MGCWh4eeXuoruha21XGvfuwMbmc5gcGa5kFu10P9vDB1GT/Tog0cUnN1iG+2itk/ubKozeT9oO2YGuff3oD84Ktpocr7zzfAL7Srbgv47Iw8T+np3VYEzHdDsxSHsqv1OFVQKJ60w/ReDA+MGD8Q3bv+lnEmZFrH1TkIwa3Pa1aLSp15fVOqRfI7OL9fElc2X5g6bBzuy7fv33ZobJtMVzmLbxHNXybfDCTkR0EEqHyjeVojw3HdNBojH5Vp8LLniG+fA343xEBu5scJ0p8/C0Zdi5T3yy5JBUhmPefa/uaMUecPCl6m+izmr+ZnOYLPobd3QECczfRG+zo4sxnL+PYndYoLStBVLsqtJg5qU0gzMl2O9u8ZcXcimV6kP8SHEmMGkRV03ZaGgJQ2RjwA2E5TjOd2I5RPXd9hfnrC186QvvBa7/Jf5ofxkjCkQ76Qc/zsV5BY4/ElHbIJXcMs4QNqfEfb2vW8mlHUI+9qssSAdu2K2/1NGAIABcRgA6ntuiMc6Sq7z2okDu/E4IT3s3isVaacnIWvC7fxeXduByVAZpPFFcv7wFXA+jfg+wXc3e3S7fCq/eVb64QReYYKF8m+wJGuuGTuLQJmuuP9v9nvzYR36iKorujsnb9mjMzsKWVxj7gMeAT43qy9JOdWJ/h39wu+2UJTt8NZbudTfD3HpAFPxrM1UbTV2bQixKafv81/xSK9/ZoMvu5w8QhQUSyrrDi+gNuBrHSHNzIjtTKp1J4eW/i4oAOEK447DBXJ5oqFaYrcSYUPrt4PAmP4OKF+Vc8Vk6OnwJACZVem3B+eG6O9+sgiBFE9c6WdmUu1oIa+MymOC6/kvFxu2C+ilCP1vlf2yTps2Db2ruW5E+JmT7I8PW2IMj9zmF/mY4IcA3cIct4z+TTv4NSSz5/umdd8e9vxlKH4UAEsGwSGa0dozWL+DewF8g0X5j303xusHoQ5luf54X/9zfoqOpOirQ054olyno5u1o1qO17KvlPCnrwFDr8uVtvmj94FC0jK4+DtGBhiVGcBQrltvsJh3tSxquW0ngALWMVBcSOqypzO0cI7RlzZRj1hGzCOXjYwGHylSWCKfk4cdaUbzQ8BeAJv6eld848D/NHDTe5f5q89vLFwTNT8lygOm9EEBmYk9LyxLhVrppFH5OaIRfena/2GWWr/ExewmUzVy7JiVI4UO3UNPExzE50cqon2MPacttJ8OstkbJ0h3XoY6vyoe6VntHYPLxThAs22JUjXVkoX72l1YVSNJQTPvx4bJe/HlZAn0GajJcILcRpJ2qM9WYp4AP66+jIKa/MXx5t1bwz2wQ7f4yJpp9G/mGmC2xCUfUk7NUsopkHH0rHRPyThDAp9oR5Af4gPjIa/y7VoXLg71WyHWaPi4Mrw6t8SonVxRu2qCokM5GowErb3rRqyrRZMye1U5Pkdi6JEiMMbL1smKEQeI/cGMrB2gkZMaDxRUq/didebm1ReOqwR2P1rbV74CJXejrshIbDdFaVDMDSYzX5kWohrfSxC7O+XtDXQfEs6O5k0NBkSDDoMWRisHtc84BARn9Coa+mq1Y6f5SUPWGndWonlVN0gQYas3YksaMqCmEry660NpMv69r+ms0EBde8OXqxX/CM2O7za4r3x2LpgV3dxmOXsALh8TDJJ4ef8C5sf4dH2Y0yy/692i+YB+DXPWqPjTbtGuJ1pqoCSDXcFHbGTJQIMAHGqTYmqiTscp8oOPz8oBZyrG/NuH9J3WwcTP0qRnG1Tt7/zxPdqOJJZXIYoBa2dzsI45jRjoWVHIOsNvvXtaYtbsP3HPo9YcIJjXQf1DSMOIEHY1zPT9TwFoooAyhtW2Ete0Duuz3BNqnr740XfklPfMEkMfg9XF/0Mez0QP/i44Ns5Mxp6Ss5f4M6fOxbEwNo0Z9hTBneXwRAPetUMPLB9HUrzCodWL3AnmNRgZ5b4DxIdcyKI7qTGSQhefTJ221Uh909fk6gWtXLbpy9lcoXieoY2cG1DqjZhF1A4wrOG4VeDqGrKY+ZrVtryUsfjVcgAngmj38l3BUb/tZac7eB6Nh0zX0w2lwOhqraszWuNCo+hOHXbvJgy2bxIcqpzWqGMYKo8SMjClwcxmQ4MR0eZ7r20YceNP9sNrOXiiVRZ1CL9tcz1TEV79nFFf2Re1ClNf+iHSUtFFsMNMM+TCCrqLLE/31GFeYn7yGIWJ/M+XKhDEM+qWRODbjseRljpolrvelxz2VqdA5NuZvPgIBvTQXFavyA6qa+eN7ik+SPJs3AhvFPJ4Ot9mBqyTR1ekLZP+1mv79DIP9YXnuLzkz0+AEwAec5uH2+cv8krtzVji0srJyanQho2HWvMcMcAR6ILKAlphMrks3znZ2sk7hK3H01iCf79+sIgD12n0IcpJuFxNV4MlIGv3s1S17US7UCJEBBcIb25HatfzUi6JOPtZfN0kpslVEyVJaCXxDmpMlpmPPfr3Zd4GvyupaP1kHNmJrT3Y5PI5wqSJrLPMeKPeVOYbqUHTzbvxLxiangdfgZRrNpizkXiwx5kagGs98z6OVXMGd562mq8QaJN19BefhZhTuM6xsI5+9JwNOIBKfgwSXWtoPjVOkGgtPHg5UR099R31q6fUGdxdtEGYJBUQKzD+HxkOKUHCQ+Z1litppM8XazGlvkGL8uPXBTQH/F7f2rd27ltAeNPrnOb0Jv3TEL7uw+etdRAT1J8HfV/IfTI/d0MGGNESOdrV1wbbB+R5I4Tl+LYwO2e0v0GElnD1BnwEcjf1iBP58LkMagGlf/ZV38vxdZLmQImyq6lf3GNPLExlUQT8sXOH4DTkS9BoI4v+yn5xTbCgQSFyHo9gZEQN6m5kQEwLr3gi+aQXN1aCeYExrP4TSyDdDiL9CPUqnFksVaTIoaPAW27QkFXsfbNK+x4pQ0FlqpYrKLkVFy6rT7Yu/MvUuVfl+N91MAK+RuRVHKJ0vXq1/p7pVstLAHRldGxC6sQ8zABAvL4AVK4SBvh9GO6Jk19XAKozfrXUnqrGwto/OgSXmvSVVjlROfBOfvxvJP7bAeezwMjPFiEiiiBqtE46PR03kHO/v+dBjFP3Y8kfeBkTI6vdXWH8u9vvIarpopZyDem4laBDL4RbsZcpf/lx2ZJBoi6UTNlXNjnpyH1sT7fr9KLE7Sb4EUyL55FWphtfO/Uit/koMjtgkP1e7nitXheSV35vQyily/7AG/jVLhtUGfHvxeuivb/4bVSrqH2JjNu1rKelMZgH5+dVFZoB8RQFkc0ccOXe4KyRspqOdxgCje2Mo4yj1zy5C+vjcI0Odx3j5muDOu5QpPwGioseiOO/k6fHh2ulXkYbdPsX4tAOxPGnhsJBGiTDTz0bEEQYQeykT+NzfuBZz8LZZCJxupQWiI8VLSK9z9suf2rAC+WpwI3k8MAGCKaLZEE6uH4Ac1t0/k4jD2uisD/orAqX0SPK3wttajKiHBS/UnK/2WH4cL2kHqNE2D3HCNaEp1CMo4GY4lrvjg5YbCBMa1vWJYwOJYIpKQujAFYIu/gx+l+zlZxBBlyQa1FSuSBHTHa7tjcdYkAupxi/rmSlVsPL014qTDnVOAYTw31djB0mwMKlG77n5K2uLq3FUpw9LgPg8qVYkxW4lEZ2kpAEGiCfROO4RXF5dCIDKMsMDUIYP9XlpXYBlRluITZb8CqhIhL4C00R4XRBcDUXVe/XzC4ndjltBzcuS2Jy7XtYj3pq16o51rQZ7MUWNhUWr/DgLyk/oN1d8PCPZwRuvEAQQb1MXxK/fXpVIZtdu05/aapjwMJo1bF+vn5wAJfQVDdzc5USr7IgaAuS1rHy5XHShGv8ZlQ5WsBH/wT/t7zJyzMuiw4h/6Imp9LU3nqqEu+Ax+X1Aqi6l00lwaujbBSQC6xR+BSPNVGYbwRn/AfY2FaNbpdC/ycv8Jiq58NeFFibmqEgVSEUm/KICo4IBghlFeEJhJtFHLG/oQ8MalqNCWweotm6/fLeZWYuChidoCB0RfoxpxcQiXmLI71yXsybzG7iLFnD+nqzQP/GoU17XJ5TdRxuP+cOosD20gLS0VgYY1TcrF34NWXOuogt6fQFi4BlgS5gVcmNDH5xARhLUmksQT3N+cTtFHY87HcR3VYH/nFDdVythDdsKzvClAbWSLQNvf2BFXoZExj1YxnfKdYcCLSKsabb3MAnVfMI8u6+6UhJ5+siO3/Ihhmlh4J1BId1Zks+xINk8boHDfPJy0BjgVhGmy7OtCSocsE3JH/4uOqDrbv+GSJKCcqJX7V5OB6RL4OWCqk+DugsvvhWt6ciK15hL+D1rS/4urGmqmV2oa22CCMH1OmwyjDm4ueEHqynAFDByotT1QkxOi4U8Bi082pdrlSoOrYfx85buC8jOMq9mx69LLpFHGwx21Bco5ARBwa+48xhDe+z8N2y9Jpf/+ze1O0Ljic56VxTCl5PPB6THvmtOnTDPyq/EveYQ477uu27dJ1Ib6Mvj98I6lCdOuBM2ksG8ArbC0/4leWX981Yqy12xn/k4pXKSeaDw9da/tjRa6PJ25qr4YWTotH8NIu8vEwYxAseYuVp5wCOJHd4Ur0ujviD+8UeNHYZRGG/D2RfYV+E0wWPnPUcz1dX0NnoHM+5b6eyuOzNfI6j2Z+CyQdj2BVqq3U5Yu7ygZgGj713vINE7p6cHIRKFbsPJoz5mylMf6q9o7lKZ1yz6v2Q+xJhij7vZZ1SOA0uF9QgHNWf7v9IdnK4gn6gqBNM+8C2uskxUakWj+s+hObbeXennWAq0fxsjkeTX0dxiebyGyKO444PwdSr9DnLb4tBzdD94juuZtrz5/W3OX1pm5uQJU2/MAFs4fOTl53fQrJyB96g15HL4OF+aSd3QdibmAC859F2CYmXoT+148QNDMvYDF8kiI1sdouXHiTWV+3xP8rr8nj+p957I1TZ1tfh+lfNT3CyfuG1WN9/BdFrgM2Ln2PfVM34MSui0UHuncef4ujxJSu7TXDCHhQM6zuy+ynl1dZHSr2Nen+A1awsa6F6IKYHlcl+ByUxO0Jlg8Ce1mwgbl28QQX2IIzNcGzBXg/oVKrX+Abatxy2RhAmYi88rOsCJIa6LAGnwPy6nFRmi6BYzv6vde4+Swr+9zzcHTumWPjnRcEtwnBSopOhIs/2gKGwSNg8ouTtBVuzu9LrXHaL518hmN1K5BQ9H3Ci2hkn2Nb6SYJcS/3JdMFIzGP/qzsGZYGFyGmUB3ag8DEkyWB5P14fI9FC1jHSRrDWFw5aQyUR/SEBgiV7ar6EF+NVhAig8Gg/PCKUPlHVO4Bjfas/Kq3d7zWWXifh0k+I6/c0x1rx9vlTuDA/hLGzUgVOtnfOTOq3wCsjf+QHxnrjz5nZ+grvaCFybOobPiN/hpyXhOkoB+3ZSTbUlVguN/loXNhCo8T45KIJPzffYO5QoBDwc798KFnP6w/g7vc6CaYnWPCqvTWaidsP9qzLgfoNn6P3P3xxF/UzrpJ3u2v8p9z1fVBDAyzKi+n3euntewscI2p47BePvkgyy6N+l9KkL7bgcr5P16mJNQM/dn90fDcJVtNTRaiTQMsOGrCBiX9G1MYhsyqRzo6dGwUQzRoQHsPW4OlKiadvss5bIsTgf3u+UDyTXxUi5IHo+RroA47L/2oGCmxHSycr4q0G4rCUuaGwLZsIbNOY1pLHW7aP3f3HIbn5GD4sjBEF66eYE5m/KrO+5VrBFH8oCIQyJngKbrdpy7uVHgNYmCGu+8TqLqaON6bTfuNZNgH/KtmPDBB36zv5aRQ6NdKla+JGb3RiBjz13j5n4Rqd4LMwONIliV04U9nZCvpDn8BUYeX3JNgtQiByFbRVRuO65IBYxFdSasWPehM6ceShIecZr/8xibb5iRvjCwwMRWG8ZBF5XXNuAqB+IeK0nnC+EyfGai1Lgh3L6sAlWhrhy73l3TH4JrahJw3/DFOIXrPsTUkRoJtjL+7t5tj7CbMWHvxNQff3Jl92FbXKEMxNWFwnHSfSvFd3XnPbchA3qC4QZBWrZK0MPz3xNTnnX5WKbHj+CaZKV56y1K/MHeeSv+okfWABzixiKOVrziM5JdrT5DwjdIAntgBOFn2pK8NEESvGa6EALHSi0SHlxK/VPEURNAm332XfEDQ1/XWbmoIx0T3cTas+TuAjW3kXemWqeErEYxosWnnaF4walk0VN4XKP4uA/6N2Dc8DUwI1Ik8LZO+l+bhXNMpqDzC/jvXRULNKsHJdgcFDAcUIe+1OU9wg4arenzA/7Jbbo0tK33VpJuFfuAStBdbeSbbodxD+9TfFXXsYK3lgA3JAJf0b9WUo4GsRg6ZNubDWwHh3j9y9rDfFDUBXvM/2PpetadhSJob8EJj+Sk8lgwhs552i+fmjfqdrZ2rp7x+ButXTOkVrqAaDraWmWJsTJDjJW/c8eVxfyscPpdhT8CbzuFXe9bPifonwvLxtPvemX7R15Vh+lO8SdnZQR1aPV+mVSw0e/L+EJ50TzulwiGhwWZp0B4rWe+Maar9D9BqzfoaAm9TIJvczQ/8D8QoChDIKVcr26ZFyHxYzHydRgW69mpC0lbgMWV2gjbnjVUKJjQkwTSwQO+WqQv+s3dNxz+Z0QG7JRr66ivnRXcM6rpt+mGu3wnjdUu3LG6fst6zMeb2G6PURRHk8VaCfK/ZAqOlyLhCJwbyuPsZrj/qWJxp6VwkQt3Vr96FXYMM/4fQdFjjUhW7xGdPQ5QrGVok0qGnnJEzu8syK/az69JxOjWebX2fSt7SATxweGkZO2JJK+9uFpegPhOz9vrA/xWRxATYCtwfLXldQwmkaGt+MPh946pOf2jaW01ZYtk9oZ7PlQgmVO37+JAZXa8EpxzQ0WKIJVSGrIhaLcJn2vm+i+YqxnNKAjGdhiwlFKhEae2c777TWKoIBk9PtDkpwhEvonmgx7eK/iJkD5sxL+uqXexwCZOEWvZzP2/fdrEamW7RahNuH1puj2CMK1/xYP8Cm2pXZOKAuJdPw4w+he0Dp/loKSaJo1LKr3aYslLcmGfgNQzlpWGQ2swKd2Mpva9kLlAWqNhi1Nr8dZv9uIq+7ws+ClU42EojzgvO16eKUXaCk/VdaPb20Wz1f0+HDHyQKMmvC4+0tHpOTEoS+zeQ0qpb0TV6F5iePuflGv0Q21E1Y+PAmRjw3v6gsOKZG2X3Z4uZj7xEsVzFthOo+NlJf5Io4A0wwK5ldNq/WAoicJp6vQcfiSH6n9G9JfdPuylvRFafqjLZ9pxatUNopnF6G6GsZBPr30OKO4CLsOT8L4lk9wjG7MXWcaZzoUc6HUeKuNXA/ZTn7L3h3GM+EjdFTDfvClA0ikoyU0SlNLgFQqvUu0jDhPPq0OghQLeApwGAW6ujHkyLhW7Hx1AvGXdt5aVnyCrACKxJjndA3kGERn/k45bRI1iQ1n9uZrR9LgHJeUr6djI0AouoXOho5bW+3Rloa+bzc7y5HdblorSffLMrWu+vXv0vwfBRCkKGbctnzck4TaTRt7jliEmeD5ntjPk+q/xF+f6SeMDyo7znEIdXdI/DgjDIubUwAUDrP1dRgB2MMG0zy9cRkOZiCt+q7cusCZkMOWNFZ9RT3xR47XQhHktI46lam46Kjbw5uWUpngPWy1tB8YRxjtoCm6Oi+b8Ca/+JE0ljly4vfUxhusZQ2/HGNAj988+vSngHthY4XpUlOS88F7ZxPJzcjHLoGNYL98uQ0zbhRoR3WV0n99mFzM692GNw8rvo8RKfdJ4ZAUnFwtXtPLI23B27xcyfn2OTvzV4NVC5hpX6ZVyVXiRUVaZbuJeEVQZn0jHK1/xbDQIt7Izat0030X59vFL6rrM+z+oLCD98MixkEBnHgTlaWgEQ3KhlFgaIcnO6nkvC2eVgTCeOn0/dHzL8R8pVLrtVvhH9B2fCiMi9NKdXTr5ZgODAaJSB/B9yzqJpRRr6CQ6LbWhDOWhdKUabffCHQ+M9vgTwqJDNfvwXsiouWBHLLwxA3oTjMbUtuwVm5oeU7wDqrXEBh2XXYg1rzF0oXTXuLWlObJxhMW3+evmUeb92NLO3I7joQ0HPPD/3H5Bm4clplOiJZkb3PZTwT7Pb7H2m1wDVXpNVBmkASb3yec7gnwH86CIBwPW32HvYlbH+6Q9RF/EuMPKpo+Kb6lnmleanboMr5SWX1KOvmqAgN7c8U3xU7bDLRVz2EfQk7mfuFvZVmP7HfFXeyOJh1pPC3A15Yc7ysMKWBVrhFKTSKVjRM3MZ8u4fb4Om3tnK/gmPao83xL5kDdAzCLApcGQXkGY28y3DmyGnVXf/vjdBwIPBUGDDTCIGRsXd06Yf1N54w8UxyO4LpqNBYytVmgD68gG2e76CW/FpwF9f6Mc/m25k36aitNKMKRVKoPY4rTd7vAWsvQfjwupSyFxUs74puPVnXvB9GGfcfG9JBRUW5A6VpME2OsBC9rykHWaYpGCHyvydRoSYKMR/VqC1SGopsS/ER5jto2YYLG0JZ+K7PB9A3xZhtFwU/elrHozXlumH9HrauA5xIspuN/k58u7CToa3Pdlnrg4PkqAnMpgyl8exHuWlEo7NfZ+h4PHdrpf2h2BiuG+HuZrLUzJGVLy6YpnjPE06zN0PqB6ynwqs3x/nNHNPYgrTL1k9Gets96KWLWZXaYLZ85h4C3PNBGfxd0OCCCPXzGyKMMk6kq+stj5pkKW+Tbbbj63tbefb3Dke4e9NA95Kaa1XSy1q/eeJLAt9qZky/R1McGCNMab6U01k3ZVIMdF/zt4YbP/od3CLebTKXH+zY+9nHJ200pyRJABL2YYUpsUzAQyMRPk9+ZbiG5NRui6JiOKlnXSDnlQxGjcF/oyHRsHP7yQP4zLm9/rr8OsC4mkijxPiQaN12j/vV4j7qEoxImaYwHNJGl0ALoSYfTZO6cQOrwUUk8TKELyywlTNM0ei9j1zxxj4pmC6ZeIK3VURRDAOivj/VL50scAyL464hmJxC/hzpz+jejIXmNo+3dhYnvrv76SbbFZtC7hqjxtF7X982Zn/ny7OSLSHwLM3mvIK0Tlkyvu63Kzhklexjuu6h7f5AQseek7bVSO0MNz78rPspdGkQ1tQSvqUURCjPdtR+7ZasXiLFsAcakESaV/KAAzy5Q3wIhHFTPXyrKW/YafRL1nWhwNGkkLZSm5YM8j63MFHUXm9bW7T39ruS/kkR22s0+qvjDby400EB+MLJ9d8RmKeQwkOKy4Z1rK/upPds2Fa3tyn8RS0dGE7Hsqgsecq60wVaGdKvaK6nsBAOxLt4gJ2nmNnCg1yFng1OnMiIX8/3inl0Li2fF9DnB9xkiqQuW/f0JYqHq6eWG9fbQq91UQz4csHNgXCTnZhjuKMQw67Z+PeBl/aL8+hVVFAQxlUu5Pyw0qYKmxpOhA1GX+0ahjyxSKnZLJXbwupigyjJaBsb03hhTIvzIBeQ34tBLD8H10azQ2wUCkZIoLDHd9Vro3d40mn3f0PBrFz+ml4I74wIlp+AhPQTRVQGkTI+JSxlmaRX8ztRtYRAPa10mjNtN+lse6kxV3fd6goZCyA/lDJLVbc1Wp15jGxExA1koo+ehHzboGz8xnIp9bBN5i63b56H0/lXfyjcXUZWIm2hYTaEhj84zRPHzfu2D6OwFE3xeiVgyZ1nzHLmHZKfIpay598xdN6f7fwomnaAeXdegeG5XJ5S49/05vK8uPR6uZleSGJvGQTA0SohRO23j5HpItn1+7f6z8xcT7285J2CCCqPV76HSkRt5sWPE2z9Ytow2rmlOK7UKq5CiRuwsyl+/OhI4wEKIr2RBkT477GtQkINqVrzEDuLaEp6uT+ThH4iqDX0/1FMijwp2CjVia0D1msTUNqDWYja8ttmv8Kti2W15+bTDrwzzPhvQe5+h7bEiqjoZYxjt6nic71rceDoxdEkcuyd4RMaxZVLfKC0a7tqluLh0/3Tjr0mM4oV4DCp9U8MmLT7/edsCxwT0V4+ZkJJ852Su+W1xgcmXfSqbtN9+JYn1r/3xLga2wv6sx/ifBmYQL5hDcCDc0sjQ2V0+wu9hkRSYbUGvql/tB/t4cVB3QZgSHi85Gg8qx7W9fHvQ/JLZ/SNOuGdsiM+UmqZKntEIYmAia74mH/ZOHyTRSXzH0g3ulo5h9flELRQZRTsHNtq0FfGlyYU+NP4brXd1XcHMF+Ydz9Ygl0nyanDmTDnCuC67KzoLAfcUit9gCXxtTjZ60I4YKee89G/xwcViwL3247fWnhW5gv5RR/9sRLsn+MXeZjnQWjmVF/l1ptIdq+VuCeCXhYPvFjGMGhVJIKkCyQDwHvTtf+tgMnyiFz9xoavveq7ziBtmxarevEvKUCtPRl6SFPOgDPtys4gRuOG9L2ICkFe6WScWIc3ZlQrHsG+d4Ibtz94/cCsVpHsXb6k+4hVTQ3vmdaAPRiMIE9pOjYgTut5Dx7f+Y2eD7Lw/9v1JYmuKLdGWXrlHvrKPgWt6dMWMFlxJGz0gOrF2/xNpv46jCuNg+2Xqv1PcVvzrZAWYndlqHGm6E3hrnNAm+nw1ekg+gq551q+xvKhPhP3taxhb09A8Xtbx0L70teO1qiIyCsaUnP6beT0ebwzTvMcmpta+kr5MruJVsC9PRJU7mHq7DKb+dPIL372PmgymaVE12ck8SFx1VTao4mfk0IadYYj8zXwEl/uoA+jHQrQTVeVUKgsWbK6B3sWh1rv81Wm2TltP8ASKjs8mNwAWZ/D9ZXFWWIrC14WVUTN886ZPxnbWAMeHHAVDbnvyOxoS6ejAc1DCWTHAnGS8IN9ag70NDv3wVZ6lgmbG+Net6RInT3YQ210fsc5vLuu7xKnH0l2gN/YwIbugUPRl8Pvm/+W3ms/3gql7bUY0o/h+YL9pvD/+QS2SUsOzTrO3M+fTjlx0t8M/S55prJoEfoHHcEP+etAo89k2XVVNmYAvfrc7rtO2lehPuhutGGUjnh1ZtgHMZWFeiv3OTpX782URao/12MrVqO/rK1mxgr/O6wU0nhHUn+SfZVu68Mu+2ukzjehX2yCZNT/uEGxLNGYQhg0YSaNl/LIm/iyJuFwVMxKjfpND8avc38jJ+NFd9Tfe+80+tIY3dlS9nw0w+arUCuEUVTASi9GWY1qFbSScB59lIUlYLQ/nd0HvjM3vkvn2kgH9bKXP7qxxJKsiSWC/DE2orzN8bbfJW0bOWeAGOfp5pX9Hp6mB1PZNHBS1ohdNPyj7+SOrl6/h88J2QuCjVy3M/fB6XNh4BT0mDB4QEtj4QxwSpj2xui4QGiSnLL0RmVdWZqnr8dG3eVibH0nNO7peoVRa+/9b7VaZX2Hme1gFWohundSxVKXxZxCbnEyQSkE8IBbQa6yrfhcnZOBYE5XuPapjxDSXQyJPiKZwtlxsgi9uWKu7FBD0a4oakFYkea8gCQRNrz/TwgilUbJ0bfGzZ+t/QGKZMuUDonrAaGZnJVpUZdL0sgKUMZ8z6DjSzXsezBzlY3K4Snc7XkFdOLPWUV6lCsHGyrb+6y2+FK8UB7ctyRzHP2t9RLiFx6dIvyxLFxxEIkMvon0nKXW3/LElrZQ6ktYa2I3cBx6ug5Iy9Fs+teSDLWooi1rRA1j6hJNlxaorldIIcqni4E60oJLP+qFFe8VhapPygK7odEOx7U0gBARmczOGHNMly72SZ2tOrn254U4rT6jrzrH6jQkh8KyU8tsb8t50kFnmD8A1lRj2fKVWRJwoyuRFqN2Da8v8jXhvWEF6Juatu5mu72tvDvpD0uNLu0Wmjm1+S0wYRSEUt1G/76yIpktHWE3uE8EQfVZa8qI/9yypSYREl8bcnm30d79yZlrzkeUAQV8VdB7KTnDnE/Nk6koC6D1ntN5AL0veEll2P6+xt5NN/5ib0I9z/fKu1znZJqLxEVTpyS1rE1vdYaPwqeDLH8zL4WaBVXT+xuG9gtvJF4MGBvSW35fezIysrwsTfENcGdu7RJF3IWrtt7s+bdvHot4DujBneeidDx2EPGYIXgRsWz5mi4HGTkrrEqYXcI52XwrdnPkAjlVqqg2kB0LFkNjq/hn7eFY3OpY63MkfhVK+0Ulh7gdSABSfprxirp1QjGkSnpgMKuJcVvteDXRiwBnPQ+SU3bzhthpNrc9xMIFNIvE68hqXYsllN2b/QxdW+9gH4eSloaoB7QWvMkQd545r9oF5pRcWH9BmUMgnKVS+WdmfEOh51+taYH2VOR43YATo9gr/3jqPe3eIIo7rYNvp4xXWu8X7Tqd/YezxFvVvWoXZkOpjYhBMHRlbr1Tn2csXbi2xKhnEt6rMfoOf/zzXedEjl1l7cVOc3rlUM9ZmipDMfeN6Lkz4lQf+zeowIstV1rT3G/JY23nhYgmNEzdtAvqgEf2VmUy4kxee9DOLpt7ijRszHK9NSeud9rx4APAjv7WHMQk8QPnp4V3Ru8dtt/tbJURi3lE1BHaVRAW5vleKsUADZCaBk+903SRsSRbMRTK53M0LaDzfRb4efHRaDmbo8rI9OELikjSfVePedRADCOaVW8NRqBKil2L22kcjLYZZdXPh1TKfev41vMJx1hnOMHwlRObDGRsw5Bux3wdFgunVgv24bMD2Gk2FBcSlvq5z+tXwqyUj6oWqSEQ/aXBVTfx+3oZdavTYszQETg+T/cK8Kt8Gm5ZikXAip5oBihZZskpvosF5/0WkdajWb3ZRa3yE8HESnlUsaJpnHS86kKsE0HhDm1hEtpGSWYv00vvrQeV+WatXyQ5lKDhWkpXdZyFb8RVxESkrk/US1idPNoYjxH7Ia17wUbjWP/rTk0ljDBHSNyxXfY6Z6WNqZnmEIb8BT2EOVYf99RXaYbKCaCtArydEa7SOfSbdxkNfkdniy0Dv9iM3suop5tCtrSYwbfX6rDP2HQoPdVgNHi6quUP/+MDNJoYtFfcNEnd9PReI7MVypZ1Wzso7RkERJpQWtPUnDWjFFSejLy1/AaFWnCdujL+xN9RO+i0m4cX1jhiWGO0vfUsk9WdCClmyZ3TGyFc0nrDhNRw05SLFAfEsR2iuDIxVU9BtEr9DIHFO1HKBVcNgnKFAzixIej20ztE2OAyRw0B9RFrud1XpD9KoIOWhWN4ike65b0XSwPS25cYcsVidkLDiy7rEQPK9Zsk+z/z3HNqk3t5Wo50hB06uxprlblD4ackiZypuIJTNC74si9cHhRN/vbnvpj3cxobb73k2oNxghv4y+eCfTnFMzAHbUxgXhf36Qr1K3QkmrEy/jxOjLD+q9XZUtEkfH7fgdInPY2kHs/LyZWQxyoKmysznjzSMxXJTdFQj64eJUdGor55/NgrOtJD4JHnk+JlZcqAYjjDKYvobAevDLNysrj7uCivtXwjWy49E7Zk4+uAQH9JZszQdMgGSLEg/kdMETf9b6AnheuGcs76piQM5ZVO5UFRbZb+/Sxjiz+80aCw/AJn+QrIAfGv7tlA6UM711aQszdJ4gD6I8k99W6eiqIzv15tGLcXo2lbAQ2IKoywYJRIIaWVNn+OhpwJVzkONhvFTAhG2L36jjUf0VVRpsVXrEwzoUng57ywQHr6w9pwO4UKcApMKKeYSlT6dMJC7+w07KIeVTQiLfKWwkvRQUneI/8Bk5mRvpUK48W4Kc6X1ULxGUF0Sl1MiZZvme2TXaO6kfruKP35U07Zy5fPSK96KcirtJkrx49YDUxp69e7q5rYIIBRlPrSSZCGLU8RU5xB1d0V2v24cMQn31aedKOF2BBGW37EGUFP64fUmN6GJd6dlBBV5bgPVNvkqcjXd1Gn6asvbCdKW1PyG6h9yhWQPWsjYkKrXBkZu6EUI9cm8uZjDzDdKmg/9KD2wj7PK1jJtHvvFSuYxwl86avIDVKE7K33KtqWZdQY9u09Qh0ezoAtC8jYKBroXyWKsFSNH9TAp5SI7aSB/7EYHN0VgxmlHzTwK8jOr42yzpeUvbNTc7geO+NCPbL8ThPdvsKNul7/LlRdrmgiUH8iNikVD0c5o4iI4MlatCJacIR/Zgne4uIPpcylym36kJw6oH3Szqn3OAYTFafD7RnZVt4CAiThMy1hmonJ5gMHCF+a9UDUiv6dQ3w8qxdVcRVoIuZW2K2+O7wqb7IgQSpvA1oDcK1SHe2YRDsLR+/y+Qmv9fOJ6AtwlEFfR6rplqJcCaWnZehAazjxYVL7f88XwSlZybFnbte/IdILoQZVYGc3h1VxYL06gS49bjX4ov+L4gtpq8D3ed0NT9gYLc+XjB0md1bCWEI7cti1daItQN5uolcCwndikLUUXIo/JN7HPmWEOuQtpr+elnphpqqIHycwIqinF7T9kyV+Jcern1A61YOx3AuIwo9L1eTfdhSrAUXLkgdKsfR+CdafUtXdXdhS7ueV68G7QmJgkX6/Zm0u4sEJBhWwdiNrwbhI4FJe4skwiHrjFdffjWZribbMhUMuDUvxz5kyAnThizdKJhaXkTLYBXg1HPh9V9TAQEYOtadM4vaBsTbGCuPDaB5yFef8OR9RaqGidpWhtNhEmcdgq0VxRwuMeBcUH4ztLVimSGu6HCiH8kOOs9AXy50rZkO3IM/S3ITbMr/amkrKIWFJ8YiEoQL/U8O5MddBF8rTliExgWNlN4fAz7J05DcdDv24UvrnMU45BYR3VYx3OUeV8nRXOY16Dvk304LuBxXLcvumHm3zP5mu19cLGruO6rq1ZciC6RkBYYrTHWa6v8ZfqseW6cMgBDvAr2v1Rvz62VZr3pivZ7TDotEowsktpINyrdEockUREms1tFtf7K4Yxj2kpI8iH4EMb5msMVnGFiv2SwboXDcn7VozcezMasWY/pgXCS16DlW6yIy4ZK1cjJag+Zrr4D7oHCuzdtwC5pm/i1No04nUy0860gD6C2RCvNsvLkaOXMHNNc06YWOYSLNUP6e6ZPzMrkvCyqZ681QXk1yCY+T/bMj6OcyW2yN+daC56SdUYcNLj4pbeGMWC9n2y1chfny5HYbGXDgda0ouX8mHxy3MPmG+TufhVgNqDc4fr92WxjGzJ+wYEmY+0PWTL+g6qM6i2W51vpoa86h1KURESp82JV59Y0L2rzY7ameSZI54NjR5Jmsl6si+WoSvSJTPIHzn9fDwPw9gaitYJ9k6izsMJH+47E+sgT19VYoTLrb262f6gL1nNoBoAmVVk7c5wIWmGfHcOwmV2qH4O7gNEAJ/czDUdoltg8RdFmj3Hel2jK2wdpd7roQ40EXa0yjutpS/hD6HjcUQSCmkwROQLEb1DUCoKNP1JYLIZA0yC9NwwzhN5h+AchaZ7fwnNWwzA+xk9DCYD4AmJxylcRG/mVEbQ4on1WJpR0eHvEtyAuFgZ0tiXUHDEYy1DFvr3s9hm8Ru4UxraTORvqA+HIiiz+E9/FXyKPL8vq+zfrM6EtzzJLQocrzmkolkccjjZQvN/qmAI5JvBU8C4QpSEFrHjQ9dJyLdemLIo/QaAal6nb/uIGTOa5q+PeBECcogVhXlkas3pGmXv+c1DBc6XIJnM858R/vjCbvo+ktXEfqlCFY5y+/2qaEpU9jvfDiBf69ohQT8YAWY+NM/3MsIIoPz8GkwzI7WGzINUPb9wJ4x6/v0Jj7fX5WFlZg+Kl2Hocxons2a9wziKd+COrSDUD6bN4/gAKW8MxJG36GvQxvDSbwd8vMBP3p1hSA57VRNz4jQ94eI8b0x/uBiq+sMk8ofVWRBVrcuvfjaMuG0Xo0jy7q5BINYB3ArqTzVWkgBeG6rqkIcPLw04wnZMuq9XuW7Hx1YhZYJoSLQrZQ6lpPNRzd1XJGv5Pt0pB2zK73LxY/fpRaoLgmpKtsATXCVkR58s+J95wXG5n0g0myYYXy2/iX99el8orzGS+RoaUiMfwAzCnrdXvBpioIDzN1QCLG41FdubLyyBb0a3JHsa/iloplmDoEJK55dxdIYPY+vX8sNRn+8P8kNpCUAT+cSzMyWvOKhvtigc5Tfr3jhuiK0pf/118ggFgUn6tmCN9Pg8rMdY12OZ7iXvMgcfd7RTtffBfoEoJjMGgiRyAl/UZgmCzVUIjtkzAmujPTz2Vt09NFUoSa0WNnBIYHn2uxNEVG4Z++SZKyiTCalMs2xne9+ew3QNuPAlYH7zdp881LG/19mBI32NOKFtJatWYwPNp4TUyxqoiNv3NffVBnH1N3mzjRp0n6w0IsIdtNwkX4/1vOHdEWg1BO0u32ZqUlwcADT6ju2c1qCVO7pMMfBxMzs4Dt5Rv0MzMYW0n71FKeu2Xidf7a/htz1WHRki6bozr37LwY8SHOhdLnbNbnZvb+gozcQcC13j4r/0NpNNTbAWZkDLzKoVFDlL8U2j7H0Hb5RRoQQjKQZtyrTIy7dj9KTIFwvQwS1JZqzH32lFfiMvbHpw0HwttB5c839GgNe/0v5fWJdZrVHCOQcsAs7P32D3IEiz840gEhsMgjXyMPlqNGXWHxY8nXjx9yFzUWltsF4fjkzCYha4B7DFTaXFGbTMsAm5cMgY8CIzbH2tPvcbwX2Sy2p66FKEDx8Bq1p342I++JWl0ZQHEMJ4VwDsS+sKQWePIt4HYTsHXR2cXL6eYMT2V3dm0r9PBQO9rIRU7ptRqpPfNS/6qgJIm+oiXX/jdpT4eaCEIeAiUwKcoiaLVlDFSpHjODlcs1SZ3qt4w9iX/awrZ+2Pa3wdx2cdf7Tv7ohuW2e2beCUNWjaTLQi5MvODyDoeDlfEDDMUsI428U6bcq/djUe558/LXYI3H2DPnhcxkHCbR93zrP4ZCMlv4+MYVFHt/S7LWvZct2MulvtAc1piv4Mxk9pxiCGk1qR/OxKqQhpRj57yaxN4ssRr4dv+x8DwK0RFnDMh5WigRQKp3XGkEYVPBmBYpe3DA7UxTcqCPJkXD5uZ9Xb1ftIJaKckv14elSFj4OBBcVtLZoUYN+l2nv+EvlpFbnBFajEhJm/v4RFs7OHIRabJ7rXYRvb4Vc/pcf5wEyvV6N6PUyjjhHkrb7t+vV2EK1iPOPNV1n0nPLpftimmEmRFjbAiICdD2R23W5k2Zqwa17ImM9XMyCV04HvzvqFbeBFS9xiwSnpglZ5zzBtJQIinviii+LO+DU3cFFpaw5x6l4ozH34quuPE5xU4qZv9KMUbcWmBnzsHVw0SQ4V9qtZnaG97S96fQQxdO/EejCel5vnkUQVQoW4i93EdSdvoDeq8tzu4EHIMlr9G0f7L1n8JZWF9i1FFvXsckEkRtw297qwPT4NnyHHqyOSx7GsDJ2W9LpmY9QdkK+VXh22ytbza9qiXT7+HFiTIh2MDZRVQGlLY1p7A3MMhfEGhTRIebprHNj5V2A+A4pXRTFvkRZl8Hj69BcFTt48Wm5KEV1ropJh30dW/g36Ym5nPx8gAl8qI7v9SgXnyW9v9wX46q+JbC04Xn9dDApFh/H6VLZVvfUHWxnj/V+Ebkcq6EGgA3uSvj3Fxil0NdHeO4zQqdH9gh+YoQSQVDpRM2FG+5tpMFzgiDe7OCZ6MktfS5OjlPD5NjomWTSXVeh90377O6MbgXHWoLJTeA17a6jow5VqZaP8DTO/MBv90qVzEWOHqaUkIwbntkjQ+ZjVR3yNVSWrU3GHWFeRvGkI/kN/IPaxoih8vVqV2oY0aB74bVzGLhbRShJvVw2GnUoQQEZyr/fAEr4oOZnW3fiSZ8jo8xuG3vilYO+/isvIzqF2lB1NqcABxyU8JrF6K8JxwaAxNBjWCstegzDy2fbT2ORqUfzzhj22FBiQiTaG8ejh45Wf9Klt77b3qIFhFF4Baqk20lxAzEVmno0ZvXqgWhHbf/WpiQXCRF26+aK3pxMEZdz37bK6E9ZKj6HnW6dwFKwQ2C/q9YtR9pJaqs7Zz5l17wd7SRRGviloOs0wIob7yyXhkofdQ9FerQJKGyCIgC/UAw9mq0V7qOEBHCYp/cqyruAj3whjAb8y4QwocFPz2XQoffiY4XtwF7ekfhLIpJKbMJisaoUbjdhnfvdq+iz+FFMs+loO6RPGOypXdtwfS/01o5m6mgLU9Gx9dCcVZegrBGbcClC8e1et3X63Fw5WyHyJ6CjtJRGoQ/EEdbAKj8ocVKRP0n0AeMK5dNR54MLnR3fftGO9xajpmax+p+GyeYn9i+ogG1QqdyIkfCTOJYQQOJUJX6P0y6SxBpAU7rKphqHJUW5mfliKQhlbfDWU0omFU4M7Z5D1ayyW4Nv1WqdX4s7f9nFGZ4bBC1EnaeWFpAUYLccMmMhB199Z5IevoZtcfVWL+5Fr2trIAkmKDYnWmGbsdOfcd50ozQKkI2lh1Aoxm44jNUi7pA+Z/pnDb84mSmaP17YpjM2rs4/FOE2W69eBGmgJYSxNeEkUOn46tw162M68icgrkWzHWxgizhvbuLES8rz/3oyeHhSaBp89aqv3ipiXLIpr6xHvPirdpkA8w0E9wypY2lXqAKkkCuf3ZXRqTbKqqke85QnhzbUDl04QGVlcOEWSv+E/Xm2PYhR8oJYPlJuuivorNynGPWx9yV47bFeSH1rUBLI/KfeAo6x/gG/joVYIN2f5bo3CHFhBjKqGJiQ8jUSRyKG1AaZ4rojiTw/lSuRbdafk+xuwjL/TjH577fyQ73ixfy2JtPxBGT2P4DLLZWGs9xvx8KtsbK+tgSQFQoov7T0YAV4TIr0Fe6Z7r3Qu9Tga8UNYRgCj5GPl+fwqj9mwsNx/3JzdRfAhSEpYtdrvnHIUsZTzhgvQHwHDFNGjc3FoUtc6aZZV0SUTqcYmQ0HjSlV+J5HxFbUpjc/H7Sv99RucegB3UUQIZHxtPkfbS3+W4LUQ5PuV+D7OBaRWWSgoLmB4QDSEkkkyLgTDdQQ9WsPgzwoIXWWRt53RxcXHVK1u5TUihHnU1rl5+e+gRyhgmheUv85epVLIf3Yd7+6aj1eXDqsx1c876E4kCBLSJJBDvlTl2j8p71X+rT6UXntY5Dk2FXxMA1QpSnlFZ4i9b+1TIZnvsQ4Ji7XyDpG3SDLuBG9KPPnUp8P9EhvhDnrcvgoXKZWecDG6jvM6VJELBGb5PEGfXASnZsXIuEX5u7BLQDdAR34zI1XEXRDHypuhVcjV6HmABxBz/nr2DD6weMZWoq+GRWOwkdkYgbuapoTPXSKTJDqRciW8386HrmkP5IlmRv3swa1Q7i3cl+EyOMYCuYIE63v4SiqZ46+nUG9bABK8ujajEEl/iPT4fj6bAZdNhXfQ0hUt/GYl4z7BPZ724WYOhOCAYaWPd8v/zvCYkBlIdRb2zgYLtWMcieutbrTe59Xah6vnhYA6JpyDCZNMjnfGFonoCffQG1P4tLColQRa/uoUBhr/5HWGeADwlgoz/YTmFf9WGVCHjYfOEjWZG7MAj1YLiM9FmT7AhMhh7DsiIUiYWmSZIpXFvuGGC5EmpgmE+BEB+8WluV6BsMC4r8fJUFci5TEVfJcRYpZbUEHMMQ2dcg4YrtiKdMM1heB7AlmLzY8wYQMdWbLO8r2mm7c4VF+cFj2uXljTyEB8DMqO6NZg/fyF1fiTtMM7H4thuoU5TnbybSiNuUQPz/exD/CpzR6j/EfKWX7l319xOTecM0PTrrNfYpCQAtfbr3YH3wAOrldYgvQVOI4c9Aoe3mUuruXKPHzeYrK0QQQTyLuquDP4KORb82iGdibga3ViJogK8e3v5zsJEEYhJ/gxK3Sxdwd0q3HNVwA/4fRurfGFWF/1fXAnKB4TDiZHvCZDkvH6ZsPsGTku2h+MJA3hp98lyHBLQWMaMMai6SSGZdUw+z0cyBi/Pq0xqUE0YqeonVGnuB2IrUv/Js2N88LI4B+kOuLjmEOib4rxOtzPg1yHxosrBYzLMbz/RdJML3FIvKrirQyzPV+Wg2dDnhcAm01qPqCaM306f2rDLWFeyWmUheiECjIX0MdS7RY/ZFSwfi0M23hIkxhYMjZnJsLXszkGZ8jKlsBUiXWK1mJJG0g9rPiLsSaFbBicnSwtpiUP0SIq2Sa/EO9BMex1Uq+asnNqqrYvjcjYB8KJD1GXgLeB3ryUO3pY7ByqJtrWmi4e5RYvDqPbb/0gsf03SAwcIA/11XO6T4z+Ht/aJ44Anb8orBlyyxqPK3GWZP92LVWmNZGPYX/CnSVXmpMzC7ZjUxcaJGE0/nsicWgOTsHoAUAY2/IJmK8jB0DEOApF1+iCRLkYHXJWs1sOCLe7i99LngdASULWl4yZ0zBY1BlHg88plfH62wHwq2HB4rIwZlAYuZ3JXQ3xFXVNauS7FTqXOOuACmQcxoCYR11/1sogAzJyXIXPLlRam443sK4/7BQgjwa7ee7v4zeLYGfTuGPsNqJGarL4RbAbQIm40/qqlj7meaKUfutC3AO9x4FgYBsSQf1uLCxwDuxxWZyDUw4E2iR28N08IEdcIbxMHIFzWTuGEG77BrVnVNYWEGpzC0O7hAOfMAumhQo5OqPdVtQYj2Hdl5Nt6efz9BG3p7w9tAHizd/90q/ya1zbcbVMGCfUXe4gfyGYs8eQe336yVFFhY31OOwUmj62bLkzGtoCA7TKYvJ8DsE9ngJuWFcbxApoSTDn82ciTJI+5cX2aRJ+yyD8hB46sw7NUVh+2pEHGs4GFIpqMQHarLIEEygM03h70TQCsY9wRUVD2QEXE/sY6D3s8HSlyEuFPfu9M93ZT8RdsT0igqbjwovr//R7H+qzNyPcof9gBx1d/GEX0Qlk0Alu8yURd57IFUcuqMp2vDmVDFarzCNzxMc1NiJYX2hWxkWFb59AqPem6bFOwjWmhsx95xCA58y8WrEh3vK1aDKbADbf9afIGvVBDFEDoyhvHOf6oNnYkhU9HDX8eJH/q9lEE/6Bc1E0pIRGRLQ4Dg5JSHUfIk9+cEtP3nLMuDeWscZfgDo+v+ruAJG0oDjAD3OV5TjPt2Z0+WXGufAhzJW3NB5H/K48w7dpjYs2uEQx6Qd0/YainBbzh4WYWQhm/zp9SN87uzNBZRBD/9pf3AslhS/GE+ZZ/sqQaA/8lkf7PE8PHvsNfOtFZSaqypipveukrvGIw8Tx4PI+l1+NtWx2eYCp7KKNIChOcRCdxNv33gs5tmqHSNg1C+nHYsHeouR5oF/gtotRIeEehRQNwtVQhvnf0Fk4KlpLCSShRRHt5XZZ0FzUVEM7j9NMZafZLOgeFxVXtDjipSVvGuFBKF9ve4LLjZHDr9cjbc95SsQqa7yTw/m8GvFr06mzE449MXE/doMAo+X2MxDTOO910NtU5Orvir3RdxTxUuVUrg5Kn7QZIrA1U+ShQXQ5AQUx/ZYeRgGjUlU94AwHq1hqbybk6VEiruM3Ko3E+7tkabCLiY+3hsDxIeBGxsdENc9SDJ003dXjF2HJvp997wVpMI2hevEeaYC5zIIlZIFjel/ZlzdgcMO36IJEaIYFAS6KtYoXDDOg1vs30t6Q/QBBXqA7Bk1CFXSJ8OsNnNWAfHIjPARcw19Db1xkwy8i7n3XW8XzMC8+SxXSDUw5AfqSs0II0XB7sF/D6UT1v1XZDf7VRARFsv/bkSyvDHaiNNQA+GGsMsszlc3Htf686/WKbm6hKiEjdWe19vluTT0YJFwej8f+uG3fgzbAP5P3yS13VkXo+IxGCGZIjGNgKm4E1UaMEcEwhvpv6hb99xoYAnTT6gaLmo0B9UAmeq93GU1rRpdCI+wLvUwe6tjQwNPaEZmlSDSpdpJuKum7BEKfIqbtD2NOzclpsGfJrtCdt4c/3xHc3oYpyXATX7JcO7Z+xK9vKaH7pCc2jfJWszwsqqBtn9p4GfFTtpQdIpLy1uu+8CTYxnCYUvYRVG14+8K7r2I4kBmXXa0t6+TBygqMMaH3S+ecv+8DrCuc48tlJIS/40D8aqjCvbr+yO7Hs3i0YWAARl0rmhr+gpTOiDH5hSNbQQ9lpT6o3yvOQRrKTU0Twa51cTwlySWS0v5d7WuMhNzSFUPJeDAPFK97DrY0Xdxf14v94jDV/UbKbI7G3kV2KNsMY4J4+uONUpJ0TyyU5nYTpUDMOl9ikzrvjUz4cR22sdznJh+WmmCAFEetPgIVZsb9+jt9ECnHA/NAPpvOHeYUXWN1SfYa47YCwoh2ww9YZvWvaheIZiU2BBIsTJueZBSx3telrdBOqjTo6i3a7tf4eFgc2/aLOvKMzneDUmlEOSFnT2+LzRVokRX5LQXatGuGxJYMSH5iKfBxafpCIb3w+puOT0fJcBoN+zbCZhtb9ENVnvCbOKCuZz8ooqrjWjx+RUvBQFnFlCdz0ojLsQse0IWnn1TDn3ozMfpgafaoGWxfuN74Ji9nWGBcmPM0eTwW1Sgpmzuh/AVnU5Mt7kUu1arszVoD8Ce0aT9wTf5iEKC78W0zsU5gKy/fm/E6AstxcOBpaQzmLDdwzuvEUSb5KNbpz/+/+/RQKo4OkeINFxkMCXEFDmeZF8WxvS5px6WAEDl6w2+hXEe2VFWfJDiDKwRZ6niFnZ1fL+xZ3Rg9LA6KLu7SRsn6csVCi+p00DyJQMhuWUBxBIPGYG6sqm2LbAH6zyyywSEwRn+0q69W7aU7Aaw/sX8LaTI0UvAtT4WrTs+Krz3QZVDnBZiP6foSc2Po+zqNFx5bHgH50yVBDCKhHvf66X3Uj4Xj3MsomDnXrYfpE6q+fwk6z55XrDQ0+4LbZxIgaIVlfkU7YlbCGReADTjsJtDod3H1NwFUyMAwXeYa16zk6VJecMd7I2ZJcFBI/toMAnkV/wmyd9p/RUVrXshz6MVxbIZ7SZpwy89x989yjSUj59iBc9SKNhmcrj9G/mCOiq+tZYnzF7c8eHgLv1mCSrakXeqXfZ+lccKIlEG+e3QAmZDFRO5SlF4Q/x2txRMCwsn2UtNOWTTbVcGjbGKtSIw/y9ipRDzOAWaeugoCEsPjRYxzAbSODhtfi0GE2UVoemFrVa2EgX/7p9a8z3ODMTAQm/HokrVxQLMt7z2x+xssTHOeSxOQbgdc6nLhXZbblyAibAizUkPO1tGSb78/sDYSKAKjlu3EYEhyWHtc4XslDSIOWuyhNUJ//rmyX78Rh+/lsG6pnu9/uUsnRMt4MU+JER5SYDJzGtcpwqaN8F69siCt7LouBE1NXbU8wZ5mGNJsPfkIhlGAIHgaeYPhtLidiRw5C5x+UX4D7WUFnhzEX1uV4a756bqR0ZrCJMmlkTR16oX+5oz+RicVzRemSBFJjreaP/gCiAD9sVCvDmtPcTCXcgEnDQFwrX7DxNz2vQHzR5m25ajChAj01oL4cA7qbGNBNUuCr6OsqbXP0YNzK/EO5fvM5FWMMR8muSW3+9Z5cZyM8p4rswAp+PFup06uubrIhkMPPP0XW5+DSUKKqBRB2NdU1QNs7ZuH7iLDCTbMCEIi/g1kGWZo+zpZVqug05GwgdMV521x3Q8pIVmbveUH13L740+AJgDG8QFf1XfSy3G7s+q80CLk94iYFk3vnLimlSIJ9DkmSUv7SHYLhlZo8NVhQTfO7K6AU0FMCBSQv+l++URZCMwnDy2/d2G5e9gtprS01u4dNaGmjLrOFl2I8UuJh9+0UdxW391myfD/6mvOzE+8bmZzfZAg+wdKw5sikZHZKU670HM24/sDc5qHCSpsEzp87OdaJV8OKD+Ozlgcy9fjRBYGfidfexCG/ZqNLY2fTzlz9kswsIpBKE7mziSMo0Y58lmFRFRCmhQfIpcgJP2/XlZXYcbreUgfx8X9hcBB6WQFFYwi5/4j5gyAJ7lSd1uzbbt/74NmPFzl4w6k/qXnDuVZ2jkCff2GPEWSOMN+5CDQxnhdPldeXCg5WMvgavop1q5DFcfSqZ5kG0Al+UBTiruydu2jjYqU23Ggp6dL3dMEq2wkzcXmdUZWuT7xbXsQDQDI/EuTdFIzbt1Ymxo/MHXbAwCGucjp3gHYcq8aLuBDz87qfQ38FZbfARdw6ZM2EOPB4ogUUvVAnPgvi9kcIBENwvYhzh/4O+RXhZVeBvzUyYjVnHsgR9J8fxkn9ASfNVFJd7tzBLOfx5ETRdn+rWvuA1kUgAohP3HddWHIQyPYmufadAzu69FwHIBIerM8DAiqLdrOocm29Ylwg5RB3N7jkKRGvIv0q9eOjZ+QsAE4WpAaAiSDKP7bGi8s/vUfIzH+4timeqxe3xKiduGvWcJt2YVBfNxhG6GpeMY3cANjY6wExBK9K8MiAIgChjHHCVlZblFvPwHCwxM4I52ewGNokAYrf3lAciTayP84x6YhmPSlTMMc/IWBNFcb5zBZdjy39FHQ4+kxeJBGN9XGkw+ORZeH24w2owBULl4Ng0IEGn+y7vnSFYmvy5dfo/RW+AB7BWVTa/r0hgBAcpf1cPIsGYDQ8kR8sEGSwn0C3vC884CfKPLrATn0w+rE9XAWJcniP51K+E2VP4UKI5IlIAXKP8iyKXlDUpqUohW2hfRrrhbSeX9Z5DgPvzfqwkgMjfshq4sOiiesGVyLB5FDIQZRJE2zDc/Zes88aKon2JHmCliFkwSqmai/gTU6gQFRBDmQKvDbBQ3clrBfJqmjpjlab4E4poiKdc2CFUP6lt32tqJxtS8If/EzRS2SR+RzzYxAVxpV2eQw+euc/mTH5GDjX7RSvrkuLhV7+pzz+qYsXaBlMCAHFdfHG329o3l43GjJptdjkPuQqvdPPZ6ECV8WfySEDsZvt3F0STzYDIjUp26Dt3zgbEQsUHjyvD05FgBFFFjy43S2opgcfmfCmbW/BMS34GtGHkv7QJ6GsaT3Nivd0fjXh1znE2TvdN9F9OauLuExX928GwQcCgKweESKv4NfwDwe5N0BaTXrMwaDWydGksD2eTP5oh0hN0pj5ST1gCPItAsmU0tgwQbc3A3Zvt2R+26cWodHhv88FVRDMGujQzREv+ZXgF7/0j/nFNubyAwFZs3vyB4WtQ7U5cW+xtcK0xa4cSBcMEEv8pRvSQyEpKChgE5E1fYDYRpXTfOykNRMJvQoyXaTIpB2SopGYGZMrM/+f+zXlcKQbxNXtJKC9/IhxpYdUQ7/gvm5Ki3jsbT/BY5PuKHn9Inw8uctzJmxYRobYHbFfz0IZlLXkSsqqPACiAVQ42HO3nCfXVrSIkGGxwMsQUG97TdRGpbNm4XBfn59nuohVOkfPOd+nQBRgROLNLpWfJM/ay/94+kqFl1VsujX9ByXIS7BHWZokAT3r28q53YP3uS83ASqtqy1tY+lTKqrhz/8wRWW7sx92D3C2WZPMHyQ8xP3smL+pQi0LyoIjcGmc3q6mj0uWI21Y8w5PEO3NfnYZyJ83NkRMxpp65rXR3b4hnVfjb9Lh22d93ZberGMixc/43v7djFheRhVTRP48a5sAoYfC8TLN62Q1gdrskJG/wawll2hv98pq/QVQabY4kSS1823v1k9gvQ5q1eiefCSuAmD8ehv92jAFz2t7RDygJpXcG7M+StbdH/I7wYKlbBJ/KDp0uDO9dMr7Kr7cWrK6SvL6Eb1ef5M6iXL/xwN+4eGwSHRaIuqirh9cVeY0asyMEID7mK8Lv63sUqoPsKj2kDAK0rMyJ2CoPZ1Y8m3cORR4ubf9vDf7oAmrcyT6Nc58Ak4Eb8OFHys+ntQlZC0Zq0ztOOLeQBHvT69pdZp0imOXWy8c9l5o75SJy+jwjZV3q0L2Dgxmslu5N/usTrvtIDVqWg+CcMmOD+cJgad2o6QxCeFZgb5ABCVaxnFqlNuIHH4PlM35K25Gyja5ElfvPI+Y7qDPYtfXdMR9qscHLbgcjNq9Vt1CZUKjwyOe1+0qhMmxRUSokdtd+j81LBDOhAMQxl+5tI2+6TSoe3MQMC7KPxY3OQ7aN6e5pYJOmyspSXCJvZGTCfNdaaghwyqHpBNDOu7Ym47DYWMYob13+gEB78+MV3sipt8kL/7kfJF/9N0sc53cP6snQK/mxBfRWPv3wZV72QCG0/o8FCACxfcUyJWIXPnk3mPmfTYhxsCfk/MzAFTyIym1MmjLvhutPxnzml/vpcxzkcxW+OYegNMX7aKoZHUGpdj1g63HnPRvpNFafG/+F6QRehhRbsYZZSK/v50o7vM/CKhuSQ46Vajf6+1oVUcOZA6yMMZjpkv8VH7nkB4Wu1rwiQM8jN8O5XdVHiOEKni9t4hGVSUWu97wjROMnSYdfTvNybcVhSHjFDC52Vtye90bLs1xmW89pPWI1M/qW6kEeMQDJz7pezwbWj6kpc9/Qg5ZVLSf3EM3052ePuYHPNKzXcSjezNBAs0gMnt16uUsOZNKhEP9bM4XybRU5XyNvJe9Lfri0yoP+nAhvRYjZQg5+LJbIPn8gADGNewMfbg/T2yO0dcQldBPoAHfPXIHOrCl9e99UnsbNHg5JAjNAtra7o7v2xn5vZBAXWKngEyoHobQZE8qOMbEN4JJhHje4OrZFtw2/dDZ2A33f1hlB/g1PXtYl6PRQd8qTLKtMmi4AC9Dd8qW1MJKfVe6tgCqyxp3iH1GzTjV8WqA3JpcL5A4gDpIowTkMfqAo4D+1Vz03SPyQ99db/H/+sbP1cn/VDl5atiGeAJd+jsW6Cr0jjt5l6wwAkIHACuU0AhXeffi6kx+zgYIEOKkMEoLWhIsiFNUw58CBNis8n3vWS8qDJmvDwoHCHeipTY8MeyaouERNwNSe4DyP4ovSdK+/RaOSPQL5LzJ23/+XVF0vRz8SJK2CLOvB+8awXX+2CdrnvLjwpfzmhNFH8TzpQqilBnIAGVlSO+kpARk8IW/bb2TMD8XJCc9j47lA+CIyvYp72d3ueWUzSQiaBzxpzcXhsIotFrfXgYlDcGHwWDZQLsrkEyMi8nwSjhTYFfRbvTB251o73fVODxkelyOrmf48eqjopfi/XNfe+1cZkGhK1fhg0vj3i2NqJZTWUzOQtZDA9VoIAha787eVOe6kS4+It6oF8gY+YqdZiA/eaJA1LG8qhpSUSBUvlK37g/osuvas0k3C9FGuHuoEnJOag54hjxY4NACNItFQK8ZDiu3bhG83tSR5hDb/VCs7G3rpr2fjUMqPkQJIpQY1rlwpy0ziqu4/fLfJnApFV2KSXLNlZrh0lgKYtZ8lJ/1BcPn6WOo2fkgLnXLJ7uu/0R2WT1MBi9umsFTqkfr/Q49t5c+Ryqy17P6LLlmt5FHmzL/jrosHmm1I+11/j2ufOTM9/xaNbo5gtMSaLp8rkLilbgST3xLIGJgAEWyTWL6jvSKrhcQumi4ivjuVKkqU/lmXnE32EtBGAkKuWGsDZCEek+DYj3vF8G7hHGXO6pOOblqiHFZMoqFzViEYE0o5L2DoJoYkaFWbRsnhaHYwArGkSb8+rUPx4jxwWPgOyoTqPF6yVOzqetl9Z8/jyaWWe+XKMNXoabvR4w7EskedLyxez7IV/l/ouIg5R/VevBZAtJqBYkRL7BKDTRP0n3/Ddh4IU8mAdtXNt9NKRdUr/VrqkCSTujtFw1R1s5/PukIIoN6btLqIXJ19RbnGJU2bJZVwNXgGic3lph4PSlrDMGjBMnoSjuB4deVMNRXE5C+24N7i8Vf57xwmm3rT2U1v46u0A50GQsPiYKyaH/K5d5mbAU/1kakbqyLm7LdpRAnwSwNJg30huYCfE4x47+BK7098ktQA+Jf8NQoNbEI6836hevYigi9kyy7W1Bjcf/WSUWcL3tL/DOMt/P+Ipks/Gj4mY+2Z3HH44e81fCDS/9lID9puNQdeTrUCEeIpohu5iAsZIebn8TfwxkdYPz8ZcAA52Ugu3Ox3k8T2py7VvZhmXHClTETUZ5Dd8owDg3/UZRu9lDOZ1tVREgMFd90olp67oZYn97l4VizNNdwRd6JR56L2qwHdRzjOYDGy509WZapGeGaZt5iVDtATrdD3fd9Im+8yD6zV4ug4dEuUn+CbRZXS4ouGa8bhgsYYSGg2pWcd13I4A9X/ZvVtnbs13hO6S14lGK3dllq8cYTWSavl1asIYkPI4E8ZvQQCFbLabkhCQwmpu9B7cZuuIoiUN3jlQmIcC0bMu2yWBnEQR1NAjdK2kKy5PzSYW2+FTxzDivhn2N32185VVhVbUFZlK9Msta4iLAEXUYFB7j32zyWXKPjcldwqIotVRVfFWlHP3W8NgE9tVW3HJuTroWt19F63zgT0GqJHglO0wIiSoNsDXk+7gGNb6xJWqZtdVYF0kjj//Xn/NGdEYuft3EoiO0ZKYSuUgXMPJqHMM7MBC4GlSKauGi/G2gyMj+AOoRF9ok5gIIybDHabyH9Yrero1Tos7Gv75OanWCKKpWLZKrFqRw+BgMDXtUqirRDiFkOyHTdoBZpo7XW3/xFnm4fBUYQlruekqjMj8no/eiyc5nWWFQMukBf/4jYpl85/xD5rlF0yvCHT7K3dPX4pvGLkl9i8O1anhknA+1onD+a237+YPYgQDFPVadH6Syye7beLpbvKZz4JLo4fD92Ozyb5yRKMadZ8J0hRDJfuIk1w22IwNCpZQR/9628uNum/PKqgfcBpoBh3J0ERhEf2Crm2rBY0pGxFy3T3sLsho/sSRlugj5HtwAmaax8eTRlTG3NZ9/VrW0rr4J79fYzKzq9w5PatQuVJpPAhPi7yhVPBlpLbSta91iPS9d4KMc/A0blrN7CQ8UZJ6/TY2nKT/NA1zrSSeeehZfZYJZMtZ4QTpLIbL6Yrnd2w4DQ6loImRE30re9UQOiseyeovbXWU3vI511b+WHqeVBswpwtFRgp5G++bPIDhMC66PD7B/0eHCI7hQRtmim3l7q/cqEbodDoDKxRILCEqvBKT5LPt3DmT5/JCuQsjDpBYQf/LRFbwlCQK1AGhfz/r58rClecvLsJ8rDkpQRJkCQVLaMUv2E8GL2a5GBuMGy4zxgxEynocRMExKvC8j90Qk1/W9lNVsBa6oN+el/INsbLo8ug0BO+X7QUy/XpT5cX2XU0AotHc67yQ9d8FTqU0CmG/GD/Bwsnr4qv2A4YMdRQ3PrDvlvvaRnC3dmgZeC+XODmG69dl6vpzAPysqFpp2VqLIEcJ9xsgbvpoRwI66iDDiHLH1tOR5XqqHrZjvPNodEGaRIywnRCac7ro0JEZhnBQWZZyG0/uQS9gfaTmBRyyTO8E2smhElVtB7lTxPhWHS6gs8lkpWhapawDQiHAiMMXjQUkA+SjWPX5MDn1wp3R9qrZM0sX+3g1BZ13vfVJHflFIp4UHAxA1VvJLwlMWEJHLx12G5ayAo3XKDD5DnehXAVpOfCXXKKzt9Qv/cFhUs7qSxy0+SCKtuthbgI2XSSIl0noe7hIAkMcLiE/g5DuwkSbEvt3Bucs+5UCOAAKoVbuubdHtfmMLFWHuEqSwC6huq2XiLyVJoK/bpHYK098mQmQKMX8jns09oIsG3kA/uMwEVoH/oB3SyFZr80A4BQ3FAoWRV5gE7exVqVwBiHNMGXLsd/CGcl2gTTd/vRqj84wlRKRdEjGViHsf0mvTefQKFnhULGL38bEeo1Q5aEQwSkQBFWlbl6TJ+3xlELzUvBMyC0XXaT68W+5DJb9FQKhNWbPXbSaR7Rd/Sh8SfX/9oC2F24tMRNd9zgaJHraOXss3tV4SeLWsflsTO3cqiPz3Gc8WwUinCYT5zcr3V4jljxjAv0ZLMXOSV/5RltmQhQodwaRa0EnKMqAO/9d9g9UvbwlbfFoz+fHA1WHha08/4PJ7ctyY7uGKEojRnlJQI+T0EGaBC9eLeU2vepJYWw3aJBvYcJjemRAIyvYq3tz2srSP5D9s+4odZzEP5OuGjlpJNe1UmUQo3uVnUXPUu/RS2T7HZe9XMg3Q+xsnveINxc5u+K6NbttGW0wgPMb7+Z86Lps/0OTfI0S9b1Dc86thwyF5Ql3Caxt4BNTK/kAvBKYzTzkMFdvm3KD5ijDbWcscEHKwyMfMEf9LeKBd5JMXrzOnivXctSuINKPbkA3SQKgDbLTdMPAvhap6uxEXUh8sCp0XYZ9St9gRUo7DeUzPjAcahah5lC6hd3mOPA0WxPiVzbYPX3sY9ik0Y5FyTEUyymk22yatHC2D5Kt1o7E/TMhXT60DfujoBnwwwTTfPlHIXZ9hgH4CEghYqsuM9qajtwWC4W3WtXlhdpdRuCKc5lnjCoRpAz/r8RT+hqtNY0RUQpZtY+n8q6P7XHd0veaVo37h+1Sy4MojSc3csj0WDoCEKT0X8ViF4dWkaOkUNS7bBaqiLwisI2GFutJvBU2SxHx/mYKFPB2L+WQ3UA7TUIv1x4QJFyqr0nrMA50M9j1Ju9peIk/xuDPROQTFlJ0bMqSV2hNcbOM3w0JLD4KiYX4jcOxUmlwx8crQ305Zrl8HDMcQR9PXVbUzxtWOAKDTS+AMqBc+pQfUgyhOuIP+FBbFuct8sc5HygBEdvyueUHNqkFzjVLZc0m6K9CUcjwQaqjZ9/Jt/bFj7e1F2DJlyqGNe8LLHrnw5TG6qWk4REpT/02YIGJe0mYMv0QE7+yIzEAjdZv6HgnrdBamA2bmb2H3WztOtPxHzuJPgc5MLkCkN3CLW6yAT6nsKLeIBPD3CAvMi6uzlT27GafyB6SSZT9zDNcTDp97gW1AcsxKH5zNTvoxUC9uFyzxMQ6o2Pw2ywCCc9roitr8o/DmmUYRxiEkic060gSkDIOEaPHWP1+Ow1Y5yHyPobHJF4pZ7l6A134WpblCIqqIIQtCI8+yRNObXB/if3OYQvexL0qOg0O20X33PmIw7esWx89L4DS6KzdwySVaHdcj7w89Z6Z6DdjL+I0KQC+0HaCkUIl1Tgg9fpDkD9uid8sbDxA8ps8Xy+oep8hPMAQeftnurhUZ4CQZJXPZt73AqxJC9Kalxdiq7zK8sqx9W9/k0x63bTNDEqtBn0fD2hE51+XupxZmIp1X43xT1B7/pl77M3AE5q/FgH7IccZuj5e6oLg6e+9ENAovviAwIllUQDrkaJ3lpv+6taHBlsx7pIN4Rd4PgQBusU1BJUb4AYvzRNstwy7PTsEJPzy7cj4eMCqlISf4YbFDWgec9C7vBO3YHCb4MSjdElawzowFjbwRTlmaCrODC7Gc/Bb28lNL+KYu1Jf/joNkSorXeSJOjGXWXK8HIboRH/n5kn8wrMtRLF4ti3JP8Ew9dhUQeoB09/1mli1XcG6mCfYmyPBBdb2ayH2GXVHv3RakiWsqHaZ41CAcXmCpuNxWbbDA2CpeXTzAt6WuMaZMKnfxmM8ZGQAUfv7e15BFrtx5Fn+Q6PZgyc43Ak6Z23e9P4yeRMxvPfRqtf/0VXgwzpSXsTX1Gj13NiNylQB+xZpUEEFhSQOGlA+1ePonlri02TMqacHhAttK9HPyqd8JeD/ZrpLSc3qatJEhvnXVC3zXLQcz+sd+HYgFOlgdJ7/tASMUZFawrIh70aOeSCsmDjmk7/iLT8pMXLnn+GI4OkcFr21UWMPRVRm/CP31OHl7VALiLR2ZSfiY9etGPtPxxbwfiz28QbKMVRgQwLx6aiaNHW2Sxwz7Y3+S1Ehhcj9Inxwa3vVjBBSUfoW9i8+u4GgjwRmk+fBC4iFeAJuKNTHd+CKCKuCAhX3RA9yKjfG5SnFWGHGAgvgsL3IYf/EXBt41QV5DULaYy67jRIwfuSoQnff88zgk4h5URJfsTnAcfdXbrO21j8dbpT9XbLlzhadOpSzYPN/xLl55nshzuusWCtJZiwt8smVebmitaMCveHsg25It0W75jrpsuy5HlHXhz7HAJU7oiqSPIIDw7Sy86ueNP19zTjZUwce7WCu67G3cupKFu2PYHhF5ba0Ei/x2QnMv/Jth7gkTn8qWysbFYeK+L2uY1fRq4OHxnXyTLKWMRT3rkTnFVQAJsE1e9eJjXpB+oRwhPjD/QxfON3lYvwurG/5a4o78mtywB9rkkFKRA90m2rGoiYVrmgNo4E5TDFm0CmJVByAovHgKogNghba+SL5zTtc0GYXl3YQqgf17p70fwi1Jn3t+MK1WzhnqsF2EKv28HLRhQHwoXecRq42wcpGoEI6q45ZpDy/bKNWvsZgf/Tvi0gGlnkqU9kmUUMOyYUAHIvkbh0ALU9K3A5pO8VUDJ1F0x9z4i4JWX17fWoyeNmctomnQsgfCVyIwjmvewOGvFkJJwXPyEOgf+mulDSFaIKdhvqlXalSmPFuCpaQgmPW1W+s3RJn+fIbZuPtirg7YWAgTh3GMR0P4Xpm7CHq2W/z7ufMJ0ZjTWNJpeylmNff9ZvNSVama/sbKF2EMDu/wEpJhwnn8Rhhi0KxekZtBjKhFdzu5EvWjtX4M1rqzblGLS/Cez2/R6MS1JaZjRCL0uS269aQm6wmSQ2UoEKPO2olSbb2uH2QDhOsp7rt2FzXPeJ/GLGTbZ0bZDiOQ79963M86r9si+xs1eNvtMGRfwpEBqkwRIyQHzT4Pm905FtlbE4r90M9fubVrZaJ7RKB90fPdHCAG85DRF4DTGoeUrYoM7qCJ9TjlH8EWG+IVkMLHqVrD0VA3Hz+WQxByjYd6SlDeNN/xGbyQUiw0snNssNxS3Dr6q0PNMmsSls5a0GmnS5mKLn2gqCcNAxwOG0VFbnc5p5V2r62EmTy30WLEgABnTgKkVHStP6j++8XN8AKqEMdAq85tvlvct7dNILeFuCZJhfKa1KNkk3RUOLqz/RKRIj63+RJHr4Z27ugggIR/jVd66C1l/FGcraiMObOypkNQ/MNT9hL/Cys+XCC9SJne9p0F+ahZSI6tfhP1S2UCPOcU5eDSCRCc5TreDNgoIT4gVR1UiA4OZLLsMAtvlZfvMNagptDJjAOFOiIM+y3JHGbOvJ+PXzpqnNadv1whq78yNCFbrrJ6fgeG4YIyJO4K8fqvrZSleXU3nkekTpYRdPIqNKC/wg2CplVAqZEGj6+1xNcdKAPIhKSFYghFEAiDDisPESdphaOp/Udwb3x7/QLQWNjG/mkdkFVZO8LU98uEhBeYWvkcgzte+ZSAyoTdO+NAYn5ttUCjGqy/yKDCHvvja5/coRB/YiT/jrX783VaJLlVMOqdPQuyM0pQPqmTVNwNZfYrdWqmwX17dQ9VDqNhzAZEb9Dg1wkgR4q1tDoqUEaE1k6Sex8b92u49M+O9xOJzpRc6HIlXcrCpCoQ3h1BjArAemxxhDx1CYyRx34YpubN/BZqxZBl0Ze46TZghRYKmiNY8+aBTLBXachY5QvIQJeVGcGddm8Tv/II5ReT+Dr4/gjepgC7zLxtVtCcfL0oRTGQjBUYMWQ7njThd1Db31GBFa4uLoXRbLB34N/cUmtXrnR6vLTWGmGqceEe3cAqg/xZA39369pvhDu/mjjBwdkVmnRCMCfatnrlRNUsCZ0DLB6O8Eu47t/acmDTEdB6ub2uAdDBYroea8/Y0ZVZMgZt2V3A2/eNC1e/0eaxoWIVn3B48t+Ho2QUZ6aspDtfjCTaRHOefxPyheR2I8IGWRMn9EMTxAtXij2OPiPT5esWHl9oD2Q2xEo7WBEgO05QsZqwvsEeVVaWGxdCFhFLQ7WqIO4mVsTW1JnMFqsrBeZgQnhCFPR5hHbNSShMUBavLB5X0yn7O6iLBskb829IrQjxUzN6v8LVi5CFWEexK1MIjHjhv8UZOl50j1yob+b8po7ZAHw5K5eFoV8/oB2ZZ0rEYTgdvgUOspZjF0r5OKboOvOrhgNDhaS3DpBZ9dtum30InKkERX/8c10yThjCX6v3Fu31Ug8qDjW3FPdNB/EhcTYvHXoLRPobX3uEqpsJWPhpXn20mfFXyoI1ksgr1MZVx6QjxeZK8EoF80nkoWaTiIA4Nj8eDsafPeQaPIypNfVOJahllqSHvZTYPj6Roi19O1hlbBTJTlS3Xj+n1/iiGfStHsJc+nxxZZ8LKud6EAUGlgD6qqUwtfsQVliL91IJ8nkusk/S7HY2sFXwvf+imyAEzle73ItbM/6Se3eGe81gdEUc6nMsyr9VL/XaYeMZ5Ki8R0kBYNt86voOQjOh7ezNW98Rg40jrAI2By0Q6UsPXfmATQZtzynWpU1PoNB6m45dvrD8TX0ETjvnBjtvHZiIxnvlKJvMgdAyhPCJrZHxVRzdRnwUrVONO3uRRE/+qg1peEFGTR1WovR65WxzVtahQ2j0QDs0xmIQaCNhYssdZlBy2h8U1XSlcp2CwkeL5zCV85LDtg7uEWyhan7EPDW9GPriM4Ka9SwANLEfl2Fr09Xl8fkA57vrpneEJ0OVN4XPiqfT3/q8AgupoFHZH661jkTDj4rTaNhuunwG4Dv7+bfpvja0hML706JRODOOvW0eyOYIjN4yr6tvzOe4+nPQ2kH3hIO+fUgf+5ck7eMsR636eCv9THM4NuHpKlkocAkL0bxPVpeiIBLVFH3o0vUGvlJp62YohajMzVrSHqQdFLEkk2mK855LY4MCViSxw0+cGU7/WgQTKQaEl+Rx9+YCG0eUpSQ7uoE6SKPqYZAOwqr9YvXSgdvQxqULakgoy6TAssnBfCH9XiPZWt8cVVofRN4OTvvc6lxIjOcEY+n9S++dqIL75toEb28sI3jaazkqeukWlLAfX5aeftXZkVj7atOTA2S/TQlGT2nDPs9e18Mioadf1FOxYJFtEurb/eb7ADAbGxMVCvmZfLpZrYJfhpvKZClsiG3Sre0gKq5WinTE0IgtSkg2L4qKiEFsVrCHlN0+dH6M/k4oqs2/V+5l4s+xVomAT9t+tGMnAH7/ORW1UT0cQa/tgaTCpCvW7CwYpX7M5Y1odVtIA5fk3B7yqWd81sIv5dOVaqQsHvYjqMbsGIHhE7n5CQxzi+CH4fjy5gHi3T5i2zj6l2/SuzlkIDidHIgfqQy6dQ0mYzfE0RUaJbYlZv9MsbJSn+z5DjeoSfJhcBWBZCYoCKrHdVKH2mTb3tZE6NIUSwcu8CVjqnEiF1zY+csWiioyQOOpyPfJGy2gWZFV7Hq5aKPHDHsgXK9SpVyE/NIs5ntKpb0B2OgccnJ8hIBUHlHtMQerL7TZzs2S6BLCc/b8GD4W4hbNNCrLBe9Dwawth0sQ44jvXnKdGphzTP2INuzwZAihviI3xRR8CGFPAN79wMcILxO6IoePxlGh3Dk3mIvIgHrfCjmr/XZW3fxCuf+xxW3p3GPxtNc5KvrsoomPZ/cepYBPQFk0eEE6vW6q7L3z7T/SI8iBAvlIe/513U7frtEdNQxs/4xNzybQ0MlvNvECkDmX46x83gCY3J0pCBGqR085ghx5KDpbYwetz6b8eu165Qynqc7PA1yssQyx87Jz9pdJLFESA6neOymPM4BtZKXWB2qsa6O00lxVm2kj5y1+sfiO4y+PR2DteT7SGKN7sSt9B7zgfA5TfS7jt+UKrGP/Q2AilTOpyAQED0eeWxx9DimT40hk2OAxgT4IjBIac5sUEG3ls2vn3HLbGEic6Iwb9fcCkVS0R7TxXA7QN09H+SXTCJwKs+l1DAQ1MVFd2LQZUyXv+8C0prp4yPyIDgt79dqd5raoF23+0pfptF+GhmVeqrnKiXYScCeT6mL85NMseptUfU0TDjvlwUELx3luyMiMjHb6t2VV7WXG+jUvj3VUXT30W5nJ7/qaVUDZ1l7DZ1agu+GxY9sE5VlLSQjoDGfXn31lwf2gCCugfG90SVxBbClNCMyeAfvaHnJr6misIq91PcoKrn27cB9muQxHTFXd95veIVWZBnqhoIY9tQPo7TMvSyg557cIQnvx9MtQy4vr3OG1YLme737Efx8hGe9fjpg8FbB6M3mMT6TRmTMcQ0WgSCdS8Chts5z4zF1HUysOm+LR2Qqq3N6UnQHsayH02kmVKeFrFfp+UcSXlk2KznzZhiPPPWn9pF9IJE4qraGD/o79qcaKycOvYeTMA/fXry0a0vp9XM9YnFceaR2Pfbk4kq/3JYmspn7bYbTig/HN0IJpeq51/KvHoc3ZWF5WRS5hxooo04+nluMJAMRHAykEN4jPV6sHuIukriez0b1vBkxOdNVfFOkxr4ZY3/rjy9xzENvTKlelj6ntrU362c3RTZ5jd+HjWJovRX1zC7o8KkknIDsjMPaFvIZercnN9UzeiyUuwu7+VUn9u9YSk4+97RHx3KoVrgjbguT6OuwdYO9dBD+Z4u1pj+aqR/ahzaRuKWIcJcSEQ0/tkHXZGl3Xkb7WlnDvjRdFS0kQ8K4M/NbLpb7nYH82MumIvXgY6cZf7EdCOIeSPOFGV2p+hb+hkN6vrlvAA2MGJSciFyRHcutL8BhrPKRoCsuBh9eIKf04hu/bD/YesXpAHn9bT3bywVHq5wJvUB8vMmTcCJNmfHcAeBNXfj5jALejs2UuBHRgDwPBDlMwnnLrzqnSZGG7+xnfMQ/LiJ0NLe9Je/1GYOke31jlYjkr/2K8GU6cRbrHVqmlg2e+xYMFN6tpVP/qhxzG5mA4OVy/oU6Tal27+Wb/kJgHk2vObknZqPctszlZIeghrUUXlOYMrknvkUoYXobQwNM77qog37+dXDCGv9GMb5cqYfJ8tmiLyw5EBFmjZFJyq7J0sJzrrzurFmRtxtLk8/kuM057NgOlxaaoB29/+NMkXmBdXCA6RZxPZmZ39XSHuS3Lv+z71DS8HT0Yk29KWZ1SywXVd2ERqnI7Ry8ac+xf1xkFjfHW14QjC9VBv3EJ/kVl2GBJNYVIzSK5oD+r6TACp0ZBFKHCt/UfZfwuV5PxDwUS1S+gIOGvhyY1E3zHMfimWRqm1rWQ+BbBvGYiHep+1WSvjR+1ic3iY0/oyxqCh284AHARktpRPuUSBW1dWbLc8tmVpZMjncdmF4nR+IMUh69XoYUTs5hVr5g1f7G7Luo7R+QOukUs7h45iRcMyXgoDFYpGEjoc5jytPTa5bV7TP+2Q+haXsaBQ+OzNPuUl1NBbrS9l3w83T10FcpaiBQ+3KdlP4h/thynzoYS2LtxBzhm8cRDk+MBp13aPVToywCDeqoLqPD9NWqOkycgGndmXB8QJ8oxt6ZnMQV4q/bisqjrXWkDpndABml+XImnknrdI3fj0K4zYCE3gdF4ItU5ttA8aO/6vueS9L0zyYo3dUGW3ZpJaP/dSwWoBvuQUkgX6WaHP2JP/Tbd5VZLZNsJt66vAaRknDw6D3lCvTx4WeH9mFF6P/Txg6So2zNHkYk215Io2VLGQrYt6522EIg+GYbuJ4OOQI1rERyf8JAA7OaCVDb8U1NWdgqXLKXOWJ+WzyaJFu/TyDUcTDUXG78/J99ZxyWlN1azTQ7wuLajk94Ma1CNtiJsFjZCADXV+0vZWtMhKvm1k876Nd8aRvVH77/iOFcmnSRP6P66uhv3ez69KqyEd1Qu9VeN6lAwWBdKDEG7X1m/uJ/E7A/q1/C4Exn16yT5re4w0oF0I+B+up5PT5cpIjgoC8YOAjvow0xuZe9YNXa6pnRh++i8mFRSnbeAoa6Ayl3+utpaZ9pKCMvkwEIwg5wlrcclgCgby37L1kTma2rsMVS5TKo+VHRQHffCc6O37AZcDkSr/wYEaEDVz+oxiI+P5DowdVHw0st35M8npIosA9Bdok0cZ8OHm4+ouWtIzcZfVS8Z+41ITb8i/BaDb63PLgCFN0RxJurjJ8oTGqdFwnvGQXkukMPBZJGjDT5iG9NUc076p1VAbI3q9y+vTwWiM5pr1ig3eAck3YSBkUn0tnyoru4Y+SYgxrDYpHE7eCgUnAoyzsnymB9se81e7ikIUcBkO0giAxZxPDT4/WEr/9EbkceGaYmN2V8c8dYt5w1Lu/41HdYPyVd+ggpolnmc6vtLFnXocmrWpVhWEZCqZ4yKveOqy2k92QVxkfdZvVAbOqyq340Xi7/AInu5Y150Nc4WAZHE7jCcoIsRKw6RUxZ6371OsEYZDDRkR+vqicabfnlPhfN9X1kgx/tE7Hau9i9SOi9VjziElM7u3VknczQJA142gCVGHL7bpe3EtAq29KKaQWs2e6+v/C2N11R/idEa5/RtzzQpQrr8Q981IdhB4aELyd9hqV3bPXxfDyiWpn4KGIZ/2DkxUTGN8NDxFVLDwy49Lc0BzshCou2ZRI++oSqj2ofMheop9tiLmt9ZnpslyC1ZKwnMj9z/4qkiiOzR25l9fsIVU3MXNx+XQD/wEHNIekaXzlCRjHAPmmgXbR4RIAVtsQ2EN6RfyNqnSkC7q216wb916bO4Q1XuV3tMzBpysGsbMt9V8Y1uriyqiGyBiDqhvM+iFBwPifMvwZCSXcSUTMd7n0m6oDC1Ct0ZQmffbgkbbd+HlTJJqXuA3/zQ45DMI/AfxGQvQZIXhxGz4nT+WbnmOEnvyDWtSJYIieLIL2Oipi+FgbPIfEvnoormh8AadQ0koyJd2BDfyjf/LAeoaCpRxeYfrWME2RPO5hfxrOjtEMIoipEY0d5v22EPCoaseHn52auZmGkPWgWk1JAYdbtxNKcPS0DVPb+NoGm3mnZxlQj339Tbgh0pEm+ZMGI/3+SxpS96zbf7R38PZ0GRjHheFxEtxfc1Zax6ouZSAuZu7K0ulcWjE7tFBxh9xTLImJThO//c1aCRwA9YBGhLFacTLYIepqUoDh/60quT89tZl4sl30GCO23ym+z7mdAKLwDlU+HsWKyte6fRhviCJKMlzFTdXt0ltVbPSg6PUOyil/JakBUaruunnphXjH4K2Pqo266d/K4otA8AbX86yTkse2VfMmpfmWWD9/6xe6MVsApAPBk3aJQGOLHO74Wj9GHzl9NcxcivWIaxJWTTfObXEvtyy98KEJsH88Zb7qSXyzeM4dqh/hHCuIZC5S07y44Y5+UkSiy9j02ZpualSGzkGha3sfUFYENPLts2hYz0ebfRIRU7luHiw1RahYNkYuPW69y7h3pQy8X3N3AAGPOt0VrVHASR9z55wWz3ScJjLZDFUwB6ohzC0lAnLxQ0fW9JUToPj3flNo3e6JoFJmmMd8Yw7oqlOw6z2Myd0zGN9ai+3Lj7rnnGWyNZlQiE/5bKQ4b48JamLNK9wsLKIs+SbMdlDTFaOZLSTFWZ0p2ePl8Mn6zWADxVPMZEYc06YYjq3FXSTj6AbltRuZH3fUec1hId/rtDOigXaekkgL1f7QLhsqrQFVmEH5ONldJuU1SbGtpNyCDb799adTMLwwoPFKYSJgRBSYSvN46uZCU0fgP9OdWjtF+RIdEuYjtUjjS8Nv6HNYy9kRJENFiu5X4b3oHfoZczCFPi/ebA15F+tkMR75puV3j0w4jHbkXqM/w1t+aaMyUEl06Nb62pHx8l4zdKgTBHDSNDn7kNVvmogZn8+0EH+yEmSAQi1iovTc0AiBYloRQ+SRlUnsvb+EwOBIucA7GK2OA+nIQgPrtBO4bdQ903b7ncCnrHPnCMxktZd7T7m2LHYrVG75VyrLe17p0NKMtjl3N3/zXIuWNqN95OQ9zLo2REBnT64R6SF+YgdkZJ1wBZ4Kxg/ded2uYtI0l68HOQ1h3vfpIchSEkqX4WbEBedXt+lwQE2sWy+jfh7vDVacVPnY7skvtN0GRU2Iix4y5y5MehLopgnSSxl23g3FYF9Rf0xD0sb7EiLnds8Teqre99EW0fwJBEhXdnlFEi0hVm9fapdqgEFZpse1K7g8sO33k3dm8oWnv/mkoFA8V2rtTi2pJXY6LVX2DeCXW51Z1lpeXqNlVlwHKtRFBL4UtE1TmQV/NZrcJXFltTIHeYriAU3JBc6qUwOqeBmBt7JpltNiOat391LJ9f4GrehfA9XG47JSPHixY6DOuozejUPIo9fVaxfsO3GgkEmEUPZeHs6/U0HzhBVeGuFVYguUdqd/1gC89fdArG37EEkyR9pADrLfE3nXJi2fK5BjkRcT07JLzeyfBbF4KKwSA95GBortbi9awfrkRx/JBTk2Tf28t8j/DS8QzhMOc7bmJ0EE8e8fDwyNCx0Y5H2IF8qL+5APItnEuIWnNWgwhIedKFjsbxptYEPkOyH1To/amK+75YAyM2eo/XeWEDxv8WNkcd68tVm6nHL+RRGJo0MBHcyWJgKPpOpe8sg+jSlIYeBULryZedQbGQmC+Krd483E8776oFMIIPTwDHlTqf3DcLmvkt2CwRzPYCmjBnoDb0B5mNMkYKGlDgX2Ord+VkU7SPMajf8wAGaKy8rWTm8fjC7S8AT4Uhl6zpfnt+ox8FmAIqgmIWsewfCui9Wz0+19JESbr+KQbMnqHy7bDXyEPRNzo4T5sd0pwqfWvOt2jK30YY6TWWVG08DANQS17dWsiZbj4VBpaVkd55lxDJTdwsjGlVBeTc08iWtADYXFhuf/Ba+tOfBFS4f+Gaqk03hGmv95at4ALXJpj2c1CKfW7V2PvwC24EiZnrq2+Ol+XDEfwx7wjuW60ZZjAYkG1+NZn6/qH9DxtAMmvenLqIo9DwVEboP1250bziKm9tSQWVOkI/3zJJ7tDKfdK19oNmrxZZxfVosPeGxQ7GsVZ7QXwehigNQXh5Hmxf1h5TffI5HQoz0jDsTBII4wOxpRES+KaH38fTktdIUZYK/bogheadaHMzw/l60grnui0y/iMyBJcgahiyHlNaqDYTFz/RvUa9RQK30NsUE/8dWvLrnlb/JVxBjx53U/YQhjbYd8J6P4gf/77Ay2+c3J4bpYZsMci9QG95WVkf2EjNq0H9DccIkP8r94H1HgjOuNa0QlLmQe5s2bwl/QMXjVMQatAA+Xz+jLjv+utBMzuhrf3bXFxho5G4FjW3jRZ4p56/ORDOfrCWV1KVNoQVyWrPe2WcilgW40+t+X2AEYBP2Vbbo/HZotHEm01nBJfV9H2nP0E7KG89SkTHcDuN9YDh4hYxkMks/y2LAchb9mirvSFC34/SOlqnqZjkgdt9lBheaMWhu1cReO/2gx7Z2Cd9g702jGo/BqzHy45KGXYXBPLIbP4NrOCrDAOLWnQNbOfj4KRmoCGlqCEZs/rOOF0voiqgIJIXQC8H/xspAVX7O5ytLINROHnQAXmSpgFKDkFnoOiLgdOsz42/RXy7C1ZkdEfQqA5A3whqCbn95al7OXkVZNT1C7ALx2qFVISrAbche+gpPJI8BpwZVCOLnWJVILk0XAEWCxRynDc69SoyKz2V+chxozV3PlCOQnMu/wIWBM7ZzhC7LFubwzIdt0mhpjO9rwhn3JvszxhYDMI5msmL+yvGqxyQDAHfWeVtwJsdiA4bEjQGd/nFr+wVy88T6fDX93Ozf1/+q1oXJdkyZRLGtdd8T2lV4iq/IqiyIE17RzmjcIaRQ5bgvZflN5+vZZ2+Oc4ZGPytQshbvxgpv7KHs/STIpF8Jl8RtCuELeZHRfeTRhdU4AambHTRq57FB4clIOXM6wBXk7KXlPn7fzOm3T6WZakeDWWmq47AdVYUS80xidZBSa4XuCDgQ5fiU2ND1VzUr6ohmSBxGBXMfz9kpXptUxZmMtmuMWrIPTiO7VcBuxHmMNCs3+Dz5wFXdwQAHR/PESCg38diMgIToz997xKJ4Sd2/Ao7DKhG2Ypf19fqho/BvjLx0hVyiA/hy8ROaDnW3io/xwP9ok08iZdy4j1MifFfkEdil7BCrKGVZ2l3AGSAAhwqgY7HO8I9ayesU1S8TyK173iTGY5u7ng3iRWEFZn0oG/281WVBpKYETCAra4mbxcAfeYOjzLApq9nFzbcqgiosRHnDqg2xHYRvrexQqu2cFLcdCNfntsjc7GW2vSJX8XsbzDcG/KP7npNr62Lo76Ef/aXZapQG7cHSTqd8lbXJta7iWi2DMFgWo9p7UUMPJZRAxYQhtIYn8cqoRUKFDyHMyUcUtVmLl6OpUMQvMe4JkxZL71wN+PZPOeSJNRmxdBs2jRmyUfVF7HzQmEVg9BBtozzZgpgOdjGhzO+Fk6Jw8q5Euj0OxUbqHFgd1j2BxJlhew2Xj/h0Ty3a4k8vM6HQaQZ+/UG6ZMCJWUKsudcPWJtH8QLQF4x+16SSZnTa96gSJbPkvstZtjiT5drv9Vi1DAT73Ln34z84CbUf9tv1QvX3yQVHGhexiSaiBIEDNEiLce/kT4e7R+kS73DN8GEjm0TJO/m+eWGiXE4X+wr2ybUUr+tWdAnqALr/XrxBUro5lBePUmSP8oPYsCEMsCBjgKaN74tvjT3qxuBoxqJ880SjBuoRWTxhdsZXc0g8ZGTVnsuMG98Wq54iBDMLOdXAPVBQIZXGqPNch80d2mHN2NBXoaSSdeiC2Xtd4cDYmHjp4iQgC3gEH/C2q4AwMEkonsP2XHoUYVbEiiKQNI29wTE4lAAjkQBSo3PXKGmlKpijLYTaIDh7wsRu1lGGjVWGtMtrGAIgpU50pQf1azc0vdkI0iFvDQ5Jz/Sb1jIHTdONFUdyUnOpxAG+kLA3CZxBa+w5dlyxR8F04Ll5hDDFpr6lMwGjE7B5WbaiPX9PHj8mpkcTraipQCMKBwz8r+3ibgq20lLBUWszDWLDA1RZTFkjkXz63SCFB4Ga1Yb8iXgH3FErPZYkCZfEzuUm9eQL35//vZg5sxn20sdJGRNAoDX6/FJoyUakjux6dsbVkr+UJtKDWOKWpXdEM2cHsOslrrPVYRfmDdNzohPGNU4Tvv80ZECEBSDj8NJY+uXF++SFpGQxE3ntfvTRRST7EKeJJnjPzupP1TASktMDlw4kTWtW7SJ4Ba6u1+y4fOc9RzcDp3uHWqooWlinMn2CuEo4oJ2KbVusghucOp9QXNcmrMde5IcwBeUWJLlQUgHvA89ZEku9k2174SsouWX1pcGtW3OjHqAjdaVvBTJJd9XRPKerzATM8j3w+/NIJi/5JttufYbFEEqxfe6wph8QdSLVbBXcp5f6OCeJ1qQZLJI4HBxzSKpFWDg35rZj60wL+kouXecl1tMtvMO4mntPmO4BswnS/F2UJ7nY3PAPi1RIWNtpGBumDho7PBJHGknaWq9PYOE0LMK3fEYe4c5QSWPADiLLGuKA37ObxUQPHd6yVnxrKje3wVrLHiD2cQP6JL3JARzuTUp+GJxRSEwAujBmYvs2F3DTfldbiuNXF9+aCWU2Cdq3wdiSMdvaZA4rbBHjn3vs+7mXYJvLjsVMGY6QWF+HlHgSUA5it+ZCOe++qil6bSpiiNGrGroZZC9oZpeXRAZBMv8XxnqVJ1WloBrFMFZC8EQbd2+iINMz+Ro7pFuzSTmPUSNZnwIbw6r1GxnZ8Mbsd0MjQfLoicRpLN/2aNL4QwvML/c/bWdajKGi384qjQlv+Eh1jSCmWYy1d9U3SUPBQZJP+8lXg+vKy0fiK1AfuMkvAAqosFj0uFey629aGHxPNZ+72SKxpslvh9junLDi/HuajlB8SrAam8KiJIBmiyIbIfVozvOhewBb+PZk9zIzwutpEQ+NhnGyCFCMaU5H1zfSltZ1HHqQ2Grh44c1oU4GaF8r4tFj6P2+DHaj0TFo14JzgSgHo+NeMkMN0cPoY225YFar+u7jwZJcNSuYD5vLFkZLpb7mYy1rBB39D4z6r7b7gPeK5sAvzEput7QCD36BP++AzRUHmLJ4gQPqN2lrZi3fextO1gvkvZoTEk1AsgAC1MTQxTZXHpRFfbjpH4TUQ8UVSSA8rjvv/o1flGsNoemy9IPUgSLE3hw6yNm0DienOuFQw9XBkGJiwV1SR+96dJpeNAcmJoSylgZyMxhG0B2zL87Nr6W1X+th5TnjgAREG9a4CeT6SRJrIeSNzF+89qBkX4hfoEcgMp+6x5pSiaHVnEdVnNJs11/38eSvd+puf0274Wnvl8ouEPqaIMeL38+EYE+4NVQs/xLEKP0rV+JLj0ozd/nkCDLOOfaXZVJ/G+gPWsHPU3tjKV9qOMd02Vbh3lI5UsfEWV4bZDBmsU9lC2VUfyXJkQE7q+u2oUwUPVPrbSPfghCM9mCVeLsMn4V5vGjsLr7igfyCi8bsdBKL0AnziM9BmDUr6HQPUFNH1i77eRASr9o0RxBv3Yf4JyxXDuGKOa1sFTuApnQk49sdtbL7aACX3G4tfuFVGmv+GiUvIFTI9hE1u0ujP9L03UtS6rkwF+Cxj/ivYdu4A3vvefrlzpzNyZiYswJoJyUmVJJ4VnytO+56QRMzEuefWtXFfnRPwjAvLIQFlPpikudKciMVId74UYLh78Tr56aXGIWPb2xrKpdeFlV+3KU+dovYjOpnwZEA6CG9kwDzaYDX7VGw3vxeUxLpNL6bBjFZMBhrIGRZob8QP+aTye13cRBU7WOHJipFd4wseYHM8HypyiEZWjSGDXc3CjX7989OI1ZMs6Ah6RRRC2LkPXK4VCENtUd1ZY6qPxPCmK+hk0cx5Jn1t+NhSP5CPYaNMrssASqx3FuPRxvndB6LKSeDcgcjm57v+4LGP2DzpZn7ZnzoW0kgK35aotCK72EwGElf1Zw7DLCT7ZVs6dgcb8u3k6yvca4cUoMrE5qiF7TSu6vrWm91gtx/Tdv+UIdSq1aL4TOXDLTNWeA19cnf2G9mdfNOr9dwcLE2K9UxZsNitFuWrybMuaXAk7QhNmWiND/K2e97M4tVAlAx4j6XIEbuvRKNQjGu/huB172A8rFSZRAn2PMJPtIFgbWaHiYOOms+ym/sw3s5F9X3IL2G9r40CgZJbmBzbvjqvpQ12Y0ioHD2EMDWVi87uUGgXrBTDIeOz2/fCMMFVwu2Xv5/CtnJvzlFAIv7LREdn1dltMxPQp1IRh6/N1VWj7iysH5KYWgwfdIKp1EpNT1DoTkEuKGq1TzjWnF0vXH2oFMegWr96i6cc/5d6dXHmsNOqlR7cdguoVOsYRViP0rYnu+qn8OPYaNsmCt2053gfS0HVa1AGesHKEunJAXH2AkR4CderdpVGfDX8KVR6W5uGXLuPBfUv/1FveQH446VTAq9F1fbE97LkpfTrmeTzSkBEM6rR6BC7fCdhRHiKM5nxM5bjA8hIQAipsc7UBCEBZFtObjWinhDM0jme4B0kRW53bLAzpjOmjKjJ5HY6Po+t0LbgUb9mxoR54z73/uIXz9Jnmt94/lH6u8bh5cKxDsGZpIPNfPn5bieS700pqpkGbJ76Q6CpbgB+vynyBhIv2BmsVQxgA31eFP58jHuz7i8MIt3tvyy/DWvMgD3IDg5gG0dOp+V0PTD2R5VGTA5AecoFivBK1MOCc8sVS3OSfhjaubXn+ml/0NywvbMJuUFU6u5lTIjOUvvjO19OZ5jMWfkkEYjyXu02MQIWxq1+jBu1V2rSFoafIezdYchKUMkK2TJOuUtiLnvBN/QVJgaFVqVW7wFaWjdKxwWyB6grCv/3hs3HwFDbb8LF+2BbHo+a/1cJMaQ3DW3nxE8zpJJvkwGCI0+E2sVd69btqxmhxpqeIdL2QE8hNB6gfkHz2zbFQuOBE42mYoSiLGVx0qqihPtfEZGL4oo0rMG/W+0RjveC/WGgNFxU9vjPNm0xBJsor4r/P0gTCd48Os/rnapEmuskPRCNg+aoR3kyOrL9Nu3yPfyHYNj49IQ57/ApIVdUadaMP0KKJcYCGsJcYDak7X4Z/wOK3/cvaVZ7YStDXTZDjix6M+ap0bHoNCeGjuJFbKXZmGcSwu92hI0zjWPplbQJ8zpBBX5bHarMmZrQvo43X9G15KzNduOQjN33V+/sHR4QXrAW/jnut6GD78BYkbjK9I/k8TNj2UYNZnd35OQsk8/cvdUkaIhzLjB61l+/685jLbqoRYltLZ+6quB4JjlSfi+IBL4e8zUYjK8rSlgzxc4SzWen09kCOI9F853feloixPmuZi3HWFmYwc6EtWWuv5TVKEQEYjw0zXBfLaXnMIdcrq1Fk3Vo4KCq9HP7qSSiIJXPG6XhsMmfoJ/UFyxr8qL5FeHoRL7Ge2hW7S/EWL25K8ehluicIZvZgJi7KTbj+B3df1pyqMQIx393VozsPz1zUKFYFBqrZq1oal5CQ94zDUP/md+fwHmqjfTbB7vD37B+Y/mElnwL0kDeDRFGVbt8kOz2eHBg9BnLK+14BDPejRpLNHSXj4E/TR/KBls809MOmC8lnPOd6AvCoJ12hBCfLXiFW3T7fDD+mTr4MMJUvXc5PphcmepqIOTlWJvgYXxF8HpE5eLoIS4oZVRY5GH0va9rucNht6xoLHUExMZB1gpVZr+/scfxo2myB2i5OY5slfhdp14S45zfmKETCQT+KfHyrE5jI0DlwZfcYW1kT45Et9pJ9eNwtP/ndDR+BbV+9ku7+rTSyMwNf1NR+c5RSRpTwgRSi+ssii9RwyHDbfIC/02RGuxpvtfG3m3DOukBV0UpUeLeOHJu6J8dFH2abJ809QqXAS7iF0x8GYWS0oXbmkWzN6oJ7YZh6w45eiLeLELUvEdJC/P2775RVe/tKbwjiq+JHO5IYQ5zFJBhUHv1YYTG9LMYx36JpO/bhyfS1hFSgx7O18F18hP814aQGWhnOs/kG76dakhswGtu5cmSNc1HhSeNeE9i+tq8hTu1W0uqUsM4WNA8TblpOyrMOvQSPKL4viv1kdhvtayMFjaoGy7HZSPZ4n+Puv3SjKrCYNb+FnjN2VqTPzq8hSXNFKCdsR+tltA0jvQYW1j4Jp1YOBsJ40z2tmrKOYFeidjFRuvjbkBkLO6QCcyDJ+lA7nbTTIqbyU8lOt8ufnsQ3I0S6u0P4LtmmGUM9BjDTBX1cbTKs93jzDlUxs5Pa1xMjkAhiLAnaT5lNtlJ0Bk5IobqCoK5b3fAAQ+P45rqgqnj48torEX3+BaJ+xkShiKCfXrkcdsVAvJp2OcpYxYioW2hC+w5J3ROWjGF4War2nD6ux25puCWmnDbkpsxKrlmWH7Q/Tf5CFl7LTJY1S12ta92RemO65xWJSkefF2h1HeojzRYoNX8zk9m19L+wPdS2qYTs1Mbe4qse94vUwPIrzRTBK+vP6Z3D4VGtecH60chg4DuoqrEaR9if++hU/17+Zr5n0V+01lQercZZf7Jfyzcfwv7G16H8VjWLxZQpBoQ/Tl8EoZgY3NZlatSU3XtW/vMyH/+uJBMgsFaDqE0rZRhAzREfR6PK6aO59cuZ/XQYmzRb/GpHELzPJPdL6HVqaI0szLRAfgpCVDu7UA1L7V5pp6ZCzmmox9Qj13gZgT2wpFiTLejeJWOZm12moRWiBBuudQ/4wEFOUSuebOZvMgpMs5weFU/NnIhcM+iu55bwuDPqrh1NbB5nRh6g/Em2Ios/4/EzLVkXoTAg04P3cq224pf7FfCPcNGP0AsugdYvMI3wq5Bxw5Qfy48EZNJ/r1sF4/6CDnC4ddnzrq1eO4Ac83DkV9JepPMqwL5AqTNMXFyP1BcN/ZHHKqMLrBZTyc2X4KGXLwmSKGFJ1khmjJFv16Stt75UmzTq0rnm20eIFCsq81hS2g/6uXoWibETro56ZJUCYjmfl7LWmPtEEOcAiEL87tZ6BZpIegP+KXhbUs8QpK7aWNqaVLQG8BSpYkqjwhjemMHn3mERcG3nlB5xA7P1ZxcYNAG8hRrBE1o2mGf2CN9JCgCEbG7zaYzt8nBl+Vrz1FmLLLsI9HO8rPpYJDXNRiVTRQYPzFYKoJO6FQL/ZT+8ZGXs+HBQdr8WMxyDBMCQxwc2GD+2HtFeaDdg230IOlx6E2PxqkeAtF+cXCimH89Gza0eKOndJKaEDKVSn2Ynjy5e6wLQN8pSpBT74k+M+exmN6tCbg+ky0RLXarBOeJ9utv7T91+leuXviGHPNJnGFEfkSrlmWU12qj7ivpzoFfRIlR3rGaVjiEjJU0UlFNmX7+bY5Wp7547yY/Rw5ASdIAF/3Qnut8yDD3QMFDrPvq2E9nJy/p0Pf9nnrcGBzpdrrrBCzJGxNcnjFlVPWFzAeOeHAt/VR4ZNT5JFELPFJGfdCyqAeOdx/i7zYEbyWMxpbN/m/usJR693zltm8tcYqmVTcwjAXs0naVXOcuUelcW3s/i7T04/ynfJhUWtwt5bv+Ol1LGNhpGK40c5JNyEl1ODsZemxaU8jVD2RUaJi8zMpZpP0RWpnlggQPQC0XVO58h6/TIfNUzmGUtuIz65s7ZOX4O+9Q7dsu952NJyL8owXKeXTWIXO8VxOr6zXVb5F2IY4NMdb/fZSbgoVAr8mbsmcuZkKD7PiTgGVyHZyLQ3uMXjSv5B6ddt+UZDf7mFluUdyrwE8CKmPTcnfJ1ZJvhadwcl63YsXaLdiuf1oXHL6edJuX4uVdLXZ/aMv8q2IfK1ozCI0+Y6xYbWq5gA58vMluFf6OM/aguxeiJ9amGoqkMM2jZqbgQoMmzjNviuIS0E95q0XBnUhMfV4eKHL+tSHcWDAMvKfM7feq7sRn8Jinqp+197EiDctdKDftvjGhoFZjidKsxGtzdbOUmhjJop8h2IgNhPghoQVdQWEiA2FtjlMhkPq9N/91cTOexvoNkdvUJRL/+lD5+nuYrAOvyvFDQzcIktXNILLZpcn+ovrcFDqdXlrcEQBFPaXduK3fbauf8E2OAlODcXM0hsTM7KlGTT7rzujDY6YHuEoi5ftC21Zy7Rteg6jnkbw0f/i7EBr5AskdR8LXuSfblUl2eqDfXGstpuPWcy+UUdTa9XZNN22VI2MACnOAmj0CrPm89flZHNG622vKmO05J35/8FdotwFxGy9pebnjTZ663i9aJ0On55OiTOff/YwrIQ9WIY7u96gQLtM9W5aalFcgX+UxuGdud+kYfpNvvIkkbQOZ4IKKvtFdUKXgACpjLUf5jO6SDxEBMEmjkvVdlzPXY7iU6+l1scFqJ+uhzIF7Cf6Vt9veyy3zZU5rKUBNCDRcblq4UDhkas2VppTpl6t9vBy1X/6o+S9B6Lal9mlFwHkf3du5PSupWgsShfbSHpTUuzv4KfCAK+n5ZVOg4hXFnwd21/L/wFYxLl3OaS9gjEYoyfb2YDcdhx2adErhfUcYFs64VF0ZI4mB2PaeIKmG90/ZX6BqEm1OX/bqU5PlMkQy1eAEElVkp5jF+3VbsMGrmJv8BvqMpN62kZHRwASNIdQuBSNgvyGFzTAimqKmZkI3AQDOIe1Atr6NesV04JnA5fS8fY6mxIJSqv0N3XeqSBcmLnHxGpY6jXXflrq1sw9eu5B3e44umLz8Zz/YX6vXNzmY+JoGx+BBjmO/6Qv5t3YX7+oC6iilNUM0IJCTWbHo2vrdk6Z+8E+LYORCUlRqnwXHAb0pZdtd0v1PqHWxjy44iWGiI35b129kPAiD6WjBNyakIaIfGXrY4cCJn2vLFdcBKARiLe5bdY8jV7VQdR0aVN/KSACvPG8Cq8GfbhitN2X5j0Xf8WentGUGbgXRPasmtn/ytprEIjOxRLscZj9RhQMxqt3hsp6gt/VcO/00/c1PE7NSXyg9tYnNXmzkv9up7tUXrg15Pa6SIUquz7N2q5bsafOUlfu8NgoVrbcgyPGu2DqvsMxF4rgn3F8cBNs9KegAHMMHS9Cmg1388oYoejYngdaVFJfoFFqlGb+0HKTTusrxbEBKpEvuhvjHOjHuOjRZMGswQ4aUCUEfBFBk0Y4dHITq9aECx7PYjG/BRyENVlGbPKPyyur+ZVujX+11BmXrOj8OLjHxqTHdWtwuc92TuT2t0vfcRhbs5luIDlHXlHCVLLf50fwGYTAOfnfq/Tz5EOW69tPaXrEpgchXktGtie1hl5Cg5qzjqSp69ZBa2Of+khMSsZeSMC4+GRe5rN0xkVMs+2KD5hdumRUcFM9PMpo2tGYKvlJHiIY/Zfey/Oo4AhoAAG/FxC+HlWLvgCuWqtkROXaeYYqtzGLAtpNXTX0xIK69zkEILnksva/i8+go+vAoFYvgzvXSboic4k1WkFCI8bC1y/D3uxzB17a0zkc3pHqcc3tPSh9WC1gGFX0OQYbOh0StxsoS9/1p6AHsqOpDXH/+XH57+9UUZqeaHTGbGsLM7QV2TGufK5h1q+ZfjTyIV9zNbTpcm39HObgjr2yr8bbuX+GlWMtNb5Hg2F0bpS/Un3LC7HR9J/ohe0aJ6oqn1c/YoPDdOyuFMLNQhmN5Mpad0T16HZWDa+/II7vtqt5/7fgJftZapQUs1/Iccq19DGA+yIXTbMh131tVnQcEnA0sAinX8KqN7PdNSnjYIasNkJFw6CNUoQQJeyPWfzdQ1iI5QQtQrs6FQitCICWB36q32wjCr1CCjhZuW2pdPh3P3pWorxPe4++Wv469Vt/cje8CkKzaLU/9k1LbhnBYpgw7xyolb1e7dRViXSeCvpiElCNH+Di27Cr9BF3BpDTuc6wd1gTklheDOr2zKuhRE1Qr12dYgWKWSjk3WtnXagq0diiYNWGo4PTyiyL0LGegvhmy/YslCjDICaNHzvQQe3XATJOZEiZRxnRFxQjSj1AQhPKRRe7GvdXOacybiAMXo048CRZdSqkZowVkm2jXuncfUsDTUl/z7rHeoclS0tq2wFgmbLLSqKbZqPXNNqOdkmg3p/JRuYy8oKG3qqZK8/pvn6ESyv6R1SbP84GD9wZtW1X5vGwU6UrsbEckT8vK4SUPnfAs+f1i5YQm6GEoQ4BUiUBApi+har+eWvtgYfRWLnk18J4J1d+c10N4UhRRwBrpbHcx71Xv6Fnpw4mXShMhQbbC2chFP7v5rg4+dAIENXA6McJI1omSvQf6fqjp7hSmMWEN8n2Mh3tX42edMbK4kaMpCSWbtJJUQnBDmN49sh/dKpel1WlpcisomLtP+xu8rxa6bKerO6IKmFwN2f/JiqLTYhdkfMQKkwwFBW2Db65NaSu35BjKj/LkrCS7+qJIHWFQWFHcIaGxwpVig1EbRaqm+GAKlOxwiwJNIA/hM32cLeMWB0CH1aTsuIo8xV+aLEJSFNZbcPhZTS9iHPE5qB4WPODxmY3eHpljsVP+BLOninVDHS099pgut+F8NoBcfKdOAbq33BouRSQNCRW/ZiIILhj+/ss+LW070/M96uEEsRm4eac2TQjwwR3TCjBtp3Nu578Ga67cUCZfX5Y99gjxZGh/1FOTfKJgqpUlQOlAQSLCtAGVCfEAhodBthtUvHB7Qp+5N+wxpsBcsuBgcEQsEEHch7mGGq99Hckr4Qf/UsC5a5ViXqYBQDkrxtROdUzFNrIvw7VsBbgdOkNa4rlWuVo8+21EL7Fdwt8XjzHS2vCKqjak1NV8WVrYqfTi+bRn5xZ+5k6tAdpMEHR4zy2seH4q6FnX7FxKjq9CPN6lyyGWwPCG05piAQ3Il5ZJkWxQCpTzX+cOSDFkKNJw0BWmC+b/W1Ev5+ycBjyOvsrIMYHv9Pufir5y7XhAnl4oZ9gBgjFmeDEKh1ZcPzE9sC/JuLhU3EoUcPG1nnWR+bHwEgsQ+rsL849J9CS0EbdhjWGJosLUi8rX4FsvQVyvF94FIETDeI0F23sY74Qye/rdjCt0yXzKT15QH97BIQwdUk2a+b8IpOw5kXzViNvWvtTTF/mVVcT18dRT7yyTehuIeF9DS39cFlUfaijLTKrrluD3j4ZT1OorrU1NzTiC5Jy87+GXYuNan6BgFi5lodCBhdetidpn6Ev+pAG0BstHGDcuvgvMgfUU5WTFZ7HEqX9JOZIhy30cIUIZAFwcAlJXeeSQOpaGvzTlm5koLAt0cQP8i5Eell1e+HlwuIvDNHPOpTHw6MTGrvD3rv/rfl1O/OiSTFGb94ikeN0no3nSN/h32ZHjIU7w/wxRROaqJ1SQjK5lR34ZUXSeNc+2NAexmV1edh/k4mg30wAUVbHWF1SpB714tm/+vJI+QLDDNf2GEv+mMt5Am52AfjSS/br1h610tf1T+4iTSyTjL385vFEQdy54z1HtVWIl203oz81QvXo85oiNfIyom3MhwTtWcq2bm9UY+bQLz1La2xzNNoaGCrtZnt5pw4bknWFAYW+sv1ooXt66aOXFriFxFfZNBe8lVk+0HTz/qY5Z3OtnmF5ejGT21GogAapHJZXvwg7mFCp6xY6rWlyGvkpYZUmf6i2Cf0FE83i9E/iIpMHi/yMRAAUSXkmyZslUd/fWHPjkohFdbFSORGsexs2v0rB9pIvJDjQgzGDOR47uFQbcus3hk80vl2QFdi9sAcaF2UT5HdhZq1kI/GzpH5zU0E9rIaMkMSuEehX6e5UYpRBH8bJn9ZWhCGAMlQDOcPlP7XIK2qCcMgrOjHAKYDxImi1huaNhjaCouik8oq3yLXpTn7yhRaL5jkiRoUAKwcbOMFh3FJnFrHqATbF764BbyPJTap1nX4S6dsNkHsKfNjAWzYma7pO3Yq6HBJ/SmsNe4bouCwhWMVeXGyR8kmBNX9dsS5jkKMe+49NeP/0kl0XQKNPJmzsEb4ut45Z5nkte70SSfSql61c/+qXrS9LWVG8IGVQa1n5GQ9xmEUTHJl3bdVwBCxMooNmVo11fMgEhyugdtHJZx+1BKKVNMYoC8lEcX0Dh6Spcs6A9AMxPhrrwquHNpYH5O5e0znWkYGpVrIvGF/Xb+7bNKuPAQ5vYI9JO3Qz8yMH/t39/RDUpIiwasUN+u/qrMApJQDuz+DzYhMjTNZ1zr9lRn5340W+mXr2yo9zpLEid45X6kiXwf30z+h5a6cGeh9mOPp7VeaFzs5e3/40rG/JVjvKUlzDT01oxccYXLBxkkQ/GSBmroi6BpW6C9W32EuccxST/I+bY3IGUdZH1Fmj/Eohca488kGOf2xBPBFD6bhTGAi+imfJVODzUOWqk5mGyy0gVbGYtLqWtd25mdZvy1FoCJ0G3BGr01JJBpuGMH5jb5jaU/yrTqfpP/pd246QgSRUUWpJW+a569nEYZE0h16mShLjodMvfvvMZqEuM8pjAtX3tdRVW862xAplYbgevja0b+1JIUhpyAwY7DMP8v42DymO7nw3WAle038j95/BUc/WeUyNEb0mRCYvXbNoMnWGPye+KithOGx4tFGXm84iJOnUR7E5tGRRbcSYNhILvXpwBC2A4iF5bSxId+qX4dIrqb34+Fymc9relzI4e+0lZcrhAqQDgTvWzSWf9mdTHoa2kIEYeHFBRIPEeZ/g6x+LVOUNpqF9s1FTg7Y6Dv/+ani5aoffjRGVATBl5GvR3NfXclaVOHLGcKc9BfC4Ncm7aXpwYFAP0BSWTmnpEX7ckqPQrtosrAsCvVbZ05z3vPyvmOZB6KdZ9zgIv0/kl9SK5amo9zSGL3Bvve7pm3z4q7rHDwphHDW9pwQX8zgCZ+efX5u3TYfem4+Ejr48A4bPnxSw1jO5zgJf6WQmYYv94PNCtblMJqcXzdChzjIVkECfwKscAgOWH/+Mp0y7S/5z8+qKMf1RRQpwb5hFiKKHtxyktLv+flxskWTxsSfr5fTy3i6oOM0mg3HRCi8YPQXKv1a5DIn2y5N2D2XJC2rufCNCk8xgnOwbw6mh1q1PMPDg7ogAnbL0DTbAqNs4V/LnOpLgGw783b/2rjMce4ZeVQ6pc68PsgHqsMVHqSFAHwCQmpMGZrlFhsO7IecgEJJvKZ6mAX0wB8ZSSciBqKUUv9wmhUdMBl8euzX+43eqxLmLwSa/nUywEQdVsYP40ZSYj4UQ6BSN576oCJK71UN4ojhGAxJ+Be9a6w0M0JBZpkxyJh67Elb/ITm98v4nY85FaI5FHAiSUD7UqiNcajsa0sHnoQU9dTiurr4q5x7RcGaN+R1d7k5/aLLy+i8aNaN3KK8KQ1zNqMtgTYjb7A3eMGH/otCwbLEwVgrQND2kuOaGUNSsRPmNTE0P+6OZA++4I9o8dozhXceofJoBBY4RQLyg1z6DltetiyvDLKeexPmpxF+0LDQzeZ7u6PQ7CSKx3WyoepXLAxn/aAcXcdOg+AfB1AiG6bAnngmO1B4Qv8YfKt8qnwXpvxEv2RzF+8Op9/TTm0wMlp8ztumgenbXGu6tJKk5sA0QxQuXM/t6ACFTh4TMxB4ZU1Oa6XKKFET74Ktn/ewgXPD+vW76b5BmRZhUijq34/TX6QESw5KeEJLSZ0R5QXBAV1dOHH453JEkXjBEz0Ix+ov0tx9ttdBZAYXXjRL+DpHGwq7VOXfPdfy94kN9OZHvXWUcguWah2GMhsaOibF+1BHzJOpL06LXoajDF+9W7AjPIed2MioSK5X2YaxBlwsXlArTg/stN7WkGsVHb4e/rXi6YSTSYVxZd8X69ZeAnMC6qF6uSe5SWsZohWUvP1OWXlyMX1uwlqDhdeQOffMQrqH4q/btirclAItH/JJo14rLS40LhuQ0QIv+FUOUOQMkQbv11i3SBN0BwUK6jxWrqmAgmsCRnc0Eatpdv2s5CH50ofL9wSJzBjn7ffrVHSyrZH0OYoehDPLgKwBlSAGY2Cj5l3UU+ds4JLI4hLxv+5a27MCgbFyRFoOY7Hy+uT7BZpYx5nRh3f69+nvGa1zCqhVJylsWwYUPZ4gUau6SVAXILIL/D19Izt1f6+gL+6xak2Za3VfsP2aPUu0+XCcq2JQPkdaACYWwjz937fvP6C6m3nnod8cwVtv/KuIxmHy5A/sh/h9s0AbS1r/e7qCxAWmR/MMgDm00GACKVeWQquixMClKp+Uo4aXHlUJafD1sdgswCo1jtXe2zNhE09yFmhbyjTq18wAfpv46RaDOvcCGoxXrNbmg3xmYapBvj3ghkOjW+c6uTR5KYORyOHz9zm8e/MZ7c9JhSP19U5ikgj4nAK/gBzzfO0o5DpeYP97ascdyMdlJ3FCVoUPm4qnaRFDl9uo/Ci0738jtK+jWeZl3rWmIDQvjE/Mpg2zaAV1jqk+e0H8v3kuVYab2TlG2BIOyFUMr9J7EeiLmYzDr37YUPuvPwY/ytqR9lcHlsKfkOcFEYUr753RWP4LQH1yeF0OHTxTVxQGKYh2XGfsV+TP+eJ1RmVwuwAGSACGEard9t9z5askYNIe6YblqEpuWJoNVXb+i6pg/YxPGivaYJFD244LQo/qGfxn0GzEZ9hWgw9DOmUxjBhhzqMG5+T/RiaTCE5SNLQl7RyRUZe8JozTKSAQxMoEzoX1MAphwdVPeN2hWJWiyLScm3k240s2RUg8cd0i4+RQ6G0F/ck/tmVgr/94rGLsDaepxEaYTbDP830Th9j2RrXWuAq8ntIQZ6EcyDZ0+6+4Zhh7QtNmH1H9hs9nxmSpej558qCJr3tXWa/1mFcvaykiWZ7viLxRkqxp2QM49dxzgQ4qbUOOCMnzMuYL/4fbSQWjcu0zMO4a9rJ8oX3oSUTBjBi0E1RD6G66zpZ8hiOJ2jZXTR1Puz4TBkOGLpxT0T254fWp2iP8a4vMSBzNpRnB1nQZS9Zoq6PfBGl71VJ1x6YTq1MOCVKyZcxkCWSDqd9+8DP+tBnjLsTSNqUMm4BmbzR/5TwlQrfP6vzEsJ0F3e/ehr/m9WZjhw2twvIHdC4XPvqyACGhLP7/Ztns6CUmqpocmtV7gaBfHGRuNSCYxyZXHbYZZTzGA2yAoRBcGd2RDCh78QNBMb73/c41Xz9cquIM808fs+Daxowo3bBu0iiyxs5l9tFrXYYVQ5Ey4V+X4FV/PQh/nJgMCfxl4od0bRRdtx3vncQad7bhoVABDEuCdP4ARAvpO/s1HZQ/Pvw1dt7PM9EXwsCw9DwPSo+omepXxCK+Wx2yNpvOvEkcCbMf3+UBFgFGkZqcKrzJEWdfPoDqFPzzOm11E4Y5DZ0KMIpP+BjKXzZGr/Mjo0NzhD2Gj3i9T9s0pB/85/peOTa1GIOLK7hjs70nBejQadBa6ZG9zlIsWQe9+KDK6Qd6qLRJSGGvgUkOvO+BAUwdxZJtyFQoUPUV+oxIALtGmVAuoO6QYx/+AlhL6MQ1ltpNubQb7Xm82Al85yMSMnaxhFGy9+Ifr9BoSva+AEezGCm4NKIpPzgD1uaJym60PyVjnZFR8Pm9FHKjYe/+IhjwOK/1/aaa0hh/XWjcSww5gDlt2rcvREQzUgu56c80JrwSPMDzsMsv5hrLOh0RmFqwqbKOq4F1BsdTTBYQpJV05iHFIzaorswUHWkKm5n1Ozq66bvZyc94kRTq2coWJw0U2e2HG09kp3z4IyioPZwSn4nWGDShqaM/rgZQo6xWlelx4OHJTZhrYN46hyLWC044O6RzEInBcHX6IiZLK6EFLLMOc3KQZefvN7mwmfo7d6O7VupxLMvdQGgQe4jAwmttR7xbdkD0TixftyDNQh8csVTPI3VFXRQDXQAiSzyl5/oMmiQTk2SD4bXyLk8GKnO+WmeWQcTTmUGr/c30wMQ6U7dzXRIcnyjgeyyvrlD8DHPKk7JnT8hYAqlljNlar9/Tre1YIem3/Q4miPEb+tE8qjnO58tebudDCrlKociNUuVk1+eWtHxpCuBW5yr8bCqa5PqB/V4idVA3Ce/Z8LfNLIbluQNV76egx6OQV4Xh+DNj9BcZo1yWwpb1JXhQ7ZopRu99BDmA+aZaApy+Rp6bBei1JI5RVPybFMPcFwOG7j1bOhdvoRmns/6i/YIRM4+Z8LrkeADy17tsHowf6OcUhvLcVXlsGox3eOZyTpFPhwcVnAjQsdqQNSRCoH7DCX/yqS52PzlJYiTRIsKvrDofxWcXwQAQYd4BsNskEhSxIGCAzwF+/y8Zw1YaZYjEzwQCI67Y23Q0BXmoW9bEBnvoocXrYm0Ioeka/kQTSblxhe0Wr7TyT7lc9ie9kInW0e5F/vSSaf4HmSMXOywdc+WLFVVVlJ6aB6V9vnZ7Tvo7Vk77a7XJ8XS564drnzcttB2saZ+0lr7pr4bUpbdbs2f11wVm+AN5kPRFP0n+sg+ZVCNXfBfI1s+/G8dbyosMks2gdgvh9ww8rsPq8rPZCEilFXQahdj94WqU9u/we1d1yi+ipIX8/XT05wq6DqL60mnFPkP5BU3PyZ71ByuBc9vRIlbyX8iCqsI5TGpallzlNMzCVdFmv83tP3qMZUoBkX7obn8dC0Pc+FPQpkiQSegyahdDed3o20/pFJ3L5QdbpEnZADoutK1olfXs86LMpvqwZZ9L2zsgg1s6EUypEumtK+DcQTpUfR/X7f5gb/5hlqtfiLST7EOGTQzwmI3EmJR4bFazX7YVuAtPf52pkmSnruQw2bFTjlDiI2VfqWp5RcIyu2p+5k0uBiVpoEYVSV929qcRFo1S9RL/wgm6ngQmeppypZLpWciPGl1oIFQlj2AY88I+30Xc1mMuQzmYWQsRS3X4c46omlLHrbe5EhCb2p7FNJU70bCV8YHsPoFuelRdl3eiyeQzLMzR6rOoinf1E6azLXfsYkd80Kw39vtX+Jdc8USF58hyjLT7dw8VnDHJ5+5GQ6rRZKw7haOuS9qd82Hx+mtKvuzkcP2u/GVOFRr65RJ0U1CeYfxNnuPvOsBqoplU+V83fTloQhQTEDAFwnScAPUl/wtPkW7h7SO7ABzoXK0ryIKTBjKqTLRImvFVfgqv+m7n8FyQ+kbHuPih8RlXZylQDASj+tIV1OjemZgkTVS7LJe9EK/YAGdCUmr6BJCCvjwkJ3T5V2blPuTlSmEL8ScHn6pkvvmDOhithY551Zr3VHgokEvjCajJkIsc3/RETASZdvaDewRtn0vl/eIKeJuv1ipebVPUNPZhyxZCgfrLjK8AUxkIt9rOpxlPkPgvbD9Oj2wC4bHxSpABbcr8Nhv/kRtHSUpJnqc2xtv4HMyfo3d2WtKYHMSJAySH72OkX7xqyxcQZTlVuOJV7Je8/eUdnxuyttXK7TReWdhn1NU9gBpgPZdlVQ338x64lTK5tbgcRk7MEkU59PxAFwAXSsE5a6WQe49ez9yFumk8LjQrxT8lbz+BheP3QWDdzbmsXZc+j+DYMztMjqgFWns0B3C5XEVw3RmLPrcxrRbCgW5H2RJCXtXwEWYuL6VsJt9zexutQbEVUeUmX4nRWdw4VEpbl3kBid+eRqpupbsLcfC1p5RLx03+ZMn3QGv5yYcl/HGdaOfczNcUDt77oBJ5LNGNoJ/7gXdLQtdh4r196rztkF2zX6uA2gh/JKGXYX1HcMG4iban0KqPHuIQc3AGXk/Qz9YUMY7XjO8FQEm3hay0F5Hj+FFNrjdfcIO56EagPU5KBbYRnfvQo+J9AKcIgUBpVeR6E7dTNmRU0vJFF2gvx8CX/qre/p2G/bppUflFB8OeFRmRS7f/VQ36TH7kdmPwvSLisMnpWs3ImFNEm09J+Wz07xw18gARhKxlyDjqKK8mj1KAxcF0lT6vC63PwXHcSWS3RLb7FLn+k/ytRM9fZRv9a1pC5DuomaQnBQOSbXauroGWhiRwhQSZwnkyzfiWC9UvbOaDrhxMaWDdwaVKyeTNcbelI/jeEVzo55RqZwayXyVjxFgUuJrNoi1kEfBf+/QI9kWtrAy5XSV8HH33Po13jSlhZ3GZu4P+5NnIizj+tsjrYM4+NmjxT1LgDqqlW/4hrc0DAcp+s0saZmxOfCCCsENrfzGVMgYPt36rEa9CmoGJfdgw5fWgx6evI8L/sNs8eHOZMLgu18O3A84jnphJ+25r8pOBQ/sR9olRI4n9dQd94HBNJXmc1pW9Ds5GNfbRa7mwJLluF5GZdzrNrtPAcpskGoyURO+TXg3JOTkfJ2ielHDq/pbWBhTfRS13RF4XZqYIePo4qb6glNtfIoduf/lyRM5YLQpez7vLjBkZ3EgReP2njYgA/ZT4ucC5NYyfq4fxp0eD+2Cu/NflbEo26qoVIjtBcK4qEn8lK8F/v7M7DZ/sYCw7l/6aYUW+R4g1pggoPZMm97M0l76J9m5T/Ub4LtDAsQHkwDHiC94J9rrk/Lmhw9Fl+MuBApeyenBtPAwpvBd/LevGXHpx+WZjiGD8yomzY4EHWaycSr60eRoC9VpjkI7Ql+TfxhJyBCjuqMW288mw/i64GyoeGhDX2chBEaZ+NN09muCvHtJAvIhAqPi2btOm/quYIBc0sZfPqafvf7wWFSCa+yZznARHmBsxM2ceUTtfkirHNi712GbJli/Thk0azvqXQo8Th5Quonzxs0FzPqO6zQVZ8kmAMR2Ocnkp7v11ruw2vLtnCtsgc9eNOTEaxBMQiSEnVWfvdr3uVZAZgp0BDgjJYxB4fI0R9tcpHxWQz9BENXyXmmvooUFERV5HBpeVwUUTwcB/tsuk67pSW0keNHJ/5uuF2fDXUKsFBMm9O82lsG+GxZj6vxr+KtAIM7+sFQS7/9qsMus8NMkYlbZqDLG1+I5+flEztVg1OZHR8XDvia4C0RpVrZ5fnZDHL62PpMRDEgwxkAQxs5m7jpOo+5yj+vPe2Y378bbll2bVOqj++Xy2/E+Bdz2fZVeSPrgQatgVb54pQp9f6iEHNkzfPMHxDiPdgdzbeYjYMM6G+s7Poskt/KmtcODvqh30gQSQPfqoroaqj2tw913Q7T7Xf2lyDg/uGDFgB0iaveY82cwKNLMpyMd8Fio70kvppCfB5d2ejSZHbQceRl1041hT4TaBIfiv4WaRFUkzx3Lzo65U9RZ+SRo1OjRnsc+/3idm5uaTUkatF8MdtPlzG3w4bxjlYSVHc0+mG6Yw2vPNyQKKTdkEbuJx3ahNnTA9J84392ey+uAsHi0t0QSki+XlqcW5WP7y2M7+Ev87/GMnA3oyEvmQaNOCYRTpc8NR6YuEQlxxEfrKsGfq0DxqcRr4iOjXjY7utf5lqZMg/qzojMov7JlGxt2oCJTjTJWKzfin9Gb1rZsdosUxsvBxcNIRKcFUEHjWv6wEBgBbMdUByC3Ig9PTsIlWGf9LTdEnWmCK7WNuPqs/vDuRUgOEiC21F5LnHeb6CyEzd1v9RXF3zru0eKIZVZjY02eTdgAdOkoeU4yCOv9SDXfdUe6mX9tsovC58Xdtme/ECpAFdWgbqZA9Cw5qHTqSwqiq7vsmWZ20+e5/3TYJEY+Xd32vQVPe41LQAB1CnRJOx4Acvw9DYHwc0ayKnH89xphdWGnM21Lu/d6+T42+CvFGsU8x0ebgbDD/+1QL6YlnLury4KF31H4u+3Rq00U7ZAO2qaAkuAgI0Z+31fmD9eB0AKe/fsw6ax3wDRtUGnrJvA9jx5hR3IwTM6NbdMNLC8sc9TlPgYnhNiqOhxPGwG1LwU83QMRI6jVN358XDlvZoZfnXPmn6B1zEf2K3EmizjRpZm9pOCmkLPm1fcEfbGyaf4ghjV1rgv+UOrLt4MuisG6rGJr/aNm6bsX7DdhHPD3/HAvtCGoOXLzuHYNLuAp7jBpUu4Gd5wH1sWEul4fUbMSMev46yc/Uu7NkSQaw3ID3XkzmgDS2xssdwEuxOty07FOhCfqbM9smnN0WPygF7/46patkQ0vVz4veFURcQcVFxFIVA82DGc5WM0I84S7EyqXLi5LO81B6pRVJG/46NGCilZ1/3QeUJ5Bp/dq0pSwFEyXW78jIH5td1/2D01XuzXF/RM5uDDQ7CzKaYSr43h845mKrYM36UbpHIs2JoMtMVHDY8Hf7TnFEYxQ6ozFWQ3Pic3rmy+SsGR5uGr5pTudunFhE8hwOR5tF/qazofnrHMrR+mEtv5P5gbkcX9JdMSQgFhDpPiMi+phq1oNKVH0EhI1GBrRoXE7/8Gzx/QXOuLWRq6Ap122AISK/lql6OvcDEjFZRU4Hih/L+y4HmV3ISGqFbpgX+VI81vGw49qNZLwN7PM4M/8z0Lz+K7w4BoxfUXRuM/x3BEocG188ugiyznPvjD+09Tq0pMBeMOzCTstB4HjM9CfPMUdSVKOYnzMu/5JRdRC2I6CPZU7rFzstDhSFwuOHzxR+ENtEeGTaEWIOjOxFCllIc8Xu0VEzDODFrgRiT7sFG3TWiVlp9I328fG/EAum3ubyYn6FSZsgPHsfw1fG+6tzlUtPLbnHgvz+avK6CvNY81/oMGAbNs0bzRaT2n5Xh7ZpOdwPCURiBIvlVPzrMuQLBWnCsUw4KB6lA0KAgxyKJkFhd4kXHfAo+ddU082BB22VcMbEYg9XyrEUWHj3AGgOInSlXdEQmz60binaBvarugs+84EyktNXeypZYS8UOcxbTzs4uKvXv1aJnGKLrn7lJJfq/ctTrOyJc3pV7DKmb/49lxnZetY7Ace7CThrfZFgUYmglDlWf2ipiQuCud6tzYZdtOvmTWn3tONggOjxkp2WliuvD9n3I81vF8x5nkAvh+YfQ2+hS+hxrjGPCM/scRMar/8zUewhKqy714mHSjfDgHnzb8/nlyb12F+hMIa9cAw0eSqw4mpEwtQV2CqSeFGu3DQeM2sXnp0XCuK0vEPmZdugPcXaQ5OOVeyHPw/PGKyTfBX7Hlkzf6SS/JkK3JVLT58HrbaWZI/mb6iUaNVwnLaHr/WpYi75bA0ocuPKrx2p3yHw/UMy9wpgFno/pcMONywDCMc07w9bV6MXhf9F1dm61sPbS4HD5uLBmEdKzoxYq4l7CUQZz3yfbxYUqy3Nl1E/ssb8EFE7q2CHqO6YBcwTFvlGdeP31JugIvP49vvckydRSh9TuKw1+GaK8FJ+MIPS+p5TderIl2Nm4JS2dFglU2IjZMBeOcJ6jNzZyyhUsSNHgXL3WA5nsOLK94a/+Gik4yY2YDqEaF4zkOFZqYLcSMPQfhnH/9sbtExFsF2wdEFtthY+2w15Gmf/UjosNoAt+pejZoEPe4tSam3K7GNAl5/Ch9ZfRKsTkXKzE/Z3NNzDl8jF37VvCs+9J1QJFQOntm+m4Rye/WjWl4T6BNyKOAnxWlfb89W/lnrmC5Bah0/qi/6yPQniKwfABvgaQOmv+uul2+Pa8ifQOIYaE3vS46VREJDGsJi8eWuBkScwGyR/QoFS2oGa6wg1EeuTEAiBJtNcaPf6ApUVpnCEls8RlXuxKyy8b69++z6fFPI4WjCJDYb2H741MhaFCGV+iKpzK/qk23x3SpHzLGz64aPuuoawl+NXMfIc3fC0CCf5L/uRyvZzOVJzGJ+vktF0X77YJlz7EJu7B9hsLmp9pj+c6NNwXk+Dix3LqKJC91eYWiwPrINCZDHb1nk9Oj/s1ZQEE+5a4tdfx6ahZ0Jg7o/mrSujYc/CHk1+PiKp7D0zXR1ON0baqSWS68bvlI9Ba/Uk/3bks51mIRSh+wJFfVOHJ9v4sFaYn2cmA/aTD8lwxaPkK5XnaMahdQ8AC1Xc4ZVW1fsvoc67fENt7NnUUwP64qsH4e3BU8GINNjvv/v+ffSyAYUNuWg95cqt6ZcTPPl6sb+vENgVNNid0uHxL/5be5FVONxUNZ1LQvD33X7UnzvHroTOkDHgogC0XcYO27/PEVJFLVJxek0iRr+Ig4TvNQ4apHwJ6OpG1p93M09oIRPrXTf1C5GlYiqXG9G8wqL9HZQB9/ymy/P9MEBjJydU3PnpCTfAlFvjtT0ZltIuQ8brLCNHUQw7f2mkSukD+yOwTdWtMeFptDb9VO5Rma2yX7JFQU0BLvnsNJ6RBBZ8u7AUaaKm0lLEK8ntmVhRzbAj+atlTfiJimzKp+ihA7INhw7+gahP8VugnwHQEM60hAjEmCjPH76PkpdUA546GYKv/lL5NRt71t9/7hxqnL9AInEtaCuVdzFy01j4fBTxmeWtR9TR9PHH2/6CckFLYvjWQtNAKK+PMX8WmnmqABKKn+lSwRTe4d51eRsTHQ5zXLoZxV9L8QscSsNpBIC2dYVgilrLcte0kdejWxzm+t2uqbNkpgcoESrErvwSs+P9OV3jXuyjKQTfmKPWc/rPtpaNZL8RYVkVqcoq1WyY/ENFbzNlnQqM0MSThRfYn8d+D/yHJf8r77uaHTeSdH+NYncf1AFvHuE9QRCEIV424L0hPPHrL4rnSGq1emb67kprZtlx2CSIKqAyv7SVqKI3mEiXNKBfhornkesdIJVjexm2qoavGPZ7UyIcGY+07XTIxtQm9+iV7lEBmk1aXl0cG4LrLK9IJw73Mvfq5RVctPeS8WcMjEh66MWPe4/icNFLCn76RMBllrG9zF4vFlRaL6OKYIw/rwPOkcbMCae9rH0XTEvyt8BdLy93V1wJXy3DrUwXevFSkXMBv0Er8EzqLguwjGKSCwfryunu1IbHhrVULyBnskl6hvJPQGO+JYaD31LOWKcLD/y+CGCLmP2lF0PnxmiqQniYjopPOAPjXvnTokKXY0nbGMqNfKSDo0Jo4Xa7JhDWzaMLSdHjPadjYIGoTxf8eNhZsmbFs7/K1MXWhnuHr0sPVYi7igyP+YkuYDCaLHq4pLOanrArTOBSovEic3FjBeaKtah5iQ/yvUxXcrraMerw1JXdhKy33w8uOgzJyJqyEjJVoOzw6MtjHKijaw9zZMXy1mL8WlG3fXsQNMe5r4AaFvL+4p4YETOydVHv3XMNAiAS7OYHcKV4Im5YKirFCbpz/CC9i7xIoBt2N4uco7grvvPx5JXjQLqN3Z/eBOZg2UvNmI+89/jKDYGDBha/ZZ2hb3GHz/vgEZvueLS9ih2k7kz6pcV5iEZo7w5P5MPg06sR3RAsQZgHHYxtoqN6XCl033DJI75xj213rzwlbTrw/Z+VZ2IXnDdZoBWv1rXFigNMGfAvRUTBzHmVoCNgap0P14J+Gv5K0L2VKmpo94qxsa9GpSvIFlPd3xMtav0UK6DLBCyItYgWZvDj3q2zOQUxiGulId2HIyV4X96OCgXPXvYqfuEnx14kHFWPaLmm9qTq7GVHc0FW4WkJZmd6WdmGiZiyXG9r05Gd2VbDbhwN6a8bqBmQ+mcGX8hHc49KjWab8kH6lHK6WQils5veOWahArqqmMfmOVQ2ZFm10IO1j5W7CLigVjSMKr0i3u4KGmLpNe3han15W5xQha+rFY+SHgSmz/fHjAyG5r7cHKcG02DcBQ0L4Add9wCuX+qVxB4Ttr9QedtNb8+ANnteVjUeqIvYTnF3dGGRWMJJnOx6iRqYX5UAf7YcxJxwLzoSkeN0YUAGvIaP10iPIXncZ4lxHkDQMwLzrPjEMH76MbMtbjzpKbu4RqqkpJPEg7TZDKy1c7xmyR+s4Yz6F49V7PmFTOwr85pe76hwTw7b1HFKwfDm0bflfdoVjdBjXs3Nhe6aLXTrAR4ywjET5PGQFhO9dth2OpXMwnatpcnhZV9ZJtl7WsKwQj3JZHYspAIzBLRQEyvtvK3o6JbacMNQZaodGdTQQpgGBLR3u0tU0Ba3nHC4AOX0KBsv5dG+Mvx9dBmh2g6ZK5/6tb2D5CWsF8Wo2O9twl6yNOuZgHH9FfOM97M2SkgZt1VKt5fsxE3grLYfkdw0PO6gDidCMyt7LXDOWAHDKDAoaomS3T721EGrYtQNrFJY4hFFize5jiEEmWzkUCKYF1lqDAnMRZS4PlwyKkD7cqe9pom4o78fTr+nrw+fAgsQbtgPWArYBbvZBEjWDtLg75NvzDcDlhtD2zKHGhq/1e6MBEtAJlIMSAtAUAMvtrBA+24O1QZSSEAqITNcHQXMQLeEwZ624kq/90+P8FE3telRbumQ+nz1LDd61Dt7nMulDnqet62DZ/JGt1wkIu5itp1eNxg9p1SwO6No/N6Q8EZcYozWBrBYc1lp7vBqgBEFc8RiR1Ghi5GJGB3WTRMV5oxslDYcU5Dxh0k0pYlCYAYOsqyBFQWIJzuV59pVcJan180RkxOWYlkh0/Kh7s8zPoQ4CAMJDUevzrMNGZq1yrK+CNBwhiplmaY8eEi0FFnu9M13Y5cZMgJxb9nL0RLGIdVD9o3jNC332mNh0UO7p4RvX+njjoxccuNkhWG547Z2a1Juk+aD6Lt8Lwy7RNpkRh4uCIITnJZuH3RKl81HSIOIlq98coW0UpDOWEt77aG4yqIVnXLPJj7hG6LpyIIAIlGp7fhcQgU2OaPt7U7DuFCyIPRsMyeSQM1T85CzvmiIIrqE+VXhOU4nQ3W4Q0ZsU0+33sZ9oNQdHnGjf7dsTgm60tsOIoi8fUb3qZa4F6vFJt3RyuDHOsgLsDCNhvQZwnO1IJ3Wk3yv8AvB+CNAeiME+eXpISnPuT3vR5Tw8aoH1/Ww51I63Qxaf/tGoWoNDKswTxN6IuRp03EaRJDh+/F9M6DrXLV6ezc2LqP6sHGOVNxqaa25D8ooWkfMXiLBMIhC72scAz3livajslgWPEsqbotIWGe43G2Wdgf+BFzbBairYe1MyCq+qufk1PK6z8fIpmvxKHnc6aCAx8UY/dWnpLdpILhUztBQhvTLeLo7761z2Bjy71N03v9NUYrdZOkIapPQiOXN4+6g9l6PP2ijFqSy6XMXLEZwVZexXTvzvvJIHTJNTtnUEW3Li7XxJ28jknkTV7qzWZvjGWZ/LycnAtsYHc1pvFnoOPxk21rxkrbKxhguxa1nvPsIfa4spustCd8OnvZBV/ORDCAbMoXlNoPdqlm+EZNxWUSkYk46lfdWGXtzaFvL6QJNi6gbd3/7qVjDgXi8vNqStY+yileLt1iOqGkuA91vM2EZdz6NxcZl1nI3gCkg0rnzrzXPnKzhuFbhCD2aV5cF6e7Rb2kCZxUr7zThxWnV1drW/c7VQ4C9eptaSHpGPcwuSsAZ9t7Le5TSvH7qG4MUnguBv1oWKzngucOjtLCIRQXlDbtzZwwRtQtWP6wcJEt2BhlSFDXA3rnss2pbaSpYG0p7ph8vQM8/HDZsWtZeWybYm5SAj5i6UcpbDT+KcACro7EUrJ/weC8k9zrchD4KyRL61znyBQBtIfh1WWpISZ1xRHKqIopqViSAEUGu88swpwcmjxUpDd7QLpWEMDfxItz4gL8+Q7w9ZA2z3MIDk1y22g6gKG69hh2R7m+NwQrys3pM4zUY0hoNmfeqf747s/FD51UhUKRHQfAzgLfes2FQD0MkX6AC2B2ixce0rwQlzplTQ7CTV0VkP8HP0+fZyQlCXUdW4sQTAiZQtEdEMxAPVXbjzHnZyk9iIPyduTOeigmnlmcEJ2Udkt5pM7rW+96dPug9eeqXC6NAhqC2/JoHLSFHlguN5SUM94Cc34s/z310g+r8zZXbbRDvCyGXskSP8KSBeSFsJ1eqOu+jYTgsO0RycQ9+LOLaaE/PyIyKQL8meRytkPyWdU5SFmp4b8EZjtfFLJAquuot55uPlkAd1LU4R6oYFIhmA57vymzzdg9G1bf89eaa2oC2fH57WdbJaiAZQUmmkE/QQFEVpBSzpD9z7hloMZfcg6QXyzgZxXfterl7kyLUx1I7IPHfDle/34yo7WzLfvfFlPKrSZHLciJvizFKwEzJs0O/9qT+knPXLgHAu2gvnfMbK/YDDRo1H/XQywpFpWN9YK8eHXIeyGbQp9V5adzGBJgyusYQRwIbOrXClbKiUWy2QsBMtwT1XtAksJ3HA0ud+rI0fYaUCAWUXMW7Y11a3vv+lDwAM5TcFTylKt5tqF9xKXgKuPls72a83e8JR0Xy3RAxp5SCvGmW+l4xEOZippJo6lNmUtwGLoo1hmBt1pNZVx7UjMhXmnoopKDk+cZYnp2913m15qwydIS0L1d4IsR+rJ91bpo9dloM09EtJ8J7rq2IRzyBOcuMrbKofzrtQxNkiY8Tl93nwAHpebfnwvXCGBfWzAvhrVe00wU7rzFWh1Cj/qXSJfi4mvDBmYFzKGqMb3OsP2BFchzzKj104iJdcjarDmtAV3TOA65kbh1SyCrCCM1xgWMghuxVWwMtMU3iPj5H7i3JgqM7CzK/5/wdA7GnyDUucLBrwWMYZner0KmQXnTR8Q85qVkHow0V28ICUhAOfZhou4F6RLi3t0RjUWbFsSRCboJTSfecnyTtBt8VJSn0G8I/fQFfbHx8ZINefsidEb7nCc+hIsjF88LsKTXU3cAx0onDZ1eOoqrCPHdpDlnX2iRF8UV97ytEyVK2FfXpLRzOezfUcb1uXaoeDkuaN/5dXpkZOCEUR9bhCHpXym3cgjp0bwpdPi+ZZ+Xs1CQsGZnA7RfhmUBxzsX7t4JlZSF0JTSZcHGYarA4+hkbKepzedTC1DiGkvvWTuWe9pKVgdyC+x3HA22xhEfOW1jLYS6Ev5hjEvZLMRWdkqSaMw3hXdgcVPIKUWlNPcXqWunF1okdO0dI5UYUeYEuD2ycVCuQNlilNIiX8MWEZqc4uZWDv5LThHqzBOvGCMDLO+WSU5i3/mGF8w/Ys89zjbdqPP+A5OXl+++0O8y7LUDZuy1gw2dbtmc+2p5+IPtue6ok+7MtwP7t3fbASIpjGP6nX9bootuGcJoL81QY8BIa8V7bi9Vy3E/oyduTX9BwOrDd/BMopQBPgoNDazrO6f7VIVT4CeXaXUr7Np3BhoTQ568ITH+hv36RHz28Pn7+GUGhjwNbmczFx0ES+oJhH0eLtMyLz2sTn03D6eN7/uvF3sumvG8BJCp3Lm2aX+7o/RmByuSjjW//Oz37xoEgfAPF4r/rztj8jNGfowqb03q+z/s4MM2n2vw4MJ2WF3xMwjmc5n48P7NbUc6pPYQx+GEbw+E8VszteW0ePj9O89jXKdc3/fjuA0VRms6yX3/xPkd8Uo/Nyqb55cyu70DvWd/NdnmAvmHyT2IGBf2e+jAEf0HwPzAAgf5IfRT5gv9F9MehH6d/2YY5oM77f2Ya0hjcH3QeCX/5kpV7el6OBXQp47DRwyhtrv1UzmXfnb9H/Tz37VcnME2Zgx/mfvge27IswknoD2yD0W/4dp4KvV/fYd7nbb/h8xPKfHw9g12wLTlXuqx52yBNynsggxfbKQQnPz8Z4I1XOOZx/s/Nnkak4ATOZxXPN85PpHC+mTsjudqGRaBF6TSC5d4wZEGTDClcfEdl6p4Mi2JbrFXbeV8qNtfYdtnYN8GCGpEV7LIunb7HNBcK9pt3u9eW5vSKINidfZ40KKUqloV14xyH4DxyLAt3727OcD91YPuwLFXPpxcJNHG3ZFM3v2JQ4EDh9HF0HchvZ+K+a8a9xdOAPMOD96GCsoTb7//ViYrfa8/iApThFtbf+NfWc8otv6Wn4pLlTd0uFdjHzePGss40kmE27nV4yy2YKPHmuKKCFMjG6EqI6bmQaJUylJAcdeLTiE3YnZw6GM+4U8kjWen4LFluIvecOFTKC1MH7sJLmC7kDPErM06s/mAye1hewVrpVii/E8rmYsYhxm4bKIw6jTrGjPeHqqn8Nd2Yo9jz1kI6UhF43sXlJWWyGVTkqGpDHOwhYdM+AmCzKYIJXR9Ra9fluuvxgSj4lHQGZneBpy8DSefCVUHB80usFLZ+ebvfq0mpapeUD/5QwKQNnAKCo+4NsVV/Uq579yDk3hj1O3plq+r6Xm/NLq3xkvN3vlcrfi4mkuyCZByiNYlp9zHXlkVvMCzfiSdHB2C+f9PMkCafw1OVoGSTG3nz5QB9UvfFHPMM9Dmk5W0mj1qr8sQJKPNq6mO3EH1qJ8Z4EYEjKa9hOGcxBzJb8V6cVjB7r+JhX8ZbcRoqeAazFrlPgQLLgtnwlq9f66xfN84wt9JMPU5+1oQ9sz0WZpSdwnOJH8KJYpD0Kuha7wU5D/qFjWcyHTbNMNsh5K2MujYIwiVp8xw8LJkv1Et+SJJWq80lm6SXr2nq1ef5sdX4YnEuyA1RvEDX9+ZzkTuw2D+A7iPzpsHj7Jfv3KKAzR1NuCK9d0BZYj6Wm7k+79Gq8HlLqtMVNKBbi6LR3Exc/KZ7jAH7ryMgMMChKOv788LUfeXwkfGrCMbtaDcgAUNWpFsmZYWaURbL8Tmcjm5VYI5icIGVbXK8+p7rTijliu+qin71YpSSUYHc4fdtck3pk7HO+jtLPTcdZgjjnfw51vfm1KVpZnMAP+f5OGg+JCdnTnO7rh9UwtZpZV3FKUnua2q+SPpYoOtyoxzAJwa+XEcwYSFaugiKG/po0ObdPJ5pgxwyIZ/vp8gF64ah4cEqKi3I3HYTWsckoFijz4gL2IH3zjAHmtyqU5M8byA5PoOEsBZyeelco0gVzYgc6WolLGYx15wCLZzeVvlEs25yFDJxTw5WbPsXZ/NpklkmlNDEKtAsfhiuhO6P7WgnEc8dIGwAOVz4uOQguEMIF5UOT1ZsEsiKz6jzNkG+B0cOk4H6CBGBn1yBQ4DxFwzid5Q+ZJgQubtSWBuovQYpMFaOtIxEqsuKwmriXF0YAyNAqxqy7Zxliw2+4xpGzn0uaRwBJseAdI84/UIu8nV5XJFKyBP2MEL5Qjb0/SrZUsDD93trALwN/eMohvfKQ112ns/4GKAcx0cWmDTNjdrJMBkDpx5v6uQXSr2C6YWDNLQlmuk2o0CLPArKCgcbCkTj4NkJRVZ0ocUgxbBFOCyTnohAWW+pFPZRFBPk1FpT76UAiSrU3ZWI75esq8CEy9NJlIR8VXXk2wbcyV0AofDsiNCUckerXbCTIsFNKrKKXIEgj0T6ymzuvSeM2iVAT6rtIzVrEkG2Eil4VM9u/HSzEfOVp5HTvddv2rjRKOHav4d49OQnD17O0ZIgUVpT/GuOrOM+WX2DSeNKQ1fjMUku8xBf2mk5O11G48EZskI6yHwOEZhr2AfUUE+0GmbGoNhOL6DdRyh8gwUWt2Q/KIjZoEIQe0Pxei35GBqlwNuyKbNTlGwNE9Bq3hyBKzvuHg8pSF0e3X7EwaPP6MP3sRk/+CcaaOMtxVpX5+lZXh5EAMJzZsXcomJebp8lzjovXXd4I4rZxG2Dg2x9XZem1WjLei4gZfmYdbhjVA2QwbRZhT0Enb1mIagub0HsiEMXsTPNO7l6FAlWpBAJhfH9MlidrgFF7+VAFUglX/RKmbLao24MeSnGtOklwj7m5zN2XunI86XeglibYGbFRZHARIa0iY0BahbGBLpJbV1XNQS2NeysfNSZ52vWq52EKQ0EGxFkQkNBmfbDr0NFp8dT6tjrgyTkZLDcu5rsIIwIj4srYNiD6EjZ1bm1y+xLN659gjSsazXd+wlQTcCUl6QdLL/CAur5XfdsaeQ9qbAFQKR8YWnUx90enQxAyjCZY607jH0M9wvIClYlUWXKXaJtNEB4inTS4DoNWUsd1k1/bwVAcoBOweqCyYEHQaW1r42Q2CPby448SIweIsXoo3g6Q2ktB/LIozm5z8hpXSKaz6XjDFX18F2GTV+dKlzdsmgpOUrMLqnxUvSCclJn7IW0zkuQPPO9TgjzXu0o7VN4zzEM1dWVf+9qFvO05ABVwMt5txOvDD9M/AbcZDqS0CONpgNkkaWCcahnxOzNQKVAo/enas7H132KMsUTmKxn6aOYGqw06eieWcF7D2qXxFnCrJgxIq+QuMqvx9UtmmdmFFcwB6IklOJdigJb6GklrncavYjN6HfDoTgqS1fBbFdknaEBSr6XpjGS6i0XI3CAOPMehOoFpJEqHRax5x2hbiRBRxF+keRMY5bHEjj7IbQkOugYkHUwKpJWK0VBQq+JXzCoYVUnMJkKaIk+EU7KzwAlOfUt64Vdv6cWkALnYXMGfxMpuUl8HfRCkMgWyHuE7WnFHtpkwCBVvAAA3Z+3YsZqbzeVsVcbqTJr36PK+KY30RBaR2FNlqEvDiyQHuRYL5AjZVc8m5Idv8/O5EgQEPC7y3jH6/RkUzi3H3olnFpXDiT7UrdrU7LMtRFdm1iJV4yZO2cvXrxrzCB5fKq3kcFMfpu4q+tfn1YRUlOOzOTDJ68ZcUEeMIXbT8/Un7sAhxC3Jzhsz9isPLEyEV/GCIssamnp8wrriPl89Y32SA65OCqCZ2I7AjV5VGuaDGJfnw1nz5wsH+E+hgKSquZL75hInk/PpR5vMC4btx7WNLtZYnWg1wqbn17d2WwPSE93jQcNwxMWejs5XiA9b2FJYHMq8bpES3Tlun7BHNjpSJ+p3xmZq0L0SPhIUmqITYLXNadU57hku80Xr52vDs1JkYoFj7fo2Mw9bzfzuHqna8a7XjALiwjX9GSX07gKVUQa+C2WX/N9bCFnSzbzlbExsMJJ3Lao7JWtr90XcFkDAOQUsyd0WQcTeSV9QKze/HBtd1gEUC2c5OHTeeGnByPIEUbJrkEVotSOr72+VkWXB653HySBx2LKvxPnJfutrofnpCLGw7m5qjeHNulZBnBrSkuCmlszGEBhKxs+QFYtuo25r4yTCqcilDotrDSJjQ9ctrXh5SuvojGfct1EbmXCKMx1/kX3rCO8G/cXmMK0tZsrObsUGxOlPZ+3WfP282rcuyTUdWBivj6PELA2MLcxryj2qkWg3EbMZcoBAxRS7r0TOXaaOjq5zCLsh8tuP67gJgdt9yvs6lzAY3SiizJbWGf7TQ0wP5bZi+XJrNHoRv+oR2Jm5h0TuUDzA2ZXfTfeNnoq85BwTT5ftKOsazcB6fxriBWXALZvl6ypLEwvc2xQ68smbiqo7Df0WR44qyr8bWEwUM3PWcblg4r2GfaJ7GbDz7jhMPxqZAat1NDlwSu3pnSwe9ljqjIWQu7LpaLy/VS8wN1VtWjFQl4X0GTLRuzIrOryB3ujDPbBKMTz+XRcGbdykyHG9vx8Wut4rh2z6s1n0Hbu2ivyvolufPrR13ENGW8DRnAYbQCgpfllH0Kgi1l0A/SFpNOIZUGVt4iRpgIt6nIL58vNbg0MLOAK+0Ldxpm657G54kz8hLJWwHLTDJkZ9qi6izvhjEQCil3EEK3R22BcusjfMskrm1DDpI07P0tl42kY48dkZ2asH/beGTf7UqCmVm88nbIBy34WLZuwRmwYWn421XvmdBh5hZi1RhnqFsFeCYPx3n6LnfGGFngFYjVuLmtssEskrXNTJXIHeTU5/2TNQ+Ifu3a/iUsdXwhTugEYMkBdj5LmKUvPyMZDPYdtoQ7W1sQKPba7ZL5sbHXqnKZLcVXvecqAxO/Iab5ybfvdZDGWgF3KaW90K8nB05B3QSw7bDBPw4NzMX8wnVCXTluOpYxDW4caavm8bes0PTo+ZuBwRDTLgIvrjOghSfdQ5eS9r+R3WXb2VsBFO1W6rSVAqpqFw+dLuxljbxnVQm0bxZRDJAWxuhQRGLa0ETk7FflitGqRrrZWG4VVmhEmpmLgOfcwm53D0qqW2k41K9pt4gMR4rBx4azp2TPX46FUz1MiTmS/WhA05u/f87m5aKKqoUFBS5B8aqWOVTzXJ+GDtcloUF01pGPMXTU+FxjQJwjeGWq/U0+PsAcmCDMNLbzNFktYxcZUiM722hSkLZ4/Y1fmVNCgM8Q7Injao1fUySwSgsy5U1Zfopo91oDkuQm7qdHrKBxqWBjLC3t+YR6x63OKrseXGrn4N0bvrwJz+zZDItxOjl8xgP4ol0C4qoaGabxTt7bjmjcN5x6KApJRf03CkP5+vvDX3ODXGUPkL8wYwv84Y/hVJvYPqcHf5/g+E65TESb99ks28TMnGJ/ES8e/mSz8Jv+HshQvwh99vdOV7Z6P4VB8yeMB+bKW6fbvTTnNfwJnUPgPqdxfyf01Z8gv6HdS6Sj6hfzLeIN8hzdEc16ZTcrToSNy8JH+cp5y6ecyA4NK5ziZ0vEc/i+nnpf+6uzvdDD34OaHoQHt475ty3k+efujzU/Kj2U6/a3zv8HSyZT598n9f4iPtkwS0Jwd06k8wujdFUDW0Jfd/CY6zv6EA0ENl7mfPpPF3+SOsT9Hjn/GKOz3cAEZ7W/BAmPfyfsT0F+FE/RHcAJDACjMyec3t6B/wLfvw8RwOe7NUDBv86Mtw+5EExSFcZ12fxNX/2w4IXDyyy+M+RUq+Heggv6XQgX7IajAACr57frm9RLH6fQrTqLxW07/Q/6f/Bj6bkp/vIt/NjDgEPQFw38PBor4QTBgfxUYiO/pjW9IP/ZLlwBL/yZeP85Fn/dd2Og9sNpvglXpPL8+KQZo+b25269naNMkT+3PK/zWofDbUTZeTgOWfHbwN+k/9csYp39nfPDnbPQcjnk6/50Tfxk3uLO/y88xbcK5XNPf3cf3+PLZ9ArA9hUOUOQbXwNHvmHux6A+2/3GX2Ycw9dXp32i+G9fCaG/uRJJwN8A5qPP3+Dz6zj/E4ii/r56+QpZxHPp509x+/lD/M5YH4KxYf/tx2/1BDj5P94L9i//8jaBUw3s35ACHwm6MeDJHqgDC6x9pY4+rvTDiu6/d2D/2r/9PaiY5+HfAOD7N1ZPSx+Hc/q/dVTzGHZTGL9rDU7cgbf51ItgoB6jf21M/rP3+YOU+SezSiTyN3TR1waJ+q/0TsgfcmRJIMRc32Xl2E4foADvX6PlRx1T0Pw3WHwd+/yGlP+jLgsMwb+HB0Fjv4a6/wghf5nLQv4DA/PJDZxlplcXA2L9KBbs83wAB7ZvZv48AcrK5v/DY/1Df2+llZRT/SYX2J0cOn2HFKQsfrCHd4XT6ZScTAOMgSGo/c944f+a9OnU/QtQyFHTgxntD8n5XvNtLIHdOCE0F//2z4ft78VmKPSD7jgM/2Xg/oHqybObcph+oGbya7oRP/0hv5bgKZVgP32nSI9CIpQg/hxKw9/6u/T3ImD6O1Sm/ioi/9LxV0TW0zB5+0+X0wn8n4N2+pfvn/cC/xkMwb+JQunvJKG/C3v0L2PID2Sg/yzUZ1mGxPH3UJ8QEYH/Saj/NvRC/9szhNQfM8li3zT99j8Q9sRfAXsI+oIg0G+vb5X/jybmvg3X/zwO4f9sUvAzCkNf4G+cyO8mvf5rRYH4vy0KPyM0/YWiv2EL9l+I/xqyU/pieqvQbAO3H/hLl3/+nhH4gwuLgMgPLNYWjsn0mef4D/vDn4mfaYl+nl7TnLb/m2O9P/D/Oyj5O3Na2Cmp0FePFhG/hwfxmTv8K1LV59exB7H2b3lIMM9rnIIIzvh/ ================================================ FILE: Documentation/etcd-internals/diagrams/write_workflow_leader.drawio ================================================ 7PzXsqzMli4IPk2adV/kNiCQl4hAq0DDzTG01gTq6Qufa/1b/lUnq85O666yWmJO8HB33Mf4hnbiP15sfwpLPFXamOXdfyBQdv7Hi/sPBHnhEPr8Ai3XrxYEeyG/Wsqlzn61wX9rsOs7/90I/W791lm+/kPHbRy7rZ7+sTEdhyFPt39oi5dlPP6xWzF2//jUKS7zf2mw07j711a/zrbqVyuJQX9rF/O6rP54Mgz9/qSP/+j8u2Gt4mw8/q7p9f6PF7uM4/brqj/ZvAPU+4Muv8bx/zuf/nVhSz5s/5UBEKIPrj76BnNm1UHDgjBg/4n9nmaPu+/vHf9e7Xb9QYLxu3X1kLN/pTD0Hy+mGIeNHbtx+enzev7y4LFMucRZnf/ts2EcctC97rq/686ROA6/nvZ1W8Y2/6fOWbxWefb7QXu+bPXDDzVO8s4c13qrx+H5LBm3bez/rgPd1SX4YBunpzX+fZc+a8mfuZlq67vnHv699t9Ig5E/7n/vFzwyXqdfGy3qE6yDmcYazPLen8nW35M87JzAgP4sAfT/Eh8r+pcs+R/j8D/qYd3iIc3B3P/KpT8o/iw7P/+u6TfXhHzs8225ni6/P6V+A+i3CP3n6w+IHX8DJEL8bqv+Dox/bYx/C0H517n/hpPn4jdU/hw2AYuyyPCZyqE8s7quE2Qb/vP/Rc2/EzX/Zrz8J4H/E2IIEv8L9i+YIaA/wcxfG//tmEH+55j5gzrp9UAn+2HAUdVbbk9xCtqPh2b/yJRk/D4dMzX5a0OctuUCWo1fAPyD8D+Q+UOF/wmKHnwVBQlB0L+JCThK/gXB/pEPMPyvkotSf8KF13+b5L7+X8n9v5u+/6uz8f87fY/9C2r85ZHLp4mu8jh7fqtj+fz8//i0+v/9F0A9+97+kSP/wq1/ZmpfZxkYziz5Wt9x8lde/bDmZ3sY8x8YB+b6bg9K7vxPmE38m2QZhZG/EK9/4ApM4X8V77/jC/yn0vzfxRb8fy7Mf0/0f4b5nwrjX91U6L/Cp1/C90/S/mJIjv8zmSnTCfnLXufH/+jqdft3cOZF4H/5I6L4gzPIn9g6GIX/QmD/yhr4j8Z/O2+o/7q5y+ItXrdxyf/n9u5fbNbrRVFF8S8GDv8nrvyhlv8bhAMjyb9A/ygcf0J/5M+IT/6VVf928pP/dfLX/U8MyPz8pv8Qkj81DP8XjdOfeBoJRkB/7pb8kzBBP3/+hHe/l/2Dnv940b9uEX4aHkXM1h5jWAekCOVIP390263ebvlcaeAHJ7F0+PxmN1/Bc9CBDRjJD7Tning/P4yTFjzlQBMwona798ezUOT7ygqk8rDzJZJONn0l+8N8Wrsca8lmO9uuO9t6f6COZ9523dbuOKKKB0Wn5VtO+1HcUXq/7cF+Ok1SLfN19bFY18VZn1jqyjsHy52cVhr78POR1XK9COnZz/At1mG7HnlAGBKj7nsYHuYzBX+eiub0WB4R9x38NFXk52394982kzGn9T9s9KLZLxMc3HWMrGSVVk7fqCge8qE3dUFHPrvUbaEQNH2w1+1/rWglecv1eAmpkINWpRhVy3emNNJUQ2Iy8LOWGrC3um200FIvlYkoDVyRfS2enVf2JZSVod7Pqq73qhMbxO30sjJqSBf29L2ivVE/sfgYTgYyvkYao8xxuM9uDfhG6cUJZUXmzPyg7+os+w8yENKb4zxM/OZ0sXlPR1nu8Ju5BXQ9FwBsJkfQ9zAm5D4Mper5XMS/A1JIxcN5c5Q+EVT5NqUX8QxlhLgPastxmlVqWo8Qb+6WnjkZOAcEf3kWYsvBKpnnEOLiqC2q8zKZpjEBkRe7/ix6yTncKDfcVq0EMUTZMiV7llJeuLWfD3XAsOjgM0tFDwH4QzFiipinWRag7BA78QjE6DWTztdYygLMOeW1tRF3qzRl5kakYRrqMnzxMbczbdH5x8ox4h7HW5Gy2jNhelbY1yjGp3m29cWqmJGGt8ecMGVARs+vij6wnmuvfVPNg9WMozZynxXnFrc3ZkTjgrRzeKux+/2g+JkwrKhWHd9iGY1fJt2IfDoUzeinmPsUpNkhCJvl3Tz5aLbp5CWGgqC0cqcXq3AFiiKbAcctvcJVX1dHLETyI1U9O/NZyPMvfeYH0A0Lf5181r4C10oipnSVt4mM/g0VmRF+LWOfnWSXuLIn5NUEA6j+Q1Kv0sg8zFJ9WoOD645wFHAoKcbxeTDp7Cy20EGTwJidnBr0RpEdGb6rtEPdIvL1Mk/p4TQV6koaG32KQ0z3wPe89UV6PATIN+5++iLF15s44Z9lsl0dEKnKBCdDzocK07iW7M8j732Pn1+1YRRbBM/bdt8UFxOru+Wl3bYhmTFt3nxMfs0yZ8+Ni6DuL2R+LdIFfKJh3Vzmhx78R+Xpp2FMJmU7jXvOO+QWcfH5+YhctB/oK74ZSabeIntY7941cChVKG4vgRngn//6/cqs5tEks6U/E22PAWGUmC1r10wSmTcSYqGaHf/QX2MvSTDCHW2Zy5SPJSYxnY7E9EntQHePgCLo7/rCFb6JlA83TSauBku/2FnCsTcORgKhuPWyfa4R3HsJty9KNgFkJaDl7VihwIcTly6ED+gBz2yFQYDxOgpx54u6RRjnWUeqPgdfAXuGMIyYKAWBNPr+guXMNT0YBTt4NS1k2yXDVAfsYApKbGMpKCwug3mB2GHUheii+Q1NpHmXGXNrsagTHeWYgi1EHOw4vQbwNo3hXU3l+VwOxdOfDlBAOZZLPguQEK11C1REQdf7hzqlTsomBVhMaMo32ai+IMGIMonqBnMA3JbJtzOSaKhKSR9SMEeCwSLh8yCd+pFJ9OnT5GJUkntLApTIeBOr3o6njl4MTfK0zG4mZcTVtElga/AgDhH0gjeXh9acvXtFRx+KRJZQFQ2xA0Fe8PwqbJYBZJWHDOhJuQ9zoyUQ5KiRinuphcWtlo0YV5kn7jCCDR3sotVwGzgxlszc6sPfZ7cgJuJbkru25HM762fsUGHZKcjUwlXw6JC/lMdyDqr4Sid3KirhJsotRmC2Y0KoI+dXM220RjKDWkFngJDYAb8Z7CMGUYVvGhkbQIenu1lzKbQIkX8Ua2HnL6LXDECr7XDfbD2wTjrlH0D04bzTKBwL6g4CdMNubn5FymLlaO+pHLWJ3xCPHu7x9I56VUNf3lhk7r59h+H2lxdq49YBR8V+md+uV6jPZ/72QIVtKjzQsgLIYNiMxNxvlTGL+BOa335+OmCQzg+G4RC7TxJfACtcooOgjnZ36EQg2hNZIY2oq420Fq1PWjShV0vejQJu39s8p+6VLxxXq70OBtOb5L2QyECmvEu1Ceq+tAF0k9x7nqy9mV6zizpsCz9QPle/vtc8etvIW8SVlw/WG7SxpFLLI3WMGRK4mE0fz5Gz83F5+PjWvTeKhvhAiJ7K7kNh68OyjxnSMd6nGyKAT+WNSpeg3Ay3w++XHwzD3FMIbACLEwGRCt7fTg4de3ELACnNoO+9HVAmnBz9WQLT1HhTSI5A2a8I4UjCzSNznYqevD+WCswISbCATtHuCWDFOJm3gbJA/Igcl534EJ+EPEmrC/84Q3krRuLCvUri3JDHuiQUVwo3x+pqHFBAm5tuE+9eXfWkmGTGkLVYzftRvcobeiG9e70F35B/bJcpAms+5vBZouhLlXcOyJSZcpTgAlXAieVw4leB3QZmAS+ZSoTXnSfrvT6fChXtknNCn91E5kCjj49qLpfLWZNC8t90MTLUXa0dWhtU4hSfiAIm1SMwBjcaekkIE+J38QpNr+rmQqvMxxVnpIyUfL2q0C+17rjpUC+d75ZgmG7JlRmqiTa7IdriFb2Iwnqm07LmRy4W4ACxhhPFss4Ckqswj84OQloETiUJpgtiodDf8Bu55/3uidekokDWwa4ISm4kCYn9Lr3gCiiAlYqWHtDyNSOsUD7xSfboW8aPh/HMP0AK3NBmNc7iSbHLAhXMghPIEYlngp55w9zKqsGPNWO+AEDObFUb2vqnIS2j3AmN0QY+WaeW2iVT/Lmrz/rR1K8Lvwkfcj/XF2jvHSvW7MSczV1dAQIC7ni0f1+PJ5vDpR2qzfvRumIk2Hrb713N0GbHeza+41eKGidrf/30VOhJ8Llc7RONXoM+83YvMOdPFZNriWxEGBBmgetICJOYPfuGOp9vOIbYM8Nge0M3aUbrjL+0BeaZ10fJZxNWEWO+xk4Js1us7gbn6NROnjiAJ3vDoBHbnDvW3lhRvONzid9ILhuXOtCJuD2eS7tYMCZq1ggrit19U3mi9gbdZr8dbGYEpKeGzoemaYbfo53d1wDsOJpFNivjl558E5Mdxi/qwu5ABHRbAZSaEj4icZjl5JQaOKcqbi1vac0MR8CbQyBP3UORhtmRvlbRjZ0ty7hN/3HNOM+PtveXh1tqtet12d9NQmiYlYrX5iw95B7ZYVwFkwIrnKV9/xL9ug8U5wseqwGAPGI2Q/o+GciVjRG++1vo2d70faNgSBnP7oU9HsxbTFBS9DSy4oV+uc7WbKqhjDzfmYQ3h6Zk4ODPI8ejbad5lREtdC1P9rfYJvyPBtya+iNAndVNGlDY0oFN0Kflvc44d9rN348iFAYlbhSBSW9MtJXpCqSr6oxZbLvEawz4BbNDoKv+544dzbkeUeNtxfIE9xRSbSWVebY2xT+fp7Ek8Jo8F8Y3c75jwNrIOJayIRlTSU7gWZUi6YINvnN2AcBEH1NHZfrGw0H8Pe3QBIuclDNoUNPVG6AovRd9xG1xWnKEBqnI6B9fZLRO1cawXfCN3k6UZyMliOhTDrz0OKi1LmPcM7jyq9x123qZAnRTjFZ6BNuWXnTNB1XrEp3kVj/4Qy4BT9RNnNhPUwXHl0ZnoBI+mv6LivYT9vHMYcNz2rEoZmqFRkktpIecZHW1izr1iMrSUr3LQKwlmRvX6gKra1r+k77LtoJWW9RSV2Rkj7sZi9SYkJbweZ5dT8Q+pUHjS/9cP9Y63VrXaEZjjvrB20dJPA/eSx8/2lz2mPYPYASnxQYA+naM8eOu8UAXM68D0BcSHiNWRE3ZI1qevyleFXu4/Fp2r6EccBSDd9unhXyWqbFjdDpDRf9GS8OI6Q32yXZIh/cTiUQk8+XjV/uyJk0fkuAoBL/uYgUVDva5FurOV1A6SInBKJggHv0nbg6ESM4/oza7dadw0WNFmIzRUk1TymeoOtKPw8hJ+KZ00tT2CHplNMr5p5W6i/WqsAbEauxWt+hk10jeloaMly5ydSU3M8YtcOGpOBb/bVMdNwQLwJAG6noRFF/6jrSohfKz7c/LRfsW36HwcATjstHdbUuKqvlddsqcJsEIVgkksx9Pg0EZHPZIt7eoXhCjWRPPN18P6GQ8hgdjU+6mh3dbu3291CIGHcNLk+vZOvZ1DQcupeF4QZSPBlfmhqgxQY1Q45ZjIJWOKLpn/8Z4O5eGo8cFwCE4ni/F0pbxozVf8jhIup4SIUrlb5WAbQsHXjJrVX61Xq7y3VZarfrURoLyOR/5rhMXm3t/lKYnj0fN8nafBUCEWHT5sp91HmnzDqVmfiTiQfbVg6Cx/Pm83Dpd4WXlFVWUAImPVhoYyfcCAr4Zm0gm2ZNjKkW9XeHKNw3mBME7TZ4OOfu4PdFRXCivyj9svoZldMnfyTNeWaO8x8o59URWBgMGjXeQt6+EoySvRpXhRMk+snrxchHuEcGxK2rJyXVXLjl96Y8fj9yXDlMvYCVVTfUW0QOLVkfzTVv/nCF5Ww/HTRSgPykFEK7KsWaATBBN265nWArGhpIEklH/pnwh8S/5QvJPU7bQn6UMof+uhOEfpbm/yxgyY7dxzP/zyhkYDv0F+y+VM/6syvTfVs1A/gvljDwr8z9Kd+OyVWM5DnH3/lsrk36X/aeYCKi3/CoP/yb23waoI8jL/nRp8m27flMY0P7P8u1/5GeR/zMMWcfvkv5e9p+evXn93u8WL2W+/R8R5jfFweb/Dxn8hGvxVu//eHTpf4VPf7pu/L9QQ/5vJjz63OdnvQXgAX/Bcez3ffjc/yf0F+iPe+78vYKfm+vvbsx8qR9yAGn91TY8pPk9HUT+0QDmg/8C/TD6p+FvE/7cXX9/989T/peQ8ee6CP0vIgPG/t3I+D3UBHrp71UG/I/64j9x6p/UwK9N/R7398fQ/nWqf6pvU/985OTXtv9lqh+w/nVP/wuaHv3/69rQ/1pF7/9aVchdQQ3I+1UV4ixKr+DnQlmEjvvAjPyBtNIV5T3quzX6PH6B1HvdkbkgP9rapUkREKSzNsgvbhQbSTLKutDbCLgmYt816wq0xJySwp+Wl7LXWI9sfXm0F0xpFa8xV48zSxfu26d9L1P16pNbKcqIRfGTwMj0GwScPdiGMQQUCPRBSjo1/JSl5UF5l7Ud+i3t7s109PTuwVPrcR9df8KPkS7plvYRVYzjknm7mJSSZTBQ2WsoGJyOiixr5S3+PeXztF1azWpa3kwpSZ/m67erp33MEVk95GI0BqI91qufwFzjyfvIQnv6zBZdMT8ZGvNIHfI8CPZqoEQDuZeU/7KsNhyyHqrCm6TZT3a+CTXU0iZ1my8r76fhHCB1uZZ3+p5kP4UZQWtVjqELOpVuMu/PF+mDPBS8a77Ia3y95beoD73lk8ZF6a12xuSLys9f6/c7wbEKGqUi+W1XOsHUciQSX5APhvqp0Aq9oQ916NM8oqJnSMfI/AvhrhuirhrDHtd7T72HnQYp74Mho4N7HsNgtfhNHVCDMRg/owb/zZU3MZ7Y1HaBVi52OTrIaaCPb04SkvOpxcFAJVd81hm7Y7OXIFhOEBZ1mXBfMJC9owKmuPJxMYdJRV5BhzmmmFOS9QHpYjHyPzsMfNFeq6JqanM1Nvjz1Rx+oakgGyfpsDe8vx+RdRDsiog6hyfBFh8NN0gM0R86VJEV2q4DzmDMl6Bp4LFX5jlKNJbQ7FRyPsqlDje+YlCcSRgfXxj/+97N+p0xI4c70rdJu6G/8NqPqsinpQMiHiSSToIQBsWyg6BulT2W7HG5IojHJUW/P0YJW9XUhZqqN9AMjURFy/cSht+3EBhIH4Ss8Kbf7hOxdRKNQ3QoHLQV0sxe0vz1qQeYluiSd326CyXt3dLKW2KMlcY2WqxL+vuTBXBpVTk8/kOjONfiND/QfnBwwYfNcTrYaYmTmIpkZK00s5IdpVJ824Ii00+wIU6fNf4wri0OLG1eNKlUs1IKbJmjrVbHDAIx9kyz89vG6coF2hVih/Ahs5OXybdkdLrWPyB3Wt43rcgrWOjLL8+Cy+QSpJ6fCUl6RMUW50Eykk9zOks5vqAB/hgQWjNqyhYFC/jBqDkj8S4jjXn06a/S4i2ir+hOtGaudt+qy3QhOX86vxQ6Zptpwg2yDx1XN8yGMO9SNLTRQ0zniM61xg3Fw8jlzYYzMvTo3Kj4eBR/I9LQAuE8MPqcq6ppOESow+F9CtqlBX0otYc3ld3NZ4YqpDF5IspJdhW1sC9fOL+8k9rf0ydEcf0gYU3VX4LnkHebcnP+1sgQQ0v5VPkDV0vqRYuplh1iUJ67UKSWinLBwQ+VWXB23lA/iRBSDdD0p1wCosAGhJA9qz/YCN+kJIu0FB8lwUb+W1/l7WiYqq4rDmZT/N0acp1a+inINaDdeyYd8VxC+13J8XtILZuNiLc1dnVtaTZ8uYLs4Fb07gSZK+x0UqRN0l0nmaS8ZXD7ih0qLb+YUF3G0IrfKCL74gIFaF54lHs7X5HT2kYkde2I2CDZJO2TVNtGp2ix5du9p/Jun7XLuOk2prd67GST3tsAGx91AnZBoqDdtDuudYSAjSy5zdqve2X3B4lcvs4XkOxeqwsT9c6B5hzukFpS6vhRo4pwK3w7dpg9db00d3hnRPDbmh8TgW7hikt2ukSXG6hT2nnX9HJ67JEOf5dBBsJ1sKRQAsre218KcUWVmG4NZiYYfXSFygJJcm4c57q+5EADWcTTTfiATDTbSTtOcbtpgxUVqvRuUxWL7NTOMTzucQ28E/ZHpUTnRPbILtJmDvYqzza9UPfhuc285FnqPPdLHrcyTONj4IWJh2VBBvOmB6qVS9FhlCMZNRo7DETVS+qKDeMixjnTHYROk4eUTj/P8esJDG/biINocvx9ikNvTjZrhoe+dLZ8iNNsIrbsC8uvD5SZJwx/Bxe/b1fOGzuzdAjAbUNwkfhAVK5C+neGcek1g+TbeFOOCYOaUFAgJ60PNGJz/qip1VBzfitcYSQ2Nhken8S31gdAbvxyYqHxES1PQ7FJ1iC0kkjNpX3uGjeBpGdE1fa6nXDyogDzPEdJE21agMC+kJ2UoWb1vfrFbdx6uyLzbRERiy1ED2oESHKDgpKE8O0neHwlpx7qIII27ToJ1SMX31utIMAnziv8r7IB8wB732dvOIlsj3/RH3GieWuGnE2i/ZwCMCY3t3TK2J/9r4EBX8SbABa+kF9rCGqtBQ6w4ZGGXGIyN7jGCO1Q92IDj0WjdLOuiGvz1DqPsRl4g4ERYeD8TAgx7waKPqV4BE0q9xseB8f0ciMYmodj/dKqgbylZrJH38c9/n5nM92z+34lKxlxL9Z8j+QwTHOeRgQ8bEbOd8RS+PjDmJPag5zIXqYnGFefBBprUcnwuwLtvU55r0GeS04IE9DqJisRe7bKL0BHNyDPDxQKmTLfUiRLOaLfWqm9GBOV32DFCheMMB6Ek8urNpaCItIdmkFVpG9Xkn3Ls0GuUwJJTM6DQO2rbjRJO1yvIh4diJI7F5M0aRGVwL8/hlVfZmC9IkFxi3Ps5ibNxAryAqslV7L7wom+3Gk+DDm5oEC+6u/lbWc4J8eV5z7mecriK5tx+kwt9eocLvhBd623aF0Y9WdE0E7SJEgB1I7aHNugCrmGA3UlvQQjDb3zcSd4wSCDvT5PJX2jo30O3Xd/DAoOOftpqm8O6hk782SIbK0rhCZAPDvsOJ30AxQoNHKaqcyLZmLlH6eAPFx2LnIgHjDdVwtBz5w7JW0Wx07rCyO81jT1fpz6tzaht90AUkVo1A9rb4MCGd+/4KrekCTqZX29fx39yGFcDdYtfy26nm1+k6PDAvLJFx2NoBR8CVR1b4N+pWgI5wKyIu9HASXFtK3U0B/ra99aa43TzbsoJAQJaZuDD4D/PJKx46e4tnyw5HpRb7Cjhlj9b4IlIYcQRPeTydv5qfR4xiORlMiLXRvIn+MSBQYSjXkT44V+twaxqjv6BVECYvxa+leJPtPNoinyBc+dXu2e+L/06eM4E94XIdwHNp6KF1/QCBU4Fy5EA5cazgQf4LZ3rO1Scox3TezPUbSBQsF2tKJxYBinjmF0bYXCksBNEPOw2EEC8o1D3VV5MA/x4gpeHWOa+9o5pQEJ18mkMl/xUs8TkQ3rUhOmnSBQeniJkc3FX4kPTgsENaCK2NSg7J99c/RnQ7zoM5MhSOjraj7ucNCDVVtAZE7ztjNKfYHSLzgQ0wsQMQ8XNo5OMeiuacTY2PuPjWXgufGJBnj1EbUkoj7HRRNQWzGv2DbY6UsdsgkdhpWkzBcEimak2lqpyJPufnZoEnsEMK/f6gQcyHBv8nJ82ad8DwidpJbHUCZilX8N+LFifL6q1InVCRW138F5NKeab4mT/HjsVOJGmekV+o5k3zX73tv2cr84OGqVohR84fiLeCHNq98fnpc68d2BlmB+0eGlgmCj+AIdgJtm+7NQc6rO1Ziscy3CkxhMDLDU3S2X8geeQIHu1A8i5OQjyX68PMq+WyTWKmrjVJNSf83OKWBS7CZ3oXqHuIXAggoRcjMgMhRcMKrtp00OzALOx8BmfxNywMzUrzoL83TAEfmShgp+fCQIlPKQHfZwALqXgXcvInq8PMDTdCLwnweJFFCOXXPXBT6hL/ONYcKDWVD5Ukmv4BIwR1GYFDWUeBRM8cEoV9SmDK+q5Zcr6Vf34T7vqGfXMaOlRsX6ZPZDERiwstDee2WaXPa60VGnVV/qq56im5B1jZi9WU5q28S/+frdvPtjij30Tn9ReCVMvHmXBfDUuYMAeIEy4tcuH0N0OM9jUNAMjuNgZFwrgCBOQarl5r6BPl+76fFzaZrxI6+SoxbWafYnXpdoWhDjD/3+Sc7/dPlpBwc27aqkRXD1BvH82y6fwPdv/dxvSR9/dy+7vy49nbPdFkzkhC+5i3zsjgUKQrjwjEG/LJo8voL8N6wZvb4nNlZFAlVHNoYmwe9HcydqDNWWCnCXCe8yF+A1GTQ856A69K097F0c3Ce+B4U2WUtiiT99jkxYKamrdJdluOR5tsSFX41FD6lBFYmlS1OU26iZbOsd/nXOtLd605bHTLQOoyb37JU9cpfeak9d0UU+sXSLqTd9qbd0Pba4jWr4fgwOFAbl9oxv/pj77+Z/R4HepH33rKfbk5q5IiHEQ1/es+BDSY9L+kf/P/7/sZYoqKaogepYtKCUG3cVwZ518G2CyJ36Qyuqy4WuV+/3sy+yNsvx7+chf42fPnFQdQnfvWMf7pJBnxIEqCzN+WAm+2sGo2OmXPBW37Wmh4rfFHlW+nc7+dmN4KGxH+Ku0KGRf4aW0G0/3PSpr9lot+Yc+z8+sfqnJ7L13+2TO38/2bujQOYSBAazQWYHaOWBUWPy+pAq8uupXqB3afvs+WfGz9/Pxh074JrRWl2K6FccMNDPmn5Rg1Rff+UyGNUce85ND7aYLQqsyixnJerp6MHng+lfp5CfeC9jUrqzsGCPr1uOBMnJruJLtDaNGvjxNW42XHYJ/XWOhKn8oDoKhe5S9lLVGWX4lbXDWrxY521GuvbioUEIflV7X8AxZIp894kEqGkT211wNC7YiEl2QAy7uHO+zHg/mAU4SCJ6qD0JfLw+IO7P98uhEgvYKnb8PFu9NulT22e/q4Y9X3zX44VXf8tJ6290c/Si8GhMCsVBAGvt7SNwLHOej0Lsl0lpHIxDZntNuWVttPwJ3CFjms16hsg9pnU2V+XZMBTUOmYseiftW7Vg/xE4F5Nz+SOU5ST026QwXqJ0+iMCUU0HYj/nbdIBv5JwVe6KN0elhFx4w4ce4NQkmbQzLYxMy5JdQi03oHifrSYh1l3UZo4TUqLcUS8onaYQoe4Vj2nltkjZ8Wxxjfr1yBwDnCdi4rSlWoeHGz9hq9C2mpNlrrMq3umKfhRTTyan47To6ktPnJYw/XJ0PxirvwPrTYHQsDc+QjBd4lckm1TBAkHYDFM5P9F4rsGbqGLig0Vn7F4B3+D0Xk/XI4unawnP/rdo24/ovqpdDd7HCN/sWyCQPn6AvzCjjQXeKwXQEJgQGhC227lGkF9IFcZWVf64UvFsmY2yy7LOWq2qFzql8JIUC2fVpY12TjJsmdt7dyOXWMDBToeOYau83jvaKls/fmOWz3Qzs9rpUSZXx316cXm3x487P109W3/I20qTCGJjXbF576yjY1ZxvfIiKYKfwN/0jGSCdVvdGeXDGsbJPb6y9i2v07seQ/USvVjSKHuDpCj4XPuaAvFhvvS0Cx1RRh8qusNXakkTwh1zM2s5SDbwjWjdEOZkGEJ+p0DttWlElQd5WLfWPHyPj4UaDP8kMcTqpc93eIJLwYI0+2bn08P1RP32cXg4iiirM4xYVrVLxKTxGgKpsx03HdaAsLJtrhpYO7ZMte5Dvd1GOnvN8FPHNyF4BGHN7VEc8hIRqFMgd+pU/hWmKbZ+tfF9c6erObfpvoLCY7GJUFzG42H38Qo7PqmEoJnVSXAZZGWXM+vh76fhkl51pbxx2PmTfZ6oD/dgP+croafSlwUc4Y+vuKI+kARTHmF5TYKytDNBKbFNe5WC+aejDoZLOiQKOBS/1g2GLfojRc2Gf7xPRVzd24XevgX2dzKRp0foDBn2vQsYSMfEJzovzFz1lqRFJTagSR+4Ye23ZarvVKJMpfT+1NM4G4/fALmRZ6PHvp6zELk4XzBLoX29a2rqQIQmK/H3eURhN0+Bs2/XuIaO8CnjZnQtHNuckFw0VnCq7jtR+8t3U//3GkhIKLbP/YTu+eAkSgiVFEPaL6zUy2AWR/nVW3u1lhi3LleyVoWDqe6A1sMkC7grfpJVtAOynWgroOpSpgyP7yCPq7/eGjpDHOsdM5ifr7vG8UqEjUE2/ODCIJ5opMBn7EjjuSkQu423GBkcCu6UF+4vGYO7qRCZPSKQST47/CvV1FvFa4/LHM/Yi3JeTmiG3Cuhq5Z99KZWHcwyjiKN6prmyJCKsQp1Fyfw6MKjG7T3UbMD16vfldx+DoBE1mthHNXUHenRem1X+EocY59ij0hSVLCGjZSTEue0sjO7wz7U2WhQk+mBHLTQvE2GV5w7oeLdo+WVt6QNeHyGpFi1C2Pd8F63TyDcwZ6BlBukd5meibOgBFob4+7dTWymlbHJ/uRWhhMNsfSIRIHnsZYQXlSjytVsv6cncL5zQdkzQY3QSxY/UVe/Au11BR9P+dLp2+a8kausKiBgF0mHWD5nx/D2khY+L8nCRkpiN01mWi0pc76/3sSyZEJzMyPOX9VRDpmC+6elLc5Ei4bATFRxUISoKeg1dKkCMvC6BbIn+HS/I5CMglIo7z3tie1e7enSL5pLatMg+3DTKfGMOsahPjirgMICxEl8GXHHW5M1ozJq4+4cd5oCxxci+3PyI6HCeHc7h65xNjnyat9w/ELwzrl8R/fy4cniPUc9vi08gnOUk8ewYanwfIxC6Xkoha6jn03Qk87YwOdvozE5hF+YC53He6X7XMIM7jX5fm/IK68nML69kIJRhXM8eMsfP95r9/DHQfIbZhBDs+3rMPlu+HIhfqhZtK+ia+dj2k/uA5l7F7O9uXVC31ECX41KEuGZHGVbHDeUlIWhMR4XFIGOfSPaSrcid6p9imgTT4mgJdHaroZDA7gMY3KxGeuhjGqCsipourV+iycLG0bxq8UfvJupUO2N+dFMgTpn7qU7cax9qdtHZXWfs+578juV2ibsNoNI3klwumcuTPbQJt9lgdVvN5FjrNPNiwNJLidSS96Nbj8L4xRDe+u5notlS+CWMSuhdMguhFn74r4/MdDGqJ6Z3COdhQAEIllJXmxYtvi4EPzNBuo4JyUfi/TApz6HtkjLTly+wUb/hKhRsrI3bsOpxAh3gvtf/8FLhch+P++1W2vbKxftyIGrqrIoDWSOCEUS4KYSvAu2Qbg6NUx7FJ+qNDr2XWql9FnpBu/7i5G+TGH4uUy7FYiH6qRhPZkL94+EM+t8Cag/N+m5zhWlzfgmsgvIl5eckMrHK5cX9EqhckmEbGiiaG/Qj7cP6GMujIwXCsETD7F0XmbHzkopfQUvF+tHq2nzxOajKn4CeVJFPZ3RY8iLcwKEmqh+wCGXkgBIaBvTz3mlnc9nbBBYOrymL9O+qDNaIAerr5wXqb02rqwH+IYs6cGykNhWJUvgPPXMCBrkr6XgCfobmBQtlqds1b6WI1BOoxlp+dIexHGWa+1d4tMKy0LxSfmwDd64mK1LqfEoDrHBwSHThM1NI0kb+EWIrMLQ8L69a3cZ2n1hW27XZtDlyKqfL+0TEy5FdC8WZM+czX9lWp0ro6yV7NzOPuZWsfIAKllUn39D/HpQprRtZd2nmmlxj/VQHKBP74sIRpFAMi0CqWR+WVXHrhAPcRRMk8nRbR/nNDEGTfw2ra8Y0lJg76P/bl+6rCkTA5yabu548RK/0JLIW/T+WJTTOVsQ4aIezsTYbEZsdSXECpy+W7eCJ6LcLE69MCbtBiFTX4u1xLUTpO/1cJLFZkx2+AjwZQsoDjIOJjdfG8vlO0RSm72lnZs5WGykTKh4sKN4gcgb+Hfp/QhuR2693CeQohz8nZ7RYXMg8+DX8nLMDrzR2bGEkqMwUPd2+DywF+whC6F2Fyr/ZCOAEvxx7JV2R9zP527YaqWzc1yyqh0xoSlT/5OpNJOyPvmpKGWwRIpUyYJ4KdCirdrCt1D8caSx/HjfDb3NOeaY9EIqRxiNbcJGNAasz9j1Cbc6f/gqxm0t1mx4HuxyShFerhxKOqsR9U8Ob8P4+45sWAjD4Jp5OadH5NO/QTayYetDYazurWMGS+2ZFiM+mkUg13MEXX3mohxugTX1r8AzD7RElIqYeO/jd3xLLQqsBLjyupSTOeT8RMZpmzTLpA0uHxnIs5pKknUgIg5C4g28qvyQ5IboCEm3SJRmN9WreLY0jjv/ThCnzG3be1vRXfP5prJ5ieR86q6FjVf6Qq6ZDcRjRurNexXeWyPxeQUJsNBxsvT+aJ5G8amOLd/1CVKUNWaQ+0HNIL6ij28u7VExrSKnad4z4WbyQpj2tpwoTd9vVXu4fPuEi+IZYjdDeS2nj1YS1Yav2jPngDQdhs1TwNVcMMhks909JW0tUlDvb8zso7454ddHKmuWvNE9mnleFy1sH8uxtTckE3lIrtuSo+5AqRqCllq0PDFqbgU69vIK0b0xWx1HeEDs9XYZFfocic4oQs7h76ptpZCb3iQlXcvFs99ib5gwEx0HKjJ3O20udQtTqIvVGDGbVH/efgXJ16BjbQqXBPGSV4GVUKuiPe5607Lre/6cNykNSnCxFbRFGi8O1wI8iAHAOvzxiZg2wfsZyxZ/S1hNQl65gvY2yiS1Or4a6urAfc/Tv0ftCt42KhQ8Q3jdj5rJPY5ZI/S58bpMVj6GRFq7yIvFHdLmpPPgRrAkNBtVeZct2gvBemsBwj/VkfRVID4Bte+OHKZUhTaToAYnBXELBP87ESGRJAbSi2eNkvoMU1ceNTDOaIJAC9HLdO+cCryjclV+dHiyQaleG711SVEsWr8JUUwtvlG2Pmg64w1SDM02sqTGBtRlMn/IFA4Y2vsuV1WfLH+ceq72p2EkN1TXCtJC/Pm9DUTccdkNCHvdcful9MWINWbEh5cpkzCLHF6kRt6X7B3PPF/89QTHSaW5vo/dhCArvcQWfGaZiWXeUiMECudkD9xepyJk+xeCww8aXWRmqLe5vz/wF834/rc1Bk5DkoEc4HXezQySDCYqTj2n4MC3TlMvRgkpTOxVVx8Rny37Dam6mJa8VPkQD/NwcQYLMLRFYigd38Vq3ThQvEEgAzkMfN2YlvF5fO4nYmnmUap4DkoJ4Qlf1Z9o59c/vM7MTBilPkpbMgJDCRNpnZ/yEf4RsSj0FHgkZ3J4v3XWG31g5pCFjUY1Z1/cDtOT2Yri4DbCoQ7HlwtNk/tWqPM47ArR8bWi59YYnPHnlzLtwClzF6Or4VOlX796RCpCnYba2scRXxBkD+lLnL/j7Ja4EMxEUCsUaTjOARe/vCxv7/FuTcmclF8BqJUxnxSEljJfesaKvtOEVWBZdc/t9qX4iyiwl2rxeR7Rz9GEQsSSNBX1lfI8e65/4tY1jnjYvGxdu9bptJs8tMrdFARCLMyX5jO4M1XBnCTjN3iJavtdiRQLT1BL/Oz3YHrMlAMb8WnemOLo/AxIlHRYEJifqDffcp6EHZ97gsn7nvqqAgyNVgLp9VrV4c89CY9a299H6Inf7NAwTwvtEgJlkcEvilT7VvF7fSHvTRMz8L5BbhbD6QDWHeTQbwLE5k9QUC8mNBlbhNb24+7co35AYtsw3TTSPAZjDguhFvK9QF7dmmfUi+4bhRzdcI/ppqgSNmd7yzPd1F1rxix9Z/x0sEPhBUx0U6nBzuqkINuvJbeQ9yJs9nRpHqk8H7diJtD9KSpf6m1/XZPmRYT0qzXmroOl0UFMtmadBkSrDQXqjxmUDUDN5VODDJuE8bwXgermaNwobODDFn5Biq2YqO7g2XNhCmBzBGhMcnwgfIi5wbvjjeLWGK6kmNO7+TJRyHh1AB75zzuABwkv1oh9nDBIZCFpXKyL+wpAryh4E72HYahKqJsR1zZFTvZP+D0sM5eOnW3ZYZBNswjSed0Ew8oUaNu0NTesHtcWKvoHFdbkplTv7tBNBe8tki3HUK8ggH5e/pY18dUvKoccxVc8MWfAo+3bDJ5rxy57I3hjj8F3a1mwSUsmY5swdphgMjko+vdgkKsMlWPOOw962IS1IOBT+RubQlye2d+yTXiHwUiOjQ30C+CrCh5rL++djWPH2xQbh/AYjsIgJZ54sMVg/BLGz6s8uGvDAlLVkxdS6SMGXy8ub/XHmPRDMpRL69E+/jgjb4A82bv9Qhh071O16LsjTspTSFJ+x6CEqo1nSsORXb4KM6wgj1XsTAcaxZb0w8sLwzFJEsaICnv5/rpq/dkpjyPhv+VwJqAjWkExlEgDAsUNP6gC7ih6ETu9nOeA+SBehwNiQV/9Pi7AEWngGDR/HpcSjNAQw24PzAfhlIwvtPMlIR9t3Nm2I7TXyFgDOzc3BCv5hLghRmKZfJ0AQLwdStaEfZA6esKE7HOgWnMtQ3sXmTbYHBKgUpe9sVPfbZtxSUEKrSkRYp6SrBtN6HMErO3MGOESshFj03xzPoS4pxJxoCjsth7J8HDDy+F09JUlbXeWSCZ87HGSWDHyCq57Nd1eQHMdhqHXzcNLk3ACoXcVn3c4/m0JPtwh1HRlvMC5Zw/+E+iDklTClfeys1RSZGL16C43A2lj8z04apzzJx4v73fP5UxfQFYfuyCTDRHFO6otmYapeCkYoVUS1Y4m5DSBCvR+3teldR+ziN7njEAQRvQLd+YuO9LlmXGLVfxwTRUVjNF1vmLBbQ+UGBMMDfXg8VpmipBKOochyHKvnibD5R3dwoeJ+BSJ1vct7Kx7zahqhCoCw8dYFBu8LEqWKS2Fncu0VNf2xu58/9bzOlNYMG3yOMT+NUPl6W0+8IfpwhysPP8aXW+7rH08fqhgxN+vm4TFrm2VgLfj9ihEOgxzjasbKT/yh5GryFkQFX5EoCNQiLvEnY4WUyBWsow/EYLdDqp3S10ik1xa2Yq0kZUzzpTTFWGqcLcG8Lvo1EWFilBxcCYaMGlkfKXCsuboYykN8oBrDAsXDLLgCVw3Dq0Mj6uFiecZpoDDnhfKX3zXxeVOll7YIwQKtk2JH4UrhuFnGF4jfqugnksNXKVnOc/i4Jsu5O28NnBEmVFcZYOKjyamxPhptsSVucKVMKT391mY+nNScfgsLL2Z7IqXDX9eUxa1Cw8XIhl8ewE13QSsfJB2/nlh08A6EeiJhvTB4XsVMrB5tOFaSsT2Ht8yQm0IHT/edSzNF0Ggj1ecuSKKFp5NbEyWlXhRUEVS1mfVF6u4UUe6D0lG+Txl5nYkO6JtxMw1I00cekEHRLdYSVJtdIpaxQG4FAyssXjAMuX2PhqqAicYeG7KCHwDXgFRyYne4scg88ByVs4wYDlNBK9hWr6miBik3Gj150F4S7+sStXHsHxdWJKcPji2QK1mvriN6RhQwZ85dBimySzftwG9XiI/fA3c91FInMTuQIqGQmScQmB72+RHNZle+1rWDqS6rCFJeCzXf74iAQ04ijqaRFXzbwt1t0zjg6JkaF1DoNAcSBL7WJCpu3AD9oReAC8vf3P12PogsNAcxV23SkV0mnHqAHURWnbIRRjg9b3tCZax+hR8XRwBBinIKk2sg0URO/rAWC+L7OXSKyreU+oocdki+w9cGjVABpY81HfNgcMw/wnyAsqu9nAyHyG4UZHppBzr71vjghJ21+jO3wI6jyMOSgQDi2rHYywtTAy//I3ie9ENYMqvIFaw61XgDB0jFS95SXJQCiFA1Fh01c+xq6B9p12Z1rUOrOIY3txuGk1KriXLKW6vmLWbkPmLuOi4+/kqAqUwyvwL3unmC71InA+vSx/4I4HjacdSXpS+0EWRq3JqeLqODddNPNjeTXZWvwUz0nmv45L5tZjRxKLzo3HEXKi+Lo0ZNc32/ZgNWRtgO/syByS4EBcLYUOIA/a+qO7LCVmRtz6Kq8tyVNJW2RoEYWisuV/DMvorb6wPBtIRowqNkDNioYQntqEk8rQUdQvOYXgizoLYqfNEsUINWB+mEmS2R77Ip9LdK0rct6oC5ioXm3FNC04k8YR/YRT8RJIOW+Rk8SHNiHLMELzIxyfHnLvYE6BNOYZeGwXKMiP06toXONtQKmxidrH4kLThjkrxmdPTgstqSr6AZweJudiNBkVmyUYCh6AKlcRJTVSJFVoYhhuoT2J+rscIBDY3JiLZX22Tg4RO46TSfL8un6+yYCTgiB1/HUngu5/znXiNRipDQzDKrsQxEKN/FVyWrxaDJndnk7rH3WjYbBPa3oxYTRc23K+gt/Y+1l8Pr4piIQNEwz00bi720PW8MUwc3fRaSSeHu5F3ttI4EFP10GB0+Tlso7Tt3TnEMXXaI8WLZRlCUhRhOfPRBbnW6dcBrt0vVEjDwIPZnmYpQgaZB79b1Mcv/4rIz7dCqQ6yjE9IX53R+fO+SCjWX55L0VIX1FEyy7MIrtUIhnv8uF6aGA85nO3O9W3b+hbNwUkM6iDMF+JuAJlPoB41Mqm/Lh44Bxw0qxHjRZlOfvMbKZHqo/FcXQKO7ev6NtEftZ2LFIGX35y/nSI9RFbjXnacxaXfM+qMiyLmyR78jRhBODSBQ3fE5Tf0NYpEj5k/X9ESVUyLW5+vXfS6+RahNSaVrExF4XpHOGmsBFbal0B0/feLag/FNLOG9F34OHqNJryxaCluUJ2iZoV2E0+ETMm9t5TCr2Mm/aOFlsPBdvC9SXzUDCevNzGDiuTOiB/B1yjz865Dd0sIdLtl9YQVr2wbLKSZ/N2zM8dGHQ6Q0pzo2G56OagwTL4WZRxK4EcoP9/A0hkR+uj7clRGtfyYxRNGONQ9SslBXdGMlMpneF99JnKDEN29mmcFBIrfPLVgb7+XYkY7vqYtQUaXX4hY48ijG1LKjsJiYxOLmWx3wqKO4NYbvK5RXJBGmqbTrJSqB9Ix8F/FO6zSTyMB/jRt7nH74mhAg/EoeC/oMdRSpgVzZ+juDE6dznMbhPIpZE8U9DYr5iooLTpAYG4pUoXhJ9fxr/ElxrGOYKgH7UXwRiPgjoVfB3RrBs1L2jBhP8GXKUk+YTRN3Vun2Kwuo3Ds7q4JyuJfISuoDaCpqguNF+DhjiMfTo+z6P7faLqqpVmRZvs0/z0ulw002rhzh7s7T3+ob8+ZiImR2G1F5sq10moOjdQOyOTq2x+nRlHNivWxUQYShgfHbcluhvtaIR9Q0+fC6C89BnD67JxT41iOtvLpBTTsTJLbRYNveDV9B6ZKsqJAyTd2CaJGUym+jY71QZ3LEXMdph9yu04UDoKqO3RjKzBYOk4hn5/pwxskAHjQIBficQi3cw6GNr55YoyWE+EfigEJc00xCmR6ctjVxs8IN4KfcsNXJVjuC6L/j3GWQkgrjjjyiIm5fycwzwfnvSECwCtXjsB3tD3lVa6wf9yDhpzuxSkco03pP4GftkGAw9dI6CTR8TWl1k+m7m2kfn4M9n3s3ud10GeR5aJ5Dpkvt376A8rlXyPjnxJ8z1ydeR4t2t4ajLH9nQRzuzMMvyS8r1LXT0Q3NMvMgHkBbY4gRpmFTqnHXXp+tc3ypij2PODq44VI7/FUCC/XzBffiZWMPxgD+AEJUsFw6nnRKm968LMnuZarQRqj9X5osMzoUNJ2WZJPEGN3lcu3c+n1IG3RFmPi25hTII0iMR/TKIFyddUOfQjF6L+pl3+92a9oH9oOO1cITN9w+Dn24Lg8xv58rspNhZ59oy+PYh3BJy3BQjNtzR/AAmVp9OqsyuSKEl/VMG4n+hqDIv7XpPpQSkEuVN3OhmSjRhZZn78GjjPPO2iJY/MM7eACcoEKxRAUFYzlJ9NIvaSkqesoAGCxBwTm5xtFbVHEdBSr7UKzKf5cFj5KU6SfzA5uG57gr1+K0pdsPiS2NPDgHM1JwJQzLOhL8iJBZGCf958y5SviE67e5/NKAxU0Oievwj5+a55zH2qSMtJRqPNJWDeb05uTt2G+kXWnXzddnOJVC9uJkwbSJCelNr6qbAh0IdmMliI5nvEJhNpAAQvH6idVnjGp9jtPES0qp23KrEi10MJuAmNC0UDMO69VgkEqdmtyql5dLZtbWCZPC34J6WMwWEeFClJTbeg5xbLDWtCETscuj7207Dz8YUlCuNIT7SMq0ms2FwrajZwhVXYBI7a5OVT5Md2dP2Sjxr5rm7sFWzP1viv+4mDko4nxkkHH4bwqaZF8t5waRwbO+TWPl09MpSA2BMqX8Fd0BWEbV0NkC0o2XRzWSZJRcgCWQp+GyHgfskn8VXRHfVaqi9IFpNI5xK/6mJpNFCgFOMsFnb5Mr1QLe79p7nn5KmA4vP65+u57kdQvOQMMnruKKB8r2fMPBDGTln3WU611/OmdCUpIE/owOOCQmYYe4DP1XXfsrUQ/dx0pBPkBjdjMVRGOo5WhzuEe/7XijwVeVooALiSM/pqwxKkgi60/fMfd8MLCiV+XQQvzPdNY2s9shgj8QNH0BWTJyOhfIybzI08qBuHc8C7yRiANaSETZLUc40duGkY9IdLiqJAbyPE86e+voUvdwkLVtOZziNbQafxHCGYh6T7qnxg9j0NP4lTFu7+29gpN0eqb+JfTC/jwUahHzfNC+71HCYIueEDjsu6eINATZ+vRs9Ed9MX25tc3EmWHf6uSWBGDUNGo83UvllfXgRL9Syqe8uEwB+laFoljxGOzzP9i1yQpl84Om3zVp9sPVQ8IEE+fp3nOSCJ/kbv7UHJZHQSWoighV6+ePZA48UJOqe7paaNGgV65ADqlwcosS9+/mTHF/qNJN/yFkaAM9mMpp+hIRIOZcv7izDb/XQa5Y9UatnAFFZULrCSXEpufc8M3xe1HPd6/Q1euScHS2EnO5y/eklA7sLywj4JaWarpcozSKC4HX3SQNGQg9i6kWYIpoAsZMZJ0As8QcZyyj2bT8xzGP2EQEu0+H4jqf2w+aF3hi+kMTHLy6+ZLkGd9mGXbdmJkkvTdEqjtdPfY03lCeyZlsAbFG7Q1LHkZGOw2r/gENif21G1sIfgdSM1BLwn0O/jint51twIpgzFCGmnQBRQkmwsy67oFIj58SbWcKDSF86G5YQl4vv+iUEJwBpvYrA7Fx7Xu+nQYx1Fk2L+gz6BwBnzg1js3rrdDpb4Y6Y0XYdA7mdOIM2Iw5s6tjAs+pS3G9ws0pPCGPf9xmAkrRLvUQEuy4+kAflUI4fCaLb9Wn939a9hokChxNKNGraabvu6Y92oVWzDSZ12sYfaoIiEEHUSCvzVm3NeNKlUSF2syEEysnj7MK6NExFPxiM9BhPBEbJM7oEQhs4jRpiXWVz7vatGGcdyf1T/YfNL0BY+Y5DmsaxRO8gzM9ps9opz+YHU39p2S2bzajbyH6nC+JFW9yTlxrOl7AAunEK9qKmLXRYM0iK3GTI6bPyv2fDrkA/J80fkylsKzsPvBSeQy2cfPq+5l7upkB0Yhn76Ye14MNT37zR8tIUmMzosAwSxo7WaBMk2o46yui3TxUY9hD3JJNUrHuQOx2TJvFPPolcZ10nVrwBAG++otCHCG2LD6m7cLXV8ZBLaRMQA5E4ZwOzT5GUUuAMwqiu8X5GqwvzzFJbaAQx27na9bMafrT/Rx6nsIIPfknbYJmm2YGuS2gyRWi3+Rlxe7SvXUVxih7UZ/wzqxRb+eAbfE5SyQt7X7EcwwAjMiZJZbnOsSx89P3PKxJfHQJutxw8BHPHTTc58zThSHVgs9/G0J4ZvPSVhZ8d1x3DXkLCeQc5gSVg+giSuJWDdqtiNnbqpBBUHRCCa2jy2YwZTGeh0KoCAgqINnVb5nZatsw0iQjIn46XXUD9hTSsly34Gkqp009zUd7saE0fuwov6c7e25blNraU3s8nbmMipZ6PeM1KyK24Q4PJAPOsbKXA5LOTCrGbCDNHzoFX+p0WvseR8vEm8SId7Z3AhQ1T8Nl2YRxwWnwNVh+grxeWOkz2E/eMFM0rACzvlS9roOX3Wru7grlmTPcDpy+WNyz88OQ3zPqrcqM3+beGY5NfzVBj/l9INKa+9eVSWl0LI6nr8dHfF6hQX2A0K9bVW1Bqocka03SDljvqQVdOqIjkMT0Q9KBIgG5RMYK8iHNhw3cmEiTj85+X5s1+gQ+in+8i/4bIOJY5LeUildpE+0L9xsfweJ467yafEQyvkenHqYYbxX5selIAc3i3rIK/7GuXDxunvOaUaCSmHw0rZv8x1PVlInDw71Jr32j8Bv164seCC4JVibyhBzkjRRExduWJoEMBnUSHs4gEDLmt5/bI2s5iWIW5Eg6eBCbSS7yqYufSWdB0eM/cLSsCPGaRBNrfvY0Nhw203Vvu+5tgiNt0TAgZru78EYLMsNXJfwf/PovEqiG0dC+b66qUuYmbCc1KHQ6LAnZVlIFBzBO1sAHo1qoWfre5AUEXJUHdoMk28yIt4GKJJ3EeYm2pArU1/fT6EAVxWT+WKEb9NMxcRiKu6ECLtFUuh83v8YWgMVaCEFR/kNecm3psm24xaFlZ3JwV4yvqP49BbPdti99bCDB+c1kOZC+6Da+jm8TOClXE+wyTVX2BfE9FT5nXsfBEyiw4ZY8DAWLhWbE1RbLjx71AqfOmbS3v6+s+cfUBCipSt1xXCZI6jijvOdL7Rb7P7160ov/Ba5Igvvk3/jsS7c0HD39W74qZYTe//0HOuHyz4L4jpbWRb29KEZ6p9Trv3U9M4H8awSr19SubDWNBqi4l+JKmuWeHAyjWm8QF1DkQygVILShA1BFpyJ+8Oiq6T3P4g04Q9YEME8H3YfmVT4q3cRE3w0LiF9GL1gUIaL/gX34CzIlUbx8UT0PGG0Yfv0OuVghy5ZpHsNTAnKimoC4xBZBP0CTOIkKLLPdVNo0e1Fr9SBgzgUd0be061k2QrmfKm9pl/sxlan9pEx11kIDmfdymp7AVIXcSFPzlfXcxPHyCzm/fSXXkducvvAuN1P8kxokYoT6X92quyNQ9tvezVq+XWsiWSbU5sUMc0APNCYk8DdYYegmNbTdObmsPnM3nQkroh7Rgj/bTFbcCzhcZ4KhB88i9/8E9iN5l+LpEdUyXCXHVd/J3p8g549MgI+4vcxGNnICCbINoCAD51WZxKuvWJleI9j7CWJFHKjID95OEPB5KXaCZuC8b7J2p9X94rlPBYGcHOCAUrKogi+cMRYky+yzqNA6pDL4Hwhq2ndTnRfePCgRlTCh9X+k9aAliWvS7ranItSmdgbBnqXVkA0C1XPn1UjP41UvcEpct/jngl9ZOkxahsdTlOQqQ083KvxM7BS7FHqAEuOAa1vKmpu7PrWHRSZKiWL9feAKF3fW4gy7sd9jTYOerT+ng5DLTsA1N/M016yKkiop+UoCM263K1kwfWyTcZerxJG6f1Nx98Sw2hCOa9Wplckl07L9Pjb346zwBR+I1QgRp/gGz13NAxV1RH18bi0k3+vR3ArjlxqyRdp7Cgpsyb1ZN66+GvjmmgefwPgMuY4v4WANOWDc1zGApPsNh7HbujhCC7W6pfChpyHIQsTZJ0xpDSKmWC2luklJyIgKd7Dk6LO78eI65I5F1NQTjhMTZbKXvTzxOEDOOJxBE9hfJeXjD+tnjQ1OQaIaRgfUdS54MJI8qE+y4ra4sOr9kNyr+NR+LPH0oyU8aTj+Y2Rqg/GjUyWkzENJwGAGi1fYJuCiriyy7qxrF7S75W4p0YlXgm68JAJ9eUhVfpZ3p7W0c+0VI0B6Syzyughs0qwdlsNUn5nK0c5h1jr/UIrgRp3s6ZDgfxeTTWBX3ChMvaK764kXVNRYimyverXAZYTld+LS1jJvEUBkcw8QZylAAGUcMELqTMXeUZE30fQjOmjA4uDO5uzT26A0b6fLoPlUh2D+2/emnvD8SBxCoXIcQp/69y/v9ZCh2H5Fd4ZV/CPJqucQB1MBH/uAflLiBFQr5kM7WOoMagae4G/Lz+2+v2+3umNtY9rXj6d2HUNhvrfJhQ9Qj2fTrEPsYOEEBFg13iohxP+xvwX+Sd6YknEfSj2G44qXurDwPA+wcE7vzCNK3wUAOA0jL90Bw8THEuPIH7q/tDiT+fHmn8UiBvU8KSj8Kk3C9ASN9bNxNrdZej+RrzfxJpb9VBDJFfH6OWXvc/nM9P7TXxQKw1dF0PbBUnKT8Rypc69IcqQo91qz3MN/uTmlxnuj24PEw5XjSOhwpeDdTm2ICEKCAOCfSEQc0NWd6P4yagphpS/5H0hrVRY4sXUprYSyrQbOl+QoqW9XpSemuoDQyKXPcZStcTXr4bbw62IvVMb09RSYFTRAHXJosbNiyUt/i/Uwk42b8Czm6JS/mENI+fg+au/XsZRXXX1fppl/9miVmTLeJlVqF8zktZeKopKkSMl7TrZvW1Ym6CCyl9GGS9dovMcQjBi4UVw3l6cDu+DqwH2uYdKgCyh/0Iwpfvp86N75ngp+LUNuyV4J6wM+w3y/d/USPBwOj41QVoVJ3vt90fap3JxhMl+CwOCrhIvR1RwJ9eOfD/pBhT9tIL82R2a99HPtUBrs/R7emgwd2uvVCdO7f4QUYoQzpJEVBDKoxBVwVn2XCVkX1XXXThPxxVJnPL4od/xI3NMTP6lpbsf6SEV89wlI7A0XhQry1CUCQZRtG5gVvwJlte7EjII+iy9FnE7q5gshgFsr2ZUOCruBe+GFDmJ+fCe8gGPYJf5D6prK+VecFmrpgXHqctNVth0+wrPof/ar9cbRSr99d49ZrsZuV5yJh1nm7tg3fWKT5Z+xvAZ2AWCV32nagQORssuFcjdKc4yUMdVohJkgt35iYefTiVXU03DceHNC+kzt7XlhKXw2DP4+8Zky30b5qR3C2QT+UufhDI64yi+10Yx4RBym2uIa/o3Ey+jgTNvXxWQvXbSCPGBrhH771d7Rcx13PnzfiPdqCxCHrnz/OfCs/cjc7hHojFiHMfkRiqaBjt6wRTF4+KvaiM973f5JCd0NEnMeCmO1gPlejeJlmyy15mt6kSdq0+khoVkDCcFAf47c9hQHXtRHEItHsa86L8bScv7GbxGn/T+amrs6qWtVV/8NS+KYvSgDFMi/h7l+nOn4r/GuYycebgTgFLqyjwkpLsVuSBy0SjbHijAnTFCPHuiyFnMwn+SisG+Lwgi1PPjToF1U/xWm3KImSptqV4oNfuX1ueLsp+OmaWfz4d78DcJ7y3PbbWfWPvqiNJVH9c9buXsZRQsIu1ua7Jd5ZjAoiigLxEUphIjp6v3GeBG7My7oXMXWH7CUzL33NUP4yf/qHnvYDflqLVmUqJIzqBfIuizs+XEsyRdXgjp+rdz6AavdORmd02BA87pZJG8m2Jnv+YNxRwBba40qdJVhgGnER8pnCLRLDcs9MUkRorEtP6dmKpE9HcW9GmoSAOWfk+Skl7ZBVruDTkvREgTXbhn0uwu4KQLwF8sRRmeF5WrtamNy8IQteKFYLXFXHLD4dYLmPmhw9DJka/dGDQf7ZEwQvavNpR2DangtxvzD6koQ+mGYjuTcJh0xx6uWSlf2ba/IYRHZwLvXPUEhWU+5ZZ9P2imhno7HAn2I4cmbY0p3GzA5tZfPO4s9ykQuFr0gOgMNfiqyemXi93M+RWjWD2EyGcYxBP5IKpcYmD/QhyqATqnicvrEFbhB7gGg38dGBuJmz1Y0rTlr+Ks7e2c0QViaNNV+xapzrR3MClDj3jVYdasw+dglapSzFoscx4kFfprrDWo8iLQpPfxPoV8xAJBi2QtJmmODAB3YPfG1A0b84xrzvyNRlii4PBvkyKytrkc5zONoZSncyUHjk1uUmfWyMyC5F07A3ekML/BQL4xZ05EIvcGRLN2U6BDgzsx9r4nZaHsjmXkUjzOkfQshStk57Hsw0afo0FATMFLsQ5wVuyee993rN/1/E7TCZkImIpv0TTUlh25azO1uTos5f0Lal9NPrX7q+IVuNbmh5H7W6h2WslwyGMJrXNd0TEViA+etQ9HKAqrv1ofJ3IKK51J4Pja0rMv8nC6C8XM10+8HEFqtnjZeXdf2Jwjt9tQs59nNNahOBR2GedNcePbwzq9ZK0Ik2HQoCJLWF/ExWpSeKrvpCM1CfQQvOuxXnvsEiTVVERX6JrkvwbT+j51mXObcuICFzUp+cygC5wepyqF/XXO294zSEfEVxL8rTGrqNjn6My/1rmIYDA8NN6vxbcnaYud6sYuQPQE6SqrsAmHi7ev88Pv7LZS8pWZ1fHHkOTWXTyQnJJpBSTQ5r+dDBCmXIQqT6Nax0hfB6320wy/dwNqEqTF5n6ldN7qMShwtV/gbhioTeMrwrvIvAVuNWIPdZ6wQDvdVB17sRXFW860muzvvPJyEZtkYNU3A1UCrapliOrfKL2XK98FPZcplYcloE28WHHESGK/vuhjlCvE2GsWjw9f2ETEb4A7T/yjSJw1ObryuSn1hzKmranbuFIFMZagOeg0zzvctv0gic4yseCKZksVBJEJFbRL/VkqVzou6x/IkSrUGzgj4NVfX6hkkH4L+8OuHGDImbYXNL8SuXOOXIIOFMllzGEGQ3BXwY62yBYZ+HHOmIG7PvHDvzJ0V4Y57bqWeCToZDHpxwGIfEjFkt1DfaArdqs4n4OhhAG7lHW7UGXwc179XrihkHXCEX0ULqwEbdM8Q4RgH4zSz3upMWQntnPlK5X2qTPpRH3br0N8DS0ZmAN3klxly7GPH9ryX94QVRzUbYsEGvp0HiZYD6A8gpBa838ybv4OUN2iX32CBr+VjGnwKpgfmqEr8Ol9wPmDa0YKFiIx2yDiYROKVyUmeZHRCATBzWmNgNtH53k6y8UtM0/5hnfqPEmM69eyelIRifEv/Wkfzs+A9qxjcy+Vs1ROMz5b7cczEFknOwiEeGYk5pFXl6xSQrqXLV691Rh2t1Fh8AT5vkFooLQn/vzs6tWT7shlurl1P+lpPNdafPmvfiSb20UE/XZakXIL2IkgBNL7MLQarUBgOY6st8d5Q3TKvnCfB0kaMrXgzsb1ELfl7L5TmpbRkHcgXXp+PruqIBUM2znctmBcgv/d05yonnQLFrIfql2s64r6VoUJo72k6ZHTfAkf8Ht6kd8m3onASSTGFzuFhy99h8iJnGpLVvUVVGo1ZTjoouIzZcdcpArhQNodMTOpnYc3kOjqvfdbopadig2ak0KvOWya1JcmX8ZnUzL9dODowbJBH5BYW7TIdrXiWqRmuU3pC3mX1rvmSH95D1d0VTarfiXq9rM4MX4wvMgkFuDnZKDbu8dUylmTScL4Oqz8VaWA3U5ScjAQy5Dsn7fkPE5DN+q7OE3i3iv9P/MHBF7V7bZMC4rjOJT33yNg8+HavBhA4qtxF7B+DPXUsL25BA9wreYSV+T4uVV4oscC1+f5rQWDRFbighZjdwn8CHzCUe7LrWlK/3A/exeUDlHB+J+LBwSFQvAV7MA2F/ucmmtyAqHsXjBB+l0s/pbFvNqAyRnc+8Z0sYIJiRjRkXGRvqYGTawmEYIHCWy+GFqhpBehremD8dgLp1XKmgODREZP7I7r1Jqp/EgT9Hdqa315wuZSEUnTZ73nRQiA1CKv7UuX/hUIbd3dk1sKRvMg1cicau/+5f+V33k/hrFWcKLIUmzE5PQtkiwNCyBmel2P5yVI6DhPiW9kI9wAnUIekUoxvDYmHw6Bzf1G2WHFcXjj8v3z2gOy3Hp7gILr7WzVj2ZPQ+7SyD7CQ+eT4gzDHtt+5fbT/j3OaE4SUmg9ltoyOOp3D7M/KZ7nC8PFujBgbTXcT5Hbd2ZyZ372kCTs9+tLrwC+x+SVC7RrX5gT4u3DuQK5SijO8n6kTmJ/2cuhRtjImguT5p9dxwr0MCwe73jiE0w7UauFBW9Vi2Nf92kyENuirf5bhUMj1aaJG2CnwImZ50ge6K9rRd+QH9VLK0xw+qvDnylolxJUC5/cVFjIhwMAfZQotsY/GQBGZOh6kAZRrF2mt/kN9XGj58+NJI4nONqwneMok7MdIhozojHq2taiAp8Ka6IeE6BuJSWeK2Ln729N7edGn2aaRgKrP9AbGCuHmtAySHj5+XynW5vVyPaRAzKyOMf+RqoIuXlJY9LspYveg67VP17P+1CWJRaneNYD39GGj9Gp296Wuo92mZSdfP999/vg7PMkLlizPvi6zJf3ZUu8HeV+KBkOR7LtVCaXjyguKWhqDpDWIf9uLtsSLqGc/Pj4Rk0PmPjbEkT0KTYn1ZDpN+mDFh8RBTWjak/z8KfMquUtfYfkefKayrUXAXQ4cgp61RgItAwyhXsXQtaRFdlcqQ1eOjB/O1Bd+3zqwDkd+wA5YL189EPGvJF0Z/5Hx1CFV1HAy4c7vfQKvGK2l98oM03BNUHS0IHLhcTSCkHW9I5hW2zFaGN/f5VH7FPBPb/XBYvHDVZP8Uasaw5e27RMId6db8xo9m3/iO00ojVuDFfVfXxi6+gdQdkCfY0S1KL2n0sBBm4RhnY1H68MuMOeqnZNGPPJbeoDHu2r6rrf2R9ASocjBvmeqH8syinSok9f3jsvCYujnRTedHgLtx8PYvDXhSQ4C0qSV5bzUgFUahKnz1JCxUlA/KYKcrKzLbdxxZR1xLgDWsNkd6NA89foxafon7/FN5pNBpsGksPX11wemwMQ1SGRBwuaLrxmlTFtSKhCBjC7GCF8LoqUIskXBodQtq9Np6hCwRHyAC/MJjIrGC2SiS9Fs1J1iQthD3h1gJXs7zdgqeOcOGaUrUoPjS9ImE2gJYc3XoIEfM25d2+w9dEjtBx30+NxRpNS9GAfsEA/ip94BdHIIu5ANn+7O5BXH6IGKBRvL2rQv6WZP2dMg1/2Q2lWdW25pNVPhACSwPvAa4OXPfTKdljJGGniX5ertH5JodTnAVo2F8wJOUV0qyTeoJ59jl8jL4alC/42Ostk0M79EshV7iffG+Y4owkwPbuXwoGmpjZ5TipagbIK6LFEf6Xdaf4c3r1Wn2fTRQwgCoXzP3LSo4x7SKCyvxEm927u9DdIWA1LjPgu+boWyhrHRDhx/gZ6/naCKqH/an/N1qiqMASvQmhMAlHoCBX2WLOmFqE74iMkGD84FnKSCBZA2zEbHDQAfBfIxGjW81oH+5IiBgM2cTz3A+CGB7aYFKNKjLlLrDx5E4950NZyeJ6fevMsdpt4iKaAI7lDtw0N/+3RExEuEGtkE/c4OB1CLOyWvkx3weZR0MqjEAgCHu+NN3l0G6vc2Md7MgTJ/WlxN7rxvTRVUHd/3imtR/yVFgjrE8FLT/7w8PAkwrNjx8Qm49/BgrTofgBxAVVjek8WUbLgPL2/kTbnmeCBQxzll+iK82uZ1Dc+1eKioAu0xYpDH1JK9/3ZwOs+LcQO/xePFF4DpK0ozWuYz567/awWazK3aoX6vdjRzlPs35oxh6KvBKTVrtaA4Y7slqdJfH+Rg8Bl0YuHz09bZJEK6VX7ulJhzGwR7t3uT13q6hB/zGMKx3cjNLUBTdRv1F2zCGQZwAUVTAamiN1qSfq6SFRNWxgDXf4GXnD6jF50SiXNt8hjkdrDMubtoMEeTUbc6+G1zeXFUBgOBE2LDrrqfsf00xDUjdDDxEesRYrXlSziOg6KMO9uS5LaN5Z4aqgCWBS5HDLnwa5CrBOOuxBaAccExdOCj8dBWChrYT5l1m1pvIFYOW7T53cdutXUopE+KBUVw+Hr2XTSvo80N4Pbpp4DWvf5qzVoGxZPtW5Whmxo5qzI/I7J9t9OV9r5xGIcDYBSid+uWFHLOpfiBVzU1RJXdFnKlM4WOhEKJ29f7AkcaGbIxNjg7/JA30KyVpMj1JCBbL9+jp6ShTxcxU5kCYyBJrq4n15LUQ88oIbxp/UP74iKtsC/gOsluCvJh6V/4uxnZJ3jfRFYcQQ88cAS9CRRAlyMyG/cURh/g5xI8LeKkYffKEPtQmL+HLbV5sbYXhyYbAYyvuDXBAado4ZGu3tEp1cnpxYFSgEPfeTfLQm+au2pjC8ILsOPGmhw0t/gR/08kKN+2b0Ft/qxG1p0sS3uJS73QnFnxmqvMvChm5O2zh+oeFqYpKJgGFosF7kFRWC8JRvhQXaF2aZl0jlM/Frm8V1y9ejwrLIF12WOua6VbAWjziPD4FRqnsvJ0AehMaHtZ5ZQDFJDC7TX/CtF/O+/dmBGSbSvte0SQkYOMGJHsR6O/WLGqnxJDccnMcUGqrnbx/+TdTF6VPUaoILTDSINwvSQBII6RS/H1jDRuhwL/t//N75yjXP9POhv8SCQYvjV9UZKcLUwBrvFT6dOPh8+sDzrgk2lpaJwnEoJtzxRFKLjos2XcyrlAN7K/uBFMYiILriL0oLP/fykNVGqvK1pR5f7wJtcm1EGov4cU2scf9f6Oj0KojBOSuKz/2h6Qp8JmXJPTT8+ApaB36Kh7lA+EX/zz2pijDjTDby00U4AMExTZzSKT1lemmVE3eVMaU84BMsAA5a8SLGUEhToRGGn5k6quKhmQ0M7DZXEz0kqmGPImzb6WrzaFf4DZwO87pbzyPnuNo6YzwKPnfB4GJzLyUdupj2ryXIYxomM4FmoK7oUCpzvq/WpQcvAS5dlyH8ZZmjcD3ouX7kd3IljfeVjzuf8O2lkitgStTo76EAfzQ8qL+dp0IJX1WgfUpdlT5RnezzTFgFik0owsknudKXEU0oGvTiFDsNYrgGeAyXYXxs3CL43oCgL9V2qbMe6YZobMy+a0PdztvJgStVPBvuduBnbXDobooPRZTtUi7rv6UlI4t8CdhNJbi45r/E/5aAiL6PGMageAARwhq7uj+0sq7ZPnjwjVZN8FKK7RgqbWeNUC4oV3RhXE0ut/1aFGymE8bo82LBxCEsqny4Z8D9hWuoVsyxsTn7+bXfdT2g+fdSPYSCFNgoq3vHL9UGOalFEfLznA9pfpvhq1fxvSqzEBcGs8MnjiwDUbg2Q1P6F6vDjXxYnGHgUUOAia4b8IxC54kX7qOyV5Wy/QTQZwR0grahGqHLzsI5aJUW/ek2FyiFAtQn23JI+QhnH00CgXMN/fws6fML3z4+RqhV19qQ6ZQnThUw4BXdiYx/0qz3eH5RVbv0+S+BMGbclmBINZeWLI7dRY9XMeph2SjnC0y0bqjNOMM8d6u/M0l+BsVsz+AKcG2zbNOc4gEx+Kh2gvM0SvV0KmSolNpzeXYtQNrM6I3vhIHczC1QpbWBAZNlJiIJMqEvPgG5DVXyDhEiaKZ2TF3UcbLMuRcH47q9OwzdSt9HSsK2p/F1NzJjB/TiDjYtJoPxaeB3iaG1M9MN3EERA2hquIT3bFB7hJP1JZC4XKIWgdI+mqfTWj1SlgiFNe0Qt77s+mijGVMMLQwGh14txOuNV9r7F6S86rfaziaAzA6aPWDV29OgKDM+fhaLEgOxJ+6eFWDg0eXMhO/Oj0Tp1Ly/V8IAv9y710nDH5rKi2i0AOYWUqw8sOLXDQswAOxZNzQVVDlGu85OE8/oZV0NlUnQCOE8ntRQkfbWTmHwZtYpAn6eagf/q1MHGOD2mwciWu931gbNR+k0JkgBRFHFenFjxhDM2Ms4jZktpdQkp+tznKRkD6sf+ogEXQlEU8doprq5VCUpG+hClyuCrGx8wnGzNnKIvb1uAWf2b3AdE6v07vCp/BYj3ng1ARP5TBjFkGlAWcWVbo0CeEbclPeatufZng0ldGsvks2paUirpSYZE/X3m1IAD7UIfGPcehttTyDmKMcJEO/skoHVJnzWDXDqWMz8eGKu5LSP3FXlNxIogPyIpBNCpmNAoSws+AuPKo2nP89LQ0HTKj0SAmjeesPKXoOz7rzt2KWOYio64VB8kDB/HhCbUQy0SXj9pnspQ5qWKQQPWhbiZxsMwPfTs3u2I53U+l4s8aqh/lOXlK6YuUfAPFS66KcSWvTHo9fT+NA/OCXpTjJoO0d3bUVeDokfyDcrHXsdptol83LOahvkrFqQT2F8PYXsQ88Uznb9FBtHOC3S7cZ4nCsvy+mofoThDuF0x0ZVA4aq+EpHzmjU4GEdc5YPssmIpRJwDqweM3V1v6Fbe725xRdCV91zPYnmHl53L58OWGMOKFKV0vIUsnQNSZj82lZ2INK/q14HHK4bUeHfmMHCV/4WcYcExisKgyxE6MGgGUkHMtxu6tCKxoZmsERhBkaE4kC8vT/lLxn6CkH8wdewHyTV4ZDB//tSloPWXmriOOJaYlvxMZa/COGBCIAFx3LIkZvLfF4mMhvmLbVkRS8hk+pQahyt04OIR/mMDHchEUHUrMAyRfPQ5fXGWEBtDonK/QbGvHhb0sb24vdXyhiqNKg5cBr/PKCPYSETb3tHa6iQb/r4Pt9g1RjJ4lV0+aldQZPHCSPJh+TJ9qpwrF5vM/KLbxj/L0FMB5k5wCl84PTKUcZ328C7q9Dkb/cwP6nifVPypIyC49+k5AfSNUyj0KP8naAfoBXhJ4qHM0VEMT3q84tnfD6zXU6qTV2H6msUcDRXSN5htBVxKzDOsR9j3R42OTG/9NdkDvXfVETv1yAY2dHqaZYEzvy84Krv0kyy8lmIQC+VC+5chxfNvc5U4YCUx92ix/T2uQSQfJ/39ngUK+a0MRl87fyn0OWQPj9EGurnrN6Q+nOh71ri065KTzCE0iZpZlDtU8ygCR+MWJ8lL4A1B7obgh82e0BC6ClElvMCUDS2VRXE7n1yub1BoFskLkFsVaLcI95vUfYUkPELEGPcHBgyGmJR/3Q2P+nQ0p9AMQ3UN1zjDnzM0ka4uq0Z36OQrzbJcfYShDblKoVwnnUCS//dXG1nKLCL+Nh/yWrYTO/Ed7ZW+V9nK5zxj5+XW+SeYW/Kqm2x6JaOjnpkxV12d9lHOifhd9RlG3A97YSWW4JBPsMYe2DQevvJVQcrcAYzXpnJbhcJwj887rRFdrjbQ6o6mUJQ3XQr3bTaSQEOaHUXjkTowej2xy63oVsJkjnDYxGXN6stRaLQwIogc6h9RB+JmHmjirTbkKbkYnXh5Vq9aeyLqROyXtcHG1+zwZFmepEH4JqNPMxy2VJ8gsdw4O/B2R0+KbNrwKHOS2ftO9tXO1rx1/obwggsxQhpRC7O5nB7Y51iubSXSc4Wn30COBw7PgSZ7fh1QSypUaXckWdRH0wHuBeUuuD9wuzXz6IHf6MF3bw1DzRP0z1tHKUp+5yLCG2nZfRuDfAr6xjTwzwPqh5E2L3p30/hIKezTcvmePo++fLqtzMcjWeSMl29MLOIjup9DZ757kLIbfY0jtBhXFfRUtQJ9gBlXVNPIG5uY2ltXR+kH2oqbpNx1Qdfvs/KypR8MUY9zRyfiQdGQOoXYkUITR8jnpixX4dvlJ3p70l+GZeec0BCN/D9Jmv526iACfrkKv1dHiPw1WQQlBIoRcyIKaOCGP5Ka95f88T1Bbf/4JwHLCDTfDNK0+jC3sltmPNHBATy9bAsQCCfxbpuyT2g+lIlBg6umPgVJ1c0hk8r1U5wqqF0F+86WukdEHsiTLe+PAvexsU7L8q/ewZeZ7AXh4K+2VfN/lQ2QH/hkP9pUrU4+ggjRn0H4p8hknxZNKU2wCu//W1VEd55tjkmfu8tTeCiOd3ez32s+Op4HKZ0TUyPVNCnKT/DZjNLfWHAQBM22Hv03naXP7p6fj3pqsvXl3brL4S550tHI8uQkhvJcWiQtILRExpwwTtZZ5vjbWwp9X5ow1cHffUtc0Dwts+j79fJ1WZFPGEU+Br8f3LUvKv5nTcwHhr3qUsdSaMaof+TjFe0Cj/sah//qr4a5zK5H4jpygCqJlVgU2QLB8rO8QvfHYhk5ZC8fEOALp6cPPEgtBPVZ1H9QCNBUb3M6EVLNyStUf8456NX3z4eC7bQEF2vzjXgMFR0Y+Yg3hQw6eNDxvLwDEFBmTpGW9TEKi9JXegPgQuX9vDbFK1O4e23bNpmb0Xzqi8uqMruE+YbLr4GRzwhRiNY45/57iZkte0RL4qD2wkECkBC0wKW6+rcO0Btt2gK5n7jagVHqBt+Sfxnu1fjSGt5I5VKjgJs1eGhqOFCEUWmdckVYylG5cex8FUHukPiXr6FMqFETtA/oJpQ7K0UmDD0AJfqROLnsMBh4Gh2nmO2QioUXnVsO6/ZJ4ceyhmGLX6MCXm0ZFr6ZU8rHpgdxC4TMiyTj3wDVNlmriW7zBLi93dMbcakH81UuzzCazGMQPaRC8pF2OCFUsfn75FlktsMk3aGdU86rKkvrBwypQdIpPiEYvrJRFnB4SVAKE7Y8ylItLq1PnPFnundCe7sJ5Nd9Oh/1Q3LNXI24/qssYYjoE2PEE4VnKFd6YB95n2owHuqu1D+CwduwlraYqTbfgYaNyu1WMm6UQra12FrlqqG/w1fx4LNvxEKd1Yk6VhuJ3zO0dIueP23ahcLFiYBsT+pJqNcwDLKcf+2jOP62f1DQQr1kL6dzzGToZmt+k9i1SmVwflBnxG4BPIVG49V4X0UnZ6uCnDkyFCKIsn4oYC3obTTSdYpidWIs9jXaKXuOQmujFFGh0jns2+PpdiODBmZuWrlEU3ySX+mjTtdIDzWDS5AMUmj4hHDu+//G36n01h7Ar2YDBcqMJp2GfD9a8PA0e2N5Nw9Vg1sd9HoqGFwDRQGGSre7WJ8ps2pQroSy/B+wzD6ambRDQxGaq8AQZKVr3CAP+izrXbEGNv1Ilq6DO3eZCnsyfenR5uVopF4AmkiaPYSZ1z4CQpBR51M19iWT7qioa7V3jLrP1jBbGhiXtX1x1ljJzbsNoTqqM3+psdCSkYu3HpsHy5oPTp7oliY32X5s78Sjnnx/T88rwyf/ZJgh/ZpDTeYmR//KNVhSyLN6CObZeqY27FpGlUgDZEvzktyb5OQTwQ4GdIbxpCVgNO2sOulxF1VL2U2Q419BBEtObBDrQnh1D8B/+iQzfQgqFSsvii/KtHn+1FIbjgwQf2sgylOhHRpjgQbDHvRIbBuCoCuLrzvI8Aq1RVQqMXELT9NKrD5n9BhF1Fi8airn59Rnmwt99avFnLY9o2q0kIEYl+Rjinu7136khWrAT1z+9acxaPG9O88Hxy1QhDbg3niJFsrkuM7cbt/PRPV3rwdKWuBZJxfdzY8eviz7R+H08Lu0z3mf8J90Ym8uwtWX5P7d2UNnKeFGe7Vz23f8VBpHSmn0d4FJFXydQiSKz3EvEENACjRwjci/qp6AfmiN/014Xg2xSKqR4HpR6V5DQ/eoXcZW1yXXhUp7nVIl6S9Afs5HZJUv8bh+TPJlHvP38NFVsjFJWkJ8BqOencJJ4jpetlRxOwU9RL2Lf3uCtR1QmPzJwE/70p/EVbfcCPYDJR/90VNnMZ5OijhUAQ31uRgQ2cHrQNVak+uK/2SGhFHxs+xfQE716Smw86Vw4GHzFHcFbFa/5lo+v2MmDsnhE4fXfhWaR03/Czp0QE107cE7hh6WiPf6sdY7puJD0nqhcbeIDYnV8frSNOH2GdgSYicqdSrEkIrywV7bmv2vw8XB1Eqck/8NsBH5ueKL+lU+ndJ+osMTb7vjrGAyvnE56gkY6opBmnKSyjMQoP4GX106XukUGVFTE1fcW48AmqzKIWWPIw98ecPJ31SIUnIcQkk6Gp3TLxgGgyNkcy5+KeQZg1gzlvsovh01ujFImQYycA9ivUnsmA7W8qxfMAjK+UabUvkGWfmCkWrzasENxb99n7ziXnM8iTHbQv1YKuQ3cqCMrbT+ZZA+n9+JGamTFNoV6mQxpFl+MQ4M6WzhytUvwe1kBp/mmJOSVT9UCuPgkUpT6ve3QmJnxPQfNAlcKSnOWD4PTKUJ+3SbSOKKAPeXrRKf5hADbgejJ8xYVvpflUsZX5SO9RIJMvHLaRwKRpKW76qu03WBvZutkQQL2vNG6Pyg58jGP/UpZRqmyrZRoXuVtCd5OgrdGcRFYY9WEFNzoa2btxCTvJ+5EnZf6NpLd0iGJINL7VYnbzYpTWVznacOxWGE/b44j2dZC/ol3dEEKKOr3HFb9YxrdO0jjm3xmweXiazOBmHyWRMGZTMVs2KNBjtOF8Lz+JjluLC8xMijL6yFgp7LDCSGHcU+MRrWlIUTSw7p6yPgex3pi8Mdxaw0onh92eqYHfZ2kjbBBARm9JZcOPeviB28g0PCQ2xnwbWNERUfxrGFh8jmTESTx/v24Tc31xHPcyLOW1vQco9+a+mHWjzRciCdAxo6eWmY8KsfXpP4jReG26Dm8SFegeTrxX0vLIIq80Y5rqhsMwTvGVhDx+eSGqSbTFdEex66xILLqJnyDrizNoIi4CHHSP/2QVk6LdLfsiktNgL9EiOtUYv0PX634mwG8rJcfmltsG6XtzsCd4JDj4xhzxYP9N0nt4ujtDCXdixTwcsNQdqvykbEjCqXO+OXFhBmgNRaItcKo0LFroKfhGhPz9/k8bts/EO/bnOori8szeuxBCS+x8SSepVvG8X/OuiIF0W7Oh3aPqR5WWq6nM1Rpxq0qRjaMM7NN+gD5LFJakSuuV85FXKJovCZwKOjCXsaL2HC6AvLoCoQd891Ai3NfIsHSXsQZZY5vWl18N+1tL9q8Ag1L84CQUfn4EDRTi4fW+ToqfYFTXZ1gme9XAL9Tt3nbwXfQILEhVSpE6Hg0Mykcq3ilyom6FGxp3XRshF55in+TufHUCviD/n+ylNd2W5MVURTl+1+3uWrkbR8FdTx/hY1yrG98po3tQgRJRXQKuXmpnwhAqqPKkymZkDOZlr3z/OgJxwztpaGd6O6r1nGclxX0fvOeE2ABAYUwPwwuba8ct4f1qHE47E5qy4jNpey9jo97Xf0hPzOxEEockcQKDAOK16y3vtiP6uSiYt1sOiLbE8X8Q/cWn4dApohKrVKGBv7YicIz2fB138XDTTTRNzIogb9ThZ/61jiBO3p4ngwVJJFdCKNnWNuOc2KtgnTdN2qXhlFMUTf6B3OE40i/0bamLFK1c1Mg5uayuXc/ewymrOywpcpAL7007YA/sYj2E8y6WH4dwvyE6kvp/ziALEAcbtpiuGMmBtEjkFOL0OSIQgitZTqpuMGtWKNb+p/kOYa45btWES1BqN4lgUjBmUBKYoINfIMAnEYw4KD4rAGvSUB9iQy7OHMC1pRaBEzc8NdLBTyiCnUYG/pxP5I4TECV/bSsF8ferbIoR+lksp7qbFppbBhuKsEK/jeDJSXn5RXo8fOWQXhMNecIklkF+gvk0sTmmUyoJs3Te3mIwPKT+Cv/f8LFK1laOTT5LOajpzxBuwq84VMy1VyjweW4+Vj2SQZCfWPGEyRuHM0QYG38M018FO2oUl0xv1X/8+718Q5ckqlM+6CWt4rOvzt6XudNNEEK9nLaZyYzoTyVshkJwcu5V3nH+0ExHS4Ols9rOuiJwamqRo+oCqTMlLVk9gUAGf7HEXRzPgigaAReCnQrm+0OUMEyMxQM4YYUOAl/yvy/SWyA13XGpjtoCQ6qsOvTMxPwoGhVQ0BqRIhxLmTerUSgwx2lMtQvIESM5d/55MNSEFkmpc83QHVM6W7YHozWMQjQChqW5R5qE+NJ+NjGxmCs5NwtjCWOXRTFHFCf1hMMVA0WFt7KufLfH0hIscfm2tfyQeUHeyWu9oggJMJC0oOU7SHpJlB4RkEqyPlw2iCyt7+hZQIk6EooCTZTD6GXjSJB+W5JpiIh1tHN/ylyUUoHM0pwTSF4Vr1/8j7r2XZkSRLFPyaeuwUcPIIDndw5iBvABycc/L1A/OI6MrKzK6u6dvdc0Um5MTZB9hwg7mZ2tKlamqqAQ5GhMYztEbQfovnWlnc3JtV13zp8VtH/Er6nRTTfTXwMQMd3RS4hfX7NAPUaPlfsDmFGYYWgQw7v/0y+S7zjNIlCv8+vUofblt02eut2qTma6qi6y/XSHP8uu3uq+cFlppw+UjC7iigMzmWu8VgmuY08FmgY9BpaV6iHnePDCa6cg6f8a1151KwpVjfWRL8DT5Kh0Lrkaa/XGQqyx9ge5imOq/WIrcAm4/m11ORN00y8ukpNKa1Adiyaec8pfvlg/b8N9TxajNM+dbEZue3D8bzznuU4TAdoDQFiidxil99VlUyZBktW/4O5d5cNek2V0ZEqMTEdCzb/gCWDsJdvKuxIRSN4J56n1Lne54QN17WEFPfSPAqginO3lpEqQfRwzSN2JJcBtlkQUjPkWf9BvW2g1ABKo3M+5uXL35QYdfUXrrkHTkov/pNwwpOPnaphnjvozeXUplgHT2eJhj+O1EO5qR6QecYD4MhPAZKzTkJemdaJu6g5YIgULhQtEnAMrf0gjvvAGWl4hhEBCAPRZmT/asnHFNl/MAAaUDGNUXv1LDPWtaa19hdCiPbpXYC/SzGKXgv/wBM3gfXlIUwHmEJdf3OsZr7TcDHmZJEwQxyuqKTN71OLIYqHSFlwBAC4Fh0xFVPInhTXu2Wf0dgtKWINZGOyqfLftf0bfLJQlGwil0mlHksyvQvfEdiYe+xjqaL7bRt/Kqom6vv6vUo4t95MN7PikNoMOo1d+VjbO+b6J5a7lDfdIUNFEXtfHuwo7waDL9BKD75DfGvhrmOz70rmzxwEHV16BRZPQbOl4dOSifTU1VrXzx41ldCK3J5wKBgL/0KCXkdOcg27FwDdAgaPhmbOOAzMhEw3W7Xi7R25zq699Z8UWIvC2/od78qA47WggY+7ioB6qGLXzXu0aAchRL1kh4dFZasuVmPSpzJBrL5y6OAAFbk5HfdBPHEr1DryL5aQEHlwXSjwOv8puvfL7OVH+1bELX4XtyonxPqxKrp+3lstgJ7wydayUCLi44lHh/j58ty1iJ2c7Zq8JOGLLMygasI1hGw2Po+mijXKEK74BVX/bg8KvCtXMqPOAz0K5YNm6Z/Zxn4DCYRrCsGERgHdWgCHzQrZHnG/+pqt3vOe+AfdOLw8nfAEzjF+QJWPkYg6FF8RmgG1hXICXdG4aRhVFkgqIZG+HCfoXOB/RagcYCUKeQZoo/y9N44YzT5m5e6UoW3q7Y74BxkhQGsY0E4c4fwvRyfQRU+dpevi+DP+x2Qx5LmnB99s2T3Ql5dgPBo+czYyuCGjoVBQrJACFf/sdb1IUrRmFNLI5eoakkVpilG3Ofa9xKZZIsiSrMO1URpFB3ZYKuCo/1rdO9T+7IM8AVs6vd7MJZmlm9zB+NDogUF5fC5a0qf8O9AWSDT+zPFH7Fnj0EPVJe0GD1BPEv5XdezJESHYsQoYcUYwTBSBhpe14PO4a++EfcJ8zCOa+rgVYEJTT5N6/SNUpzW1vy5/86VFm2dzYFM94kq50VEGwrGHfub65+VHc0Iu+XSGZ3brwxugIuG4CRsfKF0O0vn+35zjN1iWz8LwcOOLDJOouygjFYsQgjvgPmhSV9R+EXunKH1rl03mPDzOuar967b77UuR+IiTEBI98bmOmAFGAm+qXFfZZL5Q33XMofWxVdH1hUL7tWWOAW7AhJ1jLTpMRUuIGvjrtTK1bR6k1LfUW4da7iLbsG2L+VbM+fvm9pfI4aQ6IPhyKOjo2X4RZbXVs5Bh6FL2adv6AH+4JR7r0bwOqD9pnle+tTWLWdx0J8hyXI/seWhgOyLqF8Y2Nppgp8Owyo5P3+vx4s40RxDAMUXWeFzW/I2l/ODTQYz4PHL3x8ttBn+95uNTVZqh06KBuM5SrObInmVcvhldLkQZTfIJwmWTEIXRq0K488yRO3RUjNi/HKox2mK27BMzrnnHJN2y4/FsSIwXFvpHiPNS4etoljX31HnIGi/xCgBD0j+yzvEfUsjPkgAWYouQJtGdJcjc5/3e/NW5cCq7vhUfstW9QUpwws4p4zcyLvXn7II51wL3O3nJ9NDv8N3lhZgVHYMrb0N3sz/KI/8UIw1eZUYOn6CKuuDGEGLII/15MSVTOG1Xu/ug4ri4EVUJGBt911OHcnynTMxggjjk2L052Q+Q0ZhCn7siAHs/eu1LGq/aFrG72qiUwNnF6zu37Cm7CM3hiPgbkiDxAb7ymKQRbFSmz955Cagw6ovQ4FbPvV8zz/6qXP88Qt0MGg66ygbO/TtLRosiliEBjBisCDK34vrBQI4eVJ5TCRVYOljyMi1oXYlnLE874HMMnVHryCKCNuyhGAXteUoFzPVMnF/BdDvF5iAoCixzP9KCqBnZl+lZqPryPxGJwWqRKhW4U8FQxivXzgbo7k1HLrLB3fMsXu+8P3DDIfqt6mpWRnxKdt3WwnfwKx0Pgux9G2AA2WsuLcr4PFGFfAPfTU2L2ameRTimSpraGXFihQa7qF0uWxR/HW04dCLCPMtu5aTmHehPuYQWQAMpICHKTRRYoZDcvPrFBJ8h3trA3fqEPCyz/kW/6LdoeKSn3EFQievZdEBX3X3YsA4v0jTl3qT86cFza9CLyxhLn1EGfRYEGZefkmLTuUxISAACgMB4I3gEmVLptXTs189zMFQPCtV0YKrEYLG6Ucm9ER69Dlu49v1uWGmfFMUafaqih446+rlvAppF4wz+VgrIzbh30O7v4Ir8qgFSwsZOlI3VNmIBnGgYt8FzIcmIIlJYHJjXWQmZXlD8NK1Qu3dCwNpefVg6uXYPwq8/mlKxEhV4Fcbl9F5+6v58SYrZIq03dsFGJBdF8bUty8DBMnFSN9uVNVtcfe7hKrdYxRN8aD8ejfrX5524HIi6kaM8rlsvvmHTHrHF5UQsx33lhpmyMrytSonh5DDVU7tx+C4g+2vfLvAltTZZ/T3FRB4IIcXrLaeglnazThp5Z3ZWpI2LX8sxigSZXxDXcrhmm4H+ImMfakA4KXLVExued/kyHlROyTx+uvqShsmZNpDz3kvH4KdMHPl+fe0+Aa302CuKcdJ9rbMvSvoTKc2igd4dzkQ01HMY2oBIU76UcUXdbhb2pHwC3s9IxBiGDUxcz0JodNN/XKdNfMLS6DGIWmoP+tjmcGp8bD8QHYrWzQK60Ra69E4vflEPGzLZA+5PnOFZtHgGL0e0WffZFslQFZzAecSa3f3fWR5Jub8+pC5rNEc1zPwUVKmQnjl/ShNwsl3oFtyM/8F8mFuRFdjT6Is0eJedPI2mU0hi86J1KS9zWXRzCHA2yNr1qw7Zu0JloVpgDsS/akv5qbxM0qISdyYUFUIhI89UvazuoKHJniKbFmu+j7k9vCFylQJ5Ly/Yz6hpsoXWbtSAJXqRtY2jubT9MvulTmxxP4CweSBRWsN8xj0BwffJ/BoYDSI6xR+xJ4MIHuXsI9SlLv8JVHIdhMZH2ncdkL3nVrboAj1FzhTR/QxcVxSXLri7BhMl+oU14173U93vowi/yo61tY/TPsL9ilt7OWTzKjskbFis9LjssY7K85qQSP9V2u5b6wSzZXDWgrUCrWXzFlf+hc2ziRjsl7gzXiTCvoAlWN5YWLz6ZmvkeCkDFEQp+AyWYyXY6NNRHZiD11lLMSo/+K6SBq9xh9xXfbHcd7E9KC47sW9HRDM6nKnXJwf+VlIKIVTqkvV0wtNG+29wPCguGfRpKlJ9ibWyp/Ss2jgl0NdvM2E72/btH/oQlpAKZpIMl5ycKB0HAK90+p3GDA2arRRTRXzIihxXdkPH8aYPIq3dPDDNAKuQo4QEFE2LTEswDM+kabdhBdQjqalV7i4wNFBst/fkpZurqUmHqUPUj8Cx5boLu89KNLu3hPqw+JAGQEJ2GDkAPhrJmbuCaQ4wnPWlBnTt7/s08hGeRpeS/ZkvgUQ/PBdvg9P4ME7B/ZTkq8TEuIsJv5oLUqOIqBwdb/lGcXrcnxj0G4wB2bT2jiM02EO363oXeyby+aJS0FGpGvNvCQqQ/rJ//AwLS3WL/BbR+5XUkuxMUkMjnAM1qVSMit7FEVGfbwqKEE6g0w8yFLsoCFOOz10fHxwT9CBBg8wTP+21JUnVBB5D5ORZ/daCV/xKlmEzToeS8cM3kqqRJ+B4OJm+0RDXrZEwsLJUIjGAEhY2fYf1INEXW4N0QVQwWwwin0pFBCANe1NZNKhQFTAwxotq8DxY25/8une3jZZZd00lqcidtpNjqGUxZgzJ7wdLvBrVgFn7Sw7Tw4G2l0ixE76wu6AF/e9acV2tSEDQl7nTZ+cDLtKPesT9YuwXnpz1XcCyUT4PI9vX5mZRtKvAmjDQ4i6bldtcJjFZntow4qUovNvHhKH9oFxIibJXzEAMw+g1HyJORlWo9pq3ll7pB84mULO2P6oygcmUwe9fGlTxDi+8EI0B7gUBmE8T+SF3e2py+v8vkxJgNy7xCdBZUFsg8rTLa3f6Addxezq0sd6xbej0J3ZWLeNBrCbUYrTo0uerZWyOW97PNp3xcguLyof0q8+TmDC43Lag+VbgPX4PhlIzZEVUZuAVzg0Emvds+w2A/jQ2+M4AiaDZL45012CYPveZTlDFUZmrixNURybgliaisnAiTyjk+RYF5pC6xiZ0yk4KbF74Gq8sWLHKlCe4kEV15loQTCXd6wIiRkrmVaIQbj4xKegw4CRKeMNYxprdcmgMrECYCf8UbzOpAeXML5D8F1NGIZ0m2crgpQQ5WdC3rT7+YyEfTRvHxVR9R7TOe+DLZs9FCnvrMFxaJJegr8yxw7zgULUA5G944KNtBAnUy6oxk/vth4uasjEY+/1al+YDh3X2emhWnVLoFIrja0BuhuyuS+/vKSiDuO7+VmDdR66MeiqOmH8uMgnQHtALUPRJ+itz0i4WK5ey2pa4jh4gOYmzUkpDK5zpaBAQ7GA+pjlIX+/rHmJwFgXW2WtuAfK0YapULn/Q/pzBXgyifNZqRvimLDFUkb/ZwwTBhHqlJoJhUbnvUsdT1qv8Bj1n7EYCtjxqKcmySiN013bQFC55H1oFDJQkmyg55MyY9BhvYEAkePtPKS+Gpu8A4drKd0kUe2a3G1XqV+svrRPx0PN+CX54FOxkdO9t/6LbYhGyJjz+4XKP3rc7HcB8PVZFE6ZmVGW0ZMLwDHq6I/44UochKWoOsl7DaWWKTU/pEgssPzS4zFDeMyawtdHlSUkQppc9TYuhIE2XWJctoWNldVuN3UDscDXPBnbFs26S6+NESe3FXlxeyz5AMlm+kt6uKz5Ahl4mL3V0idrj231Lxn1aUIHhfdYGus+BDBoDtz1gW96zmQlNeFUAIpaHNrc8d7EtP5c8Nu4b559vpcxlJ0H6ZaHCQkSZtk0/iL5BVuz36ruOaYg9uoTOYRoU1Gfe9YHk3M6jkkWgYmujyu44+l5qHSzOIvjq38hJwzqGhxvyJN1Pa7fiX0E8S03u1XDXjea1clha7Jas+7mUX8b987HzVKDucm9Oz8mMODEi2aAN75Rz46/X4yjakZ9YdSFxZ8bdcWpfGBY8e6FcNo+EgYN/uQHa7+PU6HBln+QKiHDR1/z6tPJukvAiUvVMsYjN8XQ7NDp28K/3OkF4Zf+xs9EEPnyY5YOvMqIlHgzoT6PuuDcrzMKxKlT9Tt5RGAY05EwFZU34gxX+vU7wtdHx+CZPXcipW932eHMhjs4W/rHVNtbqX4bo35w0HzqNNgAYFW6cwYnBv5LXVMPHeMCgMvlkGv2C810eY4OQWf2YLmLxOmMLZjdBwOOaFv1WfSgV8cqlfS5ZxEYXKTW9KuVEFTOMXBXXEn6cQb42uCX/hA2eIclqfm0DsQRK2Xz9Oe3D5F9xOBBTal5PcsiO6aBvINJy/cjayvqms91u6XhNSmBwdiNq79xRBOSt07RV6ipWCT0/VK5pE66biipMpV/MD0uJ7R2OjgdTkNESSOmtrLu+dSutxvu3Rn7ELR/XvaqrY8S+GOXe1GavL62fL1+UffGsnnpw9DdFF6tkDrZnqJ1HhoEXd/cuMkVVx+ifBChZD28NF8wPfhOG7c6N9SkbnWSaCiLUczT/VuHYHjWQWpcUTi86UQckS0OUYwLzho/aycCIUF9X3Hs+Raqx7q3w3xdP5+PdLynqZU285tKfPDhOy9mHeUQH5M1Y99vTdBCEeleJZFJtDeo+gA2xhAzAnmlWR2rvCJeI56LROWsq3tM2F9JkRiafTvJXyfHo9zkgRiNzXkB/0CcUBEs1dVFL+fmCbXeed9xtXwBxE2I3oQcyuLwNu0WLojbWIt1zhnxOwsxc3ML0vO1/5G+eBxo0PKu52XkAx3kAWalnS5FFBmH5e37HtzHIShBJ4ZLEdZiiXOTpp7BK0TmX+gnfnXW9iiMX8cAJu/3jLkgaZWYzNlJExOQV+2WTkUm3pn+OvuPUcQNFwJdq7evh+RdUq+9f3lMr44VDs37LAPGbeEuCJzZa2ltt+AIwAvVtQuTlC6CFZFoT5Wh2Q8BiXIdXK8chD4tdk33NsCwBT3o5+tJcGmrfnXMZYohMaxx7tPeoCehBJxp7xYsK0Qfjv11fR0J1p1+zuFABPv6w7eKt/Aer0/FcQw3T9PQ7tM1bceO3quaNoT4HUpmgM4kprrpQbD4iImMzR6pn3rcDJ1rJkz30Uv3NwMsV+SC21UjhOS+8vwRsHdKAu+KHCvVpx2B1ekZlRXIKzvl74bJss+BioA5w99XmUps0kBxsNeRU8lvcKAjBjkZRGmRlTrMSqI8SXVqkVFhviw/HVyDVk5zHP0EVZIUDFA7nSN0TrdUDQ8QV3zZWdOrooZuK8umhimfK055iTYEgeeDFw6zbQzbqqGgguVwN53Q+rqW6DNHah+fhUQlcV/fZMRRW5/jy9cTy1qKHnuAPU0EE+8BJVm7n92D6nOxj7gyG1NJNB6VXw1TGFWvnotogWnjxGm/c0R8aycxCP/umMRyMVIev6G2aw+1RtmH5HdhWuc18rkr1su2+VeWalGlHtEouDDmQT90MUyB40fQV4XY4vlhmqjgi0I4ujLvvQVEbuUtt5ELIbAVThaWujxXEhMTxd94wVS8QhOo00SM6zhiwM0CcCPS9AturUqFoXK4BOjDNvHUMC3rR/GcQmMJCriy9IO9E2e7hYfX1BR2sbTEx+U2aev08Yiik/7YJDsBx2N98q5sL5tN3NBX0nVD8+mb87NI/YZ7sjM5sn/E3C7Vl6IdkGWezo48q9OScFf+9tBD+uLKUBog35B1tRhp2Hq7fzMsX2vL0j4nTqWJBU9sVFH0TaPusu1R6u9FYezyCzd4Dp+3UI1fN0+VBgJ3VwvVuokmmg5UWptqpVq6X7Pj4JZQyXT8ymU6ggA9+nhvr8zuaIodu6vJNuxKVDqaP6hsDeq1lY4whw5XlGJfw43i8jqo6tKamiIucxF/3FBueB3P8VBiN3U9IoUWsiHOOYeWg0GXNDh4TVvE4Z6CLTHzOZNI7UYvaIc72dD9eIkZyfmyjAxJN7+Wobwe2xufTa1TZ8BL9gb8zU2opQGWdSrAlYgf7Xprl5orQytaWXoOaTvGOACpZmzzEN3LkXWdWcbh1+AhD7e6xDFxhbfJT9MHt6S6LoY4Lurg61Yo1NYEt3af+fmoNSOTdvm0Li4n7cRiaMcRhAlawEvqWPfsro/pkMkn1n6sAdnq67tCNYZtcnV6O+/NBKpm5ihdVIM4y446XFOFeCWtuf5N96WMGKTAD++sapY1Ixl/eNVvL4XC8caelletuInjoDzYilSFnx9nh7qHyDgkxOjYfSfkUqOyippe5Fo4U9sxMGQxrshsImMWo4vig/DkHlHQ+7HRhtYp3IXt93qki0ivpyjEd9VojodtPp9731UjWlLwLPY6UGv/a7bRPGH7+bZ26Jib6Tum2cJ0Q6w55lDuYhousau5lLt8/LJoSPUzPqjXvBwb01WrwwbUqc9k/wbm7gb0L69JgGlkNNM0mSEM5SphsvxRKolt/R1mF32C1/oKJHlmTbNHcm8MQ+iVZLuDOxMBCPyFEbSh1vBHK62itoezV69xCeoImlmA8jnQL9iXGMLs2imOvA+mrBeWbQme1wyAvobNok3Jz0kgXZJgPzi3fBVgIFOmMmtAW83zZwSLz/aJZapzCbrtcUmjNnM+78w/ClXcZ5MUF47pzSh6Ze6qzOqn8GsObBcZL2QBTqcsNEC9L/YiLE9qGlppzK+XA1eiwL9ThQ9NO3hsJxDq7d24TGZMvo/Lsx7XswR+E+Zr7GpQ+8AgaZOLgH61kvqv9wmU6xF4Ec1eiMamaNHxVJtHpAYm0R5ETGjPtkHQxHtrNXWq0G1ZONV4EsN3/llycq9MRM2Rg4U8lGoFfrSlFxsEHxJteKBtyOiHOhxXgm2zJBsmBzvdF6QWYGemUdJJutn+1xP6gdYdRbAw9HsxhDjv5eLvYh07Wh7g1oHhaZMP/JBT5C133J97ogld4gSBasrbJ+GrdkdwE19nvjR6k68hCofDCRNy8xrukpSDfUYuoFWJC961mWhd5WtQUohRhuG6pGkGOJ9Z3SLjJnMva+IOnmsi2V5q4WN+Yvyr6IRh1F+WG0vMEvR3jI0d8Ll8CsA1Rlv0X8rZf61wG6i7NzppZSS4LsA3TI1MfMFsW0rRLuqFQglw1n0kZRI+EvIyk8hCaoYLxfnjydKmdMQwJOVkhLyTtRySl50JnDzvX+WMiaGHz3tyG05ZFa8lrlzpR1exhEufpRW+ieqIIFiMUvbr3SsIoQWzje5XsLwowwORYLx7HJZBk3C6c2677lJOwhEIY7bEbjN9pNE+SX7PV7uSMfl1Xqo8BPuSPNBWWy4DXHoOy/BVttIw2ZMkkSlx+/W14aJCh/Qo8sPNAE6r0+ecVXLGQoHYeA8n4IK3x0k134ntwayxvJoQ+XwWJpzmonTVLnpPyJJKGjUgUAIbiiLr1nuQ50p6mCqrLnz8EbPiMV4jolmPKrqQFeNQRtfZjCrTBtrtX1lCsfUbc+mUrdSly3LKD6e8kFa8AJDTbMGqg22HrNEowIjtE56Uy04QVUl6w6+8Ky1Z8ohyiztbh77ZitnxmSIGdoZh0rb07jKQMyaQHeKXAa/58NiA9I6SpEHvEIwmijhVEQjF0rJqhUIF4bvCnUEy8bjiC7SzVpIEqzufG9JaVoV0+oxtdm7+BaiYoFJQEB/ph7zPG1kM6k4KTM/fNgj8YpnKPcbIa22z2hptqHCY8XYB1D6fBGCCSPLHjk466tR1ybiXzfzCivtvbzJYDJBoLfyFKsaM2b4QFQjrM3Qt0t3rG7VLnSzutazQX9FV1MJs/bRCTt65j1KOK/bY25uh0txQJrex3f5SIL7KtG/rAj6MGaTEEQs7j9DNrQ2JXLs2S7k8O+5I2ucO2NoiSdqCgXRQwAcICNh+AWvZZIvy4Z1pdDRjyqsv3oRb3l9Yv7zw0j/NFsjSWjOfo0b6TXRJtEwy41dpFopD1joPimEylzWp9o6pKCacrZ0HWrt5o1zOxsYHQgQwin48cydnFXGaIJCzrzZaKoiJep1I3DAGtTKBLFYaw/q4PawwI3VYRtRBKsx8iqFSHFTBSTqk6nm+PX3zi/RZs53tNMEQUzEtq1idTrCsa96+iy37RNcnR0YTebFV3n71X/7NVrdpeEVWfAQhFCQdx3i1sItogIpy+8X15ZmR325G4xtbSg+QyVpxNrD9UaRDVr7QRRDv93mf9qc8bBjtvMdcASWzWH2VDKLI931a1x1YKqkSS1teYs+KBLCcmpiIBSaAXkwuHZ1gi21ReDOaN+jclmJaMKf5wxO1IwXjabg+xIPGPrC1B5/DqoNYcrFcE0PMQvI1uZpD3/E6zR8erhX9mN5oV8Q5M1R1tE/wXfr352T6upzCLWYKP73kmQEd9b+hqaCb+ebO+yXHDAF4ZM1FhyaRz3zSUBKOmcTfjhYe4euPGmXLRaKnhCB9VRuc0wVvxdwZe48lDKqSfe37fc0e8+Fqtp9aIElS3+Vc/lKVS4R2VI2bl2WbquVvd5GTJI/GTpbV5Os9zcdhb164FocGo54w02XZ2wCjlIINFAlCJPtewJk31goYH5hPn8Ru/e+cMTVpN0mEE6dSJ8bo2Ce0M8hAKUP0mEvlp/RQb/wKsUud1D5qBLpoU5BELwg0rKEZX0LpcPFyhh9hNuGffBlr/GW9Moa0xGPyUWAgy0k/2KOqFK8v3cBQSPvbo1WhGC9CS6A1nhM3DSbjrh9JHNH37I1Ix75G6hpL1IR8ppGJl3iEsP2++d47VECcpiRXGAHhfnEEXW3i+K/StdicuO4ZQo+sYwptg0usfUe6rhPpptMgyLvdO9FYxi8wc0j8wOh+EOSFe6f+JQXu2Nq9XU4JzCrh9AsqaK5665IZfdg0LdMqPeGi/SFhRx1utCXqafjiLdHmvDqWH/VFa1/23g0/COWx7oIM0vAjRfBpo4YCF0+wC28GPSeGD+NA6ygWtIsgTf2mD0YvYQRB1o02mAPqLKDHAhc+J4w5JWZetQ/xoFtwaGaY1xjPOtvbhflBwlEhzXLonV/7du6zeJflWOZY4lBsW2GHlx2BU9mU/3o7f2ZUrO7p9YqoutLeLdaS+/dEJ+Bd2sIrqttE3ZQMAchOWaG5fXhH6exaxFFZuM8vZmZv4Bd+Zq7JX1zYMwW1JwGHy6/DMp0HobpgSG6Ph9WDyo5qmcFmBlsI5xVVqM21l2zfvA/MeBQvs1yFYbg49ywnTiz7yLIIQuup1dHGvjFQpx1/GYQJuc4wshBcUDtD+iWUltHFspMj8RzqFlLy9cq/xLbIav0R0Mz5KvKarhREO2x2CYPuiqZdu8uNEWpN2N25aubYJ1Ms3TvyqOf+7n2u4yVNppJ0AHHzPP/y+BtWzf1UJsCLzk/FhQpVadP7ugpXfneL2yCY0lwj47c8nolqJnJj0TQY5PtCVhZ9pdNZtxG/JHVFSXZXQrMDjvOHZ8D77xhuIfvvlR8mbUkC+Edv1iD4pbHOviAChfXljLd8OSwVKZsYWePaCKqcR9feHbpdx8UkRiS+0AOth+vDeDSFZhrqycJFNOrG5G9qM0LXORcdTSPauSep8Jl9fhmHY/ILC0PGVmYvyhJx06eAXraXvQb7ku+JkbQxlN6H7Ae6x3xh1Rqj/ox5KSRTrsaYz6Wb1qQXgvmeBxT45cIYAPZFnlB24r1KJhn6zj7eZ3NkyNS4Wl5M94Wbt/mdGQBw1zwXTR8ELLuZU+dfh/g6pjhpHZdkzUkoT5JysJPfIO3ArGquvk30LpQ9HB/t0R43d2fUYw4aK9cYXLzjnmRC0/4QBiGA4MXjGwtTbp68V9e4yrtwc6I4SXBmMdg7Jxt9L7Kq7b2udycnv7xAa26stp4kLRr73uFwHeKNNsRXwqRJU6/A7RucOhgkdYKCruutdSo/0sc0ExDPVUxkkqwwIj8EKlUMHzkyfaWPAmJTHcResGNFZK1psNqX5LaM9H1K+3BnLBBg5eHoY0cXmJZ86txTPphL23P27dJjxKIAiaAUlWtOumvPp9gCm1DBJKNNwZJKis/PY0cn1/ThpvJNU1cODGZ7EtGZA+Euaje3pQybrCuXnH0d77hZpzBQizvw3xP83c2PpZmdjNYcHDpQ9tisLkHNurNtEhFMM1p8NeTDdKXGRyOREct83+8BtuBtnO3eh93vBa/hGZGPdp1mouNvjblBF8j96fKZx7/dk4h5aTQg/SYjvHrKCG71fY+DzmowxAhrgvd7rH0Qop8+c+J22jd6d+h9t5q8Qd4KnSck6ng8cd06nqWq+99OfszmVunMiID5NHgGvuBn1Dgs2db2+TFW0rMKJJUXmLk5W7bWLDrVJPHFoZF0DQI4XvqCHGkw9oD4lAWtSbsVaK0zYJ4NYk7yL+98MWGmmyv5nWaHDBORBA+zoJDcnIG/E89rQrRRehRpX1Ozeo75EapawkoGn+Hic68p7QR9fozei6402epe8pebutSHMsNso9wVXhnLBmwhmmxphCje2Qx+KIMj91fLPOo5mhf9oN0q6vToVQq7KzXTIr/bqSkn6fDtuczE2T/pSdd8qzcqzUJaegpXTGrNphyNX0zDr3oXoTu9cGujU4pvYWxmqdR0cbnvZjFyVTKKi4kUyYjgDkkTVHcec/adJIEjfVWsGUIswoqgQONwH6us1OCRdke1L1zeCaPXedwnO4xupLf6ikDGPDAr/26EZI9CTqch7Znh06XLRiZeG9z7GWXTw2olSde08RsAa2Sm/mCyQn95dccoM9jESiWIwQdsJN/Lupno4Q98L2xOuctpMrBG/w7ZKiFyiQwfLojFmIZDz6ocdOBLmMKnZ0ZMwLfRRhGkfBLvQ6Ja6A7D9fKzLZaNMD4WmorlZRGlaRPygYSNF5CJQ7uKvbtR1Z0982UjgvYpxtkplQzsQQpx9aKYGErm/huU2LlStSP8SvG6UQgSGQNnSlrnKrtq34PpbA6/y/UOLGe4Mi2OP4nUJL5fAcP/NciP5QMNeAHTclh8lkqtWv1Ve+rrih+56NCFzKZsknrrVgmfD0mBiFiwj2lApMW2lKRXiahxsf+p58Dy3HpK7G4rznIMrBRmOKTSCGNM66NnkvC7W/K0LZEdhe7ynptGtAPdHFocn9/Bowh4tu7tWI5saRA9lp8g+MX5ZavR9DJUwPQqF338QjRfprp3uDKxpIUn47qTSMscfcRlZPnz7otrgqK0tM8WOw0b2U60N4xVt0uLzQChUFZH0dNcf3WIhAg+81lNQ5larT8tXpA/ziWvgZIPFzloXuC1d6IRcnip8jRDDy/xpq/Nz8MyRphmrBnYaFGj96rskG85EeExUAk/Jt0Wcw3tvobLqWpqsyVoiYMVm3G5itGvaEr0fL8ag9SvJDa53GUOZe6JCL9tfHJWighF4eSxoxtW4dT6rSDwbilQmeHFjkSXnoHvnNrW2YZQtWXx8aFGWnxPbWdKrbzZyMMqVuFItIOXCQ0VC59sMynblpU2/U2d4IMdmjMkoFxHOV4S6OQFh5SmvHtbrOlb1TRroe7ISRUW/zmHvq+afpEi3GFaVZNJo1H0yE+Kwu5wa9+2X/DBynZffJxyxx4Hg2xcD0mXCS0QXhvlpLru9YRn2wpKTtZCW//4UGdwRJSUoipO2ojR/rOEGqHMCmNAH37gnrWeXY8d59XQ3UlKyP12F2BYw+iq6WLYKOB5pJ/2g3zVkOSoHdE1qqZor4XQJLd42laFZfTfzIgY/fwdOkKr5Qf3CAgmoUbjB8d+D8AG6WOYiWG3Nxf1/a7yD2LAYhWHAvLQGElKWfmBRETqUDydPCh6x2daGOix0rjNBsin6bdvSH/s7uGWSTwyHXe4Pmt9JLA3TCmvy7vdtpvkh5dEILI1dC6qNzgMHpvHSNbbl7Tjh1Q5d7A+Vh6MABsevyAlL+51sqVmS8o8jqtae0w8vlHr4I23cFU106owow6xOnYWX1HwRdcPiKaJEyF4FiLZf7jhWTqMjazw0XG93VO1nL1t7tHY6rpU0fsjWXYTlQiq+IoGThzYqnHdvfkQvRUJu2VJ32gCYW+a3XsPehpwbLN915t1j29LrDHg/aYFl5OGupVBXVKR4Ayu4ZUYOKZepbGBU+zs7OuWSuNK4hrR54P1i3Zi4szFACTB7vGJmltSGBfh4ugKc/kVfW9GhY867d1t7tFlF2mebVy4NKeeFlRj/awvZaNnguAydfczsD9yDMH0/f7CUfDNUfj3bHma+qol/1NJNSSXzOFi610gg8TQnzfVT9EVDMiYaBnzouOPUfVxXfm/pL3ZGxpNihPJ8bF+bx0lHEJvsMBAiwQHMljZ30yn1trWSvkeTleGUOxqzgF9beT8oonlncnXUU91C8CmaCuvJiGdTL4IRwB3cjFNX3kVBm9hO+mXAet+BDzcXqTmG+F+6HvHRF1etPzIB1CI6fZ9H+pFRc4bu9VloaUGhOIX9fDbNKIyh6htgqAym23sVjuyds0+rwR7qbjLFjB7skdMcKv7ppBulzzJwz++N2YieSZ1/o0mkVCLtDnPB1v4EAL0WzKEQmr7wLpdxQjgR9Ld9NCClGTilUCj8nTvX2xayZSGJP7it7WDef5j37aHC3PzLori31D+94f9N+ThltAYz1m/gjsIAv9xa8/mNTv/7hYq/BvKdaeUDV22ztfzyJ+//W84if8N//Nj15/3CIr6GwhrB/eO6ruWf9yn6b/RxB93y6wqyj/fif35ZLz8cV3895eAzeY/Xg0s+JPL2vavnvz+jUDV94/PBBzGIb01Fn1xfquqSpC1/2+gWufv28Ttlv3x3B83lvVq/7yRfYvM+fNymNdyKIY+boV/v8vOw9Z/M/Ae6Ln692fUYRifm/Bzs87W9dEmN2gk3tbhuVWuXfvnb/OhX//8JUw+18s6D03m/zkuz9iy2VmtAXjB3zDkr+vwd03Tf13z5589+F1cf3dhZnP1DFg2/3Wvfwbvj+ZQCv/rBmjvv0F/gxDqrzv/3uLv6vr7q39s838oKcuwzWn2n03Cn7O7xnORrf/Zg9gfD4IZ+U8lb87aeK327D/0418Jz58fNYfq6fS/SywJQf8grvQ/SOAfff3zc/8ghP+9I/8P5BL/n8vlCF7+6w7OPn+eiXssU+6vH/+GP89wv/v47/o/3gNx2v/8LPy7if3rZ0kM/xcNw/9w789O/NPD/7FlHGALgJAqfZZJnDxm7bBUazX0z/dJhnUduv+4Qv56lmmrAjyzgpXFxsuYpUBi8uoE64+N//x1+gghkEz2N0jZLOzZH2MF2lrKeARj2J3FHI/l39JqSQeY/tsxzM2yxn/0gs2rtuWGdph/o41CEA49pv9fa/Ov3/RDn/2LBfy/AzYJCPobKHz293IIQzD0T6AJ49A/QyaMQP+nMJP4J9nk2ir7cyH8vYQ+33T9j9P4T/Pzj/PaVd/vH5iaLdUdJ7+moL8m8r9LOxAfAKPLn2P+f2gKcBz/F1OA/fMUEP9iBtD/UxOAYP+vUFr/rKT+cQr+Ts38ByUD/0/0S7rN+69n8P8jvfLXCvif6pW/Wvw/rlcQgvwb8B/+vTzhGPG3/5pyYeY5vv7usT+XxP/wdQiK/8dXYSjxD8L3R5P/q6pLfTWqa+kqmegadt+hx3y7//ZfEM7/rXwJ+9d86d9FD0PRvxe+R0VB9P9dgvNfFkTi/5Ic4sg/SAYN/Zck8H9BRiBE7z0dVBs6v+XBwJLU4/9XZOTfGTOBEH/HmP/63X+dLf+n0oWT2D/yZwj9/3fxgoh/oM//d8Xrr9f9v1pBYv9RjmjsH1CKxLD/FTH6r+nOvxeVfzmE6P8tWUHRfzC1cPgfGvlD/v9JWP5FU/CjXREc+us/9B+kEP8votz/t3r2n78DAf3vVbT/6bL/z6T8L1On6mIwj+zvJ/OX1QT9KxPqf2qV/WtT7D+aRY/BhKI0nef/tAyIfzCt/gcG1J8d5r/xGv8byvxxiYhjXzwmZPVhDfuAFKkYgONKd7xS8IrnX+B/Ru45Jnx+8gaUfb4MI8RK0wrWx8aQ7aZZ/Gx92OtrEOiBmpvUXXM3b8R63zvEOVZk2YcvSK6sEWxhffNqxoxq6BhD6nJhL1fB/jrWMDCaI8oFZWsvV95BjKz4IXtZJFECLnAYg3pSp/sZNY35s6HkhGzzhKcEtfWZ/oHJGu57NHhAaCPhFmeLRSw0ttDOceCFg5EK6rl+Fwy4xw4mGzIqE/LgT8xzFscfPnNIzNExz4//nz77ukdNbAReS5RjynfUWxGE9xIs1kWpj3Wqhl2llNHh8wYBfUVT1mxBqYgQQi1nHfP7V2UL5CpJ2zuAwF4FGrw9lJXfXMUeWhvGz1/chHF9ICZYSO0Kvo/fJVZJvunC41OnQoXxrC8OFHqv58HzOl+Ui9bpfoWb6jf9zEKBZAURcepMv7nIs6y1cZY2fIfW1vKK74V20Y2c3FPSNF89/eFGsj/ylGI9gxB2rmxEkHeohyMue0Ul3Y4nhRPB94qod4a/4tz6pGNHjl2fKWRdq/edwWvCu/hfI8Zy/XsYSFP/ftE8daX7LaIq+QphuFApdu9mpPjMWL4qkCoUIBMrA0JTdoghPi/FWT6KhLSwaH3m1eVUZuRaE6t9zoVyCIQzF7MIRlbF5RD16PsWimVkwuYNwrItHSQJCLZzVE4Q7tguZDtnVEXQqxJ3LZLyBSUIhSaGXxc4u3sZaZUU5feNyugxiF+O7HPXJ86VgrjI6BUGUwa2HT9zlBHy1ULrxL/z9eULzstgmvdwgAIi7Aca2NJRq805bJbjhUpMRjrjMyH8IGwJGzCX7SNptWeHpOZfPaD7GiFWjxhI8jbhyU89m5WfsV2LrjhuXu/UWDNBaIj8aLPzs4GjVgHpFiAy3+bAJgJGN0zciDKT2rQhvkCiyhGBVLy9eBGcVxLvvu+gMQvHvms0ttQYttPfVAACDVgzVzrzo0OlLreatSmek6wdAYsZTZpzz76QLdPCr+gI48u1Dq2gKrbcVpz1OHpiXeMQTdNAyS2ASQfuq3hMP4XGgDUdCi5NpXF8/YpwXFaxnyi34bYSQmvZ9dyXRKu8zh/w0Xy0M4oqlDyNufi1oRS3EuqSL/pXhN+9frk1mfhm2jlhzYXMsxJTSnlj2dEpGYiEEkdup3sU1dzFj+lY2o2FsLMtW4LzVbiWJukKE0f7K0INRtiyJgqAa44NvZ6/UxDuiHXt+Sx+5lnXyqZEEYL1Tb8mOeXClDiU1KscJWODx6zBk1XmIuIorIppR0nmdAbyuMHnqNUT3zkHvVJwqED2iVd6HuxvjlXJT4OjoSn42N3vG0IrmFLlZJWgejHZCPuVlrY6ntEElEAD8/tyAgFMG5/e9KhX4ED3ImDqLD7f3GWYcILJkSeSPBHq2kDR4hNH5fpiJkztGopvwwSDlqSTqUKSQkq/uPHiTeWX8a3o1XPx8rih6YJzpd9YTofTl6i+0wSIi3YggUrGtnOqnXSM5kb49Xmj39mJF5asgpZSS80jD0LhFr2eE3AY8Szf1W/OxTC8h4NIUOPzRxApqlAnAZNRtYou9kEQYe5vgdfZ5lDDvauyBUefvucF8cvTjfNu2O1Lb/2+ZS35SkMZon6YGX9t4oeukMRdjQVEOJjOSJ3R/S78gfFS08QNgrB6pINuENr/9fz81Avph9bL18LQP5OZOWPt38fU7pXmV1f2tUYshkneiagj+LAFuqheWaRxIa8CVnAUWPKsV56c7wsv7c28jNOR6ykXaT0I9ukLoSBIZMPDW5/C6/wMLtg4/JggIRCLOUsRFfWSNDXTln5wZgvIpVLKTH8M5aY0dnE0THHL+2etsey15g16J50t77CnDJ+zk5j46frvPNk5/BIq4Va7xGmpfReRyjUGrTKwl8mzav0HlmC+5u+wnHIZvzHJ4RUBdWKSVjfZ63WJfZIxYZEI+xILqBHFKPTrT8IYmGdJUdj9+gPWU1GBk59fxsTU4sHgHP1SZWTwkl1Q1tmo9UvXnU99DU4Uvx5VlL1TC2uWxftNXFxv63tncl5mVEy1BAS+YSi/EG0gIfVQJW9hxKtdFG2rpzg0QIBNv1tqkR9NiS6UE1Un6rAhG54CkCIynUcXU4df5gueQn65SQN/RE4MAym7xLNQNdtFcVcCo96C+GnJLfjdg2ZYiQEY1legA43kG9uvuvtHmWwNfLQImC91I9kt8i/Ji11GCKkDN5XA/SDMilefkNmJ2FOqqkUXyIluhg2VAz4BbrrO1CfMGxNSydrAtJcv32RwMqTOmwYpAdA3N6zhLW7vQ7RUSpZG2nM6tHlTbMEsbHiHIJXR7E2jcXAQnphlmJIfkdcKQ3TE33G0W0CqKP+dpk9MVmLUzPNG2nlaqd8PRwGtuL9WyKcV/wCgxi08kUVdYU281MyHfrxLmcdYrkCgB3v5jDG42NuG9/i1/bx6hLrgtDa+sRwKn1vmI5tS+D2/fYEgzPcQQtj+EJ52yx4GTlQQl0kpeZsQuWULQhVeiDpmN8KI0KsCx7HY/C0Z55Hgn7PRwSAwIDw/dSQvr3rnd/rhPs4mBKUKxVenqJCYIm9pWZne8e0HSyfsWttF5XCz5Q/xq2zgTIHElUcfgMhWOvS29ytFXzi6xj2TX45gvCoTESTh4KW0DcCW7ik/ss9gW1kHELYjlydsqxa/8OJBhEOdhTKOwTGqeKliuQYC4fbB626C7Hx4xh0g1/IrU7CUJzj/1L4p+I+RHsAhGheuCdqUxX1PosG6r317vTcqXvMBhR17n5s14ns/FW9OCw6Tsqz98E20+fXPA8vqAcLgQdmNUwgNr/kmc+kCTsxA5l/ilw5koea9z1ZbEvbQhQqM07NmeE2/i1cDcoiKP+mTla36A9KwuJWFr8YVmBBopmIin/e0/iJux1+kQokwO6wr/UcHsZ89gz04JmmnaNKmogQEY95cWlaPfuiRAN/vZ1ZjoV8SoVu/ovlDLHENXaVyowOwX7Z+2SBN6guaxujgKZTkUiV6uPI7WgpbZcghfOy1l5Mc9mFdPcxehwIYx9kfAMFIiqP4i2kvAl2sxhZltpxZBHM6kQyi2zWJnGd3xhi/jhp6ofHTn7zYnznNO1O7MRL9nTtDc7weRojUyxkQvS5qvBeqge44AxPFqqG3RQgnwvvp3jiC3GrCdBlsYSZidXJ6DGETxVM8AFYRJdqa7wHaRZaOKW2R0zUs5oBY1tgNah6z1EiV79fxhfhPPUCXu4ct5zonXnzOQqK8oz7tv77jjgDlj1IyxdM8iOpQpoIL+eHwgL50O9l5KQIjb+aHSOD+frV0zxcofB/yiw84R5B81vqtyHQB8T1MysstoYQ3olQYt6fdUtYayK7zKKG3Sb89PniAdB6QWveJSzNCiaeYnKuAWJRe8GPhFaiLKhqHTOHtROCkO2PUOLTbzKWXyjqV8FAeuXvrARzdh5cULDbafKUOG+dZHXUk4aNrKWT8uvgaJ5SQTM7/h6frWpYUibFfs+9484j3vrBvQGEL7+Hrl7w9uxEzMRHdNUWRKR2do5RS7eMNmtoaODRxNygI3pmF+8UWeHX7pRWYgnvESCciUFPS3918JdVUysrQM95ZxT23hDElkwrfASjV/mtM0ciNyfWQOKmJBupUvTyia4C2zL+VaIgHsk9gSVZq/TfZc8q9DvIeiXvw47Ux3pWcNfF/EFJls84Yl4/qxrVqNWPxaO2CLMNJqICZxscfYj7O36BxgaE9xJhtJsylfHa2vAe3j7K/tZx9v9+N0JqMEPj9uNkL9qvtKhQri4cpPgB/+vzbJXS9vTkmPpUffVm6dQ7WET+e+pO2LRJDWzh0JwbxS60cbDJWromZVg2B1ck5wBoxZ3r4l9JUKf68q+NsRl/1Vjtusx6JQ0p6YQVQdHeCpQ0hcy0C9LABVt7lszMe8VAJWCnYS/aHqRMQUaMyuMQW18EPpCZRn1llZ4H3Jn8znT6njBkoEfJYTCAYu3kWVq9Cq5WQ7bm/kovfJ36uy/XaIQ+V6kUMu9Ns3PTmX8aYJANNVdb354dk3Di0Ky3VXfOrZTH110xiGLbkgF2Uxp7hTrEtbgB8JiPiv+4fMeTHL4bsL0Q6DXUyWx1ef44TXHwhz7Ft4ABR0poaqKOyi0dlsUILrJtFGBaARev1WXoTq61fvvYNj04e4QZTwO/ENeBrA5PdovoBeGloYQWvklHHQs2aE/Y3L7Qb39jvvxj2JZkRiP6uy66MTg3ZYFpdgfGyAs1i4k0pJRNCQEOqJV9W/ItM8rsDr4SqdRDOwJ3kLMXoimcOhphJhxJwBlk+kYdi+nVGJ+Mxa/kXZtD8DY9eCUFqOtensLLMWfseB/4K2dTlfZNPWHXKfGl/uYcE1J9HlHUCMHpXILYWgKDVelXhaY+y32THkr6AxtjKy3OK+7VvEKesP/Itrhy6ek4odqdWvB6uX2xLoBSMQX0HA6J/SuApVhJRIR29vsQDXs12a8nEetZW12LobwxtNTgny1/+so58ZYHd2h2QXCPFWExQSdDW5h8upyn2jQGPNNjS8+K0/fdRnik+f0O/JIVtGVFZ2zxid8uo/4b9haeg/zVY89oW64f/G7/JI7wajDeiEsSvGnRIidiuWvvKihUbK+F8isM5MNKqutPXSV7F8WGMlNf+RjxxOECglzVe4cErDMXPwJMlJ/+Ab4tHtK5Edfa4Hm5ebGQV9i+QAv2Cn3zxTGocSS1P2xPWziB7UJcOw7w/tllyFq6xlEmuwJvh1v+8LFB0/2ZWAaRPw8vS27urQqStOzr7g4Tr1BS0t9lMRnzmqNZOnwPv1AC2EznwwBs8U1XjUjxfbiytmCyDHBAVxKHiMh86qZ30+goHe8Ru94Do4o7D/WdFa5OHiDsKy/j8XgqBzr7xjQupmj5kVbwPXrPotaUy+OpKBtwtwW7xV+LlGTEa/POac99qEYufwPStDoBQA8oHqd+YCegaCxxibthDqUpJuFfOCJX6t66w5NDok3ufNX7449fuqXq0n7Mz9L6v23TnozNcS/SNYqCLpX+Na+GQDqixo7wC7lCU17Z4Jzjw+MxNBl1FTIkYeqUBrMtMdd7M7pjyIOcx2trnWtur+hqRdqoDi5v/tV6BwLaQT2F8rEwTaEEdvVr6jdqgqgTnV/MOgP5a6jwQEVS0WDMMNUqvFLHLUqiLuwnbi1T8Sd0MWuAabpCDfBjGl3RqwfHzqcQxelmgLRs8KU1KDCXlBjGUki7Kl+gPzZEXQx7bScp2cTxWkHCLUQlpB1AlwC/K5ZO9eG+cCnOyPHOs8S+Tkl96oaWCfTALoHDOzddh0wLAoZdZAYWifBB3aPDpm1CDkZRtit499XP/rrd3GZBNp0npZSEr962WwiRrlylj4C1BE1g8uMeALWHBtqWlj7y0zsWqUxlmQAxOYgRMHRn1tBa+KeQ1oxuw4J8dz9RUPEPWr7TcT38n3yPT+kZ0ns1UhPqM9SV9E5CSDarAah/gx7y/0HOxYptcmjHP9nSVjMrGhavIHhLJYHbPmR9Hgat8j77YSBfklEjgG1rPAtEWBCb4hXLQCv/FF/tMWib/MZ2iXJrLvHrp5H2ZXZoFEsfmXBuee66J83VmPSvWiSC4L6DPAJJ/YhQRXuWDpQtKno5pRa1Qi8PsMVcYqODVMda6yYUim85kjOKkEqqWP0BdqKKOUH/gbWaTcxsy0nGiIbb9m/xTZthDUBCGufWvCJ6PpHAMJjU2V4lTEMf3rk3Fx25fklP31pl+GVyoatcuqocA8RmHjGC5RDoCyKVz49/jXmF7DROGZb9VueqH5h4YbPyv2PjTFKAFfNbdvaAGPVScD7b0Lgjob25vvTSe407lfTpsb1kJw3SS1xt4aeYXo3w3xEBK80vjff1Qu7+mHvAX0kZZjPhptTocLA2QzAUL/zFi/v1HcRP7tMNcxSjoApHYhro6z3nSOiGy6HcHLbUg/v/P5z+D7uJkCAkIew0TtNRNuSDERlTBMr8PFecd63f7XD/j/YX7LYXP+ipA+fnhv2OlXyL+qSzAdETle09Y6BPPEyWhcEpvtDhxjfkwaZ+WaNrEshG+HKg5LYtJKv4DgutvmLyX8U9UO9i0FL8Y22kVIXvClJpRUac1N9udeBgIzKsfVv1M7+bTq0SRUA2Ij23QN/3GRA0/88cmgjjT7+ThQmHCvVSZkxfVsosiCFZd2mSNViaiksT9GJ0VgjQI7XgmLrw/UgV2tmXR/BE6qb9aJHCiiX7p11iAhlTLjbIjPTB8sFiJTzC7duj6ayDouIBI3LWGGlvhPBypX1cpjBiiTPOPwJIodxHMrVDQ9ArebiIhv2cWIkWJs/EXX2zTccx86E8DbmHITNsCKIn3hJYJl3H4qkVxXbD1aDjVyW+3On60bXRiFKAFlzzru/J/BBscNHTR/VdvvpqcDcjMq0q40Lc7FP424+ZcgFmgV2cq5aPAXS6vsIslk0L5bsCG5s9PN3ErFfJ3ZfPCa0u49xcrWTceOctBOswDq5haJCQPcaX57nUmPAnIYIIeTJycHX9Ic8nAvhg82KzETh2Inu+v3P/GZv2pALLVZMy6viZDHG2w/znAUD/GK+xaZR/wQASRNV6IdWLNdMg4ZYITDw8bU8gClHN+pTfuhJ2Tw+A0go2QSmE9ErDkaAeO/3pW12M3QsWX4nsmiVqEjJnusXGR9pT0Bk07JS2azyNM4PbBQcGEPfwYnFPEJ4eRQcO9nW+8qwhdD2yDEP1NdfMX64PHtuYb6OcbFpwScxnxOfcWSd28RMmgmlezwTUee2TgPjxelc5B0Yf02or7kmtAgc4o29qCbFf73RdeA1GTRSjlPuUBdQCtYRsHBmcenLPD00/Q1q/GkVgIE4klucbsTxwPobATsK7s/4Kv2NAkWpHL/VUxD3eFcdm550dToBn4SMGBBy11GHQL2It3LFElR++JZvnoL+XHWU0HjVhBhNw26D5/pf5VkfEHqY3wcmosCL3pgoMDNBCV8HevF/M79ndwBZYogrtl5jRuoqj+hqYHtVbOXYIjGCJSEwAQ3zcye9aFeDx2rgQTp7we1oc1KQCKhbTCtETSMHVuFoOfrIp9gSPfa/E3l8MrRh1cstwaMR7Pw9MqY6OdKIHwOZVxsNklaypaGnbakwZmbLEoy8cRa/LLB4HFiLcoVFxGcJN+u3U2kJHE87sxY5pbEo2CS6mpQFH+59/g4r4mtcdjU7O3gm2O+0bpAyPw29bCEvVyW078U84iujwiBxZj+RukEfe/8Pvi5boShSw0n/PTOqLhxadvjxge5QWdAxrXpqFGximT185n8AZO94DQslOrgXuDRtX7RyVUcW8MbW9pOv6lChPaDWLCgO0OBl2vhx1dh/q7cwqBA81Ooe9Xfvfq95wN4SqHY9YH526eWcsKuNpe/H7c0vmhYX+LicZ5MiBUGlrQ9t+UX33rBmAfQI/GsqrMjYGfmXzQ+60D12N5IQ5eW3vhPPW2tHpZNqv44eCHTAxSc68VtWKMfT4CD37cyCUFSXC1tbvUWHkgAgWT1R6JM2PvkvtizLcaqqOgk6Z04ZcratXhKx59hoEp2+f7dBF9cZ6l8h4K/t3tLM5v8H7cwkrbUjtWz+2kifXGQSZE+y9icb4o7sTY4vZ9tJLLP6Pt/uDDmfoRkuGXXvLDqx2Q6x50RQjnLh0LCGrZ6m4ucM3CtFYoBrGDshszLgeD+6Dah7kw0/Q8JfzdLy/nowG2HgVQKKAq+MfpCH5Kmb/D3IhTzmI5ze03oIdGXVXxerXz9GjQfxhR7pAxZug27xpMAgeryzDiRNnNY++pg5Ra4pnePowAyuzSvZIEO9G3+L4Ru/0ghiPGhvpsQ9iik92x7VEPhpRejVEzd0DFr37hOqzy41cYVqxEnGTgtQnI/TzxjyocXIP94PGYlxk82T00ijizAQMf5pW27HfTOXubXcVIsOQZ458SNh24poddxDIMVveGqQd9rfSjG1Tl9CJyKcKiGpALti/cFlokpT5R1VEZxHYZBaRWwnwiQNvYDUJpRspMW/FEfuP01AR7PG1dzPEDxA8wOJjysBuH+rDqS4kaG6APt3qG0eq7zEwW+Ve+0seC7Z8ghBQjLI9PMRzKehmQ9dmIrrG2hmn8QVki/42qmg5dGgwboSeVniID5mkQ6LY7ystEluuLmVwid0ytjlxM/g0vbWnrbKrdJ/j+D/ts/APoIOC8X7Kq6QOl8BXimzB3A5ktkRx4uhNXAnCYun73QU4JA1/7x/RZeQFdTrMaIYqbEYj5CeQ/iP/TB+bloE0kTovUvWjlt5wKRdRxArkBPrU2fwMhfk8KMRrrt2Ff8iVocevvtYSC/RUkeytYRWN0JJjNxrWkxVd/t/mnWLqjzda/kszdh9ij8IBIjxch9B8at1KmfG4YkFS7eLDpwh38oAKSlvujtbLxS9g/qVgGJMn1rflB5oo46IfJHmiSz0bVo8t6wHQo6dGnYEO7HDPxs2shjs98XJ+oZ8hbxqAeqgSNTSIWRuTSM7xhQZiFt/Ta28nOH1NrSLGcuYL1FEzaVnpAG1r5LWGAH+T+aTq1emxZYVPUDdks3uqK9ejRWxbDXH8zDIew5gdSnWJR8rsH5yCDzgbH4GJxXaXvXowa9FourhX10w0Xm9qv2mcCFs7GltkzSJgvkFn9Ncsxdz8fgxKP185LRQ3LtTpb29gddcG7tcbleR2dlrMCGmp94wfcwCaW9qtIKSz5nGRfmv2rU1EpiV4ZQ0/1WSoticu/7mw4TAi/d0MwHs9U2rKZShhv2TilGgx92kden2xu9dymijh9+JL1M65XvrByCkwqRvFDuN8FJYtf00jK+MZuk63DQx3Zxpsvd8PiKrG5tMWy0BM/59drhntcP414gCuIldP+iy7LV9y8GchSPAJLxO42i6nRKgXfz6LabPR3Zbk698PKsRr5E1bsOL7Cw+DPMQiZiS7zqwWC4DESFJPiEYwpiEy2DdRyj/vwkguQckRp63UIM/h1qyk/ttDyofxtDHsVDkw0GBZL52Ahc0hRfvdBG279+PgUtOfmrOzFcfReOl90BF0VohTpU6cVlJiDjPEUxufj7ejTJcvAO+M1vfTpeTRbirR0SmC4+hcRpEm/wHlm6kEEjo528auP0Ca5rlflXMvmCBNH8QNHsL4Hj5U7H+pblDIIdGy90Zxasc+1vvJJyY2bJvu8/xto1JssZ2gCRsscWjvPfM+ZpiD74Y9AkaP34oPDsB/WoRCS3oNvsxQyEFXVDkGz1YyfD+WUFBun5wuzdHjg/N25ReNJNxiH8uFynk4pOh1mlIigz3f9MZOMiqqAG7GRg2MzdW/ZpDfpRNlWQ/xCif3h008V/DPH7vlSX4RKXg5pcWyenf+SI+gbO2yxpLanbqk3WkhtB7GQBKpyPiJbhYHr6vdYLnbELseEXTlHsnBE/MKVTKqWOLLHX3SiCMbG3jRntAjJZ83LAthk5/DV/GY8tP2/XEQCI7lu/mQp8H6U7nXkVOQUIYQ5I2nQnM/lI3wGpydDATokoC0WDfW6eI0IZg6LviRygCbmvArw6q2h1jl04HmRUIMKQ+8Rvvz+HYiM2uiPlOtO55Va3jiuonjN4vQmfzfxPRvFi4rp3NKhrMcLPKid+gjjJClx/WgfWotIXa6v5agIxrBFNgMoqcfcfb6iktmRXoa2EsIG4mfFfdEKJsBneAzUCuHnVRW5oditAnNlnhq3FxvgBoLNMX6/U3q9B4Q2eHWqtdUy7M6I/A24gsm3snGTqdHgtP3lH7/Twq8XargvvBrq6nygxnd8uoU6pBVDgoGCMEJoxG3yKDh/jZhj6TiN5vUR7uco467rl3xQzRvLENmAm2xrA29KmASb7aF9cxs4KdHGz37glZbRmYApB2M0SQ8luR7oQ8lun1ghYje2ODJ6HdQRKb7ofRqX/IB3MPG52c/DI6xuqfXeQXaZdstU2TNR9KY8EjCPAsZFY/ayj60oPJvN91FZEs5EpTa1tQLbs4HF/1Ga7Nfy3+3kntORYgL7FVOmhc2hg7POuwkO7jnEb8jNITwur6U43y2TJxJMoPDuY0M96vMJwA1obDfK/aSPnrmMHgM8wn6KXHVVZxlcnfkD8i/8sp5bXpDQM7emWNPv6vU1n78sV+lh0VzjaZqEP4ay7eFqAc+uJZsWoEgzRIsZkvbe8frWfWjXq2uYfbW+Jxd6IW40Zu84srU28mjz6LXIOg4phO930B76CaMSTjXcSrDhAxm44b3QCvOdM7fN6rUYtx1LvLs4BC4i7XDyvLufbQWZtVtTa+Mhn8DTu4ZSWljm/FfmAwJTl9loN8PqwC7ghO8V3yFYh8+NvCKpBRVPnxCpqxePf5Qj+xiQiKcMsefTIB1qfUvZNwSzqTrVKfBx8VovSdDfpuhJryvT1HMYZqB54mNsV4sxKQiPXPChy78c/94reEp83Mez5VUL5AJp8wlLT5yv2lfwDGwNNz2pTL9T9wsTmQI/PqL6lzk+2jrUxRNhqXIBJ/W2+2rnQg53YYig8vtXHmM5y35pSOG4UjLUx+Jw5Dxfc3CBO5ADnHoZRnaiOqi1Oe70a7WPGx1Hv7Ej03pxTpQjDlX9N0t/9U8MNY2+9KKgPJKO/DV7+fPSqMPPKn4v4XB8s580Sg4Edyz1gyhWFLlj+kXGso64vxAOgBun+c53R0mxPgyUJ7EGNGXyYbgZ29rvmsA9+gJlpyo5ndPVw8Kp54+1PFS1n/9JkXQaoSQGPfWt9TVhpcAq6VCocc1YX5qmoFCROX5pF0XVyWjsaiRpwPEmOHs3a9nvZ/I9rYiJug0JN39EIdWgA9buy7bphdz/CpxuCi+K/ElelQTtQqWjwFRYgWrA0ALP6rPkme3g7D6/U6SbSxLfAFmqI3iODgK+nFlV621XsiPY8XpZKJ1Un59yrUuyCkhm0wSkSh1Md5QwyMj2FFVJfq3VTrOUUPiKzM0I3xhhMNZkgBR0tlEDB+6xZd0df931fuoXmQvjY4AcxkTLp12A8wb0hw6gIMbxi0y6O/rZM9SFJm2kJmktjAcppHfhI9FW8Y+jFl9EOf8vSWnhP+Zj2NeZA9mPYgv4Q3LwdcxmZ81z9BH9eUZ3LLowiT8h3E6PI+P9tbeuYEanyz7Wji3kfJ8tXjI4d+t05eW3pWhUYpt7ZPdOgk0eeBtzIJKHiTuXgxcT/f60N+Qw7ZcccJCm8AvMmub2hTn5Cerq4akVwbKjgKhv6l+UAe085/A1LrT7IPLxdYJMH8hpvv/xGyY4JtNwjc2ID1CEAbnxjFLYSw+ziEIcdq3jYnZnf627vAfVf9D0vEQ5k6Qmf/Xm9z6ACnn+pQ8HmZL+JnpWrsTOc0DBasTZa3wHPc6cH8lucsFauzm86mpg7kJek6bOKRQtYxPYDKhmXhg0/lZ3gT/wNXFgt9kpk77a7SoCfwme0GkoBeqeY+eufKplN+LF+Vb6FJepxtX5qXCZ3yfkJOm2BiriLzucpRVsjAwPFNbABe75cQvy88Wyxisp0BcgTsNdAhWV8B+VV+UvN+EffCiu54Hs+qd/KN73Et7MTX2xyOc3r8mwWMTzXBhiWUvpEkBG4ttatAkIfntOwTPqBkP4SkFlQ/nMeMlwK6AX8wytMbka4c/ySA2/lgGd2yK9Lx+7vJtDk5q/eg9wdKCRICu4tVTxZ3GAFUAxkmKeM2mC/10R1ftVh8CfJv1hZWguYx9TgJIGk1REomdWkaHEy37uXcWhQKySy0GkdPQdvZyYLFPCbkk1KzHFKzijaDMUTvUFGSAPl/rVR0ZLFjO1dPTyBTKcDMXsiTU97JElo17Yuq+PbzwaFL+9rKzeS9phCKTjXZB0yPawvqCQ4X6jYsG4pny2D9H5M+85nWq2H5DcZfNdfDXZN1LuiHZNtFgX3IafbOpHwci8YTw6B7vmGm/YXArNX2NppIwr3h3c83r+ndJc1an/7KYmRam2pskt5aSGSe7GSH0Ni/a4UR0NlWG9X4rsBgYlJjffTJNvWYS8jmGqy4m3Ly3ZW6cvYui7L0M/9eEPT09bVcKf5eVX+K2lG2k5GunW8Ej6MJhT6SKjh2Sl8qEptJYg1A5ySDJtQs6bwTSLwryH3VUcgnNohBpaoRCfUv1Hf4cvb+vzPQFFH2L+4XwESn70IOeK1/L75iwUyMYbXB1EA/5bJwdvJnXAX460rnOR325brCWHPZqXf6pBSCOiLnLUm3Ad3BLDNuW1diT8wanwHPGr/jo0X7Y2V9BWfR8/+l4V30DhTRzu7QuyQ+6wMZfJSV5wTu2heP7Pb9Xxg9+yTF6ys/GDiyI2mFXKBucbrTtArg+Pv6j2gksoQ1JlbpCA4NHAE+rErgv3yItD3MArGqTg4Xig6ts4OO3ufJlU+02/4JlKWu9arJwHmLJVGnynY6fTadcHSLR3hZ+VbVHs0tMU40lI2/yufC2zT3eewxBMSz9A2if2mOD+yfZd1Fiw0hcbnWP39BzIi2zWWkwl7ngVXnzEABpAmF+KK5LjFMGdrsG0OdYRjeKBzmLddL+b8z4o9eqsG6RlpSAcRBqJDPhsv73h5HDTakHrN0FrGIukGrMFcqTMk4/z3YtP7H5rcz4cTM+c7sIg7TGx5Zesq4RsED/Ta6u+WIosaJj33DzEeVKYUZCen9XhZRIpv42qX12ODUEo7ZIfiOH24wVjcDUUdGD4WK0hZakP0DDRJTRf1GHr1Ei5uv77JzPLJ5+rmSTu1Mmz1wbOrkgym0aW8NtE+pfP5NZb0NFYwNeRH+TioqG7I7l1pluDYfKe6L+pTmKAJfNTJPG4E+lKaZdfl74qfjOZOsKDfumiQVwl/MXbA7msW9efIoxQOUIlU4DjH/clANbSRC2CAxTtjRaylx+RmnCk8hmowOz/elZ+ztqO+nL8TVenWOLFgzqtTj77dChm5pa1lurw13UzO+jpDUxlX99udWa1YiTKWMFNsKw13Ajoc7lAXQiPGf6TgOEvgFrUmaQ1zvvs0i4hJ00qCMMBjrKgSwYPCMapZM+QqZXen0kHZ5q1HA7ZgDTKB7NXH3z2BH020ngzjM+yXhovIHTGp3b2E/F3KEqIP1Zd/4s1TNrOM35G7t91atyetaD8iqQXf2VRC0aFlpydo0zwVvJgcCj3kFa2ZnFkwpL3A1UebDj9vA+EIEKzrClFCf0iSJiU3yuDtqjDIUvgJ4s6Quf7U3UYio5t5IYbZFOk8OhINzrr2Dlon3ZK43vKSfM3K5qCuSoiomgstKgkmkPhwak9SOaBWONTjjBj3Qi/5pPiCTNbcmp1jqdjg+iwVsg2p1jDC8w0XPrfSR1I9hUnCth1zwwv0EdLE9nHKtmXDTOvYVzDwkjdcoVInCsrboo3DfkQmjXaC6SHIsuHmzQt8zA3dnE78y//KlKkRH5ZjB/O3+KPm4bL3w3qE34mDTPFuZervG80Zn9zbn8pq+VxNjjl/JkUnQ112cHFfhwTEpgfPieMdjlf9SZszgvgs9lpio6ngBWnhOZfuhlBsnxLHw+sRD2vUISNJ6a8SotJfUG14LX4QpjvF05fidkzMBPGc8zvCOMvhZzp729UkoAZJC2ZDSW7Nvz5sHT7nAZvpFoQa5fFP3/jj2W7StADnHqfG4tQq1KGekbzy99lUOnqDr3M/M44ZYf5JJIX2GzN1CkmJatwld7goO4hZTGX4Pje7zRQHoCTFiMy5G8w+QxZQwIiyoIyfRt3SilUxNSX2mawAOV9g4bE5eDIffl011MXXnZ8HjVss/DQQ5IflXiTbqI48/SwleE1ugJwy8QND5XXFbmXseqb3+YpQN2BYOWYlReL0vXNEU6WMecjNZ0m/PE28gYXGoh8ISM5twq6Ux46FeH931XUKVCMUvGyydI1bjeHIP0bs9TLowj7ARKTraPWfmPVFTbt5wCfXuaf4smyWAduRuqUtjKsTKc4WZSNwvUfySDRXx7QJPR3L3fyLi7iY/zL2AaJOxSg1VTWepCGEAga/yqMehozYPEdWhdOAk5qsTsAJg2/hgoS/MNxEDdSunQMSK27oXhC8KuqOt8aVyXTMfLj0EF8ouTchCCe++5clmzKxd1xPTpT6/Pq30wjUs6/PDpIrva0igjf2FLv0Byks8fiAiKoYWN0L081XlAjfxfPS/yCeX+NBC1PSEd6HaszETzr/paL2Srwoy1QPkZ/n8KgbFAhSm/gBeqMgj+lKQeazBh4cSk1eJHtyMySzbLhgG5zIGfm4T/gO5jP5n78gEI5+FzK4JzVXnmBaZghky7q0UvZXkLZH6eOo0ZzCEyb5fd13Q/F1eAw4d8kABm84btOVvyddANvBfJn8PdrPzXGjuQiuJLj18wEWvywdnaIUDCOQzMtG6IzTPh8YIVQRutCdYjxfgAgVQ5DkqE8jjUXYn1oaGvwrT60OXBwMRwgiEjNaC5h3NgtIg/iQ6h0A0JVJz7PazCBmbK2hhwRhUsGQrc+FDsmGSPjDP2EC73E5SDk0oGFnu8YuKU4AqRZ4DK+Xv4rkKaSkmVuCjzv/QQhsE//dmr3CVSypUCXryg9sWWfw6u0q/jsFig6ZYkCMcb/oqVFv5o8BOnDf+1E7K6XfaeCzSEf9IFrNgCk2s+Zk7dI8hL1BIgpjh5b+kYFoum9LGrgK8F8FNL25wuPmtUNzXWhOA0hIAVJ+a67s05dKZP5H2YzWFFlssFXRBU9Rf1xhEn9BOKldbFwGnHBBCg2fEBkLd/3RXTojJObXnVFcdLlheDhVePLRKmnrPM/YFnhN/r4tHAwyb6Xa6BXvRd78Q1UswhqAkSJ9KMBK6IZMEHwrx5y31chPeW/40CWFgjI3MLPX+nlQXtk/m4QMBu18gocMX/KeGRHbmHwawKu4Z+tNhVMMtgK2Ge/csFUy9eFDZMsr65i2zBa2c76/s+/ufDkKzhAyoGVuGQsbP+1/Iu3bbkA+P3+YtVZJdM/rNLZUUo3wGkNv/ZKsETIFui+RIbNflLE/l0cCJ44HQXTIXtXLITVkf6QUuyfm9nBEf+7STvtOJLW1X6A8SsobhjsCVQIvfAeBVudb1/ranWEeeMP4u0Jw53Pwm946ivji3mhjVi4A/Vq1Xc1TVslOLlNQZj+6xD6/Aj9r9AAMIc4S1U93TNIHF7h0Wy1DwqCNaSyQlxhvt+oeF+S4fCcvOM7vtoPuGFKrAvQGMVWuPXP6rg8Js2RA8+JGid+OsKJTtEksxzlh/Yx2PBdNhuNphZ6/ibRAwkGIiI8TiWn66CvTpzJtfhnX4DRRLP+nC+7GwIcbOlGIvSYUdr+KeBxEWAHkgnnt3xvkkadyfjCeZml9vfGEAA1mfFSD4alz/VoCZDWZPlMRPnfFr97IRUTq7v8Xb2ita9eJk48iDZ5B7bPueghkqsqZYVI97Kyr4o0wHoAWCM63aF0VMKMXn3jDtXZ1NL/6qEm5WGlKVaC7dK9teaojU+xPTb41VevhBEJgjUHj/HfifY3deOJq2VfLapSpszgA99zLzCLXSfWfMI22Gdui/uKWa8DkdvEodKEb7oFkzWQM+KHfG3MjyZcRlXFu9Wupt6uxleOw7hXbDNBltS9ikTzg+4nUy+tyMMPxSpTYfx8qIaynwSvbpbW3HdIrd31tohtqJR7RpCtE1lFUM94Ti5SeKlhSWI0yf+ZZcHlrKf9251WRPHg2tESx0FlRGtFmLT0IZdazkVgR+lkC8hpnFWtsmwtU5PjcQtKFQjvQdgmG+0RAdLw4oVS164Pcvys/sCIhKHDh6u2Cf9MFGEFGN6URRUB6N9uTviIEroR7LbuP3+1CLFBKvp7alMP5w5BQmZOgM0nJLpCBJlmvR/PnzxtEncT8NLtxx+kRNo/UE00in2YL9GtOZq1+vgc1ujVWbSeq/TLjME3vzJustrnVsobmLWrtkU0sjwvF2VYH7JrMaqfybO/o1OpoLkB8kjXoHEJPjnHSH0f9FSjhMb4VzLy4GXDfo6h25rRz+dGvo8RkKicb4mgTXjwd/odXa4PcbJcpCrFxDLlwQCaKG9Z0NXM+F7JuEiXH141Xg3H1rxD0V/Y/qCjJc/WB7BSGooUwv7FSc3ZipsgXOrFLF2wX1kBf/0+mEx2F11GG4M3JEY/fDLe8DfkqDEZfvFS7cXZhD2mGSC5Jv9ykCAqvujnL5wAL9agYSGxk0zAIuKolgwduMVf3DT10pk9+1wvQwSKkBJJ+Z+ji4z3/Ro+1w30AFDnG4VovJHh0xvA3buZVR0uiU6rsCPn8rg01zheQdseAC5FUJMdsRblsP5n6xn1dlNyA1disLtZ/s26SJLr5e0X6uVUFyzEFceWpKsBeW9mdBQuuyt5o/k8pmUB+jdbKIuwVxwtlCo4sBkbys8cCv8qUqCA9ig6IAs3QVS9D/DxvJT8jq8EveG5UFJGpWBjc5qJDLHtiyb0/bDtExwih0H6JAuBgvi6jYK8IU+Tvrwxh4821xzhyCX34I2+2M9H629onc54B2y6tOHtOF/JUoy8/y4chLmupvHLI9tjac+Pp8Mn+0LGORd7k4PS3sYFAMrDsnLy+crrT7NlLaIDlBIK0Vh05fsZXoH/CturJPmDskBqMHmVI2IAf0WtSOYlIcdIkAa/Oo1PjzyitvbZfbpCMwCYlsjcU4u+gYYP/xCM/LowkiGzvJDcXB1Pz22WY8kvp9h8wzCmVKrXz/m0KLiFn/rQIJ1qSTgNg45DtutARtG931XLg8xu1xqRTFqwxV//eZlujrkM4vhWmMwg9yrO2TU2e4sWxX3OoneVtr4Bb8BGfSAfApGvbWXuGOSc+nIuSuLpbxwVekbZTEbAgenoTo1H+azXCxfy2mKjPPrFYVpu138JC7U43aqsLlwJ0HxICZCXWD+F7pbJ5g7Y9Blx2PnEnub8Lpp6I48dyI9c+3O1hRgubZeVEmBMkDgQuzMljEN6/YslfzckP0/d04wiNw+ik39zbOS9oSneygTOvrBPwc2DP9TEPzR9ZZPVvrKkodbpQs5HVuPRbAnP/3HaXHQV4tYC5H91G7ZNyuoA5cA0vdrZz+CwUGiDw7++4lLG/0YCVBH6evuc28c+sN+taPa/25Nr2KlXaeuSIj+SyTmHG/pHbl45Cp2MWEvf5JUKq60OuiYHMk7FJyUA1vD4aftQok37zj7FYCgky1n8F5yNjnG4ljOkbIKu4M7GTiScM3HIeZd8OR9gXUeKA4Xy+WvsFDLGvBAobrLVQJK0Ps/fzQa+ovGC/TLvKNtl/IjVMXK0VM4L0DsnVi8+FzC7xKw3uFUYfKe/WogDHFXtgDdUqpRULG0/W8008Qi2NyIqyto82rTDQDgyI479lTqp4G/EMCiojfTk10egTZNdsYLD9EHF/0Yi5vJuJbapJTaTSGrLFfdXvD4fzwgjN+oUPx/MUXo9FuiCtsgL9VJPUZm99fCAihzaIy7RkqE+lRAAlCJ++kvRUrEGla2xd7Mw4NROQ4UXmTsPYvvoce7xGo9dKWzlpBBhcUx0wt96pIkMPBPvqoWq6/3mIwprECrEtHvoPkSa8ZNyuD2GYd0tROQa4t9mGd7ha+nVGxNqY+ci6RCisK3tjMvy/MU+voGCr5FVhlLFajcixCme4LfTgGSXlXSoizSPwHZWAwuIfmkQ3ds5MEaqkz4N69NMhgMuIySaUl8oVLxuU1da+9uFWK6o9rvSIGRW5TCyNSFZN7GlbM4i2BxMb8z6m7ep49PLDSt/mV++9OQWqQyX41a/jN9hDTDesbzCzwG3upXqS4OCjVtP7AuEzxV/XJpK7TPttl78QEVoTeAMe6mxWS2WjmXHOs3sOUWjtkDMVNpnhViR/Ncvopu0I4dWD0puttwp3hrO+pcowLJS8mATu0zEjGUWkKYAdUhfIK63DlKWfx1hF0zVeKOe973f8/KFtp59kud91KXp08GUHcNPxZP+JinR4H84A2EQeRi2ZnmP4HXZyYUNOenRcW3Ab/HMrfOP1E5DB1H8VsLo1pGp+T2MU4xj3fvuNssHKyAQDANWznBjvQM5pr20HpzYi4sKATJygETEvi4LAxtgiRZHQGYWejZcAQYickEaQrkn1rLHKY8Jf+M3ZebOZmk6qmRrP8GYpZeYfdQCtb4FuU+sRi6Dvp2/Ky1VRxau/FDi0uhRbKv+SqgcA6bk/gfCRfy188Gvw0iDoOY3SFnffDeiv6XErxrm3llBiMzOdmNAKVLT/+vppPzTSsZwb1qFrApRjjC7/fviDxN31YhUU6+BCLnB5GovKspdFsP62XDHW1m73sHwuR5751lyQ903MRI9jakD3k1LlEtklGrDYqYQTv4ylOimfl5YND/DwTUxwX9e14uOXV7Yw6mimJlmPiA6Jd8WfluZDjPRAaCcvd3QuT8bNOQ1LIu9iYeWKlUZfvzUT+mqmQUfDvJNGJgwr3mPFiZrEzt2Ph2CuZJX7e5M8L/YOE48isqGAoTIhktFjjZRv3wHCtC/qxJwS1dJhVzSHqoltJrCptUML2sDA/MeYEm4VxH1ZLga/rI6VF65649hg79SM1GVmEnUkk2BqfGQ5OpXwn95GsXwSovcrztTdzYC/wdp4Y3O3M0b25eXSQl/nAzwbwDPLwePTCKm/gZDoV9IGPUVTk9k032uufaQZNa/XOm4lXrAtfiNQwQ4W2qIviN6T0TZORg5BLMtq0EdpCWky2QSGbpcpmt2DLe0cBz0sxdTpRgfYp9WMiRa9seSUd55WnGcLXb9KLp8urWtGDcPiPfR+nlWGg2D6kygBO7yF9RkWo8hDsNB6eR/vaf/hGh+HEpTnJk45a5yD1BAQBAOMgwgI0Lrc6sxISXrFpmbHor99Zx7ZdiiKf5jdPUD/01cbYbz4Et1nXObKWNE8A/ETryXWccrPdjYjEwPzuKhrIMTrNTXtmyT6P0EGdWVSAfl6RIrnEq+xr425uoVrBhmzMxoVa/5vZuAH7Umyeb+9ct/QefnP9JRJNnfeYY+oviYxZdoDQIdVfBYDitM4aaPsp9/+7SZ5ZVaxCWzVnY9yiU18LfWFb8qO70mdgC8r89S/QJLuIrw41+bPGvSJLWeh+dgwkVR9jVF7ivrQLKhZ5hV4CcrWqql8jFgn71tmM/DTwa+FsBdIZYHRkKdgqIWbIS8BMIXAKn5G3AqdOVFPX+3usTdF6SimsNyt5Xtu+mXn5h00fooXRNPU8yoBebnbgaSiKtZ+Ma/2dqqzPsbAMxdnZ5//wYvRpDJf1EzawoX/7lgMM6zj/2tIx9C75zn87McsBSHfmmkt5lG8DHM1G4A86ILsoYxXor1+avqxOK1RlRZBX7Q15NQENYztmXArzRAey+j9j/ZZhQxUDi157dTAhP+G5WR2sYMHGYN/6f11Rzbvh2LvPV8K3DqNoeG9C2f8L48fwCVMS9Lx90sc7pG8YP5r3t39z9Hxg0R1jDQ37UrqXc3kiK0+5M2GrqE1EuC3Kvq/+wGmO7+91V7MtoiaR/JTlCe1YbwskwFFxw3WmDZReq/bOXgVlvXdbLlv4iPjSrjHqx39pMmguUTDvSV7On58tthA15Brforq/u/tIp8yohVbFLErhvvwxhEyyh0EL+/w6ngtbQ/xdbda2YT7T3V0V+/OrrcH4iGLowg869e+TdSCSta0Kzzu3mUljQSwYz5O+kNu+LlARVsYvlT4kSyA7/q8yK/+zP4MjKrr6p0rGFooB2eQofvcvG2bY46bLIJmsba4ADT/vQCYCRfB1aGuaPeF5OVvxx+uwI1QQAfqq/6fetntwXs98vxjh+pHAUiLuVy5oVB+zd9PL87nSdcE/5sTSOffmVpK+Vr1h8SK8pMa4jn3+q/kWdKiYHF8DPR5tLsc15e3ADQjQV5Vbz43elO0WBiLVE/eIna1/MCUQ9fTmo/moVTZDkxszCleJuV8PI7VNMlk893VZu/eu9lDaOH/byKsR5743gJgsN1kGVAE8d42wC3BDapBLwpLUb1uUqT+PCI33UeQMwXbUjfOhU7u1RN+2Y3Ji6MzQGPGJCAYxdy6pzE54+KRP5apLFTwFjcWctEe+jDs3/796AeuZ9cbj2k74KfQnSAL4aKsx52ftMxremopScnX1a+SNacMyUs2qsEnp3A5ZfEByEK+XdARN7Og1VhccHo46PHalJynVDhDvIvUAN8/mEjHalxN4qRTWofqFaiE4cKMSMnuW5FkyKLUZdVrlsP3xu+UQBvr3jFK5Jsc/dyRqsym3RHi0HuiQ2aQC13jotLBBTk3y0JGhnLw37wFLYUPB2DIbwgCs17PzO3WHCxOeMcj2UtFKYiZekCZhs7l8D1SbI17RBzVYkyiBqsplZJKip58Xdsh0zxqZzR+XkhO2qFXRmB2oa7x/TT3DszvkS/iI2qPCFfiFXW8ONagM4fH3BdjriwTvDj+6sEUB00I/4Qd5fCDj9lihQR5waxSiUTIFpNpE4Df+wDosXDpk5/wbUihXMwUrcVBb86E0UhFumrlqF7lQop49Kj35NYpw21oK9SKtfyRjjYt89ZHjzw+BX50UdJB+0V4LMeoc8utDrgmKxrV6pQXJMP9jyM8sK6A8MqY/MOT1yPOP5soBRvlOSrjZLbcaTPHKRxYtjP4D37U1SJrJ/CbfuY0AP1yeyZHxhy2vyWEWSKdEUGi9kjMaIrOgG+mPdQpduyD4FYHClgk6j6VNfvvqBUfhuc0G4exHD8Tp66mKhWYBWMXoXEiJXBnaPgAI61WEH/zE8oRx2OMzgCfnxVZphGRzv8fqNGpXRIf10kaGiyC95VEsjSu78quJ9EXL6vpxn3wZv6DSKdmDtaz6KtZrtbiRr87F5fa1hWFYjUQinyzlVweiu3qWhAAWORnSYLizK/fUDZpnXQrBk+UqAc2BcmbIKrvmimMfdLv++rCUyBWjISIme7kzyzbXEUUfFEq4Mtcce+jK22MJeJwM0P2BGoLJvL0sm7/4c+krFCCXOqwcuMZfvSijjgcDyo9gatCWaRIcuaCZLgcVS2xiM6NXeIOV/EnVYAVuTT7h76njvLnnU3vyMOuhUlTAugF5vPizY9f4GawPlkZZB2P4ka5AToutVd+LeVzFdENUj8XfdmQPR3ufYCa4v3gVzO8vGBJjdJyX/z60yguKlTroDgSjV4tcufmdkDhmLeVnRGMB+VSB7ERsD7ldErgWhJAmVQvqfrsA5dfpjSu44o//7llIAvscOonaoUHiNBEFP909yW/7Vl9JaGcGwu71oud+PRZ8HQfdRymNKMV4YQSgDnL9FBmqNEZ7C/iCteEZxZWKf0cfojUEBvAuRucHksyAghj1/5U4Pv2cD+J/oWLaLw3vqbU9lwW7VHddN0euBv/iP9ZtssA2XC1Tz/EFKJX6EV/XKyn4aKTeQ4N0GIJ5espfqtCApHEGzB5roznjDBJZh5cGPPldlJngko2550U9AFk3L0FIyp6HUnvDEOIjXD+Yk/VtRY32GIv3tuOCK/htSQ5UrNQKoOMot2QaqE/0GRDagcXjCeXElQuuM0v7OsxdTQZ5jjPRp1RrcXAJBSZsYO5gGqnkcS2R0xfYK40H7HBfirMBD3jhBbEE5drP3xq22c0hPOl580pr9hJ6Jov2u1gRQRUCNOsyeTwMdxiLGZSbFT9bUubxHbM9jR6nZKhxDOmpZF++p/YhDdcYu0xYgniBrSWu0JUSEpc8QkhRNfydvrljx/THXu77Hk46v3vp5bBu7kqL+FUDs3O/wxZYRffeub22OLm9PFzwFIzyfmubMNT2jEnfya+AGpAczEdq8LIdGda62bwSKgNAARisXx7XeVjAGhDEXyF8r8IFS/tXqw7fMiMIFsV7qA0qRm/Pl+hebuRR73KtHZzRjz5KDuQLGCWFCMy+qLGNKJf72V+hsBJ2AlGOAM7oAVS3b4o64v98j90qaQLwx3Q+iCZBX2JLON3V+kfBD9eMMC+6w0pYg89Ro8OZIgj52F0Y5sVxEwRLBwE03auhn9tTmH9S9WnX81qcV1JMdLqSSIFMGhDH0bbH9cFPatOpyZLMNlbY2+I/l7c3SKmPk5Ew0ZB8v2rFfsZ2gqNXP91Bn+k76fyG8JcYhZB79uCHG0JQRSnqnbHD41iBNp92Jsd/IatxWl9vN3v1wzmHU4AIQ8Xt524xqC7vXfBb58acGxDrzu/IsSRZ+mrYJCHEfQ9Ke8zpd0niijIfLTW+GDX/bqECRlJl45yemwr+DrTV3LkQ3SQDwnQqgP/F+td7kbVXvxsW8y2U4yAjkBmc8tO8Yxe6FX5Ob0U7Wx7IFleiFAxUkOReSMLS+cgYQRDvhP0k2B/LWEId+aCwfEH3QVBHbJsJxnN1dihKLXHTSyqHRiUb36ApVbrAixJsiE5rEKqH9Z7xYBK/0Gy1a15Gkk07d08ZpGDXQszNxfGQWgqDki3PLIzndll/Si0YF09dsyzvlRlqRL/U1gju7P79X/d9C44JWiFTY8/xmXee5pUUvP/6XpOrYj53nl09y9WllLqZVzTjvlVs7x6X/R893N+IyP7ZZIoFAFgKDmcrH0Ri8gs5zaoIUSrhHCV+9pSVpDoNZk1pVKHtmZzn60LdDQerTfy/9osalDxJYKQS2lJOR9XU53xNOnmlwlXrMtjoTC/wYM6mrenBedV3FdVqBxbJyLNdxlbAdtLUxSrjc/7Sh43FQ2hdEkA+SFe1AHVIqXYhpM6DqfERkgt6+xj7A1kM8X5VhMbm2I4tg9B0G7sFGvdh/gN1jC1OLm1+VJEJN4KnMiOiMdXMOy/QucaotKsDfjJWcQbtNT8SfKb+Wvu7mnWhlzs5h8RXBE8SOx4548OA+0Wbu/2LODMm3pYtyYi6+PiuXCYbQyPoSO3M+o8N31kqsmyZbH19DXqD94+4p9ab6onehf2M8/OYV+tv5v4tRNlFIKSpMx1uoxMYc5m+jFt5E90wQyd6p79t1Q+Gc7GCnBJyg6yK0z84H+M2KRS4V3P3FlXE6gVnLXnETR4amnDFZWXyxknF/E3CvM8wMSNFaAfGOFay4mL4dgn500g8Z4jnSyjWf6E7n6NxZceDgQ0VFWOsgu03LFJ0U5wdGWgODwc2OL4uA7XYJicb3Ph2ammX3hVFQC2447bGJaz4n1vcssKYVEpo36jEGNXyntnDrh4/Q8zoc22AFsgQGCnIgDaQ8Bdi+mn0VdDvdyNlQIROnbvtIH2Pqa9NexZ3/nHcfrLqLsZfxG7L5ot1wUyRZEFYazsqOrFOwBhqTBIg4rlMf8J6h3IuzGHX40rDiiIy0zo6W51ac/PatZ4OZpYCahZiy6/dHBgRhxaql89BxCZ80GBjoRXvAqkPg/sESwfskRYQAVqqgP55TMG/43LS3822dFwFmtR8PiXhVN8666GiwOmQ7W1aWv2Ypx9uW40g2iW4uTrq/uSGSsce/xp1aUXQsjcsFBuDxXfNe1NlHSOGWPsAJBYNupk3A/mqvYUyErXMGfIfAh2Hjpa6EVZIHiELlc5HxlYbhUyOJldLx39jc9U8u5Nbnzdcob7l71kFc0w18BBu+uLGPpcyYacnM3o8ZAHlZAJ0iBxucCQgCHNEXjzJekxOu57vqJdxsog4cgCCowmhVOpz2fFSZiZhKg7wROz3sZcmibS9MFJ0DDc5AAWoEB5aDk0hhqLUApvBDUzYK7DXjSRdv0g8/C/h2sLqcCWNuY4ucnyNiAD3MohSjxzOx6F+F+apq/EuyzTD/Qrnk49YZNy1Osos5fJ0mWbhzt5FfNsmYn9nNKDWyWq+aVtrLHzqmMH88vP6irgF3YmQjfVrLp7IWyRXXd6yFdt63zVjaS6+POcxf0FoEIUyKOjVLlCj1qbttTHEaZabMedIeNSWza4G4APuoW+Mspqh+ww3TE9bUqKvWyNgdyyuVxozYYk53B4FwNX+6D619C3iYQi8Qj4mxlHqH+hsOEvxL0Lwq1Zv/DcsIOuYJTsODE/Y+VNAqf1ZzvNjbyHXG1OI1mFzGkkmT6/gM4X2A6jVUagX1ZxfRgVUxx04Jdz+CZ2NF11U+JcBa/FyyST9qoUfLpWkpFWMS8RCi2NooSrA+JUAOx/g0rAdnPPKhM4hUCaW+E3d21iovuz5zU1dwjX9WryTk/ftxsT9qx67oCGcvTSwkD0uX5Km3lEOwV9HvoiiDXYZlUwqhmOqCooMHKFYGm+2H8jKJhcySAKv6Z+AJyS/dorsq6hOtvVu3bAre9cZgbRqPn7u7XR9fsZ1PxWCtrN9c+5Vd4jeGMlsqKTEi8M/ZrDhrmGWqS5VcWNXh+Qyqd65oZln/mcSzwVfmvhkd2NNfNAet9FAREJRwCWN/Y4DegAKV/hMf1OV0LoN7PbNX817WCfJfibqJ1E+6ljCgZ0AHmZABWo11+AeZGNjcumdXY8KjYgClcfJbDx8bIFW1Wphcu9+7fKXuDDzHPTVYRs3iosy6Beolbk45ifs8an5euYv/qC5LMDoSTIZk+DXr0qn4P9cG/1t0QxSfHuJqIp3sJ/cTDlFBd4iY6wgJVKh0PJQEsrf2JojQ/86m42WTBoMloKEeW0bLjxeAzfAjvgrHz1e0N1falT24Q5hAKx+gtlv5W195zYBI2NbFLsWlrygTCKZfGiNfhpUdvxAe8Z4uNVy4SnqmrptTlJ2l6jwrT65Ds+M/m7Fejva8nPSsSdZ8F+DFXdLBG/pUv/V5r2rlQ+6fAGGnfgs2Besw3LFT5a8t2+r8r7iPESyK6m77wl9y/IIRoLvKKg9LN3Irva6Js4pBg6xpoDNBDASL950siLYlEGo5T+cQm1JqiBGkY01zxWtHxhvwC5PiXbVqd2DM2LeQgTXlxiSgPDkTBBMlCD5oin0FLgwxS4XfMYl18LBQogYP1Ja78ewsoWxjy8vL+fXhbjrqgrCdDXmZ6AOrYQSA9iI9k7YxfwxdO+NX5SUEQ4sDsQe3gmIzQVqogMccxzvp87mT8yHvcjHnUEgGInV8ZBV9uOUcRPOfV8K/ar1IBPgX754bJX/ZddqqJxCrjYQLivv2ybf7Ppe+GevzjHP/m7j58tCwblMZC95PUlXiAeoO6xRL7tsbo39Elz/Epx7MBZp31V2T/5eMJiiMfo/yB40ozmBX5kruyuRTtdxBU/IHVfpLIWlS7PZIQ0IuzjbcRErdc5TykZTNVivyaJndnHOh1f2YiL3J/pV+E+4kzk1dQvETqpl7rjTtbhwQbZbukvy/GBP/uFA5uJVb4b/1iV9RqItBuJeQtjohT+tXxZX83geBX0ji/1Lo0h7RjOmdcX6ACTnDsDNrpPANbpbfBcnfnlGuVl1wgWv5ZQg+lTpXCQT3Ivj1z6Eq3Nl9edEmHcA3fNS1JkQ3XYFIrCwhEcZWeYNqzvTQOKwK5Af5kcDpI2AfTPKgDT7eVVBaZusHlzD2Dix35n/cvP8OuPw0ca+UT0RjHQAs+ubkoQ32lqVONRHbkJchrJERJ3DF0zJjFIgu1mbYnkFtO9p+OqULmfcf+I3XCbGsKdJjxGnhlzM8uVKb+jNZ+27PDfU4ghxqqTWEmXuWL4u5tj/wRWGZWzFb5S/SNNAuaq1IH3nJ1ATnVtdQOfskb4sjJvEf2U0Y3Mdl6AzrUSGDaaokok6N8SnykkJljNz0uQgcRjMNQP8CUgl5frpAO+47sr3L/THLFIla+x+AkKAO5xu4q09Y16S9gNfwEQpEXyLs48fNvJNKg14yIk9M8E+WjSun5qqTGD329tv84P1+cGgq6DDxjT8hl8tIpI70VFKaXv7npKXRUuVN8Jek4lu/q7e0uJPe9OKTRLXMWuuHrdNdyp1hnJ5wxIB27w1dH7NmRfvtTw/e8Xagbdh+wZ0k8THvyih0SZSPok2SFUTHWXTM9ujpd8LHLU8XMwfIHvraxKdYeNdeHp8NEhGLQy4EYtHArXXV/O6BFhBm4s/ADPtKozM/iOmwsj2oZPhpt4GzzYhACk4DtBWlnKGf8QRs/hb3HMBubwkUM3DnG0zPc92Wqz982OxL5CPXCPJcR8jT/9S/tD5JEt4NDDYFYkmTX9Vag94MS7KOWPpr+1eZ0uxzzh1rABjyflyJdQuc7Ko+EI/GcErHCXCVU3lovl7A6VzRqTeCf+CFv9xiNKty0IOapgMWC2wUUx/hFfeRmsOx2OOk6vT1l1vRyaPzZiQhm40FOzg3Z8J/mXgTRBn+ndcH173zHWwMCuoGCZ0r9DrHneTC/sGWFKpI7iDcqPSR/wXsxXvP/rcKgSGcBB++JlFZ+npePsuqIj14PRcsGyB7T2sWBHq7jm/hwlZRqKvrp4YUEU8FhU8tIAd6dJ6/MmH+OPmrdeIXGiKww5OIUhgxuqD3eXwRCFNCngyL0higRPLbdes773fNkAfKTKnBvPtBqpZtwJPI++Q4h+JX4tlHkhoKRLEz5Vkhr31nyMGaRfswvEsFOqm2u17gTxyLF5K8shLSN3bHI/eAN1DJ2GvHFkQcieIk2KCaeDsUpYWnRRzb+22ASg8aMFypEvmzHPtqkFi6F7vjk7InbRhOvULAX7G8gWA2OGF8v0Hkz8sD6Yf4djxEizgdEffURcSP9bEve6Fhr5Q87gNaEkA1CMPKhk/p7Sa07WFBdttl4peCX/IfPIFTxgZZSaRMhrcUDK8oNeH30U06+lPFvXHCfx44g15tHjkc2tpzd/YiEBKmf9lAUaNGIoPjLUM14AHaX1P90s61tI9t0MO4IlAc1QrdjRYUR+uJZdnjOL1T786kpfMqdUbiY6e9K0a+X1h3WqGehZy3K9lgpUg4/C7KPEx2ywST3YAB/tcQL6W8Hzm3+RHvrbUup4WpiI7jNDIRk8eD+ocu5F5v5Jayf/izJnn46G7Ge4Yk0f2rs/+qKSttSro8pp91sJxEgf/65H8eVRnYWTBSTvbY63jHaFKBCYDLKN6GANYFUSEvG90Gq7kGlEzjDw3ffOpKXCADEM4Gz8yZcWu4IDF4VFR80XrR6h5Sftg78eix6PD7HnzMiM3e9sAaBVOU1pR83404I9FZ9nclsmcci4QQwUpvVUk8pDNEAB232z9p6lLY6B5QFv+1MjEyA97x0X4n/l4BNTGGM7a8/Tm+MJUHKntgQ3+FVyEIXrGBxGJUfCStNRKNCpnAnzOUoV6R6tGeh+UNEX6mTwWuUf4V/bUzI8u/ciFvsyfo3ya+Y5wOVC1LB5+hyvzEHtGqCIYQHEEFYpg8GEkzkD70CaI0Iq4z/6RRiR5jjg4o87nCVHNscIH/F7yj2iOathhay3YS5kQoXXom+CztinfR4+E/SAGkDuDWCqnemF0qHW7wDxy8kvN+gxDz0LWialPvzN3oGRVoU8Dio4K16+vQAiPB9hsH3VpzQicxLww/xmdluy1GjLflWdTr/26F1fSlfY85gPSx2ZYHbxEj+PXyJ2+D6OuYX7p7ULSDZGdVwvcRbooRY9MkmlQ1dvXSHAQFRLesYeZXsjLfPEDyk+koM15OVVLvu5iT1ZTP3SaUjPvb5iuMOoYpRtdllt7CpNf7N7gFPMYPUmI70CvzzGzQl17uuP6TcHtTchb+ShhN1GNAO+GUJckh7vDXWONyjaB56WqIfrcyNOqYzCYb+ziB4WsOceBZZslVb38zYyK0rmuof/+b9+C95UBrRlJpHJUJmnDn1k9UALNf65G8pwEpPTx2nuOdoKlV8Rl6JPQf9OWZlhui4e94Gur+q8MuVmG3+TYLK2XgL7UmYUleaBUYzbSIHy9czfE7d24nqCbheCNAKVX+E5Fm5+W/yZ4fU+UblBV86ricawQbyGodHZlsLxRkcsLJOA/i61gFxHb17eJhy2PKJP4cuTmEq9fqcHohFWJKnHt8ItH9uxOxVgycLOrcfvvmwMmt1yG76vxjjWDVYj+smi6NV7sG0j9B6SYcYA8kxhTnj7cfySyGMiw7Wo/46SyLKQ3ta8EdDJOCIl5LpM2DLhaPdGLY4SxtlyZpeUoC4Ny3C/UMIq95Ash2NA0I/guBPGgQOYGRPUR+IMTdqv3kTRULzEqVuZm9wv+UIlZxzbVTi6bhYoBw+Dtf8Hngm8SL/J9BmyoJfpVIeL1C7HxhzHUBjZ8YGZZ/8mmYOqSAXTpWYNYaoXJkfj9MLhhJfxV8dCJZK6UuQqh3au50/n+zQcf1+ckKh3v1lmDvs3DlKhJ5kEbtuNHtFK+/zdev37/L197Xs3YTyW+THxpL6mjI/hPioH2mOb5+1gmiATEpfyPIC11zzS7VmLCNyND73hfRXryvyjvgyJVENIsCxIfq5SUUS/6jD8Z2Pcl3YbU+oGtyGACyBZpmGFOZUeZVQsfLbAnTkD5cxfeZIvshmm/xr4x+zKeTlHyDX65hL1er1DrQoN0URspD3qGrkgVAY3UG14OGYgaDQlnrwLlhiA8fU0nYMYkfrPViXQvTGsuVaUld1sR6lX8rqmdFTBYGJc0jYiC6UoETCEzK8lc7wuGgBiBKvauGc6dUkXztCw4oHhpTyGsSGPT7s1haCw1E8+N4cJtksEDoTwub1tSaV6CHwB2aPaVqor/ueN0B5UO4lOPhNjk1j6RsCpm8MSKj96lFV1YeFKjjrVfCjmJ+NTEybwQY2jSTH8ZiwCj6Ly5+iRGxmP6v5OTXtDSf8wvRqmQlGC3pfmQh6qdsGdiCkulNl1o+Eaihplod9IPFCCkuSTCPlG6KuGxU4nU9NfoT7PL95/HyyssSG+10kxew81uyoOha6Zhcbh/lyeuAOepDmn3MWRQNJcaeuxj45P2x4Yinue0YS603cEIm6bHgXaek9iwz1N+q1yGFqfwOXlUjbNCaXnRZCeYQxcnd3f3bJXUwZwYex1agpO062zqXo4GXxN3H8r+1ymhVvpRtgpPpfUF9zo/i+n2MmdSPlk1ILjuesroV0zgvJMa4MNqcmtho05/GG8irfSauwpHmpReWHpZqOEACHbiYMiKKSI503JXG99L8++TjL9ozmRtmiB3WC7BsgtRmQzMI9KDQyi2NcAMFBOhv4avLX05s1c1iuStMevJPNfh44EztemqcRD2r8uJ3LNuXGPMsVAkn9uzNBQfe1vUk9V+6ciP8KxkDAcfGNFrKRzLylyJuuZIFOSO8m4X1xCcO/44C3NaYcFEeZ1sDZzBxgfgCTvrTt2XzSX1dorsvTLozQs/CeoA4fEJNv9RLhvrLgrQkc50EljDWnclSqs/3KDJ4pcMWLbYjYwwKVRaNBnH2ZGFTt46XH+PJ32AO8rtgNPCdJ6lP7HaHjUvB3LUtZUCQjkl2MHHDZF/Dig8IiBRZpp3ZDU5jK6Bnz9yWeOsI2Wv46AfQTTyGqGRqnrcxRMVrTwhxWiTl5lsnncCqH+45I1wdQXKSI4TgI4PgDTTCevxLU3PsWyTsMAFSO/ZeX+huU0z4uODFxeVu/Sd9dt+t0EreIUV+Q3WAtNPqudgwyyzSONgX/UxjYq+fLa2TCtIej55JZNL91AgfxMGJo9g3fPTln/IpNKyYbIPiCVIXka3Jv6/ltpimFfs3UqHHXXz7H4mOIOtpIp2iLXH/V5+V4j6c5/H1q3lq0nwnyiHJsM7MxwTrl3bQJgouUyPO8mmNEgODnXfu7bUxmogfu3BW7PoPUChUuf22n0yFKZwSkISzhwB7NN+5DgQQFseSp874Nv7zytYCapBB431yhSlWe/cf4V9r9tPq7eQ91T6BP/qpYWuM793kq2qRP0NhQYjRd52t6yo9oI2Rk0LUlfayHlVaOmaZZVtdI+FbH3caX28xMt4TrFQiIaP1N1zDNi+E1htAvNG5EdG3KBt3go3pdXZv98em/NGNeUyGWlINJtKV+7qf74APZAfRX05W6i406ExBVXkn2d0gUFN2q8i+KwLa0YY32TbRYVR0PYdriDP3wNho1MukItu9ut7UKl6IfRX9GPPuA8lvcG8v32ac62VQXSkzZYWeEkXRmhCGp1oJz0zpzuxgHCbLawhavlDceLRxmN1qrIQcx4h2pZH5T2J77F4lBpYXT1MkhPpyLwRIklbnr0pHV8rDR+10t5j/ZN501Y+jHxjmZ7hmLrorhoh4RMOM+fiBIbZaFGhOmm6JtZ6QoAutHad8s970wnri1WnIhxsK+uMfBBE09VfXxppxmW6Lp9H9E93wxHp2CuHEsDGspAYTbYrsh5GuxHqj2UyQOkbppFr387+YR/loGMq62m1nmC70z83zXrve/pI2YiYHvas3xRv37idA+h+pLg/Dr4yseMZRYFpt9NPaMAAXtCWGfO+sxtCXboYr03xN3oCxc31gzwH5LqXuzV5WOlX0aN/PXyPVX9OjVvfdGfxL2KbFBisr+VFM98UAXkanXtWLnASrzB1SxoBN8jO5XLy2Hi+GB9YYiiO72mb9kLBxNFaXLGF04258GejR6NMRVbwkiBvYOR61wV998JnXuh7nHCrSG2a8/vF+Mi12Vx/JpZlgD+SyuIAQ+7odf8EOONNTPWtVtpx6VUP+RMuY0iNJE7xpgDWS1DZSvPSMJpj9MW3clLzAQajy6g7zJBoJeCY72QX77isLd6ysr/vq3Aw/5kUJF6XaxyWeGmjvyrZNoL96I5YXkKOscCkgPl/PxLnAQP8W0/qGApp2FLTYpdHXy0ncPsk0gpni2ok09e7G4ifeToODtbCAEaAl1gChfv0otRcxzYG28zIH0q/5QrFd9E39QfuT0hGvQXgKhjJvVdwU2j18xTA9WdUm07NUcQAFeadCnIlGk/FX5B7BgWOo0ZtADpQLmh2yHQMGAhL12TtPf+k/EfRnl07tUiXxKK9FkPNFZ2ESOENU/K2PUquv94B5gKPzRU2t62tXKMy1aXEM3nZGyIgmef+zXjh4uypCuZxbd8vyg6naHgkaE8L9s/mEnjm5yziJgGqqozXpl9t/RyZ0XMZwR1dJsMO3lM30CHgrZP29MpKgg9NSu2eKN1Z0cm8M9Orm+Jl10ozcsDSHZVQ1JwElAgC26lDjrmuP+W341N0NsEsI9IWelaZAeMXfEdVO6pvtZn03VLcOw1JazYrp1QL/EG1tBlLfp3gkKj0PHi6t3kSxhP3xDIGTSqmroG9Vr9lh2n9wdGh5TXXHGbvE2DXDdE1+bIElzaL+bT8R0cw7DXoO7uBcVQwZZLofGCgmcGoJwsEyltdbVS4ix+abJREn64edQ1OiP2J1j3V8OfT1Fnzy0lyP+JLkIc9nmTUv0MAoyVJO1UTQj9H7cr4lCf//Ws7rqTutrHp1+peXJNku+usyHai5leyr0yxX+u0Za0HtVOuUh5P+GhMKsO7lqRZrHVEi9b2wNkc7nJBPC/qG/IysPFHqMbUL/qSg+khdhRZDy7Fkqfb1OW0neoaWWbX6E1uzwgSyfBV/hXrAzR0PVfrA/iOgNT3TVE/QPzqzYUQEDEntZ2WsQMOIx8IWGCylW98SqFmd8K4WfXNvUMbOdEvdsfYAioaufmfAjDLvbVH90fUmyl0LDequPUiR1Z/GOM7QDHvibJ5dolWloavvVsnbuHuwZX2gvIS8dHcbdhnHXUKUtNCgq8SHAHLZfUNkBuskTdni4/PubOu6SLz8aFfIQm1cEdNFvRHHSHPxX8dSOPLG6FcaXWt5Jrg6W0ei6pPyKJs2+y4ieoXHGk/93A83TTjkKcgT+bowvOwNbsXkk8r2Z2PuwNwZyP/aGQObPCHaXWx0DMz/4hWjLy09LgujyUxWR2xDbCqbx+3hRlalB3zHzlJAapwQqfrWEE0siwVImnXUzEc9zcpur5qfOFdPaGmlCYSXRjr6MmjZSXAuiljSxDWFsPjcWA693vzIJfjZz37TBZp3J3YXSvsJo6WSm8YjDZw4vvssmcHcC3/dSGdrQPuTWZXpoRtJxYrmQvSxs43hjUyERiTi4cFG9ohEwNR8wasWvgxHlUL5xRM73TcXXqEkJa7PmZG5K6EZc6JuyTU2O18MXcdc6n8hbteoWh2GP1r9+OI37NKVWFhfbxH5YdQSgko1rpGjJYzkElbDoX8lp0BlWJkXy+rT1hdIWlEP4b/Rkx0ys5WdAUOdzXPD7ASIfC8OXcxcxzD2hrc3RFbyctPle506hypV+DmO1V1byCpI31GIxyNksaSwx9EwIMBXhv1e/nuoO6zUNyG0TsPJLxZArrFogFt5YeuVDcMzS+U3gE6Mo7LhXn5G2cQZhjHXxB2tZYyuDJoarOsqtY3b0C/BWeJnsAVuyOJg9kChdL7R0m/KDm/Ge6L2dxh7/OlG8K3Is0mn0/OjhxP+uB/LpsVb7JHR/MALn1bdKAtH66wbTOLMpiJIaPx8huFe4Oa6S60HqGqGoUFEZ+ot650QAZLMyRTza6VPOHSOBmPe9hm/ycC+ZTUBaXNyhzP7KH2Rrz18Hk6/o1SDGauWPgXH0pFdbxxm2rrEhLSpsiuIlRnr94FB5tdaN8/JOIBMbxL+n+2M0OIbGEA/EmnEr46GSuYgz/gzt7/L2tP3p3sUK1jsAcGtSLwkhtBiqlNP4IABXgmg2pfAWeTgcmQZXKckXwI9SCZunTWZC+FSiiBCom2ZX2gcvB1Iqxi6Yu+EqY9PzUHyvsmHkSlWiVwEWlzQyBVyyJ/3GX0RnURdEjE++wPtHqegKRZ6FiqQqkFUgfeCNcNzbHDVBItcXlUzCz+FvBhrL24XfxUj76u/6voqoa+yJjX584I4JO6469IOg3y8WfsojkOUM9p7tIUqCm12YQaL0Og9UkVWRKnzsYwhHZvEr9YKYX+4FSBMvXM+hrmgZTBB0sKhfhdv11VYur+IAUiErTOxqMR6GZLObZqn0nFrUMdtPfjcFEGgLf1H9lWXGDUkeTgYPesSFeoO1SaQ94mluKzhyx3QhXbmvumlbv/3R9+poTfb6ZOxO3+xpeQdaXsX4klx/gXWJPr/X+cM0vRIOsD90wLF03DY0BkI9aZYVbmLM1yZ+rTqD4w5cbVeoUEXrl9HfgLi4UCPTr/58ASJRm8G0vFhxk8Q5F69nOsQLqRBwdENHUvtVwB7lH7q93eTySGtArLEoHcJ36NfV1Y2XYjpGESI65jgZTT3f/gDnsPSO2srCQI+no+2aZ14GklTmwoY77C4goChxLvXO+qTN5rnQXIqW2fiGopZaQp0fE0IkYn0hGea0b3DBFuQZvVRlE7fUv0/MsODgiuOvdvzX26ItFes/XVdF9JeOzEUlzsxUvvAGlxBVyK90La08fg5Ubx/UHW9+dOotV6hqu/pWS59C8UiHxWUHSzyg55nibJ5B2CjG1KER2lx1Nv9GcuTe0cdVJCyBGyqnqqnaHX6rvbbEIu4nHyQIws8owLudfjQyv9FSooVMj/5KQpOkO75ycd1HdWcbREjIHAv1tV8OXsG0YYYRQRgwTkgUzlHCtTyYjXtMo3J4jtIqLKR2fl0fGk9AQi0d3E8vq0Z5c4RUvWod2u14KZ1EKFsNDokSYbT3d0FcFBFMO7nvQ3EgcxPpOy5G+9dLI7mNfisJYwO/GV3FxDeHLal2ncQ2fg7XezEr925y6MGoxIy9zer1QYBfIH0ERGkPFFUlv4+Vcpt68TQfYaZqqqI6+FHrzIi3CGnIrMFW0skrEP3Mw3QjXzizGmOQo/gkw5TXKT7lXkPQLa/Q6UqW7agnrdRV+ZfMtp4cjKGuaOFw5nh6XNx1f+WH2+pN7pwQUyRURbU6GXp9raQV74FF8HPfi7ucNkhAw8srdxfdDV1ru9K4dwc2Np/D5MhwJbNop/vZHj6ImuzXAYkuPrnBMtxHa53c31Rh9H7SdsgOdO3rR39wVrDV5HjlnecT2FeyBf8NRB4m9vfsrAZjOqbbiUHaU/mdKqwSSFxn+ikCB8YPHoxv2P5NP5MwK2Ltm4Jk1OC2r0WjTb2+rNYh/RqZXayPL5kryx80DXZm3/UbvzczTKYtnsO0jeeolm+DF3YiooNQOlS+qRTluemYDhKNybf6XHDBM8yHvxnnIzJwZ4PrTJmHpy3Dzn3ikyWHpDIc8+57dUcr9gDDl6q/iTqr+ZvNYbLob9zRESQwfxO9zY4uxnD+PordYYHSthYosatKg5mX0gzOlGC/u8VfXsilVKoP8SPFmcCkRVw1ZaOhJQyRjQE3EJbjON+J5RDVd9tfnLO28KUvvBe4/pf4o/1ljCgQ7aQf/DgX5xU4/khEbYNUcss4Q9icEvf1vm4ll3YI+divsiAdhGG3/lJHA5IAcBkB6Hhui8Y4S67y2osCufM7ITzt3SgWa6UlI2vBz/5dXNqBy1EZpPFEcf3yFgg9jPo9wHY1e3e7fCu8elf54gZdYIKF8m2yJ2isGzqJQ5usuf5s93vyYx/5iaooujsmb9ujMTsLW15h7AMeA/5qVF+WdqoT+zv8g9ttpyzZ4auxdK+7GebWA6LgX5up2mjq2hRiUUrb57/DL7XynQ267H7+AFFYIKGsO7yI3oCrcYw0NyeyM6XSmRRe/ruoCIAjhDsOG8zliYZqhdlKjAml3w4Ob/Iz6HhRzhWfpaPDlwBgUqXXFpwfrrvzzSoIUjRxrZOVTbmrhbA2LoMJruu/VGzcLqifIvSzVf7HNmnaPPim5r4V6WNCtj8ybI09MLnPKfQ3wwkBvoE7bBn/mXTyb0hiyfdP77w77v3NUPooBJAIhkUyo7VjtH6B8AZ+A4n2G/tuitcNRh/KdPvzvPjn/hYdTdVRgZ72RLlMQTdvR7MerWVfLedJWQeGWpcvb/NF6weHomV09XGIDhxYYgRHsWK5zW7S0b6k4bqVBAyoZaS6kNBhTWVu5xihLWumHLOOmEUoHx8LBFSmskQ4JQ8/1orihYa/BDTx96z8xoH/NHPQeJf7q81vL18QND8lywGm90IAmYk9LS1LhFvpplH4OaERfunZ/WKXWb7Ox+wlUDZz7ZqUIIUP3UJNEx/H5EQro36OPaQtt50Ms9oaJUt3XIc6viof6lrtHYHJxztBsmyLUTXWkYX62V9aVSBJQzHtx4fLevHnZQn0GajJcIHaRpB2qs5UY54CPqy/gYKY/sbw5d1awT+zQbT7y5ho9m3kG2K2xCYcUU/OUskqknH0rXRMyDtBAp9qR1Af4APiI6/x71oVLg/2WiHXafq4MLw6tMarnFxRuGmDpkI6G40GrLzpRa+qRJMxe1Y7PUVi65IgMcbI1suKEQaJ/8CNrRzgICEzHiyuUOnH7szLrS0aVw3ueLS2rX4HTOxCX5eV2GiI1qKaGUhivDYvQjW8lSZ2cc7fG+o6IJ4dzZ0cCooEGQZHHKkY3D7nHBCQ0a9g6KvZipXurwRVb9hZjepZ1SRNgKHWjC1pzIiaQvjqogutzfT7hqa/gwbiwgu+XL34T3hmbLfZdeW7Y9G04O4uw9ELeOGQeJjE0+MPsBvrn/kwo1l+0b9H8wX7GOSqV/Wh2aZdS7TWRE0AuYaL2s6QgQYBPtAgxdZEnYxV5gMdn5cHzFKO/R0T3n9SBxs3Q5+aYVy9s/fP82Q3mlhSiSwG6JXNzT7iOGakY0El5wC7/e5ljVm7+yA8h15/iGBSA/0HJQ0jTtDRONfzMwWshQLKEFrbRljbPqDLfi3QPn31penKL+mZJ4A8Bq+P+4M+no0e+F92bJidjDklZS33Z0ifj2Vjahg16iuEOcvjiwCEb4UaXj6Ipn6FQa0TuxeoazQyqHsDjA+5lkFxVGcigyw8nz5pq5X6oKvPNwhcu2rRlbO/QvE6QR87M6DWGTWLqBtgXMFxqyDSMWQ19TGrbXstYfGr4QJMANfs4b+Eo3rbz0pz9j4YDZuuoR9Og9PRWFVjtsaFRtWfOOzaTR5s2SQ+VDmtUcUwVhglZmRMgZvLgAQnpsvzXN824sCb7ofVdvZCqSzqFHrZ5nqmIr76PaO4si9qF6K89keko6SNYoOZZsiHEXQVXZ7o74xxhfnJ6xgi9jdTrkwYw6BfGoljMx5LXuaoWeJ6X3rcU5kKnWNj/uYjENBLc1GxKj+gq5k/vqf4JMmzeSPwUczj6XCbHbhKEl2dvkD2X6vp388w2B+W5/6KMzMNLAA+4DQPt89f5ZfcnbPCoZWVlVOjCxkNs+Y1M8AR6IHIAlpiMrku3Tjb2ck6ha/E0VuDfL5/s4oA1Gv3IchJul1MVIEnI2n0s1e37EW5UCNEBhQIb2xHatfyUy+KOvlYf90kpchWESVLaSXwDWlOlpiOPfv1Zt8Fviqra/1kHfiIrT3Z5fA4wqWKrLHMa1DuK3MM1aHo5t34l4xNTgOvwcs0mk1ZyL1YYsyNQDee+dqjlVzBneetpqvEGiTdfQXn4WYU7jOsbCOfvScDTiASn4MEl1raD41TpBoLTx4OVEdPfUd9aumNBncXbRBmCQVECsy/gMZDilBwkPmdZYraaTPF2sxpb1Bi/Lj1wU0B/5e39q3du5bQHjT65zm9Cb90xC+7sPk7u4gI6k+Cv6/kP5geu6GDDWmIHO1q64Jtg/M9kMJz/FoYHbLbX6LDSjh7gj4DMI39YgT+fC5DGoBrX/2Vd/L8XWS5kCJsqupX9xjTyxMZVEE/LFzh+A05EvQ6COL/sp+cU2woEEhch6PYGREDzjYzISYE1r0RfNMKmqtBPcGY1n4IpZFvhhB/hXqUTi2WKtJkUHDAW2zTklTsfbBJ+x4rQkFnqZUqKrsUFS2rTrcv/srUu1Tl+910MwG8RuZWHKF0vni1/p3qVslKA3dkdG1A6MY+zABAvLwAVqwQBs79MNoRJbuuBlZh/G6tO1GNhbV9dA4sMe8tqXKkcuKb+PzdSP6xBc5jh5eZKUZEEkXUaJ1wfDxqIud4f+1Dj1H0Y8sfeRsQIavfH2H9udjvI6vpopVyDuq5laBBLodbsJcpf/lz2ZFBoi2WTthUNTvqyX1sTbTr96PE7iT5EkyJ5JNXpRpeO/cjtforMThik/xc7XquXBWSV35vQiunyP3DGvjXLBlWG/Dtxeuhv7H5b1SpqH+Ijdm0r6WkM5kF5OdXF5kB6hUFkM0dceTc4a6QsJmOdhoDjO6NoYyj1D+7COnjc48MdR7j5WuCO+9SpvwEiIoei+K8k6fHh2unX0UadvsU49MOxPKkhcNCGiXCTD8bEUcYQOylTOBzf+NazMHbZiFwupUWiI4ULyG9ztkvf2rDCuSrwY3k8cAECKaIZkM4uX4Aclh3/1wiDmujsz7orwiU0iPJ3wpvazGiHha8UHO+2mP5cbykHaBH2zzECdeEplCPoICb4Vjujg9abiBMaFjXJ44NJIIpKgmhA1cIuvhz+F2yl59BBF2SaFBTuSJFTHe4tjceY0EupBq/rGemVMHK018rTjrUOQWQwn9fjR0kwcKkGr3n5q+tLa7GUZ0+LAHy86RakRS7lUR0kpIGGCCeROO4R3B5dSEAKssMD0AZPtTnpXUBlhltITZZ8iugIhH6CkwT4XVBcDUUVe/Vzy8kdjtuBT0vS2Jz7npZj3hr1qo71rUa7MUUNRYWrfLjLCg/od9c8fGMZAdvvEIQQLxNXRC/fntVIpldu01/aqthwsNo1rB9vX5yApTQVzRwc5cTrbIjagiQ17Py5XLRhWr8Z1Q6WMFG/Af/tL/LyDEviw4j/qEnptLX3niqEu6Cx+T3Aam6lE4nwamhbxeQCLxT+BWMNFOZbQRn/AfY21SMbpVC/yYv85uo5MLfKbQwMUdFqkApMuEXFTgVDBDMKMITCjOJPmJ5Qx8a1rAcFdo6QLV1++W7zcxaFDQ8QUPoiPBjTCsmFvESQ37nupw1md/AXbSA8/dkhf6JR53yuj6h7D7aeMwfRoXtoQWUpbUywKi+Wbnwa8iacxVd0OsLEAPPAFvCrJAbG/rAAhlJUGsuQTzN+cXtFHU87nQQ31UF/nNCdV+thDVsKzjDlwbUSrYMvP2BFXkZEhn3YBnfKdcdCrSIsKbZXmMSqvmEeXZfdaUk8vSRHb/lQwzTwsA7g0K6sySfY0GyedwCxnzyctAY4FYRpsuzrQkqHLBNyR/+Ljqg627/hkiSgnaiV+1eTgekS+DlgqpPg7oLL74VrenIiteYS/g9a0v+LqxpqpldqGttggzB9QZsMow5uLnhB6spwBQwcqLU9UJMTouFPAZHeLQv1ypVHFoP4+ct3ReQnWVezY5fl1wijzYY7Kgv0MgJkoJfcecxhvbY+W/Yek0u//dvaneExhOd9a4ohC8nnw9Ij33XnDphnpVfiXvNIcZ93Xfduk+kNtCXx++FdShPnHAnbCSDeQVshaf9S/LK+uetVJa7Yj/zcUrlJPNA4Rutf21ptNDl7cxV8cPI0Gn/OkTeXyYMcgSOMXO18oBHEju8Kd6QRn1B/uOPGjsMozDehrMvsK/CaYLHznuOZqqr6W30Dmbct9LZXXdmvkbQ7c/AZYOw7Qu0VLudsHZ5Qc0CRt+73kGid05PD0IkCt2Gk0d9zJSnPtRf09ylMq9b9H/FfIgxxR53s8+oHAeWCusRDmrO9n+tOzhdQT5RVQimfeBbXGWZqNSKRvWfQ3NsvbvSz7EUaP82RiLJb6C5xfJ4HZFHcccH6etU+h3ktsWh5+h+8BzXM2158/vbnL+yzMzJE6bemAG2cPjIy8/voFk5A+9Ra8jl8HG+NJO6oe1MzAFecui7BMXK0J/a8eIHhmTsBy6SRUa2OkTLjxNrKvf5nuR1+T1/Uu+1yNU2dbX4fpXzU9wsn7htVjffwXRaEDNi59j31TN+DErotFB7p3Hn+Lo8SUru01wwh4UDOs7svsp5dXWR0q9j3pjgNWsLDtC9EFMCz+W+ApOZnKAzweBPajcRNi7fIIP6EEdmuDZgrgb1K1Rq/QNsW49bIgkTMBefV3SAE0NcFwHS4H9cTisyRNEtZn5Xu/ceJYV/e59vDpzSLX1youGWwJwUqKToSLP9oChsEjYPKLk7QVbs7vS6Nxyi+dfIZjdSuQUPR9wotoZJ9jW+kmCXEv9yXTBSMxj/+s6BTbAwOY2ygG5UHoYkGSyPp+tDZHqoWka6SNaawmFLyGSiPyQgsUQv7dfQAvzqMAE0Ho2HZ4TSB8o6J3CMb7Vn5dW7veayy0R8uklxnf7mGGvePl8qd4aHcBY26oBVa+f8pE4rvALyd35AvifuvLmdn+CuNgLXpo7hM+J3+GlJuI5SwL6dVFNtidVCo7/WhQ0EarxPDprgU/M1e4cShYCH4/1bwWJOfxh/p9dZMC3Rmkfl9clM1G64f1UG3G/wDL3//M1R1M+0Ttrprv2fct/zRQUBvCwjqt/nrbvnJXyMoO25UzD+Lskgi/5dSp+60I7L8TpZry7WBPTc/dn90SBdRUsdrUYCLTNsyAoi9hVdG4PIpkw6N3pqFEw0Y0R4AFuPqyMlmrbNPmuJHIvz4f1O+UByXYyUC7LnY6QLMC77rx8ouBkhnayMvxqky1rigsa2YCa8QWNeQxpr3T56/5eH7OZn9LA4QhCkl25OYP6mzPqeawVb9KEskMKQ6Cmw2aot515+BGhtgrDmG6+zmDramE77jWvdBPinbDs2TNCh7+yvVeTQSJeqhR+52Y0R+LPn7jET3+gUj4XZgSZR7MqJwt5OyBfyHL4CI68v2WYBCpGjsK0iCtc9F8QipoJeM3bMm9CZMw8FJc947Z9ZrM1XzAhfeHggAustg8DrimsbkPUDGa/1hPOFMDlec1EKfFNOHzbByhBX7j3vjskvoRU1afhvmEL8gnV/QooIzQR7eX83z9ZHmK348GcB1deffNld2CZHODNhdZFwnET/WtF9zWnPTdigvkCYUaCXvTL08MzX5JR3XS626fEjmCZZec5auzJ/kEf+qp/4gQUwt4ihmKM1j+icZEeb/4DQDZLQDjhR+KmmBB9NoBSviw600IFGi5QXt1L/FEHUJNB2n31H3NDwd8rMHJSR7uluQu15EhfB2rvIO1PNUyIWw3jRwtOucNygdLKoKVzuURz8B717cA6YGrgRaVI4eyfdz62iWUZzUPllvJeOikWaleMSDA4KOE7IY3+K8h4BR+32lPlhv8QWXVr6tlsrCffKPWAlqO5Wsk23g/intyn+ystYwRsLgBsy4c+oP0sJR4MYLH3Sja0G1qNj/P5lrSF+CKrifaYeELqeFmdxQpz8IBMl8PfkdyG+HU2PI+Nv4HWvpOslI/DLSl1gG8+86a/aO3JffRSfCHd2UkIUj1Zq2KQGX38u/g3nRANfLhEPzvfzdQaI03riTrRApvsNWL9DQU3m5SJ6mVHgf7iFAJcy8FbG9sqSsx2WMB4rUYNtwc1IW3LShl9cpo2k4RRDjo8JMU0s5Vnk1qBg1x/oeObqnhAbslGv/sV95a7Az39Nv0012uE9Zyj2zxmn+67qMxkffno8RJZfpAq1E2X/mCo6XIuIIp/ell9jNcf9ponGnuXSRC3dWoMYLu0PxwR9B8WONSFbssZ07B+R0IrxJpaNtBSpHT15WTw1lz2TidFf5m+yqartoBLHhYZRkLYokIHmczS9gfBdnA/WR/gsDKAnwNY+0u2KShRPI8PZic+ijw7phf1gGW21Vctkdv7xAijFcqfvVWJAxTa6MlxzwwWKPwokNuRCUW6TqesmuHCC9YwG8kgGtpifOCMio8htR1W9RuZlUIxWfZJkDYHQ/Xgy7EFdhY2HinclgnXLPN8AlThZr2czCQIVXgSq/XYLX5uf9aHo9gijtb/Ll/iU21I7J5RHRDb6zjC6F7TO/lJSIk1/DYvqA9r6kpZoQ38XoJy1pDAaWAG/dnKb2vZS4QBrjYcty64XrNU2Zn9P5C945fxGQpZfct52/WelF2ip/F/ej6o2CyccvxjuOHmIUROedP/KERk5sShsNvCgUJqaujLNiSz79ItyjW6knR/Z50iIfG14V+BPRAm0DdvR5WLuGy8VcN8K03nfWIZNmDhCTDOoD7dqWq2HFD2JOP2LHIeruJHa74i+0e3+WuKN0rSvLf604r9MMsp3F6H6N4yDdHrZccZJGXUdnkbJI53AjR7MXWcaZzoUc6HMUJVGqod8J++qd4fxTLkYHZWoHwLxACnS0eIbualFICrl3iVaRpingFYGXkx4PAM8jAJT3RhyZFwrcW6dQIKlnbf2K7xBlgdNYszrXQM5hvFZqBmrTYImfqP5+3C1I2qfAhfl29OxETAU3UJnQ8etrfZoS0PVx83PavxuD61VpHt/mfp/LF3XsqNIDP0lMPmRnDOY8EaOJifz9UP7TtXO1tbdO8Z0q6VzjtSSoQbN79L8HwUQpDhhvK563JOEOm2X+K5YRrngB774mSc1eIm/PtNPGB9UdpyTCOrviPhxRhgWN7cEKBxmm+swQ7CHLab7RusxHMxAev1duXWBc6GAbWmsPzX1xB85WUtFkLMm7lWm5uKj6Q5/Wiplgveo07PPwLjC6IRt2TdF1UY3+cWPtLWtkRO/pz7eYC0b+OWaA3r85tFnPwXcj1o7ypaGktw3/nE3kdzMYuxT2Az3K5C7KOdGgXZVT6mC15spxKLZHXjzsfL7GJFynxQOSeHJNeI1vXzSEfzNL5SC756zM391WLWBmX6qrK64WryoWK8dLxWvGMrtb4yjza8YFlrEG7l5lW777+J+++RF9Z8cu98o7OKfYRGTsAROvI2rStCJFmWjODT1w5fdTHI1m6cVgTBfBn2/jeILMV+p0j/6rfAPaDveFMYlWa26hv1yLRcGg0SktxD4NnUTymjUUET0W2fBOctCWcZ0228EOp9bXfgnhcSmF3zA90RE2wc5ZOGJG9Cd5Q6kdlGj3NDynOAdVK8hMOx57ECsRYdlC6e/xK2trJNNJiy5z18zj674jB3tyt04EtJwzA//x+UbuHFYZnohXtK9K+QgFRxt1MbGa3EdVek1VGaQBJu1E872FPgPd0EQjoftT49pxG0Md8QGSDCJyRsVrYAUNenDtC81PwwZX6m8OSWDfNWhiWlc+c2w07FCfTUKOICQk7lfuKYs65H/rriL/dFmI41nJXhtyfW/wpABVuWZkdSmUtW6SZvw2RJtj6/T1979Cq7ljAbPd2QB1D0AsyhwaRCUZzDOJsO9K6txf33uYJyOA4Gn0oSBRhhGjGOoWy+sv+mcsW+JwxFeV4MmQq62C/TmFWTjHA+95NeCs6Den3GvwNH9yVgdpY1EOJYq9WFMSaZ1C6x3DB0k41LJUlS+9CO5+XhV988gOnDgOpgRMSrKDSjdiFlqjrXg5201yAZN0QiB7w2ZmR1JkMmoXl2JylB8U0KQKs9R2yZM0BnaNm5lNplPS2hsqyj4yTsyFmuc70XFd9T7GnguwWZ6/jf56cJOgr42z+uoBw6erzK0liqcIs2Pcc+OI2G/zi7weejQz+BNszNYMSTYq3Rt3CGtOlq2LPGcIZ5mHYY2DtzIgFdtD+3PHdHYg7SqLEhHZ9re66WIeZ87Ub685wIC3vJAW0Mr6WhABGd4j7FPmRZT1/SXx6wzE7Y4cLpoDfytuz/NDseGd9BD/5CbelazyV6/RutLAt/pZ0G+RMsYWyBM67yd0Vg/5VMDdlwItocbPvsf3RHcbTKVHdptvp3jkrebUtIlhAh6saKM2KZwIJCJn6agt7xS8ho2QtExG1WyaZBqKoYyQeFPaSDTsXH4ywf5z6S6g7n5usC6mFiixPuQaNzyzObX4z3uU45KmbQ1H9BEVkIHoCcdTZO1cwJpwEct8TCFLiyzVDBN0+i9jH37xD0qnm2YeoG0Vk9RDAGgvzE2L4OvcAyI4K8jnt1Q/B7qzBnfnIbkNYk3rY/SwFuD9Z1ui8OgdwNR42m/rq/GWe/58p30i0h8BzPFR0E6N6qYj+F1KjvnlOxjeOCh3v1GIsSZ0+6jV/oZ6XjxXfFR7rMwbqglfE0dilCY5a2fsV+2ZoEY2xFgTBphUikOCvDsEg1sEMJB9fylorztrPE7VbVUh+NJJ2mhsuwA5HkcZaaou9z0runu6Xcl/5WmstttzlEnb37zoIEG8oOZ77srtkspR6GUVC3vXlv1mbqz6zLR3q7iF7EMZLQQ26n78CHnShduVUR3qrOSyk4wEOvhLXKSVuEAB3odcj64TSYjcjnfL+7Ztah8VsyYU3yfIZK6YDnYnyAWqb5RbdjHGT5qPzVQAIfsHJoXyXk5hrsKMcyGY1wPeFm/KL9+RRUFQUzlMu4PC02qoKvJZBpA1OW+cRQgi5SJ/VKLPbwuFqiyjJeBsXwNYyqEH7mQ/MYcehkRuD6al0a3QCBSEqUtZrvRCB/vY5ntvm9o9HXKH9PLwJ1xgZIz8JAPBNF1CaRMn0kqGWZpFfzO1G9RmAxrU6WM10+GJg9Nrqqetp6goRDyQzmDZPdbuzWZ3zpmTMxAFsrpefgMG/RNnhhOJQG2ibzNNt3zUHr/qpryLURUJZI2HlZLaMmj901RfGuvfRDdvWTC9ysVK+asGp4j94jsFbmSde+euevmjOBPwaRT1KebBhTP7eqEEve+P4f31WfHw9WcWhITyzwIhkYJMe6mbZw8H8m396/df37+YuL9reYUTFBh9EYbagO5kRc7xrzzg2XL6OC67nZSp7AKKerEzqL89asjgUMsgvhaFhTpvcOBDoUFqGbFK+wgri3l6eZEHv6BqPrw+QzNlMqjgp1Cgzg6UL0mMXNMqLOZDW8c9iv8qlh2R17e3fArw7zPFvTeZ2hnrIm6SccERvsmGee7ETeeTk1DEsf+CR6xeWy59GmVDo12/VI8XLp/uvHXIkbxQnwGlb6Z6ZA2X/y8bYljAvqrx0xJSb4LstCDrrzA5MtPJlt00H0liQ2u/fEuJrbCwWwk+J8GZhIvmENwINzSyNA7fTHC2rBICsx2oFfVr/aDfbw4qLsgLAlPlgJNBpXjuo98+9D8ktn9LU64b25IwFS6rkq+2QpiaCFrsaZv9s4eJNFLfM/SLe5Vrml/iolaKDKOdw5stOUo4kuXS2NoAw1tdnVdwcwXRktme5CrNH21OHNmHGFel9OXvY2Aewrlb7AEvrYnGz9oR4yVc14+mvjgYjHkXvvxW2vfjj3BeKtjcLai8yH4xdlmOdQ7OZMX+XVm0p2o1W4L4JeFg+8XMYpbFUkhqQbJAPA96Dv4NuFkBsRHfCeloWrN3BQxN8yKXWu8R8pQJ09mUZEU86AM5/LymBG4QdsXMQXIK9vsE4uR9uwrhWNYzSC4Yfuz9zfcSSXp3aUmNUeyYmrkzLwB9MF4BGFC36kRcSPPf+j49nk7+SC72tu532liT4ktOtKr8MlX/jZx3YivhNHDK+3iB0Sn9h68Y/3XcVRhXGy/LON3iruaf52sALMzW48jTfcCb48T2sbvr04P6VswdN/+NZYXjYlwvp8GxtYsso6XfTy0L3vteKOqiIyCMSVnoDGvx+ONUVZ8sIlp9K9kLJOn+DUcyBNRFy6m3h6DqT+d/MJ3/62mg2XZVEP2Mg8SV32dD6r4Hjm0ZWcYIn8zH8HlPuoA+rEQ70Rdu7XKggWbG6B3caitVb86zc7tmgmeQNHx2RYmwOIMvr9szo4qUfh6sDLqZmDd9Mk47hri+FCgYMjth/yOpkS6BvAclHDWDDAnGS9JTW8xzeTQN18XeSboVoJ/vYaucPJkB7HbjRHrg/ayv0uS+Szdh0brDBOyCwpFXya/b8Fffqt9fy+Yutd2RHOK/wzsN0v2xz+oZVrpeN7rznYWfNaTi+H1+Hspcp1V0zAo8QRuyV8PGmU+u7av6ykX8CXod9dzu64Wg8nw4hWjHMR3YtsxgbkszEtxtPxUuT9fFqPO2IydXI/Gvr7SFSv567xeQOMZQf1J8V62pY++7Kub3tOIfvUNklnr7Q3htsRjDmHYgJE0WiUve+LPikiqVbFiMf5sciR+lfsbuzk/equh4Z+g3YfO9MeeavazBSZfV3opnKIKRmIx+nJMq7CNhPvgszwiCbvj4eIu6Z1x+F2yND8d0PdWBezOmke6KpIE9svUheY6o9d2W7xtFpwNbpCj71f2d3TaBkht39RFUTt+0fSDsp8/snoFOj4vbC+EAXo1wvwZXo8LG6/wgwmDD4QENnkTh4TpT6xuSoQGySnbaEXmlVd55vl8/G0f1hbEUqvF1yuSKnv/f6vdroorygMfq0EL0a2Xepaqdf4ME4uTCVIpiQfEAnqN9fXv4oQMHGuq0h+f6hkxK+SIKFKiLd2tENvwi5v26i0lBP2aooakHUv+K0xDQTea97QwQmVWLN3Y/Ow7xh+QWKZceYOoHjK61dupHte5NL3sEGWs5wy6rnTzvg8zR/WYHK7S/Y7XUB/NrH1UV6VCsLmyXfDSxJfiV+LgdRVZ4Ph7bY4Yt/HkFOmXbRuCi0hk5Md04KaV4VU/tqRXUk/Segt7sffAw3VQMobW5FNP39iiRrKolx8AS59wsqxYfWVSFkMeVR7ciZZU+l7ftOisOExtUhHSNZ1tKLZpBEJAYDY3Y8oJXbHcK3225uS6lxfttPKEuv4c69+YEALPK6m4/aH4WC4yy/wBuKaSwH6gNIqIE2WVvgi1f3BtVWiIr8EK8mES3r7b6fq+9vag3yQ9vvRbZJrE4bfUglEUQnEHDT69HdN05Qqrxb1jGKLPWk9f9PueJTWNkfjSmdt3zM/9WTkra/jYdoGgrwoGD+UnuPOJ+TJ1pSGkzTlttNDLlrdUlr33a/w46Wa8rU34jHPz8q/XOTkWovMxVBvpLesTW99Rq/CZEMhvzC/gdoFVdP4m0b2C28kXg4YmpMnaZbQzIxvrwoTfCFfG7q5QRCtFvfv217vrPolofABdmPMi8s+HDkI+M4QvAnbsAHPEUGcnpfMIyw85V78vhW7PYgDHKrPUFjJCoWZIbPX+jH086xsdKwPu5bdCKd/4pDDvDSkAik9TUTPXTijmNAlPTAYVcR6rf68WOjHgjOchdqt+3nBHjacu4DiYwCaReB1Fg0uJ5LEbs/+hC7t77INwi8pU1ZD2w1cVoa57Jw37wLzKj8o3aDMoFJMUKd+8+pwQ6Hn3MfTQ/ipzMm7ACNDtFf196yL5eEMcc1wPO+4nWWGjX/zvdAYXxh6aaHyzOsqHzBhTk2Ca2Nw+Sn2eH/nC7SVRJZP41rX12eDnP891XozYY9aPuCnux71UK9FnipCsfeM+XJTyKw/8m91jRF6orOXsN+SzjvvCxQoaJ27aBPRBI8Yrt5hoJy88/cwsmvmLP27McLw2JWt22veTAcCP4tYfxiTwAOVnh3/F2gd3vP5vlRCJ0eJ6CJ06jUty1VaKsUEDZCaF0+903SRsSzbMxTK53O0LaDzfRb4efHTaLmYa8rI9OELi0qyYVfPeDRADCOZV2MNRqhJiVGL+2kczK4dZ9Qrh1THvZv41vMJx1h3OKHqlRB7AORsypIY42kGRYHq14DwuG7C9VldhAfGor+eeQT38asmIZqFqEjFOGlxVE79vzXQqnR4/LA2B08PkvzCvyrfJZpVYppzIqVaIomWerpJGtDgfvIisidRGYxe1wUcIHyfhWcWSpnnW9eMDuSoAjTe0TURkGymZtUk/u78+VO2Xvfq17FKmgmMVWTufPGJrviYuImNlslmi5uTJ1nSFJIh43Q/fCtcFx+f0ZdIcI4QMTNtTn2NmBZia2z5hyhrgKcyhGnCwviInSlcQbQXo9YRonTaw92Q4eBQoMlt+GUjr3nIrq75iDf3a6QLT1a/3OmPfofRRl9Xh4aLaOwqON9xuYtRRyadFkv7TzCUi+4lc66ddsPKOUVCMCZUNbZ+TBrTiStIxkJa/gNAo7hM3xt/YG2ongw6T8PLSYoYlRudL3xJJ/ZmQQlbsGZ8J8hXNJ2z4LQdNhUhxQDwrEJqrQnPVFXSbxO8QSpwbd1xoNzAYZyiQMwuSXg+tc/UNjiLkMNEAkZZbq2vjQRo1pDwUy18k0jv3rUxbmN62wpxjFmtSElYC2ZAYSL7XPN3nmf+eQ5c2m2a3+hlx4OTqrFXtJoWftixyluKFQtW+4Mu2eWNQOPHXm/tuu8NrHbj7nmcLyg1m6C+TD/7pFdfCXLA9pXlR2K8v1Ksy3HDCquz7ODHKDuLG6EZFn4zxcQtunwY8lvUwKy9fRhbjPGzr3Hr+SMNYLjdFxw2yvpkEFc3m+vDPRsG5HhHvtIjdILcqDhTDEWZVTn8jYAOYhdvVM8ZdYaX9C8FG9ZaoPRfHABziQzoblqYjJkTSBflM5DRB0/8WekK0Xjjnrho1cSCnbCkXiuqrHHzuCob48zsNOssPQKa/kDwEb+3cNkqHyrm+2oylWRoP0QdR/qlv61SWtfn9+tOoZxjdOAp4SEJhlA2jRAohnawbczJ8qFCVi0inYfyUQIT9lL/RxiP6Kuus3Or1CQZ0JbxcLQ+Fhy+sH86AcCHJgElFFHOJyiebMJC7+w07qIaVTQmbfGWwkn6gtOmR4IHJzMneSo1w492W1kobkXiNoLokqaZUyjc98Mm+1b1J/fY1f/yopmMXyvtl1LwdF1TWT5QSJJ0PpjR81Ltv2tsmgFCUB9BKkqUsTjFTn0Pc3zXZ/7pxJCT8qd/dRAm3K4iwrCU6QE3ZmzfawoIm3puWEVTkeS3UOOSrLNRsU6fpqy+aG2YdqQct9XnIFZI/aCFnI6pZWxi5oRchNCejcQmHWRpKWg/9qHywj7PKNjJtHfvFStYxwl86bosDVKG7K33Kjq1bTQ49u09Qh0+zoAtCqpklA92LZDP2ipGjeliUcpG9NJA/dmOAmyIw43ajbh0l+Z7VcXbYyg4WNm5v7w3HfBTETtALgvYb7Gg41e9y5cVaFgIVB3KjYtlStDtauAiOjN0ogi3nyFu24R0u73B6X4rcZW/piQPqG93sep8LAGFxGvy+mV/1LSBgIg7TMbaVqlwRYrDwhXk/Us04+FBoEIS14umeIi2E3EnbVbTHd4UtdkQIpUthe0DuFWqiPbcJF+HofdauyF7f76SZAHcJxVW0+34ZmqVEOlq2H4SGMw8WlW9tvhheySuOrRqnCVyZThEjrFM7pzm8nkv7xQl05XOr+Rmqrzi+oK4eAp8PvMiS/cHGPPn4QVJ3Ne0lgmOv6yoP2mLUyydqJTBsJzZpy9CFKBJSI/Y5N62h8CD99XypJ2ZaquhDMjOCakpx+w9ZildqnsY5dUMjmPudgjjMqHRz3m1/oQpwlBx5oDTr3Idg3xl17f2VH+VubYURai2aEJMUGA17cykX1SiokG1CUR+0NoUjcUlq2yKSgVs8bz+epSk1h42AWh5W4p8zZ0LsxBF7lk4sqiR3ckzw1XDk/VZVHwMRMdzaLkuyC8rXDCuJC28CwFkY7Xc44s5GRfusRHtziChNok6J55oSHvcoKAEY31mxSpk28GeoESKIOM7OXiB/rlQt2Y08Q39bYsOCem9rKY+JJcMnFoJC9EsNWm+pgyGSpyPHZArDym4JR5BjWu62HA/9ulEE1jJPBQZFTdyMTTTHtft1V7hIeB36tvGD7wYWK3Dnph9u8j3br901C5t4rud5jm7LoeiZIWGL8Z7khbEmX+qDLdeFQy5wgF/R+RzN6+3YlXVvhpLfLoNOqwQju5SFwr1Kp8QRaUxk+dzlSbO/EhjzmY4yw2II37RpvcZwFVeo3C8ZrHvZknxgJ8i9t6OZ6M5jWiC8FA1Y6TY/koqxCzVWwvptZUvwoHugwN6fDiDXTCNOvcti3iBz/cxK6C1YLfHq8qIaOXqJcs+y5pRJZC7FMuOQ7g/zZ2ZlGl0O9SFvdQH5NQhm/s+2TI7jXIktDnY3nsuPpOoMOOlJeUsaRrGgfZ9st/I3oKtRWJylx4GW9OKlYliC6txD5tvmHn6VoPbg3OFGu2yWkW1534Ag85a2h2zZ30F1B9Xx6lNjGsivtUiKy4g4HU68PqkN3bva7qiTS7414vnQGrGkW6wvB2IVeSJdMYP8lrP32/cxjG2geJ1g/ySaIprw4b5zsQmL7FWnZrTc+qufnTf6ktUcagCQWUXW6U0PkmYo8OYwWmaX+szhfYAIEJCbtWZDfAss/qJI68Oxft8aCtvEmf96qANNRD2t8m5nG0v0Q+h4EpOEQpoMEQdCTO8QlIkCTb9TmGzHEJMgozDN80S0CJyjyPLuL6H7iwl4P2NE4WQCPCHxOIWL6M2cyghaPLE+SzMqOvxdghsQD6siGvsSCo74rG3Kwkd7FtsqfwN3KlOfiUKDPtFQhlWe/OmvQkCR5/dlVx+NNZjolie5Q4HjtYZMtMpDjiZHaP9PFYyAfDP4ChhXiJLQIvZ85LkpqRmlJYvSbwCo7vfGto+YOaNZ8XqLFyEgh1hTmE9m9pytca7NGg+VOF+BZDLPv0f4HQi7FQRI3hD7pQp1NMrd96uiGVE7WrEdQL429EOCfjACzHxon/cyoxig/OIaLCsn9ZYswkw9v3AvjEbx/QmPt98XUW3lD4qXYeh9miez5h+XcRX/wF1HQagfTJvH8QFS/hiKI2/T16CP0WXcLvh4gZ/8O8eQAvbrNuHEaXrCxXnemPFwMVQNhknkD7u3Iapel1/9bBRz2y7GseTffYtArAu4FfQ51URJQ3htqbpHHj68tOAIOwnpvV7Vuh1vR4WUCaIh0amVOZLSPkB1b1+RvOM/2U65YFN+l4sfu88uUl0QVFfyBZ7gOiV7+mTB/yxKjiuCVKLZLMX4evlN/Ptk94XyOiNZr6EldfIBzCDs+XvNqxEGCjh/QyXA4tZTuWl8aQt8O3oV+aHhn4JmWQ0IKqR0fhnXYPgosX8tP1z1eX+QH8oqAJrIJ56dGXklYXOzZekqv1n35nFDbEMF66+TRyQITPrpStbMjvfDesx1PZbpXoo+d/FxR3tV1w72C0QxmTERJJVT+KI2WxAcrkZwzJkRWB+d4bG3+v5AU42S1GpjA4eEtu9ovSCicsc4J89cYZVOSG1ZVTc7+/YcpmvAhS8B85u/B+Shjp97nV04NtaYE7pOshs1MdFiSkmjaoCKuH1f86feIK75phrbqmH/ziszJrxBLyzy9ViPBu+uQKsRaHepWZlFcUkI0KiWOAWtQyt39Lli4uNm9XASavFnh2Ziiugg10Qp77ePQb66X8NvZ6x7MkKydWden60AP0pxoHd52DV7+b1p0FFZqTWWhs4lf+ltJp/acC2tkJaZVS8pcpaSm0bZ+w41lFGhFCMpBm2rrCwqzTU/pMiXC9DBbUlm7Mff6WVxIy9senDQfC20EV7zf0aAN7/S/l9Yl1m9VaK5ACwCLs7fYPcwzPJTQxCJDQfBHnmYfLW6MhsPC55OvPz7kLms9S5crzdHplE5C9wD2JK21pMcWmbYgjw4Ykx4kRm2udaA+43gPslltXx0KaOHj4BVbfpxsR78ytJoxgMIYWo1APvSukLQ+UER/42wvYuuLk4uX18wE+druDMZ3KeCgV5WQiZ/2lFq0t81L/qqQ0ifmjJbf+N2lOR5oIQh4CJTCpyiLot2WCdKWeA4OVyzVFv+q9Rg7Mu+15Wz98c1vo7jvY4/2nf3RL+tM9u1cMaaNG2lehnxVR+EEHS83C8IGFYlYZzjYb0+FV+nHo/zz5+WOwTuvkFvPKmSMOW2tzcXeXKysVLcR86wqGvYxt1VjWx7Xk7dnf6A5ixDfwYTZDRjEsNJrUhx9pVURjQjnx/JaiziyxGvh28HbxPArREWcCyAlbKFFAqnDcaURhU8GYESj7dNDtTFtyoI8mRSPW5nNbrVf0sVopyS83h6VIWPg4EFxetsmhTgwKO6e/4SxWmXhcmVqMREebC/hEV38ochlpsvetfhmNsR1D+lx33DzMeoR/V6mEaTIIimak7z0lxErxnf1Pg6j59TPt0P2xRzKdajFhgRsPOBzK/bi21HF3bdjxjreTUTUjkD+O78s7AtvOipVy44JV3QKu85pq9ESCQTX/Zx0pu/5gYeKm3tIU79C4W5N1/3n+MEJ5W46Rt9K2VXs5kJH3sPl21aQKXzald36G7ni15vQYy8O7UfjOcX1nmkcY1QEe5hN3HdqQb0RlWeux08CFlG+6Ph6OdLln9JZaHTpNimnl0uidRMuvZeF/aDT8N7KPD6iOVxrGrToCWjadgE9Qbka2dXj62y/fyavuhXgD8H1qJIF2NDZRVQ2taZztnAHENhvEEhDVKd3pqETvEVmPeA4nVZzlusxzk8ngH9RYGTt46OmzLE0Nu4YljtyKu/QV/M7e7nA0TgS2Vk77NS4Xnym+a9AF/9NZFtBNf/XBeDQvFhvt61Y9ea8WArc7z/i9DdSIUfEOjAnmSarzg4ha4W+vEPM3IbdL/gB2YoISRVbtxOmNn9ZhoMFzji7S6OqZHO0tfW5TgjAr6Lj0kWrWUVPoHlaMHOGGZong2o7BRew96ZKvpwpUbZqGDDrC/Mxr906Vwm2GHpGcmI4bktEnQ+ZvUWX2Ndy+pU3hHW1yRvmULw0B+Ifawojl6vTqW2IQvbB36bl7mLZbyShOap4bBTKQLISOF/fLCEL0pOp3U3v+QZMcaswZCGXwqm/VVcxk4BdaPs6koNDjgu4QmJNVsZjQsGjZHJsHZUfXQII59tP81NrhclOG/YZyuBAZlocxiPD3y8ipM+9U3rPj41MIzCK0At1UeaC4m5zK2zteLXB6hWxPZffWoTgbBQj26/6O0bBEGZ9317rOFGjfLB0FMzKBwFKwT2i3r9YpSzZLZqcM5zZr37wV4ShZEaBU2nFcXEcH+5NFqKqH8o2qtTQGkDBBHwhfrgwWy96A81PIDDJKVfWdYVvuUbYWzgVyacAQVuajFbLmUMbyvSBm/xKuongUwquQmDxap2tNGIcxb3R82exZ8SikVfyyG9o2RH5dpJPsfSfK14pq62BDU92ye+05oyjRUCM24FKNn9q9HvoN9LFytlvkIMlPbTGNSh+II62KVP5S4q0ifpPQA85Tw67n1w4fNteBrt2poYtx8mb7QsWjY/dX5RHWSDKuVOhZSPxbmCEAKncuFrVkGVtvYAksJ9PjUwNLnKzcwPS1Eoc0uullJ6sXQbcOcMsn+NxVJ8u17r9Eq9+ds9zujMMXghmjSr/Yi0AaPlmAETOej6O4v88DUNi2uuevHeckPbG1kiabkh8ZrQjJPtnKc1qdIuQDqSFkatEavtOVKH9Et6k9mfOfzmbKJk/nhth8LYoj4/iZhk6XL9OlADLSFKpAmviNLAT/d2QA/bmbcQeSXS7dCEIeb8sUtaOyXP+++b0dODQrPwvcddra2IdcmiuHY+oX3iymtLxDdd1DftkqU9pQmRWqJwfl9Gt9Elu64/iL88Iby9duDSCSInywunSPI3/MdvnFGMwzfU8aFy03XZfOU2w7iHrS/5a4edWgoim5pA9ifjHnCUfx7g2/qoHcHtWWmdWVoDK4hx3dKEhGexKBIFtLbAFM8VUYLpoVypfKvelH5/A5ZxLctpze/mh3wni/NrSaQXD8r48Agus1weJcZnIx5+lY/dtbWQpEBI+aX9ByPAa0pkt+DM9Mev3Es9jlZ8E7YZwij5WHkxv6pjNm2sCB435/QxfAiSEtWd/junHEUs1bzhAvRHwDBF9OlCHNrMs0+aZVV0yUWqdchI0LlKlbU0Nr+iPmXJ+bh95XP9BqcewF2UMQKZX4cv0O4yniV4LQSpvdIgwLmQ1GsbBcUFDA+IhlAxac5FYLiOYMRrFP5ZAWGoLKI5OV1efEI16lZdI0JYR2Ofm1/8DnqMAqZ5QcXr/KhUBgXPruP93fDJ6tFRPWbGeYf9iYRhSloEcsiXqlz7O+P9OrjVh9LrD4s8x7aGj2mAakWprviMMO3W3zWSBz7rkrDYKFqEaCLJeBO8KckUUO8eDypshHvocfsqXGZUdsLl6Lnu61BFLhSY5f0EfXIR3IYVY/MW5e/CLiHdAh1ZY0aqTPowSRSNoVXI0+l5gAcQc/569gwBsHjGUeKvjsVjuJH5GIO7mpaEz30qkyQ6kXItaJr7phvaB3mimVHfe3grlHcL92V6DI6xQK4gwfoegZJJ1vjrKfRxbAAJXn2XU4hkPER61J7PZsBlU0ELO7qmhd+sZDwguMfTPtzMhRAcMKzs8W7F3xkeUzIHqc7S2dlwoXaMI3GjM8zOf7865/CMohRQ14ILMGGSKfDe3GIRPeEPpGEKn5U2tZJAy1/d0kSTn7zOEA8A3jJhpp/QvOLfOgfqsPnQWaIhC3MW4NHuAPG5KCsAmBA5zH1HJASJMpusMqS2WQ1uuQhpE5pAiB8RcF5cVhg1CAuM93qcDHWlUpFQ4XcZIWa5BRXEHMs0KPeA4ZqtSS9aMwi+J5C12IIYEzbQkSXv7cBv+3lLIvXF6fHj6oU1i00kwKD8iG8dNs5fWE3eaTdoxVgO0y3MSbqTmqm01hI/PD/A3sCntnuC8m+pYPmV177icm44Z0WW0+S/xCAhhZ6/X90O3gAOr1dUgfQVOI4c9Aof3mUtnu3JPHzeYrp0YQwTiFbX3Bm+FVLTfZqh3Qn4WoOYCaJGAuf7/k4ChFHICX7MCn3i3yHd6Vz7FcBPOKNfG3wh1ldzH9wJiseEgykQv82RdLy++TD7ZoGLzhsjSVP46XcpMtxS2FomjLFoNolRVbfMfg8HMiavd2dOahiP2CnqZ9wrXg9i6/LRSGvj/Cg2+Qepjvg4FpAYWGKyDvfzIM+l8fLKAONyTf9/kTTzkTgkWVXxVobZmS/bxfOhKEqAzSa1GFDdnd59MHXRljKv9DSrUnQjBZlL6G2rTocfMirYvxaGXTJkaQIsGZtzC+Gb2RrDM2JlW2Dq1D5Fe7GlDaQeVvzF2JNCtgzOTrae0JKP6DGVblNQivegmM46qVdDOQU11duXRmTsDeHEm2gqwNtAb17KG30scQ9VFx17zRaf8soXh9Hdt3mQ2P4bJAYOkI8G6jndJ0Z/j28TEEeIzl8U1k25Y83HlbhLun/7jqqyhijG6HPCvS3XulswC7ZjUx+ZJGG2gTaRODSHp2B+AEAYu+oJmK+jAEDEPErF0OmSRLkEHQpWdzoOCLe7h99LUYRASULWl4xZ0zDY1JnEQ8Aptfn62wHwq1HJ4rIw5lAUe73FXS3xFQ1dauW7E3qPOJuQCmUcxoCYR11/1sogAzJyXI3PHlTZm4G3sGE87BQgjxa7ee7v4zebYGfLvBPsNuNWavPkRbAbQIm42wWqnj3meaKUcRtC8gF6jwvBwDYkgvrdWFjgAtjjsrgHpxwItEnsEHhFSI64Qvi5OALnsvYMIdzODWrPqLwrIdThFob2CBc+YRZMCxUKdEb7rWwwHsP6Lyc70s/nGSPuTEV36APEW7/7pV/l17i25xqZME+ov7xB/kIw54wR93p/JlcVFTYxkqhXaPrY8uXOaWgLTdAqiymKOQL3eEq4ZT19EGugJcFcwJ+pMEnGVJTbu035LYfwE3rozDq0R2kHWU8eaDSbUCSq5QRos8oSTKgwTOvvZdsKxD7CNRUPVQ9cTBJgoPewy9O1Ii819uz3zvTnZyLumv0gImg6Lry4z59+H0CfXGOEOwoe7GCgSzDsIjqBDDrBbYEk4u4TuZLYA1XZrj9nksnqtXXkrvi4xlYE6wvNyrio8B0QCKVtupEYJNxgasTcdwEBeM7Mq52Y4i1fiy6zKWDz/ecUWbM5iCFuYRTlzeNcHzSb2LJiRKOOHy/yfzWbaME/cC6KppTSiIiWx8EhKanuQ+zLD275kLecMN6N5az5F6CO96+6O0QkPSwP8MNCZTnOD+wZXX6ZcS56CHPtL63PEb8rz/Bt2eOiDx5RTsYBXb+hKKfN/GEhZhbCObjOADL23uktUBnE0L/2F/dCSdGL8YV5lr8yJDoDvxXxPs/Tg8d+A98+ojITdW3O1N73Ut/6xGHheHj57yuox0a2+iLEVHbRRxAUpySMT0ILfG0hx07tEQm7ZiF72yzYW5Q8D/QL3HY5KiT8QSFFh3A1kmH+N3QWjsvOVkJJ6FBEf3l9HrYXNTXQzuM0UztZPguGz8XlFS+ueOmpRiM8COXr7UxwtTFy9PU/SPfhfCVmlTXZyeF8vhrxa9NpsBOOPTFxP3aTAKPl9jMUs6T4GKC3qcg13xXTUC2Oeal2a88ApU/6DBHYmivy0CKGnIKCmM+WHWYJo1JdP+AMB6tY6RoT8fQoEdfxG5VG4p+7Ymmwi2mAd6bA8RHgRubbQnXfVkyDtLzV5xdhyb/vff8I0mCZQ/3ifdIEc5kFW8hD1/K/ciBvwOCGb9mHqdAOCwJcFGuXLxhmQK33b6S9KQchgrxAdwyahGroEuGXBpzVgLwLMzoEXMdfw8e8yJZfRNz/rreKF1FRvpc6oluYckP0JeelEKHR9mC/ljOI+n+rshv8q40JimT/tyNZXjnsxlmkA/DD2FVe5CpbjGvz1pr1im9uoWohJw13tff57iwjHCRcHo/H/rht38MuxN+T/y5sb1ZF6HiPZgRmSIxjaCleDDVmghHhMEbGb+oW/fc1MATopvUNFjUfQ+qBTPTe7DKaNYwhRWb0KY0qfahjSwNP68RkniHxpDpptqlk4BEIfYqYvj+MObMmt8WeJbsib94e/nzHcHebliTDbXLJcuM6xpG8vpWE7pOROjTK2+3ysKiSdgJq42UkyNhKdolYKjq//8KT4JjDYUn5W1D1QQsE7VMncCgzHrvaW97Lg52XGGNB2svg3L/3AdYVzcnlMRLC30kofnVU4V7958jvx7P4tGliAEZdK5qZwYJU7ogxxYUjW0kPVa0+qN8vz0Eaqk3NUsFpDHE8Jckj0sr5Xe1rzZTcshVDyWSwDhRvPhxs64a4v64X+8Vhqv+NlNlcnb3L/FC2GcYE8QzGG6Uk6Z5YKCucNs6AmHW+xDZztY1M+XEdtrHa57YYloZggBRHrQEClVbO/fo7vRGpwEPrQN6bwR3WFF9jfUnOmuCOAsKIfsMPWGaNr+qUiG6nDgQSLEyXnWQcs/7Xo+3ISess7Jst3u7X+HhYHNv2izqKnC52k1JpRDkhd89umy0UaJEVWZNCfdp1U2IrBiQ/sQz4uCx7oZBR+p+bTk5XyXEajT5djM0OthiHqjzhN3VBXc9+UETdJI14/IqWwoGyy6lI57QVl2MXfKALTz+phj+NdmKMwdadUTfZT+n5o0Ze7rDAuDAXWfp4LKpVMrZwI/kLzqYu29yLXOpV2du1AeBP6LLPwLXFi0GA7sZ37cS6oaO8An/Gmxgsx8GBp2UJmLPcwgVvEEeVFqPYZD//r32yQ6k5OkJKDS5zGBKSGhzOqijLY3td0o5LISFy9IbfQrWObKWqAUlwJlcKstTzCju7v17Ys7oxRlQeFF3elYOSzeWJpR432aD7EoGQ/bKA4ggGTcDcWFXfFtkG9J9ZZJNDYIx+69enXvWX4Yaw8cT+LaLJyMzAW54KV5++nVx7aMigzgswH8sLJObGUO06zRee2D4BBdMlQQwioT73+ul91I+F49zLLJm5MOyH6ROqsX8Jusifr1jraP4Ft88kQNBK2/qKTsyshDsuABtw2E2g8e/i6m8CqJCDYbrMNa55xdOVvOCuryFWRXBQRP7aDAJ5Ff8Jsnf2+YqK3r6Q59CL49gO95K20Vac4x6c1ZpIZsGxA+eqNW0xON28zeLBHDXf2MuSFC9uefDwFn3zFJUcSb/UL6udlXnCiJRDgXf0AJmQ5UTuUpxdEP8d7cUXQsLN90rXT1m0ulXB43xi7VhM3svYq0QyziFmnYYKAhLD42WCcyG0ji6bXItJRPlF6Ebp6HWjRGFwB6feaue5wRgYiM34dMU6OKDZtq9N7K6BhWnPc2lD0uuBS10uvM8L5xJEhI1gVmrJ2T46Ugs+B9bFAkVg1LKdGAxJLuuMK3yvpEkkYYc9tEb4nH+u7NdvxOU/ctR01If//HKXboRWyWKdEiM8pMBi5ixpMoTNWkFb/aok7fy6LgTNLEO1fcGZZhjSHSN9C6ZZgiB4mkWL4bS4nakcuwucfVF+A+1lBZ4cxF9bleFu+Om6kdGeojQtpJG0DOqF/uaM/kYnle0XpkgRSQ9NLR58AUSAz7FQrx7rTnGwlmoBJw0BcK3RYGLuPh8T5o8q66pRhQkR6K0l8eZc1N3GkmqXFF9HWVebgKMH91aSHSr2mSnqBGPeTHpLXv9tivI4GUWba6sEKfjx7qZebrimzIfDCH3jF1ufg0lCiqiUYfRpqPoDsHVgHYaHDCfYMDOMiOQ3kGWYoe3r5nmjgk5HwgZOV1J05XU/pIRkHfaWH1zL7Y8/AZoAGMcHfNWnl16u159170c2IWsjYtk0vXPimtWKJNDnmKYdHSD5LZh6qcNXj4X9OLO7Ak4FMSFQSP6m+xUTZSMwnz60/N6F5f7AXjlllb32WtxGujIaBlv2EcYvFR59s1bxOmP32iXH/6uvBTM/8bqdrfVBguwfKI1uikRGZqc4/ULP2UruN8zpPiaosEMY8LGfa51+OaD8uAZjcyzfjBNZmvidfp1BGPZrNrcseT7lLNgvwcAqBqE4WbiTMI465cpnHRFxBelScohcipD0/3pZQ4UZ/8NDxjgu3i8EDkovK6hglgX3HzHnADzJtbo7uuM4H20fdPPhKm9vII0vPfcoz9LuERrrN+IpksQZ9i2HoT4m6/K+ivJCycFeBk83TrHxXKo8ll71JccEKskbmjLck/VrHx1UpLyeAz09PeqeJlhlY2kuN78389oLiG/3AdEAgMy/NEkvtePWj42l8wPTdB8AwDAPOb07BFvu18MFfOjZ259AB3+F5XfABTz6pE3EfLA4IkVUMxAn/stitgdIRIOwfYjzG/4OxVVjlZ8DP3UyYj0XPsiRtN9fxgk9wWdNVNrf3hzD7Ptx5ERZdX/rWgRAFgWgQihO3PA8GPLRGLbnubFck/v6NJyEIJLeLA8DguqIjnvosmO/Y9wkZRC39yQiqRHvY+P66MfGT0jUAhwtSC0BkkEU/+3MF5b8+o+RGH9xbFs/Vm9sKdF48Neq4K7qozA57qiL0Uw8kxu4gbE1VwJiiY8nwyIAiAKGMccJ2XlhU1qQAuHhCZyxQU/gMTRIg1W/PCA5El0cvN1j0xFM+lKWaQ3BwkC6p49zlC47XtjGKBjJ9Bg8SKNbauvLB8eiy8NtRodRACoXr5ZBIQJN3nn/vHRN4uvy5dc4uxU+xF5h1Ta6MWkQAEjesh5ukacDEFqeiA82SFK4d8ibvn8e8BNFfj0gh8+wukkznGVFsvhPpxJ+U+VPocaIdAlJgQoOsmor3pSUNqNohe0g45rrhXS1L4sc5xF8zKY0U1PnfsjqosPyCWsm1+Fh7FKISZRp227Dc7a0mQdN9QQn1j0Bq3GSQHULDTawRicwIIogB1IFfrukgdsS9ssiDdSyRlsTiGOKqcTQbVgxpW/Vb5odj6tzQfiLnylqkXyimBtmBLrSqMoWh8lf9wwmJyEHB/+itfItDHGp2TPg3Nc3Y+kSrcIBOaikOTT0pcXz8LjRis2uxyD3IVPvn3o8CRO+LMFICD2M317rGpJ4sDkQqU/DAd/ygbMxsUDRyfPO5NoAFFFgyY/T3cpycvmdiWbW+RIQ34HXjH2WDoA8DWPpx9/sbEeTXx9yg0+RvTcCDzHau76Ex3wN624RcCgIwOIRKfkOQQnzeFj0B6Q3bMCYDG6fGEkC2+et9Iv2hNwqrV2Q1AOOIMspmVytgAWbcHu3ZKd5I/fdOLWJjhz/eSqogWDWQYd4iH/NrwC9/qV/zilxNpEZSsyetdgZFrUJ1eXFvsbXCtM2uHEgXDBBL/JUbGkChKSwpYBORDXOA2FaT82KqpTUXCaMOM13iyKQbkrLVmBmTGzOz//YbyilKd8WrugVBe/VQ4xtJ6Zc/gXzc13Z5mNp/wscn3BDz9kT4eW3Jsy5uWE6G2JOzX99CGYyz5VrKqzxEogFUOtj7t5y711asjJFhscDLGFJaY5GVKbt8FZpsu9fn6dmiFT6B8+5XydAVODEMouvFd/k9/qRhkhMxbp8+MMfXGGozjzGw8OdffF4wwc5P+EoSvp/ikD7IDzfGEyyJJer2dOK1mg7RazD0VRbE49/xoMnnJ0RrRG2rnlDaAcVrPtK9Fk7dO+8ym2p1TK+nNBP1f7pItzyULKcZ/DwrmjeNDflLy/btVzcHqzJ8Cn1a8BadLleVQkjDyVOJOjqhKLXLbe/W8PrNWSMXgrmyYnCzo/Gc3675wR8kMvaTz57k8sG1o2+fmWL7g/53eBAxUwcPWi6MNhr6weZ2XQ/SkwpUdOUahSf4664XtPsL9Awf2gYLBKFtIgiC/sHc/kF+ZYGimsgXEzfL/ebWMWXPf8cbWDgJSmkxEFCUKveaPzJHWkS2eU3Pfw3O6BJSvPCh215+zgcCx8HevdW/TnJko9bs9ZpyvGF7A2Hgz5XYus0yRxFLjrdmeRUiC/XsWqU6K5Ih/UFPk4IF6KbuMo9N6dKcliZ86aPaSbGuPEyUejSjheB9wm00K8egKhMS0lGmTPjFQXVlbgBZy3dSFImR/jCNxtSujuZK//VNZ3BsEnv0+ZddkGsYS+/fKnAE41h3gcp65hOMJmAqEk7HCq7NPQUzxeKIjS3sEmb9ol4agc94vAh8D8WN/sOkrWXuae8DhtbYQmwiVYv00kync6pMYXKB2Tj41aV9G0nAZ+S9Lj9b53gYN8+ovJDduP+9bc/YrbqfyddqLMDrD9jJyDuxvhH1pj7N0HVu+i3jcVUcMoghPPuJeIbn7rLRVdTKj7+4YZA3BNSc0RlIqVIZfbIL3w3WvZz55S/3OsUZZOQblFEVgDTF61saAS5RcWUtuOtR2x4HEReWNxP33unIXJa4SGEKakgvx/dyCHRPyU0E3kn2Wvk77V2pIxCB1JGabyCKfVFLmyrGcjTylDjJm4Q/fjpFGZX4CV8iSV7DA5BI4LYep8LpjCCpoK0o37PmDFblh0iRHCfk7Q1u5Op7bYIk7Daj1uPSPy4vF+NEAWg4dwvZYftYzMUnOTpZ8DKs5z81zF8Oz7gvTdZWk3MKg4n5qbfKzSCzu1ftRDRpiLkkIOGRVi+Jj6QpVwZ2SD4+/fzmhF/1oEPGdD6VYCciycxDZZJIwxgXMNE6IP3j9DunH88XTWC40AWPc3mYgjFzKzMIgtsMZ9+Ve7ZDSbpcbulqg/vfRSXyFOQD+ABXyO2hroI5HVvAxI7WzQ8OeSIrMLemu7OL8eduX1QQJ2ib4IMqNHGUCwP6vgGhHeCScT83uAq2Rbc9v3QGdh77cEwyg9w6vp2sa7HogO+VJnlq8ni8AC9Dd8qW18SUhq91LEFVtnSvEPqN2zGr4pVB+TR4HyBxAHSRZgnII/VBRwH9qvmpukekx/66n2P/9c3fq5O+qHKK1DFMsRT7jDYt0BXpXk6zb1goRsSOABcp4BChsG/F0tn9nEwQYYUIcNRWtCIZCOaplz4ECbEYdPve8l4UWWsZHlQOEK8FSl14I9t1zYJibgXkdwHkP1Rek+U/un1ckagXyTnT9r+8+uKpOnn4kWUcESceT941w6v98G6XfeWHxW+3NGeKP4m3OmlKEKdgQRUVo74SkJmQgpb/NvaMwHzc0Hyqw/YoXwQHFnBAe3v9D63nKKDTASdM9bk9fpAEI1RG8PDoPwx/CgYLBNgdw2SkXk5CWYJbwqsFe1OH7jdjc5+U6HPx5bHGeR+jh+7Oip+LdY3973XxmMaELbWTAdeHvFsHUS3m8phchayGR6qQAFD1n538qZ81Y1x8Rf1QL9AxqxV6jAB+80TB6SM5VHLlogCpfKVvvFgRJdf1ZpFeF+KNKPdRdOSc1FrxDHixwaBELy2lxDiJcNx7cY1etCTBsIcRmsUuoO9DdVy9qthQM2HIFGEmtAqF+WkfVZJnbw1S7OASaucUkqXbazWDpPAUhar5KX+qC8ePksDR8/YBXOvWfy1785HZNPVx2D06q4VOKV+vF7HsffWyudQXfZGRpct1/Qe8mBb9tdBh80zpX7svca3z52fnPVORqtGt0BgShJ9LZ+7oGgFntQTz1KYCBlgkTyrqL4jrYLLJZQuLr4ynivF6xVQeWYdyXdYCwEYiUq5IayNUUS6TxPiff+XgXuEMZd7Kkl4uWpIMZ2yykPNREQg3aykvYMgmphRYRZth6fF4RjAigbR4fz6FRyPkePCR0B21KDRQtPEyf209dJaz49HK+sszTPbUDO9THvAcCCR5EnLF7Pvh3yV+y8iDlL+VW2EkyOkkVqQEPkGo9DE4CS989+EAQ15MA/aeI73aEi7vIJWv6YKJO3M0vbUHG3l6O+Tgig2ZOAtkR6lX8tocYpRZdthPR1cAaJzRmtHoduXssGYME6chKJ4HxzSqIajuJyE9t0evF8q/jyThdNvR38orfN1d4FyoclcAkwU0sP4Vy6jWbCU/FkakbqyLmnLdpRAnwSwNJg/0huYCfE4x47+hJ7098ktRA+Jf8NQqNbEI683GhRaMRQxe6bZ9rahxuf/rBILuN72F3hnme9n1GLZaoK4uJlPdufJh6PHXEu5QTNOCdhvOolUV74OFeIhohmyiwkZO+3h9jfxx0RWLzwffwkw0Ekp2O5+3MfzvCyufSvbsOxYgYq4xSja8I1DjPNe3zhuN2cop7OtKgIE5qrPa2Laum6GJNjeZaGY83RX8IVeqY/eixpuB/Uco/XAhgtd/ZkW6Zlh2mZeYlR/gE73w103faLvPIx/s5fL8CFRXpp/Qn1WlwsKrxmvGwZLGaHhoJpVPO/dCGDPl/ObVfb2HU/4Dq9a8SnF6ZyyNRKMJjLd2C49XCMSHkeC+E1ooJCtFl/khKQwmlu9D7cZuuIoiUN3jlQWIcC07MiOxWBnEYZ1PAidljaF7cv5pEJbcqp4Zp5Xw2rjdxu1vCrsqrbBTCots+0lKUIcUYdB4TH+zaafJffZhNwlLI5ftqqKWlXK8W8Nj0NgX33FbffmpGvx+lW0zwf+FKRKgldyopSQqNIEW0O+j2tQkxtb4pZZW531kFfs8//6c96IwcjFr5tYdIWWzFQiF+kCRrTGNf0DA4GrQaWoFi7K3waKjOwPoB5JoU9iLoCQDHuc5ntYr/jtOTglGmzy6+ukVjeM42rVY7lqQQqHT8DQsEelqhLtEEJ2UvLVDjDL1Ml6Gxpvk4fHV6EpvMrdeNGozM/p6Gs02QUsKwxKJj3gL3hELJPvnH/IPLfoRkV4w0e5e/paAsvcJalvcbhWTZ9M8qFWFC7Q1rafP4gTClDSY9X5QSqH7L6Nb3iFNp0Dl8YPh+/HZpd/44xEMel8C6YrhEj3Eye5bnBcGRAqpYz597aVH2/bXC2rHnAb6iYcyfFFYBD9ge1uqgWfKRkR87z+1duQ3QSpLSnTRcj34IXINI2NL4+ejHmt9fxa1dKG+ib8X2Mzs6rfOzqpUb9QaT4JTEi+o1TxZKy30LaudYv1vHSBj3LwN2pYzuklPFSQef42Nf568dM8wLWRduJpZMlVppgtY40fvmYpQtZALLd722FgKBVdhMz4W8m7kcph8VhWf/G6q+wG7VhX42sbyavSgTlFODpO0dNs3/wZhodlw/XxAfYvPjx4BBfKKFt8M29/9bUSodvhAKhcLLGQoIxKQJrPsn/nUJbPD+kphDxMagHxJx9f4VuSIFALgPb1bJyajy3NW16G/VxxUIIiyhQIktKuVbKfGF6sdjUzGDdZZkwejJDxPIyAYVLifZm5LyK5YeylrGYrcEW9NS/lH2RjX8uj2xCwU0EQJrSmUdbHCzxOAaHQ3u38k/S9BX9JbRrCfDN+gIeT1SNQnQcMH+wo6nhm3y/u6xzp2dKtZeK1UO7sEL22PltPzQ2Ds6ISoWlnJY5dIdpnjLzhqxkB7KiLGCPOEVtPW57npXrYivXO490FYRY5xnJCZKLprktTYhTGfcGijNPw6z7kEg5GWk7hEcvkTnDMLB5R5VaQ+6X4n4rDJVQW+awUbZs0dABoRDgVmOLxoCSAfBTrHT8mhz64U7o+VVumr8X53g1BZ13vf16urFFIp0cHAxA1VvJLylM2EJErwD2G5eyQow3KCj9DnRpXAVpOAiXXKaztjQv/cFhcs4aSJy0+SCKtethbgE3NIpESaX0f9wgAyJMFxCdw8h06SBNh3+7gvGWfciBHAAHUqlPXjuh1v7GFijB3KVI4BVS31TLxl5Km0NdrXs4Lpr9NjMgUYv1GPFt7SBcNvIF+cJkJ7QL/QTukke3W4YFwCjqKhQojrzAJ2tmrUrlCEOeYMuTY7/AN5YZAW16uaY3Z+eYSIdIuiZhKJH0AGbXlPnoFCzwqFon3+FifUaocNCKYJaKAirStS1/p+9QyCF5q3o2YhaLrVz68W+5Dpb9FQKhD2bPfbRaR7Rd/Sh8SfX+DsC2F248txDACzgGJHraOteX7sjUJvFpWv+2JnTsVRP77jGeLcKRfKYQFzcr3V4TljxjAv0ZLMXNTLf8oy2zKQoWOYFIt6CRlGVCH/+u+wWrNX6IWn9ZMfjxwddj42tMPuPyeHDe+9mhFCcRsTymsEXJ6CLPARevFaJNWTxLrqGGbZgMbDdM7E0JB2bTizW2arX+k4GHbV+K6i3UgXy9y1UqqabfKJELxryCLm6PeJU1l+xyX/V/JNEDvb5z0izeUuLsZeA66bRttM6HwGO/nPw1ctn6gKbhHiHrfoLjnV8OGQ/KEeoTfNvAIqJXzgTQEpjNfOUwV2+bcpPmKsNpZz1wQcrDJx8wR/0t4oF0ckBdvMKeK9dy1K4g0o9uQDdJAqANstt0w8JpCVb3TiAtpDDaFzouwTy+v2BFSTqJ5fJ0ZDzQKUfP4tUT+5bvyNNgQE1QO2z587WHYp9CMxYtjKpJRTqvZNmnlaBkkX+0bTYJhQr7Gyz7gh45uwAcTTPPtU4XcjRkG6CckgYC9DJnR33T8tkEwvM26Ni+s7jILT4RfedZ4AmE5wM/6PIW/4WrTGRGVkGXbWDr/Gug+1x1dr3nlql/4PpUsvPJYUjOvbI+FAyBhep2LeKzCoDUvtHSLGpedAlVRDQLrSFihroxbQdM0td5fpmAh38ASPt1NlMN01GaDMWWihcqqVz3moUGG+56+utpZYl/xuTM1OATFlJ0bMqSV2hNcbBM0w0JLD4KiYX4jcOxUmlyx8Mo03m5Zrl8XDMcQRyswVLUzx9WJAaAzSuAMKA2fXgfUgyhOtIP+FBbFucvSWPcjZQAiu0HXaFCz6tBco1T2XJLhCTSlHA+EGmr2vXzbYOxYZ9MIR6YsOXJwX9CckYs0nzEsXcchUpr6b8qEMaNJmzn8EhG8uyMyA43UbRl7LKzTWVgumJm/Rd1v7TjR8h85Sz4FOjO5AJH+wC1esQI+pbKj3CISwN8jLDAaV2cre3YzTuUPSCXLfuYYridcPvdDx4TkhJU+OJud9GOgNG4XbPExDqjY/DbLAIJzOuiKOvyj8Nb5imOMQ0gSmw2kCUkZBgnR4m18vhyHrXKYBT5DY1MgFLPcaYDXfhaluSIiroghCyMzz7JUN5rcGJJ/c5gi77EvSo6DQ3bQffc/Yjjt65Ykz0vgNLorN3DJJVod1yPvDz1npnoN2cv8jQpAL7QdoLRQiXVOCSN5kOQP26J3y5sPEDymzxfL6h6nyE84hD5+Od6uFxngJBklc9m3vcCrEkL8pqXF3KrvMmhZ1r7tb/ppj9txmCFN1LDP42HtiJzrcu9TCzPxmlfzfFPUnvymXgczcATWr8WAfshxxm6Pl7qgpDp7/0R0Ci++IDAi2VRIuuRon+Vm/Lq1ocGRrHukw2RF3g+BAG6xfYFKjOgDFueJjldGXZ6dght9eHblAjxkVEpHTvCHxQ5pXXDSu7wTtOtwmBAkoHRLWME6MxY08sY4ZesqzA4exHLyW9jLTy3hm7pQX/47DpIlKX7nizgxlllzaQ9C9GI+DvIl/2BYl6NYsto25Z3gmXrsKiD0AOnu+80sW67g3EwT7E2Q0YPqejWV+wy74t6/bUgX15d0WOJRg3B4gb3E5bZrkwXGVvHr4gG+LXWNCWVRuYcnfM7IAKDw8/e+hiz25M63+YNEtwdLdoEZcsrcvuv9YfQkYn3roVer/aevwoNxprxM7KnX6blzGJGrBPBX7EkFERSWNGFI+VCLb3wSiXs1e0alLThcYFuJfk4/9TsF7yc7VVr6bk+TDjIkt6H6YeB55WDF/9ivC7FAB6vj5Lc9ZISCzAqWFXE/ftQTacXUJYfXO/nikzITV+67gRiN7lHBaxsX9nB0VcYvQn89Tt4ZlZB4S0dmEQFm/7qRz9eoMe/HYg9vkCxjFQYEMK+emklzR5v0McPB2J8kNVKY3A/SJ4eGd/0YAQWltaj38NkTXH0kOJO0Hl5IPMQLYFOxJqYbX0RQBRyycCD6gFuxCT5XL5wVRhygID7LixzGNf7CwLumiDaEZYt57DpOxPiRqwIxeD84j0Mi7kFFDMnpBNc1VqPN2l7/+LxdBnPFljtX+OpUyoLD8x3v4ZXvizxneF6hIJ29eMAn29blRfaKhvyKtweyLdkS73bgqsu2G3JM2Rf+HAtc4oShSMYIAgjfzsarft74U5tzsqEKPtnFWjFkf+PWlSy8HcP2mMhreyVY5LcTmtPwb4Z5J0x8KkcqGw+Hifu+7GFWX1cDD4/v5Jt0KWUs7lmfzCmuAkiAbfKqFx/zgvQL5QrJgQUfunC/6cP6PVjdcG1JOvJrccMe6pNLSkUOdJtox6ImFq5pDqCBO00xZNEqiF0dgKDw4imILoAV+qqRfOeenmUxCst7KVUC+/d+9UEEtyR97vnBtHo5Z6jLdjGq9PNy0KYJ8ZF0nUeiNsLKxaJCuKqB25YzaI5Zql9zsT7Gd8SlA3r5KlE6J1FCDctGIR2K5G8cAi1Mad8O6GtKrho4iaI75iZYFLT68sbWYvS0uWsRT4OePRC+EoFxXPMGjn61EMoLPCcPgf6hv1baCKIFchrmm9JeZmXJsy3YygsEs75Oa/+GKNOfzzCbd1/M1QGbC2HhMI7xaATfK3MXYc92S3A/dz4hOnOay2vaNMWq5r7fHF6qKlU33lipEebg8i4vIRkmnMdvhCEGzeoVexnEiHp8t5MnUT9aGyRgrTvrFbW4hO/5/BaNQVxbarlmLEKf26ZbX2qyniA5VIZCMe7snSjV1u/6QTZBuJ7ivmt3UfOM96+EhRznzCjHZQTy/VuP+1nndVvkYKMGf7tdhuxLODZBlSliRuSgO+fhsDvHIntrQUkQBbmW27tepoZPhPoXPd/NAWIwDxnVAJzWOaRsVWTwBl2sxyn/CI7YEFpICh+3ak1XR718/NguQcg1HhkvgvKn+U7OUENKsdDJznXAcktx6+ivATXLrEvYa9bDTj89ylIM6QPFPWma4HDYOC5yp8s5vXR6fSWs9LmNFiMGBDhzEiClomuDQQ3eGjfDC6hCHEO9Orf5bvHA2TaB3BbimiQVymvSiNNNMlDh6M72S8SK+NymJo5+De3c0UEACf8ar4zIX8rko7hbUZlzZmdNh6D4h6ecJfkXVny4wOsiZXrbdxbko2YhPbb6TdSayoR4zinKwb0mQHCW63gzYKOE+IBUdVAhOjyQyXaiLLpVXr6jRIeawiAzDhTqiDActCRzWDnzfj5+Gah52neueUJWf2VoQrZcZY38Dk3TA2VI3BXh9V9bKUvz6m4+j0idLCMY5FXoQH+FGwRNq5BSYx0etbXE1x0oA8iEvArFFIowFAYDVh4iTtIKR1P7j+De+Kb9AtBY1CbBaR+QXdk7wtS3ZkGCBqZWPsfgjVc+paAyYffPJJSYX1st0KgG6y8yrLDH/gT6J3cpJJgYKbgT/f583RZJbxWMemfPguzMEpRPGiSVdEOZ/Uqdmmnw3n7dQ5XL6BizAdEbdFg7AeR4YS2tjgqUEZG9k+TeJ+atDZfx2fF+ItGZkgtDrqRLWZiXCoR3RxCzArAeW1whf3kExshjPwxT82Z+C7USyLbpS9wMB7BCGwXNEax180Am2Ks0ZawKBGSgy8qK4U6/t4lfeYQKiknUDr4/wrclwB4zb5sdNidfL0pRDCRjh2YCOa4vTfgd1s53VGCFq4tLYXQH7B34N7fU3pXrNT1eWm/N6KVz0R7fwCqD/FkDf3f72m+EO7+6OMHh2RW6dEIwJzqOeuVE1SwpnQMsHo2wJlz3b205sOkIaL3ctGsAdLCYrsfaM058ZbaMQVt2F/D2fePC1W+0dWyoWCUnHJ389+EoGcVZL1Yy3C9GEm2qu8/vRHwhed2IsGHWJCn90ATxwpViT+LPyHT5ukXHF9pDmY2w0glXBMiOG1asLqxvsEeVleXGg5BFxF6RWlUQdxMr4ujqTGaL3ZUCczARPCEK+jxCu+YkFKUoi1c2j6uvKfs7qIsGyRvrb0itCPFTM/q/wtWLkIXEQLErUwiM0PDf4gwDL7pHLtQ3c35frtUAfDkrl42h3yCkXZlnSsRlOAO+BQ6yl2MXSvk4pvg686uGQ1OFpLcBkFn1226bfQicqQTFePxzXTJuFMFfu/cXXdPUg0oi3SvFfTNAfEicrcuA3gLx+o2vPSLVywQs+jRaH29W8pWycI0l8or0cTUw6XhhcyX4pYIFJPJQs0lEQBybHw8X488e8kwextSaer8kqGWWtIf9F7F9AuKFtvTtYpW5USQ7Ud16/ZxeE4hW2LdGBHOv54sr51xQOTfCODSxFNBX/QVTewBhhb34mkqQz3ORffrKbncDWwXf+y+6CULgfLXLvbg14y+5d2e43wxmVySRMSei/Fv1Uq8dNp5hjsp7nBYAts2nYewgNBM57t68jR0x2STGKmBz0AKRvvTQlQ/YZND2nBJD2owUiuy35TqlhuVv6iNw+jk32HkbwEQ0vpajbDqHQssQwiexRyZQcXQb8VG0TzXpnEUSffmrNqTphxk1dViJ0uuVs81Z2YcBofED7dAES0CgjYSJLXeZQcnpYFBUy5PKdQqLAC2ew1TOS47aOrxHsIWq+RHzl+Un0BefEdSqZwGgif24TEefri5Pzgc43103vWM8Haq8KQJWPN3+NuYVWEgFjcv+8Ox1JBp+VNxGx3bL4zMA39nPv033tamnFN6fNo3CmXnsbfNANldgjJbRrr6xnuPqz0FvB8MXDvoOIGPsNUnax1mOW/XxVsb5yuHEgqerZKHQI2xE9z9ZXYqCSFRT/KFLzx/4SqXtm6EUorI2e3n1IO2giCWZTlOS99wrMSlgRVIn+iSZ6fbaIlhIMSC8JI+7PxfYOKIsJTnxDdRBGlUfgwwQVu0Xu5cO3IE27rWgpoSyzAtYNjmcL6TfayRb65ujSvuDyNvB6Z9bnQuJ8d1wLP1/6b0TVfDAWpvw7Y9lDE97LcdFL92CEvWjZhuvrzq7Eutc7evkANlvXwRjvGjTOc/eMKIipadf1FOxYZFtUurb/eb7ADCbmBMVCfmZfrpZrcJfhpvKZClqiG0y7O0gKq5WiteIoTFblJBsXRQVE4PYrGAPKbt96PwYg51QVId/r5xm4c+xVqmAT9t+tGMnAH7/ORW1UX0cQa/tgaTCZCj27C4YpX6s5Y3odVtIA5fm3B7xL9/8rEVQyqcn1UhZPOxHUM3ZNUMzIHLrE5rWFsMPwwnkzQfEu33EtnGNL9+87uaQgeB0cih+pDLs1jWczN0UR09olMSRmP0zJcpKfbLnO7ywJsmHwVUEklmgIKge10kdaotte0cXoUtXbAO4QE3GVPNELrhwcs0Riio2QeOpyPfpGy2gWZFV7NI8tDEShj0QrlepUi4ifmkW6z29pL0B2OgccnJ8hIBUHlHtMRerL7TZzs2W6BLCc/b8mAEW4TbNNCrLhe9Dwewth0sQ40juXvLcGphzTP2IDuzyZAShgSI3xRR+CGFPAd79wMcILxO6IkeAJnGh3Dk3WIvIgHrfCjmr/XZXw/pCefBxxG3pvGPxde0cFWP20DTAs3uPX4BPQFk8+OFr0m6q7P3zHTzSI8ihAgVIe/513U7frjFcNQqd4Ews3yHQyM1vNvVDkDmXk6x83gCY3J0pCBGqR185whx5KDpbYwdtzJasabtRucNpqfPzABdrLkPiak7O/jKJJUpiINV7p+VxhrCDrNT6QI11bZRWmqtqsxzkvMUvltxJ8uXxGKw9z0caYww/8aTvgBdcwGFqwGX8tlyhfex/CEykcuYlMiHBw7HvFUefQ8rkuhIZNXhCoA8Co4TG2iYFRFv57No5r9w2BhInOuNG471AJBXvMW0+lwP0zTdQfsl0AqeibNKOgaAmJq4Lh7YSquSDAJjWlyEeMj+iw8JevX6/ckc0ijbXjGU6Hc3Uscx/6Z5yop0E3Mmkehg/BTSL3hZVX9OEw255cNDCcb4XMTIjo53xbVlV16zEuOblsY6qZ0RBKzP5XV+zCijb2uv4zAp0Nzx2bJugPGspCQGd4ez6s68suB8UYQWU780uTSqILaUJgdkzZLXtIbeWgSYqoq3rUVZwHTiF9zDLZTgSquq+39cdUZVlohcKathfTgi9A0azhZJzf4sgdI2nNVMtL67zBm3BciPfg5j/PkIy3r8cMXkqYPVm+hifWKczdziGikCRTqTgUdpmOQ2Yu46nVhw2xaezFVS5vSknA9jXRui1kypLwtcqCoKiSC49mxSD+bINR5572gZpv5BIklZ6Q4f9nQRTjRWTj1/DyFkHHqxfRzSl9fu4nrE4rzzWOx77ckksX+9LElld/bbDaCcHE1iRDdP0XBv410gih3OwvKyKXMLMFVGmH08txxMAiI8OUghemJxaa4S4h7w8X2bje99MmJzoqr8o0me0hljfxuPLvHMQ29MuV6VPqO2tT8bZzfFNnmN34eNYWpqivrkFXR6VpFOQnREY50K0oVdrcvN8i/cTiYuxu9cqqX/Xemrxib89Ip7btcIVUVuQXF9HvQvsvYfgJ1O8ff3RXPXIPrSV1i1FjKOEWHDkqx2yLltjGAbS1/oS7b2pUbSUhiHvycBvaR71PQfns5FpR+zFw0g3/mI/EsK5lOQLN7pSsxb9hkL6v7puAQ/NGZSciFyYHultLOFjrPGIoiksBx5eJ6bXxzWDwHmw94jVA/L423py0g+OUj8XeIP6eJEhk0aYdPO7A8CbevLzGRO4HYMtcyGkQ2cYCHaYwvGUW29+KU0WtXuQ8R3zsIzE3dDynnTtNwLL8PnGLhfbXXmN8Wc4dRfpHlullg6e+RYPFtzsplGDqx9yGJvD4eRw44Y6XaoN/eab/UNiPkyuObulZaPet8zmZIWgh7QWXVhaM7gmo0cqYdBMoYGnd9JVYb5/O7lgzGCjmcApVcLi+WzRF48diBiyR8mi5FZl6XA51193Vi3I+oy90s/nu8w47TsM9Co2RT1458OfFqGBdXGh6BZJPlmZ09XTHeWOLP+y71PT8E78YEy+KWV1etkeqL6LikiV2znWaMx1fl1nFDQmW18TrixUB/3GJfgXlWHD5aUrxMsq0gv6s5ouI3BqHMYxKnzb4FHG73I1Gf9QIFH9AgoS/XpoXlaK7zgG3zRLw9S6FhLfIpjfTKRL3VpN9vr4UZvEKj7OhGr2ED58wwWAi5DUjgoojyho+8rS5ZbPrizdHOl8NrtIjMYfpDh8/Qot3ITF7HrF7PmL3XVR3zkid9AtYkn3yEmyYEjGQ1G4SuFAQp/Dkqel1y+/3RP6tx3C0PMyCV0an6U5oPycCnOz7f304xveYahQ1kKk8OE+LftBgrPlOHU2ldDZzTvEMZsnHpqcDDjt0d6hQl8GGNRTXUCF769Rc5x8AdG5M+P6kDhRjrl1I0sowFt1jcvirvekDZjeARmk+XElvkoadY/cjUt77oBF3ARG44lU5zpC86C96/ueSzLwzzQr3tQF2U5rpZHzdy8VoBrsQ0ohQ6SbHf6IPfXbdJfbLZFtJ9x6gQ6Qknny6DzkKaX58LLC+zGj9H4Y4wd5oV7PHEUmOlxLomRLmQvZtqx/OkIoBmQUeZ8MOkI1qUVwfMJDArCbC1+yGZy6srJTtGQv6kyMaflskmjzAY1cw8FUc7Hx+3PynX1c0uvGarbJAR7Xd3QymmENq9FRhM3GRgigpnrXlK21XKKStZ1016/11jGqP/pAS5JcmQySPKH76xle0u/5pFVYCe+oXBpajRpQONgXSgxhu19Zv3if1OoP6tfwuBMZ9esk+a3uMF8D6cXA/XQ9/zo9pojhsCwYJwydsI8yuZX9Y9XZ6ZpeC9vH58W8JNV9CxjqCajc5drV1gbTVkJUpgcWgRnkLGk/LgFE2Vj2W7YWMl9T44yRymVS9aHig+o4Dc/N3nYacDkQrf4bEKADVT+rxyA+PpLrwNRFwX9dgSt/PhFVZBmA7hJt4TgbPdx8RK1dR2o2+apGyThvRGr6FeG3BHxrfXYhKLwhijNVHz9RntA4LRLeMy7Kc6EcDRaLHG34EduEpppzMj6tAmJrVL9/eWMqEIPRPatGucE/IOkmTIxM47cdQHV1J8g3BTGGxSHN28UjoeBUkHFOl8f8YJs2+7mvIEQBk+0giQxYxPHQ4PeHrYJHb0QeG6YlMedgccXbsN03LO3G13LZICK1/AQV0CzzONX3lyzqyOPUrHthWUVAqpExKvZOqi6njXQXxEXeZ/VCHeiwq343NRbXwCJ7uWM0uhpnm4BIYncZTjDEmBWH2C0Lo++0E6xRBgMN2dG+eqLxp1/eU+GCIFAWyPU/Mbudq/OLlM5L1SMuIb1m7+7skzmalAEvG8ISIw7f7dJ3YloFR9KoZtCbzdnrK39L4zXVX2K0x/n1dmaaFCFD/qHvmhCcsPDRheTvqNSv7R6+2gOKpamfQobhH3ZOTFRCIzx0fIWX6WOX8SqtAc7IQqKdmUSPvqEqs9qHzIPqKfHZi5rfWZ5bJcgt2SsJzI/c/+KpIojs0duZfX7ClVBzlzQfj0A/8JBwyOuML4OhYhnhHjTRLvo8IkAK2mIbCH94fSF7nyoB7a626YXgNqTP4g1VuV/tMTFrxMGeY8p8VyU3unmyqCKyDSLqhPI+i1JwfSTJvwRDSk6RUDKd7H0mGYLC1Cp0ZwidfbslavR9H1bKIqXuAX7zQ48jMo/BP4jJNEGSF5cRs+J0/1m55jhJ/8h1vUiXGImTOCgToqYvhYGz2HpL56KK1ofAGnUNJbMiPdgU38o3/ywHqGgqUcXhH61jBNkXzuYX8azo7RCiOE6QBNHfb8dlDwqG7GTRgkxrJmbaw1YBKTUkQb1uHK3pwxJQdc9vM2zaraY9XCWi/Tf1tmBHisRbJorZzzd9bKlGr/l2/+jv4S4okhHP6yKirQSBroxVT9Tci4C5G3urS2Xz6MRu8QFGX7EMMqZl9M4/dzXoJPADNgHaUsXpRIuwh2kpTqKHvvTq5P521uViyXeQ4E2b/Cb7fib0wg9B+VQ0uzbrGP5pthG+IOloCzNVt1d3Sa3ds5LLIxS7GKW8FmSFRuv6qSdGS9BPAdsfddv1k98VhQ4AoO1PNz2HZa+cS0adK7Md8N4/dm+2AlYBiCfjJo3SACfW+b1wlDFswXJaqxgHFcswjoRsesD8WmI1r/ytAHF4MG+85U56uQLTHK4d6h8hTGooUt6yu+yIeV5uqiTS+9iUaWo0RWJjz7S5ja0vABt6ctm2KWKkz7uND6nYsQwXH6bSKhwkExu3XufePdSDWi6+v4EDwJhvjdaq7iKIvPepBrPdJ42OtUAWXwHoiXIJW0fdvFDQ13tLi9J9eLwnt6/4ja5ZaJHmeGcM463Ya8dhFpu5czqmsR5VzUu675pnvD2SVYlA+G+pPGSKD29pyuK1V1hU2eRZku24rBFGK0daWi9Vpgy3p0+N4dPVHoCnSsaEKOzZIExRnbtK2skH0G0rKjfyvu+I29qiy393yADlIi2dhrD/q10gPFYVuiKL8WNysFLaHYpqX6Z+EzLI9ge3Xt3MwrDCA4WplIlAUBLh642jK1mJzN9Af071Kf1XZEi0i9gOlSsN2sb/sIa5N1KKiCbLtdxvwzvwO/RyhtGLeL858HVkkO1QzHuW1xU+/TDisVuR+ox+za257k4pwb2mJrDXV5AcJRM0SoEwRw0jQ595DVYFqIlZ/PtBB/shpkgMItYqL03NAIgWJaEUPkkZVJ7L2/xMLgSLnAuxitjgAZxGID67QTuG3UPdN2+53Ap6xz5wgiZLWXe095tix2K1Tu+Vcqy3ve6dAyjLY5dzb/81yHnjy2n8nYY4zadkRAZ0+uEekh/lIHZGSdcA2eCsYOPXndrmLSNJRvhzkPad7EGaHoUppC/jLNiQvOr2/C4pCLSLZfVvwt0RqNOKnwYdOyX3m6DJqLCZYMdd5MiPQ10Uwbpp6izbwHmtCuov6Il7WN5ix1zuOuJvVFvfByLaPoAhjQv/ziizRKQryurtU+1QCSo02fakdheXXb7zb+zeULT2/zWVCiaK7VypJ7Utr+ZEq7/AvBsZcmu4y0rL1W2pyoDleomgtsKXiGpwIK8WsHqFryy2voDcYYaCUHBDci//BaPzKxRzc88kq81mRPf3r4Hlswau5l0I38PjtlMyc7xoocO0j9qKT92n2DNgFfs3fKuRQIBZ9FEWzr5+T/OhG1YV7tlRBZJ7pH7XD7bwg8WgYPydSDBJ0scLYL0l+b6mnFi2fK5BTkRczw6Jrnc6/NaFoGI4SA85GJqrtXkj64crVdwg4tQ03ff2st4jvHQ8Q7jM+U6aBB3Ek0d8PDoydGz04xF2IB/qby6AfAvnEqH2nNUgAlKedGGgSbKpNYHPkByEFXp/quK+L9bEiI3ek3Ve2JAJvoXDUceqeWoz9fiFPApDkyYmgjtZTAxF3y/pO8sgujS9Ip8CofX0y86gWEjMF8VRbx7up5331AIYwYcngON6uZ88sAqa+S3YLBHM8UOasGagNvQHmc0yQQoaUOBfY6t/5WRTtI8xqN/zAAZorLyjZNbx+MLtLwBPRRGXrq/99oPGOAowBVQExSxi2T8U0H+3RnKupYWSdP1TDJg9I+XbYdrIQ/E3Pjhfn13Smipja863aMnfRhjpNZFUfTxME1BLXt1ayJ1u/iUMLCsjvfsuIZKbuFkYX1UVknNPI1vaAmBzYbnzwWvpT39SUOH+hWuqtrwIpv3eX7aCCz2HYNrPQSnOuVVjH8Aa3AgSM9dX3xyaHcAx/LHuGO5bvRlmMBiQbX41mcb+oYMPG0Iya92cuoij0PBURhg/XbnRvOIqf21JBZU6wjjfMknu0Mp9XmsdhM1eLbKKG/Hg7A2LHYxrr86CBDwMUTqC8PI8OIGsP6b65HM6EmakYdiZJBAmAGJLIyTwTQ+/T6Ylr5GiLBVauyCF5t148zLT/frSCueGIzLBIzIElyJqFLE+U9qoPhMXP9G9Tr1FArfR2xLT4B3ZsnZPa6AJV9ijx92UPYShDfadsD4Ik8e/L/DyGye352apI1sCci/QW15WNgA2UvdrUH/DMQIU/Mp9YKMHgjOuNa2QlHWQO1s2b8n4wEXjFoQaNkA+nx8j3rv++tDMTmjr/DYXV9hopp5NzW2jh/5p5G8OhLMfrOWXVKUPUUWy+vNeGacits0EU2t9H2AE4FO21c5ofrZ4tPBmMxjBY3Vj3+lP2A7K24hT0TW9Tmd9YLi4RQxlMst/y2IA8pZ92m5viDD2o7SP1m0qJn3gdh+nph/ZSeTtVQzeu/2gRzb2ad9g2oZR7ceEjWTZUSnD7oJAHpnNv6EdfpVhYFGbroHtfByc1Aw0pBQ1JGN235mn58dUBRRE8kNIc/G/kRJQtb+j2c4yGIXTBx2QJ2mZoOQQdAaKgRi6zfrc+FvEt7tgRcZwBZ3qAPSNoZaQ21+eupdTrSDjrl+AXThWO6JiXA25DdkjX+GR9DHgzKCaWeIWqwLJpekJsFigkOu+0alXkVnpqSxAjhutufOBchSac/kXsCBwzk6GOGXZOhyWGbhDCjWdGX1FuOPeZH/GwGYQztUtXty1BK9yQDIEfGeVtwlvTii6bETQGNzlF7+yVyI/T2TA3yDIrf59BVq1Lkq6ZcokjGuvB77SqsRVfkVQZUFazo5yZuEOI4cs4Xsvy28+X8s6fXOcMzH4W0WQv34xUtayh7P0kyKRfCZfMbQrhCPmR0X3k04XVOiFlmx2sVbP4oPDUpBy5g2Aq0nZT8v8/b8Z016fyLJUj6Yy01VH4AYriqXuWkTroiTXC1wY8pFH8S9zQ9VcNK6qIZkwdRkVzH8/ZKXStimLMpls1wQ15R4cx/argN0IaxhoNmjw+fOAqzsGgI5P5hgQ0O9jMRmBSdCfvnepxPATO36FHQZUo2zFrxfodcMnYF+ZeBkKOSSH8GUSN7Jde2+Vn+OBftEmnsRLOfUfpsQEGuST2CWsEGvq5Vk6HQAZoACHSqHj8Y5wzzop6xYVH5BIHbj+ZEWjl7v+TWIFYccWPRib83xVpYMkZgwMYGuo6dsDQJ+5o6MMsenrO4UDtyoCamzEuQOqDbFdjO9totCqI5wUN93Il+f22FrspbYC4lcx+xsM94aCo7u0Sdu6JO5L+Gd/WaaK9HF7kKTbKW91bRKjm4hmyxAMpo2E1jVi4LGMGrCQMJXG/DxWCa1QoOA5nCnR8FId5uLlRDoEwX+Ma8qU9dILdzOezXMuaUptdgLNlkNjtnxUfZG4GgqrGIQOsm2eN1MAy8E2AZzxtXBKHFbOlUC/vlOxgRoHdoflYCBRVshuU/sJj+57XUvk0XU+DOKVsV9/kD4voKRMQfacZ8SsE4B4AcgrZt9Lsihr0uYNimX5LLnfYoYt+XS5/lstRg0z8S53/s3ID25Cg7fzVv1o/U1SwYHmZUyqiyhBwBAt0nLyG+nj08FBetQ7ehNM5DoOQfJenl9elJqH+8W+smNBLfXbmgV9wiq035rGFyhhWEN59SRJ/ig/iAETygCHBgpo3vi2+dLar24EjmokzjdLMF6oFrHNF15ndjWDJEdO2u25wLz5abniIUIws5xfAdQHARleaYy2yn3QvaUd3owN+RlKpl2LLpS93x0OiIWDnyJCAraAQ/wJ67sCAAeTit49ZMdhxBVuS6AoAnm1uS8gNocCcCQK0Mv8zBVqSS9VTNB2Ag0w/H0hYjfLSKMmSmN5hR0OYbgyx+vFj2pWbq/35CBIhWi6nJMf6Tcs5E4aN56qjuQk91MIA30hYG6TuIJX2PJsuZKPgunhcnOI6QhNfUpWA0an4HIzbcT6fh480WYmh9OtaCkAIwrXioPvbSGeynbSUkExK3PNIkNDXNkMmWPxrJ1u+IKHwZ7VhtQE/COOiN0eC9Lka+pEcqMN+RL0528PZs58tr00QELWIgB4vR6fNNqiKXkT+3r7w0rJH2pTqWF8oXblNEQzv45hVksj4CoiKKybJmckIMxqHKd9/hhIAQiKySfRpLO15ie7pMckJHHTee3BdBHFJHuQL0nW+M9OGg8VsF8lJocenMq63i36RHAL3d2abAY8Zz8Ht0Ond0c6auq6mGSys0I4inigXUqtmyyGG5x6X9CclNbsJL4kh/AFpbZk+xDSAe9DD1mai31T7Tshq2j5pY2lQR2Hs+IeYKN1JS9F8sj3FZO8HyjMxAzy/fB7KwznL/lmW679hkX4kpJ7XWFMviBKYxVMS8/zCx3c80QLkk42CRwurtsktQIM/Fsz+3EURpOOknsnebklZDvvIJ7W7jOG68B8shTvhOV5PjYH7NMSFTLRRwrmhomDxg6fxJF206Y22jNMCSOr0B1PsHeUE1T6CIC7yLKuuODPBa0CguduL7krnhXV+7tgjQ1vMJsGIV3yvoRgHremBV8sniiEZgg9OHORXadruCm/y22lkevLD62EEvtE7ftADK/xW5okTivskWPf+6y7eZfgm8tOBYyZTlGYn0cUeBJQjhJ0FsJ5Wh+3NP1qquJIELsaehlkb6imVxdEBsGy4FeGOlWnnaXgGkVw1kI4xFu3L+Ig0zM5Wnts2DOJ+Q9Ro5kAwpvDLnXH3dnoRhwvQ5PBtulJBOnsX/boUjjTD60vd38dt5rM4eIfjipN6W94iD2NYKaZTPU3VXfpQ4FB0s/XxOvhdaUdALEVyG+SRhdARTR4TDraa7l1Fj0qnsfa7518oclmi+/HmK7coDH+XS0nKF4FWO1NAVEyQZMFke2wenTHuZA94G08e5Ib+dHQSkrlY5NhjBxiFFOa88H1rbSVRZ28AihqjciVo7oQJzOS73Wx6XHUHz9GB7Go+JSW4kwI6vHYmJesaHONCNpoRx6o9bq++2iSBEftChbw5pKV0WJ7n8lcywrxRv8zo9677T7gvbIJ8BuLousNjdGjT/HvO0Qj5SGWLE7wgNpd+or528fZtoP1Y2mPxxepxgAZYNHLwhBFtpZeVIX9OKnfRNQDRRUJoDzu+69+jV8Uu82h6bKNgxTB4gQe3PqImTSOp+d64dDDlUFQ4mJBXdLHaLrXNDxoDkxNiWSsDGXmcEwgO9bfHZtf2+6/9kPKc1eACIi3bPAn0+kkSayH0jcxfvPahZF+IX6BHIDKfuseaUomh1bxXFb3SKtdf9/Hkn3Qqbnztu6Fp75fKLwj6mjDHi9/PhGBPuDVUKv8SxCj9G1cqSE9KC3Y54ggyyTn2l2VSfxvoD3rhD1N7Yytf6jjndBlW0d5ROVLHxNldG2QyVrFPZQtlVH8lyZEBO6vrtqFKFSNT620j34IQjM5gl3i7DJ+Febxo7C6B4oP8gqag9hoZRSgE+eRHhMwam0oDF9QXw+s3XZyIKVftGiOoV+7D3DOWK4fQ5zwelQqd4FM6MnHDjsb5XZQYaC43Nr9Qqq0X3x0St7AqRFsKhtOFyXJ8RaYwPfyEZiYhzwH9qapym0gKMC8iphU49uT5qZQ0Qmtd+8kzA5OooOo74aaXxx2+MO7rjfxYVXdw1GmczvJ1aIjHQQNQDT0y7bQZLnw2egMvFXIbdkSnTdHy6oWC5SxAUaa7csd+y2fzhqnfcVt3blKbOV2csHkUu7sCCtIVYlz3+YvzPRK872Evz44nZ0L3oT7rFUlvUjR5SzhRIJWzRu0jt7p8hcKYkPTIfd9Lgv717GwZ4joLHGrTi5HYsbrVdo3L9gHtOwzZRQ9OiWD112P+wJGf2eK+V6+7HEzDhrD9nR2VaW//YwkYLW8F6B2BRlk66I7Yzx7oUd0o+IsL8I8ZBbWRi3BznGhtsfWdH7nJ4QRTWs507vaaPYDoQuPKgzd7eHl8ckhbLTTstpH+Kk4mBy+C10LVovhjJdXj1C+hLmCMyxj1zkljX/jrOfNvcQ6A+gY1e4z9hKPWegWxQWP2JzYLyIQuTjIN4jPsVZWILKNgzvqb/aVfezrfoeTA+zkbytuxQQtYyIMRqVZaeLT5nqa0TeNlQ5S7LJO30I2/lq29wqBecFsNuwbMz18I0lUQnlz14z8jTMTfzWFwAu7HVmcocfxBm6kiSHG/Zd4pEovB0Ld+SCnUSwO96w2KFTOPX9HKT4jL7jO9cAcFzxfIs6JFcqvOOOLaSt/H7+eXmVodOigB+07xOMlflRbXMRXcKbcV6ibyGWGpFVnvPO68arQL+MkdSPCBaekmAdn1CnEOMWTQFKvLk+bov8VXPl0XkprMQ+zEFJG9LX5m0J4+tDAW2HP/eJb/uXT/OGUy3GnfU6ylNsZKWi4Fde92hMCK4WSLAmTFSA0AVDc4hkXEuOkqtKlHJZaTSZoGqh8i9E2tT/eZ77BZkwXy9nB9xl8kLzg84Bb0YF9B9rQ+yi/kbeLYdBmj/WOOOG23+clgLYC0ZmgkSJK44j0nChL8SsvhQbptvIcqqviGbFznoDEGZsaN9TOpjrEhKX1vzhHOVzN/kpOwhb8tTxNfymrMiZMCG5vQEvHT3S2DHNDtk+nJkwhQINeRi3q74x3kwPPDYd3M8E8P+Pjz4z394KVmWvZVS4qt9RKOvkvTde1LKmSA38JGv+I9x66gTe8956vX+rM3ZiIiTEnuqlCJWWmVBIzlr/4ztTSm+cxFn9KBmE8lrhPj0GEsKldowevqexaQ9DS5D2arTkISxmgWidJ1iltRc55N/6CpMDQqtSq3OArSkfpWOG2QPQEYV//8di4+QoabPlZvmwLYtHz3+jhJjWG4Ky9+YjmdZJM8mEwRGjwm1irvHvDtGM1OdJSxbteyAjkJ4LUD6g/embZqFxwInC0zVCURIyvOlRUUZ5q4zMwfFFGlZg36n2jMd7xXqw1BoqKn94Y582mIZJkFfHf5OkDYTrHh1n9c7VJk1xlh6IR8H3UCO8mR1Zfpt2+R76R7RoeH5GGPP8FJCvqjDrRhulRRLnAQlhLjAfUnK7DP+FxWv/V7CvPbCVoa6bJcMSPR33UOjc8BoXw0NxJrJS7Mg3jWFzu0ZCmcax9MreAPmdIIa7KY7VZkzNbF9DH6/o3vJSYr91yEJq/6/z8g6PDC9YD3sY91/UwfPhLEjcYX5H8nyZseijBrM/u/JyEknn6l7uljBAPZcYPWsv2/XndZbZVCbEspbP3VV0PBMcqT8TxAZfC32eiEJXlaUsHdbjCWaz1+kYgRxDpv3a675eKsjxpmotx1xVmMnKgL1lprec3SRECGY0MM10XyGt7zSHUKatTZ91YOSpovB796EoqiSRwxet6fTBk6if0B8kZ/6q8RHp5EC6xn9kWuknzFy1uS/LqZbglCmf0YiYsyk66/QR239CfqjACMd7d16E5D8/f1ChUBA6p2qpZG5aSk/SMw1D/5Hfm8x9oon43we7x9uwfmP9gJp2B8JI0gEdTlG3dJjs8nx0aPARxyvpeAw71oEeTzh4l4eFP0Efzg5bNNvfApgvKZz3neAPyqiRcowUlyN8gVt0+3Q4/pE++DjKULF3PTaYXJnuaijo4VSX6OlyQfx2QOnm5CEqIG1YVORp9LGnb73LabOgZCx5DMTGRdYCVWq3t73P8adhsgtwtTmKaJ38VateFu+Q05ytGwEE+iX9+qBCby9A4cGX0GVtYE+GTL/WRfnrdLDz53w0dgW9dvZPt/q42sTACX9fXfHCWU0SW8oAUofjKIovWc8hw2HyDutBnR7gab7bz9Zlzz7hCVtBJVXq0jB+auCfGRx9lmybPP0Glwkm4h9AdB2tmtaB05ZJuzeiBemKbecCOX4q2iBO3LBHTQf7+uO2XV3j5S28K46jiRzqTG0KcxyQZVBz8WmEwvS3FMN6hazr148r1tYRVoMSwt/NdfIX8NOOlBVgazrH6B+2mW5MaMhvYunNljnBR40nhXRPav7KuIk/tVtHqlrLMFDYOkG9bTsqyDr8Ggyi/LIr/ZnUY7mshB4+pBcqy20n1eJ7g779xoyizmjS8hZ8xdlemzsyvIktxRSslbEfoZ7cNIL0HFdY+CqZVDwbSetI8r5mxjmJWoHcyUrn5+pAbCDmnA3Aiy/hROpy30SCn8lLKT7XKn5/HNqBGu7hC+y/ZphlCPQcx0gR/U20wrfZ48wxXMrGR29cSI5ML4CwK2E2aT7VRdgZcSqK4gaKuWN7zAUDg++e4oqp4+vDYKhJ/4wWifcZGooihnFy7HnXEQr2YdDrKWcaIqVhoQ/gOS94VlY9ieFmo9Z4+rMZua7olpJ025KbMSqxalh22P0z/QRZeyk6XNEpdr2ndk3lhuucWi0lFnhdrdxzpIc4XKTZ8MZPbt/W9sD/UtaiG7dTE3OKqHveKN8LwKM4XwSjpzxufweFTrXnB+dHKYRA4qKuwGkXan/jrV/xc/2a+ZtJftddUHqzGWX6xX8o3H8P/xtai/3U0isWXKQSFPkxfBqOYGdzUZGrVltx4Vf/qMh/+byYSILNUgKpPKGUbQcwQHUWjy+uiuffJmf9NGZg0W/wbRBK/zCT3SOt3aGmOLM20QHwIUlY6uFMPSO1fa6alQ85qqsXUI9R7G4A/saVYkCzrNRKxzM2u01CL0AIN1juH/GEgpyiVzjdzNpkFJ1nODwqn5s9ELhj013LLeUMY9NcPp7YOMqMPUX8k2hBFn/H5mZatitCZEGjA+7lX23BL/Yv5RrhpxugFlkHrFplH+FTIOeDKD+THgzNoPtetg/H+QQc1XTrs+NZXrxzBD3i4cyror1J5lGFfIFWYpi8uRuoLhv/I4pRRhdcLKOXnyvBRypaFyRQxpOokM0ZJturTV9reK02adWhd82yjxQsUlHmtKWwH/V29CkXZiNZHPTNLgDAdz8rZa019oglygEUgfndqPQPNJD0A/xW9LKhniVNWbC1tTCtbAkQLVLAkUeENb0xh8u4xibg28soPOIHY+7OKjRsA3kKM4BVZN5pm9AveSAsBjmxs8GqP7fBxZvhZ8dZbiC27CPdwvK/4WCY0zEUlUkUHDc5XCKKSuBcC/WY/vWdk7PlwUHS8HjMegwTDkMQENxs+tB/SXmk2wGy+hRwuPUix+dUiwVsuzi8UUg7no2fXjhR17pJSQgdSqE6zE8eXL3WBaRvkKVMLfPAnx332MhrVoTcH02WiJa7VYJ3wPt1s/afvv0r1yt8Rw55pMo0pjsiVcs2ymuxUfcR9OdEr6JEqO9YzSscQkZKnikoosi/fzbHL1fbOHeXH6OHICTpBAvG6E9xvmQcf6BgodJ59Wwnt5eT8Ox/+qs9bgwOTL9dcYYWYI2Nrksctqp6wuIDzzg8FvquPDJueJIsgZ4tJzroXVADxzuP8XebBjOSxmNPYvs39NxOOXu+ct8zkbzBUy6bmEABbzSdpVc5y5R6Vxbez+LtPTj/Kd8mFRa3C3lu/46XUsY2GkYrjRzkk3ISXU4Oxl6bFpTyNUPZFRomLzMylmk/RFameWCBB9ALRdU7nyHrjMh81TOYZS24jPrmztk5fg771Dt2y73nY0nIvyjBcp5dNYhc7xXE6vrtdVvkXYhgQ0x1v99lJuChUCvyZuyZy5mQoPs+JOAZXIdnItDe4xeNK/kHp1235RkN/uYWW5R3KvATwIqY9Nyd8nVkm+Fp3ByXrdixdot2K5/Whccvp50m5fi5V0tdn9oy/zrYh8rWjMIjT5jrFhtarmADny8yW4V/q4z9qC7F6In1qYaiqQwzaNmpuBCgybOM2+K4hLQT3mrRcGdSEx9Xh4ocv61IdxYMAr5X5nL/1XNmN/hIU9VL3v/EkQLhrpQf9tsc1NArMcDpVmI1ub7ZykkIZNVPkOxABsZ8ENSCqqC0kQGwssMtlMh5Wp//uryZy2N9Aszt6haJe/ksfPk9zFYF1+F8raGbgElu4pBdaNLk+1V9ag4dSq8tbgyEIprS7thW77bVz/wmwwUtwbi5mkNiYnJUpyabded0ZbXTA9whFXb5oW2rPXKJr0XUc8zaGj/6XYwNRIVkiqfla9iT7cqkuz1Qb6o1ltd16zmTyizqaXq/Ipu2ypWxgAE5xEkahVZ43n78uI5s3Wm15Ux2nJa/l/yV2i3AXEbL2l5ueNNnrreKNonQ6fnk6JM59/9jCshD1Yhju73qBAu0z1blpqUVyBf5TG4Z2536Rh+k2+8iSRjA5nggoq+0V1QpeAAK2MtR/mM7poPAQEwSaOS9V2XM9djuJTr6XWxwWon66HMgXsJ/pW3297LLfNlTmspQE0INFxuWrhQOGRqzZWmlOmXq328HLVf/6j5L0HotqX2aUXAeR/d27k9K6laCxKF9tIelNS7O/gp8IAr6fllU6DiFcWfB3bX8v/AVjEuXc5pL2CMRijJ9vZgNx2HHZp0SuF9RxgWrrhUXRkjiYHY9p4gqYb3T9tfoGqSbU5f9upTk+UyRDLV4AQSVWSnmMX7dVuwwauYm/wG+oyk3raRkdHABI0h1CEFI2C/IYXNMCKaoqZmQjcBAM4h7UC2vo161XTgmCDl9Lx9jqbEglKq/Q3dd6pIFyYucfEaljqNdd+WurWzD167kHd7ji6YvPxnP9hfq9c3OZj4mgbH4EGOa7/pC/m/fF/PxBXUQVp6hmhBISajY9Gl9fs3XO3gnwbR2ISkqMUuG54DakLbtqu1+o9Q+3MOTHES01RG7Ke/3sh4ARfSwZJ+TUhDRC4q9aHTkQMu15Y7vgJACDRLzLb7Hka/aqDrKiS5v4SQEV5o3hVXgz7MMVp+2+MOm7/r3o7RlBm4H3ndCWXTv7X0tjFRrZoViKNR6rx4Ca0Wj13khRX/jrGv6dfuKmjt+pKZEf3MbirDZ3XurX9WyP0oO4ntROF6FQZd+/Uct1M/7MSfr6HQYL1dqWY3jUaB903Wcg9loR7CuOB26alfYEDGCGoetVQKv5fkYROxwVw+tIi0ryCzxSjdrcD1Ju2mF9tSAm0CXyRX9jnBv1GB8tmjSYJcBJA7KMgC8yaMIIj0Z2etWCZNkbQTTmp5CDqC7LmFX+YXF9Na/SrfG/hjLzmh2FFx//0JjsqG4VPu/J3pnU7n7pIw5zcy7DBTzvyDtKkFr+G/wANpsAOD/3e51+jnTYem3rKV2XwOUozOvRgHlaZ+QpOOg560ievmYVtDr+pYfErGTkjQiMh0fuaTZPZ1TIPNui+ITZpUdGBTPRz6eMrhmBr5aT4CGO2X/9vTiPAoaABhjwcwnh51m54AvkqrVGTlymmWOochuzLKTV0F1PSyisc5NDCJ5LLmv7v/gIHr4KBGL5Mrx3mWAmOpNUpxUgPG4scP1+2Itl7thbYyKf0ztKPb6hpQ+tB6sFHLuCJsdgQ6dT4mYLffmz9gT0UHYkrTn+rz4+/+2NMlLLC53OiGVlcYa+IjPOlc891PItw59GLuxjtp4uTb6ln9sU1LFX/t1wK/fXqWKktc73aCiM1pXqT7pncTk+kv4TvaBF80RV7ePqV3xomJbFnVqoQTK7mUxJ6564Ds3GsvHlF9zx1W499/8BvGwvU4WSav4LOVa5hjYeYEfssmE+7KqvzYKBSwKWBhbp/FNA9X6moz5tFNSAzU64cJCsUYIAupTtOZuvaxAboYSoVWBHpxKhFRHA69Bf7YNlVKlHQAk3K7ctnQ7n7k/XUozvcffJX8PfrG7rR/aGT1FoFqX+z65pwT0r0AQb5pUTtarfa0ZZlUjjraQjJgnR/A0uugm/Qhdxaww5nesEd4M5JYXhzaxuy7gWRtQI9drVIVqkkI1O1rV22oGuHoklDlppOD48oci+CBnrLYRvvsBkoUYZADVp+N6DDm65CJJzIkXKOM6IuKAaUeoDEJ5SKLzY17q5zDmTcQFj9GjGgSPLqFUjNWGskmwb907j6lkaakr+fdY71DkqW1pW2QoEzZZbVBTbNB+5ptVysk0G9f5aNjCXlRU29FTJXn9M840jWF7TO6TY/nEwfuDMqmu/Po2DnShdjYnliPh5QyWg8r8Fnj+tXbCE3AwlSHEKkCgJFMT0LVbzy19vDT6KxM4nvxLAO7vym+luCkOKOAJcLY/nPOq9/Es9OXEy6UJlKDYwLZyEU/u/nuDj50AgQ1cDoxwkjWiZK9B/p+qOnuFKYxYQ3yfYyPdt/WzypjdWEjVkICWzdpNKiE4IchrHt0P6pVP1uqwsL0VkExdp/2N3lePXTJX1ZnVBUQuBuz/5MVVbbELsjpiBUmGAoaywbfTJrSV3/YIcUf9dlISXflVJAq0rCgo7hDU2OFKsUGoiaLVU3wwBUp2OEWBJpAH8J26yhb1jwOgQ+rSclhFHmavyRYlLQprKbh8KKaXtQ54nNAPHx5wfMjC7w9Mtdyp+IJZ08E6pYqSnv9ME1/0uhtEKjpXpwDdW+4JFyaWAoCO37MVABMMf39lnxa2ne39mvF0hliI2DzXnyKAfGSK6YUYNtO9s3PfgzXTbiwXK6vPHvoGNFkaH/WU5N8omCqlSVA60BBIsK0AZ0J8QCGh0G2G1S8cHtCn7k37DGpiCZReDAxKhYIMO5D3MMNX7aG5JX4i/epYFr7lWJepgFAOSvG1E51TMU2si/DtWwLeCoElrXFcq1ypHn22phfYruFvi8ea7Wl4RVEfVmpquiitbFT+dXjaN/OLO3MnUoTtIgw+OGOW1jw/FXQs7/YqJUdXpR5rVuWQz2B4Q2nJMQSC4E/PIMi2KAVKfavzhyActhBpPGgKMwHy/1ddK+PslA48hr7OzDmJ4/D/l4q+fu1wTJpSLG/YBYoxYnA1CoNaVDc9PbAvwby4WNhGHHj1sZJ1nfWx+BIDEPqzC/uLQfwotBW3YYVhjaLK0IPG2+hXI0lcox/dBSBEw3SBCd93GOuIPnfy2YgvfMl0yk9aXB/SzS0AEV5Nkv27CKzoNZ140YzX2vmtvivnLrOJ6+uoo8pFPvgnFPSykp7mtDy6LshdlpFV2zXV7IMIv63ES1aWm5p5GdEladvbPsXOpSdU3SBAz1+pAwOnSw+409SP8dQfaAGKjjRu0WwfnRf6IcrJistrjULqkn8wU4biNFqYIgSwIFi4pufNMGihFW5t3y8qVFAS+PYL4Qc6NSC+rfh+8XEDmnTniUZ/6cGBkUnt/0Hvt35ZTvzsnkhRn/OIpHjVK6zU6R/4O+zI9ZCjeHxCLKZzUROuSEJTNqe7CKy+Sxrn2x4D2Miqrz8P8nUwG+2ADirY6wuqUIPeuF83+N5NHyBcYZr6ww170x1rIE3KxD8aTXrZfsfS+L31V/+Am0sg6ydzPbxZHHMidM9Z7VFuJdNF6M/LXL1yPOqMhXicrJ97KcEzUnqlk5/ZGPW4C8da3tMYyT6Ohga3WZrabc+K4JVlTGFjor9aLFravmzpyaYlfRHyRQXvJV5HtB00/62OWdzrb5hWWoxs/tRmJAhiQymV58YO4hwmdsmKp15cir5OXGlJl+otin9BTPN0sRv8gKjJ5vMjHQAJElZBvmrBVHv3NhT07KoVUWBcjkRvFsrNp968daCPxQo4LMVgzkOO5h0O1LbN6Z/BI59sBXYnZA3OgdVE+RXYXatZCPho7R+Y3NxHYy2rIDEkQHoV+neZGKUYR/G2Y/GVpQRoCFEMxnD9Q+t+AtKomDIOwoh8DmA4QJ4pab2jaYGgrLIpOKqt8i1yX5uwrU2i9YJInalAAsHJgxgsO45I4tY5RCbYvfHELRB9LbFKt6/CXTtlsgthT5scCMNiZruk7dirocEn9Kaw17hui4LCFYxV5cbJHySYE1f12xLmOQox77j014//KSXRdAoM8mbOwRvi63j1nmeT17vRJJ9KqXrVz/6petL0tZUbwgJVBrWfkZD3GYRRMcmXdt1XAELEyig2ZWjXV8yATHK6B20clnH7UEopU0xigLyURxfQuHpKlyzoDMAzE+BuvCq4c2lgfk7l7TOdaRgalWsi8YX9Tv7ts0q48BDW9gj0k7dDPzIwf+3f39ENSkiLBqxQ367+uswCklAO7P4PNiEyNM1nXOv2VGfnfjRb6ZevbKj3OksSJ3jlfqSLfAPfTP6HlrpwZ6H2Y4+ntV5oXOzl7f/jSsb8leN9TkuYaempGLzjC5ALDSRD8ZIGauiLoGlboL1bfZS5xzFJP8n7aGpEzjrI+oswe41EKjXHnkw1y+mMJEIseTMOZwET0Uz5LpgbGQ5aqTmYbLLSBVsZi0upa13bmZ1m/LUWgInQbcEavTUkkGm4YwfmNvmNpT/KtOp+k/+l3bjpCBJFRRaklb5rnr2cRhkTSHXqZKEuOh0y99vcYTULc5xTGhSvv66iqN51tiJRKQ3A9fO3o31qSwpBTEJgxWOafZ3xsHtOdXPhusJK9Lv5H77+Co5+schkaI/pMCMxeu2YwZGsMfk981FbC8FjxaCOvNxzEydMoD2Lz6MiiWwlwbCSX+nRgCNsBxMJy2tiQb9WvQyRX0/vxcLnM53U9LuTwd9rKyxVCBSgHgvctGsu/6k4mPQ1tIYKw8OICiYcI879BVr+eKUobzUL75iInBxj6zn9+qni56ocfjREVQfJl5OvR3FdXshZV+HKGMCf9hTD4tUl7aXpwINAPkFRWzilp0b6c0qPQLposLItC/daZ05z3vLzvWOaBaOcZN7hI/4/kl9SKpekotzRGb7Dv/a5p27y46zoHTwohnLU9J8QXM3jCp2efn1u3zYeem4+EDj68w4YPn9QwlvM5TsJfK2Sm4cv9YLOCdTmMJuc3jNAhDqpVkMCfACscggPWn79Kp0z7K/7zsyrKcX0RRUqwb5iFiKIHt5yk9Ht+fpxs0aQx8ecb5fQyni7oOI1mwzERCi8Y/YVKvxa5zMm2SxN2zyVJy2oufKPCU4zgHOybg+mhVi3P8PCgL4iA3TI0zbbAKFv4NzKn+hKg2s683b8xLnOce0YelU6pM28M8oHqcIUHaSEAn4CUGlOGZrnFhgP7ISegUBKvqR5mAT3wR0bSiYiBLKXUP5xmRQdMBp8e+/V+o/eqhPkLgaZ/kwwwUYeV8cO4kZSYD8UQqNSNpz6oiNJ7VYM4YjgGQxL+Ze8aK82MUJBZZgwyph570hY/ofn9Mn7nY06FaA4FgkgS0L4UamMcKvva0oEnIUU9tbiuLv4q515RsOYNed1dbk6/6PIyOi+adSO3KG9Kw5zNaEugzcgb7A1e8KH/slCwLHEw1goQtL3kuGbGkFTshHldDM2PuyPZgy/4I1q8/kzhnUeoPBqBBU6RgPwgl77DlpctyyuDrOfehPlphB80LHSz+d7uKDQ7ieJxnWyo+hULw1k/KEfXsdMg+McBlMiGKWATz2QHCk/oH4NvlU+V78KUn+iXbO7itXD6Pe3UBiOjxee8bRqYvs21pksrSWoOTDNE4cL13I4OUOjkMTEDgVfW5LRWqowSNfEu2Pp5Dxs4N6xfv5vuG5RpESaFov59OP1FSrDkoIQntJTUGVFeEBzQ1YUTh38uRxSJFzzRg3Cs/iLN3Wd7HWRmcOFFs4Svc7ShsEtV/t1zLX+f2EBvftRbRym3YKnWYSizoaFjUrwPdcQ8mfritOhlOMrw1WuCHeE57MRGRkVyvco2jDXgYvGCWnF6YKf1toZcq+jw9fBvFE8nnEwqjCv7frFu7SVwJ6Afqpd7kpu0liFaQcnb75aVJxfT5yasNXjxGjLnnllI91D8TdtWhZtSoOVDPmnUa6XFhcZlAzJa4AW/ygGKnCHS4P0a6xZpgumgQEGdx8o1FdBwTcDojiZiNc2un5U8JF/6cPmeIJEZ47z9fp2KTrY1kj5H0YN0ZhmQNaASxGAMbNS8L/XUORuEJLK4RPxvutb2rEBgrByRlsNYrLw++X6BJtZxZvThnf799PeM1jkF1KqTFLYtA4oeT5CoVd0k6AsQ2QX+nr6Rnbq/r6Av7rFqTZlrdV+w/Zo9S7T5cJyrYlA+R1oAJhbCPP3fs+8/oLqbeeeh3xzBW2/864jGYfLkD+yH+H2zQBtLWv/7dAWJC0yP5hkAc2ihwQZSriyFVkWJgUtVPilHDS89qhLS4OljsVmAV2ocq723Z8ImnuQsMLaUadSvmQH8NvHTLQZ17gU0WK9Yrc0H+czCVIN6e8ANh0a3znVyafJSBiORw+fvcXj35jPan5MKR+rr3cQkEfA5BXEBOeb52lHIdbzA/vepHXcgH5edxAlZFT5sKp6mRQxdbqPyo9C+/63Qvo5mmZd515qC0LwwPjGbNsyiFdQ5pvrsBfH/9rlUGW5m5xhhSzggVzG8Su9FoC9mMg6/+mFD7b/xGPwoa0faXx9YCn9CnhdEFK68d0dj+S8B9cnhdTl08Jm6ojBIQbTjOmO/In/OF68zKoPbBXBAAnCMUO22/z5XvkoCJu2RbliOquSGpdlQZee/rArWz/iksaINXnJo23FB6FE9g/8Mmo34DNtq8GFIpyyGESPMedTgnPzfymQSwUmKhraknSMy6pLXhXE6BQSCWJnAubAeRiEsuPoJbzgUq1IUmZZzM89mfMmmCIknrltknBwKva2gP/nHtgzsjR+PVYy94TSV2AizCew83zdxiG1vVGuNq8DXUxriLJQD2YZu/zXXDGNPaNrsI6rf8PnMmCxVzydPHjTxde8q67Ue8+plLUUky/MdkTdKkjUtewCnnnsu0EGlbcgRIXlexnzh/3A7qWBUrn0Gxl3DXpYvtA89iSiYEYNxgmoI3U3X2ZLPcCRR2+aqqeNp12fCYMjQhXMquic3vDFVe4R/Y5EZiaO5NCPYmi5jyRptdfSbIG2vWqru2HRidcohQUq2jJksgWww9dsPfsafNmPchVjappRhE9DsjeavnadE6PZZnZ8YtrOg+93b8De83mzssKFVWP6AyeXCR18WICSUxf+/WTY7eomJqiaHZvVeIOgXB5lbDUjmsclVh21GGY/xAB9gKARXRnckA8pe/EBSjO99v3PNNw6XqjjD/NPHLLi2MSNKN6ybNIqssXOZffRal2HFUKRM+DcleNXfCMIfJyZDAn+Z+CFdG0XXbcd7J7HGnW14KFQAx5IgnT8A0UL6zn5NB+WPD3+NnffzTPSFMDAsPc+D0iNqpvoVsYivqUPWZtOZN4kjYfbj+3qAR4BRpCanCm9yxNmXD6A6Bf+8QVvdhGFOQ6cCjOITPobyV43R6/zI6NAcYY/hI17v0zYN6Qf/ub5Xjk0txuDiCu7YbO9JATp0GrRWemRvsBRL1kEvPqhy+oEeKm0SUthr4JID73tgAFNHsWQbMhUKVH2FPiMSwK9RJpQLqDvk2Ie/ANYSOnGNpXZTLu1Gex4vdgLf+YiEjF0sYZTsvfjHKzSakr0vwNEsRgoujWjKD86AtXmishvtT8lYZ2QUfH4vhdxo2Lu/CAYizut9v6mmNMbfFBr3EkMOYE6b9u0LEdGM1EJu+nONCa8ED4g87PKLucayTkcErhYYVdZxNfDO4HiKyQKStJLOPKR4xAbVlZmiI01hM7N+R0c3fTc7+RkvkkI9W9nipIEiu/1w44nslA9/BAW1h1PiM9EagyY0dfTH1QBqlNWqMj0OIjy5CXMN3FvnUMR6wQlnh3QOMjEYrk5fxGRpJbSAZ9ZhTg6y7Pz9Jhc2U3/nbnTXSj2OZbkbCA1iDxF4eK3tiNdkB0TvxPINC9Is9MERS/U8UlfURTHQBSCyxFN6rs+gSTIxSTYYXivv8mSgMuerdWYZRDydGbTa304PTKwzdTvXJcHxiQKex/LqCsXPMKc8KXv2hIwlUFrGmK31xj3d2o4Vkn7b72CCGL+hH82jmuN8vuzldj6kkKsUitwoVU52fW5Jy5emAGF1rsLPpqJJrh/Y7yVSB3WT8J4Nf2ZmMSzPHah6PwU9HoW8KgzHnxmjv8gY5bIUtqwvwYNu10wxeu9HkAPYb6olwOlr5LlZgF5L4hhFxb9JMcx9MWDo3rOlc/EWmnE66y/aLxgx85gJr0uOByB/vcvmwfiBfk5hKM9dlcemwXiHZy7nFPl0eFDBiQAdqw1ZQyIE6jec8Cef6mL3k5MkRhItIvzKqvNRfHYRDAAR5l0Au00iQRELAhb4HOD3/4oxbKVRhkj8TCAx4oq9TUdTkIe6ZU1ssIceWrwh1oYQmq7hTzSRlBtX2G7xSiv/lMtlf9ILmWgd7V7kTy+Z5n+QOXKxw9IxV75YUVVF6al50Nrna7fnpL9r5bS/UZscT5e7frj2edNC28Ga9klr6Zv+akhders1e1Z/Q2CGP5AHSV/0k+Qv+5BJNXLF9wXZ+vl343hLeZFBshn0biH8noHHdVhdfjYbAam0gk6jELs/XI3S/h1+76pO+UWUtJC/n47+XEHXQVRfOq3YZyi/oOk52bP+YCUIbjtaxEr+C1nQVTiHSU3Lkquchlm4Ktrst7n9R4+xTCkg0g/d7W9iYYgbfwraFAkyCV1G7WIorxt9+ymdonO5/GCLNCkbQMeFthWtsp59XpTZVB+27HNpewdkcEsngilVIr11BZw7SIeq7+O63R/szT/McvULkXaSfciwiQEes5EYkxKPzWr2y7YCd+HprzNVkuzUlRwmO3bKEUp8pOwrVS2vSFhmV83PvMnFoCQN1Kgi6cvO/jTColGqXuJfOEHXk8BET1OuVDI9C/lRowsNhKrkEQxjXtjnu4jbesxlKAczayFiqQ5/zhFVU+q49TZXAmJT27OYpnInGrYyPpDdJ9BNj6rr8k40mXyGhTlafRZV8a5+wnS25Y5d7IgPmvXGfv8K/5IrnqjwHFmOkXb/7qGCMyb53N1oSDWajHWncNR1SbtzPixef0PJl50crt+Vv8ypQkO/XIJuCsozjL/Jc/xdB1hNNJMq/+umLwdNiGICAqZAmI4ToL7kf+Ep0i28fWQXgAOdq3UFWXDSQEaViRZJM77KT+FV3+0cngtS3+gYFz80PuPqLAWKgWBUX7qCGt07E5OkiWqX5bIX4hUb4ExISk2fAFLQl4fkhC7/yqzch7xcKWwh/uTgU5XMN39QB6O10DGvWvOeCg8Fcmk8ATUZcpHjm56IiSDTzn5wj6Dtc6m8X1yBaPPVWsWrbYqaxj5s2UIoUH+Z8RVgKgPhVtv5NOMJCv+F7cfpkU0gPDZeCTKgTZnfZuM/cuMoSSnJ89TGeBufg/lz9M5OSxqTgzhxgOTwfYz0i1dt+QKiLKcKV7yK/ZK3v7rjc0PWtlq5ncYrC/uMuroHUAO857KsquF+3gO3Uia3FpfDyIlZoiiHnh/oAuBCKThnrRRy79HrmbtQN43HhWal+Kfk7SfwcPw+CKy7OZe169LnERx7ZofJEbVAa4/mACGXqwiuO2PR5zam1UI40O0oW0LIqxo+wszlpZTN5Htub6M1aLYiqtzkKzE6ixuHSmnrMi8g8dvTSNWtdHchDr72lHLpuMmfLPkeaC0/+bCEP64T7Zyb+ZrCwfc+qEQeS3Qj6Od+4N2S0HWYeG+fOm87ZNfs1yqgNsIfSehlWN8RXDBuou0ptOqjhzjEHJyB1xP0szVFjOM143sBUNJtISvtReQ4flST680X3GAuuhFoj5NSgW1E5z70qHgfwClCIFBaFbnexO2UDRmVtHzRBdrLMYilv6q3f6dhv2FaVH7RwbBnRUbk0u1/XYM+kx+53Rh8r4g4bHK6VjMy5hTR5lNSPhv9O0eNPEAGIWsZMo46yqvJoxRgcTBdpc/rQutzcBx3Etktke0+Ra7/JH8r0fNX2Ub/upYQ+Q5qJulJwYBim52ra6ClIQlcIUGmcJ5MM77lQvULm/mgKwdTGlh3cKlSMnlz3G3pCL53BBf6OaXamYHqV8kYMRYFoWazaAtZBPzXPj2CfVErK0NuVwkfR1/bp/GuMSXsLC5zd9CfPBt5EcffFnkDzNnHBi3+SQrcQbV0yz+ktXkgQdlvdknDjM2JD0QQdmjtL6ZSxuDh1m814lVIMzCxDxumvBH0+PR1RPgfdpsHby4TBtflevh2IHjEEzNp321NfjIIaD/CPjFqJLG/6aAPHK6pJI/TurLXwdmoxj56LReWJNftIjLzTqfZdRpYbpNEg5GS6H3SqyE5J+fjBM2TEk7d39LagOK7qOWOyBvCzBQBnz5Oqi8o5fZXyKHbX74ckTNWi4LX8+4yY0YGN1IEXv9pIyJAPyV+LnBuDePn6mH86dHgPpgr/3U5m5KNumqFyE4QnKuKxF/JSvDf7+xOwyc7GMvOpb9hWJHvEWKNKQJKz6TJ/SzNpW+ivdtUvxG+CzRwbAA5cIz4gneCvS45f27ocHQZ/nKgwaWsHlwbD0MK78XfyLoxl15cvtkYIhi/cuLsWOBBFSunki9tnoZAvdYYlCP0JflnWEKOAMUdtdh2PhnW3wV3Q8VDA+I6GzkowtSPprtHE/z1QxqIFxEIFd/WbdrUfx0T5IIm9vI59fT9j9ejAkRz32SOk+AIcyNm5swjaudLUuXYxqUe2yzZ8mXasEnDWf9K6HHikNJFlC9+NmjOZ1S3uSBLPgmwpsNRLi/Fvb/Jld2Gd/dMYRtk7roxJ0aDeAIiMeSk6uzdrte9CjJDsDPAASF5DAKPrzHC/jrlowLyGZqohu9Scw09NIioyOvI4LIyuGgiGPjPdpl0XVdqK8mDRu7PfL0wG/4aarWAJLl3p7kU9s2wGFP/18NfBRph5pe1gmD335hVZp2HJhmj0laNIbYW39HPL2qmFqsmJzI6Hu490VUgWqOq1fOrE/L4pfWRlHhIgiUGkiBmNnPXcRJ1n3NUf967u3E/3rb80qxaB90/n8+W/ynwruez7ErSBxdCDbvizTNF6PNLPeTAhumbJzjeYaQ7kHs7DxEbxtlQ3/lZNLmFP7UVDvxdtYM+kACyRx/V1VD1cQ3uvgu63ef6r0zO4cEdIwZYgKTZa86TzaxAM5uCesxnobIjvZROehJc3u3ZaHLUduBh1EU3jjUVbhMYgv8GbhZZkTRzLDc/6kpVb+GXpFGjQ3MW+/ybfWJmbj4pZdR6MdxBmz+3wYfzhlEeVnI092S6YQqjPd+cLKDYlE3gJh7XjdrUCdNz4nxzfyarD87i0dISTUC5WF6eWpyL5S+P7eyv8L/DP3YyoCcjkQ+JNi1YRpE+NxyVvkgoxBUXoa8Me6YOzaMWp4GPiH7d6Ohe61+VOgnyz4rOqPzCnmlk3I2KQDnOVKnYjH9Kb1bfutkhWhwjCx8HJx2REkwFgWf9q0pgALAVUx2A3II8OD0Nm2iV8b/SFH2iBabYPubms/rDuxMpNUCI2FJ7IXneYa6/FDJzt9VfFnfnvEuLJ5pRhYk9fTZpBzCho+QxxSio86/UcNcd5W76tc0mCp8bf9eW+U6sAFlQh7aRCtmz4KDWoSMpjKrqvm+S1Umb7/43bZMQ8Xh53+81aMp7XAoaoEOoU8LpGJDj92EIjI8jmlWR82/GGLMLK415W8q9z9v3qdFXId4o9ikm2hycDeZ/n2ohPfHMRV0ePPSO2s9ln05tumiHbMA3FZQEFwEh+vO2On+wHpwOEPTXj1lnrQOeYYNKQy+Z98PYMWYUN+PEzOgW3fDSwjJHfc5T4GK4jYrj4YQxcNtS8NMNEDGSel3T9+eFw1Z26OU5V/4pesdcRL8id5KoM02a2VsaTgopS35tX/AHG5vmH2JIY9ea4D+ljmw7+LIorNsqhuY/WrauW/F+A/YRT88/x0I7gpoDF697x+ASrsIeowbdbmDneUB/bJjL5SE1GzGjnr9J8jP1WpYsyQCWG/Dei8kckMbWeLkDeClWh5uWfSo0QX9zZtuEs9viB6Xg3V+ndJVsaKn6edG7gogrqLiIWKpioHkww9lqRogn3IVYuXR5UdJ5HkqvtCJpw9+EBky0svNv+oDyBDKtX5u2lKVgosT6HRn5Y7Prun9wusq9Oe6PyNmNgWZnQUYzTAXP+wPHXGwVrFk/SvdIpDkRdJmJCg4b/m7fKY5ojEJnNMZqaE58Ts98mZw1w8NNwzfN6dyNE4tInsPhaLPI33Q2NH+TQzlaP6zldzI/sJfjS7orhgTEAiLdZ0REH1PNelCJqo+AsNHIgBaNy+kfni2+v8AZtzZyFTTlug2wROTXMlVP535AIiaryOlA8WN53+UgswsZSa3QDfMiX4rHOh52XLuRjLeBfR5n5n8Gmtd/jRfHgPEris5thv+OQIlj44tHF0HWee7d8Ye23oCWFNgLhl3YaTkIHI+Z/uQ55kiKahTzc8blXzGqDtJ2BPSxzGn9YqfFgaZQePzwmcIPYpsIj0w7QsyBlb1IIQtprtg9OmqGAXyxK4Hc027BBp11YlYafaN9fPwvxYKpt7m8mF9h0iYIz97H8JXx/vpc5dJTS+6xIL+/nryuwjzW/Jc6DNiGTfNGs8Wktt+3Q9u0HO6HBDIxgsVyKv51GfKFgjThWCYcFI/SASHAQQ5Fk6Cwu8SLDniU/Buq6eYggrZKOGNisYcr5VgKLLw2AIaDCF1pVzTEpg+tW4q2AXtVd8FnPlBGcvpqTyUr7IUih3nraQcHd/X6NyqRU2zR1a+c5FK9f3mKlT1xTq+KXcb0zb/nMiNbz3o34HiNgLPWFwkWlQhamWP1h5aauCCY6zVtNuyiXTdvSrunHQcLRI+X7LS0XHl9yL4PaX67YM7zBHo5NP8YegtdQo9zjXlEeGaPm9B4/Z+LYg9RYd29TjxUuhkG7Jt/ez6/NKnH/gqFMeyFY6DJU4EXVyMSpq7AVpHEi3LlpvGYWbvw7LxQEKflXTIv2wbtKdYemnSsYj/8eXjGYJ3kq9j3yJr5I5Xkz1Tgrlx6+jxotbUkezR/Q6VEq4bjtD18rU8Vc8lna0CTG1d+/Uj9LoHvH5K5VwCz0PspHXa4YRlAOKZ5f9i6Gr0o/C+qzta1Ht5eChw2Fw/GPFJyZsRaTdxLIMp45vt8s6BYbWm+jPqRNeaHiNpZBRaiumMWME9Y5BvVjd9Tb4KKzOPb73NPnkQpfUzhstbgmynCS/nBDkrre07VqSNfjpmBU9rSYZVMiY2QAXvlCOsxcmcvo1DFjhwFyt1jOZzBiivfG/7io5GOm9iA6RCiec1AhmelCnIjDUP7ZRz/zzZomYpgu2DpgtpsLXy2G/I0zv6ldFhsAFv0L0fNAh/2FqXU2pTZx4AuP4UPrb+IVici5WYn7O9ouIcvkYu/a98UnntPqBIqBk5t30zDOTz70awvCfUJuBVxEuK1rrbnq38j9cwXILUOn9QX/WV7EuRXDoAN8DWA0l/1N0u3x7XlT6BxDDUm9qTHS6MgII1hMXnz1gIjT+A2SP6EAqW0AzXXEWoi1ichEAJNprnQ7vUFKitM4QgtnyMq92JXWHjfXv32fT4p5HG0YBIbDO0/fGtkLAoRyvwQVedW9Em3+e6UIudZ2PTDR911DWEvx69i5Dm64WkRTvJf9SOV7edypOYwPl8lo+m+fLFNuPYhNncP8Nlc1PpMfzjRp+G8ngYXO5ZRRYXurzG1WB5YB4XIYrat80Z0ftirKQkm3LXEr7+OTUPPhMDcH81bV0bDnoU9mvx8RFLZe2a6OpxujLRTSyTXjd8pH4PW6kn+7chnO81CKEL3BYr6pg5PtvFhrTA/z0wG7CcfkuGKR8lXKs/RjEPrHgAWqrjDK62q919BnXf5htrYs6mnBvTFVw/C24OnghFpsN9/9/376GUDChty0XrKlVvTLyd48vVif18hsCtosDulw+Nf/PfuRVbhcFPVdC4Jwd93+1F/7hy7EjpDxoCLAtB2GTts/x5HSBW1SMXpdYkY/SIOEr7XOGiQ8iWgqxtZf9HNPKGFTKz3valfiCwVU7nciOYVFu3voAy45zddnu+HARo7OaHizk9PuAGm3Bqv7cmwlHYZMl5nGTmKYtj5KyNVSh/4H4Ftqm6NCU+jtemnco/KbJX9ki0KagpwyWen8YwksODbhaVIEzWVliJeSW7PxIpqhh3JXy1rwk9UZFM+RQ8dkG04dPAPZH2K3wL9DICGcKYlRCDGRHn+8H2UvKQa8NTJEHz1l8qv29iz/v4L51Dj/CUSiWtBW6m8i5GbxsLno4jPLG89oo6mjz/e9peUC1oSw7cWmgZCeWOM+bPQzFMFUFD8TJcKtvAO967L25jocJjj0s0o/kaKX+BQGk4jALStKwRT1FqWu6aNvBHd4jDX73ZNnSUzPUCLUCF25ZeYHe/P6Rr3Yh9NIfjGHLWe03+2tWwk+40Iy6pIVVapZsPkHyp6mynrVGCEJp4svMD+PPZ74D8soU4Yz/c8om5dwcrk+3uAlOP+CvRQ9EDW3b+hRNhnefJ+0CAXVbryRx3UiPDQZlLS8cXQKbI26fgMwuTV5a/d78hQ/1rGvxz4I2rxLw29EcHgahRl7MVEADJL6FUX982ASut9UT4oHWzHhLGEvrH8Gy/b4AvSkpwTfQ/j/l7yV8QOW/825he6ObEq2Yg7oQMgk3YoIrQg6cxgYU1+4U6r/5i4FdsdaCanqBUIN4M95np8ergzZ/VjNTiA+xJgW/gW7KMQ+w6tKjL+QzVEmOECrPvg3ogKGc+e9ylU6uVCRU/zoXjHsTIIHbblC4lJ+JfT0dFI0FYDe0K3yI6imkdLIg1XnbwBO/YRaj7fQ6A5NMg0HoWRbNfiPd+U/DW7ygSQEkl3iU07OzIPtEdMI32IvzZd2Qu1U8TnSIs5+WJ0/y4u+jRBS6p84BJZIcwUjvWzTOQz9I+5MELt9Ch3NKRznSFOsez3jshpJ7ybnVE8pSXbULxhPqIIHAnmDCK4kX8CptsKIqYZcrHcJP4VeRHAN1zfIvGfypMD/9/NK9+HNBf15t8KcrCM0dJmWI4/rvnGAKCB5reMP4095nPlGIWp+V2eflTQh9D8VTN6jIOoD/Xz4JUIdS639MT5oNmHDqlo6TMN0dJGpsaOzcLUYcPz+locKZ4awP5z8zNRA+NMBnhFy7Z6tHpAyoC7ZQEBmfMmQxbwUttysipq1oMDp0Y7l5XYHWX9ZO5OoRrIFXItuDI16YMcrSBjBRHE3gUb1bnlGo7NXKMU8Fpxyq/pyXEukM6nQcDdy1HBDG713V3EEOVJdit3V0VjjAspeUmB1z3a/PW2ixMVUHm3nKMbiMHsm+nSn44IjhPUDIjjXMAGEXZeUqsU09UhEZDyC7M+pMac2uCblQL2VUF/TFlCdUfUTQ+FjPscrMFjvNJQMCKPsuB4MhKjuZWPcHPcvzPNyCrQlIZDiB8E0udXuH0mXf3e3xIjJ1OnvzsSVwAHWVcEt7diEWi4oteNSOdl/q4CeLPZOJR0Ig2hX9PhGeIqs/l3cwrLSDqYO+QIm3sWol9zrwbiI6X5TgMFvIWfe6GWmHi8TaT9EBz0Av8fc+/VJSkTZAn+mn7sPmjxiIZAB5o3INAy0PDrB4+qr8XuTJ/dedjZOpmVGSTCcTc3u9fM3A0L7OyRYfzBMasjHjwZKKe4py9JyReJB26zFVhr775WKZzs6WH9W8AqznohC3sVQTdqA5Wcn9sxNZxSMLyLxr52l1NRCS3jX6W50UN3JH47wVNBeOYHiSJpM1FrwI4HVDIbO/S2KifGubPM5xxpCcOq19NN5sBCL2CGgBbqMqVfjx2d/Vqd3hiqLK0ngxxaCFPBBB39wUgr2ua2RxwMoJyiugtyHh0bPTxnnxGa45a5+qtZvQucl7BWVbPi/MqEXbK0aoWAcaOFBfpvrY2SUPp7l/Ljkr2si73dCVOSW6bIBXk4KVrYxbXBJWPHDKPAIKkl/ZzOfeYe2lSzpmONwhJRmm7B4nu6EBeyXkIfwTRkqdMlEIuocW0yCipGx/qkg65LuXt0b2888+sPpsBihJvOG5ZidsPeDgGctZM0hecS6utbh+VOV4/Co6Yu7FWXkWAJzIkcA7MFSFAHb46wQedpTs0BXEhgVkJmsnsKiED3hM4+tsKif/XTU3zWTHWJ6iOf8pBvvvVBz9rgzGu9tfHI845980zZabaPpIQrFseDusHbc0oD+yuKZr+ChG/CyDBancBmzXWj+tPVASMKYsTiQFGJj5EfMb3ttyoqzMNslD6Zc+Dxh0k0p4lKYCYOsu2JFQWIJ4cXz/W74G3fYFhTpiRsxbYTpucTLVxXfEpwQAMJFUct79snDM3add0aAjQ9VKWu85wHi0RrkeUebH7qp8yQKeC99SinW5Il1Ag5b45T1TLo741Fb9XNidCx6NtFZu7z5mSFYbn7vQ/7pz4WNQTsu/5tDLul6mKmAS4Ighc/lu6cNEqTzSihAaPlm5DcIbUWpIdrqdeZiLss2ukz79lPSIS6aHqyIAAmKvUDX0qowH4etn24NIwLNQuoZ194qQRynrpILsaqI6rUSEpL4TlOI5PX5EJ65lBfvz3mc6JeJzzj+vi7sntmkEUfJ2AQZf9N3aWVuItVM5MeaGUKMw34BViYRhP6ofBcK0iP9SR/O/xCMB7FyKgnwL+8RJLyXfunPaKEz5YWW/vtrLX0wAxa+2Gj5GVPDKswXxP6IuRj03EaMMjkt3zfjOm2fNmjc+oHV1Bj0nl3Lh6ttLfcn55R1IFYg48Ew4CFunuWAT3li07U2CwL1pKKxyYS9kOXh8NWXYAn4NapQF4N6xRC0fBNu34eLa+FfIYcmprNUsA9AAUsF2O0a8zJ4FABuVQeaihDmjE/cOdXOofNoNBd0qf9b0WpTpOlU6j/JHomHwHngtx7LfvTN6+KVA5tHeJNj63XNvf7YLo7j7QJ05WUQ93psV2sg395B5HMt7jTg8M6HM8w5287ORHYxvTuHuPNQvcdfo6jF428Vw5G9yluf/hulIRcXS3W+5P8AJ76p1/N6DMBb8iS1McKqlWzfCd+5m0TkYZ5+ql2e2UezanvbW+IVTWl3pz7w6lYxwE+XluOZJ+z/MKbLdhsT1RVn4Hc90rYusvnmdj5zF6fOjAFRL4OodXyzDM0HNcrHKGl6+6zwN09hz1N4Kxil4MqXJzaWPaxny7XTjF2jQ61kfSKBphT1WBkWHeUzzSnee3RNzopfDcCv3oWqzmA3OFZ2ljEpuL6jbncwyHSfsPayC6Bs+RkkClHUR3UzmW/Td9LS8U6UD4y42wAPR95bNL1rLP3THx2OQHfGfWmlJ8ajqpkArujsRSsPeLx20juuv0PfVeSLYzX8+YbELSN4PdtayEl9+YZKamGqJpVkYCMCHJbGtOa35g8N6Q0BVO/NRLCvEVDePMxb30TvL9lFbP9KgBBLufVTyApbreSgcjPn8ZgBfnbRMtsxVPeognz2/Uv9Fc2izT+JcSKFFUEvwLx1kY2idtpSmUDqoDdIXp8zsdGULKSeTQEuwRNSo4L/H0wz0kuEOp7spJ9AiFmYkWNUpqBeKhxOm8t617+EhMRnozLBC9MeLQ8I3g565H0SZup1Z7n8GBQ9/PVDINRIF149fxexj0hp7YPzbWRJGdMrr/Nn9cxfUNt+RuV93sS3Y2Qa1miZ3hRQVwIO8mdap52dAyHFbdIbv7Nz1XW6v2DjMy0ijXrU2bpDsm/uc5JykZNvxKcyWxtZoU0qaX1XGhGPYF6qG9zntQwKJiaHVjfVTjm243nV2iH+9s31Qnt+fJ92fYz1GBmxDWZQyFBA0VVkVLGkuHK+Q/RYowygKSLZbyC4od+N9xgUYT23loPOP77yQrHQ0/7wbGd372YWr66HDG2R/KODKMEzJQCJwnbQBqNkrOGDxA8Q700LuzsLIxVaFZDNECNHUprz/4je+3sketEdpO27N6lcgcTY8rs61OWCmzitQpXy4pKscUOATPdE9RvQ5PY8aIIy73W2LqxQGqEAkqu4f25re3g1z6ljEGEkrPAKlXRdaBxx6X4K+Dmt3fN7HDdD0elsquLmFdLcdl1W+s2DIT5mKl81NdXZnLcARDFnhOwN+szWBYPckZki6YihRSUsjwYO3CK3z6v9lo0uoaQjmHBCyGOc/ttS9McscdimJ5meyk+cn1DRNkCYpYF2xTp+PX6SBVkic8+PnuusQfc8/7IJbvB6AZrlpXw0yvqA8GeZ8zNLbRoaDSaBN+WCd+cGXu38srwY820CFYkzzMtKdIIQzJKtmhue0J3dC1jrmbeA1LJL4QRutuAMzANWUvdY/VjmoQ7f2fuN5MFT/M2ZP3F/D0dcZbU1w04PtU4mqbVPxp0qaSLrgY+kj8t62G0/sKOpIIUhEMjE+0PkI8Ij87xUVmU2XHskyJvwWskt+QXSX3DrqJ8Ku2N8N9QwDcHn6Ni0uo/805PfnHC51URxAiCpPhKHeXqOEZ6WfId6ll8vWCeM7pb1tT+k6P49vrVFaJkqTiq9kELt/erhjrv1jHkr9tjSfPN/9IrCx0nhOouBhxBXaU+5iNuE/+t0PXXKAK7ZJfuw5KpCWC/CK8EinM+Pv4ULCsLiS+hnwUXp6UFm6M/3Eh5fbeoFZbO05UytE+qDNRLVibyiF0Xx2N1s4Wo5G2s5zAfwi/mXoTTqJZqUD656i1T4gqHh0pBJSq9qeVY2yqj2HuZ55QIqbyJqqzQLcLm5WXH0gG/KBXiJXwzodWrntEqwXfNqUJ72IL9ZgSA8p55ySnMT/+wwvMN7Nnfc/Wfany+wcwr69/3Y3eY37VAyn7XgmH4ey07Mn+ufXAg+7v2UUnO32uB7L9/194YSXEMw//LP3t00X1HeJ3BfBUG/BM60W2dze457l/QZ2yf8YKmB8AO67+AVAqwEhwc2vN5zc//dAgV/gXl+lPKxz5fQUFC6O9f/xUFwRZwzfX3AA5Dfw4c9Wet/t4Dx/4N/3telddl9c/jMPLPwWT5c6D89wf8tkr5PRY4J08u77p/WvH7HYHqz59rNKXVXNvQyNTQsfuOPObT/+s/jdiT7rGYv/P+HFjWR1X+OZB/ytz5+3EYh+cHO4/b8MnBjaHn0ziv1ViOQ9Jp4zg9B+HnYJOv6+XUN7gq2dbxOVStfff3r8s6j20e/H31p+PYYhzWv6fD2PP56ev5CsET/g1CyX8ORM+Bf32OEP9+hD//tuLPp+s/f7LyuX76KZ//OXjW659bkvjfj9HfBoHf/+NW4MM/d/pfjv4ybnP2t49CDuOQwZ7KoTw/dV2nyDr86z8nrslc5ut/MwrEn/NAR/+3wjTnXbLWe/5fmvE/k4m/l1pj/bT534UQJvF/Q2D63/9R/1UiCQT9r3f80+6/N/m/CNq/t+p/X/aQ/5ey9x9iJvzH0f8vRDHb5v33BPi/yBDyf0qG/ucnQv+/ECIKh/4fCREzz6Bw5r+fNoETlv/1Y5/n/tcHYX8V5X8I5Z9b/u+K6H/b+/+diC4PGQG/1n0Cep79/WSWKc/Wv6Of/POhqE8gRyywG3X2yGiS5p01LvVaj8Pz93Rc17H/TycwXV2CP6xAlv9KKjd24/x7NIqiNF0U/zcZJoAM1133z5l/lfZ/Fmvyn4Y+Hz7JmvwLyvz5iIjTUP4LwtU+a74PSJXKERhEw/EqwSuf38A3Iw8cEz0/eRPK/c9jLxO17QTbf2PIdtMsfnYB7A0N4MSotUn9NffzRqz3vUPcg8Xt9xEIkivrBFvan6KeMbMee8aU+kLYq1V4fxx7HBndEeWSeuuKK+/sb+M5cpBFEiXgEocxaCANephRy5z9DSW/yDZ/8YygtiE3fJhs4GFAw0eDbCTc4Wy5iKXOlvo5jbxwMFJJPZ9fJQOOsaPFRozGRDz4SnjO5vgjYA6JOXrm+fF/9FzlnnSxFXg9VY9vsaPeiiC8l2KJIUpDYlAN7KqVjI7+C8Djsq0atqQ0RIigjrOP+WUAsgkiDFl3h6AakIiGLw9l5RdXs4feRcnzH/fFuCEUUyyidhXfp8+SaCTf9tHhN5lQYzwbiCOF3ut58LzBl9Wi90ZQ45b2yfxZKJG8JGJOm+kXFz88aW2dpYtekb11vBp40bvsJ04eKOk7XwPtcxM5HEVGsZ5JCDv3YGDg+B3gmMuVuKK76aRwIvxcMfXKcSUpbD+benLqh1wlm0a77xxeU97F/+kxlhte40haxueDFpkr3S8R1UglguFSo9i9f6i/P2PFqkKaUE5K5DBgSeQOMYSvqM7iqxLSwaLtz6vLaczEdRbWBJwLFRAIK5bzL5Si4XKEevR9C+UyMVH74nndtA3Ao8LtnNRfcmm3kN2cUzVBr2rSd0jGl5QglLoYfVyQMDjISKdmKL9vVE5PYaI4csBdflKoJXGRsRKF319hcH+Oc+Ihq9D65V/FqgSCo5hM+xoPUN6E9aGRrRyt3pzjzXK8UIvpROd8LkQ+wlawCXP5PpF2d/ZIZv3TAnpoEGL1iJEkbwv+Bpn3ZuWnb9eyL4+bN3ot0a1f9udjqE5/A1w4JN3f9jhvDoQGMLplklaUmexNm6ICYuYTAml4d/GiBCqE3MPQQ1MeTUPf6mylM2xvvKjw/pVlKdTe8g2oMuROtzfVc9K1J2Axp0lrHlgF2XI9+oiOMCmufeglVbPVtuKsx9Ff1jUP0bJMlNxCmHTgoU6mzC91BszpSHBpKkuSawar9C+73E+U2/C3GkFr1Q/ch0Troike5aMHaG+WdSR5OnPxa0upbi00FV8OSozfg3G5DZkGVtY7UcNFzDMTM0p9YfnRq2CTH1GcuJ0eUFR3lyChE2k3F+Kdb/kSnkrp2rpkqEwS70qMmoyw5W0c4vzPkzXwdwacLVjfnc/kZ555rW5qHCPY0A5rWlAuTIljRSnVJJkbPOUtnq4yFxNHaddMN0kyZzCQx40BR62e+Co4SMlATrIcEEp2HuxvjDUpyMKjpSn42N3PC0JrmNLkdJWgZrHYGAO76bB2zzO6gBJoaH0UJxTAsPHZTU9GDaIGi4Bps/i8ucsw0RcmJ55Ii1RoGhNFSz+Jq1VhvpjWtxTfRSkGLWkvU6UkRZRxcdPFW78EQbIctHPxiqSl6ZJzpV9ffg9nqFBjp4kfU4cEKp263ql30jHbG+HX54lB/069qGJVtHpY9Dz9anItRjOnID51Vq/6N+ZiFN3jQaSo6f+hb6hKnQRMxvUqupiPIMI83AJvsO2hRXtf5wuOPm0vSgIYOBrnwa74y2D/3rKRArWlTNE4rJy/NtGnayR1V3MBQX7Lmagzvl9lMDJeZlm4SRD2gPTQb3eZjxcUp1FKP229fGwMbX7tYZ2pCe7j2+21HtRX/rEnLIFJ3ompI/TZEl00ryqzpJRXASs5Ckx51qtOLggERX8xink6cvMtRNoIw/37gX7VrDY8uo1vdJ3+6P5KQlk9cKJhzlLGZbOkbcN0VRCeOUhfFiuZGY6x2tT2XR4tU97y7q8Nlitr0aJ32r/lHfbU0T97iUmepgsgf+8cFeC0we1uSbJK/ywiVegMWucgjsCzWvNHl2CBHuywnHE5vzHp4ZUhdWKS3rS5olzikOZMVKbCviQCasYJCv3akzIm5tlSHPW/9oD5VNbACf5hLEwrHx1coB+qik1eepeUfbZaoxiG4zfX6MSJ8pii/JXZWLss3m/gkmZbXztT8DKjYZotIPANQ8WF6CMJaYcmeQsjXt2i6lvzTSITrGIZdlsri6Ot0IVy4vpEHTZio1MAUkRm8+RiGthUT2R4Cvk5R8NgQk4MU8ALnKWmv10Ud381zDoQkwcOot2DZlhNgDJsrtAAFikwN+AxJn31+9bBpWXIfKgbyW+RVyQvcRkhog7cUkPXR5gVr/2I2YnEU+u6QxfIiW+GjdQDPoHedJ3vkDIvTMgkewPDXimBxeBkRJ03DZaXoy9uXKNb3F6HaGuULE205/Ro+6LYklnY6AaZR+LsfSfz4CA8taooI32R10tTdEQdhB1uAanj4lcuPrXAPgS5502089yleT0YBdzF/d2FfO4SHECpcQtP5HFf2l9eaufDOF6VzGMsVyLQo3v5nDG5xNvG1/R5B0X9CHXJ6V1yYwUUPYesRzal6HN+hhJBmM8hRPDbJzz9lj0M5EkRl0WpRZcShf0WhDq6EG3Kb4QRIaVefjtLviTzPFLcP1sDdAIDEksyR/KKenBmEC6+j7ONht9mlr2qQWKGvKRlZQYneD+69Itda7doHG51/CF+1A249iWuOoYQZFPQkbe9lAxVcHRNBqa4HMFUagsRJOHgpawLgeP6lB/ZZ7CtakII25HLE7ZVTxS8fDTCoc1ClSQg1TtZ6kRugEC4Q6jcbZifD864Q+T6LTPglwok+ovdi4L/9PQInIEu3BC0JYv7nsajfV/7prw2KlmLEYWd9z63a8wPQSbenB4eFmXb+xFYaPtrnwem1aMIw0fLbpxK6HjDt7lLl3BqhTKviB86lIWG9/ytsSXsgQs16KdnzvC6cZdKKwPf40/6ZHWr/6g0LOlk4aNzJSaEuqVaiP/6riDyeE7A7r4rhNlhQx18Q90fAMBgjx6T9FO0aEtVQ4Kxbi6r6sc+DEiI7/czqokwLKnQrx/R+mkscY1ctXbjA6BftlHeoDKUAn2n+OAplOQyNX6w8iteyrfGkGP08DXFSY/3YV8DzF6HChDHORxAg5EUR/EX010EutjtW5TZamYRzOlFMoxv1yIKnt0Zc/o4WuRF5s9+8uJwFjTvfLuNkejP3Ju64w0wQmRewYA0JVHnvUgLDccZmTjRTKMrIzgVXk/zpsklqk74XiZbWqlYn5yRQNiX4ikeKFYRJbqGByvP2Ng2MLUrC7qBxQIAywa7598W6RNVvZTjA/F+M0KXu0cd5zonXvpnKVHe0Zzvf95xR4DxRymZ4mkeZHiB7UgjfjxA3XrR7WVHUQVG3iyfSOHhVjp64EsUvg9Z4UPOEaSAtX8zMlvA+jMm4+WOUKMbUWuM27N+qRqw4F98jNDLol8eHz6KdB6RxgiISzcjiaeYgquBWFRe+EPh9QuMlnnIFN59CZx0Z4yaxm6buezSWKcWHsgj9y8jhOP78NKSxaY3X2vjxnl2Tx1p9NhaCpk+Lr4mKSWkX6e5nUF9NToOTdz1WyfJzFwbmeDVrQdWYAruECMdi4BNSb8tQwuqLpWFob94Z+bXtyH0KZ5e8OWD9SQOsM0quTKZFhAHNf2KqL5Oh+hqwC2zTynq4o5sv51QFmo5cRA5mTKng5xb4m58f2SMf0v2EnsthJTpV2P000M1/VzUijF5tHoDL8NBvAAyjfafxrxtoC1FgaEdRP9aTJBJ2ddesz+5Ke1SfD2v3/TAnPQAzPtxtWasrawyEEuThyneB0fvP6OELpfzjQi39MIPSzf2ztqi67xaaV1DMbCEXbOj37qO0sYmfeHqiGleAZA6OQO6RsyYHm4TmirE1jk7zmK0RWvU/TKqkdiluBcWoIquTjDVIWDOWYBu1seKq7g3xiFu6ldPHnbi7WaqGFjUsPBPscE10EBqErUvq2wsmL2xA97ePWRMR4mAxyICwdjVMbFqERq1gCzn3RZc9DzRPc+30wxZoJSPxrA61cIN59umjEEy0FSmfX+4JPOOAqtUE+1tfNQ0osB07HXdkmwwitLYM9whNvkFFJ/BiHjb/QGG/PjBkO1RkXZNHcxaBedv4vgnn8vfyNJxoFGSihqovbTy+8ViueqbF4swLFAWjdOnyUUslnZ66ifYO3mEa0wB7cRBLqk4MOklvlygL3U1KOFF0qtIqFhjwtBfas742H7v0WEfkhkB6e+69EzpRJd1ptEUGC/KX234i1IKJoAAh3wVfFHyj2aSnxF4KFSlAXP2qwRCMZriGIMuptKu+JxOFnfooJh2HuHBOMxS/MwMmj3m0Skg6JV8q0NYWOaoPAfUsRSR9TU/b+IGZad8T/Xne4jBMqyQMg+gjJ4eiMwZaNByOcvgsEbZq9N9Th6FxljKg3Py65FvYKfMH/gWFw5dHDsQu0PNnxmunWxDoBSMQX0HA6B/SOApZhxSAR0+c4lffiWGloKJtLQpz1nXHhvaqHBGFm32oI5sYYHcWh2gXCPFmIxfStAK1lVkNMU+NuCWBku6Hz1t/U7lmdz97fIhKWzDiMrSZCG7mXoFWL4eHIL2W4fHq2uk7V47fuJbeDgYr4cFsF8VWAUjYtvL3BZWLNlICb6HOBwDIy2v9/Sx44dxuIye8GoBzD6HAw30oMYz2HmFofgvmMmSnbngbtGIVqX4+jpcD9ePbmQV9mdIAX/BDz6/p1cUSg1PWxPWfIH3oCpshnkaW88ZC1dYwsSn73zhxnMfFCi+wYUb0PRJcJpac3VlgDRVR6c/lXAeqoL2FpvKiMfs5dJpX985VKDbiQzMwAs88/WKCvF4sLG0YLIMfECUHwXKm3HpuLKT8yPs7B69uxtYl/c4XD8pWuosQN6jMI93+0AI9OvpnyiXyskly/x58JKGjywV/kdT0t+WRdgltgVeHCGjwq1TH9taiVh0+4ZndkAJgY16WKodUwFdIoFDjBW7qZdSEO8zY4Ty9etXWLJp9M4cd4lufm+bLXntjXt0utb3VZNsfHgES4E+Vgzk8/WPcM0c0gE2thenz+2K8sgWb/s7Hh2ZwaCLiCkhQy80UOsyUx4Xs9mGPMhZhDbWsVTW8nqESD1eA4sb1h+wAwzbTN657pqpKtDCa3QqqR3V4fUiOK/8bkDRn3OV+SKCiiZrBIFKaaUidmkCdVE3YVueiK3UfUFgtuYG2c+GYXxAp+rvrUfFtt7LAm1avyXxcYGhpFwjulLQefEA/aHes3zIIitO2C6KxhISLjEsIHUHrATMi2J200ff64fCHCzP7EvUplLcJidaKJiLmUALZ9z33C1aAHroQVaAoSgu8h5qfPrE1KDHRZOgV0+1v30f6zcDvOk0KT0oZOE+5ZwbZPVmigjMFr/2TZ4ATqACFixLmvvQSapMLLsXwwyIzkmMgL1G5nWYM1/n8pLSNehwd8PTVyIeAeuVauYl7cH3yLQ8Fp1n0xdCuWN1Sp8YuGT90jebG8xj3pvpb75gq1wYEc/2dBmPysoFi8juEslgVs8Zrq3AZbaFH2ykc3KKJXCHxjGBtQWGCX5UOVjp9sFn64gbJmuZTlFO9c08fOngPZmd6xkSx/pYap67z4nzNGY5StYOIbjPIXcAzj8xDAmn9EDX+QVPR7TyKlGTw6wxUxgo519jpHbTGwotOpUxipMKqJx/CnWm8ipEvYG3mFXOLEhPxomG2IYHAlGk2E1QEIa9qzb3b1dSOAaTaosrxcmPomtTp9y1mgfkVL15JB8GF8rqbeXlTQD7jEO6P58iDQqsiRr3y9t6pK84hwnD0nZRzuqmuRsGA9/mK38YAjSDc9+b41cg1ZDzwJBeOQGBT3411479norrsNneNGOG6SSn1/HCyE5G+ayIjhTGh8b76qY2b0kcMF9IC2UxolWr17CzNNBkb9Dxrh7xz5fyjq3DCrIXRkEnsMQW1FVZxpPmAZF5v9loof7Kofw5P2t1uoviISAg7BHM53gwZYIQ6WEJy/w2lJyzL5/VPVv9aeF2ScG9PAxQvlu83Rf6AeJuaQKkIyqfa8ICj7jvMA6EQ3qsxYGrjMskPdhnuI5kHZTHqQ/TZOKSd4FxbYfJeRD/RDWDRUvRo2M7tSRkR5gSI8yrpOK+VifuOgLzL5d9udMz+PQiUSRUAeBj6fRFPzZRxY/stgg/SrUrvrlAmHAnUb7xo9XSkyII9jU38RIuTEjF8dvVOzMAbhDadgxceBr5AnK2puHXFTqpPxvEt8OJfuDXmIMMUvMdpnuyY/hgshIfY1Zl09VHR0DVKEAgG/0VmcF32BOvKhMY0UWZ5m+BJVHuJJhLoaDpIbzdREJez8xEghJH7c2e2CTjmHrQjwOuQcBM6wwgiXMHpgEXUfCwRXGZsWWvuZedXe9yb2lL78TQR3Muvpen538AGwQauvDKgZZYDM765TJFPBd4VofCn3pc7RMgC/TsDKW4FbjL5AV+Y/GkUN7bZwOj9ZJVXAuFbM/0O/PqHGz9yUrmhYf2vJM2c8Mv7JXHJA9xhQHK7Qh3DDyYYLk9Tn5tb0gyScc+GDxYrMROHbCeTys3YCzGHwsgG1XGzPNjMMTe+NtvAgzVrT/ErlG2AffB+h8xmollYo1kSDllgmMHD2pDSH2Us9vCGTfCyshhsGvBQkglN28JSHIIcg7ZZ2Z1PXYhVHQqnmOQqEnImPHeVy5U74JeoWmjpFn1eITx372/UzBhDS2Dc4p4ZzAyqLiz8bVz5sHbAcMghC+QD+rNpotHlurpqPsJck6JuJRwj61BkndWoKRffhejxlUeu2UwfXi8LOydonfpkZX3A64BBDrCdG1yslmsZ1x4FVhNFqGU65AH1Aawhq1tGMQ8OHuDp1ZQl4/KkVgAE7EpvfWvN3E8hMK2z75lr/U/Yk2TaEnO1+eFOfhbGOeNu1v6l9m2J7+tXKUOgy4Be/QdS5Tx3juiUdzaA/lxVtXAEgw/RC4LPX5U/yzJyEUqPTjtCvMDZzphfwdZ7gX82arZ+Iz95Z++KYogk/ybRHUYVp/AcKDGzLhTsAVdRKpfrYTnjYyefUM8HtlnjIlTVg3LzRoUUIq5tMC0RNIwdawmgx/sC/uAiXwtefMjOPmogaS9Ro/w6DvcjTLW6oESCJ9RKQcbXbwkoqlihzWpEegvlOWjkDX42UVgMeRNChXnceTeZbN2FqCRxN1emD59GxIN/VOpKF9R/mSN0X1FqrfDJkZv+us36mul93XfaxoTi1/nu+HEH3MW0fkWOdAZM3DZ6VHfBp9HXy4LkctC7R5uY4u6Ex2eNWJ4mOV0BmBckwQqGSVMVtnu4Ayc5gCiZSVmDfc6jb6uloqp/FoZ2lqTZPy5CmP67UeEDlsd/FutaoXn/mqvjEJgX7US6PORn7Fq76Mm3spuG9XOvVfHqGSFArPp474Lu0WD/hJjlXN+WfQqmtMWuPmird1vITvgo5H8Ur61jh+pvNPbpf2W3vFC5D+y9qjzxFmT8kHZrOIFgxcwEXDNPVLUiBHmugIPGjdycU4SXGVub2osHWCB/Mls9tj+Yk+Xe2LENyqqoWANV/GGH6yolrunOPQR+IZsHc/TRfTR8yyV9ZBP/BmT72O873duJk2h7ovz7qSJdcZBJkTrZ7E4TxQ3Ymxw69ob6c3fo/Vu4d2e+hGS4Qde8sPDHZDzGjRFCL5dMuYQ1LDlVf/2KZ6WEsUgdlA2/YvL/vC+UdVlTswwHEcJ2uvB5Xw4wOatAAgFWAV/2x3BTwnzC+aGnHLk82Gs7YDuKnWW+TOr7btH/d5lRLlDxoihm6yrMQkEVudhxImi+4698xqkxBSP5PJgBEDmN90rsb8RfYNvK7FZN6LbYqS/7nUIGnSyOrbZq0GXkrPWK+byqejhL1yHlV70EMOSlYiD9J0mBr6fO2qp3MZV2PNvh3mQwZ1eQ62IX9Zn4N04k4b9rBpnrd+3osdYfI9RqwR19wXzYxaLwF/eF0zd6COlrqZTpd2LyKkI80uH3mD4gnWmRVLq49drVAaxmUcBqZQgmwiQZ3sBU5qSMtOUPJFdOD3V/hZNaxdx/ADxw6+ssYNdONQHZV9I1FgDfrhWXxgtP/OXSUPvzBZ6n7HN9QNI0YNid/NhV5ZTh0x3JbraXGum9gZlDr3HqqoadKowrAeOVDiKDJCnTqDrZisPEpnPD2Zwsdwx1WvkIhLEDKaGNo+63DyC73+6z8JdAAcB5v2QZUXvKIUvEF8H2duX2QLJwEy3o1IAE6aqnnGQE0LHl/42PFaewXKe7ytElHdKIIbryz8V/+MHxmmjdShOs9Q92spruBcUUvsB6AY4a6nBmhSxvROIUVmvCfqCB1uFiv21FJC/PYRkawQzr/WO5EJS4hrS5MsCzO4ESza0XvuHkr23IXIo3CeS/dEQWotGjZQq7gUDkGrlNzaduI3vlE/Scr83Zjp+CKuV8nlA4kxb6xYyFsRGXSa9oUk+6pcWnub9Wyd+a5O/ol2GGfjRNRDHpx6ugaWVWcPo1E0VJMCzWBCSc8/wuglhJt7QS2/FG79PjS5FcvoWzDtnkqbUfFoHuxLBQH+Qm1t3r/K2ZIVN0HfAptFalaxDj84868bSfmE4gFXPl6oEC+P2Guyd9DsLhMHF/DwL730yL79XM3EpqVbT39jUfF59KmDBV19T6wsc5jNklNAvtnX1331QovHceCmvYLl6fc117PYq59+VymVZFR6mvQAYan6iewE2v7AeRkphsXuQfWH0D09FpTh8aAw9VUehNCQut91Rc5gQfK6aYBywefi8GkoQrek4JSoMuc0tL3f6bUAZ0zxKbr5gvZTrlQ+sHAKTiGF0E+/PjJJ5W9eSMj6222CrYH+NbO18z/eKRWVscUmDpYEjusfHqYdrXNxa3DM1YpTD+lmX+SOuzm/VF/6rgs5uFou9wkXyP+78stjQBNjs9e2HhWNVshUWbN8/ws3g9z4IqYHO34cL+P6txygmRWNM9mJosI3/KraoD045By5HlDafCWH4bbcY8m0JDR/In1q3FmHHRJ1hseTrz2QGKUp77bT+rm4Pn/zmWO2FPTmO3gr7g44wkHAp1KZOzSkxAx7jKYiO29nQu4vngbfHc3rg032rlhSqyRTDcPnHIkiTdoJ4ZuJABI6OVt5We2CRXNe/5ExNvyEmjqILh7C2+beZ2S71yQsZGDq2WmnuVbL3uTz0Scn0iyb7rAdOGLY3WE5XBYyWObSy7+/1TVUF2XZvBIwcvWYPBMNarEMhJLkGz2IpZCDKshn8eq0YLxuKKc5XTstmZu5w304A5qbxuBv0XXG5jKcTik6GL0qEkPtZWmaSUfEl4HqkZyBs9toaNu4NOlbWRRc/UGy5fOKW/h9x7O4P9UGo+MGQJsdm6fHHOYI+tsMSC2q9q4Z6rIXUdBALSSArxxXZMvDfb+0ai9kK2XmfsDPjSBYOiTZYyLhsiD29vVkjcn+srVW1R5OQPNY4TaCbrAw+6/aLB5b380XEMJJpRitLvtNSmtORU55RhBBkjKRC3+xb3II72D0ZCNAu/ZYbqKjTRUtIMN8g7wsiA9rE+C4CvDhLoHY27TtOKFQgwdC5hQ+/fQYipVbalTLN7pxCzWr7rShOPdu9wV91dH31/NGKybehA1mLZnh4da9bGCdJiapbdWk1JDW5Oue9JBjdEtkUaEkt4q7jIZXMhvQytBYQNhCtGfV5Ixi/Qjij/yoR/ru8RG7INzPH3jJPjeujG+Aago0xeu4pPbPnV0B0sculUVPsSonsMbiCwTeyfpGJXuO09eFvr1ODjxOouCc8HOrsPMDGN3y6hCqgFV2CAYPQA2jELXLPOW8JmX3uOJXmtRHuv2HKnWcbu6jqjEWArGCarEsNr0oQ+6vloH196Tgp0Xpr3fBCy+iXgCkbY1RJCyS5GuhdSS+PWCBi09co1HsN5BEpnui49Zt0wTsY+Lfejt0hzG6utN5GNpl+F4mypaLoTFkoYA4FhIvGrHkbG1G4V4vvw6Ig7IlKLGptBLZnfZP/QZq0bfjPenD3YUsRgbX5lKpBvWsg1nnV/s7du/gJuG8Aj/MjKfZnTeWJBHtMONe+og7luv6W/vzzcj9po2PMo8OAGWHdefZ6v+x5eGvMT5F/4Af1XPKMBI6x1vmSfBanr/jsQblKD4vGEk3TJPwQyroFiwlmdiVZtACFqi6azBA314ZXl+ZBm1aew9d7Vdf0hh4VN+pfZ9/TpdKzcHXoJU87DsmFz2dQb/oOwgJOVNyMscGFdFx3HtUK8539berFaTBu3edoe+MQWMXQ4eRxda1l+qm5mVNj4QEfw9PTh1KSm8b3l+YDDFOXWmj3hV8DO4MI30O+A9AP7oU8JKkBGU9ugFTlo49bypY9DFDEQ4bY466RDjU/hezpglGX3cvO8XF2GieO0XZVtLjXlGnqOQzT0Sz2MLarxIgUhFvO+eDNPxj/2kp4ij3cw9P5YQvkDKnfA5buKFvUj+Do2BKsWlwaXvfaTkxkcnx3xdfPc7w3VaCJByh1MoNIvfV+uHMuB5swhFDx+aXHmPa8nSqS228pHqp9tjny+z2//glWRPs49SCM9EA1kGuzX8nHbO53uO/9yo5M40QZUYw4VPafNGmrVgxUlT61PKcckg69JX3w81y/htbM2wdw2J7RTyol+8IbbP1G5AuKXBH9aMaiCrmfCQeKG6f5znuPkmK6DJTFkQo4Zewy3Bdbm88Swz36KMrupWR0Rpc3CyeON1byUFZe9qMiyTRCcQQWOjbmx4CVHCulXaHGJWU9aZr8/IV8owd2UVQVj/r2CiUVTLwJTp/BmrfrnjxHzSOiagLind2ikKjQDqvXaVn0TG6/BKeLwvM8u2OwueEmlBoKRIUVqLotvNsx+zS+v5Z/dG57gFoEkvgYyOI1guf86lTJqVk2znrGG4LtzywLpIPqs0OuNEl+AZBZ1z75onam2wsYeGR7iiolr1Iru54LKHhI5qoHj43QGXPSgQs6XamBm8Dy4veGP9P1uqtHM+e6qwMfxkTLh5WDeAPaogNIiLG9PJWujr63FH1DkzpSk7Tk+o3k0tPxoWi9cNd+5R9EOf5xUpp4y7i6dR4ZoP0oNoOD5OBpmMWCkrXaiLaO3u2zJkxiKwTr4XBktD3y1uXMaHepa27YTH6vo8ELBucujS6d7DIVlYotYwut3o6xyfmtjh6I+Gai7s3Bs4F+WvUxOUzzIQccuCm8HDOnb/OoOfn2q/LmqQXB0j2HqE/inZQObTxn8xUuNNsg8tF5AE8f8Gk+P7ya8ffJ0N/6qkc7SMKA3tEXpbAHHqYhhdjsUkX59/31lqrLepD9B033A5RTSaqzh29+rh2wkPuP+3CQKelXkLV8S+z361PwK+SsJbr8HmcOV7LqTDCX7hucVTkwVy4vcV1lFIoWkQFkBmQzzwwafcorx2/4nDgw2uyUSh/1eisCfwqO0KkoBfKeI/sqPaphV+LR843k5qfxisrDLXGZ3ybkIOmmAizi5x1OkxLWR4YHDGvg/PfhvnPS/WBp7RQUClxB03AVv3IGvPviX/KHm3AXH/LzviGrajWX4j0n5o3M0GaTvNvvEg+zSdz3iSGmORdvAtBIfF3yBuwrKG4ZBX/Rtz8EDxVUVpRP9QcMNwJ6MvfQ6NNbJbyvPFJD2zA0YJH0NrtWcdW7KtW/fA8QOlBJ4BVcGyr/SRxABVCEJJhjT6rgfRbk5bTlLvCHQbusDH2LyMMUwKRh4IQiemYRGUo8rfvaXjjki2V82oiUjJ6tFROTpkrQzYlqxoZ4+kcYrrrCvTxBBpqHS7wSFDhjMUNNRiebId1OUcyaWMPBblnSq5mt+mr/RKNO8euDyqqtoG2GQDr+DZwO6RZUJxQwXDsqJoyriru6ROd9ecfuXkbjAucum23iw8k+oXKF9NtA82XGLfhOp34U9NQZxr2zsfNb4TWbSYHR1qZKyrjiXP71XY5flOYsD6216ooUpcqcpnchxxVMchdGakuQN/uFamigDMv1QOS3r1NifPH1NHmmScjLGCSaHDvb3JC9eXgihj7jMvRTH7R4clgvJWhNJzuDTyVdSMPRSLcEe9wH/jeRTjK8SVYqbppCKwlCLT+DJMMi5KweDCPPjWvY3opNcDaNUEMj5OIN6lEDhDJ8eEv7XhNg9AHm7bYrUPKt+RmXP5Lf10euQBZe469B1OFfP9l4Pb0G/MFIy/LNs+vd5EvBYbfqZG45CElIVHmGOhOuGSAGXxfn0pGwi1PBMeJn9bFpvmgsLqfN6tpb+loUT0fhVRyu9QO8Q+9hZU6Dkxz/mJpdcbzWa16ji1+yTJ6yvfLDG0WsAhAx/3isdQfA9e7wJ9WccAGlSKJ8a8QneNR3hCq2qvy9Z/kuruAVdVJwcNx/aes42M1mf5hEbafWv6eC1roGK74DTFkvGtzTtpLpsKodONq73EuLJs836a7z8SCk9fv0fCWzd3ccw+BPcz9Aqhs5jH+1snXlFeYv9MmGx9jdPQf8Iqu55FOB206J567oQwMw83N+hnKUILjd1Zj6jTREpXjAs9h3sl31ce3U6+zMC7hlJT8YRBoJdfhoPr1uZzDYqqTxar/R9Vl66V8T+EiZOxu/Vy/e0ftTGd/dxrTU7k4MUm8Dm9t4WSRkhfgvvTSvR5ciMxpkPfcdoizOjdBPDnexeZlEik/90s4uwwY/kDbJ88VgbXlBH94qClZgeFilIkWhDdAw0QX0Pand0qiRemvan9IaYnFn3/JLEldiZ+kjA0eXx6lFI3PwqUPtw6dy48zoqM/gdqSLnFw4dFcoN/Z0qTBMXhP9211a9LH4e+dxNG5EslDq6VWF9xI/qUztwU4/cFEnzgL+4M2OnOalaXcehKgcopIhwFHLfcAeNCxNVCIIoKiPtZCdbA9fMUcq7kD5Rv9bs9LaSzNq864CvkGxxKMPqqQ8+NTtUMzITHMpXsNv1c3XRg9nYErr/HSL/X2VjETpyxdYCHO4ELDO5QR5ITyme3eMDDxQgXKVSmptP88urAKyk7iEfqu+RBasksF9grFL2dFlaqG3e9JATLOSgyEdkFpxMWvxwLkHWGcjjRfDeCzrJNEMTGd0qEc/Eb+gKCG27Gv5a2uYpPl+8SN816A13Jb+Kl2R9OwtLGrCqNCQX3svYryRHBgE5W7STJc0Cg1YclqQ5cEGU+u4EIII9bwkFCX0syBhUnYtDNqgNofMvhfPrxE6nqZqMBTu68gNF/CmSMHeke/wqCJ7pz3aLvTPIcf1BlAUBXNlSIThmKthQdS7woOoPXDmAVvjUbbwxboRfsQnwWPma8qJ2dmOhg2izZoBWx9iBc8wU3PJ30gdcPblBwrQdc8Mj6IP5zq09kWyTgtmHsE4h5mRuvkMkChTFtwQLxryIDSt1UeR7oos7++4bpibubCT25g//leRIiXyw2L8cLSzN64qLn9WqI/5L6kbCc49WOV5ozGFQN+1CatmUTrYxdedFI0NNNnGxX4cYxKIH/6NGfW0P6+LsDjHh496oyk6mnxWnGKaf+BmCMnyJbkO6Inqu0AhNh6Y8jAtJvGElwkv+QfCPC+3+1JM74GZMJ5j2j2IPhRyJC3QGqiA6SQtGTUlvy3YdVm6uQ+d1xPVj9TT5EFVKxGRrTJGdxD1PlYWoRalCLSU5mdAnaRkeQ+9zLRHlLDD9yDiR7FZqqFRTEKWwSI9xuG1BZTJnILtOe2hozxQTmqEyJC3wuQ9pDUJgCgL0vQt3Aa1UMTEk5p6MAHkfYyGxGUg5D673XlXuZPu7v0KmjTYtYDkRyVapYvIjyzZLWV4hC4H2DJ+B/uL1xS5l7Hyk13GIUDdjmDFmBYni9LVxRF2mjLHLdWdKvxwG3mhYE7wuYxk3CJodrFrVIj3ABP4CWCMUv6gyeKtX+8MgrRPxFIPjiKsG1BMtgob67FVZ1A37g7Onr+t4siyWPnvlNQodWFYmU5wMi9qhetdSSfRNvNpEgLRsCZ+OhfxMP5BbIPE7Qrgai/WvJGaEAga/yjM69C/AMV3aJXbMYjUYpcPRBp+BBU4+Id9Jy6keNMRALXvFcVjgl9eL/tT4S/JsPVs3zVgnyg5MyCI5z4bl8arcnJXVI321Hj8C/wZJuXsw6OD9FbvRhHhC5urDfr6yddhcQERXkGtdw9O1R+lRrYnz0v8jDm/hQQNT0h7cu6LPRE8+27nk1lL0GgTpI/RnzvXKQtkiNIreIEqpWC3MGRflRkdz0+lAi+y7qlRsGk67NBlDOSXuflfeQrGXd+u51MoBx9z4R/fV688imn4QgadV6OTsL2Esi33GkeV5hCYNorPM3VdiqtAMOFXn3SWwRs+/QS2HdJ0vBHIVuevR34qjB3JWXhLtlcxE1jihzVfmwgEfd9Vw7QgOsUE14UVQhnNE9UgxmmBgnxxGBIPxb4vmRBpQ02bg2f2gcWBwMWwAyMi1aMxB1FtNYg8iDfxon/Vvzvxvh+B8Y2EtVRkDylc0hG68aDINsgIGb9QK5zoKc47IRc2LPR8x8ANxRHAzQIX0fngX4E0lIQsMkPgeacVBN86vMuu3rf/IhsKrPIVpTsyrWN4mHYZHd0MhYcsUcDGeB+0MOmHkwfAffhnORG7aUXfvX61kW70hivWB6Day5iDN0nyFLUYkCmOHhv6QgWi7p00rOEzxjwUUrf7A4+q2Q31eaI4DSHABUl57/fG2lWpTMZfnc1geZnKOl8SZXjnlWsL08v1xVPtIuHQo5zxUWxwgWUtnvdFNOiI4oteNEWxk/lRwcPDxueJeh2yxrdAsoJP6Hq0sDPxthWLr5W9EznRBVizCHICRIn0wgHLw1/VTvCtBdznYUh38SccyNICARlr4P5SL3faIbNngIDYvEonxxGjVcY93TMTgx8ReOve0ahTzsSDpYBx9sq3AcwJgugGWZxdyTZBuLCd+fmBG9YgH8IBXA6sxMVjbnmP5J+8Zcn5rxDA4r/sRTK83SzsDaU0HURr+KVX/DlEVl/zJDKot4Mits9sQ/DEaehvx9e3mAuLLf00pdjfF7OBEP8zSBtt25LaVZ6P8QtIbhisCWQIPeo99NcqWz/m2WgI89gfxNlihjvumV/xxFPGR+cFFmLiNtS/yr6raNr87TKVADP9WyHktoT2SzQAyCFKk5eWbCkkDg/xqNfKAwnBKlKaAa4wn0+YPy/JcHhGXtEVnY0Ltt0Qqxz61YjBzT9Sx2URaYwceE5Y29HdEXZ4iAaZZig/NLfOBk+3WWg4NdANYhcAnWfAIsLjVHCaBtbViV9y+VNpGHy54Ve7jwfdDT4OhnQlEXpMKXVzc3icBdiGZMJu589F0qg96R84K9LE+h80XceW47oO/CXlsFS0ZOUcdspWzvHrn9hz32rO6em2LJIoVAEgkN8YAqAm1V7qwbD0uR4NAcKaLJ+KKN9u0bsXn2JiVZu/q1e09tXLxIkHUSbnwPY5Ex3kY3/lskI+97Kyr4rUwHoAWCM61aJU9INp/Re0de5Maunb3/AjpWGlKfYDm6V9K/Xx09xie0zwra9eDkIw/5Ll4DH6y2jniR1N3E/yvkVVSpTuu/A99wKzmL/YmE/YBPvMbVFfMet1IFITW1QS83W3YJICYkb8kK217irCpVVVtBvNqqvNquVSFES9bOoxsiT2VcSK53etRL20IgtcipWnQms96Ael7Qde7TT5cfmQGLvtbCFbUwn3jCBaJ7Ky8D2jOb5I4aWGJYnRJP93LAsuYx3l3+40Ior7146WOA4qIxojxD5LH3CJYV0EdpRWuoCYxln9viz7k6jJcrgFpQqEdyBsk7TmCAFpePFC/v1sD8T4WfWBkQ+GDi5XbRPuThRh+Bhel0UVAujfbk5wxQ+6Eey27q23GoRYIxWdn8rUw5lFkJCeEWDziQ9dIYJEs07L8ydP68Rd+/zn9iIXKZHmD1RjhWIfJie6NUPTRh2fwxidXxqu5/ppU23w9FzCdVZxb7m8wbG2v00RjizPS0UZ/A7JNpivl0qzt6NTKaOZBuJI16BwMT5Zx0jlD3p+w5jG+Fcy/jWyCvo5gm5jRl33RvJH80lUyrZYUCb8r4V3GF62B3GSVCRfiokkyoEBNFHOsqCrnvK9nHKhKj38V3s1HPvjLYrOYdNFR0OaDRewUhoKZcJso/jHmbIdI1ziRCxdsLkkg/9+H0zGu40uo4nBGxKhLh+PN5wHHDXGQxst1V6cddBjivY3lbYF83PEIkfdP3cCrFiBhoXETjIGi4ijSjx0H22KxE35Xiqzp+71MkSgCCmRlP4Zusg4ea55XDfQA0CdPAzQaCODp9eAuXcz+7W4ODyNwgyty+GSTOF4GW16ALgUQU1myBqUxXru1jPf207IDbTEYHe9BC9lxvH18vYLdTKq8xfiiiLjo3598t708ChsdpezWvF4TEl9FOQGkzTEXnG0UF/BgvVIk1t9KLyrSIAC2sPwgAxcB171PsCvZ+XH6/hKUGueCz7yKBdspE8zkSKmedGEuh+m+dfhPvCTJ14IFPjXbRSkDXnq5OWNGXw0mWIJR/axD17ri/18lP6G1umMdsCmSxPejvOVLMXIe+/CQZhtKwq/PJI5lub8OCp8si9knHOx1xko7a1tAKA8LMknn628+tRb2iB/A7mEQtQWVc7d4RX4r7C9SpI/KNAfWoxf5YhowF5RI5T4j5BhJAiDX53CJ0cWUlvz7B5doaCnNWuIzD016Oto+OAPwcjchpEUmaWF5ObqeHpuMyxDejnF5mmaNiWf3+qCFsegHS7l0iCcanxwGgY3DtmuAxFF+35XLfNTs1l/yEenBVNse/dluhlmM4jlGUE8g9irOKfXWO8NWhT3OYvOVZrqBqwBG9WBfAhEuraVuSMQc+rLuSiJp79xVOgZedMZAQdHR7V+eJjN6m/hAl5ZTJRHcxympWb9F7D4FqddldWFyz6aDQkB4hKrW6h2GW/2gE3uiMOWGzmK1V409Xoe05ce6efN1RZg+Ge7jIQwATcZiN2aYsYinf7FkgSE5Z/n19OMLNUPopKgibIo7TVN8UYqcOaFuQU3D97wN5UZ2JFVGs0rS2pqnS7kfKRvNOoN4Xgtp8xFVyH2T4C8XDVhU6eMDlAOTFGrnXUHi4UCEyT/+opLGC8PBagi1PX2OLuPPHB+t6LeAYvBfrD1Wz9bFxfZEU/WOdzQP3LzylHoZMTfJ49fqbCa30FVJF/CqeikBMAaHi9pHko0ac/ap2gBbIUz+BzkRscoWMsZkjdBlXFrYycSzpgo4JxLuiwXnK4jwYFCcf8udgopo18IFNXpqiFx8jvP9mZ9T1Z4wXyZd5juEn5E3zG0lETKCnB3TqxefC5gdolYZ7CrwM+nv1qIA6SqdsAbqu8nrljafLYfU0cj2N6QqChjc2jdDHzhSLUo8lbqpHwRHGRQUBuqcduH4Jomu2IFh6nDFwdEpsuk3YhNXYlNJv58G664c/FyXUcLQjvsZC8b9PHzWizQBU2RFd/re4ry7KzHX5/7oTmiEi0Zyq0EH6AU0aovRUvEH6hsjZybhQGntmoquMjMehDTQ49zj9Zo7EphKyeZCIpjomP+VkNFZOCZeFct+NpOOx9h8AOuQky6h+4DpB7dhMPNMQh+3UKEtib+bZbmHJ6SXL02oSZ2Lh8VQmS2Ma1xWZ4/38fXkJ9raaXJVfTtRoQ4xRN8dxqQ7LL6HN/lM4/g7Kwa5hP9UiOqs3P2kXy6j1uzHs2kOOAyQqzIvwuFitdsfpXStLsQSRXV5CsNXGZVDiP7Iz7GTWwJm7EINvvT67P+5sGq+PRyw8pb5pcvPZlBysNl2VWb8jsMZtaxY3kF7gE3qpGoS/3XmX09sRwInytybZpKzDPptl50oSIwJpDDXn7Y/C2WjmXHX5Kac4KGTYHoyWefZWJFsrZfRDtuRg6tHpTcTKmTnTWY1ZwowLJS0mASu0REjKEXkCIDdUhfwK83FlKWfzfCLpj64fX3vO/9npcc2nr2iZ/3UZeiTgdTdgw/FU/STp9Ygf/hDIRB5KGZiuE8gtOlJxfU5KSGx7UBu8VT+5e5n2YaOojitxJGt45M9PzQTjGKVCffTZb3V0AgGAasnGZHagdiTHtpPDixFxcV/I11AiQi8lRJGFgfi5UIzCMUCzUdLh8DHrkgNaHcY2PZo4THBOXvjjRzp/NnOqp4a1x/TJNLTN1vgRp5Qe4Tq5DLoG5neyXl15KEKzvkqNR6FNuqvxIqS4MpqW+Bu4hyMxu8XxAqEFS3wyft63wj+vsTe1XN3DsrCKHemXYEKEWie393OinvNOIx2OtGJqtClELMbP4+2GWirhqRauoV4CE3mFzN5Ytyl8GwXjrc0Vb+bOdg+EyNnPMsueHX1xESPrWuAt5NfyibSKmvCYupTFjZy1DCm2qdoKhbzcIVMcZbp+tFyywv7OG+opjqejYgKiXdBn4bqQoz4QGgnL3twLrdDRqyHyyJvY4HxvdTpfjRft3S/qYGfFhIHjMwoV/zHi5M2sRmZLkdgtkfp9rtmeDbSDtOPAzLmgKEyIRLWQo3Ub08C/LRv1YJuKF+SZlckh76fdBqCupG0Zy08TXMecBJwp2K+E2areAvq0Ollbv+GDb4r28qfj/MJCrxJsPUeHykqi3hvziNrDmlQe7XnX53NgR/QRp4rTJ3/fr25WVSwvCvBSgLjqrwcvBQJyIKKOwAzSFhVFc4OZFN9bj62gOSWf9ipeNWqj7X4DcOESC3VBN9R/SOiLKzP3IIZhpGjVpIQ3wunYkl6LKZrt4x3FCCcVDPXkzkYnyIfVrJgGjYliXDrHOU4jgb7Gopuny6takYO/OJ99HqeVYKDYPqTKAE7rL1f2TyGwMchv3Syv7unv4TotlxyHVxpuKU2fI9QD4BQTiIMICICK3OjcIElKQaZKY7KPZ359wpgwZN8JZRvy4M9mKph/Pgy+86ZyZTRojgHYgZOy+zjlZ6MLEZmR6cxQNJBRmsxFO2dPvQ+wkiqiuRDPLTxUYwlfwPy03MVitY1vSImdHqt2b3rgN+1Ogkm3lXm7V+52Utackfydt5hj7C6JjFl2gNAh1W8FgOK0zhuoey7r992vTySgziklgjvR75+tRw/lNlryo79UfsAHhfm6X6Bf7gX4Qf/67JszpNUut5OBYmXBRlXlNov7IOBBt6hlkFfjLCpVoqDwPnszc1/Xn4ScPXApgrxPLgkFCnIH8LNkReAuEJgNQw4MdCV17U89fVJepyEIqqD8PeVrbvpjY7sc9Fq+PnmniaYkbF1927HkgiqmYhj9rZ2KrUAcdt4a5OzXLz78YppPM5qqd1YeOtDXq+P/vY3yriEmpnPW5rWGApDvVSSGfTNd/V9MSsAfOiC/IHY/wnUuf8qxKL02hhZRT4QV9PTEFYz5iGBr/SAO2dlNr/ZJtWREDh/ByvmWKY8F6vjPxMTMNhVvNapa/myPTMSOSNJ69A1m0OtE9ePsF9Od7wN7hvfnA7Ta2ulj1//ru9u3vukXJDiNUM9Nd2JXHu+iMLzf4ktYIuAfWSIPuq+r9zA47u/vdRezyaImke8U5QjtEE8LJMBecfN1pg6UWqbbpycKOs6zqZ0p/Hx8YvYx+sc/aTIoLlEw70lezJ+fLbYQNWQa3qK6v7v7CKdEqIUWyfkF033oMxiJZQ6CDav+SU/560P8XW3WtqEs09/cK/++rocrsQDV0YQWa5Wnk3UgkrWtCs1d48Sn8UEsG0OZ/Uml3x8oAKNja8KbZCyYJf9XmR+f4MnoTM31dVWsYw1NAOT4HFd5l4myZHHSZZ+3VtbLCPKX96ATCS3ILlYe6o98Uk+S+G36xATRDAhn7X733rZzcFrG0zvONHKkOBiEu4jHlh0Gwn1/G603qCNebPRteyqS1LUy7fY+2SWFGmSk08/1b/9TxTQgwshp+xMpd6n/HSYvuAbizIq+LFfKc7WYGJtUQ9/yVqueP4ohq8nNR8FAOnyHJiZmFK8CYt4aU9vrpNxm6+fuu/eu9lDcKHdV/F+Bt77XgJgsV1kKFBE8c42wA3BDZ9CXiTG4zqsy9N4sMj5us8AJ8vmpC6dV/s7JJv0te7NnFBpA94yOB/49nJqbNijz8qEvm7Io2dAsbi1lrGykMfjtnu+UE9Uj/Z3Hp88gU/hfAAHwwV52/Y+U3FlLqjlp6cPEnOkbQ+Z0pYlFcJPDuBSy+J9wMU8m6fCJ2dB6vC4oLWR0eP/ciPbQUyd5B/jhrgc4uNdPiNulEMTVJxoZ8cnjhUiCk5Sb9G1CmyGFXpy3Xr4TlDHvrw9opXvCLJJrMvazQqvU52tBikntigCdRyZ7i4hEBB/nVJUMhIGvaDp7Cl4OkIzFIGXmje+5m5xYKL9BnneCxtoCARKUMVMFPbuRj+nST7oy1iripRAl6DVb5VnIhyVvyl7ZApOuUzPN0XssNG2OURqG24e3QvyZwz5Us0R0z0yxPShRjlD35sA9D5wwXtcsSFtfyW768SQLVfj/hD3F0CW/yUyp+QODeIlSsJjKAWJ1Kl/6bn+USDB/Uvaf1rRQrrYD7dVhT8ak0UhRik9zU01am+kDwuPZqfxDptqAHlcilfy+vhYM88Z2lwwONXpKWPkvaby8dnNUSfXWhUwDFZ26y+QnFNHtjzIMwK4/Y1o4z0OzhxNeT4s4YSvJbjXBk/dseRHnOQ2olhrcY7pltUsaSewm16mNAD9cnsqedrUlK3ywgiRaosgcXskQhRZZUAH8w7qNxtqUsgBkcK2CR+Parrd0+QK6/xT2jXD2I42pOnLib8yfAXwo4HEkNWAj1HQQKONVhBdecnkMIOxxkcAV++KlNMocMdfj9RoRI6oHMb8Wua7Px3lQSydO78C/qTiEv+Wpp2H7yu3sDTiZml9CzaKKa9lajGz/aVG8OyfoFILeQi62wZp7dym4oaFDAW6amzsCjxmwvKNo2DZvXg+fjygeUwYRJclaOpwtwv/b6v2tcFaklJiJzN7uPoTYOjyBePlZ+/xfbYl5HRFPoyEbgOZtKLUFnWl6GSd/8PfT7aCsXM+fVfZiyZl1JEPofjfrXX6I9gFgkyjJkgCR5HJWM8wlOxh4jzRNxqBHCKPNreA8+xZ8kx7ro9Ir9bUUI3AHqx2bwo0/PnqAmcj1cGafaT+IGYAP1rVBtut5LJRVSBxPa6Nw2i8+XaC6wp3gdyGctHBxrfJCXJAFt1oLipU6qA4EoUeDXLVk/NAUMxZys6zZ+PSiQPYiPg/UrplUCUOIZSKNuTdViHLjv0z7uOKP/+5xSDDzGDsJmqBB5DQRAT1a1vw8tNCb0/QzDWl3Mtl73x6LNg6D4qGUwp2itDCNmHs5foIPVRojPYX8QWrxBODayT+yhpCRTQGx+5a1waCzJEyKMt26+fnzXsuWFeNIjMO2s7J5JmN98eVXXd6oG9ec+nnU299OUJ/2aZS3xK/AqMsM3IfhoqNpaiTAcunlzShuq3wi8sQTAFk+vOaMIEm2DmwY4cW2InaSagdHuSTUYX7JOhp6BNRa9awY1xEKloViu2rKiwnsUQf31uOCK7hkSTpOqbglAdpBfNglQx30KhCagcXjCOVH2gZMdpfmdZg/lB7jBHeziqjGouACA/qR5ZmAOoehZ+yO6I6BP4hSYfF2CvwkDcO0JsfjB1kfLHr7ZxSk44W9rPmLTDToThfv++NSSD+YMip5iTTuDjOETYzCTY+fWULmsQ09HY0eh2SoUQzpiWRcnVPzGI7rhBmmLIE8QPUhrlCVAhLjNEJ4UTX8nb6ZYse/Tv3N9jyUdX7+SOXfr2ZH3bhfh2dnp4Y8II7e9WN7vHFjuji9YCSM/H+rmzNU8oxB23dfSA0ACmY7vTBZBozz+lm8EioDQAEYrF8a29SkaDUIYi+QtlWghVb+U3mOZ5EZhANitdQEn8Y7z5foXm7oQO9yrR2U4Z/eSg7kCxglhQjEt/FzEkE/9aK/UBaSEBKzWwriDPWLLDH3V9uUfmlSaF5DDcDYENglXYE88mdudI+SDq8boF9llpShZ56j3w5EiCOHYahDuyXYXPEP7CTTRpqnr4d805+LXR1/pXk1pcR3y8lOoDkX/zs+hbY/vjorC86nBmMjSbNRX6DqX85ugE0bNzJmoy8pftWa/IS9HkU8+/55fi7Sd3Q68hxCFiLfy6IcRSlgBIeebXZPCpQJxI2xdj2pNT2434ady//nL1oP+CASDk8fK2G1cQdP/9NfDlSwOOVGB155+XKPokaWQU4jiCpt3yOl/SeaKMgkhPbwQPfpmrRZCUHjvlJCXD/jcMUFeVDNkgBfhzIoB632t/apfZYbUXrnmT8XaSIYgJSHxmmBGOmQu9Ijennl8TSx9YohcCVJxkUEjO2PLCGQgY4YD/xN3kS7khDNlWXzgg/uBWgW+WDMs5Zn3FWiA63UEjy5eODar/vkBlFytCrDEyoVn0BdS//O0GAcv9BktGtWRJKNH35+IVhRroSJi5vzIKQFEzRLilkZ3vyizpRaH9z9VvyzhnR1mSNgXKGIrwdttX/99+bYNXCldYc7xnXOa5p0UlOTVXiOXXewGZ5dQG8ymRGiV99Z6WpDU+9JrMulJ9R35msh9jfxhoPVru8mEtNnWI3NJPUMspBXmcK+iOdPp0k6vke2yLI6GJvwaDupo358XkVVyXFSgcG+diDfcvvoOyFjYp11ucdgx83fRrfkaTCtAX7kEeUCleimmwoevAIzpAbl/j8GdrIF8syrGY3NqQpLF7DpJxEaNe7T4gbrCEqSXMr8lTwCeJdOZETEY5hIZnOweMaovA5ExxvL4ZRNjMVPyJ8lv5q27u6faLu1lMvSI4osWR3AnvOzgPtFm7v9izg7Ft6eLCmEuvjUrlIuCMMj6kjt7PqIjd9ZKrJsmWx9ew91DDRPuKfXm+6J3sX9jP4ZzG4K3/6zh1k6WcgtRkjLd6TM5hzid6wTVfzzSBzJ3qnn83FPnZDk7JyAmSDt/WmcVA/xmxJKSfdz8JZVxOoFZy15wkyRHppwxWXl8sdJxfxNwr3PMDChRWgHhjRWgu/l2Oj3128gwK4wXKyTaR7U/06l9fcBHhQEZHWekgusx8KzEpygmJtgQ4h58bW7SA3OkSFIvrwTDDTjP/wqmkBLYdd/jEtp4T63uXWXIKSWwb9RmLGb9S3gV1IsbpeRyYMfgBbIEBnJxEAGkPAXYvpfCiLod7ORv2CSSZa1/pA876mvTXsWd/9x3H6y6i7GX8Ruy+aLdcNMUXZBWGs7JjqxzsAY6mwSINK5THIhzUOxl24448Gl4c0ZGWmdEywuozcM9r1nqgQB6LoWYsug2DoYmsNLV0PnoOqfNmgwCdiCxEFcjiH1iieL/k6GcAGaqoD+eUyhvxNy0t8ttn5UPwWo+Fxb0qmuZddTVYAjodvKvLnNlKccYJQukG0a3FSddXdySx1rj3xFMryq6FEbUQwF2eK7HrWpsoaZzyR1gBJ7Dt9Em6sOYq9lR8FaEQzxDYEGK89LXQCqrACIhaLmq+sjBcKnTxMibeO5tLz9Rybu3b+TrtDXeveugrmhHug4B3V5ax9AUTC4W5mzFjoA4rYBK0wOJzAS5AQJuiceZLVuL1XHf9JLoNpMFD4AQVBMsKp9MeeEXImJ0+EDeB2/Nehh7a5jJMIXyg4TkoAK3gAOUg5dIYav2BUmQh6ZsHow1EysXaFCbmz84NVpfTAaJtbPHzE3RswMMcWiFLIjO73kWFn5rmrwSDl+lvXNjh1Bs+LU+xSrp4nRRVunG0U5yaZc1O7ueUGvj8rZpX2n49fk6/xPH88oO+CsRFnIn0bSWbzv5Ttpiuez2k67Z13spGCX3cee6C3RIQYUok8FGqXKFHz217SsP4ZdusB9VhYxKbNpgNIEbdgnCCovoBP0xHXF+rotIva3Mgp1weN2qDMdlZHMnV8OU+hM6R320Cvkg6IsFW5hHqbyRMxCvB/rxQa/Y/PCftUCgEBQ9OwoetpFHErBZ8t7FRbiTU4jSaXcLRSv4y9x/A+R+203il+fAvq5gevIppYVrw6xk8Ez+6rvopEcET94JH35Mxaox6upZWUR41LwmKrY2mPxZMofRArn/NSkD0Mw8qk3yFQNobYXd3reJi+zMndTX3KKd6NTXnx0+Y7Uk7dl1XIGN5ejlhQbg8X+WtHIK9gn4PU5HUOiyTShrVzAQ0HTR4uaLQdD+sn9EMYo4kUMU/k1hAbOkezVVZl3D9zap9WwjCdQLuhtHoubvL+dia/Ww6Hmtl7ebap/2KqHGC1dKv8iVl0Rn7NQcF8yw9fb+vLGqI/IZUJtc1Myz/jsexIFflvxoe3bFcNwe89zHgEJVwCBB944PfgAGU/pGe0OdM/QH5fnar5r+qFZRbiruJ1u1zL2VEfwEdYE8WYDXW5RdgblRzE7JZjY2ISQ3owiVmOXJs7LdizMr0wuXe/Tvlb/AQ89y+KmoWD33WJVAvcWsyUSzuWeOL8lXsnL6gyexABBVS6dNgR6/q91Af4nu6G7KAc1yoyXi6l9BPPFwJ1SVuoiMsMKXSiVD+gKW14ShK8zOfiptPFhyajIZ2vl+s7EQpgAeY9C4EP1/d3tBtX/rUBuEOqQis3uLpb3XtPQdHwqYnfik2bU3Z4HN+S2Mk6vDSo9fjA96zxcYrF0nP1FVT7vKTMr1HRZh1SHbiZwv2q9He15OfFY06eAF2LBQdolF/6Uu/15p2LtT+KXBW3rdgc6Ae9w0LU/7Ksp0e+I8yQr0kYrqJQzhq/xusqLnoKw5KN3Mrsa/JsolDkq9roDFADcXf0FuOQlsKjTSCoPOJT+g1xUjKMKa5ErWiE43vC5DjX7RpdWLP2LRQgDTlxSWyPATgBRM0Cz1oinwWKw0qSD+/Y5bqArYwoAQO3peF8u8toGxhqcvL+/fL29+oC8p6Mr7LzAxAHTsopAfxkayd8WvEwgk5XZwUFCUP3B7UDompCGvlCpJyAhcsGL6TEf7ucTPmUUsGwHdyXzD9Sry/OYYSuaiGf9l+lQ6IKdjhG6F+GbfsdBNJVSYiJCRw/bJt/s9l7oZ+/OMc//ruPmK0LBuUxp/uJ6sr+QD1BnWLJfVtjTO/o0ueAy7HswHHOuuvyP6Lx5O0QD1G+QPXlWbQK/Ild2VzKdrvIOkYRtR+kqlaUrs9klFQi7ONtxGS97fKRUjLZrqUxDVN7s44sOuGZzIvcn9lXoT7STObV1C8ROqmXutNOFuHBhttu5S/L8aE/O4UCW4lVkSufrErajUJaLcS8hZHImj96sSyv5vg41fyOL/UujSHtGM7Z1xfoAJGcOws1ukii1iltyHf7s5p1yqvb4FqObyEHkafKk2AfJB9e+bQlW5tvrzoko/PNXBrWlISH67BpFYWEIjSKj/BtGd7aRxWBGID4skSTJDwD655UAe+3VbSWWTqhpCz94yAnft5/+Iz/PrTwLVWMZGMcQy0AM7NRRnqK02daiSzIy9BXCMhS/KOoWPGLR5d6M20vQ+15VQPd2wVsu879rDcfWZbU6DDjNfAK2NxdqEy9Wes9tueH+5zAjHUUG0KM/EqX5J2b3u+8IdnZ8Vslb9A38j8m4fsIFuuLiCmupbaIS55Qx45lffofn6xTUq23oAONfqwbbVEtCnQPi09csjOsZseF6kDDybgmB/gSsGsL1dIh31H91e5w9O34lEr32NwE5SFXGN3lWnrmvQX8BpxAqEofqi7OInzryXSoNesRFDTPJPlo8rp+aqkxg99vbb/OL9YnBoGqgw8Y0+oZfLSKaO8FSSml7++6Sl0VLlTcLJ8HAu3enu7f5L7XhzK6JY5C93wNbpruVO8sxPBGNCO35GrI/fsSLn+1Ig9bxf6RtwH7FkSD9OevGKHwvgIgpOsMCrWumu2x1anC2C7PFXcHCx/EGsbn2LtUXN9eDpcQmkWuxyIxQq30lX3twNaRJqBO39+wEYalf1ZQoeP5VEtA6wxBsE3LwahCAXYXpB2hnLGMNb4KeI9htnYNCHhBPA7zIz0fZnqM9dmR/I9Qr0wz2WEPM1/7Uv7gyTJ7ZBQQyGeovh1vRXofVCCw2rpY+lfbk63yzF/6L8R9Q/8UqTr0/mOKqLhSD6nTK6IUH0qb62X67M6VzRqTeCfxPHd7jEaVaRpgc9TAYsF0wUUx/hFfeRmyNftCMp1envKrOnl0MSzkxHCx8M3OTd0I36ae5FkG/zd1r1BTqMTrQEF1UDBM6V+h9rzPJgcYlmhiuYO6o1KD3058F6s1/y/VBgk6Sxg4D2ZMsrP8/LxqzrSo9dD0fIBuseMdgmghuvgEh+pklJNJT89vJBkKyRs6i9agHcXqSsz5p+jj1o3XqExoisCuQSNo4Mbao/354FQBdTpYCizoUqEjG23nvN+9yJVgPikCsxbDLRa6SYCjTw43yGUuBLfNorcUHCKR2jfChmNm2UPZxf5x/4iCeyk2uZ6TThxLNFs/spCSNv4HY9cmGiglrXTSCyOPJDAS7RBMYlMKE0Jz0g+uolcg8ssFrNeqJD5sh37aFNauBS641OzJ20bQ75CwV7wv4ZgNbhifL1A583og+iH+Xc95hMJPiDqq49KG+VnW/J6x1orf/gBtCaEbhCKUw+T1Nwlt+5gQXXZZuOVgj/yHzGDMMUHWkplTJSyFg+sqDAQ9dFPOfVSxr92wX0eO59vvXnUeGRjK9jdj0woEPppD0WBFo0Mir8I1UwEYHcp/U8329o28k2HEM6H9qDm0+14UeGkvniWHZ7zC9X+fGqKmApnFC5m+rtSjPPSusMb9Sz0rMX4Hi8l2hHnz9cnyA7dEEp4cIC/WuKFDNeBe5s/yd5621JqpJr4CGkzA6V4Irh/2HLuxWZypPXTnyXZU7izUesZnkjzp8b+L6+otC3t+rhy2s12kgH6Z5/7cVxpZGfBRLPZe1bHO8aaAmQITFbhEhqcJhAKaan4PijVPeh0And4xI6ro+8SAYB4JnB33kRKyx3BgVclxQeFF63eoSXc1oFfj0VPxOf4c0Z0Fq4X1iAQqrymFHYz4YRAbRXnTGbLPhaFJICR2ryWekphSAa4aLPDa+vR2uocUBb8tjMxsg+y56X7Svy/AGxifsbY5vxxen0sBUL25Ib6jqhCFrbgBU8g2PeR8dJENTpkC3fCXYF2JbrHeh6aYTLi5O4LXqP8S/xrY0KVf/dG3GJP1r9OfsU8H9i3oBRiji6XiwWgVRMcJT2ACJ9lgnEQYKJ+2BVAa0RaZfxPp5A7yh4wJomEI1Tf2BYA+St+R7FHjGg1zCfbTUQY6XARlYhb+BHv5McjfrIGSBvArRFkvTO9UDrCEh0kfiHh/QEt5aFvQdOk3PBf6xkMbTHA46BCtOoJ7gEQEfuMgJ+tBKmTmZeGMAnPfLflmNGWYqs6nc91WF1fCmfMGaKHxa4sSJsYyb8vXxI2GF/H/sLdk7sFBDujGqmXeEuUEI/gbFL50NVLdxhQ4NWyjv2usp2J9gnm2IrqKzFc76uk2nU3J6Uvm7lPKhOJsS9WgnB8qhhTm/3rFja9xr/ZPZApZtEa19FeQX5+g6XUetc1TH3bg5678FcySKIOA9YBuyxBDGmPt8Yah3uUzENPSwzWytyoYyaTEejvDoKnNexJZJH1tWqLy4yN2rqiqf7xb9GP/4IHpRFNqXlUEmTGmVM/WQ3Acq1P8ZYDvPT01HGKe46mUiVm9JXYc9CfY1ZmqE64521g+6sKOaHEbfOvE1TOx1toT58pdeX5w2qmTeZg+XpWzOl7OzE9AeOFAK1Q9eeTPKsw/3X+7NA63+i8EEvH9SQj2EBc4/CobGuhOEMC/qszAL6udUBdR+8eEaEdvnxi+NClKUzlXp/TA7VIS/bUg4tA+edGzl41eN+PLuyHbz78l7c6dDf9X4wLvBqsx3VTxdEq92DaR2i9pEOKgeSYwpz19mP5pRAuRAfv0X+VJRHtYT3z8UdDIpFIlJMJHvDlIrBuDFuCZ4yy5E0vKYDfm5bP/UNJq95AsB2LA1I/guBPGgQOYGRPUR+oMTdqv3kTTUHzEqVuZm9Iv+UonZxzbVTS6bh4oBw+gdTiHngm+SI/HGgzbSGvUimPF6hdGMFdB9DYmbVB2ie/pllAK8hFUiXmjSEqV/YnEsyCYySn+KsDIXIpcySl2qG92/kDZ4dO6PeTkwr97i/L3mHnzlHy6SketetGs1es8mDOrd/PFWvuPdm7CeW3JI6NJfc1bcKk9KiwPMe3z1tBNEAmrS9UeZFgo5dqzXhWEhhi7gv5L19X5B3JsSVZDRLAsSH6uUkFppADJD24+SjXhd/2hK7BNARwEhiebajPnCqvEipWcVuAjvwRX1yfBUosstmm/sr4x2wKxe8PkOt1zOVq9XoHWpSbpsnvJ+8x1ciDT2F0B92CL8cOJI219EN0wRIbBK6WtmOQO1bvwboUkjeWrdBSuqpL9Sj/Ul7PjJ4uSFyaQ9JG9U8JUiQi+UW20hkeFysAURJVLZwzvZq+144yiOKBJqWiBvFhTwy7tYXgcpQIfjaHSTZ/SJ0NEfPirEklewh8wOyxTQv1dd+LBkgPfnsZCX6TYzN4+rqAiYsBCbVfPaqq+rDQhWC9Cn6U8rP5ktNm8IHNoMlxPCaigmcJ+VOUqM3uZzU/p6a97kRc2F4ts4/RgtpXNoJe6raBHQjp7lTZFZYxDaPM8rAPNF6oz5Ik00j7hqTrRgVu59OTHxG+KG6eOJ/8V+bD/S6SYnYea3ZUHQ9ds4uNw3w5PTAHPUhz+JwlyUBTwqmrsU9OmA9PPCV8z0hivYkbMlGXjegiLb1niaX/Wr0WOULvr+OyEnmbxuSy0+JTHmGM3t3dn11yF1NGimFsNWrKj5OtCyk2eFnMJY7P2a6gWfFWugFOqf859TU3Cu59jpnUjZxPSv1xPGd1LbRzXkiOCWWwBTWx1aA5j9eVV/lOWYUlz0stKT881XSUBDh0s2FAFtU30kVTltZL/6uTj7Nszxhh/FrMoE6QfQOkNgOKXYQHg0Z2cYwLIDgIZwNbTf5qerNmDstVadpDdLLZzwNn4sdL8zTywYyfsAvZpty4Z7mfQFb/ZiYo2L62N6Xnyp2T8V/CGAg4Ib6x4msks2gp301XskAn5XeTiL64PsO/64C3NaYCFEeZ1iDZzB6gfwCbvrTt2XzKX1dorsvTLozQs4iepA8fEBOueolwX1nI1gSO82AyzptTOSrV2XJflsgUpBKlNkTtYYHKotEgwb5MHKr28dJjYvm77AFeV+oGUZBl9an9jtQJOfgby1IWNMVKVBejB1L2BbL4ILFIg0Xa6d3QFLYyetb8ceRTR/jGfDkngH7S+YlqliEYK3NUnNG0MEdUck6eZfIFgs6RviPT9QEUFy1iJA4CJIahCSHyV4Kae9+ieYcDgMrx/+JSf41y2scFNyYub+s3mdt1u04naYtY9QXZDdFCo+9qx6CyTBMY8+PDhYG/er68RjZMeyR6ri+P5bdOEsAfRizDv+67p+ZMXPFpxb8GcL4gVCH72re39fw205TGODM1asL1F/hYfBxVRxvtFG351pz6vBzv8TRHvE/NW4sWniCPLMc2MxsTrFPeTdvn46Il+jyv5hhRIPhF1+a2jc1M7CCcu+LXZ5DbT0V8OdvpdIjW2Q/akNbnwB/NN+5DgT4Kan2nzuMacXnlawE1SfERfXOFKlV59h/rX2n302pu8x76nkCd/FXxjCZ27vNUjMmcoLChxBmmztf0/D6SjVKRwdSWDFsPL68CO03zV12jD1cddxtfbjOz3RKuV/BBJeuvu4ZpXqyosaR+YXEjYWtTNtiGHNVr6trsj0/PMax5TYVU0g4uM5YK308HEwPVAfRX05W+i40+E+BVXkn2d0kUJN2q8s+LILa84Y3GJVqsqo6Hsm1xhn54G40amUyE2He321pFyNGPZuCRyGCQfot7Y+GefaqTTXWhxPw6/Iyyss6OCCTXWnBuWmduF+ugQVZb+OKV303ECofdjdZqqEGKREcu2d8UtufOoTHItAiaOjkkLLg4IkNymbsuE1mtiBi939VS/vv6prNmLPPYhPBletZiqmK46EcCzLiPHwhSm2Whx4TtpmjbWTmKwPrRGpflvhfGk7BWS/6J8bAv7nEwQVFPVcHelDN8Szad/o/oni/GY1MQN46F4y39Ae622G4I5SzeA9l+miIgSjfNov/+mzwiXstAxdV2s8t8YXdmnu/a9T5H2aiZGMSu1oJo1L+fBO1zqL40iLhgX/HIocSz2OyjsWc/UNCeEA7fWY9jLdUOVaT/nrgDaeH6xpsB8Vta3Zu9qnS87NO4mTkj11/Ro1f33hv9SdqnzAcp9vWnmu7JB7rITL2uFT8PkJk/oIoHleBjdL96aTlcnAis1xVBTLfP4vXFw9FUMaaMsUWw/WlgRqPHQkL1liBiEe9w1Ipw9c1nU+d+2HusQGmY/drD+49x8avyWD7DDmvwPYsrCIGN+yEHfsmRh/pZq7rt1KP61H+kjD0NsjSxuwZYA1ltA+Vrz8of0x+mrbuSFxhINR7d4bt9DRS7EgLrg/z2FUW411dW/NVvBx76oz4VrdvF9j0zzNxRrk6ivXg9lhdS41cXMEB6hFyM948AiVPM6DANNO382WKTxlYnL333oNoEYotnK9rUsxdLmEQ/CQrRzgbyAy2hDhCF86vUUqQ8B6dN/Aog/Ko/NO9VXOIPyo+annAN2utDKuNm9V2BzyMnhenBqy6Flb2aAyggKg2CKwpDy1+Vw4AFI3KnsYMeKBU4fuh2fGgEkLD3nDMMV/+JOI5V4N6lSxQurUT7EonOIyZ6hJgOr6xRq673Q3qAoQisp9b0tKuVZ1q0uIZuOiNtRTIy/3jOjh4hytCuZxfd8vyg6naHhkaU9Dk+h/lJYJpcsEiEgSp6s16Z/Xd1chclnGAltTQbXHv5TJ+AL4Xu8OsTaToIPbVrtnjjdSfH53CPTqGvKRfbmA1PQ+jrqob8IShAgC2mlAXrmuOeKznNzVCbggjvk/PyNMiPlDvSuild0/0seFN1yzAstRWsmGkdUC/x+lbg5W2md4LCE7DxEupdokrED18XCJmMqhr6RveaPZYdnLtDI+KqK834Ld2mAcY9ibUJgjSH9rvFREo35zDsNbiLe1FxdPh+y6GxQpKghyAcLFNprXX1EnJsuDSZaFk//ByKGv2RunOs+8thrqfok4fxctSfZBdlL9u8GZkZxs8XqqnaKJoReh/3a6LQ37l6VlfdaX3NY1JOXp5ss75Xl/lQLaR8T4d+uSJ/Y6Q/eq/K53cIxb8moQjvTq5aUeYxFXLvG1tDpvM5fcnPDjPcyH8HGjvGNmH+VJQYfZfPiqLl2fN0+lqdtlKiw8gt3/xIrdmRA13ghViR/mNnjoap/WDDqOQNT3TVE/QPzqzYUQEDkvqvstfAYcRj4H8aIaR53ZOqWpqJrfz8vrVNHzPfKXHP1wdIErr6mX1+pGF3m+qPri/L9lJoeG/1UYqm7izdcYZ1wAJ/8+SSrTINTW2/WtbO3YM/4wvrZfSlo8O42wjhGqq8hQZNJz4EmMP2Cyo7wLbvhB8e8f39dR13qZcfjQp1SM0rArroN2IEZQ7+q3hq5zvxuhXGl1reSa4OltHouqz8iibNuGXEztA448n/m0DztFOOgRiBvxvjy87AVmwehXI3G3swf+Mg9mNvKGT+jGB3hdUxcBMmLlRbXn5akmSXn6qE3obUVghD3MeLqmwN6o7Zp4TUOCUxidMSQSrJBE/ZdNbNRDrPyW2uWpw6V0pra2RIhZclO+JYNW3kuP5IWtLENoTz+dxYLLLe/comxNnMfdMGm3UmdxfK+4pgpZOZxiMN8BxeYpdNYHaC2PdyGdrQPuTWZXpYRjFxYrmQvSx843hjU6ERhTrE56J7RSMRej4QzIpfAyPLoXz9yDffN5VYoyYlrc2ak7kpoRt1IS7lm5oar0cs4q514MhbteqWhmGP1r96OE2Am1Iri4tvYj+sOhJQycY1UqwU8RyCSkTyr+Q0mAwvkyJ5bdrioLQF6RCRi57smMm1hAcUc+DjQt4HSGL8GTjBXaQw9z5tbY7ux8spW+x14fxUudLPYaz2ykpdQfK6WjwGMZsljWWWmckPQkfE79Wvp7ojes0ActsE/PelYugVVi0QC68vvfIhOGb55BLkxGkaP+7VZ+VtnIEb413iwVve2MqgiZGqjnLrmB39ArwVWSZ7wJcsDmYPBErXCyvdpoQJM94TvbfT2BNfI4p35RtLTBo9P2Y4ib/xQD4z1mqfhO4PQZG84qokkKy/ajBNMJuCLOkRhj/BvSLNcZVCD0LXKE2HisoyHOadEwmQzcoU6WgnuJw7VgY+j7sGLnmEl8wmICwu7VBmc18Y3drz1yHUK3o1iLXaL2zgAjPp1dYJhq1rfMhICp9iRIlTXj84dF6tdeO8vBPIxAb17+mGjYbAsRgSgVgzbmU8VCqXCNafof1d3p6x4e5drGC9AwC3Jv2SEFKLoUo5DRgFuBJEsymHtyQi4cg2hErL/gf8Kp3wedpkJkRMJYZ+AnXT7EqDiXKg5GLsgrkbrjI2PQ8j9iobRqFUZWb9INKSRuaHkO1Jv4kX0XnMBR4Dzhdkh5WKqTD0WehIroKvCqQPspGOe5uj9pGp9UUlk/RzhMtAYXm7iLsUaZz+ru+riLrGnvjoJwbumPDjqkM/CPr94s9PeT5UOYO953uIlpFm/8wgUHqdB6Z8VYkufBw2PkdmiSv9gphf7gUIEy9CL2CuZBlsEHSIpF+F2/XVVi6v4gBSIStM/GpxEYG+ZjfNcuk5taTjtp/8bhog0Bb+opr7flk3pEQkGTzokRb6ddYmmfaop7ntx/l2bBcylfuqm7b12x9zr47WZK9Nxu7EZU8rOtDyKsaX5PoLosvMyV3nD9f06nOA/WECgWfitmFw4Oops6wIE2c5m/y16gyuOwi1XWGfKlo5Vn8d4uJCzZd59ecLEInaDKblxYqbJM65eD3boV5Ih4CjGzqa2q8C9mj/0O3tppZHXgNyjSX5+HBDv66ubrwU0zGKENVxx8kY+uH6A9zD0jt6KwsDO56OsWuRfRlIUpkLH+6IuwCHosS53Dvrkzab50JzKVlm4xuKWmoJfcImhMrk+kIyImhccCEW5Bm9XGWTsNQ/OGZ5cHHF8Vc7/qtt0ZaK95+uqyKGYyJzUckzMxUO2ZASoovvK11LK4+fA9PbB3PHWxydessVutquvtXSp1A8yuGJr4MnHtDzbHE2z/DZaNbUoRHaXHU2/1py5N7Rx1X0WQI3VE5VU7U75Kq9tqQi7icfBAhCePwgu53CGpXfWCkzn0yP/lJCk6w7vnIJHay6sw08JGSOhfqeXwFZQbdhlpWAGzBOSPqco0xoeTAb95hG5fAcpVVYaO38uj40noCCWia4n/6rGuUtkHL1qnVot+OldJJP2WpISJYoq71/C/yihOLaKXAPLYDITaTvhBTtnJdG3zb6rRSCD+JmdBUb3wK+pNp1ktsIH673Ylbu3dTQg1aJGX+b1WuDAL9A+AiI0h4oqur7fq1U2NRLZMQIN1VTldTBj1pnRr3lk4bsGmwlk7wC0c88XDfyRTCrMQYxCjgZprxOiSn3GpJpRYVJV6psRz1p5a7KOSrbemowhrpiPoczx9PjEq77K2Fhq7dv54S4ImMqptXJ0OtrJa9ED06EOPe9tH/TBg0YZHnl7qK7oWttVxr37sDH5nOYAhWuVBbtTD/bA4yqyX4dkOQSkxssw3201in8dRXG7idth+zA1r5+9IfgP7aaHK+883wS52T747+OyMOl/p6d1WBNx3Q7KUh7Or9ThVcCWehMP0WRwPghg8GF7V/3Mxm3It6+aeiLGcLGWQzW1OvLah3Kr9HZxfv4+gpl+YOmwc7su37992aGybTFc5i28RzV39sQPzsZMUEoH6rYVIry3EzMBInG5lt9LsTHM8xHvFkHlliks8E4U/YRGcuwc5+Es+SQVVZg332v7mjFH3Dw5eqvo85q/mZzmCyGizsmgj7sX0dvs2OKMZy5R7E7PFDa1gIpdlVpcPN/LF3VlpzaFv2a+w4U+oi7O2+4u/P1FzpnjHRG0ql0UXsvmXPpJTeDPcVIdbfoiwvZhEi0IXrEKOWpJI/KpmhUuIAAvNGhBkAyFOU6oRjC+m77i7XXFrq0hXN9x6OxP9hfRD8ZIO0ERI9zsV+C441Y2Da/UmopewiaU2Rpl3ZKqbACwEOq0gS0zw07NU0czRcEgIrwMx3PbZIIa0plVruhL3Vexwentev5Yq6kqKft99q/xaXdtxyV+jWuIKw0Z36uh1Lo47uuZu9uh2v5l+/KNKqTOcKbMNfGewxHmq7hKLBJquPNVr/HFfNIT1iG4d1RWdsejdGZyPISY+/DMd9PDevLVE9lYqrDO9jdsouCGWiVIXvNSRGnHn4ySltU2YZT1yYAAxPqPv81v9QyPetk0VXeABCIL8KMM7wWvflW4+hJZkx4Z4iFPcmc9Leo6DOOAGrbjD8XJxwoJWLJEcIXXjvYnMHNX8WLfK7oLB4duvifTSq12oSyw3F2rll5XgwntrXToil2JefXxqEQ3nG8F4qN2wX0UwiDW+mBlkGSxsE1NUuXuIfw6f5IkDn2n8iBJ9/fFMv76PbtsKW8Z9LwvyGJBdc/vf3euPs3QwmUsY8i6CZOjeaOkNr1ubfvf/zC/UboTXa7Qe8DiWwr140qp1o0OFFGGXjaE2ZTGd7cHU57uJY8pZgneR0oYl1ozuLy1vMPWU3JErSx7mtYonhbNiOpTW/cVmlcd5xS/ASopcQ6F+FhTSR2Zym+LWqqGNMOmwUgGx/zc6hUaQpQgh9epOb5axr+AtDY37NyG/v9pZn9xr2cqjboXroAYH4Khv2Q3msC8FToSXFZQtRMNpVAzwkO0UtL79d2GcWrfNRefMxmrh2D4MXgIVugaaLjmOxwpRTw2APSdNpJN8qtkdNkRzWg48riIa7V2n8Q/rjnFyzbIliJtN9CVBZNKjIgqjCiVlywrBd3XiZPnr4SD9eX2/CTTtGocsySDw9rr6PApr8xfFm3llBlND/1pikDTulGugFqiy3MFrT4LOS0xClb2wrbANzzC+AT7fjlBzgfA6U1qq5VZjN/r2V8nSbQgaDVJlVOYaWSQA3rKyok01FvvpM33PBllXA8ps9qJaeAbV3sx/oYWlpRUvwgciDUWPLxNRJS48GgMpGAVmdcTm2SqKKzx6O2bVkdELbzfV2UQqP+1BZWDV8UorV5LVTDmUls5edM30DXfeTZVp3JJoCQl6CvxZGIvu1z9gF8NPolDH05m5Hc/aWg6g05y1E5yxonsW+oNWWJKjXCBh+8vOiCayOhX9f012ggLBzvSeVr/zHXiKw2va5st02S5J3doVhy+T5wgD1U7GoR+MmN+U98qNEoaPjv0TzeOgap7BVtaLZpV2O1NWDjM7m6A1v2kH4FApyvArKlChoeKRQIHOCLA2YxQ/7ahPdK7CD9pshT1fWrt/f+eZ70hmNTLH6L/tXKZkYfsiw1khGv4LOP3F73osa03b3PPQdufwjfpAbyz5Q0lDABR2NfT2XwSAv4hM63lvVjLOsALuuVQOv0lBemy1XcU48PuBRaHzcIP64FH+hfdGyY7ZQ6RXkt9mdIHtC0ECUIG+Ulwqzpcrn/uW+ZGF48CCdeiQCtHTnXl9dopC/v/dn4gG0pGIU1KtTx3PXIkzRbsfe7+nydwLUrJlna+0sUr/OrY6cG2DzDZhE0/RtXcNzK5+kovJz6iFG3vRaR6OVwPsJ/a/bQKmaJ3vLSwphdECEhw9G1w25QMhzLckzXKFeJGoyCrt2kwZIMDCSKaQ1LijKDMDZCffKdTPpAcGw4HMf2bSMMnOGAjLozF0ykYSeTyzbXMxFyZfWMwsq8VjsXpLU/Qg3GLRgZjCT9gRSvKfDyhH89xiXixa9iCMjfTLkipnSdfGEkisxoJLqpraSx49LkuCcSEdjHRv3NR8CAF+bCQlmAX1Uzd9Cn8MTxs7njp6OIy5HBNttQGceaMtEf7b9Ww7ufYbBAhmP/kjMz+UkAdEBJFmzgX+YX3+2zRIGVkeRTJXMJDtLmFbMPI5ADlvqkSKVSXThRujOTefK0yJJb8wPpv1lFn6lX74OX4mS7qLD8ngwnYXAvb8kNM77+YenHQDh9OxKrlp56kZXJQ/rrxglZMvMwXgozhm5AtdPYsK3ZqzfrztFVXh2zkrRPRyz1SS+bQ39sIksqQ70C5bw0R1dsgmzei3/B2GQ30Oq/SKPZ5AXf8yVCnPCrxjNeeTTjy7+zrFU1BVv9uLsv/zyclEA9ipGsH7j3uM/yWOyxAO8QSwuSKIErEf9kwUB05NR3BFiLrze4u3ADEJPPAZyn/jk0DpD5nAUMepYIYieNBGlTu72/FCPo1Ac7+dxf3Nozd/daAmtQycq1ewN64YhXdEHz17v445VKhOiX8h9Uj9zAwQQkgI9WuXX+tkHZ7ovBOdImQgbM9hfoMGPWmgBw+ERjvyieO59LF4dPta/+yjppphdJysUQmcr65T369OJECpZhkIFKFL0BWwReBfl5VVpJGcEEPPaL6mAUOj2kvt5mKkB437w3jGtaXnVUoMcow9wPvtCzTecjmq9H8VQjscQNCv4avIU2KXDZ2gcLt+6xxGR4FluxJNJLVuCi7DTr4q5UuQtFut9LN+IP10jsiv4Ijctfrn8nmlkw4sAeKVnrALwxDzV8Rry4PluxAsjX90OpRxjvmuKbuV7danfCKgOp+2gfSGzcW1xmv9KObgz820gOWjzrMsOLzGQ9xLE8bNSOP0CXmPA52l/50CIYBi0JlLbhx6f1+xLGm/P9PtKazFsxY4GeXTHyi+WwC/IiZZo7l/03iKTJkDGTKEZHPJmHrLF6VRUhdCfOFd+USC5+WarutnM/Equ3YoMtNHHlqNdzZQofv/R741sp+d0V0kBVs6RIrUO3G62H9vrmv1GlggZiG7WptCknM576OFjVeap/+Yr8o80ddmTs4awAvxm2euoDBO+NLo+j2D+7AGjjc48UcR7j5am8M+9iKlc8QISPSbDuyZHjw7ZTVeK61T75+LQDtjxJbjOASggQ1c96yGL6R/YSyvfYv3EtxuBuM+/b3UryWIcLF59c5+wVldIwPP5ycD1+3G8CBJWHs86fbD98dFhz/lQiCmq9M0G4yn25cHG8WqFtzUfYRfzX1Jwv91gqlhPV46vRNg5hQlW+yZXDz6FmOJa74/yWHTADGNb1iSL9F0IEEQfAgcoYmf8p/C5aS6VjfhfHKtCUjkBg0x2s7Y1GiJ/xicot65nKpb9yJG1GcQfbJ/+F8N+PxgwibyJiDd9z81fWFpXjqEwgg33xeVwpcYLZCiw8cVH9ECAah+O4h1BxdcFnqEwjOD7IABLgC+t8JNXbXGjSuMqBPOb78psmwmk876gwrNyrl12/yOnY9at5WWKLddbLfIRbNVfNNq9VZy4qr5Egb+WKNYHsBKq55KL5lx6c/hLBz8RbxAVwK90rIk7t6m14U1sOExqEs4rs61VJ8ceEaEFHjV2K1dIKicH/vZqVLZcDL0TjPaPcQTIyohVUqX/LyBE3DQ89quATUchrb1xFDnbepbL7ABRNTKYTY5XAs3JA+LSTr3JKnInU0v0z+jPY25SPTpkA/yYvc5sgZ/xfF1oQG6Msll8qMuYW5VMq6LNgeh6cQJCK5BFJG/yQkIpkMN/WPqyuW5XtFjWrod9wGAnA448bI1I2kJATKZye62JWJW77dtF+mL/HS/iPPGqE2/UxYfXhxiHeMMpMDyxfWlotfITom5UNaF1S7Svv/F5bPjLwDJDJzzK+MYH3SSAl8krNxj9XtauoncKOQ+0O4LoyRys7UPbVjBndMv0zeGFALafLwFkgJEvLEEuoC0noTjjOkMN5iDTN9goTX84nxDH7qskFliWPZHstFyCIGvju6efincbZHPGixaHmJ8wnJ/mN/m0Vobos3Rq/RD+0KXrD36IDsu52OvjFyVdO9LLdy+4+6uK7Ga9o06Ds/Gvf8tawJdltjCWgz9qU6IUxDCW1cmWtjS9CcL0OGw8iFmpu6EFq4kMKCD4Rynr9DFaN+Cz6WnhUmm3lMgrMh/KyluxzwEpTt2ZG2sGX0CV1Cjnq6yvk/IKCtLBzCEW6zPw3bL3Gl//9m9odwtFEpr0j8MGLyecD0CLPMaaOn2e5KlC3OYSor/uuW/cJVwfycrk9Nw/5iWL2hPR4MC6fKdGkf0FeUVfuSqSZI/QzFyVEhlMPELzeumoLvQUud6eukhtGikz6VyGy/jKgL0Zg6zNby8/3SEKHNvnr0gj6i3/8QWObomTK3VDmNewrfxrfY2c9S1Ll1fQWfPsz6pnJ7Kw7NV/jV+1PQUXzY9rX0BLtdkLq5fo18yH63nEPHL4zcnp+WCyTbTC5BGgkHAESf0Vzl0K9atH/JfMByhB61EnBUT4OJOHXIxiUjOn/SndQsgQ8rCx/iApCt7BKElYqJQlrlU2yTL07YmWbMrDTjR6L0utobqE4XkXkYNT2vvB1IlYHvm1R4Nqa5z/H9Uxb1lR/l/OXlplZaUKUG9G/KxxAaam8Dpjl03cfpQYcFh3nSzWIG9jO2BigJQPoxc9XigRr240eCJCQ6lsk+xuZ8hBML4rNqdjne5LWpXr+qN4rkatlaEpO0/IJ5jfDxU6b1g09GHb7+YzIPvZ9dfWKgjGN5Gv31O8MXZcnTvB9mnPqMNEPjlO7p7BuXV64WHXU6xPcZm2/BrrXxBSf5rI0T6UGy2uUP3iT0k2YhUr3F0F9sCPVHetDrjpR5Qqx/hlsS4taLA7iby4+J2ufnRiiOvd/DfqH5dQ8/cmaSc3vaffuIydQtffZZkMJ2ZInK+hO8YmTDBQEGaqW5+e5hUPGAcR3x0uy1Z1u97pDOKP1dHZChV3QYET1fGuoeF+jK/Z3MfYux/lGavrjX935JxMMhE+jxMMbkQUBjvvL42raEBourBShJuC1KrPIElCp4A3xF1gil5bWVR+9OoT/Co/Gw9UDEQTSzvZtnS73tLh6p1cdZpkwsJtkx+5vljLnDaSJzB4ezF6YsPukWj3nJ7Fb/iWQ1Ql+8Z6oc+d2fvy73DBUnTqKS7Hq8JICc2w5hzwrLqfaFMqFhKvWgfQf0Lhg9hXBJ8Yr9jYh8D4HRTtdQkJGgpS3k+vMG6ZgzqP86mQqqDfUvywD6jdoBt7f/uYoamdSx+10114l3/d8Eb4PLcsIa/d5a8558aDutz178vrfkgw879+j9IgL7tgMreP16iKVh8/dm52K/MJVpNiRSsiTEsUEDC8gtOBYCIA3Rdw54VPD30QzSoCG7+pRZSQEw7KYZy1+x2KDnNfJICDV+Ug4X/R8DDUeQiXv1QMZNcJfJ8ljVX/hsha7gLHNqQlt4IhTf425bqDW/8Uhu/kZXSQKf79fL94sT/1NmfVcx/S3ECTML4QhkpNvMWVbzL308MDa+EHNNW5nUnW4UZ1ajWvd+ChYtB0TxPDQdxZt5hkwkoViokdmdGP4/dhzd6mJazSCQ4L0gOMwcqRYZm474HJpDl6CkdWXZDGfFcJHflsFGKp71o8ERPlqzZgxawJ7Tl34S3lGa//MQm28ZIanoeEBMKQ3dQytS7ZtvqjfF/FaTyhbMIPlVAcmvm9KycPESBGg8r1n3TF5BbDCBgn9DVOIXmPdn4AsADPGXO7f5tn6CNIVHf4koKS9yZOchWmyH2vEjCZgth1rtBne15z07IQMymsIU+KrZS91LTizNT6lXZPybXq8ECJxRprT1iqNCnDxqqwEEOK/uUUUQR2tcYTnJNnq/GcIHT8OLJ8V+EoxROhofDl/VXQg+e4rtEg4YSs0MPfDJga2++w77AaGvy4zY5BHsie7CbbmSVh4c+9C90xUVw4ZBOEEE0263Hb8wk7DJnfYR7bRCnjv4BwQxXdC3CBQ5o67yinDWYKzL/NLuS8cFfIkLcbFH2z4wzgBh/wxynv8MGq3J1SFVLElOKRIt1sr8vfKPt9JEN0tp5tm+VGltQn60stIRhvzM26/CX1G7VkKKBwEf+njbmzV7zw6yutf1BqgB6/ILjj1H6DrSWEWpp+dHXgs+94eV9fPs8LpsSX0dbzOFXe9qPteUSoLZKGpO/1le0eW1kbhCVF7x8Wf7JJyDRnE4GnPxb3uHGugy8GiwaZB2h4AVu2xO1Z9iey3T/ptAmhSNxPgywh9D2QX7FvKwJkp08tLxnRITLmMSAyWCTUjaUpxG9CoROpxw8q6FB3TzzCQhGN+twr4u/YAxzOX9/SzAAt26yrqS2f99Lxq+m2q4Q7tWV22Knuc7rusz3h8uOlxf5L0WqpAPWHmD6nCw7UI8A/sLekVVmPcbxJrrFkqDNjUzNWPoMICWcrvOyCyzem3xWtERt4R8q0QbULRiEueWOGTFflTs+kzGQhJU3+TTRV1/zJxbKDrOW4JPO6rHkuS2+e+8/NB+hCd+eGrCbBUULwdQQ6jaaRYK/YY+NEALbceJCXNtmyp1MpA1wcSJLP7XsEGWGjDK0VVJ1iACJQBocEXgnCaVFk33oFipKfUL46kI4sBRikW6nlm2YriNhInfcloxcNxRucxzYsm3RqUld84IH9Pwl+31PX0LxMnafVsxL6vQAtPtHS3cLUBrg9BtkcQrv1dvMCn2JbaPoEsxNLRs4fRuYB19paCEEiS1k2i90mTxk3BAv4WoJy1KFPqdwJebWcWse2FzH6oNRq2NL1eY620EVM9obegpV2NmCS94LztenAlF2ApvSrrR0Wd+ROKXhtu21mAEBMad//SESk+MTBkNNAgE6qSOBLJCgzz9It8jU6onqDksTiAvzK8yxAYEjxpQVZ4OYjz+kv527dCdS4dSZABYUeAqDoBsquq1lpAkJOAklVo22zJjsR+h+QNbzdtCjdMkp66eNOKVqmoF+8tAnU1jIN4uulxRnERdh2ahPEjnp8aPYizziRKdTDiAKmuyI1YD9mO32XvDOOZsBE8ymE/+MLxhUhHk2ukphY+Uin1DtZS/Dz5pDxwQsyh6YfDiG+qG4WPlGPG9q1hP39p562l+dfJcl+RGPVq14CPQXTmSsqoE68KdDjTD1vbggrmqCDdroaMH0LRTHjWNdTcapc0VVh5nOwsR3p7SLXEnZumak3267+m+X8UgBOimHLa8jVPAmw1bezafBFmnOu7fD9Psg/xf3OmXzc+yPQ4xyHQPSH2xxlBkN/s4kPhIF1fhx58d9ggqqs1DsWAFKBW98qsC5hxOWgKY9VXxOt/xHgtJE5M66iTqYqJjro93GkppQncw1ZN+4GyudEKmqKr87IJH/xGj6QxjZHh71Mdn+8saxCy9QE+/vbRp38RcDdszDBdakKwPbS3Nx7f9HzsElAP9ssX2zBjRo60ZUcqfcijcj6vdwvcXKS4XyGSnpNAASE4mZq/JsjFLc7d3FzK2fbVnflWQdn8xLQv06pkKv4iIrWynIS/IiAz7wiF679iWGDhn9/DymTT3Yt9dzFEdH2GPB4M2mg/LHwcFJ8Rb6Ky5FSsgekwCnT1cEU7FWzFZEmJw3RIIx9Py2+AuoVS7dVHYl/QdngEwsRpJduaCdmGDX6LRASP812TeDBp1CogxLqtNcCMpoE0pdrtbwU6mxlt8C8UEumO33/P+eNN98shc6/fAJ40swC5DWvpAZZXg/eveu0Hgo5DD9iat0i6MCrEb01pnHQ8IfFz/g3zaPN+bElbbMcRE4Zjfvk/Kj6fGQdFquOiJdnbXPQTzlJGZaydBlVhmVwDaf6SYLNygumefPbDXn4/hgXNvkMU7NGGJ6T9nz/xsQfzho/zitBTDSRnhyaiK5HVp6DhUBXoiMIUd4qclhGoq5aDPvA7qQdCFWlZj+yvxZ3vjiYdSTQtvo8t2O7NDenHqhw9FJpEKBs7bmI2XcLttXXq2tk3ZxvWqLFsi+dfdO+DWcTXNPiVZ1DWJoKdLcpRd/WPP07H8QOnQge/GGEQUpYmbx23/m3njFyDH47gumo45jK5WQCPlX4bYznwJUILSn/1/pR9+ZbqTtpqSU3Ig5FQyi9jilOlXUC1pUg/HpdSFMICUo/4YaNV3vuBt0DfthAtpGSYGWCy5tNEHyvOzZpyEDWSIH8Yutd4qrc4hsejfLUFLALRQ3B+Ir2qtk0Ip1KkqT3SrFN9gyl0I0noyVoiEimM64T5Papd9VkuzqQ69m/z04WcGHltjtMSLxw8oSIwljKYQsWNUMeMQm6/ztZ3WeBQT98j6fk7sZ+/l8la20NStqRoGPw5AyxJWxSpHaiWfla1OZR/5ohEXqRVpn4yWtPmrZfEZ11mhdnizTnwWcsDbjSlIMPhx1mDN0YuoRtUVZE3ixhnym2Rb7Xh6rtb+/T1Dkaac5BD95KbapbTyVxvrXEFjm3VM8ch3tDG5gtMq6yZkkg3ZVP93Tjnby83fO8/fEKw3UQiPZRH96zjEreHkJIlADByMcIU26ZgwH4TO01+ZziF4NR0CMNjOsp4Xf/KKR+KGAb7QvtNx8agkPvlP+Py8ef6tj/poiKB4J9DIFHD0eu/Ge9RlzBEQiWN/oImvOTaD3qS4TQZO8PhGnhUAgsS8EJTSwmSJAk/y9g1r98jotkECehLa3UEQWEf9NfGGtLYEkW+IDh0RLMd8Pchz4x2ZyQgrnG0KV2Y+M7qr16yLRYFPzVAjKcJXbfCGN58uVZy/wS2Bam8l36tHZZUrzmtTM8ZIboI6juw83i/8GfNSdurpXqGKprfKzqKXRpENbEE0NTCPwIxnLUfu2WrF4AyLQ5EhBHEpfwgPp5dwL75ufCvev6SYda01shLZCVRwWhScZIrDdP/8jyWNBPEU2xqW7fP9NeSDyWJaLebdVSxx24OMJBf+EHP9t3mm6UQw0CIy4a1r63sp/Zs25Q3tyv/81jabzR+plV1wUvOpTbYypBsZWvFpR2jANpBm9+JG7n1GdDrELPBrlPxJxbzAzHvrYXFe2LanKD7DODEBYr+/jqxUHa1ckN6a+jlbqoBHwzoOdAvnHEyBLUlbJg1S7te8LLeMLvevAx/TkxmUuYfFppkTpXjSde+oC5zR6H/W4SU75aK78B1Mb4qy2gZKMNVEKr8sSMT4HfEwJcWfu2jWaG1C/B5Sqww+XTXaq53ekNv9n2Dw9sq/phe+vWMc4SYfm/SAwBZFV8o06XiUgRpUv5eM3VbGMTDWpcJ5XSTpohDncmyo6znN1Do94dyBsHstmarU7ex9Aibv7BQRs5DP2zAHb8+nIh9ZONZk67b903J/ZYV6c55WMbiJhpWg2vwo3N1nvcUaB94ey+owIMSvqTOsmYZfA/xThJLUXWembkeRvP/RTDJBHbJuv6K53Z5grFn31/lhbr0eLmaVQl8bOgHRpEwxkfttI2T4/6yzfsb95+dfz7xucs5+TaoUGqtDJX2e34QPUas9QfLltFCVdVuhVaiJZxXsZ2G2euvjgQMkBBgK5GTBG8HfRUI8q+aFS2RA7u2hCXr8/fyj5+sDn0/1FMijhJycvXPUr+o18Snlg60JrWhtUXf3F8Vy26Ji9cOf2WYz9l8s/cp0horrKqTMQbhro7H+an5jSUTXRP4sXudR6QfWyb0jdTC4a5ekoMKz1/c+Dawkb9+LgULd6pbuMnmf9a2QBEO/qvHTHBBfHI8V/22uL7Nl30qGqTf3oJA+9f+WhcdWUF/1mL0XwxMxyCQ+aFf4Jb8DZ3V5SOoDIsggXT7zar6q/2gXyv+1V1ghoDGSw7Hg8wwbS8+LjBDIr17/IS6+vbzqVJVZcHVG44PjN+ar4lHP+mLJDqB7WiyQZ3S1s0+n4iFwKNoZ76LNiyJh1Sx0IbGV+B6l9f12/lCKfFsDmKZJFCDUmfKYPp1WV3Rmb+vT6H4WyyBrs1JRy/a4SPpnJde4V9czAcMtB9/Z+2akcNpnjz6Z8NbPcYu1jaLgdqKqbiI0JkKTyyXu8l9L+YOtlv4MGrkXwII1ZcM+J6DfPy7Dibdx3reiwtNVuq5ziNmmCWzUlgHF4FWnPS8xAnqRRnW5WQRxTGDsi988iGvdDNPJPo1Z1dKDEUrGsYM2z9598BWKHDnKRShPuIVkUNrZrUvPhiNn5tQd2L82aHjvnR86z0rG0Rb8azHS2Jzik3eEqDcxaHM01FVi66YUoMraaMXRCfm7nuR+jdxVKJsZL8M7U+L24qFTpoD6ZmuxpEkO441xwluIu9WySHxOE11zb/B8rw2Ydbd1yCypqFxQObx0r4U2tFaln8i/K0pOX2Fgl6LN4Zp3iMTVau3oC2TI7kV6IsTVuU2Ij8Ohch/cfIL3V1PTgbDMIka70T2S1x1VTbIvDcycEPPIID/7Xz8mvuI44sfc9GOVZVdyfR3YHP9xbsY2FTKvzrN1m7rCZy+ouOzyfUPi1PoDpmMGZY8dzugNKq6bzzkSVn2GqDokMPfktsev0ddwG3tsxwEd1bUJ04iWuCK2iCKzsAeW+VZyqlGjN5OTZYoftID3+7aiHR+c5n3EqcuTXaB1ljD9Ns5iSAvnd03/19+q/HuCySetRnhjGD7gb7TeH/tg1wkpYpmnWptZ86mHb5oTod6S56ptJwEfoHGYIP/zaCR5rNtuqqaMg5d/G63HbttK96fNCdaEcL6uVZkWvonLgsFSZaSnTLzz5ZFsDXWYytWo7avULIiBXudF/TFeMav/iT3lm3pwpuG2smbRvhWN0CkDc8Zgm2JxgxAkAHBSbiMIXNizxKLy1UyIj7qNzHkb+m5IztjR2fVFLT3m31odXfsiHo/m0/kq1ItuJOXv5VYlLoc08ptI2a/+CwLccxsWTB/CnKnLHYXDMVNBtjbSp/eaf1IVkkQvvvSVa6+zhDaHoM19Zwxvw5y2IPSf6rT1F+o7U5sGDYjiCRflP1+ifLlq+i80B0X+PBVc3M/QK8JG6+gR7jB/QIJdOxhh4Cor6+uix/5JadMreEpKCuz1HHZ6G5e1uZHQqNEFxQKpbn/19VulvkVZr6LVN8I0a0TOpqoVPYMYoMRMVwqsBfEfvQa6aq/xgnxM6yJTPYu0VF8moshlidYU9hbzjfBjerm6iwFAPwNRQ1wMxJcKEgCTtVqb1oortRLmqxNdnYt7R+QWKZM8j6vHlCq0ZmJGlWZMEFmAFPGq4O2LTys64LUUb4ih8pkt6MV0IUzbR7lVcoAqK9060MKD0luyQ9OW+I5inprfUSoicYnT0KmqXH2T8BDNyJ9Oyk1p/xjS2opdDipNqATOS88XAcppUhFPNXEQxY5FHm16D9Y+rqTZUWqKxXSCHCI4mBOuCASb/VI3lpRkNiEPCArMt1gZFOwHwZ8u7kpXYzJkmag5L2ak2khJ9xJ6XV13TlWf2tCMDQrhfxxh7w37N8sssfHNaUYdH2plngUK8oEwuTuxbVlrvxcBZR+PRWz5tNM1w3tzUF6ODlC6sNTdWyxW2KAMAzAqAX7fWdGJFna3GowXgQC5FmpCUR6zyzISfSLLpV6XEvvn35ljLRmI9P+Avoyp7FAdn49n4grElcSAMqckVoDQKa4JaLoeNDYW8mmecbG9eNcQ+4FnZNl/FQ2AioteUR1oqsnbCQ25XzRQ9wcbBZQhuc7Dp/1606+KDjQAUVULq2ZKVFbFyq4Q1Qa26eEf0rBq+3dXV7b9jGv9R9dmLM8dM+XDgIuNQQQBlqmj1h8oNKT1DqY4QaMrT6XRDZnPnxqlRpyA2gBV1E4sjr/hH08qwceSw3sRE8ipDs6CcTxAOmD4tOUV9S1Y5I+Tdzrk7+KOIdW76sBTuQzxvMQ2WU3b6glR1PrMwyIIROPQUdeo0IsOPRG7f/Qhdm+8oHZeanLckC6AVSGsG0/cU2/MK90w8L7xgxy+SSE0p2V/Ql8M+96TQ3MW5rjcfuEAN6g8N9T53HvDFHEMB1o2X28glq3uPd0+hdCHwqv3WkVZkOqjYmOUXWkb71UnWcvXqi5xLKgY3dVGf0Gvn8813nRIodae36T7N6+ZCNWZwITjH1jeiZM2JX97JvZIViWy7Rh7Q/g0pYNoXwJjBMzbRz8ohENygwq3PELTfqZhlN3cceNGg5ok9J6J103Hj74kT/qy5g49kP56eFekdKjltP9O6WfQClRNQRWlUQFviorQZnfAGQqAZN7uh4cNAUTZCIRX54G+mI89yJeLz46TRvRNXHZXhwhMEmaz7L+7NrnAzAKys3hKGThp5V8Bu2jnhbDLDs5B7WUV89/A69QlLaHMwyhBMt8MKMDCld+lnIQ+Le9mrNek/2xvUaVQe7nELdjn341/NWSYfVCVPhPO8mvVY2/PUW3SpUce5oEPu2hsj83L4uPTqclXyQMz8hGAMNFlqyCgjUo60NYWodyrdCLXKMjgI4T955iQZIsbbvR8bvKDxpvcBPzv20kRNrE3fS5XaDcL3N1K9EmdAlFSryy+iykK7bCLiylRbxewvpk8Ua3udgPWdUNPIlp/aM/XRHXx/CH+7rpyK+aGT4iZ6aL6aLy8RTqkDXQX6HQCpP187YcAL0uWiU1xJs0Cw19SaSLmwKU1hMbUXYlY+jWVuWotoK8dUbuoXBhm1bB4SKaJ/QPD2w2PmyJuG9+cdfXc/ET3Vis1NPMaXFHCCBCuNIEtv4kP1pxxcnoC8s/h1BL9us3xr+1N8SO+y0ioMWlRBSNjdZNPgJO/BMhCS/pMzrj383rr9twGwaYcp5gvuBZ/iOZMtBXVYK3ib+HQGDsqGUCswa/dYYcPtNf0uuldba6gWH4O3TY/wnLo1SV9iKNCpBeiuUuAu6c+1YkDUhuW67PEY3UCQ5KvqgJFCA+a5bs88ze59Am9aaYjXqGzKe5Km2Uu06gpynyjCE5AVc2EHiZJqsNEsP/zeZ+mvZwGgts7/NsvnKDGfiXyf9+dZJtIPZ3PYV+EcjfXCio1OxgQsr0fo0YYfpRrbWjpE7a+JoFu0t8Fkk7kBaXmxL5KAuaKjPeL2EYi+UhyKj+rR4Vw7xeXz37XhSYqSHmJXlk+5lRMl8xHKaXxfRvBawP0mCzOtq4S7Sw3wColZ5A7Bk/+p8SH8JZ0yQZUsEvWX79hE8TMP03Qo8L1wtl7FUhJubLKRvSBcPqKvr9U4IAe97ToNLs8IXpr18WfJ/aekyYDKRzhZqUJmkSDeAXUf6Lvq1TUVT6fbvTqKYIWVvS9yYxgRAmCGMJ8GtFVZvjoScCWcxDlQTRU/g8bF/8rTYeYaio0mKr1tcZkCUH2UoWcC9fWHtGA1AuTj+RCgnq4qU+nZAvd/e37KAcVjrBTBxKQSnpgaTufv4Lk6mTfqTqx4xPUxgrqYX8NX7VJXE5JUK2qb6Ld43qTPLdVezxRzUtM5c8SKtYM8qJtJsIyY9b99vS0MtPVzePiX2BoswHVhwvRH6KqOocou6p8O5vGkeMg33ltRPBPTbHg6ISqx9qSj1Wa3IDmFhnWsavIs9pgNrCoSKX002epltdFDtIW1z1G6J/ydUve9FCRodEvTbg7wEgjKtPSmFiBjEUGDde+lG63z3OMl2LpHHsFy0YxwjeZNTkx1eFbq/kKVqmatQZ8N4+RhwuSX9TEBJFLyjgWQSTMlcEH+XDIKQL74QB/2M32tcpAlJ2O6rGUeDeLI+zRZemv9BR8zgeGLGhH1l+x3HK32JHzSr/misv2jB+QH78HpgvGoK0RwPlP5Uxa4kzxezniSa4g8UTTN4liW3qCa8fkD14M6t9zj8Ii5Lf6/Xsqh7u923EoVrKNBKZyQME5G6QdUNZj/yegH0/qCRHdSRhwcRW2K68Oe4VNOjxh0ltAprD71mBOtwzE7N/DLnPyhWaq+fF9fRxl4BfebPrlqFeil9LiuaL0FDqxaLio8wXxUpZydBlbdW+LZLJTwuqxMxIBq3mwoQYjixdZtX7obz5EQLaavBd1ndCQ3QHE3HE4w+S2qtuLiEYOW1bOsAWwU42ESuGIDu2CVsKL1ge4wq2z5luDLkDqND7UK/PNGTeBURq/Kop+e0/yJJDiX5q59QONafvT/L5YUom6/NpuguWPkPJ4AdM0tZzcOaTEtfeXdlR7MaWa4HSwDE2Cb5W0w+TMGEFfxWydcCrg9IkYMgvcWUaWDwwi+Psx3s0hWLR4RctD0r+nzGnAuREf+YsnEhYCvZk6d+joT/Pk2UX+TxisDVtGqcXkK0pUmAXWvsfZ6GUP+WIWhPmzbPkzc3CwiQOWymaK4J7zSMn+d/6zpKWiqQG+6H6YX7IMGYKfflzqWzwdmQp8m6wDfGrvamELMKWFJ1oAAjgmxiUzpAHjcdPS4zwBASl3eAOP0OUzG4YFvibRuEbyzzlCBDWUT3W4RxV9m2vYB6zKnA30YvvBhrJUeshX25yn81ttvVCx47tOI6lmmLAO3qAmXy0x1murfFN9MhyXShgfwbw5q3+qCHPMkvj2TQpe2wKnlYB/O1CGnDPKpwCgyURlmZzm8X1DsUg4lItoQf5EHikbkBjsPIrUOyX+J170eCsb8a/Z29GPVatV7Q+95LX30k32RGXlJnLkRRUnpEu/ovuvwjs07cfck0V7FTbNGI1PFPPtAA8zmgwqM3ycmTIJcwcw5gTKhaZBEm1Q3h66p+YFUl4WUSPP/Ly5dcAkPpvt2V8HOeKbZG/29Fc9IKsUp+mx8UjKAhBf+P7RLMRb58sR26xlg79YkkQK+TD4pfnHlB3kznoVXy1B+cO1spl0pRoivv2BWQ8YXvJlnkPsj3IllOdClUDbqWEQlSE2Gkx/NUnJvDscrPDVia4xohmQ6NFgmrQrujzZejwZEkNoiemnue6CELXQLROoHtidR5O6PA8GV8HeQpViR4ujwp1s+XBkChnQP0BmZWnrU53AGEGfGcOwmW2iX4OnuPzAD6+GWs6RA9HoxCBGz1Du12jSXQdpS70UgcSCztSZu3W1JbwD6GjcYRjEq5TWORzEbkDQMpzJOklIN6MASIAWq7r5/lTwk+PQsN5bkx1F/3j/ZQWBpP+4QmBRQmUhx/qlMZvxBPt0iQlw8O/Jrjh5yBlSCI3JqE/lzZ1keuV97CN4m/hTqmrM5YrQB8ORVBm8b/4K+cT+HlDZtkrtEaFjziJLfwZXmNIeaM4xHCyuOa/rYLhF74ZXOlbVwjjwMJ3bOjYCa5ohSHywt8CUNXttG0fEX2G0xzy+AvjfgdfEYiLp+acrlGmzAoLFChbfslklvVG0PO53fD9X1Zj+yVzVTiK7X3LcIpVlpJvxxe+1tRDAP5gxLfzoXk/lx5GH8rPr8EwMlxt8DxI5fMGO27U8vsv8Pi4XR5WRvaieBEEvFM/qTXrbcqW3AO1LelH/MG0eRxfIOWOAT+yJnkN6hhe2mN/P55jJ/fJkF8OulUTM/w0ve7iPB9Ee7kYLPvDxLOH2ZkAUa3LX/1sGDHbzkeR4D5d8wNo++NWQH/KsZQE4NoQVfd7+fDSfCpsxbgDQeW6HZ4lA9IEkABvVdIcCknnw6qzr7+sZft0J+zvUv6ai1+5Ty9cXn6wKmULOIFVgnfkSX//mBcMk/uJQNJpgrDV8rfxr0+fC2ZVSjCgocFV/AXMn9tz94qVQ+Qr4PxbKvEdbjUVm8IWJsc2o1PiPQn+RdAMo/6cCi6cN2VrFBvG5t/ID1t+P/+XH0rLDzThrz87U/yKg/qhi8KW/nbd68cD0DXhr3+TPEKOo5K+LWg9PbyX9ejreizTs+RdZqPjDneyqhz0/QXFREr//RIxAS9iMznOYqofiljzD1RHa3jlrXp6YKpgnFhNZGB+gelaSsfxsNhS1slSV1Am068yjLKdrX17lekaUO7GQHZzdx8/5LF/1tkGI22NGK5tBbOWYx3OpwTXyvqLIm43NPfVBjD1nSh0Iwedl5V6hDmDmhs49EqPAu42R8rhN+5SMVKDYOLgQ6NKbOWkCqzM0WWSjo6b0YFxoET9DszYFJJ+pvBC1m29hkPt38Bva6w6PPyl605B/ZZ/30rQL97lINfsZM+mAEdpJMZYaCoT/0tvU9nUBGthBKRIrWpB4LMQPyRMP0+gwJQMJAhOUHBTpkVeKrbe4zxbLF8c3BREynztnVrkzw9CphcHzddCasE1/8cI0PqvtP/PrYu02kjhnH8sAszPv8XuQZBmp/L7CXQwcObIgjjUqNKsvSx4OtHi3w+Zi0ptg/XyGDwJi5ljXsAWN5UaZ8AygwbggCGlg4tI0fW1+szfCu4TX1bDhZcifPnId6p1Ny7Gi19pEk7ZD0LoSvWBfWFdAeDs4Z/r/ejOhlcbxZfb5fTYujV7xv3nlJBvlhWXin0zCnXy1+ZFXlUAqFNdpOvfuh0pft9QQH5fI1PyGUVV5M2giqUiR1F8uGahMlyoUEDkpr11Zcz9NY3QcXjr+Ef7ng7rtnWm2wZMaZ0kjUQtQrbs/AAADsi+P4dhlALCWA7SqVN+W9V4nP/sabEDX+8b4KFxGQcJs3nOnGfxSUdS/hwZRcO2ZmpPW9ai6TgZ8bTqC5rTFP4TGD8lKR0bTmL95WdXCkVIUuLZC0ZtYDeDQS/f9j39g1sjyKGID0pFA0gESmqULozy984/IHZYU2e+uvhG/pw8Hpev2Vm1dnU9ofxJp2C9lh6WweOgQE5yWpPEOdB3iPaZbyw/zSLXmQIWqDDzd4hbVCt7GWKxubxzHZa+HX71F+mxPZDqtWqUr5dp1PHvp8iKVUOK/VMrytUVtsqiV8un52WbfCZEath8QvTJ+YBn1+NEpqVyu+qGlPF+NB2QGe2z3Vm/0A24qIlTLCghXMAq7hmirliAxRNbdFHc6X/DDRxY2JqDnzoIBhmPrbr+OD9NxR7ygT2paCs61cFj78CiSXKgsKBmtYf2sW748jg+dJ7EfDGemxvnkUTVjwhRB3mw60mUL94oi3O7f2/0W0azV1C4v/HiX1KZaxUhMon3lgss0eO2edaF7tFp8IYcrY5IHMey0jVS0OqajmFn+N1menXIKprvy9RFvXz0VViDwG2EDqSVg0lTpVpr+/YYcuPzFdL8ytNZ48DKb47yBhitimLeIjXKwPH0yRv+jLxxtMyU/jS1iUqKVo6s/Lfoi3rs/XyBCHjJlOj0KxGcJ7spDvTx1b8hsjVnu/11UTAQHTrkVZZZKdqLrfTx+S8I3Y5E0H+O7ruTVHElCyXg1YB799BDu4b3C3xhhhQAQmlHzYTo7d9Og+H6VLzZ+THRklm4TVWMUsxn2+iYRN5YVq73DUvxd0rTA/2sv8pODhr2VpfhlyvV0kb4G2LcIB39pUvnIkYOQ01xig/ObRGA8xUrj4fGqhLlqXhCpKtw1tA5/6U/AP1KURRCUCsT25AGzQu/9Uvf+SJacUxx5GDYieT3kZHc7d3vCCFCTKZ112/8DCltVkBAQS8JUf5VXEZWDrSjaKtS9Sk4KqAxjtRbEY4LAoyhTtFmWPYqgODvtZ/6JlaL5J8P6NIlR32ZaH0Yjx48oPwkT3VT2t4lBoqSWOmLlqojyQTYXGTG2RgR1H9RK2z7L/rUxBxmwA7Z3PDjahhG6M/zOLRmh7XUI/CpaAQKfyf03RcB/fkoa0lNWWOsV2ed58VeAoHgCgFMpxFG2PDcTBIuedi9FA1qpa+0AQAw8ILd743palFfanh8BhMX/sqyrsATnx9lfnZlQqmvwE3OZ8MmtMEzQmVwFqck/kIgk4xv3GDQshlu5M8686eX0/fwp5igYWg5BC+Md1isrLg/lvo2opm4muKr6dn66EkqQtdW4NtxywHx7l61+vjdXthIIbLlT4NJN4m+OhSXkwezcInMhnnyxJ0XgCeMQ0ad+zV8epqjkLap8FHTU1mtpOGyuYn159W/bFApPQmXsBE/l8APQ4mMu/XSL5PGHL6kcJdNNQhMtvRQ88tSJELf4qshpI4v7PrrOQPMv8FiCbpd0DpBiTPf7WuMzgwBF6xO0soNcfNjtAw1IDwDXP90kR1uXTOY+qoWxxNr0tzw4pcU2y9aY5Ky0p1xlDqRmuULHQkLJVc/o+kYXAXUS/Dw9J84/O3ZhPHstdoWgdB5dfYxH6fJcv1NoP5iCWEsTGiJFRp62o/1zbCdWeMnrliyHQo3RIw7tnFjJvj5/HsycnpRaBp4e9RWyvozLpHn19bFlD4qnab4uboNu7pZ0KQj1cGvEgiU3ZfRrlXBrKr+5y6vC2+u/TPpGJbhxYUSOP63/MetrZGPAg9o2UB6yKqob7FJEeZl60sG7aBVCX5oEtOX/UmZFxxl/Qt8Gxc2Q7A5S6XVC2OgOT6qGhIT0DTieSwH1uYTxXP9Sf70Uq5EfGRnSu6/Bcuokmak4rbzS77jxfobSaTmL8ro2R8q0kwWxlq/YS+/ysb22hpAkIBfcZPuixHANcHSh7NmsndL+5KPo+E9zNQDEMZfKc9nqDxm3URy/zVzVheBBydIYdWqf3rKENhSzhvKAf8IGCLxLpnzQ5M65knStAwvGU80Fh5yKlPKopJE+s2rUxqfr9mX+utvcerxmYsi+gH6bbE53F7aewTQguEKlPg+ygS4WpnwV1xAsR/R4EoqyZjwW67DadEaBv+kANNk+qdYGVlcbEzU8lZe4w8zjto8Nzf/U/QI/pjmBeTQ2ctECvjvraPdU7Px6pBhNaba+QTd+QuCBDew3yFesnTtXsq6lf/IL6VXXxZ5jk0FHtMAVJJUXtEZIsqjetUv813axkG+lpTwp/A45UzgJsWTT3gd6pfICHbAa/b/z9NVLDoKZcGvmT0uS9xdAuzQACG4f/1w83pm04t0XhKu1Kk6qsFVTucHXA2+5yG7JvGRyM7hY/SpWfQaTkqsW1KumZsjpgV+ZJ0d6CrtojRVdZbRIN9gph7ugc3569nTv8CJZ101uQw8GaKVKoYE1GraMjF1mUJR2EgptajrXsg0TADiRBOrhVt0q7R/i/dp+SyBc8BdQYH13V9qLtvDr6fQ13UAJUC6T0GjsvkI6UF/PpsFxaaiHn2YmhF/s5KJF8k/SPtoMw9CCaCw8gfdyr87PGRUAUKdlbtx0UxvOE8R5se0PkGIfNzdN8tKxDwbLsGESbYkOmtNJOyAv5COq0JeOfRCAV/+4lUWlv7c6yz5EOA1FyfmMc0LcdUF8A5bj5wlG6q0JhEenA8QPidtvwAnRHdr21AZRePcod45WjucDrd8jLYpQ6LkTwi4CJ+XZg3MAusjD8jQZyaXKR1d8wCx8y1qwObYlkl7OwzXXE358ZJD8D2CqMX6SnBxBR1Zis55BW03rWmsIbyRPFAvLnlioS8cKvbkNmDz+JnVNMw+vV4OVT/e4pRmG6VbamvPyaPzX3gIMLXdUkwI5ZITFkG/pPlYCd6ObbcpfoFBUo78YDs/G3gCODqR+A3CV+A68hASPbrLnn3HVwT4uKVs/kQJTKJ6XfNHFKqUbgQMy3gjwFqTnEiyRl/uFV6jCOE0eoCXObFLgztiPgbfXiJ4hTe7pSFmckGae+cPkDwm7myJBm2BZsN5Ff0UWCUhuSFOUZb4899laH/LUWtbMM5h+SjF77plt7vf0SFFwo81alEy4IdkHEmn+h2wrfNXp+yVD+LEEh6mOhDDUELSy5bSpb+fL/I9hqjOHCguzwr+JUmzX5lH00WTbrWf3Ol0PKLoy7IC3GzUyh4zvDHsXuMnXjMWyQ7rXUlerKJTBYWO5n6IXcFE59fC8JP2eZaCk4xPhY0KzWQP0RFziiOydeYckjM78gpCDwuBsM6oUi1LcKNjpIwcoEZCZ+v4qqS7Vy13GbWzod2SHuv1YlAFDyGCDMnmDXQb6M1L+0OAp96uGZLrLPkc0H6F8DjzuZqHiW2/QWLgAgXYSzvG+8CZa7+aF7lH2HRhsGEpH856oMSbs+3qPvQ7b8hyiL8H3DlKbXglO+MbPnaxRZFW+9JHioCm6BCtLyAIw+f9GExkLwERsfZKNQ2mojA+xfqSM9wPDxy3m0/cc1lGwJOELoiC22PfO/SRJv2LV2sL+dsB8Na44ghFHAooTvzO5s+WvCTTkFvl/oidTx5NREcKAePAmUeff6eVRXt04PmamHzo7awm0cKm+ahTwDxa/Bb4v49fHZKbbOtO8dtKWrktUoTkVsASCe/z0oz8OZ4HRpu3KaZf4O/xIBicDZmkfxULM1yC8zjP3s6rOwqtMte//DKiBkIlg0IaALgsHUuKt3uD3DO6+FQQ5vIzy/ikBx8wB6aFiiU2Yd1aNbiA493FK678wzxzINyx/OxGDwn2r770Un+Nazu+UUjrgLrT75ULgnl3iHkk/I6eJqlcaqZxpzLMvhbzXTDQGlmgVRZbllMM6ngquOV8o5dq4EuC+ZdwZOIom2NZrWGbCWsBEQf0yJmlb/fKeeUdtWPxZEGxpFUjkM0aR7KRyrJtsFVtK5LbANd00r87ADHpCwe9hz2BqVVlrvFnvze2O74jedfcF5VA03ER4b9//vsX9C10Vrzj18MdTGx+9ZuEjSCCTvLrS5YI77FcaeKDrGwvmHLZ4oza3gtPeqCxlcD6QpM6zBp8v0iU1lfDTE0KbnAtZu+7hAA9Z6fFSS3pVs7ZULgMqPnue0ic1exkn7QwhgnWfiwPm00dRTXjwSB2hPqXzSbZ8I+cS5IlZwwqYdW+82hGaVufBMrDW77UraSsf+MFZ/0ZqD38ZXdHqGxE1Q5eLDWO54OXM2HzLzLOx49groO5DXjyV/IM37YzzEbvk9Vo7tD5G4pyOOwfF2InMZpe5/GCzK1zOxtkBrHMr/3FPdNyjLCBOE3KpUCS2wtrmWzTND587Dfw7SupE1nX1kRvXSd3bUDuNkFEZxCer3poFLsrI1zjZmMARnFMo+Qg9Vegz9Tw0TpUxs9JzEOHA3uLUceOXQC2q0Gl4C8GqQZEaLECC7+hs3BSfRw1ksUPhhqI3xVRe9JjA20CwbC1mxeTaAZ8Up3J7EmnkekMKgBTvtzuCL9XVomv4It+vnygJpy6pBvVH89PI39tOk1uJPDHJm77ZpFgtNx2RFKell8T9DaV+OZacB3Tk0SQa6/2TZD6ZEwQiS+FqvQtaioZSIj5rvluVTAm1/VDzgiwim9DZ2OBGWTy3H+j0ijie785Buxi9iI+lsgLMdBGVmhjRuColknZ/hIIszgXV7htX1HubauvESGgLDCXWXTEIvLs4FJeygoOXH9VXZSJbT+jAKI4p0JgmAW53r+R9pbyilAUAd0xGAqqoVOCER2AVY+GpRXvImEQSP+1TqoVZokIruXWiDIuq3CuY6aFaS/CEKWoxBiL14f7tbxJ1v9ald3gnzYhaYr7145kRgrYS/LYAOSHdd5FWWhcOSxNqDfLmdz8TNdiQZne4mzT/bHNqJcJZdif88ev2xZ9IiIcg7B0/EmToD0crBjMkBiGyFb9BGqsFCejfojN39Qt5u9n4Cjwm9Y3WNRiiOiHMjFbsylY3rCmHFvxtzLf2SMdWwYgrZtQRY4mo+Zm+apRL59EmUPCje1RzLk9ei3+LNkZ+9P66Oc7gT+3ZcsK3KanojSea+4pcr1lbBvNzGUwwWnnR0VVjPuiV0FBXzn3VjwykctP0F3wKLpWv9tyEYqa0esvUf/WKRwprM8tzlp0Su8UFc7akI6YvPf3POB0xVN6+qyMCncaSZeBqTzSfffifpAlYCwLBzTqXLDces3o2xtwtjwJdK2Y/l1rD+sPqqOX+/eq5ZnoNqY0HLLsk9nb/ZX2tVZGrfmCY1Ta2ztGNF8edgxT2pAT4S4CprvfSJnVM7i7KnZ1nWBclI7XcGO0LN8jB+Wl2yY5cGYdiNTmnr5SmTAs/Tq8t6kt+7khWeCKo5cXClV2wf/6O4WoXBKRvaPhavK7PSbnUJ+yu6SEqwIzYtzwQ5Y589LcCjWczIVAgIX95AeVJFxw+YwTu1mdR12zJuuNDA/CEvi6nfReFky5WbTGoOoBeVt+O1ypQrOiKrocGeNmWDL3ZkHwE88BxuU5gkFmFXxvJj08tSAYLP5+Enxy8dncNfUxv5kH8nq2nSbrJm2k/Ze0FPW0U41lNmWtNO+bGAC/8Phz1QiH2Y6s2TuGOxgW9638YNCp0+tnmBCnMs8exKJbNedKL1YucDcNxeERaq4XdWuXBpA/8ZN/e74tERYFfjfh046cF7kq8gomoknAcuw8+LY8BXOWW7gUTHJ/Z+UgNfkP//Vvvqs1z8RopcNVAUNiWoPL+S6ral+RU94IOSIlnlmJW3wvA/fWtBdF8hZfiYrcCSo3eb9e2JO2smZc7TRT3W8Xo5rTlyojafLeCGQSpbp5BskRLJaCubGasc6KA+Q/OysWj8I4Exrnt14MxPQi2Hxs/xozVGzl4CkPla+PwEnPLTIVkOcFlI/tv2T2xjH9PCyESJ2AhF7jKUMsKmMBj/z8ffRPhRM8YlXsVJrOo/RJzdwukimL5yfWBlZcoPpMBgKtcuxLchN2Ib1hBtyAx28SS36Fq78JoGIBhumy57AUb4F5KzPhBTpqv0keiqlfm0HgXiV+Dtk7/16SarQI+lx6aRja/p6zNl7LY9hex3tJZavkuZ73tJqxWYJpQqt8OEctNM48pyXCzw8fXuOryDDZlY1Tuzj9eFsHjMoF9PL3DjATqhqpTU7yExKuwZkDMSK9YnsbxqFI9mdRiaQYOSeR0nAeOo1MhynC7cPUgEFiBaJKCT6ClsHj0nO2yLg4ScOsXKNu1Dh63a/DaPXjWGEcDMRmA+bNuQSQ2U6gj9ymg4Vpj2NuI8rvAKTOJ9EVpXuKEsrFMCe31OTsH0p/fXf8k4g0idPzeuAwJHucOyzwvVAWmUYf/JE14vf4g7JfvxFP+Cpx86G/wvcXu/Ri7J3O9iGz4iMKbHbK0yZHubwV9SV4V5RTnOeJYrltak4guuMEQ4ZrZqFoWRUwgodVtjjBSOuRKYk3w/mFCStoLysKVC/92qr0dyOM540OzhhnWSkPlG3SCPabM/obnVS1F0xTEprtulY+/AI4Ab77TCMd/jmk3p7fM7hpKKBrjQ6T0+f7tWBhf+ef96DBpAT8rRUZ8h7mrUNFt3NGLINiaM2LZ3rvVtMNKreJLesUZ0M2u2W/u5qy2g9W1afarkAIfrg/Y6c0fFMV/W5Ggfmzrc/FpCBVUqso/jZ0/QXc+mXvpo/2B9gwK4rJ9DeQpZ+g9fKKotFApyNxBbcrLT/VeT+ihOJc7lYeXstvD54AnwAYxwew6tvJiOd3R90FsUMq+oDaDsNsvLTktSqLzDFk2Yd5ocUtWkZlwGeHR90wcZsKbgU5olBE/ab7lSPtoLCQPbL83sT5/sJ+NeZvZ+n0pI0NdTBNrupiXJjfRHzlrep/zM1v54L4530t2emx1+1kLw8T5P5IaXzTFDqwG80bJ3ZMdnqHMG8EuKjBLmnC+3YsdXbxwPPjmazDc0IzjFRlEXd2ub3Yb+dkrXn6fMpRchfJwhoOYQRVeqM4DAbtKUcdk8kbMuR0l/gMpZh/+bKmBrPBV4DMYZj9nwns1U5RMdGqSv4fYy4AeVJqbXMN13W/+tYb1qNVQr+nzIuZOkzgGG+PzOWKBZqiCJYLlSgyhnSZw7OsTozqnbn3DfOQGt+jq33utEB2LeAlCaExJ3zFOLfBxSTa73jQ09On73GENS6Rp2oNOquo/Rd5fb7AGgCS+Rcm6eR2WLuhsQ2hZ5vPFxAw3EcP/47Algd1fwIMPTrn+zLAn3DCBrSAzxyMhVoPF0flmG568iB+Ucx2B4FoYLZ3aQrhqy/PGn8HBcCpg5XqqQxAjKS9fhEn7ACfNdJZd/tTAnPhA+Rk9f78rWv5Am5RQCrE8iBM34ehAEtgZ5oa27P4K2DgNAKW9OYEGAhUV3K93VBcJ0wIi1KA3d7SmKIHokvM82vsqzCicQt4tCi3JAgG0cL1sRA8/fUfo3Dh5Lm2fk69uWZk48OX/YY/7y6O0v2OPwmWS0d6AxgYWmshIY78+gosAYIo4ji7H5BTlA6tvzLgeHgMZ2IyI/gaBoTB3r84IDWQn+QVevtqoLh80bZl96+ZhQzfGKY4mzeidMxBNNPxOfAgjG5rbaDsPIfNj7YZXFYFrFw6WxaDSCwNi+556JoilvkSliS/VSHCkejdNoY56hAgSP687F5ZZD1wtDwWH2yQrPJhJFhBcOzwY0V+PSD7b794adMf1ZviiJ+fSvxNlT/EGiezOaJE+rVT7/YtWLLa5jSjch/IPKd6pjz94tD92F9fq6mszDL4H7M6mah6zJrFf4go8WjUIqusbdf+uVv6JICmeqKbGL6I1wRFYoaNvVawRgc4QDRJ9ZQGcLtiAGyJ22lTJmbbg6OL5D4mdGoaDqxa8vXuVt1JhsU9IQIRJpqe5YAsp4YdgF9p0BSbx5XLO16jm1K9S1xYrV6lKc01d7x4D7lyjqmwd9SjO502u44hejL1D4y+ufx8DuTW59r98x6P4kjM82sgxQ4mbr/1TFnauQI4qQ/TBb/yobMJOUPxIQju6DmAFNFgyffDW6tq9ISNjSfOvUhI+IDHTAKOeQH3NIxn32B18g1Lf33ITSFDt858+ajZ3vUpPsfXtO8WBZeCBCoeldOrf1WwQERlt0NGw71YiyWcA6cocPYFO7uwjlRatXVKin7IEWS7FVtob3CCLbi9W+qj+wN/rbzWxHtB/JAKaiCYc7E+6ZNf8ysgr3/hn2NM3VVi+wp3Jj1x+1lrIm1GOGRAFphxQMWBeMIkMytjuWYpcCRFLQ38RHTjPhSm9bW8fFeyViikmWTFZtMk+hmzqhXZCZea4/vP9ptqZSm3TajGm4a39yOMHTehPQGBhal+O9Zz0v4lOD7mhpnyx8IroS5OhbXiBhfhbi1cAQSzue8pNR3VRAWcBVAb4N7W8uEmz3mVof2DAHNU0bqrk2/LcQW7srjw1+ep6WON+dFz/tcJEBN5qcqTcyFWJVy+ch9LmVRXj374oyss/bH2YfcJd5t9wQxAzE/cy4r5FyLQv6ggNCabzunp6c64YDXWjjHn8gzd1uSDz8TrMWdHzOikY+h+HzmvN2wEavxdPtj28d9eSy+2efFiN7637ycmbB+jqmkCX/4pm5DhxwLx800vpPXhmqyQ0b8GrOWnMN7vlFX6iiBTbHEjyf/Md7DZPYL0OWtUonXwkrgJg/nc389zA77oaW+HkIfUvIJ1Y85f2qL3Y343uFAJm8QPmy5N7ly7XmFXI4hTS061LKMbNeD5M6mXLP8zNOwfGwaLRKMtqiri9sU9YUavysQIHZiL8br438QqoeqE52qDA15RYkbuFAS12o0l38KVR4mbf9PDf7MDmrSyTqJf5zAg4ET8ulDY2fX3oCohaa3aYGg3EPMQjnpjekut26RTHHvYeOey+0YDpU40s8I2Vd7tC2CcGM3kZ+Tf3rG677SA1alouoRhE5wfTguDTn1HSKJLoZlBOkCicj2jWHXKTSR+vc/Ue/H2/Bko2uLJQLzyPmM+B3sWv7ym49Wvcng4gsfNqN1v1SVUKjwyOO5/0apOmBRXSIge9d2l81PHDulAMAxl+JlL26xLpUPfmYGAd1H4qbgpcNG8Pa0tEwzYXEtbhC3sjVhumhtMQQ8ZVD0kmxjWd8XcTvoSMooZ1n+tE1z86mK62BUv6ZC//ZHyxfi76WKd72D9WScFdjchvorO3r8Jqv7JhA6e0K9DASZc8E6JWIXMm0/mPWbSgw83BOyemFkDppAZTamTT13w3ej5D87pYL6XMc5HMVvjmHoDTl+2iqmT1BqXY9YOtxFz0b6TRWnzP/9emEXoYUe7GGWUiv5eutFdZn6e0FwS3HSr0b/H2tAqjlxIHeThfI1ZIPFR+56Ae1rta8IiTLIbvh+V3VR4jhCp4vbeJRlUlFr/e8I0TjL0K/vQv++YcEdRXDJCiYCX9SW/07H9rDEu43WQtD6ZBkl1I40Yv0DDuV/IDt+Gpi952TeOF6dMSvrPjxE4yQ5vncUxWmq9k2hkbyZcoAF0br+0UsKaN6lEPNTP4nxZRE9VytvMezHYri8yocFkAAzpsRopQczFl9kGz+UBBjSuYWPs4ft75HxccXl5CtIBHfA1Imuoi0Be9zYgsbNFw5NDjpdV2FvzufPLcWduHxSQp+ibIAJqtBEUyYM6voHgnWASMb832Eq2Bbt9P3IG9tI9GEb5IU6fvl2s60F0oJcqs0ybLAoPUNvwrbI1lZDS6KUPW2CVLc07pH7DZvyqWHVAHg3WF5w4ILoI8wTisbqA4cB+2dw03WPyI1+97/H//Mbu+kg/VnkFqliGeMIdBvsW6Ko0T6e5Fyx0QwIHhOsUUMgw+Pdi6cw+DiaIkCJkOEoL+iLZF01TLnwIE+Kwyfe9ZLyoMla8PCwcId6KlDhwZ9u1TUIi7r1IrgNif5TeE6V3vV7OCPTz5Pydtv/8qiJp+tl4ESUcEWfeD9+1w+t9sO7n85afK3y5oz1R/E24U6ooQp2BAFRWjvhKQmZMClv0m9ozAfi5IDntA3YoHwZHVnBA+zu9zy2n6CASQeeMNXm9PhBEY9TG8Cgofww7BYNlAsyuQTIyLyfBLOFNgbWi3ekDtz+js99U6POR5XEGuZ9jZ1dHxa/F+ua+99p4TAPc1prpwMtzPFsH0e2mcpichWyGhyqQwJC13528KV91I1z8eT3QLzhj1ip9MAH79RMHoozlUcuWiAKl8pW+8WBEl1/WmkV4X4o0X7uLJiXnotaIY8RPDYJDkG6pEOIlw3HtxjV60JMGwhxGaxS6g70N1XL2q2FAzocgUYQa0yr3ykn7rOI6fmuWZgFIq5xSSpZtrNYPJoGhLFbJS/1RXzx8lgaOnpEL+l6zeLrvTieyyepjMHp9rhUYpX680uPYe2vlc6gueyOjy5Zreg95uC37q6DD5plSO3uv8a2785Oz3vFo1egWCExJounS3QVFK/CknniWwETIAETyrKL6jrQKNpdQPlHxlfFcKdI0oPLMOuLvsBYCAIlKuSGsjVBEuk8T4n3/F4F7DmMu91Qc83LVkGIyZZWHmrGIQLpZSfsHgmhiRoVZtB2eFodjACMaRIfz6zQ4HpDjwueA7KhBo4WmiZPbtfXSWs/Lo5V9LM0z21AzvUx7yHAgkeRJyxez74d8lfvPIw5C/lVthJMjJC+1ICHyDVqhicFJeue/DgMa8nAetPEc77kh7ZIGrX5NFQjamaXtqTnayq+/dwqi2JCBt7z0V/K1jBanGFW2HdbTwRYgOme09it0+1I2GBPGiZNQFK/DIY1qOIrLSWjf7cH7heLPM144/Xb0R9I6X3cXKBeazCXARCE5jH/pMpoFS/Ef0ojUlX3itmxHCdRJAKTB/JHeQE+Ixzh+6C70pL93biF6SPwbhkK1Jp7zeqNBoRVDEbFnkm1vG2p8/g+VWKD1tj/HO8t8u1GLZKsJouJmuuzO446jx1xLuEEzTgngNx2/VFe+DhXiIaIZsosJGTvp4fbX8cdEVi88H3sJONBJKdjudu5jeVKLa9/KNiw7VqAibjGKNnyjEOO89BtF7eYM5XS2VUUAx1zVpRPT1nUzxMH2LgvFnKe7gi/0Snz0XtRwO6hnGa2HNlzo6s+0SM8M0zbzEqH6Q3Q+P9510yf6zsPo13u5DB8R5SV5F+qzulxQeM143TBYwggNB9Ws4nnvRgBzvpxfr7K373jCd0hrxacU5+OUrRFjNJHpxnbp4foi4XEkiF+HBgrZajElJySB0dzqfbjN0BVHSRy6c6SyCAGmZUd2LAY7izCso0H4aElT2L6cTyq0xaeKZ+Z5Naw2frdRy6vCrmob9KTSMtte4iLEEXUYFB7j32zSLbnPxuQuYVGU2qoqalUpR78xPA6BffUVt92bk67F61fRPh/6U5AqCR7JeSWERJUmmBryfUyDGt/YErXM2uqsh6SRz/+rz3kjBiMXv2pi0RVaMlOJXKQLGNEa1/QPDDiuBpWiWrgofxMoMrI/wPWIC30ScwG4ZNjjNN/DekVvz8Ep0WDjX10ntbphFFWrHslVC0I4fAyahj1XqirRD0LITkKm7QCzTB2vt6HxNnl4fBWaQlruRkqjMj8no6/R5CdgWWFQMukhf8FzxDL5zvlHzHOLblSEN3TK3dPXEljmLkl9i8O1avpknA+1onCBtrb93CFOKEBxj1Vnh1QO+fk2vuEV2nQOXBI9Gr4fm13+tTMSxfjjWzBdIUSynzjJfQbHlYGgUsqIf29b2Xnb5mpZ9ZDbUDfhlxxdBAbRHWx/plrwmZIRMc/r096G7CZIbEmZLkK+By9EpmlsfHn0ZMxrrefPqpY21Dfh/wqbmVX93q+TGvULleaTwIT4O0oVT0Z6C23rWrdYz0sXeCsHf18Nyzm9hIcKMs/fpsbTlJ/mAa6N5COeRhZfZYLZMtb4YTpLL2QNxHK7tx0GQKnoImRG30rejUQOiwdZ/cX7XOVn0I51Nb62EaeVDuAU4egoQU+zffNnGB6WDddHB/AvOjx4BBvKKFt0M29/9bUSodvhAKxcLLGQoIxKQJpu2b9zKMtnR3oKIQ+TWkD8yUdX+JYkCOQCoH09G6fmY0vzlpdhP1ccpKCIMgWcpLRrlWwXwYvVrmYG4ybLjPHDETKehxHQTEq8LzP3RSQ3jL2U1WwFpqi35qX8o2xsujx3GwI4FQRhTGsaZXVe4HEKcIX27sc/Sd9b8FRqkxDmm7EDFk5Wj0B1HjJ8sKOo45l9p9zXOZKzpVvLxGuh3NnhlW59tp6aGwZnRcVC085KFLnCa58x8oavZgS0oy4ijDhHbD1teZ6X6lEr1juPdhe4WeQIywmReU13XZoSozBuCosyTsPpfcglHIy0nMAjlskfwTGzaESVW0HuVPG7isMlVBb5rBRtmzR0QGhEOBGY4rGgJKB8FOsdPyWHPrxTurqqLZN0cb53Q9DZp/e71JU1Cvnor4MBjBor+SXhKRsckSvAPYbl7JCjDcoKu6FOjKsAJSeBkusU1vbGhXccFtWsoeRxiw+SSKse9hZgU7NIpERa38c9AhDyeAH+CZx8hw7SvLDv5+C8ZZ9ycI4AA6hVp64d0fv82hYqwvxJkMIpoLqtlom/lCSBvl6TOilMf5sIkSnE+rV4tvaQLhp4A/XgMhPaBf6jdkgj263Dg8Mp6CgWKoy8wiQoZ69K5QqBn2PKkGO/wzeUGwJtebmmNebHN5cXIu2SiKlE3AeQUVvuc69ggUfFIvYeG+szSpWDQgSzRBSQkbZ9kjR5n1oGwUvNuy9moeg6zYd3y3VU8hsEhDqUPfufzSKy/eJPqSPR9zcI21K4/chCDCPgHBDoYetIW76prUng0bL6bU/s/FGB57/PeLYIRzpNICxoVr6/Xlj+HAP4V2gpZm6i5Z2yzKYsVOgIOtWCSlKWAXn4v+obrNb85dXi05rJjwWuDhtfe/ohl9+T48Z0f60ogZjtKYU1Qk6PYBa413ox2qTVk8Q6atgm2cC+humdCaGgbFrx5jbN1jspeNT2FbvuYh3I13u5aiXVtFtlEqH4V5BFzVHvkqayfY7L/i9lGrD3N076xRuK3d0MPAfdto22mVB4wPv5TwOXrR9pCu4Rot43SO755bDhkDyhHuG3DTwCaeV0kIbAdOYrh6li25ybNF8RVjvrmQtcDjb5wBzxv4AH+okC8uIN5lSxnrt2BZFmdBuyQRoIdYDN9jMMvKZQVe804kIag02h8yLsU+oVO0LK8Wse0zPjwY1C1DxKl5d/+a48DTbEBJXDto9eexT2KTRjkXJMRTLKaTXbJq0cLYPgq32jcTBMyNdI7QN+5OgGbDDBNN8+UcjdmGHAfkISHLDUkBn9TUdvGzjD2+zT5oX1uczCE+E0zxpPICwH2Fmfp/A3XG06I6ISsmwbS+dfA93n+kPXa1656he+TyULrzyS1Mwr22PhAEmY0nMRj1UYtCZFS7eocdkpUBXVIDCOhBXqyrgVNEkS6/1lChbyDSzmk91EOUxHbTYYE+a1UFmV1mMeGmS470n6qZ0l8hWfOxODQ1BM2bkhQ1qpPcHGNkEzLLT0MCga5jcCx06lyRULr0zj7Zbl+nVBcwxxtAJDVT/muDoRIHRGCYwBpeFTekA98OK8dlCfwqI4d1ka63ZSBiiyG3waDWpWHZprlMqeTTI8gaaU46FQQ82+l28bjB/W2TTCkSlLfjm4L2jOyL00nzEsXcchUpr6b8KEEaNJmzn8AhG8uyMyA43UbRl7JKzTWVgu6Jm/vT6/seNEy3dyFncFOjO5AJH+wC1esQI9pbKj3CIS4N8jLDAaV2cre35mnMofkkqW/cwxXE+4fO6HjgnJMSt1OJud9ANQGrcLtviAAyo2v8kyQOCcDrqiDv9ceOtMowjjEJLEZgNpQlKGQUC0eBvdl+OwVQ6zwGdobAqEYpY/GtC13aI014uIKmLIwpeZZ1miG01uDPG/Pkwv78EXJcfBIjvovvudGE77usXx8xA4je7KDUxyiVbH9Zz3R54zU72G7GX+WgWgF9oOUFKoxDonhBE/TPLHbdG75c2HCB5T98WyuscpsguH0Mcvx9v1IgOaJKNkLvu2F3hUQojetLSYW/VdBi3L2rf9Tbr2uB2HGZJYDfs8GtYPkXOf3OtqYSbSeTXPN0Xt8a/rdTADQ2D9SgzoRxxn7PZYqQuKq7P3T0Sn8OILHCOSTYWkS472WW7Gr1obGhzJukc6jFfk/QgIYBbbFGRivDowOE90vPL1ybNTcF8dz65cgIeMSunICb5Y/CCtC1Z6l3eCdh0OE4IYpG4JKxhnxoJC3ginbF2F2cGDWE5+C3vZ1RK+qQv15b/jIFmS4n98ESfGMmsu7WGIXsRHQb7kHYZ9chSLV9umvBP8ph67Cgg9QLj7fjPLlis4N9MEexPk62F1vZrIfYZdUe/fNqSLayodlnjUwB1eYKm43HZtsgBsFb8uHuLbUtcYUxaVe3jM54wMCAo/f+9ryCJP/vg2f5Do9nDJT2CGnDK373p/FD2JWN966NVq/91X4eE4U17G9tTr9PxxGJGrBPAt9qQCDwpLmjCkdNTiG10scWmzZ1TSgsUF2Er0c9LV7wQ8n+xUSem7PU06yBDfhuqHgeeVgxX9U78uxII7WB0nv+0hIxRkVrCsiPvRcz2RVkxcckjf8ReflJm4ct8NxNfoHhW8tlFhD8enyvhF6K/HyDujEhJv6cgsIsDsXzXymY4a834Qe3iDYBmrMMCBefXUTJo72iQPDAdjf5LUSGFyP0hdDg3v+gEBBaW1V+/hsye4+khwJmk9upB4hBfgpmJNTDe+iCALOGThQPSBtmJjfK5SnBVGHLAgPsuLHMY1/sLAsyaINoRli3nsOk7E2MlVgRi8H5zHIRH3oCKG5HwE1zVWo83aXu983i6DuWLLnSt8dSplweH5D+/hle+LPGd4XqEgH3vxgE22rct72Ssa8iveHsi2ZEu024GrLttuyBFlX/izLHCJE4YiGSNwIHw/Nl7188af2pyTDVXw8S7WiiH7G7euZOHtGLZHRF7bK8Eiv5nQnIZ/M8w7YaKrHKlsPBwm7vuyh1lNrwYeHtvJN8lSyljUsz6ZU1wFmADb5FUvPvCC9AvlCvGBBR1duN/kUf0erG64tsQf8mtxwx7qk0tKRQ7uNtGORU0sXNMc4AbuNMWQRasgdnUAgcKLpyC6gFboq0byH/f0LItRWN5LqBLg3zvtgxfckvS55wfT6uWcoS77iVCln5eDNk2If0nXecRqI6xcJCqEqxq4bTmD5pil+jUXqzO+Iy4dUOqrROmcRAk1LPsK6VAkf+0QaGFK+nZA0ym+amAkis8xN8GioNWXN7YWo6fNXYtoGvTsofCVCMBxzRv49cuFUFLwO3kI1A/9ldK+IFogp2G+KS01K0uebcFWUuDM+jqt/WuiTHfdMJt3X8zVAZsLYeEwjvHoC75X5i7Cnv0swf3s+YTozGku6bRpilXNfb85vFRVqm68sVIjzMHlXV5CMkw4j18LQwya1SvyMogR9ehuJ0+ifrI2iMFYd9YranEJ3/P5LRqDuLbEcs1IhLrbpltfarKeIDlUhkIx+tg7Uaqt/+kH2QTueor7rp+Lmme8T2MWcpwzoxyXEcj3bzxut87rtsjBRg3+drsM2ZdwZIIsU8R8kYPunIfD7hyL7K0FxcEryLXc3vUyMXwi1L/o+W4O4IN5xKgG6LTOIWWrIoM36GI9TnknOGJDaCEpdG7Vmq6OevnY2S5ByDX+MlKC8qf5js9QQ0qx0MmP64DhluL2ob8G1CyzLmHprIcf/fQoSzGkDop60jTB4rBRVOTOJ+f00un1lbCSZzdajBgQYMxJwJSKTxsMavDWuBleQBbiGOrVuc13iwfOtgnkthDXJKlQXpNGlGySgQrH52y/RKSIz25q4ujX0M4dHwgw4V/hlfHylzLuFHcrKnPO7Kz5ICje8ZSzxP/cio8WSC9Sprd9Z0E8ahaSY6vfRK2pTIjnnKIcXDoBgbNcx5sBEyXEh6SqgwrR4YFMtvPKXrfKy/cr1qGmMMiMA4k6IgwHLckcVs68n7dfBmqe9p1rnpDVXxmakC1XWSO/Q9P0QBoSd73w+q+slKV5dTefn0idLCMY5FXo4P4KN3CaViGlRjo8amuJrzu4DCASkhaKKRRhKAwGrDxCnKQVjqb2n8C98U37OaCxVxsHp31AdmXvCFPfmgUJGuha+SyDN175lIDMhN0/41BifmW14EY1WH+RYYU9+BPoXe5SSDAxUnDH+t193RZJbhW0emfPgvyYJUifNEgq/gxl9kt1aqbBe/t1D1Uuo2PMBo7eoMPaCShHirW0OipQRrzsnST3PjZvbbiMbsf7iURnSi4MuZIuZWFSFRzeHUHMCtB6bHGFPPUIjJHHfhim5s38BmrFkG3Tl7gZDlCFNgqKI1jr5sGZYK/SlLEqEJCBLisrgj/6vU38yiNUUEyidvD9Eb4tAfaYedvssDn5elGKYiAZOzRjyHF9acLvsHa+owIrXF1cCqM7YO7Av76l9q5c6fRYab01X6nOvfboBqgM4mcN/N3ta78R7vzq4gSH56fQpROCOdFx1CsnqmZJ6Bxw8dcIa8J1/8aWA0xHQOnlpl0DkIPFdD1ozzjRldkyBm3ZXcDb940LV7/R1rGhYhWf8Ovkv49GySjOSlnJcL8YSbSJ7j5/8+ILyfuMCBtmTZzQj0wQL1wp9jjqRuaTr9vr+EJ7KLMvrHTCFQFnxw0rVhfWN5ijyspy40HIImLpS60qiLuJFXF0dSazxf6UAnMwL3hCFPT5Ce2ak9ArQVm8snlcTafsb6EuGgRvrL8mtSLET83o/xJXL0IWYgPFrkwhMELDf4MzDLz4POdCfTPnN3WtBvDLWblsDP0GIe3KPFMiLsMZ8C1wkL0cu1DKxzFF15lfNRyaKiS9DcDMqt9026wjcKYSFOOxz3XJuK8X/LV7f9E1TT2o+KV7pbhvBvAPibN1GdBbINJf+9rjpXqZgL26RuujzYq/UhaukUReL31cDUw6UmyuBL9UsIBEHmk2iQjwY/Pj4WL82UOeycOYWlPvVIJaZkl62E+JrQuIFG3p28Uqc6NIdqI+6/Uzek0gWmHfGi+YS58PrpxzQeXcCKPQxBIgX/UUpvYAwgp78TWVIJ/fRfZJmt3uBqYKvvefdxO4wPlql3txa8ZfcO/OcL8ZzE8Rv4w5FuXfqJd6/WDjGeaovEdJAWjbfBrGDlwzL8fdm7exIyYbR1gFMActEOlLD5/yIZsM2p5TbEibkUAv+225Tqlh+ZvqBE4/5wY7bwNARONrOcomcyi0DCF0sT0ygYqj24iPon2q8cdZJNGXv2pDmn6YUdMHK1F6vXK2OSv7MCA0eqgdGmMxcLSRMLHlLjMoOR0Mimp5UrlOYRGgxbOYynnJr7YO7xFMoWp+wjy1/Bj64jOCWvUsADaxH5fp6NP1yePzIc735zO9IzwZqrwpAlY83f425hUgpIJGZX949joSDT8qbqNju+XxGaDvbPdv0n1t6gmF96dNo3BmHnvbPJTNFRijZbSrb6xnufpz0NvB8IWDvgPIGHtNkvZxlqNWfayVcaY5HFvwdJUsFHqEjeh+l9WlKIhENUUdXXr+wFcqbd8MpRCVtdlL2oOwgyKWZDJNcd5zaWxSAEUS59XFmen22iJYSDEgvCSPuz8X2DiiLCU50Q2ugzSqPgYZwK3aL3YvHbgDbVy6oKaEskwKkE0O5wvp9xrJ1vrmqNLuEHk7OL271bmQGN8Nx9L/F947UQUPrLUJ3/5YRvC013JU9NItKK9+1Gwj/aqzK7HO1aYnB8R+mxKMkdKmc569YbyKhJ5+Xk/FhkW2Sajv59ffB5DZ2Jyol5CfSfeZ1Sr8RbipTJZeDbFNhr0dRMXVSpGOGBqxRQnJ1kVRETGIzQrmkLJbR+fHGOyEojr8e+U0C3+WtUoEfNr2ox0/AtD33amojerjCHptDyUVJkOxZ3fBKLWzljei120hDVySc/uLT32zW4uglE9PqpGyeNSPoJqza4ZmQORWF5rWFsGPwgnkzQfCu32ObeMaX75J7+aQwcH5yKHYSWX4WddwMndTHD2hUWJHYvZuipWV6rLnM7ywJslHwVUEklkgIage10kdaotte0cXoUtXbAOYQE3GVPNELrhwcs0RiioyQeGpyPfJGy2gWZFV7NI8tDFihj0QrlepUi5e/NIs1ntKpb0B3OgccnJ8DgGpPEe1x1ysvtBmOzdboksIz9mzMwPshds006gsF74PBbO3HC6BjyO+e8lzawDnmNqJDuzy5AtCA0VuiinsCGFPAN/t4GOElwldkSNA46hQ7pwbrEVkQL5vhZzVfrurYX2hPOgccVs+3rH4unaOijF7aBLg2b1HKdATUBYNfphO2k2VvX++g+f0CHKoQAHSnn9Vt9P30xiu+gqd4Iwt3yHQl5vfbOKHIHIux1n5PAGA3J0pCBGqR185whx5JDpbYwdtzJasabtRucNpqfPzAy7WXIbY1Zyc/UUSS5TEQKj3TsrjDGEHWan1oRrr2iitNFfVZjnIeYtfLL7j+MvjERh7no80xhh+7EnfAS+4gMPUgMv4bblC+9j/GJhI5UwqMiHBw5HvFUefQ8rkuhL5avCYQB8GRgmNtU0K8Lby2bVzXrltDCROdMaNxnuBSCraI9p8NgfcN99A+SXTCZx6ZZN2DAQ1MVFdOLQVUyUfBABaU0M8ZH5Eh4W9ev1Oc0c0ijbXjGU6Hc3UscxPdU850Y8EzMmkehg/BTSL3hZVX9OEw255cNDCcb73YmRGRj/Gt2VVXbNi45qXBx1Vz3gFrczkd33NKpBsa6/jMyvQn+HBsW2C8qylJARUhrPrD19ZsD8owgoo35ufJK4gtpQmBGbPkNW2R9xaBhqriLauR1nBdeAU3qMsl+GIqerz/ab3i6osE71QkMOeOiH0DhjNFkrO/Q2C0DWe1ky1vLiPN2gLlhv5HkT89zkk4/2LEZOnAkZvJg/4RDqducMxVASKfEQKHqVtlpOAuetoasVhU3w6W0GW25tyMsB9bYReP1JlSfhavYKgKOJLzybFYL5sw5HnnrRB0i8kEieV3tBhf8fBVGPF5OPXMHLWgQfr1xFNaf0+pmcsziuP9A+Pfbk4kq/3JYmsrn7bYbTjgwmslw3T9Fwb+NeIXw7nYHlZFbmEmSuiTD+dWo4nIBCdDkIIXhifWmuEuIekni+z0b1vJkxOdNVfFOkzWkOsb+OxZd45iO1pl6vSx9T21ifj/MzRTZ7j58LHsbQ0RX1zC7o8V5JOQHRGYJwL0YZercnN8y3ejyUuwu5eq6T+XeuJxcf+9hzx3K4Vrni1Bcn19at3Ad57CH4yxdvXn5urHllHW0ndUsQ4SogFv3z1g6zL1hiGgfS1vrz23tQoWkrCkPdkYLc0j/qeg9NtZPIh9uJRpBt/sZ2EcC4l+cKNrtSsvX5NIf1fXreAh+YMUk5ELkyO5DaW8AFr/EXRFJYDC68TU9q5ZhA4D/cesXpAHntbT07S4Sj1M4E3yI8XGTJuhEk3vzsgvIknP+8xgdkx2DIXQjp0hoFghykcT7n15lRpsle7Bxn/YR6VEbsbWt6Trv1aYBk+39jlYrsrrzH+DCfuIt1jq9TSwTPf4uGCm900anD1Qw5jczicHG7c0EeXakO/+WbvSMyHyTVnt6Rs1PuW2ZysEPSQ1uITltYMtsnokUoYNFNo4Okdf6ow378fuWDMYKOZwClVwuL5bNEXjx2ICLJHyaLkVmXpcDnXX3VWLcj6jKVJ132XGad9h4HSYlPUg3c6/rQIDYyLC0W3iPPJypxPPd2v3JHlX/R9ahreiR6OyTelrE6p7YHsu1fxUuV2jjQac51f1RkFjfHW14QrC9VBv3EJ/nll2HBJdYVIrSK5oD/UdBmBU6MwilDh2wbPZfwuV5PxjwQS1S+QIK9fDU1qJfiOY/BNszRMrWsh8S2C+c1EutSt1WSvj53axFbROROq2UP46A0XEC5CUj9UQHlEQdtXliy3fH7K0s2Rj89mF4nR+MMUh69foYUbs5hdr5g9f7G7Luo7R+QPdItY/HnOSbxgSMZDr3CVwoGEusOSp6XXL7/dY/o3HcLQ8zIOXRqfpTmg/JwKc7Pt/aTzDe8wVChrIVLouK5lOyQ4W45TZ1MJnd28QxyzeeKRyfGA0x7tHSr0ZQCgnuoCMnx/hZrj5AuIzp0Z14fEiXLMrRtZTAHdqmtcFn16T9oA9A7IIM2PKfFV0qh75G5c2nMH7MVNoDWeSH1cR2getnd933NJBv6ZZMWbuiDbaa3k5fztSwWkBvuIUsgQ6WaHO7GnfpPucrslsu2EWy/QAVMyTx6dhzyhNB9eVng/ZpTeD2PskBT1euYoMtHhWhIlW8pcyLZl/dMRQjEgXy+vy6AjVONaBMsnPCIAu7kwlc3g1JWVnV5LllJnbExLt0mizQc0cg0HU83Fxu/Pyn/s45LSG6vZJgd8XN/RyWiGNaxGRxE2GxshwJrqXVO21nKJStZ20l2/1lvHqP7oAy2Oc2UySPKE7q9neHG/55NWYSW8o3JpaDVqQOFgXygxhO1+Zf3idYnVH9Sv4HEnMupXSfIb3WGmA+lFwPx8ej49PaaI4LAsGCcMnbB/ZXIr+8eqs9M1pQvbR+fFpJLqvgUM9QRU/uTa1dYG01bCq0wO7AV6kLOk/ZgE4GVj2W/ZWsh8TY0zvlQuk6qOig7qw2l4bva204DNgWj1X4MAHVz1s3oA8bGR3Ad0XRT89ApcueteVJFlgLpLtIXj7OvR5iNq7TpSs/FXNUrGeSNS068Iv8XgU+vzE4LEG6I4E/WxE+UJjdMi4T3jojwXyq/BYpGjDTuxjWmqOSejaxXgW6P6/csbU4EYjO5ZNcoN/gFJN2FiZBK97QCqqztGvgnwMSwOad4u/hIKTgUR52R54AfbtNnPfQUhCphsB0lkwCCORwa/O7YKnnsj8tgwLbE5B4sr3obtvmFpN76WywYvUstPkAHNMo9RfX/Jon55nJp9UiyrCEg1MkbF3nH1yWkj2QVxkfdZvVAHOuyq302NxTUwyF7+MBpdjbNNQCSxuwwnGGLEikPkloXRf7QTjFEGDQ3Z0b56ovGnX9xT4YIgUBbI9buI3c7V+XlK56XqEZeQ0tm7P/bJHE3CgIcNYYkRh+926TvxX5qua1tWYwf+Ejk8knMOA7yR0wBDDl9/6X18H+xlH3sxdKMuVUlqad4ER9KodtLb3TmaO6+l3z03A/Gzf0taOwtNipAh/7HvhhCcsPDRleSfT6nf+zMN2kuKpXmcQ4bhX3VOzFRMIzx0DkJq+thtpKU1wRlZSLSzkOg5tlRlVseUeVAzxz57U0ud5blVgtySvZEAfuTxL54qgsgevV/Z98+4Ymrp4/brEegXnmIOSa/oNhgqkhHuZRPdqi8/BFhBV+wT4U/pANnHXAlof3ftKASPIX1Xb6rK4+7Omdk+HOw5psz3VfyguyeLKiLbIKJOKPVVlILrI3E+EAwpOUVMyXR8jJlkCArTqNCTIXQ29Oun1Y9j2iiLlPqX+C2vPP6QeQT+gphMEyR5dRkxKy73P5Rrz4v0z1zXi2SNkCiOgjImGvpWGDiLrFq6VlW0vgTWqlsomRXpwaZYK0P+XU9Q0VSiisO/p44RZF+42r+IZ0Xvp/CJohiJEb2uHZc9KRiy41ULMq2dmfkIOwWk1JAY9frfz5q/LAFVz1KbYdvtDe3hKvE5/rreFuyPIvGO+UTsd0heLNXoLd+fP/l7uiuKZMS7XES0lSDQlV81Eg2XEjD3YLW6VjaPzuwenaD1Fcsgv6T81Pn3qSadBH7AJsC1VHG+0CIcYVqK4s8rX0Z1dv9m1uViyfeQ4M27XJPjuBB64YegfOqzuDbrGP5ldh98RZKfLSxU0939LXX2yEouj1DsapTyVpAV+tm2bzMzWox+C9j+qvuhX/yhKHQACO14uck1rUfl3DLq3JntgHX/qXuzE7AKUDwZN2mUBjyxyZ+Vo4xpD9bL2sQoqFiGcSRk1wPm70qs5pV/I0AcHvQb77iLXu/ANKf7gMbXCOMG+ii17K4HYl63myixVJ+7Ms+tpkhs5Jk2t7PNDWjDSK77Pn8Y6Vt30SkVB5bh4qtUOoWDZGLntvs6+ld6UOvNjw9wABgzNGij6i6CyMeYaDDbf5PPuRXI6iuAPVEuYeuomxcKmtZ7UpTuq+M9uUujGt2y0CLN35MxjLdh6YHDLLZw13zOv+anal7cD1ue8faPrEoEwv+GykOm+OqWtizSo8I+lU1eJdn91u2D0cqZlFaqypThjvSlMXyy2RPwVPEvJgp7MQhTVJe+kg7yJXT7hsqtfBwH4na26PLDARmgXKSjkxD2/2oXCI9Vhb7IIvycHayUDoeiutTUH0IG2f7g0auHWRlWeKkwlTAfEJRE+Gbn6EpWPuZfQ39O9Sn9r8iQ6FaxmypXmrSd/+Ma5tFKCSKaLNdxfxPegd+h1yv8pERdc+BxZJAdUMR7ltcXPv0q4l+/Ic31+bvcmuvunBBcOreBvaVBfJZM0CoFwpwNjExj5rVYFaAmZvH1yw6OU0yQCESsVV6a2wkILUpCKXyWMqi81tr8zi4Ei5wLsYrY4gGcfEB8docODHumZmxrudwL+sC+cIzGa9n0tPfXxY7FGp0+KuXcHns7egdIlheXc+/4uyDn/VKn9Q8a4jSfkhEZyOlXe0j+JwexM0q6J8gGewUbf7dTu7xjJMkI/xyk/cRHkCRnYQpJalwFG5J3013DmoBAu1hW/3W4OwN13vDLoCOn5P46aDIqbMbY+RQ58qehbopg3SRx1n3ivE4F9Rf0zL0qb7UjLncd8a9V2zgGItq9hCGJCv/JKLNEpPuTNfu3OqASVGiy3UUdLi67fO8/2LOjaOP/d6lUMFHs4Eo9bmx5M2da/QvMux9D7gx33Wi5eixVmbBcLxHUVvgSUQ0O5NUCVq/wjcW2FNgdZigIBbckl/opjC5pKObmkUlWly2I7h+DgeWLBj5NXQjD6XH7JZk5XnTQadpnY0WX7lPsFbCK/dd8q5VAgFn0URbOBn+k+dANqwr37E8Fknuk/jQvt/CD1aBgvI4lmCTpMwVcb42HdM6Jdc+XBuRExO3qkc9dJ9PfuBBUDCfpFQdTe3c2b2TjdCeKG3w4NUmOo7ut+gevPc8QLnPVcRujk3jxiI9/zgz9tfr5GjuwD/WvL4D8CNf6Qe0la0AEpLzowkDjeFcbAl8gOQgr9PlWxfPcrIkRO33E27KyIRMMhcNR56Z5ajuP+I28B4YmTUwE32Q1MRStU2lYZBBdmtOPT4HQejKwCygWEvNVcdSHh8f54D21ACD46gSwXan7zQOroJm/AZslgjl+SBPWAo4N/UUWs4yRggYS+O9iq3/nZFt0Lxg09TKBBhob7yiZdb6+cP8XgKc+Hy7Z0uPxg9Y4C9AFVATFLGI5vhLQrzsjvrbSQkm6+TsYMHt9lKHHtB8PRUN0cr6+uKQ1V8beXrVoyUMr/OgtllT9d5omkJa8uneQOz98KkwsKyOjW5cQyc3cIvzSqgrJZaSRPekAsbmx3PnijfTv/CSgwn2AG6qxvA9M+6O/7gUXeg7BdN+TUpxrr35jAGtwK0jM0txje2p2AEfw13oieOz0dlpAY0C2/avJNI4vHXzZEJJZ6+HUVfwJLU9lhPF3Vh40r7jK3zpSQaWeMK5aJskD2rhvujVB2B7VKqu4EU3O0bLYybj25qxIwMMQpSMILy+TE8j6C9UXn9MfYUFahl1IAmECYLY0QgLf9Or7eF7zBinKUqG1G1Jo3o12LzPdwZc2ODcckQlekyG4BFE/H9ZnShvVF+LmZ3rUqVokcBt9LDEJ6o8ta8+8BZpwhyN6Pm05QhjaYsOMjUEYv/59hde/dnJHbpY6sscg9wLV8rqxAcBI3W9A/Q3HCFDwV+4DGyMwnN/W0ApJWSd5sGVbS8YXLlq3INSwBfb5/jHi1c3gQws7o53zN7m4wn5m4tnU0rV66F9GXnMgnP1yLb+kKn36VCSrv+vKOBWxbSaYO2t4iRGgT9neOD/zu0c/C293gxE8VjeOg/6G3aTURpSIrun1OusD4OJWMZTJLP8bFgOYt+zTdvdAhHGcpX12blsxyUu3xygx/Y8df7yjisC6uy96Zr8xGVtM2zGq+5qwEa8HKmXYUxDIa7P5ENrhoEwTi9p0A7DzdXBSO9GQUjSQjNljb16eH1EVOCCSH0Kai/9rKQFVR/1Z7CyDUTh52QF5kZYJSg7BzUAxEEO33d4vXov4/hSsyBiuoFM9oL4R1BFy95enHuVEK8ioH1eAC+dmf6gIV0NuR46Pr/BI8gI4M6lmFrvFpkByaXoCLBYo5Lo1Oo8qsigjlQXI+aANd71UjkJzLh+ACgL77GSIU5adw2GZgTuk0NCZMVaE+zva7B8Y2AzCubrFi4cW41UORIaAH6xSm/DuhKLLfggag/v85jf2juX3jQx4CILcGus70KptVZI9U2bht4164CudStzlIIIqC9JyDpQzC3f6ccga1kdZDvlyr9s85DhnYvBQfSB/GzBS1rJXs4yzIpF8Jt8RdCiEI+ZnRY+zThdU6IWWbPaR1iziy8MSkHLmDcCrSdlPyrz+f49pb4xlWWp+prLQVU/gBiuKpe5aROeiJDcKXBjyH4/iU3NH1Vw07qolmTBxGRX0fz9lpdL2OftkMtltMWrKI9iO/a8CdiesaaLZoMWX70uunggQOj5eIiBAhxcxGYGJ0b/z3icSw8/sbxAOGEiNshMHL9Cblo/BvDLxNhRyik9hYGL3Y7v20Sl/jgf6izbxJF7Kif8qJSbQIJ/EbmGDWFMvr9LpAckABThUAp2vd4RH1klYt6j4gESawPVn6/Pzctd/SKwg7MiiJ2N33kdVOkhiRgAAO0NNag8Qfeb5nGWIzYPvFA7cqQiosRGXHhxtiO0j/OhihVYd4aK4+UEGnjsia7XXxgqIv4rZv8ZwNRSc/a3N2t7H0VjCf/jLMtVH/+0vk3R7pVa3Njb6mWj3DMFg2ohpXSMmHsuoCQsJU2nN74tKaIWCA57DmfKZUtVhbl6OpVMQ/BdcE6Zs1lF42t/VvvuSJNRux9BiOTRmy2c1FrGrobCKQegk2+b1MAVADrYN4IxvhEvisHKpBDod5mIHNQ7sAcvBRKKskD2m9mc8uu/1HZF/7utVEGnGDv4kfVNwSJmCHDnPiFgnAPECkFfMhluyKGvWlh2KZPkqub/BDHv87XP9b7QYNS1EXR58zcgvb0KD2qlV/7P9dVLBwcnLmEQXUYKAIVqk5fivpY9PByfpUfWnJpiP6zgEyXt5fnufxDzdARtkx4I66m9qFvQNq9CuNY0vUMKwpvIeSZL8k/wgBkwoExwaKJB5v9rmS+u4+x9wVD/iqlmC8UK1iGy+8HqzbxgkPnPS7q4V5s1vxxWvEIKZ9RoEUB8EbHijMdoqj0n31m6qGRvyM5RM+g5dKft4ehwICwe/RIQEagGH+AvWDwUQDiYRvWfKztOIKtyWQFEEkna5LyA2hwJyJApQan6XCrWkVBVjtJvBBRj+uRGxX2SkVWOltbzCDqcw3JgzTfmfmpV7Ws8OglSIpss5+ZX+moU8cetGc9WTnOR+C2GibwT0bRI3sIQ9z9Y7/iqYHq4Ph5iO0DaXZLWgdQout/NObPX74rG2MDmc7EVHARpRuFYUDI+FeCrbS2sFRazMtasMTVFlM2SORYt2uWEKT5O9qC2pCfhX/CF2d65Im2+J85FbbcrXYLz+5mDmzHc/SgMkZC0CkNf79Uk/WzQlb2bT2p82Sv5Su0pNvxS1K6cl2iU9p0UtjYCriKCwHppckIAwq99vPpavgRRAoJh8/Jl1ttH8+JD0iIQkbr7uI5hvophlD/Ilyfr9h5PGKwXstMTk0IMTWdf7VZ8JbqX7R5PNgOfsd+MO6PKej46aui7GmexsEI4iHrgupTZtFsEtTtU3tMSltTixL8khfEOJLdk+hPTA+9BTluTi2FbHQcgqWg60sbao43BWNAJutG3krUgeWd8RyfuBwszMJD+vvrfCcBnImu24bgiLMJXiZ9tgTL4hSmMVTEuua4BO7n2jFUlmmwQOF9dtktoAB/4bM/t1FEaTzpKr47zcY7JbDhBP644Fw3UAnyzFO2F5XS/mgHlaokLG+o+CuWnmoF+Pz+KPdpO2MborTAgjq9ADj7H6kxNU8hqAu8qyrrjg54JOAcFzd5TcDc+Kqh5WrLXhHWaTIKRL3pcQzOO2pOCL1ROF0Ayhl2eusuv0LTfnT7lvNHIP/NRJKHHM1HFMxJT+htIkcVphzxwbnqvpl0OCHy67FNBmOkFhfvmhwJOAcpSgtxDO08aoo+m0rYozRuxqGmWQvaHaUV0RGQTLgr8y1Lm67CwBn1EEey2EU7T3xypOMr2QP+uIDHshMf8VajQTQHh72qXuuAf7eRDHy9B4sm16FkE6+y97dCuc6YfWwD2D41azOd38q1GlOflrHmLPP9DTTKbGh2r65JXAIOnna+L96rrSDoDZCuQQJ58bsCIavCb9ORq5c1b9U7yvdTwHmaLxbov1C6YbN2mM/1TrBYpXAVerKWBKJrhkQWQHrJ79ea3kCHQbz17kTn41tJIS+dxlGCOnCMWU9np5fSftZdHEaQB9OuPjyp+mEGfzIz/batO/n/76MTqIRMWntARnQlCPx0a8ZH121/hAO+3IE7Xd93D8TJLgqEPBAt5cs/Kz2t53NreyQryf/11Qr+76L1hXNgN9Y1F0s6MReo4JPtQh+lFeYcniBA+k3a1vmL9/nX0/WT+SjuiXkmoEmAH2SS0MUWRrHUVVOM6L+uuIeqKoIgGWxw3/1a/xq2J3OTTftnGSIhicwIOv/sNMGseTa7tx6NXKIChxs6Au6Wu0fTpPL5sDXVM+MlaGMnM6JrAd6983NgfbHgf7FeW5K0AExFs2+MlkvkgSG6GkJn5D3rgwMq7EXyAHsLK/cY80JZNTp3guq3uk1W1/z2PJMejV3KmtZ+WpYYDC50OdXTji5Z9PRKAvWBpqlf8SxCj9GHdiSC9LC47lQ5BlnHPdocok/q+hPeuEI00djK1/qbOO6bJrPvmHytcxIsrPvUMmaxXPVHZURvEDTYgIPN59dQifUDW+jdK950MQ2tkR7BJn19+gMK8fhdUjUHyQV9AcxEYrowA3cV7rMYGi1qbC8AU1fWntfpATKf1Fi5YI+rvuA5wzluvnFMW8/imVp0Bm9OIjh12Mcj+pMFBcbuv/Qqq0X3x1St7BrhFsIhtO/4njsxaYwPfyH4CYVzwH9q6pymMgKOC8ihhXv9qTlrZQ0RltDu8izB6OPyfRPC21pBx2+lPdNLv4qqr+1SjztV/kZtEfHQQNQDR0YDtotlz4anUG3ivksWyJztuzY1WLBYexBSDNjuWB/Q2fzlqnS6Ou6V0lsnI7vmFyLQ/2BytIVYnL2OUpZnqlWa/h3z04nV0K3oTHrFMlvUjQ9SrhWII2zZu0nj7o8i8UxIamQx7HUhb2342FI0NEZ406dXY5EjPStLQfXrBPaD0WyihGdI4nr79f9wVA/2CK5VkH9nwYB41ge776qtJrPyMJWC2fFRy7ggyybdWdX7R4oUf0P8VZU8I8ZRbWflqMXb+V2l+s6f3ejwnjM2/lQh9qq9kvhS48qjB0d4TX1yeHsNHN62af4bfiYHIaVroRrA7DGS+vXqNMhaWCMyxjtyUhjf/aWS+7e4tNBtgxqj1X5MUes9IdigsesTuRX3xA5OIkaxCfY62sQGQbB99ofNg0+9r3U4ezA3DybypuxQQdYyIMRiVZaeLz7nqaMbatlUxS5LLO2EE2nq57vUGgXzCbTcfOzK/eiGOVUGruXpB/7czEv5pC4IXdniyu0ON4AzeS2BCjcSBeq9LLiVAPPshpFIvCI2sMCpVzzz9Qis/IG25yPTB/K56vH86JFMqvOGPAtI1/zr87vcrU6tBJT9owRb9b/Kq2uIppcCXcIDTtx2WmuFMXvPf6312hA+PETSvCBackmAdn1CVEOMWTwFLvPk/aYvwruPLpvJS2YpkWIaSMz2DzD4Xw9KmBVWHv98X3fOCT/NWU6/kkY06ylNsbCbhwK25HdcQEVgolWRImK0BoDKi4xTMuJEZxVSVrOa2NGs/QPFH5HqFdYn+97/KAyZgulrOT7zP4JHnB9yW3ogP7DrSjz1kOH+8Qw6DLXvT+cMJj19ctgGsFojNDP4oojfOj50RZioO8Fhqk28q7qa6KZ8TBeQISZWxiPFC3mOoUEZY2/sU5yulujzS+CFvwt/Iy/bWsyogwIbh7gCz9fT9XxzAPZPt0YsIUAk5QajSiXme8G594bji8mwnm9f29/syohxtWFq5jN7mo3FIr6Zid6k96F1rtz/OUSh+1gHABz7xnwCFS3LRvZ0Svqex6RzLyz390R3dRjjZBtU6Wrb+8l3j33fgLkiNTb3K78aJQko/ateNtgZgfhIfB43NpF4o6bAdFuWwLajPz3+jhLjfH6Gz9+Ujm9Sdb1MPiqNgRN7k25fd1067dlWhPV+96ITNSngTSEFB/9MyK2XjgRBBYX2AYhZqhNjZ0VZ9aF7AwfNFmk1k35ofJlO7EILU6CyXVx+jM8+byGM2Khvxv8vSBsl83gDkDufqsy676i2EJwD56gneLp5qQ7bfwKDeqX+MDkRjID15CsmLuZJB9nB9VUoochPfkdEDd6bnCEx+n/V/NvvrMdob1Vp6NR/r4NKK1pemzGETE1k7htfKt8zhNpeWeTPk3TW1AlTaIz5lyTGjK1Gz2z53tC8TH2/YzvpJYaL16FLu/6/zCQ2DjS9YjwSF8z/NxYvxLEne40FDCX0zY8jGSXZ/d/bgZrQjMp/RqBSUf2kofrFWcG3nhstiajFyW2t2Hpm1HkufUJ+GFiM/h8PnRqMYJjG2AOlzxrNZ2fT2QK0rMXzvd90clRfnpuofz1xUXCnpgr1jp7efzkxMUMjsFZr/fSFn7a46hr7q6bfGdGlcDjdeTD9PINZlFnnRdLwZDlnFCf5ScDa7Gz+RXBxEyh8yO+P3pwaKnfU1dgwL3ZOVOfsrGVf2V7yCDvdf15xqMQqx/D21szePzNzUKkwAgNVsz6+NS87JR8DgWnMLOIv+RJvpzk9yebs+OwAKCW0wB3EvWAR1N0459W9z4IDs0+ijq1u29RjzmQ48unwNGweNfQB8rD0ax+tIHmy6qyHrO6QbCq7J4TTaUoX+DWA3n9L7EISPlOipQtnwH/mf5cbbnuWSAU1VjL+CC/OuIttmrRTBS2vCmKrEEseVtv+vf5kDPVAk4hkuZYgCu1Ov9cJ/TR8dnC+RuCQrXfSVU6d0Q75rX3VBKAEA+WXAidIzPdWwehDoFrCOumYiUS3vkyGBYla/8u6EjCr1nfBVnuJtNqswoMIy1HN3llNClPiBVrEJF4rB2jlken29QF/rsKN8S3Xa+mDkPrCcWFZM1tc8oxKFLe2YixqQ4DHX+BVQagoIHCNsJsGZOj2pPqZneSh5oILdZAOr4lWiL9OOXJWG/ULA/Xh8KqqCEzKayriYh8pndEOo+FsVi0hi0KosbfS3F6Q5dv9M4rtJYa1gDkRjudsMlUCmkmy49wvN4TrU/ave7dbmjipFrv57Ckx5mPjm862L/V9ZVlbnTq3rb07aVw+YB8m3LSdv2EbRgEGXIYcRn1sbxvhZq9NlWpG2n/2m+IJDC/TduFGNXi4G3GJlSb2XbwgpVRU4bRq1hJ8GQ3TFB6D1q8P5Rcb15cJDWk+d5Lcx1kooKu7OJLq0XQ24QyDldwBM5Nkjy8bzNDj3VV1IizaogH5/rQI12dcXOX7JNN8V2jlK0i/6m2uB66wvWGa9U5qB3oGdmoVQALCrYyzqk2WinAJCSqV6kaiteDkIEGPiOHFfSVM8QH1tDEa+/QHVk6mSaHOuf57STgdqYn1Lul3aXKWEbDtpQ4Ytn74rqRzX9ItYH3xhXc3d0wxbzrz6WlsLJnFbXX3x/2AFBF0EuTo8ya8NoGcNXBPF3zz2eUqoyL/buuvJDni9T7IRqprawD/x4OLS1asbt1KXS5puB8KvXwwgYIVTRJBvP65/B4dPseSGEyS5h4Djoq7I7Vd6fNAwaYW4/s9Cy+afZW7qMVvOsQ/yTCx1iBmFqL8ZfR6NUepVCVBnjL2Rxmp3BTU221RzZS1ftry7zEf5mIgExS0eY9sRysZHkDDFJMnmCIVn7kJ3l35SBn+5If4NI0leZlD5lfw49L9Gl+y2QEIOUlQHu1ANR+9eaafmiZ/NrpdwntXsbAZ44cirKtv0aiVSX1verYzapRzpsfF3qg4Ocoly7YeFuCgdOslIeNEHPyI9acOiv5Zb7ujDorx9Oax9UwRyS8ciMKUkBGwgzo9gNabAxiAHv595s4y0PL+eb4K6bkpdYRr1XFT4Z0DHvgis/UJCO7qgH/HcdzfcfDFDTZcBuYIdG44pBJMBft4H+KpUnBQ5ESoMZ5uJTtL1g+E8s/gq68gcRo4NSHRG17jmYylFTbk6qYNVsa5Ch0fdB7fLii7WtwHV6ukBRXba6yn2hv6tXsaSYyfpoZ2GLEG4QRT37vWX8GJIaYQkEv79aO4OYSX4A/Sv5RdTOMq+u+Fo7uF73JPAWmGjLkiqY/pTD1D3gMnlt1FUecAZxN7JKnRcB3UJO4BPZN5YXzEveKBsFQDZ1RLOnTvy4M/ysRO8v5FZcpHe4fig9tgWNc9VIdPWFRjcUo6Qm74XEwuJjDKyCPwgPJceLmOkUZTiOZha42YAwQcz4tdUBswkrJV4GkGILmkWGt1KaXyqkHi5iFNeOVm3pUXLGRHKs/WY3Ta9A/kaWY1KnQi/wIZw8j+x1MmnjYI2WxyZL2mrR+iOGfHOMj7F/Gs2vP0cK+5bFdpY0oVfOd8tqcb8GkfblxK5oQJviWM8kn2JUzp4mqaHEuQKvxC9P37/epDzmACdu9BVl4K+/ohfWZYRAx0hj8xw4auwsJx/c5fhXfd6bPJh8uZYqJ6Y8ldo/ZdqS5omrC4B3eajw3SAKbPmyIoGcLS67617RESS4j/t3mQc3s8dmT3MLu/tvJhyz3qVgW9nfYKiey60xArZa/uRVPeuVfzSO2M7q7z4586jhUoqL1sSDv4bTpbapg8WJRhBHPWb8j6h/Hc5dup7Wym+CihCdZD6xCo/ukOpb5UZmgwTRS0TXOZ8T+/XLQtKxhW8upYMG1M45BnONxja4TM+952HL672q43j9vWoSv7hfmubTu9t1U4YQywKf7vp7wP3Ei8bkKJj560fNvAKl5/kjj9FTKS6xnA3uibRRPlAeer3Q6dintLG6vmNFkAFfxPXn5sXQnRVSaA1vVIvvjudLstvpvD4MYbvD/FOvj0fXzIXMvvnX2TZGQyeJozTvrlPqGKNJSXC+rGIZ/6U+/pO2EGdkMtKKY9McUtT3SXejICLDdV5H7DraQ/Cgy8tVQF18XF9CQoS6rbVJOkjwWVnk/Kznym1MSNL0K93/xpOAwF0vP1jYH9fYqTDLG3RldYazOepJiXXS/ZLAhUiIQzLMhOiqtdEIdfDIqZef+XAG83d/NVPi4QYxu2NQafrVv8wRCAzfkPiX+GsFzY585oiX/FKLrjR+bcjo8FjrbX3rMATBtH63jur0g37uHxE2BRkurcWKMgdXijqnuPx7XnfBmF+APWLV1i/blvuzlJlW8lzXus0RMf5ybMArZEsid6Ht/JRAqbXl+bWmduNF6/S++7OERZssf1AVy/G4WjFxQKd4Gaexpiw75K/LyOZPdl/f9JfXs9fy/xK7VbxLKNUGy838dMUf7Or1okw+hQITk+e+I464LGS7mKb3uV6iwARsc256blN8RXy0jmW8eViU8XdbQ2LLE5gcT0a03Q+qZkcvAQFbGRsf3OANUHiIiyLDnpem7qWRel+ZycLLqw4b1ZBvCcIXcFAYW3u96nLYNkzhi5wC1INDpyXU4xHHEs7q7bykLeO7O9GrVf/6j1LMnkraUBe00kaJE+7fk9a/K8ngSbk6YjZYtu6EYpCJIrGftl27LileRfR3bX+vggVnM/Xc5prxSdRmzU9gFSN5OGk95GRpVPRxgWrrhcOwmjzYnUgZ8orYMLn+Wn2DVBPmCX+30tyArbKxlS7AoDI7p302aPumX0ad2qRPFHR04+Xtb5lcAhBIyhtj4FI2G/JZQtcjOWkaduIScBBM8h61C++YF9YbtwZOR2jlY+oNLqYzTVCZb2g/8ki7qftPiLQpNBieEjraFv2G9dyjO16J/OVn07l+YuPe+bkup0xUtyABCvNdfyzc3fthPsGoLZJG0HQ3QRkFdZuRTC/WbF93/4rwbR+oRsms2hCl6HWUo3hav1+Y/Y+3sBTiSrYWozftvziLkDBqTDXrxryWUWZM/lWrowdK5YNgbhecRWCQiH8FPZ6F1qAZICu69FmQVVBl3TjRxDfLPXx1Ot5Lk8L170NvzwTaDLzfhLGd1t3/Whpr0MSN1VKt6dQ8JtRNZm8MZo4F4l/X8PD3kTZtCn9djX7gPpVmrbvL2riuZ3vUAfj1rHW/CQY1zv2Z9NKwUmTO8hd3WDzWWkdJ4UlnAtB1n4W4a0XxUJoOwrIa/YlYoAxjz29ArCZEJgk/XA0n2kRPaioEiNRiDv+B1JtxuUCryB/oEvmyvyktzXZKjx7LOtwW4awDWUagF1ksY8VHp75G04Nk2etBdPajUqOkLctUNMFh80Mzr/KtC5+OtsqWm8SXH3+wlPrS31VE3pO9s7nz/eSPNM7duYwXQN5JcNUot4PX+QFu9gPk/Nzv9fdx5cMxWsfImbYGkKOyL6IB87TPxFcJ0HPWlX1jLRpodYPLiMlZLagbFVmfSLzT6p6v2aDz7EjSExeXkZgNzCafgDa/3QSwWsmihzzm4MV7aZ5EHAUNMODnEmPkWfkoBOGqtUVPQmHYY2xKB7dttNex3chrKG5Li0dJgc8ue/t/8BG8fBOJ5BKygn9ZYCY6mzWnHaECYS5w+z7s5TJ36q8pWc75neS+0DEywhjRagNgV7HsGB3odGvC6qFQOFtfxA51R/OWF/7q48vP3qkTvbzU6Uw4TpFmKJTYaW4C/qGXsI4/OrVwj9X7hvwLbOPcflGb+vXfDbd6f0EVp+x1vidTZfVvrX3ke5aWA5GNj+RHPVZmmuYc17ASY8f2HOG2YguS2d3PkvXvk7ax1dkOsXyiO736beD/P4CXGxS6UnM9eCnHqrTQJgDuiF8OLMTfJnQ4MHBJxPPIptx/EVBjmJlkyDsVM2HrK14ESNaoUQRd6vacXeiZ5EaqMWZX+PHVyNhOSIA6TKgjeEHXRgIi4Vbj9bX7Jfgb+fY0G/j8fQrX+Der2/5QgxnQNFYkefBxWkb0zgY0wYYF9cTs5vOaUdFk8nSr+YTLYjKH0cV0cSh+E35NIffrudHd4W5N40Q3a9syrZWZdGK7ftsYq3LIwX72tX71A1t9Cs9crNEJYnxiiXsZMj7YqNCFwGShTh2BNOmEwYcOfrlIincTVS543kz4qJkwGgEMT61UQRpaw1rmki34iDUHrODBkWW1ppO7ONUork8Ht/OMIo91tQyf9Y4Nni6WnlO3CsWK5ZZU1bGsR2kZrf45Fov5fy0b2MsuKgd6mmxvEct6/QhetswOqU5wHGwQubPmOS+m8bCb5Kv543gyfV5XCaT8Z4FnpHcqjlS6sQYpThGSZJGG2KHHW2H5660hJIn0DahQBnxnVz8z8/3FMU0eEaHVx3Me7V7/pZ7cNPsZYmOqDjAtgoJz57+e4BNyoJBpaJFZj7JO9uwVGZ9T8ybf9OSpiMjwiTbq/Vofh7qZjZMlHR0p2Wq9rBGTE4Lczg2cmHnlVLsuKyfICdWlVT58uF3jhbXQFKNbPVDUQhLeR3kszZG6GL8TdqQ1GHAoO+474+e1sreGIEc0hIuaCfKnqSkQ60qiyolhnYuOHK/Ulox6PTc2U4Q098uKsCwxgP6TN9XD/jHiTAwhPa8X5FGXmnLR0pJRlro7h0rJef9Q5wnNAPjYE6Ei63v4hu39qg/wJV94pzUpMfLPaYHrfhfL6hXPKUwUmKtzwZLs0SCgo/TcxUIkKxzhHHDSNjBDMLP+rpJLlVqHVvJUNEwsmdwwq0V6OJv3Pfoz0w9ShXHGjDg3sNHK/OJ/Wc6NdshKblSNBy2BRNuOMBb0JwQBNKZP8NZj0gPa1P3Jw7gFpmA71eiCRCjYoAN9DzNMDwFW2nIICdfAceAzt5pMH6xqQrK/TdicS2Vu/8jgTlXwq8BpMjr/rdVrVRJkW1qxD0Vvy3zBelcrqKLmanrXMk11Fasa5L9XTaOf9GvtVO4yX0iHD56clHVID9VbKycPpcxs2hyRZ22uuQJ2RpSxXUsUSf7EfarOq2qEtKeZPgSKYJXYEllHghGY768Geg2HIRX5LHWdX/sgxyf4i1z89XNXWtKCSmnDERCMkaqzQ0nMvorx+Uh9Bf7Mw+Mu4bFjgM3i69uII0yAkDiHXTkhAf0XoaWhDT9Me4otjhFlwdFCkaoDlXaDALgUETdMMvbWbWoT4TCosJd6+FaYmv3pQ31AH6cGQnC1KC70MkE1GLjwkxlv8fdb+79UuKwmbX+hgaGIcgpdLO1xJT/dbSOEIil+UlB2/e2u2wceflmPk2wuLbf2PGFqynaKf8DO5xbd3iBBzF6rCwHQZcbd7dpH/OsOtAHGxpg3aLcOzouCSEq24oo2EFC+5EhhSXDaJwtbxSAsCBYuq6X7/HRQirZ275bVKyWKQn9E6YOeG5lfdvu+eL2AzDt7pJPxG+KRVSj9/R/91/4dJQ++54+ipJm4BFrAzNp+jc5VwnFffg8VSzcCfDFNULpkXzKKcSX9vYjGT+RpboMpYvyCLtrzsD4nW8AB2ICqb464OWXIu9tFd/7N5BHLBYbZEHa5i0HshTohD0dwgfKL/Url93sZq/ZHN9FOMSj2fj6zNBEg3Dnjg0/3jcRUvT+jf/3CjeRrduQLskrmryzPJv2Zy07pbPTjZZBgh7U91WWejB1s9w673bybpj3FWeLIQX+1Xoy4hV7uKrUthaj0MoP+Uq6q2A+GedbHqu98dqwrricvfVorkUQwIJUvyuoD8Q8bu3XD0S+Woi/Iyx2lscNFc0/sq75hVVNwkA2VPX4S4CABoslomGdcUyZ/c2HPL51DGmxIicRPUv11GO+vHWgnC2JJiClYMwjH8w+P6VthD+7oU274BXEldo+skTEk5ZS4XWw5G0V0bk6ssLRQ2C9ayIop4B7FYf3NnVpNEvi38RcsSw/SEKAYiuWDkTb+BqQ1LWmapJ18WKB0QHCiao2OYUyWseOq+sp1U26J5zG8cxUqY1Rs9iQdBghWCcx4IWBCln69azaiE4ghYQPvY0tdrn+/xCunHC5DnV8RpCIw2JlpmTt1G+jwKOOp7DUdOrLi8YXnVGVxi0ctfihmBP1E8F8aNe958LVC+CsnMQwZDPJkz8qe4Ot695xjsxfdmZPJ5FW7Wvf+NIPk+FvOTuAFG5Nez8QtBpzHaZji63bom4glU3WSOiq3W3oQQCY4XiNvSGo4R7QaSjTLHKGQlsnq9y4eUuTLPiMwDMT8G68Krhw6+JBSpXf8zrVOTFqz0XnD/6Z+f4uffpUxqOkVnTHrx2FmZ+LYw903DlnNqoxocsJq/7rOApJSj9z+jA4rsS3BFt/eHa7CLP9utDCvWt9W+XGXLM2MrxvKDfU6uI+BxLa38lZkDHFJ5HfQ6H7qltyNCLXrhDX43r8sL3Xs1M1BdMWfBwwnQ4mTA9HUFcXWuME+qfYuc0lTjn6y92lrQs0ExgWoOvusT6sMzp9PMSr5hyOBL3pwnWAjCzVO5azZFhgPVWsGVWyw2Ed6nUpZb+jf/mshyxr2NIlJ0G3CBbN2NZnphGlGZ5iEU+38lFtzkWz4GHdpuWICUUlDa7VgWedn4FCWQvMdepUoR02HQr/295hdRt7nL04rT9nXSdNupthQOZfH6HqE1jXCVpbjmFdRmDU59h8yPo6AG24phhusFi/Ef5j9U/HMUzQey+DkUIiRNejXDIZsTdHnSY/WzlgBrx59EoyOh3jlNymj1D0Guhh2BoCN4vOAiUxxO0CwsP5tXCz0WuiS2dUNQTpeHou80ONBrnDnvbJcMVSBciB435Kp/qvuZPPT1Bcyiis/rdB0TPAgjIr2RaYk73QbG7qL+rnA0HcB+WjS5WmIMJkTJoHkyyS0k7Wvnmwvmhjypjhnw4WyxLXJe235cCQyDwiprLxbM5JzubVPY9/kZ+NFEhu3wZ7WvJf1faeKAIJ2vnmDi/T/RH5Nr3ieT0rP4MwGB/7n+m2bn36/X5fIKjGe9b0kpZcz+CIycM/Ha/sOYeYOkbExgHfYDOCTHqd6Pqef+NcKme2Eej+4ouI8Hmeo+XUjTEyAahU0Cn5AFY7RARvPX6VTof8V/wVFk5SEsUgSLTo3zEFkNYBbTnIensiHV2yGMn/C+Xo5o05/F3ScZrcRuATFF4x9YnVYq1LhFcdjSGfgs6zndA++MfGpJnAO9s3FjVhvlmd8BNAXRMRvBfrNjsiqW/w3MqcJSVBtZ93e3xiXOS19s0xqtzbY1wcFIOpwxQdlo4CfgJQaW8dWvaWmCwcxL2JQlq65ERcRMwpHQTGZhIMspTw8vG4nB0xFyIB/hqAzBk3Gg4XE8r9JBrhkwOqEsF4iZ9ZDsyQmf6fTGDVUHfymQ10pnqIxi/+yd52dF2YsKhw7RQXbTgPlSEhshSEbfAPcbVDdpYETySImkGN9SmN1X3sm8mW0an89YWhLsCqlX1WcdUP+9643d1gMZZncl816iVfVN63j7mb2NYjNKBvsj36EMH9ZKFiReRjvRQjaXnHcslNMqU7GvhDDCNPuys4YiMGEVS+eqYL7iI3PoLDIqzIIPyh14HL15SjKyqLruXdxeZoxgsWVYXXh7U1it1MYkbbZhmmhVJnuimA806ZuhxKICySRA9PAJp6fE6kCaSCm0KtIU+7irzyxkOru6rVw5j3t9Aajky2UgmOZuLHNrW7IK0XpLsywZOXB7dxPLojQKVNmRaKgrtlpr3SdZFrmX7D98R8ucm/YuD43M3QY26NsDiXD+3LGy5Rg2cVIX+xp+WsmZUXyIK4ungT88XiyyvzoSR6U54yXae4BNxggM0OIL5slA4NnTJVbmvrvnmv9QVITu4XJ6F213qKlWcexLsaOSSnpPrQJ9xU6JBjJLwiMFZrXBL+k73I/LjEbih80rmPtkZCql9RKvwd2e3/rqLVJjsCI/0bxfMWTzcVp5d4fNuy9BnAC+qH6pS97WW+bkh3VgvNuWX3yKXNu4tqCD6+jc+lblXyP1d+0bU28aRVaEOrJk0GvbT42LweI0YqohFWJMPSM0Y4Y1tSwKQtMBwUR1HlqPEsFDddEnPkyZKrlxfWxs4cS6gCu3xMksVNa9mHoNky2rYmMHNUA0pl1RLVASpCjOXJJ937U0+Ad4JKo6pKIv+la27OCAGPjSowSp1LjD1kYgpjYl7cSRHCH9+nvGW1LGkSrTkrctgJE9ASSwuzmpkBfgMSpiPf0Tdzv+/cTzMU/dqurc6vtC75fs29LjhBPc1ONKnLkFVBiMSww/737/gFRd6v8+lhYokTvT38d0Xhc+QUjh5CfsIj0qWaMv6eraFrhRjLPgJhDCwM2kPYUObYbWoo8ugkoJekE+dHUmAFvn0rdAlCpc+3+3p4f/hMo3gZjS9lOC60C8Lef8LulqC39iAHrlZq1Q1BkFn8tqLcH2nDsDPtcfx5DXepoZkr8/L2O4N1CwQRz1hBoe72bmGUiMefAL6DHPF87BnmuHzn/nvrlDxTxuJ/0Q1dViLtGYBgJx5bbbIIkdu5/K3Suo1vmZd71riJ1P05P3GFMq+pFbU7poXhJ/L99rjWWn7k5RbkajqhViq/afxnoy5nMI2g++NgGrz8G/yvnJPpfH1iaeGJBECUMbvx3R1PlLwGFlPC6HAZ4pqGqLFqR/bTO+Kcqn/Pl66zGEk4FAEgEwAi1Xv/vucpVkzDlTEzH8XSjdBzDxRo3/2VV8GEmfjonOeAjx46TVqSRtDP4j1G3kci4raYQx0zO4Tg5wbxPj+4p/K1MoVCCohloy/o5oZJv9kIYb9AgQJCqP3Au7IdVSRtuPuLrDqWmliS2573Cd9hAdmhSFsjrlli3hGJ/qxikRBzbxF//8djVNJhu10idOFvAzst9k8bU8Set1fkG/Dyto+5Cu5BjGs5fc8049cWuLxBJC+MHmXFFbh6kzB4sCwz/qtu1ncrmVS1VoijznVA3RlEto/iAp557KTJRo2/okaBlWadCFXwIJ2tgTGkDFiY801mWENrHgUJV3EzBOEEthu7u+3XkgOUpsnWsVdem02nPjMXR8RvPueSd/Pj6VP0R/41FZmWe4fOC5FqmTmV7crQp6KK8v1q5uVPLTbVfCYlythXszxapDtfCYQwK4XRY866k2rHkAv+BmL3Z/bXzlEnDOZsTSWGniL6fexv/htdbnRN3jAYrCJhcLiLGsoBAQl39/5cV68ssKdm01Nit/ksEg+qgSrsDyTwuu9q4L2jzMR+AAaZK8nVyJwqQ7NUHJMWEIQi+nvX64VqTZlh4hpQD1zZmVP2O6yZPEmfufOEcg/4t8Gqscjb+mxK8Gq8HEY4TVyBRuCzikK+NZtr+K/gnuaZfx/QxqALAkqHfYARBCzmcg5aJ6o8QfzqnHOaZHCpxZDlmnkd1QLVCCxpykV5Th+zNYQr/J02kNUzv5wGIAGNoS/0aoitRd18QIHUq4XmdtraJ45zHbgMUBRI/pvpXjTEYwsQa0Jzgjxmg/hAwDgMZh4Bc4VXivx5nCWkFd2y296SAOHQe9XZ+FK+zlGrOxS4hakrmgR467zJK3FsAyZEfHjjg1EkqO6ZCxyLdXnHASiTANdqCShHzxhJHhAtwLfErrancb+ql39ggENVOEruQUJC5SzWMUYOffgSVwXJqCEQ4maVEJeQJy4XRHfG+zDRuY4JfNrUFlUTI55WQGwP7d4jiwOO86BvmutqZf1NovEuKecA5HSZwLlTCCkqP+d8fNGaCGj3A83DLJ+U72z5dCUAtMKriy7cAncHxlLIFJGllg30o6UhN+lsXqoF2lcPOxp0c31+4OdnHfJkU5jvqlmYdlDg9wk8nutMBjIgq5oynLBSSPUVdbBnYh28B1aibVWMHAnh4ahPnFsDb16XJ9YIz3omZEmRicEL7hajFMWpsA2Q2YF6JiuL8fH4ebOXBzt/YrtdGmirKdyR1iDskgPB6/yVfkx1R4yvVr1uQZ3GIjlRu54m+km+SgrgARNVEzsztGXVZIWXZBsNr41++AqLM5WqfRQGRz9eKev1vp0c2Ndi2n9ua5IVMBe9j+22DEWdc0r5cPHtGpTIoLWOt3n79nmFvxwrJn+1zsFFK3NCHETDddZGQu7xvAKnUKscSP8mNW1zILevl0lXArc5NjGwalpXGgX9eIXXQNwXvxfhnZjbLCfyBafdTMdNRKavK8sJZsMbLjDG+yGHbDkkBdLtmq8l/H0GNYL/pngSnr1PmbgHxWorAaTr9/FTT2hcThu69WL4e0UMzwRTDxQQVKxU++yPamhcAyV/vuntwYWSeUxzrc9eUqetwwRXYyz0lIR8fTHQTIMdaU9HRBIWGjSCDX0B/Uw8pKQqnyB4VP3XzDTBi9lAcEBH2XQC3/SSSJhcULPA5wN//K8Zw1E4dEwn5gcSIJw0Ok/yiMjZs+8dFe+xj1etiHQhlmBZGkh9Fe2mD77ag9spHvTzuI7+UiTGw78v8maXQAwSdEw8/bAP3lIuTNE2Sn1YArX1Cpz9/xrtWXv8btckLTL0bh+ecNyP2X1jXkbyVw/zTQtoyOL01cMbrAgvigXxIDjEkK1/1oVBa4knvB3KM8+/G8ZYLEosW8//Ye69uR5E0UfTX9LpPXQsPesR7EAKEebkL743w4tcfQplZJjO7p2a6a865Z+6u2rkFgoD44vMuQO8W0u0YeJj72eZfZi2gpZbTSRjgb4SrMNp9B893WSX8JEpawL/PlkYOv22hW1c8GrFLMX7Ckn20XvqJF0C4rVgeKZkXsKCrcAZTmpbGRzH2L+EoabNbXs0X8xhPlRyi3MBePjsWBoTx8aCNoSBT0GFUNo7xutE1SPHIW5vLNjZP4qIG5rjQNOK9qF4uL8psovdLihza2gI3+F0n/TFRQr2xBYLbqMetem/H2/Zg5+Xhd1s/UGml2JMK6gjoYxYa4VLssGnFPtlG4A4i8VpTpahWnal+tKJHMUCxixZdqarFEQrTy1azPasz0S8oAzPKUHqyL3ccYNEoVCd2D4Kkq1FgwrMu5ls8nhOFqOGB+UJZ8CiOM5fa59qo3TjMYSgb89IC9K4++P0V3qqbOiydxRXAsKmsl5gkcisaljKckNXF0JseVNvmH+Fo8ikeZFiJTKriHN2I62zDbavYkgiWdsb69nL3kEueLIkMnbaBtj91qIDGJJd71xpaDiZzfydw2LZxs3IuLB6fTcmnleoP78guy6nEAreY/Hb0iz2InvG5fcoBZhNLpdJ92sllg8ZkPgIHpkCaj4ePuZL7hMdQvxPNKdtAOdC5SlfQiaAMdFCZcJI046l4Cq+6dvvgOT9xjZaxiU3jU65KE+AxEIzySZdQrTt7bFI0Wa6yXHRCNOM9nApxoekj0BT06aQ4oc2eMit3AS+XCpuLnuwjZcE8sxN74LQWPMyj0pyzJAKBmmpHwEyGmuToTY/kSFJJa52EQ9LWPpWOF5VA2jy1RnEq63Ybhy5o2FzIMXd6ETPQqQyUm60HUg87SPwXFo/TQ4tEeXw4YrTH6iJ7m7V7yvVDiQtJfo1NRDTR3pveQ2+tpKBx2Y/iB3A5PE8jeRJlU1wKUZrdcls88vWQl0/e8b6gc1PO3EoT5R1HBl1dfagG3HOaZtWwkYvg5pvJzfnxYOTYLDCMw3YEOoByoeTcYy4Vau2w43y1gW4apw29lPyLJ2/dAYfj115g7eVx3FddQk7hYb3YfnyImq81W70BkcuVJNfukehyC9NoAezrVphOAeSUNR/i5nSZlPXoOnZnYRVotiKq3OgqEfYSFw6TksZmLoXEbXYjUZfCXoXIf1pjwiXDIiNp/NywSj6zfgo8rhWtjHvx1Y0Azz0xidqm8I1iyPuE17uEzf3IO+vYOssm22Y3l/5tId2Bgi4L6zmAAuM6XM5cKxE9ICBm4wyiGiHP0hQxiuaU7wRgki4TVWqXRk4QWznazuuAa9zGFhLrCErK8YVs7ZMeFAcBNkUAHJT3kprf5PtR1FRY0PJB51gnR0CWemVnebthXWJaVLxwY9i9pEJqatdP1yBkdEO7HfznEZKbRY3HbIbGK0G11y4pyEJ7+6BRG4ggpA1DRWF7cypqKwRY7E1b6bIq17oMkONKoetdZFskz3RPcpcC273SMrqLtQTos1dTSY9zBiTbrFxVAV8aGsMl6qcK58g0495tqLrUZt5vi96Uetbu7Vshmbw5rJa0+c93COf6PibanoLsV8kYcBYDoma503d0EgivOTsUf2L3tAi4VSVdArtwnyba2pTwPT/M9YF58svI8ih6NuglYPYuMmjx41LgtltDN/xJ3RcHBCi7xSpomLE48YRI0gru66VTKYN/cvOzHIgyoBmYXPsFVy4JuiFdFZIuwi6v3nkVMUPoctU/WyA8opEZtecyx54MBJpHWjt+Gyj8szvoCQdzIsnDOM/ssXEWprGnXsn5XZKrZhKZ10on6bEbeGZRZI1TkuggyVFT3CPjoxjL4gJObG9qLGDi29jdHtBLhJkJCkYfRtUVlGL5JHLo1pMvBnSP1Dzn9aw9zIiRQUWKwOueNqAC5CnReQC6NQzP1oMI6TD/vTFH5rUZm1C1Omu5yI4QnKmKxB/xTPLP58seeyTdmLuVSZ/NsELXIcUKVwSMflEm5901m36TzbtJ9DfKt74GyAYYBw8jOuCVZI9Dzs43tD10GX5yoMGlrG5cE/V9Aq/5Z8u6IZMuvXyxcFQwvGLkrEjgQRYrp1KX2Tz2vnrMEUhH6Arqg1hChgKPO3Znm9fOsO4q2AsmbhpwrrPhA0OZ6tR0e6v9Tz+knrw0AqHkm6pJ6urTMUHOaXItzl1Pri8ujgo0mvebyggKkDA34GbGnKK2X0aqHFmE1OHLXb67Mm1YlPGYPyn0BLlJySTKB/8yaM5lVLs+oLu8k2BO20M5nIRwPjtXtgvRvl83fIHMVTdesVGjjoBKDDWqOvtu5uM9CzJDsi+gBwTU1gs8MUco67UKogLjMzAxjVil+ug7qBcxkdfR3mZlUGgiGIRn2Uwyz/NtKaiNRt/I67jUbPhpqOUEguTOO8mkoKv7yRi7Tw9/FfgIU7eoFBR/f7ZZZeZXX8dDWFiq0Uf3yX3o+xMzkzurxjs6PBzCOcMjR7VaVcvTq2Jq85JqiwsioMAUfUkQU4t5V1Ectsg+qJ5zQTfqhrclX2ZWpYPunyeyZB8PvO24LDtT9MYFUM3ORH2OIXZ6iYNueD8+s5ggWpyye2ptXn3IBlHaV+9sz+vsTpzVPej5d9n0ek8BlT1EVFvD1NM2uPc7p5v1VX3S5B48qDFiAAZImjVnPFW/FOjFJiAf85xu6ZYcSiudMSGv1suoM8x6wP2gi3YUaSrcxDAEfzbczNM8rl+RXHu3I1GdiZ/iWg037TFZ+2fvEzO1s1EpwsaJ4BZa3FfjI5zTD3I/U4O5xuMbvuG045rjHXhsitq3Y4drB21shfHcCb5+I+O98/f81JICi0G6WFbsWpSJhZdFVvpJ/G8JxIp7bGck6qSwugHTyJPzDYeFK5IKeUR54Cr9mqp9far5bhADqh9vbLCP+ZOlToH4s6IzKj+xexIa71pFoYxgykSsh4+nN63eutmiWhShEx/5Ox1SEnzzfef+JSuBAYqtmOhAyc2pjdOToA5nmfikpugjLTD5gpiLy+onb4+UVANHxJJYE8XzD+b4hJCZd1N+orgr5xxaNNKMKozs7rJx04MdOgoeV4z8tn9SDVf9obzrbm7S8Ua8anfVptc7vvvohD1oCy3RNfW329y31A2/lVXX1fH8SOrn+tltkxSJaLrW9+g15SKXnAbaIdQqwbj16OYhDInzUUizKrp/9hhjVmGmcWdJuOt9uy4xujIgasXaxVh7+XuNu8+znChH3DNRl3sHe4cNclj7ozJtrEUXwJvymwTnPim6r2V+fNR6QB1A6M+IWaXNA7zDAhWGXjDXYOwQMYqdcmJqtJNuOEl+Nwf9lSWAxXDLLYr6HcZBtaXgJgswxKjbxZqenhP0S9Fih/M4MiTvHuYkuiW1UmSVatKLfUv9fkOLgp+bS/mDjUVzNzGg8WOOCU+pQsvynywG65aKY5lHy/fjrThejyPi7rj7kGubX3Gg8Lp7GFzMlfhpVKDbDfw4T9AfG+YyuU/MWkxv52cn+dftwixZkoFabsBrJ8YvnzKW2skewC7Fq2DRUqTEYsx7pZZFPlZLRLAbvLrzmMySBU1l95r0NiejEsoPMpLKCPg8mH5vNCMgYu5A75l0OGHcOg5Gz7Qiaf1nhwZcvKf7Z/cB5fRlWj8WbSoKwcTI+TkwMmKx87wiBF1mzivqtvCxGj3NvgQZS3EVvK8HyFxsFLyeEaU9JcocSbpIRYWADXe13gmBaoxCpzTOalhGIrtjXpbc/QX3bxp+05zOvQlyEqm93x7aS+TfdNrXn51DOVrf7pO3Mx6A5XAZ3SVDAcMCouxzQEUXV82qV8myC4Fjo5aBWTRMu7s5lnj9B2j8vlCzoCnH2wBTRL2GKTs6c30KNVlFTvobPxTvd9HL7ESFUiO0/WuSD8VhHw6+HasRD28DR87Hi/cMLKs+jRcHn3HLG51ZDP8cgCeOjQ4emwRZ57kL4id9vwRanOOXMmzDj4aDAHm8aCTL8IekqEb+Oveo+CSj6iBsR0LI3RznJ77fOdAUiohOPlX4Xmxi4ZTphxBxYGaXppAGNJevDh3WfQ8ebEsg9rTeYYNOWzEtjK7WEJf4hFhw9W1Ol86vMEntB3vn4sTMOJ8+V5l0VpK9Taj36clrK8x5f31Chz5bs0lWa5YYV9a1OrRFy8G6SSASI9xZTiWeNkNdqiBNPu4m7Oen0gJHwAPdFE2CgvYQD9rnMeqzqaadAQnaKMELF/M1mG+PuwILFw6AzUGEtrBKGmKTk9bvirYAfFVXwWUQKKU4fbbGghXWXJGDrHG0jYPbav5slcgplmjrR0Zxid5ddso9PaOMnhWriOg3f9FlSjXO/QLAdiEBd58vTTAvRdDKHK8QWqqjnGSOC7XZoA1X3XzftPe4EmCC2HYZOw0tl04XsNdLms/Wf2VZDF02NH8aegMdQkdwtbmFRGoNi1A73YdFsZuosPZaxQ4mvRkGwM19Oy4/1YnDernCGNbEMdDoqICLqyEF3w7fUtHYCTPlTRMRM7fB3jqBII7TNWVetgzaUe5rYNKRinvEefKMwT7ip2K9B9bMTqmgPFOB22Lq6H2j1eYuWYPp9aUSzhpB0Fb/vCNlxMXIUoMmN7Z88ZHqmgLfnRTznoGahb3P4sH2b1gGKhxTXxffj1rPc/eJqa/7MW/OWggc/spPnDmleE/JuRy5y4AoohffZcsditSG5ouwG1jjdZJh81IBhqj2kPrMGeTZcmuH567Xfkll0dvtMkceRSk5TeG4z/4zVYTL5AcQlOaLTtWxpS4bMwVU2tBBGY+xhVI+e2Qo6zBya02DUEYPOfSVd4dncAortvxeiEs/GuiojgyYDiCa1wy0P+dbTi2UYWheyvFfcIOWbyFs5Syd3xZLC87lDTkaZ3kJHeQL0C26y0ZNfRd2JqXQmoRZB58ukNyFZi+k1ZFMuNcj6N5h/+6fZCZ6x7ooPHdRqBIoBnFbnqlGcETq0awrCdUObCtyJ8Vjni3HVT9b6pmXgtQ8+Lg66CfbUSC+sgHdgJh9KPHKz166HaFNHwfNw1Ajco07ojByEtIYFpcXZ85xagdsg+J3yFcKy1czHb2N5HzGJEpi8fjKtfd8KSozfCNQWt4HTO7ENr8TXXN0y/NEEsjhaMEkFxhaPWKpZTwM0JuJkGVrl/RON9n6KETOueOjRwy6bRvCWgxPxcgybCGSPBjlT/bjLV33aUvMfjifSkrTXXHpNsHcBfirPQHP5sLGZbrtESI153Q0KOyYBhUT2k9jarHY8BYK0Mlsmscl0fl+LcfYHwn7Lj7deahr+kUKzBvRnHlmNPyc2K3O9lOklLVjxqMl6NpIWrVAM93wdnnrtUaPs2dLnctu5kIe2JeiqC9qf6YLH1QK4zlm3OOevEmGLW4FX6o8RzMPWneAYqGKKzzTqvr+JNQ5h2uotfUy9cSAnsTsQESz8Td/QGvc+1rv34WXNaCwARfOu1zaFX3ZBGc2H6z3FHyrhHqrVVoi8qLP2ouswhGmqulcHIDj1TpVz35FtoS9IKMnRAH4dhkraD6vIySKmifieLFEnL40Dgp+z5Ffo8VlgM52eP9IN3OHJiq+X+umPiGqUEzlsEOaV1ise/uFz53eeDiuG/hY9MhIlXh4esz18M2uiMoajbvSTH3K6ywjh2EEPz5ppErhAv4jsHXZzhHpaLQ2eip3qsxSWpexdYPqHBT5rDSRUiTuP9ugEGmyuiWFSJSS3TGRoppBS/FHw5rwGebpmI3hSftUE/Qt7IGoT+5NkGcAbYhgGlIEzpgwy06+C+PLqAZ26mgIruol8sU21rR7f8Q5VD8+gUTymLBGKt75wI1D7vJhyKd3Z97Clqa3j932Ccr5DYUTSwONPalcMsb07ljqqAJIKD7HQwUgfAdr22ZNRLYEzHHJYuSfLcUPQJTGoxaAtq0rJJNXWprZpoVeEv3O4bbbrpr6ksxkAy1ChciWL8Nsu67TNe7SfTSF5Gtz0DpO96z7tFDsMyTv95JSZfVWL7jsYaKzmLJ+843AJOKJF1jPYZ8b4eHxbYeJbM3C21tX8CJ+eidw5dhejm2K7su6/dmUCEemM+t6DbIxpS2823YbUB5azJu0PXFsDO+LtCG9MDpV4TXrOzTUT8v4ywZGRC3yksAZUBwuB1HGL50IqMwSdlT5+82ATOt1UhCM9pdtxFlSX1j+kpeN/wRhSe4RPjfj/Tzkp4hvlv6szSf05sSyYENuhzagmTR9HmI5RacGC2vype40usdEjdiswGeyi1qOci8AY64jxpPbM1bfZoMDel8McItY/HUQIvdBq4pMeJiGCi84B/PeuEuiQsa5Zl0CFXox3cKzRm7843FPIaxfpickxsEnpqNjoaDNBn4Gdp5uefka7hJl2Oro9Pi2DlCNPDeB5jA/1XgMRtNVi9ZsUbIL7UoTqJRoskps0lqhuWEdahrJSX7adKWXqp2gLkfdmZ3PB/tTuOjSJC2p8kZIVIkyYzBU5zRSZ9+d5sQI1aPDuK2mHsceEDeWfb5DalxJ582+MCKhJctQnP61hSEgCWb3Q7iWPQHXLQUVkxQ9WG4UP0leJOANxzOP3bN0ZN/9UnnlupBmY87Lm0EMljEa2gyKwePqZwQUNND8lnHHocNdrhjCIDGf09kNCnaSmjtrRodz0A25eQ48k4HOZXc9fiBYitDBLZy6VEO1pJZvQ8umQfJgg/143jlK3DWg+79qz8QMnDMZwBXv1r3DyhOEDLi3LKAgcl6n6AQWtSnGe3l76f5G3AYrk5XIHmR9Z96tcqshW8g0/0jVuPMzrISMGUgQaxUsTOemo98Wcw4TYNeKY3aMZ0ZwvrSfNQpqLwcFN7jZtVcRR5UzXu+ZPSsaYxxowUsKPK/h4s5vK98xAZPX+2Nre7I3u3o89LMl/W0HOQPi8MphgwxaJ67UG9NWAelT8qVmIZTG7FrvmqUC4KpgHlMUUNWSVd1BAWOfG2vwOK/UNxiVB1l4ODIaYdk9G+B6e3t7klKlryk1h5IeBMLnR7Ago64+388Cp0ZTp58rGpVAD7ofIdy8lTuJBTN2vFFpP0zvyAE3exmbkoyUIXRz0p99VKYWfwEnvxtxC3ObHOKvjoXoC93LnkSkJFtp4AFv4PM93aaIPJ1FpN0AEHpOYJ6VXDiMX3rMYgs7R3ryIWyxIsrZLHLAbbYAae2e70X0R2u8rP7VY2R7eSMz8869dtB6KjrS0zY1nJIxvA2GrnLmQ1YJLeGUwlxvfbtHz2aEx5xwzRQJAnE10XuP7ZdSSa9M31mqFBnHxtDpMdxEDCuVC0xmz0AKEEOAC7WJ3C37hk7PSh0fGCrPjSuBHFoIUwGBDs/eiMubxa4XOhiAOQVV62UcOtS6f0xPmq/3U2Krl3bvHOC8hLWynGT7s03YWxIXLecxdrhjnv6ptZEjSn9sYra/JTdpQ3ez/Zhk5zFwQB5OjOZW/l7hgrZCmpZhkNQSp4d9HpmL1uWk6VgtM0QQx6s3P12dD3NJL6CUNw1JbHURxCIqXBuNnArRoTpuXtvG7Dk4pzsc2fuLToGFCDseJyyGzIo9bAI4a0dx9I/Z15eHDkutru65S42t36kOLcIioIkMA9QCMKiFV5tfoeMwx3oHLiRAlZAZba4MItAdoTOXrLjfPvunx/ikmeocVHs2Zj5Xv6r9Nmm9PS3V2oQDx9nWydFFq1lPJCYcId8vrRvMnpVr+LmgaPLZkPBBGAl2U0fQrLmq1ef4boEQBTFioaeo6ImRqRCf1kMVZPqybOQumjLg8YdJNLsRJU+PLGRZIyPwEEf2Csd2G++uL69fYrogLNmyIrrjIs1fFnyMcGAGEiqO3t1XF9E3xqqqxuCh8TJVqirLOFAkWgkMe+nmh35INBkDu7capHiNkogaIPvBsqpaeN25MuipOhnh2/fb6SATmz5YSaYZ9nxs/ZZW+6z6wPquPo1h11idzdjDeZ53w0vSHaNGaZIZRDdg0XK1T26QWvHiZWup7yMSNkmw4ovumdQnfF0wXYnngSUqdj1XiCjPpJe1vTs3GOcrBpieXe7GIsh5agMpH8qWKGMjKu4yx7IaGSmjA+mJTb2ezT4dI6Uc8ITrw+fO9qKg+20/gAVRdK/YmRuRfTNqYt76mzz6iQb8Agx8Q6PbZcKzDS9e0pP8dPiFYDwIkUGPgH95DkT5tXTX+wgiPt218L6d9lKJl5px0z66UaRYI83I9MuEXgh5yXT8BizI6FO+b4a3plCswT70nc2pIWrdMxP2Rtwa9gtkZLUnFi8VYRhYoc6WJIBPPQU7qC2GAbWkwr4KhHWZy/1uqQ7QJ+DGLkFeDWPnfF5zdbOkF5fXfC5Bdk1NJtFjLwUFlIvR2nvISG9XgXEpX6ahBGnGdKk7n61zmATynTm+3v8hy+VhMrcY6tJIT6TdYx2Qe68lX2CjlKS8a0sfrnp4V9ap23rT2Tikiei2oGzqjPf1zdj4i7MR0XwI2623GZvlaPr4tJMTgGyMz/YS3gx0nn66751gZJ280/qTYrfL3g0in63K+f5Io4+Cp36BqxmkI/CGzFG1L2C3aoZrhXRaVwGp6QtOldPJ02COXWe5faiqMfVgnY+eirUssMeruy1axyQpeL16q+UKqvqkIeexEJbucFkitE96qw4diAIiW3r/3nD0tTQs28ksocXL9mSAu3vyuxuBM7JV9Cr/ZtX6bu3b4bDNGGLvwaZW8ragHmaXFVgZxhmkI85unHbxG53kXyuBvzsGq1igucOTuDKIRYXVA3PYy4aIuxVrAqsAzpKDRsYMRXWwdy7zqrtOnEvGhrKBHiYD8PnAZaK2Y+yto8OjzQj4TKgHJX/YcFBGI+iOxlCwdqHHp5Hc+3ymt7MULX54XzNfAaKtBLetawPJmTtNSEHVRFkvsghwhJeawhiX7MSkqSbF0Ru7tRYR+iEY/IMLufsrwrtTUjHrWXogyGUr3QiS4rZ71BPZ8eEYDC+96mCe7uGYNWhEf7r++c+FSQKNU/hQFoOS4BaA3trARGEzjrFkQCWQO0SHT9lQ83JS0BeHYGavjslhhl+XznOQM4Q+XUlOUo8P6VBWg/hGQxxU2627FFUnvYiR8A/aoT0F4y8uT/Nuxrjk7biZ8b05jv7SQZ30pRkGLUM6r3TcVoQdIcXWE5oqI4qOkFw+zZ+XIX5ATfFZlcdjFJyVkCpJvE3wrIK4EHaQG1Vf79HSLJafArk+T24qk0bvLs3IjMtQu6dFEm+Q9KF1VpRXavxswRlN99UskTq+ax3rm0FHoC76tFhXrGkUkGYL6rty23w44aT4lr89nqY6oh1XPN6WdS01oIywIjPIJ26AUZWkmDCkv7DPy9CijcKDxDdDuznF9d1mON4s8825Ni5w/Hfj3R92Pe5627I/Y9GV9G4zxFgvzNsTjOIxU/TsyG88cTAK9t6nAPEM9a2xfmslfqhCk+qjHmpsUFy51hfcayaXXEayHbV5c98qu9MhJk9PfUxinoncRmYrSVYpJt8gIKY7gvo0NAltNwiwzG2MtR1ypEIowORq7jk1leV93k8uQhChZO+gSlVwbGjYcDF88bj56hwz2R0nZalYcnQBcysxLNp2bZyahrAnZsqpqrwkOsNtoKJYUwR6s16LdedAzoh0v1GBTPJyUey05dn5p8+rteS1riGkbdzhmRCGqXk1hWkO2CUxTFez3Bgf2K4mgmQGMcucqfN4eLldoPKSyCXpkzmW0AXu+efARptB6wZjFiX/4SvqpYJdz5jqk29Q36g1ET7vJnyyZuiespLg+5JoASyLrmvexUAjDNEomLw+rRHd0KUI2Yp+9EgpKQjNt6cBJ4AMmbu6hWpqmoQzvSb2Q8m8q7krsnxi/q6O2HP81A04PNQwGMfludfoXIrvW9lzgZQ2jIvddAXboxKSERYNTLTbQT4iPNh7qjIoveFYGiMP3q1Fp+BmUX3AjiynpfZAuJfP46uNT0E+atUXutOjT5zwmiqCGJ4X5S+xpRwdx0g3iV59NQmKAnOs0Z6SpnZphuKr8tlXiJLEfC+bS1s43c9uqNN23/tMOV2GNB/cJ70y13GCL8+8xxHUkat92sMmej7kW/Uycs8qmLlNGTI2gdovwAuB4uwTHz4MlpH46Cmi6YwL49yA5uiXbSQrrzVo+Ll1dbnwrYMqPPUtySO5h46D46G6WnxQcBbWsdgTwt/0OfOHUc5lL6eZ6s5j5PC7i4peKcidqWVY08iD0LmJaxcIKT+IsijRNcCmWbFCcYcVSoU4EV9NaHHLa7UK8FuxKt/sFm89aB5oeRddsjL94T8Mf/0Cefb1Wv3DGq9fQHlF9fm95A79uRdg2edesAxf72UG+su9lx7IfO69WJL99V6A+4/PvSdGUixNc3/71qPr1rWE2xr0S6bBD98KTmOvVseyf0Ovtb3WCxovBbZf/gZSKUAlODi1ZdOSHb87hfJ/Q9nuELOhyxawISH07dvb7ReSvP32Q34Z4f3l67+TJPTlxF6lS/nlJAn9gmFfzpZZVZRfn018vTWavxwXvz7s0zbl8wrAUXmwWdt+e6PPZwSq0i/3+CzGIr01Fn1xpFVVxcjS/x3Bv84qai/p+bnuy4l5udjmlxNZWmT218NhWsqhGPqo5X87y0zD2qcZeA50Hf12jTYM43USvk7W2bK87eoEg0TrMlynyqVrv347L9PQZN5XOFwwZfKhX75eDpPX8bUM09sHT/gF/3YYfH3g54A7/nD0/nqUrNP2eTPwmOyoFv93n4PfxruOfhsBHHwb4B8iwjysU5L9M9h+Xd4lmors670QYvSuMXgmc6TlTsOi2ON/x9EvFwJA/1PEmrI2Wqot+8N7/Awnvt56H6rrpX9FyL+jKPkLifyGkGBfmt9jJI4Sfxzzyxy/DvMbqv0wMoxDfxwJwb4b6QsQfhiJniawl+Kvl43ggvkfTwFB8T8+CPv6yr8RwZchfyOJX+H3p6hEkxvNsQyNjA0dO8/ApdPu78gPRAL/ch3LfTJ0FdheHdqnCqQ2XCv0WrP5emGivVacSatL4ycK8LGswLygbEnS60/xuLPgqE/HLxP8cn08fbv825nrfX83yA+0ebGi5Y+kFLVV0V+fkwtfs+k6ARhWlUQt/fWLrkrTL1SbzdUZxZ+hAKZ/hf01Ls78DefAWBehzl+J8DuaxP49PPLvJPId5lDEj1wR/sYpf88TMehf54k/Xe2fscQflhP9fwAG3LNprubPwi5l9o/W7Ieblynq5yhZqqEHdw5/fvl/GOr6G3UXk2X6eAZ/vmDZAHaAhjxa+5+LWBj6HWKhyE8QC/4JYhF/FWKR/7GsnS8rF3xMoyWal2EC4Nwv1pHZY/QRNvsUjT+TnezQDtNnDBRFb7c8/7lUrdr225X90Gc/EbT/Btgj1HegJ3D4l2+axu+gj/wE+CjyC/4XgZ/68+CvugiIYubzl57HLFm+4nL07SCvDqBX/EoFWhRn7f3C6g9Vo1w8LMvQ/YRMFqAT/bhqeR7jJPTDqsHod8t2XQp9fn6ydl9f+4M9f0PpL4eIMAIRxVZPxnzskCoWA1B3Ddstebe4PungH05m6eD6yy6eSmTgAtZnZM/Xr08kf/1jHrT4VHcsBndUbstbzweGrGiaI+UTP1CJctJxlW2LsRq7GCrZZlvbrlr7wVtQKzC8XTWVOwyY+oTC4+E9nMZS3UHmebu3r4tGuVKEqrQerOsSrEdOVfk8+oc7Ope50QWWpWjF/CaB0dOv+dwv7wTkElL47Tz7HoSSc+E4VN3p8Cwkz9P/nCopi3/88b8mVXCn8Sw2RGl2Zfyde+8DKz+KR3bZCJK0K7tRgy1TPXaqmlwlaXpn36e3PsKZEh7uU5CREtlpTY4wreBTtZbHCpLiXnjpiQk/Z7cJJ1ru5CKW5J7L0/UhsK+ZRcWiNDVgmb/52SAXiNvoaWa0gM7tcX2HW61ZkfSJ3ZqrmUQYs+8gB/mynzF6cgJFVbh7ttNneRSdhfSkzHPcE5fWjM4XkPyqKC1xMqeIzccEEJvJEIzvh5ja+r7Qnh4XCrxPiYm0Ozx3M0byVvB3GQWlwowYdX71cJx6luvmSUond8ogPwLOAMDR5wOxFX+W70cfENKgT5qD3pm6vn9am9qVNRkF53CDUnNLOZNkH6bTGG9pcnsGS2NZtx2GJYd4sbcQpNbtqhndyNf4UkQo3aVW2n0pRF+Us5pTkYMxx6x6LOTZqHWRuiFl3k1t6ldiyOxUnwwB+GykLYqWPGFBECk5ysvgzD8Ns2xjepSXTQgvIEGg8ClQy1DSO95xzXtbtPvO6uZemZnHSq+GsBdmwKKcsjN4qfCTv7AYxJfKW6MNvFSEw8okC5mNu6qb3RhxVk7dWwRh06x9jR6WLgb1lgJRVBulNfJZfPuqqtx9jps6lStX10AeiOyFmna0X/vJgn11AOoGuTePHmu/ffcRh0zhqvwdGbwTylMzWB/m9nLiTeaKjlTmO7jh1lnUDS3M9Ik/NI/WYf99hgQGVijOh+F6MOVsLD7Rfh3DuB0fOsRjyIb06yxvUDtJQjW9xmR36hJzZZ0NrXyXks33ns8ZpZ7CJ4Fx2LwEpSSUJw/485psW/lkojH+wVCvXYNpQv/EWc5tA+7ryjTzJYRfy3KeNy4iZ3fJCrtpAiplmqy27sKcps6WmW/ydq7QfX1QLlgnGjbuE8gNECxNAHmEQzyqy2Ger6xFTomQrn8vkgu3HUOjk5GVGy+x+4PvXJOAEvXGbZ/cys8mbCeaPuqLk7weIA69gNirGrFF5d7jWBHMmJxu9UZY9GpuBQXucAdb4VLVekhxRCcDOVqJ7Rvu7t9Iep1RQhXqULW4cbwTmj91k53GHHsCDx0Il8KnUQA/KkI8UfH0JNkmAa34tLLsM+R7cOzSOUhFFBD4xZY4BBbewCDuQG+nBBMC68iltYMyJxBtYqRYzUmkNjYUVlL3/oQxMAO0biDbLhim3GEHVzFyGQpRZQmQhwKoe8Jvb8SQ7mtwR2q+SJlTjySDbG/OXbTFkIMdp9MBvo1DcJbjp8lfn1/X0z4GIMdysQXykwq9cXNMwsCl5wc6hUEpdxDJP0ldXePl1uUUuKOIw6rGwd498TR6dkqR9a1UE+DN32MclkhPQKB8sBQKBEXqTAoLamuoT9ddoo6050YkjpH3NchteLmpnJLvuol9W4d7qQ8hFF5cAZoz9uxUA7sgEj7EMq/JDRDyRGTv3GY/268pfQr4pNIFmdmQCLJXSMmhWv7g5oeNmO8ii93+0ypxZye9ghvfifD4xc0evF6zBdaE0FDce4mt05mtocXEabtBdz2YxScdCG/1kpy9JqHJ6I55KZ5ksUQIzLZMALXUC63HhdYpptdK6PARCt9hnsEtyQ9LYtGpCLi5oWS7V1wCTWLo7fmc2xlKdroJYLXsLs9WPeskYwaihGd/nEkYDPnt9H1swU/uhYbq9Miw7qlxt0VaAyIEnnB6w55lTb+fQ56627L2/elNKGYTjx0O8+19X9tOvVnWawXRwWDR4J5WVAAG02Zk5uQ15p5HoJCrA25aHDKE3jQdcvMoEjR/EgiZ9v0q3Ny+BfVl1UiVSC0ZWi3PeeNRD5o0yilrB5Gwz+X1Stx3NnFcpXXArU3Qi/xEkdBExqxN9BFqV9oEvEnpnk9F55lOt/MqaHLPV613N/NzFvI2wkuEioKKqMBvIlm7TRfVMfeAJKR0tJ6Okh7AYxedxpPHsIDoSempsVuf20Y/bUOKtMzTavtPswWVx+S3qJ4Mt8E86vl9/+puyCd+v4eApHx+bZXAsSc3Byilm/S5NT3GBKNjgABcXRF1LjvizUZDhKNINwvv85h31Gk9tM+uOyQL4BRuTxCHDwgqa3x1goQB2d927EFCHAgUrU3CpQxljRRKE4cW5LEgl3SJb1whnhxraNGn4ul2d+toe1ZlR0lxavZpg1eCF1azsmBvpHPfvOiZn5Zc9KexYDZk8FFgGKopG/fZQDThbqILWAEnFf1BvHP8NPEH0JJvsYieWTyfIGArlrRLvWL6aEcqAxx9uFhzMb2dOc5lj6fzgbmd5dxilXmLndwKb0CkPkmcIcyanmLyDgmb9A7uz7J95Xp5B+kGckrJnlGW2HqbN+Lu3FBDaCe/H0/ZVZhbHS52TTY5GqLkpwucntYfupiAAsSaThgpBojY1BosYC8HoR4kcYtj3BClXKXXYA3d4+Q7Eh01DNA6mBV5U2pZRiKvTd4wKBdRZpC3BGCJvhBWLC77JL34LeNF/XBkFqACN7BZnXsIlNSmvgZGIUhkD6Ujxo6sZk511mEQlV0BAjmvR7lgjXeY8jQorVibje9RVfLQ2niMrLO0ZkvXVhfmSQ9yrTcIRzIbns/pgTuLO7siBAjcedLe+b402Qwu7ECr+YvrSqFoG023tRVD31vhaRMb8U4w82Dt1UsOlR5Fj8u0Ltbp2e/S5/b07y+rjKi5QBYy8Ml7ThhIAFO4/fJM7XXwcASxR4rD9oIt8gurUuGtT7DAoJaave6whpiv99CqQXpK5VkTHJ3YMUh/pzrTpBH7/mpZe2El6YyOKeKRTDHfWk/H0nJpLs30gHFJfwywqtrtmijjbaux5eU1vc0MAPS3vvWgcXzB/GCn5xtEwi0sDW1WId5GvMZ3th9WzIXdnvTp5hP8uMvEgERBmlFjYhKcprqVsiQV0+++cO99ZWwviNQMqCTVsIV9PR7mefcu1Yx7euHCrwLc3Ga7mqeNr2NSxx+J9F6cqYPcPd3Nd84kQAqnSdehkld1vuqs4LE6QJCLzF6QsY0m8k6HkNi8JXjaz3HlQWFOWkQv941fGgwvxRglPXWqFMRueh/NvS77Inx6zijyHJZQvkNcjxz2phlfs4Logft4Kt4S2aRn6UCtqSwRah/tqAOGLe/4CFmN8GzNY6PdjL8YodirUa2KTHLikq2Ob19+l635kpo2ftYmjMJs7xuaZ52RoztvkC1kq4+n6B5ios+U+no9FtU7rqexn+qLpwsTy/11RmBpQ3Ofippi7moMMluFQqJcMEE+YyeAmNgl6m6psQiwH62HHdzBS47q4dfY3TVAxbrwROk9avLjoYSYn0iMYXkSo7eaPgTNRCz0cmACG6p+SB+K/0z2/TZXRUQ8Ta5Y1bNqmmcKIuf3CCuNELYfRt7WFqZVBTYqjbELuwKK6HRtkUbWqkt/X2kMFM6xlm58gaJ9mX0Cs9vwK2lZDL/ruX6TG8gIOPnRVi7mVAOmyFPJF75UyQo3zOUbvF3dCFbCF00JzbakJ67EKE/uZB6UzgS0TLxeL/cp4VZh0sTUXZ8vaZ0sjWvWg/kKu/65DbJ07MIzufTo+7RFtLcDIThONkCgtf225S/gxQy6A/hC4iXE8rAuOkTPMv4maFIHF+vD7nQM9EqHfb7pklw5isTccDp5QXnHY4VpRvQCe1TTJz1/WSIhxaxChDboY9SNPvb3XPSqNlIxcWevz2LVeipG+wnZmznjR4N32c2+GCqZNegvt2pBh+2yY1JGT3RdLa5btYG+FEZOJha1lcemQ7B3SmOcdzwSd3qgJV4DW41dqgYb7QrJmsJUiMJF3m3BvRjzFLngUJ2HsDaJQZjiA6AhDdj1JKqevA60pAfKNW0LdbGuITYo2B3RfNvY5jbF7VYJm+IUGQ1irBOr+vK9Gw6TwRgCflJu97h1ohS+dOngharHRvMSPDibcCfd803ldtVUSTi096iuVK/Hvs1z0HMJDUcTolo6XN4XRIvI2wDVbjH4cuFIknt0PC7YmdzvHQGiwgwcvd7qQ58GS69Xat8puhpjMUyUtYzBtMWdKJi5LFa9U8pss9VGL63KjDEhE0LPdaJ8cU9LrTtqv9isYHepD0iIxaaVtebXQN/PQK5fF0VcmP3ugNFYfL4vltZQBUVFw/ImQtLFlXpG9p4+CZ+MTcaj8lSiW4I9N5UreBqMCYx3mjoc6uUR9kiHUa6ipbfbQgUr2JTx8XW/OodZhxev5CmxCrih1wUH4T01GGRlNsuUIAv2otW3oOTBFpIcO2MPJX6fpUuNK2150cCtdJA8fVbWtMRoEMN/0Npw5+nH9x4S/nGt+B0D2B8XIjBXlUg39U+U1Haf5kPF2UCWgTPqL/IXYj/3F/7qG/y9xxD56zyGMPyDyxD7o9v/R9f+AAb9OOH/J7vf0e+97yT+o/ed+u+M6sA/BvF+snwIWF5QXRpN6fyvxmYeNKhBgeY1/vv8npes+x+LDn9HQWLDd6FbAkN/IX+kchj9i9Dip7H3Hyn8JwuJA6xghz6vpm7+RHnH9oL1l/jdn435/f+M4bPq8HeMAf9ZvBf/70SBnzGG/9NSYLB/lMoCAUH5u2yWf5rL8msSDfwLBKF/+30azS8EjPztn6fSfI4u0VddYAeI9ufTY/5p1svv02P+aR7NX54e870agsHfody/K4kFpr570Ncp/jSJ5U8k2xDoH9/zH6Tt/BfSYX66csSf4ZkE4Jl6NDXfJUf8F+ToH9MlovlP33j9fa0D+JQMXVcty0U+v576n8JwkdufSYSA/qJEiJ8iEIr+CZbbpxcRDDsAdBvNc5X8R/zyN5aI3X7PE+Ff4H+e4gcO/n18DaV+5Gs/vZD8k2ztd8v0M7n47dy/yv1g9Hu+8o0t/afTAS8YfI9z5J9jpf8uFoX+jEd9h2G/Q6Y0mstfhfcn32b+Y77NdxkYCEXiPPLzNA7w8wOtM9AlcS9dl70ELQr+APJH2M8X8Oc08t3Z20/Pfob4/srbPxiY/Nx9ff+TQeDvziHUd9d+2NO3fJjuKC44lL80a5xN/UUm8y9VAtJcmHGSPx+4L4mN/zq7uvTBX76pBr/lD/1q4/8xV/rbhX/gWv8Wf8A/Je3fYRWY9t/n7NLMpv/7RAdOUL9A360FfvtRetx+wpa+5x3/tkXAbn9CePxOX/+a5/bfoKD/V3PUf+U+8L8keMgf5c5P9enbn5Q7/6pAwRH0Fwj5h+UPPxh0f7FI+DH7Un+y7A+oM+9V10YflPnD4kfTt7VGwJqVw1Sd1/pH365IyqpNteg9rAD2y5Rl3w5+d61znf666N8TeAIuqPpsct4flrt8Lv0J1v0k3xNFPyLnhzTB5Pq5zl+8O60udPqZmPq3sAkK/YFNYD8z63+W8PmrNfNPMOuRJUvUF232u4d+p9XC0E/40s/Sxr/nS1F7ceI+WjIGMIj5r8C8P5F4+h/k+H63smmUUXnyM1wgEiqLf76u/5Qo/vRiY9SvPruvkMd+hPs/cNT/VRLhTwiE/yR88YxKsZ/Bl0JilCD+Qvh+Vw7xa4L6f5A2/ZfB9tvz/6y0zdp42H/vGPucuL74pun8EfB/lMr/OaD+hwLxzxpi/3aB+K9B/Gcu6f+vojP1fxo6I//3o/M3BPrvx+efBt7+c5b4D/UVf8TZr8r8ZZemn2WA/oxR9aXi4juiQBmKE+Cf2LhFMiK/bFW2/79tNS//Bv0I/T7k8XNzFiF/QX9S+ov+FiT7t1MD/p+khv8tphSE/lr/C4ypv19niF/P/GdCFb/5B8k/uAf/qW/wH67/fym+8fML/6wj8F80yGAS/wUUpX/7+S4IQX4XRPiLzTH8T/iA/zeWY/2DIjrib3+qiO6/VogFfmmp/1KIxZlQ9kxpmo/U5lux1Xlj8KP1YLevQcIlel/F7j1100os57lBrG2F1mP3eNGRdIIprDSvJsysho42xS7nt3LhH6ltDQOt24JUUA9ddqQNdKwXnmQvCSRKwAUOY1BPGrd+Qu/m9FxR8oWs0wtPCGrtM+MJkzXc96h/sYaVhFucKWah0JlCP8aB43daLKjrWClocI4Z7kxAa3TAgf8jjrVYbvfoXaT3jr7+/G+9Vj5HXWh4To/V/ZVvqLsgCOfGWGQIYh8ZVA07aimhw1MBXUiKpqyZgtIQPoBa1tonxQC1CCA5MGlPHwIFE6ivuCgjKWzF7HobRNc/7Atje1+IsYDaVHwb0znSSK7pgv1ZJ3yFcYwnDBR6LsfOcQZXlLPeGV6F37U0eU58gWQFEbLadFPY0LWspbHnNlACa2051XODR9GNrNRT4mt697cnO5L9nicU45oEv7Fl8+mv18Mhm8lheWvHg8IJP32HlJLhcpRbz2TsyLHrM5Wsa+08M3iJOQf/BjGG7ZVhIO9GmqJ54oinIqAaKQcwXGgUs3UTUjwnLF9USOOLUQ5sGpSTbRBNPGXVnp+qiLSwYD2nxWE1emTbO1Z7rAPl0Kc4a/p0rNRwKUDd23nyxTzSQaNwnG5aBmhX46/HqH728Ghnsp0yqiJuixp1LZJwBcXzhS4EqQMS73oJadXkUsxWKruNfiTbkse+n1GuFsSbDOXAf2WfTdqmMCOkdwstL07JF9njbdmkG2XYGVCm84QGprS1arX3B8NyfCXE4y3jMj54IkwJmzCbbSNptUeHJPdvb3Dra4RYXGIgyfMOv7zEfTDSBdul6Ir95IxOi/T7Z5ONS5IczxWUr/mkU3ya6rGgpAi7NXTUCBKdPG6m8Cm9GxFIw9s3J3wy8M6+76AxC8a+a3Sm1GmmMxTKP0G3w3uudvenAZWG1OrWqrp2vHQELGQ38j71jIysmR6kgs2PsmPtekFVTLkuOOOytxfjmLtwv5soufowacN9FY3Js9BpQNMB79yoJIreE9gM+W0V24GyK/5QA2gpu55NSbTK6/xiPrqHdmZRBaKr029uaSjVqfi65IpeDvGzN95OTcbePensoGYD+qLEhFIVLNs7NXuAMq6R3W49iurO7EW3SNzMmXhkazb7h1w4li4aKh2FmxyiJs2vWRP6IHTBBG7PnaAAScC69riIn77oWl3VMESwvumXOKccmBKGkpLLUTRXeMwaPF4kNiT2wqrodhQl1qAhlx08llpcQclZSE5A2aXkEXJy7MxnjTXRS/y9uVHwvjmpAqEVTGlSvIhQPd+ZEItApqvVcbTOowTq31PZ9nmwbFxy3kajAs0ZZx7TJuGauUPTwQsmR46I85ivaxNFi2cUlotMvzCtayiuDWIMmuNOogpRDCjjzY5v7v7Zh4Eseu2Y3TxqbreCdcQPLF+73Zeosd0+qa82xFPx2HZ2tZG22ZwIt1xP9LpH7AYlo6Kl2FLTyIEuerNRTzFoA3qUSvVZcyEIzmEnYtR8fkl3RlXqIGAyrBbBwZ4Iwk/9yXMG0+xasHVVNuPo9e55QQABd8M5J+i2ubc+s6xFT20oUzD2e8a9V+F5q5DYWcwZFNHc7ZE6wlMpvIF2k/sdNwnC6pEOOsGuganr5YdRiB9uPacWhtaf92HssfbO/dVule5V7yy1RiyCSc4Oqd1/MgU6a25ZJFEhLTxWsKBujWHc8mA9j5d1hZbNw5bqVy7cDN/fXimEgh6hKx6cxit4H8/BAYVCz3sHilUwey7Cop7jpqbb0vOPDOwSI5QS3e9DuarNo9gbujil7bnUWCYveYOecfeQNthVh+fRiXR0vToPEvWPQf4UXVntHCWlns4Cles0WmWgXSPHaPUXXoJ5urfBUsJm3ErHu1v41IGJet1ksvwW+jijgyLmtzniUTOMUOjzPjFtYq4lhkH3eR9ATwVIaWBS+o5pxcWDczSlytDkxEdBWUej1bJh2M/6PdhhJF+iKFMSC2vm2f0sXFSvi7LROSfRGqZZPAKfMJS/EX0gIW3XRHemhXc7q/pav6LABJVR/WZpRb43JTpTdlgdqM0ETHDwAIvIZBodTAMFeQLNUcinB53vjciBYTKYwFFo+sNBcUf8dIgD5YGgD9fmQhOsRoAZ1m/fABLJM1dQXkA+1ddDB7cWPp1SJ5KdAieLbuTQfEDt+F31nSdCL3j1DOiNiFy1qlp0huzwpJlA3eED8E3HfvUxrWB8IlorWPZS9u40TgbUcd7ALr6owg5LcAqrsguWRknieHPtDm0UiinomQlO0OBdmNzXaO4shMf/i6fr2HYcxpVf8/bKYamcsxV3kqxo5Sx9/Yi3Z96iT5+23QokUKgCScCu45wMRN6oLNETDXBI6xGQJilpUFs0s0G758L3J9p7r9KqL0cBV/n8XYV8rxKeANS4lSeKpK+cmZd+y2meai3zGMtVCPRiL18wFpf6+6hOXzcsm9eoK87o0gcrofj9yH5tU4q/13eoEIT5nkIMuwHhG4/sY+99WOK2Ka3sMqJ0XEFo4hvRp+JBGBFSmhVU6CxVybrODA+unwkGgfk7s+xJftkM3gKq8j7n9YsHMPpKr+mQmCOqtG7M4IXui6Uzdm/dqnO43fGn+NV2cMxV4upziEDRajr2d1XJUQVHt3RgytsTLKWxEUESTl7KuwjUB7zk1/YZbK/bCMIO5PaFfTNSBa9eRDj1RajTFJxrSdcmlVtgEJ8hUp5fVFwvz3gi5P7r5sSvNeinJHYqBf8b6RFs0//ALUHbsngcWTI6z33sirpT6VaOKOy5x/LbEn4Ic/HhjOi0Kcc5ztBGf3/P5wO3eoEwelF25zTCwFv+V3zoCs7sSOYV8UtHstDyfrC3joS9dKEB4/T6DG+YT6X8ZFDi7c/6ZG1v/kEalnay8DW4ChMiw9ZsJFDnDRxeuyYQd90aYQ7Y1IbA1I6XADDYi2OScYk2bWtaRDD2w+V188aHAYnw43lnNRWGNRP67Svaf4glbvFHaz7JCdgv2youOMSrQPOUnDyFklyuJS9XVpO1cnWGHGOMQxQvO93TuQeYvU8NMI5rOAGCkRRH8TfT3QS6Oj9XlNl6YRHM60UySp6PTZQ8ezDW9PX02I+tv/jJi8NV0rw3dzsj0d+ltwzPH2CEyP2SAdXgRYP3Yz0yPW9kklS3zK6K4UxQ38ebpg9Rd8J8W2xlZ2JzcWYKYTPFUzwAVhElupYHDf7YxDExratKuoXFEhDLFnsWYNLURNWqcn4hPmhH6P4cccd9vAuvgquSKP9sL/d/73ggIPijlEzxNP8FB8nniov58fRBvPz0sqdoAiPvdkBk8PAoHT3wFQo/p6zwEecJUsg6fx6Zr+BQK5Pzckdo8YNoDcYdeb/WLeirLL5BSLVp1eejF0iXEWnNkLgNK5Z4iim5BphF7Ud/LLxRwWxZp0zh3Uzg5GfBqGns9oXLb531GuGlPHKvmhGcPKefVSw2uXyjjzvnOz11ZvEbaylk+n7wLc0oIZu99vEGTW0NHJq4+68dJbNwv9gCr26/tAJTcI8Y6UQEakraQCOFkmoqZWXoGe+s4p5bwpiSSYXvABwG9EBs1siNyfWQOKmJBupUvTyia4C2zL+VaIgHsv81nF+p9cLBKdAp9zrIeyTuwY/XxnhXctbE/0FIlc06Y1w+qhvXqtWMxaO1C7IMJ6ECZhoff4j5OAAtRYGhPcSYbSbMpXx2tvxfCfDfWs6+3+9GaE1GCPx+3OwF+9V2FYqVxcMUH4BPn3+zhK63N8fEp/KjL0u3zsE64sdTf9K2RWJoC4fuxH/tsyoHm4yVa2KmVUNgdXIOsEbMmR7+pTRVij/v6jib0Ve91Y7brEfikJJeWAEU3Z1gaUPIXIsAPWyAlXf57IxHPFQCRgr2kv1h6gRE1KgMLrHFQcUQkZpEfWaVnQXem3jg7T+njBkoEfJYTCAYu3kWVq9Cq5WQ7bm/kovfO36uy/XaIQ+V6kUMu9Ns3PTmX8aYJANNVdb354dk3Di0Ky3VXfOrZTEF3LE3DFtywCxKY89wp9gWNwA+kxHxX/ePGPLjF0P2FyKdhjqZrQ6vP8cJLr6Q59g2cIAoaU0N1FHZxaOyWKEF1s0iDAvAovX6LL2J1dYvX/uGRyePcIMp4Dlx0LJDHJjsFtUPwEtDCyt4lYw6FmrWnDD0rwL6+MZ+/8WwL8mMQPR3XXZldGrIBtPqCoyXlQh4900pJRNCQEOqJV9W/ItM8jsDr4SqdRDOwOE2lmJ0xTMHQ8ykQwk4gyyfyEMx/Tqjk/GYtfwLM2j+hkevhCA1netTWFnmrH2PA18hm7q8b/IJq06ZL+0v95CA09YRZZ0AjN4RiK0FIGi1XlV42qPsN9mxpC+gMbby8pzifu0bxCnrj3yLK4eunhOK3akVr4frF9sSKAVjUN/BgOifEriLlURUSEevL/GAV7PdWjKxnrXVtRj6G0NbDc7J8pe/rCNfWWC3dgck10gxFhNUErSB9lU5TbFvDHikwZaeF6ftv5/yTPH5a6YuKWzLiMra5hG7W0YNVL4RnoL+1+6Q17ZYP/zf+E0e4dVgvBGVIH7VoNmYiO2qta+sWLGxEs6nOJwDI62qO32d5FUcH8ZIea0EYZ/DAQK9rPEKD15hKH4Gniw5+eevoMGI1pWozh7Xw82LjazC/gVSoF/wky+eSY0jqeVpe8LaGWQP6tJhmPdhmyVn4RpLmeQKvBlu/c/LAkUX/McdIH0aXpbe3l0VIm3d0dkfJFynpqC9zWYy4jNHtXb6HHinBrCdyIEH3uCeqhqX4vlyY2nFZBnkgKggDhWX+dBJ7aTXVzjYI3a7B0QXdxzuPytamzxE3FFYxuf3Ugh09o1vXEjV9CGr4r3xmkWvLZXBV1cy4G4Jdou/Ei/PiNHgn9ec+1aLWPwEpm91AIQaULuF+o2ZgK6xwCHmhj2UqpSEe+WMUKl/4wpLDo0+ufdZ44c/fu2eqkf7OTtD7/u6TXc+OsO1RN8o9l7s7l/jWjikA2rsKK+AOxTltS3eCQ48PnOTQVcRUyKGXmkA6zJTnTezO6Y8yHmMtva51vaqvkaknerA4qb9j+yAwLaQT2F8rEwTaEEdvVr6jdqgqgTnV/PfQfxrqfNARFDRYs0w1Ci9UsQuS6Eu7iZsL1LxJ3UzqC3RcIMc5MMwvqRTC46fTyWO0csCbdl/nYeTEkNJuUEMpaSL8iX6Q3PkxZDHdpKyXRyPFSTcYlRC2gFUCfCLcvlkL94bp8KcLM8ca/zLpOSXXmipYB/MAiicc/N12LQAcOhlVkChKB/EHRp8+ibUYCRlm6J3T/1coIoblwH5c5qUXhayct9qKUyydpkyBt4SNIHFEyAJVMKCbUtLH3lpnYtVpzLMgBicxAiYOjLqaS18U8hrRjdgwD87nqmpeIasX2m5n/5Ovkem9Y3oPJupCPUZ60v6JiAlG1SB1T7Aj3l/oedixTa5NGOe7ekqGZWNC1eRPSSSweyeMz+OAlf5Hn2xkS7IKZHAFVrPAtEWBCb4hXLQUPCLL/aZtEz+YzpFuTSXefXSyfsyuzSgJkZzrg3PPdfE+TqznhXrRBDcF9BnAMk/MYoIr/LB0AUlT8e0olaoxWH2mCsMVPDqGGvd5EKRTWcyRnFSCVXLH6AuVFFHqD/wNrPJuQ0Z6TjRENvywCDKDHsICsIwt/4VwfORFI7BpMbmKnEK4vjetan42O1LcureOtMvgwtV7dpF9RAgPuOQESyXSEcAuXTurzz+a33lNUwYlv1W5aofmntgMPG/YuNPU4AW8Ft394IadHTgfDCld0FA4F9BvTSe407lfTpsb1kJw3SS1xt4aeYXo3w3xEBK80vjff1Qu7+mHvAX0kZZjPhptTocLA2QzAUD/zFi3gDH4BP7tMNcxSjoApHYhro6z3nSOiGy6HcHLbUg/v/f5z+D7uJkCAkIew3z/TycckGIjaiCZX4fKs471u/2uX7G+4T7LYXP+ipA+fnhv2OlXyL+qSzAdETle09Y6BPPEyWhcEpvtDhxjfkwaZ+WaNrEshG+HKg5LYtJKv4DgutvmLyX8U9UO9i0FL8Y22kVIXvClJpRUac1N9udeBgIzKsfVv1M7+TTq0SRUA2Ij23QN/3GRA0/88cmgjjT7+ThQmHCvVSZkxfVsosiCFZd2mSNViaiksT9GJ0VgjQI7XgmLrwPCY5Ms1sWzR+hk/qrRQInmuiXfo1/JXgsN8qO9MDwwWIlPsHs2qHrr4Gg4/LXVrg11NgK5+FI/bpKYcQQZZp/BJZEuYtgboWCplfwdhMJ+T2zEClKnI2/+GKbjmPmQ38acAtDZtoWQEm8J7RMuIzDVy2K64KtR8OpTn671fGjbaMTowAtuORZ35H/I9hgoaGL7gKgxGpy9l/J+JjnQt/uUPjbjJtzAWaBXp2plI8Cd7m8wi6WTArluwEbmj8/3cStVMjflc0Lry3h3l+sZN145CwH6TAPrGJqkZA8xJXmO9eZ8CQggwmKM+Dk7PhDmksG9sXgwWYldupA9HyfcgfB4q8kiEe2moxZ19dkiKMN9j8HGOrHeIVdq+wDHoA2a2K8EOvEmumQccoEJx4eNqaQBSjn/Epv3Ak7J4fBaQQbIZXCeiRgyRFo7cC+ntX12I1Q8aX4nkmiFiFjpntsXKQ9Jb1B005Ji+bzCBO4fXBQMGEPPwbnFPHJYWTQcG/nG+8qQtcD0yBEKqj64y/WB49tzTfQzzcsOCXmMuJz7i2SunmJkkE1r2aDazz2yMB9eLwqnYOiD+m1Ffcl14ACnVG2tQXZrvY7L7wGoiaLUMp9ygPqAFrDNg68/RW62OHpJ2jrV+NILISJxJJcY/YnjodQ2AlYV/Z/wVdsaBKtyOX+qpiHu8K47Nzzo/8aCByg2AdLSx0G3QL24h1LVMnRe6JZPvpL+XFW00G1ryBCbhs9/6T+VZHxB6mN8HJqLAi96YKDAxQxKuHvXi/md+zv4AosEVSBE+c0bqKo/oamB7VWzl2CIxgiUhMAEN83MnvWhXg8dq4EE6e8HtaHNSkAioW0wrRE0jB1bhaDn6yKfYEj32vR/gmcYtRBb4TWiPF4Hp5WGRvtRAmEz6mMg80uWVPR0rDTnrQYjBfK8nHEmvzyQWAx4i0KFZdx5Nyq3TobyEji+d2YMc0tiUbBpdRUoCh/PEOk+5rUHo9Nzd4KtjnuG6UPjMBvWwtL1MttOfFPOYvo8ogcGIwFpOyMuP+F3xcv15UoZKH5nJ/WEQ0vPn17xPAoL+gc0Lg2DTUyTpm8dj6DN3C6B4SWnVoN3Bs0qt4/KqGKe2Noe0vT8S9VmNBuEBMGbHdgJxJ72NF1qL87pxA40OwU+n7ld65+z9kQrnI4Zn1w7uaZtaxQwJu+H7d0fmjY32Kicd5fsyINLWgbXHzVt+6vXzDQo7GsKnNj4GcmH/R+638dDnkhDl5be+E89ba0elk2q/jh4IdMDFJzrxW1Yox9PgIPHm7kkoIkuNraXWqsPBCBgslqj8SZsXfIfTHmWw3VURdQFxd+uaJWHb7i0WcYmLJ9vncX0RfnWSrvoYD4NyfzG7wft7DSttSO1XM7aWK9cZAJ0f6LWJwvijsxtrh9H63k8s9ouz/4cKZ+hGT4pZf88GoH5LoHXRHCuUvHAoJatrqbC7SEmdYKxSB2UHZjxuVgcB9U+zAXZpqep4S/++XlfDTA1qMACgVUBf84HcFPKfO3mBtxylksp7n9BvTQqKsqXq92nh4N+g8jyh0yxgzd5l2DSWBhdRlGnCi7eew9dZBSSzzT24cRQJlduleSYCf6Ft83YrcfxHDE2FCfbQhbdLI7tj3qwZDSqzFq5g6o+NUvXIdVfvwKw4qViJMMvDYBuZ8n/lGFg2uwHzwe8zKDJ7uHRhFnNmDgw7zSlv1uOmdvs6sYCZY8Y/xTwqabgX8sYhkGq3vD1IO+VvrRDapyehG5FGFRDcgF0xduCy2SUp+o6qgMYruMAlIrYT4RoJ3JDUJpRspMW/FEfuP01AR7PG1dzPEDxA8wWJjysBuH+rDqS4kaG6APt3qG0eq7zEwW+Ve+0seC7Z8ghBQjLI9PMRzKehmQ9dmIrrG2hmn8QVki/42qmg5dGgwboSeVniID5mkQ6LY7ystEluuLmVwid0ytjlxMgjWDqaWts6l2n+D7P+yz8c9fqVjgNGRV0wdK4SvEN2HuBjJbIjnwdCeuBOAwdf3Og5wSBr72j+mz8gKK/MxqhChuRiDmJ5D/IP5PH5iXgzaROC1S96KV33IqFFEHqAfEgl+tDWj9Jf6eFGI01m/DvuRLUDmyv9cSCvZXkOytYBWN0ZFcREpcS1p8VQLvTrF0R5utfyWZuw+xR+EBkR4vQug/NG6lTPncMCCpdvFg04U7+EEFJC33R2tl45ewf1KxDEiS61vzg8wVcdAPkz3QJJ+NqkeX9fy14330KdjQLsdM/OxaiOMzH9dBB8u8ZQzqoUqw3UzEwohceoY3LAiz8JZeezvZ+WNqDSmWM1ewnoJJ20oPaEMrvyUM8IPcP02nVo8tK2yKuiGbxVtdsR49estimOtvhuEQ1vxAqlMsSn734Bxk0NlgGVwsrqv03YtRg17LxbWifrrhYlP7VftMwMLZ2DJ7BgnzBTIr6G9t6+7nY1Di8dp5qahhuVZnaxu7oy54t9a4PK+j03JWQEOtb/ysIOaX9qtIKSz5nGRfmv2rU1EpiV4ZQ0/1WSoticu/7mw4TAi/d0MwHs9U2rKZShhv2TilGgx92kden2xu9dymijh9+JL1M65XvrByCkwqRvFDuN8FJYtf00jK+MZuk63DQx3Zxpsvd8PiKrG5tMWy0BM/59drhntcP4145FrMKKf9F12Wr7h5f8318AgMEbvbLKZGqxR8P4tqs5EFuJk698PKsRr5E1bsOL7Cw+DPMQiZiS7zqwWC4DESFJPiMSF7MTLZNlDLPe7DSy5AyhGlrdchzODXrab82ELLh/K3MexVODDRYFgsnYOFzCFF+d0Hbbj14+NT0J6bs7IXx9F76XzREQYWLkX61GkFJeYgYzyF8fl4O/p0yTLwznhNL316Hs2WIi2dEhiu/kUEadIvsJ6ZehCBo6Nd/OojtEmu61U517I5wsRR/MARrO/BY+XOh/oWpQwCHVtvNKdW7HOtr3xScuOmyT7vQRKG7U2WMzQBo2UOrZ1nvudMU5D98EegyNF78cFi2A/rUAhJ78G3WQoZiKpqh6DZasbPh3JKio3T84VZOjxwwLFzkcaTbjAO5cPlPJ1SdDrMKBFBn+/6YyYZFVUBN2IjB8tm6t6ySW/SibKthviFEvvDp58q+GeO3fOlvgiVvBzS4tg8O/8lR9A3dthiSW1P3VJvtJDaDmIhCezK+YhsFQauq99judgRuxwTduUcycIR8QtXMqla4sgef9GJIhgbe9Oc0SIknzUvC2CTncNX85vx0Pb/chEJjOS6+ZOlwPtRuteRU5FThBDmjKRBcz6Xj/AZnJ4MBeiQ/ro6aajXxWtEMHNY9CWRAzQx51WAV28Ntc6hA8+LhBpsPPQe4cvv34HIqI3+SLnudF6p5Y3jKorXLE5v8ncT37NRvKiYzi0dynq8wIPaqY8wTpIS14/2obWI1OX6Wo6KYAxbZDOAknrM3ecrKpkd6WVoKyFsIH5W3BetYAJ8hsdArRB+XlWRG4rdKjBX5qlxe7EBbiDYHOP3mtLrPSC0watTra2WYXdG5G/AFUy+lY2bTI0Gp+0v//idFn69UMN94dVQV+cDNb7j0y3UIa0YEgwUhBFCI26TR8H5a8QcS8dpNK+PcD9HGXddv+SDat5YhsgG3GRbG3hTwiTYbA/tm9vASYk2fvYDr7SMzgRMORijSXooyfVAH0p2+8QKEbuxxZHR62AfkeKL3qdxyQ94BxOfm/08PMLqllrvHWSXabdMlT0TRW/KIwHzQFlflsbsZR9bUXg2m++jsiSciUptamsFtmcDi/+jNNmv5b/byT2nI8UE9iumTAubQwdrnXcTHNxziN+Qm0N4XF5Lcb5bJk8kqNnt3ceGetTnE+zZX35e7id99Mxl9BjgEfZT5KqrOsvg6swfkH/hl/Xc8oKEnrk1xZp+V6+v+fxluUoPi+YaT9Mk/DGUbQ9XC3h2Ldm0AEWaIVrMkLT3jte37kO7Xl3D7Kv1PbnQC3GjMXvHka21kUebR69F1nFIIXy/g/bQTxiVcKrhVoINH8jADe+FVpjvnLltVq/FuO1Y4t3FIR7obpw87+5nW0Fm7dbU2njIJ/D0jqGUFpY5/23zAYGpy2y0m2F1YBewwveK7xCMw+dGXpHUgh1PnxCpqxePf5Qj+xiQiKcMsefTIB1qfUvZNwSzqTrVKfBx8VovSdDfpuhJryvT1HMYZqB54mNsV4sxKQiPXPChy78c/94reEp83Mez5VUL5AJp8wlLT5yv2lfwDGwNNz2pTL9T9wsTmQI/PqL6lzk+2jrUxRNhqXIBK/W2+2rnQg53YYig8vu3PcZylv3SkMJxpWSoj8XhyHm+5uAClWADnHoZRnaiOthrc9zp12ofNzqOfmNHpvXinChHHKr6b5b+6p8Yahp96UVBeSQd+Wv28uelUYefVfxewuH4Zj9plBwI7ljqB1GsKHLH9IuMZR1xfyEcADdO853vjpJifRgoT2INaMrkw3AztrXfNYF79AXKTlVyOqerh4VTzx9reahqP/+TIuk0QkkMCri21teElQKrpEOhxjVjfWmagkJF5vilXRRVJ6Oxq5GkAceb4OydrGW/n8n3tCIm6jYk3PwRhVSDDli7L9uml3/VlO2bwosif5JXJUG7UOkoMBVWoJpf6T+e1WfJM9vB2X1+p0g3lyS+AbJUR3AfHQR8ObOq1tuuZEew4/WyUDqpPj/lWpdkFZDMpglIlTqY7ihhkJHtKaqS/FqrnWYpofAVmZsRvjHCYKzJACnobKMGbgJFgd0df931fuoXmQvjY4AcxkTLp12A9Qb0hw5gQ4zjF5l0d/SzZ6gLTdpITdJaGA9SSO/AR6Kt4h9HLb6Icv4vSWnhP+Zj2NeZA9mPYgv4kBx8HbPZWfMcfUR/ntEdiy5M4k8It9PjyHh/7a0rmNHpso+1Yws532eLlwzO3TpdefltKRqV2OYe2b2TYJMH3sYciORh4s7l4MVEvz/tDTlM+yUHHKQp/AKzprl9YU5+grp6eGpFsOwoIOqb+hdlQDvPOXyNC+0+iHx8nSDTB3Ka719+wwTHZBqusRnxATZhQG48oxT20sMsohCHXeu4mN3ZX+su78HuP2h6XqKcSVKTv3rzex9AhTz/0oeDTElAQwmVK7HzHFCwGnH2Gt9BjzPnR7KbXLDWbg6vuhqYu5DXpKlzCkXL2AQ2A3YzLwwaf6u7wB/4mjgw2+yUSV/tdhWBvwRP6DSUAvueY+eufKplN+LF+Vb6FJepxtX5qXCZ3yfkJOm2BiriLzucpRVsjAwPFNbABe75cQvy88WyxispUIhNnIa7BCoq4T8qr8pfbsI/+FBczwPZ9U//ULzvJbyZm/pikc9vXpNhsYjnuTDEspbSJYCMxLe1aBMQ/PacgmfUDYbwlYLKhvKZ8ZLhVkAv5hlaY3I1wp/lkRp+LQNOjIv0vnzs8m4OTWr+9nuApQONBFnBraWKP4sDrACKkRTznEkT/O+KqN6vOgT+NOkPK0NzGfuYApQ0DJJQRM+sIkOJl/3cu4pDgVgll4NI6eg7ejkxWaaE3ZJqVmKKV3BG0WYonOoLMkAeLvWrj4yWLGZq6ejlC2Q4GYrZE2t62CNLRr2wdV8f33g0KH57WVm9l7TDEEjHuyDpkO1hfUEhw/1GxYJxTflsH6LzZ95zOtVsPyC5y+a7+Gqyb6TcEe2aaLEuuA0/2dSPgpF5w3h0DnbNNd6wuRSav8bSSBlXvDu45/X8W6W5qlP/2U1NilJtTZNbykkNk9yNkfoaFu1xozoaKsN6vxTZDQxKTG6+mSbfsgh5HcNUlxNvX1qyt05fxNB3XoZ+6sMfnp62qoQ/y8uv8FtLN9JyNNKt4ZH0YTCn0kVGD8lK5UNTaC1BqB3kkGTahJw3g2kWhXkPu6s4BOfQCDW0QiE+pfqP/g5f3tbnewKKPsT8w/kIlPzoQc4Vr+X3zVkokI03uDqIBvw3Tg7eTOqAvxxpXeciv922WEsOezQv/1SDkEZEXeSoN+G6Cdbgm/JaOxL+4FR4jvhVfx2aL1ubK2irvo8ffa+Kb6DwJg739gXZIXfYmMvkJC84p/ZQPP/nt+r4wW9ZJi/Z2fjBRRG7BEIsON9o3QFyfXj8RbUXXEIZkipzgwQEjwaeUCd2XbhHXhziBl7RIAUPxwNV38bBaXfny6Tab/oFz1TSetdi5TzAlK3S4JqOnU6nXR8g0d4Vfla2RbFLT1OMJyFt8zvytcw+3XkOQzAt/QBpn9hjgvsn23dRY8FKX2x0jt3TcyAvsllrMZW441V48REDaABhfimuSI5TBHe6BtPmWEc0igc6i3XT/W7O+6DUq7NukJaVgnAQaSQy4LP99oaTw6AjfOs3QWsYi6Qas/VXNPjJx/nuxSd2v7U5Hw6mZ053YZD2mNjyS9ZVQjaIn+m1VV8sRRY0zHtuHuI8KcwoSM/P6vAyiZTfRtWvLseGIJR2yQ/EcPvxgjG4GgpOYPhYrSFlqQ/QMNElNF/UYevUSLm6/vsnM8snn6uZJO7UybPXBs6uSDKbRpbw20T6l8/k1lvQ0firJ09+kIuLhu6O5NaZbg2GyXsCtXHYSAywZH6KJB53Il0p7fLr0lfFbyZTR3jQL100iKuEv3h7IJd16/pThBEqR6hkCnD8474EwFqaqEWwgKK90UL28iNSE45UPgMVmP3fmZWfs7ajvhwa0BsUS7x4UKfVyWefDsXM3LLWUh3+Tt3MDnp6A1PZ17dbnVmtGIky1hlECGu4EXDO5QL7QnjM8J8EGXgAgXKdSVrjvPcu7RJy0qSCMBzgKAtOyeABwTiV7BkytdL7M+lgTbOWwyEbkEb5YPbqg9+e4JyNNN4M47Osl8bLX1OEUzv7ifhbFCXEH6uu/401TNrOM35GbgOehtuzFmy/IunFX1nUglGhJWfnKBO8lTz4ry8GaWVrFkcmLHk/sMuDDaef94EQRGiWNaUooV8ECZPye2XQFnU4ZAn8ZFFH6HwfVYeh6NhGbrhBNkUKj450o7OOnYP2aac0vqecNDtgURTMVRERRWOhRSXRHAoPVu1BMu+vVQzlCDPWjfBrPimeMLMlp1bneDo2iA5rhWxzijW8wEzDpf9dqQPJvuJEAbvumeEF+mhpIvtYJfuyYeY1jGtYGKlbrhCJc2XFTfGmIR9Cs0Z7gfRQZPlwk6ZlHubGLm5n/uVfRYqUyC+L8cP5W/xx03D5u0F9ws+kYaY493KV943GDAJj90tZLY+zwSnnz6TobKjLDi7245iQwPzwOWG0y/mqN2FzXgCfzU5TdDwFrDglNP/SzQiS5Vv6eGAk6nmFImw8MeVVWkzqC6oFr8UXwny/cPpKzJ6BmTCeY35HGH8p5Ex/ADVQATNIWjIbSnZt+PNh6fY5Dd5ItSDWLot/ANNEZLtK0AOsep8bi1CrUoZ6RvMLkE5SurpDLzO/M07ZYT6J5AU2WzN1iknJKlylNzioe0hZzCU4vvc7DZQH4KTFiAz5G0w+Q9aQgIiyYJu+jTulFCpi6kttM1iA8r5BQ+JysOS+fLrrqQsvOz6PGrZZeOghyY9KvEk3UZx5etjK8BpdAbhl4oaHyuuK3MtY9c1v8xSg7kCwcszKi0Xp+uYIJ8uY85GaThP+eBt5g96lIl/ISM6tgu6Uh05FeA84QQB6Y4hS8bLJ0jVuN4cg/Ruz1MujCPsBEpOto9Z+Y9UVNu3nAL9e5p/iybJYB25G6pS2MqxMpzhZlI3C9R/JINFfHtAkBFbD2uQdXMTH+JexDRJ3KECrqaz1IA0hEDT+VRj1NGbA4ju0LpwErNRidwBMGn4NFST4h+MgbqR06RiQWndD8YTgV1V1vjWuSqZj5Mehg/hEybkJQTz33bks2ZSLu+N6dKbW51XwNUzK+ZdHB8nVnlYR4Rtb6h2ag3T2WFxABDVsjO7lqcYLauTv4nmJXzDv7yBByxPSkV7H6kwEz7q/5WK2Cjy0BbaP0d+nMCgb7BClN/ACdUbBn9KUA01mDLy4lBq8yHZkZslm2XBAtzmQM/PwH3AN5rO5Hz+gUA4+lzI4Z7VXXmAaZsiki3r0UraXUPbHqeOo0RwC02b5fV33Q3E1WEwA6cxykcEbvuNkxd9JN/BWIH8Gf7/2U2PsSC6CKzl+zUzgiB/Wzg4RCsZxaKZlQ3SGCZ8PrBDKaF2oDjHeDwCkymFIMpTHseZCrA8NbQ2+1Yc2BxYuhgMEEakZzSWMG7tF5EF8CJVuQKjqxOd5DSYwU9bWkCOicMlA6NaHYsckY2ScoZ9woZe4HIRcOrDQ8x0DtxRHgDQLXMbXy38F0lRSssxNgee9nyAE9unfTu0+gUq2FDjlK0pPbNnn8CrtKj67BYpOWaJAjPG/aGnRryYPQfrw33EidtfLvlPB5JAP+sA1GwBS7efMyVskeYl6AsQUR48tfaMC0fReFjXwlWA+Cmn784VHzeqG5rpQnIYQkIKkfNfdWaeulMn8L2YzWFFlssFXRBU9Rf1xhEn9BOKldbFwGnHBBCg2fEBkLd/3RXTojJObXnVFcdLlheDhVePLRKmnrPM/YFnhN/r4tHAwyb6Xa6BXvRd78Q1Uswj2BIgS6UcDVkQzYILgjx5y31chPeW/5UCWFgjI3MLP39bLg/bI/J2gv85YlVfgiPlTxiM7cguDXxNwDf9stalgksFWwDz7lWuCcIIghkmWV1exbRitbGd9/8gNa5Kv4AApB1bikrGw/dfyL9625b9+P+8Tq84qmf5hlc6OUroBVmv4tVeCJUK2QPclMmz2kyL27+JA8MTpKAyClysWwupIf0gp9s/N7GCJ/52knXYcSetqP8D4FWxuGOwJ7BB64T0KtjrfvtbV6gjzxh/E2xOGO5+F3/DUV8YX80IbsXAH6tWq72qatkqwcpuCMP13QujzI/S/jQaAOcRZqurpnkHi8AqPZqt9sCFYQyorxBXm+42K9yUZDs/JO77jq/2AYgtiXYCDUWyF/+vZwXJ5TJojB+4TNU78dIQTnaJJZjnKD+1jsOE7bDYaTS30gLULwM5zEBHhcSo5XQfn6sSZXIv/+1/z+E8068/5srshwMGUbiRCjxml7Z8CHhcBdiCZcH7L9yZp1JmML5yXWWp/bwwBUJMZL/VgWPpcj5YAaU2Wz0SU/23xOxdSMbG6y9/VK1r76mXixINok3dg+5yLHiK5qlJWiHQvK/uqSAOMB4A1otMdSkclzOjVN+5QnU0t/a8ealIeVppiJdgu3Vtrjtr4FNtjg6e+eiWMSBCsOXiM/1a0v6kbT1wt+2pRlTJlBh/4nnuBWew6seYTtsE8c1vcV8x6HYjcJg6VJnzTLZisgZwRP+RrY3404TKqKt6tdjX1djW+chzGvWKbCbKk7lUkmh90P5l6aUUefihWmQrj50M1lP0keHWztOa+Q2rtrrdFbEOl3DOCbJ3IKoJ6xnNykcJLDUsSo0n+zywLLmc97d/stCKKB9eOljgOdka0VoRJSx9yqeVcBHaUTraAnMZZ1SrL1jI1OR63oFSB8B6EbbLRHhEgDS9eKHXt+iDHz+oPjEgYOny4apvwz0QRVoDhTVlUEYD+7eaEjyihG8Fu6/7zV4sQG6Siv6c29XDuECRk5gSYfEKiK0SQadb78fzJ0yZxNwEv3X78QUqk/QPVRKPYh/kS3ZqjWauPz2GNXp1F67lKv8wYfPMr4yarfW6lvIFZu2pbRCPL83JRhvUhuxaj+pk8+zs6lQqaGyCPdA0al+CTc4zU90FPNUpojH8lIw9eNuznGLqtGf18buT7GAGJyvmWCNqEB3+r39Hl+hAny0WqUkwsUx4MoInylgVdzYzvlYyLdPnhVePVcGzNOxT9he0POlrybH0AK6WhSCHsX5zUnK24CcKlXszSBfuVFfD1e2My2V10GW0M3pAY/fDJeMPfkKPGZPjFS7UXZxP2mAa6K7HyLwcJouKLfv7CCfBiDRoWEjvJBAwijmrJ0EnGFIubpl46s2ef62WIQBFSIin/c3SR8b5fw+e6gR4A6nyjEI03Mnx6A7h7N7OqwyXRaRV25Fwel+Yaxyto2wPApQhqsiPWohzW/2w9o95uSm6gJAa7myV4KTtJrpe3X6iXU12wEFccW5KuBuS9mdFRuOyu5I3m85iWBShYG0yzCHvF0UKpggObsaH8zKHwryIFCmiPogOycBNE1fsAP89Lye/4StAbngslZVQKNjanmcgQ275oQt8P2z7BInIYpE+yEKCGKbuNgrwhT5O+vDGHjzbXHOHIJffgjb7Yz0frb2idzngHbLq04e04X8lSjLz/DhyEua6m8csj22Npz4+nwyf7QsY5F3uTg629jQsAlIdl5eTzldefZstaRAcoJRSisejK9zO8Av8VtldJ8gdlgdRg8ipHxAD+ilqRzEtCjpEgDX51Gp8eeURt7bP7dIVmADAtkbmnFn0DDR/+IRj5dWEkQ2Z5Ibm5Op6e2yzHkl9OsfmGYUypVK+f82lR0N2O+tAgnWpJOA2DE4ds14GMonu/o5YHmd2uNSKZtGCLv/7zMt0ccxnE8a0wmUHuVZyza2z2Fi2K+5xF7yptfQPegI36QD4EIl/bytwxyDn15VyUxNPfOCr0jLKZjIAD09GdGo/yWa8XLuS1xUZ59IvDtNyu/xIWanG6VVlduBKg+ZASIC+xfgrdLZPNHbDpM+Kw84k9zfldNPVGHjuQH7n252oLMVzaLislbMBNBmJ3poRxSK9/sSQFafnnqXuaUeTmQXQS1CUW5b2hKd7KBM6+sE/BzYM/1MQ/NH1lk9W+sqSh1ulCzkdW49FsCc//cdpcdBXi1gLkf3Ubtk3KAlUaWUzTq539DA4LhTZY/OsrLmX8byRAFaGvt8+5fQz6wIpb0eyAxWA17NSrtHVJkR/J5JzDDf0jN68chU5GrKVv8kqF1VYHXZMDGafikxIAa3j8tH0o0aZ9Z5/iBbAVzuK/YG10jMO1nCFlE3QFdzZ2IuGciUPOu+TL+QDrOlIcKJTP38FOIWPMC4HiJlsNJEnr8/zdbOArGi/YL/OOsl3Gj1gdI0dL5bwAZ+fE6sXnAmaXmPUGtwqD7/S3F+IAS1U74A2VKiUVS9vPVjNNPILpjYiKsjaPNu0wEI7MiGN/pU4qEIEhgw21kZ78+ggc02RXrOAwfVBxQGS6XN6txDa1xGYSSW254v6K1+fjGWHkRp3i54M5Sq/HAl3QFnmhXuopKrO3Hh5QkUN7xCVaMtSnEgKAUsRPfylaKtZgZ2vs3SwMOLXTUOFF5s6D2D56nHu8xmNXCls5KURYHBOd8LceaSIDz8Q7aqHqer/5iMIahAox7R66D5Fm/KQcbo9hWHcLEbmG+DdZhnf4Wnr1xoTa2LlIOoQobGs747I8f7GPb6Dga2SVoVSx2o0IcYoneHYakOyykg51keYR2M5qYAHRLw2iezvnHqnUSZ+G9WkmwwGXERJNqS8UKl63qSut/e1CLFdU+11pEDKrchjZmpCsm9hSNmcRbA6mN2YBn5F1fHq5YeUv88uXntwileFy3OqX8TusAcY7llf4OeBWt1J9aVAwceuJfYHwueKPS1Opfabd1osfqAitCaxhLzU2q8XSsexYp5k9p2jUFoiZSvusECuS//pFdJN25NDqQcnNljvFW8NZ/xIFGFZKHmxil4mYscwC0hSgDukLxPXWQcry70TYBVM13qjnfe/3vHyhrWef5HlvdWn6dDBlx/BT8aS/SUo0+B/OQBhEHoatWd4jeF12cmFDTnp0XBvwWzxz6/wjtdPQQRS/lTC6dWRqfg/jFONY9767zfLBCggEw4CRM9xY70COaS+tByf24qJCgIwcIBGxr8vCwAZYosWgCapY6NlwBRiIyAVpCOWeWMsepzwmaH9npJk7m6XpqJKt/QRjll5i9lEL1PoW5D6xGrkM+nb+rrRUHVm48kOJS6NHsa3620LlGDAl9z8QLuKvnQ9+HUYaBDW/Qcr65rsR/S0lftUw984KQmR2thsDSpGa/t+ZTso/rWQM96ZVyKoQ5Qiz278Lf5i4q0akmnoNRMgNJld7UVHushjWz4Y73sra9Q6Gz/XYO8+SG+q+iZHoaUwd8G5aolwio1QbFjOFcPKXoUQ39fPCovkZDq6JCf7zul507PLCHk4Vxcw08wHRKfm28NvKdJiJjvSvmbwbOvdng4a8hmWxN/HQUqUqw4+f+ildNbPgw0G+CQMT5jXv0cJkbWLHzqdDMFfyqt2dCf4XG8eJR1HZUIAQ2XCpyNEm6pfvQAH6VyoBt3SVVMgl7aFaQqspbFrN8LI2MDDvAZaEexVRTwboDZd6qLxy1x/DBl+pmahKzCRqyabA1HhIcvUr4b88jWJ4pUXu152pOxuB/0FaeKMzd/PG9uVlUsIfJwP8G8Dzy8Ejk4gpoLBD9AsJo77C6Ylsus811x6SzPqXKx23Ug+4Fr9x6K9fdkP0HdF7IsrOwcghmG1ZDeogLSFdJpPI0OUyXbNjuKWF46CfvZgqxfgQ+7SSIdGyP5aM8s7TiuNssetH0eXTrW3FuHlAvLfWz7PSaBjszgRK4C5/QU2m9RjiMByUTv539vSfEM2PQ2mKMxOn3FXuAQoICMJBhgFkRGh9bjUmpGTdInPTQ7G/M+deGbZoiv8YXf3AYC6WZjgPvlTXObeZMkYE/0DsxHuZdbzSg43NyPTgLB7KOljBSn1tyzaJ3k+QUV2JdFCeLrHCqeRr7Gtjrl7BimHGzIxW9Zrfuwn4UWuSbO5fv/wXdH7+Ix1Fkv2dZ+gjio9ZfInWINBRBY/lsMIUbvoo+/k3T5tZXqlFXDJrZdejXFIDf2td8auy02tiB8D7+izVL7CEqwg//h2TZ02apNbz8BxMuCjKvqbIfWUdSDb0DLMK/GRFS7VUPgbss7cN83n4ycDXArgrxPLASKhTUNSCjZCXQPgCIDUM+Fjoyot6/qq6xN0XpKKaw3K3le276ZefmHTR+ihdE09TzKgF5uduBpKIq1n4xr/Z2qrMA+a2cFen51/778QpZPJf1MyawsV/bgvWKvaxv3XkQ+id83x+lgOG4tAvjfQ20wg+hpnaDWBedEHWMMZLsT5/VZ1YvNaIKqvAD/p6EgrCesa2DPiVBmjvZdT+J9uMIgYKp/b8dkpgwn+jMlLbmIHDrOH/tL6aY9u3Y5G3nm8FVt3m0JC+5RPel+cPYGfMy9JxN8ucrlH8YP47vbv7nyPjhghrGOiv7Erq3Y2kCO3+pI2GLiH1kiD3qvo/uwGmu/9dak9GWyTtI9kJyrPaEF6WqeCC40YLLLtI/ZetHNxq67pOtvwX8bFRZdyD9c5+0kQwfMKBvpI9PV9+O2zAK6hVf2V1/5dWkU8ZsYpNith1430Yg2gZhQ7i97c4FbyW9qfYunvNbKK9pzr6O6+OLvcHoqELI8j8q1f+jVTCihY06/xuHqUljUQwY/5OesOueHlABZtY/pQ4kezAr/q8yO/+DL6MzOqrKh1rGBpoh6fQ4btcvG2bow6bbIKmsTY4wLQ/vQAYydeBlWHuqPfFZOUvh9+uQE0QwIfqq37f+tltAfv9crzjRypHgYhLuZx5YdD+TR/P707nCdeEP1vTyKdfWdpK+Zr1h8SKMtMa4vk3+m/kmVJiYDH8TLS5NPuclxc3AHRjQV4VL353ulM0mFhL1A9eovb1vEDUw5eT2o9m4RRZTswsTCneZiW8/A7VdMnk813V5m+/97KG0cN+XsVYj71xvATB4TrIMqCJY7xtgFsCm1QC3pQWo/pcpUl8eMTvOg8g5os2pG+dip1dqqZ9sxsTF8bmgEcM/tfzm5w6J/H5oyKRvyPS2ClgLO6sZaI99OHZv/17UI/cTy63HtJ3wU8hOsCFoeKsh53fdExrOmrpycmXlS+SNedMCYv2KoFnJ3D5JfFBiEL+HRCRt/NgVFhcMPr46LGalFwnVLiD/AvUAJ9/2EhHatyNYmST2geqlejEoULMyEmuW9GkyGLUZZXr1sP3hm8UwNsrXvGKJNvcvZzRqswm3dFikHtigyawlzvHxSUCCvKvSoJGxvKwHzyFLQVPxwaIgcAt935mbrHgYnPGOR7LWihMRcrSBcw2di6B65Nka9oh5qoSZRA1WE2tklRU8uJv2Q6Z4lM5o/PzQnbUCrsyArUNd4/pp7l3ZnyJfhEbVXlCvhCrrOHHtQCdPz6gXI64sE7w4/urBFAdNCP+EHeXwg4/ZYoUEecGsUolEyBaTaROA3/sA6LFw6ZOf8G1IoVzMFK3FQW/OhNFIRbpq5ahe5UKKePSo9+TWKcNtaCvUirX8kY42LfPWR48cPsV+dFHSQftFeCzHqHPLrQ64Jisa1eqUFyTD+Y8jPLCugPDKmPzDk9cjzj+bKAUb5Tkq42S23GkzxykcWLYz+A9+1NUiayfwm37mNAD9cnsmR8Yctr8lhFkinRFBoPZIzGiKzoBLsx7qNJt2YdALI4UsElUfarrd19QKr8NTmg3D2I4fidPXUxUK7AKYccDiRErg5qjYAGOtVhB/8xPKEcdjjM4Ah6+KjNMo6Mdfq+oUSkd0l8XCRqa7IJ3lASy9O6vCuqTiMv39TTjPnhTv0GkE3NH61m01Wx3K1GDn93raw3LqgKRWihF3rkKTm/lNhUN2MBYZKfJwqLMbx+wbdM6aNYMHylQDuwLEzbBVV8005j7pd/31QSmQC0ZCZGz3Ume2bY4iqh4otXBlrhjX8ZWW5jLRODmB8wIVJbNZenk3f9DH8lYoYQ51eBlxrJ9aUUccDgeVHuD1gSzyJBlzQRJ8DgqW+MRnZo7xJwv4k4rACvyaXcPfc+dZc+6m98RB92KEqYF0IvN50Wbnr9ATeB8sjJIu59EDXICdN3qLvzbSuYrohok/q57MyD6u1x7gbXFe0MuZ/n4QJObpGQFYKsJFDd1yhUQXKkGr3b5MzN7wFDM24rOCOajEsmD2Ah4vzJ6JRAtSaAMyvd0Hdahyw9TescR5d8vpwRcxA6jdqpSeIwEQUz1T3Nb/teW0VsawrG5vGu53I1HnwVD91HLYUozXhlCKAGcv0QHaY4SncH8Iq54RXBmYZ3Sx+mPQAG9CZC7weWxICOEPH7lTw2+ZwP7n+g/NF3HlqTIsvyatyfRLCHRWqsdOtFafv0lqudtps/UqcqEiHBzM1eRFw0ssc7azomo2Y3cI6quWz2wN+8R2tnUS1+aMDnLXFwosSswwjYj+mmomFiMMh24eGJJG7LfCr+wOM7kzG93RhPK2Tg9D3bk2CIziTMOpduTbBKyoEKGnJw2Fb1qBTf6hQhFs1q+ZXiF8Swa/5tz88Wza0g0UaxkcJMfD+lFs8BVzLZQaAIqhxW0I1YClOwYxe4MY9A/yB3maA9HlVbNBQCkkOqRhTqAqmehQHRHRJ3ALzT5uAB75Qb83mF884Opi5Q/frWNU3J+sqUVxqQddjwM9/sn15AErtjlv4o56Tg2jkOEznSCnrKndFkDm47GjEa3kyoEf41pWZRc/RODyI4ZhMmHLI7/IKVRngDh4jKDdYI7sZW4nW7JskeX5/4eSza6eid37NK3J0tuF1zu7PTwxoTm2t+tbnaPLnZGFa0FkJ6N9XNnahZX8Dtu6+gBoQFUR3enCyDenn9KN4NFQCgAIiSDYVt7lbQGITRJsBdCtxCi3spvMM3zwlGOaFaqgJL4R3vz/QrN3Qmd76tEZzul9fMLdQeCFviCoN/0d+FDMrGvtZICSAtxaKmBdQV5xpIZ/qjryz0yrzRJOP98uiGwQbAKfeLZRO8cLh9YPV63wDwrRUo8S74HnhgJEMdOg3CHt6vwadxfvhNFmKoe/rU5B782kq1/NanFdcTHS6kEiOBBUoa6NaY/LhLNqw6jJ0OzGVOh7lDM7y+VwHp2znhNRP6yPesVeSmSCPX8e34p1gq5G3oNzg8RY2HXDcGWsgRAytO/JvucCvTlKfuiTXtyarvhhcb9my9XD/ovGABCHi9vuzEFRvbf3wBftjQ+kQqs7vzzEkWfJI2EQN8vTlFueZ0v6TwRWoHFpzeCB7vM1cIJUo+dchKTYV/Bx+uqksEbpAB/jgdQ73vtT+0yO6z2wjVvIt5OIgQxAZHNDDPCUHOhVvj+qqdsounzEakFBxUnGRQSM7q8cAYCRhjgP3E3+WJucEO21RcGiD/oKvDNkma+jllfsRbwTndQ8CJTsUH28gtUdrHC+BrDE5JFMqD+5W838I/Ubx/RqJYsCUXqFi5WUciBirj5+1dGAShqBnO3ODLzXZkltSiUL1z9toxzdpQlYZOgjKEIb7d99f/t1zZ4pXD9aI73jMs89xSvJKfmcrH0ei8gs5zaoIUSrhHCV+9pSVpDoNZk1pVKHtmZzn60LdDQerTfy/9osalDxJYKQS2lJOR9XU53xNOnmlwl3mNbHAmF/w0Y1NW8OS86r+K6rEDh2DgXa7jL2A7KWpikXG9+2lHwuKlsCqNJBsgL9yAPqBQvxTSY0HU+IzJAbl9jH2FrIJ8vyrGY3NoQxbF7DoJ2YaNe7T7Ab7CEqcXNr8mTwCfxVOZEdEY6uIZl+xcY1RaVYG/GS84g3Kan4k+U38pfdXNPtTLmZjH5iuCI4kdixz15cB5os3Z/sWcHZdrSxbgxF18bFcuFw2hlfAgduZ9R4bvrJVdNki2Pr6Hvof7g7Sv2pfmidqJ/YT//5BT62fq/iVM3UUopSE3GWKvHxBzmbKIX30b2TBPI3Knu2XdD4Z/tYKQEnyDpILfOzAf6z4hFLhXe/cSVcTmBWsldcxJFh6eeMlhZfbGQcX4Rc68wzw9IUFgB4o0VrrmYvByCfXbSDArjOdLJNp7pT+TqX19w4eFAREdZ6SC6TMsVnxTlBEdbApzDz40tioPvdAmKxfU+H5qZZvaFU1EJbDvusIlpPSfW9y6zpBQSmTbqMwY1fqW0c+qEj9PzOB/aYAewBQZwciIOpD0E2L2YfhZ1OdzL2VAhEKVv+0ofcNbXpL+OPfvrdxyvu4iyl/Ebsfui3XJRJFsQVRjOyo6uUrAHGJIGizisUB7zn6DeibAbd/jRsOKIjrTMjJbmVp/+9KxmrQcC5DEfasai2x8dNMSIU0vlo+cQOms2MNCJ8IJXgcT/gSWC9UuOCAPIUEV9OKdk3vC/aWnh3z4rAs5qPRoW96pomnfV1WBxyHSwri59zVaMsy/HlW4Q3VqcdH11RyJjjXuPP7Wi7FoYkQsO3OW54ruutYmSxil7hBVwAttOnYT70VzFngpZ4Qr+DIENwcZLXwutIAsUh8jlIucrC8OlQhYvo+O9s7/pmVrOrcmdr1PecPeqh7yiGf4KMHh3ZRlLnzPRkJu7GTUG8rACOkEKND4X4AI4pCkaZ74kJV7PdddPvNtAGjwETlCB0axwOu35rDARM5MAfSfQPe9lyKFtLk0XnAANz0ECaAUHKAcpl8ZQawFK4YWgbhbctMyTLtqmH3wW9u9gdTkVwNrGFD8/QcYGfJlDKUSJZ2bXuwj3U9P8lWCfZfqBcs3DqTdsWp5iFXX+OkmydONoJ79qljU7sZ9TamCzXDWvtJU9dk5l/Hh++UFdBezCzkT4tpJNZy+ULarrXg/pum2dt7KRXB93nrugtwhEmBJxbJQqV+hRc9ue4jDKTJv1oDpsTGLTBncD8FG3wF9OUf2AHaYjrq9VUamXtTmQUy6PG7XBmOwMBudq+HIfXP8S8jYBXyQeEWcr8wj1Nxwm/JWgf16oNfsflhN2yBWcggUn7n+spFH4rOZ8t7GR74irxWk0u4ghlSTT9x/A+QLTaazSCOzLKqYHq2KKmxbsegbPxI6uq35KhLP4vWCRfNJGjZJP11IqwiLmJUKxtVGUYH1IhBqI9W9YCYh+5kFlEq8QSHsj7O6uVVx0f+akruYe+apeTc758eNme9KOXdcVyFieXkoYEC7PV2krh2CvoN9DVwS5DsukEkY10wFFBQ1Wrgg03Q/jZxQNmyMBVPHPxBcQW7pHc1XWJVx/s2rfFgx/Ow5zw2j03N39+uia/WwqHmtl7ebap/wKrzGc0VJZkQmJd8Z+zUHBPENNsvzKogbPb0ilc10zw/LveBwLfFX+q+GRHc11c8B6HwUOUQmHANY3NvgNKEDpH+FxfU7XAsj3M1s1/1WtIN+luJto3YR7KSNKBnSAORmA1WiXX4C5kc2NS2Y1NjwqNmAKF5/l8LExckWblemFy737d8re4EvMc5NVxCwe6qxLoF7i1qSjmN+zxuelq9i/+oIkswPhZEimT4MevarfQ33w7+luiOKTY1xNxNO9hH7iYUqoLnETHWGBKpWOh5IAltb+RFGan/lU3GyyYNBkNJQjy2jZ8WLwGT6Ed8HY+er2hmr70ic3CHMIhWP0Fkt/q2vvOTgSNjWxS7Fpa8oEwimXxojX4aVHr8cHvGeLjVcuEp6pq6bU5Sdpeo8K0+uQ7PjP5uxXo72vJz0rEnWfBdgxV3SwRv6lL/1ea9q5UPunwBhp34LNgXrMNyxU+SvLdnrgP8oI8ZKI7qYv/CX3L3Ahmou84qB0M7fi+5oomzgk2LoGGgPUUABP//mSSEsikYbjVD6xCbWmKEEaxjRXvFZ0vCG/ADn+RZtWJ/aMTQs5SFNeXCLKgwNeMEGy0IOmyGfQ0iCDVPgds1gXHwsFSuBgfYkr/94CyhaGvLy8fx/elqMuKOvJkJeZHoA6dhBID+IjWTvj1/CFE351flIQhDgwe1A7OCYjtJUqSMxxjLM+nzsZP/IeN2MetUQAfOdXRsE/t5yjCJ7zaviX7VepAJ+C/XPD5C/7LjvVRGKV8TABcd9+2Tb/59J3Qz3+cY5/c3cfPlqWDUpjoftJ6ko8QL1B3WKJfVtj9O/okuf4lOPZgGOd9Vdk/8XjCYojH6P8gXalGcyKfMld2VyK9jsIKv7Aaj9JZC2q3R5JCKjF2cbbCIlbrnIe0rKZKkV+TZO7Mw70uj8zkRe5v9Ivwv3EmckrKF4idVOv9cadrUOCjbJd0t8XY4J/dwoHtxIr/Ld+sStqNRFotxLyFkfEKf3q+LK/m0DwK2mcX2pdmkPaMZ0zri9QASM4dgbtdJ6BrdLbYLm7c8q1yksuEC3/LKGHUqdK4SAfZN+eOXSlW5svL7qkQ7iG75qWpMiGazCplQUEorhKTzDt2V4ahxWB2AB/MjgdJOyDaR7UgafbSiqLTN3gcuaeYbBzP+9ffIZdfxpoa+UT0RjHQAs+ubkoQ32lqVONRHbkJYhrJERJ3DF0zJjFIgu1mbYnkFtO9p+OqULmfcf+I3XCbGsKdJjxGnhlzM8uVKb+jNZ+27PDfU4ghhqqTWEmXuWL4u5tj/wRWGZWzFb5C/SNNAuKq1IH3nJ1ATHVtdQOfskb4sjJvEf2U0Y3Mdl6AzrUSGDaaokok6N8SnykkJljNz0uQgcejMNQP8CUgl5frpAO+47sr3L/THLFIla+x6ATlIFcY3eVaeua9BewGn4CocgL5F2c+Pk3EmnQa0bEyWmeifJRpfR8VVLjh75e23+cny9ODQVVBp6xJ+QyeemUkd4KEtPL39z0FDqq3Cm+knQcy3f19nYXkvteHNLoljkL3fA1umu5U6yzE84YkI7d4asj9uxIv/2p4XveLtQNuw/YsyQepj15xQ6JshH0SbLCqBjrrpkeXZ0u+NjlqWLmYPkDX9vYFGuPmuvD02EiQjHo5UAMWriVrrq/HdAiwgzcWfgBG2lU5mdxHTaWR7UMH402cLZ5MQiBScD2grQzlDP+oI2fwt5jmI1N4SIGbmvl6Rnu+zLV52+bHYl8hHphnssIeZr/2pf2B0mi28GhhkAsSbLreivQ+0UJ9lFLH03/cnO6XY75Qy1gA57PS5EuofMdlUfCkXhOiVhhrhIqb62XS1idKxq1JvBP/JC3e4xGFW5a4PNUwGLB7QKKY/yiPnIzWHY7nHSd3p4ya3o5NP7sRASz8SAn54Zs+E9zL4Jog79u3RvkNDreGsCVakzwTKnfIfY8D+YXtqxQRXIH8Ualh+QveC/Ga/6/VBgk6Sxg4D2R0srP8/JRVh3x0euhaNkA2WNauzhQw3V8Ex+uklJNRT89vJBgKjhsahkpwLvz5JUZ88/RR60br9AYkRWGXJzCkMENtcf780CIAup0UITeECWCx7Zbz3m/e54sQHxSBebNB1qtdBOORN4n3yEEvxLfNorcUDCShSnfCmntO0sexizSj/lFIthJtc31GnfiWKSY/JWFkLaxOxa5H7yBWsZOI7448kAEL9EGxcTToTglLC36yMZ/G0xi0JjxQoXIl+3YR5vUwqXQHZ+cPXHbaOIVCvaC/Q0Eq0GL8fUCnTcjD6wf5l97jBBxPiDqq4+IG+lnW/J6x1orf9gBtCaEbBCCkQ+d1N9Lat3BguqyzcYrBX/kP3wGoYoPtJRKmwhpLR5YUW7A66OfcvKljH/jgvs8dgS53jxyPLKx5ezuRyQkCP20h6JAi0YExV+EasYDsLuk/qebbW0b2aaDcUegPKgRuh0rKozQF8+yw3N+odqfT03hU+6MwsVMf1eKfr207rBGPQs9a1G2x0qRcvhZkH2c6JANJrkHA/irJV5IfzvQt/kT7a23LaWGq4mN4DYzEJLFg/uHLudebOaXsH76syR7+ulsxHqGJ9L8qbH/yysqbUu5PqacdrOdRID82ed+HFca2VkwUUz2ntXxjtGmABkCk1G+CQVOEwiFtGR8H6TqHlQ6gR4evvvWkbxEACCeCfTOm3BpuSM48Kqo+KDwotU7pPy0deDXY9Hj8Tn+nBGZueuFNQiEKq8p/bgZd0KgturrTGbLPBYJJ4CR2qyWekphiAZotNk/a+tR2uocUBb8tjMxMgHe89J9Jf5fADYxhTG2v/44vT6WBCF7YkN8h1chC12wgsVhVH4krDQRjQqZwp0wl6NckerRnoXmDxF9pU4Gr1H+Jf61MSHLv74Rt9iT9W+SXzHPByoXpILP0eV+Yw5o1QRDCA8ggrBMHwwEmMgfegXQGhFWGf/TKcSOMMcHFXnc4So5tjlA/orfUewRzVsNLWS7CXMjFS68En0XdsQ66fHwn6QB0gZwawRZ70wvlA63eAeOX0h4f0CJeehb0DQp9+dv9AyKtCjgcVDBW/X06QEQ4fsMg5+tOKETmZeGH+Izs92Wo0Zb8q3qdP63Q+v6Ur7GnMF6WOzKAreJkfx7+BK3wfV1zC/cPalbQLAzquF6ibdECbHok00qG7p66Q4DArxa1jHyKtkZb58heEj1lRiuJyupdt3NSerLZu6TSkd87PMVxx1CFaNqs8tuYVNr/JvdA55iBqkxHekV+Oc3aEqud11/SLk9qLkLfyUNJ+owoB2wyxLEkPZ4a6xxuEfRPPS0RD9amRt1TGcSDP31IHhaw5x4FlmyVVvfzNjIrSua6h//5v34L3hQGtGUmkclQmacOfWT1QAs1/rkbynASk9PHae452gqVXxGXok9B/05ZmWG6Lh73ga6v6rwy5WYbf5NgsrZeAvtSZhSV5oFRjNtIgfL1zN8Tt3bieoJuF4I0ApVf4TkWbn5b/Jnh9T5RuUFXzquJxrBBuIah0dmWwvFGRywsk4D+LrWAXEdvXt4mHLY8ok/hy5OYSr1+pweiEVYkqce3wiUf27E7FWDJws6tx+++bAya3XIbvq/GONYNViP6yaLo1XuwbSP0HpJhxgDyTGFOePtx/JLIYyLDtaj/ipLIspDe1rwR0Mk4IiXkukzYMuFo90YtjhLG2XJml5SAL83LcL9Qwir3kCwHY0DQj+C4E8aBA5gZE9RH4gxN2q/eRNFQvMSpW5mb3C/5QiVnHNtVOLpuFigHD4O1/weeCbxIv8n0GbKgl+lUh4vULsfGHMdQGNnxgZpn/yaZg6pIBdOlZg1hqhcmR+P0wuGEl/FXx0IlkrpS5CqHdq7nT+f7NBx/X5yQqHe/WWYO+zcOUqEnmQRu240e0Ur7/N16/dz+fr7nuzdhPJb5MfGkvqaMj+E+KgfaY5vn7WCaIBMSl/I8gKXYfJLtWYsI3I0PveF9JevK/KO+DIlUQ0iwLEh+rlJRRL/qMPxnY9yXdhtT6ga3IYATgLNMg0pzKnyKqFi5bcF6MgfLmP6zJF8kc02+VfGP2ZTyMs/QK7XMZeq1esdaFFuiiJkIe9R1cgDoTC6g2rBwzEDQaEt9eBdsMQGjqml7RjEjtZ7sC6F6I1ly7WkrupiPUq/lNUzo6cKAhPnkLARXShBioQnZHgrneFx0QIQJV7VwjnTq0m+doSGFQ8MKeU1iA17fNitLQTNUTz42Rwm2SwQOhPC5vW1JpXoIfABs8c0LdTXfc8bID0o9xIc/CbHprH0dQHTNwYk1H71qKrqw0IVnPUq+FHMz0Ymps1gA5tGkuN4TFgF38XlT1EiNrOf1fycmva6E35herXMBKMFta9MBL3UbQM7EFLdqTLrR0I1lDTLwz6QeCGFJUmmkfINUdeNCnTnU5Mf4T7Pbx4/n6wsseF+F0kxO481O6qOha7ZxcZhvpwemIMepPnnnEXRQFLcqauxT84PG55YivuekcR6EzdEoi4b3kVaes8iQ/2Nei1ymNpfx2Ul0jaNyWWnhVAeYYzc3d2fXXIXU0bwYWw1asqOk61zKTp4WfxNHP9ru5xmxVvpBhip/ufU19wovu/3mEndSPmk1ILjOatrIZ3zQnKMK4PNqYmtBs15vK68ynfSKixpXmpR+WGppiMEwKGbCQOiqORI501JXC/9r04+zrI9o7lRtuhBnSD7BkhtBiSzcA8KjcziGBdAcBDOBraa/NX0Zs0clqvStAfvZLOfB87EjpfmacSDGj9u57JNuTHPcoVAUv/uTFDQfW1vUs+VOyfiv4QxEHBcfKOFbCQzbynypitZoBPSu0l4X1zC8K8d8LbGlIPiKNMaOJuZA8wPYNKXtj2bT/rrCs11edqFEXoW3hPU4QNi8q1eItxXFrw1geM8qISx5lSOSnW2X5nBMwWueLENEXtYoLJoNIizLxODqn289Bhf/po9wOuK3cBzkqQ+td8ROi4Ff9eylAVFMiLZxcgBl30BLz5ILFJgkXZqNzSFqYyeMX9f4qkjbKPlrxNAP/EUopqhcdrKHBWjNS3MYZWYk2eZfA6ncrjviHR9AMVFihiOgwCOP9AE4/krQc29b5G8A9fn8jn2X1zqb1BO+7igY+Lytn6Tvrtu1+kkbhGjviC7wVpo9F3tGGSWaRxtCv6nMLBXz5fXyIRpD0fPJbNofusEDvxhxNDs6757cs74FZtWTDaA8wWhCsnX5N7W89tMUwr9mqlR466/fI7FxxB1tJFO0Ra5/qrPy/EeT3P4+9S8tWg/E+QR5dhmZmOCdcq7aRMEFymR53k1x4gAwc+79nfbmMxED9y5K3Z9BqkVKlz+2k6nQ5TOCEhDWMKBPZpv3IcCCQpiyVPnfRt+eeVrATVJIfC+uUKVqjz7j/GvtPtp9XfzHuqeQJ38VbG0xnfu81S0SZ+gsKHEaLrO1/SUH9FGyMiga0v6WA8rrRwzTbOsrpHwrY67jS+3mZluCdcrEBDR+puuYZoXw2sMoV9o3Ijo2pQNusFH9Zq6Nvvj039pxrymQiwpB5NoS/3cT/fBB7ID6K+mK3UXG3UmwKu8kuyvSRQk3aryz4vAtrRhjfZNtFhVHQ9h2uIM/fA2GjUy6Qi27263tQqXoh9Ff0Y8+4D0W9wby/fZpzrZVBdKTNlhZ4SRdGaEIanWgnPTOnO7GAcJstrCFq+UNx4tHGY3WqshBzHiHalkflPYnvsXiUGmhdPUySE+nIvBEiSVuevSkdXysNH7XS3mP9k3nTVj6MfGOZnuGYuuiuGiHhEw4z5+IEhtloUaE6abom1npCgC60dp3yz3vTCeuLVaciHGwr64x8EERT1V9fGmnGZboun0f0T3fDEenYK4cSwMaykBuNtiuyHka7EeyPZTJA6RumkWvfzv5hH+WgYyrrabWeYLvTPzfNeu97+kjZiJge9qzfFG/fuJ0D6H6kuD8OvjKx4xlFgWm3009owABe0JYZ876zG0JduhivTfE3cgLVzfWDPAfkupe7NXlY6VfRo389fI9Vf06NW990Z/EvYpsUGKyv5UUz3xQBeRqde1YucBMvMHVLGgEnyM7lcvLYeL4YH1uiKI7vaZv2QsHE0VpcsYXTjbnwZ6NHo0xFVvCSIG9g5HrXBX33wmde6HuccKlIbZrz28/xgXuyqP5dPMsAbyWVxBCGzcD7/glxxpqJ+1qttOPSqh/iNlzGkQpYneNcAayGobKF97RhJMf5i27kpeYCDUeHQHeZMNBL0SHO2D/PYVhbvXV1b81W8HHvIjhYrS7WKTzww1d+RbJ9FevB7LC8lR1jkUkB4u5+Nd4CB+imn9QwFNOwtbbFLo6uSl7x5km0BM8WxFm3r2YnET7ydBwdvZQAjQEuoAUb5+lVqKmOfgtPEyB8Kv+kOxXvVN/EH5kdMTrkF7CYQyblbfFdg8fsUwPVjVJdGyV3MABXilQZ+KRJHyV+UfwIJhqdOYQQ+UChw/ZDsECgYk7D3nNP2t/0Tcl1E+vUuVyKe0Ek3GE52FTeQIUf2zMkatut4P7gGGwh89taanXa0806LFNXTTGSkrkuD5x37t6OGiDOl6ZtEtzw+qbncoaEQI/8vmH3bi6CbnLAKmoYrarFdm/7VO7ryI4YyolmaDaS+f6RPwUMj+eX0iRQWhp3bNFm+s7uTYHO7RyfU16aIbvWFpCMmuakgCTgICbNGlxFnXHPff8qu5GWKTEO4JOStNg/SIuSOum9I13c/6bKpuGYaltpwV060D6iVe3wq8vE33TlB4HDpeXL2LZAn74esCIZNWVUPfqF6zx7L75O7Q8JjqijN2i7dpgOue+NoEQZpD+918Iqabcxj2GtzFvagYMshyOTRWSODUEISDZSqtta5eQozNN00mStIPP4eiRn/E7hzr/nLo6yn65KG9HPEnyUWYyzZvWqKHUZChmqyNohmh9+t+TRT6+7ee1VV3Wl/z6PQrLU+2WfLVZT5UcynbU6FfrvDfNdKC3qvSKQ8h/zckFGbdyVUr0jymQup9Y2uIdD4nmRD2D/0dWXmg0GNsE/pPRfGRvAgrgpRnz1Lpa3XaSvIOLbVs8yO0ZocPZPks+Ar3gp05Gqr2g/1BRG94oqueoH9wZsWOChiQ2MvKXgOHEY+BLzRcSLG6J1a1OONbKfzk2qaOme2UuGfrAyQJXf3MhB9h2N2m+qPrS5K9FBrWW32UIqk7i3ecoR2wwN88uUSrTENT26+WtXP3YM/4QnsJeenoMO42jLuGKm2hQVGJDwHmsP2Cyg7QTZ6ww8Pl39/UcZd8+dGokIfYvCKgi34jipPm4L+Kp3bkidWtML7U8k5ydbCMRtcl5Vc0afZdRvQMjTOe/L8baJ52ylEQI/B3Y3zZGdiKzSOR783E3oe9MRD7sTcEMn9GsLvc6hiY+cEvRFtefloSRJefqojchthWMI3fx4uqTA3qjpmnhNQ4JVDxqyWcWBIJljLprJuJeJ6T21w1P3WumNbWSBMKK4l29GXUtJHiWhC1pIltCGPzubEYeL37lUnws5n7pg0260zuLpT2FUZLJzONRxw+c3jxXTaBuxP4vpfK0Ib2Ibcu00Mzko4Ty4XsZWEbxxubColIxMGFi+oVjYCp+YBRK34NjCiH8vUjcr5vKr5GTUpYmzUnc1NCN+JC35RtanK8Hr6Iu9b5RN6qVbc4DHu0/tXDadynKbWyuNgm9sOqIwCVbFwjRUseyyGohEX/Sk6DzrAyKZLXpq0vlLYgHcJ/oyc7ZmItPwOCOp/jgt8vEPlYGL6cu4hh7gltbY6u4OWkzfc6dwpVrvRzGKu9spJXkLyuFotBzGZJY4mhZ0KAqQj/vfr1VHdYr2lAbpuAlV8qhlxh1QKx8PrSKx+CY5bObwKfGEVhx736jLSNM3BjrIs/WMsaWxk0MVzVUW4ds6NfgLfCy2QP2JLFweyBQOl6oaXblB/cjPdE7+009vjXiOJdkWORTqPnRw8n/nc9kE+PtdonofuDETivvlUSiNZfNZjGmU1BlNT4+QjBvcLNcZVcD0LXCEWFisrQX9Q7JwIgm5Up4tFOn3LuGAn4vO81fJOHe8lsAsLi4g5l9lf+IFt7/jqYfEWvBjFWK38MjKMnvdo6zrB1jQ1pUWFTFC8x0usHh8qrtW6cl3cCmdgg/j3dH6PBMTSGeCDWjFsZD5XMRZzxZ2h/l7en7U/3Llaw3gGAW5N6SQihxVClnMYHAbgSRLMphbfIw+HINLhKSb4AfpVK2DxtMhPCpxJFhEDdNLvSPng5kFIxdsHcDVcZm56H4nuVDSNXqhK9CrC4pJEp4JI96Tf+IjqLusBjfPIF3j9KRVco8ixUJFWBrALpA2+E497mqAkSub6oZBJ+Dn8zUFjeLvwuRtpXf9f3VURdY09s9OMDd0zYcdWhHwT9frHwUx6BLGew92wPURLc7MIMAqXXeaCKrIpU4WMfQzgyi1+pF8T8ci9AmHjheg51RctggqCDRf0q3K6vtnJ5FQeQCllhYleL8TAkm900S6Xn1KKO2X7yuymAQFv4i+qvLDNuSPJwMnjQIy7U66xNIu0RT3NbwZE7pgvpyn3VTdv67Y++V0drstcmY3f6Zk/LO9DyKsaX5PoLrEv0+b3OH6bplXCA/aEDjqXjtqEx4OpJs6xwE2O+NvFr1Rm0O3C1XaFCFa1fRn8d4uJCjUy/+vMFiERtBtPyYsVNEudcvJ7pEC+kQsDRDR1J7VcBe5R/6PZ2k8sjrQGxxqJ0CN+hX1dXN16K6RhFiOiY42Q09Xz7A/Rh6R21lYWBHk9H2zXPvAwkqcyFDXfYXYBDUeJc6p31SZvNc6G5FC2z8Q1FLbWEOj8mhEjE+kIyzGnf4IItyDN6qcombql/n5hhQeOK4692/Ffboi0V6z9dV0X0l47MRSXOzFS+8AaXEFXIr3QtrTx+DlRvH9Qdb3506i1XqGq7+lZLn0LxSIfFZQdLPKDnmeJsnkHYKMbUoRHaXHU2/0Zy5N7Rx1UkLIEbKqeqqdodfqu9tsQi7icfBAjCzyjAu51+NDK/0VKihUyP/lJCk6Q7vnJx3Ud1Zxt4SMgcC/U9vxy8gmnDDCMCN2CckCico4RreTAb95hG5fAcpVVYSO38uj40noCEWjq4n15WjfLmCKl61Tq02/FSOolQthocEiXCaO/fAr8oIph2ct+H4kDkJtJ3XIz2r5dGchv9VhLGBn4zuoqJbw5bUu06iW38HK73Ylbu3eTQg1GJGXub1WuDAL9A+AiI0h4oqkp+HyvlNvXiaT7CTNVURXXwo9aZEW8R0pBZg62kk1cg+pmH6Ua+cGY1xiBG8UmGKa9TfMq9hqBbXqHTlSzbUU9aqavyL5ltPTkYQ13RwuHM8fS4uOv+yg+31ZvcOSGmSKiKanUy9PpaSSvegxPBz30v7nLaIAENL6/cXXQ3dK3tSuPeHdjYfA6TI8OVzKKd7md7+CBqsl8HJLr45AbLcB+tdXJ/U4XR+0nbITvQta8f/cFZwVaT45V3nk9gX8kW/NcReZjY37OzGozpmG4nBmlP5XeqsEogcZ3ppwgcGD94ML5h+zf9TMKsiLVvCpJRg9u+Fo029fqyWof0a2R2sT6+ZK4sf9A02Jl916//3swwmbZ4DtM2nqNavg1e2ImIDkLpUPmmUpTnpmM6SDQm3+pzwQXPMB/+ZpyPyMCdDa4zZR6etgw794lPlhySynDMu+/VHa3YAw6+VP1N1FnN32wOk0V/446OIIH5m+htdnQxhvP3UewOC5S2tUCKXVUazLyUZnCmBPvdLf7yQi6lUn2IHynOBCYt4qopGw0tYYhsDLiBsBzH+U4sh6i+2/7inLWFL33hvcD1v8Qf7S9jRIFoJ/3gx7k4r8DxRyJqG6SSW8YZwuaUuK/3dSu5tEPIx36VBenADbv1lzoaEASAywhAx3NbNMZZcpXXXhTInd8J4WnvRrFYKy0ZWQt+9+/i0g5cjsogjSeK65e3gOth1O8BtqvZu9vlW+HVu8oXN+gCEyyUb5M9QWPd0Ekc2mTN9We735Mf+8hPVEXR3TF52x6N2VnY8gpjH/AY8KlRfVnaqU7s7/APbredsmSHr8bSve5mmFsPiIJ/baZqo6lrU4hFKW2f/5pfauU7G3TZ/fwBorBAQll3eBG9AVfjGGluTmRnSqUzKbz8d1ERAEcIdxw2mMsTDdUKs5UYE0q/HRze5GdQ8aKcKz5LR4cvAcCkSq8tOD9cd+ebVRCkaOJaJyubclcLYW1cBhNc13+p2LhdUD9F6Ger/I9t0rR58E3NfSvSx4Rsf2TYGntw5D6n0N8MJwT4Bu6wZfxn0sm/IYkl3z+98+649zdD6aMQQCIYFsmM1o7R+gXcG/gLJNpv7LspXjcYfSjT7c/z4p/7W3Q0VUcFetoT5TIF3bwdzXq0ln21nCdlHRhqXb68zRetHxyKltHVxyE60LDECI5ixXKb3aSjfUnDdSsJHKCWkepCQoc1lbmdY4S2rJlyzDpiFqF8fCzgUJnKEuGUPPxYK4oXGv4C0MTfs/IbB/6nmYPGu9xfbX57+YKg+SlZDjC9FwLITOxpaVki3Eo3jcLPCY3wS8/uF7vM8jU+Zi+Bsplr16QEKXzoFmqa+DgmJ1oZ9XPsIW257WSY1dYoWbrjOtTxVflQ12rvCEw+3gmCZVuMqrGOLNTP/tKqAkkaimk/PlzWiz8vS6DPQE2GC+Q2grRTdaYa8xTwYf11FMT0N4Yv79YK/pkNot1fxkSzbyPfELMlNuGIenKWSlaRjKNvpWNC3gkC+FQ7gvwAHxAfeY1/16pwebDXCrlO08eF4dWhNV7l5IrCTRsUFdLZaDRg5U0velUlmozZs9rpKRJblwSJMUa2XlaMMEj8B25s5QCNhMx4sLhCpR+7My+3tmhcNbjj0dq2+h0wsQt9XVZioyFai2pmIInx2rwI1fBWmtjFOX9vqOuAeHY0d3IoKBJkGLQ4UjG4fc45ICCjX8HQV7MVK91fCqresLMa1bOqSZoAQ60ZW9KYETWF8NVFF1qb6fd1TX+NBuLCC75cvfhPeGZst9l15btj0bTg7i7D0Qt44ZB4mMTT4w84N9a/48OMZvlF/x7NF+xjkKte1Ydmm3Yt0VoTNQHkGi5qO0MGCgT4QIMUWxN1MlaZD3R8Xh4wSzn21ya8/6QONm6GPjXDuHpn75/nyW40saQSWQxQK5ubfcRxzEjHgkrOAXb73csas3b3gXsOvf4QwaQG+g9KGkacoKNxrudnClgLBZQhtLaNsLZ9QJf9nkD79NWXpiu/pGeeAPIYvD7uD/p4Nnrgf9GxYXYy5pSUtdyfIX0+lo2pYdSorxDmLI8vAuC+FWp4+SCa+hUGtU7sXiCv0cgg7w0wPuRaBsVRnYkMsvB8+qStVuqDrj5fJ3DtqkVXzv4KxesEdezMgFpn1CyiboBxBcetAk/HkNXUx6y27bWExa+GCzABXLOH/xKO6m0/K83Z+2A0bLqGfjgNTkdjVY3ZGhcaVX/isGs3ebBlk/hQ5bRGFcNYYZSYkTEFbi4DEpyYLs9zfduIA2+6H1bb2QulsqhT6GWb65mK+Or3jOLKvqhdiPLaH5GOkjaKDWaaIR9G0FV0eaK/HuMK85PXMETsb6ZcmTCGQb80EsdmPJa8zFGzxPW+9LinMhU6x8b8zUcgoJfmomJVfkBVM398T/FJkmfzRmCjmMfT4TY7cJUkujp9gey/VtO/n2GwPyzP/SVnZhqcAPiA0zzcPn+ZX3J3zgqHVlZWTo0uZDTMmveYAY5AD0QW0BKTyXXpxtnOTtYpfCWO3hrk8/2bVQSgXrsPQU7S7WKiCjwZSaOfvbplL8qFGiEyoEB4YztSu5afelHUycf66yYpRbaKKFlKK4FvSHOyxHTs2a83+y7wVVld6yfrwEZs7ckuh8cRLlVkjWXeA+W+MsdQHYpu3o1/ydjkNPAavEyj2ZSF3IslxtwIVOOZ73m0kiu487zVdJVYg6S7r+A83IzCfYaVbeSz92TACUTic5DgUkv7oXGKVGPhycOB6uip76hPLb3e4O6iDcIsoYBIgfnn0HhIEQoOMr+zTFE7baZYmzntDVKMH7c+uCng/+LWvrV71xLag0b/PKc34ZeO+GUXNn+9i4ig/iT4+0r+g+mxGzrYkIbI0a62Ltg2ON8DKTzHr4XRIbv9BTqshLMn6DOAo7FfjMCfz2VIAzDtq7/yTp6/iywXUoRNVf3qHmN6eSKDKuiHhSscvyFHgl4DQfxf9pNzig0FAonrcBQ7I2JAbzMTYkJg3RvBN62guRrUE4xp7YdQGvlmCPFXqEfp1GKpIk0GBQ3eYpuWpGLvg03a91gRCjpLrVRR2aWoaFl1un3xV6bepSrf76abCeA1MrfiCKXzxav171S3SlYauCOjawNCN/ZhBgDi5QWwYoUw0PfDaEeU7LoaWIXxu7XuRDUW1vbRObDEvLekypHKiW/i83cj+ccWOI8dXmamGBFJFFGjdcLx8aiJnOP9PR96jKIfW/7I24AIWf3+CuvPxX4fWU0XrZRzUM+tBA1iOdyCvUz5y5/LjgwSbbF0wqaq2VFP7mNrol2/HyV2J8mXYEokn7wq1fDauR+p1V+JwRGb5Odq13PlqpC88nsTWjlF7h/WwL9mybDagG8vXg/99c1/o0pF/UNszKZ9LSWdySwgP7+6yAyQryiAbO6II+cOd4WEzXS00xhgdG8MZRyl/tlFSB+fe2So8xgvXxPceZcy5SdAVPRYFOedPD0+XDv9KtKw26cYn3YglictHBbSKBFm+tmIOMIAYi9lAp/7G9diDt42C4HTrbRAdKR4Cel1zn75UxtWIF8NbiSPByZAMEU0G8LJ9QOQw7r7ZxJxWBud9UF/RaCUHkn+VnhbixH1sOCFmvPVHsuP4yXtADXa5iFOuCY0hXoEBdwMx3J3fNByA2FCw7o+cWwgEUxRSQgduELQxZ/B75K9/Awi6JJEg5rKFSliusO1vfEYC3Ih1fhlPTOlClae/lpx0qHOKYAQ/vtq7CAJFibV6D03f2VtcTWO6vRhCRCfJ9WKpNitJKKTlDTAAPEkGsc9gsurCwFQWWZ4AMrwoT4vrQuwzGgLscmSXwEVidBXYJoIrwuCq6Goeq9+fiGx23ErqHlZEptz18t6xFuzVt2xrtVgL6aosbBolR9nQfkJ/eaKj2ckO3jjFYIA4m3qgvj126sSyezabfpTWw0THkazhu3r9ZMToIS+ooGbu5xolR1RQ4C8lpUvl4suVOM/o9LBCjbiP/in/V1GjnlZdBjxDz0xlb72xlOVcBc8Jr8PSNWldDoJTg19u4BEYJ3Cr2CkmcpsIzjjP8DepmJ0qxT6N3mZ30QlF/660MLEHBWpAqnIhF9UYFQwQDCjCE8ozCT6iOUNfWhYw3JUaOsA1dbtl+82M2tR0PAEDaEjwo8xrZhYxEsM+Z3rctZkfgN30QLO35MV+icedcrr+oSy+2jjMX8YFbaHFpCW1soAo/pm5cKvIWvOVXRBry9ADDwDbAmzQm5s6IMTyEiCWnMJ4mnOL26nqONxp4P4rirwnxOq+2olrGFbwRm+NKBWsmXg7Q+syMuQyLgHy/hOue5QoEWENc32Hiahmk+YZ/dVV0oiTx/Z8Vs+xDAtDLwzKKQ7S/I5FiSbxy1wmE9eDhoD3CrCdHm2NUGFA7Yp+cPfRQd03e3fEElSUE70qt3L6YB0CbxcUPVpUHfhxbeiNR1Z8RpzCb9nbcnfhTVNNbMLda1NECG4XodNhjEHNzf8YDUFmAJGTpS6XojJabGQx6CFR/tyrVLFofUwft7SfQHZWebV7Ph1ySXyaIPBjvoChZwgKPgVdx5jaI+d/4at1+Tyf/+mdkdoPNFZ74pC+HLy+YD02HfNqRPmWfmVuNccYtzXfdet+0RqA315/F5Yh/LECXfCRjKYV8BWeNq/JK+sf95KZbkr9jMfp1ROMg8Uvt7615ZGC13ezlwVP4wMnfavQeT9ZcIgRuAYM1crD3gkscOb4nVp1BfEP/6oscMwCuNtOPsC+yqcJnjsvOdoprqa3kbvYMZ9K53ddWfmawTV/gxcNgjbvkBLtdsJa5cX1Cxg9L3rHSR65/T0IESi0G04edTHTHnqQ/0VzV0q85pF/5fMhxhT7HE3+4zKcWCpsB7hoOZs/1e6g9MV5BNVhWDaB77FVZaJSq1oVP85NMfWuyv9HEuB9m9jJJL8OppbLI/XEHkUd3wQvk6l30FuWxx6ju4Hz3E905Y3v7/N+UvLzJw8YeqNGWALh4+8/PwOmpUz8B61hlwOH+dLM6kb2s7EHOAlh75LUKwM/akdL35gSMZ+4CJZZGSrQ7T8OLGmcp/vSV6X3/Mn9d4Tudqmrhbfr3J+ipvlE7fN6uY7mE4LfEbsHPu+esaPQQmdFmrvNO4cX5cnScl9mgvmsHBAx5ndVzmvri5S+nXM6xO8Zm1BA90LMSWwXO4rMJnJCToTDP6kdhNh4/INIqgPcWSGawPmalC/QqXWP8C29bglkjABc/F5RQc4McR1ESAN/sfltCJDFN1i5ne1e+9RUvi39/nmwCnd0icnGm4JjpMClRQdabYfFIVNwuYBJXcnyIrdnV73ukM0/xrZ7EYqt+DhiBvF1jDJvsZXEuxS4l+uC0ZqBuNf3Tk4EyxMTqMsoBuVhyFJBsvj6foQmR6qlpEukrWmcNgSMpnoDwkILNFL+zW0AL86TACFR+PhGaH0gbLOCRzjW+1ZefVur7nsMhGfblJcp785xpq3z5fKneEhnIWNOnCqtXN+UqcVXgH5Oz8g3hN33tzOT3BXG4FrU8fwGfE7/LQkXEcpYN9Oqqm2xGqh0V/rwgYCNd4nB0Xwqfkee4cShYCH4/1bwWJOfxh/p9dZMC3RmkfltclM1G64f1UG3G/wDL3/+ZujqJ9pnbTTXfs/5b7niwoCeFlGVL/PW3fPS/gYQdtzp2D8XZJBFv27lD51oR2X43WyXl2sCei5+7P7o0G4ipY6Wo0EWmbYkBVE7Cu6NgaRTZl0bvTUKJhoxojwALYeV0dKNG2bfdYSORbnw/ud8oHkuhgpF0TPx0gXYFz2XztQcDNCOlkZfzUIl7XEBY1twUx4g8a8hjTWun30/i8O2c3P6GFxhCBIL92cwPxNmfU91wq26ENZIIQh0VNgs1Vbzr38CNDaBGHNN15nMXW0MZ32G9e6CfBP2XZsmKBD39lfq8ihkS5VCz9ysxsj8LHn7jET3+gUj4XZgSZR7MqJwt5OyBfyHL4CI68v2WYBCpGjsK0iCtc9F8QipoJaM3bMm9CZMw8FKc947Z9ZrM1XzAhfeHggAustg8DrimsbEPUDEa/1hPOFMDlec1EK/FBOHzbByhBX7j3vjskvoRU1afhvmEL8gnV/QooIzQR7eX83z9ZHmK348HcCqq8/+bK7sE2OcGbC6iLhOIn+taL7mtOem7BBfYEwo0Ate2Xo4ZmvySnvulxs0+NHME2y8py1dmX+II/8VT/xAwtgbhFDMUdrHtE5yY42/wGhGyShHXCi8FNNCT6aQCleEx1ooQOFFikvbqX+KYKoSaDtPvuOuKHhr8vMHJSR7uluQu15EhfB2rvIO1PNUyIWw3jRwtOucNygdLKoKVzuURz8B717cA6YGrgRaVI4eyfdz62iWUZzkPllvJeOikWaleMSDA4KOE7IY3+K8h4BR+32lPlhv8QWXVr6tlsrCffKPWAlqO5Wsk23g/h/LF3XsqNIDP0lMPmRnEwGE97IOUfz9UP7TtXO1tbdOwZ3q6VzjtRSpbcJ/tDLSMUbCzg3ZMLvUb+X4hUOkr/0cTe2GliPjvn0D2oN8EN8qx489QDQ9bQ0SxPiZAcZq/5nj6sL+djhdDsK/gRe94q7Xjb8T1G+l5eNp970y/aOPKuP0h3izk7KiOrRav0yqeGj35fwhHOieV0uEQ0OC7POAPFaT3xjzVfofgPW71BQk3qZhF5m6H9gfiHAUAbBSrleXTKuw2LG42RqsK1XM9KWErcBiyu0ETe8aijRMSGmiSUCh3w1yN/1GzruufxOiA3ZqFdXUV+6KzjnVdNvU412eM8bql054/T9lvUZj7cw3R6iKI+nCrQT5X5IFR2uRUIRuLeVx1jNcf/SRGPPSmGilm6tfvQqbJhn/L6DIseakC1eIzr6HKHYStEmFY285Ikd3lmR3zWf3pOJ0Szz62z61naQieMDw8hJWxJJX/vwNL2B8J2fN9aH+CwOoCbA1mD560pqGE0jw9vxh0NvHdJz+8ZS2mrLlkntDPZ8KMEyp+/fxIBKbXiluOYGCxTBKiQ15EJRbpO+1010XzHWMxrQkQxsMeEoJUIjz2zn/fYaRVBAMvr9IUnOEAn9E02GPbxXcROg/FkJf91S72OATJyi17MZ+/77tYhUy3aLUJvwelN0ewTh2n+LB/gU21I7J5SFRDp+nGF0L2idP0tBSTTNGhbV+7TFkpZkQ78BKGctq4wGVuBTO5lNbXuh8gC1RsOWptfjrN9txFV3+Fnw0qlGQlEecN52PbzSC7SUnyrrx7c2i+creny442QBRk143P2lI1Jy4tCX2bwGldLeiavQvMRxd7+o1+iG2gkrH56EyMeGd/UFh5RI2y87vFzMfeKlCuatMJ3HRsrLfBFHgGkGBfOrptV6QNGThNNV6Dh8yY/U/g3pL7p9WUv6ojT90ZbPtOJVKhvFs4tQXQ3jIJ9eepxRXIRdhydhfMsnOEY35q4zjTMdirlQarzVRq6HbCe/Ze8O45nwETqqYT/40gEk0tESGqWpJUAqld4lWkacJ59WB0GKBTwFOIwCXd0YcmRcK3a+OoH4SztvLSs+QVYARWLMc7oGcgyiM3+nnDaJmsSGM3vztSNpcI5LytfTsREgFN1CZ0PHra32aEtD37ebneXIbjetlaT7ZZlaV/36d2n+jwIIUhQzbls+7klC7aaNPUcswkzwfE/s50n1X+Kvz/QTxgeVHec4hLo7JH6cEYbFzSkACofZ+jqMAOxhg2me3rgMBzOQVn1Xbl3gTMhhSxqrvqKe+CPHa6EIclpHncpUXHTU7eFNS6lM8B62WtoPjCOMdtAUXZ2XTXiTX/xIGsscOfF7auMN1rKGX44xoMdvHn36U8C9sLHCdKkpyfngvbOJ5GbkY5fARrBfvtyGGTcKtKO6Sum/Pkwu5vVuw5uHFd/HiJT7pHBICk6uFq/p5ZG24G1eruR8+5yd+avBqgXMtC/TquQq8aIirbLdRLwiKLO+EY7Wv2JYaBFv5OZVuum+i/Pt4hfV9Rl2f1DYwfthEeOgAE68icpS0IgGZcMoMLTDk51Uct4WTysCYbx0+v7o+RdivlKp9dqt8A9oOz4UxsVppTq69XJMBwaDRKSP4HsWdRPKqFdQSHRba8IZy0JpyrTbbwQ6n5lt8CeFRIbr9+A9EdHyQA5ZeOIGdKeZDaltWCs3tDwneAfVawgMuy47EGveYunCaS9xa0rzZOMJi+/z18yjzfuxpR25HUdCGo754f+4fAM3DstMJ0RLsre57CeC/R7fY+02uIaq9BooM0iCze8TTvcE+A9nQRCOh62+w97ErQ93yPqIP4nxBxVNnxTfUs80LzU7dBlfqaw+JZ18VYGBvbnim2KnbQbaquewDyEnc7/wt7KsR/a74i52R5OONJ4W4GtLjvcVhhSwKtcIpSaRysaJm5hPl3B7fJ22ds5XcEx71Hm+JXOg7gGYRYFLg6A8g7E3Ge4cWY26q7/9cToOBJ4KAwYaYRAytq5unbD+pnNGnikOR3BdNRoLmdos0IdXkI2zXfSSXwvOgnp/xrl8W/MmfbWVJhThSCrVhzHF6btdYK1laD8el1KWwuKlHfHNR6u694Now75jY3rIqCg3oHQtpokxVoKXNeUg6zRFIwS+12RqtCRBxqN6tQUqQ9FNCX6iPEdtmzBBY2hLv5XZYPqGeLONouAnb8tY9OY8N8y/o9ZVwHMJFtPxv8lPF3YS9LW5bks9cPB8FYG5lMEUvr0Id60oFPbrbH2Phw7t9D80O4MVQ/y9TNbaGZKypWXTFM8Z4mnWZmj9wPUUeNXmeP+5Ixp7kFaZ+sloT9tnvRQx6zI7zJbPnEPAWx5oo78LOhwQwR4+Y+RRhslUFf3lMfNMhS3y7TZcfW9r777e4Uh3D3roHnJTzWo6WetXbzxJ4FvtzMmXaOpjA4RpjbdSGuumbKrBjgv+9nDDZ//DO4TbTabS430bH/u45O2mlGQJIIJezDAltikYCGTip8nvTLeQ3JoNUXRMR5Wsa6Sc8qGIUbgvdGQ6Ng5/eSD/GZe3P9dfB1gXE0mUeB8SjZuuUf96vEddwlEJkzTGA5rIUmgB9KTDaTJ3TiB1+KgkHqbQhWWWEqZpGr2XsWueuEdFswVTL5DW6iiKIQD018f6pfMljgER/HVEsxOI30OdOf2b0ZC8xtH27sLEd1d//STbYjPoXUPUeFqv6/vmzM98eXbyRSS+hZm8V5DWCUum191WZeeMkj0M913UvT9IiNhz0vZaqZ2hhuffFR/lLg2imlqC19SiCIWZ7tqP3bLVC8RYtgBj0giTSn5QgGcXqG+BEA6q5y8V5S17jT6J+k40OJo0khZK0/JBnsdWZoq6i01r6/aeflfyX0kiO+1mH1X84TcXGmggPxjZvjtisxRyGEhx2fDOtZX91J5tm4rWduW/iKUjo4lYdtUFDzlX2mArQ7pV7ZVUdoKBWBdvkJM0cxs40OuQs8GpUxmRi/l+cc+uhcWzYvqc4PsMkdQFy/7+BLFQ9fRyw3p76NVuqiEfDtg5MC6SczMMdxRimHVbvx7wsn5Rfv2KKgqCmMql3B8WmlRBU+PJ0IGoy32j0EcWKRW7pRI7eF1MUGUZLQNjem+MKRF+5ALyG3HopYfg+mhW6O0CgUhJFJaY7not9G5vGs2+b2j4tYsf00vBnXGBklPwkB6C6KoAUqbHxKUMs7QKfmfqtjCIh7UuE8btJv0tD3Wmqu57PUFDIeSHcgbJ6rZmq1OvsY2ImIEslNHz0A8b9I2fGE7FPraJvMXW7fNQev+qb+Wbi6hKxE00rKbQkEfnGaL4eb/2QXT2ggk+r0QsmbOseY7cQ7JT5FLW3HvmrpvT/T8Fk05Qj65rUDy3qxNK3Pv+HN5Xlx4PV7MrSYxN4yAYGiXEqJ22cXI9JNs+v3b/2fmLife3nBMwQYXR6vdQ6ciNvNgx4u0fLFtGG9c0p5VahVVIUSN2FuWvXx0JHGAhxFeyoEifHfY1KMhBNSteYgdxbQlP1yfy8A9E1Ya+H+opkUcFO4UasTWgek1iahtQazEbXtvsV/hVsey2vHza4VeGeZ8N6L3P0PZYEVWdjDGMdnU8znctbjydGLokjt0TPCLj2DKpb5QWDXftUlxcun+68dckRvFCPAaVvqlhkxaf/7xtgWMC+qvHTEhJvnMy1/y2uMDkyz6VTdpvv5LE+tf+eBcDW2F/1mP8TwMziBfMITgQbmlk6OwuH+H3sEgKzLagV9Wv9oN9vDiouyBMCY+XHI0HlePaXr49aH7J7P4RJ9wzNsRnSk1TJc9oBDEwkTVfkw97pw+S6CS+Y+kGd0vHsPp8ohaKjKKdAxtt2or40uRCHxr/jda7uq5g5gvzjmdrkMskeTU4c6YcYVyX3RWdhYB7CsVvsAS+NicbPWhHjJRzXvq3+OBiMeBe+/Fba8+KXEH/qKN/NqLdE/xib7McaK2cyov8OlPpjtVytwTwy8LBd4sYRo2KJJBUgWQAeA/69r91MBk+0YufuNDVdz3XecQNs2JVb94lZaiVJyMvSYp5UIZ9uVnECNzw3hcxAcgr3awTi5Dm7EqFY9i3TnDD9mfvH7iVCtK9i7dUH/GKqaE98zrQB6MRhAltp0bECV3voeNb/7GzQXbeH/v+JLE1xZZoS6/cI1/Zx8A1PbpiRguupI0eEJ1Yu/+JtF/HUYVxsP0y9d8pbiv+dbICzM5sNY403Qm8NU5oE32+Gj0kH0HXPOvXWF7UJ8L+9jWMrWloHi/reGhf+trxWlURGQVjSk7/zbwejzeGad5jE1NrX0lfJlfxKtiXJ6LKHUy9XQZTfzr5he/eR00G07SomuxkHiSuuiobVPEzcmjDzjBE/mY+gst91AH0YyHaiapyKpUFCzbXQO/iUOtd/uo0W6etJ3gCRcdnkxsAizP4/rI4KyxF4evCyqgZvnnTJ2M7a4DjQ46CIbc9+R0NiXR04Dko4awYYE4yXpBvrcHeBod++CrPUkEzY/zr1nSJkyc7iO2uj1jnN5f1XeLUY+ku0Bt7mJBdUCj6Mvh98//yW83ne8HUvTYjmlF8P7DfNN4f/6AWSanhWafZ25nzaUcuutvhnyXPNFZNAr/AY7ghfz1olPlsm66qpkzAF7/bHddp20r0J92NVoyyEc+OLNsA5rIwL8V+Z6fK/fmyCLXHemzlatT39ZWsWMFf5/UCGs8I6k/yz7ItXfhlX+30mUb0q22QzJofdwi2JRozCMMGjKTRMn5ZE3+WRFyuihmJUb/JofhV7m/kZPzorvob7/1mH1rDGzuq3s8GmHxVaoVwiioYicVoyzGtwjYSzoPPspAkrJaH87ugd8bmd8l8e8mAfrbSZ3fWOJJVkSSwX4Ym1NcZvrbb5C0j5yxwgxz9vNK/o9PUQGr7Jg6KWtGLph+U/fyR1cvX8HlhOyHw0asW5n54PS5svIIeEwYPCAls/CEOCdOeWF0XCA2SU5beiMwrK7PU9fjo2zyszY+k5h1dr1Aqrf3/rXarzK8w8z2sAi1Et07qWKrS+DOITU4mSKUgHhAL6DXWVb+LEzJwrIlK9x7VMWKayyGRJ0RTOFsuNsEXN6zVXQoI+jVFDUgrkrxXkASCptefaWGE0ihZurb42bP1PyCxTJnyAVE9YDSzsxItqjJpelkBypjPGXQc6eY9D2aO8jE5XKW7Ha+gLpxZ6yivUoVgY2Vb//UWX4pXioPblmSO45+1PiLcwuNTpF+WpQsOIpGhF9G+k5S6W/7YklZKHUlrDexG7gMP10FJGfotn1rywRY1lEWt6AEsfcLJsmLVlUppBLlUcXAnWlDJZ/3Qor3iMLVJeUBXdLqh2PYmEAICs7kZQ47pkuVeybM1J9e+3HCnlSfUdedY/caEEHhWSvntDXlvOsgs8wfgmkoMe75SKyJOFGXyItTuwbVl/ka8N6wgPRPz1t1M1/e1Nwf9Ienxpd0iU8c2vyUmjKIQituo33dWRNOlI6wm94lgiD4rLXnRn3uW1CRCoktjbs82+rtfOTOt+chygKCvCjoPZSe484l5MnUlAfSeM1pvoJclb4ksu5/X2NvJpn/MTejHuX551+ucbBPR+Aiq9OSWtYmt7rBR+FTw5Q/m5XCzwCo6f+PwXsHt5ItBAwN6y+9Lb2ZG1teFCb4hroztXaLIuxC19ttdn7btY1HvAV2Yszz0zocOQh4zBC8Cti0fs8VAYyeldQnTCzhHuy+Fbs58AMcqNdUG0gOhYkhsdf+MfTyrGx1LHe7kj0Ip3+ikMPcDKQCKT1NeMddOKMY0CU9MBhVxLqt9rwY6MeCM5yFyym7ecFuNptbnOJjAJpF4HXmNS7Hkshuz/6ELq33sg3Dy0lDVgPaCVxmijnPHNfvAvNILiw9oMyjkkxQq36zsTwj0vOt1LbC+yhyPGzACdHuFf2+dx707RBHHdbDt9PEK693ifafTvzD2eIv6N63CbEj1MTEIpo6MrVeq8+zlC7eWWJUM4ltVZr/Bz3+e67zokcusvbgpTu9cqhlrM0VI5r5xPRcm/MoD/2Z1GJHlKmva+w15rO28cLGExombNgF90Ij+ykwm3MkLT/qZRVNv8caNGY7XpqT1TntePAD4kd/aw5gEHqD89PCu6N3jttv9rRIiMe+oGgK7SqKCXN8rxVigATKTwMl3um4StiQL5iKZXO7mBTSe7yJfDz46LQczdHnZHhwhcUmaz6px7zqIAQTzyq3hKFQJ0Usxe+2jkRbDrLq58GqZTz3/Gl7hOOsMZxi+EiLz4YwNGPKN2O+DIsH0asF+XDZge42mwgLiUl/XOf1q+NWSEfVCVSSinzS4qiZ+P2/DLjV67FkaAqeHyX5hXpVvg01LsUg4kVPNAEWLLFmlN9HgvP8i0jpU6ze7qDU+Qvg4Cc8qFjTNs44XHchVAmi8oU0sIttIyaxFeun99aByv6zVq2SHMhQcK8nK7rOQrfiKuIiUlcl6CeuTJxvDEWI/5DUv+Chc6x/96cmkMYYI6RuWqz7HzPQxNbM8wpDfgKcwh6rD/voK7TBZQbQVoNcTojVaxz6TbuOhr8hs8WWgd/uRG1n1FHPo1lYTmLZ6fdYZ+w6FhzqsBg8X1dyhf3zgZhPDlor7Bom7vp4LRPZiudJOK2flHaOgCBNKC9r6kwa04oqT0ZeWv4BQK84TN8bf2BtqJ/0Wk/DiekcMS4z2l74lkvozIYUs2TM6Y+QrGk/Y8BoOmnKR4oB4liM0VwbGqinoNonfIZA4J2q5wKphMM5QIGcWJL0eWudoGxyGyGGgPiIt97uq9AdpVJDyUCxvkUj33LciaWB623JjjlisTkhY8WVdYiD5XrNkn2f+ew5tUm9vq9HOkAMnV2PNcjco/LRkkTMVNxDK5gVflsXrg8KJv97cd9MebmPD7fc8G1BuMEN/mXzwT6c4JuaA7SmMi8J+faFepe4EE1am38eJUZYf1Xo7Ktqkj49bcLrE57G0g1l5+TKyGGVBU2Xm80caxmK5KTqqkfXDxKho1FfPPxsFZ1pIfJI8cvzMLDlQDEcYZTH9jYD1YRZuVlcfd4WV9i8E6+VHovZMHH1wiA/prFmaDpkASRakn8hpgqb/LfSEcL1wzlnf1MSBnLKpXCiqrbLf3yUM8ed3GjSWH4BMfyFZAL61fVsoHSjn+mpSlmZpPEAfRPmnvq1TUVTG9+tNo5ZidG0r4CExhVEWjBIJhLSyps/x0FOBKuehRsP4KYEI2xe/0cYj+iqqtNiq9QkGdCm8nHcWCA9fWHtOh3AhToFJhRRziUqfThjI3f2GHZTDyiaERb5SWEl6KKk7xH9gMnOyt1Ih3Hg3hbnSeiheI6guicspkbJN8z2yazR3Ur9dxR8/qmlbufJ56RVvRTmVdhOl+HHrgSkNvXp3dXNbBBCKMh9aSbKQxSliqnOIursiu183jpiE++rTTpRwO4IIy+9YA6gp/fB6k5vQxLvTMoKKPLeBapt8Fbmabuo0fbXl7QRpS2p+Q/UPuUKyBy1kbEjVawMjN/QihPpk3lzMYeYbJc2HfpQe2MdZZWuZNo/9YiXzGOEvHTX5AarQnZU+ZdvSzDqDnt0nqMOjWdAFIXkbBQPdi2Qx1oqRo3qYlHKRnTSQP3ajg5siMOO0o2YeBfmZ1XG22dLyFzZqbvcDR3zoR7bfCcL7N9hRt8vf5cqLNU0Eyg/kRsWioWhnNHERHBmrVgRLzpCPbME7XNzB9LkUuU0/0hMH1A+6WdU+5wDC4jT4fSO7qltAwEQcpmUsM1G5PMBg4QvzXqgakd9TqO8HleJqriIthNxK25U3x3eFTXZECKVNYGtA7hWqwz2zCAfh6H1+X6G1fj5xPQHuEoiraHXdMtRLgbS0bD0IDWceLCrf7/lieCUrObas7dp3ZDpB9KBKrIzm8GourBcn0KXHrUY/lF9xfEFtNfge77uhKXuDhbny8YOkzmpYSwhHbtuWLrRFqJtN1Epg2E5s0paiC5HH5JvY58wwh9yFtNfzUk/MNFXRg2RmBNWU4vYfsuSvxDj1c2qHWjD2OwFxmFHp+ryb7kIV4Cg58kBp1r4PwbpT6tq7KzuK3dxyPXg3aExMkq/X7M0lXFihoEK2DkRteDcJHIpLXFkmEQ/c4rr78SxN8bbZEKjlQSn+OXMmwE4csWbpxMJScibbAK+GI5+PqnoYiIjB1rRpnF5QtqZYQVx47QPOwrx/hyNqLVS0zlK0NpsIkzhslWiuKOFxj4Lig/GdJasUSQ33Q4UQfshxVvoC+XOlbMh25Bn62xAb5ld7U0lZRCwpPrEQFKBfanh3pjroInnackQmMKzspnD4GfbOnIbjoV83Ct9c5inHoLCO6rEO56hyvs4K5zGvQd8mevDdwGI5bt/0w02+Z/O12nphY9dxXdfWLDkQXSMgLDHa4yzX1/hL9dhyXTjkAAf4Fe3+qF8f2yrNe9OV7HYYdFolGNmlNBDuVToljkgiIs3mNovr/RXDmMe0lBHkQ/ChDfM1Bqu4QsV+yWDdi4bkfStG7r0ZjVizH9MC4SWvwUo32RGXjJWrkRJUHzNd/AfdAwX27luAXNM3cWptGvE6mWlnWkAfwWyIV5vl5cjRS5i5pjknTCxzCZbqh3T3zJ+ZFUl42VRP3uoC8msQzPyfbRkfx7kSW+TvTjQXvaRqDDjpcXFLb4xiQfs+2Wrkr0+Xo7DYS4cDLenFS/mw+OW5B8y3yVz8KkDtwbnD9fuyWEa25H0DgsxH2h6yZX0H1RlU263ON1NDXvUOpagIidPmxKtPLOje1WZH7UzyzBHPhkaPJM1kPdkXy9AV6ZIZ5I+cfj6eh2FsDUXrBHsnUefhhA/3nYl1kKevKjHC5dZe3Wx/0JesZlANgMwqsnZnuJA0Q747B+EyO1Q/B/cBIoBPbuaaDtEtsPiLIs2eY72u0RW2jlLv9VAHmgg7WuWd1tKX8IfQ8TgiCYU0GCLyhYjeISgVBZr+JDDZjAEmQXpuGOeJvENwjkLTvb+E5i0G4P2MHgaTAfCExOMULqI3cyojaPHEeizNqOjwdwluQFysDGnsSyg44rGWIQv9+1lss/gN3CkNbSbyN9SHQxGUWfynvwo+RZ7fl1X2b1Znwlue5BYFjtccUtEsDjmcbKH5P1UwBPLN4ClgXCFKQovY8aHrJORbL0xZlH4DQDWv07d9xIwZTfPXR7wIATnEisI8MrXmdI2y9/zmoQLnS5BM5vnPCH98YTd9H8lqYr9UoQpHuf1+VTQlKvudbweQr3XtkKAfjAAzH5rnexlhBFB+fg2mmZFaQ+ZBqp5fuBNGPf/+hMfb6/KwMrMHxcsw9DmNk1mz3mEcxTtwx1YQ6gfT5nF8gJQ3BuLIW/Q1aGN46bcDPl7gJ+/OMCSHvaqJOXGannBxnjemP1wMVf1hEvnD6iyIqtblVz8bRty2i1EkeXfXIBDrAG4F9acaK0kArw1VdcjDh5cGHGE7Jt3Xq1y342OrkDJBNCTalTKHUtL5qObuK5K1fJ/ulAM25Xe5+LH79CLVBUE1JVvgCa4SsqNPFvzPvOC43E8kmk0TjK+W38S/Pr0vlNcYyXwNDamRD2AGYc/bK14NMVDA+RsqARa3mortzReWwDejW5I9Df8UNNOsQVAhpfPLODrDh7H1a/nhqM/3B/mhtASgiXzi2ZmSVxzUN1sUjvKbdW8cN8TWlL/+OnmEgsAkfVuwRnp8HtZjrOuxTPeSd5mDjzvaqdr7YL9AFJMZA0ESOYEvarMEweYqBMfsGYG10R4ee6vuHpoqlKRWCxs4JLA8+90JIiq3jH3yzBWUyYRUplm2s71vz2G6Blz4EjC/ebtPHurY3+vswJG+RpzQtpJVq7GB5lNC6mUNVMTt+5r7aoO4+pu82UYNuk9WGhHhDlpukq/Het7w7gi0GoJ2l28zNSkuDgAafcd2TmvQyh1dphj4uJkdHAfvqN+hmZhC2s/eopR1W6+Tr/bX8Nseq44MkXTdmVe/5eBHCQ70Lhe7Zje7tzd0lGZijoWucfFfepvJpiZYCzOgZWbVCoqcpfimUfa+gzfKqFCCkRSDNmVa5OXbMXpS5IsF6OCWJDPW4++0Ir+RFzY9OGi+FloPrvk/I8DrX2n/L6zLrNYo4ZwDFgHn52+wexCk2flGEIkNBsEaeZh8NZoy6w8Lnk68+PuQuai0NlivD0cmYTEL3APY4qbS4gxaZtiEXDhkDHiRGba+Vp/7jeA+yWU1PXQpwoePgFWtu3ExH/zK0mjKAwhhvCsA9qV1haCzRxHvg7Cdg64OTi5fTzBi+6s7M+nfp4KBXlZCKvfNKNXJ75oXfVUBpE11ka6/cTtK/DxQwhBwkSkBTlGTRSuoYqXIcZwcrlmqTO9VvGHsy37WlbP2xzW+juOzjj/ad3dEt60z2zZwyho0bSZaEfJl5wcQdLycLwgYZilhnO1inTblX7saj/PPnxY7BO6+QR88LuMg4baPO+dZfLKRkt9HxrCoo1v63Za1bLluRt2t9oDmNEV/BuOnNGMQw0mtSH52pVSENCOfvWTWJvHliNfDt/2PAeDWCAs45sNK0UAKhdM6Y0ijCp6MQLHLWwYH6uIbFQR5Mi4ft7Pq7ep9pBJRTsl+PD2qwsfBwILithZNCrDvUu09f4n8tIrc4ApUYsLM31/CotnZwxCLzRPd67CN7fCrn9LjfGCm16tRvR6mUccI8lbfdv16O4hWMZ7x5qssek75dD9sU8ykSAsbYETAzgcyu243smxN2DUvZMznqxmQyunAd2f9wjbwoiVuseCUdEGrvGeYthIBEU980UVxZ/yaG7iotDWHOHUvFOY+fNX1xwlOKnHTN/pRirZiUwM+9g4umiSHCvvVrM7Q3vYXvT6CGLp3Yj0Yz8vN80iiCqFC3MVu4rqTN9AbVXlud/AgZBmt/o2j/Zcs/pLKQvuWIot6drkgEiNum3td2B6fhs+Q49URyeNYVoZOS3pdszHqDsjXSq8OW2Xr+TVt0S4ffw6sSZEOxgbKKqC0pTGtvYE5hsJ4g0IapDzdNQ7s/CswnwHFq6KYt0iLMng8ffqLAidvHi03pYiuNVHJsO8jK/8GfTG3s58PEIEvlZHdfqWC8+S3t/sCfPXXRLYWHK+/LgaFosN4fSrbqt76g62M8f4vQrcjFfQg0IE9Sd+eYuMUuppo7x1G6NTofsEPzFACSCqdqJkwo/3NNBgucMSbXRwTPZmlr6XJUUr4fBsdkyyayyr0vmm//Z3RjcA4a1DZKbyGvTVU9OFKtbJR/oaZX5iNfunSuYixw9RSkhGDc1sk6HzM6iO+xqqS1am4Q6yrSN40BP+hPxD7WFEUvl6tSm1DGjQP/DYuYxeLaCWJt6sGw04lCCAjudd7YAlflJxM6258yTNk9PkNQ2/8UrD3X8VlZOdQO8qOplTggOMSHpNYvRXhuGDQGBoMa4Vlr0EY+Wz7aWxytSj+ecMeWwoMyEQbw3j08PHKT/rUtnfbe9TAMAqvALVUG2kuIOYiM8/GjF49UK2I7b/61MQCYaIu3XzR29MJgjLu+3ZZ3QlrpcfQ861TOApWCOwX9frFKHtJLVXn7OfMuveDvSQKI98UNJ1mGBHD/eWScMnD7qFor1YBpQ0QRMAX6oEHs9WiPdTwAA6TlH5lWVfwkW+EsYBfmXAGFLip+Ww6lD58zPA9uItbUj8JZFLJTRhMVrXCjUbsM797NX0Wf4opFn0th/QJ4x2VKzvuj6X+mtFMXU0Banq2PrqTijL0FQIzbgUo3r2r1m6/2wsHK2S+RHSU9pII1KF4gjpYhUdlDirSJ+k+ADzhXDrqPHDh86O7b9qx3mLU9ExWv9Nw2bzE/kV1kA0qlTsREj4S5xJCCJzKhK9R+mXSWANICnfZVMPQ5Cg3Mz8sRaGMLb4aSunEwqnBnTPI+jUWS/Dteq3TK3Hnb/s4ozPD4IWok7TyQtICjJZjBkzkoOvvLPLD19BNrr6qxf3INW1tZIEkxYZEa0wzdrpz7rtOlGYB0pG0MGqFmE3HkRqkXdKHTP/M4TdnEyWzx2vbFMbm1dnHYpwmy/XrQA20hDCWJrwkCh0/ndsGPWxn3kTklUi24y0MEeeNbdxYCXnef29GTw8KTYPPHrXVe0XMSxbFtfWIdx+VblMgnuGgnmEVLO0qdYBUEoXz+zI6tSZZVdUj3vKE8ObagUsniIwsLpwiyd/wH6+2RzEKPlDLB8pNV0X9lZsU4x62vmSvHbYryQ8tagLZn5R7wFHWP8C38VArhJuzfLdGYQ6sIEZVQxMSnkaiSOTQ2gBTPFdE8aeHciXyrbpT8v0NWMbfaUa/vXZ+yHe82L+WRFr+oIyeR3CZ5bIw1vuNePhVNrbX1kCSAiHFl/YejACvCZHegj3TvVc6l3ocjfghLCOAUfKx8nx+lcdsWFjuP27O7iL4ECQlrFrtd045iljKecMF6I+AYYro0bk4NKlrnTTLquiSiVRjk6GgcaUqv5PI+IralMbn4/aV/voNTj2AuygiBDK+Np+j7aU/S/BaCPL9Snwf5wJSqywUFBcwPCAaQskkGReC4TqCHq1h8GcFhK6yyNvO6OLiY6pWt/IaEcI8auvcvPx30CMUMM0Lyl9nr1Ip5D+7jnd3zcerS4fVmOrnHXQnEgQJaRLIIV+qcu2flPcq/1YfSq89LPIcmwo+pgGqFKW8ojPE3rf2qZDM91iHhMVaeYfIWyQZd4I3JZ586tPhfomNcAc9bl+Fi5RKT7gYXcd5HarIBQKzfJ6gTy6CU7NiZNyi/F3YJaAboCO/mZEq4i6IY+XN0CrkavQ8wAOIOX89ewYfWDxjK9FXw6Ix2MhsjMBdTVPC5y6RSRKdSLkS3m/nQ9e0B/JEM6N+9uBWKPcW7stwGRxjgVxBgvU9fCWVzPHXU6i3LQAJXl2bUYikP0R6fD+fzYDLpsI7aOmKFn6zknGf4B5P+3AzB0JwwLDSx7vlf2d4TMgMpDoLe2eDhdoxjsT1Vjda7/Nq7cPV80JAHRPOwYRJJsc7Y4tE9IR76I0pfFpY1EoCLX91CgONf/I6QzwAeEuFmX5C84p/qwyow8ZDZ4mazI1ZgEerBcTnokwfYELkMPYdkRAkTC2yTJHKYt9ww4VIE9MEQvyIgP3i0lyvQFhg3NfjZKgrkfKYCr7LCDHLLagg5piGTjkHDFdsRbrhmkLwPYGsxeZHmLCBjixZZ/le081bHKovToseVy+saWQgPgZlR3RrsH7+wmr8SdrhnY/FMN3CHCc7+TaUxlyih+f72Af41GaPUf4j5Sy/8u+vuJwbzpmhadfZLzFISIHr7Ve7g28AB9crLEH6ChxHDnoFD+8yF9dyZR4+bzFZ2iCCCeRdVdwZfBTyrXk0QzsT8LU6MRNEhfj29/OdBAijkBP8mBW62LsDutW45iuAn3B6t9b4Qqyv+j64ExSPCQeTI16TIcl4fbNh9owcF+0PRpKG8NPvEmS4paAxDRhj0XQSw7JqmP0eDmSMX5/WmNQgGrFT1M6oU9wOxNalf5PmxnlhZPAPUh3xccwh0TfFeB3u50GuQ+PFlQLG5Rje/yJpppc4JF5V8VaG2Z4vy8GzIc8LgM0mNR9QzZk+nT+14ZYwr+Q0ykJ0QgWZC+hjqXaLHzIqWL8Whm08pEkMLBmbMxPh69kcgzNkZUtgqsQ6RWuxpA2kHlb8xViTQjYMzk6WFtOSh2gRlWyTX4j3oBj2OqlXTdk5NVXbl0Zk7APhxIeoS8DbQG9eyh09LHYOVRNta00Xj3KLF4fR7bd+kNj+GyQGDpCH+uo53SdGf49v7RNHgM5fFNYMuWWNx5U4S7J/u5Yq05rIx7A/4c6SK83JmQXbsakLDZIwGv89kTg0B6dg9AAgjG35BMzXkQMgYhyFomt0QaJcjA45q9ktB4Tb3cXvJc8DoCQh60vGzGkYLOqMo8HnlMp4/e0A+NWwYHFZGDMojNzO5K6G+Iq6JjXy3QqdS5x1QAUyDmNAzKOuP2tlkAEZOa7CZxcqrU3HG1jXH3YKkEeD3Tz39/GbRbCzadwxdhtRIzVZ/CLYDaBE3Gl9VUsf8zxRSr91Ie6B3uNAMLANiaB+NxYWOAf2uCzOwSkHAm0SO/huHpAjrhBeJo7AuawdQwi3fYPaMyprCwi1uYWhXcKBT5gF00KFHJ3RbitqjMew7svJtvTzefqI21PeHtoA8ebvfulX+TWu7bhaJowT6i53kL8QzNljyL0+/eSoosLGehx2Ck0fW7bcGQ1tgQFaZTF5PofgHk8BN6yrDWIFtCSY8/kzESZJn/Ji+zQJv2UQfkIPnVmH5igsP+3IAw1nAwpFtZgAbVZZggkUhmm8vWgagdhHuKKioeyAi4l9DPQedni6UuSlwp793pnu7CfirtgeEUHTceHF9X/6vQ/12ZsR7tB/sIOOLv6wi+gEMugEt/mSiDtP5IojF1RlO96cSgarVeaROeLjGhsRrC80K+OiwrdPINR70/RYJ+EaU0PmvnMIwHNmXq3YEG/5WjSZTQCb7/pTZI36IIaogVGUN45zfdBsbMmKHo4afrzI/9Vsogn/wLkoGlJCIyJaHAeHJKS6D5EnP7ilJ285Ztwby1jjL0Adn191d4BIWlAc4Ie5ynKc51szuvwy41z4EObKWxqPI35XnuHbtMZFG1yimPQDun5DUU6L+cNCzCwEs3+dPqTvnd2ZoDKIoX/tL+6FksIX4wnzLH9lSLQHfsujfZ6nB4/9Br71ojITVWXM1N51Utd4xGHieHB5n8uvxlo2uzzAVHbRRhAUpziITuLte++FHFu1QyTsmoX0Y7Fgb1HyPNAvcNvFqJBwj0KKBuFqKMP8b+gsHBWtpQSS0KKI9nK7LGguaqqhncdpprLTbBZ0j4uKK1oc8dKSN43wIJSvtz3B5cbI4dfrkbbnPCVilTXeyeF8Xo34tenU2QnHnpi4H7tBgNFy+xmIaZz3OuhtKnL1d8Xe6DuKeKlyKlcHpU/aDBHYminy0CC6nICCmH5LD6OAUamqHnCGg1UstTcT8vQoEdfxG5VG4v1dsjTYxcTHW0Pg+BBwI+NjoppnKYZOmu7q8YuwZN/PvveCNJjGUL14jzTAXGbBErLAMb2v7MsbMLjhW3RBIjTDggAXxVrFC4YZUOv9G2lvyH6AIC/QHYMmoQq6RPj1Bs5qQD65ER4CruGvoTcusuEXEfe+663ieZgXn6UK6QamnAB9yVkhhGi4Pdiv4XSi+t+q7Ab/aiKCItn/7UiWVwY7URpqAPwwVpnlmcrm41p/3vV6RTe3UJWQkbqzWvt8t6YeDBIuj8djf9y270Eb4J/J++SWO6sidHxGIwQzJMYxMBU3gmojxohgGEP9N3WL/nsNDAG6aXWDRc3GgHogE73Xu4ymNaNLoRH2hV4mD3VsaOBp7YjMUiSaVDtJN5X0XQKhTxHT9ocxp+bkNNizZFfoztvDn+8Ibm/DlGS4iS9Zrh1bP+LXt5TQfdITm0Z5q1keFlXQtk9tvIz4KVvKDhFJeet1X3gSbGM4TCn7CKo2vH3h3VcxHMiMy67WlnXyYGUFxpjQ+6Vzzt/3AdYVzvHlMhLC33EgfjVU4V5df2T341k82jAwAKOuFU0Nf0FKZ8SY/MKRraCHslIf1O8V5yAN5aamiWDXujiekuQSSWn/rvY1RkJu6YqhZDyYB4rXPQdbmi7ur+vFfnGY6n4jZTZHY+8iO5RthjFBPP3xRilJuicWSnO7iVIgZp0vsUmd90Ym/LgO21juc5MPS00wQIqjVh+BCjPjfv2dPoiU44F5IJ9N5w5ziq6xuiR7jXFbAWFEu+EHLLP6V7ULRLMSGwIJFqZNTzKKWO/r0lZoJ1UadPUWbfdrfDwsjm37RR15Rue7Qak0opyQs6e3xeYKtMiK/JYCbdo1Q2JLBiQ/sRT4uDR9oZBeeP1Nx6ejZDiNhn0bYbONLfqhKk/4TRxQ17MfFFHVcS0ev6KlYKCsYsqTOWnE5dgFD+jC00+q4U+9mRh9sDR71Ay2L1xvfJOXMywwLsx5mjwei2qUlM2dUP6Cs6nJFvcil2pV9matAfgT2rQfuCZ/MQjQ3fi2mVgnsJWX7814HYHlODjwtDQGc5YbOOd14iiTfBTr9Of/3316KBVHh0jxhosMhoS4AoezzIvi2F6XtONSQIgcveG3UK4jW6qqTxKcwRWCLHW8ws7Orxf2rG6MHhYHRRd3aaNkfblioUV1OmieRCBktyygOIJBYzA3VtW2RbYA/WcW2eAQGKM/2tVXq/bSnQDWn9i/hTQZGin4lqfCVadnxdce6DKo8wLMx3R9ibkx9H2dxguPLY+A/OmSIAaRUI97/fQ+6sfCce5lFMyc69bD9AlV378EnWfPK1Yamn3B7TMJELTCMr+iHTEr4YwLwAYcdhNo9Lu4+psAKmRgmC5zjWtW8nQpL7jjvRGzJDgoJH9tBoG8iv8E2Tvtv6KiNS/kOfTiODbDvSRNuOXnuPtnucaSkXPswDlqRZsMTtcfI38wR8XX1rLE+YtbHjy8hd8sQSVb0i71y77P0jhhRMog3z06gEzIYiJ3KUoviP+O1uIJAeFke6lppyya7argUTaxViTGn2XsVCIe5wAzT10FAYnh8SLGuQBaR4eNr8UgwuwiNL2wtapWwsC//VNr3ue5wRgYiM14dMnaOKDZlvee2P0NFqY5z6UJSLcDLnW58C7L7UsQETaEWakhZ+toybffH1gbCRSBUct2YjAkOaw9rvC9kgYRBy320BqhP/9c2a/fiMP3cli3VM/3v9ylE6JlvJinxAgPKTCZOY3rFGHTRnivXlmQVnZdF4Kmpq5anmBPMwxptp58BMMoQBA8jbzBcFrczkSOnAVOvyi/gfayAk8O4q+tynDX/HTdyGhNYZLk0kiaOvVCf3NGf6OTiuYLU6SIJMdbzR98AUSA/lioV4e1pziYS7mAk4YAuFa/YWJu+96A+aNM23JUYUIEemtBfDgHdbaxoJolwddR1tTa5+jBuZV4h/J9ZvIqxpgPk9yS233rvDhORnnPlVmAFPx4t1Mn11xdZMOhB57+i63PwSQhRVSKIOxrquoBtvbNQ3eR4QQbZgQhEf8GsgwztH2dLKtV0OlI2MDpivO2uO6HlJCszd7yg2u5/fEnQBMA4/iAr+o76eW43Vl1XmgR8ntETIumd05c00qRBPock6SlfSS7BUMrNPjqsKAbZ3ZXwKkgJgQKyN90v3yiLATmk4eW37uw3D3sFlNaWmv3jppQU0ZdZ4suxPilxMNv2ihuq+9us2T4f/U1Z+YnXjezuT5IkP0DpeFNkcjI7BSnXeg5m/H9gTnNwwQVtgkdPvZzrZIvB5QfR2csjuXrcSILA7+Trz0Iw37NxpbGz6ecOfslGFjFIBQnc2cSxlGjHPmsQiIqIU2KD5FLEJL+Xy+rqzDj9Tykj+Pi/kLgoHSyggpGkXP/EXMGwJNcqbut2bbdv/dBMx6u8nEHUv/Sc4fyLO0cgb5+Q54iSZxhP3IQaGO8Lp8rLy6UHKxlcDX9FGvXoYpj6VRPsg2gknygKcVdWbv20UZFyu040NPTpe5pglU2kuZi8zojq1yf+LY9iAYAZP6lSTqpGbdurE2NH5i67QEAw1zkdO8AbLlXDRfwoWdn9b4G/grL74ALuPRJG4jxYHFECql6IE78l8VsDpCIBmH7EOcP/B3yq8JKLwN+6mTEas49kCNpvr+ME3qCz5qopLvdOYLZz+PIiaJs/9Y194EsCkCFkJ+47row5KERbM1zbToG9/VoOA5AJL1ZHgYE1RZt59Bk2/pEuEHKIG7vcUhSI95F+tVrx8ZPSNgAHC1IDQGSQRT/bY0XFv/6j5EYf3FsUz1Wr28JUbvw1yzhtuzCID7usI3QVDzjG7iBsTFWAmKJ3pVhEQBEAcOY44SsLLeot58A4eEJnJFOT+AxNEiDlb88IDkSbeR/nGPTEEz6UqZhDv7CQJqrjXOYLDueW/oo6PH0GDxIo5tq48kHx6LLw21Gm1EAKhevhkEhAo0/Wfd86YrE1+XLr1F6K3yAvYKyqTV9ekMAILnLejh5lgxAaHkiPtggSeE+AW943nnATxT59YAc+mF14no4i5Jk8Z9OJfymyp9ChRHJEpAC5R9k2ZS8ISlNStEK20L6NVcL6by/LHKch98bdWEkhsb9kNVFB8UT1gyuxYPIoRCDKJKm2YbnbL1nHjTVE+xIcwWswkkC1UzU38AancCAKIIcSBX47YIGbkvYL5PUUdMcrbdAHFNExbpmwYohfctue1vRuNoXhL/4maIWySPyuWZGoCuNqmxymPx1Tn+yY3Kw8S9aKd9cF5eKPX3OeX1Tli7QMhiQg4rr442+3tE8PG60ZNPrMch9SNX7px5PwoQviz8SQgfjt9s4uiQebAZE6lO3wVs+cDYiFig8ed6eHAuAIgos+XE6W1FMDr8z4czaXwLiW/A1I4+lfSBPw1jSe5uV7mj860Ou8wmyd7rvInpzV5fwmK9u3g0CDgUBWDwixd/BL2AeD/LugLSa9RmDwa0TI0lg+7yZfNGOkBulsXKSesARZNoFk6klsGADbu6GbN/uyH03Tq3DI8N/ngqqIZi10SEaol/zK0Cvf+mfc4rtTWSGArPmd2QPi1oH6vJiX+NrhWkL3DgQLpigF3nKtyQGQlLQUEAnomr7gTCNq6Z5WUhqJhN6lGS7SRFIOyVFIzAzJtZn/z/260phyLeJK1pJwXv5EGPLjiiHf8H8XJWW8Vja/wLHJ9zQc/pEePnzFubM2DCNDTC74r8eBDOp68gVFVR4AcQCqPEwZ2+4zy4t6T+ermLRVSWLfk3PcRniEtxhhgZJcP/6pnJu9+BNzstNoGrLWlurDO0fCzBHFa05Gvk2bUewKpMLfnOemj5+MT94zv8mAWIiL1V5ci7EqgTLV+5jKZPq6uEPf3CFpTtzH3aPcLbZEwwf5PzEvayYfykC7YsKQmOw6ZyermaPC1Zj7RhzDs/QbU0+9pkIH3d2xIxG2rrm9ZEdvmHdV+Pv0mFb573dll4s4+LFz/jevl1MWB5GVdMEfrwrm4DhxwLx8k0rpPXBmqyQ0b8BrGVX6O93yip9RZAptjiR5HXz7W9WjyB9zuqVaB68JG7CYDz62z0a8EVPazuEPKDmFZwbc/7KFt0f8ruBQiVsEj9oujS4c/30CrvqfpyacvrKMrpRfZ4/k3rJ8j9Hw/6hYXBINNqiqiJuX9wVZvSqDIzQgLsYr4v/bawSqo/wqDYQ8IoSM3KnIKh93VjyLRx5lLj5tz38tzugSSvzJPp1DnwCTsSvAwUfq/4eVCUkrVnrDO34Yh7AUa9Pb6l1mnSKYxcb71x23qiv1MnLqLBNlXfrAjZOjGayG/m3e6zOOy1gdSqaT8KwCc4Pp4lBp7YjJPFJoZlBPgBE5VpGseqUG0gcvs/UDXlr7gaKNnnSF6+8z5juYM/iV9d0hP0qB4ctuNyMWv1WXUKlwiOD494XreqESXGFhOhR2x06PzXskA4Ew1CGn7m0zT6pdGg7MxDwLgo/Fjf5Dpq3p7llgg4ba2mJsIm9EdNJc50p6CGDqgdkE8P6rpjbTkMho5hh/Tc6wcGvT0wXu+ImH+TvfqR80f80XazzHZw/a6fA7ybEV9HY+7dB1TuZwMYTOjwU4MIF95SIVcjc+WTeYyY99uGGgN8TM3PAFDKjKXXyqAu+Gy3/mXPan+9ljPNRzNY4pt4A05etYmgktcblmLXDrcdctO9kUVr8L74XZBF6WNEuRhmlor8/3eguM79IaC4JTrrV6N9rbWgVRw6kDvJwhmPmS3zUvicQnlb7mjAJg/wM305lNxWeI0SquL13SAYVpdb7njCNkwwdZh39+40JtxXFISOU8HlZW/I7HdtujXEZr/2k9cjUT6obacQ4BAPnfik7fBuavuRlTz9CTpmU9F8cw7eTHd4+Jse8UvOdRCN7M8ECDWBy+/UqJax5k0rEQ/0szpdJ9FSlvI28F/3t+iIT6k86sCE9ViMlyLl4MtvguTzAAMY1bIw9eH+P7M4Rl9BVkA/gAV89Moe68OV1b30SO1s0ODnkCM3C2pruzi/bmbl9UECdomeADKjeRlAkD+r4BoR3gknE+N7gKtkW3Pb90BnYTXd/GOUHOHV9u5jXY9EBX6qMMm2yKDhAb8O3ytZUQkq9lzq2wCpLmndI/QbN+FWx6oBcGpwvkDhAugjjBOSxuoDjwH7V3DTdY/JDX93v8f/6xs/VST9UefmqWAZ4wh06+xboqjROu7kXLHACAgeA6xRQSNf592JqzD4OBsiQImQwSgsakmxI05QDH8KE2GzyfS8ZL6qMGS8PCkeItyIlNvyxrNoiIRF3Q5L7ALI/Su+J0j69Vs4I9Ivk/Enbf35dkTT9XLyIEraIM+8H71rB9T5Yp+ve8qPClzNaE8XfhDOliiLUGUhAZeWIryRkxKSwRb+tPRMwPxckp73PDuWD4MgK9mlvp/e55RQNZCLonDEnt9cGgmj0Wh8eBuWNwUfBYJkAu2uQjMzLSTBKeFPgV9Hu9IFb3WjvNxV4fGS6nE7u5/ixqqPi12J9c997bVymAWHrl2HDyyOerY1oVlPZTM5CFsNDFShgyNrvTt6UpzoRLv6iHugXyJi5Sh0mYL954oCUsTxqWhJRoFS+0jfuj+jyq1ozCfdLkUa4O2hScg5qjjhG/NggEIJ0S4UALxmOazeu0fye1BHm0Fu90GzsraumvV8NA2o+BIki1JhWuTAnrbOK6/j9Ml8mMGmVXUrJso3V2mESWMpilrzUH/XFw2ep4+gZOWDuNYun+25/RDZZPQxGr+5agVPqxys9jr03Vz6H6rLXM7psuaZ3kQfbsr8OOmyeKfVj7TW+fe785Mx3PJo1uvkCU5JounzugqIVeFJPPEtgImCARXLNovqOtAoul1C6qPjKeK4UaepTeWYe8XdYCwEYiUq5IayNUES6TwPiPe+XgXuEMZd7Ko55uWpIMZmyykWNWEQgzaikvYMgmphRYRYtm6fF4RjAigbR5rw69Y/HyHHBIyA7qtNo8XqJk/Np66U1nz+PZtaZL9dog5fhZq8HDPsSSZ60fDH7fshXuf8i4iDlX9V6MNlCEqoFCZFvMApN9E/SPf9NGHghD+ZBG9d2Hw1pl9RvtWuqQNLOKC1XzdFWDv8+KYhiQ/ruEmph8jX1FqcYVbZs1tXAFSAap7dWGDh9KeuMAePESSiK+8GhF9VwFJeT0L5bg/tLxZ9nvHDabWsPpbW/zi5QDjQZi4+JQnLo/8plXiYsxX+WRqSurIvbsh0l0CcBLA3mjfQGZkI8zrGjP4Er/X1yC9BD4t8wFKg18cjrjfrFqxiKiD2TbHtbUOPxf1aJBVxv+wu8s8z3M74i2Wz8qLiZT3bn8Yejx/yVcMNLPyVgv+k4VB35OlSIh4hmyC4mYKykh9vfxB8DWd3gfPwlwEAnpWC783Eez5OaXPtWtmHZsQIVcZNRXsM3CjDOTb9R1G72UE5nW1UECMxVn3Ri2rpuhtjf3mWhGPN0V/CFXomH3osabAf1HKP5wIYLXb2ZFumZYdpmXiJUe4BO98NdN32i7zyIfrOXy+AhUW6SfwJtVpcLCq4ZrxsGSxih4aCaVVz33Qhgz5f9m1X29mxX+A5prXiUYnd22eoxRhOZpm+XFqwhCY8jQfwmNFDIVospOSEJjOZm78Fthq44SuLQnSOVSQgwLduybTLYWQRBHQ1C90qawvLkfFKhLT5VPDPOq2Ff43cbX3lVWFVtgZlUr8yylrgIcEQdBoXH+DebfJbcY2Nyl7AoSi1VFV9VKUe/NTw2gX21Fbecm5Ouxe1X0Tof+FOQKgleyQ4TQqJKA2wN+T6uQY1vbIlaZm011kXSyOP/9ee8EZ2Ri183segILZmpRC7SBYy8GsfwDgwErgaVolq4KH8bKDKyP4B6xIU2ibkAQjLscRrvYb2it2vjlKiz8a+vk1qdIIqqVYvkqgUpHD4GQ8MelapKtEMI2U7ItB1glqnj9dZfvEUeLl8FhpCWu57SqMzPyei9aLLzWVYYlEx6wJ//iFgm3zn/kHlu0fSKcIePcvf0tfimsUtS3+JwrRoeGedDrSic/1rbfv4gdiBAcY9V5wepbLL7Np7uFq/pHLgkejh8Pza7/BtnJIpx55kwXSFEsp84yXWD7ciAUCllxL+3rfy42+a8suoBt4FmwKEcXQQG0R/Y6qZa8JiSETHX7dPegqzGTyxJmS5Cvgc3QKZpbDx5dGXMbc3nn1Utratvwvs1NjOr+r3Dkxq1C5Xmk8CE+DtKFU9GWgtt61q3WM9LF/goB3/DhuXsXsIDBZnnb1PjacpP8wDXetKJp57FV5lglow1XpDOUoisvlhu97bDwFAqmggZ0beSdz2Rg+KxrN7idlfZDa9jXfWvpcdppQFzinB0lKCn0b75MwgO04Lr4wPsX3S48AgulFG26Gbe3uq9SoRuhwOgcrHEAoLSKwFpPsv+nQNZPj+kqxDyMKkFxJ98dAVvSYJALQDa17N+vjxsad7yMuznioMSFFGmQJCUdsyS/UTwYrarkcG4wTJj/GCEjOdhBAyTEu/LyD0RyXV9L2U1W4Er6s15Kf8gG5suj25DwE75fhDTrxdlflzf5RQQCu2dzjtJz13wVGqTAOab8QM8nKwevmo/YPhgR1HDM+tOua99JGdLt6aB10K5s0OYbn22ni8n8M+KioWmnZUocoRwnzHyhq9mBLCjLiKMOEdsPS15npfqYSvmO492B4RZ5AjLCZEJp7suDYlRGCeFRRmn4fQ+5BL2R1pO4BHL5E6wjSwaUeVWkDtVvE/F4RIqi3xWipZF6hoANCKcCEzxeFASQD6KdY8fk0Mf3Cldn6otk3Sxv3dD0FnXe5/UkV8U0mnhwQBEjZX8kvCUBUTk8nGXYTkr4GidMoPPUCf6VYCWE1/JNQpre/3CPxwW1ayu5HGLD5JIqy72FmDjZZJIibSeh7sEAOTxAuITOPkObKQJsW93cO6yTzmQI4AAatWua1t0u9/YQkWYuwQp7AKq22qZ+EtJEujrNqmdwvS3iRCZQszfiGdzD+iigTfQDy4zgVXgP2iHNLLV2jwQTkFDsUBh5BUmQTt7VSpXAOIcU4Yc+x28oVwXaNPNX6/G6DxjCRFpl0RMJeLeh/TadB69ggUeFYvYfXysxyhVDhoRjBJRQEXa1iVp8j5fGQQvNe+EzELRdZoP75b7UMlvERBqU9bsdZtJZPvFn9KHRN9fP2hL4fYiE9F1n7NBooeto9fyTa2XBF4tq9/WxM6dCiL/fcazRTDSaQJhfrPy/RVi+SMG8K/RUsyc5JV/lGU2ZKFCRzCpFnSSsgyow/9132D1y1vCFp/WTH48cHVY+NrTD7j8nhw3pnu4ogRitKcU1Ag5PYRZ4ML1Yl7Tq54k1laDNskGNhymdyYEgrK9ije3vSztI/kP275ix1nMA/m6oaNWUk07VSYRinf5WdQc9S69VLbPcdn7lUwD9P7GSa94Q7GzG75ro9u20RYTCI/xfv6njsvmDzT59whR7xsU9/xq2HBInlCX8NoGHgG1sj/QC4HpzFMOQ8W2OTdoviLMdtYyB4QcLPIxc8T/Eh5oF/nkxevMqWI9d+0KIs3oNmSDNBDqABttNwz8S6Gq3m7EhdQHi0LnRdin1C12hJTjcB7TM+OBRiFqHqVL6F2eI0+DBTF+ZbPtw9cehn0KzVikHFORjHKazbZJK0fLIPlq3WjsDxPy1VPrgB86ugEfTDDNt08UctdnGKCfgAQCluoyo73p6G2BYHibdW1emN1lFK4Ip3nWuAJh2sDPejyFv+Fq0xgRlZBl21g6/+roPtcdXa955ahf+D6VLLjySFIzt2yPhQMgYUrPRTxWYXg1KVo6RY3LdoGq6AsC60hYoa70W0GTJDHfX6ZgIU/HYj7ZDZTDNNRi/TFhwoXKqrQe80Ang31P0q62l8hTPO5MdA5BMWXnhgxppfYEF9v4zbDQ0oOgaJjfCBw7lSZXTLwy9LdTluvXAcMxxNH0dVXtjHG1IwDo9BI4A+qFT+kB9SCKE+6gP4VFce4yX6zzkTIAkR2/a15Qs2rQXKNU9lyS7go0pRwPhBpq9r18W3/sWHt7EbZMmXJo457wskcufHmMbmoaDpHS1H8TJoiYl7QZwy8RwTs7IjPQSN2mvkfCOp2F6YCZ+VvY/daOEy3/kbP4U6AzkwsQ6Q3c4hYr4FMqO8otIgH8PcIC8+LqbGXPbsap/AGpZNnPHMP1hMPnXmAbkByz0gdns5N+DNSL2wVLfIwDKja/zTKA4Jw2uqI2/yi8eaZRhHEISWKzjjQBKcMgIVq89c+X47BVDjLfY2hs8oVilrsX4LWfRWmukIgqYsiC0MizLNH0JteH+N8cptB97IuS4+CQbXTfvY8YTPu6xfHzEjiN7soNXHKJVsf1yPtDz5mpXgP2Mn6jAtALbQcoKVRinRNCjx8k+cO26N3yxgMEj+nzxbK6xynyEwyBh1+2u2tFBjhJRslc9m0v8KqEEL1paTG26rsMryxr39Y3+bTHbdvMkMRq0OfRsHZEznW5+6mFmUjn1TjfFLXHv6nX/gwcgflrMaAfcpyx2+OlLiiuzt47EY3Ciy8IjEgWFZAOOVpnuem/bm1osCXzHukgXpH3QyCAW2xTUIkRfsDiPNF2y7DLs1Nwwg/PrpyPB4xKacgJfljskNYBJ73LO0E7NocJfgxKt4QVrDNjQSNvhFOWpsLs4EIsJ7+FvfzUEr6pC/Xlv+MgmZLidZ6IE2OZNdfrQYhuxEd+vuQfDOtyFItXy6LcEzxTj10FhB4g3X2/mWXLFZybaYK9CTJ8UF2vJnKfYVfUe7cFaeKaSocpHjUIhxdYKi63VRssMLaKVxcP8G2pa4wpk8pdPOZzRgYAhZ+/9zVkkSt3nsUfJLo9WLLzjYBT5vZd7w+jJxHzWw+9Wu0/fRUejDPlZWxNvUbPnc2IXCWAX7EmFURQWNKAIeVDLZ7+iSUubfaMSlpwuMC2Ev2cfOp3At5Ptquk9JyeJm1kiG9d9QLfdcvBjP6xXwdigQ5Wx8lve8AIBZkVLCviXvSoJ9KKiUMO6Tv+4pMyE1fuOb4Yjs5RwWsbFdZwdFXGL0J/PU7eHpWAeEtHZhI+Zv26kc90fDHvx2IPb5AsYxUGBDCvnppJY0eb5DHD/tifJDVSmNwP0ieHhnf9GAEFpV9h7+KzKzjaSHAGaT68kHiIF8CmYk1MN76IoAo4YGFf9AC3YmN8rlKcFUYcoCA+y4scxl/8hYF3TZDXEJQt5rLrOBHjR64KROc9/zwOibgHFdEluxMcR1/1Nmt77ePxVunPFVvuXOGpUykLNs93vItXnifynO66hYJ01uICn2yZlxtaKxrwK94eyLZkS7RbvqMu267LEWVd+HMscIkTuiLpIwggfDsLr/p548/XnJMNVfDxLtaKLnsbt65k4e4YtkdEXlsrwSK/ndDcC/9mmHvCxKeypbJxcZi478saZjW9Gnh4fCffJEspY1HPemROcRVAAmyTV734mBekXyhHiA/M/9CF800e1u/C6oa/lrgjvyY37IE2OaRU5EC3iXYsamLhmuYAGrjTFEMWrYJY1QEICi+egugAWKGtL5LvnNM1TUZheTehSmD/3mnvh3BL0ueeH0yrlXOGOmwXoUo/LwdtGBAfStd5xGojrFwkKoSj6rhl2sPLNkr1ayzmR/+OuHRAqacSpX0SJdSwbBjQgUj+xiHQwpT07YCmU3zVwEkU3TE3/qKg1ZfXtxajp81Zi2gatOyB8JUIjOOaN3D4q4VQUvCcPAT6h/5aaUOIFshpmG/qlRqVKc+WYCkpCGZ97db6DVGmP59hNu6+mKsDNhbCxGEc49EQvlfmLoKe7Rb/fu58QjTmNJZ02l6KWc19v9m8VFWqpr+x8kUYg8M7vIRkmHAevxGGGDSrV+RmECNq0d1OrkT9aK0fg7XurFvU4hK85/NbNDpxbYnpGJEIfW6Lbj2pyXqC5FAZCsSos3aiVFuv6wfZAOF6ivuu3UXNM96nMQvZ9plRtsMI5Pu3Hvezzuu2yP5GDd52OwzZl3BkgCpTxAjJQbPPw2Z3jkX21oRiP/TzV27tWpnoHhFoX/R8NweIwTxk9AXgtMYhZasigztoYj1O+UewxYZ4BaTwcarWcDTUzceP5RCEXOOhnhKUN813fAYvpBQLjewcGyy3FLeO/upQs8yahKWzFnTa6VKmoksfKOpJwwCHw0ZRkdtdzmml3WsrYSbPbbQYMSDAmZMAKRVd6w+q/35xM7yAKsQx0Kpzm+8W9+1tE8htIa5JUqG8JvUo2SQdFY7ubL9EpIjPbb7E0auhnTs6CCDhX+OVHnpLGX8UZysqY86srOkQFP/wlL3E/8KKDxdIL1Kmt31nQT5qFpJjq99E/VKZAM85RTm4dAIEZ7mONwM2SogPSFUHFaKDA5ksO8zCW+XlO4w1qCl0MuNAoY4Iw35LMoeZM+/n45eOGqd15y9XyOqvDE3Ilqusnt+BYbigDIm7Qrz+aytlaV7djecRqZNlBJ28Cg3or3CDoGkVUGqkweNrLfF1B8oAMiFpoRhCEQTCoMPKQ8RJWuFoav8R3BvfXr8ANBa2sX9aB2RV1o4w9f0yIeEFplY+x+COVz4loDJh9844kJhfWy3QqAbrLzKosMf++NondyjEnxjJv2Pt/nydFkluFYx6Z8+C7IwSlE/qJBV3Q5n9Sp2aaXDfXt1DlcNoGLMB0Rs0+HUCyJFiLa2OCpQRobWT5N7Hxv0aLv2z4/1EojMlF7pcSZeyMKkKhHdHEKMCsB5bHCFPXQJj5LEfhql5M7+FWjFkWfQlbroNWKGFguYI1rx5IBPsVRoyVvkCMtBlZUZwp93bxK88QvnFJL4Ovj+CtynALjNvmxU0J18vSlEMJGMFRgzZjidN+B3U9ndUYIWri0thNBvsHfg3t9TalSudHi+ttUaYaly4RzewyiB/1sDf3br2G+HOryZOcHB2hSadEMyJtq1eOVE1S0LnAIuHI/wSrvu3thzYdAS0Xm6vawB0sJiux9ozdnRlloxBW3YX8PZ948LVb7R5bKhYxSccnvz34SgZxZkpK+nOFyOJNtGc59+EfCG53YiwQdbECf3QBPHClWKPo8/IdPm6hccX2gOZDbHSDlYEyI4TVKwmrG+wR5WV5caFkEXE0lCtKoi7iRWxNXUms8XqSoE5mBCeEAV9HqFdcxIKE5TFK4vH1XTK/g7qokHyxvwbUitC/NSM3q9w9SJkIdZR7MoUAiNe+G9xho4X3SMX6ps5v6ljNgBfzsplYejXD2hH5pkScRhOh2+Bg6zl2IVSPo4pus78quHAUCHprQNkVv2222YfAmcqQdEf/1yXjBOG8NfqvUV7vdSDikPNLcV900F8SJzNS4feApH+xtceoepmAhZ+mlcfbWb8lbJgjSTyCrVx1THpSLG5ErxSwXwSeajZJCIgjs2Ph4PxZw+5Bg9jak29UwlqmSXpYS8lto9PpGhL3w5WGRtFshPVrdfP6TW+aAZ9q4cwlz5fXNnngsq5HkSBgSWAvmopTO0+hBXW4r1Ugnyei+yTNLudDWwVfO+/6CYIgfPVLvfi1oy/5N6d4V4zGF0Rh/oci/Jv1Uu9dth4Bjkq71FSANg2n7q+g9BMaDt789Z3xGDjCKuAzUELRPrSQ1c+YJNB23OKdWnTEyi03qZjly8sf1MfgdPOucHOWwcmovFeOcomcyC0DCF8YmtkfBVHtxEfRetU485eJNGTv2pDGl6QUVOHlSi9XjnbnJV16BAaPdAOjbEYBNpImNhyhxmUnPYHRTVdqVynoPDR4jlM5bzksK2DewRbqJofMU9NL4a++IygZj0LAE3sx2XY2nR1eXw+wPnuuukd4clQ5U3hs+Lp9Lc+r8BCKmhU9odrrSPR8KPiNBq2my6fAfjOfv5tuq8NLaHw/rRoFM6MY2+bB7I5AqO3zOvqG/M5rv4ctHbQPeGgbx/Sx/4lSfs4y1GrPt5KP9Mcjk14ukoWClzCQjTvk9WlKIhENUUfunS9ga9U2roZSiEqc7OWtAdpB0UsyWSa4rzn0tiggBVJ7PATZ4bTvxbBRIoB4SV53L25wMYRZSnJjm6gDtKoehikg7Bqv1i9dOA2tHHpghoSyjIpsGxyMF9Iv9dIttY3R5XWB5G3g9M+tzoXEuM5wVh6/9J7J6rgvrk2wdsbywie9lqOil66BSXsx5elp191diTWvtr05ADZb1OC0VPasM+z1/WwSOjpF/VULFhkm4T6dr/5PgDMxsZEhUJ+Jp9uVqvgl+GmMlkKG2KbdGs7iIqrlSIdMTRiixKSzYuiImIQmxXsIWW3D50fo78Timrz75V7mfhzrFUi4NO2H+3YCYDff05FbVQPR9BreyCpMOmKNTsLRqkfc3kjWt0W0sAlObeHfOoZn7XwS/l0pRopi4f9CKoxO0Zg+ERufgLD3CL4YTi+vHmAeLeP2DaO/uWb9G4OGQhOJwfiRyqDbl2DydgNcXSFRoltidk/U6ys1Cd7vsMNapJ8GFxFIJkJCoLqcZ3UoTbZtrc1Ebo0xdKBC3zJmGqcyAUXdv6yhaKKDNB4KvJ98kYLaFZkFbteLtroMcMeCNerVCkXIb80i/meUmlvADY6h5wcHyEglUdUe8zB6gtttnOzJLqE8Jw9P4aPhbhFM43KcsH7UDBry+ESxDjiu5dcpwbmHFM/og07PBlCqK/ITTEFH0LYE4B3P/AxwsuErsjho3FUKHfODeYiMqDet0LOar+dVTe/UO5/bHFbOvdYPO11joo+u2ji49m9RyngE1AWDV6QTq+bKnvvfPuP9AhyoEA+0p5/XbfTt2t0Rw0D2z9j07MJNHTym028AGTO5TgrnzcAJndnCkKE6tFTjiBHHorO1thB67Mpv167XjnDaarz8wAXayxD7LzsnP1lEkuUxECq907K4wxgG1mp9YEa69oorTRX1WbayHmLXyy+4/jL4xFYe56PNMboXuxK3wEvOJ/DVJ/L+G25AuvY/xCYSOVMKjIBwcOR5xZHn0PK5DgSGTZ4TKAPAqOExtwmBURb+ezaObfcNgYSJzrjRv29QCQV7RFtPJcD9M3TUX7JNAKnwmx6HQNBTUxUFzZtxlTJ+z4wrakuHjI/osPCXr12p7kt6kWbv/RlOu2XoWGZl2qucqKdBNzJpLoYP/k0i94mVV/ThMNOeXDQwnGeGzIyI6Od/m1ZVXuZsX7Ny2MdVVcP/VZm8ru+ZhVQtrXX8JkV6G547Ng2QXnWUhICOsPZ9WdfWXA/KMIKKN8bXRJXEFtKEwKzZ8C+tofcmjoaq8hrXY+ygmvfLtyHWS7DEVNV9/2md0hVpoFeKKhhT+0AevvMyxJKzvktgtBePP0y1PLiOnd4LViu57sf8d9HSMb7lyMmTwWs3kwe4xNpdOYMx1ARKNKJFDxK2ywnPnPX0dSKw6Z4dLaCKrc3ZWcA+1oIvXZSZUr4WoW+XxTxpWWTojNftuHIc09aP+kXEomTSmvooL9jf6qxYvLwaxg588D99WuLhrR+H9czFueVR1rHY18ujuTrfUkiq6nfdhit+GB8M7Rgmp5rHf/qcWhzNpaXVZFLmLEiyvTjqeV4AgDx0UAKwQ3i89XqAe4iqevJbHTvmwGTE131F0V6zKsh1rf++DL3HMT2tMpV6WNqe2uTfnZzdJPn2F34OJbmS1Hf3IIuj0rSCcjOCIx9Ia+hV2tycz2T92KJi7C7f1VS/661xORjb3tEPLdqhSvCtiC5vg57B9h7F8FPpnh72qO56pF9aDOpW4oYRwkx4dBTO2RdtkbXdaSvtSXce+NF0VISBLwrA7/1cqnvOdifjUw6Yi8eRrrxF/uREM6hJE+40ZWaX+FvKKT3q+sW8MCYQcmJyAXJkdz6EjzGGg8pmsJy4OE1Yko/juH79oO9R6wekMff1pOdfHCU+rnAG9THiwwZN8KkGd8dAN7ElZ/PGMDt6GyZCwEd2MNAsMMUjKfcunOqNFnY7n7Gd8zDMmJnQ8t70l6/EVi6xzdWuVjOyr8Yb4YTZ5HusVVq6eCZb/Fgwc1qGtW/+iGHsTkYTg7Xb6jTpFrXbr7ZPyTmweSas1tSNup9y2xOVgh6SGvRBaU5g2vSe6QShpchNPD0jrsqyPdvJxeM4W8049ulSpg8ny3a4rIDEUHWKJmU3KosHSzn+uvOqgVZm7E0+Xy+y4zTns1AabEp6sHbH/40iRdYFxeIThHnk5nZXT3dYW7L8i/7PjUNb0cPxuSbUlan1HJB9V1YhKrcztGLxhz713VGQWO89TXhyEJ10G9cgn9RGTZYUk0hUrNILujPajqMwKlREEWo8G39Rxm/y9Vk/EOBRPULKEj466FJzQTfcQy+aZaGqXUtJL5FMK+ZSIe6XzXZa+NHbWKz+NgT+rKG4OEbDgBchKR2lE+5REFbV5Yst3x2ZenkSOex2UViNP4gxeHrVWjhxCxm1StmzV/srov6zhG5g24Ri7tHTuIFQzIeCoNVCgYS+hymPC29dnntHtO/7RC6lpdx4ND4LM0+5eVUkBtt7yUfT3cPXYWyFiKFD/dp2Q/iny3HqbOhBPZu3AGOWTzx0OR4wGmXdg8V+jLAoJ7qAip8f42a4+QJiMadGdcHxIlyzK3pWUwB3qq9uCzqelfagOkdkEGaH1fiqaRe98jdOLTrDFjITWA0nkh1ji00D9q7vu+5JH3vTLLiTV2QZbdmEtp/91IBqsE+pBTSRbrZ4Y/YU79Nd7nVEtl2wq3rawApGSePzkOeUC8PXlZ4P2aU3g99/CAp6vbMUWSizbUkSraUsZBty3qnLQSiT4ah+8mgI1DjWgTHJzwkALu5IJUN/9SUlZ3CJUupM9an5bNJosX7NHINB1PNxcbvz8l31nFJ6Y3VbJMDPK7t6KQ3wxpUo60Im4WNEEBN9f5SttZ0iEp+7aSzfs23hlH90fuvOM6VSSfJE7q/ru7G/Z5Prwor4R2VS/1VozoUDNaFEkPQ7lfWL+4nMfuD+jU87kRG/TpJfqs7jHQg3Qi4n67n09NliggOyoKxg8AO+jCTW9k7Vo2drild2D46LyaVVOctYKgroHKXv6621pm2EsIyObAQzCBnSetxCSDKxrLfsjWR+ZoaewxVLpOqDxUdVMe98NzoLbsBlwPR6r8BARpQ9bN6DOLjI7kOTF0UvPTyHfnzCakiywB0l2gTx9nw4eYjau4aUrPxV9VLxn4jUtOvCL/F4FvrswtA4Q1RnIn6+InyhMZpkfCecVCeC+RwMFnkaIOP2MY01ZyT/mkVEFuj+v3L61OB6IzmmjXKDd4BSTdhYGQSvS0fqqs7Rr4JiDEsNmncDh4KBaeCjHOyPOYH216zl3sKQhQw2Q6SyIBFHA8Nfn/Yyn/0RuSxYVpiY/YXR7x1y3nD0q5/TYf1Q/KVn6ACmmUep/r+kkUdupyadSmWVQSk6hmjYu+46nJaT3ZBXOR9Vi/Uhg6r6nfjxeIvsMhe7pgXXY2zRUAksTsMJ+hixIpD5JSF3nevE6xRBgMN2dG6eqLxpl/eU+F831cWyPE+Ebudq/2LlM5L1SMOIaWze3fWyRxNwoCXDWCJEYfvdmk7Ma2CLb2oZtCazd7rK39L4zXVX2K0xjl92zNNipAu/9B3TQh2UHjoQvJ3WGrXdg/f1wOKpamfAobhH3ZOTFRMIzx0fIXU8LBLT0tzgDOykGh7JtGjb6jKqPYhc6F6ij32ouZ3ludmCXJL1koC8yP3v3iqCCJ79HZmn59wxdTcxc3HJdAPPMQckp7RpTNUJCPcgybaRZtHBEhBW2wD4Q3pF7L2qRLQ7mqbXvBvXfos7lCV+9UeE7OGHOzahsx3VXyjmyuLKiJbIKJOKO+zKAXHQ+L8SzCkZBcxJdPx3meSLihMrUJ3htDZt1vCRtv3YaVMUuoe4Dc/9Dgk8wj8BzHZS5DkxWHErDidf1auOU7SO3JNK5IlQqI48suYqOlLYeAsMt/Suaii+SGwRl0DyahIFzbEt/LNP8sBKppKVLH5R+sYQfaEs/lFPCt6O4QwimIkRrT323bYg4IhK15efvZqJmbag1YBKTUkRt1uHM3pwxJQdc9vI2jaraZdXCXC/Tf1tmBHisRbJozYzzd5bOmLXvPt/tHfw1lQJCOe10VES/F9TRmrnqi5lIC5G3urS2Xx6MRu0QFGX7EMMiZl+M4/dzVoJPADFgHaUsXpRIugh2kpisOHvvTq5Px21uViyXeQ4E6b/Cb7fia0wgtA+VQ4OxZr695ptCG+IMloCTNVt1d3Sa3Vs5LDIxS76KW8FmSFhuv6qSfmFaOfArY+6rZrJ78rCu0DQNufTnIOy17Zl4zaV2bZ4L1/7N5oBawCEE/GDRqlAU6s83vhKH3Y/OU0VzHyK5ZhbAnZNJ/5tcS+3PK3AsTmwbzxljvp5fINY7h2qH+EMK6hUHnLzrIjxnk5iRJL72NTpql5KRIbuYbFbWx9AdjQk8u2TSEjfd5tdEjFjmW4+DCVVuEgmdi49Tr37qEe1HLx/Q0cAMZ8a7RWNQdB5L1PXjDbfZLwWAtk8RSAniiHsDTUyQsFTd9bUpTOw+NduU2jN7pmgUka450xjLti6Y7DLDZz53RMYz2qLzfuvmue8dZIViUC4b+l8pAhPrylKYt0r7CwssizJNtxWUOMVo6kNFNVpnSnp88XwyerNQBPFY8xUVizThiiOneVtJMPoNtWVG7kfd8Rp7VEh//ukA7KRVo6CWDvV7tAuKwqdEUW4cdkY6W02xTVpoZ2EzLI9vu3Vt3MwrDCA4WphAlBUBLh642jK1kJjd9Af071KO1XZEi0i9gOlSMNr43/YQ1jb6QEEQ2Wa7nfhnfgd+jlDMKUeL858HWkn+1QxLum2xUe/TDisVuR+gx/za255kwJwaVT41tr6sdHyfiNUiDMUcPI0Gdug1U+amAm/37QwX6ICRKBiLXKS1MzAKJFSSiFT1IGlefyNj6TA8Ei50CsIja4DychiM9u0I5h91D3zVsut4LesQ8co/FS1h3t/qbYsVit0XulHOttrXtnA8ry2OXc3X8Ncu6Y2o230xD38igZkQGdfriH5IU5iJ1R0jVAFjgrWP91p7Z5y0iSHvwcpHXHu58kR2EISaqfBRuQV92e3yUBgXaxrP5NuDt8dVrxU6cju+R+EzQZFTZi7LiLHPlxqIsiWCdJ7GUbOLdVQf0FPXEPy1usiMsdW/yNaut7X0TbBzAkUeHdGWWUiHSFWb19qh0qQYUm257U7uCyw3fejd0bitbev6ZSwUCxnSu1uLbk1Zho9ReYd0JdbnVnWWm5uk1VGbBcKxHUUvgSUXUO5NV8VqvwlcXWFMgdpisIBTckl3opjM5pIObGnklmm82I5u1fHcvnF7iadyF8D5fbTsnI8aKFDsM6ajM6NY9iT59VrN/wrUYCAWbRQ1k4+3o9zQdOUFW4a4UVSO6R2l0/2MLzF52C8XcswSRJHynAekv8TaecWLZ8rkFORFzPDgmvdzL81oWgYjBIDzkYmqu1eD3rhytRHD/k1CTZ9/Yy3yO8dDxDOMz5jpsYHcSTRzw8PDJ0bLTjEXYgH+pvLoB8C+cSotac1SACUp50oaNxvKk1gc+Q7AcVen+q4r4v1sCIjd7jdV7YgPG/hc1Rx/py1Wbq8Qt5FIYmDUwEd7IYGIq+U+k7yyC6NKWhR4HQevJlZ1AsJOaLYqs3D/fTzrtqAYzgwxPAcaXOJ/fNgmZ+CzZLBLO9gCbMGagN/UFmo4yRggYU+NfY6l052RTtYwzq9zyAARorbyuZeTy+cPsLwFNhyCVrut+e3+hHAaaAiqCYRSz7hwJ671aPz7U0UZKuf4oBs2eofDvsNfJQ9I0OztNmhzSnSt+a8y2a8rcRRnqNJVUbD8MA1JJXtxZypptPhYFlZaR33iVEchM3C2NaVQE59zSyJS0ANheW2x+8lv70JwEV7l+4pmrTDWHa671lK7jAtQmm/RyUYp9bNfY+/IIbQWLm+uqb42X5cAR/zDuC+1ZrhhkMBmSbX02mvn9o/8MGkMyaN6cu4ig0PJUR+k9XbjSvuMpbW1JBpY7Qz7dMkju0cp90rf2g2atFVnE9Guy9YbGDcazVXhCfhyFKQxBengfbl7XHVJ98TofCjDQMO5MEwvhAbGmEBL7p4ffxtOQ1UpSlQr8uSKF5J9rczHC+nrTCuW6LjP+IDMEliBqGrMeUFqrNxMVPdK9Rb5HALfQ2xcR/h5b8uqfVfwlX0KPH3ZQ9hKEN9p2w3g/ix78v8PIbJ7fnRqkhWwxyL9BbXlbWBzZS82pQf8MxAuT/yn1gvQeCM641rZCUeZA7WzZvSf/AReMUhBo0QD6fPyPuu/560MxOaGv/NhdX2GgkrkXNbaMF3qnnbw6Esx+s5ZVUpQ1hRbLa814ZpyKWxfhTa34fYATgU7bV9mh8tmg08WbTGcFlNX3f6U/QDspbjxLRMdxOYz1guLhFDGQyy3/LYgDylj3aam+I0PejtI7WaSomeeB2HyWGF1px6O5VBN67/aBHNvZJ32CvDaPajwHr8bKjUobdBYE8Mpt/Ayv4KsPAohZdA9v5ODipGWhIKWpIxqy+M07Xi6gKKIjkBdDLwf9GSkDV/g5nK8tgFE4edECepGmAkkPQGSj6YuA063PjbxHf7oIVGd0RNKoD0DeCWkJuf3nqXk5eBRl1/QLswrFaIRXhasBtyB56Co8kjwFnBtXIYqdYFUguDVeAxQKFHOeNTr2KzEpPZT5y3GjNnQ+Uo9Ccy7+ABYFztjPELsvW5rBMx21SqOlM7yvCGfcm+zMGFoNwjmby4v6K8SoHJEPAd1Z5G/BmB6LDhgSNwV1+8St7xfLzRDr89f3c7N+X/6rWRUm2TJmEce0131NalbjKrwiqLEjT3lHOKJxh5JAleO9l+c3na1mnb45zBgZ/qxDy1i9Gyq/s4Sz9pEgkn8lXBO0KYYv5UdH9pNEFFbiBKRtd9Kpn8cFhCUg58zrA1aTsJWX+/t+MabePZVmqR0OZ6aojcJ0VxVJzTKJ1UJLrBS4I+NCl+NTYUDUX9atqSCZIHEYF898PWale25SFmUy2a4wacg+OY/tVwG6EOQw06zf4/HnA1R0BQMfHcwQI6PexmIzAxOhP37tEYviJHb/CDgOqUbbi1/W1uuFjsK9MvHSFHOJD+DKxE1qOtbfKz/FAv2gTT+KlnHgPU2L8F+SR2CWsEGto5VnaHQAZoACHSqDj8Y5wz9oJ6xQV75NI7TveZIajmzveTWIFYUUmPeib/XxVpYEkZgQMYKurydsFQJ+5w6MMsOnr2YUNtyoCamzEuQOqDbFdhO9trNCqLZwUN93Il+f2yFyspTZ94lcx+xsM94b8o7te02vr4qgv4Z/9ZZkq1MbtQZJOp7zVtYn1biKaLUMwmNZjWnsRA49l1IAFhKE0xuexSmiFAgXP4UwJh1S1mYuXY+kQBO8xrglT1ksv3M14Ns+5JAm1WTE0mzaNWfJR9UXsvFBYxSB0kC3jvJkCWA628eGMr4VT4rByrgQ6/U7FBmoc2B2W/YFEWSG7jddPeDTP7VoiD6/zYRBpxn69QfqkQEmZguw5V49Y2wfxApBXzL6XZFLm9Jo3KJLls+R+ixm2+NPl2m+1GDXMxLvc+TcjP7gJ9d/2W/XC9TdJBQealzGJJqIEAUO0SMvxb6SPR/sH6VLv8E0woWPbBMm7eX65YWIczhf7yrYJtdRvaxb0CarAer9efIESujmUV0+S5I/ygxgwoQxwoKOA5o1viy/N/epG4KhG4nyzBOMGahFZfOF2RlczSHzkpNWeC8wbn5YrHiIEM8v5FUB9EJDhlcZos9wHzV3a4c1YkJehZNK16EJZ+93hgFjY+CkiJGALOMSfsLYrAHAwiejeQ3YcelThlgSKIpC0zT0BsTgUgCNRgFLjM1eoKaWqGKPtBBpg+PtCxG6WkUaNlcZ0CysYgmBljjTlRzUrt/Q92QhSIS9NzsmP9BsWcseNE01VR3KS8ymEgb4QMLdJXMErbHm2XPFHwbRguTnEsIWmPiWzAaNTcLmZNmJ9Pw8ev2Ymh5OtaCkAIwrHjPzvbSKuynbSUkERK3PNIkNDVFkMmWPR/DqdIIWHwZrVhnwJ+EccEas9FqTJ18QO5eY15Ivfn789mDnz2fZSBwlZkwDg9Xp80miJhuRObPr2hpWSP9SmUsOYolZlN0Qzp8cwq6XucxXhF+ZNkzPiE0Y1jtM+f3SkAATF4ONw0tj65cW7pEUkJHHTee3+dBHFJLuQJ0nm+M9O6g8VsNISkwMXTmRN6xZtIriF7u6XbPg8Zz0Ht0One4caamiaGGeyvUI4irigXUqtmyyCG5x6X9Acl+Zsx54kB/AFJZZkeRDSAe9DD1mSi31T7Tshq2j5pfWlQW2bM6MeYKN1JS9Fcsn3FZG85yvMxAzy/fB7MwjmL/lmW679BkWQSvG9rjAmXxD1YhXslZznFzq454kWJJksEjhcXLNIagUY+Ldm9mMrzEs6Su4d5+UWk+28g3hau88YrgHzyVK8HZTn+dgcsE9LVMhYGymYGyYOGjt8EkfaSZpab88gIfSsQnc8xt5hTlDJIwDOIsua4oCf81sFBM+dXnJWPCuq93fBGgveYDbxA7rkPQnBXG5NCr5YXFEIjAB6cOYiO3bXcFN+l9tKI9eXH1oJJfaJ2veBGNLxWxokTivskWPf+6y7eZfgm8tOBYyZTlCYn0cUeBJQjuJ3JsK5rz5qaTptquKIEasaehlkb6imVxdEBsEy/1eGOlWnlSXgGkVw1kIwRFu3L+Ig0zM5mnukWzOJeQ9RoxkfwpvDKjXb2dnwRmw3Q+PBsuhJBOnsX/boUjjDC8wvd39tp5qM4eIfjipNyW94iDWNYKaZTPU3VXfJQ4FB0s97idfD60rLB2IrkN84CS+AimjwmHS413JrL1pYPI+13zuZovFmie/HmK7c8GK8u1pOULwKsNqbAqJkgCYLItth9eiOcyF7wNt49iQ38vNCKymRj02GMXKIUExpzgfXt9JWFnWc+lDY6qEjh3UhTkYo3+ti0eOoPX6M9iNR8ahXgjMBqMdjI14yw83RQ2ijbXmg1uv67qNBEhy1K5jPG0tWhovlfiZjLSvEHb3PjLrvtvuA98omwG9Miq43NEKPPsG/7wANlYdYsjjBA2p3aSvmbR972w7Wi6Q9GlNSjQAywMLUxBBFNpdeVIX9OKnfRNQDRRUJoDzu+69+jV8Uq82h6bL0gxTB4gQe3PqIGTSOJ+d64dDDlUFQ4mJBXdJHb7p0Gh40B6amhDJWBjJz2AaQHfPvjo2vZfVf6yHluSNABMSbFvjJZDpJEuuh5E2M37x2YKRfiF8gB6Cy37pHmpLJoVVch9Vc0mzX3/exZO93am6/zXvhqe8XCu6QOtqgx8ufT0SgD3g11Cz/EsQofetXoksPSvP3OSTIMs65dldlEv8baM/aQU9TO2NpH+p4x3TZ1mEeUvnSR0QZXhtksGZxD2VLZRT/pQkRgfurq3YhDFT9Uyvtox+C0Ey2YJU4u4xfhXn8KKzuvuKBvMLLRiy00gvQifNIjwEY9WsodE9Q0wfWbjs5kNIvWjRH0K/dBzhnLNeOIfovTde1LKmSA38JGv+I9x66gTe8956vX+rM3ZiIiTEngHJSZkolhZz2y+Un+8zIxQU2s+j5fpJfX3bYrf2TVCkv6zRS2sGs4Uwk6Xb7C8Oz5Gnfc9MJmJiXPPvWriryo38QgHllISym0hWXOlOQGakO98KNFg5/J149NbnELHp6Y1lVu/CyqvblKPO1X8RmUj8NiAZADe2ZBppNB75qjYb34vOYlkil9dkwismAw1gDI80M+YH+NZ9OaruJg6ZqHTkwUyu8YWLND2aC5U9RCMvQpDFquLlRrt+/e3Aas2ScAQ9Jo4haFiHrlcOhCG2qO6otdVD5nxTEfA2bOI4lz6y/GwtH8hHsNWiU2WEJVI/j3Ho43jqh9VhIPRuQORzd9n7dFzD6B50tz9oz50PbSABb89UWhVZ6CYHDSv6s4NhlhJ9sq2ZPweJ+XbydZHuNceOUGFid1BC9ppXcX1vTeq0X4vpv3vKFOpRatV4InblkpmvOAK+vT/7CejOvm3V+u4KFibFfqYo3GxSj3bR4N2XMLwWcoAmzLRGh/1fOetmdW6gSgI4R9bkCN3TplWoQjHfx3Q687AeUi5MogT7HmEn2kSwMrNHwMHHSWfdTfmcb2Mm/rrgF7Te08aFRMkpyA5t3x1X1oa7NaBQDh7GHBrKweN3LDQL1gplkPHZ6fvlGGCq4XLL38vlXzkz4yykEXthpiez6uiynY3oU6kIw9Pi7q7R8xJWD81MKQYPvkVQ6iUip6x0IySXEDVep5hvTiqXrj7UDmfQKVu9RdeOe8+9OrzzWGnRSo9qPwXQLnWIJqxD7V8T2fFX/HHoMG2XBWred7gLpaTusagHOWDlCXTghLz7ASI4AO/Vu06jOhr+EK49Kc3HLlnHhv6T+6y3uIT8cdapgVOi7vtie9lyUvpxyPZ9oSAmGdFo9Ahduhe0ojhBHcz4nctxgeAgJARQ3OdqBhCAsimjNx7VSwhmaRzLdA6SJrM7tlgd0xnTQlBk9j8ZG0fW7F9wKNuzZ0I48Z97/3EP4+k3yWu8fyz9Wed08uFYg2DM0kXiunz8txfNc6KU1UyHNkt9JdRQswQ/W5T9BwkT6AzWLoYwBbqrDn86Rj3d9xOGFW7y35ZfhrXmRB7gBwc0DaOnU/a6Gph/I8qjIgMkPOEGxXglamXBOeGKpbnNOwhtXN73+TC/7G5YXtmE2KSucXM2pkBnLX3xnaunN8xiLPyWDMB5L3KfHIELY1K7Rg3er7FpD0NLkPZqtOQhLGSBbJ0nWKW1Fznkn/oKkwNCq1Krc4CtKR+lY4bZA9ARhX//x2Lj5Chps+Vm+bAti0fNf6+EmNYbgrL35iOZ1kkzyYTBEaPCbWKu8e920YzU50lLFO17ICOQngtQPyD96ZtmoXHAicLTNUJREjK86VFRRnmrjMzB8UUaVmDfqfaMx3vFerDUGioqf3hjnzaYhkmQV8V/n6QNhOseHWf1ztUmTXGWHohGwfdQI7yZHVl+m3b5HvpHtGh4fkYY8/wUkK+qMOtGG6VFEucBCWEuMB9ScrsM/4XFa/+XsK89sJWhrpslwxI9HfdQ6NzwGhfDQ3EmslLsyDeNYXO7RkKZxrH0yt4A+Z0ghrspjtVmTM1sX0Mfr+je8lJiv3XIQmr/r/PyDo8ML1gPexj3X9TB8+AsSNxhfkfyfJmx6KMGsz+78nISSefqXu6WMEA9lxg9ay/b9ec1ltlUJsSyls/dVXQ8ExypPxPEBl8LfZ6IQleVpSwd5uMJZrPX6eiBHEOm/crrvS0VZnjTNxbjrCjMZOdCXrLTW85ukCIGMRoaZrgvktb3mEOqU1amzbqwcFRRej350JZVEErjidb02GDL1E/qD5Ix/VV4ivTwIl9jPbAvdpPmLFrclefUy3BKFM3oxExZlJ91+Aruv609VGIEY7+7r0JyH569rFCoCg1Rt1awNS8lJesZhqH/yO/P5DzRRv5tg93h79g/MfzCTzoB7SRrAoynKtm6THZ7PDg0egjhlfa8Bh3rQo0lnj5Lw8Cfoo/lBy2abe2DSBeWznnO8AXlVEq7RghLkrxGrbp9uhx/SJ18HGUqWrucm0wuTPU1FHZyqEn0NLoi/DkidvFwEJcQNq4ocjT6WtO13OW029IwFj6GYmMg6wEqt1vb3Of40bDZB7BYnMc2Tvwq168JdcprzFSNgIJ/EPz9UiM1laBy4MvqMLayJ8MmX+kg/vW4Wnvzvho7At67eyXZ/V5tYGIGv62s+OMspIkt5QIpQfGWRRes5ZDhsvkFe6LMjXI032/nazLlnXCEr6KQqPVrGD03cE+Ojj7JNk+efoFLhJNxD6I6DMbNaULpySbdm9EA9sc08YMcvRVvEiVuWiOkgf3/c9ssrvPylN4VxVPEjnckNIc5jkgwqDn6tMJjelmIY79A1nfpx5fpawipQYtjb+S6+Qn6a8dICLA3nWP2DdtOtSQ2ZDWzduTJHuKjxpPCuCe1fWleRp3araHVLWWYKGweIty0nZVmHX4NGlF8WxX+zOgz3tZCDx9QCZdntpHo8T/D3X7tRlFlNGt7Czxi7K1Nn5leRpbiilRK2I/Sz2waQ3oMKax8F06oHA2E9aZ7XzFhHMSvQOxmp3HxtyA2EnNMBOJFl/CgdzttokFN5KeWnWuXPz2MbkKNdXKH9F2zTDKGegxhpgr+uNphWe7x5hiuZ2Mjta4mRyQUwFgXsJs2n2ig7AyYlUdxAUVcs7/kAIPD9c1xRVTx9eGwVib/+AtE+YyNRxFBOrl2POmKhXkw6HeUsY8RULLQhfIcl74jKRzG8LNR6Tx9WY7c13RLSThtyU2YlVi3LDtsfpv8gCy9lp0sapa7XtO7JvDDdc4vFpCLPi7U7jvQQ54sUG76Yye3b+l7YH+paVMN2amJucVWPe8XrYXgU54tglPTn9c/g8KnWvOD8aOUwcBzUVViNIu1P/PUrfq5/M18z6a/aayoPVuMsv9gv5ZuP4X9ja9H/KhrF4ssUgkIfpi+DUcwMbmoytWpLbryqf3mZD//XEwmQWSpA1SeUso0gZoiOotHlddHc++TM/7oMTJot/jUiiV9mknuk9Tu0NEeWZlogPgQhKx3cqQek9q8009IhZzXVYuoR6r0NwJ7YUixIlvVuErHMza7TUIvQAg3WO4f8YSCmKJXON3M2mQUnWc4PCqfmz0QuGPRXcst5XRj0Vw+ntg4yow9RfyTaEEWf8fmZlq2K0JkQaMD7uVfbcEv9i/lGuGnG6AWWQesWmUf4VMg54MoP5MeDM2g+162D8f5BBzldOuz41levHMEPeLhzKugvU3mUYV8gVZimLy5G6guG/8jilFGF1wso5efK8FHKloXJFDGk6iQzRkm26tNX2t4rTZp1aF3zbKPFCxSUea0pbAf9Xb0KRdmI1kc9M0uAMB3PytlrTX2iCXKARSB+d2o9A80kPQD/Fb0sqGeJU1ZsLW1MK1sCeAtUsCRR4Q1vTGHy7jGJuDbyyg84gdj7s4qNGwDeQoxgiawbTTP6BW+khQBDNjZ4tcd2+Dgz/Kx46y3Ell2EezjeV3wsExrmohKpooMG5ysEUUncC4F+s5/eMzL2fDgoOl6LGY9BgmFIYoKbDR/aD2mvNBuwbb6FHC49CLH51SLBWy7OLxRSDuejZ9eOFHXuklJCB1KoTrMTx5cvdYFpG+QpUwt88CfHffYyGtWhNwfTZaIlrtVgnfA+3Wz9p++/SvXK3xHDnmkyjSmOyJVyzbKa7FR9xH050SvokSo71jNKxxCRkqeKSiiyL9/NscvV9s4d5cfo4cgJOkEC/roT3G+ZBx/oGCh0nn1bCe3l5Pw7H/6yz1uDA50v11xhhZgjY2uSxy2qnrC4gPHODwW+q48Mm54kiyBmi0nOuhdUAPHO4/xd5sGM5LGY09i+zf3XE45e75y3zOSvMVTLpuYQgL2aT9KqnOXKPSqLb2fxd5+cfpTvkguLWoW9t37HS6ljGw0jFcePcki4CS+nBmMvTYtLeRqh7IuMEheZmUs1n6IrUj2xQIDoBaLrnM6R9fplPmqYzDOW3EZ8cmdtnb4GfesdumXf87Cl5V6UYbhOL5vELnaK43R8Z7us8i/EMMCnO97us5NwUagU+DN3TeTMyVB8nhNxDK5CspFpb3CLx5X8g9Kv2/KNhv5yCy3LO5R5CeBFTHtuTvg6s0zwte4OStbtWLpEuxXP60PjltPPk3L9XKqkr8/sGX+VbUPka0dhEKfNdYoNrVcxAc6XmS3Dv9DHf9QWYvVE+tTCUFWHGLRt1NwIUGTYxm3wXUNaCO41abkyqAmPq8PFD1/WpTqKBwGWlfmcv/Vc2Y3+EhT1Uve/9iRAuGulB/22xzU0CsxwOlWYjW5vtnKSQhk1U+Q7EAGxnwQ1IKqoLSRAbCywy2UyHlan/+6vJnLY30CzO3qFol7+Sx8+T3MVgXX4XyloZuASW7ikF1o0uT7VX1qDh1Kry1uDIQimtLu2FbvttXP/CbDBS3BuLmaQ2JiclSnJpt153RltdMD2CEVdvmhbas9comvRdRzzNoaP/hdjA14hWSKp+Vr2JPtyqS7PVBvqjWW13XrOZPKLOpper8im7bKlbGAATnESRqFVnjefvyojmzdabXlTHacl787/C+wW4S4iZO0vNz1pstdbxetF6XT88nRInPv+sYVlIerFMNzf9QIF2meqc9NSi+QK/Kc2DO3O/SIP0232kSWNoHM8EVBW2yuqFbwABExlqP8wndNB4iEmCDRzXqqy53rsdhKdfC+3OCxE/XQ5kC9gP9O3+nrZZb9tqMxlKQmgB4uMy1cLBwyNWLO10pwy9W63g5er/tUfJek9FtW+zCi5DiL7u3cnpXUrQWNRvtpC0puWZn8FPxEEfD8tq3QcQriy4O/a/l74C8YkyrnNJe0RiMUYP9/MBuKw47JPiVwvqOMC2dYLi6IlcTA7HtPEFTDf6Por9Q1CTajL/91Kc3ymSIZavACCSqyU8hi/bqt2GTRyE3+B31CVm9bTMjo4AJCkO4TApWwW5DG4pgVSVFXMyEbgIBjEPagX1tCvWa+cEjgdvpaOsdXZkEpUXqG7r/VIA+XEzj8iUsdQr7vy11a3YOrXcw/ucMXTF5+N5/oL9Xvn5jIfE0HZ/AgwzHf8IX8378L8/EFdRBWnqGaEEhJqNj0aX1uzdc7eCfBtHYhKSoxS4bngNqQtu2q7X6j1D7cw5McRLTVEbsp77eyHgBF9LBkn5NSENELiL1sdORAy7Xlju+AkAI1EvMtvseRr9qoOoqJLm/hJARXmjeFVeDPswxWn7b4w6bv+LfT2jKDMwLsmtGXXzv5X0liFRnYolmKNx+oxoGY0Wr03UtQX/qqGf6efuKnjd2pK5Ae3sTirzZ2X+nU926P0wK8ntdNFKFTZ92/Uct2MP3OSvnaHwUK1tuUYHjXaB1X3GYi9VgT7iuOBm2alPQEDmGHoehXQar6fUcQOR8XwOtKikvwCi1SjNveDlJt2WF8tiAlUiXzR3xjnRj3GR4smDWYJcNKAKCPgiwyaMMKjkZ1etSBY9noQjfkp5CCqyzJmlX9YXF/Nq3Rr/K+hzLxmR+HFxz80JjuqW4XPe7J3JrW7X/qIw9ycy3AByzvyjhKklv86P4DNJgDOz/1ep58jHbZe23pK1yUwOQrzWjSwPa0z8hQc1Jx1JE9fswpaHf/SQ2JWMvJGBMbDI/c0m6czKmSebVF8wuzSI6OCmejnU0bXjMBWy0nwEMfsv/ZenEcBQ0ABDPi5hPDzrFzwBXLVWiMnLtPMMVS5jVkW0mrorqclFNa5ySEEzyWXtf1ffAQfXwUCsXwZ3rtM0BOdSarTChAeNxa4fh/2Ypk79taYyOf0jlKPb2jpQ+vBagHDrqDJMdjQ6ZS42UJf/qw9AT2UHUlrjv/Lj89/e6OM1PJCpzNiWVmcoa/IjHPlcw+1fMvwp5EL+5itp0uTb+nnNgV17JV/N9zK/TWqGGmt8z0aCqN1pfqT7llcjo+k/0QvaNE8UVX7uPoVHxqmZXGnFmoQzG4mU9K6J65Ds7FsfPkFd3y1W8/9vwEv28tUoaSa/0KOVa6hjQfYEbtsmA+76muzoOGSgKWBRTr/FFC9n+moTxsFNWCzEy4cBGuUIIAuZXvO5usaxEYoIWoV2NGpRGhFBLA69Ff7YBlV6hFQws3KbUunw7n707UU43vcffLX8Ner2/qRveFTFJpFqf+za1pwzwoUwYZ55USt6vduo6xKpPFW0hGThGj+BhfdhF+hi7g1hpzOdYK7wZySwvBmVrdlXAsjaoR67eoQLVLIRifrWjvtQFePxBIHrTQcH55QZF+EjPUWwjdfsGWhRhkANWn43oMObrkIknMiRco4zoi4oBpR6gMQnlIovNjXurnMOZNxAWP0aMaBI8uoVSM1YaySbBv3TuPqWRpqSv591jvUOSpbWlbZCgTNlltUFNs0H7mm1XKyTQb1/ko2MJeVFTb0VMlef0zz9SNYXtM7pNj+cTB+4Myqa782jYOdKF2NieWI+HldJaDyvwWeP61dsITcDCUIcQqQKAkUxPQtVvPLX20NPorEzie/EsA7u/Kb6W4KQ4o4Alwtj+c86r38Cz05cTLpQmUoNthaOAmn9n81wcfPgUCGrgZGOUga0TJXoP9O1R09w5XGLCC+T7CR72r9bPKmN1YSNWQgJbN2k0qITghyGse3Q/qlU/W6rCwvRWQTF2n/Y3eV49dMlfVmdUFSC4G7P/kxVVtsQuyOmIFSYYChrLBt9MmtJXf9ghhR/12UhJd+VUkCrSsKCjuENTY4UqxQaiJotVTfDAFSnY4RYEmkAfwnbrKFvWPA6BD6tJyWEUeZq/JFiUtCmspuHwoppe1Dnic0A8PHnB8yMLvD0y13Kn7Al3TwTqlipKe/0wTX/S6G0QqOlenAN1b7gkXJpYCgI7fsxUAEwx/f2WfFrad7f2a8XSGWIjYPNefIoB8ZIrphRg2072zc9+DNdNuLBcrq88e+wR4tjA77i3JulE0UUqWoHCgJJFhWgDKgPiEQ0Og2wmqXjg9oU/Yn/YY12AqWXQwOCISCCTqQ9zDDVO+juSV9If7qWRYsc61K1MEoBiR524jOqZin1kT4d6yAtwKnSWtcVyrXKkefbamF9iu4W+Lx5jtaXhFUR9Wamq6KK1sVP51eNo384s7cydShO0iDD44Y5bWPD8VdCzv9iolR1elHmtW5ZDPYHhDackxBILgT88gyLYoBUp9q/OHIBy2EGk8aArTAfN/qayX8/ZKBx5DX2VkHMTz+n3LxV89drgkTysUN+wAxRizOBiFQ68qG5ye2Bfg3FwubiEOPHjayzrM+Nj8CQGIfVmF/ceg/hZaCNuwwrDE0WVqQeFv9CmTpK5Tj+8ClCJhuEKG7bmMd8YdOfluxhW+ZLplJ68sD+tklIIKrSbJfN+EVnYYzL5qxGnvX2pti/jKruJ6+Oop85JNvQnEPC+lpbuuDy6LsRRlplV1z3R7w8Mt6nER1qam5pxFdkpad/TPsXGpS9Q0CxMy1OhAwuvSwO039CH/VgTaA2GjjBuXWwXmRP6KcrJis9jiULuknM0U4bqOFKUIgC4KBS0ruPJMGUtHW5p2yciUFgW+PIH6QcyPSy6rfDy8XEHlnjnjUpz4cGJnU3h/03v1vy6nfnRNJijN+8RSPGqX1bjpH/g77Mj1kKN4f4IspnNRE65IQlM2p7sIrL5LGufbHgPYyKqvPw/ydTAb7YAKKtjrC6pQg964Xzf7Xk0fIFxhmvrDDXvTHWsgTcrEPxpNetl+x9K6Xvqp/cBNpZJ1k7uc3iyMO5M4Z6z2qrUS6aL0Z+asXrked0RCvkZUTb2U4JmrPVLJze6MeN4F461taY5mn0dDAVmsz2805cdySrCkMLPSX60UL29dNHbm0xC8ivsigveSryPaDpp/1Mcs7nW3zCsvRjZ/ajEQBNEjlsrz4QdzDhE5ZsdRrS5HXyEsNqTL9RbFP6CmebhajfxAVmTxe5GMgAKJKyDdN2CqP/vrCnh2VQiqsi5HIjWLZ2bT7Vw60kXghx4UYjBnI8dzDodqWWb0zeKTz7YCuxOyBOdC6KJ8iuws1ayEfjZ0j85ubCOxlNWSGJHCPQr9Oc6MUowj+Nkz+srQgDAGSoRjOHyj9r0FaVROGQVjRjwFMB4gTRa03NG0wtBUWRSeVVb5Frktz9pUptF4wyRM1KABYOdjGCw7jkji1jlEJti98cQt4H0tsUq3r8JdO2WyC2FPmxwLYsDNd03fsVNDhkvpTWGvcN0TBYQvHKvLiZI+STQiq++2Icx2FGPfce2rG/6WT6LoEGnkyZ2GN8HW9c84yyWvd6ZNOpFW9auf+Vb1oe1vKjOADK4Naz8jJeozDKJjkyrpvq4AhYmUUGzK1aqrnQSQ4XAO3j0o4/aglFKmmMUBfSiKK6R08JEuXdQagGYjx114VXDm0sT4mc/eYzrWMDEq1kHnD/rp+d9mkXXkIcnoFe0jaoZ+ZGT/27+7ph6QkRYJXKW7Wf1VnAUgpB3Z/BpsRmRpnsq51+isz8r8bLfTL1rdVepwliRO9c75SRb4O7qd/QstdOTPQ+zDH09uvNC92cvb+8KVjf0uw3lOS5hp6akYvOMLkgo2TIPjJAjV1RdA1rNBfrL7DXOKYpZ7kfdoakTOOsj6izB7jUQqNceeTDXL6Ywngix5Mw5nARPRTPkumBpuHLFWdzDZYaAOtjMWk1bWu7czPsn5bikBF6DbgjF6bkkg03DCC8xt9x9Ke5Ft1Pkn/0+/cdIQIIqOKUkveNM9fzyIMiaQ79DJRlhwPmXr332M0CXGfUxgXrryvo6redLYhUioNwfXwtaN/a0kKQ05BYMZgmX+W8bF5THdy4bvBSvaa+B+9/wqOfrLKZWiM6DMhMHvtmkGTrTH4PfFRWwnDY8WjjbzecBAnT6M8iM2jI4tuJcCwkVzq04EhbAcQC8tpY0O+Vb8OkVxN78fD5TKf1/S4kMPfaSsvVwgVIB0I3rdoLP+yO5n0NLSFCMLCiwskHiLM/wZZ/VqmKG00C+2bi5wcsNF3/vNTxctVP/xojKgIgi8jX4/mvrqStajClzOEOekvhMGvTdpL04MDgX6ApLJyTkmL9uWUHoV20WRhWRTqt86c5rzn5X3HMg9EO8+4wUX6fyS/pFYsTUe5pTF6g33vd03b5sVd1zl4UgjhrO05Ib6YwRM+Pfv83LptPvTcfCR08OEdNnz4pIaxnM9xEv5KITMNX+4HmxWsy2E0Ob9uhA5xkK2CBP4EWOEQHLD+/GU6Zdpf8p+fVVGO64soUoJ9wyxEFD245SSl3/Pz42SLJo2JP18vp5fxdEHHaTQbjolQeMHoL1T6tchlTrZdmrB7LklaVnPhGxWeYgTnYN8cTA+1anmGhwd1QQTslqFptgVG2cK/ljnVlwDZdubt/rVxmePcM/KodEqdeX2QD1SHKzxICwH4BITUmDI0yy02HNgPOQGFknhN9TAL6IE/MpJORAxEKaX+4TQrOmAy+PTYr/cbvVclzF8INP3rZICJOqyMH8aNpMR8KIZApW489UFFlN6rGsQRwzEYkvAvetdYaWaEgswyY5Ax9diTtvgJze+X8TsfcypEcyjgRJKA9qVQG+NQ2deWDjwJKeqpxXV18Vc594qCNW/I6+5yc/pFl5fRedGsG7lFeVMa5mxGWwJtRt5gb/CCD/0XhYJliYOxVoCg7SXHNTOGpGInzGtiaH7cHckefMEf0eK1ZwrvPELl0QgscIoE5Ae59B22vGxZXhlkPfcmzE8j/KBhoZvN93ZHodlJFI/rZEPVr1gYzvpBObqOnQbBPw6gRDZMgT3xTHag8IT+MfhW+VT5Lkz5iX7J5i7eHU6/p53aYGS0+Jy3TQPTt7nWdGklSc2BaYYoXLie29EBCp08JmYg8MqanNZKlVGiJt4FWz/vYQPnhvXrd9N9gzItwqRQ1L8fp79ICZYclPCElpI6I8oLggO6unDi8M/liCLxgid6EI7VX6S5+2yvg8gMLrxolvB1jjYUdqnKv3uu5e8TG+jNj3rrKOUWLNU6DGU2NHRMivehjpgnU1+cFr0MRxm+erdgR3gOO7GRUZFcr7INYw24WLygVpwe2Gm9rSHXKjp8PfxrxdMJJ5MK48q+L9atvQTmBNRD9XJPcpPWMkQrKHn7nbLy5GL63IS1BguvIXPumYV0D8Vft21VuCkFWj7kk0a9VlpcaFw2IKMFXvCrHKDIGSIN3q+xbpEm6A4KFNR5rFxTAQXXBIzuaCJW0+z6WclD8qUPl+8JEpkxztvv16noZFsj6XMUPQhnlgFZAypBDMbARs27qKfO2cAlkcUl4n/dtbZnBQJj5Yi0HMZi5fXJ9ws0sY4zow/v9O/T3zNa5xRQq05S2LYMKHo8QaJWdZOgLkBkF/h7+kZ26v5eQV/cY9WaMtfqvmD7NXuWaPPhOFfFoHyOtABMLIR5+r9v339AdTfzzkO/OYK33vhXEY3D5Mkf2A/x+2aBNpa0/vd0BYkLTI/mGQBzaKHBBFKuLIVWRYmBS1U+KUcNLz2qEtLg62OxWYBVahyrvbdnwiae5CzQtpRp1K+ZAfw28dMtBnXuBTQYr1itzQf5zMJUg3x7wA2HRrfOdXJp8lIGI5HD5+9zePfmM9qfkwpH6uudxCQR8DkFfgE55vnaUch1vMD+99SOO5CPy07ihKwKHzYVT9Mihi63UflRaN//RmhfR7PMy7xrTUFoXhifmE0bZtEK6hxTffaC+H/zXKoMN7NzjLAlHJCrGF6l9yLQFzMZh1/9sKH2X38MfpS1I+2vDiyFPyHPCyIKV947o7H8F4D65PC6HDp4pq4oDFIQ7bjO2K/In/PF64zK4HYBDJAADCNUu+2/58pXScCkPdINy1GV3LA0G6rs/BdVwfoZnzRWtMEih7YdF4Qe1TP4z6DZiM+wrQYfhnTKYhgxwpxHDc7J/41MJhGcpGhoS9o5IqMueU0Yp1NAIIiVCZwL62EUwoKrn/C6Q7EqRZFpOTfzbMaXbIqQeOK6RcbJodDbCvqTf2zLwF7/8VjF2BtOU4mNMJtgn+f7Jg6x7Y1qrXEVeD2lIc5COZBt6PZfcc0w9oSmzT6i+g2fz4zJUvV88uRBE1/3rrJe6zGvXtZSRLI83xF5oyRZ07IHcOq55wIdVNqGHBGS52XMF/4Pt5MKRuXaZ2DcNexl+UL70JOIghkxaCeohtDddJ0t+QxHErVtrpo6nnZ9JgyGDF04p6J7csPrU7VH+NcWmZE4mkszgq3pMpas0VZHvwnS9qql6o5NJ1anHBKkZMuYyRLIBlO//eBn/Gkzxl2IpW1KGTYBzd5o/sp5SoRun9X5iWE7C7rfvQ1/zevNxg4bWoXlD+hcLnz0ZQFCQln8/82y2dFLTFQ1OTSr9wJBvzjI3GpAMI9NrjpsM8p4jAfYAEMhuDK6IxlQ9uIHgmJ87/uda75+uFTFGeafPmbBtY0ZUbph3aRRZI2dy+yj17oMK4YiZcK/LsGr/noQ/jgxGRL4y8QP6dooum473juJNe5sw0OhAhiWBOn8AYgW0nf2azoof3z4a+y8n2eiL4SBYel5HpQeUTPVr4hFfLc6ZG02nXmTOBJmP77LAywCjCI1OVV4kyPOvnwA1Sn453Xa6iYMcxo6FWAUn/AxlL9sjF7nR0aH5gh7DB/xep+2aUg/+M/1vXJsajEGF1dwx2Z7TwrQodOgtdIje52lWLIOevFBldMP9FBpk5DCXgOTHHjfAwOYOool25CpUKDqK/QZkQB2jTKhXEDdIcc+/AWwltCJayy1m3JpN9rzeLET+M5HJGTsYgmjZO/FP16h0ZTsfQGOZjFScGlEU35wBqzNE5XdaH9Kxjojo+DzeynkRsPe/UUw4HFe6/tNNaUx/rrQuJcYcgBz2rRvX4iIZqQWctOfaUx4JXiA52GXX8w1lnU6IjC1YFNlHVcD6wyOp5gsIEgr6cxDikdsUF2ZKTrSFDYz63d0dNN3s5Of8SIp1LOVLU4aKLLbDzeeyE758EdQUHs4JT4TrTFoQlNHf1wNoEZZrSrT48DDk5sw18C8dQ5FrBeccHZI5yASg+Hq9EVMllZCC1hmHebkIMvO329yYTP1d+5Gd63U41iWu4HQIPYQgYXX2o54t+yA6J1Yvm5BmoU+OGKpnkfqirooBroARJZ4Ss/1GTRJJibJBsNr5V2eDFTmfLXOLIOIpzODVvub6YGJdaZu57okOD5RwPdYXl2h+BnmlCdlz56QsQRSyxiztV6/p1vbsULSb/sdTBDjN/SjeVRznM+XvdzOhxRylUKRG6XKya7PLWn50hTArc5V+NlUNMn1A/u9ROqgbhLes+Fvm1kMy3MHqt5PQY9HIa8Kw/FnxugvMka5LIUt60vwoNo1U4ze+whyAPNNtQQ4fY08NwvQa0kco6j4NymGuS8GDN17tnQu3kIzTmf9RfsFI2YeM+F1yfEA5K932TwYP9DPKQzluavy2DQY7/DM5Zwinw4PKjgRoGO1IWtIhED9hhP+5FNd7H5yksRIokWEX1l1PorPLoIBIMK8A2C3SSQoYkHAAJ8D/P5fMoatNMoQiZ8JBEZcsbfpaAryULesiQ320EOL18XaEELTNfyJJpJy4wrbLV5p5Z9yuexPeiETraPdi/zpJdP8DzJHLnZYOubKFyuqqig9NQ9K+3zt9pz0d6yc9tdqk+PpctcP1z5vWmg7WNM+aS19018NqUtvt2bP6q8LzPAH8iDpi36S/GUfMqlGrvgukK2ffzeOt5QXGSSbQe0Wwu8ZeFyH1eVnsxGQSivoNAqx+8PVKO3f4feu6pRfREkL+fvp6M8VdB1E9aXTin2G8guanpM96w9WAue2o0Ws5L+QBVWFc5jUtCy5ymmYhauizX6b23/0GMuUAiL90N3+OhaGuPGnoE2RIJPQZdQuhvK60bef0ik6l8sPtkiTsgF0XGhb0Srr2edFmU31Ycs+l7Z3QAa3dCKYUiXSW1fAuYN0qPo+rtv9wd78wyxXvxBpJ9mHDJsY4DEbiTEp8disZr9sK3AXnv46UyXJTl3JYbJjpxyhxEfKvlLV8oqEZXbV/MybXAxK0kCNKpK+7OxPIywapeol/oUTdD0JTPQ05Uol07OQHzW60ECoSh7BMOaFfb6LuK3HXIZyMLMWIpbq8OccUTWljltvcyUgNrU9i2kqd6JhK+MD2X0C3fSoui7vRJPJZ1iYo9VnURXv6idMZ1vu2MWO+KBZb+z3r/AvueKJCs+R5Rhp9+8eKjhjks/djYZUo8lYdwpHXZe0O+fD4vXXlHzZyeH6XfnLnCo09Msl6KagPMP4mzzH33WA1UQzqfK/bvpy0IQoJiBgCoTpOAHqS/4XniLdwttHdgE40LlaV5AFJw1kVJlokTTjq/wUXvXdzuG5IPWNjnHxQ+Mzrs5SoBgIRvWlK6jRvTMxSZqodlkueyFesQHOhKTU9AkgBX15SE7o8q/Myn3Iy5XCFuJPDj5VyXzzB3UwWgsd86o176nwUCCXxhNQkyEXOb7piZgIMu3sB/cI2j6XyvvFFfA2X61VvNqmqGnsw5YthAL1lxlfAaYyEG61nU8zniDxX9h+nB7ZBMJj45UgA9qU+W02/iM3jpKUkjxPbYy38TmYP0fv7LSkMTmIEwdIDt/HSL941ZYvIMpyqnDFq9gvefvLOz43ZG2rldtpvLKwz6irewA1wHouy6oa7uc9cCtlcmtxOYycmCWKcuj5gS4ALpSCc9ZKIfcevZ65C3XTeFxoVop/St5+AgvH74PAuptzWbsufR7BsWd2mBxRC7T2aA7gcrmK4LozFn1uY1othAPdjrIlhLyq4SPMXF5K2Uy+5/Y2WoNiK6LKTb4So7O4caiUti7zAhK/PY1U3Up3F+Lga08pl46b/MmS74HW8pMPS/jjOtHOuZmvKRy890El8liiG0E/9wPvloSuw8R7+9R52yG7Zr9WAbUR/khCL8P6juCCcRNtT6FVHz3EIebgDLyeoJ+tKWIcrxnfC4CSbgtZaS8ix/GjmlxvvuAGc9GNQHuclApsIzr3oUfF+wBOEQKB0qrI9SZup2zIqKTliy7QXo6BL/1Vvf07Dft106Lyiw6GPSsyIpdu/6sa9Jn8yO3G4HtFxGGT07WakTGniDafkvLZ6N85auQBIghZy5Bx1FFeTR6lAIuD6Sp9Xhdan4PjuJPIbols9yly/Sf5W4mev8o2+te0hMh3UDNJTwoGJNvsXF0DLQ1J4AoJMoXzZJrxLReqX9jMB105mNLAuoNLlZLJm+NuS0fwvSO40M8p1c4MZL9KxoixKHA1m0VbyCLgv/bpEeyLWlkZcrtK+Dj67n0a7xpTws7iMncH/cmzkRdx/G2R18GcfWzQ4p+kwB1US7f8Q1qbBwKU/WaXNMzYnPhABGGH1v5iKmUMHm79ViNehTQDE/uwYcrrQY9PX0eE/2G3efDmMmFwXa6HbwecRzwxk/bd1uQnA4f2I+wTo0YS++sO+sDhmkryOK0rex2cjWrso9dyYUly3S4iM+90ml2ngeU2STQYKYneJ70aknNyPk7QPCnh1P0trQ0ovota7oi8LsxMEfD0cVJ9QSm3v0QO3f7y5YicsVoUvJ53lxkzMriRIvD6TxsRAfop8XOBc2sYP1cP40+PBvfBXPmvy9mUbNRVK0R2guBcVST+SlaC/35ndxo+2cFYdi79NcOKfI8Qa0wRUHomTe5naS59E+3dpvqN8F2ggWMDyIFjxBe8E+x1yflzQ4ejy/CXAwUuZfXg2ngYUngv/lrWjbn04vLNxhDB+JUTZ8cCD7JYOZV8afM0BOq1xiAdoS/Jv40l5AhQ3FGLbeeTYf1dcDdUPDQgrrORgyJM/Wi6ezTBXz2kgXgRgVDxbd2mTf1XMUEuaGIvn1NP3/94LSpANPdN5jgJjjA3YmbOPKJ2viRVjm1c6rHNki1fpg2bNJz1L4UeJw4pXUT54meD5nxGdZsLsuSTAGM6HOXyUtz761zZbXh3zxS2QeauG3NiNIgnIBJDTqrO3u163asgMwQ7AxwQkscg8PgaI+yvUz4qIJ+hiWr4LjXX0EODiIq8jgwuK4OLJoKB/2yXSdd1pbaSPGjk/szXC7Phr6FWCwiSe3eaS2HfDIsx9X81/FWgEWZ+WSsIdv+1WWXWeWiSMSpt1Rhia/Ed/fyiZmqxanIio+Ph3hNdBaI1qlo9vzohj19aH0mJhyQYYiAJYmYzdx0nUfc5R/XnvbMb9+Ntyy/NqnVQ/fP5bPmfAu96PsuuJH1wIdSwK948U4Q+v9RDDmyYvnmC4x1GugO5t/MQsWGcDfWdn0WTW/hTW+HA31U76AMJIHv0UV0NVR/X4O67oNt9rv/S5Bwe3DFiwA6QNHvNebKZFWhmU5CP+SxUdqSX0klPgsu7PRtNjtoOPIy66MaxpsJtAkPwX8PNIiuSZo7l5kddqeot/JI0anRozmKff71PzMzNJ6WMWi+GO2jz5zb4cN4wysNKjuaeTDdMYbTnm5MFFJuyCdzE47pRmzphek6cb+7PZPXBWTxaWqIJSBfLy1OLc7H85bGd/SX+d/jHTgb0ZCTyIdGmBcMo0ueGo9IXCYW44iL0lWHP1KF51OI08BHRrxsd3Wv9y1InQfxZ0RmVX9gzjYy7UREox5kqFZvxT+nN6ls3O0SLY2Th4+CkI1KCqSDwrH9ZCQwAtmKqA5BbkAenp2ETrTL+l5qiT7TAFNvH3HxWf3h3IqUGCBFbai8kzzvM9RdCZu62+ovi7px3afFEM6owsafPJu0AOnSUPKYYBXX+pRruuqPcTb+22UThc+Pv2jLfiRUgC+rQNlIhexYc1Dp0JIVRVd33TbI6afPd/7ptEiIeL+/6XoOmvMeloAE6hDolnI4BOX4fhsD4OKJZFTn/eowxu7DSmLel3Pu9fZ8afRXijWKfYqLNwdlg/vepFtITz1zU5cFD76j9XPbp1KaLdsgGbFNBSXAREKI/b6vzB+vB6QBOf/2YddY64Bs2qDT0knkfxo4xo7gZJ2ZGt+iGlxaWOepzngITw21UHA8njIHbloKfboCIkdRrmr4/Lxy2skMvz7nyT9E75iL6FbmTRJ1p0sze0nBSSFnya/uCP9jYNP8QQxq71gT/KXVk28GXRWHdVjE0/9Gydd2K9xuwj3h6/jkW2hHUHLh43TsGl3AV9hg1qHYDO88D6mPDXC4PqdmIGfX8dZKfqXdnyZIMYLkB772YzAFpbI2XO4CXYnW4admnQhP0N2e2TTi7LX5QCt79dUpXyYaWqp8XvSuIuIKKi4ilKgaaBzOcrWaEeMJdiJVLlxclneeh9Eorkjb8dWjARCs7/7oPKE8g0/q1aUtZCiZKrN+RkT82u677B6er3Jvj/oic3RhodhZkNMNU8L0/cMzFVsGa9aN0j0SaE0GXmajgsOHv9p3iiMYodEZjrIbmxOf0zJfJWTM83DR805zO3TixiOQ5HI42i/xNZ0Pz1zmUo/XDWn4n8wNzOb6ku2JIQCwg0n1GRPQx1awHlaj6CAgbjQxo0bic/uHZ4vsLnHFrI1dBU67bAENEfi1T9XTuByRisoqcDhQ/lvddDjK7kJHUCt0wL/KleKzjYce1G8l4G9jncWb+Z6B5/Vd4cQwYv6Lo3Gb47wiUODa+eHQRZJ3n3hl/aOt1aEmBvWDYhZ2Wg8DxmOlPnmOOpKhGMT9nXP4lo+ogbEdAH8uc1i92WhwoCoXHD58p/CC2ifDItCPEHBjZixSykOaK3aOjZhjAi10JxJ52CzborBOz0ugb7ePjfyEWTL3N5cX8CpM2QXj2PoavjPdX5yqXnlpyjwX5/dXkdRXmsea/0GHANmyaN5otJrX9rg5t03K4HxKIxAgWy6n412XIFwrShGOZcFA8SgeEAAc5FE2Cwu4SLzrgUfKvqaabAw/aKuGMicUerpRjKbDw7gHQHEToSruiITZ9aN1StA3sV3UXfOYDZSSnr/ZUssJeKHKYt552cHBXr3+tEjnFFl39ykku1fuXp1jZE+f0qthlTN/8ey4zsvWsdwKOdxNw1voiwaISQSlzrP7QUhMXBHO9W5sNu2jXzZvS7mnHwQDR4yU7LS1XXh+y70ea3y6Y8zyBXg7NP4beQpfQ41xjHhGe2eMmNF7/Z6LYQ1RYd68TD5VuhgHz5t+ezy9N6rG/QmEMe+EYaPJUYMXViISpK7BVJPGiXLlpPGbWLjw7LxTEaXmHzMu2QXuKtYcmHavYD38enjFYJ/kq9j2yZv5IJfkzFbgrl54+D1ptLckezd9QKdGq4ThtD1/rU8Vc8tkaUOTGlV87Ur9D4PuHZO4VwCz0fkqHHW5YBhCOad4ftq5GLwr/i6qzda2Ht5cCh83FgzGPlJwZsVYT9xKIMp75Pt8sKFZbmi+jfmSN+SGidlbBDlHdMQuYJyzyjerG76k3QUXm8e33uSdPopQ+pnBZa/DNFOGl/GAGpfU9p+rUkS/HzMApbemwSqbERsiAvXKE9Ri5s5dRqGJHjgLl7rEczmDFle8Nf/HRSMdNbMB0CNG8ZiDDs1IFuZGGof0yjv+3N2iZimC7YOmC2mwtfLYb8jTO/qV0WGwAW/QvR80CH/YWpdTalNnHgC4/hQ+tv4hWJyLlZifs72i4hy+Ri79r3xSee0+oEioGTm3fTMM5PPvRrC8J9Qm4FXES4rWutuerfy31zBcgtQ6f1Bf9ZXsSxFcOgA3wNYDSX/XXS7fHteVPoHEMNSb2pMdLoyAgjWExefPWAiNPYDZI/oQCpbQDNdcRaiLWJyEQAk2mudDu9QUqK0zhCC2fIyr3YldYeN9e/fZ9PinkcbRgEhsM7T98a2QsChHK/BBV51b0Sbf57pQi51nY9MNH3XUNYS/Hr2LkObrhaRFO8l/2I5Xt53Kk5jA+XyWj6b58sU249iE2dw+w2VzU+kx/ONGn4byeBhc7llFFhe6vMLVYHlgHhchitq3zenR+2KspCSbctcSvv45NQ8+EwNwfzVtXRsOehT2a/HxEUtl7Zro6nG6MtFNLJNeN3ykfg9bqSf7tyGc7zUIoQvcFivqmDk+28WGtMD/PTAbsJx+S4YpHyVcqz9GMQ+seABaquMMrrar3X0Kdd/mG2tizqacG9MVXD8Lbg6eCEWmw33/3/fvoZQMKG3LResqVW9MvJ3jy9WJ/XyGwK2iwO6XD41/8t/Yiq3C4qWo6l4Tg77v9qD93jl0JnSFjwEUBaLuMHbZ/nyOkilqk4vSaRIx+EQcJ32scNEj5EtDVjaw/72ae0EIm1rtu6hciS8VULjeieYVF+zsoA+75TZfn+2GAxk5OqLjz0xNugCm3xmt7MiylXYaM11lGjqIYdv7SSJXSB/ZHYJuqW2PC02ht+qncozJbZb9ki4KaAlzy2Wk8Iwks+HZhKdJETaWliFeS2zOxopphR/JXy5rwExXZlE/RQwdkGw4d/ANRn+K3QD8DoCGcaQkRiDFRnj98HyUvqQY8dTIEX/2l8ms29qy//9w51Dh/gUTiWtBWKu9i5Kax8Pko4jPLW4+oo+njj7f9BeWClsTwrYWmgVBeH2P+LDTzVAEkFD/TpYIpvMO96/I2Jjoc5rh0M4q/luIXOJSG0wgAbesKwRS1luWuaSOvR7c4zPW7XVNnyUwPUCJUiF35JWbH+3O6xr3YR1MIvjFHref0n/2/8r6rSXIcO/fXbEh62A5680jvk8lk0iRfbtB7k/TJXy8iq3unp7u0O9LO7N2VqqOqaQCQOA7nfDgEruNMcW5AXq8FpSkaXc244mHSfTYVg/YvD5OIRkHkvDvnroSHR/QGE+mSBvTLUPE8cr0DQDm2l2GraviKYb83JcKR8UjbTodsTG1yj17pHhWg2aTl1cWxIbjO8op04nAvc69eXsFFey8Zf8bAiKSHXvy49ygOF72k4KdPBFxmGdvL7PViQab1MqoIxvjzOuAcacyccI6Xte+CaUn+Frjr5eXuiivhq2W4lelCL14qci7gN2gFnkndZQGWUUxy4WBdOd2d2vDYsJbqBWAmm6RnKP8ENOZbYjj4LeWMdbrwwO+LgGwRs7/0YujcGE1VCA/TUfEJZ6DfK3+OqNDlWNI2hnIjH+ngqBBauN2uCYR18+hCUvR4z+kYWCDq0wU/HnaWrFnx7K8ydbG14d7h69JDFeKuIsNjfqILGIwmix4u6aymp9gVJnAp0XiRubixAnPFWtS8xAf5XqYrOV3tGHV46spuQtbb7w8XHYZkZE1ZCZkqUHZ49OUxDtTRtYc5smJ5azF+rajbvj0ImuPcV0ANC3l/cU+MiBnZuqj37rkGAVAJdvMDuFI8ETcsFZXiBN05fpDeSV4ksA27m0XOUdwV3/n48spxIN3G7k9vAnOw7KVmzEfee3zlhsBBA4vfss7Qt7jD533wiE13PNpexQ5Sdyb90uI8RCO0d4cn8mHw6dWIbgiWIMyDDsY20VE9rhS6b7jkEd+4x7a7V56SNh34/s/KM7ELzpsssIpX69pixQGmDPiXIqJg5rxK0BEwtc6Ha0E/DX8l6N5KFTW0e8XY2Fej0hVki6nu74kWtX6KFdBlAiOItYgWZvDj3q2zOQUxiGulId2HIyV4X96OCgXfXvYqfuEnx14kHFWPaLmm9qTq7GVHc0FW4WkJZmd6WdmGiZiyXG9r05Gd2VbDbhwN6a8byBmQ+mcGX8hHc49KjWab8kH6lHK6WQils5veOWahArqqmMfmOVQ2ZFm10IO1j5W7CLigVjSMKr0i3u4KGmLpNe3han15W5xQha+rFY+SHgSmz/fHjAyG5r7cHKcG02DcBQ0L4Add9wCuX+qVxB4Ttr9QedtNb8+ANXteVjUeqIvYTnF3dGGRWMJJnOx6iRqYX5UAf7YcxJziXnQkIsfpwgAEvIaP10iPIXncZ4lxHkDRMwLzrPiUYfz0Y2Zb3HjSU3ZxjVRJSSeJB7DZDEZr53jNkj9Ywxn1Lx6r2PMLmdhX5jW93lHhnhy2qeOUguHNo2/L+7QrGqHHvJqbC901W+jWAzxkhGMmyOMhLSZ67bDtdCqZhe1aS5PDy76yTLL3tIRhhXqSyexYSAXDELBCTay087aio1tqww1Dlal2ZJBDC2EaUNDe7S5RQVvccorDBRinR9l4KY/2leHvo8sI1XbIXPnUr+0dgJewXhSjYr+3CXvJ0qxnAsb1V8wz3t/aKCFl3FYp3V6yEzeBs9p+RHLT8LiDPJwIzazstcA5YwUMo8AgqSVKdvvYUwetilE3sEphiUcULd7kOoYQZLKRQ4lgXmSpMSQwF1Hi+nDJqADty532mibijv5+OP2evj58CixAuGE/YClgF+xmEwCsHaTB3yffmG8GLDeGtmUONTR+q90ZCZaATqQY0BYgQQ282MIC7bs5VBuAkIBWQma4OgqYgW4Jgz3Hiiv93j89wkfd1KZHuaVD6vPVs9zoUe/scS6XOuh53rYOnskb3XKRiLiL2XZ63aD3nFLB7oyi8XtDwhtxiTFaG8BizWWlucOrAYMomCMWO4oKXYxMxOiwbpqoMGdko7ThmALEHybRlCYKgRk4yLIGVhQgnuxUnmtXwVmeXjdHTE5YimWFTMuHuj/P+BDiIAwkNBy9Os82ZGjWKsv6IkDDGaqUZZry4CPRUmS50zffjV1myAjEvWUvR0sYh1QP2TeO07Tca4+FRQ/tnhK+faWPOzJyyY2TFYbljtvarUm5TZoPou/yvTDsEmmTGXm4IAhOcI50+6BTumw+QhpEtHzlkyuklYJ0xlraaw/FVRat6NR7NvEJ3xBNRxYEEIlKbcfnEiqwyRltb3caxoWSBaFnmzmRBHKemoec9UVDFNElzK8Kz3E6GarDHTJim3q69TbuA6Xu8Igb/btmc2rQld52EEHk7TO6T7XEvVgtNumOVgY/1gEuwMI0GtJnCM/VgnSOnuR7hV8Ixh8B0hshwJenh6Q85/Z8H1HCx6seXNfDnkvpdDNo/e0bhao1MKzCPE3oiZDnmI7TIIIM35/vmwFd56rV27uxcRnVh41zpOJWS2vNfVBG0Tpi9hIJhkEUel/jGNgpV7QflcWy4FtScVtEwjrD5W6ztDvwJ+DaLkBeDWtnQlbxVT0np5XXfT5GNl2LR8njTgcFfC7G6K8+Jb1NA8GlcoaGMqRfxtPdeW+dw8aQf5+i8/1vilLsJktHUJuERixvHncHufd6/EEbtSCVTZ+7YDGCq7qM7dqZ95VH6pBpcsqmjmhbXqyNP3kbkcybuNKdzdoczzD7ezk5EYyN0dGcgzcLHYefbFsrXtJW2RjDpbj1jHcfoc+VxXS9JeHbwdM+6Go+kgGgIVNYbjPYrZrlGzEZl0VEKuakU3lvlbE3h7a1nC7QtIi6cfe3n4o1HIjHy6stWfsoq3i1eIvliJrmMtD9NhOWcefTWGxcZi13AwwFRDp3/rXmmZM1HNcqHKFH8+qyAO4e/ZYmcFax8k4TXpxWXa1t3e9cPQTYq7ephaRn1MPsogScYe+9vEcpzeunvTFI4bkQ+KtlsZIDnjs8SguLWFRQ3rA7d8YQUbtg9cPKAViyM8iQoqgB9s5ln1XbSlPB2lDaM/14AXb+4bBh07L22jLB3qQEfMTUjVLeZvhRhANYHY2lYP0Uj/dCcq/DTeijkCyhf509X4CgLQS/LksNKakzjkhOVURRzYoEZESQ6/wyzOmByWNFSoM3tEslIcxNvAg3PuCvzxBvD1nDLLfwwCSXrbYDSIpbr2FHpPvbYrCC/Kwe03gNhrRGQ+a96p/vzmz80HlVCBTpURD8DMRb79kwqIchki9QAcYdosXHtK8EJc6Z00Kwk1dFZD/Bz9Pn2ckJQl1HVuLEEwImULRHRDMQD1V248x52cpPYiD8nbkznooJp5VnBCdlHZLeaTO61vvenT7oPXnqlwujQIagtvyaBy0hR5YLjeUlDPeAnN+LP899dIPq/M2V220Q7wshl7JEj/CkgXkhbCdXqjrfo2E4LDtEcnEPfizi2mhPz8iMikC/JnkcrZD81nVOUhZqeG/BGY7XxSyQKrrqLeebj5ZAHdS1OEeqGBSoZgO+78ps83YPRtW3/PXmmtqAtnx+e1nWyWqgGUFJppBP0MBQFaQUs6Q/c+4ZaDGX3IOkF8s4GcV37Xq5e5Mi1MdSOwD4b4er329G1Ha2Zb/bYkr51aTIZTklb4sxSsBMybNDv/ak/pJz1y4BgnfRXjrnN1bsBxo0aj7qoZcVikrH+pC9enTIeSCbQZ9W56VxGxNgyugaQxwJbOjUClfKikax2QqBYbolqPeCJoHtPB5Y6tSXpekzpEQoYOQq3h3r0vLe76fkAZih5K7gK1XxbkP9ikvBU8DNZ3s34+1+Tzgqku+GiDmlFORNs9T3ioEwFzOVRFOfMpPiNnBRrDEEa7OezLryIGdEvtLUQyEFJc83xvLs7L3OqzVnlaEjpH25whMh9mP9rHPT7LFzxDAd3XIivOfainjEE5izzNgqi/qn0z40QZb4OHHZfQ4cAM+7PReuF8a4sGZeCG+7op0u2PmMsTqEGvUvlS7Bx9WED84MnENRY3ybY/0BK5LjmFfpoRMX6ZKzWXVYA7qicx5wJXPrkEJWEUZojgscAzVkr9oaaIlpEvfxOXJvTRYc3VmQ+T3n7xiIPUWucYGDXQsewzC7W4VOhfSii45/yEnNOhhtqNgWFpCCcOjDRNsN5CPCvb0lGosyK44lEXITnEq65/wkaTf4rihJod8Q/ukL+GLj4yMb9PJD74zwPU94dhVBLp4XZk+poe4GjpFOHD67chRVFea5S3PIutYmKYov6ntfIUqWsq2oT2/hcN67oY7rdetS9XBY0rzx7/TKzMAJoTiyDkfQu1Ju4xbUoXtT6PJ5yTwrZ6cmYcnIBG6/CM8EinMu3r8NLCsLoSuhyYSLw1SDxdHP2EhRn8ujFqbGMZTct3Yq97SXrAzkFtzvOB5oiyU8ct7CWg5zIfzFHJOwX4qp6JQk1ZxpCO/C5qCSV4hKa+opVtdKL7ZO7Ng5Qio3osgLdHlg46RagbTBKqVBvIQvJjQ7xcmtHPyWnCbUmyVYN0YAXt6pl5zCvO0PK5y/YDz7WtZ4m8bzF2heXr5/z3GHedcFUvauC9jwtS7bMx91Tz+Qfdc9TZL9tS6Q/du77oGRFMcw/J++rdFFtw3hNBfmqTDgR2jEe20vVstxf0JP3p78gobTge3mP4FUCvAlOLi0puOc7t9dQoU/oVy7S2nfpjPYkBD6eheBoC84Tv/yQ3608Pq4/eezwMeFrUzm4uMiCX3BsI+rRVrmxddnE1+rhtPHef6Xh72XTXm/AgAqdy5tmm9v9D5GoDL5qAMhl8659J7J7kmxMbAkdfif8a8PW8PmHD3f5T4uTPNpNj8uTOfICw6TcA6nuR/PY3Yryjm1hzAGN7YxHM5rxdyez+bh83Cax75Oub7px3cbKIrSdJb95Y73tccn9disbJpvJbu+A61nfTfb5QHahsnfiRkU9GvqwxD8BcF/YsA3nnxPfRT5gv9R9Md/O/3LNswBdd7/M9OQxuD9oPNK+O0kK/f0fBwL6FLGYaOHUdpc+6mcy74770f9PPftdwWYpszBjbkfPmNblkU4Cf3ENhj9gW9nUej98wnzvr72W3z+hDIfp2ewC7Yl50qXNW8bpEl5D3TwYjuF4OTnkQH+8ArHPM7/udnTiBQU4HxW8XzjPCKF84+5M5KrbVgEapROI1juDUMWNMmQwsV3VKbuybAotsVatZ33pWJzjW2XjX0TLKgRWcEu69Lpe0xzoWC/ebd7bWlOrwiC3dlnoUEpVbEsrBvnOATnkWNZuHt3c4b7aQPbh2Wpej69SGCJuyWbuvkVgwQHCqePo+sAvp2J+64Z9xZPA/IMD96XCsoSbr/+Vycqfq89iwtQhltYf+NfW88pt/yWnoZLljd1u1RgHzePG8s600iG2bjX4S23YKLEm+OKClIgG6MrIabnQqJVylBCctSJTyM2YXdy6mA8404lj2Sl47NkuYncc+JQKS9MHbgLL2G6kDPEr8w4sfqDyexheQVrpVuh/AaUzcWMQ4zdNpAYdQ7qGDPeH6qm8td0Y45iz1sL6UhF4HkXl5eUyWaQkaOqDXGwh4RN+wgEm00RTOj6iFq7Ltddjw9EwaekMzC7Czx9GUg6F64KCr5fYqWw9cvb/V5NSlW7pHzwhwImbeAUEBx1b4it+pNy3bsHIffGqN/RK1tV1/d6a3ZpjZecv/O9WvFzMZFkFyTjEK1JTLuPubYseoNh+U48OToA8/2bZoY0+RyeqgQlm9zImy8H6JO6L+aYZ6DNIS1vM3nUWpUnTkCZV1Mfu4XoUzsxxosIHEl5DcM5izmAbMV7cY6C2XsVD/sy3opzoIJnMGuR+xRIsCyYDW/5+rXO+nXjDHMrzdTj5GdN2DPbY2FG2Sk8l/ghnFIMQK+CrvVekPOgX9h4JtNh0wyzHULeyqhrgyBckjbPwcOS+UK95IckabXaXLJJevmapl59nh9bjS8W54LcEMULdH1vvi5yBxb7B6L7yLxp8Dj75Tu3KGBzRxOuSO8dUJaYj+Vmrs97tCp83pLqdAUV6NaiaDQ3Exe/6R5jwP7rCAgMcCjK+v58MHVfOXxk/CqCcTvaDUjAkBXplklZoWaUxXJ8DqejWxWYoxhcYGWbHK++57oTSrniO6uiX70YpWRUIHf4/ZpcU/pkrLP+zlLPTYcZwniDP8f63py6NM1sDuDnPB8HzYfk5Mxpbtf1g0rYOq2sqzglyX1NzRdJHwt0XW6UA/jEwJfrCCYsREsXQXJDHw3avJvHM22QQybk8++pcsG6YWh4sIpKCzK33YTWMQko1ugz4gLjwHtnmANNbtVpSZ43AI7PABDWQi4vnWsUqaIZkSNdrYTFLOaaU6CG09sqn2jWTY5CJu7JwYpt/+JsPk0yy4QSmlgFmsUPw5XQ/bEd7STiuQOEDQDDhY9LDoI7hHBR6fBkxSaBrviMOm8T5Htw5DAZyI8QEfjJFTgEGH/BIH5H6UOGCZG7K4W1gdxrAIGxcqRlJFJdVhRWE+fqwhjoAVrVkG3nLFts8B3XMHLuc0njCDA5BrR7xOkXcpGvy+OKVEKesIcRyheyoe9XyZYCHr7fWwPI29A/jmJ4rzzUZWd5xscA5Tg+ssCkaW7UTobJGCh6vKmTXyj1CqYXDtLQlmim24wCNfIoKCscbCgQjYNnJxRZ0YUWA4hhi3BYJj0RgbLeUinsIykmyKm1pt5LARJVqLsrEd8vWVeBCZenkygJ+arqyLcNuJO7AELh2RGhKeWOVrtgJ0WCm1RkFbkCRR6J9JXZ3HtPGLVLgJ1U20dq1iSCbCVS8Kie3fjpZiPmK08jp3uv37Rxo1HCtX8P8ejJTx68nL0lAVBaU/xrjqzjPll9g0njSkNX4zFJLvMQX9o5cna6jMaDM2SFdJD5HCIw17APqKGeaDXMjEGxnV5Au49Q+AYLLG7JflAQs0GFIPaG4vVa8jE0SoG3ZVNmpyjZGiag1bw5Ald23D0eUgBdHt1+xMGjz+jD97EZP/gnGmjjLcVaV+fpWV4eRADCc2bF3KJiXm6fJc46L113eCOK2cRtg4NsfV2XptVoy3ouALJ8zDrcMaoGyGDarMIegs5esxBkl7cgdsShi9iZ5p1cPYoEK1KIhML4fhmsTteApPdyoAqkki96pUxZ7VE3hrwUY9r0EmEf8/MZO6905PlSb0GsTTCz4qJIYCJD2sTGADULYwLbpLauqxoC2xp2Vj7qzPM169VOwpQGgo0IMqGhIE374dehotPjqXXs9UEScjJY7l1NdhBGhMfFFTDsQXSk7Orc2mX2pRvXPkEa1rWa7v0FqCZgykvSDpZfYQH1/K57tjTynlTYAqBSvrA06uNuj04GRMowmWOtO4x9DPcLQAWrkqgy5S7RNhogPEU6aXCdhqylDuumv7cCIDlAp2B1weTAg6DS2tdGSOyR7WVHHiRGD5Fi9FE8naG0lgN55NGc3GfkHF0ims+l4wxV9fCdhk1fnSpc3bJoKTlKzC6p8VL0gnJSZ+yFtM5LkDzzvU4I817tKO1TeM8xDNXVlX/vahbztOQAU8DLebcTrww/TPwG3GQ6ktAjjaYDoMhSwTjUM2L2ZqBSYNH70zTn4+s+RZniCUzWs/RRTA1WmnR0z6zgvQe1S+IsYVbMGJFXSFzl1+PqFs0zM4ormANREkrxLkWBLfS0Etc7jV7EZvS74VAclaWrYLYrss7QACXfS9MYSfXWixE4QJx5D0L1AmCkSodF7HlHqBtJ0FGEXyQ505jlsQTOfggtiQ46BnQd9Iqk1UpRkNBr4hcMcljVCUymAlqiT4ST8jNASU57y3ph1++pBbTAedicwd9ESm4SXwetECSyBfIeYXtasYc2GTCAihcgQPfnrZix2ttNZezVRqrM2veoMr7pTTSE1lFYk2XoiwMLpAc51gtgpOyKZ1Oy4/fZmRwJAgp+dxnveJ2ebArn9kOvhNPqyoFkX+p2bUqWuTaiaxMr8Yoxc+fsxYt3jRkkj0/1NjKYyW8Td3X969MqQmrKkZl8+OQ1Iy7IA6Zw++mZ+nMX4BDi9gSH7RmblSdWJuLLGGGRRS0tfV5hHTGfr77RHskhF0dF8ExsRyAnj2pNk0Hs67Ph7JmT5SPcx1BAUtV86R0TyfPpudTjDcZl49bDmmY3S6wO9Fph89OrO5vtAenprvGgYXjCQm8nxwvA8xaWBDanEq9LtERXrusXzIGdjvSZ+o3IXBWiR8JHklJDbBK8rjmlOscl222+eO18dWhOilQs+LxFx2buebuZx9U7XTPe9YJZWES4pie7nMZVqCLSwG+x/JrvYws5W7KZr4yNwSicxG2Lyl7Z+tp9AY81gICcavaELutgIq+kD4jVmx+u7Q6LALKFkzx8Oi/89GAEOcIo2TWoQpTa8bXX16ro8sD17oMk8FhM+XfifGS/1fXwnFTEeDg3V/Xm0CY9ywBuTWlJUHNrBgMYbGXDB8iqRbcx95VxUuE0hFKnhZUmsfGBy7Y2vHzlVTTmU66byK1MGIW5zr/onnWEd+P+AlOYtnZzJWeXYmOitOfzNmvefj6Ne6eEug5MzNfnEQLWBuY25hXFXrUIpNuIuUw5oINCyr13IsfOoY5OLrMI++Gy248reMlB2/0KuzoX8Bmd6KLMFtbZflMDzI9l9mJ5Mms0utE/6pGYmXnHRC7Q/IDZVd+Nt42eyjwkXJPPF+0o69pNAJx/DbHiEsD27ZI1lYXpZY4Nan3ZxE0Fmf2GPssDZ1WFvy0MBrL5Ocu4fFDRPsM+kd1s+Bk3HIZfjcyglRq6PHjl1pQOdi97TFXGQsh9uVRUvp+KF3i7qhatWMjrApps2YgdmVVd/mBvlME+GIV4Pp+OK+NWbjLE2J7H52gdz7VjVr35DNrOXXtF3jfRjU8/+jquIeNtYBAcRhsI0NJ824cQ2GIW3QB9IekcxLKgylvESFOBFnW5hfPlZrcGBhZwhX2hbuNM3fPYXHEmfkJZK2C5aYbMDHtU3cWdcEYiAcUuYojW6G0wLl3kb5nklU2oYdLGncdS2Xgaxvgx2ZkZ64e9d8bNvhSoqdUbT6dswLKfRcsmrBEbhpafVfWeOR1GXiFmrVGGukWwV8JgvLffYme8oQVegViNm8saG+wSSevcVIncQV5Nzj9Z85D4x67db+JSxxfClG5ADBlgrkdJ85SlZ2TjoZ7dtlAHa2tihR7bXTJfNrY6dU7Tpbiq9zxlAPA7cpqvXNt+N1mMJWCXctob3Upy8DTkXRDLDhvMc+DBuZg/mE6oS6ctx1LGoa1DDbV83rZ1mh4dHzNwOCKaZcDFdUb0kKR7qHLy3lfyuyw7eyvgop0q3dYSAKpm4fD50m7G2FtGtVDbRjHlEElBrC5FBLotbUTOTkW+GK1apKut1UZhlWaEiakYeM49zGbnsLSqpbbTzIp2m/hAhThsXDhrevbM9Xgo1fPUiFOyXy0IGvP3/XxuLpqoamhQ0BIkn1apYxXP9Un4YG0yGlRXDekYc1eNzwUGtAmCd4ba79TTI+yBCcJMQwtvs8USVrExFaKzvjYFaYvnz9iVORVU6Azxjgie9ugVdTKLhCBz7tTVl6hmjzUgeW7Cbmr0OgqHGhbG8sKeX5hH7PqcouvxpUYu/o3R+6vA3H5ESITbyfErBqQ/yiUQrqqhYRpv6NZ2XPOm4dxDUQAY9ccAhvTneOFfsMHvEUPkD0QMyb+NGH6HxP4EDf4a4/sKuE5FmPTbNzTxKyYYn8RLx/8SLPwB/0NZihfhj7becGW752M4FF/yeEC+rGW6/b+mnObfgTMo/BOU+xdyf88Z8gv6CZSOol/IP4w31Ce8IZrzyWxSng4dkYND8stZ5NLPZQY6lc5xMqXj2f1Pis49eM1haEDJuG/bcp5PLn4teL7jd2U/qX7SeCzT6a+X//HqD7J0MmX+Nbj/N+WjLZMEVGfHdCqPMHo3BSRr6MtufhMdZ/+EA0UNl7mfvoLFP2DH2O+jx3/GflRkgGj/KCww9gnuT0B/kJwQ8G+REwrICXMy/81C6Adm/iglhstxb86BCZqvZaLxbzH7p7bC7hQvKArjOu3+hqD9bxccAse/4OgPsoN/IjufzRn9cbLz2YzdT2ykgezkt+tbJJY4Tqfp7xCKkx1D303/Dbn63yYLGI1/wfBfywJF/EZZwH4HWeiwlw+betsLKBbFZNB1afDn3zB5eLZSDtNvmLL9nmrEz8N7gqdUgn02R0ghEUoQvxOdUfQHY01RnxCZ/oTI1B9F5J9ttZ6GCRiuz0E8Sf95ZJ3+dv71XeDfgyEI/aPU4z+7WjD62az5H8UQ4h8n9VmWIXH8mdQnRETgv5PUwyj9haR+TWb6k+SQP8xJ+ZTMyE9kFvum6bd/Qskn/gDJh3HoC/T9zw+eAA19wqDP9AD5oxj0WbDxL60H5/0vMPZrMqOf2f9/qB7Q/7f14Ay3v6DED0z5zPP5h8r+Nwjgv/KCv2MH8Vz6+Sth/vxBOOYsAGPD/svNH/1ZUPh/3gr6b//2jt2mGgRuQ/oWlRsDvkiFOrAw6Hdu88eTfrND/v+3Y//ev3EKqJjn4T/O/9+R55ieIWoczum/aq/mMeymMH7nyJ1yB/7MpxSDjnqM/n3Q8/e+52+kzD+LPfmdoifsB58e+wyAof6RNh3+2aj/PHZ2CTOOb1Q0bsJpKuO/PmQCfOblfyXz++QBTgDa+PWc37+/y7++ne3l7H9tFBw/vrv+SxVw8q3Gz0m0STgVAN59N/JfsmzqlzH+2j2fwziks4a8y/ekLMsImbs/f7PXczjm6fzXCn5tMU3y9K+KwHcsxj/h8LdrY9qEc7mmv3rdz9j+9QlXIMbfhSkk8WsZQ3+MuT+6/rXaL9LzSUs/oMsI/UNLH7T5qaW3HP6l43+H2/03RravNgJnmenVxWfBfkjH8KvtCrP5bZ8JoNr/c7yH67usHNvpwxCCv99byN+KJILqv5jC79HrX6zj/1E4if7BoSJh9GeLSP5BFjF9onwk40kslYldPnv0KvmfoBx/Tep+ka6/KQb2WR5IAts3M38WgM7Q4u8BqN9ynpRT/abWSSwEOk1HCmaWfmML70T003Sd/AI8gSGo/XvA0X9P+nTq/g3YkKjpQeLhh9L8xurbWALP6RSoufiPf2WN+En8P1GSvwa2kz+B7ehvDbFh+L+vFufp2AMz9IvxBtOVxumegxL/CQ== ================================================ FILE: Documentation/postmortems/v3.5-data-inconsistency.md ================================================ # v3.5 data inconsistency postmortem | | | |---------|------------| | Authors | serathius@ | | Date | 2022-04-20 | | Status | published | ## Summary | | | |---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Summary | Code refactor in v3.5.0 resulted in consistent index not being saved atomically. Independent crash could lead to committed transactions are not reflected on all the members. | | Impact | No user reported problems in production as triggering the issue required frequent crashes, however issue was critical enough to motivate a public statement. Main impact comes from loosing user trust into etcd reliability. | ## Background etcd v3 state is preserved on disk in two forms write ahead log (WAL) and database state (DB). etcd v3.5 also still maintains v2 state, however it's deprecated and not relevant to the issue in this postmortem. WAL stores history of changes for etcd state and database represents state at one point. To know which point of history database is representing, it stores consistent index (CI). It's a special metadata field that points to last entry in WAL that it has seen. When etcd is updating database state, it replays entries from WAL and updates the consistent index to point to new entry. This operation is required to be [atomic](https://en.wikipedia.org/wiki/Atomic_commit). A partial fail would mean that database and WAL would no longer match, so some entries would be either skipped (if only CI is updated) or executed twice (if only changes are applied). This is especially important for distributed system like etcd, where there are multiple cluster members, each applying the WAL entries to their database. Correctness of the system depends on assumption that every member of the cluster, while replying WAL entries, will reach the same state. ## Root cause To simplify managing consistency index, etcd has introduced backend hooks in https://github.com/etcd-io/etcd/pull/12855. Goal was to ensure that consistency index is always updated, by automatically triggering update during commit. Implementation was as follows, before applying the WAL entries, etcd updated in memory value of consistent index. As part of transaction commit process, a database hook would read the value of consistent index and store it to database. Problem is that in memory value of consistent index is shared, and there might be other in flight transactions apart from serial WAL apply flow. So if we imagine scenario: 1. etcd server starts an apply workflow, and it just sets a new consistent index value. 2. The periodic commit is triggered, and it executes the backend hook and saves consistent index from apply workflow. 3. etcd server finished an apply workflow, saves new changes and saves same value of consistent index again. Between second and third point there is a very small window where consistent index is increased without applying entry from WAL. ## Trigger If etcd crashed after consistency index is saved, but before to apply workflow finished it would lead to data inconsistency. When recovering the data etcd would skip executing changes from failed apply workflow, assuming they have been already executed. This follows the issue reports and code used to reproduce the issue where trigger was etcd crashing under high request load. Etcd v3.5.0 was released with bug (https://github.com/etcd-io/etcd/pull/13505) that could cause etcd to crash that was fixed in v3.5.1. Apart from that all reports described etcd running under high memory pressure, causing it to go out of memory from time to time. Reproduction run etcd under high stress and randomly killed one of the members using SIGKILL signal (not recoverable immediate process death). ## Detection For single member cluster it is totally undetectable. There is no mechanism or tool for verifying that state database matches WAL. In cluster with multiple members it would mean that one of the members that crashed, will missing changes from failed apply workflow. This means that it will have different state of database and will return different hash via `HashKV` grpc call. There is an automatic mechanism to detect data inconsistency. It can be executed during etcd start via `--experimental-initial-corrupt-check` and periodically via `--experimental-corrupt-check-time`. Both checks however have a flaw, they depend on `HashKV` grpc method, which might fail causing the check to pass. In multi member etcd cluster, each member can run with different performance and be at different stage of applying the WAL log. Comparing database hashes between multiple etcd members requires all hashes to be calculated at the same change. This is done by requesting hash for the same `revision` (version of key value store). However, it will not work if the provided revision is not available on the members. This can happen on very slow members, or in cases where corruption has lead revision numbers to diverge. This means that for this issue, the corrupt check is only reliable during etcd start just after etcd crashes. ## Impact We are not aware any cases of users reporting a data corruption in production environment. However, issue was critical enough to motivate a public statement. Main impact comes from loosing user trust into etcd reliability. ## Lessons learned ### What went well * Multiple maintainers were able to work effectively on reproducing and fixing the issue. As they are in different timezones, there was always someone working on the issue. * When fixing the main data inconsistency we have found multiple other edge cases that could lead to data corruption (https://github.com/etcd-io/etcd/issues/13514, https://github.com/etcd-io/etcd/issues/13922, https://github.com/etcd-io/etcd/issues/13937). ### What went wrong * No users enable data corruption detection as it is still an experimental feature introduced in v3.3. All reported cases where detected manually making it almost impossible to reproduce. * etcd has functional tests designed to detect such problems, however they are unmaintained, flaky and are missing crucial scenarios. * etcd v3.5 release was not qualified as comprehensive as previous ones. Older maintainers run manual qualification process that is no longer known or executed. * etcd apply code is so complicated that fixing the data inconsistency took almost 2 weeks and multiple tries. Fix needed to be so complicated that we needed to develop automatic validation for it (https://github.com/etcd-io/etcd/pull/13885). * etcd v3.5 was recommended for production without enough insight on the production adoption. Production ready recommendations based on after some internal feedback... to get diverse usage, but the user's hold on till someone else will discover issues. ### Where we got lucky * We reproduced the issue using etcd functional only because weird partition setup on workstation. Functional tests store etcd data under `/tmp` usually mounted to in memory filesystem. Problem was reproduced only because one of the maintainers has `/tmp` mounted to standard disk. ## Action items Action items should directly address items listed in lessons learned. We should double down on things that went well, fix things that went wrong, and stop depending on luck. Action fall under three types, and we should have at least one item per type. Types: * Prevent - Prevent similar issues from occurring. In this case, what testing we should introduce to find data inconsistency issues before release, preventing publishing broken release. * Detect - Be more effective in detecting when similar issues occur. In this case, improve mechanism to detect data inconsistency issue so users will be automatically informed. * Mitigate - Reduce time to recovery for users. In this case, how we ensure that users are able to quickly fix data inconsistency. Actions should not be restricted to fixing the immediate issues and also propose long term strategic improvements. To reflect this action items should have assigned priority: * P0 - Critical for reliability of the v3.5 release. Should be prioritized this over all other work and backported to v3.5. * P1 - Important for long term success of the project. Blocks v3.6 release. * P2 - Stretch goals that would be nice to have for v3.6, however should not be blocking. | Action Item | Type | Priority | Bug | Status | |-------------------------------------------------------------------------------------|----------|----------|----------------------------------------------|--------| | etcd testing can reproduce historical data inconsistency issues | Prevent | P0 | https://github.com/etcd-io/etcd/issues/14045 | DONE | | etcd detects data corruption by default | Detect | P0 | https://github.com/etcd-io/etcd/issues/14039 | DONE | | etcd testing is high quality, easy to maintain and expand | Prevent | P1 | https://github.com/etcd-io/etcd/issues/13637 | | | etcd apply code should be easy to understand and validate correctness | Prevent | P1 | | | | Critical etcd features are not abandoned when contributors move on | Prevent | P1 | https://github.com/etcd-io/etcd/issues/13775 | DONE | | etcd is continuously qualified with failure injection | Prevent | P1 | https://github.com/etcd-io/etcd/pull/14911 | DONE | | etcd can reliably detect data corruption (hash is linearizable) | Detect | P1 | | | | etcd checks consistency of snapshots sent between leader and followers | Detect | P1 | https://github.com/etcd-io/etcd/issues/13973 | DONE | | etcd recovery from data inconsistency procedures are documented and tested | Mitigate | P1 | | | | etcd can imminently detect and recover from data corruption (implement Merkle root) | Mitigate | P2 | https://github.com/etcd-io/etcd/issues/13839 | | ## Timeline | Date | Event | |------------|-----------------------------------------------------------------------------------------------------------------------| | 2021-05-08 | Pull request that caused data corruption was merged - https://github.com/etcd-io/etcd/pull/12855 | | 2021-06-16 | Release v3.5.0 with data corruption was published - https://github.com/etcd-io/etcd/releases/tag/v3.5.0 | | 2021-12-01 | Report of data corruption - https://github.com/etcd-io/etcd/issues/13514 | | 2021-01-28 | Report of data corruption - https://github.com/etcd-io/etcd/issues/13654 | | 2022-03-08 | Report of data corruption - https://github.com/etcd-io/etcd/issues/13766 | | 2022-03-25 | Corruption confirmed by one of the maintainers - https://github.com/etcd-io/etcd/issues/13766#issuecomment-1078897588 | | 2022-03-29 | Statement about the corruption was sent to etcd-dev@googlegroups.com and dev@kubernetes.io | | 2022-04-24 | Release v3.5.3 with fix was published - https://github.com/etcd-io/etcd/releases/tag/v3.5.3 | ================================================ FILE: GOVERNANCE.md ================================================ # etcd Governance ## Principles The etcd community adheres to the following principles: - Open: etcd is open source. - Welcoming and respectful: See [Code of Conduct]. - Transparent and accessible: Changes to the etcd code repository and CNCF related activities (e.g. level, involvement, etc) are done in public. - Merit: Ideas and contributions are accepted according to their technical merit for the betterment of the project. For specific guidance on practical contribution steps please see [contributor guide] guide. ## Roles and responsibilities Etcd project roles along with their requirements and responsibilities are defined in [community membership]. ## Decision making process Decisions are built on consensus between [maintainers] publicly. Proposals and ideas can either be submitted for agreement via a GitHub issue or PR, or by sending an email to `etcd-maintainers@googlegroups.com`. ## Conflict resolution In general, we prefer that technical issues and maintainer membership are amicably worked out between the persons involved. However, any technical dispute that has reached an impasse with a subset of the community, any contributor may open a GitHub issue or PR or send an email to `etcd-maintainers@googlegroups.com`. If the maintainers themselves cannot decide an issue, the issue will be resolved by a supermajority of the maintainers with a fallback on lazy consensus after three business weeks inactive voting period and as long as two maintainers are on board. ## Changes in Governance Changes in project governance could be initiated by opening a GitHub PR. ## SIG-etcd Governance [SIG-etcd Governance] is documented in the Kubernetes/community repository. [community membership]: /Documentation/contributor-guide/community-membership.md [Code of Conduct]: /code-of-conduct.md [contributor guide]: /CONTRIBUTING.md [maintainers]: /OWNERS [SIG-etcd Governance]: https://github.com/kubernetes/community/blob/master/sig-etcd/charter.md#deviations-from-sig-governance ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2013 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: Makefile ================================================ REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel) .PHONY: all all: build include $(REPOSITORY_ROOT)/tests/robustness/Makefile .PHONY: build build: GO_BUILD_FLAGS="${GO_BUILD_FLAGS} -v -mod=readonly" ./scripts/build.sh .PHONY: install-benchmark install-benchmark: build ifeq (, $(shell command -v benchmark)) @echo "Installing etcd benchmark tool..." go install -v ./tools/benchmark else @echo "benchmark tool already installed..." endif .PHONY: bench-put bench-put: build install-benchmark @echo "Running benchmark: put $(ARGS)" ./scripts/benchmark_test.sh put $(ARGS) PLATFORMS=linux-amd64 linux-386 linux-arm linux-arm64 linux-ppc64le linux-s390x darwin-amd64 darwin-arm64 windows-amd64 windows-arm64 .PHONY: build-all build-all: @for platform in $(PLATFORMS); do \ $(MAKE) build-$${platform}; \ done .PHONY: build-% build-%: GOOS=$$(echo $* | cut -d- -f 1) GOARCH=$$(echo $* | cut -d- -f 2) GO_BUILD_FLAGS="${GO_BUILD_FLAGS} -v -mod=readonly" ./scripts/build.sh .PHONY: tools tools: GO_BUILD_FLAGS="${GO_BUILD_FLAGS} -v -mod=readonly" ./scripts/build_tools.sh # Tests GO_TEST_FLAGS?= .PHONY: test test: PASSES="unit integration release e2e" ./scripts/test.sh $(GO_TEST_FLAGS) .PHONY: test-unit test-unit: PASSES="unit" ./scripts/test.sh $(GO_TEST_FLAGS) .PHONY: test-integration test-integration: PASSES="integration" ./scripts/test.sh $(GO_TEST_FLAGS) .PHONY: test-e2e test-e2e: build PASSES="e2e" ./scripts/test.sh $(GO_TEST_FLAGS) .PHONY: test-grpcproxy-integration test-grpcproxy-integration: PASSES="grpcproxy_integration" ./scripts/test.sh $(GO_TEST_FLAGS) .PHONY: test-grpcproxy-e2e test-grpcproxy-e2e: build PASSES="grpcproxy_e2e" ./scripts/test.sh $(GO_TEST_FLAGS) .PHONY: test-e2e-release test-e2e-release: build PASSES="release e2e" ./scripts/test.sh $(GO_TEST_FLAGS) # When we release the first 3.7.0-alpha.0, we can remove `VERSION="3.7.99"` below. .PHONY: test-release test-release: PASSES="release_tests" VERSION="3.7.99" ./scripts/test.sh $(GO_TEST_FLAGS) .PHONY: test-robustness test-robustness: PASSES="robustness" ./scripts/test.sh $(GO_TEST_FLAGS) .PHONY: test-coverage test-coverage: COVERDIR=covdir PASSES="build cov" ./scripts/test.sh $(GO_TEST_FLAGS) .PHONY: upload-coverage-report upload-coverage-report: return_code=0; \ $(MAKE) test-coverage || return_code=$$?; \ COVERDIR=covdir ./scripts/codecov_upload.sh; \ exit $$return_code .PHONY: fuzz fuzz: ./scripts/fuzzing.sh # Static analysis .PHONY: verify verify: verify-bom verify-lint verify-dep verify-shellcheck verify-mod-tidy \ verify-shellws verify-proto-annotations verify-genproto verify-yamllint \ verify-markdown-marker verify-go-versions verify-gomodguard \ verify-go-workspace verify-grpc-experimental .PHONY: fix fix: fix-mod-tidy fix-bom fix-lint fix-yamllint sync-toolchain-directive \ update-go-workspace fix-shell-ws .PHONY: verify-bom verify-bom: PASSES="bom" ./scripts/test.sh .PHONY: fix-bom fix-bom: ./scripts/fix/bom.sh .PHONY: verify-dep verify-dep: PASSES="dep" ./scripts/test.sh .PHONY: verify-lint verify-lint: install-golangci-lint PASSES="lint" ./scripts/test.sh .PHONY: fix-lint fix-lint: install-golangci-lint PASSES="lint_fix" ./scripts/test.sh .PHONY: verify-shellcheck verify-shellcheck: PASSES="shellcheck" ./scripts/test.sh .PHONY: verify-mod-tidy verify-mod-tidy: PASSES="mod_tidy" ./scripts/test.sh .PHONY: fix-mod-tidy fix-mod-tidy: ./scripts/fix/mod-tidy.sh .PHONY: verify-shellws verify-shellws: PASSES="shellws" ./scripts/test.sh .PHONY: fix-shell-ws fix-shell-ws: ./scripts/fix/shell_ws.sh .PHONY: verify-proto-annotations verify-proto-annotations: PASSES="proto_annotations" ./scripts/test.sh .PHONY: verify-genproto verify-genproto: PASSES="genproto" ./scripts/test.sh .PHONY: verify-yamllint verify-yamllint: ifeq (, $(shell command -v yamllint)) @echo "Installing yamllint..." tmpdir=$$(mktemp -d); \ trap "rm -rf $$tmpdir" EXIT; \ python3 -m venv $$tmpdir; \ $$tmpdir/bin/python3 -m pip install yamllint; \ $$tmpdir/bin/yamllint --config-file tools/.yamllint . else @echo "yamllint already installed..." yamllint --config-file tools/.yamllint . endif .PHONY: verify-markdown-marker verify-markdown-marker: PASSES="markdown_marker" ./scripts/test.sh .PHONY: fix-yamllint fix-yamllint: ./scripts/fix/yamllint.sh .PHONY: run-govulncheck run-govulncheck: PASSES="govuln" ./scripts/test.sh # Tools .PHONY: install-golangci-lint install-golangci-lint: ./scripts/verify_golangci-lint_version.sh .PHONY: install-lazyfs install-lazyfs: bin/lazyfs bin/lazyfs: rm /tmp/lazyfs -rf git clone --depth 1 --branch 0.2.0 https://github.com/dsrhaslab/lazyfs /tmp/lazyfs cd /tmp/lazyfs/libs/libpcache; ./build.sh cd /tmp/lazyfs/lazyfs; ./build.sh mkdir -p ./bin cp /tmp/lazyfs/lazyfs/build/lazyfs ./bin/lazyfs # Cleanup .PHONY: clean clean: rm -f ./codecov rm -rf ./covdir rm -f ./bin/Dockerfile-release rm -rf ./bin/etcd* rm -rf ./bin/lazyfs rm -rf ./bin/python rm -rf ./default.etcd rm -rf ./tests/e2e/default.etcd rm -rf ./release rm -rf ./coverage/*.err ./coverage/*.out rm -rf ./tests/e2e/default.proxy rm -rf ./bin/shellcheck* find ./ -name "127.0.0.1:*" -o -name "localhost:*" -o -name "*.log" -o -name "agent-*" -o -name "*.coverprofile" -o -name "testname-proxy-*" -delete .PHONY: verify-go-versions verify-go-versions: ./scripts/verify_go_versions.sh .PHONY: verify-gomodguard verify-gomodguard: PASSES="gomodguard" ./scripts/test.sh .PHONY: verify-go-workspace verify-go-workspace: PASSES="go_workspace" ./scripts/test.sh .PHONY: verify-grpc-experimental verify-grpc-experimental: ./scripts/verify_grpc_experimental.sh .PHONY: sync-toolchain-directive sync-toolchain-directive: ./scripts/sync_go_toolchain_directive.sh .PHONY: markdown-diff-lint markdown-diff-lint: ./scripts/markdown_diff_lint.sh .PHONY: update-go-workspace update-go-workspace: ./scripts/update_go_workspace.sh ================================================ FILE: OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners approvers: - sig-etcd-chairs # Defined in OWNERS_ALIASES - sig-etcd-tech-leads # Defined in OWNERS_ALIASES - spzala # Sahdev Zala emeritus_approvers: - bdarnell # Ben Darnell - fanminshi # Fanmin Shi - gyuho # Gyuho Lee - hexfusion # Sam Batschelet - heyitsanthony # Anthony Romano - jingyih # Jingyi Hu - jpbetz # Joe Betz - mitake # Hitoshi Mitake - philips # Brandon Philips - ptabor # Piotr Tabor - wenjiaswe # Wenjia Zhang - xiang90 # Xiang Li ================================================ FILE: OWNERS_ALIASES ================================================ aliases: sig-etcd-chairs: - ivanvc # Ivan Valdes - jmhbnz # James Blair - siyuanfoundation # Siyuan Zhang sig-etcd-tech-leads: - ahrtr # Benjamin Wang - fuweid # Wei Fu - serathius # Marek Siarkowicz ================================================ FILE: Procfile ================================================ # Use goreman to run `go install github.com/mattn/goreman@latest` # Change the path of bin/etcd if etcd is located elsewhere etcd1: bin/etcd --name infra1 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:2379 --listen-peer-urls http://127.0.0.1:12380 --initial-advertise-peer-urls http://127.0.0.1:12380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof --logger=zap --log-outputs=stderr etcd2: bin/etcd --name infra2 --listen-client-urls http://127.0.0.1:22379 --advertise-client-urls http://127.0.0.1:22379 --listen-peer-urls http://127.0.0.1:22380 --initial-advertise-peer-urls http://127.0.0.1:22380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof --logger=zap --log-outputs=stderr etcd3: bin/etcd --name infra3 --listen-client-urls http://127.0.0.1:32379 --advertise-client-urls http://127.0.0.1:32379 --listen-peer-urls http://127.0.0.1:32380 --initial-advertise-peer-urls http://127.0.0.1:32380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof --logger=zap --log-outputs=stderr #proxy: bin/etcd grpc-proxy start --endpoints=127.0.0.1:2379,127.0.0.1:22379,127.0.0.1:32379 --listen-addr=127.0.0.1:23790 --advertise-client-url=127.0.0.1:23790 --enable-pprof # A learner node can be started using the below Procfile.learner (uncomment and run) # Use goreman to run `go install github.com/mattn/goreman@latest` # 1. Start the cluster using Procfile # 2. Add learner node to the cluster # % etcdctl member add infra4 --peer-urls="http://127.0.0.1:42380" --learner=true # 3. Start learner node with goreman # Change the path of bin/etcd if etcd is located elsewhere # uncomment below to setup # etcd4: bin/etcd --name infra4 --listen-client-urls http://127.0.0.1:42379 --advertise-client-urls http://127.0.0.1:42379 --listen-peer-urls http://127.0.0.1:42380 --initial-advertise-peer-urls http://127.0.0.1:42380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra4=http://127.0.0.1:42380,infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state existing --enable-pprof --logger=zap --log-outputs=stderr # 4. The learner node can be promoted to voting member by the command # % etcdctl member promote ================================================ FILE: README.md ================================================ # etcd [![Go Report Card](https://goreportcard.com/badge/github.com/etcd-io/etcd?style=flat-square)](https://goreportcard.com/report/github.com/etcd-io/etcd) [![Coverage](https://codecov.io/gh/etcd-io/etcd/branch/main/graph/badge.svg)](https://app.codecov.io/gh/etcd-io/etcd/tree/main) [![Tests](https://github.com/etcd-io/etcd/actions/workflows/tests.yaml/badge.svg)](https://github.com/etcd-io/etcd/actions/workflows/tests.yaml) [![codeql-analysis](https://github.com/etcd-io/etcd/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/etcd-io/etcd/actions/workflows/codeql-analysis.yml) [![Docs](https://img.shields.io/badge/docs-latest-green.svg)](https://etcd.io/docs) [![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godocs.io/go.etcd.io/etcd/v3) [![Releases](https://img.shields.io/github/release/etcd-io/etcd/all.svg?style=flat-square)](https://github.com/etcd-io/etcd/releases) [![LICENSE](https://img.shields.io/github/license/etcd-io/etcd.svg?style=flat-square)](https://github.com/etcd-io/etcd/blob/main/LICENSE) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/etcd-io/etcd/badge)](https://scorecard.dev/viewer/?uri=github.com/etcd-io/etcd) **Note**: The `main` branch may be in an *unstable or even broken state* during development. For stable versions, see [releases][github-release]. etcd logo etcd is a distributed reliable key-value store for the most critical data of a distributed system, with a focus on being: * *Simple*: well-defined, user-facing API (gRPC) * *Secure*: automatic TLS with optional client cert authentication * *Fast*: benchmarked 10,000 writes/sec * *Reliable*: properly distributed using Raft etcd is written in Go and uses the [Raft][] consensus algorithm to manage a highly-available replicated log. etcd is used [in production by many companies](./ADOPTERS.md), and the development team stands behind it in critical deployment scenarios, where etcd is frequently teamed with applications such as [Kubernetes][k8s], [locksmith][], [vulcand][], [Doorman][], and many others. Reliability is further ensured by rigorous [**robustness testing**](https://github.com/etcd-io/etcd/tree/main/tests/robustness). See [etcdctl][etcdctl] for a simple command line client. ![etcd reliability is important](logos/etcd-xkcd-2347.png) Original image credited to xkcd.com/2347, alterations by Josh Berkus. [raft]: https://raft.github.io/ [k8s]: http://kubernetes.io/ [doorman]: https://github.com/youtube/doorman [locksmith]: https://github.com/coreos/locksmith [vulcand]: https://github.com/vulcand/vulcand [etcdctl]: https://github.com/etcd-io/etcd/tree/main/etcdctl ## Documentation The most common API documentation you'll need can be found here: * [go.etcd.io/etcd/api/v3](https://godocs.io/go.etcd.io/etcd/api/v3) * [go.etcd.io/etcd/client/pkg/v3](https://godocs.io/go.etcd.io/etcd/client/pkg/v3) * [go.etcd.io/etcd/client/v3](https://godocs.io/go.etcd.io/etcd/client/v3) * [go.etcd.io/etcd/etcdctl/v3](https://godocs.io/go.etcd.io/etcd/etcdctl/v3) * [go.etcd.io/etcd/pkg/v3](https://godocs.io/go.etcd.io/etcd/pkg/v3) * [go.etcd.io/etcd/raft/v3](https://godocs.io/go.etcd.io/etcd/raft/v3) * [go.etcd.io/etcd/server/v3](https://godocs.io/go.etcd.io/etcd/server/v3) ## Maintainers [Maintainers](OWNERS) strive to shape an inclusive open source project culture where users are heard and contributors feel respected and empowered. Maintainers aim to build productive relationships across different companies and disciplines. Read more about [Maintainers role and responsibilities](Documentation/contributor-guide/community-membership.md#maintainers). ## Getting started ### Getting etcd The easiest way to get etcd is to use one of the pre-built release binaries which are available for OSX, Linux, Windows, and Docker on the [release page][github-release]. For more installation guides, please check out [play.etcd.io](http://play.etcd.io) and [operating etcd](https://etcd.io/docs/latest/op-guide). [github-release]: https://github.com/etcd-io/etcd/releases ### Running etcd First start a single-member cluster of etcd. If etcd is installed using the [pre-built release binaries][github-release], run it from the installation location as below: ```bash /tmp/etcd-download-test/etcd ``` The etcd command can be simply run as such if it is moved to the system path as below: ```bash mv /tmp/etcd-download-test/etcd /usr/local/bin/ etcd ``` This will bring up etcd listening on port 2379 for client communication and on port 2380 for server-to-server communication. Next, let's set a single key, and then retrieve it: ```bash etcdctl put mykey "this is awesome" etcdctl get mykey ``` etcd is now running and serving client requests. For more, please check out: * [Interactive etcd playground](http://play.etcd.io) * [Animated quick demo](https://etcd.io/docs/latest/demo) ### etcd TCP ports The [official etcd ports][iana-ports] are 2379 for client requests, and 2380 for peer communication. [iana-ports]: http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.txt ### Running a local etcd cluster First install [goreman](https://github.com/mattn/goreman), which manages Procfile-based applications. Our [Procfile script](./Procfile) will set up a local example cluster. Start it with: ```bash goreman start ``` This will bring up 3 etcd members `infra1`, `infra2` and `infra3` and optionally etcd `grpc-proxy`, which runs locally and composes a cluster. Every cluster member and proxy accepts key value reads and key value writes. Follow the comments in [Procfile script](./Procfile) to add a learner node to the cluster. ### Install etcd client v3 ```bash go get go.etcd.io/etcd/client/v3 ``` ### Next steps Now it's time to dig into the full etcd API and other guides. * Read the full [documentation]. * Review etcd [frequently asked questions]. * Explore the full gRPC [API]. * Set up a [multi-machine cluster][clustering]. * Learn the [config format, env variables and flags][configuration]. * Find [language bindings and tools][integrations]. * Use TLS to [secure an etcd cluster][security]. * [Tune etcd][tuning]. [documentation]: https://etcd.io/docs/latest [api]: https://etcd.io/docs/latest/learning/api [clustering]: https://etcd.io/docs/latest/op-guide/clustering [configuration]: https://etcd.io/docs/latest/op-guide/configuration [integrations]: https://etcd.io/docs/latest/integrations [security]: https://etcd.io/docs/latest/op-guide/security [tuning]: https://etcd.io/docs/latest/tuning ## Contact * Email: [etcd-dev](https://groups.google.com/g/etcd-dev) * Slack: [#sig-etcd](https://kubernetes.slack.com/archives/C3HD8ARJ5) channel on Kubernetes ([get an invite](http://slack.kubernetes.io/)) * [Community meetings](#community-meetings) ### Community meetings etcd contributors and maintainers meet every week at `11:00` AM (USA Pacific) on Thursday and meetings alternate between community meetings and issue triage meetings. Meeting agendas are recorded in a [shared Google doc][shared-meeting-notes] and everyone is welcome to suggest additional topics or other agendas. Issue triage meetings are aimed at getting through our backlog of PRs and Issues. Triage meetings are open to any contributor; you don't have to be a reviewer or approver to help out! They can also be a good way to get started contributing. The meeting lead role is rotated for each meeting between etcd maintainers or sig-etcd leads and is recorded in a [shared Google sheet][shared-rotation-sheet]. Meeting recordings are uploaded to the official etcd [YouTube channel]. Get calendar invitations by joining [etcd-dev](https://groups.google.com/g/etcd-dev) mailing group. Join the CNCF-funded Zoom channel: [zoom.us/my/cncfetcdproject](https://zoom.us/my/cncfetcdproject) [shared-meeting-notes]: https://docs.google.com/document/d/16XEGyPBisZvmmoIHSZzv__LoyOeluC5a4x353CX0SIM/edit [shared-rotation-sheet]: https://docs.google.com/spreadsheets/d/1jodHIO7Dk2VWTs1IRnfMFaRktS9IH8XRyifOnPdSY8I/edit [YouTube channel]: https://www.youtube.com/@etcdio ## Contributing See [CONTRIBUTING](CONTRIBUTING.md) for details on setting up your development environment, submitting patches and the contribution workflow. Please refer to [community-membership.md](Documentation/contributor-guide/community-membership.md#member) for information on becoming an etcd project member. We welcome and look forward to your contributions to the project! Please also refer to [roadmap](Documentation/contributor-guide/roadmap.md) to get more details on the priorities for the next few major or minor releases. ## Reporting bugs See [reporting bugs](https://github.com/etcd-io/etcd/blob/main/Documentation/contributor-guide/reporting_bugs.md) for details about reporting any issues. Before opening an issue please check it is not covered in our [frequently asked questions]. [frequently asked questions]: https://etcd.io/docs/latest/faq ## Reporting a security vulnerability See [security disclosure and release process](security/README.md) for details on how to report a security vulnerability and how the etcd team manages it. ## Issue and PR management See [issue triage guidelines](https://github.com/etcd-io/etcd/blob/main/Documentation/contributor-guide/triage_issues.md) for details on how issues are managed. See [PR management](https://github.com/etcd-io/etcd/blob/main/Documentation/contributor-guide/triage_prs.md) for guidelines on how pull requests are managed. ## etcd Emeritus Maintainers etcd [emeritus maintainers](OWNERS) dedicated a part of their career to etcd and reviewed code, triaged bugs and pushed the project forward over a substantial period of time. Their contribution is greatly appreciated. ### License etcd is under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details. ================================================ FILE: api/.gomodguard.yaml ================================================ --- blocked: modules: - go.etcd.io/etcd: reason: "Forbidden dependency" - go.etcd.io/etcd/api/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/pkg/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/server/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/tests/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/v3: reason: "Forbidden dependency" ================================================ FILE: api/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: api/authpb/auth.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: auth.proto package authpb import ( fmt "fmt" io "io" math "math" math_bits "math/bits" proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Permission_Type int32 const ( Permission_READ Permission_Type = 0 Permission_WRITE Permission_Type = 1 Permission_READWRITE Permission_Type = 2 ) var Permission_Type_name = map[int32]string{ 0: "READ", 1: "WRITE", 2: "READWRITE", } var Permission_Type_value = map[string]int32{ "READ": 0, "WRITE": 1, "READWRITE": 2, } func (x Permission_Type) String() string { return proto.EnumName(Permission_Type_name, int32(x)) } func (Permission_Type) EnumDescriptor() ([]byte, []int) { return fileDescriptor_8bbd6f3875b0e874, []int{2, 0} } type UserAddOptions struct { NoPassword bool `protobuf:"varint,1,opt,name=no_password,json=noPassword,proto3" json:"no_password,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *UserAddOptions) Reset() { *m = UserAddOptions{} } func (m *UserAddOptions) String() string { return proto.CompactTextString(m) } func (*UserAddOptions) ProtoMessage() {} func (*UserAddOptions) Descriptor() ([]byte, []int) { return fileDescriptor_8bbd6f3875b0e874, []int{0} } func (m *UserAddOptions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *UserAddOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_UserAddOptions.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *UserAddOptions) XXX_Merge(src proto.Message) { xxx_messageInfo_UserAddOptions.Merge(m, src) } func (m *UserAddOptions) XXX_Size() int { return m.Size() } func (m *UserAddOptions) XXX_DiscardUnknown() { xxx_messageInfo_UserAddOptions.DiscardUnknown(m) } var xxx_messageInfo_UserAddOptions proto.InternalMessageInfo func (m *UserAddOptions) GetNoPassword() bool { if m != nil { return m.NoPassword } return false } // User is a single entry in the bucket authUsers type User struct { Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Password []byte `protobuf:"bytes,2,opt,name=password,proto3" json:"password,omitempty"` Roles []string `protobuf:"bytes,3,rep,name=roles,proto3" json:"roles,omitempty"` Options *UserAddOptions `protobuf:"bytes,4,opt,name=options,proto3" json:"options,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *User) Reset() { *m = User{} } func (m *User) String() string { return proto.CompactTextString(m) } func (*User) ProtoMessage() {} func (*User) Descriptor() ([]byte, []int) { return fileDescriptor_8bbd6f3875b0e874, []int{1} } func (m *User) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *User) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_User.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *User) XXX_Merge(src proto.Message) { xxx_messageInfo_User.Merge(m, src) } func (m *User) XXX_Size() int { return m.Size() } func (m *User) XXX_DiscardUnknown() { xxx_messageInfo_User.DiscardUnknown(m) } var xxx_messageInfo_User proto.InternalMessageInfo func (m *User) GetName() []byte { if m != nil { return m.Name } return nil } func (m *User) GetPassword() []byte { if m != nil { return m.Password } return nil } func (m *User) GetRoles() []string { if m != nil { return m.Roles } return nil } func (m *User) GetOptions() *UserAddOptions { if m != nil { return m.Options } return nil } // Permission is a single entity type Permission struct { PermType Permission_Type `protobuf:"varint,1,opt,name=permType,proto3,enum=authpb.Permission_Type" json:"permType,omitempty"` Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` RangeEnd []byte `protobuf:"bytes,3,opt,name=range_end,json=rangeEnd,proto3" json:"range_end,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Permission) Reset() { *m = Permission{} } func (m *Permission) String() string { return proto.CompactTextString(m) } func (*Permission) ProtoMessage() {} func (*Permission) Descriptor() ([]byte, []int) { return fileDescriptor_8bbd6f3875b0e874, []int{2} } func (m *Permission) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Permission) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Permission.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Permission) XXX_Merge(src proto.Message) { xxx_messageInfo_Permission.Merge(m, src) } func (m *Permission) XXX_Size() int { return m.Size() } func (m *Permission) XXX_DiscardUnknown() { xxx_messageInfo_Permission.DiscardUnknown(m) } var xxx_messageInfo_Permission proto.InternalMessageInfo func (m *Permission) GetPermType() Permission_Type { if m != nil { return m.PermType } return Permission_READ } func (m *Permission) GetKey() []byte { if m != nil { return m.Key } return nil } func (m *Permission) GetRangeEnd() []byte { if m != nil { return m.RangeEnd } return nil } // Role is a single entry in the bucket authRoles type Role struct { Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` KeyPermission []*Permission `protobuf:"bytes,2,rep,name=keyPermission,proto3" json:"keyPermission,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Role) Reset() { *m = Role{} } func (m *Role) String() string { return proto.CompactTextString(m) } func (*Role) ProtoMessage() {} func (*Role) Descriptor() ([]byte, []int) { return fileDescriptor_8bbd6f3875b0e874, []int{3} } func (m *Role) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Role) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Role.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Role) XXX_Merge(src proto.Message) { xxx_messageInfo_Role.Merge(m, src) } func (m *Role) XXX_Size() int { return m.Size() } func (m *Role) XXX_DiscardUnknown() { xxx_messageInfo_Role.DiscardUnknown(m) } var xxx_messageInfo_Role proto.InternalMessageInfo func (m *Role) GetName() []byte { if m != nil { return m.Name } return nil } func (m *Role) GetKeyPermission() []*Permission { if m != nil { return m.KeyPermission } return nil } func init() { proto.RegisterEnum("authpb.Permission_Type", Permission_Type_name, Permission_Type_value) proto.RegisterType((*UserAddOptions)(nil), "authpb.UserAddOptions") proto.RegisterType((*User)(nil), "authpb.User") proto.RegisterType((*Permission)(nil), "authpb.Permission") proto.RegisterType((*Role)(nil), "authpb.Role") } func init() { proto.RegisterFile("auth.proto", fileDescriptor_8bbd6f3875b0e874) } var fileDescriptor_8bbd6f3875b0e874 = []byte{ // 342 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xcf, 0x4e, 0xf2, 0x40, 0x14, 0xc5, 0x19, 0x5a, 0xf8, 0xda, 0xcb, 0x07, 0x21, 0x37, 0x46, 0x1b, 0x8d, 0xb5, 0xe9, 0xaa, 0x71, 0xd1, 0x2a, 0x6c, 0xdc, 0x62, 0x64, 0xe1, 0x4a, 0x32, 0xc1, 0x98, 0xb8, 0x21, 0xc5, 0x4e, 0xb0, 0x01, 0x66, 0x9a, 0x99, 0xaa, 0x61, 0xe3, 0x73, 0xb8, 0xf0, 0x81, 0x5c, 0xfa, 0x08, 0x06, 0x5f, 0xc4, 0xb4, 0xc3, 0x9f, 0x10, 0x5d, 0xf5, 0x9e, 0xd3, 0x73, 0xee, 0xfc, 0x32, 0x03, 0x10, 0x3f, 0xe5, 0x8f, 0x61, 0x26, 0x45, 0x2e, 0xb0, 0x5e, 0xcc, 0xd9, 0xd8, 0x3f, 0x87, 0xd6, 0xad, 0x62, 0xb2, 0x97, 0x24, 0x37, 0x59, 0x9e, 0x0a, 0xae, 0xf0, 0x04, 0x1a, 0x5c, 0x8c, 0xb2, 0x58, 0xa9, 0x17, 0x21, 0x13, 0x87, 0x78, 0x24, 0xb0, 0x28, 0x70, 0x31, 0x58, 0x39, 0xfe, 0x2b, 0x98, 0x45, 0x05, 0x11, 0x4c, 0x1e, 0xcf, 0x59, 0x99, 0xf8, 0x4f, 0xcb, 0x19, 0x0f, 0xc1, 0xda, 0x34, 0xab, 0xa5, 0xbf, 0xd1, 0xb8, 0x07, 0x35, 0x29, 0x66, 0x4c, 0x39, 0x86, 0x67, 0x04, 0x36, 0xd5, 0x02, 0xcf, 0xe0, 0x9f, 0xd0, 0x27, 0x3b, 0xa6, 0x47, 0x82, 0x46, 0x67, 0x3f, 0xd4, 0x68, 0xe1, 0x2e, 0x17, 0x5d, 0xc7, 0xfc, 0x77, 0x02, 0x30, 0x60, 0x72, 0x9e, 0x2a, 0x95, 0x0a, 0x8e, 0x5d, 0xb0, 0x32, 0x26, 0xe7, 0xc3, 0x45, 0xa6, 0x51, 0x5a, 0x9d, 0x83, 0xf5, 0x86, 0x6d, 0x2a, 0x2c, 0x7e, 0xd3, 0x4d, 0x10, 0xdb, 0x60, 0x4c, 0xd9, 0x62, 0x85, 0x58, 0x8c, 0x78, 0x04, 0xb6, 0x8c, 0xf9, 0x84, 0x8d, 0x18, 0x4f, 0x1c, 0x43, 0xa3, 0x97, 0x46, 0x9f, 0x27, 0xfe, 0x29, 0x98, 0x65, 0xcd, 0x02, 0x93, 0xf6, 0x7b, 0x57, 0xed, 0x0a, 0xda, 0x50, 0xbb, 0xa3, 0xd7, 0xc3, 0x7e, 0x9b, 0x60, 0x13, 0xec, 0xc2, 0xd4, 0xb2, 0xea, 0x0f, 0xc1, 0xa4, 0x62, 0xc6, 0xfe, 0xbc, 0x9e, 0x0b, 0x68, 0x4e, 0xd9, 0x62, 0x8b, 0xe5, 0x54, 0x3d, 0x23, 0x68, 0x74, 0xf0, 0x37, 0x30, 0xdd, 0x0d, 0x5e, 0x46, 0x1f, 0x4b, 0x97, 0x7c, 0x2e, 0x5d, 0xf2, 0xb5, 0x74, 0xc9, 0xdb, 0xb7, 0x5b, 0xb9, 0x3f, 0x9e, 0x88, 0x90, 0xe5, 0x0f, 0x49, 0x98, 0x8a, 0xa8, 0xf8, 0x46, 0x71, 0x96, 0x46, 0xcf, 0xdd, 0x48, 0xaf, 0x1a, 0xd7, 0xcb, 0x77, 0xee, 0xfe, 0x04, 0x00, 0x00, 0xff, 0xff, 0xc0, 0x1b, 0x2e, 0xdd, 0xf5, 0x01, 0x00, 0x00, } func (m *UserAddOptions) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *UserAddOptions) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *UserAddOptions) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.NoPassword { i-- if m.NoPassword { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *User) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *User) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *User) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Options != nil { { size, err := m.Options.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintAuth(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x22 } if len(m.Roles) > 0 { for iNdEx := len(m.Roles) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Roles[iNdEx]) copy(dAtA[i:], m.Roles[iNdEx]) i = encodeVarintAuth(dAtA, i, uint64(len(m.Roles[iNdEx]))) i-- dAtA[i] = 0x1a } } if len(m.Password) > 0 { i -= len(m.Password) copy(dAtA[i:], m.Password) i = encodeVarintAuth(dAtA, i, uint64(len(m.Password))) i-- dAtA[i] = 0x12 } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintAuth(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *Permission) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Permission) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Permission) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.RangeEnd) > 0 { i -= len(m.RangeEnd) copy(dAtA[i:], m.RangeEnd) i = encodeVarintAuth(dAtA, i, uint64(len(m.RangeEnd))) i-- dAtA[i] = 0x1a } if len(m.Key) > 0 { i -= len(m.Key) copy(dAtA[i:], m.Key) i = encodeVarintAuth(dAtA, i, uint64(len(m.Key))) i-- dAtA[i] = 0x12 } if m.PermType != 0 { i = encodeVarintAuth(dAtA, i, uint64(m.PermType)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *Role) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Role) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Role) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.KeyPermission) > 0 { for iNdEx := len(m.KeyPermission) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.KeyPermission[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintAuth(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintAuth(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func encodeVarintAuth(dAtA []byte, offset int, v uint64) int { offset -= sovAuth(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) return base } func (m *UserAddOptions) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.NoPassword { n += 2 } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *User) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovAuth(uint64(l)) } l = len(m.Password) if l > 0 { n += 1 + l + sovAuth(uint64(l)) } if len(m.Roles) > 0 { for _, s := range m.Roles { l = len(s) n += 1 + l + sovAuth(uint64(l)) } } if m.Options != nil { l = m.Options.Size() n += 1 + l + sovAuth(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *Permission) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.PermType != 0 { n += 1 + sovAuth(uint64(m.PermType)) } l = len(m.Key) if l > 0 { n += 1 + l + sovAuth(uint64(l)) } l = len(m.RangeEnd) if l > 0 { n += 1 + l + sovAuth(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *Role) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovAuth(uint64(l)) } if len(m.KeyPermission) > 0 { for _, e := range m.KeyPermission { l = e.Size() n += 1 + l + sovAuth(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func sovAuth(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } func sozAuth(x uint64) (n int) { return sovAuth(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (m *UserAddOptions) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: UserAddOptions: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: UserAddOptions: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field NoPassword", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.NoPassword = bool(v != 0) default: iNdEx = preIndex skippy, err := skipAuth(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthAuth } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *User) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: User: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: User: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthAuth } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthAuth } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) if m.Name == nil { m.Name = []byte{} } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Password", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthAuth } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthAuth } if postIndex > l { return io.ErrUnexpectedEOF } m.Password = append(m.Password[:0], dAtA[iNdEx:postIndex]...) if m.Password == nil { m.Password = []byte{} } iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Roles", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthAuth } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthAuth } if postIndex > l { return io.ErrUnexpectedEOF } m.Roles = append(m.Roles, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Options", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthAuth } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthAuth } if postIndex > l { return io.ErrUnexpectedEOF } if m.Options == nil { m.Options = &UserAddOptions{} } if err := m.Options.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipAuth(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthAuth } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *Permission) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: Permission: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: Permission: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field PermType", wireType) } m.PermType = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.PermType |= Permission_Type(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthAuth } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthAuth } if postIndex > l { return io.ErrUnexpectedEOF } m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) if m.Key == nil { m.Key = []byte{} } iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RangeEnd", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthAuth } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthAuth } if postIndex > l { return io.ErrUnexpectedEOF } m.RangeEnd = append(m.RangeEnd[:0], dAtA[iNdEx:postIndex]...) if m.RangeEnd == nil { m.RangeEnd = []byte{} } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipAuth(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthAuth } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *Role) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: Role: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: Role: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthAuth } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthAuth } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) if m.Name == nil { m.Name = []byte{} } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field KeyPermission", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowAuth } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthAuth } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthAuth } if postIndex > l { return io.ErrUnexpectedEOF } m.KeyPermission = append(m.KeyPermission, &Permission{}) if err := m.KeyPermission[len(m.KeyPermission)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipAuth(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthAuth } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func skipAuth(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowAuth } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } wireType := int(wire & 0x7) switch wireType { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowAuth } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } iNdEx++ if dAtA[iNdEx-1] < 0x80 { break } } case 1: iNdEx += 8 case 2: var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowAuth } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { break } } if length < 0 { return 0, ErrInvalidLengthAuth } iNdEx += length case 3: depth++ case 4: if depth == 0 { return 0, ErrUnexpectedEndOfGroupAuth } depth-- case 5: iNdEx += 4 default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { return 0, ErrInvalidLengthAuth } if depth == 0 { return iNdEx, nil } } return 0, io.ErrUnexpectedEOF } var ( ErrInvalidLengthAuth = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowAuth = fmt.Errorf("proto: integer overflow") ErrUnexpectedEndOfGroupAuth = fmt.Errorf("proto: unexpected end of group") ) ================================================ FILE: api/authpb/auth.proto ================================================ syntax = "proto3"; package authpb; option go_package = "go.etcd.io/etcd/api/v3/authpb"; message UserAddOptions { bool no_password = 1; }; // User is a single entry in the bucket authUsers message User { bytes name = 1; bytes password = 2; repeated string roles = 3; UserAddOptions options = 4; } // Permission is a single entity message Permission { enum Type { READ = 0; WRITE = 1; READWRITE = 2; } Type permType = 1; bytes key = 2; bytes range_end = 3; } // Role is a single entry in the bucket authRoles message Role { bytes name = 1; repeated Permission keyPermission = 2; } ================================================ FILE: api/authpb/deprecated.go ================================================ // Copyright 2026 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package authpb const ( // READ is an alias of Permission_READ // Deprecated: use Permission_READ instead. Will be removed in v3.8. READ = Permission_READ // WRITE is an alias of Permission_WRITE // Deprecated: use Permission_WRITE instead. Will be removed in v3.8. WRITE = Permission_WRITE // READWRITE is an alias of Permission_READWRITE // Deprecated: use Permission_READWRITE instead. Will be removed in v3.8. READWRITE = Permission_READWRITE ) ================================================ FILE: api/etcdserverpb/etcdserver.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: etcdserver.proto package etcdserverpb import ( fmt "fmt" io "io" math "math" math_bits "math/bits" proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Metadata struct { NodeID *uint64 `protobuf:"varint,1,opt,name=NodeID" json:"NodeID,omitempty"` ClusterID *uint64 `protobuf:"varint,2,opt,name=ClusterID" json:"ClusterID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Metadata) Reset() { *m = Metadata{} } func (m *Metadata) String() string { return proto.CompactTextString(m) } func (*Metadata) ProtoMessage() {} func (*Metadata) Descriptor() ([]byte, []int) { return fileDescriptor_09ffbeb3bebbce7e, []int{0} } func (m *Metadata) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Metadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Metadata.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Metadata) XXX_Merge(src proto.Message) { xxx_messageInfo_Metadata.Merge(m, src) } func (m *Metadata) XXX_Size() int { return m.Size() } func (m *Metadata) XXX_DiscardUnknown() { xxx_messageInfo_Metadata.DiscardUnknown(m) } var xxx_messageInfo_Metadata proto.InternalMessageInfo func (m *Metadata) GetNodeID() uint64 { if m != nil && m.NodeID != nil { return *m.NodeID } return 0 } func (m *Metadata) GetClusterID() uint64 { if m != nil && m.ClusterID != nil { return *m.ClusterID } return 0 } func init() { proto.RegisterType((*Metadata)(nil), "etcdserverpb.Metadata") } func init() { proto.RegisterFile("etcdserver.proto", fileDescriptor_09ffbeb3bebbce7e) } var fileDescriptor_09ffbeb3bebbce7e = []byte{ // 139 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x48, 0x2d, 0x49, 0x4e, 0x29, 0x4e, 0x2d, 0x2a, 0x4b, 0x2d, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x41, 0x88, 0x14, 0x24, 0x29, 0x39, 0x70, 0x71, 0xf8, 0xa6, 0x96, 0x24, 0xa6, 0x24, 0x96, 0x24, 0x0a, 0x89, 0x71, 0xb1, 0xf9, 0xe5, 0xa7, 0xa4, 0x7a, 0xba, 0x48, 0x30, 0x2a, 0x30, 0x6a, 0xb0, 0x04, 0x41, 0x79, 0x42, 0x32, 0x5c, 0x9c, 0xce, 0x39, 0xa5, 0xc5, 0x25, 0xa9, 0x45, 0x9e, 0x2e, 0x12, 0x4c, 0x60, 0x29, 0x84, 0x80, 0x93, 0xe9, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe3, 0xb1, 0x1c, 0x43, 0x94, 0x72, 0x7a, 0xbe, 0x1e, 0xc8, 0x12, 0xbd, 0xcc, 0x7c, 0x7d, 0x10, 0xad, 0x9f, 0x58, 0x90, 0xa9, 0x5f, 0x66, 0xac, 0x8f, 0x6c, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x5c, 0x60, 0x56, 0x96, 0x99, 0x00, 0x00, 0x00, } func (m *Metadata) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Metadata) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Metadata) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.ClusterID != nil { i = encodeVarintEtcdserver(dAtA, i, uint64(*m.ClusterID)) i-- dAtA[i] = 0x10 } if m.NodeID != nil { i = encodeVarintEtcdserver(dAtA, i, uint64(*m.NodeID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func encodeVarintEtcdserver(dAtA []byte, offset int, v uint64) int { offset -= sovEtcdserver(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) return base } func (m *Metadata) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.NodeID != nil { n += 1 + sovEtcdserver(uint64(*m.NodeID)) } if m.ClusterID != nil { n += 1 + sovEtcdserver(uint64(*m.ClusterID)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func sovEtcdserver(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } func sozEtcdserver(x uint64) (n int) { return sovEtcdserver(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (m *Metadata) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowEtcdserver } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: Metadata: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: Metadata: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field NodeID", wireType) } var v uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowEtcdserver } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= uint64(b&0x7F) << shift if b < 0x80 { break } } m.NodeID = &v case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ClusterID", wireType) } var v uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowEtcdserver } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= uint64(b&0x7F) << shift if b < 0x80 { break } } m.ClusterID = &v default: iNdEx = preIndex skippy, err := skipEtcdserver(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthEtcdserver } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func skipEtcdserver(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowEtcdserver } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } wireType := int(wire & 0x7) switch wireType { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowEtcdserver } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } iNdEx++ if dAtA[iNdEx-1] < 0x80 { break } } case 1: iNdEx += 8 case 2: var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowEtcdserver } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { break } } if length < 0 { return 0, ErrInvalidLengthEtcdserver } iNdEx += length case 3: depth++ case 4: if depth == 0 { return 0, ErrUnexpectedEndOfGroupEtcdserver } depth-- case 5: iNdEx += 4 default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { return 0, ErrInvalidLengthEtcdserver } if depth == 0 { return iNdEx, nil } } return 0, io.ErrUnexpectedEOF } var ( ErrInvalidLengthEtcdserver = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowEtcdserver = fmt.Errorf("proto: integer overflow") ErrUnexpectedEndOfGroupEtcdserver = fmt.Errorf("proto: unexpected end of group") ) ================================================ FILE: api/etcdserverpb/etcdserver.proto ================================================ syntax = "proto2"; package etcdserverpb; option go_package = "go.etcd.io/etcd/api/v3/etcdserverpb"; message Metadata { optional uint64 NodeID = 1; optional uint64 ClusterID = 2; } ================================================ FILE: api/etcdserverpb/gw/rpc.pb.gw.go ================================================ // Code generated by protoc-gen-grpc-gateway. DO NOT EDIT. // source: api/etcdserverpb/rpc.proto /* Package etcdserverpb is a reverse proxy. It translates gRPC into RESTful JSON APIs. */ package gw import ( protov1 "github.com/golang/protobuf/proto" "context" "errors" "go.etcd.io/etcd/api/v3/etcdserverpb" "io" "net/http" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/grpc-ecosystem/grpc-gateway/v2/utilities" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" ) // Suppress "imported and not used" errors var ( _ codes.Code _ io.Reader _ status.Status _ = errors.New _ = runtime.String _ = utilities.NewDoubleArray _ = metadata.Join ) func request_KV_Range_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.KVClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.RangeRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Range(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_KV_Range_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.KVServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.RangeRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Range(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_KV_Put_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.KVClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.PutRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Put(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_KV_Put_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.KVServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.PutRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Put(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_KV_DeleteRange_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.KVClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.DeleteRangeRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.DeleteRange(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_KV_DeleteRange_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.KVServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.DeleteRangeRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.DeleteRange(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_KV_Txn_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.KVClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.TxnRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Txn(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_KV_Txn_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.KVServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.TxnRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Txn(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_KV_Compact_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.KVClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.CompactionRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Compact(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_KV_Compact_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.KVServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.CompactionRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Compact(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Watch_Watch_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.WatchClient, req *http.Request, pathParams map[string]string) (etcdserverpb.Watch_WatchClient, runtime.ServerMetadata, error) { var metadata runtime.ServerMetadata stream, err := client.Watch(ctx) if err != nil { grpclog.Errorf("Failed to start streaming: %v", err) return nil, metadata, err } dec := marshaler.NewDecoder(req.Body) handleSend := func() error { var protoReq etcdserverpb.WatchRequest err := dec.Decode(protov1.MessageV2(&protoReq)) if errors.Is(err, io.EOF) { return err } if err != nil { grpclog.Errorf("Failed to decode request: %v", err) return status.Errorf(codes.InvalidArgument, "Failed to decode request: %v", err) } if err := stream.Send(&protoReq); err != nil { grpclog.Errorf("Failed to send request: %v", err) return err } return nil } go func() { for { if err := handleSend(); err != nil { break } } if err := stream.CloseSend(); err != nil { grpclog.Errorf("Failed to terminate client stream: %v", err) } }() header, err := stream.Header() if err != nil { grpclog.Errorf("Failed to get header from client: %v", err) return nil, metadata, err } metadata.HeaderMD = header return stream, metadata, nil } func request_Lease_LeaseGrant_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.LeaseClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseGrantRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.LeaseGrant(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Lease_LeaseGrant_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.LeaseServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseGrantRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.LeaseGrant(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Lease_LeaseRevoke_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.LeaseClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseRevokeRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.LeaseRevoke(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Lease_LeaseRevoke_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.LeaseServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseRevokeRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.LeaseRevoke(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Lease_LeaseRevoke_1(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.LeaseClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseRevokeRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.LeaseRevoke(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Lease_LeaseRevoke_1(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.LeaseServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseRevokeRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.LeaseRevoke(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Lease_LeaseKeepAlive_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.LeaseClient, req *http.Request, pathParams map[string]string) (etcdserverpb.Lease_LeaseKeepAliveClient, runtime.ServerMetadata, error) { var metadata runtime.ServerMetadata stream, err := client.LeaseKeepAlive(ctx) if err != nil { grpclog.Errorf("Failed to start streaming: %v", err) return nil, metadata, err } dec := marshaler.NewDecoder(req.Body) handleSend := func() error { var protoReq etcdserverpb.LeaseKeepAliveRequest err := dec.Decode(protov1.MessageV2(&protoReq)) if errors.Is(err, io.EOF) { return err } if err != nil { grpclog.Errorf("Failed to decode request: %v", err) return status.Errorf(codes.InvalidArgument, "Failed to decode request: %v", err) } if err := stream.Send(&protoReq); err != nil { grpclog.Errorf("Failed to send request: %v", err) return err } return nil } go func() { for { if err := handleSend(); err != nil { break } } if err := stream.CloseSend(); err != nil { grpclog.Errorf("Failed to terminate client stream: %v", err) } }() header, err := stream.Header() if err != nil { grpclog.Errorf("Failed to get header from client: %v", err) return nil, metadata, err } metadata.HeaderMD = header return stream, metadata, nil } func request_Lease_LeaseTimeToLive_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.LeaseClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseTimeToLiveRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.LeaseTimeToLive(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Lease_LeaseTimeToLive_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.LeaseServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseTimeToLiveRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.LeaseTimeToLive(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Lease_LeaseTimeToLive_1(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.LeaseClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseTimeToLiveRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.LeaseTimeToLive(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Lease_LeaseTimeToLive_1(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.LeaseServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseTimeToLiveRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.LeaseTimeToLive(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Lease_LeaseLeases_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.LeaseClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseLeasesRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.LeaseLeases(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Lease_LeaseLeases_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.LeaseServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseLeasesRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.LeaseLeases(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Lease_LeaseLeases_1(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.LeaseClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseLeasesRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.LeaseLeases(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Lease_LeaseLeases_1(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.LeaseServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.LeaseLeasesRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.LeaseLeases(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Cluster_MemberAdd_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.ClusterClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MemberAddRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.MemberAdd(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Cluster_MemberAdd_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.ClusterServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MemberAddRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.MemberAdd(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Cluster_MemberRemove_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.ClusterClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MemberRemoveRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.MemberRemove(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Cluster_MemberRemove_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.ClusterServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MemberRemoveRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.MemberRemove(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Cluster_MemberUpdate_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.ClusterClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MemberUpdateRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.MemberUpdate(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Cluster_MemberUpdate_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.ClusterServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MemberUpdateRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.MemberUpdate(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Cluster_MemberList_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.ClusterClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MemberListRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.MemberList(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Cluster_MemberList_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.ClusterServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MemberListRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.MemberList(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Cluster_MemberPromote_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.ClusterClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MemberPromoteRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.MemberPromote(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Cluster_MemberPromote_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.ClusterServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MemberPromoteRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.MemberPromote(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Maintenance_Alarm_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.MaintenanceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AlarmRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Alarm(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Maintenance_Alarm_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.MaintenanceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AlarmRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Alarm(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Maintenance_Status_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.MaintenanceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.StatusRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Status(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Maintenance_Status_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.MaintenanceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.StatusRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Status(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Maintenance_Defragment_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.MaintenanceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.DefragmentRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Defragment(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Maintenance_Defragment_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.MaintenanceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.DefragmentRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Defragment(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Maintenance_Hash_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.MaintenanceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.HashRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Hash(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Maintenance_Hash_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.MaintenanceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.HashRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Hash(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Maintenance_HashKV_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.MaintenanceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.HashKVRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.HashKV(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Maintenance_HashKV_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.MaintenanceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.HashKVRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.HashKV(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Maintenance_Snapshot_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.MaintenanceClient, req *http.Request, pathParams map[string]string) (etcdserverpb.Maintenance_SnapshotClient, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.SnapshotRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } stream, err := client.Snapshot(ctx, &protoReq) if err != nil { return nil, metadata, err } header, err := stream.Header() if err != nil { return nil, metadata, err } metadata.HeaderMD = header return stream, metadata, nil } func request_Maintenance_MoveLeader_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.MaintenanceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MoveLeaderRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.MoveLeader(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Maintenance_MoveLeader_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.MaintenanceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.MoveLeaderRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.MoveLeader(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Maintenance_Downgrade_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.MaintenanceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.DowngradeRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Downgrade(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Maintenance_Downgrade_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.MaintenanceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.DowngradeRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Downgrade(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_AuthEnable_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthEnableRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.AuthEnable(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_AuthEnable_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthEnableRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.AuthEnable(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_AuthDisable_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthDisableRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.AuthDisable(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_AuthDisable_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthDisableRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.AuthDisable(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_AuthStatus_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthStatusRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.AuthStatus(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_AuthStatus_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthStatusRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.AuthStatus(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_Authenticate_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthenticateRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Authenticate(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_Authenticate_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthenticateRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Authenticate(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_UserAdd_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserAddRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.UserAdd(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_UserAdd_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserAddRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.UserAdd(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_UserGet_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserGetRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.UserGet(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_UserGet_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserGetRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.UserGet(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_UserList_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserListRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.UserList(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_UserList_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserListRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.UserList(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_UserDelete_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserDeleteRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.UserDelete(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_UserDelete_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserDeleteRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.UserDelete(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_UserChangePassword_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserChangePasswordRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.UserChangePassword(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_UserChangePassword_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserChangePasswordRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.UserChangePassword(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_UserGrantRole_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserGrantRoleRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.UserGrantRole(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_UserGrantRole_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserGrantRoleRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.UserGrantRole(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_UserRevokeRole_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserRevokeRoleRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.UserRevokeRole(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_UserRevokeRole_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthUserRevokeRoleRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.UserRevokeRole(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_RoleAdd_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleAddRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.RoleAdd(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_RoleAdd_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleAddRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.RoleAdd(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_RoleGet_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleGetRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.RoleGet(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_RoleGet_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleGetRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.RoleGet(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_RoleList_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleListRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.RoleList(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_RoleList_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleListRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.RoleList(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_RoleDelete_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleDeleteRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.RoleDelete(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_RoleDelete_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleDeleteRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.RoleDelete(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_RoleGrantPermission_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleGrantPermissionRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.RoleGrantPermission(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_RoleGrantPermission_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleGrantPermissionRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.RoleGrantPermission(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Auth_RoleRevokePermission_0(ctx context.Context, marshaler runtime.Marshaler, client etcdserverpb.AuthClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleRevokePermissionRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.RoleRevokePermission(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Auth_RoleRevokePermission_0(ctx context.Context, marshaler runtime.Marshaler, server etcdserverpb.AuthServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq etcdserverpb.AuthRoleRevokePermissionRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.RoleRevokePermission(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } // etcdserverpb.RegisterKVHandlerServer registers the http handlers for service KV to "mux". // UnaryRPC :call etcdserverpb.KVServer directly. // StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906. // Note that using this registration option will cause many gRPC library features to stop working. Consider using RegisterKVHandlerFromEndpoint instead. // GRPC interceptors will not work for this type of registration. To use interceptors, you must use the "runtime.WithMiddlewares" option in the "runtime.NewServeMux" call. func RegisterKVHandlerServer(ctx context.Context, mux *runtime.ServeMux, server etcdserverpb.KVServer) error { mux.Handle(http.MethodPost, pattern_KV_Range_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.KV/Range", runtime.WithHTTPPathPattern("/v3/kv/range")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_KV_Range_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_KV_Range_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_KV_Put_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.KV/Put", runtime.WithHTTPPathPattern("/v3/kv/put")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_KV_Put_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_KV_Put_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_KV_DeleteRange_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.KV/DeleteRange", runtime.WithHTTPPathPattern("/v3/kv/deleterange")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_KV_DeleteRange_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_KV_DeleteRange_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_KV_Txn_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.KV/Txn", runtime.WithHTTPPathPattern("/v3/kv/txn")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_KV_Txn_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_KV_Txn_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_KV_Compact_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.KV/Compact", runtime.WithHTTPPathPattern("/v3/kv/compaction")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_KV_Compact_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_KV_Compact_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } // etcdserverpb.RegisterWatchHandlerServer registers the http handlers for service Watch to "mux". // UnaryRPC :call etcdserverpb.WatchServer directly. // StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906. // Note that using this registration option will cause many gRPC library features to stop working. Consider using RegisterWatchHandlerFromEndpoint instead. // GRPC interceptors will not work for this type of registration. To use interceptors, you must use the "runtime.WithMiddlewares" option in the "runtime.NewServeMux" call. func RegisterWatchHandlerServer(ctx context.Context, mux *runtime.ServeMux, server etcdserverpb.WatchServer) error { mux.Handle(http.MethodPost, pattern_Watch_Watch_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { err := status.Error(codes.Unimplemented, "streaming calls are not yet supported in the in-process transport") _, outboundMarshaler := runtime.MarshalerForRequest(mux, req) runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return }) return nil } // etcdserverpb.RegisterLeaseHandlerServer registers the http handlers for service Lease to "mux". // UnaryRPC :call etcdserverpb.LeaseServer directly. // StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906. // Note that using this registration option will cause many gRPC library features to stop working. Consider using RegisterLeaseHandlerFromEndpoint instead. // GRPC interceptors will not work for this type of registration. To use interceptors, you must use the "runtime.WithMiddlewares" option in the "runtime.NewServeMux" call. func RegisterLeaseHandlerServer(ctx context.Context, mux *runtime.ServeMux, server etcdserverpb.LeaseServer) error { mux.Handle(http.MethodPost, pattern_Lease_LeaseGrant_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseGrant", runtime.WithHTTPPathPattern("/v3/lease/grant")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Lease_LeaseGrant_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseGrant_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseRevoke_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseRevoke", runtime.WithHTTPPathPattern("/v3/lease/revoke")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Lease_LeaseRevoke_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseRevoke_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseRevoke_1, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseRevoke", runtime.WithHTTPPathPattern("/v3/kv/lease/revoke")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Lease_LeaseRevoke_1(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseRevoke_1(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseKeepAlive_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { err := status.Error(codes.Unimplemented, "streaming calls are not yet supported in the in-process transport") _, outboundMarshaler := runtime.MarshalerForRequest(mux, req) runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return }) mux.Handle(http.MethodPost, pattern_Lease_LeaseTimeToLive_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseTimeToLive", runtime.WithHTTPPathPattern("/v3/lease/timetolive")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Lease_LeaseTimeToLive_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseTimeToLive_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseTimeToLive_1, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseTimeToLive", runtime.WithHTTPPathPattern("/v3/kv/lease/timetolive")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Lease_LeaseTimeToLive_1(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseTimeToLive_1(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseLeases_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseLeases", runtime.WithHTTPPathPattern("/v3/lease/leases")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Lease_LeaseLeases_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseLeases_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseLeases_1, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseLeases", runtime.WithHTTPPathPattern("/v3/kv/lease/leases")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Lease_LeaseLeases_1(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseLeases_1(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } // etcdserverpb.RegisterClusterHandlerServer registers the http handlers for service Cluster to "mux". // UnaryRPC :call etcdserverpb.ClusterServer directly. // StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906. // Note that using this registration option will cause many gRPC library features to stop working. Consider using RegisterClusterHandlerFromEndpoint instead. // GRPC interceptors will not work for this type of registration. To use interceptors, you must use the "runtime.WithMiddlewares" option in the "runtime.NewServeMux" call. func RegisterClusterHandlerServer(ctx context.Context, mux *runtime.ServeMux, server etcdserverpb.ClusterServer) error { mux.Handle(http.MethodPost, pattern_Cluster_MemberAdd_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Cluster/MemberAdd", runtime.WithHTTPPathPattern("/v3/cluster/member/add")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Cluster_MemberAdd_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Cluster_MemberAdd_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Cluster_MemberRemove_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Cluster/MemberRemove", runtime.WithHTTPPathPattern("/v3/cluster/member/remove")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Cluster_MemberRemove_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Cluster_MemberRemove_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Cluster_MemberUpdate_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Cluster/MemberUpdate", runtime.WithHTTPPathPattern("/v3/cluster/member/update")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Cluster_MemberUpdate_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Cluster_MemberUpdate_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Cluster_MemberList_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Cluster/MemberList", runtime.WithHTTPPathPattern("/v3/cluster/member/list")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Cluster_MemberList_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Cluster_MemberList_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Cluster_MemberPromote_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Cluster/MemberPromote", runtime.WithHTTPPathPattern("/v3/cluster/member/promote")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Cluster_MemberPromote_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Cluster_MemberPromote_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } // etcdserverpb.RegisterMaintenanceHandlerServer registers the http handlers for service Maintenance to "mux". // UnaryRPC :call etcdserverpb.MaintenanceServer directly. // StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906. // Note that using this registration option will cause many gRPC library features to stop working. Consider using RegisterMaintenanceHandlerFromEndpoint instead. // GRPC interceptors will not work for this type of registration. To use interceptors, you must use the "runtime.WithMiddlewares" option in the "runtime.NewServeMux" call. func RegisterMaintenanceHandlerServer(ctx context.Context, mux *runtime.ServeMux, server etcdserverpb.MaintenanceServer) error { mux.Handle(http.MethodPost, pattern_Maintenance_Alarm_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Maintenance/Alarm", runtime.WithHTTPPathPattern("/v3/maintenance/alarm")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Maintenance_Alarm_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_Alarm_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_Status_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Maintenance/Status", runtime.WithHTTPPathPattern("/v3/maintenance/status")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Maintenance_Status_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_Status_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_Defragment_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Maintenance/Defragment", runtime.WithHTTPPathPattern("/v3/maintenance/defragment")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Maintenance_Defragment_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_Defragment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_Hash_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Maintenance/Hash", runtime.WithHTTPPathPattern("/v3/maintenance/hash")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Maintenance_Hash_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_Hash_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_HashKV_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Maintenance/HashKV", runtime.WithHTTPPathPattern("/v3/maintenance/hashkv")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Maintenance_HashKV_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_HashKV_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_Snapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { err := status.Error(codes.Unimplemented, "streaming calls are not yet supported in the in-process transport") _, outboundMarshaler := runtime.MarshalerForRequest(mux, req) runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return }) mux.Handle(http.MethodPost, pattern_Maintenance_MoveLeader_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Maintenance/MoveLeader", runtime.WithHTTPPathPattern("/v3/maintenance/transfer-leadership")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Maintenance_MoveLeader_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_MoveLeader_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_Downgrade_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Maintenance/Downgrade", runtime.WithHTTPPathPattern("/v3/maintenance/downgrade")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Maintenance_Downgrade_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_Downgrade_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } // etcdserverpb.RegisterAuthHandlerServer registers the http handlers for service Auth to "mux". // UnaryRPC :call etcdserverpb.AuthServer directly. // StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906. // Note that using this registration option will cause many gRPC library features to stop working. Consider using RegisterAuthHandlerFromEndpoint instead. // GRPC interceptors will not work for this type of registration. To use interceptors, you must use the "runtime.WithMiddlewares" option in the "runtime.NewServeMux" call. func RegisterAuthHandlerServer(ctx context.Context, mux *runtime.ServeMux, server etcdserverpb.AuthServer) error { mux.Handle(http.MethodPost, pattern_Auth_AuthEnable_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/AuthEnable", runtime.WithHTTPPathPattern("/v3/auth/enable")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_AuthEnable_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_AuthEnable_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_AuthDisable_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/AuthDisable", runtime.WithHTTPPathPattern("/v3/auth/disable")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_AuthDisable_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_AuthDisable_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_AuthStatus_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/AuthStatus", runtime.WithHTTPPathPattern("/v3/auth/status")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_AuthStatus_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_AuthStatus_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_Authenticate_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/Authenticate", runtime.WithHTTPPathPattern("/v3/auth/authenticate")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_Authenticate_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_Authenticate_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserAdd_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/UserAdd", runtime.WithHTTPPathPattern("/v3/auth/user/add")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_UserAdd_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserAdd_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserGet_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/UserGet", runtime.WithHTTPPathPattern("/v3/auth/user/get")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_UserGet_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserGet_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserList_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/UserList", runtime.WithHTTPPathPattern("/v3/auth/user/list")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_UserList_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserList_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserDelete_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/UserDelete", runtime.WithHTTPPathPattern("/v3/auth/user/delete")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_UserDelete_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserDelete_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserChangePassword_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/UserChangePassword", runtime.WithHTTPPathPattern("/v3/auth/user/changepw")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_UserChangePassword_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserChangePassword_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserGrantRole_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/UserGrantRole", runtime.WithHTTPPathPattern("/v3/auth/user/grant")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_UserGrantRole_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserGrantRole_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserRevokeRole_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/UserRevokeRole", runtime.WithHTTPPathPattern("/v3/auth/user/revoke")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_UserRevokeRole_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserRevokeRole_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleAdd_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/RoleAdd", runtime.WithHTTPPathPattern("/v3/auth/role/add")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_RoleAdd_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleAdd_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleGet_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/RoleGet", runtime.WithHTTPPathPattern("/v3/auth/role/get")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_RoleGet_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleGet_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleList_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/RoleList", runtime.WithHTTPPathPattern("/v3/auth/role/list")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_RoleList_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleList_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleDelete_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/RoleDelete", runtime.WithHTTPPathPattern("/v3/auth/role/delete")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_RoleDelete_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleDelete_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleGrantPermission_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/RoleGrantPermission", runtime.WithHTTPPathPattern("/v3/auth/role/grant")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_RoleGrantPermission_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleGrantPermission_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleRevokePermission_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/etcdserverpb.Auth/RoleRevokePermission", runtime.WithHTTPPathPattern("/v3/auth/role/revoke")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Auth_RoleRevokePermission_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleRevokePermission_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } // RegisterKVHandlerFromEndpoint is same as RegisterKVHandler but // automatically dials to "endpoint" and closes the connection when "ctx" gets done. func RegisterKVHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { conn, err := grpc.NewClient(endpoint, opts...) if err != nil { return err } defer func() { if err != nil { if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } return } go func() { <-ctx.Done() if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } }() }() return RegisterKVHandler(ctx, mux, conn) } // RegisterKVHandler registers the http handlers for service KV to "mux". // The handlers forward requests to the grpc endpoint over "conn". func RegisterKVHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { return RegisterKVHandlerClient(ctx, mux, etcdserverpb.NewKVClient(conn)) } // etcdserverpb.RegisterKVHandlerClient registers the http handlers for service KV // to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "KVClient". // Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "KVClient" // doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in // "KVClient" to call the correct interceptors. This client ignores the HTTP middlewares. func RegisterKVHandlerClient(ctx context.Context, mux *runtime.ServeMux, client etcdserverpb.KVClient) error { mux.Handle(http.MethodPost, pattern_KV_Range_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.KV/Range", runtime.WithHTTPPathPattern("/v3/kv/range")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_KV_Range_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_KV_Range_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_KV_Put_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.KV/Put", runtime.WithHTTPPathPattern("/v3/kv/put")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_KV_Put_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_KV_Put_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_KV_DeleteRange_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.KV/DeleteRange", runtime.WithHTTPPathPattern("/v3/kv/deleterange")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_KV_DeleteRange_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_KV_DeleteRange_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_KV_Txn_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.KV/Txn", runtime.WithHTTPPathPattern("/v3/kv/txn")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_KV_Txn_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_KV_Txn_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_KV_Compact_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.KV/Compact", runtime.WithHTTPPathPattern("/v3/kv/compaction")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_KV_Compact_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_KV_Compact_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } var ( pattern_KV_Range_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "kv", "range"}, "")) pattern_KV_Put_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "kv", "put"}, "")) pattern_KV_DeleteRange_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "kv", "deleterange"}, "")) pattern_KV_Txn_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "kv", "txn"}, "")) pattern_KV_Compact_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "kv", "compaction"}, "")) ) var ( forward_KV_Range_0 = runtime.ForwardResponseMessage forward_KV_Put_0 = runtime.ForwardResponseMessage forward_KV_DeleteRange_0 = runtime.ForwardResponseMessage forward_KV_Txn_0 = runtime.ForwardResponseMessage forward_KV_Compact_0 = runtime.ForwardResponseMessage ) // RegisterWatchHandlerFromEndpoint is same as RegisterWatchHandler but // automatically dials to "endpoint" and closes the connection when "ctx" gets done. func RegisterWatchHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { conn, err := grpc.NewClient(endpoint, opts...) if err != nil { return err } defer func() { if err != nil { if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } return } go func() { <-ctx.Done() if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } }() }() return RegisterWatchHandler(ctx, mux, conn) } // RegisterWatchHandler registers the http handlers for service Watch to "mux". // The handlers forward requests to the grpc endpoint over "conn". func RegisterWatchHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { return RegisterWatchHandlerClient(ctx, mux, etcdserverpb.NewWatchClient(conn)) } // etcdserverpb.RegisterWatchHandlerClient registers the http handlers for service Watch // to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "WatchClient". // Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "WatchClient" // doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in // "WatchClient" to call the correct interceptors. This client ignores the HTTP middlewares. func RegisterWatchHandlerClient(ctx context.Context, mux *runtime.ServeMux, client etcdserverpb.WatchClient) error { mux.Handle(http.MethodPost, pattern_Watch_Watch_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Watch/Watch", runtime.WithHTTPPathPattern("/v3/watch")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Watch_Watch_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Watch_Watch_0(annotatedContext, mux, outboundMarshaler, w, req, func() (proto.Message, error) { m1, err := resp.Recv() return protov1.MessageV2(m1), err }, mux.GetForwardResponseOptions()...) }) return nil } var ( pattern_Watch_Watch_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1}, []string{"v3", "watch"}, "")) ) var ( forward_Watch_Watch_0 = runtime.ForwardResponseStream ) // RegisterLeaseHandlerFromEndpoint is same as RegisterLeaseHandler but // automatically dials to "endpoint" and closes the connection when "ctx" gets done. func RegisterLeaseHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { conn, err := grpc.NewClient(endpoint, opts...) if err != nil { return err } defer func() { if err != nil { if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } return } go func() { <-ctx.Done() if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } }() }() return RegisterLeaseHandler(ctx, mux, conn) } // RegisterLeaseHandler registers the http handlers for service Lease to "mux". // The handlers forward requests to the grpc endpoint over "conn". func RegisterLeaseHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { return RegisterLeaseHandlerClient(ctx, mux, etcdserverpb.NewLeaseClient(conn)) } // etcdserverpb.RegisterLeaseHandlerClient registers the http handlers for service Lease // to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "LeaseClient". // Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "LeaseClient" // doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in // "LeaseClient" to call the correct interceptors. This client ignores the HTTP middlewares. func RegisterLeaseHandlerClient(ctx context.Context, mux *runtime.ServeMux, client etcdserverpb.LeaseClient) error { mux.Handle(http.MethodPost, pattern_Lease_LeaseGrant_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseGrant", runtime.WithHTTPPathPattern("/v3/lease/grant")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Lease_LeaseGrant_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseGrant_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseRevoke_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseRevoke", runtime.WithHTTPPathPattern("/v3/lease/revoke")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Lease_LeaseRevoke_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseRevoke_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseRevoke_1, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseRevoke", runtime.WithHTTPPathPattern("/v3/kv/lease/revoke")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Lease_LeaseRevoke_1(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseRevoke_1(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseKeepAlive_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseKeepAlive", runtime.WithHTTPPathPattern("/v3/lease/keepalive")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Lease_LeaseKeepAlive_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseKeepAlive_0(annotatedContext, mux, outboundMarshaler, w, req, func() (proto.Message, error) { m1, err := resp.Recv() return protov1.MessageV2(m1), err }, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseTimeToLive_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseTimeToLive", runtime.WithHTTPPathPattern("/v3/lease/timetolive")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Lease_LeaseTimeToLive_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseTimeToLive_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseTimeToLive_1, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseTimeToLive", runtime.WithHTTPPathPattern("/v3/kv/lease/timetolive")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Lease_LeaseTimeToLive_1(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseTimeToLive_1(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseLeases_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseLeases", runtime.WithHTTPPathPattern("/v3/lease/leases")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Lease_LeaseLeases_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseLeases_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lease_LeaseLeases_1, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Lease/LeaseLeases", runtime.WithHTTPPathPattern("/v3/kv/lease/leases")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Lease_LeaseLeases_1(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lease_LeaseLeases_1(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } var ( pattern_Lease_LeaseGrant_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "lease", "grant"}, "")) pattern_Lease_LeaseRevoke_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "lease", "revoke"}, "")) pattern_Lease_LeaseRevoke_1 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "kv", "lease", "revoke"}, "")) pattern_Lease_LeaseKeepAlive_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "lease", "keepalive"}, "")) pattern_Lease_LeaseTimeToLive_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "lease", "timetolive"}, "")) pattern_Lease_LeaseTimeToLive_1 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "kv", "lease", "timetolive"}, "")) pattern_Lease_LeaseLeases_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "lease", "leases"}, "")) pattern_Lease_LeaseLeases_1 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "kv", "lease", "leases"}, "")) ) var ( forward_Lease_LeaseGrant_0 = runtime.ForwardResponseMessage forward_Lease_LeaseRevoke_0 = runtime.ForwardResponseMessage forward_Lease_LeaseRevoke_1 = runtime.ForwardResponseMessage forward_Lease_LeaseKeepAlive_0 = runtime.ForwardResponseStream forward_Lease_LeaseTimeToLive_0 = runtime.ForwardResponseMessage forward_Lease_LeaseTimeToLive_1 = runtime.ForwardResponseMessage forward_Lease_LeaseLeases_0 = runtime.ForwardResponseMessage forward_Lease_LeaseLeases_1 = runtime.ForwardResponseMessage ) // RegisterClusterHandlerFromEndpoint is same as RegisterClusterHandler but // automatically dials to "endpoint" and closes the connection when "ctx" gets done. func RegisterClusterHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { conn, err := grpc.NewClient(endpoint, opts...) if err != nil { return err } defer func() { if err != nil { if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } return } go func() { <-ctx.Done() if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } }() }() return RegisterClusterHandler(ctx, mux, conn) } // RegisterClusterHandler registers the http handlers for service Cluster to "mux". // The handlers forward requests to the grpc endpoint over "conn". func RegisterClusterHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { return RegisterClusterHandlerClient(ctx, mux, etcdserverpb.NewClusterClient(conn)) } // etcdserverpb.RegisterClusterHandlerClient registers the http handlers for service Cluster // to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "ClusterClient". // Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "ClusterClient" // doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in // "ClusterClient" to call the correct interceptors. This client ignores the HTTP middlewares. func RegisterClusterHandlerClient(ctx context.Context, mux *runtime.ServeMux, client etcdserverpb.ClusterClient) error { mux.Handle(http.MethodPost, pattern_Cluster_MemberAdd_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Cluster/MemberAdd", runtime.WithHTTPPathPattern("/v3/cluster/member/add")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Cluster_MemberAdd_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Cluster_MemberAdd_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Cluster_MemberRemove_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Cluster/MemberRemove", runtime.WithHTTPPathPattern("/v3/cluster/member/remove")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Cluster_MemberRemove_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Cluster_MemberRemove_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Cluster_MemberUpdate_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Cluster/MemberUpdate", runtime.WithHTTPPathPattern("/v3/cluster/member/update")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Cluster_MemberUpdate_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Cluster_MemberUpdate_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Cluster_MemberList_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Cluster/MemberList", runtime.WithHTTPPathPattern("/v3/cluster/member/list")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Cluster_MemberList_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Cluster_MemberList_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Cluster_MemberPromote_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Cluster/MemberPromote", runtime.WithHTTPPathPattern("/v3/cluster/member/promote")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Cluster_MemberPromote_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Cluster_MemberPromote_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } var ( pattern_Cluster_MemberAdd_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "cluster", "member", "add"}, "")) pattern_Cluster_MemberRemove_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "cluster", "member", "remove"}, "")) pattern_Cluster_MemberUpdate_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "cluster", "member", "update"}, "")) pattern_Cluster_MemberList_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "cluster", "member", "list"}, "")) pattern_Cluster_MemberPromote_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "cluster", "member", "promote"}, "")) ) var ( forward_Cluster_MemberAdd_0 = runtime.ForwardResponseMessage forward_Cluster_MemberRemove_0 = runtime.ForwardResponseMessage forward_Cluster_MemberUpdate_0 = runtime.ForwardResponseMessage forward_Cluster_MemberList_0 = runtime.ForwardResponseMessage forward_Cluster_MemberPromote_0 = runtime.ForwardResponseMessage ) // RegisterMaintenanceHandlerFromEndpoint is same as RegisterMaintenanceHandler but // automatically dials to "endpoint" and closes the connection when "ctx" gets done. func RegisterMaintenanceHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { conn, err := grpc.NewClient(endpoint, opts...) if err != nil { return err } defer func() { if err != nil { if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } return } go func() { <-ctx.Done() if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } }() }() return RegisterMaintenanceHandler(ctx, mux, conn) } // RegisterMaintenanceHandler registers the http handlers for service Maintenance to "mux". // The handlers forward requests to the grpc endpoint over "conn". func RegisterMaintenanceHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { return RegisterMaintenanceHandlerClient(ctx, mux, etcdserverpb.NewMaintenanceClient(conn)) } // etcdserverpb.RegisterMaintenanceHandlerClient registers the http handlers for service Maintenance // to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "MaintenanceClient". // Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "MaintenanceClient" // doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in // "MaintenanceClient" to call the correct interceptors. This client ignores the HTTP middlewares. func RegisterMaintenanceHandlerClient(ctx context.Context, mux *runtime.ServeMux, client etcdserverpb.MaintenanceClient) error { mux.Handle(http.MethodPost, pattern_Maintenance_Alarm_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Maintenance/Alarm", runtime.WithHTTPPathPattern("/v3/maintenance/alarm")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Maintenance_Alarm_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_Alarm_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_Status_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Maintenance/Status", runtime.WithHTTPPathPattern("/v3/maintenance/status")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Maintenance_Status_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_Status_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_Defragment_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Maintenance/Defragment", runtime.WithHTTPPathPattern("/v3/maintenance/defragment")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Maintenance_Defragment_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_Defragment_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_Hash_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Maintenance/Hash", runtime.WithHTTPPathPattern("/v3/maintenance/hash")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Maintenance_Hash_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_Hash_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_HashKV_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Maintenance/HashKV", runtime.WithHTTPPathPattern("/v3/maintenance/hashkv")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Maintenance_HashKV_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_HashKV_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_Snapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Maintenance/Snapshot", runtime.WithHTTPPathPattern("/v3/maintenance/snapshot")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Maintenance_Snapshot_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_Snapshot_0(annotatedContext, mux, outboundMarshaler, w, req, func() (proto.Message, error) { m1, err := resp.Recv() return protov1.MessageV2(m1), err }, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_MoveLeader_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Maintenance/MoveLeader", runtime.WithHTTPPathPattern("/v3/maintenance/transfer-leadership")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Maintenance_MoveLeader_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_MoveLeader_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Maintenance_Downgrade_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Maintenance/Downgrade", runtime.WithHTTPPathPattern("/v3/maintenance/downgrade")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Maintenance_Downgrade_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Maintenance_Downgrade_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } var ( pattern_Maintenance_Alarm_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "alarm"}, "")) pattern_Maintenance_Status_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "status"}, "")) pattern_Maintenance_Defragment_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "defragment"}, "")) pattern_Maintenance_Hash_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "hash"}, "")) pattern_Maintenance_HashKV_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "hashkv"}, "")) pattern_Maintenance_Snapshot_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "snapshot"}, "")) pattern_Maintenance_MoveLeader_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "transfer-leadership"}, "")) pattern_Maintenance_Downgrade_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "maintenance", "downgrade"}, "")) ) var ( forward_Maintenance_Alarm_0 = runtime.ForwardResponseMessage forward_Maintenance_Status_0 = runtime.ForwardResponseMessage forward_Maintenance_Defragment_0 = runtime.ForwardResponseMessage forward_Maintenance_Hash_0 = runtime.ForwardResponseMessage forward_Maintenance_HashKV_0 = runtime.ForwardResponseMessage forward_Maintenance_Snapshot_0 = runtime.ForwardResponseStream forward_Maintenance_MoveLeader_0 = runtime.ForwardResponseMessage forward_Maintenance_Downgrade_0 = runtime.ForwardResponseMessage ) // RegisterAuthHandlerFromEndpoint is same as RegisterAuthHandler but // automatically dials to "endpoint" and closes the connection when "ctx" gets done. func RegisterAuthHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { conn, err := grpc.NewClient(endpoint, opts...) if err != nil { return err } defer func() { if err != nil { if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } return } go func() { <-ctx.Done() if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } }() }() return RegisterAuthHandler(ctx, mux, conn) } // RegisterAuthHandler registers the http handlers for service Auth to "mux". // The handlers forward requests to the grpc endpoint over "conn". func RegisterAuthHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { return RegisterAuthHandlerClient(ctx, mux, etcdserverpb.NewAuthClient(conn)) } // etcdserverpb.RegisterAuthHandlerClient registers the http handlers for service Auth // to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "AuthClient". // Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "AuthClient" // doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in // "AuthClient" to call the correct interceptors. This client ignores the HTTP middlewares. func RegisterAuthHandlerClient(ctx context.Context, mux *runtime.ServeMux, client etcdserverpb.AuthClient) error { mux.Handle(http.MethodPost, pattern_Auth_AuthEnable_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/AuthEnable", runtime.WithHTTPPathPattern("/v3/auth/enable")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_AuthEnable_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_AuthEnable_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_AuthDisable_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/AuthDisable", runtime.WithHTTPPathPattern("/v3/auth/disable")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_AuthDisable_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_AuthDisable_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_AuthStatus_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/AuthStatus", runtime.WithHTTPPathPattern("/v3/auth/status")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_AuthStatus_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_AuthStatus_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_Authenticate_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/Authenticate", runtime.WithHTTPPathPattern("/v3/auth/authenticate")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_Authenticate_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_Authenticate_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserAdd_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/UserAdd", runtime.WithHTTPPathPattern("/v3/auth/user/add")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_UserAdd_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserAdd_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserGet_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/UserGet", runtime.WithHTTPPathPattern("/v3/auth/user/get")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_UserGet_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserGet_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserList_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/UserList", runtime.WithHTTPPathPattern("/v3/auth/user/list")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_UserList_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserList_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserDelete_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/UserDelete", runtime.WithHTTPPathPattern("/v3/auth/user/delete")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_UserDelete_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserDelete_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserChangePassword_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/UserChangePassword", runtime.WithHTTPPathPattern("/v3/auth/user/changepw")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_UserChangePassword_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserChangePassword_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserGrantRole_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/UserGrantRole", runtime.WithHTTPPathPattern("/v3/auth/user/grant")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_UserGrantRole_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserGrantRole_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_UserRevokeRole_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/UserRevokeRole", runtime.WithHTTPPathPattern("/v3/auth/user/revoke")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_UserRevokeRole_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_UserRevokeRole_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleAdd_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/RoleAdd", runtime.WithHTTPPathPattern("/v3/auth/role/add")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_RoleAdd_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleAdd_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleGet_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/RoleGet", runtime.WithHTTPPathPattern("/v3/auth/role/get")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_RoleGet_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleGet_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleList_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/RoleList", runtime.WithHTTPPathPattern("/v3/auth/role/list")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_RoleList_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleList_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleDelete_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/RoleDelete", runtime.WithHTTPPathPattern("/v3/auth/role/delete")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_RoleDelete_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleDelete_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleGrantPermission_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/RoleGrantPermission", runtime.WithHTTPPathPattern("/v3/auth/role/grant")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_RoleGrantPermission_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleGrantPermission_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Auth_RoleRevokePermission_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/etcdserverpb.Auth/RoleRevokePermission", runtime.WithHTTPPathPattern("/v3/auth/role/revoke")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Auth_RoleRevokePermission_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Auth_RoleRevokePermission_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } var ( pattern_Auth_AuthEnable_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "auth", "enable"}, "")) pattern_Auth_AuthDisable_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "auth", "disable"}, "")) pattern_Auth_AuthStatus_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "auth", "status"}, "")) pattern_Auth_Authenticate_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "auth", "authenticate"}, "")) pattern_Auth_UserAdd_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "user", "add"}, "")) pattern_Auth_UserGet_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "user", "get"}, "")) pattern_Auth_UserList_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "user", "list"}, "")) pattern_Auth_UserDelete_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "user", "delete"}, "")) pattern_Auth_UserChangePassword_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "user", "changepw"}, "")) pattern_Auth_UserGrantRole_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "user", "grant"}, "")) pattern_Auth_UserRevokeRole_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "user", "revoke"}, "")) pattern_Auth_RoleAdd_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "role", "add"}, "")) pattern_Auth_RoleGet_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "role", "get"}, "")) pattern_Auth_RoleList_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "role", "list"}, "")) pattern_Auth_RoleDelete_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "role", "delete"}, "")) pattern_Auth_RoleGrantPermission_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "role", "grant"}, "")) pattern_Auth_RoleRevokePermission_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v3", "auth", "role", "revoke"}, "")) ) var ( forward_Auth_AuthEnable_0 = runtime.ForwardResponseMessage forward_Auth_AuthDisable_0 = runtime.ForwardResponseMessage forward_Auth_AuthStatus_0 = runtime.ForwardResponseMessage forward_Auth_Authenticate_0 = runtime.ForwardResponseMessage forward_Auth_UserAdd_0 = runtime.ForwardResponseMessage forward_Auth_UserGet_0 = runtime.ForwardResponseMessage forward_Auth_UserList_0 = runtime.ForwardResponseMessage forward_Auth_UserDelete_0 = runtime.ForwardResponseMessage forward_Auth_UserChangePassword_0 = runtime.ForwardResponseMessage forward_Auth_UserGrantRole_0 = runtime.ForwardResponseMessage forward_Auth_UserRevokeRole_0 = runtime.ForwardResponseMessage forward_Auth_RoleAdd_0 = runtime.ForwardResponseMessage forward_Auth_RoleGet_0 = runtime.ForwardResponseMessage forward_Auth_RoleList_0 = runtime.ForwardResponseMessage forward_Auth_RoleDelete_0 = runtime.ForwardResponseMessage forward_Auth_RoleGrantPermission_0 = runtime.ForwardResponseMessage forward_Auth_RoleRevokePermission_0 = runtime.ForwardResponseMessage ) ================================================ FILE: api/etcdserverpb/raft_internal.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: raft_internal.proto package etcdserverpb import ( fmt "fmt" io "io" math "math" math_bits "math/bits" proto "github.com/golang/protobuf/proto" membershippb "go.etcd.io/etcd/api/v3/membershippb" _ "go.etcd.io/etcd/api/v3/versionpb" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type RequestHeader struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` // username is a username that is associated with an auth token of gRPC connection Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"` // auth_revision is a revision number of auth.authStore. It is not related to mvcc AuthRevision uint64 `protobuf:"varint,3,opt,name=auth_revision,json=authRevision,proto3" json:"auth_revision,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *RequestHeader) Reset() { *m = RequestHeader{} } func (m *RequestHeader) String() string { return proto.CompactTextString(m) } func (*RequestHeader) ProtoMessage() {} func (*RequestHeader) Descriptor() ([]byte, []int) { return fileDescriptor_b4c9a9be0cfca103, []int{0} } func (m *RequestHeader) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *RequestHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_RequestHeader.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *RequestHeader) XXX_Merge(src proto.Message) { xxx_messageInfo_RequestHeader.Merge(m, src) } func (m *RequestHeader) XXX_Size() int { return m.Size() } func (m *RequestHeader) XXX_DiscardUnknown() { xxx_messageInfo_RequestHeader.DiscardUnknown(m) } var xxx_messageInfo_RequestHeader proto.InternalMessageInfo func (m *RequestHeader) GetID() uint64 { if m != nil { return m.ID } return 0 } func (m *RequestHeader) GetUsername() string { if m != nil { return m.Username } return "" } func (m *RequestHeader) GetAuthRevision() uint64 { if m != nil { return m.AuthRevision } return 0 } // An InternalRaftRequest is the union of all requests which can be // sent via raft. type InternalRaftRequest struct { Header *RequestHeader `protobuf:"bytes,100,opt,name=header,proto3" json:"header,omitempty"` ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Range *RangeRequest `protobuf:"bytes,3,opt,name=range,proto3" json:"range,omitempty"` Put *PutRequest `protobuf:"bytes,4,opt,name=put,proto3" json:"put,omitempty"` DeleteRange *DeleteRangeRequest `protobuf:"bytes,5,opt,name=delete_range,json=deleteRange,proto3" json:"delete_range,omitempty"` Txn *TxnRequest `protobuf:"bytes,6,opt,name=txn,proto3" json:"txn,omitempty"` Compaction *CompactionRequest `protobuf:"bytes,7,opt,name=compaction,proto3" json:"compaction,omitempty"` LeaseGrant *LeaseGrantRequest `protobuf:"bytes,8,opt,name=lease_grant,json=leaseGrant,proto3" json:"lease_grant,omitempty"` LeaseRevoke *LeaseRevokeRequest `protobuf:"bytes,9,opt,name=lease_revoke,json=leaseRevoke,proto3" json:"lease_revoke,omitempty"` Alarm *AlarmRequest `protobuf:"bytes,10,opt,name=alarm,proto3" json:"alarm,omitempty"` LeaseCheckpoint *LeaseCheckpointRequest `protobuf:"bytes,11,opt,name=lease_checkpoint,json=leaseCheckpoint,proto3" json:"lease_checkpoint,omitempty"` AuthEnable *AuthEnableRequest `protobuf:"bytes,1000,opt,name=auth_enable,json=authEnable,proto3" json:"auth_enable,omitempty"` AuthDisable *AuthDisableRequest `protobuf:"bytes,1011,opt,name=auth_disable,json=authDisable,proto3" json:"auth_disable,omitempty"` AuthStatus *AuthStatusRequest `protobuf:"bytes,1013,opt,name=auth_status,json=authStatus,proto3" json:"auth_status,omitempty"` Authenticate *InternalAuthenticateRequest `protobuf:"bytes,1012,opt,name=authenticate,proto3" json:"authenticate,omitempty"` AuthUserAdd *AuthUserAddRequest `protobuf:"bytes,1100,opt,name=auth_user_add,json=authUserAdd,proto3" json:"auth_user_add,omitempty"` AuthUserDelete *AuthUserDeleteRequest `protobuf:"bytes,1101,opt,name=auth_user_delete,json=authUserDelete,proto3" json:"auth_user_delete,omitempty"` AuthUserGet *AuthUserGetRequest `protobuf:"bytes,1102,opt,name=auth_user_get,json=authUserGet,proto3" json:"auth_user_get,omitempty"` AuthUserChangePassword *AuthUserChangePasswordRequest `protobuf:"bytes,1103,opt,name=auth_user_change_password,json=authUserChangePassword,proto3" json:"auth_user_change_password,omitempty"` AuthUserGrantRole *AuthUserGrantRoleRequest `protobuf:"bytes,1104,opt,name=auth_user_grant_role,json=authUserGrantRole,proto3" json:"auth_user_grant_role,omitempty"` AuthUserRevokeRole *AuthUserRevokeRoleRequest `protobuf:"bytes,1105,opt,name=auth_user_revoke_role,json=authUserRevokeRole,proto3" json:"auth_user_revoke_role,omitempty"` AuthUserList *AuthUserListRequest `protobuf:"bytes,1106,opt,name=auth_user_list,json=authUserList,proto3" json:"auth_user_list,omitempty"` AuthRoleList *AuthRoleListRequest `protobuf:"bytes,1107,opt,name=auth_role_list,json=authRoleList,proto3" json:"auth_role_list,omitempty"` AuthRoleAdd *AuthRoleAddRequest `protobuf:"bytes,1200,opt,name=auth_role_add,json=authRoleAdd,proto3" json:"auth_role_add,omitempty"` AuthRoleDelete *AuthRoleDeleteRequest `protobuf:"bytes,1201,opt,name=auth_role_delete,json=authRoleDelete,proto3" json:"auth_role_delete,omitempty"` AuthRoleGet *AuthRoleGetRequest `protobuf:"bytes,1202,opt,name=auth_role_get,json=authRoleGet,proto3" json:"auth_role_get,omitempty"` AuthRoleGrantPermission *AuthRoleGrantPermissionRequest `protobuf:"bytes,1203,opt,name=auth_role_grant_permission,json=authRoleGrantPermission,proto3" json:"auth_role_grant_permission,omitempty"` AuthRoleRevokePermission *AuthRoleRevokePermissionRequest `protobuf:"bytes,1204,opt,name=auth_role_revoke_permission,json=authRoleRevokePermission,proto3" json:"auth_role_revoke_permission,omitempty"` ClusterVersionSet *membershippb.ClusterVersionSetRequest `protobuf:"bytes,1300,opt,name=cluster_version_set,json=clusterVersionSet,proto3" json:"cluster_version_set,omitempty"` ClusterMemberAttrSet *membershippb.ClusterMemberAttrSetRequest `protobuf:"bytes,1301,opt,name=cluster_member_attr_set,json=clusterMemberAttrSet,proto3" json:"cluster_member_attr_set,omitempty"` DowngradeInfoSet *membershippb.DowngradeInfoSetRequest `protobuf:"bytes,1302,opt,name=downgrade_info_set,json=downgradeInfoSet,proto3" json:"downgrade_info_set,omitempty"` DowngradeVersionTest *DowngradeVersionTestRequest `protobuf:"bytes,9900,opt,name=downgrade_version_test,json=downgradeVersionTest,proto3" json:"downgrade_version_test,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *InternalRaftRequest) Reset() { *m = InternalRaftRequest{} } func (m *InternalRaftRequest) String() string { return proto.CompactTextString(m) } func (*InternalRaftRequest) ProtoMessage() {} func (*InternalRaftRequest) Descriptor() ([]byte, []int) { return fileDescriptor_b4c9a9be0cfca103, []int{1} } func (m *InternalRaftRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *InternalRaftRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_InternalRaftRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *InternalRaftRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_InternalRaftRequest.Merge(m, src) } func (m *InternalRaftRequest) XXX_Size() int { return m.Size() } func (m *InternalRaftRequest) XXX_DiscardUnknown() { xxx_messageInfo_InternalRaftRequest.DiscardUnknown(m) } var xxx_messageInfo_InternalRaftRequest proto.InternalMessageInfo func (m *InternalRaftRequest) GetHeader() *RequestHeader { if m != nil { return m.Header } return nil } func (m *InternalRaftRequest) GetID() uint64 { if m != nil { return m.ID } return 0 } func (m *InternalRaftRequest) GetRange() *RangeRequest { if m != nil { return m.Range } return nil } func (m *InternalRaftRequest) GetPut() *PutRequest { if m != nil { return m.Put } return nil } func (m *InternalRaftRequest) GetDeleteRange() *DeleteRangeRequest { if m != nil { return m.DeleteRange } return nil } func (m *InternalRaftRequest) GetTxn() *TxnRequest { if m != nil { return m.Txn } return nil } func (m *InternalRaftRequest) GetCompaction() *CompactionRequest { if m != nil { return m.Compaction } return nil } func (m *InternalRaftRequest) GetLeaseGrant() *LeaseGrantRequest { if m != nil { return m.LeaseGrant } return nil } func (m *InternalRaftRequest) GetLeaseRevoke() *LeaseRevokeRequest { if m != nil { return m.LeaseRevoke } return nil } func (m *InternalRaftRequest) GetAlarm() *AlarmRequest { if m != nil { return m.Alarm } return nil } func (m *InternalRaftRequest) GetLeaseCheckpoint() *LeaseCheckpointRequest { if m != nil { return m.LeaseCheckpoint } return nil } func (m *InternalRaftRequest) GetAuthEnable() *AuthEnableRequest { if m != nil { return m.AuthEnable } return nil } func (m *InternalRaftRequest) GetAuthDisable() *AuthDisableRequest { if m != nil { return m.AuthDisable } return nil } func (m *InternalRaftRequest) GetAuthStatus() *AuthStatusRequest { if m != nil { return m.AuthStatus } return nil } func (m *InternalRaftRequest) GetAuthenticate() *InternalAuthenticateRequest { if m != nil { return m.Authenticate } return nil } func (m *InternalRaftRequest) GetAuthUserAdd() *AuthUserAddRequest { if m != nil { return m.AuthUserAdd } return nil } func (m *InternalRaftRequest) GetAuthUserDelete() *AuthUserDeleteRequest { if m != nil { return m.AuthUserDelete } return nil } func (m *InternalRaftRequest) GetAuthUserGet() *AuthUserGetRequest { if m != nil { return m.AuthUserGet } return nil } func (m *InternalRaftRequest) GetAuthUserChangePassword() *AuthUserChangePasswordRequest { if m != nil { return m.AuthUserChangePassword } return nil } func (m *InternalRaftRequest) GetAuthUserGrantRole() *AuthUserGrantRoleRequest { if m != nil { return m.AuthUserGrantRole } return nil } func (m *InternalRaftRequest) GetAuthUserRevokeRole() *AuthUserRevokeRoleRequest { if m != nil { return m.AuthUserRevokeRole } return nil } func (m *InternalRaftRequest) GetAuthUserList() *AuthUserListRequest { if m != nil { return m.AuthUserList } return nil } func (m *InternalRaftRequest) GetAuthRoleList() *AuthRoleListRequest { if m != nil { return m.AuthRoleList } return nil } func (m *InternalRaftRequest) GetAuthRoleAdd() *AuthRoleAddRequest { if m != nil { return m.AuthRoleAdd } return nil } func (m *InternalRaftRequest) GetAuthRoleDelete() *AuthRoleDeleteRequest { if m != nil { return m.AuthRoleDelete } return nil } func (m *InternalRaftRequest) GetAuthRoleGet() *AuthRoleGetRequest { if m != nil { return m.AuthRoleGet } return nil } func (m *InternalRaftRequest) GetAuthRoleGrantPermission() *AuthRoleGrantPermissionRequest { if m != nil { return m.AuthRoleGrantPermission } return nil } func (m *InternalRaftRequest) GetAuthRoleRevokePermission() *AuthRoleRevokePermissionRequest { if m != nil { return m.AuthRoleRevokePermission } return nil } func (m *InternalRaftRequest) GetClusterVersionSet() *membershippb.ClusterVersionSetRequest { if m != nil { return m.ClusterVersionSet } return nil } func (m *InternalRaftRequest) GetClusterMemberAttrSet() *membershippb.ClusterMemberAttrSetRequest { if m != nil { return m.ClusterMemberAttrSet } return nil } func (m *InternalRaftRequest) GetDowngradeInfoSet() *membershippb.DowngradeInfoSetRequest { if m != nil { return m.DowngradeInfoSet } return nil } func (m *InternalRaftRequest) GetDowngradeVersionTest() *DowngradeVersionTestRequest { if m != nil { return m.DowngradeVersionTest } return nil } type EmptyResponse struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *EmptyResponse) Reset() { *m = EmptyResponse{} } func (m *EmptyResponse) String() string { return proto.CompactTextString(m) } func (*EmptyResponse) ProtoMessage() {} func (*EmptyResponse) Descriptor() ([]byte, []int) { return fileDescriptor_b4c9a9be0cfca103, []int{2} } func (m *EmptyResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *EmptyResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_EmptyResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *EmptyResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_EmptyResponse.Merge(m, src) } func (m *EmptyResponse) XXX_Size() int { return m.Size() } func (m *EmptyResponse) XXX_DiscardUnknown() { xxx_messageInfo_EmptyResponse.DiscardUnknown(m) } var xxx_messageInfo_EmptyResponse proto.InternalMessageInfo // What is the difference between AuthenticateRequest (defined in rpc.proto) and InternalAuthenticateRequest? // InternalAuthenticateRequest has a member that is filled by etcdserver and shouldn't be user-facing. // For avoiding misusage the field, we have an internal version of AuthenticateRequest. type InternalAuthenticateRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Password string `protobuf:"bytes,2,opt,name=password,proto3" json:"password,omitempty"` // simple_token is generated in API layer (etcdserver/v3_server.go) SimpleToken string `protobuf:"bytes,3,opt,name=simple_token,json=simpleToken,proto3" json:"simple_token,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *InternalAuthenticateRequest) Reset() { *m = InternalAuthenticateRequest{} } func (m *InternalAuthenticateRequest) String() string { return proto.CompactTextString(m) } func (*InternalAuthenticateRequest) ProtoMessage() {} func (*InternalAuthenticateRequest) Descriptor() ([]byte, []int) { return fileDescriptor_b4c9a9be0cfca103, []int{3} } func (m *InternalAuthenticateRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *InternalAuthenticateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_InternalAuthenticateRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *InternalAuthenticateRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_InternalAuthenticateRequest.Merge(m, src) } func (m *InternalAuthenticateRequest) XXX_Size() int { return m.Size() } func (m *InternalAuthenticateRequest) XXX_DiscardUnknown() { xxx_messageInfo_InternalAuthenticateRequest.DiscardUnknown(m) } var xxx_messageInfo_InternalAuthenticateRequest proto.InternalMessageInfo func (m *InternalAuthenticateRequest) GetName() string { if m != nil { return m.Name } return "" } func (m *InternalAuthenticateRequest) GetPassword() string { if m != nil { return m.Password } return "" } func (m *InternalAuthenticateRequest) GetSimpleToken() string { if m != nil { return m.SimpleToken } return "" } func init() { proto.RegisterType((*RequestHeader)(nil), "etcdserverpb.RequestHeader") proto.RegisterType((*InternalRaftRequest)(nil), "etcdserverpb.InternalRaftRequest") proto.RegisterType((*EmptyResponse)(nil), "etcdserverpb.EmptyResponse") proto.RegisterType((*InternalAuthenticateRequest)(nil), "etcdserverpb.InternalAuthenticateRequest") } func init() { proto.RegisterFile("raft_internal.proto", fileDescriptor_b4c9a9be0cfca103) } var fileDescriptor_b4c9a9be0cfca103 = []byte{ // 1077 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x56, 0x4d, 0x73, 0x1b, 0x45, 0x10, 0x45, 0xb6, 0x63, 0x5b, 0x23, 0xdb, 0x71, 0xc6, 0x26, 0x19, 0xec, 0x2a, 0xe3, 0x38, 0x10, 0x0c, 0x04, 0x39, 0xd8, 0x84, 0x03, 0x17, 0x50, 0x2c, 0x97, 0xe3, 0x54, 0x92, 0x72, 0x6d, 0x0c, 0x95, 0x82, 0xa2, 0x96, 0xd1, 0x6e, 0x5b, 0xda, 0x78, 0xb5, 0xbb, 0xcc, 0x8c, 0x1c, 0xe7, 0xca, 0x91, 0x33, 0x50, 0xf9, 0x11, 0x1c, 0xf8, 0xfc, 0x0f, 0x1c, 0xf8, 0x08, 0xf0, 0x07, 0x28, 0x73, 0xe1, 0x0e, 0xdc, 0x53, 0xf3, 0xb1, 0xb3, 0x5a, 0x69, 0xe4, 0x9b, 0xb6, 0xfb, 0xf5, 0x7b, 0x6f, 0x66, 0xbb, 0x57, 0x8d, 0x16, 0x18, 0x3d, 0x14, 0x7e, 0x94, 0x08, 0x60, 0x09, 0x8d, 0xeb, 0x19, 0x4b, 0x45, 0x8a, 0x67, 0x40, 0x04, 0x21, 0x07, 0x76, 0x0c, 0x2c, 0x6b, 0x2d, 0x55, 0x59, 0x16, 0xe8, 0xc4, 0xd2, 0xaa, 0x4c, 0x6c, 0xd0, 0x2c, 0xda, 0x38, 0x06, 0xc6, 0xa3, 0x34, 0xc9, 0x5a, 0xf9, 0x2f, 0x83, 0xb8, 0x6a, 0x11, 0x5d, 0xe8, 0xb6, 0x80, 0xf1, 0x4e, 0x94, 0x65, 0xad, 0xbe, 0x07, 0x8d, 0x5b, 0x63, 0x68, 0xd6, 0x83, 0x4f, 0x7b, 0xc0, 0xc5, 0x2d, 0xa0, 0x21, 0x30, 0x3c, 0x87, 0xc6, 0xf6, 0x9a, 0xa4, 0xb2, 0x5a, 0x59, 0x9f, 0xf0, 0xc6, 0xf6, 0x9a, 0x78, 0x09, 0x4d, 0xf7, 0xb8, 0x34, 0xd5, 0x05, 0x32, 0xb6, 0x5a, 0x59, 0xaf, 0x7a, 0xf6, 0x19, 0x5f, 0x43, 0xb3, 0xb4, 0x27, 0x3a, 0x3e, 0x83, 0xe3, 0x48, 0x6a, 0x93, 0x71, 0x59, 0x76, 0x73, 0xea, 0xf3, 0x1f, 0xc9, 0xf8, 0x56, 0xfd, 0x4d, 0x6f, 0x46, 0x66, 0x3d, 0x93, 0x7c, 0x67, 0xea, 0x33, 0x15, 0xbe, 0xbe, 0xf6, 0x64, 0x01, 0x2d, 0xec, 0x99, 0x93, 0x7a, 0xf4, 0x50, 0x18, 0x03, 0x78, 0x0b, 0x4d, 0x76, 0x94, 0x09, 0x12, 0xae, 0x56, 0xd6, 0x6b, 0x9b, 0xcb, 0xf5, 0xfe, 0xf3, 0xd7, 0x4b, 0x3e, 0x3d, 0x03, 0x1d, 0xf2, 0x7b, 0x1d, 0x9d, 0x63, 0x34, 0x69, 0x83, 0xf2, 0x52, 0xdb, 0x5c, 0x1a, 0xe0, 0x90, 0x29, 0x43, 0xe4, 0x69, 0x20, 0x7e, 0x0d, 0x8d, 0x67, 0x3d, 0x41, 0x26, 0x14, 0x9e, 0x94, 0xf1, 0xfb, 0xbd, 0xdc, 0x9d, 0x27, 0x41, 0x78, 0x1b, 0xcd, 0x84, 0x10, 0x83, 0x00, 0x5f, 0x8b, 0x9c, 0x53, 0x45, 0xab, 0xe5, 0xa2, 0xa6, 0x42, 0x94, 0xa4, 0x6a, 0x61, 0x11, 0x93, 0x82, 0xe2, 0x24, 0x21, 0x93, 0x2e, 0xc1, 0x83, 0x93, 0xc4, 0x0a, 0x8a, 0x93, 0x04, 0xbf, 0x8b, 0x50, 0x90, 0x76, 0x33, 0x1a, 0x08, 0x79, 0xbf, 0x53, 0xaa, 0xe4, 0xc5, 0x72, 0xc9, 0xb6, 0xcd, 0xe7, 0x95, 0x7d, 0x25, 0xf8, 0x3d, 0x54, 0x8b, 0x81, 0x72, 0xf0, 0xdb, 0x8c, 0x26, 0x82, 0x4c, 0xbb, 0x18, 0xee, 0x48, 0xc0, 0xae, 0xcc, 0x5b, 0x86, 0xd8, 0x86, 0xe4, 0x99, 0x35, 0x03, 0x83, 0xe3, 0xf4, 0x08, 0x48, 0xd5, 0x75, 0x66, 0x45, 0xe1, 0x29, 0x80, 0x3d, 0x73, 0x5c, 0xc4, 0xe4, 0x6b, 0xa1, 0x31, 0x65, 0x5d, 0x82, 0x5c, 0xaf, 0xa5, 0x21, 0x53, 0xf6, 0xb5, 0x28, 0x20, 0x7e, 0x80, 0xe6, 0xb5, 0x6c, 0xd0, 0x81, 0xe0, 0x28, 0x4b, 0xa3, 0x44, 0x90, 0x9a, 0x2a, 0x7e, 0xc9, 0x21, 0xbd, 0x6d, 0x41, 0x86, 0x26, 0xef, 0xc2, 0xb7, 0xbc, 0xf3, 0x71, 0x19, 0x80, 0x1b, 0xa8, 0xa6, 0xda, 0x16, 0x12, 0xda, 0x8a, 0x81, 0xfc, 0xe3, 0xbc, 0xd5, 0x46, 0x4f, 0x74, 0x76, 0x14, 0xc0, 0xde, 0x09, 0xb5, 0x21, 0xdc, 0x44, 0xaa, 0xb7, 0xfd, 0x30, 0xe2, 0x8a, 0xe3, 0xdf, 0x29, 0xd7, 0xa5, 0x48, 0x8e, 0xa6, 0x46, 0xd8, 0x4b, 0xa1, 0x45, 0x0c, 0xdf, 0x36, 0x46, 0xb8, 0xa0, 0xa2, 0xc7, 0xc9, 0xff, 0x23, 0x8d, 0xdc, 0x57, 0x80, 0x81, 0x93, 0xdd, 0xd0, 0x8e, 0x74, 0x0e, 0xdf, 0xd3, 0x8e, 0x20, 0x11, 0x51, 0x40, 0x05, 0x90, 0xff, 0x34, 0xd9, 0xab, 0x65, 0xb2, 0x7c, 0xec, 0x1a, 0x7d, 0xd0, 0xdc, 0x5a, 0xa9, 0x1e, 0xef, 0x98, 0xd9, 0x96, 0xc3, 0xee, 0xd3, 0x30, 0x24, 0x3f, 0x4f, 0x8f, 0x3a, 0xe2, 0xfb, 0x1c, 0x58, 0x23, 0x0c, 0x4b, 0x47, 0x34, 0x31, 0x7c, 0x0f, 0xcd, 0x17, 0x34, 0x7a, 0x08, 0xc8, 0x2f, 0x9a, 0xe9, 0x8a, 0x9b, 0xc9, 0x4c, 0x8f, 0x21, 0x9b, 0xa3, 0xa5, 0x70, 0xd9, 0x56, 0x1b, 0x04, 0xf9, 0xf5, 0x4c, 0x5b, 0xbb, 0x20, 0x86, 0x6c, 0xed, 0x82, 0xc0, 0x6d, 0xf4, 0x42, 0x41, 0x13, 0x74, 0xe4, 0x58, 0xfa, 0x19, 0xe5, 0xfc, 0x51, 0xca, 0x42, 0xf2, 0x9b, 0xa6, 0x7c, 0xdd, 0x4d, 0xb9, 0xad, 0xd0, 0xfb, 0x06, 0x9c, 0xb3, 0x5f, 0xa4, 0xce, 0x34, 0x7e, 0x80, 0x16, 0xfb, 0xfc, 0xca, 0x79, 0xf2, 0x59, 0x1a, 0x03, 0x79, 0xaa, 0x35, 0xae, 0x8e, 0xb0, 0xad, 0x66, 0x31, 0x2d, 0xda, 0xe6, 0x02, 0x1d, 0xcc, 0xe0, 0x8f, 0xd0, 0xf3, 0x05, 0xb3, 0x1e, 0x4d, 0x4d, 0xfd, 0xbb, 0xa6, 0x7e, 0xc5, 0x4d, 0x6d, 0x66, 0xb4, 0x8f, 0x1b, 0xd3, 0xa1, 0x14, 0xbe, 0x85, 0xe6, 0x0a, 0xf2, 0x38, 0xe2, 0x82, 0xfc, 0xa1, 0x59, 0x2f, 0xbb, 0x59, 0xef, 0x44, 0x5c, 0x94, 0xfa, 0x28, 0x0f, 0x5a, 0x26, 0x69, 0x4d, 0x33, 0xfd, 0x39, 0x92, 0x49, 0x4a, 0x0f, 0x31, 0xe5, 0x41, 0xfb, 0xea, 0x15, 0x93, 0xec, 0xc8, 0x6f, 0xaa, 0xa3, 0x5e, 0xbd, 0xac, 0x19, 0xec, 0x48, 0x13, 0xb3, 0x1d, 0xa9, 0x68, 0x4c, 0x47, 0x7e, 0x5b, 0x1d, 0xd5, 0x91, 0xb2, 0xca, 0xd1, 0x91, 0x45, 0xb8, 0x6c, 0x4b, 0x76, 0xe4, 0x77, 0x67, 0xda, 0x1a, 0xec, 0x48, 0x13, 0xc3, 0x0f, 0xd1, 0x52, 0x1f, 0x8d, 0x6a, 0x94, 0x0c, 0x58, 0x37, 0xe2, 0xea, 0x8f, 0xf5, 0x7b, 0xcd, 0x79, 0x6d, 0x04, 0xa7, 0x84, 0xef, 0x5b, 0x74, 0xce, 0x7f, 0x89, 0xba, 0xf3, 0xb8, 0x8b, 0x96, 0x0b, 0x2d, 0xd3, 0x3a, 0x7d, 0x62, 0x3f, 0x68, 0xb1, 0x37, 0xdc, 0x62, 0xba, 0x4b, 0x86, 0xd5, 0x08, 0x1d, 0x01, 0xc0, 0x9f, 0xa0, 0x85, 0x20, 0xee, 0x71, 0x01, 0xcc, 0x37, 0x4b, 0x8a, 0xcf, 0x41, 0x90, 0x2f, 0x90, 0x19, 0x81, 0xfe, 0x0d, 0xa5, 0xbe, 0xad, 0x91, 0x1f, 0x68, 0xe0, 0x7d, 0x10, 0x43, 0x5f, 0xbd, 0x0b, 0xc1, 0x20, 0x04, 0x3f, 0x44, 0x97, 0x72, 0x05, 0x4d, 0xe6, 0x53, 0x21, 0x98, 0x52, 0xf9, 0x12, 0x99, 0xef, 0xa0, 0x4b, 0xe5, 0xae, 0x8a, 0x35, 0x84, 0x60, 0x2e, 0xa1, 0xc5, 0xc0, 0x81, 0xc2, 0x1f, 0x23, 0x1c, 0xa6, 0x8f, 0x92, 0x36, 0xa3, 0x21, 0xf8, 0x51, 0x72, 0x98, 0x2a, 0x99, 0xaf, 0xb4, 0xcc, 0xcb, 0x65, 0x99, 0x66, 0x0e, 0xdc, 0x4b, 0x0e, 0x53, 0x97, 0xc4, 0x7c, 0x38, 0x80, 0xc0, 0x11, 0xba, 0x58, 0xd0, 0xe7, 0xd7, 0x25, 0x80, 0x0b, 0xf2, 0xf5, 0x5d, 0xd7, 0x17, 0xdd, 0x4a, 0x98, 0xeb, 0x38, 0x00, 0x3e, 0x28, 0xf3, 0xb6, 0xb7, 0x18, 0x3a, 0x50, 0x76, 0x21, 0xbb, 0x3d, 0x31, 0x3d, 0x36, 0x3f, 0xee, 0x8d, 0x1d, 0x6f, 0xae, 0x9d, 0x47, 0xb3, 0x3b, 0xdd, 0x4c, 0x3c, 0xf6, 0x80, 0x67, 0x69, 0xc2, 0x61, 0xed, 0x31, 0x5a, 0x3e, 0xe3, 0x3f, 0x03, 0x63, 0x34, 0xa1, 0x36, 0xc3, 0x8a, 0xda, 0x0c, 0xd5, 0x6f, 0xb9, 0x31, 0xda, 0x4f, 0xa9, 0xd9, 0x18, 0xf3, 0x67, 0x7c, 0x19, 0xcd, 0xf0, 0xa8, 0x9b, 0xc5, 0xe0, 0x8b, 0xf4, 0x08, 0xf4, 0xc2, 0x58, 0xf5, 0x6a, 0x3a, 0x76, 0x20, 0x43, 0xd6, 0xd5, 0xcd, 0x1b, 0x3f, 0x9d, 0xae, 0x54, 0x9e, 0x9e, 0xae, 0x54, 0xfe, 0x3a, 0x5d, 0xa9, 0x3c, 0xf9, 0x7b, 0xe5, 0xb9, 0x0f, 0xaf, 0xb4, 0x53, 0x75, 0xf8, 0x7a, 0x94, 0x6e, 0x14, 0x1b, 0xf0, 0xd6, 0x46, 0xff, 0x85, 0xb4, 0x26, 0xd5, 0x62, 0xbb, 0xf5, 0x2c, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x47, 0x30, 0xbe, 0x52, 0x0b, 0x00, 0x00, } func (m *RequestHeader) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *RequestHeader) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *RequestHeader) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.AuthRevision != 0 { i = encodeVarintRaftInternal(dAtA, i, uint64(m.AuthRevision)) i-- dAtA[i] = 0x18 } if len(m.Username) > 0 { i -= len(m.Username) copy(dAtA[i:], m.Username) i = encodeVarintRaftInternal(dAtA, i, uint64(len(m.Username))) i-- dAtA[i] = 0x12 } if m.ID != 0 { i = encodeVarintRaftInternal(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *InternalRaftRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *InternalRaftRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *InternalRaftRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.DowngradeVersionTest != nil { { size, err := m.DowngradeVersionTest.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x4 i-- dAtA[i] = 0xea i-- dAtA[i] = 0xe2 } if m.DowngradeInfoSet != nil { { size, err := m.DowngradeInfoSet.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x51 i-- dAtA[i] = 0xb2 } if m.ClusterMemberAttrSet != nil { { size, err := m.ClusterMemberAttrSet.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x51 i-- dAtA[i] = 0xaa } if m.ClusterVersionSet != nil { { size, err := m.ClusterVersionSet.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x51 i-- dAtA[i] = 0xa2 } if m.AuthRoleRevokePermission != nil { { size, err := m.AuthRoleRevokePermission.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x4b i-- dAtA[i] = 0xa2 } if m.AuthRoleGrantPermission != nil { { size, err := m.AuthRoleGrantPermission.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x4b i-- dAtA[i] = 0x9a } if m.AuthRoleGet != nil { { size, err := m.AuthRoleGet.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x4b i-- dAtA[i] = 0x92 } if m.AuthRoleDelete != nil { { size, err := m.AuthRoleDelete.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x4b i-- dAtA[i] = 0x8a } if m.AuthRoleAdd != nil { { size, err := m.AuthRoleAdd.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x4b i-- dAtA[i] = 0x82 } if m.AuthRoleList != nil { { size, err := m.AuthRoleList.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x45 i-- dAtA[i] = 0x9a } if m.AuthUserList != nil { { size, err := m.AuthUserList.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x45 i-- dAtA[i] = 0x92 } if m.AuthUserRevokeRole != nil { { size, err := m.AuthUserRevokeRole.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x45 i-- dAtA[i] = 0x8a } if m.AuthUserGrantRole != nil { { size, err := m.AuthUserGrantRole.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x45 i-- dAtA[i] = 0x82 } if m.AuthUserChangePassword != nil { { size, err := m.AuthUserChangePassword.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x44 i-- dAtA[i] = 0xfa } if m.AuthUserGet != nil { { size, err := m.AuthUserGet.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x44 i-- dAtA[i] = 0xf2 } if m.AuthUserDelete != nil { { size, err := m.AuthUserDelete.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x44 i-- dAtA[i] = 0xea } if m.AuthUserAdd != nil { { size, err := m.AuthUserAdd.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x44 i-- dAtA[i] = 0xe2 } if m.AuthStatus != nil { { size, err := m.AuthStatus.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x3f i-- dAtA[i] = 0xaa } if m.Authenticate != nil { { size, err := m.Authenticate.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x3f i-- dAtA[i] = 0xa2 } if m.AuthDisable != nil { { size, err := m.AuthDisable.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x3f i-- dAtA[i] = 0x9a } if m.AuthEnable != nil { { size, err := m.AuthEnable.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x3e i-- dAtA[i] = 0xc2 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x6 i-- dAtA[i] = 0xa2 } if m.LeaseCheckpoint != nil { { size, err := m.LeaseCheckpoint.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x5a } if m.Alarm != nil { { size, err := m.Alarm.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x52 } if m.LeaseRevoke != nil { { size, err := m.LeaseRevoke.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x4a } if m.LeaseGrant != nil { { size, err := m.LeaseGrant.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x42 } if m.Compaction != nil { { size, err := m.Compaction.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x3a } if m.Txn != nil { { size, err := m.Txn.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x32 } if m.DeleteRange != nil { { size, err := m.DeleteRange.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x2a } if m.Put != nil { { size, err := m.Put.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x22 } if m.Range != nil { { size, err := m.Range.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRaftInternal(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } if m.ID != 0 { i = encodeVarintRaftInternal(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *EmptyResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *EmptyResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *EmptyResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *InternalAuthenticateRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *InternalAuthenticateRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *InternalAuthenticateRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.SimpleToken) > 0 { i -= len(m.SimpleToken) copy(dAtA[i:], m.SimpleToken) i = encodeVarintRaftInternal(dAtA, i, uint64(len(m.SimpleToken))) i-- dAtA[i] = 0x1a } if len(m.Password) > 0 { i -= len(m.Password) copy(dAtA[i:], m.Password) i = encodeVarintRaftInternal(dAtA, i, uint64(len(m.Password))) i-- dAtA[i] = 0x12 } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintRaftInternal(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func encodeVarintRaftInternal(dAtA []byte, offset int, v uint64) int { offset -= sovRaftInternal(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) return base } func (m *RequestHeader) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovRaftInternal(uint64(m.ID)) } l = len(m.Username) if l > 0 { n += 1 + l + sovRaftInternal(uint64(l)) } if m.AuthRevision != 0 { n += 1 + sovRaftInternal(uint64(m.AuthRevision)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *InternalRaftRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovRaftInternal(uint64(m.ID)) } if m.Range != nil { l = m.Range.Size() n += 1 + l + sovRaftInternal(uint64(l)) } if m.Put != nil { l = m.Put.Size() n += 1 + l + sovRaftInternal(uint64(l)) } if m.DeleteRange != nil { l = m.DeleteRange.Size() n += 1 + l + sovRaftInternal(uint64(l)) } if m.Txn != nil { l = m.Txn.Size() n += 1 + l + sovRaftInternal(uint64(l)) } if m.Compaction != nil { l = m.Compaction.Size() n += 1 + l + sovRaftInternal(uint64(l)) } if m.LeaseGrant != nil { l = m.LeaseGrant.Size() n += 1 + l + sovRaftInternal(uint64(l)) } if m.LeaseRevoke != nil { l = m.LeaseRevoke.Size() n += 1 + l + sovRaftInternal(uint64(l)) } if m.Alarm != nil { l = m.Alarm.Size() n += 1 + l + sovRaftInternal(uint64(l)) } if m.LeaseCheckpoint != nil { l = m.LeaseCheckpoint.Size() n += 1 + l + sovRaftInternal(uint64(l)) } if m.Header != nil { l = m.Header.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthEnable != nil { l = m.AuthEnable.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthDisable != nil { l = m.AuthDisable.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.Authenticate != nil { l = m.Authenticate.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthStatus != nil { l = m.AuthStatus.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthUserAdd != nil { l = m.AuthUserAdd.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthUserDelete != nil { l = m.AuthUserDelete.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthUserGet != nil { l = m.AuthUserGet.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthUserChangePassword != nil { l = m.AuthUserChangePassword.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthUserGrantRole != nil { l = m.AuthUserGrantRole.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthUserRevokeRole != nil { l = m.AuthUserRevokeRole.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthUserList != nil { l = m.AuthUserList.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthRoleList != nil { l = m.AuthRoleList.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthRoleAdd != nil { l = m.AuthRoleAdd.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthRoleDelete != nil { l = m.AuthRoleDelete.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthRoleGet != nil { l = m.AuthRoleGet.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthRoleGrantPermission != nil { l = m.AuthRoleGrantPermission.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.AuthRoleRevokePermission != nil { l = m.AuthRoleRevokePermission.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.ClusterVersionSet != nil { l = m.ClusterVersionSet.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.ClusterMemberAttrSet != nil { l = m.ClusterMemberAttrSet.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.DowngradeInfoSet != nil { l = m.DowngradeInfoSet.Size() n += 2 + l + sovRaftInternal(uint64(l)) } if m.DowngradeVersionTest != nil { l = m.DowngradeVersionTest.Size() n += 3 + l + sovRaftInternal(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *EmptyResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *InternalAuthenticateRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovRaftInternal(uint64(l)) } l = len(m.Password) if l > 0 { n += 1 + l + sovRaftInternal(uint64(l)) } l = len(m.SimpleToken) if l > 0 { n += 1 + l + sovRaftInternal(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func sovRaftInternal(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } func sozRaftInternal(x uint64) (n int) { return sovRaftInternal(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (m *RequestHeader) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: RequestHeader: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: RequestHeader: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Username", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } m.Username = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field AuthRevision", wireType) } m.AuthRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.AuthRevision |= uint64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRaftInternal(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRaftInternal } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *InternalRaftRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: InternalRaftRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: InternalRaftRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Range", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.Range == nil { m.Range = &RangeRequest{} } if err := m.Range.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Put", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.Put == nil { m.Put = &PutRequest{} } if err := m.Put.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 5: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field DeleteRange", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.DeleteRange == nil { m.DeleteRange = &DeleteRangeRequest{} } if err := m.DeleteRange.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 6: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Txn", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.Txn == nil { m.Txn = &TxnRequest{} } if err := m.Txn.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 7: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Compaction", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.Compaction == nil { m.Compaction = &CompactionRequest{} } if err := m.Compaction.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 8: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field LeaseGrant", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.LeaseGrant == nil { m.LeaseGrant = &LeaseGrantRequest{} } if err := m.LeaseGrant.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 9: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field LeaseRevoke", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.LeaseRevoke == nil { m.LeaseRevoke = &LeaseRevokeRequest{} } if err := m.LeaseRevoke.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 10: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Alarm", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.Alarm == nil { m.Alarm = &AlarmRequest{} } if err := m.Alarm.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 11: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field LeaseCheckpoint", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.LeaseCheckpoint == nil { m.LeaseCheckpoint = &LeaseCheckpointRequest{} } if err := m.LeaseCheckpoint.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 100: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &RequestHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1000: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthEnable", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthEnable == nil { m.AuthEnable = &AuthEnableRequest{} } if err := m.AuthEnable.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1011: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthDisable", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthDisable == nil { m.AuthDisable = &AuthDisableRequest{} } if err := m.AuthDisable.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1012: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Authenticate", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.Authenticate == nil { m.Authenticate = &InternalAuthenticateRequest{} } if err := m.Authenticate.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1013: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthStatus", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthStatus == nil { m.AuthStatus = &AuthStatusRequest{} } if err := m.AuthStatus.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1100: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthUserAdd", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthUserAdd == nil { m.AuthUserAdd = &AuthUserAddRequest{} } if err := m.AuthUserAdd.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1101: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthUserDelete", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthUserDelete == nil { m.AuthUserDelete = &AuthUserDeleteRequest{} } if err := m.AuthUserDelete.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1102: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthUserGet", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthUserGet == nil { m.AuthUserGet = &AuthUserGetRequest{} } if err := m.AuthUserGet.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1103: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthUserChangePassword", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthUserChangePassword == nil { m.AuthUserChangePassword = &AuthUserChangePasswordRequest{} } if err := m.AuthUserChangePassword.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1104: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthUserGrantRole", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthUserGrantRole == nil { m.AuthUserGrantRole = &AuthUserGrantRoleRequest{} } if err := m.AuthUserGrantRole.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1105: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthUserRevokeRole", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthUserRevokeRole == nil { m.AuthUserRevokeRole = &AuthUserRevokeRoleRequest{} } if err := m.AuthUserRevokeRole.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1106: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthUserList", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthUserList == nil { m.AuthUserList = &AuthUserListRequest{} } if err := m.AuthUserList.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1107: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthRoleList", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthRoleList == nil { m.AuthRoleList = &AuthRoleListRequest{} } if err := m.AuthRoleList.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1200: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthRoleAdd", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthRoleAdd == nil { m.AuthRoleAdd = &AuthRoleAddRequest{} } if err := m.AuthRoleAdd.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1201: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthRoleDelete", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthRoleDelete == nil { m.AuthRoleDelete = &AuthRoleDeleteRequest{} } if err := m.AuthRoleDelete.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1202: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthRoleGet", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthRoleGet == nil { m.AuthRoleGet = &AuthRoleGetRequest{} } if err := m.AuthRoleGet.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1203: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthRoleGrantPermission", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthRoleGrantPermission == nil { m.AuthRoleGrantPermission = &AuthRoleGrantPermissionRequest{} } if err := m.AuthRoleGrantPermission.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1204: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AuthRoleRevokePermission", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.AuthRoleRevokePermission == nil { m.AuthRoleRevokePermission = &AuthRoleRevokePermissionRequest{} } if err := m.AuthRoleRevokePermission.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1300: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ClusterVersionSet", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.ClusterVersionSet == nil { m.ClusterVersionSet = &membershippb.ClusterVersionSetRequest{} } if err := m.ClusterVersionSet.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1301: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ClusterMemberAttrSet", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.ClusterMemberAttrSet == nil { m.ClusterMemberAttrSet = &membershippb.ClusterMemberAttrSetRequest{} } if err := m.ClusterMemberAttrSet.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 1302: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field DowngradeInfoSet", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.DowngradeInfoSet == nil { m.DowngradeInfoSet = &membershippb.DowngradeInfoSetRequest{} } if err := m.DowngradeInfoSet.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 9900: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field DowngradeVersionTest", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } if m.DowngradeVersionTest == nil { m.DowngradeVersionTest = &DowngradeVersionTestRequest{} } if err := m.DowngradeVersionTest.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRaftInternal(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRaftInternal } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *EmptyResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: EmptyResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: EmptyResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRaftInternal(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRaftInternal } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *InternalAuthenticateRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: InternalAuthenticateRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: InternalAuthenticateRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Password", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } m.Password = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field SimpleToken", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRaftInternal } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRaftInternal } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRaftInternal } if postIndex > l { return io.ErrUnexpectedEOF } m.SimpleToken = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRaftInternal(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRaftInternal } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func skipRaftInternal(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowRaftInternal } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } wireType := int(wire & 0x7) switch wireType { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowRaftInternal } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } iNdEx++ if dAtA[iNdEx-1] < 0x80 { break } } case 1: iNdEx += 8 case 2: var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowRaftInternal } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { break } } if length < 0 { return 0, ErrInvalidLengthRaftInternal } iNdEx += length case 3: depth++ case 4: if depth == 0 { return 0, ErrUnexpectedEndOfGroupRaftInternal } depth-- case 5: iNdEx += 4 default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { return 0, ErrInvalidLengthRaftInternal } if depth == 0 { return iNdEx, nil } } return 0, io.ErrUnexpectedEOF } var ( ErrInvalidLengthRaftInternal = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowRaftInternal = fmt.Errorf("proto: integer overflow") ErrUnexpectedEndOfGroupRaftInternal = fmt.Errorf("proto: unexpected end of group") ) ================================================ FILE: api/etcdserverpb/raft_internal.proto ================================================ syntax = "proto3"; package etcdserverpb; import "rpc.proto"; import "etcd/api/versionpb/version.proto"; import "etcd/api/membershippb/membership.proto"; option go_package = "go.etcd.io/etcd/api/v3/etcdserverpb"; message RequestHeader { option (versionpb.etcd_version_msg) = "3.0"; uint64 ID = 1; // username is a username that is associated with an auth token of gRPC connection string username = 2; // auth_revision is a revision number of auth.authStore. It is not related to mvcc uint64 auth_revision = 3 [(versionpb.etcd_version_field) = "3.1"]; } // An InternalRaftRequest is the union of all requests which can be // sent via raft. message InternalRaftRequest { option (versionpb.etcd_version_msg) = "3.0"; RequestHeader header = 100; uint64 ID = 1; reserved 2; reserved "v2"; RangeRequest range = 3; PutRequest put = 4; DeleteRangeRequest delete_range = 5; TxnRequest txn = 6; CompactionRequest compaction = 7; LeaseGrantRequest lease_grant = 8; LeaseRevokeRequest lease_revoke = 9; AlarmRequest alarm = 10; LeaseCheckpointRequest lease_checkpoint = 11 [(versionpb.etcd_version_field) = "3.4"]; AuthEnableRequest auth_enable = 1000; AuthDisableRequest auth_disable = 1011; AuthStatusRequest auth_status = 1013 [(versionpb.etcd_version_field) = "3.5"]; InternalAuthenticateRequest authenticate = 1012; AuthUserAddRequest auth_user_add = 1100; AuthUserDeleteRequest auth_user_delete = 1101; AuthUserGetRequest auth_user_get = 1102; AuthUserChangePasswordRequest auth_user_change_password = 1103; AuthUserGrantRoleRequest auth_user_grant_role = 1104; AuthUserRevokeRoleRequest auth_user_revoke_role = 1105; AuthUserListRequest auth_user_list = 1106; AuthRoleListRequest auth_role_list = 1107; AuthRoleAddRequest auth_role_add = 1200; AuthRoleDeleteRequest auth_role_delete = 1201; AuthRoleGetRequest auth_role_get = 1202; AuthRoleGrantPermissionRequest auth_role_grant_permission = 1203; AuthRoleRevokePermissionRequest auth_role_revoke_permission = 1204; membershippb.ClusterVersionSetRequest cluster_version_set = 1300 [(versionpb.etcd_version_field) = "3.5"]; membershippb.ClusterMemberAttrSetRequest cluster_member_attr_set = 1301 [(versionpb.etcd_version_field) = "3.5"]; membershippb.DowngradeInfoSetRequest downgrade_info_set = 1302 [(versionpb.etcd_version_field) = "3.5"]; DowngradeVersionTestRequest downgrade_version_test = 9900 [(versionpb.etcd_version_field) = "3.6"]; } message EmptyResponse { } // What is the difference between AuthenticateRequest (defined in rpc.proto) and InternalAuthenticateRequest? // InternalAuthenticateRequest has a member that is filled by etcdserver and shouldn't be user-facing. // For avoiding misusage the field, we have an internal version of AuthenticateRequest. message InternalAuthenticateRequest { option (versionpb.etcd_version_msg) = "3.0"; string name = 1; string password = 2; // simple_token is generated in API layer (etcdserver/v3_server.go) string simple_token = 3; } ================================================ FILE: api/etcdserverpb/raft_internal_stringer.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserverpb import ( "fmt" "strings" proto "github.com/golang/protobuf/proto" //nolint:staticcheck // TODO: remove for a supported version ) // InternalRaftStringer implements custom proto Stringer: // redact password, replace value fields with value_size fields. type InternalRaftStringer struct { Request *InternalRaftRequest } func (as *InternalRaftStringer) String() string { switch { case as.Request.LeaseGrant != nil: return fmt.Sprintf("header:<%s> lease_grant:", as.Request.Header.String(), as.Request.LeaseGrant.TTL, as.Request.LeaseGrant.ID, ) case as.Request.LeaseRevoke != nil: return fmt.Sprintf("header:<%s> lease_revoke:", as.Request.Header.String(), as.Request.LeaseRevoke.ID, ) case as.Request.Authenticate != nil: return fmt.Sprintf("header:<%s> authenticate:", as.Request.Header.String(), as.Request.Authenticate.Name, as.Request.Authenticate.SimpleToken, ) case as.Request.AuthUserAdd != nil: return fmt.Sprintf("header:<%s> auth_user_add:", as.Request.Header.String(), as.Request.AuthUserAdd.Name, ) case as.Request.AuthUserChangePassword != nil: return fmt.Sprintf("header:<%s> auth_user_change_password:", as.Request.Header.String(), as.Request.AuthUserChangePassword.Name, ) case as.Request.Put != nil: return fmt.Sprintf("header:<%s> put:<%s>", as.Request.Header.String(), NewLoggablePutRequest(as.Request.Put).String(), ) case as.Request.Txn != nil: return fmt.Sprintf("header:<%s> txn:<%s>", as.Request.Header.String(), NewLoggableTxnRequest(as.Request.Txn).String(), ) default: // nothing to redact } return as.Request.String() } // txnRequestStringer implements fmt.Stringer, a custom proto String to replace value bytes // fields with value size fields in any nested txn and put operations. type txnRequestStringer struct { Request *TxnRequest } func NewLoggableTxnRequest(request *TxnRequest) fmt.Stringer { return &txnRequestStringer{request} } func (as *txnRequestStringer) String() string { var compare []string for _, c := range as.Request.Compare { switch cv := c.TargetUnion.(type) { case *Compare_Value: compare = append(compare, newLoggableValueCompare(c, cv).String()) default: // nothing to redact compare = append(compare, c.String()) } } var success []string for _, s := range as.Request.Success { success = append(success, newLoggableRequestOp(s).String()) } var failure []string for _, f := range as.Request.Failure { failure = append(failure, newLoggableRequestOp(f).String()) } return fmt.Sprintf("compare:<%s> success:<%s> failure:<%s>", strings.Join(compare, " "), strings.Join(success, " "), strings.Join(failure, " "), ) } // requestOpStringer implements a custom proto String to replace value bytes fields with value // size fields in any nested txn and put operations. type requestOpStringer struct { Op *RequestOp } func newLoggableRequestOp(op *RequestOp) *requestOpStringer { return &requestOpStringer{op} } func (as *requestOpStringer) String() string { switch op := as.Op.Request.(type) { case *RequestOp_RequestPut: return fmt.Sprintf("request_put:<%s>", NewLoggablePutRequest(op.RequestPut).String()) case *RequestOp_RequestTxn: return fmt.Sprintf("request_txn:<%s>", NewLoggableTxnRequest(op.RequestTxn).String()) default: // nothing to redact } return as.Op.String() } // loggableValueCompare implements a custom proto String for Compare.Value union member types to // replace the value bytes field with a value size field. // To preserve proto encoding of the key and range_end bytes, a faked out proto type is used here. type loggableValueCompare struct { Result Compare_CompareResult `protobuf:"varint,1,opt,name=result,proto3,enum=etcdserverpb.Compare_CompareResult"` Target Compare_CompareTarget `protobuf:"varint,2,opt,name=target,proto3,enum=etcdserverpb.Compare_CompareTarget"` Key []byte `protobuf:"bytes,3,opt,name=key,proto3"` ValueSize int64 `protobuf:"varint,7,opt,name=value_size,proto3"` RangeEnd []byte `protobuf:"bytes,64,opt,name=range_end,proto3"` } func newLoggableValueCompare(c *Compare, cv *Compare_Value) *loggableValueCompare { return &loggableValueCompare{ c.Result, c.Target, c.Key, int64(len(cv.Value)), c.RangeEnd, } } func (m *loggableValueCompare) Reset() { *m = loggableValueCompare{} } func (m *loggableValueCompare) String() string { return proto.CompactTextString(m) } func (*loggableValueCompare) ProtoMessage() {} // loggablePutRequest implements proto.Message, a custom proto String to replace value bytes // field with a value size field. // To preserve proto encoding of the key bytes, a faked out proto type is used here. type loggablePutRequest struct { Key []byte `protobuf:"bytes,1,opt,name=key,proto3"` ValueSize int64 `protobuf:"varint,2,opt,name=value_size,proto3"` Lease int64 `protobuf:"varint,3,opt,name=lease,proto3"` PrevKv bool `protobuf:"varint,4,opt,name=prev_kv,proto3"` IgnoreValue bool `protobuf:"varint,5,opt,name=ignore_value,proto3"` IgnoreLease bool `protobuf:"varint,6,opt,name=ignore_lease,proto3"` } func NewLoggablePutRequest(request *PutRequest) proto.Message { return &loggablePutRequest{ request.Key, int64(len(request.Value)), request.Lease, request.PrevKv, request.IgnoreValue, request.IgnoreLease, } } func (m *loggablePutRequest) Reset() { *m = loggablePutRequest{} } func (m *loggablePutRequest) String() string { return proto.CompactTextString(m) } func (*loggablePutRequest) ProtoMessage() {} ================================================ FILE: api/etcdserverpb/raft_internal_stringer_test.go ================================================ // Copyright 2020 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserverpb_test import ( "testing" "github.com/stretchr/testify/assert" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) // TestInvalidGoTypeIntPanic tests conditions that caused // panic: invalid Go type int for field k8s_io.kubernetes.vendor.go_etcd_io.etcd.etcdserver.etcdserverpb.loggablePutRequest.value_size // See https://github.com/kubernetes/kubernetes/issues/91937 for more details func TestInvalidGoTypeIntPanic(t *testing.T) { assert.Empty(t, pb.NewLoggablePutRequest(&pb.PutRequest{}).String()) } ================================================ FILE: api/etcdserverpb/rpc.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: rpc.proto package etcdserverpb import ( fmt "fmt" io "io" math "math" math_bits "math/bits" proto "github.com/golang/protobuf/proto" _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2/options" authpb "go.etcd.io/etcd/api/v3/authpb" mvccpb "go.etcd.io/etcd/api/v3/mvccpb" _ "go.etcd.io/etcd/api/v3/versionpb" _ "google.golang.org/genproto/googleapis/api/annotations" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type AlarmType int32 const ( AlarmType_NONE AlarmType = 0 AlarmType_NOSPACE AlarmType = 1 AlarmType_CORRUPT AlarmType = 2 ) var AlarmType_name = map[int32]string{ 0: "NONE", 1: "NOSPACE", 2: "CORRUPT", } var AlarmType_value = map[string]int32{ "NONE": 0, "NOSPACE": 1, "CORRUPT": 2, } func (x AlarmType) String() string { return proto.EnumName(AlarmType_name, int32(x)) } func (AlarmType) EnumDescriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{0} } type RangeRequest_SortOrder int32 const ( RangeRequest_NONE RangeRequest_SortOrder = 0 RangeRequest_ASCEND RangeRequest_SortOrder = 1 RangeRequest_DESCEND RangeRequest_SortOrder = 2 ) var RangeRequest_SortOrder_name = map[int32]string{ 0: "NONE", 1: "ASCEND", 2: "DESCEND", } var RangeRequest_SortOrder_value = map[string]int32{ "NONE": 0, "ASCEND": 1, "DESCEND": 2, } func (x RangeRequest_SortOrder) String() string { return proto.EnumName(RangeRequest_SortOrder_name, int32(x)) } func (RangeRequest_SortOrder) EnumDescriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{1, 0} } type RangeRequest_SortTarget int32 const ( RangeRequest_KEY RangeRequest_SortTarget = 0 RangeRequest_VERSION RangeRequest_SortTarget = 1 RangeRequest_CREATE RangeRequest_SortTarget = 2 RangeRequest_MOD RangeRequest_SortTarget = 3 RangeRequest_VALUE RangeRequest_SortTarget = 4 ) var RangeRequest_SortTarget_name = map[int32]string{ 0: "KEY", 1: "VERSION", 2: "CREATE", 3: "MOD", 4: "VALUE", } var RangeRequest_SortTarget_value = map[string]int32{ "KEY": 0, "VERSION": 1, "CREATE": 2, "MOD": 3, "VALUE": 4, } func (x RangeRequest_SortTarget) String() string { return proto.EnumName(RangeRequest_SortTarget_name, int32(x)) } func (RangeRequest_SortTarget) EnumDescriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{1, 1} } type Compare_CompareResult int32 const ( Compare_EQUAL Compare_CompareResult = 0 Compare_GREATER Compare_CompareResult = 1 Compare_LESS Compare_CompareResult = 2 Compare_NOT_EQUAL Compare_CompareResult = 3 ) var Compare_CompareResult_name = map[int32]string{ 0: "EQUAL", 1: "GREATER", 2: "LESS", 3: "NOT_EQUAL", } var Compare_CompareResult_value = map[string]int32{ "EQUAL": 0, "GREATER": 1, "LESS": 2, "NOT_EQUAL": 3, } func (x Compare_CompareResult) String() string { return proto.EnumName(Compare_CompareResult_name, int32(x)) } func (Compare_CompareResult) EnumDescriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{9, 0} } type Compare_CompareTarget int32 const ( Compare_VERSION Compare_CompareTarget = 0 Compare_CREATE Compare_CompareTarget = 1 Compare_MOD Compare_CompareTarget = 2 Compare_VALUE Compare_CompareTarget = 3 Compare_LEASE Compare_CompareTarget = 4 ) var Compare_CompareTarget_name = map[int32]string{ 0: "VERSION", 1: "CREATE", 2: "MOD", 3: "VALUE", 4: "LEASE", } var Compare_CompareTarget_value = map[string]int32{ "VERSION": 0, "CREATE": 1, "MOD": 2, "VALUE": 3, "LEASE": 4, } func (x Compare_CompareTarget) String() string { return proto.EnumName(Compare_CompareTarget_name, int32(x)) } func (Compare_CompareTarget) EnumDescriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{9, 1} } type WatchCreateRequest_FilterType int32 const ( // filter out put event. WatchCreateRequest_NOPUT WatchCreateRequest_FilterType = 0 // filter out delete event. WatchCreateRequest_NODELETE WatchCreateRequest_FilterType = 1 ) var WatchCreateRequest_FilterType_name = map[int32]string{ 0: "NOPUT", 1: "NODELETE", } var WatchCreateRequest_FilterType_value = map[string]int32{ "NOPUT": 0, "NODELETE": 1, } func (x WatchCreateRequest_FilterType) String() string { return proto.EnumName(WatchCreateRequest_FilterType_name, int32(x)) } func (WatchCreateRequest_FilterType) EnumDescriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{21, 0} } type AlarmRequest_AlarmAction int32 const ( AlarmRequest_GET AlarmRequest_AlarmAction = 0 AlarmRequest_ACTIVATE AlarmRequest_AlarmAction = 1 AlarmRequest_DEACTIVATE AlarmRequest_AlarmAction = 2 ) var AlarmRequest_AlarmAction_name = map[int32]string{ 0: "GET", 1: "ACTIVATE", 2: "DEACTIVATE", } var AlarmRequest_AlarmAction_value = map[string]int32{ "GET": 0, "ACTIVATE": 1, "DEACTIVATE": 2, } func (x AlarmRequest_AlarmAction) String() string { return proto.EnumName(AlarmRequest_AlarmAction_name, int32(x)) } func (AlarmRequest_AlarmAction) EnumDescriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{54, 0} } type DowngradeRequest_DowngradeAction int32 const ( DowngradeRequest_VALIDATE DowngradeRequest_DowngradeAction = 0 DowngradeRequest_ENABLE DowngradeRequest_DowngradeAction = 1 DowngradeRequest_CANCEL DowngradeRequest_DowngradeAction = 2 ) var DowngradeRequest_DowngradeAction_name = map[int32]string{ 0: "VALIDATE", 1: "ENABLE", 2: "CANCEL", } var DowngradeRequest_DowngradeAction_value = map[string]int32{ "VALIDATE": 0, "ENABLE": 1, "CANCEL": 2, } func (x DowngradeRequest_DowngradeAction) String() string { return proto.EnumName(DowngradeRequest_DowngradeAction_name, int32(x)) } func (DowngradeRequest_DowngradeAction) EnumDescriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{57, 0} } type ResponseHeader struct { // cluster_id is the ID of the cluster which sent the response. ClusterId uint64 `protobuf:"varint,1,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"` // member_id is the ID of the member which sent the response. MemberId uint64 `protobuf:"varint,2,opt,name=member_id,json=memberId,proto3" json:"member_id,omitempty"` // revision is the key-value store revision when the request was applied, and it's // unset (so 0) in case of calls not interacting with key-value store. // For watch progress responses, the header.revision indicates progress. All future events // received in this stream are guaranteed to have a higher revision number than the // header.revision number. Revision int64 `protobuf:"varint,3,opt,name=revision,proto3" json:"revision,omitempty"` // raft_term is the raft term when the request was applied. RaftTerm uint64 `protobuf:"varint,4,opt,name=raft_term,json=raftTerm,proto3" json:"raft_term,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *ResponseHeader) Reset() { *m = ResponseHeader{} } func (m *ResponseHeader) String() string { return proto.CompactTextString(m) } func (*ResponseHeader) ProtoMessage() {} func (*ResponseHeader) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{0} } func (m *ResponseHeader) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *ResponseHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_ResponseHeader.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *ResponseHeader) XXX_Merge(src proto.Message) { xxx_messageInfo_ResponseHeader.Merge(m, src) } func (m *ResponseHeader) XXX_Size() int { return m.Size() } func (m *ResponseHeader) XXX_DiscardUnknown() { xxx_messageInfo_ResponseHeader.DiscardUnknown(m) } var xxx_messageInfo_ResponseHeader proto.InternalMessageInfo func (m *ResponseHeader) GetClusterId() uint64 { if m != nil { return m.ClusterId } return 0 } func (m *ResponseHeader) GetMemberId() uint64 { if m != nil { return m.MemberId } return 0 } func (m *ResponseHeader) GetRevision() int64 { if m != nil { return m.Revision } return 0 } func (m *ResponseHeader) GetRaftTerm() uint64 { if m != nil { return m.RaftTerm } return 0 } type RangeRequest struct { // key is the first key for the range. If range_end is not given, the request only looks up key. Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // range_end is the upper bound on the requested range [key, range_end). // If range_end is '\0', the range is all keys >= key. // If range_end is key plus one (e.g., "aa"+1 == "ab", "a\xff"+1 == "b"), // then the range request gets all keys prefixed with key. // If both key and range_end are '\0', then the range request returns all keys. RangeEnd []byte `protobuf:"bytes,2,opt,name=range_end,json=rangeEnd,proto3" json:"range_end,omitempty"` // limit is a limit on the number of keys returned for the request. When limit is set to 0, // it is treated as no limit. Limit int64 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` // revision is the point-in-time of the key-value store to use for the range. // If revision is less or equal to zero, the range is over the newest key-value store. // If the revision has been compacted, ErrCompacted is returned as a response. Revision int64 `protobuf:"varint,4,opt,name=revision,proto3" json:"revision,omitempty"` // sort_order is the order for returned sorted results. SortOrder RangeRequest_SortOrder `protobuf:"varint,5,opt,name=sort_order,json=sortOrder,proto3,enum=etcdserverpb.RangeRequest_SortOrder" json:"sort_order,omitempty"` // sort_target is the key-value field to use for sorting. SortTarget RangeRequest_SortTarget `protobuf:"varint,6,opt,name=sort_target,json=sortTarget,proto3,enum=etcdserverpb.RangeRequest_SortTarget" json:"sort_target,omitempty"` // serializable sets the range request to use serializable member-local reads. // Range requests are linearizable by default; linearizable requests have higher // latency and lower throughput than serializable requests but reflect the current // consensus of the cluster. For better performance, in exchange for possible stale reads, // a serializable range request is served locally without needing to reach consensus // with other nodes in the cluster. Serializable bool `protobuf:"varint,7,opt,name=serializable,proto3" json:"serializable,omitempty"` // keys_only when set returns only the keys and not the values. KeysOnly bool `protobuf:"varint,8,opt,name=keys_only,json=keysOnly,proto3" json:"keys_only,omitempty"` // count_only when set returns only the count of the keys in the range. CountOnly bool `protobuf:"varint,9,opt,name=count_only,json=countOnly,proto3" json:"count_only,omitempty"` // min_mod_revision is the lower bound for returned key mod revisions; all keys with // lesser mod revisions will be filtered away. MinModRevision int64 `protobuf:"varint,10,opt,name=min_mod_revision,json=minModRevision,proto3" json:"min_mod_revision,omitempty"` // max_mod_revision is the upper bound for returned key mod revisions; all keys with // greater mod revisions will be filtered away. MaxModRevision int64 `protobuf:"varint,11,opt,name=max_mod_revision,json=maxModRevision,proto3" json:"max_mod_revision,omitempty"` // min_create_revision is the lower bound for returned key create revisions; all keys with // lesser create revisions will be filtered away. MinCreateRevision int64 `protobuf:"varint,12,opt,name=min_create_revision,json=minCreateRevision,proto3" json:"min_create_revision,omitempty"` // max_create_revision is the upper bound for returned key create revisions; all keys with // greater create revisions will be filtered away. MaxCreateRevision int64 `protobuf:"varint,13,opt,name=max_create_revision,json=maxCreateRevision,proto3" json:"max_create_revision,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *RangeRequest) Reset() { *m = RangeRequest{} } func (m *RangeRequest) String() string { return proto.CompactTextString(m) } func (*RangeRequest) ProtoMessage() {} func (*RangeRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{1} } func (m *RangeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *RangeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_RangeRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *RangeRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_RangeRequest.Merge(m, src) } func (m *RangeRequest) XXX_Size() int { return m.Size() } func (m *RangeRequest) XXX_DiscardUnknown() { xxx_messageInfo_RangeRequest.DiscardUnknown(m) } var xxx_messageInfo_RangeRequest proto.InternalMessageInfo func (m *RangeRequest) GetKey() []byte { if m != nil { return m.Key } return nil } func (m *RangeRequest) GetRangeEnd() []byte { if m != nil { return m.RangeEnd } return nil } func (m *RangeRequest) GetLimit() int64 { if m != nil { return m.Limit } return 0 } func (m *RangeRequest) GetRevision() int64 { if m != nil { return m.Revision } return 0 } func (m *RangeRequest) GetSortOrder() RangeRequest_SortOrder { if m != nil { return m.SortOrder } return RangeRequest_NONE } func (m *RangeRequest) GetSortTarget() RangeRequest_SortTarget { if m != nil { return m.SortTarget } return RangeRequest_KEY } func (m *RangeRequest) GetSerializable() bool { if m != nil { return m.Serializable } return false } func (m *RangeRequest) GetKeysOnly() bool { if m != nil { return m.KeysOnly } return false } func (m *RangeRequest) GetCountOnly() bool { if m != nil { return m.CountOnly } return false } func (m *RangeRequest) GetMinModRevision() int64 { if m != nil { return m.MinModRevision } return 0 } func (m *RangeRequest) GetMaxModRevision() int64 { if m != nil { return m.MaxModRevision } return 0 } func (m *RangeRequest) GetMinCreateRevision() int64 { if m != nil { return m.MinCreateRevision } return 0 } func (m *RangeRequest) GetMaxCreateRevision() int64 { if m != nil { return m.MaxCreateRevision } return 0 } type RangeResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // kvs is the list of key-value pairs matched by the range request. // kvs is empty when count is requested. Kvs []*mvccpb.KeyValue `protobuf:"bytes,2,rep,name=kvs,proto3" json:"kvs,omitempty"` // more indicates if there are more keys to return in the requested range. More bool `protobuf:"varint,3,opt,name=more,proto3" json:"more,omitempty"` // count is set to the actual number of keys within the range when requested. // Unlike Kvs, it is unaffected by limits and filters (e.g., Min/Max, Create/Modify, Revisions) // and reflects the full count within the specified range. Count int64 `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *RangeResponse) Reset() { *m = RangeResponse{} } func (m *RangeResponse) String() string { return proto.CompactTextString(m) } func (*RangeResponse) ProtoMessage() {} func (*RangeResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{2} } func (m *RangeResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *RangeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_RangeResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *RangeResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_RangeResponse.Merge(m, src) } func (m *RangeResponse) XXX_Size() int { return m.Size() } func (m *RangeResponse) XXX_DiscardUnknown() { xxx_messageInfo_RangeResponse.DiscardUnknown(m) } var xxx_messageInfo_RangeResponse proto.InternalMessageInfo func (m *RangeResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *RangeResponse) GetKvs() []*mvccpb.KeyValue { if m != nil { return m.Kvs } return nil } func (m *RangeResponse) GetMore() bool { if m != nil { return m.More } return false } func (m *RangeResponse) GetCount() int64 { if m != nil { return m.Count } return 0 } type PutRequest struct { // key is the key, in bytes, to put into the key-value store. Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // value is the value, in bytes, to associate with the key in the key-value store. Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` // lease is the lease ID to associate with the key in the key-value store. A lease // value of 0 indicates no lease. Lease int64 `protobuf:"varint,3,opt,name=lease,proto3" json:"lease,omitempty"` // If prev_kv is set, etcd gets the previous key-value pair before changing it. // The previous key-value pair will be returned in the put response. PrevKv bool `protobuf:"varint,4,opt,name=prev_kv,json=prevKv,proto3" json:"prev_kv,omitempty"` // If ignore_value is set, etcd updates the key using its current value. // Returns an error if the key does not exist. IgnoreValue bool `protobuf:"varint,5,opt,name=ignore_value,json=ignoreValue,proto3" json:"ignore_value,omitempty"` // If ignore_lease is set, etcd updates the key using its current lease. // Returns an error if the key does not exist. IgnoreLease bool `protobuf:"varint,6,opt,name=ignore_lease,json=ignoreLease,proto3" json:"ignore_lease,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *PutRequest) Reset() { *m = PutRequest{} } func (m *PutRequest) String() string { return proto.CompactTextString(m) } func (*PutRequest) ProtoMessage() {} func (*PutRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{3} } func (m *PutRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *PutRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_PutRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *PutRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_PutRequest.Merge(m, src) } func (m *PutRequest) XXX_Size() int { return m.Size() } func (m *PutRequest) XXX_DiscardUnknown() { xxx_messageInfo_PutRequest.DiscardUnknown(m) } var xxx_messageInfo_PutRequest proto.InternalMessageInfo func (m *PutRequest) GetKey() []byte { if m != nil { return m.Key } return nil } func (m *PutRequest) GetValue() []byte { if m != nil { return m.Value } return nil } func (m *PutRequest) GetLease() int64 { if m != nil { return m.Lease } return 0 } func (m *PutRequest) GetPrevKv() bool { if m != nil { return m.PrevKv } return false } func (m *PutRequest) GetIgnoreValue() bool { if m != nil { return m.IgnoreValue } return false } func (m *PutRequest) GetIgnoreLease() bool { if m != nil { return m.IgnoreLease } return false } type PutResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // if prev_kv is set in the request, the previous key-value pair will be returned. PrevKv *mvccpb.KeyValue `protobuf:"bytes,2,opt,name=prev_kv,json=prevKv,proto3" json:"prev_kv,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *PutResponse) Reset() { *m = PutResponse{} } func (m *PutResponse) String() string { return proto.CompactTextString(m) } func (*PutResponse) ProtoMessage() {} func (*PutResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{4} } func (m *PutResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *PutResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_PutResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *PutResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_PutResponse.Merge(m, src) } func (m *PutResponse) XXX_Size() int { return m.Size() } func (m *PutResponse) XXX_DiscardUnknown() { xxx_messageInfo_PutResponse.DiscardUnknown(m) } var xxx_messageInfo_PutResponse proto.InternalMessageInfo func (m *PutResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *PutResponse) GetPrevKv() *mvccpb.KeyValue { if m != nil { return m.PrevKv } return nil } type DeleteRangeRequest struct { // key is the first key to delete in the range. Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // range_end is the key following the last key to delete for the range [key, range_end). // If range_end is not given, the range is defined to contain only the key argument. // If range_end is one bit larger than the given key, then the range is all the keys // with the prefix (the given key). // If range_end is '\0', the range is all keys greater than or equal to the key argument. RangeEnd []byte `protobuf:"bytes,2,opt,name=range_end,json=rangeEnd,proto3" json:"range_end,omitempty"` // If prev_kv is set, etcd gets the previous key-value pairs before deleting it. // The previous key-value pairs will be returned in the delete response. PrevKv bool `protobuf:"varint,3,opt,name=prev_kv,json=prevKv,proto3" json:"prev_kv,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *DeleteRangeRequest) Reset() { *m = DeleteRangeRequest{} } func (m *DeleteRangeRequest) String() string { return proto.CompactTextString(m) } func (*DeleteRangeRequest) ProtoMessage() {} func (*DeleteRangeRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{5} } func (m *DeleteRangeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *DeleteRangeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_DeleteRangeRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *DeleteRangeRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_DeleteRangeRequest.Merge(m, src) } func (m *DeleteRangeRequest) XXX_Size() int { return m.Size() } func (m *DeleteRangeRequest) XXX_DiscardUnknown() { xxx_messageInfo_DeleteRangeRequest.DiscardUnknown(m) } var xxx_messageInfo_DeleteRangeRequest proto.InternalMessageInfo func (m *DeleteRangeRequest) GetKey() []byte { if m != nil { return m.Key } return nil } func (m *DeleteRangeRequest) GetRangeEnd() []byte { if m != nil { return m.RangeEnd } return nil } func (m *DeleteRangeRequest) GetPrevKv() bool { if m != nil { return m.PrevKv } return false } type DeleteRangeResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // deleted is the number of keys deleted by the delete range request. Deleted int64 `protobuf:"varint,2,opt,name=deleted,proto3" json:"deleted,omitempty"` // if prev_kv is set in the request, the previous key-value pairs will be returned. PrevKvs []*mvccpb.KeyValue `protobuf:"bytes,3,rep,name=prev_kvs,json=prevKvs,proto3" json:"prev_kvs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *DeleteRangeResponse) Reset() { *m = DeleteRangeResponse{} } func (m *DeleteRangeResponse) String() string { return proto.CompactTextString(m) } func (*DeleteRangeResponse) ProtoMessage() {} func (*DeleteRangeResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{6} } func (m *DeleteRangeResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *DeleteRangeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_DeleteRangeResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *DeleteRangeResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_DeleteRangeResponse.Merge(m, src) } func (m *DeleteRangeResponse) XXX_Size() int { return m.Size() } func (m *DeleteRangeResponse) XXX_DiscardUnknown() { xxx_messageInfo_DeleteRangeResponse.DiscardUnknown(m) } var xxx_messageInfo_DeleteRangeResponse proto.InternalMessageInfo func (m *DeleteRangeResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *DeleteRangeResponse) GetDeleted() int64 { if m != nil { return m.Deleted } return 0 } func (m *DeleteRangeResponse) GetPrevKvs() []*mvccpb.KeyValue { if m != nil { return m.PrevKvs } return nil } type RequestOp struct { // request is a union of request types accepted by a transaction. // // Types that are valid to be assigned to Request: // *RequestOp_RequestRange // *RequestOp_RequestPut // *RequestOp_RequestDeleteRange // *RequestOp_RequestTxn Request isRequestOp_Request `protobuf_oneof:"request"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *RequestOp) Reset() { *m = RequestOp{} } func (m *RequestOp) String() string { return proto.CompactTextString(m) } func (*RequestOp) ProtoMessage() {} func (*RequestOp) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{7} } func (m *RequestOp) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *RequestOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_RequestOp.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *RequestOp) XXX_Merge(src proto.Message) { xxx_messageInfo_RequestOp.Merge(m, src) } func (m *RequestOp) XXX_Size() int { return m.Size() } func (m *RequestOp) XXX_DiscardUnknown() { xxx_messageInfo_RequestOp.DiscardUnknown(m) } var xxx_messageInfo_RequestOp proto.InternalMessageInfo type isRequestOp_Request interface { isRequestOp_Request() MarshalTo([]byte) (int, error) Size() int } type RequestOp_RequestRange struct { RequestRange *RangeRequest `protobuf:"bytes,1,opt,name=request_range,json=requestRange,proto3,oneof" json:"request_range,omitempty"` } type RequestOp_RequestPut struct { RequestPut *PutRequest `protobuf:"bytes,2,opt,name=request_put,json=requestPut,proto3,oneof" json:"request_put,omitempty"` } type RequestOp_RequestDeleteRange struct { RequestDeleteRange *DeleteRangeRequest `protobuf:"bytes,3,opt,name=request_delete_range,json=requestDeleteRange,proto3,oneof" json:"request_delete_range,omitempty"` } type RequestOp_RequestTxn struct { RequestTxn *TxnRequest `protobuf:"bytes,4,opt,name=request_txn,json=requestTxn,proto3,oneof" json:"request_txn,omitempty"` } func (*RequestOp_RequestRange) isRequestOp_Request() {} func (*RequestOp_RequestPut) isRequestOp_Request() {} func (*RequestOp_RequestDeleteRange) isRequestOp_Request() {} func (*RequestOp_RequestTxn) isRequestOp_Request() {} func (m *RequestOp) GetRequest() isRequestOp_Request { if m != nil { return m.Request } return nil } func (m *RequestOp) GetRequestRange() *RangeRequest { if x, ok := m.GetRequest().(*RequestOp_RequestRange); ok { return x.RequestRange } return nil } func (m *RequestOp) GetRequestPut() *PutRequest { if x, ok := m.GetRequest().(*RequestOp_RequestPut); ok { return x.RequestPut } return nil } func (m *RequestOp) GetRequestDeleteRange() *DeleteRangeRequest { if x, ok := m.GetRequest().(*RequestOp_RequestDeleteRange); ok { return x.RequestDeleteRange } return nil } func (m *RequestOp) GetRequestTxn() *TxnRequest { if x, ok := m.GetRequest().(*RequestOp_RequestTxn); ok { return x.RequestTxn } return nil } // XXX_OneofWrappers is for the internal use of the proto package. func (*RequestOp) XXX_OneofWrappers() []interface{} { return []interface{}{ (*RequestOp_RequestRange)(nil), (*RequestOp_RequestPut)(nil), (*RequestOp_RequestDeleteRange)(nil), (*RequestOp_RequestTxn)(nil), } } type ResponseOp struct { // response is a union of response types returned by a transaction. // // Types that are valid to be assigned to Response: // *ResponseOp_ResponseRange // *ResponseOp_ResponsePut // *ResponseOp_ResponseDeleteRange // *ResponseOp_ResponseTxn Response isResponseOp_Response `protobuf_oneof:"response"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *ResponseOp) Reset() { *m = ResponseOp{} } func (m *ResponseOp) String() string { return proto.CompactTextString(m) } func (*ResponseOp) ProtoMessage() {} func (*ResponseOp) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{8} } func (m *ResponseOp) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *ResponseOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_ResponseOp.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *ResponseOp) XXX_Merge(src proto.Message) { xxx_messageInfo_ResponseOp.Merge(m, src) } func (m *ResponseOp) XXX_Size() int { return m.Size() } func (m *ResponseOp) XXX_DiscardUnknown() { xxx_messageInfo_ResponseOp.DiscardUnknown(m) } var xxx_messageInfo_ResponseOp proto.InternalMessageInfo type isResponseOp_Response interface { isResponseOp_Response() MarshalTo([]byte) (int, error) Size() int } type ResponseOp_ResponseRange struct { ResponseRange *RangeResponse `protobuf:"bytes,1,opt,name=response_range,json=responseRange,proto3,oneof" json:"response_range,omitempty"` } type ResponseOp_ResponsePut struct { ResponsePut *PutResponse `protobuf:"bytes,2,opt,name=response_put,json=responsePut,proto3,oneof" json:"response_put,omitempty"` } type ResponseOp_ResponseDeleteRange struct { ResponseDeleteRange *DeleteRangeResponse `protobuf:"bytes,3,opt,name=response_delete_range,json=responseDeleteRange,proto3,oneof" json:"response_delete_range,omitempty"` } type ResponseOp_ResponseTxn struct { ResponseTxn *TxnResponse `protobuf:"bytes,4,opt,name=response_txn,json=responseTxn,proto3,oneof" json:"response_txn,omitempty"` } func (*ResponseOp_ResponseRange) isResponseOp_Response() {} func (*ResponseOp_ResponsePut) isResponseOp_Response() {} func (*ResponseOp_ResponseDeleteRange) isResponseOp_Response() {} func (*ResponseOp_ResponseTxn) isResponseOp_Response() {} func (m *ResponseOp) GetResponse() isResponseOp_Response { if m != nil { return m.Response } return nil } func (m *ResponseOp) GetResponseRange() *RangeResponse { if x, ok := m.GetResponse().(*ResponseOp_ResponseRange); ok { return x.ResponseRange } return nil } func (m *ResponseOp) GetResponsePut() *PutResponse { if x, ok := m.GetResponse().(*ResponseOp_ResponsePut); ok { return x.ResponsePut } return nil } func (m *ResponseOp) GetResponseDeleteRange() *DeleteRangeResponse { if x, ok := m.GetResponse().(*ResponseOp_ResponseDeleteRange); ok { return x.ResponseDeleteRange } return nil } func (m *ResponseOp) GetResponseTxn() *TxnResponse { if x, ok := m.GetResponse().(*ResponseOp_ResponseTxn); ok { return x.ResponseTxn } return nil } // XXX_OneofWrappers is for the internal use of the proto package. func (*ResponseOp) XXX_OneofWrappers() []interface{} { return []interface{}{ (*ResponseOp_ResponseRange)(nil), (*ResponseOp_ResponsePut)(nil), (*ResponseOp_ResponseDeleteRange)(nil), (*ResponseOp_ResponseTxn)(nil), } } type Compare struct { // result is logical comparison operation for this comparison. Result Compare_CompareResult `protobuf:"varint,1,opt,name=result,proto3,enum=etcdserverpb.Compare_CompareResult" json:"result,omitempty"` // target is the key-value field to inspect for the comparison. Target Compare_CompareTarget `protobuf:"varint,2,opt,name=target,proto3,enum=etcdserverpb.Compare_CompareTarget" json:"target,omitempty"` // key is the subject key for the comparison operation. Key []byte `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` // Types that are valid to be assigned to TargetUnion: // *Compare_Version // *Compare_CreateRevision // *Compare_ModRevision // *Compare_Value // *Compare_Lease TargetUnion isCompare_TargetUnion `protobuf_oneof:"target_union"` // range_end compares the given target to all keys in the range [key, range_end). // See RangeRequest for more details on key ranges. RangeEnd []byte `protobuf:"bytes,64,opt,name=range_end,json=rangeEnd,proto3" json:"range_end,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Compare) Reset() { *m = Compare{} } func (m *Compare) String() string { return proto.CompactTextString(m) } func (*Compare) ProtoMessage() {} func (*Compare) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{9} } func (m *Compare) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Compare) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Compare.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Compare) XXX_Merge(src proto.Message) { xxx_messageInfo_Compare.Merge(m, src) } func (m *Compare) XXX_Size() int { return m.Size() } func (m *Compare) XXX_DiscardUnknown() { xxx_messageInfo_Compare.DiscardUnknown(m) } var xxx_messageInfo_Compare proto.InternalMessageInfo type isCompare_TargetUnion interface { isCompare_TargetUnion() MarshalTo([]byte) (int, error) Size() int } type Compare_Version struct { Version int64 `protobuf:"varint,4,opt,name=version,proto3,oneof" json:"version,omitempty"` } type Compare_CreateRevision struct { CreateRevision int64 `protobuf:"varint,5,opt,name=create_revision,json=createRevision,proto3,oneof" json:"create_revision,omitempty"` } type Compare_ModRevision struct { ModRevision int64 `protobuf:"varint,6,opt,name=mod_revision,json=modRevision,proto3,oneof" json:"mod_revision,omitempty"` } type Compare_Value struct { Value []byte `protobuf:"bytes,7,opt,name=value,proto3,oneof" json:"value,omitempty"` } type Compare_Lease struct { Lease int64 `protobuf:"varint,8,opt,name=lease,proto3,oneof" json:"lease,omitempty"` } func (*Compare_Version) isCompare_TargetUnion() {} func (*Compare_CreateRevision) isCompare_TargetUnion() {} func (*Compare_ModRevision) isCompare_TargetUnion() {} func (*Compare_Value) isCompare_TargetUnion() {} func (*Compare_Lease) isCompare_TargetUnion() {} func (m *Compare) GetTargetUnion() isCompare_TargetUnion { if m != nil { return m.TargetUnion } return nil } func (m *Compare) GetResult() Compare_CompareResult { if m != nil { return m.Result } return Compare_EQUAL } func (m *Compare) GetTarget() Compare_CompareTarget { if m != nil { return m.Target } return Compare_VERSION } func (m *Compare) GetKey() []byte { if m != nil { return m.Key } return nil } func (m *Compare) GetVersion() int64 { if x, ok := m.GetTargetUnion().(*Compare_Version); ok { return x.Version } return 0 } func (m *Compare) GetCreateRevision() int64 { if x, ok := m.GetTargetUnion().(*Compare_CreateRevision); ok { return x.CreateRevision } return 0 } func (m *Compare) GetModRevision() int64 { if x, ok := m.GetTargetUnion().(*Compare_ModRevision); ok { return x.ModRevision } return 0 } func (m *Compare) GetValue() []byte { if x, ok := m.GetTargetUnion().(*Compare_Value); ok { return x.Value } return nil } func (m *Compare) GetLease() int64 { if x, ok := m.GetTargetUnion().(*Compare_Lease); ok { return x.Lease } return 0 } func (m *Compare) GetRangeEnd() []byte { if m != nil { return m.RangeEnd } return nil } // XXX_OneofWrappers is for the internal use of the proto package. func (*Compare) XXX_OneofWrappers() []interface{} { return []interface{}{ (*Compare_Version)(nil), (*Compare_CreateRevision)(nil), (*Compare_ModRevision)(nil), (*Compare_Value)(nil), (*Compare_Lease)(nil), } } // From google paxosdb paper: // Our implementation hinges around a powerful primitive which we call MultiOp. All other database // operations except for iteration are implemented as a single call to MultiOp. A MultiOp is applied atomically // and consists of three components: // 1. A list of tests called guard. Each test in guard checks a single entry in the database. It may check // for the absence or presence of a value, or compare with a given value. Two different tests in the guard // may apply to the same or different entries in the database. All tests in the guard are applied and // MultiOp returns the results. If all tests are true, MultiOp executes t op (see item 2 below), otherwise // it executes f op (see item 3 below). // 2. A list of database operations called t op. Each operation in the list is either an insert, delete, or // lookup operation, and applies to a single database entry. Two different operations in the list may apply // to the same or different entries in the database. These operations are executed // if guard evaluates to // true. // 3. A list of database operations called f op. Like t op, but executed if guard evaluates to false. type TxnRequest struct { // compare is a list of predicates representing a conjunction of terms. // If the comparisons succeed, then the success requests will be processed in order, // and the response will contain their respective responses in order. // If the comparisons fail, then the failure requests will be processed in order, // and the response will contain their respective responses in order. Compare []*Compare `protobuf:"bytes,1,rep,name=compare,proto3" json:"compare,omitempty"` // success is a list of requests which will be applied when compare evaluates to true. Success []*RequestOp `protobuf:"bytes,2,rep,name=success,proto3" json:"success,omitempty"` // failure is a list of requests which will be applied when compare evaluates to false. Failure []*RequestOp `protobuf:"bytes,3,rep,name=failure,proto3" json:"failure,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *TxnRequest) Reset() { *m = TxnRequest{} } func (m *TxnRequest) String() string { return proto.CompactTextString(m) } func (*TxnRequest) ProtoMessage() {} func (*TxnRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{10} } func (m *TxnRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *TxnRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_TxnRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *TxnRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_TxnRequest.Merge(m, src) } func (m *TxnRequest) XXX_Size() int { return m.Size() } func (m *TxnRequest) XXX_DiscardUnknown() { xxx_messageInfo_TxnRequest.DiscardUnknown(m) } var xxx_messageInfo_TxnRequest proto.InternalMessageInfo func (m *TxnRequest) GetCompare() []*Compare { if m != nil { return m.Compare } return nil } func (m *TxnRequest) GetSuccess() []*RequestOp { if m != nil { return m.Success } return nil } func (m *TxnRequest) GetFailure() []*RequestOp { if m != nil { return m.Failure } return nil } type TxnResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // succeeded is set to true if the compare evaluated to true or false otherwise. Succeeded bool `protobuf:"varint,2,opt,name=succeeded,proto3" json:"succeeded,omitempty"` // responses is a list of responses corresponding to the results from applying // success if succeeded is true or failure if succeeded is false. Responses []*ResponseOp `protobuf:"bytes,3,rep,name=responses,proto3" json:"responses,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *TxnResponse) Reset() { *m = TxnResponse{} } func (m *TxnResponse) String() string { return proto.CompactTextString(m) } func (*TxnResponse) ProtoMessage() {} func (*TxnResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{11} } func (m *TxnResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *TxnResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_TxnResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *TxnResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_TxnResponse.Merge(m, src) } func (m *TxnResponse) XXX_Size() int { return m.Size() } func (m *TxnResponse) XXX_DiscardUnknown() { xxx_messageInfo_TxnResponse.DiscardUnknown(m) } var xxx_messageInfo_TxnResponse proto.InternalMessageInfo func (m *TxnResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *TxnResponse) GetSucceeded() bool { if m != nil { return m.Succeeded } return false } func (m *TxnResponse) GetResponses() []*ResponseOp { if m != nil { return m.Responses } return nil } // CompactionRequest compacts the key-value store up to a given revision. All superseded keys // with a revision less than the compaction revision will be removed. type CompactionRequest struct { // revision is the key-value store revision for the compaction operation. Revision int64 `protobuf:"varint,1,opt,name=revision,proto3" json:"revision,omitempty"` // physical is set so the RPC will wait until the compaction is physically // applied to the local database such that compacted entries are totally // removed from the backend database. Physical bool `protobuf:"varint,2,opt,name=physical,proto3" json:"physical,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *CompactionRequest) Reset() { *m = CompactionRequest{} } func (m *CompactionRequest) String() string { return proto.CompactTextString(m) } func (*CompactionRequest) ProtoMessage() {} func (*CompactionRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{12} } func (m *CompactionRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *CompactionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_CompactionRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *CompactionRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_CompactionRequest.Merge(m, src) } func (m *CompactionRequest) XXX_Size() int { return m.Size() } func (m *CompactionRequest) XXX_DiscardUnknown() { xxx_messageInfo_CompactionRequest.DiscardUnknown(m) } var xxx_messageInfo_CompactionRequest proto.InternalMessageInfo func (m *CompactionRequest) GetRevision() int64 { if m != nil { return m.Revision } return 0 } func (m *CompactionRequest) GetPhysical() bool { if m != nil { return m.Physical } return false } type CompactionResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *CompactionResponse) Reset() { *m = CompactionResponse{} } func (m *CompactionResponse) String() string { return proto.CompactTextString(m) } func (*CompactionResponse) ProtoMessage() {} func (*CompactionResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{13} } func (m *CompactionResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *CompactionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_CompactionResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *CompactionResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_CompactionResponse.Merge(m, src) } func (m *CompactionResponse) XXX_Size() int { return m.Size() } func (m *CompactionResponse) XXX_DiscardUnknown() { xxx_messageInfo_CompactionResponse.DiscardUnknown(m) } var xxx_messageInfo_CompactionResponse proto.InternalMessageInfo func (m *CompactionResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type HashRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *HashRequest) Reset() { *m = HashRequest{} } func (m *HashRequest) String() string { return proto.CompactTextString(m) } func (*HashRequest) ProtoMessage() {} func (*HashRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{14} } func (m *HashRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *HashRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_HashRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *HashRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_HashRequest.Merge(m, src) } func (m *HashRequest) XXX_Size() int { return m.Size() } func (m *HashRequest) XXX_DiscardUnknown() { xxx_messageInfo_HashRequest.DiscardUnknown(m) } var xxx_messageInfo_HashRequest proto.InternalMessageInfo type HashKVRequest struct { // revision is the key-value store revision for the hash operation. Revision int64 `protobuf:"varint,1,opt,name=revision,proto3" json:"revision,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *HashKVRequest) Reset() { *m = HashKVRequest{} } func (m *HashKVRequest) String() string { return proto.CompactTextString(m) } func (*HashKVRequest) ProtoMessage() {} func (*HashKVRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{15} } func (m *HashKVRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *HashKVRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_HashKVRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *HashKVRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_HashKVRequest.Merge(m, src) } func (m *HashKVRequest) XXX_Size() int { return m.Size() } func (m *HashKVRequest) XXX_DiscardUnknown() { xxx_messageInfo_HashKVRequest.DiscardUnknown(m) } var xxx_messageInfo_HashKVRequest proto.InternalMessageInfo func (m *HashKVRequest) GetRevision() int64 { if m != nil { return m.Revision } return 0 } type HashKVResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // hash is the hash value computed from the responding member's MVCC keys up to a given revision. Hash uint32 `protobuf:"varint,2,opt,name=hash,proto3" json:"hash,omitempty"` // compact_revision is the compacted revision of key-value store when hash begins. CompactRevision int64 `protobuf:"varint,3,opt,name=compact_revision,json=compactRevision,proto3" json:"compact_revision,omitempty"` // hash_revision is the revision up to which the hash is calculated. HashRevision int64 `protobuf:"varint,4,opt,name=hash_revision,json=hashRevision,proto3" json:"hash_revision,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *HashKVResponse) Reset() { *m = HashKVResponse{} } func (m *HashKVResponse) String() string { return proto.CompactTextString(m) } func (*HashKVResponse) ProtoMessage() {} func (*HashKVResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{16} } func (m *HashKVResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *HashKVResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_HashKVResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *HashKVResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_HashKVResponse.Merge(m, src) } func (m *HashKVResponse) XXX_Size() int { return m.Size() } func (m *HashKVResponse) XXX_DiscardUnknown() { xxx_messageInfo_HashKVResponse.DiscardUnknown(m) } var xxx_messageInfo_HashKVResponse proto.InternalMessageInfo func (m *HashKVResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *HashKVResponse) GetHash() uint32 { if m != nil { return m.Hash } return 0 } func (m *HashKVResponse) GetCompactRevision() int64 { if m != nil { return m.CompactRevision } return 0 } func (m *HashKVResponse) GetHashRevision() int64 { if m != nil { return m.HashRevision } return 0 } type HashResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // hash is the hash value computed from the responding member's KV's backend. Hash uint32 `protobuf:"varint,2,opt,name=hash,proto3" json:"hash,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *HashResponse) Reset() { *m = HashResponse{} } func (m *HashResponse) String() string { return proto.CompactTextString(m) } func (*HashResponse) ProtoMessage() {} func (*HashResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{17} } func (m *HashResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *HashResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_HashResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *HashResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_HashResponse.Merge(m, src) } func (m *HashResponse) XXX_Size() int { return m.Size() } func (m *HashResponse) XXX_DiscardUnknown() { xxx_messageInfo_HashResponse.DiscardUnknown(m) } var xxx_messageInfo_HashResponse proto.InternalMessageInfo func (m *HashResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *HashResponse) GetHash() uint32 { if m != nil { return m.Hash } return 0 } type SnapshotRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *SnapshotRequest) Reset() { *m = SnapshotRequest{} } func (m *SnapshotRequest) String() string { return proto.CompactTextString(m) } func (*SnapshotRequest) ProtoMessage() {} func (*SnapshotRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{18} } func (m *SnapshotRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *SnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_SnapshotRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *SnapshotRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_SnapshotRequest.Merge(m, src) } func (m *SnapshotRequest) XXX_Size() int { return m.Size() } func (m *SnapshotRequest) XXX_DiscardUnknown() { xxx_messageInfo_SnapshotRequest.DiscardUnknown(m) } var xxx_messageInfo_SnapshotRequest proto.InternalMessageInfo type SnapshotResponse struct { // header has the current key-value store information. The first header in the snapshot // stream indicates the point in time of the snapshot. Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // remaining_bytes is the number of blob bytes to be sent after this message RemainingBytes uint64 `protobuf:"varint,2,opt,name=remaining_bytes,json=remainingBytes,proto3" json:"remaining_bytes,omitempty"` // blob contains the next chunk of the snapshot in the snapshot stream. Blob []byte `protobuf:"bytes,3,opt,name=blob,proto3" json:"blob,omitempty"` // local version of server that created the snapshot. // In cluster with binaries with different version, each cluster can return different result. // Informs which etcd server version should be used when restoring the snapshot. Version string `protobuf:"bytes,4,opt,name=version,proto3" json:"version,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *SnapshotResponse) Reset() { *m = SnapshotResponse{} } func (m *SnapshotResponse) String() string { return proto.CompactTextString(m) } func (*SnapshotResponse) ProtoMessage() {} func (*SnapshotResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{19} } func (m *SnapshotResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *SnapshotResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_SnapshotResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *SnapshotResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_SnapshotResponse.Merge(m, src) } func (m *SnapshotResponse) XXX_Size() int { return m.Size() } func (m *SnapshotResponse) XXX_DiscardUnknown() { xxx_messageInfo_SnapshotResponse.DiscardUnknown(m) } var xxx_messageInfo_SnapshotResponse proto.InternalMessageInfo func (m *SnapshotResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *SnapshotResponse) GetRemainingBytes() uint64 { if m != nil { return m.RemainingBytes } return 0 } func (m *SnapshotResponse) GetBlob() []byte { if m != nil { return m.Blob } return nil } func (m *SnapshotResponse) GetVersion() string { if m != nil { return m.Version } return "" } type WatchRequest struct { // request_union is a request to either create a new watcher or cancel an existing watcher. // // Types that are valid to be assigned to RequestUnion: // *WatchRequest_CreateRequest // *WatchRequest_CancelRequest // *WatchRequest_ProgressRequest RequestUnion isWatchRequest_RequestUnion `protobuf_oneof:"request_union"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *WatchRequest) Reset() { *m = WatchRequest{} } func (m *WatchRequest) String() string { return proto.CompactTextString(m) } func (*WatchRequest) ProtoMessage() {} func (*WatchRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{20} } func (m *WatchRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *WatchRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_WatchRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *WatchRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_WatchRequest.Merge(m, src) } func (m *WatchRequest) XXX_Size() int { return m.Size() } func (m *WatchRequest) XXX_DiscardUnknown() { xxx_messageInfo_WatchRequest.DiscardUnknown(m) } var xxx_messageInfo_WatchRequest proto.InternalMessageInfo type isWatchRequest_RequestUnion interface { isWatchRequest_RequestUnion() MarshalTo([]byte) (int, error) Size() int } type WatchRequest_CreateRequest struct { CreateRequest *WatchCreateRequest `protobuf:"bytes,1,opt,name=create_request,json=createRequest,proto3,oneof" json:"create_request,omitempty"` } type WatchRequest_CancelRequest struct { CancelRequest *WatchCancelRequest `protobuf:"bytes,2,opt,name=cancel_request,json=cancelRequest,proto3,oneof" json:"cancel_request,omitempty"` } type WatchRequest_ProgressRequest struct { ProgressRequest *WatchProgressRequest `protobuf:"bytes,3,opt,name=progress_request,json=progressRequest,proto3,oneof" json:"progress_request,omitempty"` } func (*WatchRequest_CreateRequest) isWatchRequest_RequestUnion() {} func (*WatchRequest_CancelRequest) isWatchRequest_RequestUnion() {} func (*WatchRequest_ProgressRequest) isWatchRequest_RequestUnion() {} func (m *WatchRequest) GetRequestUnion() isWatchRequest_RequestUnion { if m != nil { return m.RequestUnion } return nil } func (m *WatchRequest) GetCreateRequest() *WatchCreateRequest { if x, ok := m.GetRequestUnion().(*WatchRequest_CreateRequest); ok { return x.CreateRequest } return nil } func (m *WatchRequest) GetCancelRequest() *WatchCancelRequest { if x, ok := m.GetRequestUnion().(*WatchRequest_CancelRequest); ok { return x.CancelRequest } return nil } func (m *WatchRequest) GetProgressRequest() *WatchProgressRequest { if x, ok := m.GetRequestUnion().(*WatchRequest_ProgressRequest); ok { return x.ProgressRequest } return nil } // XXX_OneofWrappers is for the internal use of the proto package. func (*WatchRequest) XXX_OneofWrappers() []interface{} { return []interface{}{ (*WatchRequest_CreateRequest)(nil), (*WatchRequest_CancelRequest)(nil), (*WatchRequest_ProgressRequest)(nil), } } type WatchCreateRequest struct { // key is the key to register for watching. Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // range_end is the end of the range [key, range_end) to watch. If range_end is not given, // only the key argument is watched. If range_end is equal to '\0', all keys greater than // or equal to the key argument are watched. // If the range_end is one bit larger than the given key, // then all keys with the prefix (the given key) will be watched. RangeEnd []byte `protobuf:"bytes,2,opt,name=range_end,json=rangeEnd,proto3" json:"range_end,omitempty"` // start_revision is an optional revision to watch from (inclusive). No start_revision is "now". StartRevision int64 `protobuf:"varint,3,opt,name=start_revision,json=startRevision,proto3" json:"start_revision,omitempty"` // progress_notify is set so that the etcd server will periodically send a WatchResponse with // no events to the new watcher if there are no recent events. It is useful when clients // wish to recover a disconnected watcher starting from a recent known revision. // The etcd server may decide how often it will send notifications based on current load. ProgressNotify bool `protobuf:"varint,4,opt,name=progress_notify,json=progressNotify,proto3" json:"progress_notify,omitempty"` // filters filter the events at server side before it sends back to the watcher. Filters []WatchCreateRequest_FilterType `protobuf:"varint,5,rep,packed,name=filters,proto3,enum=etcdserverpb.WatchCreateRequest_FilterType" json:"filters,omitempty"` // If prev_kv is set, created watcher gets the previous KV before the event happens. // If the previous KV is already compacted, nothing will be returned. PrevKv bool `protobuf:"varint,6,opt,name=prev_kv,json=prevKv,proto3" json:"prev_kv,omitempty"` // If watch_id is provided and non-zero, it will be assigned to this watcher. // Since creating a watcher in etcd is not a synchronous operation, // this can be used ensure that ordering is correct when creating multiple // watchers on the same stream. Creating a watcher with an ID already in // use on the stream will cause an error to be returned. WatchId int64 `protobuf:"varint,7,opt,name=watch_id,json=watchId,proto3" json:"watch_id,omitempty"` // fragment enables splitting large revisions into multiple watch responses. Fragment bool `protobuf:"varint,8,opt,name=fragment,proto3" json:"fragment,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *WatchCreateRequest) Reset() { *m = WatchCreateRequest{} } func (m *WatchCreateRequest) String() string { return proto.CompactTextString(m) } func (*WatchCreateRequest) ProtoMessage() {} func (*WatchCreateRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{21} } func (m *WatchCreateRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *WatchCreateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_WatchCreateRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *WatchCreateRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_WatchCreateRequest.Merge(m, src) } func (m *WatchCreateRequest) XXX_Size() int { return m.Size() } func (m *WatchCreateRequest) XXX_DiscardUnknown() { xxx_messageInfo_WatchCreateRequest.DiscardUnknown(m) } var xxx_messageInfo_WatchCreateRequest proto.InternalMessageInfo func (m *WatchCreateRequest) GetKey() []byte { if m != nil { return m.Key } return nil } func (m *WatchCreateRequest) GetRangeEnd() []byte { if m != nil { return m.RangeEnd } return nil } func (m *WatchCreateRequest) GetStartRevision() int64 { if m != nil { return m.StartRevision } return 0 } func (m *WatchCreateRequest) GetProgressNotify() bool { if m != nil { return m.ProgressNotify } return false } func (m *WatchCreateRequest) GetFilters() []WatchCreateRequest_FilterType { if m != nil { return m.Filters } return nil } func (m *WatchCreateRequest) GetPrevKv() bool { if m != nil { return m.PrevKv } return false } func (m *WatchCreateRequest) GetWatchId() int64 { if m != nil { return m.WatchId } return 0 } func (m *WatchCreateRequest) GetFragment() bool { if m != nil { return m.Fragment } return false } type WatchCancelRequest struct { // watch_id is the watcher id to cancel so that no more events are transmitted. WatchId int64 `protobuf:"varint,1,opt,name=watch_id,json=watchId,proto3" json:"watch_id,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *WatchCancelRequest) Reset() { *m = WatchCancelRequest{} } func (m *WatchCancelRequest) String() string { return proto.CompactTextString(m) } func (*WatchCancelRequest) ProtoMessage() {} func (*WatchCancelRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{22} } func (m *WatchCancelRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *WatchCancelRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_WatchCancelRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *WatchCancelRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_WatchCancelRequest.Merge(m, src) } func (m *WatchCancelRequest) XXX_Size() int { return m.Size() } func (m *WatchCancelRequest) XXX_DiscardUnknown() { xxx_messageInfo_WatchCancelRequest.DiscardUnknown(m) } var xxx_messageInfo_WatchCancelRequest proto.InternalMessageInfo func (m *WatchCancelRequest) GetWatchId() int64 { if m != nil { return m.WatchId } return 0 } // Requests the a watch stream progress status be sent in the watch response stream as soon as // possible. type WatchProgressRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *WatchProgressRequest) Reset() { *m = WatchProgressRequest{} } func (m *WatchProgressRequest) String() string { return proto.CompactTextString(m) } func (*WatchProgressRequest) ProtoMessage() {} func (*WatchProgressRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{23} } func (m *WatchProgressRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *WatchProgressRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_WatchProgressRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *WatchProgressRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_WatchProgressRequest.Merge(m, src) } func (m *WatchProgressRequest) XXX_Size() int { return m.Size() } func (m *WatchProgressRequest) XXX_DiscardUnknown() { xxx_messageInfo_WatchProgressRequest.DiscardUnknown(m) } var xxx_messageInfo_WatchProgressRequest proto.InternalMessageInfo type WatchResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // watch_id is the ID of the watcher that corresponds to the response. WatchId int64 `protobuf:"varint,2,opt,name=watch_id,json=watchId,proto3" json:"watch_id,omitempty"` // created is set to true if the response is for a create watch request. // The client should record the watch_id and expect to receive events for // the created watcher from the same stream. // All events sent to the created watcher will attach with the same watch_id. Created bool `protobuf:"varint,3,opt,name=created,proto3" json:"created,omitempty"` // canceled is set to true if the response is for a cancel watch request // or if the start_revision has already been compacted. // No further events will be sent to the canceled watcher. Canceled bool `protobuf:"varint,4,opt,name=canceled,proto3" json:"canceled,omitempty"` // compact_revision is set to the minimum index if a watcher tries to watch // at a compacted index. // // This happens when creating a watcher at a compacted revision or the watcher cannot // catch up with the progress of the key-value store. // // The client should treat the watcher as canceled and should not try to create any // watcher with the same start_revision again. CompactRevision int64 `protobuf:"varint,5,opt,name=compact_revision,json=compactRevision,proto3" json:"compact_revision,omitempty"` // cancel_reason indicates the reason for canceling the watcher. CancelReason string `protobuf:"bytes,6,opt,name=cancel_reason,json=cancelReason,proto3" json:"cancel_reason,omitempty"` // framgment is true if large watch response was split over multiple responses. Fragment bool `protobuf:"varint,7,opt,name=fragment,proto3" json:"fragment,omitempty"` Events []*mvccpb.Event `protobuf:"bytes,11,rep,name=events,proto3" json:"events,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *WatchResponse) Reset() { *m = WatchResponse{} } func (m *WatchResponse) String() string { return proto.CompactTextString(m) } func (*WatchResponse) ProtoMessage() {} func (*WatchResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{24} } func (m *WatchResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *WatchResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_WatchResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *WatchResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_WatchResponse.Merge(m, src) } func (m *WatchResponse) XXX_Size() int { return m.Size() } func (m *WatchResponse) XXX_DiscardUnknown() { xxx_messageInfo_WatchResponse.DiscardUnknown(m) } var xxx_messageInfo_WatchResponse proto.InternalMessageInfo func (m *WatchResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *WatchResponse) GetWatchId() int64 { if m != nil { return m.WatchId } return 0 } func (m *WatchResponse) GetCreated() bool { if m != nil { return m.Created } return false } func (m *WatchResponse) GetCanceled() bool { if m != nil { return m.Canceled } return false } func (m *WatchResponse) GetCompactRevision() int64 { if m != nil { return m.CompactRevision } return 0 } func (m *WatchResponse) GetCancelReason() string { if m != nil { return m.CancelReason } return "" } func (m *WatchResponse) GetFragment() bool { if m != nil { return m.Fragment } return false } func (m *WatchResponse) GetEvents() []*mvccpb.Event { if m != nil { return m.Events } return nil } type LeaseGrantRequest struct { // TTL is the advisory time-to-live in seconds. Expired lease will return -1. TTL int64 `protobuf:"varint,1,opt,name=TTL,proto3" json:"TTL,omitempty"` // ID is the requested ID for the lease. If ID is set to 0, the lessor chooses an ID. ID int64 `protobuf:"varint,2,opt,name=ID,proto3" json:"ID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseGrantRequest) Reset() { *m = LeaseGrantRequest{} } func (m *LeaseGrantRequest) String() string { return proto.CompactTextString(m) } func (*LeaseGrantRequest) ProtoMessage() {} func (*LeaseGrantRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{25} } func (m *LeaseGrantRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseGrantRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseGrantRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseGrantRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseGrantRequest.Merge(m, src) } func (m *LeaseGrantRequest) XXX_Size() int { return m.Size() } func (m *LeaseGrantRequest) XXX_DiscardUnknown() { xxx_messageInfo_LeaseGrantRequest.DiscardUnknown(m) } var xxx_messageInfo_LeaseGrantRequest proto.InternalMessageInfo func (m *LeaseGrantRequest) GetTTL() int64 { if m != nil { return m.TTL } return 0 } func (m *LeaseGrantRequest) GetID() int64 { if m != nil { return m.ID } return 0 } type LeaseGrantResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // ID is the lease ID for the granted lease. ID int64 `protobuf:"varint,2,opt,name=ID,proto3" json:"ID,omitempty"` // TTL is the server chosen lease time-to-live in seconds. TTL int64 `protobuf:"varint,3,opt,name=TTL,proto3" json:"TTL,omitempty"` Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseGrantResponse) Reset() { *m = LeaseGrantResponse{} } func (m *LeaseGrantResponse) String() string { return proto.CompactTextString(m) } func (*LeaseGrantResponse) ProtoMessage() {} func (*LeaseGrantResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{26} } func (m *LeaseGrantResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseGrantResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseGrantResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseGrantResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseGrantResponse.Merge(m, src) } func (m *LeaseGrantResponse) XXX_Size() int { return m.Size() } func (m *LeaseGrantResponse) XXX_DiscardUnknown() { xxx_messageInfo_LeaseGrantResponse.DiscardUnknown(m) } var xxx_messageInfo_LeaseGrantResponse proto.InternalMessageInfo func (m *LeaseGrantResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *LeaseGrantResponse) GetID() int64 { if m != nil { return m.ID } return 0 } func (m *LeaseGrantResponse) GetTTL() int64 { if m != nil { return m.TTL } return 0 } func (m *LeaseGrantResponse) GetError() string { if m != nil { return m.Error } return "" } type LeaseRevokeRequest struct { // ID is the lease ID to revoke. When the ID is revoked, all associated keys will be deleted. ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseRevokeRequest) Reset() { *m = LeaseRevokeRequest{} } func (m *LeaseRevokeRequest) String() string { return proto.CompactTextString(m) } func (*LeaseRevokeRequest) ProtoMessage() {} func (*LeaseRevokeRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{27} } func (m *LeaseRevokeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseRevokeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseRevokeRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseRevokeRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseRevokeRequest.Merge(m, src) } func (m *LeaseRevokeRequest) XXX_Size() int { return m.Size() } func (m *LeaseRevokeRequest) XXX_DiscardUnknown() { xxx_messageInfo_LeaseRevokeRequest.DiscardUnknown(m) } var xxx_messageInfo_LeaseRevokeRequest proto.InternalMessageInfo func (m *LeaseRevokeRequest) GetID() int64 { if m != nil { return m.ID } return 0 } type LeaseRevokeResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseRevokeResponse) Reset() { *m = LeaseRevokeResponse{} } func (m *LeaseRevokeResponse) String() string { return proto.CompactTextString(m) } func (*LeaseRevokeResponse) ProtoMessage() {} func (*LeaseRevokeResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{28} } func (m *LeaseRevokeResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseRevokeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseRevokeResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseRevokeResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseRevokeResponse.Merge(m, src) } func (m *LeaseRevokeResponse) XXX_Size() int { return m.Size() } func (m *LeaseRevokeResponse) XXX_DiscardUnknown() { xxx_messageInfo_LeaseRevokeResponse.DiscardUnknown(m) } var xxx_messageInfo_LeaseRevokeResponse proto.InternalMessageInfo func (m *LeaseRevokeResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type LeaseCheckpoint struct { // ID is the lease ID to checkpoint. ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` // Remaining_TTL is the remaining time until expiry of the lease. Remaining_TTL int64 `protobuf:"varint,2,opt,name=remaining_TTL,json=remainingTTL,proto3" json:"remaining_TTL,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseCheckpoint) Reset() { *m = LeaseCheckpoint{} } func (m *LeaseCheckpoint) String() string { return proto.CompactTextString(m) } func (*LeaseCheckpoint) ProtoMessage() {} func (*LeaseCheckpoint) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{29} } func (m *LeaseCheckpoint) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseCheckpoint) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseCheckpoint.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseCheckpoint) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseCheckpoint.Merge(m, src) } func (m *LeaseCheckpoint) XXX_Size() int { return m.Size() } func (m *LeaseCheckpoint) XXX_DiscardUnknown() { xxx_messageInfo_LeaseCheckpoint.DiscardUnknown(m) } var xxx_messageInfo_LeaseCheckpoint proto.InternalMessageInfo func (m *LeaseCheckpoint) GetID() int64 { if m != nil { return m.ID } return 0 } func (m *LeaseCheckpoint) GetRemaining_TTL() int64 { if m != nil { return m.Remaining_TTL } return 0 } type LeaseCheckpointRequest struct { Checkpoints []*LeaseCheckpoint `protobuf:"bytes,1,rep,name=checkpoints,proto3" json:"checkpoints,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseCheckpointRequest) Reset() { *m = LeaseCheckpointRequest{} } func (m *LeaseCheckpointRequest) String() string { return proto.CompactTextString(m) } func (*LeaseCheckpointRequest) ProtoMessage() {} func (*LeaseCheckpointRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{30} } func (m *LeaseCheckpointRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseCheckpointRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseCheckpointRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseCheckpointRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseCheckpointRequest.Merge(m, src) } func (m *LeaseCheckpointRequest) XXX_Size() int { return m.Size() } func (m *LeaseCheckpointRequest) XXX_DiscardUnknown() { xxx_messageInfo_LeaseCheckpointRequest.DiscardUnknown(m) } var xxx_messageInfo_LeaseCheckpointRequest proto.InternalMessageInfo func (m *LeaseCheckpointRequest) GetCheckpoints() []*LeaseCheckpoint { if m != nil { return m.Checkpoints } return nil } type LeaseCheckpointResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseCheckpointResponse) Reset() { *m = LeaseCheckpointResponse{} } func (m *LeaseCheckpointResponse) String() string { return proto.CompactTextString(m) } func (*LeaseCheckpointResponse) ProtoMessage() {} func (*LeaseCheckpointResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{31} } func (m *LeaseCheckpointResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseCheckpointResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseCheckpointResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseCheckpointResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseCheckpointResponse.Merge(m, src) } func (m *LeaseCheckpointResponse) XXX_Size() int { return m.Size() } func (m *LeaseCheckpointResponse) XXX_DiscardUnknown() { xxx_messageInfo_LeaseCheckpointResponse.DiscardUnknown(m) } var xxx_messageInfo_LeaseCheckpointResponse proto.InternalMessageInfo func (m *LeaseCheckpointResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type LeaseKeepAliveRequest struct { // ID is the lease ID for the lease to keep alive. ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseKeepAliveRequest) Reset() { *m = LeaseKeepAliveRequest{} } func (m *LeaseKeepAliveRequest) String() string { return proto.CompactTextString(m) } func (*LeaseKeepAliveRequest) ProtoMessage() {} func (*LeaseKeepAliveRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{32} } func (m *LeaseKeepAliveRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseKeepAliveRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseKeepAliveRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseKeepAliveRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseKeepAliveRequest.Merge(m, src) } func (m *LeaseKeepAliveRequest) XXX_Size() int { return m.Size() } func (m *LeaseKeepAliveRequest) XXX_DiscardUnknown() { xxx_messageInfo_LeaseKeepAliveRequest.DiscardUnknown(m) } var xxx_messageInfo_LeaseKeepAliveRequest proto.InternalMessageInfo func (m *LeaseKeepAliveRequest) GetID() int64 { if m != nil { return m.ID } return 0 } type LeaseKeepAliveResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // ID is the lease ID from the keep alive request. ID int64 `protobuf:"varint,2,opt,name=ID,proto3" json:"ID,omitempty"` // TTL is the new time-to-live for the lease. TTL int64 `protobuf:"varint,3,opt,name=TTL,proto3" json:"TTL,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseKeepAliveResponse) Reset() { *m = LeaseKeepAliveResponse{} } func (m *LeaseKeepAliveResponse) String() string { return proto.CompactTextString(m) } func (*LeaseKeepAliveResponse) ProtoMessage() {} func (*LeaseKeepAliveResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{33} } func (m *LeaseKeepAliveResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseKeepAliveResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseKeepAliveResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseKeepAliveResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseKeepAliveResponse.Merge(m, src) } func (m *LeaseKeepAliveResponse) XXX_Size() int { return m.Size() } func (m *LeaseKeepAliveResponse) XXX_DiscardUnknown() { xxx_messageInfo_LeaseKeepAliveResponse.DiscardUnknown(m) } var xxx_messageInfo_LeaseKeepAliveResponse proto.InternalMessageInfo func (m *LeaseKeepAliveResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *LeaseKeepAliveResponse) GetID() int64 { if m != nil { return m.ID } return 0 } func (m *LeaseKeepAliveResponse) GetTTL() int64 { if m != nil { return m.TTL } return 0 } type LeaseTimeToLiveRequest struct { // ID is the lease ID for the lease. ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` // keys is true to query all the keys attached to this lease. Keys bool `protobuf:"varint,2,opt,name=keys,proto3" json:"keys,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseTimeToLiveRequest) Reset() { *m = LeaseTimeToLiveRequest{} } func (m *LeaseTimeToLiveRequest) String() string { return proto.CompactTextString(m) } func (*LeaseTimeToLiveRequest) ProtoMessage() {} func (*LeaseTimeToLiveRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{34} } func (m *LeaseTimeToLiveRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseTimeToLiveRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseTimeToLiveRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseTimeToLiveRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseTimeToLiveRequest.Merge(m, src) } func (m *LeaseTimeToLiveRequest) XXX_Size() int { return m.Size() } func (m *LeaseTimeToLiveRequest) XXX_DiscardUnknown() { xxx_messageInfo_LeaseTimeToLiveRequest.DiscardUnknown(m) } var xxx_messageInfo_LeaseTimeToLiveRequest proto.InternalMessageInfo func (m *LeaseTimeToLiveRequest) GetID() int64 { if m != nil { return m.ID } return 0 } func (m *LeaseTimeToLiveRequest) GetKeys() bool { if m != nil { return m.Keys } return false } type LeaseTimeToLiveResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // ID is the lease ID from the keep alive request. ID int64 `protobuf:"varint,2,opt,name=ID,proto3" json:"ID,omitempty"` // TTL is the remaining TTL in seconds for the lease; the lease will expire in under TTL+1 seconds. TTL int64 `protobuf:"varint,3,opt,name=TTL,proto3" json:"TTL,omitempty"` // GrantedTTL is the initial granted time in seconds upon lease creation/renewal. GrantedTTL int64 `protobuf:"varint,4,opt,name=grantedTTL,proto3" json:"grantedTTL,omitempty"` // Keys is the list of keys attached to this lease. Keys [][]byte `protobuf:"bytes,5,rep,name=keys,proto3" json:"keys,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseTimeToLiveResponse) Reset() { *m = LeaseTimeToLiveResponse{} } func (m *LeaseTimeToLiveResponse) String() string { return proto.CompactTextString(m) } func (*LeaseTimeToLiveResponse) ProtoMessage() {} func (*LeaseTimeToLiveResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{35} } func (m *LeaseTimeToLiveResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseTimeToLiveResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseTimeToLiveResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseTimeToLiveResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseTimeToLiveResponse.Merge(m, src) } func (m *LeaseTimeToLiveResponse) XXX_Size() int { return m.Size() } func (m *LeaseTimeToLiveResponse) XXX_DiscardUnknown() { xxx_messageInfo_LeaseTimeToLiveResponse.DiscardUnknown(m) } var xxx_messageInfo_LeaseTimeToLiveResponse proto.InternalMessageInfo func (m *LeaseTimeToLiveResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *LeaseTimeToLiveResponse) GetID() int64 { if m != nil { return m.ID } return 0 } func (m *LeaseTimeToLiveResponse) GetTTL() int64 { if m != nil { return m.TTL } return 0 } func (m *LeaseTimeToLiveResponse) GetGrantedTTL() int64 { if m != nil { return m.GrantedTTL } return 0 } func (m *LeaseTimeToLiveResponse) GetKeys() [][]byte { if m != nil { return m.Keys } return nil } type LeaseLeasesRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseLeasesRequest) Reset() { *m = LeaseLeasesRequest{} } func (m *LeaseLeasesRequest) String() string { return proto.CompactTextString(m) } func (*LeaseLeasesRequest) ProtoMessage() {} func (*LeaseLeasesRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{36} } func (m *LeaseLeasesRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseLeasesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseLeasesRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseLeasesRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseLeasesRequest.Merge(m, src) } func (m *LeaseLeasesRequest) XXX_Size() int { return m.Size() } func (m *LeaseLeasesRequest) XXX_DiscardUnknown() { xxx_messageInfo_LeaseLeasesRequest.DiscardUnknown(m) } var xxx_messageInfo_LeaseLeasesRequest proto.InternalMessageInfo type LeaseStatus struct { ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseStatus) Reset() { *m = LeaseStatus{} } func (m *LeaseStatus) String() string { return proto.CompactTextString(m) } func (*LeaseStatus) ProtoMessage() {} func (*LeaseStatus) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{37} } func (m *LeaseStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseStatus.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseStatus) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseStatus.Merge(m, src) } func (m *LeaseStatus) XXX_Size() int { return m.Size() } func (m *LeaseStatus) XXX_DiscardUnknown() { xxx_messageInfo_LeaseStatus.DiscardUnknown(m) } var xxx_messageInfo_LeaseStatus proto.InternalMessageInfo func (m *LeaseStatus) GetID() int64 { if m != nil { return m.ID } return 0 } type LeaseLeasesResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` Leases []*LeaseStatus `protobuf:"bytes,2,rep,name=leases,proto3" json:"leases,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseLeasesResponse) Reset() { *m = LeaseLeasesResponse{} } func (m *LeaseLeasesResponse) String() string { return proto.CompactTextString(m) } func (*LeaseLeasesResponse) ProtoMessage() {} func (*LeaseLeasesResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{38} } func (m *LeaseLeasesResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseLeasesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseLeasesResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseLeasesResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseLeasesResponse.Merge(m, src) } func (m *LeaseLeasesResponse) XXX_Size() int { return m.Size() } func (m *LeaseLeasesResponse) XXX_DiscardUnknown() { xxx_messageInfo_LeaseLeasesResponse.DiscardUnknown(m) } var xxx_messageInfo_LeaseLeasesResponse proto.InternalMessageInfo func (m *LeaseLeasesResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *LeaseLeasesResponse) GetLeases() []*LeaseStatus { if m != nil { return m.Leases } return nil } type Member struct { // ID is the member ID for this member. ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` // name is the human-readable name of the member. If the member is not started, the name will be an empty string. Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` // peerURLs is the list of URLs the member exposes to the cluster for communication. PeerURLs []string `protobuf:"bytes,3,rep,name=peerURLs,proto3" json:"peerURLs,omitempty"` // clientURLs is the list of URLs the member exposes to clients for communication. If the member is not started, clientURLs will be empty. ClientURLs []string `protobuf:"bytes,4,rep,name=clientURLs,proto3" json:"clientURLs,omitempty"` // isLearner indicates if the member is raft learner. IsLearner bool `protobuf:"varint,5,opt,name=isLearner,proto3" json:"isLearner,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Member) Reset() { *m = Member{} } func (m *Member) String() string { return proto.CompactTextString(m) } func (*Member) ProtoMessage() {} func (*Member) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{39} } func (m *Member) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Member) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Member.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Member) XXX_Merge(src proto.Message) { xxx_messageInfo_Member.Merge(m, src) } func (m *Member) XXX_Size() int { return m.Size() } func (m *Member) XXX_DiscardUnknown() { xxx_messageInfo_Member.DiscardUnknown(m) } var xxx_messageInfo_Member proto.InternalMessageInfo func (m *Member) GetID() uint64 { if m != nil { return m.ID } return 0 } func (m *Member) GetName() string { if m != nil { return m.Name } return "" } func (m *Member) GetPeerURLs() []string { if m != nil { return m.PeerURLs } return nil } func (m *Member) GetClientURLs() []string { if m != nil { return m.ClientURLs } return nil } func (m *Member) GetIsLearner() bool { if m != nil { return m.IsLearner } return false } type MemberAddRequest struct { // peerURLs is the list of URLs the added member will use to communicate with the cluster. PeerURLs []string `protobuf:"bytes,1,rep,name=peerURLs,proto3" json:"peerURLs,omitempty"` // isLearner indicates if the added member is raft learner. IsLearner bool `protobuf:"varint,2,opt,name=isLearner,proto3" json:"isLearner,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MemberAddRequest) Reset() { *m = MemberAddRequest{} } func (m *MemberAddRequest) String() string { return proto.CompactTextString(m) } func (*MemberAddRequest) ProtoMessage() {} func (*MemberAddRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{40} } func (m *MemberAddRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MemberAddRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MemberAddRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MemberAddRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_MemberAddRequest.Merge(m, src) } func (m *MemberAddRequest) XXX_Size() int { return m.Size() } func (m *MemberAddRequest) XXX_DiscardUnknown() { xxx_messageInfo_MemberAddRequest.DiscardUnknown(m) } var xxx_messageInfo_MemberAddRequest proto.InternalMessageInfo func (m *MemberAddRequest) GetPeerURLs() []string { if m != nil { return m.PeerURLs } return nil } func (m *MemberAddRequest) GetIsLearner() bool { if m != nil { return m.IsLearner } return false } type MemberAddResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // member is the member information for the added member. Member *Member `protobuf:"bytes,2,opt,name=member,proto3" json:"member,omitempty"` // members is a list of all members after adding the new member. Members []*Member `protobuf:"bytes,3,rep,name=members,proto3" json:"members,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MemberAddResponse) Reset() { *m = MemberAddResponse{} } func (m *MemberAddResponse) String() string { return proto.CompactTextString(m) } func (*MemberAddResponse) ProtoMessage() {} func (*MemberAddResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{41} } func (m *MemberAddResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MemberAddResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MemberAddResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MemberAddResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_MemberAddResponse.Merge(m, src) } func (m *MemberAddResponse) XXX_Size() int { return m.Size() } func (m *MemberAddResponse) XXX_DiscardUnknown() { xxx_messageInfo_MemberAddResponse.DiscardUnknown(m) } var xxx_messageInfo_MemberAddResponse proto.InternalMessageInfo func (m *MemberAddResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *MemberAddResponse) GetMember() *Member { if m != nil { return m.Member } return nil } func (m *MemberAddResponse) GetMembers() []*Member { if m != nil { return m.Members } return nil } type MemberRemoveRequest struct { // ID is the member ID of the member to remove. ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MemberRemoveRequest) Reset() { *m = MemberRemoveRequest{} } func (m *MemberRemoveRequest) String() string { return proto.CompactTextString(m) } func (*MemberRemoveRequest) ProtoMessage() {} func (*MemberRemoveRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{42} } func (m *MemberRemoveRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MemberRemoveRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MemberRemoveRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MemberRemoveRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_MemberRemoveRequest.Merge(m, src) } func (m *MemberRemoveRequest) XXX_Size() int { return m.Size() } func (m *MemberRemoveRequest) XXX_DiscardUnknown() { xxx_messageInfo_MemberRemoveRequest.DiscardUnknown(m) } var xxx_messageInfo_MemberRemoveRequest proto.InternalMessageInfo func (m *MemberRemoveRequest) GetID() uint64 { if m != nil { return m.ID } return 0 } type MemberRemoveResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // members is a list of all members after removing the member. Members []*Member `protobuf:"bytes,2,rep,name=members,proto3" json:"members,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MemberRemoveResponse) Reset() { *m = MemberRemoveResponse{} } func (m *MemberRemoveResponse) String() string { return proto.CompactTextString(m) } func (*MemberRemoveResponse) ProtoMessage() {} func (*MemberRemoveResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{43} } func (m *MemberRemoveResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MemberRemoveResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MemberRemoveResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MemberRemoveResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_MemberRemoveResponse.Merge(m, src) } func (m *MemberRemoveResponse) XXX_Size() int { return m.Size() } func (m *MemberRemoveResponse) XXX_DiscardUnknown() { xxx_messageInfo_MemberRemoveResponse.DiscardUnknown(m) } var xxx_messageInfo_MemberRemoveResponse proto.InternalMessageInfo func (m *MemberRemoveResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *MemberRemoveResponse) GetMembers() []*Member { if m != nil { return m.Members } return nil } type MemberUpdateRequest struct { // ID is the member ID of the member to update. ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` // peerURLs is the new list of URLs the member will use to communicate with the cluster. PeerURLs []string `protobuf:"bytes,2,rep,name=peerURLs,proto3" json:"peerURLs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MemberUpdateRequest) Reset() { *m = MemberUpdateRequest{} } func (m *MemberUpdateRequest) String() string { return proto.CompactTextString(m) } func (*MemberUpdateRequest) ProtoMessage() {} func (*MemberUpdateRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{44} } func (m *MemberUpdateRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MemberUpdateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MemberUpdateRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MemberUpdateRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_MemberUpdateRequest.Merge(m, src) } func (m *MemberUpdateRequest) XXX_Size() int { return m.Size() } func (m *MemberUpdateRequest) XXX_DiscardUnknown() { xxx_messageInfo_MemberUpdateRequest.DiscardUnknown(m) } var xxx_messageInfo_MemberUpdateRequest proto.InternalMessageInfo func (m *MemberUpdateRequest) GetID() uint64 { if m != nil { return m.ID } return 0 } func (m *MemberUpdateRequest) GetPeerURLs() []string { if m != nil { return m.PeerURLs } return nil } type MemberUpdateResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // members is a list of all members after updating the member. Members []*Member `protobuf:"bytes,2,rep,name=members,proto3" json:"members,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MemberUpdateResponse) Reset() { *m = MemberUpdateResponse{} } func (m *MemberUpdateResponse) String() string { return proto.CompactTextString(m) } func (*MemberUpdateResponse) ProtoMessage() {} func (*MemberUpdateResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{45} } func (m *MemberUpdateResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MemberUpdateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MemberUpdateResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MemberUpdateResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_MemberUpdateResponse.Merge(m, src) } func (m *MemberUpdateResponse) XXX_Size() int { return m.Size() } func (m *MemberUpdateResponse) XXX_DiscardUnknown() { xxx_messageInfo_MemberUpdateResponse.DiscardUnknown(m) } var xxx_messageInfo_MemberUpdateResponse proto.InternalMessageInfo func (m *MemberUpdateResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *MemberUpdateResponse) GetMembers() []*Member { if m != nil { return m.Members } return nil } type MemberListRequest struct { Linearizable bool `protobuf:"varint,1,opt,name=linearizable,proto3" json:"linearizable,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MemberListRequest) Reset() { *m = MemberListRequest{} } func (m *MemberListRequest) String() string { return proto.CompactTextString(m) } func (*MemberListRequest) ProtoMessage() {} func (*MemberListRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{46} } func (m *MemberListRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MemberListRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MemberListRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MemberListRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_MemberListRequest.Merge(m, src) } func (m *MemberListRequest) XXX_Size() int { return m.Size() } func (m *MemberListRequest) XXX_DiscardUnknown() { xxx_messageInfo_MemberListRequest.DiscardUnknown(m) } var xxx_messageInfo_MemberListRequest proto.InternalMessageInfo func (m *MemberListRequest) GetLinearizable() bool { if m != nil { return m.Linearizable } return false } type MemberListResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // members is a list of all members associated with the cluster. Members []*Member `protobuf:"bytes,2,rep,name=members,proto3" json:"members,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MemberListResponse) Reset() { *m = MemberListResponse{} } func (m *MemberListResponse) String() string { return proto.CompactTextString(m) } func (*MemberListResponse) ProtoMessage() {} func (*MemberListResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{47} } func (m *MemberListResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MemberListResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MemberListResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MemberListResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_MemberListResponse.Merge(m, src) } func (m *MemberListResponse) XXX_Size() int { return m.Size() } func (m *MemberListResponse) XXX_DiscardUnknown() { xxx_messageInfo_MemberListResponse.DiscardUnknown(m) } var xxx_messageInfo_MemberListResponse proto.InternalMessageInfo func (m *MemberListResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *MemberListResponse) GetMembers() []*Member { if m != nil { return m.Members } return nil } type MemberPromoteRequest struct { // ID is the member ID of the member to promote. ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MemberPromoteRequest) Reset() { *m = MemberPromoteRequest{} } func (m *MemberPromoteRequest) String() string { return proto.CompactTextString(m) } func (*MemberPromoteRequest) ProtoMessage() {} func (*MemberPromoteRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{48} } func (m *MemberPromoteRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MemberPromoteRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MemberPromoteRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MemberPromoteRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_MemberPromoteRequest.Merge(m, src) } func (m *MemberPromoteRequest) XXX_Size() int { return m.Size() } func (m *MemberPromoteRequest) XXX_DiscardUnknown() { xxx_messageInfo_MemberPromoteRequest.DiscardUnknown(m) } var xxx_messageInfo_MemberPromoteRequest proto.InternalMessageInfo func (m *MemberPromoteRequest) GetID() uint64 { if m != nil { return m.ID } return 0 } type MemberPromoteResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // members is a list of all members after promoting the member. Members []*Member `protobuf:"bytes,2,rep,name=members,proto3" json:"members,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MemberPromoteResponse) Reset() { *m = MemberPromoteResponse{} } func (m *MemberPromoteResponse) String() string { return proto.CompactTextString(m) } func (*MemberPromoteResponse) ProtoMessage() {} func (*MemberPromoteResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{49} } func (m *MemberPromoteResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MemberPromoteResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MemberPromoteResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MemberPromoteResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_MemberPromoteResponse.Merge(m, src) } func (m *MemberPromoteResponse) XXX_Size() int { return m.Size() } func (m *MemberPromoteResponse) XXX_DiscardUnknown() { xxx_messageInfo_MemberPromoteResponse.DiscardUnknown(m) } var xxx_messageInfo_MemberPromoteResponse proto.InternalMessageInfo func (m *MemberPromoteResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *MemberPromoteResponse) GetMembers() []*Member { if m != nil { return m.Members } return nil } type DefragmentRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *DefragmentRequest) Reset() { *m = DefragmentRequest{} } func (m *DefragmentRequest) String() string { return proto.CompactTextString(m) } func (*DefragmentRequest) ProtoMessage() {} func (*DefragmentRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{50} } func (m *DefragmentRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *DefragmentRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_DefragmentRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *DefragmentRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_DefragmentRequest.Merge(m, src) } func (m *DefragmentRequest) XXX_Size() int { return m.Size() } func (m *DefragmentRequest) XXX_DiscardUnknown() { xxx_messageInfo_DefragmentRequest.DiscardUnknown(m) } var xxx_messageInfo_DefragmentRequest proto.InternalMessageInfo type DefragmentResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *DefragmentResponse) Reset() { *m = DefragmentResponse{} } func (m *DefragmentResponse) String() string { return proto.CompactTextString(m) } func (*DefragmentResponse) ProtoMessage() {} func (*DefragmentResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{51} } func (m *DefragmentResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *DefragmentResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_DefragmentResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *DefragmentResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_DefragmentResponse.Merge(m, src) } func (m *DefragmentResponse) XXX_Size() int { return m.Size() } func (m *DefragmentResponse) XXX_DiscardUnknown() { xxx_messageInfo_DefragmentResponse.DiscardUnknown(m) } var xxx_messageInfo_DefragmentResponse proto.InternalMessageInfo func (m *DefragmentResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type MoveLeaderRequest struct { // targetID is the node ID for the new leader. TargetID uint64 `protobuf:"varint,1,opt,name=targetID,proto3" json:"targetID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MoveLeaderRequest) Reset() { *m = MoveLeaderRequest{} } func (m *MoveLeaderRequest) String() string { return proto.CompactTextString(m) } func (*MoveLeaderRequest) ProtoMessage() {} func (*MoveLeaderRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{52} } func (m *MoveLeaderRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MoveLeaderRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MoveLeaderRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MoveLeaderRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_MoveLeaderRequest.Merge(m, src) } func (m *MoveLeaderRequest) XXX_Size() int { return m.Size() } func (m *MoveLeaderRequest) XXX_DiscardUnknown() { xxx_messageInfo_MoveLeaderRequest.DiscardUnknown(m) } var xxx_messageInfo_MoveLeaderRequest proto.InternalMessageInfo func (m *MoveLeaderRequest) GetTargetID() uint64 { if m != nil { return m.TargetID } return 0 } type MoveLeaderResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *MoveLeaderResponse) Reset() { *m = MoveLeaderResponse{} } func (m *MoveLeaderResponse) String() string { return proto.CompactTextString(m) } func (*MoveLeaderResponse) ProtoMessage() {} func (*MoveLeaderResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{53} } func (m *MoveLeaderResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *MoveLeaderResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_MoveLeaderResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *MoveLeaderResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_MoveLeaderResponse.Merge(m, src) } func (m *MoveLeaderResponse) XXX_Size() int { return m.Size() } func (m *MoveLeaderResponse) XXX_DiscardUnknown() { xxx_messageInfo_MoveLeaderResponse.DiscardUnknown(m) } var xxx_messageInfo_MoveLeaderResponse proto.InternalMessageInfo func (m *MoveLeaderResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type AlarmRequest struct { // action is the kind of alarm request to issue. The action // may GET alarm statuses, ACTIVATE an alarm, or DEACTIVATE a // raised alarm. Action AlarmRequest_AlarmAction `protobuf:"varint,1,opt,name=action,proto3,enum=etcdserverpb.AlarmRequest_AlarmAction" json:"action,omitempty"` // memberID is the ID of the member associated with the alarm. If memberID is 0, the // alarm request covers all members. MemberID uint64 `protobuf:"varint,2,opt,name=memberID,proto3" json:"memberID,omitempty"` // alarm is the type of alarm to consider for this request. Alarm AlarmType `protobuf:"varint,3,opt,name=alarm,proto3,enum=etcdserverpb.AlarmType" json:"alarm,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AlarmRequest) Reset() { *m = AlarmRequest{} } func (m *AlarmRequest) String() string { return proto.CompactTextString(m) } func (*AlarmRequest) ProtoMessage() {} func (*AlarmRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{54} } func (m *AlarmRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AlarmRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AlarmRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AlarmRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AlarmRequest.Merge(m, src) } func (m *AlarmRequest) XXX_Size() int { return m.Size() } func (m *AlarmRequest) XXX_DiscardUnknown() { xxx_messageInfo_AlarmRequest.DiscardUnknown(m) } var xxx_messageInfo_AlarmRequest proto.InternalMessageInfo func (m *AlarmRequest) GetAction() AlarmRequest_AlarmAction { if m != nil { return m.Action } return AlarmRequest_GET } func (m *AlarmRequest) GetMemberID() uint64 { if m != nil { return m.MemberID } return 0 } func (m *AlarmRequest) GetAlarm() AlarmType { if m != nil { return m.Alarm } return AlarmType_NONE } type AlarmMember struct { // memberID is the ID of the member associated with the raised alarm. MemberID uint64 `protobuf:"varint,1,opt,name=memberID,proto3" json:"memberID,omitempty"` // alarm is the type of alarm which has been raised. Alarm AlarmType `protobuf:"varint,2,opt,name=alarm,proto3,enum=etcdserverpb.AlarmType" json:"alarm,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AlarmMember) Reset() { *m = AlarmMember{} } func (m *AlarmMember) String() string { return proto.CompactTextString(m) } func (*AlarmMember) ProtoMessage() {} func (*AlarmMember) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{55} } func (m *AlarmMember) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AlarmMember) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AlarmMember.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AlarmMember) XXX_Merge(src proto.Message) { xxx_messageInfo_AlarmMember.Merge(m, src) } func (m *AlarmMember) XXX_Size() int { return m.Size() } func (m *AlarmMember) XXX_DiscardUnknown() { xxx_messageInfo_AlarmMember.DiscardUnknown(m) } var xxx_messageInfo_AlarmMember proto.InternalMessageInfo func (m *AlarmMember) GetMemberID() uint64 { if m != nil { return m.MemberID } return 0 } func (m *AlarmMember) GetAlarm() AlarmType { if m != nil { return m.Alarm } return AlarmType_NONE } type AlarmResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // alarms is a list of alarms associated with the alarm request. Alarms []*AlarmMember `protobuf:"bytes,2,rep,name=alarms,proto3" json:"alarms,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AlarmResponse) Reset() { *m = AlarmResponse{} } func (m *AlarmResponse) String() string { return proto.CompactTextString(m) } func (*AlarmResponse) ProtoMessage() {} func (*AlarmResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{56} } func (m *AlarmResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AlarmResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AlarmResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AlarmResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AlarmResponse.Merge(m, src) } func (m *AlarmResponse) XXX_Size() int { return m.Size() } func (m *AlarmResponse) XXX_DiscardUnknown() { xxx_messageInfo_AlarmResponse.DiscardUnknown(m) } var xxx_messageInfo_AlarmResponse proto.InternalMessageInfo func (m *AlarmResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *AlarmResponse) GetAlarms() []*AlarmMember { if m != nil { return m.Alarms } return nil } type DowngradeRequest struct { // action is the kind of downgrade request to issue. The action may // VALIDATE the target version, DOWNGRADE the cluster version, // or CANCEL the current downgrading job. Action DowngradeRequest_DowngradeAction `protobuf:"varint,1,opt,name=action,proto3,enum=etcdserverpb.DowngradeRequest_DowngradeAction" json:"action,omitempty"` // version is the target version to downgrade. Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *DowngradeRequest) Reset() { *m = DowngradeRequest{} } func (m *DowngradeRequest) String() string { return proto.CompactTextString(m) } func (*DowngradeRequest) ProtoMessage() {} func (*DowngradeRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{57} } func (m *DowngradeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *DowngradeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_DowngradeRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *DowngradeRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_DowngradeRequest.Merge(m, src) } func (m *DowngradeRequest) XXX_Size() int { return m.Size() } func (m *DowngradeRequest) XXX_DiscardUnknown() { xxx_messageInfo_DowngradeRequest.DiscardUnknown(m) } var xxx_messageInfo_DowngradeRequest proto.InternalMessageInfo func (m *DowngradeRequest) GetAction() DowngradeRequest_DowngradeAction { if m != nil { return m.Action } return DowngradeRequest_VALIDATE } func (m *DowngradeRequest) GetVersion() string { if m != nil { return m.Version } return "" } type DowngradeResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // version is the current cluster version. Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *DowngradeResponse) Reset() { *m = DowngradeResponse{} } func (m *DowngradeResponse) String() string { return proto.CompactTextString(m) } func (*DowngradeResponse) ProtoMessage() {} func (*DowngradeResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{58} } func (m *DowngradeResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *DowngradeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_DowngradeResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *DowngradeResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_DowngradeResponse.Merge(m, src) } func (m *DowngradeResponse) XXX_Size() int { return m.Size() } func (m *DowngradeResponse) XXX_DiscardUnknown() { xxx_messageInfo_DowngradeResponse.DiscardUnknown(m) } var xxx_messageInfo_DowngradeResponse proto.InternalMessageInfo func (m *DowngradeResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *DowngradeResponse) GetVersion() string { if m != nil { return m.Version } return "" } // DowngradeVersionTestRequest is used for test only. The version in // this request will be read as the WAL record version.If the downgrade // target version is less than this version, then the downgrade(online) // or migration(offline) isn't safe, so shouldn't be allowed. type DowngradeVersionTestRequest struct { Ver string `protobuf:"bytes,1,opt,name=ver,proto3" json:"ver,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *DowngradeVersionTestRequest) Reset() { *m = DowngradeVersionTestRequest{} } func (m *DowngradeVersionTestRequest) String() string { return proto.CompactTextString(m) } func (*DowngradeVersionTestRequest) ProtoMessage() {} func (*DowngradeVersionTestRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{59} } func (m *DowngradeVersionTestRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *DowngradeVersionTestRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_DowngradeVersionTestRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *DowngradeVersionTestRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_DowngradeVersionTestRequest.Merge(m, src) } func (m *DowngradeVersionTestRequest) XXX_Size() int { return m.Size() } func (m *DowngradeVersionTestRequest) XXX_DiscardUnknown() { xxx_messageInfo_DowngradeVersionTestRequest.DiscardUnknown(m) } var xxx_messageInfo_DowngradeVersionTestRequest proto.InternalMessageInfo func (m *DowngradeVersionTestRequest) GetVer() string { if m != nil { return m.Ver } return "" } type StatusRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *StatusRequest) Reset() { *m = StatusRequest{} } func (m *StatusRequest) String() string { return proto.CompactTextString(m) } func (*StatusRequest) ProtoMessage() {} func (*StatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{60} } func (m *StatusRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *StatusRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_StatusRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *StatusRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_StatusRequest.Merge(m, src) } func (m *StatusRequest) XXX_Size() int { return m.Size() } func (m *StatusRequest) XXX_DiscardUnknown() { xxx_messageInfo_StatusRequest.DiscardUnknown(m) } var xxx_messageInfo_StatusRequest proto.InternalMessageInfo type StatusResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // version is the cluster protocol version used by the responding member. Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` // dbSize is the size of the backend database physically allocated, in bytes, of the responding member. DbSize int64 `protobuf:"varint,3,opt,name=dbSize,proto3" json:"dbSize,omitempty"` // leader is the member ID which the responding member believes is the current leader. Leader uint64 `protobuf:"varint,4,opt,name=leader,proto3" json:"leader,omitempty"` // raftIndex is the current raft committed index of the responding member. RaftIndex uint64 `protobuf:"varint,5,opt,name=raftIndex,proto3" json:"raftIndex,omitempty"` // raftTerm is the current raft term of the responding member. RaftTerm uint64 `protobuf:"varint,6,opt,name=raftTerm,proto3" json:"raftTerm,omitempty"` // raftAppliedIndex is the current raft applied index of the responding member. RaftAppliedIndex uint64 `protobuf:"varint,7,opt,name=raftAppliedIndex,proto3" json:"raftAppliedIndex,omitempty"` // errors contains alarm/health information and status. Errors []string `protobuf:"bytes,8,rep,name=errors,proto3" json:"errors,omitempty"` // dbSizeInUse is the size of the backend database logically in use, in bytes, of the responding member. DbSizeInUse int64 `protobuf:"varint,9,opt,name=dbSizeInUse,proto3" json:"dbSizeInUse,omitempty"` // isLearner indicates if the member is raft learner. IsLearner bool `protobuf:"varint,10,opt,name=isLearner,proto3" json:"isLearner,omitempty"` // storageVersion is the version of the db file. It might be updated with delay in relationship to the target cluster version. StorageVersion string `protobuf:"bytes,11,opt,name=storageVersion,proto3" json:"storageVersion,omitempty"` // dbSizeQuota is the configured etcd storage quota in bytes (the value passed to etcd instance by flag --quota-backend-bytes) DbSizeQuota int64 `protobuf:"varint,12,opt,name=dbSizeQuota,proto3" json:"dbSizeQuota,omitempty"` // downgradeInfo indicates if there is downgrade process. DowngradeInfo *DowngradeInfo `protobuf:"bytes,13,opt,name=downgradeInfo,proto3" json:"downgradeInfo,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *StatusResponse) Reset() { *m = StatusResponse{} } func (m *StatusResponse) String() string { return proto.CompactTextString(m) } func (*StatusResponse) ProtoMessage() {} func (*StatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{61} } func (m *StatusResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *StatusResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_StatusResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *StatusResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_StatusResponse.Merge(m, src) } func (m *StatusResponse) XXX_Size() int { return m.Size() } func (m *StatusResponse) XXX_DiscardUnknown() { xxx_messageInfo_StatusResponse.DiscardUnknown(m) } var xxx_messageInfo_StatusResponse proto.InternalMessageInfo func (m *StatusResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *StatusResponse) GetVersion() string { if m != nil { return m.Version } return "" } func (m *StatusResponse) GetDbSize() int64 { if m != nil { return m.DbSize } return 0 } func (m *StatusResponse) GetLeader() uint64 { if m != nil { return m.Leader } return 0 } func (m *StatusResponse) GetRaftIndex() uint64 { if m != nil { return m.RaftIndex } return 0 } func (m *StatusResponse) GetRaftTerm() uint64 { if m != nil { return m.RaftTerm } return 0 } func (m *StatusResponse) GetRaftAppliedIndex() uint64 { if m != nil { return m.RaftAppliedIndex } return 0 } func (m *StatusResponse) GetErrors() []string { if m != nil { return m.Errors } return nil } func (m *StatusResponse) GetDbSizeInUse() int64 { if m != nil { return m.DbSizeInUse } return 0 } func (m *StatusResponse) GetIsLearner() bool { if m != nil { return m.IsLearner } return false } func (m *StatusResponse) GetStorageVersion() string { if m != nil { return m.StorageVersion } return "" } func (m *StatusResponse) GetDbSizeQuota() int64 { if m != nil { return m.DbSizeQuota } return 0 } func (m *StatusResponse) GetDowngradeInfo() *DowngradeInfo { if m != nil { return m.DowngradeInfo } return nil } type DowngradeInfo struct { // enabled indicates whether the cluster is enabled to downgrade. Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"` // targetVersion is the target downgrade version. TargetVersion string `protobuf:"bytes,2,opt,name=targetVersion,proto3" json:"targetVersion,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *DowngradeInfo) Reset() { *m = DowngradeInfo{} } func (m *DowngradeInfo) String() string { return proto.CompactTextString(m) } func (*DowngradeInfo) ProtoMessage() {} func (*DowngradeInfo) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{62} } func (m *DowngradeInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *DowngradeInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_DowngradeInfo.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *DowngradeInfo) XXX_Merge(src proto.Message) { xxx_messageInfo_DowngradeInfo.Merge(m, src) } func (m *DowngradeInfo) XXX_Size() int { return m.Size() } func (m *DowngradeInfo) XXX_DiscardUnknown() { xxx_messageInfo_DowngradeInfo.DiscardUnknown(m) } var xxx_messageInfo_DowngradeInfo proto.InternalMessageInfo func (m *DowngradeInfo) GetEnabled() bool { if m != nil { return m.Enabled } return false } func (m *DowngradeInfo) GetTargetVersion() string { if m != nil { return m.TargetVersion } return "" } type AuthEnableRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthEnableRequest) Reset() { *m = AuthEnableRequest{} } func (m *AuthEnableRequest) String() string { return proto.CompactTextString(m) } func (*AuthEnableRequest) ProtoMessage() {} func (*AuthEnableRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{63} } func (m *AuthEnableRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthEnableRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthEnableRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthEnableRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthEnableRequest.Merge(m, src) } func (m *AuthEnableRequest) XXX_Size() int { return m.Size() } func (m *AuthEnableRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthEnableRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthEnableRequest proto.InternalMessageInfo type AuthDisableRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthDisableRequest) Reset() { *m = AuthDisableRequest{} } func (m *AuthDisableRequest) String() string { return proto.CompactTextString(m) } func (*AuthDisableRequest) ProtoMessage() {} func (*AuthDisableRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{64} } func (m *AuthDisableRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthDisableRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthDisableRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthDisableRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthDisableRequest.Merge(m, src) } func (m *AuthDisableRequest) XXX_Size() int { return m.Size() } func (m *AuthDisableRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthDisableRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthDisableRequest proto.InternalMessageInfo type AuthStatusRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthStatusRequest) Reset() { *m = AuthStatusRequest{} } func (m *AuthStatusRequest) String() string { return proto.CompactTextString(m) } func (*AuthStatusRequest) ProtoMessage() {} func (*AuthStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{65} } func (m *AuthStatusRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthStatusRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthStatusRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthStatusRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthStatusRequest.Merge(m, src) } func (m *AuthStatusRequest) XXX_Size() int { return m.Size() } func (m *AuthStatusRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthStatusRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthStatusRequest proto.InternalMessageInfo type AuthenticateRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Password string `protobuf:"bytes,2,opt,name=password,proto3" json:"password,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthenticateRequest) Reset() { *m = AuthenticateRequest{} } func (m *AuthenticateRequest) String() string { return proto.CompactTextString(m) } func (*AuthenticateRequest) ProtoMessage() {} func (*AuthenticateRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{66} } func (m *AuthenticateRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthenticateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthenticateRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthenticateRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthenticateRequest.Merge(m, src) } func (m *AuthenticateRequest) XXX_Size() int { return m.Size() } func (m *AuthenticateRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthenticateRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthenticateRequest proto.InternalMessageInfo func (m *AuthenticateRequest) GetName() string { if m != nil { return m.Name } return "" } func (m *AuthenticateRequest) GetPassword() string { if m != nil { return m.Password } return "" } type AuthUserAddRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Password string `protobuf:"bytes,2,opt,name=password,proto3" json:"password,omitempty"` Options *authpb.UserAddOptions `protobuf:"bytes,3,opt,name=options,proto3" json:"options,omitempty"` HashedPassword string `protobuf:"bytes,4,opt,name=hashedPassword,proto3" json:"hashedPassword,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserAddRequest) Reset() { *m = AuthUserAddRequest{} } func (m *AuthUserAddRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserAddRequest) ProtoMessage() {} func (*AuthUserAddRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{67} } func (m *AuthUserAddRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserAddRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserAddRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserAddRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserAddRequest.Merge(m, src) } func (m *AuthUserAddRequest) XXX_Size() int { return m.Size() } func (m *AuthUserAddRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserAddRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthUserAddRequest proto.InternalMessageInfo func (m *AuthUserAddRequest) GetName() string { if m != nil { return m.Name } return "" } func (m *AuthUserAddRequest) GetPassword() string { if m != nil { return m.Password } return "" } func (m *AuthUserAddRequest) GetOptions() *authpb.UserAddOptions { if m != nil { return m.Options } return nil } func (m *AuthUserAddRequest) GetHashedPassword() string { if m != nil { return m.HashedPassword } return "" } type AuthUserGetRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserGetRequest) Reset() { *m = AuthUserGetRequest{} } func (m *AuthUserGetRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserGetRequest) ProtoMessage() {} func (*AuthUserGetRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{68} } func (m *AuthUserGetRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserGetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserGetRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserGetRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserGetRequest.Merge(m, src) } func (m *AuthUserGetRequest) XXX_Size() int { return m.Size() } func (m *AuthUserGetRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserGetRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthUserGetRequest proto.InternalMessageInfo func (m *AuthUserGetRequest) GetName() string { if m != nil { return m.Name } return "" } type AuthUserDeleteRequest struct { // name is the name of the user to delete. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserDeleteRequest) Reset() { *m = AuthUserDeleteRequest{} } func (m *AuthUserDeleteRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserDeleteRequest) ProtoMessage() {} func (*AuthUserDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{69} } func (m *AuthUserDeleteRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserDeleteRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserDeleteRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserDeleteRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserDeleteRequest.Merge(m, src) } func (m *AuthUserDeleteRequest) XXX_Size() int { return m.Size() } func (m *AuthUserDeleteRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserDeleteRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthUserDeleteRequest proto.InternalMessageInfo func (m *AuthUserDeleteRequest) GetName() string { if m != nil { return m.Name } return "" } type AuthUserChangePasswordRequest struct { // name is the name of the user whose password is being changed. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // password is the new password for the user. Note that this field will be removed in the API layer. Password string `protobuf:"bytes,2,opt,name=password,proto3" json:"password,omitempty"` // hashedPassword is the new password for the user. Note that this field will be initialized in the API layer. HashedPassword string `protobuf:"bytes,3,opt,name=hashedPassword,proto3" json:"hashedPassword,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserChangePasswordRequest) Reset() { *m = AuthUserChangePasswordRequest{} } func (m *AuthUserChangePasswordRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserChangePasswordRequest) ProtoMessage() {} func (*AuthUserChangePasswordRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{70} } func (m *AuthUserChangePasswordRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserChangePasswordRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserChangePasswordRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserChangePasswordRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserChangePasswordRequest.Merge(m, src) } func (m *AuthUserChangePasswordRequest) XXX_Size() int { return m.Size() } func (m *AuthUserChangePasswordRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserChangePasswordRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthUserChangePasswordRequest proto.InternalMessageInfo func (m *AuthUserChangePasswordRequest) GetName() string { if m != nil { return m.Name } return "" } func (m *AuthUserChangePasswordRequest) GetPassword() string { if m != nil { return m.Password } return "" } func (m *AuthUserChangePasswordRequest) GetHashedPassword() string { if m != nil { return m.HashedPassword } return "" } type AuthUserGrantRoleRequest struct { // user is the name of the user which should be granted a given role. User string `protobuf:"bytes,1,opt,name=user,proto3" json:"user,omitempty"` // role is the name of the role to grant to the user. Role string `protobuf:"bytes,2,opt,name=role,proto3" json:"role,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserGrantRoleRequest) Reset() { *m = AuthUserGrantRoleRequest{} } func (m *AuthUserGrantRoleRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserGrantRoleRequest) ProtoMessage() {} func (*AuthUserGrantRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{71} } func (m *AuthUserGrantRoleRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserGrantRoleRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserGrantRoleRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserGrantRoleRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserGrantRoleRequest.Merge(m, src) } func (m *AuthUserGrantRoleRequest) XXX_Size() int { return m.Size() } func (m *AuthUserGrantRoleRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserGrantRoleRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthUserGrantRoleRequest proto.InternalMessageInfo func (m *AuthUserGrantRoleRequest) GetUser() string { if m != nil { return m.User } return "" } func (m *AuthUserGrantRoleRequest) GetRole() string { if m != nil { return m.Role } return "" } type AuthUserRevokeRoleRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Role string `protobuf:"bytes,2,opt,name=role,proto3" json:"role,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserRevokeRoleRequest) Reset() { *m = AuthUserRevokeRoleRequest{} } func (m *AuthUserRevokeRoleRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserRevokeRoleRequest) ProtoMessage() {} func (*AuthUserRevokeRoleRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{72} } func (m *AuthUserRevokeRoleRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserRevokeRoleRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserRevokeRoleRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserRevokeRoleRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserRevokeRoleRequest.Merge(m, src) } func (m *AuthUserRevokeRoleRequest) XXX_Size() int { return m.Size() } func (m *AuthUserRevokeRoleRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserRevokeRoleRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthUserRevokeRoleRequest proto.InternalMessageInfo func (m *AuthUserRevokeRoleRequest) GetName() string { if m != nil { return m.Name } return "" } func (m *AuthUserRevokeRoleRequest) GetRole() string { if m != nil { return m.Role } return "" } type AuthRoleAddRequest struct { // name is the name of the role to add to the authentication system. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleAddRequest) Reset() { *m = AuthRoleAddRequest{} } func (m *AuthRoleAddRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleAddRequest) ProtoMessage() {} func (*AuthRoleAddRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{73} } func (m *AuthRoleAddRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleAddRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleAddRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleAddRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleAddRequest.Merge(m, src) } func (m *AuthRoleAddRequest) XXX_Size() int { return m.Size() } func (m *AuthRoleAddRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleAddRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleAddRequest proto.InternalMessageInfo func (m *AuthRoleAddRequest) GetName() string { if m != nil { return m.Name } return "" } type AuthRoleGetRequest struct { Role string `protobuf:"bytes,1,opt,name=role,proto3" json:"role,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleGetRequest) Reset() { *m = AuthRoleGetRequest{} } func (m *AuthRoleGetRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleGetRequest) ProtoMessage() {} func (*AuthRoleGetRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{74} } func (m *AuthRoleGetRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleGetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleGetRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleGetRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleGetRequest.Merge(m, src) } func (m *AuthRoleGetRequest) XXX_Size() int { return m.Size() } func (m *AuthRoleGetRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleGetRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleGetRequest proto.InternalMessageInfo func (m *AuthRoleGetRequest) GetRole() string { if m != nil { return m.Role } return "" } type AuthUserListRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserListRequest) Reset() { *m = AuthUserListRequest{} } func (m *AuthUserListRequest) String() string { return proto.CompactTextString(m) } func (*AuthUserListRequest) ProtoMessage() {} func (*AuthUserListRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{75} } func (m *AuthUserListRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserListRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserListRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserListRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserListRequest.Merge(m, src) } func (m *AuthUserListRequest) XXX_Size() int { return m.Size() } func (m *AuthUserListRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserListRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthUserListRequest proto.InternalMessageInfo type AuthRoleListRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleListRequest) Reset() { *m = AuthRoleListRequest{} } func (m *AuthRoleListRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleListRequest) ProtoMessage() {} func (*AuthRoleListRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{76} } func (m *AuthRoleListRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleListRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleListRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleListRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleListRequest.Merge(m, src) } func (m *AuthRoleListRequest) XXX_Size() int { return m.Size() } func (m *AuthRoleListRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleListRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleListRequest proto.InternalMessageInfo type AuthRoleDeleteRequest struct { Role string `protobuf:"bytes,1,opt,name=role,proto3" json:"role,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleDeleteRequest) Reset() { *m = AuthRoleDeleteRequest{} } func (m *AuthRoleDeleteRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleDeleteRequest) ProtoMessage() {} func (*AuthRoleDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{77} } func (m *AuthRoleDeleteRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleDeleteRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleDeleteRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleDeleteRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleDeleteRequest.Merge(m, src) } func (m *AuthRoleDeleteRequest) XXX_Size() int { return m.Size() } func (m *AuthRoleDeleteRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleDeleteRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleDeleteRequest proto.InternalMessageInfo func (m *AuthRoleDeleteRequest) GetRole() string { if m != nil { return m.Role } return "" } type AuthRoleGrantPermissionRequest struct { // name is the name of the role which will be granted the permission. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // perm is the permission to grant to the role. Perm *authpb.Permission `protobuf:"bytes,2,opt,name=perm,proto3" json:"perm,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleGrantPermissionRequest) Reset() { *m = AuthRoleGrantPermissionRequest{} } func (m *AuthRoleGrantPermissionRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleGrantPermissionRequest) ProtoMessage() {} func (*AuthRoleGrantPermissionRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{78} } func (m *AuthRoleGrantPermissionRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleGrantPermissionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleGrantPermissionRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleGrantPermissionRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleGrantPermissionRequest.Merge(m, src) } func (m *AuthRoleGrantPermissionRequest) XXX_Size() int { return m.Size() } func (m *AuthRoleGrantPermissionRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleGrantPermissionRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleGrantPermissionRequest proto.InternalMessageInfo func (m *AuthRoleGrantPermissionRequest) GetName() string { if m != nil { return m.Name } return "" } func (m *AuthRoleGrantPermissionRequest) GetPerm() *authpb.Permission { if m != nil { return m.Perm } return nil } type AuthRoleRevokePermissionRequest struct { Role string `protobuf:"bytes,1,opt,name=role,proto3" json:"role,omitempty"` Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` RangeEnd []byte `protobuf:"bytes,3,opt,name=range_end,json=rangeEnd,proto3" json:"range_end,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleRevokePermissionRequest) Reset() { *m = AuthRoleRevokePermissionRequest{} } func (m *AuthRoleRevokePermissionRequest) String() string { return proto.CompactTextString(m) } func (*AuthRoleRevokePermissionRequest) ProtoMessage() {} func (*AuthRoleRevokePermissionRequest) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{79} } func (m *AuthRoleRevokePermissionRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleRevokePermissionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleRevokePermissionRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleRevokePermissionRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleRevokePermissionRequest.Merge(m, src) } func (m *AuthRoleRevokePermissionRequest) XXX_Size() int { return m.Size() } func (m *AuthRoleRevokePermissionRequest) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleRevokePermissionRequest.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleRevokePermissionRequest proto.InternalMessageInfo func (m *AuthRoleRevokePermissionRequest) GetRole() string { if m != nil { return m.Role } return "" } func (m *AuthRoleRevokePermissionRequest) GetKey() []byte { if m != nil { return m.Key } return nil } func (m *AuthRoleRevokePermissionRequest) GetRangeEnd() []byte { if m != nil { return m.RangeEnd } return nil } type AuthEnableResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthEnableResponse) Reset() { *m = AuthEnableResponse{} } func (m *AuthEnableResponse) String() string { return proto.CompactTextString(m) } func (*AuthEnableResponse) ProtoMessage() {} func (*AuthEnableResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{80} } func (m *AuthEnableResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthEnableResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthEnableResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthEnableResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthEnableResponse.Merge(m, src) } func (m *AuthEnableResponse) XXX_Size() int { return m.Size() } func (m *AuthEnableResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthEnableResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthEnableResponse proto.InternalMessageInfo func (m *AuthEnableResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type AuthDisableResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthDisableResponse) Reset() { *m = AuthDisableResponse{} } func (m *AuthDisableResponse) String() string { return proto.CompactTextString(m) } func (*AuthDisableResponse) ProtoMessage() {} func (*AuthDisableResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{81} } func (m *AuthDisableResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthDisableResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthDisableResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthDisableResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthDisableResponse.Merge(m, src) } func (m *AuthDisableResponse) XXX_Size() int { return m.Size() } func (m *AuthDisableResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthDisableResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthDisableResponse proto.InternalMessageInfo func (m *AuthDisableResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type AuthStatusResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` Enabled bool `protobuf:"varint,2,opt,name=enabled,proto3" json:"enabled,omitempty"` // authRevision is the current revision of auth store AuthRevision uint64 `protobuf:"varint,3,opt,name=authRevision,proto3" json:"authRevision,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthStatusResponse) Reset() { *m = AuthStatusResponse{} } func (m *AuthStatusResponse) String() string { return proto.CompactTextString(m) } func (*AuthStatusResponse) ProtoMessage() {} func (*AuthStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{82} } func (m *AuthStatusResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthStatusResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthStatusResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthStatusResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthStatusResponse.Merge(m, src) } func (m *AuthStatusResponse) XXX_Size() int { return m.Size() } func (m *AuthStatusResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthStatusResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthStatusResponse proto.InternalMessageInfo func (m *AuthStatusResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *AuthStatusResponse) GetEnabled() bool { if m != nil { return m.Enabled } return false } func (m *AuthStatusResponse) GetAuthRevision() uint64 { if m != nil { return m.AuthRevision } return 0 } type AuthenticateResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // token is an authorized token that can be used in succeeding RPCs Token string `protobuf:"bytes,2,opt,name=token,proto3" json:"token,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthenticateResponse) Reset() { *m = AuthenticateResponse{} } func (m *AuthenticateResponse) String() string { return proto.CompactTextString(m) } func (*AuthenticateResponse) ProtoMessage() {} func (*AuthenticateResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{83} } func (m *AuthenticateResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthenticateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthenticateResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthenticateResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthenticateResponse.Merge(m, src) } func (m *AuthenticateResponse) XXX_Size() int { return m.Size() } func (m *AuthenticateResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthenticateResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthenticateResponse proto.InternalMessageInfo func (m *AuthenticateResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *AuthenticateResponse) GetToken() string { if m != nil { return m.Token } return "" } type AuthUserAddResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserAddResponse) Reset() { *m = AuthUserAddResponse{} } func (m *AuthUserAddResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserAddResponse) ProtoMessage() {} func (*AuthUserAddResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{84} } func (m *AuthUserAddResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserAddResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserAddResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserAddResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserAddResponse.Merge(m, src) } func (m *AuthUserAddResponse) XXX_Size() int { return m.Size() } func (m *AuthUserAddResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserAddResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthUserAddResponse proto.InternalMessageInfo func (m *AuthUserAddResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type AuthUserGetResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` Roles []string `protobuf:"bytes,2,rep,name=roles,proto3" json:"roles,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserGetResponse) Reset() { *m = AuthUserGetResponse{} } func (m *AuthUserGetResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserGetResponse) ProtoMessage() {} func (*AuthUserGetResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{85} } func (m *AuthUserGetResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserGetResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserGetResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserGetResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserGetResponse.Merge(m, src) } func (m *AuthUserGetResponse) XXX_Size() int { return m.Size() } func (m *AuthUserGetResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserGetResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthUserGetResponse proto.InternalMessageInfo func (m *AuthUserGetResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *AuthUserGetResponse) GetRoles() []string { if m != nil { return m.Roles } return nil } type AuthUserDeleteResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserDeleteResponse) Reset() { *m = AuthUserDeleteResponse{} } func (m *AuthUserDeleteResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserDeleteResponse) ProtoMessage() {} func (*AuthUserDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{86} } func (m *AuthUserDeleteResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserDeleteResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserDeleteResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserDeleteResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserDeleteResponse.Merge(m, src) } func (m *AuthUserDeleteResponse) XXX_Size() int { return m.Size() } func (m *AuthUserDeleteResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserDeleteResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthUserDeleteResponse proto.InternalMessageInfo func (m *AuthUserDeleteResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type AuthUserChangePasswordResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserChangePasswordResponse) Reset() { *m = AuthUserChangePasswordResponse{} } func (m *AuthUserChangePasswordResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserChangePasswordResponse) ProtoMessage() {} func (*AuthUserChangePasswordResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{87} } func (m *AuthUserChangePasswordResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserChangePasswordResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserChangePasswordResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserChangePasswordResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserChangePasswordResponse.Merge(m, src) } func (m *AuthUserChangePasswordResponse) XXX_Size() int { return m.Size() } func (m *AuthUserChangePasswordResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserChangePasswordResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthUserChangePasswordResponse proto.InternalMessageInfo func (m *AuthUserChangePasswordResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type AuthUserGrantRoleResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserGrantRoleResponse) Reset() { *m = AuthUserGrantRoleResponse{} } func (m *AuthUserGrantRoleResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserGrantRoleResponse) ProtoMessage() {} func (*AuthUserGrantRoleResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{88} } func (m *AuthUserGrantRoleResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserGrantRoleResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserGrantRoleResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserGrantRoleResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserGrantRoleResponse.Merge(m, src) } func (m *AuthUserGrantRoleResponse) XXX_Size() int { return m.Size() } func (m *AuthUserGrantRoleResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserGrantRoleResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthUserGrantRoleResponse proto.InternalMessageInfo func (m *AuthUserGrantRoleResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type AuthUserRevokeRoleResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserRevokeRoleResponse) Reset() { *m = AuthUserRevokeRoleResponse{} } func (m *AuthUserRevokeRoleResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserRevokeRoleResponse) ProtoMessage() {} func (*AuthUserRevokeRoleResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{89} } func (m *AuthUserRevokeRoleResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserRevokeRoleResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserRevokeRoleResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserRevokeRoleResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserRevokeRoleResponse.Merge(m, src) } func (m *AuthUserRevokeRoleResponse) XXX_Size() int { return m.Size() } func (m *AuthUserRevokeRoleResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserRevokeRoleResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthUserRevokeRoleResponse proto.InternalMessageInfo func (m *AuthUserRevokeRoleResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type AuthRoleAddResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleAddResponse) Reset() { *m = AuthRoleAddResponse{} } func (m *AuthRoleAddResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleAddResponse) ProtoMessage() {} func (*AuthRoleAddResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{90} } func (m *AuthRoleAddResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleAddResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleAddResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleAddResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleAddResponse.Merge(m, src) } func (m *AuthRoleAddResponse) XXX_Size() int { return m.Size() } func (m *AuthRoleAddResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleAddResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleAddResponse proto.InternalMessageInfo func (m *AuthRoleAddResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type AuthRoleGetResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` Perm []*authpb.Permission `protobuf:"bytes,2,rep,name=perm,proto3" json:"perm,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleGetResponse) Reset() { *m = AuthRoleGetResponse{} } func (m *AuthRoleGetResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleGetResponse) ProtoMessage() {} func (*AuthRoleGetResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{91} } func (m *AuthRoleGetResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleGetResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleGetResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleGetResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleGetResponse.Merge(m, src) } func (m *AuthRoleGetResponse) XXX_Size() int { return m.Size() } func (m *AuthRoleGetResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleGetResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleGetResponse proto.InternalMessageInfo func (m *AuthRoleGetResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *AuthRoleGetResponse) GetPerm() []*authpb.Permission { if m != nil { return m.Perm } return nil } type AuthRoleListResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` Roles []string `protobuf:"bytes,2,rep,name=roles,proto3" json:"roles,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleListResponse) Reset() { *m = AuthRoleListResponse{} } func (m *AuthRoleListResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleListResponse) ProtoMessage() {} func (*AuthRoleListResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{92} } func (m *AuthRoleListResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleListResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleListResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleListResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleListResponse.Merge(m, src) } func (m *AuthRoleListResponse) XXX_Size() int { return m.Size() } func (m *AuthRoleListResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleListResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleListResponse proto.InternalMessageInfo func (m *AuthRoleListResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *AuthRoleListResponse) GetRoles() []string { if m != nil { return m.Roles } return nil } type AuthUserListResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` Users []string `protobuf:"bytes,2,rep,name=users,proto3" json:"users,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthUserListResponse) Reset() { *m = AuthUserListResponse{} } func (m *AuthUserListResponse) String() string { return proto.CompactTextString(m) } func (*AuthUserListResponse) ProtoMessage() {} func (*AuthUserListResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{93} } func (m *AuthUserListResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthUserListResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthUserListResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthUserListResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthUserListResponse.Merge(m, src) } func (m *AuthUserListResponse) XXX_Size() int { return m.Size() } func (m *AuthUserListResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthUserListResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthUserListResponse proto.InternalMessageInfo func (m *AuthUserListResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func (m *AuthUserListResponse) GetUsers() []string { if m != nil { return m.Users } return nil } type AuthRoleDeleteResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleDeleteResponse) Reset() { *m = AuthRoleDeleteResponse{} } func (m *AuthRoleDeleteResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleDeleteResponse) ProtoMessage() {} func (*AuthRoleDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{94} } func (m *AuthRoleDeleteResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleDeleteResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleDeleteResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleDeleteResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleDeleteResponse.Merge(m, src) } func (m *AuthRoleDeleteResponse) XXX_Size() int { return m.Size() } func (m *AuthRoleDeleteResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleDeleteResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleDeleteResponse proto.InternalMessageInfo func (m *AuthRoleDeleteResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type AuthRoleGrantPermissionResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleGrantPermissionResponse) Reset() { *m = AuthRoleGrantPermissionResponse{} } func (m *AuthRoleGrantPermissionResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleGrantPermissionResponse) ProtoMessage() {} func (*AuthRoleGrantPermissionResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{95} } func (m *AuthRoleGrantPermissionResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleGrantPermissionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleGrantPermissionResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleGrantPermissionResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleGrantPermissionResponse.Merge(m, src) } func (m *AuthRoleGrantPermissionResponse) XXX_Size() int { return m.Size() } func (m *AuthRoleGrantPermissionResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleGrantPermissionResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleGrantPermissionResponse proto.InternalMessageInfo func (m *AuthRoleGrantPermissionResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } type AuthRoleRevokePermissionResponse struct { Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *AuthRoleRevokePermissionResponse) Reset() { *m = AuthRoleRevokePermissionResponse{} } func (m *AuthRoleRevokePermissionResponse) String() string { return proto.CompactTextString(m) } func (*AuthRoleRevokePermissionResponse) ProtoMessage() {} func (*AuthRoleRevokePermissionResponse) Descriptor() ([]byte, []int) { return fileDescriptor_77a6da22d6a3feb1, []int{96} } func (m *AuthRoleRevokePermissionResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *AuthRoleRevokePermissionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_AuthRoleRevokePermissionResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *AuthRoleRevokePermissionResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_AuthRoleRevokePermissionResponse.Merge(m, src) } func (m *AuthRoleRevokePermissionResponse) XXX_Size() int { return m.Size() } func (m *AuthRoleRevokePermissionResponse) XXX_DiscardUnknown() { xxx_messageInfo_AuthRoleRevokePermissionResponse.DiscardUnknown(m) } var xxx_messageInfo_AuthRoleRevokePermissionResponse proto.InternalMessageInfo func (m *AuthRoleRevokePermissionResponse) GetHeader() *ResponseHeader { if m != nil { return m.Header } return nil } func init() { proto.RegisterEnum("etcdserverpb.AlarmType", AlarmType_name, AlarmType_value) proto.RegisterEnum("etcdserverpb.RangeRequest_SortOrder", RangeRequest_SortOrder_name, RangeRequest_SortOrder_value) proto.RegisterEnum("etcdserverpb.RangeRequest_SortTarget", RangeRequest_SortTarget_name, RangeRequest_SortTarget_value) proto.RegisterEnum("etcdserverpb.Compare_CompareResult", Compare_CompareResult_name, Compare_CompareResult_value) proto.RegisterEnum("etcdserverpb.Compare_CompareTarget", Compare_CompareTarget_name, Compare_CompareTarget_value) proto.RegisterEnum("etcdserverpb.WatchCreateRequest_FilterType", WatchCreateRequest_FilterType_name, WatchCreateRequest_FilterType_value) proto.RegisterEnum("etcdserverpb.AlarmRequest_AlarmAction", AlarmRequest_AlarmAction_name, AlarmRequest_AlarmAction_value) proto.RegisterEnum("etcdserverpb.DowngradeRequest_DowngradeAction", DowngradeRequest_DowngradeAction_name, DowngradeRequest_DowngradeAction_value) proto.RegisterType((*ResponseHeader)(nil), "etcdserverpb.ResponseHeader") proto.RegisterType((*RangeRequest)(nil), "etcdserverpb.RangeRequest") proto.RegisterType((*RangeResponse)(nil), "etcdserverpb.RangeResponse") proto.RegisterType((*PutRequest)(nil), "etcdserverpb.PutRequest") proto.RegisterType((*PutResponse)(nil), "etcdserverpb.PutResponse") proto.RegisterType((*DeleteRangeRequest)(nil), "etcdserverpb.DeleteRangeRequest") proto.RegisterType((*DeleteRangeResponse)(nil), "etcdserverpb.DeleteRangeResponse") proto.RegisterType((*RequestOp)(nil), "etcdserverpb.RequestOp") proto.RegisterType((*ResponseOp)(nil), "etcdserverpb.ResponseOp") proto.RegisterType((*Compare)(nil), "etcdserverpb.Compare") proto.RegisterType((*TxnRequest)(nil), "etcdserverpb.TxnRequest") proto.RegisterType((*TxnResponse)(nil), "etcdserverpb.TxnResponse") proto.RegisterType((*CompactionRequest)(nil), "etcdserverpb.CompactionRequest") proto.RegisterType((*CompactionResponse)(nil), "etcdserverpb.CompactionResponse") proto.RegisterType((*HashRequest)(nil), "etcdserverpb.HashRequest") proto.RegisterType((*HashKVRequest)(nil), "etcdserverpb.HashKVRequest") proto.RegisterType((*HashKVResponse)(nil), "etcdserverpb.HashKVResponse") proto.RegisterType((*HashResponse)(nil), "etcdserverpb.HashResponse") proto.RegisterType((*SnapshotRequest)(nil), "etcdserverpb.SnapshotRequest") proto.RegisterType((*SnapshotResponse)(nil), "etcdserverpb.SnapshotResponse") proto.RegisterType((*WatchRequest)(nil), "etcdserverpb.WatchRequest") proto.RegisterType((*WatchCreateRequest)(nil), "etcdserverpb.WatchCreateRequest") proto.RegisterType((*WatchCancelRequest)(nil), "etcdserverpb.WatchCancelRequest") proto.RegisterType((*WatchProgressRequest)(nil), "etcdserverpb.WatchProgressRequest") proto.RegisterType((*WatchResponse)(nil), "etcdserverpb.WatchResponse") proto.RegisterType((*LeaseGrantRequest)(nil), "etcdserverpb.LeaseGrantRequest") proto.RegisterType((*LeaseGrantResponse)(nil), "etcdserverpb.LeaseGrantResponse") proto.RegisterType((*LeaseRevokeRequest)(nil), "etcdserverpb.LeaseRevokeRequest") proto.RegisterType((*LeaseRevokeResponse)(nil), "etcdserverpb.LeaseRevokeResponse") proto.RegisterType((*LeaseCheckpoint)(nil), "etcdserverpb.LeaseCheckpoint") proto.RegisterType((*LeaseCheckpointRequest)(nil), "etcdserverpb.LeaseCheckpointRequest") proto.RegisterType((*LeaseCheckpointResponse)(nil), "etcdserverpb.LeaseCheckpointResponse") proto.RegisterType((*LeaseKeepAliveRequest)(nil), "etcdserverpb.LeaseKeepAliveRequest") proto.RegisterType((*LeaseKeepAliveResponse)(nil), "etcdserverpb.LeaseKeepAliveResponse") proto.RegisterType((*LeaseTimeToLiveRequest)(nil), "etcdserverpb.LeaseTimeToLiveRequest") proto.RegisterType((*LeaseTimeToLiveResponse)(nil), "etcdserverpb.LeaseTimeToLiveResponse") proto.RegisterType((*LeaseLeasesRequest)(nil), "etcdserverpb.LeaseLeasesRequest") proto.RegisterType((*LeaseStatus)(nil), "etcdserverpb.LeaseStatus") proto.RegisterType((*LeaseLeasesResponse)(nil), "etcdserverpb.LeaseLeasesResponse") proto.RegisterType((*Member)(nil), "etcdserverpb.Member") proto.RegisterType((*MemberAddRequest)(nil), "etcdserverpb.MemberAddRequest") proto.RegisterType((*MemberAddResponse)(nil), "etcdserverpb.MemberAddResponse") proto.RegisterType((*MemberRemoveRequest)(nil), "etcdserverpb.MemberRemoveRequest") proto.RegisterType((*MemberRemoveResponse)(nil), "etcdserverpb.MemberRemoveResponse") proto.RegisterType((*MemberUpdateRequest)(nil), "etcdserverpb.MemberUpdateRequest") proto.RegisterType((*MemberUpdateResponse)(nil), "etcdserverpb.MemberUpdateResponse") proto.RegisterType((*MemberListRequest)(nil), "etcdserverpb.MemberListRequest") proto.RegisterType((*MemberListResponse)(nil), "etcdserverpb.MemberListResponse") proto.RegisterType((*MemberPromoteRequest)(nil), "etcdserverpb.MemberPromoteRequest") proto.RegisterType((*MemberPromoteResponse)(nil), "etcdserverpb.MemberPromoteResponse") proto.RegisterType((*DefragmentRequest)(nil), "etcdserverpb.DefragmentRequest") proto.RegisterType((*DefragmentResponse)(nil), "etcdserverpb.DefragmentResponse") proto.RegisterType((*MoveLeaderRequest)(nil), "etcdserverpb.MoveLeaderRequest") proto.RegisterType((*MoveLeaderResponse)(nil), "etcdserverpb.MoveLeaderResponse") proto.RegisterType((*AlarmRequest)(nil), "etcdserverpb.AlarmRequest") proto.RegisterType((*AlarmMember)(nil), "etcdserverpb.AlarmMember") proto.RegisterType((*AlarmResponse)(nil), "etcdserverpb.AlarmResponse") proto.RegisterType((*DowngradeRequest)(nil), "etcdserverpb.DowngradeRequest") proto.RegisterType((*DowngradeResponse)(nil), "etcdserverpb.DowngradeResponse") proto.RegisterType((*DowngradeVersionTestRequest)(nil), "etcdserverpb.DowngradeVersionTestRequest") proto.RegisterType((*StatusRequest)(nil), "etcdserverpb.StatusRequest") proto.RegisterType((*StatusResponse)(nil), "etcdserverpb.StatusResponse") proto.RegisterType((*DowngradeInfo)(nil), "etcdserverpb.DowngradeInfo") proto.RegisterType((*AuthEnableRequest)(nil), "etcdserverpb.AuthEnableRequest") proto.RegisterType((*AuthDisableRequest)(nil), "etcdserverpb.AuthDisableRequest") proto.RegisterType((*AuthStatusRequest)(nil), "etcdserverpb.AuthStatusRequest") proto.RegisterType((*AuthenticateRequest)(nil), "etcdserverpb.AuthenticateRequest") proto.RegisterType((*AuthUserAddRequest)(nil), "etcdserverpb.AuthUserAddRequest") proto.RegisterType((*AuthUserGetRequest)(nil), "etcdserverpb.AuthUserGetRequest") proto.RegisterType((*AuthUserDeleteRequest)(nil), "etcdserverpb.AuthUserDeleteRequest") proto.RegisterType((*AuthUserChangePasswordRequest)(nil), "etcdserverpb.AuthUserChangePasswordRequest") proto.RegisterType((*AuthUserGrantRoleRequest)(nil), "etcdserverpb.AuthUserGrantRoleRequest") proto.RegisterType((*AuthUserRevokeRoleRequest)(nil), "etcdserverpb.AuthUserRevokeRoleRequest") proto.RegisterType((*AuthRoleAddRequest)(nil), "etcdserverpb.AuthRoleAddRequest") proto.RegisterType((*AuthRoleGetRequest)(nil), "etcdserverpb.AuthRoleGetRequest") proto.RegisterType((*AuthUserListRequest)(nil), "etcdserverpb.AuthUserListRequest") proto.RegisterType((*AuthRoleListRequest)(nil), "etcdserverpb.AuthRoleListRequest") proto.RegisterType((*AuthRoleDeleteRequest)(nil), "etcdserverpb.AuthRoleDeleteRequest") proto.RegisterType((*AuthRoleGrantPermissionRequest)(nil), "etcdserverpb.AuthRoleGrantPermissionRequest") proto.RegisterType((*AuthRoleRevokePermissionRequest)(nil), "etcdserverpb.AuthRoleRevokePermissionRequest") proto.RegisterType((*AuthEnableResponse)(nil), "etcdserverpb.AuthEnableResponse") proto.RegisterType((*AuthDisableResponse)(nil), "etcdserverpb.AuthDisableResponse") proto.RegisterType((*AuthStatusResponse)(nil), "etcdserverpb.AuthStatusResponse") proto.RegisterType((*AuthenticateResponse)(nil), "etcdserverpb.AuthenticateResponse") proto.RegisterType((*AuthUserAddResponse)(nil), "etcdserverpb.AuthUserAddResponse") proto.RegisterType((*AuthUserGetResponse)(nil), "etcdserverpb.AuthUserGetResponse") proto.RegisterType((*AuthUserDeleteResponse)(nil), "etcdserverpb.AuthUserDeleteResponse") proto.RegisterType((*AuthUserChangePasswordResponse)(nil), "etcdserverpb.AuthUserChangePasswordResponse") proto.RegisterType((*AuthUserGrantRoleResponse)(nil), "etcdserverpb.AuthUserGrantRoleResponse") proto.RegisterType((*AuthUserRevokeRoleResponse)(nil), "etcdserverpb.AuthUserRevokeRoleResponse") proto.RegisterType((*AuthRoleAddResponse)(nil), "etcdserverpb.AuthRoleAddResponse") proto.RegisterType((*AuthRoleGetResponse)(nil), "etcdserverpb.AuthRoleGetResponse") proto.RegisterType((*AuthRoleListResponse)(nil), "etcdserverpb.AuthRoleListResponse") proto.RegisterType((*AuthUserListResponse)(nil), "etcdserverpb.AuthUserListResponse") proto.RegisterType((*AuthRoleDeleteResponse)(nil), "etcdserverpb.AuthRoleDeleteResponse") proto.RegisterType((*AuthRoleGrantPermissionResponse)(nil), "etcdserverpb.AuthRoleGrantPermissionResponse") proto.RegisterType((*AuthRoleRevokePermissionResponse)(nil), "etcdserverpb.AuthRoleRevokePermissionResponse") } func init() { proto.RegisterFile("rpc.proto", fileDescriptor_77a6da22d6a3feb1) } var fileDescriptor_77a6da22d6a3feb1 = []byte{ // 4564 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3c, 0x5d, 0x6f, 0x5c, 0x49, 0x56, 0xbe, 0xdd, 0xb6, 0xdb, 0x7d, 0xfa, 0xc3, 0x9d, 0x8a, 0x93, 0x74, 0x3a, 0x89, 0xe3, 0xb9, 0x49, 0x66, 0x32, 0x99, 0x89, 0x3b, 0xb1, 0x93, 0x99, 0x25, 0x68, 0x86, 0xed, 0xd8, 0x3d, 0x89, 0x37, 0x8e, 0xed, 0xb9, 0xee, 0x64, 0x76, 0x82, 0xb4, 0xe6, 0xba, 0xbb, 0x62, 0xdf, 0x75, 0xf7, 0xbd, 0xbd, 0xf7, 0x5e, 0x77, 0xec, 0xe1, 0x61, 0x87, 0x85, 0x65, 0xb5, 0x20, 0xad, 0xc4, 0x20, 0xa1, 0x15, 0x82, 0x17, 0x40, 0x82, 0x07, 0x40, 0xf0, 0xc0, 0x03, 0x02, 0x89, 0x07, 0x78, 0x80, 0x07, 0x24, 0x24, 0xfe, 0x00, 0x0c, 0xfb, 0xc4, 0xaf, 0x58, 0xd5, 0xd7, 0xad, 0xaa, 0xfb, 0x61, 0x67, 0xd6, 0x1e, 0xed, 0x4b, 0x7c, 0xab, 0xea, 0x7c, 0xd5, 0x39, 0x55, 0xe7, 0x54, 0x9d, 0x53, 0x69, 0x28, 0xfa, 0xc3, 0xee, 0xfc, 0xd0, 0xf7, 0x42, 0x0f, 0x95, 0x71, 0xd8, 0xed, 0x05, 0xd8, 0x1f, 0x61, 0x7f, 0xb8, 0xdd, 0xa8, 0x93, 0x56, 0xd3, 0x1e, 0x3a, 0xcd, 0xc1, 0xa8, 0xdb, 0x1d, 0x6e, 0x37, 0xf7, 0x46, 0x0c, 0xae, 0xd1, 0x88, 0x46, 0xec, 0xfd, 0x70, 0x77, 0xb8, 0x4d, 0xff, 0xf0, 0xb1, 0xb9, 0x68, 0x6c, 0x84, 0xfd, 0xc0, 0xf1, 0xdc, 0xe1, 0xb6, 0xf8, 0xe2, 0x10, 0x97, 0x77, 0x3c, 0x6f, 0xa7, 0x8f, 0x19, 0xbe, 0xeb, 0x7a, 0xa1, 0x1d, 0x3a, 0x9e, 0x1b, 0xf0, 0x51, 0xf6, 0xa7, 0x7b, 0x7b, 0x07, 0xbb, 0xb7, 0xbd, 0x21, 0x76, 0xed, 0xa1, 0x33, 0x5a, 0x68, 0x7a, 0x43, 0x0a, 0x93, 0x84, 0x37, 0x7f, 0x62, 0x40, 0xd5, 0xc2, 0xc1, 0xd0, 0x73, 0x03, 0xfc, 0x18, 0xdb, 0x3d, 0xec, 0xa3, 0x2b, 0x00, 0xdd, 0xfe, 0x7e, 0x10, 0x62, 0x7f, 0xcb, 0xe9, 0xd5, 0x8d, 0x39, 0xe3, 0xe6, 0xb8, 0x55, 0xe4, 0x3d, 0x2b, 0x3d, 0x74, 0x09, 0x8a, 0x03, 0x3c, 0xd8, 0x66, 0xa3, 0x39, 0x3a, 0x3a, 0xc5, 0x3a, 0x56, 0x7a, 0xa8, 0x01, 0x53, 0x3e, 0x1e, 0x39, 0x44, 0xdc, 0x7a, 0x7e, 0xce, 0xb8, 0x99, 0xb7, 0xa2, 0x36, 0x41, 0xf4, 0xed, 0x97, 0xe1, 0x56, 0x88, 0xfd, 0x41, 0x7d, 0x9c, 0x21, 0x92, 0x8e, 0x0e, 0xf6, 0x07, 0x0f, 0x0a, 0x3f, 0xf8, 0x87, 0x7a, 0x7e, 0x71, 0xfe, 0x8e, 0xf9, 0xaf, 0x13, 0x50, 0xb6, 0x6c, 0x77, 0x07, 0x5b, 0xf8, 0x7b, 0xfb, 0x38, 0x08, 0x51, 0x0d, 0xf2, 0x7b, 0xf8, 0x90, 0xca, 0x51, 0xb6, 0xc8, 0x27, 0x23, 0xe4, 0xee, 0xe0, 0x2d, 0xec, 0x32, 0x09, 0xca, 0x84, 0x90, 0xbb, 0x83, 0xdb, 0x6e, 0x0f, 0xcd, 0xc0, 0x44, 0xdf, 0x19, 0x38, 0x21, 0x67, 0xcf, 0x1a, 0x9a, 0x5c, 0xe3, 0x31, 0xb9, 0x96, 0x00, 0x02, 0xcf, 0x0f, 0xb7, 0x3c, 0xbf, 0x87, 0xfd, 0xfa, 0xc4, 0x9c, 0x71, 0xb3, 0xba, 0x70, 0x7d, 0x5e, 0xb5, 0xe5, 0xbc, 0x2a, 0xd0, 0xfc, 0xa6, 0xe7, 0x87, 0xeb, 0x04, 0xd6, 0x2a, 0x06, 0xe2, 0x13, 0x7d, 0x04, 0x25, 0x4a, 0x24, 0xb4, 0xfd, 0x1d, 0x1c, 0xd6, 0x27, 0x29, 0x95, 0x1b, 0xc7, 0x50, 0xe9, 0x50, 0x60, 0x8b, 0xb2, 0x67, 0xdf, 0xc8, 0x84, 0x72, 0x80, 0x7d, 0xc7, 0xee, 0x3b, 0x9f, 0xd9, 0xdb, 0x7d, 0x5c, 0x2f, 0xcc, 0x19, 0x37, 0xa7, 0x2c, 0xad, 0x8f, 0xcc, 0x7f, 0x0f, 0x1f, 0x06, 0x5b, 0x9e, 0xdb, 0x3f, 0xac, 0x4f, 0x51, 0x80, 0x29, 0xd2, 0xb1, 0xee, 0xf6, 0x0f, 0xa9, 0xf5, 0xbc, 0x7d, 0x37, 0x64, 0xa3, 0x45, 0x3a, 0x5a, 0xa4, 0x3d, 0x74, 0xf8, 0x2e, 0xd4, 0x06, 0x8e, 0xbb, 0x35, 0xf0, 0x7a, 0x5b, 0x91, 0x42, 0x80, 0x28, 0xe4, 0x61, 0xe1, 0xf7, 0xa8, 0x05, 0xee, 0x5a, 0xd5, 0x81, 0xe3, 0x3e, 0xf5, 0x7a, 0x96, 0xd0, 0x0f, 0x41, 0xb1, 0x0f, 0x74, 0x94, 0x52, 0x1c, 0xc5, 0x3e, 0x50, 0x51, 0xde, 0x87, 0xb3, 0x84, 0x4b, 0xd7, 0xc7, 0x76, 0x88, 0x25, 0x56, 0x59, 0xc7, 0x3a, 0x33, 0x70, 0xdc, 0x25, 0x0a, 0xa2, 0x21, 0xda, 0x07, 0x09, 0xc4, 0x4a, 0x1c, 0xd1, 0x3e, 0xd0, 0x11, 0xcd, 0xf7, 0xa1, 0x18, 0xd9, 0x05, 0x4d, 0xc1, 0xf8, 0xda, 0xfa, 0x5a, 0xbb, 0x36, 0x86, 0x00, 0x26, 0x5b, 0x9b, 0x4b, 0xed, 0xb5, 0xe5, 0x9a, 0x81, 0x4a, 0x50, 0x58, 0x6e, 0xb3, 0x46, 0xae, 0x51, 0xf8, 0x82, 0xaf, 0xb7, 0x27, 0x00, 0xd2, 0x14, 0xa8, 0x00, 0xf9, 0x27, 0xed, 0x4f, 0x6b, 0x63, 0x04, 0xf8, 0x79, 0xdb, 0xda, 0x5c, 0x59, 0x5f, 0xab, 0x19, 0x84, 0xca, 0x92, 0xd5, 0x6e, 0x75, 0xda, 0xb5, 0x1c, 0x81, 0x78, 0xba, 0xbe, 0x5c, 0xcb, 0xa3, 0x22, 0x4c, 0x3c, 0x6f, 0xad, 0x3e, 0x6b, 0xd7, 0xc6, 0x23, 0x62, 0x72, 0x15, 0xff, 0x89, 0x01, 0x15, 0x6e, 0x6e, 0xb6, 0xb7, 0xd0, 0x3d, 0x98, 0xdc, 0xa5, 0xfb, 0x8b, 0xae, 0xe4, 0xd2, 0xc2, 0xe5, 0xd8, 0xda, 0xd0, 0xf6, 0xa0, 0xc5, 0x61, 0x91, 0x09, 0xf9, 0xbd, 0x51, 0x50, 0xcf, 0xcd, 0xe5, 0x6f, 0x96, 0x16, 0x6a, 0xf3, 0xcc, 0x93, 0xcc, 0x3f, 0xc1, 0x87, 0xcf, 0xed, 0xfe, 0x3e, 0xb6, 0xc8, 0x20, 0x42, 0x30, 0x3e, 0xf0, 0x7c, 0x4c, 0x17, 0xfc, 0x94, 0x45, 0xbf, 0xc9, 0x2e, 0xa0, 0x36, 0xe7, 0x8b, 0x9d, 0x35, 0xa4, 0x78, 0xff, 0x69, 0x00, 0x6c, 0xec, 0x87, 0xd9, 0x5b, 0x6c, 0x06, 0x26, 0x46, 0x84, 0x03, 0xdf, 0x5e, 0xac, 0x41, 0xf7, 0x16, 0xb6, 0x03, 0x1c, 0xed, 0x2d, 0xd2, 0x40, 0x73, 0x50, 0x18, 0xfa, 0x78, 0xb4, 0xb5, 0x37, 0xa2, 0xdc, 0xa6, 0xa4, 0x9d, 0x26, 0x49, 0xff, 0x93, 0x11, 0xba, 0x05, 0x65, 0x67, 0xc7, 0xf5, 0x7c, 0xbc, 0xc5, 0x88, 0x4e, 0xa8, 0x60, 0x0b, 0x56, 0x89, 0x0d, 0xd2, 0x29, 0x29, 0xb0, 0x8c, 0xd5, 0x64, 0x2a, 0xec, 0x2a, 0x19, 0x93, 0xf3, 0xf9, 0xdc, 0x80, 0x12, 0x9d, 0xcf, 0x89, 0x94, 0xbd, 0x20, 0x27, 0x92, 0xa3, 0x68, 0x09, 0x85, 0x27, 0xa6, 0x26, 0x45, 0x70, 0x01, 0x2d, 0xe3, 0x3e, 0x0e, 0xf1, 0x49, 0x9c, 0x97, 0xa2, 0xca, 0x7c, 0xaa, 0x2a, 0x25, 0xbf, 0xbf, 0x30, 0xe0, 0xac, 0xc6, 0xf0, 0x44, 0x53, 0xaf, 0x43, 0xa1, 0x47, 0x89, 0x31, 0x99, 0xf2, 0x96, 0x68, 0xa2, 0x7b, 0x30, 0xc5, 0x45, 0x0a, 0xea, 0xf9, 0xf4, 0x65, 0x28, 0xa5, 0x2c, 0x30, 0x29, 0x03, 0x29, 0xe6, 0x3f, 0xe5, 0xa0, 0xc8, 0x95, 0xb1, 0x3e, 0x44, 0x2d, 0xa8, 0xf8, 0xac, 0xb1, 0x45, 0xe7, 0xcc, 0x65, 0x6c, 0x64, 0xfb, 0xc9, 0xc7, 0x63, 0x56, 0x99, 0xa3, 0xd0, 0x6e, 0xf4, 0xab, 0x50, 0x12, 0x24, 0x86, 0xfb, 0x21, 0x37, 0x54, 0x5d, 0x27, 0x20, 0x97, 0xf6, 0xe3, 0x31, 0x0b, 0x38, 0xf8, 0xc6, 0x7e, 0x88, 0x3a, 0x30, 0x23, 0x90, 0xd9, 0xfc, 0xb8, 0x18, 0x79, 0x4a, 0x65, 0x4e, 0xa7, 0x92, 0x34, 0xe7, 0xe3, 0x31, 0x0b, 0x71, 0x7c, 0x65, 0x10, 0x2d, 0x4b, 0x91, 0xc2, 0x03, 0x16, 0x5f, 0x12, 0x22, 0x75, 0x0e, 0x5c, 0x4e, 0x44, 0x68, 0x6b, 0x51, 0x91, 0xad, 0x73, 0xe0, 0x46, 0x2a, 0x7b, 0x58, 0x84, 0x02, 0xef, 0x36, 0xff, 0x23, 0x07, 0x20, 0x2c, 0xb6, 0x3e, 0x44, 0xcb, 0x50, 0xf5, 0x79, 0x4b, 0xd3, 0xdf, 0xa5, 0x54, 0xfd, 0x71, 0x43, 0x8f, 0x59, 0x15, 0x81, 0xc4, 0xc4, 0xfd, 0x10, 0xca, 0x11, 0x15, 0xa9, 0xc2, 0x8b, 0x29, 0x2a, 0x8c, 0x28, 0x94, 0x04, 0x02, 0x51, 0xe2, 0x27, 0x70, 0x2e, 0xc2, 0x4f, 0xd1, 0xe2, 0x1b, 0x47, 0x68, 0x31, 0x22, 0x78, 0x56, 0x50, 0x50, 0xf5, 0xf8, 0x48, 0x11, 0x4c, 0x2a, 0xf2, 0x62, 0x8a, 0x22, 0x19, 0x90, 0xaa, 0xc9, 0x48, 0x42, 0x4d, 0x95, 0x40, 0xc2, 0x3e, 0xeb, 0x37, 0xff, 0x6a, 0x1c, 0x0a, 0x4b, 0xde, 0x60, 0x68, 0xfb, 0x64, 0x11, 0x4d, 0xfa, 0x38, 0xd8, 0xef, 0x87, 0x54, 0x81, 0xd5, 0x85, 0x6b, 0x3a, 0x0f, 0x0e, 0x26, 0xfe, 0x5a, 0x14, 0xd4, 0xe2, 0x28, 0x04, 0x99, 0x47, 0xf9, 0xdc, 0x6b, 0x20, 0xf3, 0x18, 0xcf, 0x51, 0x84, 0x43, 0xc8, 0x4b, 0x87, 0xd0, 0x80, 0x02, 0x3f, 0xe0, 0x31, 0x67, 0xfd, 0x78, 0xcc, 0x12, 0x1d, 0xe8, 0x6d, 0x98, 0x8e, 0x87, 0xc2, 0x09, 0x0e, 0x53, 0xed, 0xea, 0x91, 0xf3, 0x1a, 0x94, 0xb5, 0x08, 0x3d, 0xc9, 0xe1, 0x4a, 0x03, 0x25, 0x2e, 0x9f, 0x17, 0x6e, 0x9d, 0x1c, 0x2b, 0xca, 0x8f, 0xc7, 0x84, 0x63, 0xbf, 0x2a, 0x1c, 0xfb, 0x94, 0x1a, 0x68, 0x89, 0x5e, 0xb9, 0x8f, 0xbf, 0xae, 0x7a, 0xad, 0x6f, 0x12, 0xe4, 0x08, 0x48, 0xba, 0x2f, 0xd3, 0x82, 0x8a, 0xa6, 0x32, 0x12, 0x23, 0xdb, 0x1f, 0x3f, 0x6b, 0xad, 0xb2, 0x80, 0xfa, 0x88, 0xc6, 0x50, 0xab, 0x66, 0x90, 0x00, 0xbd, 0xda, 0xde, 0xdc, 0xac, 0xe5, 0xd0, 0x79, 0x28, 0xae, 0xad, 0x77, 0xb6, 0x18, 0x54, 0xbe, 0x51, 0xf8, 0x63, 0xe6, 0x49, 0x64, 0x7c, 0xfe, 0x34, 0xa2, 0xc9, 0x43, 0xb4, 0x12, 0x99, 0xc7, 0x94, 0xc8, 0x6c, 0x88, 0xc8, 0x9c, 0x93, 0x91, 0x39, 0x8f, 0x10, 0x4c, 0xac, 0xb6, 0x5b, 0x9b, 0x34, 0x48, 0x33, 0xd2, 0x8b, 0xc9, 0x68, 0xfd, 0xb0, 0x0a, 0x65, 0x66, 0x9e, 0xad, 0x7d, 0x97, 0x1c, 0x26, 0xfe, 0xda, 0x00, 0x90, 0x1b, 0x16, 0x35, 0xa1, 0xd0, 0x65, 0x22, 0xd4, 0x0d, 0xea, 0x01, 0xcf, 0xa5, 0x5a, 0xdc, 0x12, 0x50, 0xe8, 0x2e, 0x14, 0x82, 0xfd, 0x6e, 0x17, 0x07, 0x22, 0x72, 0x5f, 0x88, 0x3b, 0x61, 0xee, 0x10, 0x2d, 0x01, 0x47, 0x50, 0x5e, 0xda, 0x4e, 0x7f, 0x9f, 0xc6, 0xf1, 0xa3, 0x51, 0x38, 0x9c, 0xf4, 0xb1, 0x7f, 0x66, 0x40, 0x49, 0xd9, 0x16, 0xbf, 0x60, 0x08, 0xb8, 0x0c, 0x45, 0x2a, 0x0c, 0xee, 0xf1, 0x20, 0x30, 0x65, 0xc9, 0x0e, 0xf4, 0x1e, 0x14, 0xc5, 0x4e, 0x12, 0x71, 0xa0, 0x9e, 0x4e, 0x76, 0x7d, 0x68, 0x49, 0x50, 0x29, 0x64, 0x07, 0xce, 0x50, 0x3d, 0x75, 0xc9, 0xed, 0x43, 0x68, 0x56, 0x3d, 0x96, 0x1b, 0xb1, 0x63, 0x79, 0x03, 0xa6, 0x86, 0xbb, 0x87, 0x81, 0xd3, 0xb5, 0xfb, 0x5c, 0x9c, 0xa8, 0x2d, 0xa9, 0x6e, 0x02, 0x52, 0xa9, 0x9e, 0x44, 0x01, 0x92, 0xe8, 0x79, 0x28, 0x3d, 0xb6, 0x83, 0x5d, 0x2e, 0xa4, 0xec, 0xbf, 0x07, 0x15, 0xd2, 0xff, 0xe4, 0xf9, 0x6b, 0x88, 0x2f, 0xb0, 0x16, 0xcd, 0x7f, 0x36, 0xa0, 0x2a, 0xd0, 0x4e, 0x64, 0x20, 0x04, 0xe3, 0xbb, 0x76, 0xb0, 0x4b, 0x95, 0x51, 0xb1, 0xe8, 0x37, 0x7a, 0x1b, 0x6a, 0x5d, 0x36, 0xff, 0xad, 0xd8, 0xbd, 0x6b, 0x9a, 0xf7, 0x47, 0x7b, 0xff, 0x5d, 0xa8, 0x10, 0x94, 0x2d, 0xfd, 0x1e, 0x24, 0xb6, 0xf1, 0x7b, 0x56, 0x79, 0x97, 0xce, 0x39, 0x2e, 0xbe, 0x0d, 0x65, 0xa6, 0x8c, 0xd3, 0x96, 0x5d, 0xea, 0xb5, 0x01, 0xd3, 0x9b, 0xae, 0x3d, 0x0c, 0x76, 0xbd, 0x30, 0xa6, 0xf3, 0x45, 0xf3, 0xef, 0x0d, 0xa8, 0xc9, 0xc1, 0x13, 0xc9, 0xf0, 0x16, 0x4c, 0xfb, 0x78, 0x60, 0x3b, 0xae, 0xe3, 0xee, 0x6c, 0x6d, 0x1f, 0x86, 0x38, 0xe0, 0xd7, 0xd7, 0x6a, 0xd4, 0xfd, 0x90, 0xf4, 0x12, 0x61, 0xb7, 0xfb, 0xde, 0x36, 0x77, 0xd2, 0xf4, 0x1b, 0xbd, 0xa1, 0x7b, 0xe9, 0xa2, 0xd4, 0x9b, 0xe8, 0x97, 0x32, 0xff, 0x34, 0x07, 0xe5, 0x4f, 0xec, 0xb0, 0x2b, 0x56, 0x10, 0x5a, 0x81, 0x6a, 0xe4, 0xc6, 0x69, 0x0f, 0x97, 0x3b, 0x76, 0xe0, 0xa0, 0x38, 0xe2, 0x5e, 0x23, 0x0e, 0x1c, 0x95, 0xae, 0xda, 0x41, 0x49, 0xd9, 0x6e, 0x17, 0xf7, 0x23, 0x52, 0xb9, 0x6c, 0x52, 0x14, 0x50, 0x25, 0xa5, 0x76, 0xa0, 0x6f, 0x43, 0x6d, 0xe8, 0x7b, 0x3b, 0x3e, 0x0e, 0x82, 0x88, 0x18, 0x0b, 0xe1, 0x66, 0x0a, 0xb1, 0x0d, 0x0e, 0x1a, 0x3b, 0xc5, 0xdc, 0x7b, 0x3c, 0x66, 0x4d, 0x0f, 0xf5, 0x31, 0xe9, 0x58, 0xa7, 0xe5, 0x79, 0x8f, 0x79, 0xd6, 0x1f, 0xe5, 0x01, 0x25, 0xa7, 0xf9, 0x55, 0x8f, 0xc9, 0x37, 0xa0, 0x1a, 0x84, 0xb6, 0x9f, 0x58, 0xf3, 0x15, 0xda, 0x1b, 0xad, 0xf8, 0xb7, 0x20, 0x92, 0x6c, 0xcb, 0xf5, 0x42, 0xe7, 0xe5, 0x21, 0xbb, 0xa0, 0x58, 0x55, 0xd1, 0xbd, 0x46, 0x7b, 0xd1, 0x1a, 0x14, 0x5e, 0x3a, 0xfd, 0x10, 0xfb, 0x41, 0x7d, 0x62, 0x2e, 0x7f, 0xb3, 0xba, 0xf0, 0xce, 0x71, 0x86, 0x99, 0xff, 0x88, 0xc2, 0x77, 0x0e, 0x87, 0xea, 0xe9, 0x97, 0x13, 0x51, 0x8f, 0xf1, 0x93, 0xe9, 0x37, 0x22, 0x13, 0xa6, 0x5e, 0x11, 0xa2, 0x5b, 0x4e, 0x8f, 0xc6, 0xe2, 0x68, 0x1f, 0xde, 0xb3, 0x0a, 0x74, 0x60, 0xa5, 0x87, 0xae, 0xc1, 0xd4, 0x4b, 0xdf, 0xde, 0x19, 0x60, 0x37, 0x64, 0xb7, 0x7c, 0x09, 0x13, 0x0d, 0x98, 0xf3, 0x00, 0x52, 0x14, 0x12, 0xf9, 0xd6, 0xd6, 0x37, 0x9e, 0x75, 0x6a, 0x63, 0xa8, 0x0c, 0x53, 0x6b, 0xeb, 0xcb, 0xed, 0xd5, 0x36, 0x89, 0x8d, 0x22, 0xe6, 0xdd, 0x95, 0x9b, 0xae, 0x25, 0x0c, 0xa1, 0xad, 0x09, 0x55, 0x2e, 0x43, 0xbf, 0x74, 0x0b, 0xb9, 0x04, 0x89, 0xbb, 0xe6, 0x55, 0x98, 0x49, 0x5b, 0x1a, 0x02, 0xe0, 0x9e, 0xf9, 0x6f, 0x39, 0xa8, 0xf0, 0x8d, 0x70, 0xa2, 0x9d, 0x7b, 0x51, 0x91, 0x8a, 0x5f, 0x4f, 0x84, 0x92, 0xea, 0x50, 0x60, 0x1b, 0xa4, 0xc7, 0xef, 0xbf, 0xa2, 0x49, 0x9c, 0x33, 0x5b, 0xef, 0xb8, 0xc7, 0xcd, 0x1e, 0xb5, 0x53, 0xdd, 0xe6, 0x44, 0xa6, 0xdb, 0x8c, 0x36, 0x9c, 0x1d, 0xf0, 0x83, 0x55, 0x51, 0x9a, 0xa2, 0x2c, 0x36, 0x15, 0x19, 0xd4, 0x6c, 0x56, 0xc8, 0xb0, 0x19, 0xba, 0x01, 0x93, 0x78, 0x84, 0xdd, 0x30, 0xa8, 0x97, 0x68, 0x20, 0xad, 0x88, 0x0b, 0x55, 0x9b, 0xf4, 0x5a, 0x7c, 0x50, 0x9a, 0xea, 0x43, 0x38, 0x43, 0xef, 0xbb, 0x8f, 0x7c, 0xdb, 0x55, 0xef, 0xec, 0x9d, 0xce, 0x2a, 0x0f, 0x3b, 0xe4, 0x13, 0x55, 0x21, 0xb7, 0xb2, 0xcc, 0xf5, 0x93, 0x5b, 0x59, 0x96, 0xf8, 0xbf, 0x6f, 0x00, 0x52, 0x09, 0x9c, 0xc8, 0x16, 0x31, 0x2e, 0x42, 0x8e, 0xbc, 0x94, 0x63, 0x06, 0x26, 0xb0, 0xef, 0x7b, 0x3e, 0x73, 0x94, 0x16, 0x6b, 0x48, 0x69, 0x6e, 0x73, 0x61, 0x2c, 0x3c, 0xf2, 0xf6, 0x22, 0x0f, 0xc0, 0xc8, 0x1a, 0x49, 0xe1, 0x3b, 0x70, 0x56, 0x03, 0x3f, 0x9d, 0x10, 0xbf, 0x0e, 0xd3, 0x94, 0xea, 0xd2, 0x2e, 0xee, 0xee, 0x0d, 0x3d, 0xc7, 0x4d, 0x48, 0x80, 0xae, 0x11, 0xdf, 0x25, 0xc2, 0x05, 0x99, 0x22, 0x9b, 0x73, 0x39, 0xea, 0xec, 0x74, 0x56, 0xe5, 0x52, 0xdf, 0x86, 0xf3, 0x31, 0x82, 0x62, 0x66, 0xbf, 0x06, 0xa5, 0x6e, 0xd4, 0x19, 0xf0, 0x13, 0xe4, 0x15, 0x5d, 0xdc, 0x38, 0xaa, 0x8a, 0x21, 0x79, 0x7c, 0x1b, 0x2e, 0x24, 0x78, 0x9c, 0x86, 0x3a, 0xee, 0x99, 0x77, 0xe0, 0x1c, 0xa5, 0xfc, 0x04, 0xe3, 0x61, 0xab, 0xef, 0x8c, 0x8e, 0x37, 0xcb, 0x21, 0x9f, 0xaf, 0x82, 0xf1, 0xf5, 0x2e, 0x2b, 0xc9, 0xba, 0xcd, 0x59, 0x77, 0x9c, 0x01, 0xee, 0x78, 0xab, 0xd9, 0xd2, 0x92, 0x40, 0xbe, 0x87, 0x0f, 0x03, 0x7e, 0x7c, 0xa4, 0xdf, 0xd2, 0x7b, 0xfd, 0xad, 0xc1, 0xd5, 0xa9, 0xd2, 0xf9, 0x9a, 0xb7, 0xc6, 0x2c, 0xc0, 0x0e, 0xd9, 0x83, 0xb8, 0x47, 0x06, 0x58, 0x6e, 0x4e, 0xe9, 0x89, 0x04, 0x26, 0x51, 0xa8, 0x1c, 0x17, 0xf8, 0x0a, 0xdf, 0x38, 0xf4, 0x9f, 0x20, 0x71, 0x52, 0x7a, 0x13, 0x4a, 0x74, 0x64, 0x33, 0xb4, 0xc3, 0xfd, 0x20, 0xcb, 0x72, 0x8b, 0xe6, 0x8f, 0x0c, 0xbe, 0xa3, 0x04, 0x9d, 0x13, 0xcd, 0xf9, 0x2e, 0x4c, 0xd2, 0x1b, 0xa2, 0xb8, 0xe9, 0x5c, 0x4c, 0x59, 0xd8, 0x4c, 0x22, 0x8b, 0x03, 0x2a, 0xe7, 0x24, 0x03, 0x26, 0x9f, 0xd2, 0xca, 0x81, 0x22, 0xed, 0xb8, 0xb0, 0x9c, 0x6b, 0x0f, 0x58, 0xfa, 0xb1, 0x68, 0xd1, 0x6f, 0x7a, 0x21, 0xc0, 0xd8, 0x7f, 0x66, 0xad, 0xb2, 0x1b, 0x48, 0xd1, 0x8a, 0xda, 0x44, 0xb1, 0xdd, 0xbe, 0x83, 0xdd, 0x90, 0x8e, 0x8e, 0xd3, 0x51, 0xa5, 0x07, 0xdd, 0x80, 0xa2, 0x13, 0xac, 0x62, 0xdb, 0x77, 0x79, 0x8a, 0x5f, 0x71, 0xcc, 0x72, 0x44, 0xae, 0xb1, 0xef, 0x40, 0x8d, 0x49, 0xd6, 0xea, 0xf5, 0x94, 0xd3, 0x7e, 0xc4, 0xdf, 0x88, 0xf1, 0xd7, 0xe8, 0xe7, 0x8e, 0xa7, 0xff, 0x77, 0x06, 0x9c, 0x51, 0x18, 0x9c, 0xc8, 0x04, 0xef, 0xc2, 0x24, 0xab, 0xbf, 0xf0, 0xa3, 0xe0, 0x8c, 0x8e, 0xc5, 0xd8, 0x58, 0x1c, 0x06, 0xcd, 0x43, 0x81, 0x7d, 0x89, 0x6b, 0x5c, 0x3a, 0xb8, 0x00, 0x92, 0x22, 0xcf, 0xc3, 0x59, 0x3e, 0x86, 0x07, 0x5e, 0xda, 0x9e, 0x1b, 0xd7, 0x3d, 0xc4, 0x0f, 0x0d, 0x98, 0xd1, 0x11, 0x4e, 0x34, 0x4b, 0x45, 0xee, 0xdc, 0x57, 0x92, 0xfb, 0x5b, 0x42, 0xee, 0x67, 0xc3, 0x9e, 0x72, 0xe4, 0x8c, 0xaf, 0x38, 0xd5, 0xba, 0x39, 0xdd, 0xba, 0x92, 0xd6, 0x4f, 0xa2, 0x39, 0x09, 0x62, 0x27, 0x9a, 0xd3, 0xfb, 0xaf, 0x35, 0x27, 0xe5, 0x08, 0x96, 0x98, 0xdc, 0x8a, 0x58, 0x46, 0xab, 0x4e, 0x10, 0x45, 0x9c, 0x77, 0xa0, 0xdc, 0x77, 0x5c, 0x6c, 0xfb, 0xbc, 0x86, 0x64, 0xa8, 0xeb, 0xf1, 0xbe, 0xa5, 0x0d, 0x4a, 0x52, 0xbf, 0x6d, 0x00, 0x52, 0x69, 0xfd, 0x72, 0xac, 0xd5, 0x14, 0x0a, 0xde, 0xf0, 0xbd, 0x81, 0x17, 0x1e, 0xb7, 0xcc, 0xee, 0x99, 0xbf, 0x6b, 0xc0, 0xb9, 0x18, 0xc6, 0x2f, 0x43, 0xf2, 0x7b, 0xe6, 0x65, 0x38, 0xb3, 0x8c, 0xc5, 0x19, 0x2f, 0x91, 0x3b, 0xd8, 0x04, 0xa4, 0x8e, 0x9e, 0xce, 0x29, 0xe6, 0x1b, 0x70, 0xe6, 0xa9, 0x37, 0x22, 0x8e, 0x9c, 0x0c, 0x4b, 0x37, 0xc5, 0x92, 0x59, 0x91, 0xbe, 0xa2, 0xb6, 0x74, 0xbd, 0x9b, 0x80, 0x54, 0xcc, 0xd3, 0x10, 0x67, 0xd1, 0xfc, 0x5f, 0x03, 0xca, 0xad, 0xbe, 0xed, 0x0f, 0x84, 0x28, 0x1f, 0xc2, 0x24, 0xcb, 0xcc, 0xf0, 0x34, 0xeb, 0x9b, 0x3a, 0x3d, 0x15, 0x96, 0x35, 0x5a, 0x2c, 0x8f, 0xc3, 0xb1, 0xc8, 0x54, 0x78, 0x65, 0x79, 0x39, 0x56, 0x69, 0x5e, 0x46, 0xb7, 0x61, 0xc2, 0x26, 0x28, 0x34, 0xbc, 0x56, 0xe3, 0xe9, 0x32, 0x4a, 0x8d, 0x5c, 0x89, 0x2c, 0x06, 0x65, 0x7e, 0x00, 0x25, 0x85, 0x03, 0x2a, 0x40, 0xfe, 0x51, 0x9b, 0x5f, 0x93, 0x5a, 0x4b, 0x9d, 0x95, 0xe7, 0x2c, 0x85, 0x58, 0x05, 0x58, 0x6e, 0x47, 0xed, 0x5c, 0x4a, 0x61, 0xcf, 0xe6, 0x74, 0x78, 0xdc, 0x52, 0x25, 0x34, 0xb2, 0x24, 0xcc, 0xbd, 0x8e, 0x84, 0x92, 0xc5, 0x6f, 0x19, 0x50, 0xe1, 0xaa, 0x39, 0x69, 0x68, 0xa6, 0x94, 0x33, 0x42, 0xb3, 0x32, 0x0d, 0x8b, 0x03, 0x4a, 0x19, 0xfe, 0xc5, 0x80, 0xda, 0xb2, 0xf7, 0xca, 0xdd, 0xf1, 0xed, 0x5e, 0xb4, 0x07, 0x3f, 0x8a, 0x99, 0x73, 0x3e, 0x96, 0xe9, 0x8f, 0xc1, 0xcb, 0x8e, 0x98, 0x59, 0xeb, 0x32, 0x97, 0xc2, 0xe2, 0xbb, 0x68, 0x9a, 0xdf, 0x84, 0xe9, 0x18, 0x12, 0x31, 0xd0, 0xf3, 0xd6, 0xea, 0xca, 0x32, 0x31, 0x08, 0xcd, 0xf7, 0xb6, 0xd7, 0x5a, 0x0f, 0x57, 0xdb, 0xbc, 0x2a, 0xdb, 0x5a, 0x5b, 0x6a, 0xaf, 0x4a, 0x43, 0xdd, 0x17, 0x33, 0xb8, 0x6f, 0xf6, 0xe1, 0x8c, 0x22, 0xd0, 0x49, 0x8b, 0x63, 0xe9, 0xf2, 0x4a, 0x6e, 0xdf, 0x80, 0x4b, 0x11, 0xb7, 0xe7, 0x6c, 0xb0, 0x83, 0x03, 0xf5, 0xb2, 0x36, 0xe2, 0x4c, 0x8b, 0x16, 0xf9, 0x14, 0x98, 0xef, 0x99, 0x75, 0xa8, 0xf0, 0xf3, 0x51, 0xdc, 0x65, 0xfc, 0xf9, 0x38, 0x54, 0xc5, 0xd0, 0xd7, 0x23, 0x3f, 0x3a, 0x0f, 0x93, 0xbd, 0xed, 0x4d, 0xe7, 0x33, 0x51, 0xd1, 0xe5, 0x2d, 0xd2, 0xdf, 0x67, 0x7c, 0xd8, 0x3b, 0x0d, 0xde, 0x42, 0x97, 0xd9, 0x13, 0x8e, 0x15, 0xb7, 0x87, 0x0f, 0xe8, 0x31, 0x6a, 0xdc, 0x92, 0x1d, 0x34, 0x1d, 0xca, 0xdf, 0x73, 0xd0, 0x5b, 0xb2, 0xf2, 0xbe, 0x03, 0x2d, 0x42, 0x8d, 0x7c, 0xb7, 0x86, 0xc3, 0xbe, 0x83, 0x7b, 0x8c, 0x00, 0xb9, 0x20, 0x8f, 0xcb, 0x73, 0x52, 0x02, 0x00, 0x5d, 0x85, 0x49, 0x7a, 0x79, 0x0c, 0xea, 0x53, 0x24, 0x22, 0x4b, 0x50, 0xde, 0x8d, 0xde, 0x86, 0x12, 0x93, 0x78, 0xc5, 0x7d, 0x16, 0x60, 0xfa, 0xda, 0x41, 0xc9, 0xa4, 0xa8, 0x63, 0xfa, 0x09, 0x0d, 0xb2, 0x4e, 0x68, 0xa8, 0x09, 0xd5, 0x20, 0xf4, 0x7c, 0x7b, 0x47, 0x98, 0x91, 0x3e, 0x75, 0x50, 0xd2, 0x7d, 0xb1, 0x61, 0x29, 0xc2, 0xc7, 0xfb, 0x5e, 0x68, 0xeb, 0x4f, 0x1c, 0xde, 0xb3, 0xd4, 0x31, 0xf4, 0x2d, 0xa8, 0xf4, 0xc4, 0x22, 0x59, 0x71, 0x5f, 0x7a, 0xf4, 0x59, 0x43, 0xa2, 0x7a, 0xb7, 0xac, 0x82, 0x48, 0x4a, 0x3a, 0xaa, 0x7a, 0x93, 0xad, 0x68, 0x18, 0xc4, 0xda, 0xd8, 0x25, 0xa1, 0x9d, 0x65, 0x70, 0xa6, 0x2c, 0xd1, 0x44, 0xd7, 0xa1, 0xc2, 0x22, 0xc1, 0x73, 0x6d, 0x35, 0xe8, 0x9d, 0x24, 0x8e, 0xb5, 0xf6, 0xc3, 0xdd, 0x36, 0x45, 0x4a, 0x2c, 0xca, 0x2b, 0x80, 0xc8, 0xe8, 0xb2, 0x13, 0xa4, 0x0e, 0x73, 0xe4, 0xd4, 0x15, 0x7d, 0xdf, 0x5c, 0x83, 0xb3, 0x64, 0x14, 0xbb, 0xa1, 0xd3, 0x55, 0x8e, 0x62, 0xe2, 0xb0, 0x6f, 0xc4, 0x0e, 0xfb, 0x76, 0x10, 0xbc, 0xf2, 0xfc, 0x1e, 0x17, 0x33, 0x6a, 0x4b, 0x6e, 0xff, 0x68, 0x30, 0x69, 0x9e, 0x05, 0xda, 0x41, 0xfd, 0x2b, 0xd2, 0x43, 0xbf, 0x02, 0x05, 0xfe, 0x40, 0x8a, 0xe7, 0x3f, 0xcf, 0xcf, 0xb3, 0x87, 0x59, 0xf3, 0x9c, 0xf0, 0x3a, 0x1b, 0x55, 0x72, 0x74, 0x1c, 0x9e, 0x2c, 0x97, 0x5d, 0x3b, 0xd8, 0xc5, 0xbd, 0x0d, 0x41, 0x5c, 0xcb, 0x0e, 0xdf, 0xb7, 0x62, 0xc3, 0x52, 0xf6, 0xbb, 0x52, 0xf4, 0x47, 0x38, 0x3c, 0x42, 0x74, 0xb5, 0xfe, 0x70, 0x4e, 0xa0, 0xf0, 0xb2, 0xe9, 0xeb, 0x60, 0xfd, 0xd8, 0x80, 0x2b, 0x02, 0x6d, 0x69, 0xd7, 0x76, 0x77, 0xb0, 0x10, 0xe6, 0x17, 0xd5, 0x57, 0x72, 0xd2, 0xf9, 0xd7, 0x9c, 0xf4, 0x13, 0xa8, 0x47, 0x93, 0xa6, 0xb9, 0x28, 0xaf, 0xaf, 0x4e, 0x62, 0x3f, 0x88, 0x9c, 0x24, 0xfd, 0x26, 0x7d, 0xbe, 0xd7, 0x8f, 0xae, 0x81, 0xe4, 0x5b, 0x12, 0x5b, 0x85, 0x8b, 0x82, 0x18, 0x4f, 0x0e, 0xe9, 0xd4, 0x12, 0x73, 0x3a, 0x92, 0x1a, 0xb7, 0x07, 0xa1, 0x71, 0xf4, 0x52, 0x4a, 0x45, 0xd1, 0x4d, 0x48, 0xb9, 0x18, 0x69, 0x5c, 0x66, 0xd9, 0x0e, 0x20, 0x32, 0x2b, 0x27, 0xf6, 0xc4, 0x38, 0x21, 0x99, 0x3a, 0xce, 0x97, 0x00, 0x19, 0x4f, 0x2c, 0x81, 0x6c, 0xae, 0x18, 0x66, 0x23, 0x41, 0x89, 0xda, 0x37, 0xb0, 0x3f, 0x70, 0x82, 0x40, 0x29, 0xc4, 0xa5, 0xa9, 0xeb, 0x4d, 0x18, 0x1f, 0x62, 0x7e, 0x7c, 0x29, 0x2d, 0x20, 0xb1, 0x27, 0x14, 0x64, 0x3a, 0x2e, 0xd9, 0x0c, 0xe0, 0xaa, 0x60, 0xc3, 0x0c, 0x92, 0xca, 0x27, 0x2e, 0xa6, 0x48, 0xfe, 0xe7, 0x32, 0x92, 0xff, 0x79, 0x3d, 0xf9, 0xaf, 0x1d, 0xa9, 0x55, 0x47, 0x75, 0x3a, 0x47, 0xea, 0x0e, 0x33, 0x40, 0xe4, 0xdf, 0x4e, 0x87, 0xea, 0x1f, 0x70, 0x47, 0x75, 0x5a, 0xe1, 0x5c, 0x38, 0xf8, 0x9c, 0xee, 0xe0, 0x4d, 0x28, 0x13, 0x23, 0x59, 0x6a, 0x55, 0x64, 0xdc, 0xd2, 0xfa, 0xa4, 0x33, 0xde, 0x83, 0x19, 0xdd, 0x19, 0x9f, 0x48, 0xa8, 0x19, 0x98, 0x08, 0xbd, 0x3d, 0x2c, 0x62, 0x0a, 0x6b, 0x24, 0xd4, 0x1a, 0x39, 0xea, 0xd3, 0x51, 0xeb, 0x77, 0x25, 0x55, 0xba, 0x01, 0x4f, 0x3a, 0x03, 0xb2, 0x1c, 0xc5, 0xed, 0x9f, 0x35, 0x24, 0xaf, 0x4f, 0xe0, 0x7c, 0xdc, 0xf9, 0x9e, 0xce, 0x24, 0xb6, 0xd8, 0xe6, 0x4c, 0x73, 0xcf, 0xa7, 0xc3, 0xe0, 0x85, 0xf4, 0x93, 0x8a, 0xd3, 0x3d, 0x1d, 0xda, 0xbf, 0x0e, 0x8d, 0x34, 0x1f, 0x7c, 0xaa, 0x7b, 0x31, 0x72, 0xc9, 0xa7, 0x43, 0xf5, 0x87, 0x86, 0x24, 0xab, 0xae, 0x9a, 0x0f, 0xbe, 0x0a, 0x59, 0x11, 0xeb, 0xee, 0x44, 0xcb, 0xa7, 0x19, 0x79, 0xcb, 0x7c, 0xba, 0xb7, 0x94, 0x28, 0x14, 0x50, 0xec, 0x3f, 0xe9, 0xea, 0xbf, 0xce, 0xd5, 0xcb, 0x99, 0xc9, 0xb8, 0x73, 0x52, 0x66, 0x24, 0x3c, 0x47, 0xcc, 0x68, 0x23, 0xb1, 0x55, 0xd4, 0x20, 0x75, 0x3a, 0xa6, 0xfb, 0x0d, 0x19, 0x60, 0x12, 0x71, 0xec, 0x74, 0x38, 0xd8, 0x30, 0x97, 0x1d, 0xc2, 0x4e, 0x85, 0xc5, 0xad, 0x16, 0x14, 0xa3, 0xbb, 0xbf, 0xf2, 0x52, 0xb9, 0x04, 0x85, 0xb5, 0xf5, 0xcd, 0x8d, 0xd6, 0x12, 0xb9, 0xda, 0xce, 0x40, 0x61, 0x69, 0xdd, 0xb2, 0x9e, 0x6d, 0x74, 0xc8, 0xdd, 0x36, 0xfe, 0x70, 0x69, 0xe1, 0x67, 0x79, 0xc8, 0x3d, 0x79, 0x8e, 0x3e, 0x85, 0x09, 0xf6, 0x70, 0xee, 0x88, 0xf7, 0x93, 0x8d, 0xa3, 0xde, 0x06, 0x9a, 0x17, 0x7e, 0xf0, 0xdf, 0x3f, 0xfb, 0xc3, 0xdc, 0x19, 0xb3, 0xdc, 0x1c, 0x2d, 0x36, 0xf7, 0x46, 0x4d, 0x1a, 0x64, 0x1f, 0x18, 0xb7, 0xd0, 0xc7, 0x90, 0xdf, 0xd8, 0x0f, 0x51, 0xe6, 0xbb, 0xca, 0x46, 0xf6, 0x73, 0x41, 0xf3, 0x1c, 0x25, 0x3a, 0x6d, 0x02, 0x27, 0x3a, 0xdc, 0x0f, 0x09, 0xc9, 0xef, 0x41, 0x49, 0x7d, 0xec, 0x77, 0xec, 0x63, 0xcb, 0xc6, 0xf1, 0x0f, 0x09, 0xcd, 0x2b, 0x94, 0xd5, 0x05, 0x13, 0x71, 0x56, 0xec, 0x39, 0xa2, 0x3a, 0x8b, 0xce, 0x81, 0x8b, 0x32, 0x9f, 0x62, 0x36, 0xb2, 0xdf, 0x16, 0x26, 0x66, 0x11, 0x1e, 0xb8, 0x84, 0xe4, 0x77, 0xf9, 0x23, 0xc2, 0x6e, 0x88, 0xae, 0xa6, 0xbc, 0x02, 0x53, 0x5f, 0x37, 0x35, 0xe6, 0xb2, 0x01, 0x38, 0x93, 0xcb, 0x94, 0xc9, 0x79, 0xf3, 0x0c, 0x67, 0xd2, 0x8d, 0x40, 0x1e, 0x18, 0xb7, 0x16, 0xba, 0x30, 0x41, 0xab, 0xe7, 0xe8, 0x85, 0xf8, 0x68, 0xa4, 0xbc, 0x4b, 0xc8, 0x30, 0xb4, 0x56, 0x77, 0x37, 0x67, 0x28, 0xa3, 0xaa, 0x59, 0x24, 0x8c, 0x68, 0xed, 0xfc, 0x81, 0x71, 0xeb, 0xa6, 0x71, 0xc7, 0x58, 0xf8, 0x9b, 0x09, 0x98, 0xa0, 0x55, 0x1a, 0xb4, 0x07, 0x20, 0xab, 0xc4, 0xf1, 0xd9, 0x25, 0x0a, 0xd0, 0xf1, 0xd9, 0x25, 0x0b, 0xcc, 0x66, 0x83, 0x32, 0x9d, 0x31, 0xa7, 0x09, 0x53, 0x5a, 0xfc, 0x69, 0xd2, 0x5a, 0x17, 0xd1, 0xe3, 0x8f, 0x0d, 0x5e, 0xae, 0x62, 0xdb, 0x0c, 0xa5, 0x51, 0xd3, 0x2a, 0xc4, 0xf1, 0xe5, 0x90, 0x52, 0x14, 0x36, 0xef, 0x53, 0x86, 0x4d, 0xb3, 0x26, 0x19, 0xfa, 0x14, 0xe2, 0x81, 0x71, 0xeb, 0x45, 0xdd, 0x3c, 0xcb, 0xb5, 0x1c, 0x1b, 0x41, 0xdf, 0x87, 0xaa, 0x5e, 0xcb, 0x44, 0xd7, 0x52, 0x78, 0xc5, 0x6b, 0xa3, 0x8d, 0xeb, 0x47, 0x03, 0x71, 0x99, 0x66, 0xa9, 0x4c, 0x9c, 0x39, 0xe3, 0xbc, 0x87, 0xf1, 0xd0, 0x26, 0x40, 0xdc, 0x06, 0xe8, 0x4f, 0x0d, 0x5e, 0x8e, 0x96, 0xa5, 0x48, 0x94, 0x46, 0x3d, 0x51, 0xf1, 0x6c, 0xdc, 0x38, 0x06, 0x8a, 0x0b, 0xf1, 0x01, 0x15, 0xe2, 0x7d, 0x73, 0x46, 0x0a, 0x11, 0x3a, 0x03, 0x1c, 0x7a, 0x5c, 0x8a, 0x17, 0x97, 0xcd, 0x0b, 0x9a, 0x72, 0xb4, 0x51, 0x69, 0x2c, 0x56, 0x32, 0x4c, 0x35, 0x96, 0x56, 0x95, 0x4c, 0x35, 0x96, 0x5e, 0x6f, 0x4c, 0x33, 0x16, 0x2f, 0x10, 0xa6, 0x18, 0x2b, 0x1a, 0x59, 0xf8, 0xff, 0x71, 0x28, 0x2c, 0xb1, 0xff, 0x8c, 0x84, 0x3c, 0x28, 0x46, 0x45, 0x34, 0x34, 0x9b, 0x96, 0xa7, 0x97, 0x57, 0xb9, 0xc6, 0xd5, 0xcc, 0x71, 0x2e, 0xd0, 0x1b, 0x54, 0xa0, 0x4b, 0xe6, 0x79, 0xc2, 0x99, 0xff, 0x7f, 0xa7, 0x26, 0xcb, 0xe6, 0x36, 0xed, 0x5e, 0x8f, 0x28, 0xe2, 0x37, 0xa1, 0xac, 0x96, 0xb4, 0xd0, 0x1b, 0xa9, 0xb5, 0x01, 0xb5, 0x3e, 0xd6, 0x30, 0x8f, 0x02, 0xe1, 0x9c, 0xaf, 0x53, 0xce, 0xb3, 0xe6, 0xc5, 0x14, 0xce, 0x3e, 0x05, 0xd5, 0x98, 0xb3, 0xda, 0x53, 0x3a, 0x73, 0xad, 0xc8, 0x95, 0xce, 0x5c, 0x2f, 0x5d, 0x1d, 0xc9, 0x7c, 0x9f, 0x82, 0x12, 0xe6, 0x01, 0x80, 0x2c, 0x0e, 0xa1, 0x54, 0x5d, 0x2a, 0x17, 0xd6, 0xb8, 0x73, 0x48, 0xd6, 0x95, 0x4c, 0x93, 0xb2, 0xe5, 0xeb, 0x2e, 0xc6, 0xb6, 0xef, 0x04, 0x21, 0xdb, 0x98, 0x15, 0xad, 0xb4, 0x83, 0x52, 0xe7, 0xa3, 0x57, 0x8a, 0x1a, 0xd7, 0x8e, 0x84, 0xe1, 0xdc, 0x6f, 0x50, 0xee, 0x57, 0xcd, 0x46, 0x0a, 0xf7, 0x21, 0x83, 0x25, 0x8b, 0xed, 0xf3, 0x02, 0x94, 0x9e, 0xda, 0x8e, 0x1b, 0x62, 0xd7, 0x76, 0xbb, 0x18, 0x6d, 0xc3, 0x04, 0x8d, 0xdd, 0x71, 0x47, 0xac, 0x56, 0x32, 0xe2, 0x8e, 0x58, 0x4b, 0xe5, 0x9b, 0x73, 0x94, 0x71, 0xc3, 0x3c, 0x47, 0x18, 0x0f, 0x24, 0xe9, 0x26, 0x2b, 0x02, 0x18, 0xb7, 0xd0, 0x4b, 0x98, 0xe4, 0x25, 0xfc, 0x18, 0x21, 0x2d, 0xa9, 0xd6, 0xb8, 0x9c, 0x3e, 0x98, 0xb6, 0x96, 0x55, 0x36, 0x01, 0x85, 0x23, 0x7c, 0x46, 0x00, 0xb2, 0x22, 0x15, 0xb7, 0x68, 0xa2, 0x92, 0xd5, 0x98, 0xcb, 0x06, 0x48, 0xd3, 0xa9, 0xca, 0xb3, 0x17, 0xc1, 0x12, 0xbe, 0xdf, 0x81, 0xf1, 0xc7, 0x76, 0xb0, 0x8b, 0x62, 0xb1, 0x57, 0x79, 0x71, 0xdb, 0x68, 0xa4, 0x0d, 0x71, 0x2e, 0x57, 0x29, 0x97, 0x8b, 0xcc, 0x95, 0xa9, 0x5c, 0xe8, 0x9b, 0x52, 0xa6, 0x3f, 0xf6, 0xdc, 0x36, 0xae, 0x3f, 0xed, 0xed, 0x6e, 0x5c, 0x7f, 0xfa, 0x0b, 0xdd, 0x6c, 0xfd, 0x11, 0x2e, 0x7b, 0x23, 0xc2, 0x67, 0x08, 0x53, 0xe2, 0x61, 0x2a, 0x8a, 0x3d, 0xe7, 0x89, 0xbd, 0x66, 0x6d, 0xcc, 0x66, 0x0d, 0x73, 0x6e, 0xd7, 0x28, 0xb7, 0x2b, 0x66, 0x3d, 0x61, 0x2d, 0x0e, 0xf9, 0xc0, 0xb8, 0x75, 0xc7, 0x40, 0xdf, 0x07, 0x90, 0x45, 0xbb, 0xc4, 0x1e, 0x8c, 0x17, 0x02, 0x13, 0x7b, 0x30, 0x51, 0xef, 0x33, 0xe7, 0x29, 0xdf, 0x9b, 0xe6, 0xb5, 0x38, 0xdf, 0xd0, 0xb7, 0xdd, 0xe0, 0x25, 0xf6, 0x6f, 0xb3, 0xbc, 0x7f, 0xb0, 0xeb, 0x0c, 0xc9, 0x94, 0x7d, 0x28, 0x46, 0xb9, 0xe6, 0xb8, 0xbf, 0x8d, 0x57, 0x7f, 0xe2, 0xfe, 0x36, 0x51, 0x8c, 0xd1, 0x1d, 0x8f, 0xb6, 0x5e, 0x04, 0x28, 0xd9, 0x82, 0x7f, 0x59, 0x83, 0x71, 0x72, 0x24, 0x27, 0xc7, 0x13, 0x99, 0xee, 0x89, 0xcf, 0x3e, 0x91, 0xb1, 0x8e, 0xcf, 0x3e, 0x99, 0x29, 0xd2, 0x8f, 0x27, 0xe4, 0xba, 0xd6, 0x64, 0x79, 0x14, 0x32, 0x53, 0x0f, 0x4a, 0x4a, 0x1a, 0x08, 0xa5, 0x10, 0xd3, 0x33, 0xe0, 0xf1, 0x80, 0x97, 0x92, 0x43, 0x32, 0x2f, 0x51, 0x7e, 0xe7, 0x58, 0xc0, 0xa3, 0xfc, 0x7a, 0x0c, 0x82, 0x30, 0xe4, 0xb3, 0xe3, 0x3b, 0x3f, 0x65, 0x76, 0xfa, 0xee, 0x9f, 0xcb, 0x06, 0xc8, 0x9c, 0x9d, 0xdc, 0xfa, 0xaf, 0xa0, 0xac, 0xa6, 0x7e, 0x50, 0x8a, 0xf0, 0xb1, 0x1c, 0x7d, 0x3c, 0x92, 0xa4, 0x65, 0x8e, 0x74, 0xdf, 0x46, 0x59, 0xda, 0x0a, 0x18, 0x61, 0xdc, 0x87, 0x02, 0x4f, 0x01, 0xa5, 0xa9, 0x54, 0x4f, 0xe3, 0xa7, 0xa9, 0x34, 0x96, 0x3f, 0xd2, 0xcf, 0xcf, 0x94, 0x23, 0xb9, 0x8a, 0x8a, 0x68, 0xcd, 0xb9, 0x3d, 0xc2, 0x61, 0x16, 0x37, 0x99, 0xb6, 0xcd, 0xe2, 0xa6, 0x64, 0x08, 0xb2, 0xb8, 0xed, 0xe0, 0x90, 0xfb, 0x03, 0x71, 0xbd, 0x46, 0x19, 0xc4, 0xd4, 0x08, 0x69, 0x1e, 0x05, 0x92, 0x76, 0xbd, 0x91, 0x0c, 0x45, 0x78, 0x3c, 0x00, 0x90, 0xe9, 0xa8, 0xf8, 0x99, 0x35, 0xb5, 0x52, 0x10, 0x3f, 0xb3, 0xa6, 0x67, 0xb4, 0x74, 0x1f, 0x2b, 0xf9, 0xb2, 0xdb, 0x15, 0xe1, 0xfc, 0x85, 0x01, 0x28, 0x99, 0xb0, 0x42, 0xef, 0xa4, 0x53, 0x4f, 0xad, 0x3a, 0x34, 0xde, 0x7d, 0x3d, 0xe0, 0x34, 0x87, 0x2c, 0x45, 0xea, 0x52, 0xe8, 0xe1, 0x2b, 0x22, 0xd4, 0xe7, 0x06, 0x54, 0xb4, 0x24, 0x17, 0x7a, 0x33, 0xc3, 0xa6, 0xb1, 0xd2, 0x43, 0xe3, 0xad, 0x63, 0xe1, 0xd2, 0x0e, 0xf3, 0xca, 0x0a, 0x10, 0xb7, 0x9a, 0xdf, 0x31, 0xa0, 0xaa, 0xe7, 0xc2, 0x50, 0x06, 0xed, 0x44, 0xc5, 0xa2, 0x71, 0xf3, 0x78, 0xc0, 0xa3, 0xcd, 0x23, 0x2f, 0x34, 0x7d, 0x28, 0xf0, 0xa4, 0x59, 0xda, 0xc2, 0xd7, 0x4b, 0x1c, 0x69, 0x0b, 0x3f, 0x96, 0x71, 0x4b, 0x59, 0xf8, 0xbe, 0xd7, 0xc7, 0xca, 0x36, 0xe3, 0xb9, 0xb4, 0x2c, 0x6e, 0x47, 0x6f, 0xb3, 0x58, 0x22, 0x2e, 0x8b, 0x9b, 0xdc, 0x66, 0x22, 0x65, 0x86, 0x32, 0x88, 0x1d, 0xb3, 0xcd, 0xe2, 0x19, 0xb7, 0x94, 0x6d, 0x46, 0x19, 0x2a, 0xdb, 0x4c, 0xa6, 0xb2, 0xd2, 0xb6, 0x59, 0xa2, 0x1a, 0x93, 0xb6, 0xcd, 0x92, 0xd9, 0xb0, 0x14, 0x3b, 0x52, 0xbe, 0xda, 0x36, 0x3b, 0x9b, 0x92, 0xec, 0x42, 0xef, 0x66, 0x28, 0x31, 0xb5, 0xb6, 0xd3, 0xb8, 0xfd, 0x9a, 0xd0, 0x99, 0x6b, 0x9c, 0xa9, 0x5f, 0xac, 0xf1, 0x3f, 0x32, 0x60, 0x26, 0x2d, 0x3f, 0x86, 0x32, 0xf8, 0x64, 0x94, 0x82, 0x1a, 0xf3, 0xaf, 0x0b, 0x7e, 0xb4, 0xb6, 0xa2, 0x55, 0xff, 0x70, 0xe7, 0x8b, 0x56, 0xf3, 0xc5, 0x55, 0xb8, 0x02, 0x93, 0xad, 0xa1, 0xf3, 0x04, 0x1f, 0xa2, 0xb3, 0x53, 0xb9, 0x46, 0x85, 0xd0, 0xf5, 0x7c, 0xe7, 0x33, 0xfa, 0xab, 0x17, 0x73, 0xb9, 0xed, 0x32, 0x40, 0x04, 0x30, 0xf6, 0xef, 0x5f, 0xce, 0x1a, 0xff, 0xf5, 0xe5, 0xac, 0xf1, 0x3f, 0x5f, 0xce, 0x1a, 0x3f, 0xfd, 0xbf, 0xd9, 0xb1, 0x17, 0xd7, 0x76, 0x3c, 0x2a, 0xd6, 0xbc, 0xe3, 0x35, 0xe5, 0x2f, 0x71, 0x2c, 0x36, 0x55, 0x51, 0xb7, 0x27, 0xe9, 0x4f, 0x67, 0x2c, 0xfe, 0x3c, 0x00, 0x00, 0xff, 0xff, 0x08, 0x5e, 0xc8, 0xca, 0xfb, 0x43, 0x00, 0x00, } func (m *ResponseHeader) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *ResponseHeader) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ResponseHeader) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.RaftTerm != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.RaftTerm)) i-- dAtA[i] = 0x20 } if m.Revision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Revision)) i-- dAtA[i] = 0x18 } if m.MemberId != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.MemberId)) i-- dAtA[i] = 0x10 } if m.ClusterId != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ClusterId)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *RangeRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *RangeRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *RangeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.MaxCreateRevision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.MaxCreateRevision)) i-- dAtA[i] = 0x68 } if m.MinCreateRevision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.MinCreateRevision)) i-- dAtA[i] = 0x60 } if m.MaxModRevision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.MaxModRevision)) i-- dAtA[i] = 0x58 } if m.MinModRevision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.MinModRevision)) i-- dAtA[i] = 0x50 } if m.CountOnly { i-- if m.CountOnly { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x48 } if m.KeysOnly { i-- if m.KeysOnly { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x40 } if m.Serializable { i-- if m.Serializable { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x38 } if m.SortTarget != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.SortTarget)) i-- dAtA[i] = 0x30 } if m.SortOrder != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.SortOrder)) i-- dAtA[i] = 0x28 } if m.Revision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Revision)) i-- dAtA[i] = 0x20 } if m.Limit != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Limit)) i-- dAtA[i] = 0x18 } if len(m.RangeEnd) > 0 { i -= len(m.RangeEnd) copy(dAtA[i:], m.RangeEnd) i = encodeVarintRpc(dAtA, i, uint64(len(m.RangeEnd))) i-- dAtA[i] = 0x12 } if len(m.Key) > 0 { i -= len(m.Key) copy(dAtA[i:], m.Key) i = encodeVarintRpc(dAtA, i, uint64(len(m.Key))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *RangeResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *RangeResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *RangeResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Count != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Count)) i-- dAtA[i] = 0x20 } if m.More { i-- if m.More { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x18 } if len(m.Kvs) > 0 { for iNdEx := len(m.Kvs) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Kvs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *PutRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *PutRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *PutRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.IgnoreLease { i-- if m.IgnoreLease { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x30 } if m.IgnoreValue { i-- if m.IgnoreValue { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x28 } if m.PrevKv { i-- if m.PrevKv { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x20 } if m.Lease != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Lease)) i-- dAtA[i] = 0x18 } if len(m.Value) > 0 { i -= len(m.Value) copy(dAtA[i:], m.Value) i = encodeVarintRpc(dAtA, i, uint64(len(m.Value))) i-- dAtA[i] = 0x12 } if len(m.Key) > 0 { i -= len(m.Key) copy(dAtA[i:], m.Key) i = encodeVarintRpc(dAtA, i, uint64(len(m.Key))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *PutResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *PutResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *PutResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.PrevKv != nil { { size, err := m.PrevKv.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *DeleteRangeRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *DeleteRangeRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *DeleteRangeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.PrevKv { i-- if m.PrevKv { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x18 } if len(m.RangeEnd) > 0 { i -= len(m.RangeEnd) copy(dAtA[i:], m.RangeEnd) i = encodeVarintRpc(dAtA, i, uint64(len(m.RangeEnd))) i-- dAtA[i] = 0x12 } if len(m.Key) > 0 { i -= len(m.Key) copy(dAtA[i:], m.Key) i = encodeVarintRpc(dAtA, i, uint64(len(m.Key))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *DeleteRangeResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *DeleteRangeResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *DeleteRangeResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.PrevKvs) > 0 { for iNdEx := len(m.PrevKvs) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.PrevKvs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } } if m.Deleted != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Deleted)) i-- dAtA[i] = 0x10 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *RequestOp) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *RequestOp) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *RequestOp) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Request != nil { { size := m.Request.Size() i -= size if _, err := m.Request.MarshalTo(dAtA[i:]); err != nil { return 0, err } } } return len(dAtA) - i, nil } func (m *RequestOp_RequestRange) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *RequestOp_RequestRange) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.RequestRange != nil { { size, err := m.RequestRange.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *RequestOp_RequestPut) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *RequestOp_RequestPut) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.RequestPut != nil { { size, err := m.RequestPut.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } return len(dAtA) - i, nil } func (m *RequestOp_RequestDeleteRange) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *RequestOp_RequestDeleteRange) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.RequestDeleteRange != nil { { size, err := m.RequestDeleteRange.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } return len(dAtA) - i, nil } func (m *RequestOp_RequestTxn) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *RequestOp_RequestTxn) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.RequestTxn != nil { { size, err := m.RequestTxn.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x22 } return len(dAtA) - i, nil } func (m *ResponseOp) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *ResponseOp) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ResponseOp) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Response != nil { { size := m.Response.Size() i -= size if _, err := m.Response.MarshalTo(dAtA[i:]); err != nil { return 0, err } } } return len(dAtA) - i, nil } func (m *ResponseOp_ResponseRange) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ResponseOp_ResponseRange) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.ResponseRange != nil { { size, err := m.ResponseRange.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *ResponseOp_ResponsePut) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ResponseOp_ResponsePut) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.ResponsePut != nil { { size, err := m.ResponsePut.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } return len(dAtA) - i, nil } func (m *ResponseOp_ResponseDeleteRange) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ResponseOp_ResponseDeleteRange) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.ResponseDeleteRange != nil { { size, err := m.ResponseDeleteRange.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } return len(dAtA) - i, nil } func (m *ResponseOp_ResponseTxn) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ResponseOp_ResponseTxn) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.ResponseTxn != nil { { size, err := m.ResponseTxn.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x22 } return len(dAtA) - i, nil } func (m *Compare) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Compare) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Compare) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.RangeEnd) > 0 { i -= len(m.RangeEnd) copy(dAtA[i:], m.RangeEnd) i = encodeVarintRpc(dAtA, i, uint64(len(m.RangeEnd))) i-- dAtA[i] = 0x4 i-- dAtA[i] = 0x82 } if m.TargetUnion != nil { { size := m.TargetUnion.Size() i -= size if _, err := m.TargetUnion.MarshalTo(dAtA[i:]); err != nil { return 0, err } } } if len(m.Key) > 0 { i -= len(m.Key) copy(dAtA[i:], m.Key) i = encodeVarintRpc(dAtA, i, uint64(len(m.Key))) i-- dAtA[i] = 0x1a } if m.Target != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Target)) i-- dAtA[i] = 0x10 } if m.Result != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Result)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *Compare_Version) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Compare_Version) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) i = encodeVarintRpc(dAtA, i, uint64(m.Version)) i-- dAtA[i] = 0x20 return len(dAtA) - i, nil } func (m *Compare_CreateRevision) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Compare_CreateRevision) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) i = encodeVarintRpc(dAtA, i, uint64(m.CreateRevision)) i-- dAtA[i] = 0x28 return len(dAtA) - i, nil } func (m *Compare_ModRevision) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Compare_ModRevision) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) i = encodeVarintRpc(dAtA, i, uint64(m.ModRevision)) i-- dAtA[i] = 0x30 return len(dAtA) - i, nil } func (m *Compare_Value) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Compare_Value) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.Value != nil { i -= len(m.Value) copy(dAtA[i:], m.Value) i = encodeVarintRpc(dAtA, i, uint64(len(m.Value))) i-- dAtA[i] = 0x3a } return len(dAtA) - i, nil } func (m *Compare_Lease) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Compare_Lease) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) i = encodeVarintRpc(dAtA, i, uint64(m.Lease)) i-- dAtA[i] = 0x40 return len(dAtA) - i, nil } func (m *TxnRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *TxnRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *TxnRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Failure) > 0 { for iNdEx := len(m.Failure) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Failure[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } } if len(m.Success) > 0 { for iNdEx := len(m.Success) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Success[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } } if len(m.Compare) > 0 { for iNdEx := len(m.Compare) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Compare[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } } return len(dAtA) - i, nil } func (m *TxnResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *TxnResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *TxnResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Responses) > 0 { for iNdEx := len(m.Responses) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Responses[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } } if m.Succeeded { i-- if m.Succeeded { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x10 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *CompactionRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *CompactionRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *CompactionRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Physical { i-- if m.Physical { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x10 } if m.Revision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Revision)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *CompactionResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *CompactionResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *CompactionResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *HashRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *HashRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *HashRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *HashKVRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *HashKVRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *HashKVRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Revision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Revision)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *HashKVResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *HashKVResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *HashKVResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.HashRevision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.HashRevision)) i-- dAtA[i] = 0x20 } if m.CompactRevision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.CompactRevision)) i-- dAtA[i] = 0x18 } if m.Hash != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Hash)) i-- dAtA[i] = 0x10 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *HashResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *HashResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *HashResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Hash != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Hash)) i-- dAtA[i] = 0x10 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *SnapshotRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *SnapshotRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *SnapshotRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *SnapshotResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *SnapshotResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *SnapshotResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Version) > 0 { i -= len(m.Version) copy(dAtA[i:], m.Version) i = encodeVarintRpc(dAtA, i, uint64(len(m.Version))) i-- dAtA[i] = 0x22 } if len(m.Blob) > 0 { i -= len(m.Blob) copy(dAtA[i:], m.Blob) i = encodeVarintRpc(dAtA, i, uint64(len(m.Blob))) i-- dAtA[i] = 0x1a } if m.RemainingBytes != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.RemainingBytes)) i-- dAtA[i] = 0x10 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *WatchRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *WatchRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *WatchRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.RequestUnion != nil { { size := m.RequestUnion.Size() i -= size if _, err := m.RequestUnion.MarshalTo(dAtA[i:]); err != nil { return 0, err } } } return len(dAtA) - i, nil } func (m *WatchRequest_CreateRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *WatchRequest_CreateRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.CreateRequest != nil { { size, err := m.CreateRequest.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *WatchRequest_CancelRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *WatchRequest_CancelRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.CancelRequest != nil { { size, err := m.CancelRequest.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } return len(dAtA) - i, nil } func (m *WatchRequest_ProgressRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *WatchRequest_ProgressRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) if m.ProgressRequest != nil { { size, err := m.ProgressRequest.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } return len(dAtA) - i, nil } func (m *WatchCreateRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *WatchCreateRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *WatchCreateRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Fragment { i-- if m.Fragment { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x40 } if m.WatchId != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.WatchId)) i-- dAtA[i] = 0x38 } if m.PrevKv { i-- if m.PrevKv { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x30 } if len(m.Filters) > 0 { dAtA22 := make([]byte, len(m.Filters)*10) var j21 int for _, num := range m.Filters { for num >= 1<<7 { dAtA22[j21] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 j21++ } dAtA22[j21] = uint8(num) j21++ } i -= j21 copy(dAtA[i:], dAtA22[:j21]) i = encodeVarintRpc(dAtA, i, uint64(j21)) i-- dAtA[i] = 0x2a } if m.ProgressNotify { i-- if m.ProgressNotify { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x20 } if m.StartRevision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.StartRevision)) i-- dAtA[i] = 0x18 } if len(m.RangeEnd) > 0 { i -= len(m.RangeEnd) copy(dAtA[i:], m.RangeEnd) i = encodeVarintRpc(dAtA, i, uint64(len(m.RangeEnd))) i-- dAtA[i] = 0x12 } if len(m.Key) > 0 { i -= len(m.Key) copy(dAtA[i:], m.Key) i = encodeVarintRpc(dAtA, i, uint64(len(m.Key))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *WatchCancelRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *WatchCancelRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *WatchCancelRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.WatchId != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.WatchId)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *WatchProgressRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *WatchProgressRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *WatchProgressRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *WatchResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *WatchResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *WatchResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Events) > 0 { for iNdEx := len(m.Events) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Events[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x5a } } if m.Fragment { i-- if m.Fragment { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x38 } if len(m.CancelReason) > 0 { i -= len(m.CancelReason) copy(dAtA[i:], m.CancelReason) i = encodeVarintRpc(dAtA, i, uint64(len(m.CancelReason))) i-- dAtA[i] = 0x32 } if m.CompactRevision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.CompactRevision)) i-- dAtA[i] = 0x28 } if m.Canceled { i-- if m.Canceled { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x20 } if m.Created { i-- if m.Created { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x18 } if m.WatchId != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.WatchId)) i-- dAtA[i] = 0x10 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *LeaseGrantRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseGrantRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseGrantRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x10 } if m.TTL != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.TTL)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *LeaseGrantResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseGrantResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseGrantResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Error) > 0 { i -= len(m.Error) copy(dAtA[i:], m.Error) i = encodeVarintRpc(dAtA, i, uint64(len(m.Error))) i-- dAtA[i] = 0x22 } if m.TTL != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.TTL)) i-- dAtA[i] = 0x18 } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x10 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *LeaseRevokeRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseRevokeRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseRevokeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *LeaseRevokeResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseRevokeResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseRevokeResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *LeaseCheckpoint) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseCheckpoint) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseCheckpoint) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Remaining_TTL != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Remaining_TTL)) i-- dAtA[i] = 0x10 } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *LeaseCheckpointRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseCheckpointRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseCheckpointRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Checkpoints) > 0 { for iNdEx := len(m.Checkpoints) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Checkpoints[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } } return len(dAtA) - i, nil } func (m *LeaseCheckpointResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseCheckpointResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseCheckpointResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *LeaseKeepAliveRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseKeepAliveRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseKeepAliveRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *LeaseKeepAliveResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseKeepAliveResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseKeepAliveResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.TTL != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.TTL)) i-- dAtA[i] = 0x18 } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x10 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *LeaseTimeToLiveRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseTimeToLiveRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseTimeToLiveRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Keys { i-- if m.Keys { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x10 } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *LeaseTimeToLiveResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseTimeToLiveResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseTimeToLiveResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Keys) > 0 { for iNdEx := len(m.Keys) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Keys[iNdEx]) copy(dAtA[i:], m.Keys[iNdEx]) i = encodeVarintRpc(dAtA, i, uint64(len(m.Keys[iNdEx]))) i-- dAtA[i] = 0x2a } } if m.GrantedTTL != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.GrantedTTL)) i-- dAtA[i] = 0x20 } if m.TTL != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.TTL)) i-- dAtA[i] = 0x18 } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x10 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *LeaseLeasesRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseLeasesRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseLeasesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *LeaseStatus) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseStatus) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *LeaseLeasesResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseLeasesResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseLeasesResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Leases) > 0 { for iNdEx := len(m.Leases) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Leases[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *Member) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Member) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Member) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.IsLearner { i-- if m.IsLearner { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x28 } if len(m.ClientURLs) > 0 { for iNdEx := len(m.ClientURLs) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.ClientURLs[iNdEx]) copy(dAtA[i:], m.ClientURLs[iNdEx]) i = encodeVarintRpc(dAtA, i, uint64(len(m.ClientURLs[iNdEx]))) i-- dAtA[i] = 0x22 } } if len(m.PeerURLs) > 0 { for iNdEx := len(m.PeerURLs) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.PeerURLs[iNdEx]) copy(dAtA[i:], m.PeerURLs[iNdEx]) i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerURLs[iNdEx]))) i-- dAtA[i] = 0x1a } } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0x12 } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *MemberAddRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MemberAddRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MemberAddRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.IsLearner { i-- if m.IsLearner { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x10 } if len(m.PeerURLs) > 0 { for iNdEx := len(m.PeerURLs) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.PeerURLs[iNdEx]) copy(dAtA[i:], m.PeerURLs[iNdEx]) i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerURLs[iNdEx]))) i-- dAtA[i] = 0xa } } return len(dAtA) - i, nil } func (m *MemberAddResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MemberAddResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MemberAddResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Members) > 0 { for iNdEx := len(m.Members) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Members[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } } if m.Member != nil { { size, err := m.Member.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *MemberRemoveRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MemberRemoveRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MemberRemoveRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *MemberRemoveResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MemberRemoveResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MemberRemoveResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Members) > 0 { for iNdEx := len(m.Members) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Members[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *MemberUpdateRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MemberUpdateRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MemberUpdateRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.PeerURLs) > 0 { for iNdEx := len(m.PeerURLs) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.PeerURLs[iNdEx]) copy(dAtA[i:], m.PeerURLs[iNdEx]) i = encodeVarintRpc(dAtA, i, uint64(len(m.PeerURLs[iNdEx]))) i-- dAtA[i] = 0x12 } } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *MemberUpdateResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MemberUpdateResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MemberUpdateResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Members) > 0 { for iNdEx := len(m.Members) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Members[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *MemberListRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MemberListRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MemberListRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Linearizable { i-- if m.Linearizable { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *MemberListResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MemberListResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MemberListResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Members) > 0 { for iNdEx := len(m.Members) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Members[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *MemberPromoteRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MemberPromoteRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MemberPromoteRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.ID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *MemberPromoteResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MemberPromoteResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MemberPromoteResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Members) > 0 { for iNdEx := len(m.Members) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Members[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *DefragmentRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *DefragmentRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *DefragmentRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *DefragmentResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *DefragmentResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *DefragmentResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *MoveLeaderRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MoveLeaderRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MoveLeaderRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.TargetID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.TargetID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *MoveLeaderResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *MoveLeaderResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *MoveLeaderResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AlarmRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AlarmRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AlarmRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Alarm != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Alarm)) i-- dAtA[i] = 0x18 } if m.MemberID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.MemberID)) i-- dAtA[i] = 0x10 } if m.Action != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Action)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *AlarmMember) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AlarmMember) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AlarmMember) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Alarm != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Alarm)) i-- dAtA[i] = 0x10 } if m.MemberID != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.MemberID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *AlarmResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AlarmResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AlarmResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Alarms) > 0 { for iNdEx := len(m.Alarms) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Alarms[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *DowngradeRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *DowngradeRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *DowngradeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Version) > 0 { i -= len(m.Version) copy(dAtA[i:], m.Version) i = encodeVarintRpc(dAtA, i, uint64(len(m.Version))) i-- dAtA[i] = 0x12 } if m.Action != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Action)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *DowngradeResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *DowngradeResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *DowngradeResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Version) > 0 { i -= len(m.Version) copy(dAtA[i:], m.Version) i = encodeVarintRpc(dAtA, i, uint64(len(m.Version))) i-- dAtA[i] = 0x12 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *DowngradeVersionTestRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *DowngradeVersionTestRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *DowngradeVersionTestRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Ver) > 0 { i -= len(m.Ver) copy(dAtA[i:], m.Ver) i = encodeVarintRpc(dAtA, i, uint64(len(m.Ver))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *StatusRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *StatusRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *StatusRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *StatusResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *StatusResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *StatusResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.DowngradeInfo != nil { { size, err := m.DowngradeInfo.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x6a } if m.DbSizeQuota != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.DbSizeQuota)) i-- dAtA[i] = 0x60 } if len(m.StorageVersion) > 0 { i -= len(m.StorageVersion) copy(dAtA[i:], m.StorageVersion) i = encodeVarintRpc(dAtA, i, uint64(len(m.StorageVersion))) i-- dAtA[i] = 0x5a } if m.IsLearner { i-- if m.IsLearner { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x50 } if m.DbSizeInUse != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.DbSizeInUse)) i-- dAtA[i] = 0x48 } if len(m.Errors) > 0 { for iNdEx := len(m.Errors) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Errors[iNdEx]) copy(dAtA[i:], m.Errors[iNdEx]) i = encodeVarintRpc(dAtA, i, uint64(len(m.Errors[iNdEx]))) i-- dAtA[i] = 0x42 } } if m.RaftAppliedIndex != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.RaftAppliedIndex)) i-- dAtA[i] = 0x38 } if m.RaftTerm != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.RaftTerm)) i-- dAtA[i] = 0x30 } if m.RaftIndex != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.RaftIndex)) i-- dAtA[i] = 0x28 } if m.Leader != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.Leader)) i-- dAtA[i] = 0x20 } if m.DbSize != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.DbSize)) i-- dAtA[i] = 0x18 } if len(m.Version) > 0 { i -= len(m.Version) copy(dAtA[i:], m.Version) i = encodeVarintRpc(dAtA, i, uint64(len(m.Version))) i-- dAtA[i] = 0x12 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *DowngradeInfo) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *DowngradeInfo) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *DowngradeInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.TargetVersion) > 0 { i -= len(m.TargetVersion) copy(dAtA[i:], m.TargetVersion) i = encodeVarintRpc(dAtA, i, uint64(len(m.TargetVersion))) i-- dAtA[i] = 0x12 } if m.Enabled { i-- if m.Enabled { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *AuthEnableRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthEnableRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthEnableRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *AuthDisableRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthDisableRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthDisableRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *AuthStatusRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthStatusRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthStatusRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *AuthenticateRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthenticateRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthenticateRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Password) > 0 { i -= len(m.Password) copy(dAtA[i:], m.Password) i = encodeVarintRpc(dAtA, i, uint64(len(m.Password))) i-- dAtA[i] = 0x12 } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserAddRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserAddRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserAddRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.HashedPassword) > 0 { i -= len(m.HashedPassword) copy(dAtA[i:], m.HashedPassword) i = encodeVarintRpc(dAtA, i, uint64(len(m.HashedPassword))) i-- dAtA[i] = 0x22 } if m.Options != nil { { size, err := m.Options.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } if len(m.Password) > 0 { i -= len(m.Password) copy(dAtA[i:], m.Password) i = encodeVarintRpc(dAtA, i, uint64(len(m.Password))) i-- dAtA[i] = 0x12 } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserGetRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserGetRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserGetRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserDeleteRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserDeleteRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserDeleteRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserChangePasswordRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserChangePasswordRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserChangePasswordRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.HashedPassword) > 0 { i -= len(m.HashedPassword) copy(dAtA[i:], m.HashedPassword) i = encodeVarintRpc(dAtA, i, uint64(len(m.HashedPassword))) i-- dAtA[i] = 0x1a } if len(m.Password) > 0 { i -= len(m.Password) copy(dAtA[i:], m.Password) i = encodeVarintRpc(dAtA, i, uint64(len(m.Password))) i-- dAtA[i] = 0x12 } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserGrantRoleRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserGrantRoleRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserGrantRoleRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Role) > 0 { i -= len(m.Role) copy(dAtA[i:], m.Role) i = encodeVarintRpc(dAtA, i, uint64(len(m.Role))) i-- dAtA[i] = 0x12 } if len(m.User) > 0 { i -= len(m.User) copy(dAtA[i:], m.User) i = encodeVarintRpc(dAtA, i, uint64(len(m.User))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserRevokeRoleRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserRevokeRoleRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserRevokeRoleRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Role) > 0 { i -= len(m.Role) copy(dAtA[i:], m.Role) i = encodeVarintRpc(dAtA, i, uint64(len(m.Role))) i-- dAtA[i] = 0x12 } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthRoleAddRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleAddRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleAddRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthRoleGetRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleGetRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleGetRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Role) > 0 { i -= len(m.Role) copy(dAtA[i:], m.Role) i = encodeVarintRpc(dAtA, i, uint64(len(m.Role))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserListRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserListRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserListRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *AuthRoleListRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleListRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleListRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } return len(dAtA) - i, nil } func (m *AuthRoleDeleteRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleDeleteRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleDeleteRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Role) > 0 { i -= len(m.Role) copy(dAtA[i:], m.Role) i = encodeVarintRpc(dAtA, i, uint64(len(m.Role))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthRoleGrantPermissionRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleGrantPermissionRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleGrantPermissionRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Perm != nil { { size, err := m.Perm.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintRpc(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthRoleRevokePermissionRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleRevokePermissionRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleRevokePermissionRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.RangeEnd) > 0 { i -= len(m.RangeEnd) copy(dAtA[i:], m.RangeEnd) i = encodeVarintRpc(dAtA, i, uint64(len(m.RangeEnd))) i-- dAtA[i] = 0x1a } if len(m.Key) > 0 { i -= len(m.Key) copy(dAtA[i:], m.Key) i = encodeVarintRpc(dAtA, i, uint64(len(m.Key))) i-- dAtA[i] = 0x12 } if len(m.Role) > 0 { i -= len(m.Role) copy(dAtA[i:], m.Role) i = encodeVarintRpc(dAtA, i, uint64(len(m.Role))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthEnableResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthEnableResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthEnableResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthDisableResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthDisableResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthDisableResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthStatusResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthStatusResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthStatusResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.AuthRevision != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.AuthRevision)) i-- dAtA[i] = 0x18 } if m.Enabled { i-- if m.Enabled { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x10 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthenticateResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthenticateResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthenticateResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Token) > 0 { i -= len(m.Token) copy(dAtA[i:], m.Token) i = encodeVarintRpc(dAtA, i, uint64(len(m.Token))) i-- dAtA[i] = 0x12 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserAddResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserAddResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserAddResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserGetResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserGetResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserGetResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Roles) > 0 { for iNdEx := len(m.Roles) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Roles[iNdEx]) copy(dAtA[i:], m.Roles[iNdEx]) i = encodeVarintRpc(dAtA, i, uint64(len(m.Roles[iNdEx]))) i-- dAtA[i] = 0x12 } } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserDeleteResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserDeleteResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserDeleteResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserChangePasswordResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserChangePasswordResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserChangePasswordResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserGrantRoleResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserGrantRoleResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserGrantRoleResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserRevokeRoleResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserRevokeRoleResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserRevokeRoleResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthRoleAddResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleAddResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleAddResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthRoleGetResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleGetResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleGetResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Perm) > 0 { for iNdEx := len(m.Perm) - 1; iNdEx >= 0; iNdEx-- { { size, err := m.Perm[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthRoleListResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleListResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleListResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Roles) > 0 { for iNdEx := len(m.Roles) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Roles[iNdEx]) copy(dAtA[i:], m.Roles[iNdEx]) i = encodeVarintRpc(dAtA, i, uint64(len(m.Roles[iNdEx]))) i-- dAtA[i] = 0x12 } } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthUserListResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthUserListResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthUserListResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Users) > 0 { for iNdEx := len(m.Users) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Users[iNdEx]) copy(dAtA[i:], m.Users[iNdEx]) i = encodeVarintRpc(dAtA, i, uint64(len(m.Users[iNdEx]))) i-- dAtA[i] = 0x12 } } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthRoleDeleteResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleDeleteResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleDeleteResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthRoleGrantPermissionResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleGrantPermissionResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleGrantPermissionResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *AuthRoleRevokePermissionResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *AuthRoleRevokePermissionResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *AuthRoleRevokePermissionResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRpc(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func encodeVarintRpc(dAtA []byte, offset int, v uint64) int { offset -= sovRpc(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) return base } func (m *ResponseHeader) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ClusterId != 0 { n += 1 + sovRpc(uint64(m.ClusterId)) } if m.MemberId != 0 { n += 1 + sovRpc(uint64(m.MemberId)) } if m.Revision != 0 { n += 1 + sovRpc(uint64(m.Revision)) } if m.RaftTerm != 0 { n += 1 + sovRpc(uint64(m.RaftTerm)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *RangeRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Key) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.RangeEnd) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.Limit != 0 { n += 1 + sovRpc(uint64(m.Limit)) } if m.Revision != 0 { n += 1 + sovRpc(uint64(m.Revision)) } if m.SortOrder != 0 { n += 1 + sovRpc(uint64(m.SortOrder)) } if m.SortTarget != 0 { n += 1 + sovRpc(uint64(m.SortTarget)) } if m.Serializable { n += 2 } if m.KeysOnly { n += 2 } if m.CountOnly { n += 2 } if m.MinModRevision != 0 { n += 1 + sovRpc(uint64(m.MinModRevision)) } if m.MaxModRevision != 0 { n += 1 + sovRpc(uint64(m.MaxModRevision)) } if m.MinCreateRevision != 0 { n += 1 + sovRpc(uint64(m.MinCreateRevision)) } if m.MaxCreateRevision != 0 { n += 1 + sovRpc(uint64(m.MaxCreateRevision)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *RangeResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Kvs) > 0 { for _, e := range m.Kvs { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.More { n += 2 } if m.Count != 0 { n += 1 + sovRpc(uint64(m.Count)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *PutRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Key) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.Value) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.Lease != 0 { n += 1 + sovRpc(uint64(m.Lease)) } if m.PrevKv { n += 2 } if m.IgnoreValue { n += 2 } if m.IgnoreLease { n += 2 } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *PutResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.PrevKv != nil { l = m.PrevKv.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *DeleteRangeRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Key) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.RangeEnd) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.PrevKv { n += 2 } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *DeleteRangeResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.Deleted != 0 { n += 1 + sovRpc(uint64(m.Deleted)) } if len(m.PrevKvs) > 0 { for _, e := range m.PrevKvs { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *RequestOp) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Request != nil { n += m.Request.Size() } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *RequestOp_RequestRange) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.RequestRange != nil { l = m.RequestRange.Size() n += 1 + l + sovRpc(uint64(l)) } return n } func (m *RequestOp_RequestPut) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.RequestPut != nil { l = m.RequestPut.Size() n += 1 + l + sovRpc(uint64(l)) } return n } func (m *RequestOp_RequestDeleteRange) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.RequestDeleteRange != nil { l = m.RequestDeleteRange.Size() n += 1 + l + sovRpc(uint64(l)) } return n } func (m *RequestOp_RequestTxn) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.RequestTxn != nil { l = m.RequestTxn.Size() n += 1 + l + sovRpc(uint64(l)) } return n } func (m *ResponseOp) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Response != nil { n += m.Response.Size() } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *ResponseOp_ResponseRange) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ResponseRange != nil { l = m.ResponseRange.Size() n += 1 + l + sovRpc(uint64(l)) } return n } func (m *ResponseOp_ResponsePut) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ResponsePut != nil { l = m.ResponsePut.Size() n += 1 + l + sovRpc(uint64(l)) } return n } func (m *ResponseOp_ResponseDeleteRange) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ResponseDeleteRange != nil { l = m.ResponseDeleteRange.Size() n += 1 + l + sovRpc(uint64(l)) } return n } func (m *ResponseOp_ResponseTxn) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ResponseTxn != nil { l = m.ResponseTxn.Size() n += 1 + l + sovRpc(uint64(l)) } return n } func (m *Compare) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Result != 0 { n += 1 + sovRpc(uint64(m.Result)) } if m.Target != 0 { n += 1 + sovRpc(uint64(m.Target)) } l = len(m.Key) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.TargetUnion != nil { n += m.TargetUnion.Size() } l = len(m.RangeEnd) if l > 0 { n += 2 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *Compare_Version) Size() (n int) { if m == nil { return 0 } var l int _ = l n += 1 + sovRpc(uint64(m.Version)) return n } func (m *Compare_CreateRevision) Size() (n int) { if m == nil { return 0 } var l int _ = l n += 1 + sovRpc(uint64(m.CreateRevision)) return n } func (m *Compare_ModRevision) Size() (n int) { if m == nil { return 0 } var l int _ = l n += 1 + sovRpc(uint64(m.ModRevision)) return n } func (m *Compare_Value) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Value != nil { l = len(m.Value) n += 1 + l + sovRpc(uint64(l)) } return n } func (m *Compare_Lease) Size() (n int) { if m == nil { return 0 } var l int _ = l n += 1 + sovRpc(uint64(m.Lease)) return n } func (m *TxnRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if len(m.Compare) > 0 { for _, e := range m.Compare { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if len(m.Success) > 0 { for _, e := range m.Success { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if len(m.Failure) > 0 { for _, e := range m.Failure { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *TxnResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.Succeeded { n += 2 } if len(m.Responses) > 0 { for _, e := range m.Responses { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *CompactionRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Revision != 0 { n += 1 + sovRpc(uint64(m.Revision)) } if m.Physical { n += 2 } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *CompactionResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *HashRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *HashKVRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Revision != 0 { n += 1 + sovRpc(uint64(m.Revision)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *HashKVResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.Hash != 0 { n += 1 + sovRpc(uint64(m.Hash)) } if m.CompactRevision != 0 { n += 1 + sovRpc(uint64(m.CompactRevision)) } if m.HashRevision != 0 { n += 1 + sovRpc(uint64(m.HashRevision)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *HashResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.Hash != 0 { n += 1 + sovRpc(uint64(m.Hash)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *SnapshotRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *SnapshotResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.RemainingBytes != 0 { n += 1 + sovRpc(uint64(m.RemainingBytes)) } l = len(m.Blob) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.Version) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *WatchRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.RequestUnion != nil { n += m.RequestUnion.Size() } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *WatchRequest_CreateRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.CreateRequest != nil { l = m.CreateRequest.Size() n += 1 + l + sovRpc(uint64(l)) } return n } func (m *WatchRequest_CancelRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.CancelRequest != nil { l = m.CancelRequest.Size() n += 1 + l + sovRpc(uint64(l)) } return n } func (m *WatchRequest_ProgressRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ProgressRequest != nil { l = m.ProgressRequest.Size() n += 1 + l + sovRpc(uint64(l)) } return n } func (m *WatchCreateRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Key) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.RangeEnd) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.StartRevision != 0 { n += 1 + sovRpc(uint64(m.StartRevision)) } if m.ProgressNotify { n += 2 } if len(m.Filters) > 0 { l = 0 for _, e := range m.Filters { l += sovRpc(uint64(e)) } n += 1 + sovRpc(uint64(l)) + l } if m.PrevKv { n += 2 } if m.WatchId != 0 { n += 1 + sovRpc(uint64(m.WatchId)) } if m.Fragment { n += 2 } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *WatchCancelRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.WatchId != 0 { n += 1 + sovRpc(uint64(m.WatchId)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *WatchProgressRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *WatchResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.WatchId != 0 { n += 1 + sovRpc(uint64(m.WatchId)) } if m.Created { n += 2 } if m.Canceled { n += 2 } if m.CompactRevision != 0 { n += 1 + sovRpc(uint64(m.CompactRevision)) } l = len(m.CancelReason) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.Fragment { n += 2 } if len(m.Events) > 0 { for _, e := range m.Events { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseGrantRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.TTL != 0 { n += 1 + sovRpc(uint64(m.TTL)) } if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseGrantResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if m.TTL != 0 { n += 1 + sovRpc(uint64(m.TTL)) } l = len(m.Error) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseRevokeRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseRevokeResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseCheckpoint) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if m.Remaining_TTL != 0 { n += 1 + sovRpc(uint64(m.Remaining_TTL)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseCheckpointRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if len(m.Checkpoints) > 0 { for _, e := range m.Checkpoints { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseCheckpointResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseKeepAliveRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseKeepAliveResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if m.TTL != 0 { n += 1 + sovRpc(uint64(m.TTL)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseTimeToLiveRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if m.Keys { n += 2 } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseTimeToLiveResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if m.TTL != 0 { n += 1 + sovRpc(uint64(m.TTL)) } if m.GrantedTTL != 0 { n += 1 + sovRpc(uint64(m.GrantedTTL)) } if len(m.Keys) > 0 { for _, b := range m.Keys { l = len(b) n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseLeasesRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseStatus) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseLeasesResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Leases) > 0 { for _, e := range m.Leases { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *Member) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } l = len(m.Name) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if len(m.PeerURLs) > 0 { for _, s := range m.PeerURLs { l = len(s) n += 1 + l + sovRpc(uint64(l)) } } if len(m.ClientURLs) > 0 { for _, s := range m.ClientURLs { l = len(s) n += 1 + l + sovRpc(uint64(l)) } } if m.IsLearner { n += 2 } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MemberAddRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if len(m.PeerURLs) > 0 { for _, s := range m.PeerURLs { l = len(s) n += 1 + l + sovRpc(uint64(l)) } } if m.IsLearner { n += 2 } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MemberAddResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.Member != nil { l = m.Member.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Members) > 0 { for _, e := range m.Members { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MemberRemoveRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MemberRemoveResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Members) > 0 { for _, e := range m.Members { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MemberUpdateRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if len(m.PeerURLs) > 0 { for _, s := range m.PeerURLs { l = len(s) n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MemberUpdateResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Members) > 0 { for _, e := range m.Members { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MemberListRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Linearizable { n += 2 } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MemberListResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Members) > 0 { for _, e := range m.Members { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MemberPromoteRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovRpc(uint64(m.ID)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MemberPromoteResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Members) > 0 { for _, e := range m.Members { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *DefragmentRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *DefragmentResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MoveLeaderRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.TargetID != 0 { n += 1 + sovRpc(uint64(m.TargetID)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *MoveLeaderResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AlarmRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Action != 0 { n += 1 + sovRpc(uint64(m.Action)) } if m.MemberID != 0 { n += 1 + sovRpc(uint64(m.MemberID)) } if m.Alarm != 0 { n += 1 + sovRpc(uint64(m.Alarm)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AlarmMember) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.MemberID != 0 { n += 1 + sovRpc(uint64(m.MemberID)) } if m.Alarm != 0 { n += 1 + sovRpc(uint64(m.Alarm)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AlarmResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Alarms) > 0 { for _, e := range m.Alarms { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *DowngradeRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Action != 0 { n += 1 + sovRpc(uint64(m.Action)) } l = len(m.Version) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *DowngradeResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } l = len(m.Version) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *DowngradeVersionTestRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Ver) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *StatusRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *StatusResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } l = len(m.Version) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.DbSize != 0 { n += 1 + sovRpc(uint64(m.DbSize)) } if m.Leader != 0 { n += 1 + sovRpc(uint64(m.Leader)) } if m.RaftIndex != 0 { n += 1 + sovRpc(uint64(m.RaftIndex)) } if m.RaftTerm != 0 { n += 1 + sovRpc(uint64(m.RaftTerm)) } if m.RaftAppliedIndex != 0 { n += 1 + sovRpc(uint64(m.RaftAppliedIndex)) } if len(m.Errors) > 0 { for _, s := range m.Errors { l = len(s) n += 1 + l + sovRpc(uint64(l)) } } if m.DbSizeInUse != 0 { n += 1 + sovRpc(uint64(m.DbSizeInUse)) } if m.IsLearner { n += 2 } l = len(m.StorageVersion) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.DbSizeQuota != 0 { n += 1 + sovRpc(uint64(m.DbSizeQuota)) } if m.DowngradeInfo != nil { l = m.DowngradeInfo.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *DowngradeInfo) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Enabled { n += 2 } l = len(m.TargetVersion) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthEnableRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthDisableRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthStatusRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthenticateRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.Password) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserAddRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.Password) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.Options != nil { l = m.Options.Size() n += 1 + l + sovRpc(uint64(l)) } l = len(m.HashedPassword) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserGetRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserDeleteRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserChangePasswordRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.Password) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.HashedPassword) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserGrantRoleRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.User) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.Role) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserRevokeRoleRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.Role) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleAddRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleGetRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Role) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserListRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleListRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleDeleteRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Role) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleGrantPermissionRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.Perm != nil { l = m.Perm.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleRevokePermissionRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Role) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.Key) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } l = len(m.RangeEnd) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthEnableResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthDisableResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthStatusResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.Enabled { n += 2 } if m.AuthRevision != 0 { n += 1 + sovRpc(uint64(m.AuthRevision)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthenticateResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } l = len(m.Token) if l > 0 { n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserAddResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserGetResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Roles) > 0 { for _, s := range m.Roles { l = len(s) n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserDeleteResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserChangePasswordResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserGrantRoleResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserRevokeRoleResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleAddResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleGetResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Perm) > 0 { for _, e := range m.Perm { l = e.Size() n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleListResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Roles) > 0 { for _, s := range m.Roles { l = len(s) n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthUserListResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if len(m.Users) > 0 { for _, s := range m.Users { l = len(s) n += 1 + l + sovRpc(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleDeleteResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleGrantPermissionResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *AuthRoleRevokePermissionResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovRpc(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func sovRpc(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } func sozRpc(x uint64) (n int) { return sovRpc(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (m *ResponseHeader) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: ResponseHeader: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: ResponseHeader: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ClusterId", wireType) } m.ClusterId = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ClusterId |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field MemberId", wireType) } m.MemberId = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.MemberId |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Revision", wireType) } m.Revision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Revision |= int64(b&0x7F) << shift if b < 0x80 { break } } case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field RaftTerm", wireType) } m.RaftTerm = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.RaftTerm |= uint64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *RangeRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: RangeRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: RangeRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) if m.Key == nil { m.Key = []byte{} } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RangeEnd", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.RangeEnd = append(m.RangeEnd[:0], dAtA[iNdEx:postIndex]...) if m.RangeEnd == nil { m.RangeEnd = []byte{} } iNdEx = postIndex case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) } m.Limit = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Limit |= int64(b&0x7F) << shift if b < 0x80 { break } } case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Revision", wireType) } m.Revision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Revision |= int64(b&0x7F) << shift if b < 0x80 { break } } case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field SortOrder", wireType) } m.SortOrder = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.SortOrder |= RangeRequest_SortOrder(b&0x7F) << shift if b < 0x80 { break } } case 6: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field SortTarget", wireType) } m.SortTarget = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.SortTarget |= RangeRequest_SortTarget(b&0x7F) << shift if b < 0x80 { break } } case 7: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Serializable", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Serializable = bool(v != 0) case 8: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field KeysOnly", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.KeysOnly = bool(v != 0) case 9: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CountOnly", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.CountOnly = bool(v != 0) case 10: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field MinModRevision", wireType) } m.MinModRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.MinModRevision |= int64(b&0x7F) << shift if b < 0x80 { break } } case 11: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field MaxModRevision", wireType) } m.MaxModRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.MaxModRevision |= int64(b&0x7F) << shift if b < 0x80 { break } } case 12: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field MinCreateRevision", wireType) } m.MinCreateRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.MinCreateRevision |= int64(b&0x7F) << shift if b < 0x80 { break } } case 13: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field MaxCreateRevision", wireType) } m.MaxCreateRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.MaxCreateRevision |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *RangeResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: RangeResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: RangeResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Kvs", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Kvs = append(m.Kvs, &mvccpb.KeyValue{}) if err := m.Kvs[len(m.Kvs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field More", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.More = bool(v != 0) case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) } m.Count = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Count |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *PutRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: PutRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: PutRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) if m.Key == nil { m.Key = []byte{} } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Value = append(m.Value[:0], dAtA[iNdEx:postIndex]...) if m.Value == nil { m.Value = []byte{} } iNdEx = postIndex case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Lease", wireType) } m.Lease = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Lease |= int64(b&0x7F) << shift if b < 0x80 { break } } case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field PrevKv", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.PrevKv = bool(v != 0) case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field IgnoreValue", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.IgnoreValue = bool(v != 0) case 6: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field IgnoreLease", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.IgnoreLease = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *PutResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: PutResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: PutResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PrevKv", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.PrevKv == nil { m.PrevKv = &mvccpb.KeyValue{} } if err := m.PrevKv.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *DeleteRangeRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: DeleteRangeRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: DeleteRangeRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) if m.Key == nil { m.Key = []byte{} } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RangeEnd", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.RangeEnd = append(m.RangeEnd[:0], dAtA[iNdEx:postIndex]...) if m.RangeEnd == nil { m.RangeEnd = []byte{} } iNdEx = postIndex case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field PrevKv", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.PrevKv = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *DeleteRangeResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: DeleteRangeResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: DeleteRangeResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType) } m.Deleted = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Deleted |= int64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PrevKvs", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.PrevKvs = append(m.PrevKvs, &mvccpb.KeyValue{}) if err := m.PrevKvs[len(m.PrevKvs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *RequestOp) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: RequestOp: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: RequestOp: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RequestRange", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := &RangeRequest{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } m.Request = &RequestOp_RequestRange{v} iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RequestPut", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := &PutRequest{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } m.Request = &RequestOp_RequestPut{v} iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RequestDeleteRange", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := &DeleteRangeRequest{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } m.Request = &RequestOp_RequestDeleteRange{v} iNdEx = postIndex case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RequestTxn", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := &TxnRequest{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } m.Request = &RequestOp_RequestTxn{v} iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *ResponseOp) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: ResponseOp: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: ResponseOp: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ResponseRange", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := &RangeResponse{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } m.Response = &ResponseOp_ResponseRange{v} iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ResponsePut", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := &PutResponse{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } m.Response = &ResponseOp_ResponsePut{v} iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ResponseDeleteRange", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := &DeleteRangeResponse{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } m.Response = &ResponseOp_ResponseDeleteRange{v} iNdEx = postIndex case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ResponseTxn", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := &TxnResponse{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } m.Response = &ResponseOp_ResponseTxn{v} iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *Compare) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: Compare: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: Compare: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Result", wireType) } m.Result = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Result |= Compare_CompareResult(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Target", wireType) } m.Target = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Target |= Compare_CompareTarget(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) if m.Key == nil { m.Key = []byte{} } iNdEx = postIndex case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) } var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int64(b&0x7F) << shift if b < 0x80 { break } } m.TargetUnion = &Compare_Version{v} case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CreateRevision", wireType) } var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int64(b&0x7F) << shift if b < 0x80 { break } } m.TargetUnion = &Compare_CreateRevision{v} case 6: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ModRevision", wireType) } var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int64(b&0x7F) << shift if b < 0x80 { break } } m.TargetUnion = &Compare_ModRevision{v} case 7: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := make([]byte, postIndex-iNdEx) copy(v, dAtA[iNdEx:postIndex]) m.TargetUnion = &Compare_Value{v} iNdEx = postIndex case 8: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Lease", wireType) } var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int64(b&0x7F) << shift if b < 0x80 { break } } m.TargetUnion = &Compare_Lease{v} case 64: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RangeEnd", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.RangeEnd = append(m.RangeEnd[:0], dAtA[iNdEx:postIndex]...) if m.RangeEnd == nil { m.RangeEnd = []byte{} } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *TxnRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: TxnRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: TxnRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Compare", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Compare = append(m.Compare, &Compare{}) if err := m.Compare[len(m.Compare)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Success", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Success = append(m.Success, &RequestOp{}) if err := m.Success[len(m.Success)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Failure", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Failure = append(m.Failure, &RequestOp{}) if err := m.Failure[len(m.Failure)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *TxnResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: TxnResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: TxnResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Succeeded", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Succeeded = bool(v != 0) case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Responses", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Responses = append(m.Responses, &ResponseOp{}) if err := m.Responses[len(m.Responses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *CompactionRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: CompactionRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: CompactionRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Revision", wireType) } m.Revision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Revision |= int64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Physical", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Physical = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *CompactionResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: CompactionResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: CompactionResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *HashRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: HashRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: HashRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *HashKVRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: HashKVRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: HashKVRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Revision", wireType) } m.Revision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Revision |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *HashKVResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: HashKVResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: HashKVResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType) } m.Hash = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Hash |= uint32(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CompactRevision", wireType) } m.CompactRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.CompactRevision |= int64(b&0x7F) << shift if b < 0x80 { break } } case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field HashRevision", wireType) } m.HashRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.HashRevision |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *HashResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: HashResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: HashResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType) } m.Hash = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Hash |= uint32(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *SnapshotRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: SnapshotRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: SnapshotRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *SnapshotResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: SnapshotResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: SnapshotResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field RemainingBytes", wireType) } m.RemainingBytes = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.RemainingBytes |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Blob", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Blob = append(m.Blob[:0], dAtA[iNdEx:postIndex]...) if m.Blob == nil { m.Blob = []byte{} } iNdEx = postIndex case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Version = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *WatchRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: WatchRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: WatchRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field CreateRequest", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := &WatchCreateRequest{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } m.RequestUnion = &WatchRequest_CreateRequest{v} iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field CancelRequest", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := &WatchCancelRequest{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } m.RequestUnion = &WatchRequest_CancelRequest{v} iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ProgressRequest", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } v := &WatchProgressRequest{} if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } m.RequestUnion = &WatchRequest_ProgressRequest{v} iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *WatchCreateRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: WatchCreateRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: WatchCreateRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) if m.Key == nil { m.Key = []byte{} } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RangeEnd", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.RangeEnd = append(m.RangeEnd[:0], dAtA[iNdEx:postIndex]...) if m.RangeEnd == nil { m.RangeEnd = []byte{} } iNdEx = postIndex case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field StartRevision", wireType) } m.StartRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.StartRevision |= int64(b&0x7F) << shift if b < 0x80 { break } } case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ProgressNotify", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.ProgressNotify = bool(v != 0) case 5: if wireType == 0 { var v WatchCreateRequest_FilterType for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= WatchCreateRequest_FilterType(b&0x7F) << shift if b < 0x80 { break } } m.Filters = append(m.Filters, v) } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ packedLen |= int(b&0x7F) << shift if b < 0x80 { break } } if packedLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + packedLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } var elementCount int if elementCount != 0 && len(m.Filters) == 0 { m.Filters = make([]WatchCreateRequest_FilterType, 0, elementCount) } for iNdEx < postIndex { var v WatchCreateRequest_FilterType for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= WatchCreateRequest_FilterType(b&0x7F) << shift if b < 0x80 { break } } m.Filters = append(m.Filters, v) } } else { return fmt.Errorf("proto: wrong wireType = %d for field Filters", wireType) } case 6: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field PrevKv", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.PrevKv = bool(v != 0) case 7: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field WatchId", wireType) } m.WatchId = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.WatchId |= int64(b&0x7F) << shift if b < 0x80 { break } } case 8: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Fragment", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Fragment = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *WatchCancelRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: WatchCancelRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: WatchCancelRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field WatchId", wireType) } m.WatchId = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.WatchId |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *WatchProgressRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: WatchProgressRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: WatchProgressRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *WatchResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: WatchResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: WatchResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field WatchId", wireType) } m.WatchId = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.WatchId |= int64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Created", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Created = bool(v != 0) case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Canceled", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Canceled = bool(v != 0) case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CompactRevision", wireType) } m.CompactRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.CompactRevision |= int64(b&0x7F) << shift if b < 0x80 { break } } case 6: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field CancelReason", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.CancelReason = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 7: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Fragment", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Fragment = bool(v != 0) case 11: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Events", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Events = append(m.Events, &mvccpb.Event{}) if err := m.Events[len(m.Events)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseGrantRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseGrantRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseGrantRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field TTL", wireType) } m.TTL = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.TTL |= int64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseGrantResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseGrantResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseGrantResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= int64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field TTL", wireType) } m.TTL = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.TTL |= int64(b&0x7F) << shift if b < 0x80 { break } } case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Error = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseRevokeRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseRevokeRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseRevokeRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseRevokeResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseRevokeResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseRevokeResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseCheckpoint) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseCheckpoint: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseCheckpoint: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= int64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Remaining_TTL", wireType) } m.Remaining_TTL = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Remaining_TTL |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseCheckpointRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseCheckpointRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseCheckpointRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Checkpoints", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Checkpoints = append(m.Checkpoints, &LeaseCheckpoint{}) if err := m.Checkpoints[len(m.Checkpoints)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseCheckpointResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseCheckpointResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseCheckpointResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseKeepAliveRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseKeepAliveRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseKeepAliveRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseKeepAliveResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseKeepAliveResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseKeepAliveResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= int64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field TTL", wireType) } m.TTL = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.TTL |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseTimeToLiveRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseTimeToLiveRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseTimeToLiveRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= int64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Keys = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseTimeToLiveResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseTimeToLiveResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseTimeToLiveResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= int64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field TTL", wireType) } m.TTL = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.TTL |= int64(b&0x7F) << shift if b < 0x80 { break } } case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field GrantedTTL", wireType) } m.GrantedTTL = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.GrantedTTL |= int64(b&0x7F) << shift if b < 0x80 { break } } case 5: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Keys = append(m.Keys, make([]byte, postIndex-iNdEx)) copy(m.Keys[len(m.Keys)-1], dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseLeasesRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseLeasesRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseLeasesRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseStatus) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseStatus: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseStatus: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseLeasesResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseLeasesResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseLeasesResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Leases", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Leases = append(m.Leases, &LeaseStatus{}) if err := m.Leases[len(m.Leases)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *Member) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: Member: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: Member: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PeerURLs", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.PeerURLs = append(m.PeerURLs, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ClientURLs", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.ClientURLs = append(m.ClientURLs, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field IsLearner", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.IsLearner = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MemberAddRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MemberAddRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MemberAddRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PeerURLs", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.PeerURLs = append(m.PeerURLs, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field IsLearner", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.IsLearner = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MemberAddResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MemberAddResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MemberAddResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Member", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Member == nil { m.Member = &Member{} } if err := m.Member.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Members", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Members = append(m.Members, &Member{}) if err := m.Members[len(m.Members)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MemberRemoveRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MemberRemoveRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MemberRemoveRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= uint64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MemberRemoveResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MemberRemoveResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MemberRemoveResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Members", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Members = append(m.Members, &Member{}) if err := m.Members[len(m.Members)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MemberUpdateRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MemberUpdateRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MemberUpdateRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PeerURLs", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.PeerURLs = append(m.PeerURLs, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MemberUpdateResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MemberUpdateResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MemberUpdateResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Members", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Members = append(m.Members, &Member{}) if err := m.Members[len(m.Members)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MemberListRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MemberListRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MemberListRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Linearizable", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Linearizable = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MemberListResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MemberListResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MemberListResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Members", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Members = append(m.Members, &Member{}) if err := m.Members[len(m.Members)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MemberPromoteRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MemberPromoteRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MemberPromoteRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= uint64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MemberPromoteResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MemberPromoteResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MemberPromoteResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Members", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Members = append(m.Members, &Member{}) if err := m.Members[len(m.Members)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *DefragmentRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: DefragmentRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: DefragmentRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *DefragmentResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: DefragmentResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: DefragmentResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MoveLeaderRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MoveLeaderRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MoveLeaderRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field TargetID", wireType) } m.TargetID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.TargetID |= uint64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *MoveLeaderResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: MoveLeaderResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: MoveLeaderResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AlarmRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AlarmRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AlarmRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Action", wireType) } m.Action = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Action |= AlarmRequest_AlarmAction(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field MemberID", wireType) } m.MemberID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.MemberID |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Alarm", wireType) } m.Alarm = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Alarm |= AlarmType(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AlarmMember) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AlarmMember: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AlarmMember: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field MemberID", wireType) } m.MemberID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.MemberID |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Alarm", wireType) } m.Alarm = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Alarm |= AlarmType(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AlarmResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AlarmResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AlarmResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Alarms", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Alarms = append(m.Alarms, &AlarmMember{}) if err := m.Alarms[len(m.Alarms)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *DowngradeRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: DowngradeRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: DowngradeRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Action", wireType) } m.Action = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Action |= DowngradeRequest_DowngradeAction(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Version = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *DowngradeResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: DowngradeResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: DowngradeResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Version = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *DowngradeVersionTestRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: DowngradeVersionTestRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: DowngradeVersionTestRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Ver", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Ver = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *StatusRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: StatusRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: StatusRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *StatusResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: StatusResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: StatusResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Version = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field DbSize", wireType) } m.DbSize = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.DbSize |= int64(b&0x7F) << shift if b < 0x80 { break } } case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Leader", wireType) } m.Leader = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Leader |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field RaftIndex", wireType) } m.RaftIndex = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.RaftIndex |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 6: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field RaftTerm", wireType) } m.RaftTerm = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.RaftTerm |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 7: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field RaftAppliedIndex", wireType) } m.RaftAppliedIndex = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.RaftAppliedIndex |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 8: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Errors", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Errors = append(m.Errors, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex case 9: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field DbSizeInUse", wireType) } m.DbSizeInUse = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.DbSizeInUse |= int64(b&0x7F) << shift if b < 0x80 { break } } case 10: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field IsLearner", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.IsLearner = bool(v != 0) case 11: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field StorageVersion", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.StorageVersion = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 12: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field DbSizeQuota", wireType) } m.DbSizeQuota = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.DbSizeQuota |= int64(b&0x7F) << shift if b < 0x80 { break } } case 13: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field DowngradeInfo", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.DowngradeInfo == nil { m.DowngradeInfo = &DowngradeInfo{} } if err := m.DowngradeInfo.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *DowngradeInfo) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: DowngradeInfo: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: DowngradeInfo: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Enabled", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Enabled = bool(v != 0) case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field TargetVersion", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.TargetVersion = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthEnableRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthEnableRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthEnableRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthDisableRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthDisableRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthDisableRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthStatusRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthStatusRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthStatusRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthenticateRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthenticateRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthenticateRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Password", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Password = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserAddRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserAddRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserAddRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Password", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Password = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Options", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Options == nil { m.Options = &authpb.UserAddOptions{} } if err := m.Options.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field HashedPassword", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.HashedPassword = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserGetRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserGetRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserGetRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserDeleteRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserDeleteRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserDeleteRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserChangePasswordRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserChangePasswordRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserChangePasswordRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Password", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Password = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field HashedPassword", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.HashedPassword = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserGrantRoleRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserGrantRoleRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserGrantRoleRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.User = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Role", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Role = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserRevokeRoleRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserRevokeRoleRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserRevokeRoleRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Role", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Role = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleAddRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleAddRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleAddRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleGetRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleGetRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleGetRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Role", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Role = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserListRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserListRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserListRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleListRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleListRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleListRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleDeleteRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleDeleteRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleDeleteRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Role", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Role = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleGrantPermissionRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleGrantPermissionRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleGrantPermissionRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Perm", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Perm == nil { m.Perm = &authpb.Permission{} } if err := m.Perm.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleRevokePermissionRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleRevokePermissionRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleRevokePermissionRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Role", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Role = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) if m.Key == nil { m.Key = []byte{} } iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RangeEnd", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.RangeEnd = append(m.RangeEnd[:0], dAtA[iNdEx:postIndex]...) if m.RangeEnd == nil { m.RangeEnd = []byte{} } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthEnableResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthEnableResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthEnableResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthDisableResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthDisableResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthDisableResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthStatusResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthStatusResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthStatusResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Enabled", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Enabled = bool(v != 0) case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field AuthRevision", wireType) } m.AuthRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.AuthRevision |= uint64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthenticateResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthenticateResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthenticateResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Token", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Token = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserAddResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserAddResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserAddResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserGetResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserGetResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserGetResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Roles", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Roles = append(m.Roles, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserDeleteResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserDeleteResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserDeleteResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserChangePasswordResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserChangePasswordResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserChangePasswordResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserGrantRoleResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserGrantRoleResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserGrantRoleResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserRevokeRoleResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserRevokeRoleResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserRevokeRoleResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleAddResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleAddResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleAddResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleGetResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleGetResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleGetResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Perm", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Perm = append(m.Perm, &authpb.Permission{}) if err := m.Perm[len(m.Perm)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleListResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleListResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleListResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Roles", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Roles = append(m.Roles, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthUserListResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthUserListResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthUserListResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Users", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } m.Users = append(m.Users, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleDeleteResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleDeleteResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleDeleteResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleGrantPermissionResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleGrantPermissionResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleGrantPermissionResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *AuthRoleRevokePermissionResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: AuthRoleRevokePermissionResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: AuthRoleRevokePermissionResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRpc } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRpc } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRpc } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRpc } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func skipRpc(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowRpc } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } wireType := int(wire & 0x7) switch wireType { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowRpc } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } iNdEx++ if dAtA[iNdEx-1] < 0x80 { break } } case 1: iNdEx += 8 case 2: var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowRpc } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { break } } if length < 0 { return 0, ErrInvalidLengthRpc } iNdEx += length case 3: depth++ case 4: if depth == 0 { return 0, ErrUnexpectedEndOfGroupRpc } depth-- case 5: iNdEx += 4 default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { return 0, ErrInvalidLengthRpc } if depth == 0 { return iNdEx, nil } } return 0, io.ErrUnexpectedEOF } var ( ErrInvalidLengthRpc = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowRpc = fmt.Errorf("proto: integer overflow") ErrUnexpectedEndOfGroupRpc = fmt.Errorf("proto: unexpected end of group") ) ================================================ FILE: api/etcdserverpb/rpc.proto ================================================ syntax = "proto3"; package etcdserverpb; import "etcd/api/mvccpb/kv.proto"; import "etcd/api/authpb/auth.proto"; import "etcd/api/versionpb/version.proto"; // for grpc-gateway import "google/api/annotations.proto"; import "protoc-gen-openapiv2/options/annotations.proto"; option go_package = "go.etcd.io/etcd/api/v3/etcdserverpb"; option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { security_definitions: { security: { key: "ApiKey"; value: { type: TYPE_API_KEY; in: IN_HEADER; name: "Authorization"; } } } security: { security_requirement: { key: "ApiKey"; value: {}; } } }; service KV { // Range gets the keys in the range from the key-value store. rpc Range(RangeRequest) returns (RangeResponse) { option (google.api.http) = { post: "/v3/kv/range" body: "*" }; } // Put puts the given key into the key-value store. // A put request increments the revision of the key-value store // and generates one event in the event history. rpc Put(PutRequest) returns (PutResponse) { option (google.api.http) = { post: "/v3/kv/put" body: "*" }; } // DeleteRange deletes the given range from the key-value store. // A delete request increments the revision of the key-value store // and generates a delete event in the event history for every deleted key. rpc DeleteRange(DeleteRangeRequest) returns (DeleteRangeResponse) { option (google.api.http) = { post: "/v3/kv/deleterange" body: "*" }; } // Txn processes multiple requests in a single transaction. // A txn request increments the revision of the key-value store // and generates events with the same revision for every completed request. // It is not allowed to modify the same key several times within one txn. rpc Txn(TxnRequest) returns (TxnResponse) { option (google.api.http) = { post: "/v3/kv/txn" body: "*" }; } // Compact compacts the event history in the etcd key-value store. The key-value // store should be periodically compacted or the event history will continue to grow // indefinitely. rpc Compact(CompactionRequest) returns (CompactionResponse) { option (google.api.http) = { post: "/v3/kv/compaction" body: "*" }; } } service Watch { // Watch watches for events happening or that have happened. Both input and output // are streams; the input stream is for creating and canceling watchers and the output // stream sends events. One watch RPC can watch on multiple key ranges, streaming events // for several watches at once. The entire event history can be watched starting from the // last compaction revision. rpc Watch(stream WatchRequest) returns (stream WatchResponse) { option (google.api.http) = { post: "/v3/watch" body: "*" }; } } service Lease { // LeaseGrant creates a lease which expires if the server does not receive a keepAlive // within a given time to live period. All keys attached to the lease will be expired and // deleted if the lease expires. Each expired key generates a delete event in the event history. rpc LeaseGrant(LeaseGrantRequest) returns (LeaseGrantResponse) { option (google.api.http) = { post: "/v3/lease/grant" body: "*" }; } // LeaseRevoke revokes a lease. All keys attached to the lease will expire and be deleted. rpc LeaseRevoke(LeaseRevokeRequest) returns (LeaseRevokeResponse) { option (google.api.http) = { post: "/v3/lease/revoke" body: "*" additional_bindings { post: "/v3/kv/lease/revoke" body: "*" } }; } // LeaseKeepAlive keeps the lease alive by streaming keep alive requests from the client // to the server and streaming keep alive responses from the server to the client. rpc LeaseKeepAlive(stream LeaseKeepAliveRequest) returns (stream LeaseKeepAliveResponse) { option (google.api.http) = { post: "/v3/lease/keepalive" body: "*" }; } // LeaseTimeToLive retrieves lease information. rpc LeaseTimeToLive(LeaseTimeToLiveRequest) returns (LeaseTimeToLiveResponse) { option (google.api.http) = { post: "/v3/lease/timetolive" body: "*" additional_bindings { post: "/v3/kv/lease/timetolive" body: "*" } }; } // LeaseLeases lists all existing leases. rpc LeaseLeases(LeaseLeasesRequest) returns (LeaseLeasesResponse) { option (google.api.http) = { post: "/v3/lease/leases" body: "*" additional_bindings { post: "/v3/kv/lease/leases" body: "*" } }; } } service Cluster { // MemberAdd adds a member into the cluster. rpc MemberAdd(MemberAddRequest) returns (MemberAddResponse) { option (google.api.http) = { post: "/v3/cluster/member/add" body: "*" }; } // MemberRemove removes an existing member from the cluster. rpc MemberRemove(MemberRemoveRequest) returns (MemberRemoveResponse) { option (google.api.http) = { post: "/v3/cluster/member/remove" body: "*" }; } // MemberUpdate updates the member configuration. rpc MemberUpdate(MemberUpdateRequest) returns (MemberUpdateResponse) { option (google.api.http) = { post: "/v3/cluster/member/update" body: "*" }; } // MemberList lists all the members in the cluster. rpc MemberList(MemberListRequest) returns (MemberListResponse) { option (google.api.http) = { post: "/v3/cluster/member/list" body: "*" }; } // MemberPromote promotes a member from raft learner (non-voting) to raft voting member. rpc MemberPromote(MemberPromoteRequest) returns (MemberPromoteResponse) { option (google.api.http) = { post: "/v3/cluster/member/promote" body: "*" }; } } service Maintenance { // Alarm activates, deactivates, and queries alarms regarding cluster health. rpc Alarm(AlarmRequest) returns (AlarmResponse) { option (google.api.http) = { post: "/v3/maintenance/alarm" body: "*" }; } // Status gets the status of the member. rpc Status(StatusRequest) returns (StatusResponse) { option (google.api.http) = { post: "/v3/maintenance/status" body: "*" }; } // Defragment defragments a member's backend database to recover storage space. rpc Defragment(DefragmentRequest) returns (DefragmentResponse) { option (google.api.http) = { post: "/v3/maintenance/defragment" body: "*" }; } // Hash computes the hash of whole backend keyspace, // including key, lease, and other buckets in storage. // This is designed for testing ONLY! // Do not rely on this in production with ongoing transactions, // since Hash operation does not hold MVCC locks. // Use "HashKV" API instead for "key" bucket consistency checks. rpc Hash(HashRequest) returns (HashResponse) { option (google.api.http) = { post: "/v3/maintenance/hash" body: "*" }; } // HashKV computes the hash of all MVCC keys up to a given revision. // It only iterates "key" bucket in backend storage. rpc HashKV(HashKVRequest) returns (HashKVResponse) { option (google.api.http) = { post: "/v3/maintenance/hashkv" body: "*" }; } // Snapshot sends a snapshot of the entire backend from a member over a stream to a client. rpc Snapshot(SnapshotRequest) returns (stream SnapshotResponse) { option (google.api.http) = { post: "/v3/maintenance/snapshot" body: "*" }; } // MoveLeader requests current leader node to transfer its leadership to transferee. rpc MoveLeader(MoveLeaderRequest) returns (MoveLeaderResponse) { option (google.api.http) = { post: "/v3/maintenance/transfer-leadership" body: "*" }; } // Downgrade requests downgrades, verifies feasibility or cancels downgrade // on the cluster version. // Supported since etcd 3.5. rpc Downgrade(DowngradeRequest) returns (DowngradeResponse) { option (google.api.http) = { post: "/v3/maintenance/downgrade" body: "*" }; } } service Auth { // AuthEnable enables authentication. rpc AuthEnable(AuthEnableRequest) returns (AuthEnableResponse) { option (google.api.http) = { post: "/v3/auth/enable" body: "*" }; } // AuthDisable disables authentication. rpc AuthDisable(AuthDisableRequest) returns (AuthDisableResponse) { option (google.api.http) = { post: "/v3/auth/disable" body: "*" }; } // AuthStatus displays authentication status. rpc AuthStatus(AuthStatusRequest) returns (AuthStatusResponse) { option (google.api.http) = { post: "/v3/auth/status" body: "*" }; } // Authenticate processes an authenticate request. rpc Authenticate(AuthenticateRequest) returns (AuthenticateResponse) { option (google.api.http) = { post: "/v3/auth/authenticate" body: "*" }; } // UserAdd adds a new user. User name cannot be empty. rpc UserAdd(AuthUserAddRequest) returns (AuthUserAddResponse) { option (google.api.http) = { post: "/v3/auth/user/add" body: "*" }; } // UserGet gets detailed user information. rpc UserGet(AuthUserGetRequest) returns (AuthUserGetResponse) { option (google.api.http) = { post: "/v3/auth/user/get" body: "*" }; } // UserList gets a list of all users. rpc UserList(AuthUserListRequest) returns (AuthUserListResponse) { option (google.api.http) = { post: "/v3/auth/user/list" body: "*" }; } // UserDelete deletes a specified user. rpc UserDelete(AuthUserDeleteRequest) returns (AuthUserDeleteResponse) { option (google.api.http) = { post: "/v3/auth/user/delete" body: "*" }; } // UserChangePassword changes the password of a specified user. rpc UserChangePassword(AuthUserChangePasswordRequest) returns (AuthUserChangePasswordResponse) { option (google.api.http) = { post: "/v3/auth/user/changepw" body: "*" }; } // UserGrantRole grants a role to a specified user. rpc UserGrantRole(AuthUserGrantRoleRequest) returns (AuthUserGrantRoleResponse) { option (google.api.http) = { post: "/v3/auth/user/grant" body: "*" }; } // UserRevokeRole revokes a role of specified user. rpc UserRevokeRole(AuthUserRevokeRoleRequest) returns (AuthUserRevokeRoleResponse) { option (google.api.http) = { post: "/v3/auth/user/revoke" body: "*" }; } // RoleAdd adds a new role. Role name cannot be empty. rpc RoleAdd(AuthRoleAddRequest) returns (AuthRoleAddResponse) { option (google.api.http) = { post: "/v3/auth/role/add" body: "*" }; } // RoleGet gets detailed role information. rpc RoleGet(AuthRoleGetRequest) returns (AuthRoleGetResponse) { option (google.api.http) = { post: "/v3/auth/role/get" body: "*" }; } // RoleList gets lists of all roles. rpc RoleList(AuthRoleListRequest) returns (AuthRoleListResponse) { option (google.api.http) = { post: "/v3/auth/role/list" body: "*" }; } // RoleDelete deletes a specified role. rpc RoleDelete(AuthRoleDeleteRequest) returns (AuthRoleDeleteResponse) { option (google.api.http) = { post: "/v3/auth/role/delete" body: "*" }; } // RoleGrantPermission grants a permission of a specified key or range to a specified role. rpc RoleGrantPermission(AuthRoleGrantPermissionRequest) returns (AuthRoleGrantPermissionResponse) { option (google.api.http) = { post: "/v3/auth/role/grant" body: "*" }; } // RoleRevokePermission revokes a key or range permission of a specified role. rpc RoleRevokePermission(AuthRoleRevokePermissionRequest) returns (AuthRoleRevokePermissionResponse) { option (google.api.http) = { post: "/v3/auth/role/revoke" body: "*" }; } } message ResponseHeader { option (versionpb.etcd_version_msg) = "3.0"; // cluster_id is the ID of the cluster which sent the response. uint64 cluster_id = 1; // member_id is the ID of the member which sent the response. uint64 member_id = 2; // revision is the key-value store revision when the request was applied, and it's // unset (so 0) in case of calls not interacting with key-value store. // For watch progress responses, the header.revision indicates progress. All future events // received in this stream are guaranteed to have a higher revision number than the // header.revision number. int64 revision = 3; // raft_term is the raft term when the request was applied. uint64 raft_term = 4; } message RangeRequest { option (versionpb.etcd_version_msg) = "3.0"; enum SortOrder { option (versionpb.etcd_version_enum) = "3.0"; NONE = 0; // default, no sorting ASCEND = 1; // lowest target value first DESCEND = 2; // highest target value first } enum SortTarget { option (versionpb.etcd_version_enum) = "3.0"; KEY = 0; VERSION = 1; CREATE = 2; MOD = 3; VALUE = 4; } // key is the first key for the range. If range_end is not given, the request only looks up key. bytes key = 1; // range_end is the upper bound on the requested range [key, range_end). // If range_end is '\0', the range is all keys >= key. // If range_end is key plus one (e.g., "aa"+1 == "ab", "a\xff"+1 == "b"), // then the range request gets all keys prefixed with key. // If both key and range_end are '\0', then the range request returns all keys. bytes range_end = 2; // limit is a limit on the number of keys returned for the request. When limit is set to 0, // it is treated as no limit. int64 limit = 3; // revision is the point-in-time of the key-value store to use for the range. // If revision is less or equal to zero, the range is over the newest key-value store. // If the revision has been compacted, ErrCompacted is returned as a response. int64 revision = 4; // sort_order is the order for returned sorted results. SortOrder sort_order = 5; // sort_target is the key-value field to use for sorting. SortTarget sort_target = 6; // serializable sets the range request to use serializable member-local reads. // Range requests are linearizable by default; linearizable requests have higher // latency and lower throughput than serializable requests but reflect the current // consensus of the cluster. For better performance, in exchange for possible stale reads, // a serializable range request is served locally without needing to reach consensus // with other nodes in the cluster. bool serializable = 7; // keys_only when set returns only the keys and not the values. bool keys_only = 8; // count_only when set returns only the count of the keys in the range. bool count_only = 9; // min_mod_revision is the lower bound for returned key mod revisions; all keys with // lesser mod revisions will be filtered away. int64 min_mod_revision = 10 [(versionpb.etcd_version_field)="3.1"]; // max_mod_revision is the upper bound for returned key mod revisions; all keys with // greater mod revisions will be filtered away. int64 max_mod_revision = 11 [(versionpb.etcd_version_field)="3.1"]; // min_create_revision is the lower bound for returned key create revisions; all keys with // lesser create revisions will be filtered away. int64 min_create_revision = 12 [(versionpb.etcd_version_field)="3.1"]; // max_create_revision is the upper bound for returned key create revisions; all keys with // greater create revisions will be filtered away. int64 max_create_revision = 13 [(versionpb.etcd_version_field)="3.1"]; } message RangeResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // kvs is the list of key-value pairs matched by the range request. // kvs is empty when count is requested. repeated mvccpb.KeyValue kvs = 2; // more indicates if there are more keys to return in the requested range. bool more = 3; // count is set to the actual number of keys within the range when requested. // Unlike Kvs, it is unaffected by limits and filters (e.g., Min/Max, Create/Modify, Revisions) // and reflects the full count within the specified range. int64 count = 4; } message PutRequest { option (versionpb.etcd_version_msg) = "3.0"; // key is the key, in bytes, to put into the key-value store. bytes key = 1; // value is the value, in bytes, to associate with the key in the key-value store. bytes value = 2; // lease is the lease ID to associate with the key in the key-value store. A lease // value of 0 indicates no lease. int64 lease = 3; // If prev_kv is set, etcd gets the previous key-value pair before changing it. // The previous key-value pair will be returned in the put response. bool prev_kv = 4 [(versionpb.etcd_version_field)="3.1"]; // If ignore_value is set, etcd updates the key using its current value. // Returns an error if the key does not exist. bool ignore_value = 5 [(versionpb.etcd_version_field)="3.2"]; // If ignore_lease is set, etcd updates the key using its current lease. // Returns an error if the key does not exist. bool ignore_lease = 6 [(versionpb.etcd_version_field)="3.2"]; } message PutResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // if prev_kv is set in the request, the previous key-value pair will be returned. mvccpb.KeyValue prev_kv = 2 [(versionpb.etcd_version_field)="3.1"]; } message DeleteRangeRequest { option (versionpb.etcd_version_msg) = "3.0"; // key is the first key to delete in the range. bytes key = 1; // range_end is the key following the last key to delete for the range [key, range_end). // If range_end is not given, the range is defined to contain only the key argument. // If range_end is one bit larger than the given key, then the range is all the keys // with the prefix (the given key). // If range_end is '\0', the range is all keys greater than or equal to the key argument. bytes range_end = 2; // If prev_kv is set, etcd gets the previous key-value pairs before deleting it. // The previous key-value pairs will be returned in the delete response. bool prev_kv = 3 [(versionpb.etcd_version_field)="3.1"]; } message DeleteRangeResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // deleted is the number of keys deleted by the delete range request. int64 deleted = 2; // if prev_kv is set in the request, the previous key-value pairs will be returned. repeated mvccpb.KeyValue prev_kvs = 3 [(versionpb.etcd_version_field)="3.1"]; } message RequestOp { option (versionpb.etcd_version_msg) = "3.0"; // request is a union of request types accepted by a transaction. oneof request { RangeRequest request_range = 1; PutRequest request_put = 2; DeleteRangeRequest request_delete_range = 3; TxnRequest request_txn = 4 [(versionpb.etcd_version_field)="3.3"]; } } message ResponseOp { option (versionpb.etcd_version_msg) = "3.0"; // response is a union of response types returned by a transaction. oneof response { RangeResponse response_range = 1; PutResponse response_put = 2; DeleteRangeResponse response_delete_range = 3; TxnResponse response_txn = 4 [(versionpb.etcd_version_field)="3.3"]; } } message Compare { option (versionpb.etcd_version_msg) = "3.0"; enum CompareResult { option (versionpb.etcd_version_enum) = "3.0"; EQUAL = 0; GREATER = 1; LESS = 2; NOT_EQUAL = 3 [(versionpb.etcd_version_enum_value)="3.1"]; } enum CompareTarget { option (versionpb.etcd_version_enum) = "3.0"; VERSION = 0; CREATE = 1; MOD = 2; VALUE = 3; LEASE = 4 [(versionpb.etcd_version_enum_value)="3.3"]; } // result is logical comparison operation for this comparison. CompareResult result = 1; // target is the key-value field to inspect for the comparison. CompareTarget target = 2; // key is the subject key for the comparison operation. bytes key = 3; oneof target_union { // version is the version of the given key int64 version = 4; // create_revision is the creation revision of the given key int64 create_revision = 5; // mod_revision is the last modified revision of the given key. int64 mod_revision = 6; // value is the value of the given key, in bytes. bytes value = 7; // lease is the lease id of the given key. int64 lease = 8 [(versionpb.etcd_version_field)="3.3"]; // leave room for more target_union field tags, jump to 64 } // range_end compares the given target to all keys in the range [key, range_end). // See RangeRequest for more details on key ranges. bytes range_end = 64 [(versionpb.etcd_version_field)="3.3"]; // TODO: fill out with most of the rest of RangeRequest fields when needed. } // From google paxosdb paper: // Our implementation hinges around a powerful primitive which we call MultiOp. All other database // operations except for iteration are implemented as a single call to MultiOp. A MultiOp is applied atomically // and consists of three components: // 1. A list of tests called guard. Each test in guard checks a single entry in the database. It may check // for the absence or presence of a value, or compare with a given value. Two different tests in the guard // may apply to the same or different entries in the database. All tests in the guard are applied and // MultiOp returns the results. If all tests are true, MultiOp executes t op (see item 2 below), otherwise // it executes f op (see item 3 below). // 2. A list of database operations called t op. Each operation in the list is either an insert, delete, or // lookup operation, and applies to a single database entry. Two different operations in the list may apply // to the same or different entries in the database. These operations are executed // if guard evaluates to // true. // 3. A list of database operations called f op. Like t op, but executed if guard evaluates to false. message TxnRequest { option (versionpb.etcd_version_msg) = "3.0"; // compare is a list of predicates representing a conjunction of terms. // If the comparisons succeed, then the success requests will be processed in order, // and the response will contain their respective responses in order. // If the comparisons fail, then the failure requests will be processed in order, // and the response will contain their respective responses in order. repeated Compare compare = 1; // success is a list of requests which will be applied when compare evaluates to true. repeated RequestOp success = 2; // failure is a list of requests which will be applied when compare evaluates to false. repeated RequestOp failure = 3; } message TxnResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // succeeded is set to true if the compare evaluated to true or false otherwise. bool succeeded = 2; // responses is a list of responses corresponding to the results from applying // success if succeeded is true or failure if succeeded is false. repeated ResponseOp responses = 3; } // CompactionRequest compacts the key-value store up to a given revision. All superseded keys // with a revision less than the compaction revision will be removed. message CompactionRequest { option (versionpb.etcd_version_msg) = "3.0"; // revision is the key-value store revision for the compaction operation. int64 revision = 1; // physical is set so the RPC will wait until the compaction is physically // applied to the local database such that compacted entries are totally // removed from the backend database. bool physical = 2; } message CompactionResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message HashRequest { option (versionpb.etcd_version_msg) = "3.0"; } message HashKVRequest { option (versionpb.etcd_version_msg) = "3.3"; // revision is the key-value store revision for the hash operation. int64 revision = 1; } message HashKVResponse { option (versionpb.etcd_version_msg) = "3.3"; ResponseHeader header = 1; // hash is the hash value computed from the responding member's MVCC keys up to a given revision. uint32 hash = 2; // compact_revision is the compacted revision of key-value store when hash begins. int64 compact_revision = 3; // hash_revision is the revision up to which the hash is calculated. int64 hash_revision = 4 [(versionpb.etcd_version_field)="3.6"]; } message HashResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // hash is the hash value computed from the responding member's KV's backend. uint32 hash = 2; } message SnapshotRequest { option (versionpb.etcd_version_msg) = "3.3"; } message SnapshotResponse { option (versionpb.etcd_version_msg) = "3.3"; // header has the current key-value store information. The first header in the snapshot // stream indicates the point in time of the snapshot. ResponseHeader header = 1; // remaining_bytes is the number of blob bytes to be sent after this message uint64 remaining_bytes = 2; // blob contains the next chunk of the snapshot in the snapshot stream. bytes blob = 3; // local version of server that created the snapshot. // In cluster with binaries with different version, each cluster can return different result. // Informs which etcd server version should be used when restoring the snapshot. string version = 4 [(versionpb.etcd_version_field)="3.6"]; } message WatchRequest { option (versionpb.etcd_version_msg) = "3.0"; // request_union is a request to either create a new watcher or cancel an existing watcher. oneof request_union { WatchCreateRequest create_request = 1; WatchCancelRequest cancel_request = 2; WatchProgressRequest progress_request = 3 [(versionpb.etcd_version_field)="3.4"]; } } message WatchCreateRequest { option (versionpb.etcd_version_msg) = "3.0"; // key is the key to register for watching. bytes key = 1; // range_end is the end of the range [key, range_end) to watch. If range_end is not given, // only the key argument is watched. If range_end is equal to '\0', all keys greater than // or equal to the key argument are watched. // If the range_end is one bit larger than the given key, // then all keys with the prefix (the given key) will be watched. bytes range_end = 2; // start_revision is an optional revision to watch from (inclusive). No start_revision is "now". int64 start_revision = 3; // progress_notify is set so that the etcd server will periodically send a WatchResponse with // no events to the new watcher if there are no recent events. It is useful when clients // wish to recover a disconnected watcher starting from a recent known revision. // The etcd server may decide how often it will send notifications based on current load. bool progress_notify = 4; enum FilterType { option (versionpb.etcd_version_enum) = "3.1"; // filter out put event. NOPUT = 0; // filter out delete event. NODELETE = 1; } // filters filter the events at server side before it sends back to the watcher. repeated FilterType filters = 5 [(versionpb.etcd_version_field)="3.1"]; // If prev_kv is set, created watcher gets the previous KV before the event happens. // If the previous KV is already compacted, nothing will be returned. bool prev_kv = 6 [(versionpb.etcd_version_field)="3.1"]; // If watch_id is provided and non-zero, it will be assigned to this watcher. // Since creating a watcher in etcd is not a synchronous operation, // this can be used ensure that ordering is correct when creating multiple // watchers on the same stream. Creating a watcher with an ID already in // use on the stream will cause an error to be returned. int64 watch_id = 7 [(versionpb.etcd_version_field)="3.4"]; // fragment enables splitting large revisions into multiple watch responses. bool fragment = 8 [(versionpb.etcd_version_field)="3.4"]; } message WatchCancelRequest { option (versionpb.etcd_version_msg) = "3.1"; // watch_id is the watcher id to cancel so that no more events are transmitted. int64 watch_id = 1 [(versionpb.etcd_version_field)="3.1"]; } // Requests the a watch stream progress status be sent in the watch response stream as soon as // possible. message WatchProgressRequest { option (versionpb.etcd_version_msg) = "3.4"; } message WatchResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // watch_id is the ID of the watcher that corresponds to the response. int64 watch_id = 2; // created is set to true if the response is for a create watch request. // The client should record the watch_id and expect to receive events for // the created watcher from the same stream. // All events sent to the created watcher will attach with the same watch_id. bool created = 3; // canceled is set to true if the response is for a cancel watch request // or if the start_revision has already been compacted. // No further events will be sent to the canceled watcher. bool canceled = 4; // compact_revision is set to the minimum index if a watcher tries to watch // at a compacted index. // // This happens when creating a watcher at a compacted revision or the watcher cannot // catch up with the progress of the key-value store. // // The client should treat the watcher as canceled and should not try to create any // watcher with the same start_revision again. int64 compact_revision = 5; // cancel_reason indicates the reason for canceling the watcher. string cancel_reason = 6 [(versionpb.etcd_version_field)="3.4"]; // framgment is true if large watch response was split over multiple responses. bool fragment = 7 [(versionpb.etcd_version_field)="3.4"]; repeated mvccpb.Event events = 11; } message LeaseGrantRequest { option (versionpb.etcd_version_msg) = "3.0"; // TTL is the advisory time-to-live in seconds. Expired lease will return -1. int64 TTL = 1; // ID is the requested ID for the lease. If ID is set to 0, the lessor chooses an ID. int64 ID = 2; } message LeaseGrantResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // ID is the lease ID for the granted lease. int64 ID = 2; // TTL is the server chosen lease time-to-live in seconds. int64 TTL = 3; string error = 4; } message LeaseRevokeRequest { option (versionpb.etcd_version_msg) = "3.0"; // ID is the lease ID to revoke. When the ID is revoked, all associated keys will be deleted. int64 ID = 1; } message LeaseRevokeResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message LeaseCheckpoint { option (versionpb.etcd_version_msg) = "3.4"; // ID is the lease ID to checkpoint. int64 ID = 1; // Remaining_TTL is the remaining time until expiry of the lease. int64 remaining_TTL = 2; } message LeaseCheckpointRequest { option (versionpb.etcd_version_msg) = "3.4"; repeated LeaseCheckpoint checkpoints = 1; } message LeaseCheckpointResponse { option (versionpb.etcd_version_msg) = "3.4"; ResponseHeader header = 1; } message LeaseKeepAliveRequest { option (versionpb.etcd_version_msg) = "3.0"; // ID is the lease ID for the lease to keep alive. int64 ID = 1; } message LeaseKeepAliveResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // ID is the lease ID from the keep alive request. int64 ID = 2; // TTL is the new time-to-live for the lease. int64 TTL = 3; } message LeaseTimeToLiveRequest { option (versionpb.etcd_version_msg) = "3.1"; // ID is the lease ID for the lease. int64 ID = 1; // keys is true to query all the keys attached to this lease. bool keys = 2; } message LeaseTimeToLiveResponse { option (versionpb.etcd_version_msg) = "3.1"; ResponseHeader header = 1; // ID is the lease ID from the keep alive request. int64 ID = 2; // TTL is the remaining TTL in seconds for the lease; the lease will expire in under TTL+1 seconds. int64 TTL = 3; // GrantedTTL is the initial granted time in seconds upon lease creation/renewal. int64 grantedTTL = 4; // Keys is the list of keys attached to this lease. repeated bytes keys = 5; } message LeaseLeasesRequest { option (versionpb.etcd_version_msg) = "3.3"; } message LeaseStatus { option (versionpb.etcd_version_msg) = "3.3"; int64 ID = 1; // TODO: int64 TTL = 2; } message LeaseLeasesResponse { option (versionpb.etcd_version_msg) = "3.3"; ResponseHeader header = 1; repeated LeaseStatus leases = 2; } message Member { option (versionpb.etcd_version_msg) = "3.0"; // ID is the member ID for this member. uint64 ID = 1; // name is the human-readable name of the member. If the member is not started, the name will be an empty string. string name = 2; // peerURLs is the list of URLs the member exposes to the cluster for communication. repeated string peerURLs = 3; // clientURLs is the list of URLs the member exposes to clients for communication. If the member is not started, clientURLs will be empty. repeated string clientURLs = 4; // isLearner indicates if the member is raft learner. bool isLearner = 5 [(versionpb.etcd_version_field)="3.4"]; } message MemberAddRequest { option (versionpb.etcd_version_msg) = "3.0"; // peerURLs is the list of URLs the added member will use to communicate with the cluster. repeated string peerURLs = 1; // isLearner indicates if the added member is raft learner. bool isLearner = 2 [(versionpb.etcd_version_field)="3.4"]; } message MemberAddResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // member is the member information for the added member. Member member = 2; // members is a list of all members after adding the new member. repeated Member members = 3; } message MemberRemoveRequest { option (versionpb.etcd_version_msg) = "3.0"; // ID is the member ID of the member to remove. uint64 ID = 1; } message MemberRemoveResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // members is a list of all members after removing the member. repeated Member members = 2; } message MemberUpdateRequest { option (versionpb.etcd_version_msg) = "3.0"; // ID is the member ID of the member to update. uint64 ID = 1; // peerURLs is the new list of URLs the member will use to communicate with the cluster. repeated string peerURLs = 2; } message MemberUpdateResponse{ option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // members is a list of all members after updating the member. repeated Member members = 2 [(versionpb.etcd_version_field)="3.1"]; } message MemberListRequest { option (versionpb.etcd_version_msg) = "3.0"; bool linearizable = 1 [(versionpb.etcd_version_field)="3.5"]; } message MemberListResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // members is a list of all members associated with the cluster. repeated Member members = 2; } message MemberPromoteRequest { option (versionpb.etcd_version_msg) = "3.4"; // ID is the member ID of the member to promote. uint64 ID = 1; } message MemberPromoteResponse { option (versionpb.etcd_version_msg) = "3.4"; ResponseHeader header = 1; // members is a list of all members after promoting the member. repeated Member members = 2; } message DefragmentRequest { option (versionpb.etcd_version_msg) = "3.0"; } message DefragmentResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message MoveLeaderRequest { option (versionpb.etcd_version_msg) = "3.3"; // targetID is the node ID for the new leader. uint64 targetID = 1; } message MoveLeaderResponse { option (versionpb.etcd_version_msg) = "3.3"; ResponseHeader header = 1; } enum AlarmType { option (versionpb.etcd_version_enum) = "3.0"; NONE = 0; // default, used to query if any alarm is active NOSPACE = 1; // space quota is exhausted CORRUPT = 2 [(versionpb.etcd_version_enum_value)="3.3"]; // kv store corruption detected } message AlarmRequest { option (versionpb.etcd_version_msg) = "3.0"; enum AlarmAction { option (versionpb.etcd_version_enum) = "3.0"; GET = 0; ACTIVATE = 1; DEACTIVATE = 2; } // action is the kind of alarm request to issue. The action // may GET alarm statuses, ACTIVATE an alarm, or DEACTIVATE a // raised alarm. AlarmAction action = 1; // memberID is the ID of the member associated with the alarm. If memberID is 0, the // alarm request covers all members. uint64 memberID = 2; // alarm is the type of alarm to consider for this request. AlarmType alarm = 3; } message AlarmMember { option (versionpb.etcd_version_msg) = "3.0"; // memberID is the ID of the member associated with the raised alarm. uint64 memberID = 1; // alarm is the type of alarm which has been raised. AlarmType alarm = 2; } message AlarmResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // alarms is a list of alarms associated with the alarm request. repeated AlarmMember alarms = 2; } message DowngradeRequest { option (versionpb.etcd_version_msg) = "3.5"; enum DowngradeAction { option (versionpb.etcd_version_enum) = "3.5"; VALIDATE = 0; ENABLE = 1; CANCEL = 2; } // action is the kind of downgrade request to issue. The action may // VALIDATE the target version, DOWNGRADE the cluster version, // or CANCEL the current downgrading job. DowngradeAction action = 1; // version is the target version to downgrade. string version = 2; } message DowngradeResponse { option (versionpb.etcd_version_msg) = "3.5"; ResponseHeader header = 1; // version is the current cluster version. string version = 2; } // DowngradeVersionTestRequest is used for test only. The version in // this request will be read as the WAL record version.If the downgrade // target version is less than this version, then the downgrade(online) // or migration(offline) isn't safe, so shouldn't be allowed. message DowngradeVersionTestRequest { option (versionpb.etcd_version_msg) = "3.6"; string ver = 1; } message StatusRequest { option (versionpb.etcd_version_msg) = "3.0"; } message StatusResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // version is the cluster protocol version used by the responding member. string version = 2; // dbSize is the size of the backend database physically allocated, in bytes, of the responding member. int64 dbSize = 3; // leader is the member ID which the responding member believes is the current leader. uint64 leader = 4; // raftIndex is the current raft committed index of the responding member. uint64 raftIndex = 5; // raftTerm is the current raft term of the responding member. uint64 raftTerm = 6; // raftAppliedIndex is the current raft applied index of the responding member. uint64 raftAppliedIndex = 7 [(versionpb.etcd_version_field)="3.4"]; // errors contains alarm/health information and status. repeated string errors = 8 [(versionpb.etcd_version_field)="3.4"]; // dbSizeInUse is the size of the backend database logically in use, in bytes, of the responding member. int64 dbSizeInUse = 9 [(versionpb.etcd_version_field)="3.4"]; // isLearner indicates if the member is raft learner. bool isLearner = 10 [(versionpb.etcd_version_field)="3.4"]; // storageVersion is the version of the db file. It might be updated with delay in relationship to the target cluster version. string storageVersion = 11 [(versionpb.etcd_version_field)="3.6"]; // dbSizeQuota is the configured etcd storage quota in bytes (the value passed to etcd instance by flag --quota-backend-bytes) int64 dbSizeQuota = 12 [(versionpb.etcd_version_field)="3.6"]; // downgradeInfo indicates if there is downgrade process. DowngradeInfo downgradeInfo = 13 [(versionpb.etcd_version_field)="3.6"]; } message DowngradeInfo { // enabled indicates whether the cluster is enabled to downgrade. bool enabled = 1; // targetVersion is the target downgrade version. string targetVersion = 2; } message AuthEnableRequest { option (versionpb.etcd_version_msg) = "3.0"; } message AuthDisableRequest { option (versionpb.etcd_version_msg) = "3.0"; } message AuthStatusRequest { option (versionpb.etcd_version_msg) = "3.5"; } message AuthenticateRequest { option (versionpb.etcd_version_msg) = "3.0"; string name = 1; string password = 2; } message AuthUserAddRequest { option (versionpb.etcd_version_msg) = "3.0"; string name = 1; string password = 2; authpb.UserAddOptions options = 3 [(versionpb.etcd_version_field)="3.4"]; string hashedPassword = 4 [(versionpb.etcd_version_field)="3.5"]; } message AuthUserGetRequest { option (versionpb.etcd_version_msg) = "3.0"; string name = 1; } message AuthUserDeleteRequest { option (versionpb.etcd_version_msg) = "3.0"; // name is the name of the user to delete. string name = 1; } message AuthUserChangePasswordRequest { option (versionpb.etcd_version_msg) = "3.0"; // name is the name of the user whose password is being changed. string name = 1; // password is the new password for the user. Note that this field will be removed in the API layer. string password = 2; // hashedPassword is the new password for the user. Note that this field will be initialized in the API layer. string hashedPassword = 3 [(versionpb.etcd_version_field)="3.5"]; } message AuthUserGrantRoleRequest { option (versionpb.etcd_version_msg) = "3.0"; // user is the name of the user which should be granted a given role. string user = 1; // role is the name of the role to grant to the user. string role = 2; } message AuthUserRevokeRoleRequest { option (versionpb.etcd_version_msg) = "3.0"; string name = 1; string role = 2; } message AuthRoleAddRequest { option (versionpb.etcd_version_msg) = "3.0"; // name is the name of the role to add to the authentication system. string name = 1; } message AuthRoleGetRequest { option (versionpb.etcd_version_msg) = "3.0"; string role = 1; } message AuthUserListRequest { option (versionpb.etcd_version_msg) = "3.0"; } message AuthRoleListRequest { option (versionpb.etcd_version_msg) = "3.0"; } message AuthRoleDeleteRequest { option (versionpb.etcd_version_msg) = "3.0"; string role = 1; } message AuthRoleGrantPermissionRequest { option (versionpb.etcd_version_msg) = "3.0"; // name is the name of the role which will be granted the permission. string name = 1; // perm is the permission to grant to the role. authpb.Permission perm = 2; } message AuthRoleRevokePermissionRequest { option (versionpb.etcd_version_msg) = "3.0"; string role = 1; bytes key = 2; bytes range_end = 3; } message AuthEnableResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message AuthDisableResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message AuthStatusResponse { option (versionpb.etcd_version_msg) = "3.5"; ResponseHeader header = 1; bool enabled = 2; // authRevision is the current revision of auth store uint64 authRevision = 3; } message AuthenticateResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; // token is an authorized token that can be used in succeeding RPCs string token = 2; } message AuthUserAddResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message AuthUserGetResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; repeated string roles = 2; } message AuthUserDeleteResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message AuthUserChangePasswordResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message AuthUserGrantRoleResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message AuthUserRevokeRoleResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message AuthRoleAddResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message AuthRoleGetResponse { ResponseHeader header = 1 [(versionpb.etcd_version_field)="3.0"]; repeated authpb.Permission perm = 2 [(versionpb.etcd_version_field)="3.0"]; } message AuthRoleListResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; repeated string roles = 2; } message AuthUserListResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; repeated string users = 2; } message AuthRoleDeleteResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message AuthRoleGrantPermissionResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } message AuthRoleRevokePermissionResponse { option (versionpb.etcd_version_msg) = "3.0"; ResponseHeader header = 1; } ================================================ FILE: api/etcdserverpb/rpc_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.1 // - protoc v3.20.3 // source: rpc.proto package etcdserverpb import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.64.0 or later. const _ = grpc.SupportPackageIsVersion9 const ( KV_Range_FullMethodName = "/etcdserverpb.KV/Range" KV_Put_FullMethodName = "/etcdserverpb.KV/Put" KV_DeleteRange_FullMethodName = "/etcdserverpb.KV/DeleteRange" KV_Txn_FullMethodName = "/etcdserverpb.KV/Txn" KV_Compact_FullMethodName = "/etcdserverpb.KV/Compact" ) // KVClient is the client API for KV service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type KVClient interface { // Range gets the keys in the range from the key-value store. Range(ctx context.Context, in *RangeRequest, opts ...grpc.CallOption) (*RangeResponse, error) // Put puts the given key into the key-value store. // A put request increments the revision of the key-value store // and generates one event in the event history. Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error) // DeleteRange deletes the given range from the key-value store. // A delete request increments the revision of the key-value store // and generates a delete event in the event history for every deleted key. DeleteRange(ctx context.Context, in *DeleteRangeRequest, opts ...grpc.CallOption) (*DeleteRangeResponse, error) // Txn processes multiple requests in a single transaction. // A txn request increments the revision of the key-value store // and generates events with the same revision for every completed request. // It is not allowed to modify the same key several times within one txn. Txn(ctx context.Context, in *TxnRequest, opts ...grpc.CallOption) (*TxnResponse, error) // Compact compacts the event history in the etcd key-value store. The key-value // store should be periodically compacted or the event history will continue to grow // indefinitely. Compact(ctx context.Context, in *CompactionRequest, opts ...grpc.CallOption) (*CompactionResponse, error) } type kVClient struct { cc grpc.ClientConnInterface } func NewKVClient(cc grpc.ClientConnInterface) KVClient { return &kVClient{cc} } func (c *kVClient) Range(ctx context.Context, in *RangeRequest, opts ...grpc.CallOption) (*RangeResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RangeResponse) err := c.cc.Invoke(ctx, KV_Range_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *kVClient) Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PutResponse) err := c.cc.Invoke(ctx, KV_Put_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *kVClient) DeleteRange(ctx context.Context, in *DeleteRangeRequest, opts ...grpc.CallOption) (*DeleteRangeResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(DeleteRangeResponse) err := c.cc.Invoke(ctx, KV_DeleteRange_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *kVClient) Txn(ctx context.Context, in *TxnRequest, opts ...grpc.CallOption) (*TxnResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(TxnResponse) err := c.cc.Invoke(ctx, KV_Txn_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *kVClient) Compact(ctx context.Context, in *CompactionRequest, opts ...grpc.CallOption) (*CompactionResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CompactionResponse) err := c.cc.Invoke(ctx, KV_Compact_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } // KVServer is the server API for KV service. // All implementations must embed UnimplementedKVServer // for forward compatibility. type KVServer interface { // Range gets the keys in the range from the key-value store. Range(context.Context, *RangeRequest) (*RangeResponse, error) // Put puts the given key into the key-value store. // A put request increments the revision of the key-value store // and generates one event in the event history. Put(context.Context, *PutRequest) (*PutResponse, error) // DeleteRange deletes the given range from the key-value store. // A delete request increments the revision of the key-value store // and generates a delete event in the event history for every deleted key. DeleteRange(context.Context, *DeleteRangeRequest) (*DeleteRangeResponse, error) // Txn processes multiple requests in a single transaction. // A txn request increments the revision of the key-value store // and generates events with the same revision for every completed request. // It is not allowed to modify the same key several times within one txn. Txn(context.Context, *TxnRequest) (*TxnResponse, error) // Compact compacts the event history in the etcd key-value store. The key-value // store should be periodically compacted or the event history will continue to grow // indefinitely. Compact(context.Context, *CompactionRequest) (*CompactionResponse, error) mustEmbedUnimplementedKVServer() } // UnimplementedKVServer must be embedded to have // forward compatible implementations. // // NOTE: this should be embedded by value instead of pointer to avoid a nil // pointer dereference when methods are called. type UnimplementedKVServer struct{} func (UnimplementedKVServer) Range(context.Context, *RangeRequest) (*RangeResponse, error) { return nil, status.Error(codes.Unimplemented, "method Range not implemented") } func (UnimplementedKVServer) Put(context.Context, *PutRequest) (*PutResponse, error) { return nil, status.Error(codes.Unimplemented, "method Put not implemented") } func (UnimplementedKVServer) DeleteRange(context.Context, *DeleteRangeRequest) (*DeleteRangeResponse, error) { return nil, status.Error(codes.Unimplemented, "method DeleteRange not implemented") } func (UnimplementedKVServer) Txn(context.Context, *TxnRequest) (*TxnResponse, error) { return nil, status.Error(codes.Unimplemented, "method Txn not implemented") } func (UnimplementedKVServer) Compact(context.Context, *CompactionRequest) (*CompactionResponse, error) { return nil, status.Error(codes.Unimplemented, "method Compact not implemented") } func (UnimplementedKVServer) mustEmbedUnimplementedKVServer() {} func (UnimplementedKVServer) testEmbeddedByValue() {} // UnsafeKVServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to KVServer will // result in compilation errors. type UnsafeKVServer interface { mustEmbedUnimplementedKVServer() } func RegisterKVServer(s grpc.ServiceRegistrar, srv KVServer) { // If the following call panics, it indicates UnimplementedKVServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { t.testEmbeddedByValue() } s.RegisterService(&KV_ServiceDesc, srv) } func _KV_Range_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RangeRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(KVServer).Range(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: KV_Range_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(KVServer).Range(ctx, req.(*RangeRequest)) } return interceptor(ctx, in, info, handler) } func _KV_Put_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(PutRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(KVServer).Put(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: KV_Put_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(KVServer).Put(ctx, req.(*PutRequest)) } return interceptor(ctx, in, info, handler) } func _KV_DeleteRange_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DeleteRangeRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(KVServer).DeleteRange(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: KV_DeleteRange_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(KVServer).DeleteRange(ctx, req.(*DeleteRangeRequest)) } return interceptor(ctx, in, info, handler) } func _KV_Txn_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(TxnRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(KVServer).Txn(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: KV_Txn_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(KVServer).Txn(ctx, req.(*TxnRequest)) } return interceptor(ctx, in, info, handler) } func _KV_Compact_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CompactionRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(KVServer).Compact(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: KV_Compact_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(KVServer).Compact(ctx, req.(*CompactionRequest)) } return interceptor(ctx, in, info, handler) } // KV_ServiceDesc is the grpc.ServiceDesc for KV service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var KV_ServiceDesc = grpc.ServiceDesc{ ServiceName: "etcdserverpb.KV", HandlerType: (*KVServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "Range", Handler: _KV_Range_Handler, }, { MethodName: "Put", Handler: _KV_Put_Handler, }, { MethodName: "DeleteRange", Handler: _KV_DeleteRange_Handler, }, { MethodName: "Txn", Handler: _KV_Txn_Handler, }, { MethodName: "Compact", Handler: _KV_Compact_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "rpc.proto", } const ( Watch_Watch_FullMethodName = "/etcdserverpb.Watch/Watch" ) // WatchClient is the client API for Watch service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type WatchClient interface { // Watch watches for events happening or that have happened. Both input and output // are streams; the input stream is for creating and canceling watchers and the output // stream sends events. One watch RPC can watch on multiple key ranges, streaming events // for several watches at once. The entire event history can be watched starting from the // last compaction revision. Watch(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[WatchRequest, WatchResponse], error) } type watchClient struct { cc grpc.ClientConnInterface } func NewWatchClient(cc grpc.ClientConnInterface) WatchClient { return &watchClient{cc} } func (c *watchClient) Watch(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[WatchRequest, WatchResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Watch_ServiceDesc.Streams[0], Watch_Watch_FullMethodName, cOpts...) if err != nil { return nil, err } x := &grpc.GenericClientStream[WatchRequest, WatchResponse]{ClientStream: stream} return x, nil } // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type Watch_WatchClient = grpc.BidiStreamingClient[WatchRequest, WatchResponse] // WatchServer is the server API for Watch service. // All implementations must embed UnimplementedWatchServer // for forward compatibility. type WatchServer interface { // Watch watches for events happening or that have happened. Both input and output // are streams; the input stream is for creating and canceling watchers and the output // stream sends events. One watch RPC can watch on multiple key ranges, streaming events // for several watches at once. The entire event history can be watched starting from the // last compaction revision. Watch(grpc.BidiStreamingServer[WatchRequest, WatchResponse]) error mustEmbedUnimplementedWatchServer() } // UnimplementedWatchServer must be embedded to have // forward compatible implementations. // // NOTE: this should be embedded by value instead of pointer to avoid a nil // pointer dereference when methods are called. type UnimplementedWatchServer struct{} func (UnimplementedWatchServer) Watch(grpc.BidiStreamingServer[WatchRequest, WatchResponse]) error { return status.Error(codes.Unimplemented, "method Watch not implemented") } func (UnimplementedWatchServer) mustEmbedUnimplementedWatchServer() {} func (UnimplementedWatchServer) testEmbeddedByValue() {} // UnsafeWatchServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to WatchServer will // result in compilation errors. type UnsafeWatchServer interface { mustEmbedUnimplementedWatchServer() } func RegisterWatchServer(s grpc.ServiceRegistrar, srv WatchServer) { // If the following call panics, it indicates UnimplementedWatchServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { t.testEmbeddedByValue() } s.RegisterService(&Watch_ServiceDesc, srv) } func _Watch_Watch_Handler(srv interface{}, stream grpc.ServerStream) error { return srv.(WatchServer).Watch(&grpc.GenericServerStream[WatchRequest, WatchResponse]{ServerStream: stream}) } // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type Watch_WatchServer = grpc.BidiStreamingServer[WatchRequest, WatchResponse] // Watch_ServiceDesc is the grpc.ServiceDesc for Watch service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var Watch_ServiceDesc = grpc.ServiceDesc{ ServiceName: "etcdserverpb.Watch", HandlerType: (*WatchServer)(nil), Methods: []grpc.MethodDesc{}, Streams: []grpc.StreamDesc{ { StreamName: "Watch", Handler: _Watch_Watch_Handler, ServerStreams: true, ClientStreams: true, }, }, Metadata: "rpc.proto", } const ( Lease_LeaseGrant_FullMethodName = "/etcdserverpb.Lease/LeaseGrant" Lease_LeaseRevoke_FullMethodName = "/etcdserverpb.Lease/LeaseRevoke" Lease_LeaseKeepAlive_FullMethodName = "/etcdserverpb.Lease/LeaseKeepAlive" Lease_LeaseTimeToLive_FullMethodName = "/etcdserverpb.Lease/LeaseTimeToLive" Lease_LeaseLeases_FullMethodName = "/etcdserverpb.Lease/LeaseLeases" ) // LeaseClient is the client API for Lease service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type LeaseClient interface { // LeaseGrant creates a lease which expires if the server does not receive a keepAlive // within a given time to live period. All keys attached to the lease will be expired and // deleted if the lease expires. Each expired key generates a delete event in the event history. LeaseGrant(ctx context.Context, in *LeaseGrantRequest, opts ...grpc.CallOption) (*LeaseGrantResponse, error) // LeaseRevoke revokes a lease. All keys attached to the lease will expire and be deleted. LeaseRevoke(ctx context.Context, in *LeaseRevokeRequest, opts ...grpc.CallOption) (*LeaseRevokeResponse, error) // LeaseKeepAlive keeps the lease alive by streaming keep alive requests from the client // to the server and streaming keep alive responses from the server to the client. LeaseKeepAlive(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[LeaseKeepAliveRequest, LeaseKeepAliveResponse], error) // LeaseTimeToLive retrieves lease information. LeaseTimeToLive(ctx context.Context, in *LeaseTimeToLiveRequest, opts ...grpc.CallOption) (*LeaseTimeToLiveResponse, error) // LeaseLeases lists all existing leases. LeaseLeases(ctx context.Context, in *LeaseLeasesRequest, opts ...grpc.CallOption) (*LeaseLeasesResponse, error) } type leaseClient struct { cc grpc.ClientConnInterface } func NewLeaseClient(cc grpc.ClientConnInterface) LeaseClient { return &leaseClient{cc} } func (c *leaseClient) LeaseGrant(ctx context.Context, in *LeaseGrantRequest, opts ...grpc.CallOption) (*LeaseGrantResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(LeaseGrantResponse) err := c.cc.Invoke(ctx, Lease_LeaseGrant_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *leaseClient) LeaseRevoke(ctx context.Context, in *LeaseRevokeRequest, opts ...grpc.CallOption) (*LeaseRevokeResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(LeaseRevokeResponse) err := c.cc.Invoke(ctx, Lease_LeaseRevoke_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *leaseClient) LeaseKeepAlive(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[LeaseKeepAliveRequest, LeaseKeepAliveResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Lease_ServiceDesc.Streams[0], Lease_LeaseKeepAlive_FullMethodName, cOpts...) if err != nil { return nil, err } x := &grpc.GenericClientStream[LeaseKeepAliveRequest, LeaseKeepAliveResponse]{ClientStream: stream} return x, nil } // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type Lease_LeaseKeepAliveClient = grpc.BidiStreamingClient[LeaseKeepAliveRequest, LeaseKeepAliveResponse] func (c *leaseClient) LeaseTimeToLive(ctx context.Context, in *LeaseTimeToLiveRequest, opts ...grpc.CallOption) (*LeaseTimeToLiveResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(LeaseTimeToLiveResponse) err := c.cc.Invoke(ctx, Lease_LeaseTimeToLive_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *leaseClient) LeaseLeases(ctx context.Context, in *LeaseLeasesRequest, opts ...grpc.CallOption) (*LeaseLeasesResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(LeaseLeasesResponse) err := c.cc.Invoke(ctx, Lease_LeaseLeases_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } // LeaseServer is the server API for Lease service. // All implementations must embed UnimplementedLeaseServer // for forward compatibility. type LeaseServer interface { // LeaseGrant creates a lease which expires if the server does not receive a keepAlive // within a given time to live period. All keys attached to the lease will be expired and // deleted if the lease expires. Each expired key generates a delete event in the event history. LeaseGrant(context.Context, *LeaseGrantRequest) (*LeaseGrantResponse, error) // LeaseRevoke revokes a lease. All keys attached to the lease will expire and be deleted. LeaseRevoke(context.Context, *LeaseRevokeRequest) (*LeaseRevokeResponse, error) // LeaseKeepAlive keeps the lease alive by streaming keep alive requests from the client // to the server and streaming keep alive responses from the server to the client. LeaseKeepAlive(grpc.BidiStreamingServer[LeaseKeepAliveRequest, LeaseKeepAliveResponse]) error // LeaseTimeToLive retrieves lease information. LeaseTimeToLive(context.Context, *LeaseTimeToLiveRequest) (*LeaseTimeToLiveResponse, error) // LeaseLeases lists all existing leases. LeaseLeases(context.Context, *LeaseLeasesRequest) (*LeaseLeasesResponse, error) mustEmbedUnimplementedLeaseServer() } // UnimplementedLeaseServer must be embedded to have // forward compatible implementations. // // NOTE: this should be embedded by value instead of pointer to avoid a nil // pointer dereference when methods are called. type UnimplementedLeaseServer struct{} func (UnimplementedLeaseServer) LeaseGrant(context.Context, *LeaseGrantRequest) (*LeaseGrantResponse, error) { return nil, status.Error(codes.Unimplemented, "method LeaseGrant not implemented") } func (UnimplementedLeaseServer) LeaseRevoke(context.Context, *LeaseRevokeRequest) (*LeaseRevokeResponse, error) { return nil, status.Error(codes.Unimplemented, "method LeaseRevoke not implemented") } func (UnimplementedLeaseServer) LeaseKeepAlive(grpc.BidiStreamingServer[LeaseKeepAliveRequest, LeaseKeepAliveResponse]) error { return status.Error(codes.Unimplemented, "method LeaseKeepAlive not implemented") } func (UnimplementedLeaseServer) LeaseTimeToLive(context.Context, *LeaseTimeToLiveRequest) (*LeaseTimeToLiveResponse, error) { return nil, status.Error(codes.Unimplemented, "method LeaseTimeToLive not implemented") } func (UnimplementedLeaseServer) LeaseLeases(context.Context, *LeaseLeasesRequest) (*LeaseLeasesResponse, error) { return nil, status.Error(codes.Unimplemented, "method LeaseLeases not implemented") } func (UnimplementedLeaseServer) mustEmbedUnimplementedLeaseServer() {} func (UnimplementedLeaseServer) testEmbeddedByValue() {} // UnsafeLeaseServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to LeaseServer will // result in compilation errors. type UnsafeLeaseServer interface { mustEmbedUnimplementedLeaseServer() } func RegisterLeaseServer(s grpc.ServiceRegistrar, srv LeaseServer) { // If the following call panics, it indicates UnimplementedLeaseServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { t.testEmbeddedByValue() } s.RegisterService(&Lease_ServiceDesc, srv) } func _Lease_LeaseGrant_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(LeaseGrantRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(LeaseServer).LeaseGrant(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Lease_LeaseGrant_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(LeaseServer).LeaseGrant(ctx, req.(*LeaseGrantRequest)) } return interceptor(ctx, in, info, handler) } func _Lease_LeaseRevoke_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(LeaseRevokeRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(LeaseServer).LeaseRevoke(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Lease_LeaseRevoke_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(LeaseServer).LeaseRevoke(ctx, req.(*LeaseRevokeRequest)) } return interceptor(ctx, in, info, handler) } func _Lease_LeaseKeepAlive_Handler(srv interface{}, stream grpc.ServerStream) error { return srv.(LeaseServer).LeaseKeepAlive(&grpc.GenericServerStream[LeaseKeepAliveRequest, LeaseKeepAliveResponse]{ServerStream: stream}) } // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type Lease_LeaseKeepAliveServer = grpc.BidiStreamingServer[LeaseKeepAliveRequest, LeaseKeepAliveResponse] func _Lease_LeaseTimeToLive_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(LeaseTimeToLiveRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(LeaseServer).LeaseTimeToLive(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Lease_LeaseTimeToLive_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(LeaseServer).LeaseTimeToLive(ctx, req.(*LeaseTimeToLiveRequest)) } return interceptor(ctx, in, info, handler) } func _Lease_LeaseLeases_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(LeaseLeasesRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(LeaseServer).LeaseLeases(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Lease_LeaseLeases_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(LeaseServer).LeaseLeases(ctx, req.(*LeaseLeasesRequest)) } return interceptor(ctx, in, info, handler) } // Lease_ServiceDesc is the grpc.ServiceDesc for Lease service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var Lease_ServiceDesc = grpc.ServiceDesc{ ServiceName: "etcdserverpb.Lease", HandlerType: (*LeaseServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "LeaseGrant", Handler: _Lease_LeaseGrant_Handler, }, { MethodName: "LeaseRevoke", Handler: _Lease_LeaseRevoke_Handler, }, { MethodName: "LeaseTimeToLive", Handler: _Lease_LeaseTimeToLive_Handler, }, { MethodName: "LeaseLeases", Handler: _Lease_LeaseLeases_Handler, }, }, Streams: []grpc.StreamDesc{ { StreamName: "LeaseKeepAlive", Handler: _Lease_LeaseKeepAlive_Handler, ServerStreams: true, ClientStreams: true, }, }, Metadata: "rpc.proto", } const ( Cluster_MemberAdd_FullMethodName = "/etcdserverpb.Cluster/MemberAdd" Cluster_MemberRemove_FullMethodName = "/etcdserverpb.Cluster/MemberRemove" Cluster_MemberUpdate_FullMethodName = "/etcdserverpb.Cluster/MemberUpdate" Cluster_MemberList_FullMethodName = "/etcdserverpb.Cluster/MemberList" Cluster_MemberPromote_FullMethodName = "/etcdserverpb.Cluster/MemberPromote" ) // ClusterClient is the client API for Cluster service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type ClusterClient interface { // MemberAdd adds a member into the cluster. MemberAdd(ctx context.Context, in *MemberAddRequest, opts ...grpc.CallOption) (*MemberAddResponse, error) // MemberRemove removes an existing member from the cluster. MemberRemove(ctx context.Context, in *MemberRemoveRequest, opts ...grpc.CallOption) (*MemberRemoveResponse, error) // MemberUpdate updates the member configuration. MemberUpdate(ctx context.Context, in *MemberUpdateRequest, opts ...grpc.CallOption) (*MemberUpdateResponse, error) // MemberList lists all the members in the cluster. MemberList(ctx context.Context, in *MemberListRequest, opts ...grpc.CallOption) (*MemberListResponse, error) // MemberPromote promotes a member from raft learner (non-voting) to raft voting member. MemberPromote(ctx context.Context, in *MemberPromoteRequest, opts ...grpc.CallOption) (*MemberPromoteResponse, error) } type clusterClient struct { cc grpc.ClientConnInterface } func NewClusterClient(cc grpc.ClientConnInterface) ClusterClient { return &clusterClient{cc} } func (c *clusterClient) MemberAdd(ctx context.Context, in *MemberAddRequest, opts ...grpc.CallOption) (*MemberAddResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MemberAddResponse) err := c.cc.Invoke(ctx, Cluster_MemberAdd_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *clusterClient) MemberRemove(ctx context.Context, in *MemberRemoveRequest, opts ...grpc.CallOption) (*MemberRemoveResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MemberRemoveResponse) err := c.cc.Invoke(ctx, Cluster_MemberRemove_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *clusterClient) MemberUpdate(ctx context.Context, in *MemberUpdateRequest, opts ...grpc.CallOption) (*MemberUpdateResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MemberUpdateResponse) err := c.cc.Invoke(ctx, Cluster_MemberUpdate_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *clusterClient) MemberList(ctx context.Context, in *MemberListRequest, opts ...grpc.CallOption) (*MemberListResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MemberListResponse) err := c.cc.Invoke(ctx, Cluster_MemberList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *clusterClient) MemberPromote(ctx context.Context, in *MemberPromoteRequest, opts ...grpc.CallOption) (*MemberPromoteResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MemberPromoteResponse) err := c.cc.Invoke(ctx, Cluster_MemberPromote_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } // ClusterServer is the server API for Cluster service. // All implementations must embed UnimplementedClusterServer // for forward compatibility. type ClusterServer interface { // MemberAdd adds a member into the cluster. MemberAdd(context.Context, *MemberAddRequest) (*MemberAddResponse, error) // MemberRemove removes an existing member from the cluster. MemberRemove(context.Context, *MemberRemoveRequest) (*MemberRemoveResponse, error) // MemberUpdate updates the member configuration. MemberUpdate(context.Context, *MemberUpdateRequest) (*MemberUpdateResponse, error) // MemberList lists all the members in the cluster. MemberList(context.Context, *MemberListRequest) (*MemberListResponse, error) // MemberPromote promotes a member from raft learner (non-voting) to raft voting member. MemberPromote(context.Context, *MemberPromoteRequest) (*MemberPromoteResponse, error) mustEmbedUnimplementedClusterServer() } // UnimplementedClusterServer must be embedded to have // forward compatible implementations. // // NOTE: this should be embedded by value instead of pointer to avoid a nil // pointer dereference when methods are called. type UnimplementedClusterServer struct{} func (UnimplementedClusterServer) MemberAdd(context.Context, *MemberAddRequest) (*MemberAddResponse, error) { return nil, status.Error(codes.Unimplemented, "method MemberAdd not implemented") } func (UnimplementedClusterServer) MemberRemove(context.Context, *MemberRemoveRequest) (*MemberRemoveResponse, error) { return nil, status.Error(codes.Unimplemented, "method MemberRemove not implemented") } func (UnimplementedClusterServer) MemberUpdate(context.Context, *MemberUpdateRequest) (*MemberUpdateResponse, error) { return nil, status.Error(codes.Unimplemented, "method MemberUpdate not implemented") } func (UnimplementedClusterServer) MemberList(context.Context, *MemberListRequest) (*MemberListResponse, error) { return nil, status.Error(codes.Unimplemented, "method MemberList not implemented") } func (UnimplementedClusterServer) MemberPromote(context.Context, *MemberPromoteRequest) (*MemberPromoteResponse, error) { return nil, status.Error(codes.Unimplemented, "method MemberPromote not implemented") } func (UnimplementedClusterServer) mustEmbedUnimplementedClusterServer() {} func (UnimplementedClusterServer) testEmbeddedByValue() {} // UnsafeClusterServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ClusterServer will // result in compilation errors. type UnsafeClusterServer interface { mustEmbedUnimplementedClusterServer() } func RegisterClusterServer(s grpc.ServiceRegistrar, srv ClusterServer) { // If the following call panics, it indicates UnimplementedClusterServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { t.testEmbeddedByValue() } s.RegisterService(&Cluster_ServiceDesc, srv) } func _Cluster_MemberAdd_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MemberAddRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ClusterServer).MemberAdd(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Cluster_MemberAdd_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ClusterServer).MemberAdd(ctx, req.(*MemberAddRequest)) } return interceptor(ctx, in, info, handler) } func _Cluster_MemberRemove_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MemberRemoveRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ClusterServer).MemberRemove(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Cluster_MemberRemove_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ClusterServer).MemberRemove(ctx, req.(*MemberRemoveRequest)) } return interceptor(ctx, in, info, handler) } func _Cluster_MemberUpdate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MemberUpdateRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ClusterServer).MemberUpdate(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Cluster_MemberUpdate_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ClusterServer).MemberUpdate(ctx, req.(*MemberUpdateRequest)) } return interceptor(ctx, in, info, handler) } func _Cluster_MemberList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MemberListRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ClusterServer).MemberList(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Cluster_MemberList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ClusterServer).MemberList(ctx, req.(*MemberListRequest)) } return interceptor(ctx, in, info, handler) } func _Cluster_MemberPromote_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MemberPromoteRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ClusterServer).MemberPromote(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Cluster_MemberPromote_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ClusterServer).MemberPromote(ctx, req.(*MemberPromoteRequest)) } return interceptor(ctx, in, info, handler) } // Cluster_ServiceDesc is the grpc.ServiceDesc for Cluster service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var Cluster_ServiceDesc = grpc.ServiceDesc{ ServiceName: "etcdserverpb.Cluster", HandlerType: (*ClusterServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "MemberAdd", Handler: _Cluster_MemberAdd_Handler, }, { MethodName: "MemberRemove", Handler: _Cluster_MemberRemove_Handler, }, { MethodName: "MemberUpdate", Handler: _Cluster_MemberUpdate_Handler, }, { MethodName: "MemberList", Handler: _Cluster_MemberList_Handler, }, { MethodName: "MemberPromote", Handler: _Cluster_MemberPromote_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "rpc.proto", } const ( Maintenance_Alarm_FullMethodName = "/etcdserverpb.Maintenance/Alarm" Maintenance_Status_FullMethodName = "/etcdserverpb.Maintenance/Status" Maintenance_Defragment_FullMethodName = "/etcdserverpb.Maintenance/Defragment" Maintenance_Hash_FullMethodName = "/etcdserverpb.Maintenance/Hash" Maintenance_HashKV_FullMethodName = "/etcdserverpb.Maintenance/HashKV" Maintenance_Snapshot_FullMethodName = "/etcdserverpb.Maintenance/Snapshot" Maintenance_MoveLeader_FullMethodName = "/etcdserverpb.Maintenance/MoveLeader" Maintenance_Downgrade_FullMethodName = "/etcdserverpb.Maintenance/Downgrade" ) // MaintenanceClient is the client API for Maintenance service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type MaintenanceClient interface { // Alarm activates, deactivates, and queries alarms regarding cluster health. Alarm(ctx context.Context, in *AlarmRequest, opts ...grpc.CallOption) (*AlarmResponse, error) // Status gets the status of the member. Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) // Defragment defragments a member's backend database to recover storage space. Defragment(ctx context.Context, in *DefragmentRequest, opts ...grpc.CallOption) (*DefragmentResponse, error) // Hash computes the hash of whole backend keyspace, // including key, lease, and other buckets in storage. // This is designed for testing ONLY! // Do not rely on this in production with ongoing transactions, // since Hash operation does not hold MVCC locks. // Use "HashKV" API instead for "key" bucket consistency checks. Hash(ctx context.Context, in *HashRequest, opts ...grpc.CallOption) (*HashResponse, error) // HashKV computes the hash of all MVCC keys up to a given revision. // It only iterates "key" bucket in backend storage. HashKV(ctx context.Context, in *HashKVRequest, opts ...grpc.CallOption) (*HashKVResponse, error) // Snapshot sends a snapshot of the entire backend from a member over a stream to a client. Snapshot(ctx context.Context, in *SnapshotRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SnapshotResponse], error) // MoveLeader requests current leader node to transfer its leadership to transferee. MoveLeader(ctx context.Context, in *MoveLeaderRequest, opts ...grpc.CallOption) (*MoveLeaderResponse, error) // Downgrade requests downgrades, verifies feasibility or cancels downgrade // on the cluster version. // Supported since etcd 3.5. Downgrade(ctx context.Context, in *DowngradeRequest, opts ...grpc.CallOption) (*DowngradeResponse, error) } type maintenanceClient struct { cc grpc.ClientConnInterface } func NewMaintenanceClient(cc grpc.ClientConnInterface) MaintenanceClient { return &maintenanceClient{cc} } func (c *maintenanceClient) Alarm(ctx context.Context, in *AlarmRequest, opts ...grpc.CallOption) (*AlarmResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AlarmResponse) err := c.cc.Invoke(ctx, Maintenance_Alarm_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *maintenanceClient) Status(ctx context.Context, in *StatusRequest, opts ...grpc.CallOption) (*StatusResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(StatusResponse) err := c.cc.Invoke(ctx, Maintenance_Status_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *maintenanceClient) Defragment(ctx context.Context, in *DefragmentRequest, opts ...grpc.CallOption) (*DefragmentResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(DefragmentResponse) err := c.cc.Invoke(ctx, Maintenance_Defragment_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *maintenanceClient) Hash(ctx context.Context, in *HashRequest, opts ...grpc.CallOption) (*HashResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(HashResponse) err := c.cc.Invoke(ctx, Maintenance_Hash_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *maintenanceClient) HashKV(ctx context.Context, in *HashKVRequest, opts ...grpc.CallOption) (*HashKVResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(HashKVResponse) err := c.cc.Invoke(ctx, Maintenance_HashKV_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *maintenanceClient) Snapshot(ctx context.Context, in *SnapshotRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SnapshotResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Maintenance_ServiceDesc.Streams[0], Maintenance_Snapshot_FullMethodName, cOpts...) if err != nil { return nil, err } x := &grpc.GenericClientStream[SnapshotRequest, SnapshotResponse]{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } if err := x.ClientStream.CloseSend(); err != nil { return nil, err } return x, nil } // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type Maintenance_SnapshotClient = grpc.ServerStreamingClient[SnapshotResponse] func (c *maintenanceClient) MoveLeader(ctx context.Context, in *MoveLeaderRequest, opts ...grpc.CallOption) (*MoveLeaderResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MoveLeaderResponse) err := c.cc.Invoke(ctx, Maintenance_MoveLeader_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *maintenanceClient) Downgrade(ctx context.Context, in *DowngradeRequest, opts ...grpc.CallOption) (*DowngradeResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(DowngradeResponse) err := c.cc.Invoke(ctx, Maintenance_Downgrade_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } // MaintenanceServer is the server API for Maintenance service. // All implementations must embed UnimplementedMaintenanceServer // for forward compatibility. type MaintenanceServer interface { // Alarm activates, deactivates, and queries alarms regarding cluster health. Alarm(context.Context, *AlarmRequest) (*AlarmResponse, error) // Status gets the status of the member. Status(context.Context, *StatusRequest) (*StatusResponse, error) // Defragment defragments a member's backend database to recover storage space. Defragment(context.Context, *DefragmentRequest) (*DefragmentResponse, error) // Hash computes the hash of whole backend keyspace, // including key, lease, and other buckets in storage. // This is designed for testing ONLY! // Do not rely on this in production with ongoing transactions, // since Hash operation does not hold MVCC locks. // Use "HashKV" API instead for "key" bucket consistency checks. Hash(context.Context, *HashRequest) (*HashResponse, error) // HashKV computes the hash of all MVCC keys up to a given revision. // It only iterates "key" bucket in backend storage. HashKV(context.Context, *HashKVRequest) (*HashKVResponse, error) // Snapshot sends a snapshot of the entire backend from a member over a stream to a client. Snapshot(*SnapshotRequest, grpc.ServerStreamingServer[SnapshotResponse]) error // MoveLeader requests current leader node to transfer its leadership to transferee. MoveLeader(context.Context, *MoveLeaderRequest) (*MoveLeaderResponse, error) // Downgrade requests downgrades, verifies feasibility or cancels downgrade // on the cluster version. // Supported since etcd 3.5. Downgrade(context.Context, *DowngradeRequest) (*DowngradeResponse, error) mustEmbedUnimplementedMaintenanceServer() } // UnimplementedMaintenanceServer must be embedded to have // forward compatible implementations. // // NOTE: this should be embedded by value instead of pointer to avoid a nil // pointer dereference when methods are called. type UnimplementedMaintenanceServer struct{} func (UnimplementedMaintenanceServer) Alarm(context.Context, *AlarmRequest) (*AlarmResponse, error) { return nil, status.Error(codes.Unimplemented, "method Alarm not implemented") } func (UnimplementedMaintenanceServer) Status(context.Context, *StatusRequest) (*StatusResponse, error) { return nil, status.Error(codes.Unimplemented, "method Status not implemented") } func (UnimplementedMaintenanceServer) Defragment(context.Context, *DefragmentRequest) (*DefragmentResponse, error) { return nil, status.Error(codes.Unimplemented, "method Defragment not implemented") } func (UnimplementedMaintenanceServer) Hash(context.Context, *HashRequest) (*HashResponse, error) { return nil, status.Error(codes.Unimplemented, "method Hash not implemented") } func (UnimplementedMaintenanceServer) HashKV(context.Context, *HashKVRequest) (*HashKVResponse, error) { return nil, status.Error(codes.Unimplemented, "method HashKV not implemented") } func (UnimplementedMaintenanceServer) Snapshot(*SnapshotRequest, grpc.ServerStreamingServer[SnapshotResponse]) error { return status.Error(codes.Unimplemented, "method Snapshot not implemented") } func (UnimplementedMaintenanceServer) MoveLeader(context.Context, *MoveLeaderRequest) (*MoveLeaderResponse, error) { return nil, status.Error(codes.Unimplemented, "method MoveLeader not implemented") } func (UnimplementedMaintenanceServer) Downgrade(context.Context, *DowngradeRequest) (*DowngradeResponse, error) { return nil, status.Error(codes.Unimplemented, "method Downgrade not implemented") } func (UnimplementedMaintenanceServer) mustEmbedUnimplementedMaintenanceServer() {} func (UnimplementedMaintenanceServer) testEmbeddedByValue() {} // UnsafeMaintenanceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to MaintenanceServer will // result in compilation errors. type UnsafeMaintenanceServer interface { mustEmbedUnimplementedMaintenanceServer() } func RegisterMaintenanceServer(s grpc.ServiceRegistrar, srv MaintenanceServer) { // If the following call panics, it indicates UnimplementedMaintenanceServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { t.testEmbeddedByValue() } s.RegisterService(&Maintenance_ServiceDesc, srv) } func _Maintenance_Alarm_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AlarmRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(MaintenanceServer).Alarm(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Maintenance_Alarm_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(MaintenanceServer).Alarm(ctx, req.(*AlarmRequest)) } return interceptor(ctx, in, info, handler) } func _Maintenance_Status_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(StatusRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(MaintenanceServer).Status(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Maintenance_Status_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(MaintenanceServer).Status(ctx, req.(*StatusRequest)) } return interceptor(ctx, in, info, handler) } func _Maintenance_Defragment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DefragmentRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(MaintenanceServer).Defragment(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Maintenance_Defragment_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(MaintenanceServer).Defragment(ctx, req.(*DefragmentRequest)) } return interceptor(ctx, in, info, handler) } func _Maintenance_Hash_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(HashRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(MaintenanceServer).Hash(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Maintenance_Hash_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(MaintenanceServer).Hash(ctx, req.(*HashRequest)) } return interceptor(ctx, in, info, handler) } func _Maintenance_HashKV_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(HashKVRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(MaintenanceServer).HashKV(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Maintenance_HashKV_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(MaintenanceServer).HashKV(ctx, req.(*HashKVRequest)) } return interceptor(ctx, in, info, handler) } func _Maintenance_Snapshot_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(SnapshotRequest) if err := stream.RecvMsg(m); err != nil { return err } return srv.(MaintenanceServer).Snapshot(m, &grpc.GenericServerStream[SnapshotRequest, SnapshotResponse]{ServerStream: stream}) } // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type Maintenance_SnapshotServer = grpc.ServerStreamingServer[SnapshotResponse] func _Maintenance_MoveLeader_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MoveLeaderRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(MaintenanceServer).MoveLeader(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Maintenance_MoveLeader_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(MaintenanceServer).MoveLeader(ctx, req.(*MoveLeaderRequest)) } return interceptor(ctx, in, info, handler) } func _Maintenance_Downgrade_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DowngradeRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(MaintenanceServer).Downgrade(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Maintenance_Downgrade_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(MaintenanceServer).Downgrade(ctx, req.(*DowngradeRequest)) } return interceptor(ctx, in, info, handler) } // Maintenance_ServiceDesc is the grpc.ServiceDesc for Maintenance service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var Maintenance_ServiceDesc = grpc.ServiceDesc{ ServiceName: "etcdserverpb.Maintenance", HandlerType: (*MaintenanceServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "Alarm", Handler: _Maintenance_Alarm_Handler, }, { MethodName: "Status", Handler: _Maintenance_Status_Handler, }, { MethodName: "Defragment", Handler: _Maintenance_Defragment_Handler, }, { MethodName: "Hash", Handler: _Maintenance_Hash_Handler, }, { MethodName: "HashKV", Handler: _Maintenance_HashKV_Handler, }, { MethodName: "MoveLeader", Handler: _Maintenance_MoveLeader_Handler, }, { MethodName: "Downgrade", Handler: _Maintenance_Downgrade_Handler, }, }, Streams: []grpc.StreamDesc{ { StreamName: "Snapshot", Handler: _Maintenance_Snapshot_Handler, ServerStreams: true, }, }, Metadata: "rpc.proto", } const ( Auth_AuthEnable_FullMethodName = "/etcdserverpb.Auth/AuthEnable" Auth_AuthDisable_FullMethodName = "/etcdserverpb.Auth/AuthDisable" Auth_AuthStatus_FullMethodName = "/etcdserverpb.Auth/AuthStatus" Auth_Authenticate_FullMethodName = "/etcdserverpb.Auth/Authenticate" Auth_UserAdd_FullMethodName = "/etcdserverpb.Auth/UserAdd" Auth_UserGet_FullMethodName = "/etcdserverpb.Auth/UserGet" Auth_UserList_FullMethodName = "/etcdserverpb.Auth/UserList" Auth_UserDelete_FullMethodName = "/etcdserverpb.Auth/UserDelete" Auth_UserChangePassword_FullMethodName = "/etcdserverpb.Auth/UserChangePassword" Auth_UserGrantRole_FullMethodName = "/etcdserverpb.Auth/UserGrantRole" Auth_UserRevokeRole_FullMethodName = "/etcdserverpb.Auth/UserRevokeRole" Auth_RoleAdd_FullMethodName = "/etcdserverpb.Auth/RoleAdd" Auth_RoleGet_FullMethodName = "/etcdserverpb.Auth/RoleGet" Auth_RoleList_FullMethodName = "/etcdserverpb.Auth/RoleList" Auth_RoleDelete_FullMethodName = "/etcdserverpb.Auth/RoleDelete" Auth_RoleGrantPermission_FullMethodName = "/etcdserverpb.Auth/RoleGrantPermission" Auth_RoleRevokePermission_FullMethodName = "/etcdserverpb.Auth/RoleRevokePermission" ) // AuthClient is the client API for Auth service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type AuthClient interface { // AuthEnable enables authentication. AuthEnable(ctx context.Context, in *AuthEnableRequest, opts ...grpc.CallOption) (*AuthEnableResponse, error) // AuthDisable disables authentication. AuthDisable(ctx context.Context, in *AuthDisableRequest, opts ...grpc.CallOption) (*AuthDisableResponse, error) // AuthStatus displays authentication status. AuthStatus(ctx context.Context, in *AuthStatusRequest, opts ...grpc.CallOption) (*AuthStatusResponse, error) // Authenticate processes an authenticate request. Authenticate(ctx context.Context, in *AuthenticateRequest, opts ...grpc.CallOption) (*AuthenticateResponse, error) // UserAdd adds a new user. User name cannot be empty. UserAdd(ctx context.Context, in *AuthUserAddRequest, opts ...grpc.CallOption) (*AuthUserAddResponse, error) // UserGet gets detailed user information. UserGet(ctx context.Context, in *AuthUserGetRequest, opts ...grpc.CallOption) (*AuthUserGetResponse, error) // UserList gets a list of all users. UserList(ctx context.Context, in *AuthUserListRequest, opts ...grpc.CallOption) (*AuthUserListResponse, error) // UserDelete deletes a specified user. UserDelete(ctx context.Context, in *AuthUserDeleteRequest, opts ...grpc.CallOption) (*AuthUserDeleteResponse, error) // UserChangePassword changes the password of a specified user. UserChangePassword(ctx context.Context, in *AuthUserChangePasswordRequest, opts ...grpc.CallOption) (*AuthUserChangePasswordResponse, error) // UserGrantRole grants a role to a specified user. UserGrantRole(ctx context.Context, in *AuthUserGrantRoleRequest, opts ...grpc.CallOption) (*AuthUserGrantRoleResponse, error) // UserRevokeRole revokes a role of specified user. UserRevokeRole(ctx context.Context, in *AuthUserRevokeRoleRequest, opts ...grpc.CallOption) (*AuthUserRevokeRoleResponse, error) // RoleAdd adds a new role. Role name cannot be empty. RoleAdd(ctx context.Context, in *AuthRoleAddRequest, opts ...grpc.CallOption) (*AuthRoleAddResponse, error) // RoleGet gets detailed role information. RoleGet(ctx context.Context, in *AuthRoleGetRequest, opts ...grpc.CallOption) (*AuthRoleGetResponse, error) // RoleList gets lists of all roles. RoleList(ctx context.Context, in *AuthRoleListRequest, opts ...grpc.CallOption) (*AuthRoleListResponse, error) // RoleDelete deletes a specified role. RoleDelete(ctx context.Context, in *AuthRoleDeleteRequest, opts ...grpc.CallOption) (*AuthRoleDeleteResponse, error) // RoleGrantPermission grants a permission of a specified key or range to a specified role. RoleGrantPermission(ctx context.Context, in *AuthRoleGrantPermissionRequest, opts ...grpc.CallOption) (*AuthRoleGrantPermissionResponse, error) // RoleRevokePermission revokes a key or range permission of a specified role. RoleRevokePermission(ctx context.Context, in *AuthRoleRevokePermissionRequest, opts ...grpc.CallOption) (*AuthRoleRevokePermissionResponse, error) } type authClient struct { cc grpc.ClientConnInterface } func NewAuthClient(cc grpc.ClientConnInterface) AuthClient { return &authClient{cc} } func (c *authClient) AuthEnable(ctx context.Context, in *AuthEnableRequest, opts ...grpc.CallOption) (*AuthEnableResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthEnableResponse) err := c.cc.Invoke(ctx, Auth_AuthEnable_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) AuthDisable(ctx context.Context, in *AuthDisableRequest, opts ...grpc.CallOption) (*AuthDisableResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthDisableResponse) err := c.cc.Invoke(ctx, Auth_AuthDisable_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) AuthStatus(ctx context.Context, in *AuthStatusRequest, opts ...grpc.CallOption) (*AuthStatusResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthStatusResponse) err := c.cc.Invoke(ctx, Auth_AuthStatus_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) Authenticate(ctx context.Context, in *AuthenticateRequest, opts ...grpc.CallOption) (*AuthenticateResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthenticateResponse) err := c.cc.Invoke(ctx, Auth_Authenticate_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) UserAdd(ctx context.Context, in *AuthUserAddRequest, opts ...grpc.CallOption) (*AuthUserAddResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthUserAddResponse) err := c.cc.Invoke(ctx, Auth_UserAdd_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) UserGet(ctx context.Context, in *AuthUserGetRequest, opts ...grpc.CallOption) (*AuthUserGetResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthUserGetResponse) err := c.cc.Invoke(ctx, Auth_UserGet_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) UserList(ctx context.Context, in *AuthUserListRequest, opts ...grpc.CallOption) (*AuthUserListResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthUserListResponse) err := c.cc.Invoke(ctx, Auth_UserList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) UserDelete(ctx context.Context, in *AuthUserDeleteRequest, opts ...grpc.CallOption) (*AuthUserDeleteResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthUserDeleteResponse) err := c.cc.Invoke(ctx, Auth_UserDelete_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) UserChangePassword(ctx context.Context, in *AuthUserChangePasswordRequest, opts ...grpc.CallOption) (*AuthUserChangePasswordResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthUserChangePasswordResponse) err := c.cc.Invoke(ctx, Auth_UserChangePassword_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) UserGrantRole(ctx context.Context, in *AuthUserGrantRoleRequest, opts ...grpc.CallOption) (*AuthUserGrantRoleResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthUserGrantRoleResponse) err := c.cc.Invoke(ctx, Auth_UserGrantRole_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) UserRevokeRole(ctx context.Context, in *AuthUserRevokeRoleRequest, opts ...grpc.CallOption) (*AuthUserRevokeRoleResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthUserRevokeRoleResponse) err := c.cc.Invoke(ctx, Auth_UserRevokeRole_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) RoleAdd(ctx context.Context, in *AuthRoleAddRequest, opts ...grpc.CallOption) (*AuthRoleAddResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthRoleAddResponse) err := c.cc.Invoke(ctx, Auth_RoleAdd_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) RoleGet(ctx context.Context, in *AuthRoleGetRequest, opts ...grpc.CallOption) (*AuthRoleGetResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthRoleGetResponse) err := c.cc.Invoke(ctx, Auth_RoleGet_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) RoleList(ctx context.Context, in *AuthRoleListRequest, opts ...grpc.CallOption) (*AuthRoleListResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthRoleListResponse) err := c.cc.Invoke(ctx, Auth_RoleList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) RoleDelete(ctx context.Context, in *AuthRoleDeleteRequest, opts ...grpc.CallOption) (*AuthRoleDeleteResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthRoleDeleteResponse) err := c.cc.Invoke(ctx, Auth_RoleDelete_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) RoleGrantPermission(ctx context.Context, in *AuthRoleGrantPermissionRequest, opts ...grpc.CallOption) (*AuthRoleGrantPermissionResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthRoleGrantPermissionResponse) err := c.cc.Invoke(ctx, Auth_RoleGrantPermission_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *authClient) RoleRevokePermission(ctx context.Context, in *AuthRoleRevokePermissionRequest, opts ...grpc.CallOption) (*AuthRoleRevokePermissionResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(AuthRoleRevokePermissionResponse) err := c.cc.Invoke(ctx, Auth_RoleRevokePermission_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } // AuthServer is the server API for Auth service. // All implementations must embed UnimplementedAuthServer // for forward compatibility. type AuthServer interface { // AuthEnable enables authentication. AuthEnable(context.Context, *AuthEnableRequest) (*AuthEnableResponse, error) // AuthDisable disables authentication. AuthDisable(context.Context, *AuthDisableRequest) (*AuthDisableResponse, error) // AuthStatus displays authentication status. AuthStatus(context.Context, *AuthStatusRequest) (*AuthStatusResponse, error) // Authenticate processes an authenticate request. Authenticate(context.Context, *AuthenticateRequest) (*AuthenticateResponse, error) // UserAdd adds a new user. User name cannot be empty. UserAdd(context.Context, *AuthUserAddRequest) (*AuthUserAddResponse, error) // UserGet gets detailed user information. UserGet(context.Context, *AuthUserGetRequest) (*AuthUserGetResponse, error) // UserList gets a list of all users. UserList(context.Context, *AuthUserListRequest) (*AuthUserListResponse, error) // UserDelete deletes a specified user. UserDelete(context.Context, *AuthUserDeleteRequest) (*AuthUserDeleteResponse, error) // UserChangePassword changes the password of a specified user. UserChangePassword(context.Context, *AuthUserChangePasswordRequest) (*AuthUserChangePasswordResponse, error) // UserGrantRole grants a role to a specified user. UserGrantRole(context.Context, *AuthUserGrantRoleRequest) (*AuthUserGrantRoleResponse, error) // UserRevokeRole revokes a role of specified user. UserRevokeRole(context.Context, *AuthUserRevokeRoleRequest) (*AuthUserRevokeRoleResponse, error) // RoleAdd adds a new role. Role name cannot be empty. RoleAdd(context.Context, *AuthRoleAddRequest) (*AuthRoleAddResponse, error) // RoleGet gets detailed role information. RoleGet(context.Context, *AuthRoleGetRequest) (*AuthRoleGetResponse, error) // RoleList gets lists of all roles. RoleList(context.Context, *AuthRoleListRequest) (*AuthRoleListResponse, error) // RoleDelete deletes a specified role. RoleDelete(context.Context, *AuthRoleDeleteRequest) (*AuthRoleDeleteResponse, error) // RoleGrantPermission grants a permission of a specified key or range to a specified role. RoleGrantPermission(context.Context, *AuthRoleGrantPermissionRequest) (*AuthRoleGrantPermissionResponse, error) // RoleRevokePermission revokes a key or range permission of a specified role. RoleRevokePermission(context.Context, *AuthRoleRevokePermissionRequest) (*AuthRoleRevokePermissionResponse, error) mustEmbedUnimplementedAuthServer() } // UnimplementedAuthServer must be embedded to have // forward compatible implementations. // // NOTE: this should be embedded by value instead of pointer to avoid a nil // pointer dereference when methods are called. type UnimplementedAuthServer struct{} func (UnimplementedAuthServer) AuthEnable(context.Context, *AuthEnableRequest) (*AuthEnableResponse, error) { return nil, status.Error(codes.Unimplemented, "method AuthEnable not implemented") } func (UnimplementedAuthServer) AuthDisable(context.Context, *AuthDisableRequest) (*AuthDisableResponse, error) { return nil, status.Error(codes.Unimplemented, "method AuthDisable not implemented") } func (UnimplementedAuthServer) AuthStatus(context.Context, *AuthStatusRequest) (*AuthStatusResponse, error) { return nil, status.Error(codes.Unimplemented, "method AuthStatus not implemented") } func (UnimplementedAuthServer) Authenticate(context.Context, *AuthenticateRequest) (*AuthenticateResponse, error) { return nil, status.Error(codes.Unimplemented, "method Authenticate not implemented") } func (UnimplementedAuthServer) UserAdd(context.Context, *AuthUserAddRequest) (*AuthUserAddResponse, error) { return nil, status.Error(codes.Unimplemented, "method UserAdd not implemented") } func (UnimplementedAuthServer) UserGet(context.Context, *AuthUserGetRequest) (*AuthUserGetResponse, error) { return nil, status.Error(codes.Unimplemented, "method UserGet not implemented") } func (UnimplementedAuthServer) UserList(context.Context, *AuthUserListRequest) (*AuthUserListResponse, error) { return nil, status.Error(codes.Unimplemented, "method UserList not implemented") } func (UnimplementedAuthServer) UserDelete(context.Context, *AuthUserDeleteRequest) (*AuthUserDeleteResponse, error) { return nil, status.Error(codes.Unimplemented, "method UserDelete not implemented") } func (UnimplementedAuthServer) UserChangePassword(context.Context, *AuthUserChangePasswordRequest) (*AuthUserChangePasswordResponse, error) { return nil, status.Error(codes.Unimplemented, "method UserChangePassword not implemented") } func (UnimplementedAuthServer) UserGrantRole(context.Context, *AuthUserGrantRoleRequest) (*AuthUserGrantRoleResponse, error) { return nil, status.Error(codes.Unimplemented, "method UserGrantRole not implemented") } func (UnimplementedAuthServer) UserRevokeRole(context.Context, *AuthUserRevokeRoleRequest) (*AuthUserRevokeRoleResponse, error) { return nil, status.Error(codes.Unimplemented, "method UserRevokeRole not implemented") } func (UnimplementedAuthServer) RoleAdd(context.Context, *AuthRoleAddRequest) (*AuthRoleAddResponse, error) { return nil, status.Error(codes.Unimplemented, "method RoleAdd not implemented") } func (UnimplementedAuthServer) RoleGet(context.Context, *AuthRoleGetRequest) (*AuthRoleGetResponse, error) { return nil, status.Error(codes.Unimplemented, "method RoleGet not implemented") } func (UnimplementedAuthServer) RoleList(context.Context, *AuthRoleListRequest) (*AuthRoleListResponse, error) { return nil, status.Error(codes.Unimplemented, "method RoleList not implemented") } func (UnimplementedAuthServer) RoleDelete(context.Context, *AuthRoleDeleteRequest) (*AuthRoleDeleteResponse, error) { return nil, status.Error(codes.Unimplemented, "method RoleDelete not implemented") } func (UnimplementedAuthServer) RoleGrantPermission(context.Context, *AuthRoleGrantPermissionRequest) (*AuthRoleGrantPermissionResponse, error) { return nil, status.Error(codes.Unimplemented, "method RoleGrantPermission not implemented") } func (UnimplementedAuthServer) RoleRevokePermission(context.Context, *AuthRoleRevokePermissionRequest) (*AuthRoleRevokePermissionResponse, error) { return nil, status.Error(codes.Unimplemented, "method RoleRevokePermission not implemented") } func (UnimplementedAuthServer) mustEmbedUnimplementedAuthServer() {} func (UnimplementedAuthServer) testEmbeddedByValue() {} // UnsafeAuthServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to AuthServer will // result in compilation errors. type UnsafeAuthServer interface { mustEmbedUnimplementedAuthServer() } func RegisterAuthServer(s grpc.ServiceRegistrar, srv AuthServer) { // If the following call panics, it indicates UnimplementedAuthServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { t.testEmbeddedByValue() } s.RegisterService(&Auth_ServiceDesc, srv) } func _Auth_AuthEnable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthEnableRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).AuthEnable(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_AuthEnable_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).AuthEnable(ctx, req.(*AuthEnableRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_AuthDisable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthDisableRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).AuthDisable(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_AuthDisable_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).AuthDisable(ctx, req.(*AuthDisableRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_AuthStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthStatusRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).AuthStatus(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_AuthStatus_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).AuthStatus(ctx, req.(*AuthStatusRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_Authenticate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthenticateRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).Authenticate(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_Authenticate_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).Authenticate(ctx, req.(*AuthenticateRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_UserAdd_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthUserAddRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).UserAdd(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_UserAdd_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).UserAdd(ctx, req.(*AuthUserAddRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_UserGet_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthUserGetRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).UserGet(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_UserGet_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).UserGet(ctx, req.(*AuthUserGetRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_UserList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthUserListRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).UserList(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_UserList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).UserList(ctx, req.(*AuthUserListRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_UserDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthUserDeleteRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).UserDelete(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_UserDelete_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).UserDelete(ctx, req.(*AuthUserDeleteRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_UserChangePassword_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthUserChangePasswordRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).UserChangePassword(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_UserChangePassword_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).UserChangePassword(ctx, req.(*AuthUserChangePasswordRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_UserGrantRole_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthUserGrantRoleRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).UserGrantRole(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_UserGrantRole_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).UserGrantRole(ctx, req.(*AuthUserGrantRoleRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_UserRevokeRole_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthUserRevokeRoleRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).UserRevokeRole(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_UserRevokeRole_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).UserRevokeRole(ctx, req.(*AuthUserRevokeRoleRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_RoleAdd_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthRoleAddRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).RoleAdd(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_RoleAdd_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).RoleAdd(ctx, req.(*AuthRoleAddRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_RoleGet_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthRoleGetRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).RoleGet(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_RoleGet_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).RoleGet(ctx, req.(*AuthRoleGetRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_RoleList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthRoleListRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).RoleList(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_RoleList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).RoleList(ctx, req.(*AuthRoleListRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_RoleDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthRoleDeleteRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).RoleDelete(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_RoleDelete_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).RoleDelete(ctx, req.(*AuthRoleDeleteRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_RoleGrantPermission_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthRoleGrantPermissionRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).RoleGrantPermission(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_RoleGrantPermission_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).RoleGrantPermission(ctx, req.(*AuthRoleGrantPermissionRequest)) } return interceptor(ctx, in, info, handler) } func _Auth_RoleRevokePermission_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(AuthRoleRevokePermissionRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(AuthServer).RoleRevokePermission(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Auth_RoleRevokePermission_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AuthServer).RoleRevokePermission(ctx, req.(*AuthRoleRevokePermissionRequest)) } return interceptor(ctx, in, info, handler) } // Auth_ServiceDesc is the grpc.ServiceDesc for Auth service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var Auth_ServiceDesc = grpc.ServiceDesc{ ServiceName: "etcdserverpb.Auth", HandlerType: (*AuthServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "AuthEnable", Handler: _Auth_AuthEnable_Handler, }, { MethodName: "AuthDisable", Handler: _Auth_AuthDisable_Handler, }, { MethodName: "AuthStatus", Handler: _Auth_AuthStatus_Handler, }, { MethodName: "Authenticate", Handler: _Auth_Authenticate_Handler, }, { MethodName: "UserAdd", Handler: _Auth_UserAdd_Handler, }, { MethodName: "UserGet", Handler: _Auth_UserGet_Handler, }, { MethodName: "UserList", Handler: _Auth_UserList_Handler, }, { MethodName: "UserDelete", Handler: _Auth_UserDelete_Handler, }, { MethodName: "UserChangePassword", Handler: _Auth_UserChangePassword_Handler, }, { MethodName: "UserGrantRole", Handler: _Auth_UserGrantRole_Handler, }, { MethodName: "UserRevokeRole", Handler: _Auth_UserRevokeRole_Handler, }, { MethodName: "RoleAdd", Handler: _Auth_RoleAdd_Handler, }, { MethodName: "RoleGet", Handler: _Auth_RoleGet_Handler, }, { MethodName: "RoleList", Handler: _Auth_RoleList_Handler, }, { MethodName: "RoleDelete", Handler: _Auth_RoleDelete_Handler, }, { MethodName: "RoleGrantPermission", Handler: _Auth_RoleGrantPermission_Handler, }, { MethodName: "RoleRevokePermission", Handler: _Auth_RoleRevokePermission_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "rpc.proto", } ================================================ FILE: api/go.mod ================================================ module go.etcd.io/etcd/api/v3 go 1.26 toolchain go1.26.1 require ( github.com/coreos/go-semver v0.3.1 github.com/golang/protobuf v1.5.4 github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 github.com/stretchr/testify v1.11.1 google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 google.golang.org/grpc v1.79.2 google.golang.org/protobuf v1.36.11 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) ================================================ FILE: api/go.sum ================================================ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: api/membershippb/membership.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: membership.proto package membershippb import ( fmt "fmt" io "io" math "math" math_bits "math/bits" proto "github.com/golang/protobuf/proto" _ "go.etcd.io/etcd/api/v3/versionpb" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package // RaftAttributes represents the raft related attributes of an etcd member. type RaftAttributes struct { // peerURLs is the list of peers in the raft cluster. PeerUrls []string `protobuf:"bytes,1,rep,name=peer_urls,json=peerUrls,proto3" json:"peer_urls,omitempty"` // isLearner indicates if the member is raft learner. IsLearner bool `protobuf:"varint,2,opt,name=is_learner,json=isLearner,proto3" json:"is_learner,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *RaftAttributes) Reset() { *m = RaftAttributes{} } func (m *RaftAttributes) String() string { return proto.CompactTextString(m) } func (*RaftAttributes) ProtoMessage() {} func (*RaftAttributes) Descriptor() ([]byte, []int) { return fileDescriptor_949fe0d019050ef5, []int{0} } func (m *RaftAttributes) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *RaftAttributes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_RaftAttributes.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *RaftAttributes) XXX_Merge(src proto.Message) { xxx_messageInfo_RaftAttributes.Merge(m, src) } func (m *RaftAttributes) XXX_Size() int { return m.Size() } func (m *RaftAttributes) XXX_DiscardUnknown() { xxx_messageInfo_RaftAttributes.DiscardUnknown(m) } var xxx_messageInfo_RaftAttributes proto.InternalMessageInfo func (m *RaftAttributes) GetPeerUrls() []string { if m != nil { return m.PeerUrls } return nil } func (m *RaftAttributes) GetIsLearner() bool { if m != nil { return m.IsLearner } return false } // Attributes represents all the non-raft related attributes of an etcd member. type Attributes struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` ClientUrls []string `protobuf:"bytes,2,rep,name=client_urls,json=clientUrls,proto3" json:"client_urls,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Attributes) Reset() { *m = Attributes{} } func (m *Attributes) String() string { return proto.CompactTextString(m) } func (*Attributes) ProtoMessage() {} func (*Attributes) Descriptor() ([]byte, []int) { return fileDescriptor_949fe0d019050ef5, []int{1} } func (m *Attributes) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Attributes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Attributes.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Attributes) XXX_Merge(src proto.Message) { xxx_messageInfo_Attributes.Merge(m, src) } func (m *Attributes) XXX_Size() int { return m.Size() } func (m *Attributes) XXX_DiscardUnknown() { xxx_messageInfo_Attributes.DiscardUnknown(m) } var xxx_messageInfo_Attributes proto.InternalMessageInfo func (m *Attributes) GetName() string { if m != nil { return m.Name } return "" } func (m *Attributes) GetClientUrls() []string { if m != nil { return m.ClientUrls } return nil } type Member struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` RaftAttributes *RaftAttributes `protobuf:"bytes,2,opt,name=raft_attributes,json=raftAttributes,proto3" json:"raft_attributes,omitempty"` MemberAttributes *Attributes `protobuf:"bytes,3,opt,name=member_attributes,json=memberAttributes,proto3" json:"member_attributes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Member) Reset() { *m = Member{} } func (m *Member) String() string { return proto.CompactTextString(m) } func (*Member) ProtoMessage() {} func (*Member) Descriptor() ([]byte, []int) { return fileDescriptor_949fe0d019050ef5, []int{2} } func (m *Member) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Member) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Member.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Member) XXX_Merge(src proto.Message) { xxx_messageInfo_Member.Merge(m, src) } func (m *Member) XXX_Size() int { return m.Size() } func (m *Member) XXX_DiscardUnknown() { xxx_messageInfo_Member.DiscardUnknown(m) } var xxx_messageInfo_Member proto.InternalMessageInfo func (m *Member) GetID() uint64 { if m != nil { return m.ID } return 0 } func (m *Member) GetRaftAttributes() *RaftAttributes { if m != nil { return m.RaftAttributes } return nil } func (m *Member) GetMemberAttributes() *Attributes { if m != nil { return m.MemberAttributes } return nil } type ClusterVersionSetRequest struct { Ver string `protobuf:"bytes,1,opt,name=ver,proto3" json:"ver,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *ClusterVersionSetRequest) Reset() { *m = ClusterVersionSetRequest{} } func (m *ClusterVersionSetRequest) String() string { return proto.CompactTextString(m) } func (*ClusterVersionSetRequest) ProtoMessage() {} func (*ClusterVersionSetRequest) Descriptor() ([]byte, []int) { return fileDescriptor_949fe0d019050ef5, []int{3} } func (m *ClusterVersionSetRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *ClusterVersionSetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_ClusterVersionSetRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *ClusterVersionSetRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_ClusterVersionSetRequest.Merge(m, src) } func (m *ClusterVersionSetRequest) XXX_Size() int { return m.Size() } func (m *ClusterVersionSetRequest) XXX_DiscardUnknown() { xxx_messageInfo_ClusterVersionSetRequest.DiscardUnknown(m) } var xxx_messageInfo_ClusterVersionSetRequest proto.InternalMessageInfo func (m *ClusterVersionSetRequest) GetVer() string { if m != nil { return m.Ver } return "" } type ClusterMemberAttrSetRequest struct { Member_ID uint64 `protobuf:"varint,1,opt,name=member_ID,json=memberID,proto3" json:"member_ID,omitempty"` MemberAttributes *Attributes `protobuf:"bytes,2,opt,name=member_attributes,json=memberAttributes,proto3" json:"member_attributes,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *ClusterMemberAttrSetRequest) Reset() { *m = ClusterMemberAttrSetRequest{} } func (m *ClusterMemberAttrSetRequest) String() string { return proto.CompactTextString(m) } func (*ClusterMemberAttrSetRequest) ProtoMessage() {} func (*ClusterMemberAttrSetRequest) Descriptor() ([]byte, []int) { return fileDescriptor_949fe0d019050ef5, []int{4} } func (m *ClusterMemberAttrSetRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *ClusterMemberAttrSetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_ClusterMemberAttrSetRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *ClusterMemberAttrSetRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_ClusterMemberAttrSetRequest.Merge(m, src) } func (m *ClusterMemberAttrSetRequest) XXX_Size() int { return m.Size() } func (m *ClusterMemberAttrSetRequest) XXX_DiscardUnknown() { xxx_messageInfo_ClusterMemberAttrSetRequest.DiscardUnknown(m) } var xxx_messageInfo_ClusterMemberAttrSetRequest proto.InternalMessageInfo func (m *ClusterMemberAttrSetRequest) GetMember_ID() uint64 { if m != nil { return m.Member_ID } return 0 } func (m *ClusterMemberAttrSetRequest) GetMemberAttributes() *Attributes { if m != nil { return m.MemberAttributes } return nil } type DowngradeInfoSetRequest struct { Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"` Ver string `protobuf:"bytes,2,opt,name=ver,proto3" json:"ver,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *DowngradeInfoSetRequest) Reset() { *m = DowngradeInfoSetRequest{} } func (m *DowngradeInfoSetRequest) String() string { return proto.CompactTextString(m) } func (*DowngradeInfoSetRequest) ProtoMessage() {} func (*DowngradeInfoSetRequest) Descriptor() ([]byte, []int) { return fileDescriptor_949fe0d019050ef5, []int{5} } func (m *DowngradeInfoSetRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *DowngradeInfoSetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_DowngradeInfoSetRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *DowngradeInfoSetRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_DowngradeInfoSetRequest.Merge(m, src) } func (m *DowngradeInfoSetRequest) XXX_Size() int { return m.Size() } func (m *DowngradeInfoSetRequest) XXX_DiscardUnknown() { xxx_messageInfo_DowngradeInfoSetRequest.DiscardUnknown(m) } var xxx_messageInfo_DowngradeInfoSetRequest proto.InternalMessageInfo func (m *DowngradeInfoSetRequest) GetEnabled() bool { if m != nil { return m.Enabled } return false } func (m *DowngradeInfoSetRequest) GetVer() string { if m != nil { return m.Ver } return "" } func init() { proto.RegisterType((*RaftAttributes)(nil), "membershippb.RaftAttributes") proto.RegisterType((*Attributes)(nil), "membershippb.Attributes") proto.RegisterType((*Member)(nil), "membershippb.Member") proto.RegisterType((*ClusterVersionSetRequest)(nil), "membershippb.ClusterVersionSetRequest") proto.RegisterType((*ClusterMemberAttrSetRequest)(nil), "membershippb.ClusterMemberAttrSetRequest") proto.RegisterType((*DowngradeInfoSetRequest)(nil), "membershippb.DowngradeInfoSetRequest") } func init() { proto.RegisterFile("membership.proto", fileDescriptor_949fe0d019050ef5) } var fileDescriptor_949fe0d019050ef5 = []byte{ // 401 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x52, 0x3f, 0xcf, 0xd2, 0x40, 0x18, 0xf7, 0xda, 0x37, 0xd0, 0x3e, 0x18, 0xc4, 0x5b, 0x6c, 0x44, 0x6b, 0x83, 0x0b, 0x53, 0x9b, 0x48, 0x58, 0xdc, 0x54, 0x18, 0x30, 0xe2, 0x70, 0x06, 0x07, 0x17, 0x72, 0x85, 0x07, 0xbc, 0xa4, 0xb4, 0xf5, 0xee, 0x8a, 0xbb, 0xa3, 0x9f, 0xc0, 0x6f, 0xe1, 0xe4, 0x77, 0x70, 0xf4, 0x23, 0x18, 0xfc, 0x22, 0xa6, 0xd7, 0x42, 0x4b, 0x74, 0x7a, 0xb7, 0xe7, 0x7e, 0xb9, 0xe7, 0xf7, 0x2f, 0x0f, 0x0c, 0x0e, 0x78, 0x88, 0x51, 0xaa, 0x8f, 0x22, 0x0f, 0x73, 0x99, 0xe9, 0x8c, 0xde, 0x6d, 0x90, 0x3c, 0x7e, 0x18, 0xa0, 0xde, 0x6c, 0x23, 0x9e, 0x8b, 0xe8, 0x88, 0x52, 0x89, 0x2c, 0xcd, 0xe3, 0xf3, 0x54, 0xfd, 0x1f, 0xad, 0xa0, 0xcf, 0xf8, 0x4e, 0xbf, 0xd0, 0x5a, 0x8a, 0xb8, 0xd0, 0xa8, 0xe8, 0x10, 0xdc, 0x1c, 0x51, 0xae, 0x0b, 0x99, 0x28, 0x8f, 0x04, 0xf6, 0xd8, 0x65, 0x4e, 0x09, 0xac, 0x64, 0xa2, 0xe8, 0x63, 0x00, 0xa1, 0xd6, 0x09, 0x72, 0x99, 0xa2, 0xf4, 0xac, 0x80, 0x8c, 0x1d, 0xe6, 0x0a, 0xf5, 0xa6, 0x02, 0x9e, 0x77, 0xbf, 0xfc, 0xf0, 0xec, 0x49, 0x38, 0x1d, 0xbd, 0x06, 0x68, 0x51, 0x52, 0xb8, 0x49, 0xf9, 0x01, 0x3d, 0x12, 0x90, 0xb1, 0xcb, 0xcc, 0x4c, 0x9f, 0x40, 0x6f, 0x93, 0x08, 0x4c, 0x75, 0x25, 0x64, 0x19, 0x21, 0xa8, 0xa0, 0x52, 0xaa, 0xe1, 0xfa, 0x4e, 0xa0, 0xb3, 0x34, 0xa9, 0x68, 0x1f, 0xac, 0xc5, 0xcc, 0xd0, 0xdc, 0x30, 0x6b, 0x31, 0xa3, 0x73, 0xb8, 0x27, 0xf9, 0x4e, 0xaf, 0xf9, 0x45, 0xcb, 0x78, 0xea, 0x3d, 0x7b, 0x14, 0xb6, 0x7b, 0x08, 0xaf, 0x23, 0xb2, 0xbe, 0xbc, 0x8e, 0x3c, 0x87, 0xfb, 0xd5, 0xf7, 0x36, 0x91, 0x6d, 0x88, 0xbc, 0x6b, 0xa2, 0x16, 0x49, 0xdd, 0x7d, 0x83, 0x34, 0x8e, 0xa7, 0xe0, 0xbd, 0x4a, 0x0a, 0xa5, 0x51, 0xbe, 0xaf, 0xca, 0x7e, 0x87, 0x9a, 0xe1, 0xa7, 0x02, 0x95, 0xa6, 0x03, 0xb0, 0x8f, 0x28, 0xeb, 0x2a, 0xca, 0xb1, 0x59, 0xfb, 0x4a, 0x60, 0x58, 0xef, 0x2d, 0x2f, 0xdc, 0xad, 0xd5, 0x21, 0xb8, 0xb5, 0xcd, 0x4b, 0x09, 0x4e, 0x05, 0x98, 0x2a, 0xfe, 0x93, 0xc1, 0xba, 0x7d, 0x86, 0xb7, 0xf0, 0x60, 0x96, 0x7d, 0x4e, 0xf7, 0x92, 0x6f, 0x71, 0x91, 0xee, 0xb2, 0x96, 0x0f, 0x0f, 0xba, 0x98, 0xf2, 0x38, 0xc1, 0xad, 0x71, 0xe1, 0xb0, 0xf3, 0xf3, 0x1c, 0xce, 0xfa, 0x37, 0xdc, 0xcb, 0xe9, 0xcf, 0x93, 0x4f, 0x7e, 0x9d, 0x7c, 0xf2, 0xfb, 0xe4, 0x93, 0x6f, 0x7f, 0xfc, 0x3b, 0x1f, 0x9e, 0xee, 0xb3, 0xb0, 0xbc, 0xcf, 0x50, 0x64, 0x51, 0x73, 0xa7, 0x93, 0xa8, 0x6d, 0x36, 0xee, 0x98, 0x33, 0x9d, 0xfc, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x15, 0x23, 0xc3, 0x3f, 0xea, 0x02, 0x00, 0x00, } func (m *RaftAttributes) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *RaftAttributes) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *RaftAttributes) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.IsLearner { i-- if m.IsLearner { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x10 } if len(m.PeerUrls) > 0 { for iNdEx := len(m.PeerUrls) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.PeerUrls[iNdEx]) copy(dAtA[i:], m.PeerUrls[iNdEx]) i = encodeVarintMembership(dAtA, i, uint64(len(m.PeerUrls[iNdEx]))) i-- dAtA[i] = 0xa } } return len(dAtA) - i, nil } func (m *Attributes) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Attributes) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Attributes) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.ClientUrls) > 0 { for iNdEx := len(m.ClientUrls) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.ClientUrls[iNdEx]) copy(dAtA[i:], m.ClientUrls[iNdEx]) i = encodeVarintMembership(dAtA, i, uint64(len(m.ClientUrls[iNdEx]))) i-- dAtA[i] = 0x12 } } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintMembership(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *Member) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Member) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Member) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.MemberAttributes != nil { { size, err := m.MemberAttributes.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintMembership(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } if m.RaftAttributes != nil { { size, err := m.RaftAttributes.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintMembership(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } if m.ID != 0 { i = encodeVarintMembership(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *ClusterVersionSetRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *ClusterVersionSetRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ClusterVersionSetRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Ver) > 0 { i -= len(m.Ver) copy(dAtA[i:], m.Ver) i = encodeVarintMembership(dAtA, i, uint64(len(m.Ver))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *ClusterMemberAttrSetRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *ClusterMemberAttrSetRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ClusterMemberAttrSetRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.MemberAttributes != nil { { size, err := m.MemberAttributes.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintMembership(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } if m.Member_ID != 0 { i = encodeVarintMembership(dAtA, i, uint64(m.Member_ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *DowngradeInfoSetRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *DowngradeInfoSetRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *DowngradeInfoSetRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Ver) > 0 { i -= len(m.Ver) copy(dAtA[i:], m.Ver) i = encodeVarintMembership(dAtA, i, uint64(len(m.Ver))) i-- dAtA[i] = 0x12 } if m.Enabled { i-- if m.Enabled { dAtA[i] = 1 } else { dAtA[i] = 0 } i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func encodeVarintMembership(dAtA []byte, offset int, v uint64) int { offset -= sovMembership(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) return base } func (m *RaftAttributes) Size() (n int) { if m == nil { return 0 } var l int _ = l if len(m.PeerUrls) > 0 { for _, s := range m.PeerUrls { l = len(s) n += 1 + l + sovMembership(uint64(l)) } } if m.IsLearner { n += 2 } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *Attributes) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovMembership(uint64(l)) } if len(m.ClientUrls) > 0 { for _, s := range m.ClientUrls { l = len(s) n += 1 + l + sovMembership(uint64(l)) } } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *Member) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovMembership(uint64(m.ID)) } if m.RaftAttributes != nil { l = m.RaftAttributes.Size() n += 1 + l + sovMembership(uint64(l)) } if m.MemberAttributes != nil { l = m.MemberAttributes.Size() n += 1 + l + sovMembership(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *ClusterVersionSetRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Ver) if l > 0 { n += 1 + l + sovMembership(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *ClusterMemberAttrSetRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Member_ID != 0 { n += 1 + sovMembership(uint64(m.Member_ID)) } if m.MemberAttributes != nil { l = m.MemberAttributes.Size() n += 1 + l + sovMembership(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *DowngradeInfoSetRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Enabled { n += 2 } l = len(m.Ver) if l > 0 { n += 1 + l + sovMembership(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func sovMembership(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } func sozMembership(x uint64) (n int) { return sovMembership(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (m *RaftAttributes) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: RaftAttributes: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: RaftAttributes: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PeerUrls", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthMembership } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthMembership } if postIndex > l { return io.ErrUnexpectedEOF } m.PeerUrls = append(m.PeerUrls, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field IsLearner", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.IsLearner = bool(v != 0) default: iNdEx = preIndex skippy, err := skipMembership(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMembership } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *Attributes) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: Attributes: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: Attributes: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthMembership } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthMembership } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ClientUrls", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthMembership } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthMembership } if postIndex > l { return io.ErrUnexpectedEOF } m.ClientUrls = append(m.ClientUrls, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipMembership(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMembership } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *Member) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: Member: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: Member: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field RaftAttributes", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthMembership } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthMembership } if postIndex > l { return io.ErrUnexpectedEOF } if m.RaftAttributes == nil { m.RaftAttributes = &RaftAttributes{} } if err := m.RaftAttributes.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field MemberAttributes", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthMembership } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthMembership } if postIndex > l { return io.ErrUnexpectedEOF } if m.MemberAttributes == nil { m.MemberAttributes = &Attributes{} } if err := m.MemberAttributes.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipMembership(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMembership } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *ClusterVersionSetRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: ClusterVersionSetRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: ClusterVersionSetRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Ver", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthMembership } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthMembership } if postIndex > l { return io.ErrUnexpectedEOF } m.Ver = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipMembership(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMembership } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *ClusterMemberAttrSetRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: ClusterMemberAttrSetRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: ClusterMemberAttrSetRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Member_ID", wireType) } m.Member_ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Member_ID |= uint64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field MemberAttributes", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthMembership } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthMembership } if postIndex > l { return io.ErrUnexpectedEOF } if m.MemberAttributes == nil { m.MemberAttributes = &Attributes{} } if err := m.MemberAttributes.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipMembership(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMembership } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *DowngradeInfoSetRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: DowngradeInfoSetRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: DowngradeInfoSetRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Enabled", wireType) } var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int(b&0x7F) << shift if b < 0x80 { break } } m.Enabled = bool(v != 0) case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Ver", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMembership } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } intStringLen := int(stringLen) if intStringLen < 0 { return ErrInvalidLengthMembership } postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthMembership } if postIndex > l { return io.ErrUnexpectedEOF } m.Ver = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipMembership(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMembership } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func skipMembership(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowMembership } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } wireType := int(wire & 0x7) switch wireType { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowMembership } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } iNdEx++ if dAtA[iNdEx-1] < 0x80 { break } } case 1: iNdEx += 8 case 2: var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowMembership } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { break } } if length < 0 { return 0, ErrInvalidLengthMembership } iNdEx += length case 3: depth++ case 4: if depth == 0 { return 0, ErrUnexpectedEndOfGroupMembership } depth-- case 5: iNdEx += 4 default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { return 0, ErrInvalidLengthMembership } if depth == 0 { return iNdEx, nil } } return 0, io.ErrUnexpectedEOF } var ( ErrInvalidLengthMembership = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowMembership = fmt.Errorf("proto: integer overflow") ErrUnexpectedEndOfGroupMembership = fmt.Errorf("proto: unexpected end of group") ) ================================================ FILE: api/membershippb/membership.proto ================================================ syntax = "proto3"; package membershippb; import "etcd/api/versionpb/version.proto"; option go_package = "go.etcd.io/etcd/api/v3/membershippb"; // RaftAttributes represents the raft related attributes of an etcd member. message RaftAttributes { option (versionpb.etcd_version_msg) = "3.5"; // peerURLs is the list of peers in the raft cluster. repeated string peer_urls = 1; // isLearner indicates if the member is raft learner. bool is_learner = 2; } // Attributes represents all the non-raft related attributes of an etcd member. message Attributes { option (versionpb.etcd_version_msg) = "3.5"; string name = 1; repeated string client_urls = 2; } message Member { option (versionpb.etcd_version_msg) = "3.5"; uint64 ID = 1; RaftAttributes raft_attributes = 2; Attributes member_attributes = 3; } message ClusterVersionSetRequest { option (versionpb.etcd_version_msg) = "3.5"; string ver = 1; } message ClusterMemberAttrSetRequest { option (versionpb.etcd_version_msg) = "3.5"; uint64 member_ID = 1; Attributes member_attributes = 2; } message DowngradeInfoSetRequest { option (versionpb.etcd_version_msg) = "3.5"; bool enabled = 1; string ver = 2; } ================================================ FILE: api/mvccpb/deprecated.go ================================================ // Copyright 2026 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvccpb const ( // PUT is an alias of Event_PUT // Deprecated: use Event_PUT instead. Will be removed in v3.8. PUT = Event_PUT // DELETE is an alias of Permission_WRITE // Deprecated: use Event_DELETE instead. Will be removed in v3.8. DELETE = Event_DELETE ) ================================================ FILE: api/mvccpb/kv.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: kv.proto package mvccpb import ( fmt "fmt" io "io" math "math" math_bits "math/bits" proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Event_EventType int32 const ( Event_PUT Event_EventType = 0 Event_DELETE Event_EventType = 1 ) var Event_EventType_name = map[int32]string{ 0: "PUT", 1: "DELETE", } var Event_EventType_value = map[string]int32{ "PUT": 0, "DELETE": 1, } func (x Event_EventType) String() string { return proto.EnumName(Event_EventType_name, int32(x)) } func (Event_EventType) EnumDescriptor() ([]byte, []int) { return fileDescriptor_2216fe83c9c12408, []int{1, 0} } type KeyValue struct { // key is the key in bytes. An empty key is not allowed. Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // create_revision is the revision of last creation on this key. CreateRevision int64 `protobuf:"varint,2,opt,name=create_revision,json=createRevision,proto3" json:"create_revision,omitempty"` // mod_revision is the revision of last modification on this key. ModRevision int64 `protobuf:"varint,3,opt,name=mod_revision,json=modRevision,proto3" json:"mod_revision,omitempty"` // version is the version of the key. A deletion resets // the version to zero and any modification of the key // increases its version. Version int64 `protobuf:"varint,4,opt,name=version,proto3" json:"version,omitempty"` // value is the value held by the key, in bytes. Value []byte `protobuf:"bytes,5,opt,name=value,proto3" json:"value,omitempty"` // lease is the ID of the lease that attached to key. // When the attached lease expires, the key will be deleted. // If lease is 0, then no lease is attached to the key. Lease int64 `protobuf:"varint,6,opt,name=lease,proto3" json:"lease,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *KeyValue) Reset() { *m = KeyValue{} } func (m *KeyValue) String() string { return proto.CompactTextString(m) } func (*KeyValue) ProtoMessage() {} func (*KeyValue) Descriptor() ([]byte, []int) { return fileDescriptor_2216fe83c9c12408, []int{0} } func (m *KeyValue) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *KeyValue) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_KeyValue.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *KeyValue) XXX_Merge(src proto.Message) { xxx_messageInfo_KeyValue.Merge(m, src) } func (m *KeyValue) XXX_Size() int { return m.Size() } func (m *KeyValue) XXX_DiscardUnknown() { xxx_messageInfo_KeyValue.DiscardUnknown(m) } var xxx_messageInfo_KeyValue proto.InternalMessageInfo func (m *KeyValue) GetKey() []byte { if m != nil { return m.Key } return nil } func (m *KeyValue) GetCreateRevision() int64 { if m != nil { return m.CreateRevision } return 0 } func (m *KeyValue) GetModRevision() int64 { if m != nil { return m.ModRevision } return 0 } func (m *KeyValue) GetVersion() int64 { if m != nil { return m.Version } return 0 } func (m *KeyValue) GetValue() []byte { if m != nil { return m.Value } return nil } func (m *KeyValue) GetLease() int64 { if m != nil { return m.Lease } return 0 } type Event struct { // type is the kind of event. If type is a PUT, it indicates // new data has been stored to the key. If type is a DELETE, // it indicates the key was deleted. Type Event_EventType `protobuf:"varint,1,opt,name=type,proto3,enum=mvccpb.Event_EventType" json:"type,omitempty"` // kv holds the KeyValue for the event. // A PUT event contains current kv pair. // A PUT event with kv.Version=1 indicates the creation of a key. // A DELETE/EXPIRE event contains the deleted key with // its modification revision set to the revision of deletion. Kv *KeyValue `protobuf:"bytes,2,opt,name=kv,proto3" json:"kv,omitempty"` // prev_kv holds the key-value pair before the event happens. PrevKv *KeyValue `protobuf:"bytes,3,opt,name=prev_kv,json=prevKv,proto3" json:"prev_kv,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Event) Reset() { *m = Event{} } func (m *Event) String() string { return proto.CompactTextString(m) } func (*Event) ProtoMessage() {} func (*Event) Descriptor() ([]byte, []int) { return fileDescriptor_2216fe83c9c12408, []int{1} } func (m *Event) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Event) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Event.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Event) XXX_Merge(src proto.Message) { xxx_messageInfo_Event.Merge(m, src) } func (m *Event) XXX_Size() int { return m.Size() } func (m *Event) XXX_DiscardUnknown() { xxx_messageInfo_Event.DiscardUnknown(m) } var xxx_messageInfo_Event proto.InternalMessageInfo func (m *Event) GetType() Event_EventType { if m != nil { return m.Type } return Event_PUT } func (m *Event) GetKv() *KeyValue { if m != nil { return m.Kv } return nil } func (m *Event) GetPrevKv() *KeyValue { if m != nil { return m.PrevKv } return nil } func init() { proto.RegisterEnum("mvccpb.Event_EventType", Event_EventType_name, Event_EventType_value) proto.RegisterType((*KeyValue)(nil), "mvccpb.KeyValue") proto.RegisterType((*Event)(nil), "mvccpb.Event") } func init() { proto.RegisterFile("kv.proto", fileDescriptor_2216fe83c9c12408) } var fileDescriptor_2216fe83c9c12408 = []byte{ // 308 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xc1, 0x4e, 0xb3, 0x40, 0x14, 0x85, 0x3b, 0xa5, 0xa5, 0xfd, 0x6f, 0x9b, 0xfe, 0x64, 0x62, 0x22, 0x1b, 0x09, 0x76, 0x63, 0x8d, 0x09, 0x24, 0xed, 0x1b, 0x18, 0x59, 0xd5, 0x85, 0x21, 0xe8, 0xc2, 0x4d, 0x43, 0xe1, 0xc6, 0x10, 0x4a, 0x67, 0x42, 0xf1, 0x26, 0xbc, 0x89, 0x7b, 0xf7, 0x3e, 0x87, 0x4b, 0x1f, 0xc1, 0xe0, 0x8b, 0x18, 0x66, 0xa4, 0x6e, 0xdc, 0xc0, 0x9c, 0x73, 0xbe, 0xcc, 0x3d, 0x37, 0x03, 0xe3, 0x9c, 0x3c, 0x59, 0x8a, 0x4a, 0x70, 0xb3, 0xa0, 0x24, 0x91, 0xdb, 0xf9, 0x1b, 0x83, 0xf1, 0x1a, 0xeb, 0x87, 0x78, 0xf7, 0x8c, 0xdc, 0x02, 0x23, 0xc7, 0xda, 0x66, 0x2e, 0x5b, 0x4c, 0xc3, 0xf6, 0xc8, 0x2f, 0xe0, 0x7f, 0x52, 0x62, 0x5c, 0xe1, 0xa6, 0x44, 0xca, 0x0e, 0x99, 0xd8, 0xdb, 0x7d, 0x97, 0x2d, 0x8c, 0x70, 0xa6, 0xed, 0xf0, 0xc7, 0xe5, 0xe7, 0x30, 0x2d, 0x44, 0xfa, 0x4b, 0x19, 0x8a, 0x9a, 0x14, 0x22, 0x3d, 0x22, 0x36, 0x8c, 0x08, 0x4b, 0x95, 0x0e, 0x54, 0xda, 0x49, 0x7e, 0x02, 0x43, 0x6a, 0x0b, 0xd8, 0x43, 0x35, 0x59, 0x8b, 0xd6, 0xdd, 0x61, 0x7c, 0x40, 0xdb, 0x54, 0xb4, 0x16, 0xf3, 0x57, 0x06, 0xc3, 0x80, 0x70, 0x5f, 0xf1, 0x2b, 0x18, 0x54, 0xb5, 0x44, 0x55, 0x77, 0xb6, 0x3c, 0xf5, 0xf4, 0x46, 0x9e, 0x0a, 0xf5, 0x37, 0xaa, 0x25, 0x86, 0x0a, 0xe2, 0x2e, 0xf4, 0x73, 0x52, 0xdd, 0x27, 0x4b, 0xab, 0x43, 0xbb, 0xc5, 0xc3, 0x7e, 0x4e, 0xfc, 0x12, 0x46, 0xb2, 0x44, 0xda, 0xe4, 0xa4, 0xca, 0xff, 0x85, 0x99, 0x2d, 0xb0, 0xa6, 0xb9, 0x0b, 0xff, 0x8e, 0xf7, 0xf3, 0x11, 0x18, 0x77, 0xf7, 0x91, 0xd5, 0xe3, 0x00, 0xe6, 0x4d, 0x70, 0x1b, 0x44, 0x81, 0xc5, 0xae, 0xfd, 0xf7, 0xc6, 0x61, 0x1f, 0x8d, 0xc3, 0x3e, 0x1b, 0x87, 0xbd, 0x7c, 0x39, 0xbd, 0xc7, 0xb3, 0x27, 0xe1, 0x61, 0x95, 0xa4, 0x5e, 0x26, 0xfc, 0xf6, 0xef, 0xc7, 0x32, 0xf3, 0x69, 0xe5, 0xeb, 0x19, 0x5b, 0x53, 0x3d, 0xcb, 0xea, 0x3b, 0x00, 0x00, 0xff, 0xff, 0xcb, 0xc0, 0x08, 0x63, 0xa2, 0x01, 0x00, 0x00, } func (m *KeyValue) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *KeyValue) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *KeyValue) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Lease != 0 { i = encodeVarintKv(dAtA, i, uint64(m.Lease)) i-- dAtA[i] = 0x30 } if len(m.Value) > 0 { i -= len(m.Value) copy(dAtA[i:], m.Value) i = encodeVarintKv(dAtA, i, uint64(len(m.Value))) i-- dAtA[i] = 0x2a } if m.Version != 0 { i = encodeVarintKv(dAtA, i, uint64(m.Version)) i-- dAtA[i] = 0x20 } if m.ModRevision != 0 { i = encodeVarintKv(dAtA, i, uint64(m.ModRevision)) i-- dAtA[i] = 0x18 } if m.CreateRevision != 0 { i = encodeVarintKv(dAtA, i, uint64(m.CreateRevision)) i-- dAtA[i] = 0x10 } if len(m.Key) > 0 { i -= len(m.Key) copy(dAtA[i:], m.Key) i = encodeVarintKv(dAtA, i, uint64(len(m.Key))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *Event) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Event) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Event) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.PrevKv != nil { { size, err := m.PrevKv.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintKv(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } if m.Kv != nil { { size, err := m.Kv.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintKv(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } if m.Type != 0 { i = encodeVarintKv(dAtA, i, uint64(m.Type)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func encodeVarintKv(dAtA []byte, offset int, v uint64) int { offset -= sovKv(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) return base } func (m *KeyValue) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Key) if l > 0 { n += 1 + l + sovKv(uint64(l)) } if m.CreateRevision != 0 { n += 1 + sovKv(uint64(m.CreateRevision)) } if m.ModRevision != 0 { n += 1 + sovKv(uint64(m.ModRevision)) } if m.Version != 0 { n += 1 + sovKv(uint64(m.Version)) } l = len(m.Value) if l > 0 { n += 1 + l + sovKv(uint64(l)) } if m.Lease != 0 { n += 1 + sovKv(uint64(m.Lease)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *Event) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Type != 0 { n += 1 + sovKv(uint64(m.Type)) } if m.Kv != nil { l = m.Kv.Size() n += 1 + l + sovKv(uint64(l)) } if m.PrevKv != nil { l = m.PrevKv.Size() n += 1 + l + sovKv(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func sovKv(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } func sozKv(x uint64) (n int) { return sovKv(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (m *KeyValue) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowKv } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: KeyValue: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: KeyValue: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowKv } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthKv } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthKv } if postIndex > l { return io.ErrUnexpectedEOF } m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) if m.Key == nil { m.Key = []byte{} } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field CreateRevision", wireType) } m.CreateRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowKv } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.CreateRevision |= int64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ModRevision", wireType) } m.ModRevision = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowKv } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ModRevision |= int64(b&0x7F) << shift if b < 0x80 { break } } case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) } m.Version = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowKv } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Version |= int64(b&0x7F) << shift if b < 0x80 { break } } case 5: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowKv } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthKv } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthKv } if postIndex > l { return io.ErrUnexpectedEOF } m.Value = append(m.Value[:0], dAtA[iNdEx:postIndex]...) if m.Value == nil { m.Value = []byte{} } iNdEx = postIndex case 6: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Lease", wireType) } m.Lease = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowKv } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Lease |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipKv(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthKv } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *Event) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowKv } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: Event: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: Event: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) } m.Type = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowKv } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Type |= Event_EventType(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Kv", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowKv } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthKv } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthKv } if postIndex > l { return io.ErrUnexpectedEOF } if m.Kv == nil { m.Kv = &KeyValue{} } if err := m.Kv.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PrevKv", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowKv } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthKv } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthKv } if postIndex > l { return io.ErrUnexpectedEOF } if m.PrevKv == nil { m.PrevKv = &KeyValue{} } if err := m.PrevKv.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipKv(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthKv } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func skipKv(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowKv } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } wireType := int(wire & 0x7) switch wireType { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowKv } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } iNdEx++ if dAtA[iNdEx-1] < 0x80 { break } } case 1: iNdEx += 8 case 2: var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowKv } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { break } } if length < 0 { return 0, ErrInvalidLengthKv } iNdEx += length case 3: depth++ case 4: if depth == 0 { return 0, ErrUnexpectedEndOfGroupKv } depth-- case 5: iNdEx += 4 default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { return 0, ErrInvalidLengthKv } if depth == 0 { return iNdEx, nil } } return 0, io.ErrUnexpectedEOF } var ( ErrInvalidLengthKv = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowKv = fmt.Errorf("proto: integer overflow") ErrUnexpectedEndOfGroupKv = fmt.Errorf("proto: unexpected end of group") ) ================================================ FILE: api/mvccpb/kv.proto ================================================ syntax = "proto3"; package mvccpb; option go_package = "go.etcd.io/etcd/api/v3/mvccpb"; message KeyValue { // key is the key in bytes. An empty key is not allowed. bytes key = 1; // create_revision is the revision of last creation on this key. int64 create_revision = 2; // mod_revision is the revision of last modification on this key. int64 mod_revision = 3; // version is the version of the key. A deletion resets // the version to zero and any modification of the key // increases its version. int64 version = 4; // value is the value held by the key, in bytes. bytes value = 5; // lease is the ID of the lease that attached to key. // When the attached lease expires, the key will be deleted. // If lease is 0, then no lease is attached to the key. int64 lease = 6; } message Event { enum EventType { PUT = 0; DELETE = 1; } // type is the kind of event. If type is a PUT, it indicates // new data has been stored to the key. If type is a DELETE, // it indicates the key was deleted. EventType type = 1; // kv holds the KeyValue for the event. // A PUT event contains current kv pair. // A PUT event with kv.Version=1 indicates the creation of a key. // A DELETE/EXPIRE event contains the deleted key with // its modification revision set to the revision of deletion. KeyValue kv = 2; // prev_kv holds the key-value pair before the event happens. KeyValue prev_kv = 3; } ================================================ FILE: api/v3rpc/rpctypes/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package rpctypes has types and values shared by the etcd server and client for v3 RPC interaction. package rpctypes ================================================ FILE: api/v3rpc/rpctypes/error.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rpctypes import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // server-side error var ( ErrGRPCEmptyKey = status.Error(codes.InvalidArgument, "etcdserver: key is not provided") ErrGRPCKeyNotFound = status.Error(codes.InvalidArgument, "etcdserver: key not found") ErrGRPCValueProvided = status.Error(codes.InvalidArgument, "etcdserver: value is provided") ErrGRPCLeaseProvided = status.Error(codes.InvalidArgument, "etcdserver: lease is provided") ErrGRPCTooManyOps = status.Error(codes.InvalidArgument, "etcdserver: too many operations in txn request") ErrGRPCDuplicateKey = status.Error(codes.InvalidArgument, "etcdserver: duplicate key given in txn request") ErrGRPCInvalidClientAPIVersion = status.Error(codes.InvalidArgument, "etcdserver: invalid client api version") ErrGRPCInvalidSortOption = status.Error(codes.InvalidArgument, "etcdserver: invalid sort option") ErrGRPCCompacted = status.Error(codes.OutOfRange, "etcdserver: mvcc: required revision has been compacted") ErrGRPCFutureRev = status.Error(codes.OutOfRange, "etcdserver: mvcc: required revision is a future revision") ErrGRPCNoSpace = status.Error(codes.ResourceExhausted, "etcdserver: mvcc: database space exceeded") ErrGRPCLeaseNotFound = status.Error(codes.NotFound, "etcdserver: requested lease not found") ErrGRPCLeaseExist = status.Error(codes.FailedPrecondition, "etcdserver: lease already exists") ErrGRPCLeaseTTLTooLarge = status.Error(codes.OutOfRange, "etcdserver: too large lease TTL") ErrGRPCWatchCanceled = status.Error(codes.Canceled, "etcdserver: watch canceled") ErrGRPCMemberExist = status.Error(codes.FailedPrecondition, "etcdserver: member ID already exist") ErrGRPCPeerURLExist = status.Error(codes.FailedPrecondition, "etcdserver: Peer URLs already exists") ErrGRPCMemberNotEnoughStarted = status.Error(codes.FailedPrecondition, "etcdserver: re-configuration failed due to not enough started members") ErrGRPCMemberBadURLs = status.Error(codes.InvalidArgument, "etcdserver: given member URLs are invalid") ErrGRPCMemberNotFound = status.Error(codes.NotFound, "etcdserver: member not found") ErrGRPCMemberNotLearner = status.Error(codes.FailedPrecondition, "etcdserver: can only promote a learner member") ErrGRPCLearnerNotReady = status.Error(codes.FailedPrecondition, "etcdserver: can only promote a learner member which is in sync with leader") ErrGRPCTooManyLearners = status.Error(codes.FailedPrecondition, "etcdserver: too many learner members in cluster") ErrGRPCClusterIDMismatch = status.Error(codes.FailedPrecondition, "etcdserver: cluster ID mismatch") //revive:disable:var-naming // Deprecated: Please use ErrGRPCClusterIDMismatch. ErrGRPCClusterIdMismatch = ErrGRPCClusterIDMismatch //revive:enable:var-naming ErrGRPCRequestTooLarge = status.Error(codes.InvalidArgument, "etcdserver: request is too large") ErrGRPCRequestTooManyRequests = status.Error(codes.ResourceExhausted, "etcdserver: too many requests") ErrGRPCRootUserNotExist = status.Error(codes.FailedPrecondition, "etcdserver: root user does not exist") ErrGRPCRootRoleNotExist = status.Error(codes.FailedPrecondition, "etcdserver: root user does not have root role") ErrGRPCUserAlreadyExist = status.Error(codes.FailedPrecondition, "etcdserver: user name already exists") ErrGRPCUserEmpty = status.Error(codes.InvalidArgument, "etcdserver: user name is empty") ErrGRPCUserNotFound = status.Error(codes.FailedPrecondition, "etcdserver: user name not found") ErrGRPCRoleAlreadyExist = status.Error(codes.FailedPrecondition, "etcdserver: role name already exists") ErrGRPCRoleNotFound = status.Error(codes.FailedPrecondition, "etcdserver: role name not found") ErrGRPCRoleEmpty = status.Error(codes.InvalidArgument, "etcdserver: role name is empty") ErrGRPCAuthFailed = status.Error(codes.InvalidArgument, "etcdserver: authentication failed, invalid user ID or password") ErrGRPCPermissionNotGiven = status.Error(codes.InvalidArgument, "etcdserver: permission not given") ErrGRPCPermissionDenied = status.Error(codes.PermissionDenied, "etcdserver: permission denied") ErrGRPCRoleNotGranted = status.Error(codes.FailedPrecondition, "etcdserver: role is not granted to the user") ErrGRPCPermissionNotGranted = status.Error(codes.FailedPrecondition, "etcdserver: permission is not granted to the role") ErrGRPCAuthNotEnabled = status.Error(codes.FailedPrecondition, "etcdserver: authentication is not enabled") ErrGRPCInvalidAuthToken = status.Error(codes.Unauthenticated, "etcdserver: invalid auth token") ErrGRPCInvalidAuthMgmt = status.Error(codes.InvalidArgument, "etcdserver: invalid auth management") ErrGRPCAuthOldRevision = status.Error(codes.InvalidArgument, "etcdserver: revision of auth store is old") ErrGRPCNoLeader = status.Error(codes.Unavailable, "etcdserver: no leader") ErrGRPCNotLeader = status.Error(codes.FailedPrecondition, "etcdserver: not leader") ErrGRPCLeaderChanged = status.Error(codes.Unavailable, "etcdserver: leader changed") ErrGRPCNotCapable = status.Error(codes.FailedPrecondition, "etcdserver: not capable") ErrGRPCStopped = status.Error(codes.Unavailable, "etcdserver: server stopped") ErrGRPCTimeout = status.Error(codes.Unavailable, "etcdserver: request timed out") ErrGRPCTimeoutDueToLeaderFail = status.Error(codes.Unavailable, "etcdserver: request timed out, possibly due to previous leader failure") ErrGRPCTimeoutDueToConnectionLost = status.Error(codes.Unavailable, "etcdserver: request timed out, possibly due to connection lost") ErrGRPCTimeoutWaitAppliedIndex = status.Error(codes.Unavailable, "etcdserver: request timed out, waiting for the applied index took too long") ErrGRPCUnhealthy = status.Error(codes.Unavailable, "etcdserver: unhealthy cluster") ErrGRPCCorrupt = status.Error(codes.DataLoss, "etcdserver: corrupt cluster") ErrGRPCNotSupportedForLearner = status.Error(codes.FailedPrecondition, "etcdserver: rpc not supported for learner") ErrGRPCBadLeaderTransferee = status.Error(codes.FailedPrecondition, "etcdserver: bad leader transferee") ErrGRPCWrongDowngradeVersionFormat = status.Error(codes.InvalidArgument, "etcdserver: wrong downgrade target version format") ErrGRPCInvalidDowngradeTargetVersion = status.Error(codes.InvalidArgument, "etcdserver: invalid downgrade target version") ErrGRPCClusterVersionUnavailable = status.Error(codes.FailedPrecondition, "etcdserver: cluster version not found during downgrade") ErrGRPCDowngradeInProcess = status.Error(codes.FailedPrecondition, "etcdserver: cluster has a downgrade job in progress") ErrGRPCNoInflightDowngrade = status.Error(codes.FailedPrecondition, "etcdserver: no inflight downgrade job") ErrGRPCCanceled = status.Error(codes.Canceled, "etcdserver: request canceled") ErrGRPCDeadlineExceeded = status.Error(codes.DeadlineExceeded, "etcdserver: context deadline exceeded") errStringToError = map[string]error{ ErrorDesc(ErrGRPCEmptyKey): ErrGRPCEmptyKey, ErrorDesc(ErrGRPCKeyNotFound): ErrGRPCKeyNotFound, ErrorDesc(ErrGRPCValueProvided): ErrGRPCValueProvided, ErrorDesc(ErrGRPCLeaseProvided): ErrGRPCLeaseProvided, ErrorDesc(ErrGRPCTooManyOps): ErrGRPCTooManyOps, ErrorDesc(ErrGRPCDuplicateKey): ErrGRPCDuplicateKey, ErrorDesc(ErrGRPCInvalidSortOption): ErrGRPCInvalidSortOption, ErrorDesc(ErrGRPCCompacted): ErrGRPCCompacted, ErrorDesc(ErrGRPCFutureRev): ErrGRPCFutureRev, ErrorDesc(ErrGRPCNoSpace): ErrGRPCNoSpace, ErrorDesc(ErrGRPCLeaseNotFound): ErrGRPCLeaseNotFound, ErrorDesc(ErrGRPCLeaseExist): ErrGRPCLeaseExist, ErrorDesc(ErrGRPCLeaseTTLTooLarge): ErrGRPCLeaseTTLTooLarge, ErrorDesc(ErrGRPCMemberExist): ErrGRPCMemberExist, ErrorDesc(ErrGRPCPeerURLExist): ErrGRPCPeerURLExist, ErrorDesc(ErrGRPCMemberNotEnoughStarted): ErrGRPCMemberNotEnoughStarted, ErrorDesc(ErrGRPCMemberBadURLs): ErrGRPCMemberBadURLs, ErrorDesc(ErrGRPCMemberNotFound): ErrGRPCMemberNotFound, ErrorDesc(ErrGRPCMemberNotLearner): ErrGRPCMemberNotLearner, ErrorDesc(ErrGRPCLearnerNotReady): ErrGRPCLearnerNotReady, ErrorDesc(ErrGRPCTooManyLearners): ErrGRPCTooManyLearners, ErrorDesc(ErrGRPCClusterIDMismatch): ErrGRPCClusterIDMismatch, ErrorDesc(ErrGRPCRequestTooLarge): ErrGRPCRequestTooLarge, ErrorDesc(ErrGRPCRequestTooManyRequests): ErrGRPCRequestTooManyRequests, ErrorDesc(ErrGRPCRootUserNotExist): ErrGRPCRootUserNotExist, ErrorDesc(ErrGRPCRootRoleNotExist): ErrGRPCRootRoleNotExist, ErrorDesc(ErrGRPCUserAlreadyExist): ErrGRPCUserAlreadyExist, ErrorDesc(ErrGRPCUserEmpty): ErrGRPCUserEmpty, ErrorDesc(ErrGRPCUserNotFound): ErrGRPCUserNotFound, ErrorDesc(ErrGRPCRoleAlreadyExist): ErrGRPCRoleAlreadyExist, ErrorDesc(ErrGRPCRoleNotFound): ErrGRPCRoleNotFound, ErrorDesc(ErrGRPCRoleEmpty): ErrGRPCRoleEmpty, ErrorDesc(ErrGRPCAuthFailed): ErrGRPCAuthFailed, ErrorDesc(ErrGRPCPermissionDenied): ErrGRPCPermissionDenied, ErrorDesc(ErrGRPCRoleNotGranted): ErrGRPCRoleNotGranted, ErrorDesc(ErrGRPCPermissionNotGranted): ErrGRPCPermissionNotGranted, ErrorDesc(ErrGRPCAuthNotEnabled): ErrGRPCAuthNotEnabled, ErrorDesc(ErrGRPCInvalidAuthToken): ErrGRPCInvalidAuthToken, ErrorDesc(ErrGRPCInvalidAuthMgmt): ErrGRPCInvalidAuthMgmt, ErrorDesc(ErrGRPCAuthOldRevision): ErrGRPCAuthOldRevision, ErrorDesc(ErrGRPCNoLeader): ErrGRPCNoLeader, ErrorDesc(ErrGRPCNotLeader): ErrGRPCNotLeader, ErrorDesc(ErrGRPCLeaderChanged): ErrGRPCLeaderChanged, ErrorDesc(ErrGRPCNotCapable): ErrGRPCNotCapable, ErrorDesc(ErrGRPCStopped): ErrGRPCStopped, ErrorDesc(ErrGRPCTimeout): ErrGRPCTimeout, ErrorDesc(ErrGRPCTimeoutDueToLeaderFail): ErrGRPCTimeoutDueToLeaderFail, ErrorDesc(ErrGRPCTimeoutDueToConnectionLost): ErrGRPCTimeoutDueToConnectionLost, ErrorDesc(ErrGRPCUnhealthy): ErrGRPCUnhealthy, ErrorDesc(ErrGRPCCorrupt): ErrGRPCCorrupt, ErrorDesc(ErrGRPCNotSupportedForLearner): ErrGRPCNotSupportedForLearner, ErrorDesc(ErrGRPCBadLeaderTransferee): ErrGRPCBadLeaderTransferee, ErrorDesc(ErrGRPCClusterVersionUnavailable): ErrGRPCClusterVersionUnavailable, ErrorDesc(ErrGRPCWrongDowngradeVersionFormat): ErrGRPCWrongDowngradeVersionFormat, ErrorDesc(ErrGRPCInvalidDowngradeTargetVersion): ErrGRPCInvalidDowngradeTargetVersion, ErrorDesc(ErrGRPCDowngradeInProcess): ErrGRPCDowngradeInProcess, ErrorDesc(ErrGRPCNoInflightDowngrade): ErrGRPCNoInflightDowngrade, } ) // client-side error var ( ErrEmptyKey = Error(ErrGRPCEmptyKey) ErrKeyNotFound = Error(ErrGRPCKeyNotFound) ErrValueProvided = Error(ErrGRPCValueProvided) ErrLeaseProvided = Error(ErrGRPCLeaseProvided) ErrTooManyOps = Error(ErrGRPCTooManyOps) ErrDuplicateKey = Error(ErrGRPCDuplicateKey) ErrInvalidSortOption = Error(ErrGRPCInvalidSortOption) ErrCompacted = Error(ErrGRPCCompacted) ErrFutureRev = Error(ErrGRPCFutureRev) ErrNoSpace = Error(ErrGRPCNoSpace) ErrLeaseNotFound = Error(ErrGRPCLeaseNotFound) ErrLeaseExist = Error(ErrGRPCLeaseExist) ErrLeaseTTLTooLarge = Error(ErrGRPCLeaseTTLTooLarge) ErrMemberExist = Error(ErrGRPCMemberExist) ErrPeerURLExist = Error(ErrGRPCPeerURLExist) ErrMemberNotEnoughStarted = Error(ErrGRPCMemberNotEnoughStarted) ErrMemberBadURLs = Error(ErrGRPCMemberBadURLs) ErrMemberNotFound = Error(ErrGRPCMemberNotFound) ErrMemberNotLearner = Error(ErrGRPCMemberNotLearner) ErrMemberLearnerNotReady = Error(ErrGRPCLearnerNotReady) ErrTooManyLearners = Error(ErrGRPCTooManyLearners) ErrRequestTooLarge = Error(ErrGRPCRequestTooLarge) ErrTooManyRequests = Error(ErrGRPCRequestTooManyRequests) ErrRootUserNotExist = Error(ErrGRPCRootUserNotExist) ErrRootRoleNotExist = Error(ErrGRPCRootRoleNotExist) ErrUserAlreadyExist = Error(ErrGRPCUserAlreadyExist) ErrUserEmpty = Error(ErrGRPCUserEmpty) ErrUserNotFound = Error(ErrGRPCUserNotFound) ErrRoleAlreadyExist = Error(ErrGRPCRoleAlreadyExist) ErrRoleNotFound = Error(ErrGRPCRoleNotFound) ErrRoleEmpty = Error(ErrGRPCRoleEmpty) ErrAuthFailed = Error(ErrGRPCAuthFailed) ErrPermissionDenied = Error(ErrGRPCPermissionDenied) ErrRoleNotGranted = Error(ErrGRPCRoleNotGranted) ErrPermissionNotGranted = Error(ErrGRPCPermissionNotGranted) ErrAuthNotEnabled = Error(ErrGRPCAuthNotEnabled) ErrInvalidAuthToken = Error(ErrGRPCInvalidAuthToken) ErrAuthOldRevision = Error(ErrGRPCAuthOldRevision) ErrInvalidAuthMgmt = Error(ErrGRPCInvalidAuthMgmt) ErrClusterIDMismatch = Error(ErrGRPCClusterIDMismatch) //revive:disable:var-naming // Deprecated: Please use ErrClusterIDMismatch. ErrClusterIdMismatch = ErrClusterIDMismatch //revive:enable:var-naming ErrNoLeader = Error(ErrGRPCNoLeader) ErrNotLeader = Error(ErrGRPCNotLeader) ErrLeaderChanged = Error(ErrGRPCLeaderChanged) ErrNotCapable = Error(ErrGRPCNotCapable) ErrStopped = Error(ErrGRPCStopped) ErrTimeout = Error(ErrGRPCTimeout) ErrTimeoutDueToLeaderFail = Error(ErrGRPCTimeoutDueToLeaderFail) ErrTimeoutDueToConnectionLost = Error(ErrGRPCTimeoutDueToConnectionLost) ErrTimeoutWaitAppliedIndex = Error(ErrGRPCTimeoutWaitAppliedIndex) ErrUnhealthy = Error(ErrGRPCUnhealthy) ErrCorrupt = Error(ErrGRPCCorrupt) ErrBadLeaderTransferee = Error(ErrGRPCBadLeaderTransferee) ErrClusterVersionUnavailable = Error(ErrGRPCClusterVersionUnavailable) ErrWrongDowngradeVersionFormat = Error(ErrGRPCWrongDowngradeVersionFormat) ErrInvalidDowngradeTargetVersion = Error(ErrGRPCInvalidDowngradeTargetVersion) ErrDowngradeInProcess = Error(ErrGRPCDowngradeInProcess) ErrNoInflightDowngrade = Error(ErrGRPCNoInflightDowngrade) ) // EtcdError defines gRPC server errors. // (https://github.com/grpc/grpc-go/blob/master/rpc_util.go#L319-L323) type EtcdError struct { code codes.Code desc string } // Code returns grpc/codes.Code. // TODO: define clientv3/codes.Code. func (e EtcdError) Code() codes.Code { return e.code } func (e EtcdError) Error() string { return e.desc } func Error(err error) error { if err == nil { return nil } verr, ok := errStringToError[ErrorDesc(err)] if !ok { // not gRPC error return err } ev, ok := status.FromError(verr) var desc string if ok { desc = ev.Message() } else { desc = verr.Error() } return EtcdError{code: ev.Code(), desc: desc} } func ErrorDesc(err error) string { if s, ok := status.FromError(err); ok { return s.Message() } return err.Error() } ================================================ FILE: api/v3rpc/rpctypes/error_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rpctypes import ( "errors" "testing" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) func TestConvert(t *testing.T) { e1 := status.Error(codes.InvalidArgument, "etcdserver: key is not provided") e2 := ErrGRPCEmptyKey var e3 EtcdError errors.As(ErrEmptyKey, &e3) require.Equal(t, e1.Error(), e2.Error()) if ev1, ok := status.FromError(e1); ok { require.Equal(t, ev1.Code(), e3.Code()) } require.NotEqual(t, e1.Error(), e3.Error()) if ev2, ok := status.FromError(e2); ok { require.Equal(t, ev2.Code(), e3.Code()) } } ================================================ FILE: api/v3rpc/rpctypes/md.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rpctypes var ( MetadataRequireLeaderKey = "hasleader" MetadataHasLeader = "true" MetadataClientAPIVersionKey = "client-api-version" ) ================================================ FILE: api/v3rpc/rpctypes/metadatafields.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rpctypes var ( TokenFieldNameGRPC = "token" TokenFieldNameSwagger = "authorization" ) // TokenFieldNameGRPCKey is used as a key of context to store token. type TokenFieldNameGRPCKey struct{} ================================================ FILE: api/version/version.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package version implements etcd version parsing and contains latest version // information. package version import ( "fmt" "strings" "github.com/coreos/go-semver/semver" ) var ( // MinClusterVersion is the min cluster version this etcd binary is compatible with. MinClusterVersion = "3.0.0" Version = "3.7.0-alpha.0" APIVersion = "unknown" // Git SHA Value will be set during build GitSHA = "Not provided (use ./build instead of go build)" ) // Get all constant versions defined in a centralized place. var ( V3_0 = semver.Version{Major: 3, Minor: 0} V3_1 = semver.Version{Major: 3, Minor: 1} V3_2 = semver.Version{Major: 3, Minor: 2} V3_3 = semver.Version{Major: 3, Minor: 3} V3_4 = semver.Version{Major: 3, Minor: 4} V3_5 = semver.Version{Major: 3, Minor: 5} V3_6 = semver.Version{Major: 3, Minor: 6} V3_7 = semver.Version{Major: 3, Minor: 7} V3_8 = semver.Version{Major: 3, Minor: 8} V4_0 = semver.Version{Major: 4, Minor: 0} // AllVersions keeps all the versions in ascending order. AllVersions = []semver.Version{V3_0, V3_1, V3_2, V3_3, V3_4, V3_5, V3_6, V3_7, V4_0} ) func init() { ver, err := semver.NewVersion(Version) if err == nil { APIVersion = fmt.Sprintf("%d.%d", ver.Major, ver.Minor) } } type Versions struct { Server string `json:"etcdserver"` Cluster string `json:"etcdcluster"` Storage string `json:"storage"` // TODO: raft state machine version } // Cluster only keeps the major.minor. func Cluster(v string) string { vs := strings.Split(v, ".") if len(vs) <= 2 { return v } return fmt.Sprintf("%s.%s", vs[0], vs[1]) } func Compare(ver1, ver2 semver.Version) int { return ver1.Compare(ver2) } func LessThan(ver1, ver2 semver.Version) bool { return ver1.LessThan(ver2) } func Equal(ver1, ver2 semver.Version) bool { return ver1.Equal(ver2) } ================================================ FILE: api/version/version_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package version import ( "testing" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" ) func TestVersionCompare(t *testing.T) { cases := []struct { name string ver1 semver.Version ver2 semver.Version expectedCompareResult int expectedLessThanResult bool expectedEqualResult bool }{ { name: "ver1 should be great than ver2", ver1: V3_5, ver2: V3_4, expectedCompareResult: 1, expectedLessThanResult: false, expectedEqualResult: false, }, { name: "ver1(4.0) should be great than ver2", ver1: V4_0, ver2: V3_7, expectedCompareResult: 1, expectedLessThanResult: false, expectedEqualResult: false, }, { name: "ver1 should be less than ver2", ver1: V3_5, ver2: V3_6, expectedCompareResult: -1, expectedLessThanResult: true, expectedEqualResult: false, }, { name: "ver1 should be less than ver2 (4.0)", ver1: V3_5, ver2: V4_0, expectedCompareResult: -1, expectedLessThanResult: true, expectedEqualResult: false, }, { name: "ver1 should be equal to ver2", ver1: V3_5, ver2: V3_5, expectedCompareResult: 0, expectedLessThanResult: false, expectedEqualResult: true, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { compareResult := Compare(tc.ver1, tc.ver2) lessThanResult := LessThan(tc.ver1, tc.ver2) equalResult := Equal(tc.ver1, tc.ver2) assert.Equal(t, tc.expectedCompareResult, compareResult) assert.Equal(t, tc.expectedLessThanResult, lessThanResult) assert.Equal(t, tc.expectedEqualResult, equalResult) }) } } ================================================ FILE: api/versionpb/version.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: version.proto package versionpb import ( fmt "fmt" math "math" proto "github.com/golang/protobuf/proto" descriptorpb "google.golang.org/protobuf/types/descriptorpb" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package var E_EtcdVersionMsg = &proto.ExtensionDesc{ ExtendedType: (*descriptorpb.MessageOptions)(nil), ExtensionType: (*string)(nil), Field: 50000, Name: "versionpb.etcd_version_msg", Tag: "bytes,50000,opt,name=etcd_version_msg", Filename: "version.proto", } var E_EtcdVersionField = &proto.ExtensionDesc{ ExtendedType: (*descriptorpb.FieldOptions)(nil), ExtensionType: (*string)(nil), Field: 50001, Name: "versionpb.etcd_version_field", Tag: "bytes,50001,opt,name=etcd_version_field", Filename: "version.proto", } var E_EtcdVersionEnum = &proto.ExtensionDesc{ ExtendedType: (*descriptorpb.EnumOptions)(nil), ExtensionType: (*string)(nil), Field: 50002, Name: "versionpb.etcd_version_enum", Tag: "bytes,50002,opt,name=etcd_version_enum", Filename: "version.proto", } var E_EtcdVersionEnumValue = &proto.ExtensionDesc{ ExtendedType: (*descriptorpb.EnumValueOptions)(nil), ExtensionType: (*string)(nil), Field: 50003, Name: "versionpb.etcd_version_enum_value", Tag: "bytes,50003,opt,name=etcd_version_enum_value", Filename: "version.proto", } func init() { proto.RegisterExtension(E_EtcdVersionMsg) proto.RegisterExtension(E_EtcdVersionField) proto.RegisterExtension(E_EtcdVersionEnum) proto.RegisterExtension(E_EtcdVersionEnumValue) } func init() { proto.RegisterFile("version.proto", fileDescriptor_7d2c07d79758f814) } var fileDescriptor_7d2c07d79758f814 = []byte{ // 271 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2d, 0x4b, 0x2d, 0x2a, 0xce, 0xcc, 0xcf, 0xd3, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x84, 0x72, 0x0b, 0x92, 0xa4, 0x14, 0xd2, 0xf3, 0xf3, 0xd3, 0x73, 0x52, 0xf5, 0xc1, 0x12, 0x49, 0xa5, 0x69, 0xfa, 0x29, 0xa9, 0xc5, 0xc9, 0x45, 0x99, 0x05, 0x25, 0xf9, 0x45, 0x10, 0xc5, 0x56, 0x7e, 0x5c, 0x02, 0xa9, 0x25, 0xc9, 0x29, 0xf1, 0x50, 0x3d, 0xf1, 0xb9, 0xc5, 0xe9, 0x42, 0xf2, 0x7a, 0x10, 0x6d, 0x7a, 0x30, 0x6d, 0x7a, 0xbe, 0xa9, 0xc5, 0xc5, 0x89, 0xe9, 0xa9, 0xfe, 0x05, 0x25, 0x99, 0xf9, 0x79, 0xc5, 0x12, 0x17, 0xda, 0x98, 0x15, 0x18, 0x35, 0x38, 0x83, 0xf8, 0x40, 0x5a, 0xc3, 0x20, 0x3a, 0x7d, 0x8b, 0xd3, 0x3b, 0x18, 0x19, 0xad, 0x02, 0xb8, 0x84, 0x50, 0xcc, 0x4b, 0xcb, 0x4c, 0xcd, 0x49, 0x11, 0x92, 0xc5, 0x30, 0xd1, 0x0d, 0x24, 0x0e, 0x33, 0xef, 0x22, 0xd4, 0x3c, 0x01, 0x24, 0xf3, 0xc0, 0x0a, 0x40, 0x26, 0xfa, 0x72, 0x09, 0xa2, 0x98, 0x98, 0x9a, 0x57, 0x9a, 0x2b, 0x24, 0x83, 0x61, 0xa0, 0x6b, 0x5e, 0x69, 0x2e, 0xcc, 0xbc, 0x4b, 0x50, 0xf3, 0xf8, 0x91, 0xcc, 0x03, 0xc9, 0x83, 0x8c, 0x8b, 0xe5, 0x12, 0xc7, 0x30, 0x2e, 0xbe, 0x2c, 0x31, 0xa7, 0x34, 0x55, 0x48, 0x11, 0xab, 0xa1, 0x61, 0x20, 0x39, 0x98, 0xc9, 0x97, 0xa1, 0x26, 0x8b, 0xa0, 0x99, 0x0c, 0x56, 0xd4, 0xc1, 0xc8, 0xe8, 0x64, 0x74, 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, 0x1e, 0xc9, 0x31, 0xce, 0x78, 0x2c, 0xc7, 0x10, 0xa5, 0x90, 0x9e, 0xaf, 0x07, 0x52, 0xad, 0x97, 0x99, 0xaf, 0x0f, 0xa2, 0xf5, 0x13, 0x0b, 0x32, 0xf5, 0xcb, 0x8c, 0xf5, 0xe1, 0xb1, 0x94, 0xc4, 0x06, 0xb6, 0xcf, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x16, 0x4f, 0x52, 0x12, 0xc8, 0x01, 0x00, 0x00, } ================================================ FILE: api/versionpb/version.proto ================================================ syntax = "proto3"; package versionpb; import "google/protobuf/descriptor.proto"; option go_package = "go.etcd.io/etcd/api/v3/versionpb"; // Indicates etcd version that introduced the message, used to determine minimal etcd version required to interpret wal that includes this message. extend google.protobuf.MessageOptions { optional string etcd_version_msg = 50000; } // Indicates etcd version that introduced the field, used to determine minimal etcd version required to interpret wal that sets this field. extend google.protobuf.FieldOptions { optional string etcd_version_field = 50001; } // Indicates etcd version that introduced the enum, used to determine minimal etcd version required to interpret wal that uses this enum. extend google.protobuf.EnumOptions { optional string etcd_version_enum = 50002; } // Indicates etcd version that introduced the enum value, used to determine minimal etcd version required to interpret wal that sets this enum value. extend google.protobuf.EnumValueOptions { optional string etcd_version_enum_value = 50003; } ================================================ FILE: bill-of-materials.json ================================================ [ { "project": "github.com/VividCortex/ewma", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/anishathalye/porcupine", "licenses": [ { "type": "MIT License", "confidence": 0.96875 } ] }, { "project": "github.com/antithesishq/antithesis-sdk-go", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/beorn7/perks/quantile", "licenses": [ { "type": "MIT License", "confidence": 0.9891304347826086 } ] }, { "project": "github.com/bgentry/speakeasy", "licenses": [ { "type": "MIT License", "confidence": 0.9441624365482234 } ] }, { "project": "github.com/cenkalti/backoff/v5", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/cespare/xxhash/v2", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/cheggaaa/pb/v3", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9916666666666667 } ] }, { "project": "github.com/clipperhouse/displaywidth", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/clipperhouse/stringish", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/clipperhouse/uax29/v2", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/coreos/go-semver/semver", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/coreos/go-systemd/v22", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9966703662597114 } ] }, { "project": "github.com/creack/pty", "licenses": [ { "type": "MIT License", "confidence": 0.9891304347826086 } ] }, { "project": "github.com/davecgh/go-spew/spew", "licenses": [ { "type": "ISC License", "confidence": 0.9850746268656716 } ] }, { "project": "github.com/dustin/go-humanize", "licenses": [ { "type": "MIT License", "confidence": 0.96875 } ] }, { "project": "github.com/fatih/color", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/go-logr/logr", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/go-logr/stdr", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/gogo/protobuf", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9163346613545816 } ] }, { "project": "github.com/golang-jwt/jwt/v5", "licenses": [ { "type": "MIT License", "confidence": 0.9891304347826086 } ] }, { "project": "github.com/golang/protobuf", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "github.com/google/go-cmp/cmp", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "github.com/google/uuid", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "github.com/gorilla/websocket", "licenses": [ { "type": "BSD 2-clause \"Simplified\" License", "confidence": 0.9852216748768473 } ] }, { "project": "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/grpc-ecosystem/grpc-gateway/v2", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.979253112033195 } ] }, { "project": "github.com/inconshreveable/mousetrap", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/jonboulle/clockwork", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/mattn/go-colorable", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/mattn/go-isatty", "licenses": [ { "type": "MIT License", "confidence": 0.9587628865979382 } ] }, { "project": "github.com/mattn/go-runewidth", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/munnerz/goautoneg", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9794238683127572 } ] }, { "project": "github.com/olekukonko/cat", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/olekukonko/errors", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/olekukonko/ll", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/olekukonko/tablewriter", "licenses": [ { "type": "MIT License", "confidence": 0.9891304347826086 } ] }, { "project": "github.com/pmezard/go-difflib/difflib", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9830508474576272 } ] }, { "project": "github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "github.com/prometheus/client_golang/prometheus", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/prometheus/client_model/go", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/prometheus/common", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/prometheus/procfs", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/sirupsen/logrus", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/soheilhy/cmux", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/spf13/cobra", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9573241061130334 } ] }, { "project": "github.com/spf13/pflag", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "github.com/stretchr/testify", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "github.com/tmc/grpc-websocket-proxy/wsproxy", "licenses": [ { "type": "MIT License", "confidence": 0.9891304347826086 } ] }, { "project": "github.com/xiang90/probing", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "go.etcd.io/bbolt", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "go.etcd.io/etcd/api/v3", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9988925802879292 } ] }, { "project": "go.etcd.io/etcd/cache/v3", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9988925802879292 } ] }, { "project": "go.etcd.io/etcd/client/pkg/v3", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9988925802879292 } ] }, { "project": "go.etcd.io/etcd/client/v3", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9988925802879292 } ] }, { "project": "go.etcd.io/etcd/etcdctl/v3", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9988925802879292 } ] }, { "project": "go.etcd.io/etcd/etcdutl/v3", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9988925802879292 } ] }, { "project": "go.etcd.io/etcd/pkg/v3", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9988925802879292 } ] }, { "project": "go.etcd.io/etcd/server/v3", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9988925802879292 } ] }, { "project": "go.etcd.io/etcd/tests/v3", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9988925802879292 } ] }, { "project": "go.etcd.io/etcd/v3", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9988925802879292 } ] }, { "project": "go.etcd.io/gofail/runtime", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "go.etcd.io/raft/v3", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "go.opentelemetry.io/auto/sdk", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9647812166488794 } ] }, { "project": "go.opentelemetry.io/otel", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9647812166488794 } ] }, { "project": "go.opentelemetry.io/otel/exporters/otlp/otlptrace", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9647812166488794 } ] }, { "project": "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9647812166488794 } ] }, { "project": "go.opentelemetry.io/otel/metric", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9647812166488794 } ] }, { "project": "go.opentelemetry.io/otel/sdk", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9647812166488794 } ] }, { "project": "go.opentelemetry.io/otel/trace", "licenses": [ { "type": "Apache License 2.0", "confidence": 0.9647812166488794 } ] }, { "project": "go.opentelemetry.io/proto/otlp", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "go.uber.org/multierr", "licenses": [ { "type": "MIT License", "confidence": 0.9891304347826086 } ] }, { "project": "go.uber.org/zap", "licenses": [ { "type": "MIT License", "confidence": 0.9891304347826086 } ] }, { "project": "go.yaml.in/yaml/v2", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 }, { "type": "MIT License", "confidence": 0.8975609756097561 } ] }, { "project": "golang.org/x/crypto", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "golang.org/x/mod/semver", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "golang.org/x/net", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "golang.org/x/sync/errgroup", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "golang.org/x/sys", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "golang.org/x/text", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "golang.org/x/time/rate", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "golang.org/x/tools", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "google.golang.org/genproto/googleapis/api", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "google.golang.org/genproto/googleapis/rpc", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "google.golang.org/grpc", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "google.golang.org/protobuf", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "gopkg.in/natefinch/lumberjack.v2", "licenses": [ { "type": "MIT License", "confidence": 1 } ] }, { "project": "gopkg.in/yaml.v3", "licenses": [ { "type": "MIT License", "confidence": 0.7469879518072289 } ] }, { "project": "k8s.io/utils/internal/third_party/forked/golang/golang-lru", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 0.9663865546218487 } ] }, { "project": "k8s.io/utils/lru", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "k8s.io/utils/third_party/forked/golang/btree", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "sigs.k8s.io/yaml", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 }, { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 1 } ] } ] ================================================ FILE: bill-of-materials.override.json ================================================ [ { "project": "sigs.k8s.io/yaml", "licenses": [ { "type": "BSD 3-clause \"New\" or \"Revised\" License" } ] }, { "project": "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors", "licenses": [ { "type": "Apache License 2.0", "confidence": 1 } ] }, { "project": "github.com/inconshreveable/mousetrap", "licenses": [ { "type": "Apache License 2.0" } ] } ] ================================================ FILE: cache/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: cache/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/cache ================================================ FILE: cache/README.md ================================================ # etcd cache Experimental etcd client cache library. ================================================ FILE: cache/cache.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import ( "bytes" "context" "errors" "fmt" "sync" "time" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" ) var ( // Returned when an option combination isn’t yet handled by the cache (e.g. WithPrevKV, WithProgressNotify for Watch(), WithCountOnly for Get()). ErrUnsupportedRequest = errors.New("cache: unsupported request parameters") // Returned when the requested key or key‑range is invalid (empty or reversed) or lies outside c.prefix. ErrKeyRangeInvalid = errors.New("cache: invalid or out‑of‑range key range") ) // Cache buffers a single etcd Watch for a given key‐prefix and fan‑outs local watchers. type Cache struct { prefix string // prefix is the key-prefix this shard is responsible for ("" = root). cfg Config // immutable runtime configuration watcher clientv3.Watcher kv clientv3.KV demux *demux // demux fans incoming events out to active watchers and manages resync. store *store // last‑observed snapshot ready *ready stop context.CancelFunc waitGroup sync.WaitGroup internalCtx context.Context } // New builds a cache shard that watches only the requested prefix. // For the root cache pass "". func New(client *clientv3.Client, prefix string, opts ...Option) (*Cache, error) { cfg := defaultConfig() for _, opt := range opts { opt(&cfg) } if cfg.HistoryWindowSize <= 0 { return nil, fmt.Errorf("invalid HistoryWindowSize %d (must be > 0)", cfg.HistoryWindowSize) } if cfg.BTreeDegree < 2 { return nil, fmt.Errorf("invalid BTreeDegree %d (must be >= 2)", cfg.BTreeDegree) } internalCtx, cancel := context.WithCancel(context.Background()) cache := &Cache{ prefix: prefix, cfg: cfg, watcher: client.Watcher, kv: client.KV, store: newStore(cfg.BTreeDegree, cfg.HistoryWindowSize), ready: newReady(), stop: cancel, internalCtx: internalCtx, } cache.demux = NewDemux(internalCtx, &cache.waitGroup, cfg.HistoryWindowSize, cfg.ResyncInterval) cache.waitGroup.Add(1) go func() { defer cache.waitGroup.Done() cache.getWatchLoop() }() return cache, nil } // Watch registers a cache-backed watcher for a given key or prefix. // It returns a WatchChan that streams WatchResponses containing events. func (c *Cache) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan { if err := c.WaitReady(ctx); err != nil { emptyWatchChan := make(chan clientv3.WatchResponse) close(emptyWatchChan) return emptyWatchChan } op := clientv3.OpWatch(key, opts...) startRev := op.Rev() pred, err := c.validateWatch(key, op) if err != nil { ch := make(chan clientv3.WatchResponse, 1) ch <- clientv3.WatchResponse{Canceled: true, CancelReason: err.Error()} close(ch) return ch } w := newWatcher(c.cfg.PerWatcherBufferSize, pred) c.demux.Register(w, startRev) responseChan := make(chan clientv3.WatchResponse) c.waitGroup.Add(1) go func() { defer c.waitGroup.Done() defer close(responseChan) defer c.demux.Unregister(w) for { select { case <-ctx.Done(): return case <-c.internalCtx.Done(): return case resp, ok := <-w.respCh: if !ok { if w.cancelResp != nil { select { case <-ctx.Done(): case <-c.internalCtx.Done(): case responseChan <- *w.cancelResp: } } return } select { case <-ctx.Done(): return case <-c.internalCtx.Done(): return case responseChan <- resp: } } } }() return responseChan } func (c *Cache) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { if c.store.LatestRev() == 0 { if err := c.WaitReady(ctx); err != nil { return nil, err } } op := clientv3.OpGet(key, opts...) if _, err := c.validateGet(key, op); err != nil { return nil, err } startKey := []byte(key) endKey := op.RangeBytes() requestedRev := op.Rev() kvs, latestRev, err := c.store.Get(startKey, endKey, requestedRev) if err != nil { return nil, err } return &clientv3.GetResponse{ Header: &pb.ResponseHeader{Revision: latestRev}, Kvs: kvs, Count: int64(len(kvs)), }, nil } // Ready returns true if the snapshot has been loaded and the first watch has been confirmed. func (c *Cache) Ready() bool { return c.ready.Ready() } // WaitReady blocks until the cache is ready or the ctx is cancelled. func (c *Cache) WaitReady(ctx context.Context) error { return c.ready.WaitReady(ctx) } func (c *Cache) WaitForRevision(ctx context.Context, rev int64) error { for { if c.store.LatestRev() >= rev { return nil } select { case <-time.After(10 * time.Millisecond): case <-ctx.Done(): return ctx.Err() } } } // Close cancels the private context and blocks until all goroutines return. func (c *Cache) Close() { c.stop() c.waitGroup.Wait() } func (c *Cache) getWatchLoop() { cfg := defaultConfig() ctx := c.internalCtx backoff := cfg.InitialBackoff for { if err := ctx.Err(); err != nil { return } if err := c.getWatch(); err != nil { fmt.Printf("getWatch failed, will retry after %v: %v\n", backoff, err) } select { case <-ctx.Done(): return case <-time.After(backoff): } } } func (c *Cache) getWatch() error { getResp, err := c.get(c.internalCtx) if err != nil { return err } return c.watch(getResp.Header.Revision + 1) } func (c *Cache) get(ctx context.Context) (*clientv3.GetResponse, error) { resp, err := c.kv.Get(ctx, c.prefix, clientv3.WithPrefix()) if err != nil { return nil, err } c.store.Restore(resp.Kvs, resp.Header.Revision) return resp, nil } func (c *Cache) watch(rev int64) error { readyOnce := sync.Once{} for { storeW := newWatcher(c.cfg.PerWatcherBufferSize, nil) c.demux.Register(storeW, rev) applyErr := make(chan error, 1) c.waitGroup.Add(1) go func() { defer c.waitGroup.Done() if err := c.applyStorage(storeW); err != nil { applyErr <- err } close(applyErr) }() err := c.watchEvents(rev, applyErr, &readyOnce) c.demux.Unregister(storeW) if err != nil { return err } } } func (c *Cache) applyStorage(storeW *watcher) error { for { select { case <-c.internalCtx.Done(): return nil case resp, ok := <-storeW.respCh: if !ok { return nil } if err := c.store.Apply(resp); err != nil { return err } } } } func (c *Cache) watchEvents(rev int64, applyErr <-chan error, readyOnce *sync.Once) error { watchCh := c.watcher.Watch( c.internalCtx, c.prefix, clientv3.WithPrefix(), clientv3.WithRev(rev), clientv3.WithProgressNotify(), clientv3.WithCreatedNotify(), ) for { select { case <-c.internalCtx.Done(): return c.internalCtx.Err() case resp, ok := <-watchCh: if !ok { return nil } readyOnce.Do(func() { c.demux.Init(rev) c.ready.Set() }) if err := resp.Err(); err != nil { c.ready.Reset() return err } err := c.demux.Broadcast(resp) if err != nil { c.ready.Reset() return err } case err := <-applyErr: c.ready.Reset() return err } } } func (c *Cache) validateWatch(key string, op clientv3.Op) (pred KeyPredicate, err error) { switch { case op.IsPrevKV(): return nil, fmt.Errorf("%w: PrevKV not supported", ErrUnsupportedRequest) case op.IsFragment(): return nil, fmt.Errorf("%w: Fragment not supported", ErrUnsupportedRequest) case op.IsProgressNotify(): return nil, fmt.Errorf("%w: ProgressNotify not supported", ErrUnsupportedRequest) case op.IsCreatedNotify(): return nil, fmt.Errorf("%w: CreatedNotify not supported", ErrUnsupportedRequest) case op.IsFilterPut(): return nil, fmt.Errorf("%w: FilterPut not supported", ErrUnsupportedRequest) case op.IsFilterDelete(): return nil, fmt.Errorf("%w: FilterDelete not supported", ErrUnsupportedRequest) } startKey := []byte(key) endKey := op.RangeBytes() // nil = single key, {0}=FromKey, else explicit range if err := c.validateRange(startKey, endKey); err != nil { return nil, err } return KeyPredForRange(startKey, endKey), nil } func (c *Cache) validateGet(key string, op clientv3.Op) (KeyPredicate, error) { switch { case op.IsCountOnly(): return nil, fmt.Errorf("%w: CountOnly not supported", ErrUnsupportedRequest) case op.IsPrevKV(): return nil, fmt.Errorf("%w: PrevKV not supported", ErrUnsupportedRequest) case op.IsSortSet(): return nil, fmt.Errorf("%w: SortSet not supported", ErrUnsupportedRequest) case op.Limit() != 0: return nil, fmt.Errorf("%w: Limit(%d) not supported", ErrUnsupportedRequest, op.Limit()) case op.MinModRev() != 0: return nil, fmt.Errorf("%w: MinModRev(%d) not supported", ErrUnsupportedRequest, op.MinModRev()) case op.MaxModRev() != 0: return nil, fmt.Errorf("%w: MaxModRev(%d) not supported", ErrUnsupportedRequest, op.MaxModRev()) case op.MinCreateRev() != 0: return nil, fmt.Errorf("%w: MinCreateRev(%d) not supported", ErrUnsupportedRequest, op.MinCreateRev()) case op.MaxCreateRev() != 0: return nil, fmt.Errorf("%w: MaxCreateRev(%d) not supported", ErrUnsupportedRequest, op.MaxCreateRev()) // cache now only serves serializable reads of the latest revision (rev == 0). case !op.IsSerializable(): return nil, fmt.Errorf("%w: non-serializable request", ErrUnsupportedRequest) } startKey := []byte(key) endKey := op.RangeBytes() if err := c.validateRange(startKey, endKey); err != nil { return nil, err } return KeyPredForRange(startKey, endKey), nil } func (c *Cache) validateRange(startKey, endKey []byte) error { prefixStart := []byte(c.prefix) prefixEnd := []byte(clientv3.GetPrefixRangeEnd(c.prefix)) isSingleKey := len(endKey) == 0 isFromKey := len(endKey) == 1 && endKey[0] == 0 switch { case isSingleKey: if c.prefix == "" { return nil } if bytes.Compare(startKey, prefixStart) < 0 || bytes.Compare(startKey, prefixEnd) >= 0 { return ErrKeyRangeInvalid } return nil case isFromKey: if c.prefix != "" { return ErrKeyRangeInvalid } return nil default: if bytes.Compare(endKey, startKey) <= 0 { return ErrKeyRangeInvalid } if c.prefix == "" { return nil } if bytes.Compare(startKey, prefixStart) < 0 || bytes.Compare(endKey, prefixEnd) > 0 { return ErrKeyRangeInvalid } return nil } } // WaitForNextResync blocks until the next resync loop iteration is complete. func (c *Cache) WaitForNextResync(ctx context.Context) error { return c.demux.WaitForNextResync(ctx) } ================================================ FILE: cache/cache_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import ( "context" "sync" "testing" "time" "github.com/google/go-cmp/cmp" pb "go.etcd.io/etcd/api/v3/etcdserverpb" mvccpb "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" ) func TestCacheWatchAtomicOrderedDelivery(t *testing.T) { tests := []struct { name string sentBatches [][]*clientv3.Event wantBatch []*clientv3.Event }{ { name: "single_event", sentBatches: [][]*clientv3.Event{ {event(mvccpb.Event_PUT, "/a", 5)}, }, wantBatch: []*clientv3.Event{ event(mvccpb.Event_PUT, "/a", 5), }, }, { name: "same_revision_batch", sentBatches: [][]*clientv3.Event{ { event(mvccpb.Event_PUT, "/a", 10), event(mvccpb.Event_PUT, "/b", 10), }, }, wantBatch: []*clientv3.Event{ event(mvccpb.Event_PUT, "/a", 10), event(mvccpb.Event_PUT, "/b", 10), }, }, { name: "mixed_revisions_in_single_response", sentBatches: [][]*clientv3.Event{ { event(mvccpb.Event_PUT, "/a", 11), event(mvccpb.Event_PUT, "/b", 11), event(mvccpb.Event_PUT, "/c", 12), }, }, wantBatch: []*clientv3.Event{ event(mvccpb.Event_PUT, "/a", 11), event(mvccpb.Event_PUT, "/b", 11), event(mvccpb.Event_PUT, "/c", 12), }, }, { name: "mixed_event_types_same_revision", sentBatches: [][]*clientv3.Event{ { event(mvccpb.Event_PUT, "/x", 5), event(mvccpb.Event_PUT, "/y", 6), event(mvccpb.Event_DELETE, "/x", 6), }, }, wantBatch: []*clientv3.Event{ event(mvccpb.Event_PUT, "/x", 5), event(mvccpb.Event_PUT, "/y", 6), event(mvccpb.Event_DELETE, "/x", 6), }, }, { name: "all_events_in_one_response", sentBatches: [][]*clientv3.Event{ { event(mvccpb.Event_PUT, "/a", 2), event(mvccpb.Event_PUT, "/b", 2), event(mvccpb.Event_PUT, "/c", 3), event(mvccpb.Event_PUT, "/d", 4), event(mvccpb.Event_PUT, "/e", 4), event(mvccpb.Event_PUT, "/f", 5), event(mvccpb.Event_PUT, "/g", 6), event(mvccpb.Event_PUT, "/h", 6), event(mvccpb.Event_PUT, "/i", 7), event(mvccpb.Event_PUT, "/j", 7), }, }, wantBatch: []*clientv3.Event{ event(mvccpb.Event_PUT, "/a", 2), event(mvccpb.Event_PUT, "/b", 2), event(mvccpb.Event_PUT, "/c", 3), event(mvccpb.Event_PUT, "/d", 4), event(mvccpb.Event_PUT, "/e", 4), event(mvccpb.Event_PUT, "/f", 5), event(mvccpb.Event_PUT, "/g", 6), event(mvccpb.Event_PUT, "/h", 6), event(mvccpb.Event_PUT, "/i", 7), event(mvccpb.Event_PUT, "/j", 7), }, }, { name: "one_revision_group_per_response", sentBatches: [][]*clientv3.Event{ {event(mvccpb.Event_PUT, "/a", 2), event(mvccpb.Event_PUT, "/b", 2)}, {event(mvccpb.Event_PUT, "/c", 3)}, {event(mvccpb.Event_PUT, "/d", 4), event(mvccpb.Event_PUT, "/e", 4)}, {event(mvccpb.Event_PUT, "/f", 5)}, {event(mvccpb.Event_PUT, "/g", 6), event(mvccpb.Event_PUT, "/h", 6)}, {event(mvccpb.Event_PUT, "/i", 7), event(mvccpb.Event_PUT, "/j", 7)}, }, wantBatch: []*clientv3.Event{ event(mvccpb.Event_PUT, "/a", 2), event(mvccpb.Event_PUT, "/b", 2), event(mvccpb.Event_PUT, "/c", 3), event(mvccpb.Event_PUT, "/d", 4), event(mvccpb.Event_PUT, "/e", 4), event(mvccpb.Event_PUT, "/f", 5), event(mvccpb.Event_PUT, "/g", 6), event(mvccpb.Event_PUT, "/h", 6), event(mvccpb.Event_PUT, "/i", 7), event(mvccpb.Event_PUT, "/j", 7), }, }, { name: "two_revision_groups_per_response", sentBatches: [][]*clientv3.Event{ {event(mvccpb.Event_PUT, "/a", 2), event(mvccpb.Event_PUT, "/b", 2), event(mvccpb.Event_PUT, "/c", 3)}, {event(mvccpb.Event_PUT, "/d", 4), event(mvccpb.Event_PUT, "/e", 4), event(mvccpb.Event_PUT, "/f", 5)}, {event(mvccpb.Event_PUT, "/g", 6), event(mvccpb.Event_PUT, "/h", 6)}, {event(mvccpb.Event_PUT, "/i", 7), event(mvccpb.Event_PUT, "/j", 7)}, }, wantBatch: []*clientv3.Event{ event(mvccpb.Event_PUT, "/a", 2), event(mvccpb.Event_PUT, "/b", 2), event(mvccpb.Event_PUT, "/c", 3), event(mvccpb.Event_PUT, "/d", 4), event(mvccpb.Event_PUT, "/e", 4), event(mvccpb.Event_PUT, "/f", 5), event(mvccpb.Event_PUT, "/g", 6), event(mvccpb.Event_PUT, "/h", 6), event(mvccpb.Event_PUT, "/i", 7), event(mvccpb.Event_PUT, "/j", 7), }, }, { name: "three_revision_groups_per_response", sentBatches: [][]*clientv3.Event{ { event(mvccpb.Event_PUT, "/a", 2), event(mvccpb.Event_PUT, "/b", 2), event(mvccpb.Event_PUT, "/c", 3), event(mvccpb.Event_PUT, "/d", 4), event(mvccpb.Event_PUT, "/e", 4), }, { event(mvccpb.Event_PUT, "/f", 5), event(mvccpb.Event_PUT, "/g", 6), event(mvccpb.Event_PUT, "/h", 6), event(mvccpb.Event_PUT, "/i", 7), event(mvccpb.Event_PUT, "/j", 7), }, }, wantBatch: []*clientv3.Event{ event(mvccpb.Event_PUT, "/a", 2), event(mvccpb.Event_PUT, "/b", 2), event(mvccpb.Event_PUT, "/c", 3), event(mvccpb.Event_PUT, "/d", 4), event(mvccpb.Event_PUT, "/e", 4), event(mvccpb.Event_PUT, "/f", 5), event(mvccpb.Event_PUT, "/g", 6), event(mvccpb.Event_PUT, "/h", 6), event(mvccpb.Event_PUT, "/i", 7), event(mvccpb.Event_PUT, "/j", 7), }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mw := newMockWatcher(16) fakeClient := &clientv3.Client{ Watcher: mw, KV: newKVStub(), } cache, err := New(fakeClient, "") if err != nil { t.Fatalf("New cache: %v", err) } if err != nil { t.Fatalf("New cache: %v", err) } defer cache.Close() mw.responses <- clientv3.WatchResponse{} <-mw.registered ctxWait, cancelWait := context.WithTimeout(t.Context(), time.Second) if err := cache.WaitReady(ctxWait); err != nil { t.Fatalf("cache did not become Ready(): %v", err) } cancelWait() ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) defer cancel() watchCh := cache.Watch(ctx, "", clientv3.WithPrefix()) for _, batch := range tt.sentBatches { mw.responses <- clientv3.WatchResponse{Events: batch} } close(mw.responses) got := collectAndAssertAtomicEvents(ctx, t, watchCh, len(tt.wantBatch)) if diff := cmp.Diff(tt.wantBatch, got); diff != "" { t.Fatalf("event mismatch (-want +got):\n%s", diff) } }) } } func TestValidateWatchRange(t *testing.T) { type tc struct { name string watchKey string opts []clientv3.OpOption cachePrefix string wantErr bool } tests := []tc{ { name: "single key", watchKey: "/a", cachePrefix: "", wantErr: false, }, { name: "prefix single key", watchKey: "/foo/a", cachePrefix: "/foo", wantErr: false, }, { name: "single key outside prefix returns error", watchKey: "/z", cachePrefix: "/foo", wantErr: true, }, { name: "explicit range", watchKey: "/a", opts: []clientv3.OpOption{clientv3.WithRange("/b")}, cachePrefix: "", wantErr: false, }, { name: "exact prefix range", watchKey: "/a", opts: []clientv3.OpOption{clientv3.WithRange("/b")}, cachePrefix: "/a", wantErr: false, }, { name: "prefix subrange", watchKey: "/foo", opts: []clientv3.OpOption{clientv3.WithRange("/foo/a")}, cachePrefix: "/foo", wantErr: false, }, { name: "reverse range returns error", watchKey: "/b", opts: []clientv3.OpOption{clientv3.WithRange("/a")}, cachePrefix: "", wantErr: true, }, { name: "empty range returns error", watchKey: "/foo", opts: []clientv3.OpOption{clientv3.WithRange("/foo")}, cachePrefix: "", wantErr: true, }, { name: "range starting below cache prefix returns error", watchKey: "/a", opts: []clientv3.OpOption{clientv3.WithRange("/foo")}, cachePrefix: "/foo", wantErr: true, }, { name: "range encompassing cache prefix returns error", watchKey: "/a", opts: []clientv3.OpOption{clientv3.WithRange("/z")}, cachePrefix: "/foo", wantErr: true, }, { name: "range crossing prefixEnd returns error", watchKey: "/foo", opts: []clientv3.OpOption{clientv3.WithRange("/z")}, cachePrefix: "/foo", wantErr: true, }, { name: "empty prefix", watchKey: "", opts: []clientv3.OpOption{clientv3.WithPrefix()}, cachePrefix: "", wantErr: false, }, { name: "empty prefix with cachePrefix returns error", watchKey: "", opts: []clientv3.OpOption{clientv3.WithPrefix()}, cachePrefix: "/foo", wantErr: true, }, { name: "prefix watch matches cachePrefix exactly", watchKey: "/foo", opts: []clientv3.OpOption{clientv3.WithPrefix()}, cachePrefix: "/foo", wantErr: false, }, { name: "prefix watch inside cachePrefix", watchKey: "/foo/bar", opts: []clientv3.OpOption{clientv3.WithPrefix()}, cachePrefix: "/foo", wantErr: false, }, { name: "prefix starting below cachePrefix returns error", watchKey: "/a", opts: []clientv3.OpOption{clientv3.WithPrefix()}, cachePrefix: "/foo", wantErr: true, }, { name: "prefix starting above shard prefixEnd returns error", watchKey: "/fop", opts: []clientv3.OpOption{clientv3.WithPrefix()}, cachePrefix: "/foo", wantErr: true, }, { name: "fromKey open‑ended", watchKey: "/a", opts: []clientv3.OpOption{clientv3.WithFromKey()}, cachePrefix: "", wantErr: false, }, { name: "fromKey starting at prefix start", watchKey: "/foo", opts: []clientv3.OpOption{clientv3.WithFromKey()}, cachePrefix: "/foo", wantErr: true, }, { name: "fromKey starting below prefixEnd", watchKey: "/a", opts: []clientv3.OpOption{clientv3.WithFromKey()}, cachePrefix: "/foo", wantErr: true, }, { name: "fromKey starting above prefixEnd returns error", watchKey: "/fop", opts: []clientv3.OpOption{clientv3.WithFromKey()}, cachePrefix: "/foo", wantErr: true, }, } for _, c := range tests { t.Run(c.name, func(t *testing.T) { dummyCache := &Cache{prefix: c.cachePrefix} op := clientv3.OpGet(c.watchKey, c.opts...) err := dummyCache.validateRange([]byte(c.watchKey), op.RangeBytes()) if gotErr := err != nil; gotErr != c.wantErr { t.Fatalf("validateWatchRange(%q, %q, %v) err=%v, wantErr=%v", c.cachePrefix, c.watchKey, c.opts, err, c.wantErr) } }) } } func TestCacheCompactionResync(t *testing.T) { firstSnapshot := &clientv3.GetResponse{ Header: &pb.ResponseHeader{Revision: 5}, Kvs: []*mvccpb.KeyValue{ {Key: []byte("foo"), Value: []byte("old_value"), ModRevision: 5, CreateRevision: 5, Version: 1}, {Key: []byte("bar"), Value: []byte("old_bar"), ModRevision: 3, CreateRevision: 3, Version: 1}, }, } secondSnapshot := &clientv3.GetResponse{ Header: &pb.ResponseHeader{Revision: 20}, Kvs: []*mvccpb.KeyValue{ {Key: []byte("foo"), Value: []byte("new_value"), ModRevision: 20, CreateRevision: 5, Version: 2}, {Key: []byte("baz"), Value: []byte("new_baz"), ModRevision: 18, CreateRevision: 18, Version: 1}, }, } fakeClient := &clientv3.Client{ Watcher: newMockWatcher(16), KV: newKVStub(firstSnapshot, secondSnapshot), } cache, err := New(fakeClient, "") if err != nil { t.Fatalf("New cache: %v", err) } defer cache.Close() mw := fakeClient.Watcher.(*mockWatcher) t.Log("Phase 1: initial getWatch bootstrap") mw.triggerCreatedNotify() <-mw.registered if err = cache.WaitReady(t.Context()); err != nil { t.Fatalf("initial WaitReady: %v", err) } verifySnapshot(t, cache, []*mvccpb.KeyValue{ {Key: []byte("bar"), Value: []byte("old_bar"), ModRevision: 3, CreateRevision: 3, Version: 1}, {Key: []byte("foo"), Value: []byte("old_value"), ModRevision: 5, CreateRevision: 5, Version: 1}, }) t.Log("Phase 2: simulate compaction") mw.errorCompacted(10) waitUntil(t, time.Second, 10*time.Millisecond, func() bool { return !cache.Ready() }) ctxGet, cancelGet := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancelGet() snapshot, err := cache.Get(ctxGet, "foo", clientv3.WithSerializable()) if err != nil { t.Fatalf("expected Get() to serve from cached snapshot after compaction, got %v", err) } if got := snapshot.Header.Revision; got != firstSnapshot.Header.Revision { t.Fatalf("expected cached revision %d after compaction, got %d", firstSnapshot.Header.Revision, got) } if string(snapshot.Kvs[0].Value) != "old_value" { t.Fatalf("expected cached value 'old_value' during compaction, got %q", string(snapshot.Kvs[0].Value)) } t.Log("Phase 3: resync after compaction") mw.resetRegistered() mw.triggerCreatedNotify() <-mw.registered expectSnapshotRev := int64(20) ctxResync, cancelResync := context.WithTimeout(t.Context(), time.Second) defer cancelResync() if err = cache.WaitForRevision(ctxResync, expectSnapshotRev); err != nil { t.Fatalf("cache failed to resync to rev=%d within 1s: %v", expectSnapshotRev, err) } expectedWatchStart := secondSnapshot.Header.Revision + 1 if gotWatchStart := mw.getLastStartRev(); gotWatchStart != expectedWatchStart { t.Errorf("Watch started at rev=%d; want %d", gotWatchStart, expectedWatchStart) } gotSnapshot, err := cache.Get(t.Context(), "foo", clientv3.WithSerializable()) if err != nil { t.Fatalf("Get after resync: %v", err) } if gotSnapshot.Header.Revision != expectSnapshotRev { t.Errorf("unexpected Snapshot revision: got=%d, want=%d", gotSnapshot.Header.Revision, expectSnapshotRev) } verifySnapshot(t, cache, []*mvccpb.KeyValue{ {Key: []byte("baz"), Value: []byte("new_baz"), ModRevision: 18, CreateRevision: 18, Version: 1}, {Key: []byte("foo"), Value: []byte("new_value"), ModRevision: 20, CreateRevision: 5, Version: 2}, }) } func waitUntil(t *testing.T, timeout, poll time.Duration, cond func() bool) { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { if cond() { return } time.Sleep(poll) } t.Fatalf("condition not satisfied within %s", timeout) } type mockWatcher struct { responses chan clientv3.WatchResponse registered chan struct{} closeOnce sync.Once wg sync.WaitGroup mu sync.Mutex lastStartRev int64 } func newMockWatcher(buf int) *mockWatcher { return &mockWatcher{ responses: make(chan clientv3.WatchResponse, buf), registered: make(chan struct{}), } } func (m *mockWatcher) Watch(ctx context.Context, _ string, opts ...clientv3.OpOption) clientv3.WatchChan { rev := m.extractRev(opts) m.recordStartRev(rev) m.signalRegistration() out := make(chan clientv3.WatchResponse) m.wg.Add(1) go m.streamResponses(ctx, out) return out } func (m *mockWatcher) RequestProgress(_ context.Context) error { return nil } func (m *mockWatcher) Close() error { m.closeOnce.Do(func() { close(m.responses) }) m.wg.Wait() return nil } func (m *mockWatcher) triggerCreatedNotify() { m.responses <- clientv3.WatchResponse{} } func (m *mockWatcher) errorCompacted(compRev int64) { m.responses <- clientv3.WatchResponse{ Canceled: true, CompactRevision: compRev, } } func (m *mockWatcher) extractRev(opts []clientv3.OpOption) int64 { var op clientv3.Op for _, o := range opts { o(&op) } return op.Rev() } func (m *mockWatcher) recordStartRev(rev int64) { m.mu.Lock() defer m.mu.Unlock() m.lastStartRev = rev } func (m *mockWatcher) getLastStartRev() int64 { m.mu.Lock() defer m.mu.Unlock() return m.lastStartRev } func (m *mockWatcher) signalRegistration() { m.mu.Lock() defer m.mu.Unlock() select { case <-m.registered: default: close(m.registered) } } func (m *mockWatcher) resetRegistered() { m.mu.Lock() defer m.mu.Unlock() m.registered = make(chan struct{}) } func (m *mockWatcher) streamResponses(ctx context.Context, out chan<- clientv3.WatchResponse) { defer func() { close(out) m.wg.Done() }() for { select { case <-ctx.Done(): return case resp, ok := <-m.responses: if !ok { return } out <- resp if resp.Canceled { return } } } } type kvStub struct { queued []*clientv3.GetResponse defaultResp *clientv3.GetResponse } func newKVStub(resps ...*clientv3.GetResponse) *kvStub { queue := append([]*clientv3.GetResponse(nil), resps...) return &kvStub{ queued: queue, defaultResp: &clientv3.GetResponse{Header: &pb.ResponseHeader{Revision: 0}}, } } func (s *kvStub) Get(ctx context.Context, key string, _ ...clientv3.OpOption) (*clientv3.GetResponse, error) { if len(s.queued) > 0 { next := s.queued[0] s.queued = s.queued[1:] return next, nil } return s.defaultResp, nil } func (s *kvStub) Put(ctx context.Context, key, val string, _ ...clientv3.OpOption) (*clientv3.PutResponse, error) { return nil, nil } func (s *kvStub) Delete(ctx context.Context, key string, _ ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { return nil, nil } func (s *kvStub) Compact(ctx context.Context, rev int64, _ ...clientv3.CompactOption) (*clientv3.CompactResponse, error) { return nil, nil } func (s *kvStub) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) { return clientv3.OpResponse{}, nil } func (s *kvStub) Txn(ctx context.Context) clientv3.Txn { return nil } func event(eventType mvccpb.Event_EventType, key string, rev int64) *clientv3.Event { return &clientv3.Event{ Type: eventType, Kv: &mvccpb.KeyValue{ Key: []byte(key), ModRevision: rev, CreateRevision: rev, Version: 1, }, } } func collectAndAssertAtomicEvents(ctx context.Context, t *testing.T, watchCh clientv3.WatchChan, wantCount int) []*clientv3.Event { t.Helper() var events []*clientv3.Event var lastRevision int64 for { select { case <-ctx.Done(): t.Fatalf("timed out waiting for events (%d/%d received)", len(events), wantCount) case resp, ok := <-watchCh: if !ok { return events } if len(resp.Events) != 0 && resp.Events[0].Kv.ModRevision == lastRevision { t.Fatalf("same revision found as in previous response: %d", lastRevision) } for _, ev := range resp.Events { if ev.Kv.ModRevision < lastRevision { t.Fatalf("revision went backwards: last %d, now %d", lastRevision, ev.Kv.ModRevision) } events = append(events, ev) lastRevision = ev.Kv.ModRevision } if wantCount != 0 && len(events) >= wantCount { return events } } } } func verifySnapshot(t *testing.T, cache *Cache, want []*mvccpb.KeyValue) { resp, err := cache.Get(t.Context(), "", clientv3.WithPrefix(), clientv3.WithSerializable()) if err != nil { t.Fatalf("Get all keys: %v", err) } if diff := cmp.Diff(want, resp.Kvs); diff != "" { t.Fatalf("cache snapshot mismatch (-want +got):\n%s", diff) } } ================================================ FILE: cache/config.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import "time" type Config struct { // PerWatcherBufferSize caps each watcher’s buffered channel. // Bigger values tolerate brief client slow-downs at the cost of extra memory. PerWatcherBufferSize int // HistoryWindowSize is the max events kept in memory for replay. // It defines how far back the cache can replay events to lagging watchers HistoryWindowSize int // ResyncInterval controls how often the demux attempts to catch a lagging watcher up by replaying events from History. ResyncInterval time.Duration // InitialBackoff is the first delay to wait before retrying an upstream etcd Watch after it ends with an error. InitialBackoff time.Duration // MaxBackoff caps the exponential back-off between successive upstream watch retries. MaxBackoff time.Duration // GetTimeout is the timeout applied to the first Get() used to bootstrap the cache. GetTimeout time.Duration // BTreeDegree controls the degree (branching factor) of the in-memory B-tree store. BTreeDegree int } // TODO: tune via performance/load tests. func defaultConfig() Config { return Config{ PerWatcherBufferSize: 10, HistoryWindowSize: 2048, ResyncInterval: 50 * time.Millisecond, InitialBackoff: 50 * time.Millisecond, MaxBackoff: 2 * time.Second, GetTimeout: 5 * time.Second, BTreeDegree: 32, } } type Option func(*Config) func WithPerWatcherBufferSize(n int) Option { return func(c *Config) { c.PerWatcherBufferSize = n } } func WithHistoryWindowSize(n int) Option { return func(c *Config) { c.HistoryWindowSize = n } } func WithResyncInterval(d time.Duration) Option { return func(c *Config) { c.ResyncInterval = d } } func WithInitialBackoff(d time.Duration) Option { return func(c *Config) { c.InitialBackoff = d } } func WithMaxBackoff(d time.Duration) Option { return func(c *Config) { c.MaxBackoff = d } } func WithGetTimeout(d time.Duration) Option { return func(c *Config) { c.GetTimeout = d } } func WithBTreeDegree(n int) Option { return func(c *Config) { c.BTreeDegree = n } } ================================================ FILE: cache/demux.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import ( "context" "errors" "sync" "time" "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" ) type demux struct { mu sync.RWMutex // activeWatchers & laggingWatchers hold the first revision the watcher still needs (nextRev). activeWatchers map[*watcher]int64 laggingWatchers map[*watcher]int64 resyncInterval time.Duration // Range of revisions maintained for demux operations, inclusive. Broader than history as event revision is not contious. // maxRev tracks highest seen revision; minRev sets watcher compaction threshold (updated to evictedRev+1 on history overflow) minRev, maxRev int64 // History stores events within [minRev, maxRev]. history ringBuffer[[]*clientv3.Event] // resynced is used to notify that resync loop was completed. resynced *notifier } func NewDemux(ctx context.Context, wg *sync.WaitGroup, historyWindowSize int, resyncInterval time.Duration) *demux { d := newDemux(historyWindowSize, resyncInterval) wg.Add(1) go func() { defer wg.Done() d.resyncLoop(ctx) }() return d } func newDemux(historyWindowSize int, resyncInterval time.Duration) *demux { return &demux{ activeWatchers: make(map[*watcher]int64), laggingWatchers: make(map[*watcher]int64), history: *newRingBuffer(historyWindowSize, func(batch []*clientv3.Event) int64 { return batch[0].Kv.ModRevision }), resyncInterval: resyncInterval, resynced: newNotifier(), } } // resyncLoop periodically tries to catch lagging watchers up by replaying events from History. func (d *demux) resyncLoop(ctx context.Context) { timer := time.NewTimer(d.resyncInterval) defer timer.Stop() for { select { case <-ctx.Done(): return case <-timer.C: d.resyncLaggingWatchers() d.resynced.notify() timer.Reset(d.resyncInterval) } } } // WaitForNextResync blocks until the next resync loop iteration is complete. func (d *demux) WaitForNextResync(ctx context.Context) error { return d.resynced.wait(ctx) } func (d *demux) Register(w *watcher, startingRev int64) { d.mu.Lock() defer d.mu.Unlock() if d.maxRev == 0 { if startingRev == 0 { d.activeWatchers[w] = 0 } else { d.laggingWatchers[w] = startingRev } return } // Special case: 0 means “newest”. if startingRev == 0 { startingRev = d.maxRev + 1 } if startingRev <= d.maxRev { d.laggingWatchers[w] = startingRev } else { d.activeWatchers[w] = startingRev } } func (d *demux) Unregister(w *watcher) { func() { d.mu.Lock() defer d.mu.Unlock() delete(d.activeWatchers, w) delete(d.laggingWatchers, w) }() w.Stop() } func (d *demux) Init(minRev int64) { d.mu.Lock() defer d.mu.Unlock() if minRev == 0 { return } if d.minRev == 0 { // Watch started for empty demux d.minRev = minRev return } if d.maxRev == 0 { // Watch started on initialized demux that never got any event. d.purge() d.minRev = minRev return } if minRev == d.maxRev+1 { // Watch continuing from last revision it observed. return } // Watch opened on revision mismatching dmux last observed revision. d.purge() d.minRev = minRev } func (d *demux) Broadcast(resp clientv3.WatchResponse) error { d.mu.Lock() defer d.mu.Unlock() if d.minRev == 0 { return errors.New("demux: not initialized") } err := validateRevisions(resp, d.maxRev) if err != nil { return err } d.updateStoreLocked(resp) d.broadcastLocked(resp) return nil } func (d *demux) LatestRev() int64 { d.mu.RLock() defer d.mu.RUnlock() return d.maxRev } func (d *demux) updateStoreLocked(resp clientv3.WatchResponse) { if resp.IsProgressNotify() { d.maxRev = resp.Header.Revision return } if len(resp.Events) == 0 { return } events := resp.Events batchStart := 0 for end := 1; end < len(events); end++ { if events[end].Kv.ModRevision != events[batchStart].Kv.ModRevision { if end > batchStart { if end+1 == len(events) && d.history.full() { d.minRev = d.history.PeekOldest() + 1 } d.history.Append(events[batchStart:end]) } batchStart = end } } if batchStart < len(events) { if d.history.full() { d.minRev = d.history.PeekOldest() + 1 } d.history.Append(events[batchStart:]) } d.maxRev = events[len(events)-1].Kv.ModRevision } func (d *demux) broadcastLocked(resp clientv3.WatchResponse) { switch { case resp.IsProgressNotify(): d.broadcastProgressLocked(resp.Header.Revision) case len(resp.Events) != 0: d.broadcastEventsLocked(resp.Events) default: } } func (d *demux) broadcastProgressLocked(progressRev int64) { for w, nextRev := range d.activeWatchers { if nextRev >= progressRev { continue } resp := clientv3.WatchResponse{ Header: etcdserverpb.ResponseHeader{ Revision: progressRev, }, } if w.enqueueResponse(resp) { d.activeWatchers[w] = progressRev + 1 } } } func (d *demux) broadcastEventsLocked(events []*clientv3.Event) { firstRev := events[0].Kv.ModRevision lastRev := events[len(events)-1].Kv.ModRevision for w, nextRev := range d.activeWatchers { if nextRev != 0 && firstRev > nextRev { d.laggingWatchers[w] = nextRev delete(d.activeWatchers, w) continue } sendStart := len(events) for i, ev := range events { if ev.Kv.ModRevision >= nextRev { sendStart = i break } } if sendStart == len(events) { continue } if !w.enqueueResponse(clientv3.WatchResponse{ Events: events[sendStart:], }) { // overflow → lagging d.laggingWatchers[w] = nextRev delete(d.activeWatchers, w) } else { d.activeWatchers[w] = lastRev + 1 } } } // Purge stops all watchers and rebase history on watch errors func (d *demux) Purge() { d.mu.Lock() defer d.mu.Unlock() d.purge() } func (d *demux) purge() { d.maxRev = 0 d.minRev = 0 d.history.RebaseHistory() for w := range d.activeWatchers { w.Stop() } for w := range d.laggingWatchers { w.Stop() } d.activeWatchers = make(map[*watcher]int64) d.laggingWatchers = make(map[*watcher]int64) } // Compact is called when etcd reports a compaction at compactRev to rebase history; // it keeps provably-too-old watchers for later cancellation, stops others, and clients should resubscribe. func (d *demux) Compact(compactRev int64) { d.mu.Lock() defer d.mu.Unlock() d.purge() } func (d *demux) resyncLaggingWatchers() { d.mu.Lock() defer d.mu.Unlock() if d.minRev == 0 { return } for w, nextRev := range d.laggingWatchers { if nextRev < d.minRev { w.Compact(nextRev) delete(d.laggingWatchers, w) continue } // TODO: re-enable key‐predicate in Filter when non‐zero startRev or performance tuning is needed resyncSuccess := true d.history.AscendGreaterOrEqual(nextRev, func(rev int64, eventBatch []*clientv3.Event) bool { resp := clientv3.WatchResponse{ Events: eventBatch, } if !w.enqueueResponse(resp) { // buffer overflow: watcher still lagging resyncSuccess = false return false } nextRev = rev + 1 return true }) // Send progress to just resync. if resyncSuccess { resp := clientv3.WatchResponse{ Header: etcdserverpb.ResponseHeader{Revision: d.maxRev}, } if d.maxRev > nextRev && w.enqueueResponse(resp) { nextRev = d.maxRev + 1 } delete(d.laggingWatchers, w) d.activeWatchers[w] = nextRev } else { d.laggingWatchers[w] = nextRev } } } func newNotifier() *notifier { return ¬ifier{ ch: make(chan struct{}), } } type notifier struct { mu sync.RWMutex ch chan struct{} } func (n *notifier) notify() { n.mu.Lock() defer n.mu.Unlock() previous := n.ch n.ch = make(chan struct{}) close(previous) } func (n *notifier) wait(ctx context.Context) error { n.mu.RLock() ch := n.ch n.mu.RUnlock() select { case <-ch: return nil case <-ctx.Done(): return ctx.Err() } } ================================================ FILE: cache/demux_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import ( "testing" "time" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" ) func TestInit(t *testing.T) { type want struct { min int64 max int64 historyRevs []int64 } tests := []struct { name string capacity int initRev int64 eventRevs []int64 shouldReinit bool reinitRev int64 want want }{ { name: "first init sets only min", capacity: 8, initRev: 5, eventRevs: nil, shouldReinit: false, want: want{min: 5, max: 0, historyRevs: nil}, }, { name: "init on empty demux with events", capacity: 8, initRev: 5, eventRevs: []int64{7, 9, 13}, shouldReinit: false, want: want{min: 5, max: 13, historyRevs: []int64{7, 9, 13}}, }, { name: "continuation at max+1 preserves range and history", capacity: 8, initRev: 10, eventRevs: []int64{13, 15, 21}, shouldReinit: true, reinitRev: 22, want: want{min: 10, max: 21, historyRevs: []int64{13, 15, 21}}, }, { name: "gap from max triggers purge and clears history", capacity: 8, initRev: 10, eventRevs: []int64{13, 15, 21}, shouldReinit: true, reinitRev: 30, want: want{min: 30, max: 0, historyRevs: nil}, }, { name: "idempotent reinit at same revision clears history", capacity: 8, initRev: 7, eventRevs: []int64{8, 9, 10}, shouldReinit: true, reinitRev: 7, want: want{min: 7, max: 0, historyRevs: nil}, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { d := newDemux(tt.capacity, 10*time.Millisecond) d.Init(tt.initRev) if len(tt.eventRevs) > 0 { if err := d.Broadcast(respWithEventRevs(tt.eventRevs...)); err != nil { t.Fatalf("Broadcast(%v) failed: %v", tt.eventRevs, err) } } if tt.shouldReinit { d.Init(tt.reinitRev) } if d.minRev != tt.want.min || d.maxRev != tt.want.max { t.Fatalf("revision range: got(min=%d, max=%d), want(min=%d, max=%d)", d.minRev, d.maxRev, tt.want.min, tt.want.max) } var actualHistoryRevs []int64 d.history.AscendGreaterOrEqual(0, func(rev int64, events []*clientv3.Event) bool { actualHistoryRevs = append(actualHistoryRevs, rev) return true }) if diff := cmp.Diff(tt.want.historyRevs, actualHistoryRevs); diff != "" { t.Fatalf("history validation failed (-want +got):\n%s", diff) } }) } } func TestBroadcast(t *testing.T) { type want struct { min int64 max int64 shouldError bool } tests := []struct { name string capacity int initRev int64 initialRevs []int64 followupRevs []int64 want want }{ { name: "history not full", capacity: 2, initRev: 1, initialRevs: []int64{2}, want: want{min: 1, max: 2, shouldError: false}, }, { name: "history at exact capacity", capacity: 2, initRev: 1, initialRevs: []int64{2, 3}, want: want{min: 1, max: 3, shouldError: false}, }, { name: "history overflow with eviction", capacity: 2, initRev: 1, initialRevs: []int64{2, 3, 4}, want: want{min: 3, max: 4, shouldError: false}, }, { name: "history overflow not continuous", capacity: 2, initRev: 2, initialRevs: []int64{4, 8, 16}, want: want{min: 5, max: 16, shouldError: false}, }, { name: "empty broadcast is no-op", capacity: 8, initRev: 10, initialRevs: []int64{}, want: want{min: 10, max: 0, shouldError: false}, }, { name: "revisions below maxRev are rejected", capacity: 8, initRev: 4, initialRevs: []int64{5, 6}, followupRevs: []int64{4}, want: want{shouldError: true}, }, { name: "revisions equal to maxRev are rejected", capacity: 8, initRev: 4, initialRevs: []int64{5, 6}, followupRevs: []int64{6}, want: want{shouldError: true}, }, { name: "revisions above maxRev are accepted", capacity: 8, initRev: 4, initialRevs: []int64{5, 6}, followupRevs: []int64{9, 14, 17}, want: want{min: 4, max: 17, shouldError: false}, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { d := newDemux(tt.capacity, 10*time.Millisecond) d.Init(tt.initRev) if len(tt.initialRevs) > 0 { if err := d.Broadcast(respWithEventRevs(tt.initialRevs...)); err != nil { t.Fatalf("unexpected error broadcasting initial revisions %v: %v", tt.initialRevs, err) } } if len(tt.followupRevs) > 0 { err := d.Broadcast(respWithEventRevs(tt.followupRevs...)) if tt.want.shouldError { require.Error(t, err) return } require.NoError(t, err) } if d.minRev != tt.want.min || d.maxRev != tt.want.max { t.Fatalf("revision range: got(min=%d, max=%d), want(min=%d, max=%d)", d.minRev, d.maxRev, tt.want.min, tt.want.max) } }) } } func TestBroadcastBatching(t *testing.T) { tests := []struct { name string input []int64 wantRevs []int64 wantSizes []int }{ { name: "two groups", input: []int64{14, 14, 15, 15, 15}, wantRevs: []int64{14}, wantSizes: []int{5}, }, { name: "single group", input: []int64{7, 7, 7}, wantRevs: []int64{7}, wantSizes: []int{3}, }, { name: "all distinct", input: []int64{1, 2, 3}, wantRevs: []int64{1}, wantSizes: []int{3}, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { d := newDemux(16, 10*time.Millisecond) w := newWatcher(len(tt.input)+1, nil) d.Init(1) d.Register(w, 0) d.Broadcast(respWithEventRevs(tt.input...)) gotRevs, gotSizes := readBatches(t, w, len(tt.wantRevs)) if diff := cmp.Diff(tt.wantRevs, gotRevs); diff != "" { t.Fatalf("revision mismatch (-want +got):\n%s", diff) } if diff := cmp.Diff(tt.wantSizes, gotSizes); diff != "" { t.Fatalf("batch size mismatch (-want +got):\n%s", diff) } }) } } func TestSlowWatcherResync(t *testing.T) { tests := []struct { name string input []int64 wantInitialRevs []int64 wantInitialSizes []int wantResyncRevs []int64 wantResyncSizes []int }{ { name: "single event overflow", input: []int64{1, 2, 3}, wantInitialRevs: []int64{1}, wantInitialSizes: []int{3}, wantResyncRevs: []int64{}, wantResyncSizes: []int{}, }, { name: "multi events batch overflow", input: []int64{10, 10, 11, 12, 12}, wantInitialRevs: []int64{10}, wantInitialSizes: []int{5}, wantResyncRevs: []int64{}, wantResyncSizes: []int{}, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { d := newDemux(16, 10*time.Millisecond) w := newWatcher(1, nil) d.Init(1) d.Register(w, 0) d.Broadcast(respWithEventRevs(tt.input...)) gotInitRevs, gotInitSizes := readBatches(t, w, len(tt.wantInitialRevs)) if diff := cmp.Diff(tt.wantInitialRevs, gotInitRevs); diff != "" { t.Fatalf("initial revs mismatch (-want +got):\n%s", diff) } if diff := cmp.Diff(tt.wantInitialSizes, gotInitSizes); diff != "" { t.Fatalf("initial batch sizes mismatch (-want +got):\n%s", diff) } gotRevs, gotSizes := make([]int64, 0, len(tt.wantResyncRevs)), make([]int, 0, len(tt.wantResyncRevs)) for len(gotRevs) < len(tt.wantResyncRevs) { d.resyncLaggingWatchers() revs, sizes := readBatches(t, w, 1) gotRevs = append(gotRevs, revs...) gotSizes = append(gotSizes, sizes...) } if diff := cmp.Diff(tt.wantResyncRevs, gotRevs); diff != "" { t.Fatalf("resync revs mismatch (-want +got):\n%s", diff) } if diff := cmp.Diff(tt.wantResyncSizes, gotSizes); diff != "" { t.Fatalf("resync batch sizes mismatch (-want +got):\n%s", diff) } }) } } func respWithEventRevs(revs ...int64) clientv3.WatchResponse { events := make([]*clientv3.Event, 0, len(revs)) for _, r := range revs { kv := &mvccpb.KeyValue{ Key: []byte("k"), Value: []byte("v"), ModRevision: r, } events = append(events, &clientv3.Event{ Type: clientv3.EventTypePut, Kv: kv, }) } return clientv3.WatchResponse{Events: events} } func readBatches(t *testing.T, w *watcher, n int) (revs []int64, sizes []int) { t.Helper() timeout := time.After(2 * time.Second) for len(revs) < n { select { case resp := <-w.respCh: if resp.Canceled { t.Fatalf("unexpected canceled response in test: %v", resp.CancelReason) } if len(resp.Events) == 0 { continue } revs = append(revs, resp.Events[0].Kv.ModRevision) sizes = append(sizes, len(resp.Events)) case <-timeout: t.Fatalf("timed out waiting for %d batches; got %d", n, len(revs)) } } return revs, sizes } ================================================ FILE: cache/go.mod ================================================ module go.etcd.io/etcd/cache/v3 go 1.26 toolchain go1.26.1 require ( github.com/google/go-cmp v0.7.0 github.com/stretchr/testify v1.11.1 go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/v3 v3.6.0-alpha.0 k8s.io/utils v0.0.0-20260108192941-914a6e750570 ) require ( github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/grpc v1.79.2 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace ( go.etcd.io/etcd/api/v3 => ../api go.etcd.io/etcd/client/pkg/v3 => ../client/pkg go.etcd.io/etcd/client/v3 => ../client/v3 ) ================================================ FILE: cache/go.sum ================================================ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/utils v0.0.0-20260108192941-914a6e750570 h1:JT4W8lsdrGENg9W+YwwdLJxklIuKWdRm+BC+xt33FOY= k8s.io/utils v0.0.0-20260108192941-914a6e750570/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= ================================================ FILE: cache/predicate.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import "bytes" type Prefix string func (prefix Prefix) Match(key []byte) bool { if prefix == "" { return true } prefixLen := len(prefix) return len(key) >= prefixLen && string(key[:prefixLen]) == string(prefix) } func ExactKey(key []byte) KeyPredicate { return func(k []byte) bool { return bytes.Equal(k, key) } } func FromKey(start []byte) KeyPredicate { return func(k []byte) bool { return bytes.Compare(k, start) >= 0 } } func Range(start, end []byte) KeyPredicate { return func(k []byte) bool { return bytes.Compare(k, start) >= 0 && bytes.Compare(k, end) < 0 } } func KeyPredForRange(start, end []byte) KeyPredicate { if len(end) == 0 { return ExactKey(start) } if len(end) == 1 && end[0] == 0 { return FromKey(start) } return Range(start, end) } ================================================ FILE: cache/ready.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import ( "context" "sync" ) // ready tracks readiness state changes and allows callers to wait for a target state type ready struct { mu sync.Mutex isReady bool stateCh chan struct{} // closed on any state transition, then replaced immediately } func newReady() *ready { return &ready{stateCh: make(chan struct{})} } func (r *ready) Ready() bool { r.mu.Lock() defer r.mu.Unlock() return r.isReady } func (r *ready) WaitReady(ctx context.Context) error { return r.waitForState(ctx, func() bool { return r.isReady }) } func (r *ready) WaitNotReady(ctx context.Context) error { return r.waitForState(ctx, func() bool { return !r.isReady }) } func (r *ready) Set() { r.mu.Lock() defer r.mu.Unlock() if !r.isReady { r.isReady = true close(r.stateCh) r.stateCh = make(chan struct{}) } } func (r *ready) Reset() { r.mu.Lock() defer r.mu.Unlock() if r.isReady { r.isReady = false close(r.stateCh) r.stateCh = make(chan struct{}) } } func (r *ready) waitForState(ctx context.Context, pred func() bool) error { for { r.mu.Lock() if pred() { r.mu.Unlock() return ctx.Err() } stateChCopy := r.stateCh r.mu.Unlock() select { case <-stateChCopy: case <-ctx.Done(): return ctx.Err() } } } ================================================ FILE: cache/ready_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import ( "context" "errors" "sync" "testing" "time" ) func TestWaitMethods(t *testing.T) { tests := []struct { name string initialReady bool expectReady bool waitMethod string expectBlock bool }{ { name: "not_ready_testing_WaitNotReady", initialReady: false, expectReady: false, waitMethod: "not_ready", expectBlock: false, }, { name: "not_ready_testing_WaitReady", initialReady: false, expectReady: false, waitMethod: "ready", expectBlock: true, }, { name: "ready_testing_WaitReady", initialReady: true, expectReady: true, waitMethod: "ready", expectBlock: false, }, { name: "ready_testing_WaitNotReady", initialReady: true, expectReady: true, waitMethod: "not_ready", expectBlock: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { r := newReady() if tt.initialReady { r.Set() } if got := r.Ready(); got != tt.expectReady { t.Fatalf("Ready() = %t; want %t", got, tt.expectReady) } ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() var err error switch tt.waitMethod { case "ready": err = r.WaitReady(ctx) case "not_ready": err = r.WaitNotReady(ctx) default: t.Fatalf("invalid waitMethod: %s", tt.waitMethod) } if tt.expectBlock { if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("expected timeout but got: %v", err) } } else { if err != nil { t.Fatalf("expected immediate return but got error: %v", err) } } }) } } func TestSetUnblocksWaiters(t *testing.T) { testStateTransitionUnblocksWaiters(t, false, true, (*ready).Set, true, "WaitReady") } func TestResetUnblocksWaiters(t *testing.T) { testStateTransitionUnblocksWaiters(t, true, false, (*ready).Reset, false, "WaitNotReady") } func testStateTransitionUnblocksWaiters(t *testing.T, initialSet bool, waitForReady bool, transition func(*ready), expectedReady bool, waitMethodName string) { cases := []struct { name string n int }{ {"one_waiter", 1}, {"several_waiters", 16}, {"many_waiters", 128}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { r := newReady() if initialSet { r.Set() } var startWg sync.WaitGroup var readyWg sync.WaitGroup errs := make(chan error, tc.n) for i := 0; i < tc.n; i++ { startWg.Add(1) readyWg.Add(1) go func() { defer readyWg.Done() startWg.Done() ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() if waitForReady { errs <- r.WaitReady(ctx) } else { errs <- r.WaitNotReady(ctx) } }() } startWg.Wait() time.Sleep(50 * time.Millisecond) transition(r) readyWg.Wait() for i := 0; i < tc.n; i++ { if err := <-errs; err != nil { t.Fatalf("waiter %d: %s = %v; want: nil", i, waitMethodName, err) } } if r.Ready() != expectedReady { t.Fatalf("Ready() = %t after transition; want %t", r.Ready(), expectedReady) } if waitForReady { if err := r.WaitReady(t.Context()); err != nil { t.Fatalf("immediate WaitReady() after transition = %v; want: nil", err) } } else { if err := r.WaitNotReady(t.Context()); err != nil { t.Fatalf("immediate WaitNotReady() after transition = %v; want: nil", err) } } }) } } func TestIdempotentStateTransitions(t *testing.T) { r := newReady() r.Set() r.Set() if !r.Ready() { t.Fatalf("Ready() = false after double Set(); want: true") } if err := r.WaitReady(t.Context()); err != nil { t.Fatalf("WaitReady() after double Set() = %v; want: nil", err) } r.Reset() r.Reset() if r.Ready() { t.Fatalf("Ready() = true after double Reset(); want: false") } if err := r.WaitNotReady(t.Context()); err != nil { t.Fatalf("WaitNotReady() after double Reset() = %v; want ", err) } } ================================================ FILE: cache/ringbuffer.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache type ringBuffer[T any] struct { buffer []entry[T] // head is the index immediately after the last non-empty entry in the buffer (i.e., the next write position). head, tail, size int revisionOf RevisionOf[T] } type entry[T any] struct { revision int64 item T } type ( KeyPredicate = func([]byte) bool RevisionOf[T any] func(T) int64 IterFunc[T any] func(rev int64, item T) bool ) func newRingBuffer[T any](capacity int, revisionOf RevisionOf[T]) *ringBuffer[T] { // assume capacity > 0 – validated by Cache return &ringBuffer[T]{ buffer: make([]entry[T], capacity), revisionOf: revisionOf, } } func (r *ringBuffer[T]) Append(item T) { entry := entry[T]{revision: r.revisionOf(item), item: item} if r.full() { r.tail = (r.tail + 1) % len(r.buffer) } else { r.size++ } r.buffer[r.head] = entry r.head = (r.head + 1) % len(r.buffer) } func (r *ringBuffer[T]) full() bool { return r.size == len(r.buffer) } // AscendGreaterOrEqual iterates through entries in ascending order starting from the first entry with revision >= pivot. // TODO: use binary search on the ring buffer to locate the first entry >= nextRev instead of a full scan func (r *ringBuffer[T]) AscendGreaterOrEqual(pivot int64, iter IterFunc[T]) { if r.size == 0 { return } for n, i := 0, r.tail; n < r.size; n, i = n+1, (i+1)%len(r.buffer) { entry := r.buffer[i] if entry.revision < pivot { continue } if !iter(entry.revision, entry.item) { return } } } // AscendLessThan iterates in ascending order over entries with revision < pivot. func (r *ringBuffer[T]) AscendLessThan(pivot int64, iter IterFunc[T]) { if r.size == 0 { return } for n, i := 0, r.tail; n < r.size; n, i = n+1, (i+1)%len(r.buffer) { entry := r.buffer[i] if entry.revision >= pivot { return } if !iter(entry.revision, entry.item) { return } } } // DescendGreaterThan iterates in descending order over entries with revision > pivot. func (r *ringBuffer[T]) DescendGreaterThan(pivot int64, iter IterFunc[T]) { if r.size == 0 { return } for n, i := 0, r.moduloIndex(r.head-1); n < r.size; n, i = n+1, r.moduloIndex(i-1) { entry := r.buffer[i] if entry.revision <= pivot { return } if !iter(entry.revision, entry.item) { return } } } // DescendLessOrEqual iterates in descending order over entries with revision <= pivot. func (r *ringBuffer[T]) DescendLessOrEqual(pivot int64, iter IterFunc[T]) { if r.size == 0 { return } for n, i := 0, r.moduloIndex(r.head-1); n < r.size; n, i = n+1, r.moduloIndex(i-1) { entry := r.buffer[i] if entry.revision > pivot { continue } if !iter(entry.revision, entry.item) { return } } } // PeekLatest returns the most recently-appended revision (or 0 if empty). func (r *ringBuffer[T]) PeekLatest() int64 { if r.size == 0 { return 0 } idx := (r.head - 1 + len(r.buffer)) % len(r.buffer) return r.buffer[idx].revision } // PeekOldest returns the oldest revision currently stored (or 0 if empty). func (r *ringBuffer[T]) PeekOldest() int64 { if r.size == 0 { return 0 } return r.buffer[r.tail].revision } func (r *ringBuffer[T]) RebaseHistory() { r.head, r.tail, r.size = 0, 0, 0 for i := range r.buffer { r.buffer[i] = entry[T]{} } } func (r *ringBuffer[T]) moduloIndex(index int) int { return (index + len(r.buffer)) % len(r.buffer) } ================================================ FILE: cache/ringbuffer_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import ( "fmt" "testing" "github.com/google/go-cmp/cmp" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" ) func TestPeekLatestAndOldest(t *testing.T) { tests := []struct { name string capacity int revs []int64 wantLatestRev int64 wantOldestRev int64 }{ { name: "empty_buffer", capacity: 4, revs: nil, wantLatestRev: 0, wantOldestRev: 0, }, { name: "single_element", capacity: 8, revs: []int64{1}, wantLatestRev: 1, wantOldestRev: 1, }, { name: "ascending_fill", capacity: 4, revs: []int64{1, 2, 3, 4}, wantLatestRev: 4, wantOldestRev: 1, }, { name: "overwrite_when_full", capacity: 3, revs: []int64{5, 6, 7, 8}, wantLatestRev: 8, wantOldestRev: 6, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { rb := newRingBuffer(tt.capacity, func(batch []*clientv3.Event) int64 { return batch[0].Kv.ModRevision }) for _, r := range tt.revs { batch, err := makeEventBatch(r, "k", 1) if err != nil { t.Fatalf("makeEventBatch(%d, k, 1) failed: %v", r, err) } rb.Append(batch) } latestRev := rb.PeekLatest() oldestRev := rb.PeekOldest() gotLatestRev := latestRev gotOldestRev := oldestRev if tt.wantLatestRev != gotLatestRev { t.Fatalf("PeekLatest()=%d, want=%d", gotLatestRev, tt.wantLatestRev) } if tt.wantOldestRev != gotOldestRev { t.Fatalf("PeekOldest()=%d, want=%d", gotOldestRev, tt.wantOldestRev) } }) } } func TestIterationMethods(t *testing.T) { type iterTestCase struct { method iterMethod pivot int64 wantIterRevisions []int64 } tests := []struct { name string capacity int setupRevisions []int64 cases []iterTestCase }{ { name: "empty_buffer", capacity: 4, setupRevisions: nil, cases: []iterTestCase{ {ascendGTE, 0, []int64{}}, {ascendLT, 10, []int64{}}, {descendGT, 0, []int64{}}, {descendLTE, 10, []int64{}}, }, }, { name: "basic_filtering", capacity: 5, setupRevisions: []int64{1, 2, 3}, cases: []iterTestCase{ {ascendGTE, 0, []int64{1, 2, 3}}, {ascendGTE, 2, []int64{2, 3}}, {ascendGTE, 100, []int64{}}, {ascendLT, 3, []int64{1, 2}}, {ascendLT, 1, []int64{}}, {ascendLT, 100, []int64{1, 2, 3}}, {descendGT, 1, []int64{3, 2}}, {descendGT, 3, []int64{}}, {descendGT, 0, []int64{3, 2, 1}}, {descendLTE, 2, []int64{2, 1}}, {descendLTE, 3, []int64{3, 2, 1}}, {descendLTE, 0, []int64{}}, }, }, { name: "overflowed stores only entries within capacity", capacity: 3, setupRevisions: []int64{20, 21, 22, 23, 24}, // stored: 22, 23, 24 cases: []iterTestCase{ {ascendGTE, 23, []int64{23, 24}}, {ascendGTE, 0, []int64{22, 23, 24}}, {ascendLT, 23, []int64{22}}, {ascendLT, 25, []int64{22, 23, 24}}, {descendGT, 22, []int64{24, 23}}, {descendGT, 25, []int64{}}, {descendLTE, 23, []int64{23, 22}}, {descendLTE, 24, []int64{24, 23, 22}}, }, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { rb := setupRingBuffer(t, tt.capacity, tt.setupRevisions) for _, tc := range tt.cases { tc := tc t.Run(fmt.Sprintf("%s_pivot_%d", tc.method, tc.pivot), func(t *testing.T) { got := collectRevisions(rb, tc.method, tc.pivot) if diff := cmp.Diff(tc.wantIterRevisions, got); diff != "" { t.Fatalf("%s(%d) mismatch (-want +got):\n%s", tc.method, tc.pivot, diff) } }) } }) } } func TestIterationWithBatching(t *testing.T) { rb := newRingBuffer(6, func(batch []*clientv3.Event) int64 { return batch[0].Kv.ModRevision }) batchA := []*clientv3.Event{ {Kv: &mvccpb.KeyValue{Key: []byte("key-a"), ModRevision: 5}}, } batchB := []*clientv3.Event{ {Kv: &mvccpb.KeyValue{Key: []byte("key-b-1"), ModRevision: 10}}, {Kv: &mvccpb.KeyValue{Key: []byte("key-b-2"), ModRevision: 10}}, {Kv: &mvccpb.KeyValue{Key: []byte("key-b-3"), ModRevision: 10}}, } batchC := []*clientv3.Event{ {Kv: &mvccpb.KeyValue{Key: []byte("key-c"), ModRevision: 12}}, } rb.Append(batchA) rb.Append(batchB) rb.Append(batchC) tests := []struct { name string method iterMethod pivot int64 want [][]*clientv3.Event }{ { name: "ascending_gte_includes_batched_revision", method: ascendGTE, pivot: 10, want: [][]*clientv3.Event{ { {Kv: &mvccpb.KeyValue{Key: []byte("key-b-1"), ModRevision: 10}}, {Kv: &mvccpb.KeyValue{Key: []byte("key-b-2"), ModRevision: 10}}, {Kv: &mvccpb.KeyValue{Key: []byte("key-b-3"), ModRevision: 10}}, }, { {Kv: &mvccpb.KeyValue{Key: []byte("key-c"), ModRevision: 12}}, }, }, }, { name: "ascending_lt_stops_before_batched_revision", method: ascendLT, pivot: 10, want: [][]*clientv3.Event{ { {Kv: &mvccpb.KeyValue{Key: []byte("key-a"), ModRevision: 5}}, }, }, }, { name: "all_revisions_with_proper_batch_sizes", method: ascendGTE, pivot: 0, want: [][]*clientv3.Event{ { {Kv: &mvccpb.KeyValue{Key: []byte("key-a"), ModRevision: 5}}, }, { {Kv: &mvccpb.KeyValue{Key: []byte("key-b-1"), ModRevision: 10}}, {Kv: &mvccpb.KeyValue{Key: []byte("key-b-2"), ModRevision: 10}}, {Kv: &mvccpb.KeyValue{Key: []byte("key-b-3"), ModRevision: 10}}, }, { {Kv: &mvccpb.KeyValue{Key: []byte("key-c"), ModRevision: 12}}, }, }, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { var got [][]*clientv3.Event rb.iterate(tt.method, tt.pivot, func(rev int64, events []*clientv3.Event) bool { got = append(got, events) return true }) if diff := cmp.Diff(tt.want, got); diff != "" { t.Fatalf("Events mismatch (-want +got):\n%s", diff) } }) } } func TestIterationEarlyStop(t *testing.T) { rb := setupRingBuffer(t, 5, []int64{5, 10, 15, 20}) tests := []struct { name string method iterMethod pivot int64 stopAfter int want []int64 }{ { name: "find_first_match_ascending", method: ascendGTE, pivot: 10, stopAfter: 1, want: []int64{10}, }, { name: "find_first_two_ascending_lt", method: ascendLT, pivot: 20, stopAfter: 2, want: []int64{5, 10}, }, { name: "find_first_two_descending_gt", method: descendGT, pivot: 5, stopAfter: 2, want: []int64{20, 15}, }, { name: "find_first_match_descending_lte", method: descendLTE, pivot: 15, stopAfter: 1, want: []int64{15}, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { var collected []int64 callCount := 0 rb.iterate(tt.method, tt.pivot, func(rev int64, events []*clientv3.Event) bool { collected = append(collected, rev) callCount++ shouldContinue := callCount < tt.stopAfter if !shouldContinue { t.Logf("Stopping early after %d items (callback returned false)", callCount) } return shouldContinue }) if diff := cmp.Diff(tt.want, collected); diff != "" { t.Fatalf("Early stop failed.\nExpected: \nDiff (-want +got):\n%s", diff) } if callCount != tt.stopAfter { t.Fatalf("Expected exactly %d callback calls, got %d", tt.stopAfter, callCount) } t.Logf("Successfully stopped early: collected %v after %d callbacks", collected, callCount) }) } } type iterMethod string const ( ascendGTE iterMethod = "AscendGreaterOrEqual" ascendLT iterMethod = "AscendLessThan" descendGT iterMethod = "DescendGreaterThan" descendLTE iterMethod = "DescendLessOrEqual" ) func (r *ringBuffer[T]) iterate(method iterMethod, pivot int64, fn IterFunc[T]) { switch method { case ascendGTE: r.AscendGreaterOrEqual(pivot, fn) case ascendLT: r.AscendLessThan(pivot, fn) case descendGT: r.DescendGreaterThan(pivot, fn) case descendLTE: r.DescendLessOrEqual(pivot, fn) default: panic(fmt.Sprintf("unknown iteration method: %s", method)) } } func TestAtomicOrdered(t *testing.T) { tests := []struct { name string capacity int inputs []struct { rev int64 key string size int } wantRev []int64 wantSize []int }{ { name: "unfiltered", capacity: 5, inputs: []struct { rev int64 key string size int }{ {5, "a", 1}, {10, "b", 3}, {15, "c", 7}, {20, "d", 11}, }, wantRev: []int64{5, 10, 15, 20}, wantSize: []int{1, 3, 7, 11}, }, { name: "across_wrap", capacity: 3, inputs: []struct { rev int64 key string size int }{ {1, "a", 2}, {2, "b", 1}, {3, "c", 3}, {4, "d", 7}, }, wantRev: []int64{2, 3, 4}, wantSize: []int{1, 3, 7}, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() rb := newRingBuffer(tt.capacity, func(batch []*clientv3.Event) int64 { return batch[0].Kv.ModRevision }) for _, in := range tt.inputs { batch, err := makeEventBatch(in.rev, in.key, in.size) if err != nil { t.Fatalf("makeEventBatch(%d, k, 1) failed: %v", in.rev, err) } rb.Append(batch) } gotRevs := []int64{} var gotSizes []int rb.AscendGreaterOrEqual(0, func(rev int64, events []*clientv3.Event) bool { gotRevs = append(gotRevs, rev) gotSizes = append(gotSizes, len(events)) return true }) if len(gotRevs) != len(tt.wantRev) { t.Fatalf("len(got) = %d, want %d", len(gotRevs), len(tt.wantRev)) } for i := range gotRevs { if gotRevs[i] != tt.wantRev[i] { t.Errorf("at idx %d: rev = %d, want %d", i, gotRevs[i], tt.wantRev[i]) } if gotSizes[i] != tt.wantSize[i] { t.Errorf("at rev %d: events.len = %d, want %d", gotRevs[i], gotSizes[i], tt.wantSize[i]) } } }) } } func TestRebaseHistory(t *testing.T) { tests := []struct { name string revs []int64 }{ { name: "rebase_empty_buffer", revs: nil, }, { name: "rebase_after_data", revs: []int64{7, 8, 9}, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() rb := newRingBuffer(4, func(batch []*clientv3.Event) int64 { return batch[0].Kv.ModRevision }) for _, r := range tt.revs { batch, err := makeEventBatch(r, "k", 1) if err != nil { t.Fatalf("makeEventBatch(%d, k, 1) failed: %v", r, err) } rb.Append(batch) } rb.RebaseHistory() oldestRev := rb.PeekOldest() latestRev := rb.PeekLatest() if oldestRev != 0 { t.Fatalf("PeekOldest()=%d, want=%d", oldestRev, 0) } if latestRev != 0 { t.Fatalf("PeekLatest()=%d, want=%d", latestRev, 0) } gotRevs := []int64{} rb.AscendGreaterOrEqual(0, func(rev int64, events []*clientv3.Event) bool { gotRevs = append(gotRevs, rev) return true }) if len(gotRevs) != 0 { t.Fatalf("AscendGreaterOrEqual() len(events)=%d, want=%d", len(gotRevs), 0) } }) } } func TestFull(t *testing.T) { tests := []struct { name string capacity int numAppends int expectedFull bool }{ { name: "empty_buffer", capacity: 3, numAppends: 0, expectedFull: false, }, { name: "partially_filled", capacity: 5, numAppends: 3, expectedFull: false, }, { name: "exactly_at_capacity", capacity: 3, numAppends: 3, expectedFull: true, }, { name: "beyond_capacity_wrapping", capacity: 3, numAppends: 5, expectedFull: true, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { rb := newRingBuffer(tt.capacity, func(batch []*clientv3.Event) int64 { return batch[0].Kv.ModRevision }) for i := 1; i <= tt.numAppends; i++ { batch, err := makeEventBatch(int64(i), "k", 1) if err != nil { t.Fatalf("makeEventBatch(%d, k, 1) failed: %v", i, err) } rb.Append(batch) } if got := rb.full(); got != tt.expectedFull { t.Fatalf("full()=%t, want=%t (capacity=%d, appends=%d)", got, tt.expectedFull, tt.capacity, tt.numAppends) } }) } } func setupRingBuffer(t *testing.T, capacity int, revs []int64) *ringBuffer[[]*clientv3.Event] { rb := newRingBuffer(capacity, func(batch []*clientv3.Event) int64 { return batch[0].Kv.ModRevision }) for _, r := range revs { batch, err := makeEventBatch(r, "key", 1) if err != nil { t.Fatalf("makeEventBatch(%d, %s, %d) failed: %v", r, "key", 1, err) } rb.Append(batch) } return rb } func collectRevisions(rb *ringBuffer[[]*clientv3.Event], method iterMethod, pivot int64) []int64 { revs := []int64{} rb.iterate(method, pivot, func(rev int64, events []*clientv3.Event) bool { revs = append(revs, rev) return true }) return revs } func makeEventBatch(rev int64, key string, batchSize int) ([]*clientv3.Event, error) { if batchSize < 0 { return nil, fmt.Errorf("invalid batchSize %d", batchSize) } events := make([]*clientv3.Event, batchSize) for i := range events { events[i] = &clientv3.Event{ Kv: &mvccpb.KeyValue{ Key: []byte(fmt.Sprintf("%s-%d", key, i)), ModRevision: rev, }, } } return events, nil } ================================================ FILE: cache/snapshot.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import ( "k8s.io/utils/third_party/forked/golang/btree" "go.etcd.io/etcd/api/v3/mvccpb" ) // snapshot captures a full, point-in-time view of the KV state at rev. type snapshot struct { rev int64 tree *btree.BTree[*kvItem] } func newClonedSnapshot(rev int64, t *btree.BTree[*kvItem]) *snapshot { return &snapshot{rev: rev, tree: t.Clone()} } func (s *snapshot) Range(startKey, endKey []byte) []*mvccpb.KeyValue { var out []*mvccpb.KeyValue switch { case len(endKey) == 0: if item, ok := s.tree.Get(probeItemFromKey(startKey)); ok { out = append(out, item.kv) } case isPrefixScan(endKey): s.tree.AscendGreaterOrEqual(probeItemFromKey(startKey), func(item *kvItem) bool { out = append(out, item.kv) return true }) default: s.tree.AscendRange( probeItemFromKey(startKey), probeItemFromKey(endKey), func(item *kvItem) bool { out = append(out, item.kv) return true }, ) } return out } func isPrefixScan(endKey []byte) bool { return len(endKey) == 1 && endKey[0] == 0 } func probeItemFromKey(key []byte) *kvItem { return &kvItem{key: string(key)} } ================================================ FILE: cache/store.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import ( "errors" "fmt" "sync" "k8s.io/utils/third_party/forked/golang/btree" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" ) var ErrNotReady = fmt.Errorf("cache: store not ready") // The store keeps a bounded history of snapshots using ringBuffer so that // reads at historical revisions can be served until they fall out of the window. type store struct { mu sync.RWMutex degree int latest snapshot // latest is the mutable working snapshot history ringBuffer[*snapshot] // history stores immutable cloned snapshots } func newStore(degree int, historyCapacity int) *store { tree := btree.New[*kvItem](degree, kvItemLess) return &store{ degree: degree, latest: snapshot{rev: 0, tree: tree}, history: *newRingBuffer(historyCapacity, func(s *snapshot) int64 { return s.rev }), } } type kvItem struct { key string kv *mvccpb.KeyValue } func newKVItem(kv *mvccpb.KeyValue) *kvItem { return &kvItem{key: string(kv.Key), kv: kv} } func kvItemLess(a, b *kvItem) bool { return a.key < b.key } func (s *store) Get(startKey, endKey []byte, rev int64) ([]*mvccpb.KeyValue, int64, error) { snapshot, latestRev, err := s.getSnapshot(rev) if err != nil { return nil, 0, err } return snapshot.Range(startKey, endKey), latestRev, nil } func (s *store) getSnapshot(rev int64) (*snapshot, int64, error) { s.mu.RLock() defer s.mu.RUnlock() if s.latest.rev == 0 { return nil, 0, ErrNotReady } if rev < 0 { return nil, 0, fmt.Errorf("invalid revision: %d", rev) } if rev == 0 { rev = s.latest.rev } if rev > s.latest.rev { return nil, 0, rpctypes.ErrFutureRev } oldestRev := s.history.PeekOldest() if rev < oldestRev { return nil, 0, rpctypes.ErrCompacted } var targetSnapshot *snapshot s.history.AscendGreaterOrEqual(rev, func(rev int64, snap *snapshot) bool { targetSnapshot = snap return false }) // If s.history < rev < s.latest.rev serve latest. if targetSnapshot == nil { targetSnapshot = &s.latest } return targetSnapshot, s.latest.rev, nil } // Restore replaces state with the bootstrap snapshot and resets history. func (s *store) Restore(kvs []*mvccpb.KeyValue, rev int64) { s.mu.Lock() defer s.mu.Unlock() s.latest.tree = btree.New[*kvItem](s.degree, kvItemLess) for _, kv := range kvs { s.latest.tree.ReplaceOrInsert(newKVItem(kv)) } s.history.RebaseHistory() s.latest.rev = rev s.history.Append(newClonedSnapshot(rev, s.latest.tree)) } func (s *store) Apply(resp clientv3.WatchResponse) error { if resp.Canceled { return errors.New("canceled") } s.mu.Lock() defer s.mu.Unlock() if err := validateRevisions(resp, s.latest.rev); err != nil { return err } switch { case resp.IsProgressNotify(): s.applyProgressNotifyLocked(resp.Header.Revision) return nil case len(resp.Events) != 0: return s.applyEventsLocked(resp.Events) default: return nil } } func (s *store) applyProgressNotifyLocked(revision int64) { if s.latest.rev == 0 { return } s.latest.rev = revision } func (s *store) applyEventsLocked(events []*clientv3.Event) error { for i := 0; i < len(events); { rev := events[i].Kv.ModRevision for i < len(events) && events[i].Kv.ModRevision == rev { ev := events[i] switch ev.Type { case clientv3.EventTypeDelete: if _, ok := s.latest.tree.Delete(&kvItem{key: string(ev.Kv.Key)}); !ok { return fmt.Errorf("cache: delete non-existent key %s", string(ev.Kv.Key)) } case clientv3.EventTypePut: s.latest.tree.ReplaceOrInsert(newKVItem(ev.Kv)) } i++ } s.latest.rev = rev s.history.Append(newClonedSnapshot(rev, s.latest.tree)) } return nil } func (s *store) LatestRev() int64 { s.mu.RLock() defer s.mu.RUnlock() return s.latest.rev } func validateRevisions(resp clientv3.WatchResponse, latestRev int64) error { if resp.IsProgressNotify() { if resp.Header.Revision < latestRev { return fmt.Errorf("cache: progress notification out of order (progress %d < latest %d)", resp.Header.Revision, latestRev) } return nil } events := resp.Events if len(events) == 0 { return nil } for _, ev := range events { r := ev.Kv.ModRevision if r < latestRev { return fmt.Errorf("cache: stale event batch (rev %d < latest %d)", r, latestRev) } if r == latestRev { return fmt.Errorf("cache: duplicate revision batch breaks atomic guarantee (rev %d == latest %d)", r, latestRev) } } return nil } ================================================ FILE: cache/store_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import ( "errors" "testing" "github.com/google/go-cmp/cmp" mvccpb "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" ) func TestStoreGet(t *testing.T) { tests := []struct { name string initialKVs []*mvccpb.KeyValue initialRev int64 start []byte end []byte expectedKVs []*mvccpb.KeyValue expectedRev int64 expectedErr error }{ { name: "empty_store_returns_ErrNotReady", initialKVs: nil, start: []byte("a"), expectedErr: ErrNotReady, }, { name: "Get_single_key_hit", initialKVs: []*mvccpb.KeyValue{makeKV("/b", "2", 5), makeKV("/a", "1", 5), makeKV("/c", "3", 5)}, initialRev: 5, start: []byte("/b"), expectedKVs: []*mvccpb.KeyValue{makeKV("/b", "2", 5)}, expectedRev: 5, }, { name: "Get_single_key_miss_returns_empty", initialKVs: []*mvccpb.KeyValue{makeKV("/b", "2", 5), makeKV("/a", "1", 5), makeKV("/c", "3", 5)}, initialRev: 5, start: []byte("/zzz"), expectedKVs: nil, expectedRev: 5, }, { name: "Get_explicit_range", initialKVs: []*mvccpb.KeyValue{makeKV("/a", "1", 10), makeKV("/b", "2", 10), makeKV("/c", "3", 10), makeKV("/d", "4", 10)}, initialRev: 10, start: []byte("/b"), end: []byte("/d"), expectedKVs: []*mvccpb.KeyValue{makeKV("/b", "2", 10), makeKV("/c", "3", 10)}, expectedRev: 10, }, { name: "Get_range_includes_prefix_excludes_end", initialKVs: []*mvccpb.KeyValue{makeKV("/a", "1", 4), makeKV("/aa", "2", 4), makeKV("/ab", "3", 4), makeKV("/b", "4", 4)}, initialRev: 4, start: []byte("/a"), end: []byte("/b"), expectedKVs: []*mvccpb.KeyValue{makeKV("/a", "1", 4), makeKV("/aa", "2", 4), makeKV("/ab", "3", 4)}, expectedRev: 4, }, { name: "Get_empty_range_returns_empty", initialKVs: []*mvccpb.KeyValue{makeKV("/a", "1", 2), makeKV("/b", "2", 2)}, initialRev: 2, start: []byte("/a"), end: []byte("/a"), expectedKVs: nil, expectedRev: 2, }, { name: "Get_invalid_range_returns_empty", initialKVs: []*mvccpb.KeyValue{makeKV("/a", "1", 6), makeKV("/z", "9", 6)}, initialRev: 6, start: []byte("/z"), end: []byte("/a"), expectedKVs: nil, expectedRev: 6, }, { name: "Get_fromKey_scans_ordered", initialKVs: []*mvccpb.KeyValue{makeKV("/a", "1", 7), makeKV("/b", "2", 7), makeKV("/c", "3", 7), makeKV("/d", "4", 7)}, initialRev: 7, start: []byte("/b"), end: []byte{0}, expectedKVs: []*mvccpb.KeyValue{makeKV("/b", "2", 7), makeKV("/c", "3", 7), makeKV("/d", "4", 7)}, expectedRev: 7, }, { name: "Get_fromKey_with_no_results", initialKVs: []*mvccpb.KeyValue{makeKV("/a", "1", 9), makeKV("/b", "2", 9)}, initialRev: 9, start: []byte("/zzz"), end: []byte{0}, expectedKVs: nil, expectedRev: 9, }, } for _, tt := range tests { test := tt t.Run(test.name, func(t *testing.T) { s := newStore(8, 32) if test.initialKVs != nil { s.Restore(test.initialKVs, test.initialRev) } kvs, rev, err := s.Get(test.start, test.end, 0) if test.expectedErr != nil { if !errors.Is(err, test.expectedErr) { t.Fatalf("Get error = %v; want %v", err, test.expectedErr) } return } if err != nil { t.Fatalf("Get returned unexpected error: %v", err) } if rev != test.expectedRev { t.Fatalf("revision=%d; want %d", rev, test.expectedRev) } if diff := cmp.Diff(test.expectedKVs, kvs); diff != "" { t.Fatalf("Get mismatch (-want +got):\n%s", diff) } }) } } func TestStoreApply(t *testing.T) { type testCase struct { name string initialKVs []*mvccpb.KeyValue initialRev int64 eventBatches [][]*clientv3.Event expectedLatestRev int64 expectedSnapshot []*mvccpb.KeyValue expectErr bool } tests := []testCase{ { name: "put_overwrites_key", initialKVs: []*mvccpb.KeyValue{makeKV("/k", "v1", 10)}, initialRev: 10, eventBatches: [][]*clientv3.Event{{makePutEvent("/k", "v2", 11)}}, expectedLatestRev: 11, expectedSnapshot: []*mvccpb.KeyValue{makeKV("/k", "v2", 11)}, }, { name: "put_contiguous_revision", initialKVs: []*mvccpb.KeyValue{makeKV("/a", "A1", 20)}, initialRev: 20, eventBatches: [][]*clientv3.Event{ {makePutEvent("/a", "A2", 21)}, {makePutEvent("/b", "B1", 22)}, {makePutEvent("/c", "C1", 23)}, }, expectedLatestRev: 23, expectedSnapshot: []*mvccpb.KeyValue{makeKV("/a", "A2", 21), makeKV("/b", "B1", 22), makeKV("/c", "C1", 23)}, }, { name: "put_single_non_contiguous_batch", initialKVs: []*mvccpb.KeyValue{makeKV("/a", "A1", 20)}, initialRev: 20, eventBatches: [][]*clientv3.Event{{makePutEvent("/a", "A2", 25)}}, expectedLatestRev: 25, expectedSnapshot: []*mvccpb.KeyValue{makeKV("/a", "A2", 25)}, }, { name: "put_multiple_non_contiguous_batches", initialKVs: []*mvccpb.KeyValue{makeKV("/a", "A1", 21), makeKV("/b", "B1", 22)}, initialRev: 22, eventBatches: [][]*clientv3.Event{ {makePutEvent("/a", "A2", 25)}, {makePutEvent("/b", "B2", 27)}, }, expectedLatestRev: 27, expectedSnapshot: []*mvccpb.KeyValue{makeKV("/a", "A2", 25), makeKV("/b", "B2", 27)}, }, { name: "apply_mixed_operations", initialKVs: []*mvccpb.KeyValue{makeKV("/a", "A1", 20)}, initialRev: 20, eventBatches: [][]*clientv3.Event{ {makePutEvent("/a", "A2", 21), makePutEvent("/b", "B1", 21), makePutEvent("/c", "C1", 21)}, {makePutEvent("/b", "B2", 22)}, {makeDelEvent("/c", 23), makePutEvent("/a", "A3", 23)}, {makePutEvent("/b", "B3", 24)}, }, expectedLatestRev: 24, expectedSnapshot: []*mvccpb.KeyValue{makeKV("/a", "A3", 23), makeKV("/b", "B3", 24)}, }, { name: "delete_same_key", initialKVs: []*mvccpb.KeyValue{makeKV("/a", "X", 10)}, initialRev: 10, eventBatches: [][]*clientv3.Event{ {makeDelEvent("/a", 11)}, }, expectedLatestRev: 11, expectedSnapshot: nil, }, { name: "delete_nonexistent_returns_error", initialKVs: []*mvccpb.KeyValue{makeKV("/p", "X", 5)}, initialRev: 5, eventBatches: [][]*clientv3.Event{{makeDelEvent("/zzz", 6)}}, expectedLatestRev: 5, expectedSnapshot: []*mvccpb.KeyValue{makeKV("/p", "X", 5)}, expectErr: true, }, { name: "mixed_delete_nonexistent_returns_error", initialKVs: []*mvccpb.KeyValue{makeKV("/p", "X", 5)}, initialRev: 5, eventBatches: [][]*clientv3.Event{{makeDelEvent("/zzz", 6), makePutEvent("/r", "Y", 6)}}, expectedLatestRev: 5, expectedSnapshot: []*mvccpb.KeyValue{makeKV("/p", "X", 5)}, expectErr: true, }, { name: "delete_then_delete_again_returns_error", initialKVs: []*mvccpb.KeyValue{makeKV("/p", "X", 5)}, initialRev: 5, eventBatches: [][]*clientv3.Event{ {makeDelEvent("/p", 6)}, {makeDelEvent("/p", 7)}, }, expectedLatestRev: 6, expectedSnapshot: nil, expectErr: true, }, { name: "stale_batch_rejected", initialKVs: []*mvccpb.KeyValue{makeKV("/x", "1", 20)}, initialRev: 20, eventBatches: [][]*clientv3.Event{{makePutEvent("/x", "2", 19)}}, expectedLatestRev: 20, expectedSnapshot: []*mvccpb.KeyValue{makeKV("/x", "1", 20)}, expectErr: true, }, { name: "mixed_stale_batch_returns_error", initialKVs: []*mvccpb.KeyValue{makeKV("/x", "1", 20)}, initialRev: 20, eventBatches: [][]*clientv3.Event{ {makePutEvent("/x", "should-not-apply", 19)}, {makeDelEvent("/x", 21), makePutEvent("/y", "new", 21)}, {makeDelEvent("/y", 22)}, }, expectedLatestRev: 20, expectedSnapshot: []*mvccpb.KeyValue{makeKV("/x", "1", 20)}, expectErr: true, }, } for _, tt := range tests { test := tt t.Run(test.name, func(t *testing.T) { s := newStore(4, 32) s.Restore(test.initialKVs, test.initialRev) var gotErr error for batchIndex, batch := range test.eventBatches { resp := clientv3.WatchResponse{Events: batch} if err := s.Apply(resp); err != nil { gotErr = err if !test.expectErr { t.Fatalf("Apply(batch %d) unexpected error: %v", batchIndex, err) } break } } if test.expectErr && gotErr == nil { t.Fatalf("expected Apply() to error, but got nil") } if latest := s.LatestRev(); latest != test.expectedLatestRev { t.Fatalf("LatestRev=%d; want %d", latest, test.expectedLatestRev) } verifyStoreSnapshot(t, s, test.expectedSnapshot, test.expectedLatestRev, 0) }) } } func TestStoreRestore(t *testing.T) { type restoreSeq struct { kvs []*mvccpb.KeyValue rev int64 } tests := []struct { name string seq []restoreSeq expectedSnap []*mvccpb.KeyValue expectedRev int64 }{ { name: "rebuilds_tree_and_resets_rev", seq: []restoreSeq{ {[]*mvccpb.KeyValue{makeKV("/a", "1", 3), makeKV("/b", "2", 3)}, 3}, {[]*mvccpb.KeyValue{makeKV("/c", "3", 15)}, 15}, }, expectedSnap: []*mvccpb.KeyValue{makeKV("/c", "3", 15)}, expectedRev: 15, }, { name: "restore_to_revision_zero_returns_ErrNotReady", seq: []restoreSeq{ {[]*mvccpb.KeyValue{makeKV("/a", "1", 5)}, 5}, {nil, 0}, }, expectedSnap: nil, expectedRev: 0, }, { name: "restore_empty_ready", seq: []restoreSeq{{nil, 5}}, expectedSnap: nil, expectedRev: 5, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := newStore(8, 32) for _, step := range tt.seq { s.Restore(step.kvs, step.rev) } if tt.expectedRev == 0 { if _, _, err := s.Get([]byte("/"), []byte{0}, 0); !errors.Is(err, ErrNotReady) { t.Fatalf("Get after restore to rev=0 err=%v; want %v", err, ErrNotReady) } return } verifyStoreSnapshot(t, s, tt.expectedSnap, tt.expectedRev, 0) }) } } func TestRestoreAppendCloneImmutability(t *testing.T) { tests := []struct { name string initialKVs []*mvccpb.KeyValue initialRev int64 events []*clientv3.Event requestedRev int64 expectedSnap []*mvccpb.KeyValue expectedLatestSnap []*mvccpb.KeyValue expectedLatestRev int64 }{ { name: "put_overwrites_key", initialKVs: []*mvccpb.KeyValue{makeKV("/k", "v1", 5)}, initialRev: 5, events: []*clientv3.Event{makePutEvent("/k", "v2", 6)}, requestedRev: 5, expectedSnap: []*mvccpb.KeyValue{makeKV("/k", "v1", 5)}, expectedLatestSnap: []*mvccpb.KeyValue{makeKV("/k", "v2", 6)}, expectedLatestRev: 6, }, { name: "delete_key", initialKVs: []*mvccpb.KeyValue{makeKV("/k", "v1", 5)}, initialRev: 5, events: []*clientv3.Event{makeDelEvent("/k", 6)}, requestedRev: 5, expectedSnap: []*mvccpb.KeyValue{makeKV("/k", "v1", 5)}, expectedLatestSnap: nil, expectedLatestRev: 6, }, } for _, tt := range tests { test := tt t.Run(test.name, func(t *testing.T) { s := newStore(8, 32) if test.initialKVs != nil { s.Restore(test.initialKVs, test.initialRev) } if len(test.events) > 0 { resp := clientv3.WatchResponse{Events: test.events} if err := s.Apply(resp); err != nil { t.Fatalf("Apply failed: %v", err) } } if test.requestedRev != 0 { verifyStoreSnapshot(t, s, test.expectedSnap, test.expectedLatestRev, test.requestedRev) } verifyStoreSnapshot(t, s, test.expectedLatestSnap, test.expectedLatestRev, test.expectedLatestRev) }) } } func makeKV(key, val string, rev int64) *mvccpb.KeyValue { return &mvccpb.KeyValue{Key: []byte(key), Value: []byte(val), ModRevision: rev, CreateRevision: rev, Version: 1} } func makePutEvent(key, val string, rev int64) *clientv3.Event { return &clientv3.Event{Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{Key: []byte(key), Value: []byte(val), ModRevision: rev, CreateRevision: rev, Version: 1}} } func makeDelEvent(key string, rev int64) *clientv3.Event { return &clientv3.Event{Type: clientv3.EventTypeDelete, Kv: &mvccpb.KeyValue{Key: []byte(key), ModRevision: rev}} } func verifyStoreSnapshot(t *testing.T, s *store, want []*mvccpb.KeyValue, wantRev int64, requestedRev int64) { kvs, headerRev, err := s.Get([]byte("/"), []byte{0}, requestedRev) if err != nil { t.Fatalf("Get all keys (rev=%d): got error: %v", requestedRev, err) } latestRev := s.LatestRev() if headerRev != latestRev { t.Fatalf("header rev=%d; want latest %d (requestedRev=%d)", latestRev, wantRev, requestedRev) } if diff := cmp.Diff(want, kvs); diff != "" { t.Fatalf("snapshot mismatch (requestedRev=%d) (-want +got):\n%s", requestedRev, diff) } } ================================================ FILE: cache/watcher.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cache import ( "sync" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" ) // watcher holds one client’s buffered stream of events. type watcher struct { respCh chan clientv3.WatchResponse cancelResp *clientv3.WatchResponse keyPred KeyPredicate stopOnce sync.Once } func newWatcher(bufSize int, pred KeyPredicate) *watcher { return &watcher{ respCh: make(chan clientv3.WatchResponse, bufSize), keyPred: pred, } } // true -> events delivered (or filtered/duplicate) // false -> buffer full (caller should mark watcher “lagging”) func (w *watcher) enqueueResponse(resp clientv3.WatchResponse) bool { if !resp.IsProgressNotify() && w.keyPred != nil { filtered := make([]*clientv3.Event, 0, len(resp.Events)) for _, event := range resp.Events { if w.keyPred(event.Kv.Key) { filtered = append(filtered, event) } } if len(filtered) == 0 { return true } resp.Events = filtered } select { case w.respCh <- resp: return true default: return false } } func (w *watcher) Compact(compactRev int64) { resp := &clientv3.WatchResponse{ Canceled: true, CompactRevision: compactRev, CancelReason: rpctypes.ErrCompacted.Error(), } w.stopOnce.Do(func() { w.cancelResp = resp close(w.respCh) }) } // Stop closes the event channel atomically. func (w *watcher) Stop() { w.stopOnce.Do(func() { close(w.respCh) }) } ================================================ FILE: client/pkg/.gomodguard.yaml ================================================ --- blocked: modules: - go.etcd.io/etcd: reason: "Forbidden dependency" - go.etcd.io/etcd/api/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/pkg/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/server/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/tests/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/v3: reason: "Forbidden dependency" ================================================ FILE: client/pkg/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: client/pkg/fileutil/dir_unix.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows package fileutil import "os" const ( // PrivateDirMode grants owner to make/remove files inside the directory. PrivateDirMode = 0o700 ) // OpenDir opens a directory for syncing. func OpenDir(path string) (*os.File, error) { return os.Open(path) } ================================================ FILE: client/pkg/fileutil/dir_windows.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows package fileutil import ( "os" "syscall" ) const ( // PrivateDirMode grants owner to make/remove files inside the directory. PrivateDirMode = 0o777 ) // OpenDir opens a directory in windows with write access for syncing. func OpenDir(path string) (*os.File, error) { fd, err := openDir(path) if err != nil { return nil, err } return os.NewFile(uintptr(fd), path), nil } func openDir(path string) (fd syscall.Handle, err error) { if len(path) == 0 { return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND } pathp, err := syscall.UTF16PtrFromString(path) if err != nil { return syscall.InvalidHandle, err } access := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE) sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE) createmode := uint32(syscall.OPEN_EXISTING) fl := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS) return syscall.CreateFile(pathp, access, sharemode, nil, createmode, fl, 0) } ================================================ FILE: client/pkg/fileutil/doc.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package fileutil implements utility functions related to files and paths. package fileutil ================================================ FILE: client/pkg/fileutil/filereader.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "bufio" "io" "io/fs" "os" ) // FileReader is a wrapper of io.Reader. It also provides file info. type FileReader interface { io.Reader FileInfo() (fs.FileInfo, error) } type fileReader struct { *os.File } func NewFileReader(f *os.File) FileReader { return &fileReader{f} } func (fr *fileReader) FileInfo() (fs.FileInfo, error) { return fr.Stat() } // FileBufReader is a wrapper of bufio.Reader. It also provides file info. type FileBufReader struct { *bufio.Reader fi fs.FileInfo } func NewFileBufReader(fr FileReader) *FileBufReader { bufReader := bufio.NewReader(fr) fi, err := fr.FileInfo() if err != nil { // This should never happen. panic(err) } return &FileBufReader{bufReader, fi} } func (fbr *FileBufReader) FileInfo() fs.FileInfo { return fbr.fi } ================================================ FILE: client/pkg/fileutil/filereader_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "os" "strings" "testing" "github.com/stretchr/testify/assert" ) func TestFileBufReader(t *testing.T) { f, err := os.CreateTemp(t.TempDir(), "wal") if err != nil { t.Errorf("Unexpected error: %v", err) } fi, err := f.Stat() if err != nil { t.Errorf("Unexpected error: %v", err) } fbr := NewFileBufReader(NewFileReader(f)) if !strings.HasPrefix(fbr.FileInfo().Name(), "wal") { t.Errorf("Unexpected file name: %s", fbr.FileInfo().Name()) } assert.Equal(t, fi.Size(), fbr.FileInfo().Size()) assert.Equal(t, fi.IsDir(), fbr.FileInfo().IsDir()) assert.Equal(t, fi.Mode(), fbr.FileInfo().Mode()) assert.Equal(t, fi.ModTime(), fbr.FileInfo().ModTime()) } ================================================ FILE: client/pkg/fileutil/fileutil.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "fmt" "io" "io/fs" "os" "path/filepath" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/verify" ) const ( // PrivateFileMode grants owner to read/write a file. PrivateFileMode = 0o600 ) // IsDirWriteable checks if dir is writable by writing and removing a file // to dir. It returns nil if dir is writable. func IsDirWriteable(dir string) error { f, err := filepath.Abs(filepath.Join(dir, ".touch")) if err != nil { return err } if err := os.WriteFile(f, []byte(""), PrivateFileMode); err != nil { return err } return os.Remove(f) } // TouchDirAll is similar to os.MkdirAll. It creates directories with 0700 permission if any directory // does not exists. TouchDirAll also ensures the given directory is writable. func TouchDirAll(lg *zap.Logger, dir string) error { verify.Assert(lg != nil, "nil log isn't allowed") // If path is already a directory, MkdirAll does nothing and returns nil, so, // first check if dir exists with an expected permission mode. if Exist(dir) { err := CheckDirPermission(dir, PrivateDirMode) if err != nil { lg.Warn("check file permission", zap.Error(err)) } } else { err := os.MkdirAll(dir, PrivateDirMode) if err != nil { // if mkdirAll("a/text") and "text" is not // a directory, this will return syscall.ENOTDIR return err } } return IsDirWriteable(dir) } // CreateDirAll is similar to TouchDirAll but returns error // if the deepest directory was not empty. func CreateDirAll(lg *zap.Logger, dir string) error { err := TouchDirAll(lg, dir) if err == nil { var ns []string ns, err = ReadDir(dir) if err != nil { return err } if len(ns) != 0 { err = fmt.Errorf("expected %q to be empty, got %q", dir, ns) } } return err } // Exist returns true if a file or directory exists. func Exist(name string) bool { _, err := os.Stat(name) return err == nil } // DirEmpty returns true if a directory empty and can access. func DirEmpty(name string) bool { ns, err := ReadDir(name) return len(ns) == 0 && err == nil } // ZeroToEnd zeros a file starting from SEEK_CUR to its SEEK_END. May temporarily // shorten the length of the file. func ZeroToEnd(f *os.File) error { // TODO: support FALLOC_FL_ZERO_RANGE off, err := f.Seek(0, io.SeekCurrent) if err != nil { return err } lenf, lerr := f.Seek(0, io.SeekEnd) if lerr != nil { return lerr } if err = f.Truncate(off); err != nil { return err } // make sure blocks remain allocated if err = Preallocate(f, lenf, true); err != nil { return err } _, err = f.Seek(off, io.SeekStart) return err } // CheckDirPermission checks permission on an existing dir. // Returns error if dir is empty or exist with a different permission than specified. func CheckDirPermission(dir string, perm os.FileMode) error { if !Exist(dir) { return fmt.Errorf("directory %q empty, cannot check permission", dir) } // check the existing permission on the directory dirInfo, err := os.Stat(dir) if err != nil { return err } dirMode := dirInfo.Mode().Perm() if dirMode != perm { err = fmt.Errorf("directory %q exist, but the permission is %q. The recommended permission is %q to prevent possible unprivileged access to the data", dir, dirInfo.Mode(), os.FileMode(PrivateDirMode)) return err } return nil } // RemoveMatchFile deletes file if matchFunc is true on an existing dir // Returns error if the dir does not exist or remove file fail func RemoveMatchFile(lg *zap.Logger, dir string, matchFunc func(fileName string) bool) error { if lg == nil { lg = zap.NewNop() } if !Exist(dir) { return fmt.Errorf("directory %s does not exist", dir) } fileNames, err := ReadDir(dir) if err != nil { return err } var removeFailedFiles []string for _, fileName := range fileNames { if matchFunc(fileName) { file := filepath.Join(dir, fileName) if err = os.Remove(file); err != nil { removeFailedFiles = append(removeFailedFiles, fileName) lg.Error("remove file failed", zap.String("file", file), zap.Error(err)) } } } if len(removeFailedFiles) != 0 { return fmt.Errorf("remove file(s) %v error", removeFailedFiles) } return nil } // ListFiles lists files if matchFunc is true on an existing dir // Returns error if the dir does not exist func ListFiles(dir string, matchFunc func(fileName string) bool) ([]string, error) { var files []string err := filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { if matchFunc(path) { files = append(files, path) } return nil }) return files, err } ================================================ FILE: client/pkg/fileutil/fileutil_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "fmt" "io" "math/rand" "os" "os/user" "path/filepath" "runtime" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" ) func TestIsDirWriteable(t *testing.T) { tmpdir := t.TempDir() require.NoErrorf(t, IsDirWriteable(tmpdir), "unexpected IsDirWriteable error") require.NoErrorf(t, os.Chmod(tmpdir, 0o444), "unexpected os.Chmod error") me, err := user.Current() if err != nil { // err can be non-nil when cross compiled // http://stackoverflow.com/questions/20609415/cross-compiling-user-current-not-implemented-on-linux-amd64 t.Skipf("failed to get current user: %v", err) } if me.Name == "root" || runtime.GOOS == "windows" { // ideally we should check CAP_DAC_OVERRIDE. // but it does not matter for tests. // Chmod is not supported under windows. t.Skipf("running as a superuser or in windows") } require.Errorf(t, IsDirWriteable(tmpdir), "expected IsDirWriteable to error") } func TestCreateDirAll(t *testing.T) { tmpdir := t.TempDir() tmpdir2 := filepath.Join(tmpdir, "testdir") require.NoError(t, CreateDirAll(zaptest.NewLogger(t), tmpdir2)) require.NoError(t, os.WriteFile(filepath.Join(tmpdir2, "text.txt"), []byte("test text"), PrivateFileMode)) if err := CreateDirAll(zaptest.NewLogger(t), tmpdir2); err == nil || !strings.Contains(err.Error(), "to be empty, got") { t.Fatalf("unexpected error %v", err) } } func TestExist(t *testing.T) { fdir := filepath.Join(os.TempDir(), fmt.Sprint(time.Now().UnixNano()+rand.Int63n(1000))) os.RemoveAll(fdir) if err := os.Mkdir(fdir, 0o666); err != nil { t.Skip(err) } defer os.RemoveAll(fdir) require.Truef(t, Exist(fdir), "expected Exist true, got %v", Exist(fdir)) f, err := os.CreateTemp(os.TempDir(), "fileutil") require.NoError(t, err) f.Close() if g := Exist(f.Name()); !g { t.Errorf("exist = %v, want true", g) } os.Remove(f.Name()) if g := Exist(f.Name()); g { t.Errorf("exist = %v, want false", g) } } func TestDirEmpty(t *testing.T) { dir := t.TempDir() require.Truef(t, DirEmpty(dir), "expected DirEmpty true, got %v", DirEmpty(dir)) file, err := os.CreateTemp(dir, "new_file") require.NoError(t, err) file.Close() require.Falsef(t, DirEmpty(dir), "expected DirEmpty false, got %v", DirEmpty(dir)) require.Falsef(t, DirEmpty(file.Name()), "expected DirEmpty false, got %v", DirEmpty(file.Name())) } func TestZeroToEnd(t *testing.T) { f, err := os.CreateTemp(os.TempDir(), "fileutil") require.NoError(t, err) defer os.Remove(f.Name()) defer f.Close() // Ensure 0 size is a nop so zero-to-end on an empty file won't give EINVAL. require.NoError(t, ZeroToEnd(f)) b := make([]byte, 1024) for i := range b { b[i] = 12 } _, err = f.Write(b) require.NoError(t, err) _, err = f.Seek(512, io.SeekStart) require.NoError(t, err) require.NoError(t, ZeroToEnd(f)) off, serr := f.Seek(0, io.SeekCurrent) require.NoError(t, serr) require.Equalf(t, int64(512), off, "expected offset 512, got %d", off) b = make([]byte, 512) _, err = f.Read(b) require.NoError(t, err) for i := range b { if b[i] != 0 { t.Errorf("expected b[%d] = 0, got %d", i, b[i]) } } } func TestDirPermission(t *testing.T) { tmpdir := t.TempDir() tmpdir2 := filepath.Join(tmpdir, "testpermission") // create a new dir with 0700 require.NoError(t, CreateDirAll(zaptest.NewLogger(t), tmpdir2)) // check dir permission with mode different than created dir if err := CheckDirPermission(tmpdir2, 0o600); err == nil { t.Errorf("expected error, got nil") } } func TestRemoveMatchFile(t *testing.T) { tmpdir := t.TempDir() f, err := os.CreateTemp(tmpdir, "tmp") require.NoError(t, err) f.Close() f, err = os.CreateTemp(tmpdir, "foo.tmp") require.NoError(t, err) f.Close() err = RemoveMatchFile(zaptest.NewLogger(t), tmpdir, func(fileName string) bool { return strings.HasPrefix(fileName, "tmp") }) if err != nil { t.Errorf("expected nil, got error") } fnames, err := ReadDir(tmpdir) require.NoError(t, err) if len(fnames) != 1 { t.Errorf("expected exist 1 files, got %d", len(fnames)) } f, err = os.CreateTemp(tmpdir, "tmp") require.NoError(t, err) f.Close() err = RemoveMatchFile(zaptest.NewLogger(t), tmpdir, func(fileName string) bool { os.Remove(filepath.Join(tmpdir, fileName)) return strings.HasPrefix(fileName, "tmp") }) if err == nil { t.Errorf("expected error, got nil") } } func TestTouchDirAll(t *testing.T) { tmpdir := t.TempDir() assert.Panicsf(t, func() { TouchDirAll(nil, tmpdir) }, "expected panic with nil log") assert.NoError(t, TouchDirAll(zaptest.NewLogger(t), tmpdir)) } ================================================ FILE: client/pkg/fileutil/lock.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "errors" "os" ) var ErrLocked = errors.New("fileutil: file already locked") type LockedFile struct{ *os.File } ================================================ FILE: client/pkg/fileutil/lock_flock.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows && !plan9 && !solaris package fileutil import ( "errors" "os" "syscall" ) func flockTryLockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { f, err := os.OpenFile(path, flag, perm) if err != nil { return nil, err } if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { f.Close() if errors.Is(err, syscall.EWOULDBLOCK) { err = ErrLocked } return nil, err } return &LockedFile{f}, nil } func flockLockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { f, err := os.OpenFile(path, flag, perm) if err != nil { return nil, err } if err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil { f.Close() return nil, err } return &LockedFile{f}, err } ================================================ FILE: client/pkg/fileutil/lock_linux.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build linux package fileutil import ( "errors" "fmt" "io" "os" "syscall" "golang.org/x/sys/unix" ) // This used to call syscall.Flock() but that call fails with EBADF on NFS. // An alternative is lockf() which works on NFS but that call lets a process lock // the same file twice. Instead, use Linux's non-standard open file descriptor // locks which will block if the process already holds the file lock. var ( wrlck = syscall.Flock_t{ Type: syscall.F_WRLCK, Whence: int16(io.SeekStart), Start: 0, Len: 0, } linuxTryLockFile = flockTryLockFile linuxLockFile = flockLockFile ) func init() { // use open file descriptor locks if the system supports it getlk := syscall.Flock_t{Type: syscall.F_RDLCK} if err := syscall.FcntlFlock(0, unix.F_OFD_GETLK, &getlk); err == nil { linuxTryLockFile = ofdTryLockFile linuxLockFile = ofdLockFile } } func TryLockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { return linuxTryLockFile(path, flag, perm) } func ofdTryLockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { f, err := os.OpenFile(path, flag, perm) if err != nil { return nil, fmt.Errorf("ofdTryLockFile failed to open %q (%w)", path, err) } flock := wrlck if err = syscall.FcntlFlock(f.Fd(), unix.F_OFD_SETLK, &flock); err != nil { f.Close() if errors.Is(err, syscall.EWOULDBLOCK) { err = ErrLocked } return nil, err } return &LockedFile{f}, nil } func LockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { return linuxLockFile(path, flag, perm) } func ofdLockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { f, err := os.OpenFile(path, flag, perm) if err != nil { return nil, fmt.Errorf("ofdLockFile failed to open %q (%w)", path, err) } flock := wrlck err = syscall.FcntlFlock(f.Fd(), unix.F_OFD_SETLKW, &flock) if err != nil { f.Close() return nil, err } return &LockedFile{f}, nil } ================================================ FILE: client/pkg/fileutil/lock_linux_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build linux package fileutil import "testing" // TestLockAndUnlockSyscallFlock tests the fallback flock using the flock syscall. func TestLockAndUnlockSyscallFlock(t *testing.T) { oldTryLock, oldLock := linuxTryLockFile, linuxLockFile defer func() { linuxTryLockFile, linuxLockFile = oldTryLock, oldLock }() linuxTryLockFile, linuxLockFile = flockTryLockFile, flockLockFile TestLockAndUnlock(t) } ================================================ FILE: client/pkg/fileutil/lock_plan9.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "os" "syscall" "time" ) func TryLockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { if err := os.Chmod(path, syscall.DMEXCL|PrivateFileMode); err != nil { return nil, err } f, err := os.Open(path, flag, perm) if err != nil { return nil, ErrLocked } return &LockedFile{f}, nil } func LockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { if err := os.Chmod(path, syscall.DMEXCL|PrivateFileMode); err != nil { return nil, err } for { f, err := os.OpenFile(path, flag, perm) if err == nil { return &LockedFile{f}, nil } time.Sleep(10 * time.Millisecond) } } ================================================ FILE: client/pkg/fileutil/lock_solaris.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build solaris package fileutil import ( "os" "syscall" ) func TryLockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { var lock syscall.Flock_t lock.Start = 0 lock.Len = 0 lock.Pid = 0 lock.Type = syscall.F_WRLCK lock.Whence = 0 lock.Pid = 0 f, err := os.OpenFile(path, flag, perm) if err != nil { return nil, err } if err := syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &lock); err != nil { f.Close() if err == syscall.EAGAIN { err = ErrLocked } return nil, err } return &LockedFile{f}, nil } func LockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { var lock syscall.Flock_t lock.Start = 0 lock.Len = 0 lock.Pid = 0 lock.Type = syscall.F_WRLCK lock.Whence = 0 f, err := os.OpenFile(path, flag, perm) if err != nil { return nil, err } if err = syscall.FcntlFlock(f.Fd(), syscall.F_SETLKW, &lock); err != nil { f.Close() return nil, err } return &LockedFile{f}, nil } ================================================ FILE: client/pkg/fileutil/lock_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "os" "testing" "time" "github.com/stretchr/testify/require" ) func TestLockAndUnlock(t *testing.T) { f, err := os.CreateTemp(t.TempDir(), "lock") require.NoError(t, err) f.Close() defer func() { require.NoError(t, os.Remove(f.Name())) }() // lock the file l, err := LockFile(f.Name(), os.O_WRONLY, PrivateFileMode) require.NoError(t, err) // try lock a locked file _, err = TryLockFile(f.Name(), os.O_WRONLY, PrivateFileMode) require.ErrorIs(t, err, ErrLocked) // unlock the file require.NoError(t, l.Close()) // try lock the unlocked file dupl, err := TryLockFile(f.Name(), os.O_WRONLY, PrivateFileMode) if err != nil { t.Errorf("err = %v, want %v", err, nil) } // blocking on locked file locked := make(chan struct{}, 1) go func() { bl, blerr := LockFile(f.Name(), os.O_WRONLY, PrivateFileMode) if blerr != nil { t.Error(blerr) } locked <- struct{}{} if blerr = bl.Close(); blerr != nil { t.Error(blerr) } }() select { case <-locked: t.Error("unexpected unblocking") case <-time.After(100 * time.Millisecond): } // unlock require.NoError(t, dupl.Close()) // the previously blocked routine should be unblocked select { case <-locked: case <-time.After(1 * time.Second): t.Error("unexpected blocking") } } ================================================ FILE: client/pkg/fileutil/lock_unix.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows && !plan9 && !solaris && !linux package fileutil import ( "os" ) func TryLockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { return flockTryLockFile(path, flag, perm) } func LockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { return flockLockFile(path, flag, perm) } ================================================ FILE: client/pkg/fileutil/lock_windows.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows package fileutil import ( "errors" "fmt" "os" "syscall" "golang.org/x/sys/windows" ) var errLocked = errors.New("the process cannot access the file because another process has locked a portion of the file") func TryLockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { f, err := open(path, flag, perm) if err != nil { return nil, err } if err := lockFile(windows.Handle(f.Fd()), windows.LOCKFILE_FAIL_IMMEDIATELY); err != nil { f.Close() return nil, err } return &LockedFile{f}, nil } func LockFile(path string, flag int, perm os.FileMode) (*LockedFile, error) { f, err := open(path, flag, perm) if err != nil { return nil, err } if err := lockFile(windows.Handle(f.Fd()), 0); err != nil { f.Close() return nil, err } return &LockedFile{f}, nil } func open(path string, flag int, perm os.FileMode) (*os.File, error) { if path == "" { return nil, errors.New("cannot open empty filename") } var access uint32 switch flag { case syscall.O_RDONLY: access = syscall.GENERIC_READ case syscall.O_WRONLY: access = syscall.GENERIC_WRITE case syscall.O_RDWR: access = syscall.GENERIC_READ | syscall.GENERIC_WRITE case syscall.O_WRONLY | syscall.O_CREAT: access = syscall.GENERIC_ALL default: panic(fmt.Errorf("flag %v is not supported", flag)) } fd, err := syscall.CreateFile(&(syscall.StringToUTF16(path)[0]), access, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, nil, syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) if err != nil { return nil, err } return os.NewFile(uintptr(fd), path), nil } func lockFile(fd windows.Handle, flags uint32) error { if fd == windows.InvalidHandle { return nil } err := windows.LockFileEx(fd, flags|windows.LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &windows.Overlapped{}) if err == nil { return nil } else if err.Error() == errLocked.Error() { return ErrLocked } else if err != windows.ERROR_LOCK_VIOLATION { return err } return nil } ================================================ FILE: client/pkg/fileutil/preallocate.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "io" "os" ) // Preallocate tries to allocate the space for given file. This // operation is only supported on darwin and linux by a few // filesystems (APFS, btrfs, ext4, etc.). // If the operation is unsupported, no error will be returned. // Otherwise, the error encountered will be returned. func Preallocate(f *os.File, sizeInBytes int64, extendFile bool) error { if sizeInBytes == 0 { // fallocate will return EINVAL if length is 0; skip return nil } if extendFile { return preallocExtend(f, sizeInBytes) } return preallocFixed(f, sizeInBytes) } func preallocExtendTrunc(f *os.File, sizeInBytes int64) error { curOff, err := f.Seek(0, io.SeekCurrent) if err != nil { return err } size, err := f.Seek(sizeInBytes, io.SeekEnd) if err != nil { return err } if _, err = f.Seek(curOff, io.SeekStart); err != nil { return err } if sizeInBytes > size { return nil } return f.Truncate(sizeInBytes) } ================================================ FILE: client/pkg/fileutil/preallocate_darwin.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build darwin package fileutil import ( "errors" "os" "syscall" "golang.org/x/sys/unix" ) func preallocExtend(f *os.File, sizeInBytes int64) error { if err := preallocFixed(f, sizeInBytes); err != nil { return err } return preallocExtendTrunc(f, sizeInBytes) } func preallocFixed(f *os.File, sizeInBytes int64) error { // allocate all requested space or no space at all // TODO: allocate contiguous space on disk with F_ALLOCATECONTIG flag fstore := &unix.Fstore_t{ Flags: unix.F_ALLOCATEALL, Posmode: unix.F_PEOFPOSMODE, Length: sizeInBytes, } err := unix.FcntlFstore(f.Fd(), unix.F_PREALLOCATE, fstore) if err == nil || errors.Is(err, unix.ENOTSUP) { return nil } // wrong argument to fallocate syscall if err == unix.EINVAL { // filesystem "st_blocks" are allocated in the units of // "Allocation Block Size" (run "diskutil info /" command) var stat syscall.Stat_t syscall.Fstat(int(f.Fd()), &stat) // syscall.Statfs_t.Bsize is "optimal transfer block size" // and contains matching 4096 value when latest OS X kernel // supports 4,096 KB filesystem block size var statfs syscall.Statfs_t syscall.Fstatfs(int(f.Fd()), &statfs) blockSize := int64(statfs.Bsize) if stat.Blocks*blockSize >= sizeInBytes { // enough blocks are already allocated return nil } } return err } ================================================ FILE: client/pkg/fileutil/preallocate_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "os" "testing" "github.com/stretchr/testify/require" ) func TestPreallocateExtend(t *testing.T) { pf := func(f *os.File, sz int64) error { return Preallocate(f, sz, true) } tf := func(t *testing.T, f *os.File) { t.Helper() testPreallocateExtend(t, f, pf) } runPreallocTest(t, tf) } func TestPreallocateExtendTrunc(t *testing.T) { tf := func(t *testing.T, f *os.File) { t.Helper() testPreallocateExtend(t, f, preallocExtendTrunc) } runPreallocTest(t, tf) } func testPreallocateExtend(t *testing.T, f *os.File, pf func(*os.File, int64) error) { t.Helper() size := int64(64 * 1000) require.NoError(t, pf(f, size)) stat, err := f.Stat() require.NoError(t, err) if stat.Size() != size { t.Errorf("size = %d, want %d", stat.Size(), size) } } func TestPreallocateFixed(t *testing.T) { runPreallocTest(t, testPreallocateFixed) } func testPreallocateFixed(t *testing.T, f *os.File) { t.Helper() size := int64(64 * 1000) require.NoError(t, Preallocate(f, size, false)) stat, err := f.Stat() require.NoError(t, err) if stat.Size() != 0 { t.Errorf("size = %d, want %d", stat.Size(), 0) } } func runPreallocTest(t *testing.T, test func(*testing.T, *os.File)) { t.Helper() p := t.TempDir() f, err := os.CreateTemp(p, "") require.NoError(t, err) test(t, f) } ================================================ FILE: client/pkg/fileutil/preallocate_unix.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build linux package fileutil import ( "errors" "os" "syscall" ) func preallocExtend(f *os.File, sizeInBytes int64) error { // use mode = 0 to change size err := syscall.Fallocate(int(f.Fd()), 0, 0, sizeInBytes) if err != nil { var errno syscall.Errno // not supported; fallback // fallocate EINTRs frequently in some environments; fallback if errors.As(err, &errno) && (errno == syscall.ENOTSUP || errno == syscall.EINTR) { return preallocExtendTrunc(f, sizeInBytes) } } return err } func preallocFixed(f *os.File, sizeInBytes int64) error { // use mode = 1 to keep size; see FALLOC_FL_KEEP_SIZE err := syscall.Fallocate(int(f.Fd()), 1, 0, sizeInBytes) if err != nil { var errno syscall.Errno // treat not supported as nil error if errors.As(err, &errno) && errno == syscall.ENOTSUP { return nil } } return err } ================================================ FILE: client/pkg/fileutil/preallocate_unsupported.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !linux && !darwin package fileutil import "os" func preallocExtend(f *os.File, sizeInBytes int64) error { return preallocExtendTrunc(f, sizeInBytes) } func preallocFixed(f *os.File, sizeInBytes int64) error { return nil } ================================================ FILE: client/pkg/fileutil/purge.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "os" "path/filepath" "strings" "time" "go.uber.org/zap" ) func PurgeFile(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) <-chan error { return purgeFile(lg, dirname, suffix, max, interval, stop, nil, nil, true) } func PurgeFileWithDoneNotify(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) (<-chan struct{}, <-chan error) { doneC := make(chan struct{}) errC := purgeFile(lg, dirname, suffix, max, interval, stop, nil, doneC, true) return doneC, errC } func PurgeFileWithoutFlock(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) (<-chan struct{}, <-chan error) { doneC := make(chan struct{}) errC := purgeFile(lg, dirname, suffix, max, interval, stop, nil, doneC, false) return doneC, errC } // purgeFile is the internal implementation for PurgeFile which can post purged files to purgec if non-nil. // if donec is non-nil, the function closes it to notify its exit. func purgeFile(lg *zap.Logger, dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}, purgec chan<- string, donec chan<- struct{}, flock bool) <-chan error { if lg == nil { lg = zap.NewNop() } errC := make(chan error, 1) lg.Info("started to purge file", zap.String("dir", dirname), zap.String("suffix", suffix), zap.Uint("max", max), zap.Duration("interval", interval)) go func() { if donec != nil { defer close(donec) } for { fnamesWithSuffix, err := readDirWithSuffix(dirname, suffix) if err != nil { errC <- err return } nPurged := 0 for nPurged < len(fnamesWithSuffix)-int(max) { f := filepath.Join(dirname, fnamesWithSuffix[nPurged]) var l *LockedFile if flock { l, err = TryLockFile(f, os.O_WRONLY, PrivateFileMode) if err != nil { lg.Warn("failed to lock file", zap.String("path", f), zap.Error(err)) break } } if err = os.Remove(f); err != nil { lg.Error("failed to remove file", zap.String("path", f), zap.Error(err)) errC <- err return } if flock { if err = l.Close(); err != nil { lg.Error("failed to unlock/close", zap.String("path", l.Name()), zap.Error(err)) errC <- err return } } lg.Info("purged", zap.String("path", f)) nPurged++ } if purgec != nil { for i := 0; i < nPurged; i++ { purgec <- fnamesWithSuffix[i] } } select { case <-time.After(interval): case <-stop: return } } }() return errC } func readDirWithSuffix(dirname string, suffix string) ([]string, error) { fnames, err := ReadDir(dirname) if err != nil { return nil, err } // filter in place (ref. https://go.dev/wiki/SliceTricks#filtering-without-allocating) fnamesWithSuffix := fnames[:0] for _, fname := range fnames { if strings.HasSuffix(fname, suffix) { fnamesWithSuffix = append(fnamesWithSuffix, fname) } } return fnamesWithSuffix, nil } ================================================ FILE: client/pkg/fileutil/purge_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "fmt" "os" "path/filepath" "reflect" "testing" "time" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" ) func TestPurgeFile(t *testing.T) { dir := t.TempDir() // minimal file set for i := 0; i < 3; i++ { f, ferr := os.Create(filepath.Join(dir, fmt.Sprintf("%d.test", i))) require.NoError(t, ferr) f.Close() } stop, purgec := make(chan struct{}), make(chan string, 10) // keep 3 most recent files errch := purgeFile(zaptest.NewLogger(t), dir, "test", 3, time.Millisecond, stop, purgec, nil, false) select { case f := <-purgec: t.Errorf("unexpected purge on %q", f) case <-time.After(10 * time.Millisecond): } // rest of the files for i := 4; i < 10; i++ { go func(n int) { f, ferr := os.Create(filepath.Join(dir, fmt.Sprintf("%d.test", n))) if ferr != nil { t.Error(ferr) } f.Close() }(i) } // watch files purge away for i := 4; i < 10; i++ { select { case <-purgec: case <-time.After(time.Second): t.Errorf("purge took too long") } } fnames, rerr := ReadDir(dir) require.NoError(t, rerr) wnames := []string{"7.test", "8.test", "9.test"} if !reflect.DeepEqual(fnames, wnames) { t.Errorf("filenames = %v, want %v", fnames, wnames) } // no error should be reported from purge routine select { case f := <-purgec: t.Errorf("unexpected purge on %q", f) case err := <-errch: t.Errorf("unexpected purge error %v", err) case <-time.After(10 * time.Millisecond): } close(stop) } func TestPurgeFileHoldingLockFile(t *testing.T) { dir := t.TempDir() for i := 0; i < 10; i++ { var f *os.File f, err := os.Create(filepath.Join(dir, fmt.Sprintf("%d.test", i))) require.NoError(t, err) f.Close() } // create a purge barrier at 5 p := filepath.Join(dir, fmt.Sprintf("%d.test", 5)) l, err := LockFile(p, os.O_WRONLY, PrivateFileMode) require.NoError(t, err) stop, purgec := make(chan struct{}), make(chan string, 10) errch := purgeFile(zaptest.NewLogger(t), dir, "test", 3, time.Millisecond, stop, purgec, nil, true) for i := 0; i < 5; i++ { select { case <-purgec: case <-time.After(time.Second): t.Fatalf("purge took too long") } } fnames, rerr := ReadDir(dir) require.NoError(t, rerr) wnames := []string{"5.test", "6.test", "7.test", "8.test", "9.test"} if !reflect.DeepEqual(fnames, wnames) { t.Errorf("filenames = %v, want %v", fnames, wnames) } select { case s := <-purgec: t.Errorf("unexpected purge %q", s) case err = <-errch: t.Errorf("unexpected purge error %v", err) case <-time.After(10 * time.Millisecond): } // remove the purge barrier require.NoError(t, l.Close()) // wait for rest of purges (5, 6) for i := 0; i < 2; i++ { select { case <-purgec: case <-time.After(time.Second): t.Fatalf("purge took too long") } } fnames, rerr = ReadDir(dir) require.NoError(t, rerr) wnames = []string{"7.test", "8.test", "9.test"} if !reflect.DeepEqual(fnames, wnames) { t.Errorf("filenames = %v, want %v", fnames, wnames) } select { case f := <-purgec: t.Errorf("unexpected purge on %q", f) case err := <-errch: t.Errorf("unexpected purge error %v", err) case <-time.After(10 * time.Millisecond): } close(stop) } ================================================ FILE: client/pkg/fileutil/read_dir.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "os" "path/filepath" "sort" ) // ReadDirOp represents an read-directory operation. type ReadDirOp struct { ext string } // ReadDirOption configures archiver operations. type ReadDirOption func(*ReadDirOp) // WithExt filters file names by their extensions. // (e.g. WithExt(".wal") to list only WAL files) func WithExt(ext string) ReadDirOption { return func(op *ReadDirOp) { op.ext = ext } } func (op *ReadDirOp) applyOpts(opts []ReadDirOption) { for _, opt := range opts { opt(op) } } // ReadDir returns the filenames in the given directory in sorted order. func ReadDir(d string, opts ...ReadDirOption) ([]string, error) { op := &ReadDirOp{} op.applyOpts(opts) dir, err := os.Open(d) if err != nil { return nil, err } defer dir.Close() names, err := dir.Readdirnames(-1) if err != nil { return nil, err } sort.Strings(names) if op.ext != "" { tss := make([]string, 0) for _, v := range names { if filepath.Ext(v) == op.ext { tss = append(tss, v) } } names = tss } return names, nil } ================================================ FILE: client/pkg/fileutil/read_dir_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package fileutil import ( "os" "path/filepath" "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestReadDir(t *testing.T) { tmpdir := t.TempDir() files := []string{"def", "abc", "xyz", "ghi"} for _, f := range files { writeFunc(t, filepath.Join(tmpdir, f)) } fs, err := ReadDir(tmpdir) require.NoErrorf(t, err, "error calling ReadDir") wfs := []string{"abc", "def", "ghi", "xyz"} require.Truef(t, reflect.DeepEqual(fs, wfs), "ReadDir: got %v, want %v", fs, wfs) files = []string{"def.wal", "abc.wal", "xyz.wal", "ghi.wal"} for _, f := range files { writeFunc(t, filepath.Join(tmpdir, f)) } fs, err = ReadDir(tmpdir, WithExt(".wal")) require.NoErrorf(t, err, "error calling ReadDir") wfs = []string{"abc.wal", "def.wal", "ghi.wal", "xyz.wal"} require.Truef(t, reflect.DeepEqual(fs, wfs), "ReadDir: got %v, want %v", fs, wfs) } func writeFunc(t *testing.T, path string) { t.Helper() fh, err := os.Create(path) require.NoErrorf(t, err, "error creating file") assert.NoErrorf(t, fh.Close(), "error closing file") } ================================================ FILE: client/pkg/fileutil/sync.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !linux && !darwin package fileutil import "os" // Fsync is a wrapper around file.Sync(). Special handling is needed on darwin platform. func Fsync(f *os.File) error { return f.Sync() } // Fdatasync is a wrapper around file.Sync(). Special handling is needed on linux platform. func Fdatasync(f *os.File) error { return f.Sync() } ================================================ FILE: client/pkg/fileutil/sync_darwin.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build darwin package fileutil import ( "os" "golang.org/x/sys/unix" ) // Fsync on HFS/OSX flushes the data on to the physical drive but the drive // may not write it to the persistent media for quite sometime and it may be // written in out-of-order sequence. Using F_FULLFSYNC ensures that the // physical drive's buffer will also get flushed to the media. func Fsync(f *os.File) error { _, err := unix.FcntlInt(f.Fd(), unix.F_FULLFSYNC, 0) return err } // Fdatasync on darwin platform invokes fcntl(F_FULLFSYNC) for actual persistence // on physical drive media. func Fdatasync(f *os.File) error { return Fsync(f) } ================================================ FILE: client/pkg/fileutil/sync_linux.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build linux package fileutil import ( "os" "syscall" ) // Fsync is a wrapper around file.Sync(). Special handling is needed on darwin platform. func Fsync(f *os.File) error { return f.Sync() } // Fdatasync is similar to fsync(), but does not flush modified metadata // unless that metadata is needed in order to allow a subsequent data retrieval // to be correctly handled. func Fdatasync(f *os.File) error { return syscall.Fdatasync(int(f.Fd())) } ================================================ FILE: client/pkg/go.mod ================================================ module go.etcd.io/etcd/client/pkg/v3 go 1.26 toolchain go1.26.1 require ( github.com/coreos/go-systemd/v22 v22.7.0 github.com/stretchr/testify v1.11.1 go.uber.org/zap v1.27.1 golang.org/x/sys v0.41.0 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect go.uber.org/multierr v1.11.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) ================================================ FILE: client/pkg/go.sum ================================================ github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: client/pkg/logutil/doc.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package logutil includes utilities to facilitate logging. package logutil ================================================ FILE: client/pkg/logutil/log_format.go ================================================ // Copyright 2019 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package logutil import "fmt" const ( JSONLogFormat = "json" ConsoleLogFormat = "console" //revive:disable:var-naming // Deprecated: Please use JSONLogFormat. JsonLogFormat = JSONLogFormat //revive:enable:var-naming ) var DefaultLogFormat = JSONLogFormat // ConvertToZapFormat converts and validated log format string. func ConvertToZapFormat(format string) (string, error) { switch format { case ConsoleLogFormat: return ConsoleLogFormat, nil case JSONLogFormat: return JSONLogFormat, nil case "": return DefaultLogFormat, nil default: return "", fmt.Errorf("unknown log format: %s, supported values json, console", format) } } ================================================ FILE: client/pkg/logutil/log_format_test.go ================================================ // Copyright 2019 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package logutil import ( "testing" ) func TestLogFormat(t *testing.T) { tests := []struct { given string want string errExpected bool }{ {"json", JSONLogFormat, false}, {"console", ConsoleLogFormat, false}, {"", JSONLogFormat, false}, {"konsole", "", true}, } for i, tt := range tests { got, err := ConvertToZapFormat(tt.given) if got != tt.want { t.Errorf("#%d: ConvertToZapFormat failure: want=%v, got=%v", i, tt.want, got) } if err != nil { if !tt.errExpected { t.Errorf("#%d: ConvertToZapFormat unexpected error: %v", i, err) } } } } ================================================ FILE: client/pkg/logutil/log_level.go ================================================ // Copyright 2019 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package logutil import ( "go.uber.org/zap/zapcore" ) var DefaultLogLevel = "info" // ConvertToZapLevel converts log level string to zapcore.Level. func ConvertToZapLevel(lvl string) zapcore.Level { var level zapcore.Level if err := level.Set(lvl); err != nil { panic(err) } return level } ================================================ FILE: client/pkg/logutil/zap.go ================================================ // Copyright 2019 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package logutil import ( "slices" "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) // CreateDefaultZapLogger creates a logger with default zap configuration func CreateDefaultZapLogger(level zapcore.Level) (*zap.Logger, error) { lcfg := DefaultZapLoggerConfig lcfg.Level = zap.NewAtomicLevelAt(level) c, err := lcfg.Build() if err != nil { return nil, err } return c, nil } // DefaultZapLoggerConfig defines default zap logger configuration. var DefaultZapLoggerConfig = zap.Config{ Level: zap.NewAtomicLevelAt(ConvertToZapLevel(DefaultLogLevel)), Development: false, Sampling: &zap.SamplingConfig{ Initial: 100, Thereafter: 100, }, Encoding: DefaultLogFormat, // copied from "zap.NewProductionEncoderConfig" with some updates EncoderConfig: zapcore.EncoderConfig{ TimeKey: "ts", LevelKey: "level", NameKey: "logger", CallerKey: "caller", MessageKey: "msg", StacktraceKey: "stacktrace", LineEnding: zapcore.DefaultLineEnding, EncodeLevel: zapcore.LowercaseLevelEncoder, // Custom EncodeTime function to ensure we match format and precision of historic capnslog timestamps EncodeTime: func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { enc.AppendString(t.Format("2006-01-02T15:04:05.000000Z0700")) }, EncodeDuration: zapcore.StringDurationEncoder, EncodeCaller: zapcore.ShortCallerEncoder, }, // Use "/dev/null" to discard all OutputPaths: []string{"stderr"}, ErrorOutputPaths: []string{"stderr"}, } // MergeOutputPaths merges logging output paths, resolving conflicts. func MergeOutputPaths(cfg zap.Config) zap.Config { cfg.OutputPaths = mergePaths(cfg.OutputPaths) cfg.ErrorOutputPaths = mergePaths(cfg.ErrorOutputPaths) return cfg } func mergePaths(old []string) []string { if len(old) == 0 { // the original implementation ensures the result is non-nil return []string{} } // use "/dev/null" to discard all if slices.Contains(old, "/dev/null") { return []string{"/dev/null"} } // clone a new one; don't modify the original, in case it matters. dup := slices.Clone(old) slices.Sort(dup) return slices.Compact(dup) } ================================================ FILE: client/pkg/logutil/zap_journal.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows package logutil import ( "bytes" "encoding/json" "fmt" "io" "os" "path/filepath" "github.com/coreos/go-systemd/v22/journal" "go.uber.org/zap/zapcore" "go.etcd.io/etcd/client/pkg/v3/systemd" ) // NewJournalWriter wraps "io.Writer" to redirect log output // to the local systemd journal. If journald send fails, it fails // back to writing to the original writer. // The decode overhead is only <30µs per write. // Reference: https://github.com/coreos/pkg/blob/master/capnslog/journald_formatter.go func NewJournalWriter(wr io.Writer) (io.Writer, error) { return &journalWriter{Writer: wr}, systemd.DialJournal() } type journalWriter struct { io.Writer } // WARN: assume that etcd uses default field names in zap encoder config // make sure to keep this up-to-date! type logLine struct { Level string `json:"level"` Caller string `json:"caller"` } func (w *journalWriter) Write(p []byte) (int, error) { line := &logLine{} if err := json.NewDecoder(bytes.NewReader(p)).Decode(line); err != nil { return 0, err } var pri journal.Priority switch line.Level { case zapcore.DebugLevel.String(): pri = journal.PriDebug case zapcore.InfoLevel.String(): pri = journal.PriInfo case zapcore.WarnLevel.String(): pri = journal.PriWarning case zapcore.ErrorLevel.String(): pri = journal.PriErr case zapcore.DPanicLevel.String(): pri = journal.PriCrit case zapcore.PanicLevel.String(): pri = journal.PriCrit case zapcore.FatalLevel.String(): pri = journal.PriCrit default: panic(fmt.Errorf("unknown log level: %q", line.Level)) } err := journal.Send(string(p), pri, map[string]string{ "PACKAGE": filepath.Dir(line.Caller), "SYSLOG_IDENTIFIER": filepath.Base(os.Args[0]), }) if err != nil { // "journal" also falls back to stderr // "fmt.Fprintln(os.Stderr, s)" return w.Writer.Write(p) } return 0, nil } ================================================ FILE: client/pkg/logutil/zap_journal_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows package logutil import ( "bytes" "testing" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) func TestNewJournalWriter(t *testing.T) { buf := bytes.NewBuffer(nil) jw, err := NewJournalWriter(buf) if err != nil { t.Skip(err) } syncer := zapcore.AddSync(jw) cr := zapcore.NewCore( zapcore.NewJSONEncoder(DefaultZapLoggerConfig.EncoderConfig), syncer, zap.NewAtomicLevelAt(zap.InfoLevel), ) lg := zap.New(cr, zap.AddCaller(), zap.ErrorOutput(syncer)) defer lg.Sync() lg.Info("TestNewJournalWriter") if buf.String() == "" { // check with "journalctl -f" t.Log("sent logs successfully to journald") } } ================================================ FILE: client/pkg/logutil/zap_test.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package logutil import ( "bytes" "encoding/json" "regexp" "slices" "testing" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) type commonLogFields struct { Level string `json:"level"` Timestamp string `json:"ts"` Message string `json:"msg"` } const ( fractionSecondsPrecision = 6 // MicroSeconds ) func TestEncodeTimePrecisionToMicroSeconds(t *testing.T) { buf := bytes.NewBuffer(nil) syncer := zapcore.AddSync(buf) zc := zapcore.NewCore( zapcore.NewJSONEncoder(DefaultZapLoggerConfig.EncoderConfig), syncer, zap.NewAtomicLevelAt(zap.InfoLevel), ) lg := zap.New(zc) lg.Info("TestZapLog") fields := commonLogFields{} require.NoError(t, json.Unmarshal(buf.Bytes(), &fields)) // example 1: 2024-06-06T23:37:21.948385Z // example 2 with zone offset: 2024-06-06T16:16:44.176778-0700 regex := `\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.(\d+)(Z|[+-]\d{4})` re := regexp.MustCompile(regex) matches := re.FindStringSubmatch(fields.Timestamp) require.Len(t, matches, 3) require.Lenf(t, matches[1], fractionSecondsPrecision, "unexpected timestamp %s", fields.Timestamp) } func TestMergeOutputPaths(t *testing.T) { tests := []struct { name string cfg zap.Config want zap.Config }{ { name: "OutputPaths /dev/null", cfg: zap.Config{ OutputPaths: []string{"c", "/dev/null"}, ErrorOutputPaths: []string{"c", "a", "a", "b"}, }, want: zap.Config{ OutputPaths: []string{"/dev/null"}, ErrorOutputPaths: []string{"a", "b", "c"}, }, }, { name: "ErrorOutputPaths /dev/null", cfg: zap.Config{ OutputPaths: []string{"c", "a", "a", "b"}, ErrorOutputPaths: []string{"/dev/null", "c"}, }, want: zap.Config{ OutputPaths: []string{"a", "b", "c"}, ErrorOutputPaths: []string{"/dev/null"}, }, }, { name: "empty slice", cfg: zap.Config{ OutputPaths: []string{}, ErrorOutputPaths: []string{"c", "a", "a", "b"}, }, want: zap.Config{ OutputPaths: []string{}, ErrorOutputPaths: []string{"a", "b", "c"}, }, }, { name: "nil slice", cfg: zap.Config{ OutputPaths: []string{"c", "a", "a", "b"}, ErrorOutputPaths: nil, }, want: zap.Config{ OutputPaths: []string{"a", "b", "c"}, ErrorOutputPaths: []string{}, }, }, { name: "normal", cfg: zap.Config{ OutputPaths: []string{"c", "a", "a", "b"}, ErrorOutputPaths: []string{"c", "a", "a", "b"}, }, want: zap.Config{ OutputPaths: []string{"a", "b", "c"}, ErrorOutputPaths: []string{"a", "b", "c"}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { outputPaths := slices.Clone(tt.cfg.OutputPaths) errorOutputPaths := slices.Clone(tt.cfg.ErrorOutputPaths) require.Equal(t, tt.want, MergeOutputPaths(tt.cfg)) // ensure the OutputPaths and ErrorOutputPaths have not been modified require.Equal(t, outputPaths, tt.cfg.OutputPaths) require.Equal(t, errorOutputPaths, tt.cfg.ErrorOutputPaths) }) } } ================================================ FILE: client/pkg/pathutil/path.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package pathutil implements utility functions for handling slash-separated // paths. package pathutil import "path" // CanonicalURLPath returns the canonical url path for p, which follows the rules: // 1. the path always starts with "/" // 2. replace multiple slashes with a single slash // 3. replace each '.' '..' path name element with equivalent one // 4. keep the trailing slash // The function is borrowed from stdlib http.cleanPath in server.go. func CanonicalURLPath(p string) string { if p == "" { return "/" } if p[0] != '/' { p = "/" + p } np := path.Clean(p) // path.Clean removes trailing slash except for root, // put the trailing slash back if necessary. if p[len(p)-1] == '/' && np != "/" { np += "/" } return np } ================================================ FILE: client/pkg/pathutil/path_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pathutil import "testing" func TestCanonicalURLPath(t *testing.T) { tests := []struct { p string wp string }{ {"/a", "/a"}, {"", "/"}, {"a", "/a"}, {"//a", "/a"}, {"/a/.", "/a"}, {"/a/..", "/"}, {"/a/", "/a/"}, {"/a//", "/a/"}, } for i, tt := range tests { if g := CanonicalURLPath(tt.p); g != tt.wp { t.Errorf("#%d: canonical path = %s, want %s", i, g, tt.wp) } } } ================================================ FILE: client/pkg/srv/srv.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package srv looks up DNS SRV records. package srv import ( "fmt" "net" "net/url" "strings" "go.etcd.io/etcd/client/pkg/v3/types" ) var ( // indirection for testing lookupSRV = net.LookupSRV // net.DefaultResolver.LookupSRV when ctxs don't conflict resolveTCPAddr = net.ResolveTCPAddr ) // GetCluster gets the cluster information via DNS discovery. // Also sees each entry as a separate instance. func GetCluster(serviceScheme, service, name, dns string, apurls types.URLs) ([]string, error) { tcp2ap := make(map[string]url.URL) // First, resolve the apurls for _, url := range apurls { tcpAddr, err := resolveTCPAddr("tcp", url.Host) if err != nil { return nil, err } tcp2ap[tcpAddr.String()] = url } var ( tempName int stringParts []string ) updateNodeMap := func(service, scheme string) error { _, addrs, err := lookupSRV(service, "tcp", dns) if err != nil { return err } for _, srv := range addrs { port := fmt.Sprintf("%d", srv.Port) host := net.JoinHostPort(srv.Target, port) tcpAddr, terr := resolveTCPAddr("tcp", host) if terr != nil { err = terr continue } n := "" url, ok := tcp2ap[tcpAddr.String()] if ok { n = name } if n == "" { n = fmt.Sprintf("%d", tempName) tempName++ } // SRV records have a trailing dot but URL shouldn't. shortHost := strings.TrimSuffix(srv.Target, ".") urlHost := net.JoinHostPort(shortHost, port) if ok && url.Scheme != scheme { err = fmt.Errorf("bootstrap at %s from DNS for %s has scheme mismatch with expected peer %s", scheme+"://"+urlHost, service, url.String()) } else { stringParts = append(stringParts, fmt.Sprintf("%s=%s://%s", n, scheme, urlHost)) } } if len(stringParts) == 0 { return err } return nil } err := updateNodeMap(service, serviceScheme) if err != nil { return nil, fmt.Errorf("error querying DNS SRV records for _%s %w", service, err) } return stringParts, nil } type SRVClients struct { Endpoints []string SRVs []*net.SRV } // GetClient looks up the client endpoints for a service and domain. func GetClient(service, domain string, serviceName string) (*SRVClients, error) { var ( urls []*url.URL srvs []*net.SRV ) updateURLs := func(service, scheme string) error { _, addrs, err := lookupSRV(service, "tcp", domain) if err != nil { return err } for _, srv := range addrs { urls = append(urls, &url.URL{ Scheme: scheme, Host: net.JoinHostPort(srv.Target, fmt.Sprintf("%d", srv.Port)), }) } srvs = append(srvs, addrs...) return nil } errHTTPS := updateURLs(GetSRVService(service, serviceName, "https"), "https") errHTTP := updateURLs(GetSRVService(service, serviceName, "http"), "http") if errHTTPS != nil && errHTTP != nil { return nil, fmt.Errorf("dns lookup errors: %w and %w", errHTTPS, errHTTP) } endpoints := make([]string, len(urls)) for i := range urls { endpoints[i] = urls[i].String() } return &SRVClients{Endpoints: endpoints, SRVs: srvs}, nil } // GetSRVService generates a SRV service including an optional suffix. func GetSRVService(service, serviceName string, scheme string) (SRVService string) { if scheme == "https" { service = fmt.Sprintf("%s-ssl", service) } if serviceName != "" { return fmt.Sprintf("%s-%s", service, serviceName) } return service } ================================================ FILE: client/pkg/srv/srv_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package srv import ( "errors" "fmt" "net" "reflect" "strings" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func notFoundErr(service, proto, domain string) error { name := fmt.Sprintf("_%s._%s.%s", service, proto, domain) return &net.DNSError{Err: "no such host", Name: name, Server: "10.0.0.53:53", IsTimeout: false, IsTemporary: false, IsNotFound: true} } func TestSRVGetCluster(t *testing.T) { defer func() { lookupSRV = net.LookupSRV resolveTCPAddr = net.ResolveTCPAddr }() hasErr := func(err error) bool { return err != nil } name := "dnsClusterTest" dns := map[string]string{ "1.example.com.:2480": "10.0.0.1:2480", "2.example.com.:2480": "10.0.0.2:2480", "3.example.com.:2480": "10.0.0.3:2480", "4.example.com.:2380": "10.0.0.3:2380", } srvAll := []*net.SRV{ {Target: "1.example.com.", Port: 2480}, {Target: "2.example.com.", Port: 2480}, {Target: "3.example.com.", Port: 2480}, } var srvNone []*net.SRV tests := []struct { service string scheme string withSSL []*net.SRV withoutSSL []*net.SRV urls []string expected string werr bool }{ { "etcd-server-ssl", "https", srvNone, srvNone, nil, "", true, }, { "etcd-server-ssl", "https", srvAll, srvNone, nil, "0=https://1.example.com:2480,1=https://2.example.com:2480,2=https://3.example.com:2480", false, }, { "etcd-server", "http", srvNone, srvAll, nil, "0=http://1.example.com:2480,1=http://2.example.com:2480,2=http://3.example.com:2480", false, }, { "etcd-server-ssl", "https", srvAll, srvNone, []string{"https://10.0.0.1:2480"}, "dnsClusterTest=https://1.example.com:2480,0=https://2.example.com:2480,1=https://3.example.com:2480", false, }, // matching local member with resolved addr and return unresolved hostnames { "etcd-server-ssl", "https", srvAll, srvNone, []string{"https://10.0.0.1:2480"}, "dnsClusterTest=https://1.example.com:2480,0=https://2.example.com:2480,1=https://3.example.com:2480", false, }, // reject if apurls are TLS but SRV is only http { "etcd-server", "http", srvNone, srvAll, []string{"https://10.0.0.1:2480"}, "0=http://2.example.com:2480,1=http://3.example.com:2480", false, }, } resolveTCPAddr = func(network, addr string) (*net.TCPAddr, error) { if strings.Contains(addr, "10.0.0.") { // accept IP addresses when resolving apurls return net.ResolveTCPAddr(network, addr) } if dns[addr] == "" { return nil, errors.New("missing dns record") } return net.ResolveTCPAddr(network, dns[addr]) } for i, tt := range tests { lookupSRV = func(service string, proto string, domain string) (string, []*net.SRV, error) { if service == "etcd-server-ssl" { if len(tt.withSSL) > 0 { return "", tt.withSSL, nil } return "", nil, notFoundErr(service, proto, domain) } if service == "etcd-server" { if len(tt.withoutSSL) > 0 { return "", tt.withoutSSL, nil } return "", nil, notFoundErr(service, proto, domain) } return "", nil, errors.New("unknown service in mock") } urls := testutil.MustNewURLs(t, tt.urls) str, err := GetCluster(tt.scheme, tt.service, name, "example.com", urls) require.Equalf(t, hasErr(err), tt.werr, "%d: err = %#v, want = %#v", i, err, tt.werr) require.Equalf(t, tt.expected, strings.Join(str, ","), "#%d: cluster = %s, want %s", i, str, tt.expected) } } func TestSRVDiscover(t *testing.T) { defer func() { lookupSRV = net.LookupSRV }() hasErr := func(err error) bool { return err != nil } tests := []struct { withSSL []*net.SRV withoutSSL []*net.SRV expected []string werr bool }{ { []*net.SRV{}, []*net.SRV{}, []string{}, true, }, { []*net.SRV{}, []*net.SRV{ {Target: "10.0.0.1", Port: 2480}, {Target: "10.0.0.2", Port: 2480}, {Target: "10.0.0.3", Port: 2480}, }, []string{"http://10.0.0.1:2480", "http://10.0.0.2:2480", "http://10.0.0.3:2480"}, false, }, { []*net.SRV{ {Target: "10.0.0.1", Port: 2480}, {Target: "10.0.0.2", Port: 2480}, {Target: "10.0.0.3", Port: 2480}, }, []*net.SRV{}, []string{"https://10.0.0.1:2480", "https://10.0.0.2:2480", "https://10.0.0.3:2480"}, false, }, { []*net.SRV{ {Target: "10.0.0.1", Port: 2480}, {Target: "10.0.0.2", Port: 2480}, {Target: "10.0.0.3", Port: 2480}, }, []*net.SRV{ {Target: "10.0.0.1", Port: 7001}, }, []string{"https://10.0.0.1:2480", "https://10.0.0.2:2480", "https://10.0.0.3:2480", "http://10.0.0.1:7001"}, false, }, { []*net.SRV{ {Target: "10.0.0.1", Port: 2480}, {Target: "10.0.0.2", Port: 2480}, {Target: "10.0.0.3", Port: 2480}, }, []*net.SRV{ {Target: "10.0.0.1", Port: 7001}, }, []string{"https://10.0.0.1:2480", "https://10.0.0.2:2480", "https://10.0.0.3:2480", "http://10.0.0.1:7001"}, false, }, { []*net.SRV{ {Target: "a.example.com", Port: 2480}, {Target: "b.example.com", Port: 2480}, {Target: "c.example.com.", Port: 2480}, }, []*net.SRV{}, []string{"https://a.example.com:2480", "https://b.example.com:2480", "https://c.example.com.:2480"}, false, }, } for i, tt := range tests { lookupSRV = func(service string, proto string, domain string) (string, []*net.SRV, error) { if service == "etcd-client-ssl" { if len(tt.withSSL) > 0 { return "", tt.withSSL, nil } return "", nil, notFoundErr(service, proto, domain) } if service == "etcd-client" { if len(tt.withoutSSL) > 0 { return "", tt.withoutSSL, nil } return "", nil, notFoundErr(service, proto, domain) } return "", nil, errors.New("unknown service in mock") } srvs, err := GetClient("etcd-client", "example.com", "") require.Equalf(t, hasErr(err), tt.werr, "%d: err = %#v, want = %#v", i, err, tt.werr) if srvs == nil { if len(tt.expected) > 0 { t.Errorf("#%d: srvs = nil, want non-nil", i) } } else { if !reflect.DeepEqual(srvs.Endpoints, tt.expected) { t.Errorf("#%d: endpoints = %v, want = %v", i, srvs.Endpoints, tt.expected) } } } } func TestGetSRVService(t *testing.T) { tests := []struct { scheme string serviceName string expected string }{ { "https", "", "etcd-client-ssl", }, { "http", "", "etcd-client", }, { "https", "foo", "etcd-client-ssl-foo", }, { "http", "bar", "etcd-client-bar", }, } for i, tt := range tests { service := GetSRVService("etcd-client", tt.serviceName, tt.scheme) if strings.Compare(service, tt.expected) != 0 { t.Errorf("#%d: service = %s, want %s", i, service, tt.expected) } } } ================================================ FILE: client/pkg/systemd/doc.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package systemd provides utility functions for systemd. package systemd ================================================ FILE: client/pkg/systemd/journal.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package systemd import "net" // DialJournal returns no error if the process can dial journal socket. // Returns an error if dial failed, which indicates journald is not available // (e.g. run embedded etcd as docker daemon). // Reference: https://github.com/coreos/go-systemd/blob/master/journal/journal.go. func DialJournal() error { conn, err := net.Dial("unixgram", "/run/systemd/journal/socket") if conn != nil { defer conn.Close() } return err } ================================================ FILE: client/pkg/testutil/assert.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutil import ( "testing" "github.com/stretchr/testify/assert" ) // AssertNil // Deprecated: use github.com/stretchr/testify/assert.Nil instead. func AssertNil(t *testing.T, v any) { t.Helper() assert.Nil(t, v) } // AssertNotNil // Deprecated: use github.com/stretchr/testify/require.NotNil instead. func AssertNotNil(t *testing.T, v any) { t.Helper() if v == nil { t.Fatalf("expected non-nil, got %+v", v) } } // AssertTrue // Deprecated: use github.com/stretchr/testify/assert.True instead. func AssertTrue(t *testing.T, v bool, msg ...string) { t.Helper() assert.True(t, v, msg) //nolint:testifylint } // AssertFalse // Deprecated: use github.com/stretchr/testify/assert.False instead. func AssertFalse(t *testing.T, v bool, msg ...string) { t.Helper() assert.False(t, v, msg) //nolint:testifylint } ================================================ FILE: client/pkg/testutil/before.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutil import ( "log" "os" "testing" "go.etcd.io/etcd/client/pkg/v3/verify" ) func BeforeTest(tb testing.TB) { tb.Helper() RegisterLeakDetection(tb) revertVerifyFunc := verify.EnableAllVerifications() tempDir := tb.TempDir() tb.Chdir(tempDir) tb.Logf("Changing working directory to: %s", tempDir) tb.Cleanup(func() { revertVerifyFunc() }) } func BeforeIntegrationExamples(*testing.M) func() { ExitInShortMode("Skipping: the tests require real cluster") tempDir, err := os.MkdirTemp(os.TempDir(), "etcd-integration") if err != nil { log.Printf("Failed to obtain tempDir: %v", tempDir) os.Exit(1) } err = os.Chdir(tempDir) if err != nil { log.Printf("Failed to change working dir to: %s: %v", tempDir, err) os.Exit(1) } log.Printf("Running tests (examples) in dir(%v): ...", tempDir) return func() { os.RemoveAll(tempDir) } } ================================================ FILE: client/pkg/testutil/leak.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutil import ( "fmt" "net/http" "os" "regexp" "runtime" "sort" "strings" "testing" "time" ) // TODO: Replace with https://github.com/uber-go/goleak. /* CheckLeakedGoroutine verifies tests do not leave any leaky goroutines. It returns true when there are goroutines still running(leaking) after all tests. import "go.etcd.io/etcd/client/pkg/v3/testutil" func TestMain(m *testing.M) { testutil.MustTestMainWithLeakDetection(m) } func TestSample(t *testing.T) { RegisterLeakDetection(t) ... } */ var normalizedRegexp = regexp.MustCompile(`\(0[0-9a-fx, ]*\)`) func CheckLeakedGoroutine() bool { gs := interestingGoroutines() if len(gs) == 0 { return false } stackCount := make(map[string]int) for _, g := range gs { // strip out pointer arguments in first function of stack dump normalized := string(normalizedRegexp.ReplaceAll([]byte(g), []byte("(...)"))) stackCount[normalized]++ } fmt.Fprint(os.Stderr, "Unexpected goroutines running after all test(s).\n") for stack, count := range stackCount { fmt.Fprintf(os.Stderr, "%d instances of:\n%s\n", count, stack) } return true } // CheckAfterTest returns an error if AfterTest would fail with an error. // Waits for go-routines shutdown for 'd'. func CheckAfterTest(d time.Duration) error { http.DefaultTransport.(*http.Transport).CloseIdleConnections() var bad string // Presence of these goroutines causes immediate test failure. badSubstring := map[string]string{ ").writeLoop(": "a Transport", "created by net/http/httptest.(*Server).Start": "an httptest.Server", "timeoutHandler": "a TimeoutHandler", "net.(*netFD).connect(": "a timing out dial", ").noteClientGone(": "a closenotifier sender", ").readLoop(": "a Transport", ".grpc": "a gRPC resource", ").sendCloseSubstream(": "a stream closing routine", } var stacks string begin := time.Now() for time.Since(begin) < d { bad = "" goroutines := interestingGoroutines() if len(goroutines) == 0 { return nil } stacks = strings.Join(goroutines, "\n\n") for substr, what := range badSubstring { if strings.Contains(stacks, substr) { bad = what } } // Undesired goroutines found, but goroutines might just still be // shutting down, so give it some time. runtime.Gosched() time.Sleep(50 * time.Millisecond) } return fmt.Errorf("appears to have leaked %s:\n%s", bad, stacks) } // RegisterLeakDetection is a convenient way to register before-and-after code to a test. // If you execute RegisterLeakDetection, you don't need to explicitly register AfterTest. func RegisterLeakDetection(t TB) { if err := CheckAfterTest(10 * time.Millisecond); err != nil { t.Skip("Found leaked goroutined BEFORE test", err) return } t.Cleanup(func() { afterTest(t) }) } // afterTest is meant to run in a defer that executes after a test completes. // It will detect common goroutine leaks, retrying in case there are goroutines // not synchronously torn down, and fail the test if any goroutines are stuck. func afterTest(t TB) { // If the test fails, the leaked goroutines list may hide the real // source of problem. if !t.Failed() { if err := CheckAfterTest(1 * time.Second); err != nil { t.Errorf("Test %v", err) } } } func interestingGoroutines() (gs []string) { buf := make([]byte, 2<<20) buf = buf[:runtime.Stack(buf, true)] for _, g := range strings.Split(string(buf), "\n\n") { sl := strings.SplitN(g, "\n", 2) if len(sl) != 2 { continue } stack := strings.TrimSpace(sl[1]) if stack == "" { continue } shouldSkip := func() bool { uninterestingMsgs := [...]string{ "sync.(*WaitGroup).Done", "os.(*file).close", "os.(*Process).Release", "created by os/signal.init", "runtime/panic.go", "created by testing.RunTests", "created by testing.runTests", "created by testing.(*T).Run", "testing.Main(", "runtime.goexit", "go.etcd.io/etcd/client/pkg/v3/testutil.interestingGoroutines", "go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop", "github.com/golang/glog.(*loggingT).flushDaemon", "created by runtime.gc", "created by text/template/parse.lex", "runtime.MHeap_Scavenger", "rcrypto/internal/boring.(*PublicKeyRSA).finalize", "net.(*netFD).Close(", "testing.(*T).Run", "crypto/tls.(*certCache).evict", } for _, msg := range uninterestingMsgs { if strings.Contains(stack, msg) { return true } } return false }() if shouldSkip { continue } gs = append(gs, stack) } sort.Strings(gs) return gs } func MustCheckLeakedGoroutine() { http.DefaultTransport.(*http.Transport).CloseIdleConnections() CheckAfterTest(5 * time.Second) // Let the other goroutines finalize. runtime.Gosched() if CheckLeakedGoroutine() { os.Exit(1) } } // MustTestMainWithLeakDetection expands standard m.Run with leaked // goroutines detection. func MustTestMainWithLeakDetection(m *testing.M) { v := m.Run() if v == 0 { MustCheckLeakedGoroutine() } os.Exit(v) } ================================================ FILE: client/pkg/testutil/leak_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutil import ( "fmt" "os" "testing" ) // so tests pass if given a -run that doesn't include TestSample var ranSample = false func TestMain(m *testing.M) { m.Run() isLeaked := CheckLeakedGoroutine() if ranSample && !isLeaked { fmt.Fprintln(os.Stderr, "expected leaky goroutines but none is detected") os.Exit(1) } os.Exit(0) } func TestSample(t *testing.T) { SkipTestIfShortMode(t, "Counting leaked routines is disabled in --short tests") defer afterTest(t) ranSample = true for range make([]struct{}, 100) { go func() { select {} }() } } ================================================ FILE: client/pkg/testutil/pauseable_handler.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutil import ( "net/http" "sync" ) type PauseableHandler struct { Next http.Handler mu sync.Mutex paused bool } func (ph *PauseableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ph.mu.Lock() paused := ph.paused ph.mu.Unlock() if !paused { ph.Next.ServeHTTP(w, r) } else { hj, ok := w.(http.Hijacker) if !ok { panic("webserver doesn't support hijacking") } conn, _, err := hj.Hijack() if err != nil { panic(err.Error()) } conn.Close() } } func (ph *PauseableHandler) Pause() { ph.mu.Lock() defer ph.mu.Unlock() ph.paused = true } func (ph *PauseableHandler) Resume() { ph.mu.Lock() defer ph.mu.Unlock() ph.paused = false } ================================================ FILE: client/pkg/testutil/recorder.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutil import ( "errors" "fmt" "sync" "time" ) type Action struct { Name string Params []any } type Recorder interface { // Record publishes an Action (e.g., function call) which will // be reflected by Wait() or Chan() Record(a Action) // Wait waits until at least n Actions are available or returns with error Wait(n int) ([]Action, error) // Action returns immediately available Actions Action() []Action // Chan returns the channel for actions published by Record Chan() <-chan Action } // RecorderBuffered appends all Actions to a slice type RecorderBuffered struct { sync.Mutex actions []Action } func (r *RecorderBuffered) Record(a Action) { r.Lock() r.actions = append(r.actions, a) r.Unlock() } func (r *RecorderBuffered) Action() []Action { r.Lock() cpy := make([]Action, len(r.actions)) copy(cpy, r.actions) r.Unlock() return cpy } func (r *RecorderBuffered) Wait(n int) (acts []Action, err error) { // legacy racey behavior WaitSchedule() acts = r.Action() if len(acts) < n { err = newLenErr(n, len(acts)) } return acts, err } func (r *RecorderBuffered) Chan() <-chan Action { ch := make(chan Action) go func() { acts := r.Action() for i := range acts { ch <- acts[i] } close(ch) }() return ch } // RecorderStream writes all Actions to an unbuffered channel type recorderStream struct { ch chan Action waitTimeout time.Duration } func NewRecorderStream() Recorder { return NewRecorderStreamWithWaitTimout(5 * time.Second) } func NewRecorderStreamWithWaitTimout(waitTimeout time.Duration) Recorder { return &recorderStream{ch: make(chan Action), waitTimeout: waitTimeout} } func (r *recorderStream) Record(a Action) { r.ch <- a } func (r *recorderStream) Action() (acts []Action) { for { select { case act := <-r.ch: acts = append(acts, act) default: return acts } } } func (r *recorderStream) Chan() <-chan Action { return r.ch } func (r *recorderStream) Wait(n int) ([]Action, error) { acts := make([]Action, n) var timeoutC <-chan time.Time if r.waitTimeout != 0 { timeoutC = time.After(r.waitTimeout) } for i := 0; i < n; i++ { select { case acts[i] = <-r.ch: case <-timeoutC: acts = acts[:i] return acts, newLenErr(n, i) } } // extra wait to catch any Action spew select { case act := <-r.ch: acts = append(acts, act) case <-time.After(10 * time.Millisecond): } return acts, nil } func newLenErr(expected int, actual int) error { s := fmt.Sprintf("len(actions) = %d, expected >= %d", actual, expected) return errors.New(s) } ================================================ FILE: client/pkg/testutil/testingtb.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutil import ( "log" "os" ) // TB is a subset of methods of testing.TB interface. // We cannot implement testing.TB due to protection, so we expose this simplified interface. type TB interface { Cleanup(func()) Error(args ...any) Errorf(format string, args ...any) Fail() FailNow() Failed() bool Fatal(args ...any) Fatalf(format string, args ...any) Logf(format string, args ...any) Name() string TempDir() string Helper() Skip(args ...any) } // NewTestingTBProthesis creates a fake variant of testing.TB implementation. // It's supposed to be used in contexts were real testing.T is not provided, // e.g. in 'examples'. // // The `closef` goroutine should get executed when tb will not be needed any longer. // // The provided implementation is NOT thread safe (Cleanup() method). func NewTestingTBProthesis(name string) (tb TB, closef func()) { testtb := &testingTBProthesis{name: name} return testtb, testtb.close } type testingTBProthesis struct { name string failed bool cleanups []func() } func (t *testingTBProthesis) Helper() { // Ignored } func (t *testingTBProthesis) Skip(args ...any) { t.Log(append([]any{"Skipping due to: "}, args...)) } func (t *testingTBProthesis) Cleanup(f func()) { t.cleanups = append(t.cleanups, f) } func (t *testingTBProthesis) Error(args ...any) { log.Println(args...) t.Fail() } func (t *testingTBProthesis) Errorf(format string, args ...any) { log.Printf(format, args...) t.Fail() } func (t *testingTBProthesis) Fail() { t.failed = true } func (t *testingTBProthesis) FailNow() { t.failed = true panic("FailNow() called") } func (t *testingTBProthesis) Failed() bool { return t.failed } func (t *testingTBProthesis) Fatal(args ...any) { log.Fatalln(args...) } func (t *testingTBProthesis) Fatalf(format string, args ...any) { log.Fatalf(format, args...) } func (t *testingTBProthesis) Logf(format string, args ...any) { log.Printf(format, args...) } func (t *testingTBProthesis) Log(args ...any) { log.Println(args...) } func (t *testingTBProthesis) Name() string { return t.name } func (t *testingTBProthesis) TempDir() string { dir, err := os.MkdirTemp("", t.name) if err != nil { t.Fatal(err) } t.cleanups = append([]func(){func() { t.Logf("Cleaning UP: %v", dir) os.RemoveAll(dir) }}, t.cleanups...) return dir } func (t *testingTBProthesis) close() { for i := len(t.cleanups) - 1; i >= 0; i-- { t.cleanups[i]() } } ================================================ FILE: client/pkg/testutil/testutil.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package testutil provides test utility functions. package testutil import ( "flag" "log" "net/url" "os" "runtime" "testing" "time" ) // WaitSchedule briefly sleeps in order to invoke the go scheduler. // TODO: improve this when we are able to know the schedule or status of target go-routine. func WaitSchedule() { time.Sleep(10 * time.Millisecond) } func MustNewURLs(t *testing.T, urls []string) []url.URL { t.Helper() if urls == nil { return nil } var us []url.URL for _, url := range urls { u := MustNewURL(t, url) us = append(us, *u) } return us } func MustNewURL(t *testing.T, s string) *url.URL { t.Helper() u, err := url.Parse(s) if err != nil { t.Fatalf("parse %v error: %v", s, err) } return u } // FatalStack helps to fatal the test and print out the stacks of all running goroutines. func FatalStack(t *testing.T, s string) { t.Helper() stackTrace := make([]byte, 1024*1024) n := runtime.Stack(stackTrace, true) t.Errorf("---> Test failed: %s", s) t.Error(string(stackTrace[:n])) t.Fatal(s) } // ConditionFunc returns true when a condition is met. type ConditionFunc func() (bool, error) // Poll calls a condition function repeatedly on a polling interval until it returns true, returns an error // or the timeout is reached. If the condition function returns true or an error before the timeout, Poll // immediately returns with the true value or the error. If the timeout is exceeded, Poll returns false. func Poll(interval time.Duration, timeout time.Duration, condition ConditionFunc) (bool, error) { timeoutCh := time.After(timeout) ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-timeoutCh: return false, nil case <-ticker.C: success, err := condition() if err != nil { return false, err } if success { return true, nil } } } } func SkipTestIfShortMode(t TB, reason string) { if t != nil { t.Helper() if testing.Short() { t.Skip(reason) } } } // ExitInShortMode closes the current process (with 0) if the short test mode detected. // // To be used in Test-main, where test context (testing.TB) is not available. func ExitInShortMode(reason string) { // Calling testing.Short() requires flags to be parsed before. if !flag.Parsed() { flag.Parse() } if testing.Short() { log.Println(reason) os.Exit(0) } } ================================================ FILE: client/pkg/testutil/var.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutil import "time" var ( ApplyTimeout = time.Second RequestTimeout = 3 * time.Second ) ================================================ FILE: client/pkg/tlsutil/cipher_suites.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package tlsutil import ( "crypto/tls" "fmt" ) // GetCipherSuite returns the corresponding cipher suite, // and boolean value if it is supported. func GetCipherSuite(s string) (uint16, bool) { for _, c := range tls.CipherSuites() { if s == c.Name { return c.ID, true } } for _, c := range tls.InsecureCipherSuites() { if s == c.Name { return c.ID, true } } switch s { case "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305": return tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, true case "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305": return tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, true } return 0, false } // GetCipherSuites returns list of corresponding cipher suite IDs. func GetCipherSuites(ss []string) ([]uint16, error) { cs := make([]uint16, len(ss)) for i, s := range ss { var ok bool cs[i], ok = GetCipherSuite(s) if !ok { return nil, fmt.Errorf("unexpected TLS cipher suite %q", s) } } return cs, nil } ================================================ FILE: client/pkg/tlsutil/cipher_suites_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package tlsutil import ( "crypto/tls" "testing" "github.com/stretchr/testify/require" ) func TestGetCipherSuite_not_existing(t *testing.T) { _, ok := GetCipherSuite("not_existing") require.Falsef(t, ok, "Expected not ok") } func CipherSuiteExpectedToExist(tb testing.TB, cipher string, expectedID uint16) { tb.Helper() vid, ok := GetCipherSuite(cipher) if !ok { tb.Errorf("Expected %v cipher to exist", cipher) } if vid != expectedID { tb.Errorf("For %v expected=%v found=%v", cipher, expectedID, vid) } } func TestGetCipherSuite_success(t *testing.T) { CipherSuiteExpectedToExist(t, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA) CipherSuiteExpectedToExist(t, "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) // Explicit test for legacy names CipherSuiteExpectedToExist(t, "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256) CipherSuiteExpectedToExist(t, "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256) } func TestGetCipherSuite_insecure(t *testing.T) { CipherSuiteExpectedToExist(t, "TLS_ECDHE_RSA_WITH_RC4_128_SHA", tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA) } ================================================ FILE: client/pkg/tlsutil/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package tlsutil provides utility functions for handling TLS. package tlsutil ================================================ FILE: client/pkg/tlsutil/tlsutil.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package tlsutil import ( "crypto/tls" "crypto/x509" "encoding/pem" "os" ) // NewCertPool creates x509 certPool with provided CA files. func NewCertPool(CAFiles []string) (*x509.CertPool, error) { certPool := x509.NewCertPool() for _, CAFile := range CAFiles { pemByte, err := os.ReadFile(CAFile) if err != nil { return nil, err } for { var block *pem.Block block, pemByte = pem.Decode(pemByte) if block == nil { break } cert, err := x509.ParseCertificate(block.Bytes) if err != nil { return nil, err } certPool.AddCert(cert) } } return certPool, nil } // NewCert generates TLS cert by using the given cert,key and parse function. func NewCert(certfile, keyfile string, parseFunc func([]byte, []byte) (tls.Certificate, error)) (*tls.Certificate, error) { cert, err := os.ReadFile(certfile) if err != nil { return nil, err } key, err := os.ReadFile(keyfile) if err != nil { return nil, err } if parseFunc == nil { parseFunc = tls.X509KeyPair } tlsCert, err := parseFunc(cert, key) if err != nil { return nil, err } return &tlsCert, nil } ================================================ FILE: client/pkg/tlsutil/versions.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package tlsutil import ( "crypto/tls" "fmt" ) type TLSVersion string // Constants for TLS versions. const ( TLSVersionDefault TLSVersion = "" TLSVersion12 TLSVersion = "TLS1.2" TLSVersion13 TLSVersion = "TLS1.3" ) // GetTLSVersion returns the corresponding tls.Version or error. func GetTLSVersion(version string) (uint16, error) { var v uint16 switch version { case string(TLSVersionDefault): v = 0 // 0 means let Go decide. case string(TLSVersion12): v = tls.VersionTLS12 case string(TLSVersion13): v = tls.VersionTLS13 default: return 0, fmt.Errorf("unexpected TLS version %q (must be one of: TLS1.2, TLS1.3)", version) } return v, nil } ================================================ FILE: client/pkg/tlsutil/versions_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package tlsutil import ( "crypto/tls" "testing" "github.com/stretchr/testify/assert" ) func TestGetVersion(t *testing.T) { tests := []struct { name string version string want uint16 expectError bool }{ { name: "TLS1.2", version: "TLS1.2", want: tls.VersionTLS12, }, { name: "TLS1.3", version: "TLS1.3", want: tls.VersionTLS13, }, { name: "Empty version", version: "", want: 0, }, { name: "Converting invalid version string to TLS version", version: "not_existing", expectError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := GetTLSVersion(tt.version) if err != nil { assert.Truef(t, tt.expectError, "GetTLSVersion() returned error while expecting success: %v", err) return } assert.Equal(t, tt.want, got) }) } } ================================================ FILE: client/pkg/transport/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package transport implements various HTTP transport utilities based on Go // net package. package transport ================================================ FILE: client/pkg/transport/keepalive_listener.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "crypto/tls" "errors" "fmt" "net" "time" ) // NewKeepAliveListener returns a listener that listens on the given address. // Be careful when wrap around KeepAliveListener with another Listener if TLSInfo is not nil. // Some pkgs (like go/http) might expect Listener to return TLSConn type to start TLS handshake. // http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html // // Note(ahrtr): // only `net.TCPConn` supports `SetKeepAlive` and `SetKeepAlivePeriod` // by default, so if you want to wrap multiple layers of net.Listener, // the `keepaliveListener` should be the one which is closest to the // original `net.Listener` implementation, namely `TCPListener`. func NewKeepAliveListener(l net.Listener, scheme string, tlscfg *tls.Config) (net.Listener, error) { kal := &keepaliveListener{ Listener: l, } if scheme == "https" { if tlscfg == nil { return nil, errors.New("cannot listen on TLS for given listener: KeyFile and CertFile are not presented") } return newTLSKeepaliveListener(kal, tlscfg), nil } return kal, nil } type keepaliveListener struct{ net.Listener } func (kln *keepaliveListener) Accept() (net.Conn, error) { c, err := kln.Listener.Accept() if err != nil { return nil, err } kac, err := createKeepaliveConn(c) if err != nil { return nil, fmt.Errorf("create keepalive connection failed, %w", err) } // detection time: tcp_keepalive_time + tcp_keepalive_probes + tcp_keepalive_intvl // default on linux: 30 + 8 * 30 // default on osx: 30 + 8 * 75 if err := kac.SetKeepAlive(true); err != nil { return nil, fmt.Errorf("SetKeepAlive failed, %w", err) } if err := kac.SetKeepAlivePeriod(30 * time.Second); err != nil { return nil, fmt.Errorf("SetKeepAlivePeriod failed, %w", err) } return kac, nil } func createKeepaliveConn(c net.Conn) (*keepAliveConn, error) { tcpc, ok := c.(*net.TCPConn) if !ok { return nil, ErrNotTCP } return &keepAliveConn{tcpc}, nil } type keepAliveConn struct { *net.TCPConn } // SetKeepAlive sets keepalive func (l *keepAliveConn) SetKeepAlive(doKeepAlive bool) error { return l.TCPConn.SetKeepAlive(doKeepAlive) } // A tlsKeepaliveListener implements a network listener (net.Listener) for TLS connections. type tlsKeepaliveListener struct { net.Listener config *tls.Config } // Accept waits for and returns the next incoming TLS connection. // The returned connection c is a *tls.Conn. func (l *tlsKeepaliveListener) Accept() (net.Conn, error) { c, err := l.Listener.Accept() if err != nil { return nil, err } c = tls.Server(c, l.config) return c, nil } // newTLSKeepaliveListener creates a Listener which accepts connections from an inner // Listener and wraps each connection with Server. // The configuration config must be non-nil and must have // at least one certificate. func newTLSKeepaliveListener(inner net.Listener, config *tls.Config) net.Listener { l := &tlsKeepaliveListener{} l.Listener = inner l.config = config return l } ================================================ FILE: client/pkg/transport/keepalive_listener_openbsd.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build openbsd package transport import "time" // SetKeepAlivePeriod sets keepalive period func (l *keepAliveConn) SetKeepAlivePeriod(d time.Duration) error { // OpenBSD has no user-settable per-socket TCP keepalive options. // Refer to https://github.com/etcd-io/etcd/issues/15811. return nil } ================================================ FILE: client/pkg/transport/keepalive_listener_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "crypto/tls" "net" "net/http" "testing" "github.com/stretchr/testify/require" ) // TestNewKeepAliveListener tests NewKeepAliveListener returns a listener // that accepts connections. // TODO: verify the keepalive option is set correctly func TestNewKeepAliveListener(t *testing.T) { ln, err := net.Listen("tcp", "127.0.0.1:0") require.NoErrorf(t, err, "unexpected listen error") ln, err = NewKeepAliveListener(ln, "http", nil) require.NoErrorf(t, err, "unexpected NewKeepAliveListener error") go http.Get("http://" + ln.Addr().String()) conn, err := ln.Accept() require.NoErrorf(t, err, "unexpected Accept error") _, ok := conn.(*keepAliveConn) require.Truef(t, ok, "Unexpected conn type: %T, wanted *keepAliveConn", conn) conn.Close() ln.Close() ln, err = net.Listen("tcp", "127.0.0.1:0") require.NoErrorf(t, err, "unexpected Listen error") // tls tlsinfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create tmpfile") tlsInfo := TLSInfo{CertFile: tlsinfo.CertFile, KeyFile: tlsinfo.KeyFile} tlsInfo.parseFunc = fakeCertificateParserFunc(nil) tlscfg, err := tlsInfo.ServerConfig() require.NoErrorf(t, err, "unexpected serverConfig error") tlsln, err := NewKeepAliveListener(ln, "https", tlscfg) require.NoErrorf(t, err, "unexpected NewKeepAliveListener error") go http.Get("https://" + tlsln.Addr().String()) conn, err = tlsln.Accept() require.NoErrorf(t, err, "unexpected Accept error") if _, ok := conn.(*tls.Conn); !ok { t.Errorf("failed to accept *tls.Conn") } conn.Close() tlsln.Close() } func TestNewKeepAliveListenerTLSEmptyConfig(t *testing.T) { ln, err := net.Listen("tcp", "127.0.0.1:0") require.NoErrorf(t, err, "unexpected listen error") _, err = NewKeepAliveListener(ln, "https", nil) if err == nil { t.Errorf("err = nil, want not presented error") } } ================================================ FILE: client/pkg/transport/keepalive_listener_unix.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !openbsd package transport import "time" // SetKeepAlivePeriod sets keepalive period func (l *keepAliveConn) SetKeepAlivePeriod(d time.Duration) error { return l.TCPConn.SetKeepAlivePeriod(d) } ================================================ FILE: client/pkg/transport/limit_listen.go ================================================ // Copyright 2013 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package transport provides network utility functions, complementing the more // common ones in the net package. package transport import ( "errors" "net" "sync" "time" ) var ErrNotTCP = errors.New("only tcp connections have keepalive") // LimitListener returns a Listener that accepts at most n simultaneous // connections from the provided Listener. func LimitListener(l net.Listener, n int) net.Listener { return &limitListener{l, make(chan struct{}, n)} } type limitListener struct { net.Listener sem chan struct{} } func (l *limitListener) acquire() { l.sem <- struct{}{} } func (l *limitListener) release() { <-l.sem } func (l *limitListener) Accept() (net.Conn, error) { l.acquire() c, err := l.Listener.Accept() if err != nil { l.release() return nil, err } return &limitListenerConn{Conn: c, release: l.release}, nil } type limitListenerConn struct { net.Conn releaseOnce sync.Once release func() } func (l *limitListenerConn) Close() error { err := l.Conn.Close() l.releaseOnce.Do(l.release) return err } // SetKeepAlive sets keepalive // // Deprecated: use (*keepAliveConn) SetKeepAlive instead. func (l *limitListenerConn) SetKeepAlive(doKeepAlive bool) error { tcpc, ok := l.Conn.(*net.TCPConn) if !ok { return ErrNotTCP } return tcpc.SetKeepAlive(doKeepAlive) } // SetKeepAlivePeriod sets keepalive period // // Deprecated: use (*keepAliveConn) SetKeepAlivePeriod instead. func (l *limitListenerConn) SetKeepAlivePeriod(d time.Duration) error { tcpc, ok := l.Conn.(*net.TCPConn) if !ok { return ErrNotTCP } return tcpc.SetKeepAlivePeriod(d) } ================================================ FILE: client/pkg/transport/listener.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" "errors" "fmt" "math/big" "net" "os" "path/filepath" "strings" "time" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/tlsutil" "go.etcd.io/etcd/client/pkg/v3/verify" ) // NewListener creates a new listner. func NewListener(addr, scheme string, tlsinfo *TLSInfo) (l net.Listener, err error) { return newListener(addr, scheme, WithTLSInfo(tlsinfo)) } // NewListenerWithOpts creates a new listener which accepts listener options. func NewListenerWithOpts(addr, scheme string, opts ...ListenerOption) (net.Listener, error) { return newListener(addr, scheme, opts...) } func newListener(addr, scheme string, opts ...ListenerOption) (net.Listener, error) { if scheme == "unix" || scheme == "unixs" { // unix sockets via unix://laddr return NewUnixListener(addr) } lnOpts := newListenOpts(opts...) switch { case lnOpts.IsSocketOpts(): // new ListenConfig with socket options. lnOpts.ListenConfig = newListenConfig(lnOpts.socketOpts) // check for timeout fallthrough case lnOpts.IsTimeout(), lnOpts.IsSocketOpts(): // timeout listener with socket options. ln, err := newKeepAliveListener(&lnOpts.ListenConfig, addr) if err != nil { return nil, err } lnOpts.Listener = &rwTimeoutListener{ Listener: ln, readTimeout: lnOpts.readTimeout, writeTimeout: lnOpts.writeTimeout, } case lnOpts.IsTimeout(): ln, err := newKeepAliveListener(nil, addr) if err != nil { return nil, err } lnOpts.Listener = &rwTimeoutListener{ Listener: ln, readTimeout: lnOpts.readTimeout, writeTimeout: lnOpts.writeTimeout, } default: ln, err := newKeepAliveListener(nil, addr) if err != nil { return nil, err } lnOpts.Listener = ln } // only skip if not passing TLSInfo if lnOpts.skipTLSInfoCheck && !lnOpts.IsTLS() { return lnOpts.Listener, nil } return wrapTLS(scheme, lnOpts.tlsInfo, lnOpts.Listener) } func newKeepAliveListener(cfg *net.ListenConfig, addr string) (net.Listener, error) { var ln net.Listener var err error if cfg != nil { ln, err = cfg.Listen(context.TODO(), "tcp", addr) } else { ln, err = net.Listen("tcp", addr) } if err != nil { return nil, err } return NewKeepAliveListener(ln, "tcp", nil) } func wrapTLS(scheme string, tlsinfo *TLSInfo, l net.Listener) (net.Listener, error) { if scheme != "https" && scheme != "unixs" { return l, nil } if tlsinfo != nil && tlsinfo.SkipClientSANVerify { return NewTLSListener(l, tlsinfo) } return newTLSListener(l, tlsinfo, checkSAN) } func newListenConfig(sopts *SocketOpts) net.ListenConfig { lc := net.ListenConfig{} if sopts != nil { ctls := getControls(sopts) if len(ctls) > 0 { lc.Control = ctls.Control } } return lc } type TLSInfo struct { // CertFile is the _server_ cert, it will also be used as a _client_ certificate if ClientCertFile is empty CertFile string // KeyFile is the key for the CertFile KeyFile string // ClientCertFile is a _client_ cert for initiating connections when ClientCertAuth is defined. If ClientCertAuth // is true but this value is empty, the CertFile will be used instead. ClientCertFile string // ClientKeyFile is the key for the ClientCertFile ClientKeyFile string TrustedCAFile string ClientCertAuth bool CRLFile string InsecureSkipVerify bool SkipClientSANVerify bool // ServerName ensures the cert matches the given host in case of discovery / virtual hosting ServerName string // HandshakeFailure is optionally called when a connection fails to handshake. The // connection will be closed immediately afterwards. HandshakeFailure func(*tls.Conn, error) // CipherSuites is a list of supported cipher suites. // If empty, Go auto-populates it by default. // Note that cipher suites are prioritized in the given order. CipherSuites []uint16 // MinVersion is the minimum TLS version that is acceptable. // If not set, the minimum version is TLS 1.2. MinVersion uint16 // MaxVersion is the maximum TLS version that is acceptable. // If not set, the default used by Go is selected (see tls.Config.MaxVersion). MaxVersion uint16 selfCert bool // parseFunc exists to simplify testing. Typically, parseFunc // should be left nil. In that case, tls.X509KeyPair will be used. parseFunc func([]byte, []byte) (tls.Certificate, error) // AllowedCN is a CN which must be provided by a client. // // Deprecated: use AllowedCNs instead. AllowedCN string // AllowedHostname is an IP address or hostname that must match the TLS // certificate provided by a client. // // Deprecated: use AllowedHostnames instead. AllowedHostname string // AllowedCNs is a list of acceptable CNs which must be provided by a client. AllowedCNs []string // AllowedHostnames is a list of acceptable IP addresses or hostnames that must match the // TLS certificate provided by a client. AllowedHostnames []string // Logger logs TLS errors. // If nil, all logs are discarded. Logger *zap.Logger // EmptyCN indicates that the cert must have empty CN. // If true, ClientConfig() will return an error for a cert with non empty CN. EmptyCN bool // LocalAddr is the local IP address to use when communicating with a peer. LocalAddr string } func (info TLSInfo) String() string { return fmt.Sprintf("cert = %s, key = %s, client-cert=%s, client-key=%s, trusted-ca = %s, client-cert-auth = %v, crl-file = %s", info.CertFile, info.KeyFile, info.ClientCertFile, info.ClientKeyFile, info.TrustedCAFile, info.ClientCertAuth, info.CRLFile) } func (info TLSInfo) Empty() bool { return info.CertFile == "" && info.KeyFile == "" } func SelfCert(lg *zap.Logger, dirpath string, hosts []string, selfSignedCertValidity uint, additionalUsages ...x509.ExtKeyUsage) (TLSInfo, error) { verify.Assert(lg != nil, "nil log isn't allowed") var err error info := TLSInfo{Logger: lg} if selfSignedCertValidity == 0 { err = errors.New("selfSignedCertValidity is invalid,it should be greater than 0") info.Logger.Warn( "cannot generate cert", zap.Error(err), ) return info, err } err = fileutil.TouchDirAll(lg, dirpath) if err != nil { info.Logger.Warn( "cannot create cert directory", zap.Error(err), ) return info, err } certPath, err := filepath.Abs(filepath.Join(dirpath, "cert.pem")) if err != nil { return info, err } keyPath, err := filepath.Abs(filepath.Join(dirpath, "key.pem")) if err != nil { return info, err } _, errcert := os.Stat(certPath) _, errkey := os.Stat(keyPath) if errcert == nil && errkey == nil { info.CertFile = certPath info.KeyFile = keyPath info.ClientCertFile = certPath info.ClientKeyFile = keyPath info.selfCert = true return info, err } serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) if err != nil { info.Logger.Warn( "cannot generate random number", zap.Error(err), ) return info, err } tmpl := x509.Certificate{ SerialNumber: serialNumber, Subject: pkix.Name{Organization: []string{"etcd"}}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Duration(selfSignedCertValidity) * 365 * (24 * time.Hour)), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCRLSign, ExtKeyUsage: append([]x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, additionalUsages...), BasicConstraintsValid: true, IsCA: true, } info.Logger.Warn( "automatically generate certificates", zap.Time("certificate-validity-bound-not-after", tmpl.NotAfter), ) for _, host := range hosts { h, _, _ := net.SplitHostPort(host) if ip := net.ParseIP(h); ip != nil { tmpl.IPAddresses = append(tmpl.IPAddresses, ip) } else { tmpl.DNSNames = append(tmpl.DNSNames, h) } } priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader) if err != nil { info.Logger.Warn( "cannot generate ECDSA key", zap.Error(err), ) return info, err } derBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv) if err != nil { info.Logger.Warn( "cannot generate x509 certificate", zap.Error(err), ) return info, err } certOut, err := os.Create(certPath) if err != nil { info.Logger.Warn( "cannot cert file", zap.String("path", certPath), zap.Error(err), ) return info, err } pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) certOut.Close() info.Logger.Info("created cert file", zap.String("path", certPath)) b, err := x509.MarshalECPrivateKey(priv) if err != nil { return info, err } keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { info.Logger.Warn( "cannot key file", zap.String("path", keyPath), zap.Error(err), ) return info, err } pem.Encode(keyOut, &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}) keyOut.Close() info.Logger.Info("created key file", zap.String("path", keyPath)) return SelfCert(lg, dirpath, hosts, selfSignedCertValidity) } // baseConfig is called on initial TLS handshake start. // // Previously, // 1. Server has non-empty (*tls.Config).Certificates on client hello // 2. Server calls (*tls.Config).GetCertificate iff: // - Server's (*tls.Config).Certificates is not empty, or // - Client supplies SNI; non-empty (*tls.ClientHelloInfo).ServerName // // When (*tls.Config).Certificates is always populated on initial handshake, // client is expected to provide a valid matching SNI to pass the TLS // verification, thus trigger server (*tls.Config).GetCertificate to reload // TLS assets. However, a cert whose SAN field does not include domain names // but only IP addresses, has empty (*tls.ClientHelloInfo).ServerName, thus // it was never able to trigger TLS reload on initial handshake; first // ceritifcate object was being used, never being updated. // // Now, (*tls.Config).Certificates is created empty on initial TLS client // handshake, in order to trigger (*tls.Config).GetCertificate and populate // rest of the certificates on every new TLS connection, even when client // SNI is empty (e.g. cert only includes IPs). func (info TLSInfo) baseConfig() (*tls.Config, error) { if info.KeyFile == "" || info.CertFile == "" { return nil, fmt.Errorf("KeyFile and CertFile must both be present[key: %v, cert: %v]", info.KeyFile, info.CertFile) } if info.Logger == nil { info.Logger = zap.NewNop() } _, err := tlsutil.NewCert(info.CertFile, info.KeyFile, info.parseFunc) if err != nil { return nil, err } // Perform prevalidation of client cert and key if either are provided. This makes sure we crash before accepting any connections. if (info.ClientKeyFile == "") != (info.ClientCertFile == "") { return nil, fmt.Errorf("ClientKeyFile and ClientCertFile must both be present or both absent: key: %v, cert: %v]", info.ClientKeyFile, info.ClientCertFile) } if info.ClientCertFile != "" { _, err := tlsutil.NewCert(info.ClientCertFile, info.ClientKeyFile, info.parseFunc) if err != nil { return nil, err } } var minVersion uint16 if info.MinVersion != 0 { minVersion = info.MinVersion } else { // Default minimum version is TLS 1.2, previous versions are insecure and deprecated. minVersion = tls.VersionTLS12 } cfg := &tls.Config{ MinVersion: minVersion, MaxVersion: info.MaxVersion, ServerName: info.ServerName, } if len(info.CipherSuites) > 0 { cfg.CipherSuites = info.CipherSuites } // Client certificates may be verified by either an exact match on the CN, // or a more general check of the CN and SANs. var verifyCertificate func(*x509.Certificate) bool if info.AllowedCN != "" && len(info.AllowedCNs) > 0 { return nil, fmt.Errorf("AllowedCN and AllowedCNs are mutually exclusive (cn=%q, cns=%q)", info.AllowedCN, info.AllowedCNs) } if info.AllowedHostname != "" && len(info.AllowedHostnames) > 0 { return nil, fmt.Errorf("AllowedHostname and AllowedHostnames are mutually exclusive (hostname=%q, hostnames=%q)", info.AllowedHostname, info.AllowedHostnames) } if info.AllowedCN != "" && info.AllowedHostname != "" { return nil, fmt.Errorf("AllowedCN and AllowedHostname are mutually exclusive (cn=%q, hostname=%q)", info.AllowedCN, info.AllowedHostname) } if len(info.AllowedCNs) > 0 && len(info.AllowedHostnames) > 0 { return nil, fmt.Errorf("AllowedCNs and AllowedHostnames are mutually exclusive (cns=%q, hostnames=%q)", info.AllowedCNs, info.AllowedHostnames) } if info.AllowedCN != "" { info.Logger.Warn("AllowedCN is deprecated, use AllowedCNs instead") verifyCertificate = func(cert *x509.Certificate) bool { return info.AllowedCN == cert.Subject.CommonName } } if info.AllowedHostname != "" { info.Logger.Warn("AllowedHostname is deprecated, use AllowedHostnames instead") verifyCertificate = func(cert *x509.Certificate) bool { return cert.VerifyHostname(info.AllowedHostname) == nil } } if len(info.AllowedCNs) > 0 { verifyCertificate = func(cert *x509.Certificate) bool { for _, allowedCN := range info.AllowedCNs { if allowedCN == cert.Subject.CommonName { return true } } return false } } if len(info.AllowedHostnames) > 0 { verifyCertificate = func(cert *x509.Certificate) bool { for _, allowedHostname := range info.AllowedHostnames { if cert.VerifyHostname(allowedHostname) == nil { return true } } return false } } if verifyCertificate != nil { cfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { for _, chains := range verifiedChains { if len(chains) != 0 { if verifyCertificate(chains[0]) { return nil } } } return errors.New("client certificate authentication failed") } } // this only reloads certs when there's a client request // TODO: support server-side refresh (e.g. inotify, SIGHUP), caching cfg.GetCertificate = func(clientHello *tls.ClientHelloInfo) (cert *tls.Certificate, err error) { cert, err = tlsutil.NewCert(info.CertFile, info.KeyFile, info.parseFunc) if os.IsNotExist(err) { info.Logger.Warn( "failed to find peer cert files", zap.String("cert-file", info.CertFile), zap.String("key-file", info.KeyFile), zap.Error(err), ) } else if err != nil { info.Logger.Warn( "failed to create peer certificate", zap.String("cert-file", info.CertFile), zap.String("key-file", info.KeyFile), zap.Error(err), ) } return cert, err } cfg.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (cert *tls.Certificate, err error) { certfile, keyfile := info.CertFile, info.KeyFile if info.ClientCertFile != "" { certfile, keyfile = info.ClientCertFile, info.ClientKeyFile } cert, err = tlsutil.NewCert(certfile, keyfile, info.parseFunc) if os.IsNotExist(err) { info.Logger.Warn( "failed to find client cert files", zap.String("cert-file", certfile), zap.String("key-file", keyfile), zap.Error(err), ) } else if err != nil { info.Logger.Warn( "failed to create client certificate", zap.String("cert-file", certfile), zap.String("key-file", keyfile), zap.Error(err), ) } return cert, err } return cfg, nil } // cafiles returns a list of CA file paths. func (info TLSInfo) cafiles() []string { cs := make([]string, 0) if info.TrustedCAFile != "" { cs = append(cs, info.TrustedCAFile) } return cs } // ServerConfig generates a tls.Config object for use by an HTTP server. func (info TLSInfo) ServerConfig() (*tls.Config, error) { cfg, err := info.baseConfig() if err != nil { return nil, err } if info.Logger == nil { info.Logger = zap.NewNop() } cfg.ClientAuth = tls.NoClientCert if info.TrustedCAFile != "" || info.ClientCertAuth { cfg.ClientAuth = tls.RequireAndVerifyClientCert } cs := info.cafiles() if len(cs) > 0 { info.Logger.Info("Loading cert pool", zap.Strings("cs", cs), zap.Any("tlsinfo", info)) cp, err := tlsutil.NewCertPool(cs) if err != nil { return nil, err } cfg.ClientCAs = cp } // "h2" NextProtos is necessary for enabling HTTP2 for go's HTTP server cfg.NextProtos = []string{"h2"} return cfg, nil } // ClientConfig generates a tls.Config object for use by an HTTP client. func (info TLSInfo) ClientConfig() (*tls.Config, error) { var cfg *tls.Config var err error if !info.Empty() { cfg, err = info.baseConfig() if err != nil { return nil, err } } else { cfg = &tls.Config{ServerName: info.ServerName} } cfg.InsecureSkipVerify = info.InsecureSkipVerify cs := info.cafiles() if len(cs) > 0 { cfg.RootCAs, err = tlsutil.NewCertPool(cs) if err != nil { return nil, err } } if info.selfCert { cfg.InsecureSkipVerify = true } if info.EmptyCN { hasNonEmptyCN := false cn := "" _, err := tlsutil.NewCert(info.CertFile, info.KeyFile, func(certPEMBlock []byte, keyPEMBlock []byte) (tls.Certificate, error) { var block *pem.Block block, _ = pem.Decode(certPEMBlock) cert, err := x509.ParseCertificate(block.Bytes) if err != nil { return tls.Certificate{}, err } if len(cert.Subject.CommonName) != 0 { hasNonEmptyCN = true cn = cert.Subject.CommonName } return tls.X509KeyPair(certPEMBlock, keyPEMBlock) }) if err != nil { return nil, err } if hasNonEmptyCN { return nil, fmt.Errorf("cert has non empty Common Name (%s): %s", cn, info.CertFile) } } return cfg, nil } // IsClosedConnError returns true if the error is from closing listener, cmux. // copied from golang.org/x/net/http2/http2.go func IsClosedConnError(err error) bool { // 'use of closed network connection' (Go <=1.8) // 'use of closed file or network connection' (Go >1.8, internal/poll.ErrClosing) // 'mux: listener closed' (cmux.ErrListenerClosed) return err != nil && strings.Contains(err.Error(), "closed") } ================================================ FILE: client/pkg/transport/listener_opts.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "net" "time" ) type ListenerOptions struct { Listener net.Listener ListenConfig net.ListenConfig socketOpts *SocketOpts tlsInfo *TLSInfo skipTLSInfoCheck bool writeTimeout time.Duration readTimeout time.Duration } func newListenOpts(opts ...ListenerOption) *ListenerOptions { lnOpts := &ListenerOptions{} lnOpts.applyOpts(opts) return lnOpts } func (lo *ListenerOptions) applyOpts(opts []ListenerOption) { for _, opt := range opts { opt(lo) } } // IsTimeout returns true if the listener has a read/write timeout defined. func (lo *ListenerOptions) IsTimeout() bool { return lo.readTimeout != 0 || lo.writeTimeout != 0 } // IsSocketOpts returns true if the listener options includes socket options. func (lo *ListenerOptions) IsSocketOpts() bool { if lo.socketOpts == nil { return false } return lo.socketOpts.ReusePort || lo.socketOpts.ReuseAddress } // IsTLS returns true if listner options includes TLSInfo. func (lo *ListenerOptions) IsTLS() bool { if lo.tlsInfo == nil { return false } return !lo.tlsInfo.Empty() } // ListenerOption are options which can be applied to the listener. type ListenerOption func(*ListenerOptions) // WithTimeout allows for a read or write timeout to be applied to the listener. func WithTimeout(read, write time.Duration) ListenerOption { return func(lo *ListenerOptions) { lo.writeTimeout = write lo.readTimeout = read } } // WithSocketOpts defines socket options that will be applied to the listener. func WithSocketOpts(s *SocketOpts) ListenerOption { return func(lo *ListenerOptions) { lo.socketOpts = s } } // WithTLSInfo adds TLS credentials to the listener. func WithTLSInfo(t *TLSInfo) ListenerOption { return func(lo *ListenerOptions) { lo.tlsInfo = t } } // WithSkipTLSInfoCheck when true a transport can be created with an https scheme // without passing TLSInfo, circumventing not presented error. Skipping this check // also requires that TLSInfo is not passed. func WithSkipTLSInfoCheck(skip bool) ListenerOption { return func(lo *ListenerOptions) { lo.skipTLSInfoCheck = skip } } ================================================ FILE: client/pkg/transport/listener_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "crypto/rand" "crypto/tls" "crypto/x509" "encoding/pem" "errors" "math/big" "net" "net/http" "os" "path/filepath" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" ) func createSelfCert(t *testing.T) (*TLSInfo, error) { t.Helper() return createSelfCertEx(t, "127.0.0.1") } func createSelfCertEx(t *testing.T, host string, additionalUsages ...x509.ExtKeyUsage) (*TLSInfo, error) { t.Helper() d := t.TempDir() info, err := SelfCert(zaptest.NewLogger(t), d, []string{host + ":0"}, 1, additionalUsages...) if err != nil { return nil, err } return &info, nil } func fakeCertificateParserFunc(err error) func(certPEMBlock, keyPEMBlock []byte) (tls.Certificate, error) { return func(certPEMBlock, keyPEMBlock []byte) (tls.Certificate, error) { return tls.Certificate{}, err } } // TestNewListenerTLSInfo tests that NewListener with valid TLSInfo returns // a TLS listener that accepts TLS connections. func TestNewListenerTLSInfo(t *testing.T) { tlsInfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create cert") testNewListenerTLSInfoAccept(t, *tlsInfo) } func TestNewListenerWithOpts(t *testing.T) { tlsInfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create cert") tests := map[string]struct { opts []ListenerOption scheme string expectedErr bool }{ "https scheme no TLSInfo": { opts: []ListenerOption{}, expectedErr: true, scheme: "https", }, "https scheme no TLSInfo with skip check": { opts: []ListenerOption{WithSkipTLSInfoCheck(true)}, expectedErr: false, scheme: "https", }, "https scheme empty TLSInfo with skip check": { opts: []ListenerOption{ WithSkipTLSInfoCheck(true), WithTLSInfo(&TLSInfo{}), }, expectedErr: false, scheme: "https", }, "https scheme empty TLSInfo no skip check": { opts: []ListenerOption{ WithTLSInfo(&TLSInfo{}), }, expectedErr: true, scheme: "https", }, "https scheme with TLSInfo and skip check": { opts: []ListenerOption{ WithSkipTLSInfoCheck(true), WithTLSInfo(tlsInfo), }, expectedErr: false, scheme: "https", }, } for testName, test := range tests { t.Run(testName, func(t *testing.T) { ln, err := NewListenerWithOpts("127.0.0.1:0", test.scheme, test.opts...) if ln != nil { defer ln.Close() } require.Falsef(t, test.expectedErr && err == nil, "expected error") if !test.expectedErr { require.NoErrorf(t, err, "unexpected error: %v", err) } }) } } func TestNewListenerWithSocketOpts(t *testing.T) { tlsInfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create cert") tests := map[string]struct { opts []ListenerOption scheme string expectedErr bool }{ "nil socketopts": { opts: []ListenerOption{WithSocketOpts(nil)}, expectedErr: true, scheme: "http", }, "empty socketopts": { opts: []ListenerOption{WithSocketOpts(&SocketOpts{})}, expectedErr: true, scheme: "http", }, "reuse address": { opts: []ListenerOption{WithSocketOpts(&SocketOpts{ReuseAddress: true})}, scheme: "http", expectedErr: true, }, "reuse address with TLS": { opts: []ListenerOption{ WithSocketOpts(&SocketOpts{ReuseAddress: true}), WithTLSInfo(tlsInfo), }, scheme: "https", expectedErr: true, }, "reuse address and port": { opts: []ListenerOption{WithSocketOpts(&SocketOpts{ReuseAddress: true, ReusePort: true})}, scheme: "http", expectedErr: false, }, "reuse address and port with TLS": { opts: []ListenerOption{ WithSocketOpts(&SocketOpts{ReuseAddress: true, ReusePort: true}), WithTLSInfo(tlsInfo), }, scheme: "https", expectedErr: false, }, "reuse port with TLS and timeout": { opts: []ListenerOption{ WithSocketOpts(&SocketOpts{ReusePort: true}), WithTLSInfo(tlsInfo), WithTimeout(5*time.Second, 5*time.Second), }, scheme: "https", expectedErr: false, }, "reuse port with https scheme and no TLSInfo skip check": { opts: []ListenerOption{ WithSocketOpts(&SocketOpts{ReusePort: true}), WithSkipTLSInfoCheck(true), }, scheme: "https", expectedErr: false, }, "reuse port": { opts: []ListenerOption{WithSocketOpts(&SocketOpts{ReusePort: true})}, scheme: "http", expectedErr: false, }, } for testName, test := range tests { t.Run(testName, func(t *testing.T) { ln, err := NewListenerWithOpts("127.0.0.1:0", test.scheme, test.opts...) require.NoErrorf(t, err, "unexpected NewListenerWithSocketOpts error") defer ln.Close() ln2, err := NewListenerWithOpts(ln.Addr().String(), test.scheme, test.opts...) if ln2 != nil { ln2.Close() } if test.expectedErr { require.Errorf(t, err, "expected error") } if !test.expectedErr { require.NoErrorf(t, err, "unexpected error: %v", err) } if test.scheme == "http" { lnOpts := newListenOpts(test.opts...) if !lnOpts.IsSocketOpts() && !lnOpts.IsTimeout() { _, ok := ln.(*keepaliveListener) require.Truef(t, ok, "ln: unexpected listener type: %T, wanted *keepaliveListener", ln) } } }) } } func testNewListenerTLSInfoAccept(t *testing.T, tlsInfo TLSInfo) { t.Helper() ln, err := NewListener("127.0.0.1:0", "https", &tlsInfo) require.NoErrorf(t, err, "unexpected NewListener error") defer ln.Close() tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} cli := &http.Client{Transport: tr} go cli.Get("https://" + ln.Addr().String()) conn, err := ln.Accept() require.NoErrorf(t, err, "unexpected Accept error") defer conn.Close() if _, ok := conn.(*tls.Conn); !ok { t.Error("failed to accept *tls.Conn") } } // TestNewListenerTLSInfoSkipClientSANVerify tests that if client IP address mismatches // with specified address in its certificate the connection is still accepted // if the flag SkipClientSANVerify is set (i.e. checkSAN() is disabled for the client side) func TestNewListenerTLSInfoSkipClientSANVerify(t *testing.T) { tests := []struct { skipClientSANVerify bool goodClientHost bool acceptExpected bool }{ {false, true, true}, {false, false, false}, {true, true, true}, {true, false, true}, } for _, test := range tests { testNewListenerTLSInfoClientCheck(t, test.skipClientSANVerify, test.goodClientHost, test.acceptExpected) } } func testNewListenerTLSInfoClientCheck(t *testing.T, skipClientSANVerify, goodClientHost, acceptExpected bool) { t.Helper() tlsInfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create cert") host := "127.0.0.222" if goodClientHost { host = "127.0.0.1" } clientTLSInfo, err := createSelfCertEx(t, host, x509.ExtKeyUsageClientAuth) require.NoErrorf(t, err, "unable to create cert") tlsInfo.SkipClientSANVerify = skipClientSANVerify tlsInfo.TrustedCAFile = clientTLSInfo.CertFile rootCAs := x509.NewCertPool() loaded, err := os.ReadFile(tlsInfo.CertFile) require.NoErrorf(t, err, "unexpected missing certfile") rootCAs.AppendCertsFromPEM(loaded) clientCert, err := tls.LoadX509KeyPair(clientTLSInfo.CertFile, clientTLSInfo.KeyFile) require.NoErrorf(t, err, "unable to create peer cert") tlsConfig := &tls.Config{} tlsConfig.InsecureSkipVerify = false tlsConfig.Certificates = []tls.Certificate{clientCert} tlsConfig.RootCAs = rootCAs ln, err := NewListener("127.0.0.1:0", "https", tlsInfo) require.NoErrorf(t, err, "unexpected NewListener error") defer ln.Close() tr := &http.Transport{TLSClientConfig: tlsConfig} cli := &http.Client{Transport: tr} chClientErr := make(chan error, 1) go func() { _, err := cli.Get("https://" + ln.Addr().String()) chClientErr <- err }() chAcceptErr := make(chan error, 1) chAcceptConn := make(chan net.Conn, 1) go func() { conn, err := ln.Accept() if err != nil { chAcceptErr <- err } else { chAcceptConn <- conn } }() select { case <-chClientErr: if acceptExpected { t.Errorf("accepted for good client address: skipClientSANVerify=%t, goodClientHost=%t", skipClientSANVerify, goodClientHost) } case acceptErr := <-chAcceptErr: t.Fatalf("unexpected Accept error: %v", acceptErr) case conn := <-chAcceptConn: defer conn.Close() if _, ok := conn.(*tls.Conn); !ok { t.Errorf("failed to accept *tls.Conn") } if !acceptExpected { t.Errorf("accepted for bad client address: skipClientSANVerify=%t, goodClientHost=%t", skipClientSANVerify, goodClientHost) } } } func TestNewListenerTLSEmptyInfo(t *testing.T) { _, err := NewListener("127.0.0.1:0", "https", nil) if err == nil { t.Errorf("err = nil, want not presented error") } } func TestNewTransportTLSInfo(t *testing.T) { tlsinfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create cert") tests := []TLSInfo{ {}, { CertFile: tlsinfo.CertFile, KeyFile: tlsinfo.KeyFile, }, { CertFile: tlsinfo.CertFile, KeyFile: tlsinfo.KeyFile, TrustedCAFile: tlsinfo.TrustedCAFile, }, { TrustedCAFile: tlsinfo.TrustedCAFile, }, } for i, tt := range tests { tt.parseFunc = fakeCertificateParserFunc(nil) trans, err := NewTransport(tt, time.Second) require.NoErrorf(t, err, "Received unexpected error from NewTransport") require.NotNilf(t, trans.TLSClientConfig, "#%d: want non-nil TLSClientConfig", i) } } func TestTLSInfoNonexist(t *testing.T) { tlsInfo := TLSInfo{CertFile: "@badname", KeyFile: "@badname"} _, err := tlsInfo.ServerConfig() werr := &os.PathError{ Op: "open", Path: "@badname", Err: errors.New("no such file or directory"), } if err.Error() != werr.Error() { t.Errorf("err = %v, want %v", err, werr) } } func TestTLSInfoEmpty(t *testing.T) { tests := []struct { info TLSInfo want bool }{ {TLSInfo{}, true}, {TLSInfo{TrustedCAFile: "baz"}, true}, {TLSInfo{CertFile: "foo"}, false}, {TLSInfo{KeyFile: "bar"}, false}, {TLSInfo{CertFile: "foo", KeyFile: "bar"}, false}, {TLSInfo{CertFile: "foo", TrustedCAFile: "baz"}, false}, {TLSInfo{KeyFile: "bar", TrustedCAFile: "baz"}, false}, {TLSInfo{CertFile: "foo", KeyFile: "bar", TrustedCAFile: "baz"}, false}, } for i, tt := range tests { got := tt.info.Empty() if tt.want != got { t.Errorf("#%d: result of Empty() incorrect: want=%t got=%t", i, tt.want, got) } } } func TestTLSInfoMissingFields(t *testing.T) { tlsinfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create cert") tests := []TLSInfo{ {CertFile: tlsinfo.CertFile}, {KeyFile: tlsinfo.KeyFile}, {CertFile: tlsinfo.CertFile, TrustedCAFile: tlsinfo.TrustedCAFile}, {KeyFile: tlsinfo.KeyFile, TrustedCAFile: tlsinfo.TrustedCAFile}, } for i, info := range tests { if _, err = info.ServerConfig(); err == nil { t.Errorf("#%d: expected non-nil error from ServerConfig()", i) } if _, err = info.ClientConfig(); err == nil { t.Errorf("#%d: expected non-nil error from ClientConfig()", i) } } } func TestTLSInfoParseFuncError(t *testing.T) { tlsinfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create cert") tests := []struct { info TLSInfo }{ { info: *tlsinfo, }, { info: TLSInfo{CertFile: "", KeyFile: "", TrustedCAFile: tlsinfo.CertFile, EmptyCN: true}, }, } for i, tt := range tests { tt.info.parseFunc = fakeCertificateParserFunc(errors.New("fake")) if _, err = tt.info.ServerConfig(); err == nil { t.Errorf("#%d: expected non-nil error from ServerConfig()", i) } if _, err = tt.info.ClientConfig(); err == nil { t.Errorf("#%d: expected non-nil error from ClientConfig()", i) } } } func TestTLSInfoConfigFuncs(t *testing.T) { ln := zaptest.NewLogger(t) tlsinfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create cert") tests := []struct { info TLSInfo clientAuth tls.ClientAuthType wantCAs bool }{ { info: TLSInfo{CertFile: tlsinfo.CertFile, KeyFile: tlsinfo.KeyFile, Logger: ln}, clientAuth: tls.NoClientCert, wantCAs: false, }, { info: TLSInfo{CertFile: tlsinfo.CertFile, KeyFile: tlsinfo.KeyFile, TrustedCAFile: tlsinfo.CertFile, Logger: ln}, clientAuth: tls.RequireAndVerifyClientCert, wantCAs: true, }, } for i, tt := range tests { tt.info.parseFunc = fakeCertificateParserFunc(nil) sCfg, err := tt.info.ServerConfig() if err != nil { t.Errorf("#%d: expected nil error from ServerConfig(), got non-nil: %v", i, err) } if tt.wantCAs != (sCfg.ClientCAs != nil) { t.Errorf("#%d: wantCAs=%t but ClientCAs=%v", i, tt.wantCAs, sCfg.ClientCAs) } cCfg, err := tt.info.ClientConfig() if err != nil { t.Errorf("#%d: expected nil error from ClientConfig(), got non-nil: %v", i, err) } if tt.wantCAs != (cCfg.RootCAs != nil) { t.Errorf("#%d: wantCAs=%t but RootCAs=%v", i, tt.wantCAs, sCfg.RootCAs) } } } func TestNewListenerUnixSocket(t *testing.T) { l, err := NewListener("testsocket", "unix", nil) if err != nil { t.Errorf("error listening on unix socket (%v)", err) } l.Close() } // TestNewListenerTLSInfoSelfCert tests that a new certificate accepts connections. func TestNewListenerTLSInfoSelfCert(t *testing.T) { tmpdir := t.TempDir() tlsinfo, err := SelfCert(zaptest.NewLogger(t), tmpdir, []string{"127.0.0.1"}, 1) require.NoError(t, err) require.Falsef(t, tlsinfo.Empty(), "tlsinfo should have certs (%+v)", tlsinfo) testNewListenerTLSInfoAccept(t, tlsinfo) assert.Panicsf(t, func() { SelfCert(nil, tmpdir, []string{"127.0.0.1"}, 1) }, "expected panic with nil log") } func TestIsClosedConnError(t *testing.T) { l, err := NewListener("testsocket", "unix", nil) if err != nil { t.Errorf("error listening on unix socket (%v)", err) } l.Close() _, err = l.Accept() require.Truef(t, IsClosedConnError(err), "expect true, got false (%v)", err) } func TestSocktOptsEmpty(t *testing.T) { tests := []struct { sopts SocketOpts want bool }{ {SocketOpts{}, true}, {SocketOpts{ReuseAddress: true, ReusePort: false}, false}, {SocketOpts{ReusePort: true}, false}, } for i, tt := range tests { got := tt.sopts.Empty() if tt.want != got { t.Errorf("#%d: result of Empty() incorrect: want=%t got=%t", i, tt.want, got) } } } // TestNewListenerWithACRLFile tests when a revocation list is present. func TestNewListenerWithACRLFile(t *testing.T) { clientTLSInfo, err := createSelfCertEx(t, "127.0.0.1", x509.ExtKeyUsageClientAuth) require.NoErrorf(t, err, "unable to create client cert") loadFileAsPEM := func(fileName string) []byte { loaded, readErr := os.ReadFile(fileName) require.NoErrorf(t, readErr, "unable to read file %q", fileName) block, _ := pem.Decode(loaded) return block.Bytes } clientCert, err := x509.ParseCertificate(loadFileAsPEM(clientTLSInfo.CertFile)) require.NoErrorf(t, err, "unable to parse client cert") tests := map[string]struct { expectHandshakeError bool revokedCertificateEntries []x509.RevocationListEntry revocationListContents []byte }{ "empty revocation list": { expectHandshakeError: false, }, "client cert is revoked": { expectHandshakeError: true, revokedCertificateEntries: []x509.RevocationListEntry{ { SerialNumber: clientCert.SerialNumber, RevocationTime: time.Now(), }, }, }, "invalid CRL file content": { expectHandshakeError: true, revocationListContents: []byte("@invalidcontent"), }, } for testName, test := range tests { t.Run(testName, func(t *testing.T) { tmpdir := t.TempDir() tlsInfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create server cert") tlsInfo.TrustedCAFile = clientTLSInfo.CertFile tlsInfo.CRLFile = filepath.Join(tmpdir, "revoked.r0") cert, err := x509.ParseCertificate(loadFileAsPEM(tlsInfo.CertFile)) require.NoErrorf(t, err, "unable to decode server cert") key, err := x509.ParseECPrivateKey(loadFileAsPEM(tlsInfo.KeyFile)) require.NoErrorf(t, err, "unable to parse server key") revocationListContents := test.revocationListContents if len(revocationListContents) == 0 { tmpl := &x509.RevocationList{ RevokedCertificateEntries: test.revokedCertificateEntries, ThisUpdate: time.Now(), NextUpdate: time.Now().Add(time.Hour), Number: big.NewInt(1), } revocationListContents, err = x509.CreateRevocationList(rand.Reader, tmpl, cert, key) require.NoErrorf(t, err, "unable to create revocation list") } err = os.WriteFile(tlsInfo.CRLFile, revocationListContents, 0o600) require.NoErrorf(t, err, "unable to write revocation list") chHandshakeFailure := make(chan error, 1) tlsInfo.HandshakeFailure = func(_ *tls.Conn, err error) { if err != nil { chHandshakeFailure <- err } } rootCAs := x509.NewCertPool() rootCAs.AddCert(cert) clientCert, err := tls.LoadX509KeyPair(clientTLSInfo.CertFile, clientTLSInfo.KeyFile) require.NoErrorf(t, err, "unable to create peer cert") ln, err := NewListener("127.0.0.1:0", "https", tlsInfo) require.NoErrorf(t, err, "unable to start listener") tlsConfig := &tls.Config{} tlsConfig.InsecureSkipVerify = false tlsConfig.Certificates = []tls.Certificate{clientCert} tlsConfig.RootCAs = rootCAs tr := &http.Transport{TLSClientConfig: tlsConfig} cli := &http.Client{Transport: tr, Timeout: 5 * time.Second} var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() if _, gerr := cli.Get("https://" + ln.Addr().String()); gerr != nil { t.Logf("http GET failed: %v", gerr) } }() chAcceptConn := make(chan net.Conn, 1) go func() { defer wg.Done() conn, err := ln.Accept() if err == nil { chAcceptConn <- conn } }() timer := time.NewTimer(5 * time.Second) defer func() { if !timer.Stop() { <-timer.C } }() select { case err := <-chHandshakeFailure: if !test.expectHandshakeError { t.Errorf("expecting no handshake error, got: %v", err) } case conn := <-chAcceptConn: if test.expectHandshakeError { t.Errorf("expecting handshake error, got nothing") } conn.Close() case <-timer.C: t.Error("timed out waiting for closed connection or handshake error") } ln.Close() wg.Wait() }) } } ================================================ FILE: client/pkg/transport/listener_tls.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "context" "crypto/tls" "crypto/x509" "fmt" "net" "os" "strings" "sync" ) // tlsListener overrides a TLS listener so it will reject client // certificates with insufficient SAN credentials or CRL revoked // certificates. type tlsListener struct { net.Listener connc chan net.Conn donec chan struct{} err error handshakeFailure func(*tls.Conn, error) check tlsCheckFunc } type tlsCheckFunc func(context.Context, *tls.Conn) error // NewTLSListener handshakes TLS connections and performs optional CRL checking. func NewTLSListener(l net.Listener, tlsinfo *TLSInfo) (net.Listener, error) { check := func(context.Context, *tls.Conn) error { return nil } return newTLSListener(l, tlsinfo, check) } func newTLSListener(l net.Listener, tlsinfo *TLSInfo, check tlsCheckFunc) (net.Listener, error) { if tlsinfo == nil || tlsinfo.Empty() { l.Close() return nil, fmt.Errorf("cannot listen on TLS for %s: KeyFile and CertFile are not presented", l.Addr().String()) } tlscfg, err := tlsinfo.ServerConfig() if err != nil { return nil, err } hf := tlsinfo.HandshakeFailure if hf == nil { hf = func(*tls.Conn, error) {} } if len(tlsinfo.CRLFile) > 0 { prevCheck := check check = func(ctx context.Context, tlsConn *tls.Conn) error { if err := prevCheck(ctx, tlsConn); err != nil { return err } st := tlsConn.ConnectionState() if certs := st.PeerCertificates; len(certs) > 0 { return checkCRL(tlsinfo.CRLFile, certs) } return nil } } tlsl := &tlsListener{ Listener: tls.NewListener(l, tlscfg), connc: make(chan net.Conn), donec: make(chan struct{}), handshakeFailure: hf, check: check, } go tlsl.acceptLoop() return tlsl, nil } func (l *tlsListener) Accept() (net.Conn, error) { select { case conn := <-l.connc: return conn, nil case <-l.donec: return nil, l.err } } func checkSAN(ctx context.Context, tlsConn *tls.Conn) error { st := tlsConn.ConnectionState() if certs := st.PeerCertificates; len(certs) > 0 { addr := tlsConn.RemoteAddr().String() return checkCertSAN(ctx, certs[0], addr) } return nil } // acceptLoop launches each TLS handshake in a separate goroutine // to prevent a hanging TLS connection from blocking other connections. func (l *tlsListener) acceptLoop() { var wg sync.WaitGroup var pendingMu sync.Mutex pending := make(map[net.Conn]struct{}) ctx, cancel := context.WithCancel(context.Background()) defer func() { cancel() pendingMu.Lock() for c := range pending { c.Close() } pendingMu.Unlock() wg.Wait() close(l.donec) }() for { conn, err := l.Listener.Accept() if err != nil { l.err = err return } pendingMu.Lock() pending[conn] = struct{}{} pendingMu.Unlock() wg.Add(1) go func() { defer func() { if conn != nil { conn.Close() } wg.Done() }() tlsConn := conn.(*tls.Conn) herr := tlsConn.Handshake() pendingMu.Lock() delete(pending, conn) pendingMu.Unlock() if herr != nil { l.handshakeFailure(tlsConn, herr) return } if err := l.check(ctx, tlsConn); err != nil { l.handshakeFailure(tlsConn, err) return } select { case l.connc <- tlsConn: conn = nil case <-ctx.Done(): } }() } } func checkCRL(crlPath string, cert []*x509.Certificate) error { // TODO: cache crlBytes, err := os.ReadFile(crlPath) if err != nil { return err } certList, err := x509.ParseRevocationList(crlBytes) if err != nil { return err } revokedSerials := make(map[string]struct{}) for _, rc := range certList.RevokedCertificateEntries { revokedSerials[string(rc.SerialNumber.Bytes())] = struct{}{} } for _, c := range cert { serial := string(c.SerialNumber.Bytes()) if _, ok := revokedSerials[serial]; ok { return fmt.Errorf("transport: certificate serial %x revoked", serial) } } return nil } func checkCertSAN(ctx context.Context, cert *x509.Certificate, remoteAddr string) error { if len(cert.IPAddresses) == 0 && len(cert.DNSNames) == 0 { return nil } h, _, herr := net.SplitHostPort(remoteAddr) if herr != nil { return herr } if len(cert.IPAddresses) > 0 { cerr := cert.VerifyHostname(h) if cerr == nil { return nil } if len(cert.DNSNames) == 0 { return cerr } } if len(cert.DNSNames) > 0 { ok, err := isHostInDNS(ctx, h, cert.DNSNames) if ok { return nil } errStr := "" if err != nil { errStr = " (" + err.Error() + ")" } return fmt.Errorf("tls: %q does not match any of DNSNames %q"+errStr, h, cert.DNSNames) } return nil } func isHostInDNS(ctx context.Context, host string, dnsNames []string) (ok bool, err error) { // reverse lookup var names []string var wildcards []string for _, dns := range dnsNames { if strings.HasPrefix(dns, "*.") { wildcards = append(wildcards, dns[1:]) } else { names = append(names, dns) } } lnames, lerr := net.DefaultResolver.LookupAddr(ctx, host) for _, name := range lnames { // strip trailing '.' from PTR record if name[len(name)-1] == '.' { name = name[:len(name)-1] } for _, wc := range wildcards { if strings.HasSuffix(name, wc) { return true, nil } } for _, n := range names { if n == name { return true, nil } } } err = lerr // forward lookup for _, dns := range names { addrs, lerr := net.DefaultResolver.LookupHost(ctx, dns) if lerr != nil { err = lerr continue } for _, addr := range addrs { if addr == host { return true, nil } } } return false, err } func (l *tlsListener) Close() error { err := l.Listener.Close() <-l.donec return err } ================================================ FILE: client/pkg/transport/sockopt.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "syscall" ) type Controls []func(network, addr string, conn syscall.RawConn) error func (ctls Controls) Control(network, addr string, conn syscall.RawConn) error { for _, s := range ctls { if err := s(network, addr, conn); err != nil { return err } } return nil } type SocketOpts struct { // ReusePort enables socket option SO_REUSEPORT [1] which allows rebind of // a port already in use. User should keep in mind that flock can fail // in which case lock on data file could result in unexpected // condition. User should take caution to protect against lock race. // [1] https://man7.org/linux/man-pages/man7/socket.7.html ReusePort bool `json:"reuse-port"` // ReuseAddress enables a socket option SO_REUSEADDR which allows // binding to an address in `TIME_WAIT` state. Useful to improve MTTR // in cases where etcd slow to restart due to excessive `TIME_WAIT`. // [1] https://man7.org/linux/man-pages/man7/socket.7.html ReuseAddress bool `json:"reuse-address"` } func getControls(sopts *SocketOpts) Controls { ctls := Controls{} if sopts.ReuseAddress { ctls = append(ctls, setReuseAddress) } if sopts.ReusePort { ctls = append(ctls, setReusePort) } return ctls } func (sopts *SocketOpts) Empty() bool { return !sopts.ReuseAddress && !sopts.ReusePort } ================================================ FILE: client/pkg/transport/sockopt_solaris.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build solaris package transport import ( "errors" "syscall" "golang.org/x/sys/unix" ) func setReusePort(network, address string, c syscall.RawConn) error { return errors.New("port reuse is not supported on Solaris") } func setReuseAddress(network, address string, conn syscall.RawConn) error { return conn.Control(func(fd uintptr) { syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEADDR, 1) }) } ================================================ FILE: client/pkg/transport/sockopt_unix.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows && !solaris && !wasm && !js package transport import ( "syscall" "golang.org/x/sys/unix" ) func setReusePort(network, address string, conn syscall.RawConn) error { return conn.Control(func(fd uintptr) { syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1) }) } func setReuseAddress(network, address string, conn syscall.RawConn) error { return conn.Control(func(fd uintptr) { syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEADDR, 1) }) } ================================================ FILE: client/pkg/transport/sockopt_wasm.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build wasm || js package transport import ( "errors" "syscall" ) func setReusePort(network, address string, c syscall.RawConn) error { return errors.New("port reuse is not supported on WASM") } func setReuseAddress(network, addr string, conn syscall.RawConn) error { return errors.New("address reuse is not supported on WASM") } ================================================ FILE: client/pkg/transport/sockopt_windows.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows package transport import ( "errors" "syscall" ) func setReusePort(network, address string, c syscall.RawConn) error { return errors.New("port reuse is not supported on Windows") } // Windows supports SO_REUSEADDR, but it may cause undefined behavior, as // there is no protection against port hijacking. func setReuseAddress(network, addr string, conn syscall.RawConn) error { return errors.New("address reuse is not supported on Windows") } ================================================ FILE: client/pkg/transport/timeout_conn.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "net" "time" ) type timeoutConn struct { net.Conn writeTimeout time.Duration readTimeout time.Duration } func (c timeoutConn) Write(b []byte) (n int, err error) { if c.writeTimeout > 0 { if err := c.SetWriteDeadline(time.Now().Add(c.writeTimeout)); err != nil { return 0, err } } return c.Conn.Write(b) } func (c timeoutConn) Read(b []byte) (n int, err error) { if c.readTimeout > 0 { if err := c.SetReadDeadline(time.Now().Add(c.readTimeout)); err != nil { return 0, err } } return c.Conn.Read(b) } ================================================ FILE: client/pkg/transport/timeout_dialer.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "net" "time" ) type rwTimeoutDialer struct { wtimeoutd time.Duration rdtimeoutd time.Duration net.Dialer } func (d *rwTimeoutDialer) Dial(network, address string) (net.Conn, error) { conn, err := d.Dialer.Dial(network, address) tconn := &timeoutConn{ readTimeout: d.rdtimeoutd, writeTimeout: d.wtimeoutd, Conn: conn, } return tconn, err } ================================================ FILE: client/pkg/transport/timeout_dialer_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "errors" "net" "testing" "time" "github.com/stretchr/testify/require" ) func TestReadWriteTimeoutDialer(t *testing.T) { stop := make(chan struct{}) ln, err := net.Listen("tcp", "127.0.0.1:0") require.NoErrorf(t, err, "unexpected listen error") defer func() { stop <- struct{}{} }() ts := testBlockingServer{ln, 2, stop} go ts.Start(t) d := rwTimeoutDialer{ wtimeoutd: 10 * time.Millisecond, rdtimeoutd: 10 * time.Millisecond, } conn, err := d.Dial("tcp", ln.Addr().String()) require.NoErrorf(t, err, "unexpected dial error") defer conn.Close() // fill the socket buffer data := make([]byte, 5*1024*1024) done := make(chan struct{}, 1) go func() { _, err = conn.Write(data) done <- struct{}{} }() select { case <-done: // Wait 5s more than timeout to avoid delay in low-end systems; // the slack was 1s extra, but that wasn't enough for CI. case <-time.After(d.wtimeoutd*10 + 5*time.Second): t.Fatal("wait timeout") } var operr *net.OpError if !errors.As(err, &operr) || operr.Op != "write" || !operr.Timeout() { t.Errorf("err = %v, want write i/o timeout error", err) } conn, err = d.Dial("tcp", ln.Addr().String()) require.NoErrorf(t, err, "unexpected dial error") defer conn.Close() buf := make([]byte, 10) go func() { _, err = conn.Read(buf) done <- struct{}{} }() select { case <-done: case <-time.After(d.rdtimeoutd * 10): t.Fatal("wait timeout") } if !errors.As(err, &operr) || operr.Op != "read" || !operr.Timeout() { t.Errorf("err = %v, want read i/o timeout error", err) } } type testBlockingServer struct { ln net.Listener n int stop chan struct{} } func (ts *testBlockingServer) Start(t *testing.T) { t.Helper() for i := 0; i < ts.n; i++ { conn, err := ts.ln.Accept() if err != nil { t.Error(err) } defer conn.Close() } <-ts.stop } ================================================ FILE: client/pkg/transport/timeout_listener.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "net" "time" ) // NewTimeoutListener returns a listener that listens on the given address. // If read/write on the accepted connection blocks longer than its time limit, // it will return timeout error. func NewTimeoutListener(addr string, scheme string, tlsinfo *TLSInfo, readTimeout, writeTimeout time.Duration) (net.Listener, error) { return newListener(addr, scheme, WithTimeout(readTimeout, writeTimeout), WithTLSInfo(tlsinfo)) } type rwTimeoutListener struct { net.Listener writeTimeout time.Duration readTimeout time.Duration } func (rwln *rwTimeoutListener) Accept() (net.Conn, error) { c, err := rwln.Listener.Accept() if err != nil { return nil, err } return timeoutConn{ Conn: c, writeTimeout: rwln.writeTimeout, readTimeout: rwln.readTimeout, }, nil } ================================================ FILE: client/pkg/transport/timeout_listener_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "errors" "net" "testing" "time" "github.com/stretchr/testify/require" ) // TestNewTimeoutListener tests that NewTimeoutListener returns a // rwTimeoutListener struct with timeouts set. func TestNewTimeoutListener(t *testing.T) { l, err := NewTimeoutListener("127.0.0.1:0", "http", nil, time.Hour, time.Hour) require.NoErrorf(t, err, "unexpected NewTimeoutListener error") defer l.Close() tln := l.(*rwTimeoutListener) if tln.readTimeout != time.Hour { t.Errorf("read timeout = %s, want %s", tln.readTimeout, time.Hour) } if tln.writeTimeout != time.Hour { t.Errorf("write timeout = %s, want %s", tln.writeTimeout, time.Hour) } } func TestWriteReadTimeoutListener(t *testing.T) { ln, err := net.Listen("tcp", "127.0.0.1:0") require.NoErrorf(t, err, "unexpected listen error") wln := rwTimeoutListener{ Listener: ln, writeTimeout: 10 * time.Millisecond, readTimeout: 10 * time.Millisecond, } blocker := func(stopCh <-chan struct{}) { conn, derr := net.Dial("tcp", ln.Addr().String()) if derr != nil { t.Errorf("unexpected dail error: %v", derr) } defer conn.Close() // block the receiver until the writer timeout <-stopCh } writerStopCh := make(chan struct{}, 1) go blocker(writerStopCh) conn, err := wln.Accept() if err != nil { writerStopCh <- struct{}{} t.Fatalf("unexpected accept error: %v", err) } defer conn.Close() // fill the socket buffer data := make([]byte, 5*1024*1024) done := make(chan struct{}, 1) go func() { _, err = conn.Write(data) done <- struct{}{} }() select { case <-done: // It waits 1s more to avoid delay in low-end system. case <-time.After(wln.writeTimeout*10 + time.Second): writerStopCh <- struct{}{} t.Fatal("wait timeout") } var operr *net.OpError if !errors.As(err, &operr) || operr.Op != "write" || !operr.Timeout() { t.Errorf("err = %v, want write i/o timeout error", err) } writerStopCh <- struct{}{} readerStopCh := make(chan struct{}, 1) go blocker(readerStopCh) conn, err = wln.Accept() if err != nil { readerStopCh <- struct{}{} t.Fatalf("unexpected accept error: %v", err) } buf := make([]byte, 10) go func() { _, err = conn.Read(buf) done <- struct{}{} }() select { case <-done: case <-time.After(wln.readTimeout * 10): readerStopCh <- struct{}{} t.Fatal("wait timeout") } if !errors.As(err, &operr) || operr.Op != "read" || !operr.Timeout() { t.Errorf("err = %v, want read i/o timeout error", err) } readerStopCh <- struct{}{} } ================================================ FILE: client/pkg/transport/timeout_transport.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "net" "net/http" "time" ) // NewTimeoutTransport returns a transport created using the given TLS info. // If read/write on the created connection blocks longer than its time limit, // it will return timeout error. // If read/write timeout is set, transport will not be able to reuse connection. func NewTimeoutTransport(info TLSInfo, dialtimeoutd, rdtimeoutd, wtimeoutd time.Duration) (*http.Transport, error) { tr, err := NewTransport(info, dialtimeoutd) if err != nil { return nil, err } if rdtimeoutd != 0 || wtimeoutd != 0 { // the timed out connection will timeout soon after it is idle. // it should not be put back to http transport as an idle connection for future usage. tr.MaxIdleConnsPerHost = -1 } else { // allow more idle connections between peers to avoid unnecessary port allocation. tr.MaxIdleConnsPerHost = 1024 } tr.Dial = (&rwTimeoutDialer{ //nolint:staticcheck // TODO: remove for a supported version Dialer: net.Dialer{ Timeout: dialtimeoutd, KeepAlive: 30 * time.Second, }, rdtimeoutd: rdtimeoutd, wtimeoutd: wtimeoutd, }).Dial return tr, nil } ================================================ FILE: client/pkg/transport/timeout_transport_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "bytes" "io" "net/http" "net/http/httptest" "testing" "time" "github.com/stretchr/testify/require" ) // TestNewTimeoutTransport tests that NewTimeoutTransport returns a transport // that can dial out timeout connections. func TestNewTimeoutTransport(t *testing.T) { tr, err := NewTimeoutTransport(TLSInfo{}, time.Hour, time.Hour, time.Hour) require.NoError(t, err) remoteAddr := func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(r.RemoteAddr)) } srv := httptest.NewServer(http.HandlerFunc(remoteAddr)) defer srv.Close() conn, err := tr.Dial("tcp", srv.Listener.Addr().String()) //nolint:staticcheck // TODO: remove for a supported version require.NoError(t, err) defer conn.Close() tconn, ok := conn.(*timeoutConn) require.Truef(t, ok, "failed to dial out *timeoutConn") if tconn.readTimeout != time.Hour { t.Errorf("read timeout = %s, want %s", tconn.readTimeout, time.Hour) } if tconn.writeTimeout != time.Hour { t.Errorf("write timeout = %s, want %s", tconn.writeTimeout, time.Hour) } // ensure not reuse timeout connection req, err := http.NewRequest(http.MethodGet, srv.URL, nil) require.NoError(t, err) resp, err := tr.RoundTrip(req) require.NoError(t, err) addr0, err := io.ReadAll(resp.Body) resp.Body.Close() require.NoError(t, err) resp, err = tr.RoundTrip(req) require.NoError(t, err) addr1, err := io.ReadAll(resp.Body) resp.Body.Close() require.NoError(t, err) if bytes.Equal(addr0, addr1) { t.Errorf("addr0 = %s addr1= %s, want not equal", addr0, addr1) } } ================================================ FILE: client/pkg/transport/tls.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "context" "errors" "fmt" "strings" "time" ) // ValidateSecureEndpoints scans the given endpoints against tls info, returning only those // endpoints that could be validated as secure. func ValidateSecureEndpoints(tlsInfo TLSInfo, eps []string) ([]string, error) { t, err := NewTransport(tlsInfo, 5*time.Second) if err != nil { return nil, err } defer t.CloseIdleConnections() var errs []string var endpoints []string for _, ep := range eps { if !strings.HasPrefix(ep, "https://") { errs = append(errs, fmt.Sprintf("%q is insecure", ep)) continue } conn, cerr := t.DialContext(context.Background(), "tcp", ep[len("https://"):]) if cerr != nil { errs = append(errs, fmt.Sprintf("%q failed to dial (%v)", ep, cerr)) continue } conn.Close() endpoints = append(endpoints, ep) } if len(errs) != 0 { err = errors.New(strings.Join(errs, ",")) } return endpoints, err } ================================================ FILE: client/pkg/transport/tls_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "net/http" "net/http/httptest" "reflect" "testing" "github.com/stretchr/testify/require" ) func TestValidateSecureEndpoints(t *testing.T) { tlsInfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create cert") remoteAddr := func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(r.RemoteAddr)) } srv := httptest.NewServer(http.HandlerFunc(remoteAddr)) defer srv.Close() tests := map[string]struct { endPoints []string expectedEndpoints []string expectedErr bool }{ "invalidEndPoints": { endPoints: []string{ "invalid endpoint", }, expectedEndpoints: nil, expectedErr: true, }, "insecureEndpoints": { endPoints: []string{ "http://127.0.0.1:8000", "http://" + srv.Listener.Addr().String(), }, expectedEndpoints: nil, expectedErr: true, }, "secureEndPoints": { endPoints: []string{ "https://" + srv.Listener.Addr().String(), }, expectedEndpoints: []string{ "https://" + srv.Listener.Addr().String(), }, expectedErr: false, }, "mixEndPoints": { endPoints: []string{ "https://" + srv.Listener.Addr().String(), "http://" + srv.Listener.Addr().String(), "invalid end points", }, expectedEndpoints: []string{ "https://" + srv.Listener.Addr().String(), }, expectedErr: true, }, } for name, test := range tests { t.Run(name, func(t *testing.T) { secureEps, err := ValidateSecureEndpoints(*tlsInfo, test.endPoints) if test.expectedErr != (err != nil) { t.Errorf("Unexpected error, got: %v, want: %v", err, test.expectedErr) } if !reflect.DeepEqual(test.expectedEndpoints, secureEps) { t.Errorf("expected endpoints %v, got %v", test.expectedEndpoints, secureEps) } }) } } ================================================ FILE: client/pkg/transport/transport.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "context" "net" "net/http" "strings" "time" ) type unixTransport struct{ *http.Transport } func NewTransport(info TLSInfo, dialtimeoutd time.Duration) (*http.Transport, error) { cfg, err := info.ClientConfig() if err != nil { return nil, err } var ipAddr net.Addr if info.LocalAddr != "" { ipAddr, err = net.ResolveTCPAddr("tcp", info.LocalAddr+":0") if err != nil { return nil, err } } t := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: dialtimeoutd, LocalAddr: ipAddr, // value taken from http.DefaultTransport KeepAlive: 30 * time.Second, }).DialContext, // value taken from http.DefaultTransport TLSHandshakeTimeout: 10 * time.Second, TLSClientConfig: cfg, } dialer := &net.Dialer{ Timeout: dialtimeoutd, KeepAlive: 30 * time.Second, } dialContext := func(ctx context.Context, net, addr string) (net.Conn, error) { return dialer.DialContext(ctx, "unix", addr) } tu := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: dialContext, TLSHandshakeTimeout: 10 * time.Second, TLSClientConfig: cfg, // Cost of reopening connection on sockets is low, and they are mostly used in testing. // Long living unix-transport connections were leading to 'leak' test flakes. // Alternatively the returned Transport (t) should override CloseIdleConnections to // forward it to 'tu' as well. IdleConnTimeout: time.Microsecond, } ut := &unixTransport{tu} t.RegisterProtocol("unix", ut) t.RegisterProtocol("unixs", ut) return t, nil } func (urt *unixTransport) RoundTrip(req *http.Request) (*http.Response, error) { url := *req.URL req.URL = &url req.URL.Scheme = strings.Replace(req.URL.Scheme, "unix", "http", 1) return urt.Transport.RoundTrip(req) } ================================================ FILE: client/pkg/transport/transport_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "crypto/tls" "net/http" "strings" "testing" "time" "github.com/stretchr/testify/require" ) // TestNewTransportTLSInvalidCipherSuitesTLS12 expects a client with invalid // cipher suites fail to handshake with the server. func TestNewTransportTLSInvalidCipherSuitesTLS12(t *testing.T) { tlsInfo, err := createSelfCert(t) require.NoErrorf(t, err, "unable to create cert") cipherSuites := []uint16{ tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, } // make server and client have unmatched cipher suites srvTLS, cliTLS := *tlsInfo, *tlsInfo srvTLS.CipherSuites, cliTLS.CipherSuites = cipherSuites[:2], cipherSuites[2:] ln, err := NewListener("127.0.0.1:0", "https", &srvTLS) require.NoError(t, err) defer ln.Close() donec := make(chan struct{}) go func() { ln.Accept() donec <- struct{}{} }() go func() { tr, err := NewTransport(cliTLS, 3*time.Second) tr.TLSClientConfig.MaxVersion = tls.VersionTLS12 if err != nil { t.Errorf("unexpected NewTransport error: %v", err) } cli := &http.Client{Transport: tr} _, gerr := cli.Get("https://" + ln.Addr().String()) if gerr == nil || !strings.Contains(gerr.Error(), "tls: handshake failure") { t.Error("expected client TLS handshake error") } ln.Close() donec <- struct{}{} }() <-donec <-donec } ================================================ FILE: client/pkg/transport/unix_listener.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package transport import ( "net" "os" ) type unixListener struct{ net.Listener } func NewUnixListener(addr string) (net.Listener, error) { if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { return nil, err } l, err := net.Listen("unix", addr) if err != nil { return nil, err } return &unixListener{l}, nil } func (ul *unixListener) Close() error { if err := os.Remove(ul.Addr().String()); err != nil && !os.IsNotExist(err) { return err } return ul.Listener.Close() } ================================================ FILE: client/pkg/types/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package types declares various data types and implements type-checking // functions. package types ================================================ FILE: client/pkg/types/id.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package types import ( "strconv" "strings" ) // ID represents a generic identifier which is canonically // stored as a uint64 but is typically represented as a // base-16 string for input/output type ID uint64 func (i ID) String() string { return strconv.FormatUint(uint64(i), 16) } // IDFromString attempts to create an ID from a base-16 string. func IDFromString(s string) (ID, error) { i, err := strconv.ParseUint(s, 16, 64) return ID(i), err } // IDSlice implements the sort interface type IDSlice []ID func (p IDSlice) Len() int { return len(p) } func (p IDSlice) Less(i, j int) bool { return uint64(p[i]) < uint64(p[j]) } func (p IDSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p IDSlice) String() string { var b strings.Builder if p.Len() > 0 { b.WriteString(p[0].String()) } for i := 1; i < p.Len(); i++ { b.WriteString(",") b.WriteString(p[i].String()) } return b.String() } ================================================ FILE: client/pkg/types/id_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package types import ( "reflect" "sort" "testing" "github.com/stretchr/testify/require" ) func TestIDString(t *testing.T) { tests := []struct { input ID want string }{ { input: 12, want: "c", }, { input: 4918257920282737594, want: "444129853c343bba", }, } for i, tt := range tests { got := tt.input.String() if tt.want != got { t.Errorf("#%d: ID.String failure: want=%v, got=%v", i, tt.want, got) } } } func TestIDFromString(t *testing.T) { tests := []struct { input string want ID }{ { input: "17", want: 23, }, { input: "612840dae127353", want: 437557308098245459, }, } for i, tt := range tests { got, err := IDFromString(tt.input) if err != nil { t.Errorf("#%d: IDFromString failure: err=%v", i, err) continue } if tt.want != got { t.Errorf("#%d: IDFromString failure: want=%v, got=%v", i, tt.want, got) } } } func TestIDFromStringFail(t *testing.T) { tests := []string{ "", "XXX", "612840dae127353612840dae127353", } for i, tt := range tests { _, err := IDFromString(tt) require.Errorf(t, err, "#%d: IDFromString expected error", i) } } func TestIDSlice(t *testing.T) { g := []ID{10, 500, 5, 1, 100, 25} w := []ID{1, 5, 10, 25, 100, 500} sort.Sort(IDSlice(g)) if !reflect.DeepEqual(g, w) { t.Errorf("slice after sort = %#v, want %#v", g, w) } } ================================================ FILE: client/pkg/types/set.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package types import ( "reflect" "sort" "sync" ) type Set interface { Add(string) Remove(string) Contains(val ...string) bool Equals(Set) bool Length() int Values() []string Copy() Set Sub(Set) Set // ContainsAll returns whether the set contains all given values // Deprecated: Use Contains instead. ContainsAll(values []string) bool } type ThreadsafeSet interface { Set } func NewUnsafeSet(values ...string) Set { set := &unsafeSet{make(map[string]struct{})} for _, v := range values { set.Add(v) } return set } func NewThreadsafeSet(values ...string) ThreadsafeSet { us := NewUnsafeSet(values...) return &tsafeSet{us, sync.RWMutex{}} } var _ Set = (*unsafeSet)(nil) type unsafeSet struct { d map[string]struct{} } // Add adds a new value to the set (no-op if the value is already present) func (us *unsafeSet) Add(value string) { us.d[value] = struct{}{} } // Remove removes the given value from the set func (us *unsafeSet) Remove(value string) { delete(us.d, value) } // Contains returns whether the set contains the given value func (us *unsafeSet) Contains(values ...string) (exists bool) { for _, value := range values { if _, exists := us.d[value]; !exists { return false } } return true } // ContainsAll returns whether the set contains all given values // Deprecated: Use Contains instead. func (us *unsafeSet) ContainsAll(values []string) bool { for _, s := range values { if !us.Contains(s) { return false } } return true } // Equals returns whether the contents of two sets are identical func (us *unsafeSet) Equals(other Set) bool { v1 := sort.StringSlice(us.Values()) v2 := sort.StringSlice(other.Values()) v1.Sort() v2.Sort() return reflect.DeepEqual(v1, v2) } // Length returns the number of elements in the set func (us *unsafeSet) Length() int { return len(us.d) } // Values returns the values of the Set in an unspecified order. func (us *unsafeSet) Values() (values []string) { values = make([]string, 0, len(us.d)) for val := range us.d { values = append(values, val) } return values } // Copy creates a new Set containing the values of the first func (us *unsafeSet) Copy() Set { cp := NewUnsafeSet() for val := range us.d { cp.Add(val) } return cp } // Sub removes all elements in other from the set func (us *unsafeSet) Sub(other Set) Set { oValues := other.Values() result := us.Copy().(*unsafeSet) for _, val := range oValues { if _, ok := result.d[val]; !ok { continue } delete(result.d, val) } return result } var _ ThreadsafeSet = (*tsafeSet)(nil) type tsafeSet struct { us Set m sync.RWMutex } func (ts *tsafeSet) Add(value string) { ts.m.Lock() defer ts.m.Unlock() ts.us.Add(value) } func (ts *tsafeSet) Remove(value string) { ts.m.Lock() defer ts.m.Unlock() ts.us.Remove(value) } func (ts *tsafeSet) Contains(values ...string) (exists bool) { ts.m.RLock() defer ts.m.RUnlock() return ts.us.Contains(values...) } // ContainsAll returns whether the set contains all given values // Deprecated: Use Contains instead. func (ts *tsafeSet) ContainsAll(values []string) bool { ts.m.RLock() defer ts.m.RUnlock() return ts.us.ContainsAll(values) } func (ts *tsafeSet) Equals(other Set) bool { ts.m.RLock() defer ts.m.RUnlock() // If ts and other represent the same variable, avoid calling // ts.us.Equals(other), to avoid double RLock bug if _other, ok := other.(*tsafeSet); ok { if _other == ts { return true } } return ts.us.Equals(other) } func (ts *tsafeSet) Length() int { ts.m.RLock() defer ts.m.RUnlock() return ts.us.Length() } func (ts *tsafeSet) Values() (values []string) { ts.m.RLock() defer ts.m.RUnlock() return ts.us.Values() } func (ts *tsafeSet) Copy() Set { ts.m.RLock() defer ts.m.RUnlock() usResult := ts.us.Copy().(*unsafeSet) return &tsafeSet{usResult, sync.RWMutex{}} } func (ts *tsafeSet) Sub(other Set) Set { ts.m.RLock() defer ts.m.RUnlock() // If ts and other represent the same variable, avoid calling // ts.us.Sub(other), to avoid double RLock bug if _other, ok := other.(*tsafeSet); ok { if _other == ts { usResult := NewUnsafeSet() return &tsafeSet{usResult, sync.RWMutex{}} } } usResult := ts.us.Sub(other).(*unsafeSet) return &tsafeSet{usResult, sync.RWMutex{}} } ================================================ FILE: client/pkg/types/set_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package types import ( "reflect" "sort" "testing" "github.com/stretchr/testify/require" ) func TestUnsafeSet(t *testing.T) { driveSetTests(t, NewUnsafeSet()) } func TestThreadsafeSet(t *testing.T) { driveSetTests(t, NewThreadsafeSet()) } // Check that two slices contents are equal; order is irrelevant func equal(a, b []string) bool { as := sort.StringSlice(a) bs := sort.StringSlice(b) as.Sort() bs.Sort() return reflect.DeepEqual(as, bs) } func driveSetTests(t *testing.T, s Set) { t.Helper() // Verify operations on an empty set values := s.Values() require.Emptyf(t, values, "Expect values=%v got %v", []string{}, values) l := s.Length() require.Equalf(t, 0, l, "Expected length=0, got %d", l) for _, v := range []string{"foo", "bar", "baz"} { require.Falsef(t, s.Contains(v), "Expect s.Contains(%q) to be fale, got true", v) } // Add three items, ensure they show up s.Add("foo") s.Add("bar") s.Add("baz") eValues := []string{"foo", "bar", "baz"} values = s.Values() require.Truef(t, equal(values, eValues), "Expect values=%v got %v", eValues, values) for _, v := range eValues { require.Truef(t, s.Contains(v), "Expect s.Contains(%q) to be true, got false", v) } l = s.Length() require.Equalf(t, 3, l, "Expected length=3, got %d", l) // Add the same item a second time, ensuring it is not duplicated s.Add("foo") values = s.Values() require.Truef(t, equal(values, eValues), "Expect values=%v got %v", eValues, values) l = s.Length() require.Equalf(t, 3, l, "Expected length=3, got %d", l) // Remove all items, ensure they are gone s.Remove("foo") s.Remove("bar") s.Remove("baz") eValues = []string{} values = s.Values() require.Truef(t, equal(values, eValues), "Expect values=%v got %v", eValues, values) l = s.Length() require.Equalf(t, 0, l, "Expected length=0, got %d", l) // Create new copies of the set, and ensure they are unlinked to the // original Set by making modifications s.Add("foo") s.Add("bar") cp1 := s.Copy() cp2 := s.Copy() s.Remove("foo") cp3 := s.Copy() cp1.Add("baz") for i, tt := range []struct { want []string got []string }{ {[]string{"bar"}, s.Values()}, {[]string{"foo", "bar", "baz"}, cp1.Values()}, {[]string{"foo", "bar"}, cp2.Values()}, {[]string{"bar"}, cp3.Values()}, } { require.Truef(t, equal(tt.want, tt.got), "case %d: expect values=%v got %v", i, tt.want, tt.got) } for i, tt := range []struct { want bool got bool }{ {true, s.Equals(cp3)}, {true, cp3.Equals(s)}, {false, s.Equals(cp2)}, {false, s.Equals(cp1)}, {false, cp1.Equals(s)}, {false, cp2.Equals(s)}, {false, cp2.Equals(cp1)}, } { require.Equalf(t, tt.want, tt.got, "case %d: want %t, got %t", i, tt.want, tt.got) } // Subtract values from a Set, ensuring a new Set is created and // the original Sets are unmodified sub1 := cp1.Sub(s) sub2 := cp2.Sub(cp1) for i, tt := range []struct { want []string got []string }{ {[]string{"foo", "bar", "baz"}, cp1.Values()}, {[]string{"foo", "bar"}, cp2.Values()}, {[]string{"bar"}, s.Values()}, {[]string{"foo", "baz"}, sub1.Values()}, {[]string{}, sub2.Values()}, } { require.Truef(t, equal(tt.want, tt.got), "case %d: expect values=%v got %v", i, tt.want, tt.got) } } func TestUnsafeSetContainsAll(t *testing.T) { vals := []string{"foo", "bar", "baz"} s := NewUnsafeSet(vals...) tests := []struct { strs []string wcontain bool }{ {[]string{}, true}, {vals[:1], true}, {vals[:2], true}, {vals, true}, {[]string{"cuz"}, false}, {[]string{vals[0], "cuz"}, false}, } for i, tt := range tests { if g := s.ContainsAll(tt.strs); g != tt.wcontain { t.Errorf("#%d: ok = %v, want %v", i, g, tt.wcontain) } } } ================================================ FILE: client/pkg/types/slice.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package types // Uint64Slice implements sort interface type Uint64Slice []uint64 func (p Uint64Slice) Len() int { return len(p) } func (p Uint64Slice) Less(i, j int) bool { return p[i] < p[j] } func (p Uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } ================================================ FILE: client/pkg/types/slice_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package types import ( "reflect" "sort" "testing" ) func TestUint64Slice(t *testing.T) { g := Uint64Slice{10, 500, 5, 1, 100, 25} w := Uint64Slice{1, 5, 10, 25, 100, 500} sort.Sort(g) if !reflect.DeepEqual(g, w) { t.Errorf("slice after sort = %#v, want %#v", g, w) } } ================================================ FILE: client/pkg/types/urls.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package types import ( "errors" "fmt" "net" "net/url" "sort" "strings" ) type URLs []url.URL func NewURLs(strs []string) (URLs, error) { all := make([]url.URL, len(strs)) if len(all) == 0 { return nil, errors.New("no valid URLs given") } for i, in := range strs { in = strings.TrimSpace(in) u, err := url.Parse(in) if err != nil { return nil, err } switch u.Scheme { case "http", "https": if _, _, err := net.SplitHostPort(u.Host); err != nil { return nil, fmt.Errorf(`URL address does not have the form "host:port": %s`, in) } if u.Path != "" { return nil, fmt.Errorf("URL must not contain a path: %s", in) } case "unix", "unixs": default: return nil, fmt.Errorf("URL scheme must be http, https, unix, or unixs: %s", in) } all[i] = *u } us := URLs(all) us.Sort() return us, nil } func MustNewURLs(strs []string) URLs { urls, err := NewURLs(strs) if err != nil { panic(err) } return urls } func (us URLs) String() string { return strings.Join(us.StringSlice(), ",") } func (us *URLs) Sort() { sort.Sort(us) } func (us URLs) Len() int { return len(us) } func (us URLs) Less(i, j int) bool { return us[i].String() < us[j].String() } func (us URLs) Swap(i, j int) { us[i], us[j] = us[j], us[i] } func (us URLs) StringSlice() []string { out := make([]string, len(us)) for i := range us { out[i] = us[i].String() } return out } ================================================ FILE: client/pkg/types/urls_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package types import ( "reflect" "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func TestNewURLs(t *testing.T) { tests := []struct { strs []string wurls URLs }{ { []string{"http://127.0.0.1:2379"}, testutil.MustNewURLs(t, []string{"http://127.0.0.1:2379"}), }, // it can trim space { []string{" http://127.0.0.1:2379 "}, testutil.MustNewURLs(t, []string{"http://127.0.0.1:2379"}), }, // it does sort { []string{ "http://127.0.0.2:2379", "http://127.0.0.1:2379", }, testutil.MustNewURLs(t, []string{ "http://127.0.0.1:2379", "http://127.0.0.2:2379", }), }, } for i, tt := range tests { urls, _ := NewURLs(tt.strs) if !reflect.DeepEqual(urls, tt.wurls) { t.Errorf("#%d: urls = %+v, want %+v", i, urls, tt.wurls) } } } func TestURLsString(t *testing.T) { tests := []struct { us URLs wstr string }{ { URLs{}, "", }, { testutil.MustNewURLs(t, []string{"http://127.0.0.1:2379"}), "http://127.0.0.1:2379", }, { testutil.MustNewURLs(t, []string{ "http://127.0.0.1:2379", "http://127.0.0.2:2379", }), "http://127.0.0.1:2379,http://127.0.0.2:2379", }, { testutil.MustNewURLs(t, []string{ "http://127.0.0.2:2379", "http://127.0.0.1:2379", }), "http://127.0.0.2:2379,http://127.0.0.1:2379", }, } for i, tt := range tests { g := tt.us.String() if g != tt.wstr { t.Errorf("#%d: string = %s, want %s", i, g, tt.wstr) } } } func TestURLsSort(t *testing.T) { g := testutil.MustNewURLs(t, []string{ "http://127.0.0.4:2379", "http://127.0.0.2:2379", "http://127.0.0.1:2379", "http://127.0.0.3:2379", }) w := testutil.MustNewURLs(t, []string{ "http://127.0.0.1:2379", "http://127.0.0.2:2379", "http://127.0.0.3:2379", "http://127.0.0.4:2379", }) gurls := URLs(g) gurls.Sort() if !reflect.DeepEqual(g, w) { t.Errorf("URLs after sort = %#v, want %#v", g, w) } } func TestURLsStringSlice(t *testing.T) { tests := []struct { us URLs wstr []string }{ { URLs{}, []string{}, }, { testutil.MustNewURLs(t, []string{"http://127.0.0.1:2379"}), []string{"http://127.0.0.1:2379"}, }, { testutil.MustNewURLs(t, []string{ "http://127.0.0.1:2379", "http://127.0.0.2:2379", }), []string{"http://127.0.0.1:2379", "http://127.0.0.2:2379"}, }, { testutil.MustNewURLs(t, []string{ "http://127.0.0.2:2379", "http://127.0.0.1:2379", }), []string{"http://127.0.0.2:2379", "http://127.0.0.1:2379"}, }, } for i, tt := range tests { g := tt.us.StringSlice() if !reflect.DeepEqual(g, tt.wstr) { t.Errorf("#%d: string slice = %+v, want %+v", i, g, tt.wstr) } } } func TestNewURLsFail(t *testing.T) { tests := [][]string{ // no urls given {}, // missing protocol scheme {"://127.0.0.1:2379"}, // unsupported scheme {"mailto://127.0.0.1:2379"}, // not conform to host:port {"http://127.0.0.1"}, // contain a path {"http://127.0.0.1:2379/path"}, } for i, tt := range tests { _, err := NewURLs(tt) if err == nil { t.Errorf("#%d: err = nil, but error", i) } } } ================================================ FILE: client/pkg/types/urlsmap.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package types import ( "fmt" "sort" "strings" ) // URLsMap is a map from a name to its URLs. type URLsMap map[string]URLs // NewURLsMap returns a URLsMap instantiated from the given string, // which consists of discovery-formatted names-to-URLs, like: // mach0=http://1.1.1.1:2380,mach0=http://2.2.2.2::2380,mach1=http://3.3.3.3:2380,mach2=http://4.4.4.4:2380 func NewURLsMap(s string) (URLsMap, error) { m := parse(s) cl := URLsMap{} for name, urls := range m { us, err := NewURLs(urls) if err != nil { return nil, err } cl[name] = us } return cl, nil } // NewURLsMapFromStringMap takes a map of strings and returns a URLsMap. The // string values in the map can be multiple values separated by the sep string. func NewURLsMapFromStringMap(m map[string]string, sep string) (URLsMap, error) { var err error um := URLsMap{} for k, v := range m { um[k], err = NewURLs(strings.Split(v, sep)) if err != nil { return nil, err } } return um, nil } // String turns URLsMap into discovery-formatted name-to-URLs sorted by name. func (c URLsMap) String() string { var pairs []string for name, urls := range c { for _, url := range urls { pairs = append(pairs, fmt.Sprintf("%s=%s", name, url.String())) } } sort.Strings(pairs) return strings.Join(pairs, ",") } // URLs returns a list of all URLs. // The returned list is sorted in ascending lexicographical order. func (c URLsMap) URLs() []string { var urls []string for _, us := range c { for _, u := range us { urls = append(urls, u.String()) } } sort.Strings(urls) return urls } // Len returns the size of URLsMap. func (c URLsMap) Len() int { return len(c) } // parse parses the given string and returns a map listing the values specified for each key. func parse(s string) map[string][]string { m := make(map[string][]string) for s != "" { key := s if i := strings.IndexAny(key, ","); i >= 0 { key, s = key[:i], key[i+1:] } else { s = "" } if key == "" { continue } value := "" if i := strings.Index(key, "="); i >= 0 { key, value = key[:i], key[i+1:] } m[key] = append(m[key], value) } return m } ================================================ FILE: client/pkg/types/urlsmap_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package types import ( "reflect" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func TestParseInitialCluster(t *testing.T) { c, err := NewURLsMap("mem1=http://10.0.0.1:2379,mem1=http://128.193.4.20:2379,mem2=http://10.0.0.2:2379,default=http://127.0.0.1:2379") require.NoError(t, err) wc := URLsMap(map[string]URLs{ "mem1": testutil.MustNewURLs(t, []string{"http://10.0.0.1:2379", "http://128.193.4.20:2379"}), "mem2": testutil.MustNewURLs(t, []string{"http://10.0.0.2:2379"}), "default": testutil.MustNewURLs(t, []string{"http://127.0.0.1:2379"}), }) if !reflect.DeepEqual(c, wc) { t.Errorf("cluster = %+v, want %+v", c, wc) } } func TestParseInitialClusterBad(t *testing.T) { tests := []string{ // invalid URL "%^", // no URL defined for member "mem1=,mem2=http://128.193.4.20:2379,mem3=http://10.0.0.2:2379", "mem1,mem2=http://128.193.4.20:2379,mem3=http://10.0.0.2:2379", // bad URL for member "default=http://localhost/", } for i, tt := range tests { if _, err := NewURLsMap(tt); err == nil { t.Errorf("#%d: unexpected successful parse, want err", i) } } } func TestNameURLPairsString(t *testing.T) { cls := URLsMap(map[string]URLs{ "abc": testutil.MustNewURLs(t, []string{"http://1.1.1.1:1111", "http://0.0.0.0:0000"}), "def": testutil.MustNewURLs(t, []string{"http://2.2.2.2:2222"}), "ghi": testutil.MustNewURLs(t, []string{"http://3.3.3.3:1234", "http://127.0.0.1:2380"}), // no PeerURLs = not included "four": testutil.MustNewURLs(t, []string{}), "five": testutil.MustNewURLs(t, nil), }) w := "abc=http://0.0.0.0:0000,abc=http://1.1.1.1:1111,def=http://2.2.2.2:2222,ghi=http://127.0.0.1:2380,ghi=http://3.3.3.3:1234" g := cls.String() require.Equalf(t, g, w, "NameURLPairs.String():\ngot %#v\nwant %#v", g, w) } func TestParse(t *testing.T) { tests := []struct { s string wm map[string][]string }{ { "", map[string][]string{}, }, { "a=b", map[string][]string{"a": {"b"}}, }, { "a=b,a=c", map[string][]string{"a": {"b", "c"}}, }, { "a=b,a1=c", map[string][]string{"a": {"b"}, "a1": {"c"}}, }, } for i, tt := range tests { m := parse(tt.s) if !reflect.DeepEqual(m, tt.wm) { t.Errorf("#%d: m = %+v, want %+v", i, m, tt.wm) } } } // TestNewURLsMapIPV6 is only tested in Go1.5+ because Go1.4 doesn't support literal IPv6 address with zone in // URI (https://github.com/golang/go/issues/6530). func TestNewURLsMapIPV6(t *testing.T) { c, err := NewURLsMap("mem1=http://[2001:db8::1]:2380,mem1=http://[fe80::6e40:8ff:feb1:58e4%25en0]:2380,mem2=http://[fe80::92e2:baff:fe7c:3224%25ext0]:2380") require.NoError(t, err) wc := URLsMap(map[string]URLs{ "mem1": testutil.MustNewURLs(t, []string{"http://[2001:db8::1]:2380", "http://[fe80::6e40:8ff:feb1:58e4%25en0]:2380"}), "mem2": testutil.MustNewURLs(t, []string{"http://[fe80::92e2:baff:fe7c:3224%25ext0]:2380"}), }) if !reflect.DeepEqual(c, wc) { t.Errorf("cluster = %#v, want %#v", c, wc) } } func TestNewURLsMapFromStringMapEmpty(t *testing.T) { mss := make(map[string]string) urlsMap, err := NewURLsMapFromStringMap(mss, ",") if err != nil { t.Errorf("Unexpected error: %v", err) } s := "" um, err := NewURLsMap(s) if err != nil { t.Errorf("Unexpected error: %v", err) } if um.String() != urlsMap.String() { t.Errorf("Expected:\n%+v\ngot:\n%+v", um, urlsMap) } } func TestNewURLsMapFromStringMapNormal(t *testing.T) { mss := make(map[string]string) mss["host0"] = "http://127.0.0.1:2379,http://127.0.0.1:2380" mss["host1"] = "http://127.0.0.1:2381,http://127.0.0.1:2382" mss["host2"] = "http://127.0.0.1:2383,http://127.0.0.1:2384" mss["host3"] = "http://127.0.0.1:2385,http://127.0.0.1:2386" urlsMap, err := NewURLsMapFromStringMap(mss, ",") if err != nil { t.Errorf("Unexpected error: %v", err) } s := "host0=http://127.0.0.1:2379,host0=http://127.0.0.1:2380," + "host1=http://127.0.0.1:2381,host1=http://127.0.0.1:2382," + "host2=http://127.0.0.1:2383,host2=http://127.0.0.1:2384," + "host3=http://127.0.0.1:2385,host3=http://127.0.0.1:2386" um, err := NewURLsMap(s) if err != nil { t.Errorf("Unexpected error: %v", err) } if um.String() != urlsMap.String() { t.Errorf("Expected:\n%+v\ngot:\n%+v", um, urlsMap) } } ================================================ FILE: client/pkg/verify/verify.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package verify import ( "fmt" "os" "strings" ) const envVerify = "ETCD_VERIFY" type VerificationType string const ( envVerifyValueAll VerificationType = "all" envVerifyValueAssert VerificationType = "assert" ) func getEnvVerify() string { return strings.ToLower(os.Getenv(envVerify)) } func IsVerificationEnabled(verification VerificationType) bool { env := getEnvVerify() return env == string(envVerifyValueAll) || env == strings.ToLower(string(verification)) } // EnableVerifications sets `envVerify` and returns a function that // can be used to bring the original settings. func EnableVerifications(verification VerificationType) func() { previousEnv := getEnvVerify() os.Setenv(envVerify, string(verification)) return func() { os.Setenv(envVerify, previousEnv) } } // EnableAllVerifications enables verification and returns a function // that can be used to bring the original settings. func EnableAllVerifications() func() { return EnableVerifications(envVerifyValueAll) } // DisableVerifications unsets `envVerify` and returns a function that // can be used to bring the original settings. func DisableVerifications() func() { previousEnv := getEnvVerify() os.Unsetenv(envVerify) return func() { os.Setenv(envVerify, previousEnv) } } // Verify performs verification if the assertions are enabled. // In the default setup running in tests and skipped in the production code. func Verify(msg string, f VerifyFunc) { if IsVerificationEnabled(envVerifyValueAssert) { ok, details := f() verifier(ok, msg, details) } } type VerifyFunc func() (condition bool, details map[string]any) func verifier(condition bool, msg string, details map[string]any) { if !condition { panic(fmt.Sprintf("%s. details: %v.", msg, details)) } } // Assert will panic with a given formatted message if the given condition is false. func Assert(condition bool, msg string, v ...any) { if !condition { panic(fmt.Sprintf("assertion failed: "+msg, v...)) } } ================================================ FILE: client/v3/.gomodguard.yaml ================================================ --- blocked: modules: - go.etcd.io/etcd: reason: "Forbidden dependency" - go.etcd.io/etcd/pkg/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/server/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/tests/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/v3: reason: "Forbidden dependency" ================================================ FILE: client/v3/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: client/v3/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/clientv3 ================================================ FILE: client/v3/README.md ================================================ # etcd/client/v3 [![Docs](https://img.shields.io/badge/docs-latest-green.svg)](https://etcd.io/docs) [![Godoc](https://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/go.etcd.io/etcd/client/v3) `etcd/clientv3` is the official Go etcd client for v3. ## Install ```bash go get go.etcd.io/etcd/client/v3 ``` ## Get started Create client using `clientv3.New`: ```go import clientv3 "go.etcd.io/etcd/client/v3" func main() { cli, err := clientv3.New(clientv3.Config{ Endpoints: []string{"localhost:2379", "localhost:22379", "localhost:32379"}, DialTimeout: 5 * time.Second, }) if err != nil { // handle error! } defer cli.Close() } ``` etcd v3 uses [`gRPC`](https://www.grpc.io) for remote procedure calls. And `clientv3` uses [`grpc-go`](https://github.com/grpc/grpc-go) to connect to etcd. Make sure to close the client after using it. If the client is not closed, the connection will have leaky goroutines. To specify client request timeout, pass `context.WithTimeout` to APIs: ```go ctx, cancel := context.WithTimeout(context.Background(), timeout) resp, err := cli.Put(ctx, "sample_key", "sample_value") cancel() if err != nil { // handle error! } // use the response ``` For full compatibility, it is recommended to install released versions of clients using go modules. ## Error Handling etcd client returns 2 types of errors: 1. context error: canceled or deadline exceeded. 2. gRPC error: see [api/v3rpc/rpctypes](https://godoc.org/go.etcd.io/etcd/api/v3rpc/rpctypes). Here is the example code to handle client errors: ```go resp, err := cli.Put(ctx, "", "") if err != nil { switch err { case context.Canceled: log.Fatalf("ctx is canceled by another routine: %v", err) case context.DeadlineExceeded: log.Fatalf("ctx is attached with a deadline is exceeded: %v", err) case rpctypes.ErrEmptyKey: log.Fatalf("client-side error: %v", err) default: log.Fatalf("bad cluster endpoints, which are not etcd servers: %v", err) } } ``` ## Metrics The etcd client optionally exposes RPC metrics through [go-grpc-prometheus](https://github.com/grpc-ecosystem/go-grpc-prometheus). See the [examples](https://github.com/etcd-io/etcd/blob/main/tests/integration/clientv3/examples/example_metrics_test.go). ## Namespacing The [namespace](https://godoc.org/go.etcd.io/etcd/client/v3/namespace) package provides `clientv3` interface wrappers to transparently isolate client requests to a user-defined prefix. ## Request size limit Client request size limit is configurable via `clientv3.Config.MaxCallSendMsgSize` and `MaxCallRecvMsgSize` in bytes. If none given, client request send limit defaults to 2 MiB including gRPC overhead bytes. And receive limit defaults to `math.MaxInt32`. ## Examples More code [examples](https://github.com/etcd-io/etcd/tree/main/tests/integration/clientv3/examples) can be found at [GoDoc](https://pkg.go.dev/go.etcd.io/etcd/client/v3). ================================================ FILE: client/v3/auth.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "fmt" "strings" "google.golang.org/grpc" "go.etcd.io/etcd/api/v3/authpb" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) type ( AuthEnableResponse pb.AuthEnableResponse AuthDisableResponse pb.AuthDisableResponse AuthStatusResponse pb.AuthStatusResponse AuthenticateResponse pb.AuthenticateResponse AuthUserAddResponse pb.AuthUserAddResponse AuthUserDeleteResponse pb.AuthUserDeleteResponse AuthUserChangePasswordResponse pb.AuthUserChangePasswordResponse AuthUserGrantRoleResponse pb.AuthUserGrantRoleResponse AuthUserGetResponse pb.AuthUserGetResponse AuthUserRevokeRoleResponse pb.AuthUserRevokeRoleResponse AuthRoleAddResponse pb.AuthRoleAddResponse AuthRoleGrantPermissionResponse pb.AuthRoleGrantPermissionResponse AuthRoleGetResponse pb.AuthRoleGetResponse AuthRoleRevokePermissionResponse pb.AuthRoleRevokePermissionResponse AuthRoleDeleteResponse pb.AuthRoleDeleteResponse AuthUserListResponse pb.AuthUserListResponse AuthRoleListResponse pb.AuthRoleListResponse PermissionType authpb.Permission_Type Permission authpb.Permission ) const ( PermRead = authpb.Permission_READ PermWrite = authpb.Permission_WRITE PermReadWrite = authpb.Permission_READWRITE ) type UserAddOptions authpb.UserAddOptions type Auth interface { // Authenticate login and get token Authenticate(ctx context.Context, name string, password string) (*AuthenticateResponse, error) // AuthEnable enables auth of an etcd cluster. AuthEnable(ctx context.Context) (*AuthEnableResponse, error) // AuthDisable disables auth of an etcd cluster. AuthDisable(ctx context.Context) (*AuthDisableResponse, error) // AuthStatus returns the status of auth of an etcd cluster. AuthStatus(ctx context.Context) (*AuthStatusResponse, error) // UserAdd adds a new user to an etcd cluster. UserAdd(ctx context.Context, name string, password string) (*AuthUserAddResponse, error) // UserAddWithOptions adds a new user to an etcd cluster with some options. UserAddWithOptions(ctx context.Context, name string, password string, opt *UserAddOptions) (*AuthUserAddResponse, error) // UserDelete deletes a user from an etcd cluster. UserDelete(ctx context.Context, name string) (*AuthUserDeleteResponse, error) // UserChangePassword changes a password of a user. UserChangePassword(ctx context.Context, name string, password string) (*AuthUserChangePasswordResponse, error) // UserGrantRole grants a role to a user. UserGrantRole(ctx context.Context, user string, role string) (*AuthUserGrantRoleResponse, error) // UserGet gets a detailed information of a user. UserGet(ctx context.Context, name string) (*AuthUserGetResponse, error) // UserList gets a list of all users. UserList(ctx context.Context) (*AuthUserListResponse, error) // UserRevokeRole revokes a role of a user. UserRevokeRole(ctx context.Context, name string, role string) (*AuthUserRevokeRoleResponse, error) // RoleAdd adds a new role to an etcd cluster. RoleAdd(ctx context.Context, name string) (*AuthRoleAddResponse, error) // RoleGrantPermission grants a permission to a role. RoleGrantPermission(ctx context.Context, name string, key, rangeEnd string, permType PermissionType) (*AuthRoleGrantPermissionResponse, error) // RoleGet gets a detailed information of a role. RoleGet(ctx context.Context, role string) (*AuthRoleGetResponse, error) // RoleList gets a list of all roles. RoleList(ctx context.Context) (*AuthRoleListResponse, error) // RoleRevokePermission revokes a permission from a role. RoleRevokePermission(ctx context.Context, role string, key, rangeEnd string) (*AuthRoleRevokePermissionResponse, error) // RoleDelete deletes a role. RoleDelete(ctx context.Context, role string) (*AuthRoleDeleteResponse, error) } type authClient struct { remote pb.AuthClient callOpts []grpc.CallOption } func NewAuth(c *Client) Auth { api := &authClient{remote: RetryAuthClient(c)} if c != nil { api.callOpts = c.callOpts } return api } func NewAuthFromAuthClient(remote pb.AuthClient, c *Client) Auth { api := &authClient{remote: remote} if c != nil { api.callOpts = c.callOpts } return api } func (auth *authClient) Authenticate(ctx context.Context, name string, password string) (*AuthenticateResponse, error) { resp, err := auth.remote.Authenticate(ctx, &pb.AuthenticateRequest{Name: name, Password: password}, auth.callOpts...) return (*AuthenticateResponse)(resp), ContextError(ctx, err) } func (auth *authClient) AuthEnable(ctx context.Context) (*AuthEnableResponse, error) { resp, err := auth.remote.AuthEnable(ctx, &pb.AuthEnableRequest{}, auth.callOpts...) return (*AuthEnableResponse)(resp), ContextError(ctx, err) } func (auth *authClient) AuthDisable(ctx context.Context) (*AuthDisableResponse, error) { resp, err := auth.remote.AuthDisable(ctx, &pb.AuthDisableRequest{}, auth.callOpts...) return (*AuthDisableResponse)(resp), ContextError(ctx, err) } func (auth *authClient) AuthStatus(ctx context.Context) (*AuthStatusResponse, error) { resp, err := auth.remote.AuthStatus(ctx, &pb.AuthStatusRequest{}, auth.callOpts...) return (*AuthStatusResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserAdd(ctx context.Context, name string, password string) (*AuthUserAddResponse, error) { resp, err := auth.remote.UserAdd(ctx, &pb.AuthUserAddRequest{Name: name, Password: password, Options: &authpb.UserAddOptions{NoPassword: false}}, auth.callOpts...) return (*AuthUserAddResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserAddWithOptions(ctx context.Context, name string, password string, options *UserAddOptions) (*AuthUserAddResponse, error) { resp, err := auth.remote.UserAdd(ctx, &pb.AuthUserAddRequest{Name: name, Password: password, Options: (*authpb.UserAddOptions)(options)}, auth.callOpts...) return (*AuthUserAddResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserDelete(ctx context.Context, name string) (*AuthUserDeleteResponse, error) { resp, err := auth.remote.UserDelete(ctx, &pb.AuthUserDeleteRequest{Name: name}, auth.callOpts...) return (*AuthUserDeleteResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserChangePassword(ctx context.Context, name string, password string) (*AuthUserChangePasswordResponse, error) { resp, err := auth.remote.UserChangePassword(ctx, &pb.AuthUserChangePasswordRequest{Name: name, Password: password}, auth.callOpts...) return (*AuthUserChangePasswordResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserGrantRole(ctx context.Context, user string, role string) (*AuthUserGrantRoleResponse, error) { resp, err := auth.remote.UserGrantRole(ctx, &pb.AuthUserGrantRoleRequest{User: user, Role: role}, auth.callOpts...) return (*AuthUserGrantRoleResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserGet(ctx context.Context, name string) (*AuthUserGetResponse, error) { resp, err := auth.remote.UserGet(ctx, &pb.AuthUserGetRequest{Name: name}, auth.callOpts...) return (*AuthUserGetResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserList(ctx context.Context) (*AuthUserListResponse, error) { resp, err := auth.remote.UserList(ctx, &pb.AuthUserListRequest{}, auth.callOpts...) return (*AuthUserListResponse)(resp), ContextError(ctx, err) } func (auth *authClient) UserRevokeRole(ctx context.Context, name string, role string) (*AuthUserRevokeRoleResponse, error) { resp, err := auth.remote.UserRevokeRole(ctx, &pb.AuthUserRevokeRoleRequest{Name: name, Role: role}, auth.callOpts...) return (*AuthUserRevokeRoleResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleAdd(ctx context.Context, name string) (*AuthRoleAddResponse, error) { resp, err := auth.remote.RoleAdd(ctx, &pb.AuthRoleAddRequest{Name: name}, auth.callOpts...) return (*AuthRoleAddResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleGrantPermission(ctx context.Context, name string, key, rangeEnd string, permType PermissionType) (*AuthRoleGrantPermissionResponse, error) { perm := &authpb.Permission{ Key: []byte(key), RangeEnd: []byte(rangeEnd), PermType: authpb.Permission_Type(permType), } resp, err := auth.remote.RoleGrantPermission(ctx, &pb.AuthRoleGrantPermissionRequest{Name: name, Perm: perm}, auth.callOpts...) return (*AuthRoleGrantPermissionResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleGet(ctx context.Context, role string) (*AuthRoleGetResponse, error) { resp, err := auth.remote.RoleGet(ctx, &pb.AuthRoleGetRequest{Role: role}, auth.callOpts...) return (*AuthRoleGetResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleList(ctx context.Context) (*AuthRoleListResponse, error) { resp, err := auth.remote.RoleList(ctx, &pb.AuthRoleListRequest{}, auth.callOpts...) return (*AuthRoleListResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleRevokePermission(ctx context.Context, role string, key, rangeEnd string) (*AuthRoleRevokePermissionResponse, error) { resp, err := auth.remote.RoleRevokePermission(ctx, &pb.AuthRoleRevokePermissionRequest{Role: role, Key: []byte(key), RangeEnd: []byte(rangeEnd)}, auth.callOpts...) return (*AuthRoleRevokePermissionResponse)(resp), ContextError(ctx, err) } func (auth *authClient) RoleDelete(ctx context.Context, role string) (*AuthRoleDeleteResponse, error) { resp, err := auth.remote.RoleDelete(ctx, &pb.AuthRoleDeleteRequest{Role: role}, auth.callOpts...) return (*AuthRoleDeleteResponse)(resp), ContextError(ctx, err) } func StrToPermissionType(s string) (PermissionType, error) { val, ok := authpb.Permission_Type_value[strings.ToUpper(s)] if ok { return PermissionType(val), nil } return PermissionType(-1), fmt.Errorf("invalid permission type: %s", s) } ================================================ FILE: client/v3/client.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "errors" "fmt" "strings" "sync" "sync/atomic" "time" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" grpccredentials "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" healthpb "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/status" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/client/pkg/v3/verify" "go.etcd.io/etcd/client/v3/credentials" "go.etcd.io/etcd/client/v3/internal/endpoint" "go.etcd.io/etcd/client/v3/internal/resolver" ) var ( ErrNoAvailableEndpoints = errors.New("etcdclient: no available endpoints") ErrOldCluster = errors.New("etcdclient: old cluster version") ErrMutuallyExclusiveCfg = errors.New("Username/Password and Token configurations are mutually exclusive") ) // Client provides and manages an etcd v3 client session. type Client struct { Cluster KV Lease Watcher Auth Maintenance conn *grpc.ClientConn cfg Config creds grpccredentials.TransportCredentials resolver *resolver.EtcdManualResolver epMu *sync.RWMutex endpoints []string ctx context.Context cancel context.CancelFunc // Username is a user name for authentication. Username string // Password is a password for authentication. Password string // Token is a JWT used for authentication instead of a password. Token string authTokenBundle credentials.PerRPCCredentialsBundle callOpts []grpc.CallOption lg atomic.Pointer[zap.Logger] } // New creates a new etcdv3 client from a given configuration. func New(cfg Config) (*Client, error) { if len(cfg.Endpoints) == 0 { return nil, ErrNoAvailableEndpoints } return newClient(&cfg) } // NewCtxClient creates a client with a context but no underlying grpc // connection. This is useful for embedded cases that override the // service interface implementations and do not need connection management. func NewCtxClient(ctx context.Context, opts ...Option) *Client { cctx, cancel := context.WithCancel(ctx) c := &Client{ctx: cctx, cancel: cancel, epMu: new(sync.RWMutex)} for _, opt := range opts { opt(c) } if c.lg.Load() == nil { c.lg.Store(zap.NewNop()) } return c } // Option is a function type that can be passed as argument to NewCtxClient to configure client type Option func(*Client) // NewFromURL creates a new etcdv3 client from a URL. func NewFromURL(url string) (*Client, error) { return New(Config{Endpoints: []string{url}}) } // NewFromURLs creates a new etcdv3 client from URLs. func NewFromURLs(urls []string) (*Client, error) { return New(Config{Endpoints: urls}) } // WithZapLogger is a NewCtxClient option that overrides the logger func WithZapLogger(lg *zap.Logger) Option { return func(c *Client) { c.lg.Store(lg) } } // WithLogger overrides the logger. // // Deprecated: Please use WithZapLogger or Logger field in clientv3.Config // // Does not changes grpcLogger, that can be explicitly configured // using grpc_zap.ReplaceGrpcLoggerV2(..) method. func (c *Client) WithLogger(lg *zap.Logger) *Client { c.lg.Store(lg) return c } // GetLogger gets the logger. // NOTE: This method is for internal use of etcd-client library and should not be used as general-purpose logger. func (c *Client) GetLogger() *zap.Logger { return c.lg.Load() } // Close shuts down the client's etcd connections. func (c *Client) Close() error { c.cancel() if c.Watcher != nil { c.Watcher.Close() } if c.Lease != nil { c.Lease.Close() } if c.conn != nil { return ContextError(c.ctx, c.conn.Close()) } return c.ctx.Err() } // Ctx is a context for "out of band" messages (e.g., for sending // "clean up" message when another context is canceled). It is // canceled on client Close(). func (c *Client) Ctx() context.Context { return c.ctx } // Endpoints lists the registered endpoints for the client. func (c *Client) Endpoints() []string { // copy the slice; protect original endpoints from being changed c.epMu.RLock() defer c.epMu.RUnlock() eps := make([]string, len(c.endpoints)) copy(eps, c.endpoints) return eps } // SetEndpoints updates client's endpoints. func (c *Client) SetEndpoints(eps ...string) { c.epMu.Lock() defer c.epMu.Unlock() c.endpoints = eps c.resolver.SetEndpoints(eps) } // Sync synchronizes client's endpoints with the known endpoints from the etcd membership. func (c *Client) Sync(ctx context.Context) error { mresp, err := c.MemberList(ctx) if err != nil { return err } var eps []string for _, m := range mresp.Members { if len(m.Name) != 0 && !m.IsLearner { eps = append(eps, m.ClientURLs...) } } // The linearizable `MemberList` returned successfully, so the // endpoints shouldn't be empty. verify.Verify("empty endpoints returned from etcd cluster", func() (bool, map[string]any) { return len(eps) > 0, nil }) c.SetEndpoints(eps...) c.GetLogger().Debug("set etcd endpoints by autoSync", zap.Strings("endpoints", eps)) return nil } func (c *Client) autoSync() { if c.cfg.AutoSyncInterval == time.Duration(0) { return } for { select { case <-c.ctx.Done(): return case <-time.After(c.cfg.AutoSyncInterval): ctx, cancel := context.WithTimeout(c.ctx, 5*time.Second) err := c.Sync(ctx) cancel() if err != nil && !errors.Is(err, c.ctx.Err()) { c.GetLogger().Info("Auto sync endpoints failed.", zap.Error(err)) } } } } // dialSetupOpts gives the dial opts prior to any authentication. func (c *Client) dialSetupOpts(creds grpccredentials.TransportCredentials, dopts ...grpc.DialOption) []grpc.DialOption { var opts []grpc.DialOption if c.cfg.DialKeepAliveTime > 0 { params := keepalive.ClientParameters{ Time: c.cfg.DialKeepAliveTime, Timeout: c.cfg.DialKeepAliveTimeout, PermitWithoutStream: c.cfg.PermitWithoutStream, } opts = append(opts, grpc.WithKeepaliveParams(params)) } opts = append(opts, dopts...) if creds != nil { opts = append(opts, grpc.WithTransportCredentials(creds)) } else { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } unaryMaxRetries := defaultUnaryMaxRetries if c.cfg.MaxUnaryRetries > 0 { unaryMaxRetries = c.cfg.MaxUnaryRetries } backoffWaitBetween := defaultBackoffWaitBetween if c.cfg.BackoffWaitBetween > 0 { backoffWaitBetween = c.cfg.BackoffWaitBetween } backoffJitterFraction := defaultBackoffJitterFraction if c.cfg.BackoffJitterFraction > 0 { backoffJitterFraction = c.cfg.BackoffJitterFraction } // Interceptor retry and backoff. // TODO: Replace all of clientv3/retry.go with RetryPolicy: // https://github.com/grpc/grpc-proto/blob/cdd9ed5c3d3f87aef62f373b93361cf7bddc620d/grpc/service_config/service_config.proto#L130 rrBackoff := withBackoff(c.roundRobinQuorumBackoff(backoffWaitBetween, backoffJitterFraction)) opts = append(opts, // Disable stream retry by default since go-grpc-middleware/retry does not support client streams. // Streams that are safe to retry are enabled individually. grpc.WithStreamInterceptor(c.streamClientInterceptor(withMax(0), rrBackoff)), grpc.WithUnaryInterceptor(c.unaryClientInterceptor(withMax(unaryMaxRetries), rrBackoff)), ) return opts } // Dial connects to a single endpoint using the client's config. func (c *Client) Dial(ep string) (*grpc.ClientConn, error) { creds := c.credentialsForEndpoint(ep) // Using ad-hoc created resolver, to guarantee only explicitly given // endpoint is used. return c.dial(creds, grpc.WithResolvers(resolver.New(ep))) } func (c *Client) getToken(ctx context.Context) error { var err error // return last error in a case of fail if c.Token != "" { c.authTokenBundle.UpdateAuthToken(c.Token) return nil } if c.Username == "" || c.Password == "" { return nil } resp, err := c.Auth.Authenticate(ctx, c.Username, c.Password) if err != nil { if errors.Is(err, rpctypes.ErrAuthNotEnabled) { c.authTokenBundle.UpdateAuthToken("") return nil } return err } c.authTokenBundle.UpdateAuthToken(resp.Token) return nil } // dialWithBalancer dials the client's current load balanced resolver group. The scheme of the host // of the provided endpoint determines the scheme used for all endpoints of the client connection. func (c *Client) dialWithBalancer(dopts ...grpc.DialOption) (*grpc.ClientConn, error) { creds := c.credentialsForEndpoint(c.Endpoints()[0]) opts := append(dopts, grpc.WithResolvers(c.resolver)) return c.dial(creds, opts...) } // dial configures and dials any grpc balancer target. func (c *Client) dial(creds grpccredentials.TransportCredentials, dopts ...grpc.DialOption) (*grpc.ClientConn, error) { opts := c.dialSetupOpts(creds, dopts...) if c.authTokenBundle != nil { opts = append(opts, grpc.WithPerRPCCredentials(c.authTokenBundle.PerRPCCredentials())) } opts = append(opts, c.cfg.DialOptions...) target := fmt.Sprintf("%s://%p/%s", resolver.Schema, c, authority(c.endpoints[0])) conn, err := grpc.NewClient(target, opts...) if err != nil { return nil, err } if dialTimeout := c.cfg.DialTimeout; dialTimeout > 0 { dctx, cancel := context.WithTimeout(c.ctx, dialTimeout) defer cancel() if err := waitForConnection(dctx, conn); err != nil { conn.Close() return nil, err } } return conn, nil } func waitForConnection(ctx context.Context, conn *grpc.ClientConn) error { cli := healthpb.NewHealthClient(conn) // Use WaitForReady to wait until the connection is ready. The health check // may return Unimplemented if the server does not expose the health endpoint, // or FailedPrecondition if the leader has not yet applied the configuration // change that enables it. In both cases, we can still treat the connection as // healthy enough to proceed. // // Use withMax to disable retrying on Unimplemented, so that we can // return the original error immediately. _, err := cli.Check(ctx, &healthpb.HealthCheckRequest{}, grpc.WaitForReady(true), withMax(0)) if err == nil { return nil } if cerr := ctx.Err(); cerr != nil { if serr, ok := status.FromError(err); ok && serr.Message() != "" { return fmt.Errorf("etcdclient: failed to connect to the etcd server: %s: %w", serr.Message(), cerr) } return fmt.Errorf("etcdclient: failed to connect to the etcd server: %w", cerr) } serr, ok := status.FromError(err) if ok { switch serr.Code() { case codes.Unimplemented, codes.FailedPrecondition: return nil } } return fmt.Errorf("etcdclient: failed to dial by invoking health endpoint: %w", err) } func authority(endpoint string) string { spl := strings.SplitN(endpoint, "://", 2) if len(spl) < 2 { if strings.HasPrefix(endpoint, "unix:") { return endpoint[len("unix:"):] } if strings.HasPrefix(endpoint, "unixs:") { return endpoint[len("unixs:"):] } return endpoint } return spl[1] } func (c *Client) credentialsForEndpoint(ep string) grpccredentials.TransportCredentials { r := endpoint.RequiresCredentials(ep) switch r { case endpoint.CredsDrop: return nil case endpoint.CredsOptional: return c.creds case endpoint.CredsRequire: if c.creds != nil { return c.creds } return credentials.NewTransportCredential(nil) default: panic(fmt.Errorf("unsupported CredsRequirement: %v", r)) } } func newClient(cfg *Config) (*Client, error) { if cfg == nil { cfg = &Config{} } var creds grpccredentials.TransportCredentials if cfg.TLS != nil { creds = credentials.NewTransportCredential(cfg.TLS) } if cfg.Token != "" && (cfg.Username != "" || cfg.Password != "") { return nil, ErrMutuallyExclusiveCfg } // use a temporary skeleton client to bootstrap first connection baseCtx := context.TODO() if cfg.Context != nil { baseCtx = cfg.Context } ctx, cancel := context.WithCancel(baseCtx) client := &Client{ conn: nil, cfg: *cfg, creds: creds, ctx: ctx, cancel: cancel, epMu: new(sync.RWMutex), callOpts: defaultCallOpts, } var err error var lg *zap.Logger if cfg.Logger != nil { lg = cfg.Logger } else if cfg.LogConfig != nil { lg, err = cfg.LogConfig.Build() } else { lg, err = logutil.CreateDefaultZapLogger(ClientLogLevel()) if lg != nil { lg = lg.Named("etcd-client") } } if err != nil { return nil, err } client.lg.Store(lg) if cfg.Username != "" && cfg.Password != "" { client.Username = cfg.Username client.Password = cfg.Password client.authTokenBundle = credentials.NewPerRPCCredentialBundle() } if cfg.Token != "" { client.Token = cfg.Token client.authTokenBundle = credentials.NewPerRPCCredentialBundle() } if cfg.MaxCallSendMsgSize > 0 || cfg.MaxCallRecvMsgSize > 0 { if cfg.MaxCallRecvMsgSize > 0 && cfg.MaxCallSendMsgSize > cfg.MaxCallRecvMsgSize { return nil, fmt.Errorf("gRPC message recv limit (%d bytes) must be greater than send limit (%d bytes)", cfg.MaxCallRecvMsgSize, cfg.MaxCallSendMsgSize) } callOpts := []grpc.CallOption{ defaultWaitForReady, defaultMaxCallSendMsgSize, defaultMaxCallRecvMsgSize, } if cfg.MaxCallSendMsgSize > 0 { callOpts[1] = grpc.MaxCallSendMsgSize(cfg.MaxCallSendMsgSize) } if cfg.MaxCallRecvMsgSize > 0 { callOpts[2] = grpc.MaxCallRecvMsgSize(cfg.MaxCallRecvMsgSize) } client.callOpts = callOpts } client.resolver = resolver.New(cfg.Endpoints...) if len(cfg.Endpoints) < 1 { client.cancel() return nil, errors.New("at least one Endpoint is required in client config") } client.SetEndpoints(cfg.Endpoints...) // Use a provided endpoint target so that for https:// without any tls config given, then // grpc will assume the certificate server name is the endpoint host. conn, err := client.dialWithBalancer() if err != nil { client.cancel() client.resolver.Close() // TODO: Error like `fmt.Errorf(dialing [%s] failed: %v, strings.Join(cfg.Endpoints, ";"), err)` would help with debugging a lot. return nil, err } client.conn = conn client.Cluster = NewCluster(client) client.KV = NewKV(client) client.Lease = NewLease(client) client.Watcher = NewWatcher(client) client.Auth = NewAuth(client) client.Maintenance = NewMaintenance(client) // get token with established connection ctx, cancel = client.ctx, func() {} if client.cfg.DialTimeout > 0 { ctx, cancel = context.WithTimeout(ctx, client.cfg.DialTimeout) } err = client.getToken(ctx) if err != nil { client.Close() cancel() // TODO: Consider fmt.Errorf("communicating with [%s] failed: %v", strings.Join(cfg.Endpoints, ";"), err) return nil, err } cancel() if cfg.RejectOldCluster { if err := client.checkVersion(); err != nil { client.Close() return nil, err } } go client.autoSync() return client, nil } // roundRobinQuorumBackoff retries against quorum between each backoff. // This is intended for use with a round robin load balancer. func (c *Client) roundRobinQuorumBackoff(waitBetween time.Duration, jitterFraction float64) backoffFunc { return func(attempt uint) time.Duration { // after each round robin across quorum, backoff for our wait between duration n := uint(len(c.Endpoints())) quorum := (n/2 + 1) if attempt%quorum == 0 { c.GetLogger().Debug("backoff", zap.Uint("attempt", attempt), zap.Uint("quorum", quorum), zap.Duration("waitBetween", waitBetween), zap.Float64("jitterFraction", jitterFraction)) return jitterUp(waitBetween, jitterFraction) } c.GetLogger().Debug("backoff skipped", zap.Uint("attempt", attempt), zap.Uint("quorum", quorum)) return 0 } } // minSupportedVersion returns the minimum version supported, which is the previous minor release. func minSupportedVersion() *semver.Version { ver := semver.Must(semver.NewVersion(version.Version)) // consider only major and minor version ver = &semver.Version{Major: ver.Major, Minor: ver.Minor} for i := range version.AllVersions { if version.AllVersions[i].Equal(*ver) { if i == 0 { return ver } return &version.AllVersions[i-1] } } panic("current version is not in the version list") } func (c *Client) checkVersion() (err error) { var wg sync.WaitGroup eps := c.Endpoints() errc := make(chan error, len(eps)) ctx, cancel := context.WithCancel(c.ctx) if c.cfg.DialTimeout > 0 { cancel() ctx, cancel = context.WithTimeout(c.ctx, c.cfg.DialTimeout) } wg.Add(len(eps)) for _, ep := range eps { // if cluster is current, any endpoint gives a recent version go func(e string) { defer wg.Done() resp, rerr := c.Status(ctx, e) if rerr != nil { errc <- rerr return } vs, serr := semver.NewVersion(resp.Version) if serr != nil { errc <- serr return } if vs.LessThan(*minSupportedVersion()) { rerr = ErrOldCluster } errc <- rerr }(ep) } // wait for success for range eps { if err = <-errc; err != nil { break } } cancel() wg.Wait() return err } // ActiveConnection returns the current in-use connection func (c *Client) ActiveConnection() *grpc.ClientConn { return c.conn } // isHaltErr returns true if the given error and context indicate no forward // progress can be made, even after reconnecting. func isHaltErr(ctx context.Context, err error) bool { if ctx != nil && ctx.Err() != nil { return true } if err == nil { return false } ev, _ := status.FromError(err) // Unavailable codes mean the system will be right back. // (e.g., can't connect, lost leader) // Treat Internal codes as if something failed, leaving the // system in an inconsistent state, but retrying could make progress. // (e.g., failed in middle of send, corrupted frame) // TODO: are permanent Internal errors possible from grpc? return ev.Code() != codes.Unavailable && ev.Code() != codes.Internal } // isUnavailableErr returns true if the given error is an unavailable error func isUnavailableErr(ctx context.Context, err error) bool { if ctx != nil && ctx.Err() != nil { return false } if err == nil { return false } ev, ok := status.FromError(err) if ok { // Unavailable codes mean the system will be right back. // (e.g., can't connect, lost leader) return ev.Code() == codes.Unavailable } return false } // ContextError converts the error into an EtcdError if the error message matches one of // the defined messages; otherwise, it tries to retrieve the context error. func ContextError(ctx context.Context, err error) error { if err == nil { return nil } err = rpctypes.Error(err) var serverErr rpctypes.EtcdError if errors.As(err, &serverErr) { return err } if ev, ok := status.FromError(err); ok { code := ev.Code() switch code { case codes.DeadlineExceeded: fallthrough case codes.Canceled: if ctx.Err() != nil { err = ctx.Err() } } } return err } func canceledByCaller(stopCtx context.Context, err error) bool { if stopCtx.Err() == nil || err == nil { return false } return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) } // IsConnCanceled returns true, if error is from a closed gRPC connection. // ref. https://github.com/grpc/grpc-go/pull/1854 func IsConnCanceled(err error) bool { if err == nil { return false } // >= gRPC v1.23.x s, ok := status.FromError(err) if ok { // connection is canceled or server has already closed the connection return s.Code() == codes.Canceled || s.Message() == "transport is closing" } // >= gRPC v1.10.x if errors.Is(err, context.Canceled) { return true } // <= gRPC v1.7.x returns 'errors.New("grpc: the client connection is closing")' return strings.Contains(err.Error(), "grpc: the client connection is closing") } ================================================ FILE: client/v3/client_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "errors" "io" "net" "sync" "testing" "time" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" "google.golang.org/grpc" "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func NewClient(t *testing.T, cfg Config) (*Client, error) { t.Helper() if cfg.Logger == nil { cfg.Logger = zaptest.NewLogger(t).Named("client") } return New(cfg) } func TestDialNotImplemented(t *testing.T) { testutil.RegisterLeakDetection(t) ln, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) defer ln.Close() srv := grpc.NewServer() serveDone := make(chan error) go func() { defer close(serveDone) srv.Serve(ln) }() defer func() { srv.Stop() <-serveDone }() ep := ln.Addr().String() cfg := Config{ Endpoints: []string{ep}, DialTimeout: 10 * time.Second, } c, err := NewClient(t, cfg) require.NoError(t, err) defer c.Close() _, err = c.Get(t.Context(), "foo") require.ErrorContains(t, err, "code = Unimplemented desc = unknown service etcdserverpb.KV") } func TestDialCancel(t *testing.T) { testutil.RegisterLeakDetection(t) // Start a real gRPC endpoint with health service so initial dial readiness // check succeeds before switching endpoints below. ln, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) defer ln.Close() srv := grpc.NewServer() healthpb.RegisterHealthServer(srv, health.NewServer()) serveDone := make(chan error) go func() { defer close(serveDone) srv.Serve(ln) }() defer func() { srv.Stop() <-serveDone }() ep := ln.Addr().String() cfg := Config{ Endpoints: []string{ep}, DialTimeout: 30 * time.Second, } c, err := NewClient(t, cfg) require.NoError(t, err) // connect to ipv4 black hole so dial blocks c.SetEndpoints("http://254.0.0.1:12345") // issue Get to force redial attempts getc := make(chan struct{}) go func() { defer close(getc) // Get may hang forever on grpc's Stream.Header() if its // context is never canceled. c.Get(c.Ctx(), "abc") }() // wait a little bit so client close is after dial starts time.Sleep(100 * time.Millisecond) donec := make(chan struct{}) go func() { defer close(donec) c.Close() }() select { case <-time.After(5 * time.Second): t.Fatalf("failed to close") case <-donec: } select { case <-time.After(5 * time.Second): t.Fatalf("get failed to exit") case <-getc: } } func TestDialTimeout(t *testing.T) { testutil.RegisterLeakDetection(t) wantError := context.DeadlineExceeded testCfgs := []Config{ { Endpoints: []string{"http://254.0.0.1:12345"}, DialTimeout: 2 * time.Second, }, { Endpoints: []string{"http://254.0.0.1:12345"}, DialTimeout: time.Second, Username: "abc", Password: "def", }, } for i, cfg := range testCfgs { donec := make(chan error, 1) go func(cfg Config, i int) { // without timeout, dial continues forever on ipv4 black hole c, err := NewClient(t, cfg) if c != nil || err == nil { t.Errorf("#%d: new client should fail", i) } donec <- err }(cfg, i) time.Sleep(10 * time.Millisecond) select { case err := <-donec: t.Errorf("#%d: dial didn't wait (%v)", i, err) default: } select { case <-time.After(5 * time.Second): t.Errorf("#%d: failed to timeout dial on time", i) case err := <-donec: if !errors.Is(err, wantError) { t.Errorf("#%d: unexpected error '%v', want '%v'", i, err, wantError) } } } } func TestDialNoTimeout(t *testing.T) { cfg := Config{Endpoints: []string{"127.0.0.1:12345"}} c, err := NewClient(t, cfg) require.NotNilf(t, c, "new client with DialNoWait should succeed, got %v", err) require.NoErrorf(t, err, "new client with DialNoWait should succeed") c.Close() } func TestMaxUnaryRetries(t *testing.T) { maxUnaryRetries := uint(10) cfg := Config{ Endpoints: []string{"127.0.0.1:12345"}, MaxUnaryRetries: maxUnaryRetries, } c, err := NewClient(t, cfg) require.NoError(t, err) require.NotNil(t, c) defer c.Close() require.Equal(t, maxUnaryRetries, c.cfg.MaxUnaryRetries) } func TestBackoff(t *testing.T) { backoffWaitBetween := 100 * time.Millisecond cfg := Config{ Endpoints: []string{"127.0.0.1:12345"}, BackoffWaitBetween: backoffWaitBetween, } c, err := NewClient(t, cfg) require.NoError(t, err) require.NotNil(t, c) defer c.Close() require.Equal(t, backoffWaitBetween, c.cfg.BackoffWaitBetween) } func TestBackoffJitterFraction(t *testing.T) { backoffJitterFraction := float64(0.9) cfg := Config{ Endpoints: []string{"127.0.0.1:12345"}, BackoffJitterFraction: backoffJitterFraction, } c, err := NewClient(t, cfg) require.NoError(t, err) require.NotNil(t, c) defer c.Close() require.InDelta(t, backoffJitterFraction, c.cfg.BackoffJitterFraction, 0.01) } func TestIsHaltErr(t *testing.T) { assert.Truef(t, isHaltErr(t.Context(), errors.New("etcdserver: some etcdserver error")), "error created by errors.New should be unavailable error", ) assert.Falsef(t, isHaltErr(t.Context(), rpctypes.ErrGRPCStopped), `error "%v" should not be halt error`, rpctypes.ErrGRPCStopped, ) assert.Falsef(t, isHaltErr(t.Context(), rpctypes.ErrGRPCNoLeader), `error "%v" should not be halt error`, rpctypes.ErrGRPCNoLeader, ) ctx, cancel := context.WithCancel(t.Context()) assert.Falsef(t, isHaltErr(ctx, nil), "no error and active context should be halt error", ) cancel() assert.Truef(t, isHaltErr(ctx, nil), "cancel on context should be halt error", ) } func TestIsUnavailableErr(t *testing.T) { assert.Falsef(t, isUnavailableErr(t.Context(), errors.New("etcdserver: some etcdserver error")), "error created by errors.New should not be unavailable error", ) assert.Truef(t, isUnavailableErr(t.Context(), rpctypes.ErrGRPCStopped), `error "%v" should be unavailable error`, rpctypes.ErrGRPCStopped, ) assert.Falsef(t, isUnavailableErr(t.Context(), rpctypes.ErrGRPCNotCapable), "error %v should not be unavailable error", rpctypes.ErrGRPCNotCapable, ) ctx, cancel := context.WithCancel(t.Context()) assert.Falsef(t, isUnavailableErr(ctx, nil), "no error and active context should not be unavailable error", ) cancel() assert.Falsef(t, isUnavailableErr(ctx, nil), "cancel on context should not be unavailable error", ) } func TestCloseCtxClient(t *testing.T) { ctx := t.Context() c := NewCtxClient(ctx) err := c.Close() // Close returns ctx.toErr, a nil error means an open Done channel if err == nil { t.Errorf("failed to Close the client. %v", err) } } func TestWithLogger(t *testing.T) { ctx := t.Context() c := NewCtxClient(ctx) if c.lg.Load() == nil { t.Errorf("unexpected nil in *zap.Logger") } c.WithLogger(nil) if c.GetLogger() != nil { t.Errorf("WithLogger should modify *zap.Logger") } } func TestZapWithLogger(t *testing.T) { ctx := t.Context() lg := zap.NewNop() c := NewCtxClient(ctx, WithZapLogger(lg)) if c.GetLogger() != lg { t.Errorf("WithZapLogger should modify *zap.Logger") } } func TestAuthTokenBundleNoOverwrite(t *testing.T) { // This call in particular changes working directory to the tmp dir of // the test. The `etcd-auth-test:0` can be created in local directory, // not exceeding the longest allowed path on OsX. testutil.BeforeTest(t) // Create a mock AuthServer to handle Authenticate RPCs. lis, err := net.Listen("unix", "etcd-auth-test:0") require.NoError(t, err) defer lis.Close() addr := "unix://" + lis.Addr().String() srv := grpc.NewServer() etcdserverpb.RegisterAuthServer(srv, mockAuthServer{}) go srv.Serve(lis) defer srv.Stop() // Create a client, which should call Authenticate on the mock server to // exchange username/password for an auth token. c, err := NewClient(t, Config{ DialTimeout: 5 * time.Second, Endpoints: []string{addr}, Username: "foo", Password: "bar", }) require.NoError(t, err) defer c.Close() oldTokenBundle := c.authTokenBundle // Call the public Dial again, which should preserve the original // authTokenBundle. gc, err := c.Dial(addr) require.NoError(t, err) defer gc.Close() newTokenBundle := c.authTokenBundle if oldTokenBundle != newTokenBundle { t.Error("Client.authTokenBundle has been overwritten during Client.Dial") } } func TestNewWithOnlyJWT(t *testing.T) { // This call in particular changes working directory to the tmp dir of // the test. The `etcd-auth-test:1` can be created in local directory, // not exceeding the longest allowed path on OsX. testutil.BeforeTest(t) // Create a mock AuthServer to handle Authenticate RPCs. lis, err := net.Listen("unix", "etcd-auth-test:1") if err != nil { t.Fatal(err) } defer lis.Close() addr := "unix://" + lis.Addr().String() srv := grpc.NewServer() // Having a token removes the need to ever call Authenticate on the // server. If that happens then this will cause a connection failure. etcdserverpb.RegisterAuthServer(srv, mockFailingAuthServer{}) go srv.Serve(lis) defer srv.Stop() c, err := NewClient(t, Config{ DialTimeout: 5 * time.Second, Endpoints: []string{addr}, Token: "foo", }) if err != nil { t.Fatal(err) } defer c.Close() meta, err := c.authTokenBundle.PerRPCCredentials().GetRequestMetadata(t.Context(), "") if err != nil { t.Errorf("Error building request metadata: %s", err) } if tok, ok := meta[rpctypes.TokenFieldNameGRPC]; !ok { t.Error("Token was not successfully set in the auth bundle") } else if tok != "foo" { t.Errorf("Incorrect token set in auth bundle, got '%s', expected 'foo'", tok) } } func TestNewOnlyJWTExclusivity(t *testing.T) { testutil.BeforeTest(t) // Create a mock AuthServer to handle Authenticate RPCs. lis, err := net.Listen("unix", "etcd-auth-test:1") if err != nil { t.Fatal(err) } defer lis.Close() addr := "unix://" + lis.Addr().String() srv := grpc.NewServer() // Having a token removes the need to ever call Authenticate on the // server. If that happens then this will cause a connection failure. etcdserverpb.RegisterAuthServer(srv, mockFailingAuthServer{}) go srv.Serve(lis) defer srv.Stop() _, err = NewClient(t, Config{ DialTimeout: 5 * time.Second, Endpoints: []string{addr}, Token: "foo", Username: "user", Password: "pass", }) require.ErrorIs(t, ErrMutuallyExclusiveCfg, err) } func TestSyncFiltersMembers(t *testing.T) { c, _ := NewClient(t, Config{Endpoints: []string{"http://254.0.0.1:12345"}}) defer c.Close() c.Cluster = &mockCluster{ []*etcdserverpb.Member{ {ID: 0, Name: "", ClientURLs: []string{"http://254.0.0.1:12345"}, IsLearner: false}, {ID: 1, Name: "isStarted", ClientURLs: []string{"http://254.0.0.2:12345"}, IsLearner: true}, {ID: 2, Name: "isStartedAndNotLearner", ClientURLs: []string{"http://254.0.0.3:12345"}, IsLearner: false}, }, } c.Sync(t.Context()) endpoints := c.Endpoints() if len(endpoints) != 1 || endpoints[0] != "http://254.0.0.3:12345" { t.Error("Client.Sync uses learner and/or non-started member client URLs") } } func TestMinSupportedVersion(t *testing.T) { testutil.BeforeTest(t) tests := []struct { name string currentVersion semver.Version minSupportedVersion semver.Version }{ { name: "v3.6 client should accept v3.5", currentVersion: version.V3_6, minSupportedVersion: version.V3_5, }, { name: "v3.7 client should accept v3.6", currentVersion: version.V3_7, minSupportedVersion: version.V3_6, }, { name: "first minor version should accept its previous version", currentVersion: version.V4_0, minSupportedVersion: version.V3_7, }, { name: "first version in list should not accept previous versions", currentVersion: version.V3_0, minSupportedVersion: version.V3_0, }, } versionBackup := version.Version t.Cleanup(func() { version.Version = versionBackup }) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { version.Version = tt.currentVersion.String() require.True(t, minSupportedVersion().Equal(tt.minSupportedVersion)) }) } } func TestClientRejectOldCluster(t *testing.T) { testutil.BeforeTest(t) tests := []struct { name string endpoints []string versions []string expectedError error }{ { name: "all new versions with the same value", endpoints: []string{"192.168.3.41:22379", "192.168.3.41:22479", "192.168.3.41:22579"}, versions: []string{version.Version, version.Version, version.Version}, expectedError: nil, }, { name: "all new versions with different values", endpoints: []string{"192.168.3.41:22379", "192.168.3.41:22479", "192.168.3.41:22579"}, versions: []string{version.Version, minSupportedVersion().String(), minSupportedVersion().String()}, expectedError: nil, }, { name: "all old versions with different values", endpoints: []string{"192.168.3.41:22379", "192.168.3.41:22479", "192.168.3.41:22579"}, versions: []string{"3.3.0", "3.3.0", "3.4.0"}, expectedError: ErrOldCluster, }, { name: "all old versions with the same value", endpoints: []string{"192.168.3.41:22379", "192.168.3.41:22479", "192.168.3.41:22579"}, versions: []string{"3.3.0", "3.3.0", "3.3.0"}, expectedError: ErrOldCluster, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if len(tt.endpoints) != len(tt.versions) || len(tt.endpoints) == 0 { t.Errorf("Unexpected endpoints and versions length, len(endpoints):%d, len(versions):%d", len(tt.endpoints), len(tt.versions)) return } endpointToVersion := make(map[string]string) for j := range tt.endpoints { endpointToVersion[tt.endpoints[j]] = tt.versions[j] } c := &Client{ ctx: t.Context(), endpoints: tt.endpoints, epMu: new(sync.RWMutex), Maintenance: &mockMaintenance{ Version: endpointToVersion, }, } if err := c.checkVersion(); !errors.Is(err, tt.expectedError) { t.Errorf("checkVersion err:%v", err) } }) } } type mockMaintenance struct { Version map[string]string } func (mm mockMaintenance) Status(ctx context.Context, endpoint string) (*StatusResponse, error) { return &StatusResponse{Version: mm.Version[endpoint]}, nil } func (mm mockMaintenance) AlarmList(ctx context.Context) (*AlarmResponse, error) { return nil, nil } func (mm mockMaintenance) AlarmDisarm(ctx context.Context, m *AlarmMember) (*AlarmResponse, error) { return nil, nil } func (mm mockMaintenance) Defragment(ctx context.Context, endpoint string) (*DefragmentResponse, error) { return nil, nil } func (mm mockMaintenance) HashKV(ctx context.Context, endpoint string, rev int64) (*HashKVResponse, error) { return nil, nil } func (mm mockMaintenance) SnapshotWithVersion(ctx context.Context) (*SnapshotResponse, error) { return nil, nil } func (mm mockMaintenance) Snapshot(ctx context.Context) (io.ReadCloser, error) { return nil, nil } func (mm mockMaintenance) MoveLeader(ctx context.Context, transfereeID uint64) (*MoveLeaderResponse, error) { return nil, nil } func (mm mockMaintenance) Downgrade(ctx context.Context, action DowngradeAction, version string) (*DowngradeResponse, error) { return nil, nil } type mockFailingAuthServer struct { etcdserverpb.UnimplementedAuthServer } func (mockFailingAuthServer) Authenticate(context.Context, *etcdserverpb.AuthenticateRequest) (*etcdserverpb.AuthenticateResponse, error) { return nil, errors.New("this auth server always fails") } type mockAuthServer struct { etcdserverpb.UnimplementedAuthServer } func (mockAuthServer) Authenticate(context.Context, *etcdserverpb.AuthenticateRequest) (*etcdserverpb.AuthenticateResponse, error) { return &etcdserverpb.AuthenticateResponse{Token: "mock-token"}, nil } type mockCluster struct { members []*etcdserverpb.Member } func (mc *mockCluster) MemberList(ctx context.Context, opts ...OpOption) (*MemberListResponse, error) { return &MemberListResponse{Members: mc.members}, nil } func (mc *mockCluster) MemberAdd(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) { return nil, nil } func (mc *mockCluster) MemberAddAsLearner(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) { return nil, nil } func (mc *mockCluster) MemberRemove(ctx context.Context, id uint64) (*MemberRemoveResponse, error) { return nil, nil } func (mc *mockCluster) MemberUpdate(ctx context.Context, id uint64, peerAddrs []string) (*MemberUpdateResponse, error) { return nil, nil } func (mc *mockCluster) MemberPromote(ctx context.Context, id uint64) (*MemberPromoteResponse, error) { return nil, nil } ================================================ FILE: client/v3/clientv3util/example_key_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3util_test import ( "context" "log" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/clientv3util" ) func ExampleKeyMissing() { cli, err := clientv3.New(clientv3.Config{ Endpoints: []string{"127.0.0.1:2379"}, }) if err != nil { log.Fatal(err) } defer cli.Close() kvc := clientv3.NewKV(cli) // perform a put only if key is missing // It is useful to do the check atomically to avoid overwriting // the existing key which would generate potentially unwanted events, // unless of course you wanted to do an overwrite no matter what. _, err = kvc.Txn(context.Background()). If(clientv3util.KeyMissing("purpleidea")). Then(clientv3.OpPut("purpleidea", "hello world")). Commit() if err != nil { log.Fatal(err) } } func ExampleKeyExists() { cli, err := clientv3.New(clientv3.Config{ Endpoints: []string{"127.0.0.1:2379"}, }) if err != nil { log.Fatal(err) } defer cli.Close() kvc := clientv3.NewKV(cli) // perform a delete only if key already exists _, err = kvc.Txn(context.Background()). If(clientv3util.KeyExists("purpleidea")). Then(clientv3.OpDelete("purpleidea")). Commit() if err != nil { log.Fatal(err) } } ================================================ FILE: client/v3/clientv3util/util.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package clientv3util contains utility functions derived from clientv3. package clientv3util import ( clientv3 "go.etcd.io/etcd/client/v3" ) // KeyExists returns a comparison operation that evaluates to true iff the given // key exists. It does this by checking if the key `Version` is greater than 0. // It is a useful guard in transaction delete operations. func KeyExists(key string) clientv3.Cmp { return clientv3.Compare(clientv3.Version(key), ">", 0) } // KeyMissing returns a comparison operation that evaluates to true iff the // given key does not exist. func KeyMissing(key string) clientv3.Cmp { return clientv3.Compare(clientv3.Version(key), "=", 0) } ================================================ FILE: client/v3/cluster.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "google.golang.org/grpc" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/types" ) type ( Member pb.Member MemberListResponse pb.MemberListResponse MemberAddResponse pb.MemberAddResponse MemberRemoveResponse pb.MemberRemoveResponse MemberUpdateResponse pb.MemberUpdateResponse MemberPromoteResponse pb.MemberPromoteResponse ) type Cluster interface { // MemberList lists the current cluster membership. MemberList(ctx context.Context, opts ...OpOption) (*MemberListResponse, error) // MemberAdd adds a new member into the cluster. MemberAdd(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) // MemberAddAsLearner adds a new learner member into the cluster. MemberAddAsLearner(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) // MemberRemove removes an existing member from the cluster. MemberRemove(ctx context.Context, id uint64) (*MemberRemoveResponse, error) // MemberUpdate updates the peer addresses of the member. MemberUpdate(ctx context.Context, id uint64, peerAddrs []string) (*MemberUpdateResponse, error) // MemberPromote promotes a member from raft learner (non-voting) to raft voting member. MemberPromote(ctx context.Context, id uint64) (*MemberPromoteResponse, error) } type cluster struct { remote pb.ClusterClient callOpts []grpc.CallOption } func NewCluster(c *Client) Cluster { api := &cluster{remote: RetryClusterClient(c)} if c != nil { api.callOpts = c.callOpts } return api } func NewClusterFromClusterClient(remote pb.ClusterClient, c *Client) Cluster { api := &cluster{remote: remote} if c != nil { api.callOpts = c.callOpts } return api } func (c *cluster) MemberAdd(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) { return c.memberAdd(ctx, peerAddrs, false) } func (c *cluster) MemberAddAsLearner(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) { return c.memberAdd(ctx, peerAddrs, true) } func (c *cluster) memberAdd(ctx context.Context, peerAddrs []string, isLearner bool) (*MemberAddResponse, error) { // fail-fast before panic in rafthttp if _, err := types.NewURLs(peerAddrs); err != nil { return nil, err } r := &pb.MemberAddRequest{ PeerURLs: peerAddrs, IsLearner: isLearner, } resp, err := c.remote.MemberAdd(ctx, r, c.callOpts...) if err != nil { return nil, ContextError(ctx, err) } return (*MemberAddResponse)(resp), nil } func (c *cluster) MemberRemove(ctx context.Context, id uint64) (*MemberRemoveResponse, error) { r := &pb.MemberRemoveRequest{ID: id} resp, err := c.remote.MemberRemove(ctx, r, c.callOpts...) if err != nil { return nil, ContextError(ctx, err) } return (*MemberRemoveResponse)(resp), nil } func (c *cluster) MemberUpdate(ctx context.Context, id uint64, peerAddrs []string) (*MemberUpdateResponse, error) { // fail-fast before panic in rafthttp if _, err := types.NewURLs(peerAddrs); err != nil { return nil, err } // it is safe to retry on update. r := &pb.MemberUpdateRequest{ID: id, PeerURLs: peerAddrs} resp, err := c.remote.MemberUpdate(ctx, r, c.callOpts...) if err == nil { return (*MemberUpdateResponse)(resp), nil } return nil, ContextError(ctx, err) } func (c *cluster) MemberList(ctx context.Context, opts ...OpOption) (*MemberListResponse, error) { opt := OpGet("", opts...) resp, err := c.remote.MemberList(ctx, &pb.MemberListRequest{Linearizable: !opt.serializable}, c.callOpts...) if err == nil { return (*MemberListResponse)(resp), nil } return nil, ContextError(ctx, err) } func (c *cluster) MemberPromote(ctx context.Context, id uint64) (*MemberPromoteResponse, error) { r := &pb.MemberPromoteRequest{ID: id} resp, err := c.remote.MemberPromote(ctx, r, c.callOpts...) if err != nil { return nil, ContextError(ctx, err) } return (*MemberPromoteResponse)(resp), nil } ================================================ FILE: client/v3/compact_op.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) // CompactOp represents a compact operation. type CompactOp struct { revision int64 physical bool } // CompactOption configures compact operation. type CompactOption func(*CompactOp) func (op *CompactOp) applyCompactOpts(opts []CompactOption) { for _, opt := range opts { opt(op) } } // OpCompact wraps slice CompactOption to create a CompactOp. func OpCompact(rev int64, opts ...CompactOption) CompactOp { ret := CompactOp{revision: rev} ret.applyCompactOpts(opts) return ret } func (op CompactOp) toRequest() *pb.CompactionRequest { return &pb.CompactionRequest{Revision: op.revision, Physical: op.physical} } // WithCompactPhysical makes Compact wait until all compacted entries are // removed from the etcd server's storage. func WithCompactPhysical() CompactOption { return func(op *CompactOp) { op.physical = true } } ================================================ FILE: client/v3/compact_op_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "reflect" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/etcdserverpb" ) func TestCompactOp(t *testing.T) { req1 := OpCompact(100, WithCompactPhysical()).toRequest() req2 := &etcdserverpb.CompactionRequest{Revision: 100, Physical: true} require.Truef(t, reflect.DeepEqual(req1, req2), "expected %+v, got %+v", req2, req1) } ================================================ FILE: client/v3/compare.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) type ( CompareTarget int CompareResult int ) const ( CompareVersion CompareTarget = iota CompareCreated CompareModified CompareValue ) type Cmp pb.Compare func Compare(cmp Cmp, result string, v any) Cmp { var r pb.Compare_CompareResult switch result { case "=": r = pb.Compare_EQUAL case "!=": r = pb.Compare_NOT_EQUAL case ">": r = pb.Compare_GREATER case "<": r = pb.Compare_LESS default: panic("Unknown result op") } cmp.Result = r switch cmp.Target { case pb.Compare_VALUE: val, ok := v.(string) if !ok { panic("bad compare value") } cmp.TargetUnion = &pb.Compare_Value{Value: []byte(val)} case pb.Compare_VERSION: cmp.TargetUnion = &pb.Compare_Version{Version: mustInt64(v)} case pb.Compare_CREATE: cmp.TargetUnion = &pb.Compare_CreateRevision{CreateRevision: mustInt64(v)} case pb.Compare_MOD: cmp.TargetUnion = &pb.Compare_ModRevision{ModRevision: mustInt64(v)} case pb.Compare_LEASE: cmp.TargetUnion = &pb.Compare_Lease{Lease: mustInt64orLeaseID(v)} default: panic("Unknown compare type") } return cmp } func Value(key string) Cmp { return Cmp{Key: []byte(key), Target: pb.Compare_VALUE} } func Version(key string) Cmp { return Cmp{Key: []byte(key), Target: pb.Compare_VERSION} } func CreateRevision(key string) Cmp { return Cmp{Key: []byte(key), Target: pb.Compare_CREATE} } func ModRevision(key string) Cmp { return Cmp{Key: []byte(key), Target: pb.Compare_MOD} } // LeaseValue compares a key's LeaseID to a value of your choosing. The empty // LeaseID is 0, otherwise known as `NoLease`. func LeaseValue(key string) Cmp { return Cmp{Key: []byte(key), Target: pb.Compare_LEASE} } // KeyBytes returns the byte slice holding with the comparison key. func (cmp *Cmp) KeyBytes() []byte { return cmp.Key } // WithKeyBytes sets the byte slice for the comparison key. func (cmp *Cmp) WithKeyBytes(key []byte) { cmp.Key = key } // ValueBytes returns the byte slice holding the comparison value, if any. func (cmp *Cmp) ValueBytes() []byte { if tu, ok := cmp.TargetUnion.(*pb.Compare_Value); ok { return tu.Value } return nil } // WithValueBytes sets the byte slice for the comparison's value. func (cmp *Cmp) WithValueBytes(v []byte) { cmp.TargetUnion.(*pb.Compare_Value).Value = v } // WithRange sets the comparison to scan the range [key, end). func (cmp Cmp) WithRange(end string) Cmp { cmp.RangeEnd = []byte(end) return cmp } // WithPrefix sets the comparison to scan all keys prefixed by the key. func (cmp Cmp) WithPrefix() Cmp { cmp.RangeEnd = getPrefix(cmp.Key) return cmp } // mustInt64 panics if val isn't an int or int64. It returns an int64 otherwise. func mustInt64(val any) int64 { if v, ok := val.(int64); ok { return v } if v, ok := val.(int); ok { return int64(v) } panic("bad value") } // mustInt64orLeaseID panics if val isn't a LeaseID, int or int64. It returns an // int64 otherwise. func mustInt64orLeaseID(val any) int64 { if v, ok := val.(LeaseID); ok { return int64(v) } return mustInt64(val) } ================================================ FILE: client/v3/concurrency/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package concurrency implements concurrency operations on top of // etcd such as distributed locks, barriers, and elections. package concurrency ================================================ FILE: client/v3/concurrency/election.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency import ( "context" "errors" "fmt" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" v3 "go.etcd.io/etcd/client/v3" ) var ( ErrElectionNotLeader = errors.New("election: not leader") ErrElectionNoLeader = errors.New("election: no leader") ) type Election struct { session *Session keyPrefix string leaderKey string leaderRev int64 leaderSession *Session hdr *pb.ResponseHeader } // NewElection returns a new election on a given key prefix. func NewElection(s *Session, pfx string) *Election { return &Election{session: s, keyPrefix: pfx + "/"} } // ResumeElection initializes an election with a known leader. func ResumeElection(s *Session, pfx string, leaderKey string, leaderRev int64) *Election { return &Election{ keyPrefix: pfx, session: s, leaderKey: leaderKey, leaderRev: leaderRev, leaderSession: s, } } // Campaign puts a value as eligible for the election on the prefix // key. // Multiple sessions can participate in the election for the // same prefix, but only one can be the leader at a time. // // If the context is 'context.TODO()/context.Background()', the Campaign // will continue to be blocked for other keys to be deleted, unless server // returns a non-recoverable error (e.g. ErrCompacted). // Otherwise, until the context is not cancelled or timed-out, Campaign will // continue to be blocked until it becomes the leader. func (e *Election) Campaign(ctx context.Context, val string) error { s := e.session client := e.session.Client() k := fmt.Sprintf("%s%x", e.keyPrefix, s.Lease()) txn := client.Txn(ctx).If(v3.Compare(v3.CreateRevision(k), "=", 0)) txn = txn.Then(v3.OpPut(k, val, v3.WithLease(s.Lease()))) txn = txn.Else(v3.OpGet(k)) resp, err := txn.Commit() if err != nil { return err } e.leaderKey, e.leaderRev, e.leaderSession = k, resp.Header.Revision, s if !resp.Succeeded { kv := resp.Responses[0].GetResponseRange().Kvs[0] e.leaderRev = kv.CreateRevision if string(kv.Value) != val { if err = e.Proclaim(ctx, val); err != nil { e.Resign(ctx) return err } } } err = waitDeletes(ctx, client, e.keyPrefix, e.leaderRev-1) if err != nil { // clean up in case of context cancel select { case <-ctx.Done(): e.Resign(client.Ctx()) default: e.leaderSession = nil } return err } e.hdr = resp.Header return nil } // Proclaim lets the leader announce a new value without another election. func (e *Election) Proclaim(ctx context.Context, val string) error { if e.leaderSession == nil { return ErrElectionNotLeader } client := e.session.Client() cmp := v3.Compare(v3.CreateRevision(e.leaderKey), "=", e.leaderRev) txn := client.Txn(ctx).If(cmp) txn = txn.Then(v3.OpPut(e.leaderKey, val, v3.WithLease(e.leaderSession.Lease()))) tresp, terr := txn.Commit() if terr != nil { return terr } if !tresp.Succeeded { e.leaderKey = "" return ErrElectionNotLeader } e.hdr = tresp.Header return nil } // Resign lets a leader start a new election. func (e *Election) Resign(ctx context.Context) (err error) { if e.leaderSession == nil { return nil } client := e.session.Client() cmp := v3.Compare(v3.CreateRevision(e.leaderKey), "=", e.leaderRev) resp, err := client.Txn(ctx).If(cmp).Then(v3.OpDelete(e.leaderKey)).Commit() if err == nil { e.hdr = resp.Header } e.leaderKey = "" e.leaderSession = nil return err } // Leader returns the leader value for the current election. func (e *Election) Leader(ctx context.Context) (*v3.GetResponse, error) { client := e.session.Client() resp, err := client.Get(ctx, e.keyPrefix, v3.WithFirstCreate()...) if err != nil { return nil, err } else if len(resp.Kvs) == 0 { // no leader currently elected return nil, ErrElectionNoLeader } return resp, nil } // Observe returns a channel that reliably observes ordered leader proposals // as GetResponse values on every current elected leader key. It will not // necessarily fetch all historical leader updates, but will always post the // most recent leader value. // // The channel closes when the context is canceled or the underlying watcher // is otherwise disrupted. func (e *Election) Observe(ctx context.Context) <-chan v3.GetResponse { retc := make(chan v3.GetResponse) go e.observe(ctx, retc) return retc } func (e *Election) observe(ctx context.Context, ch chan<- v3.GetResponse) { client := e.session.Client() defer close(ch) for { resp, err := client.Get(ctx, e.keyPrefix, v3.WithFirstCreate()...) if err != nil { return } var kv *mvccpb.KeyValue var hdr *pb.ResponseHeader if len(resp.Kvs) == 0 { cctx, cancel := context.WithCancel(ctx) // wait for first key put on prefix opts := []v3.OpOption{v3.WithRev(resp.Header.Revision), v3.WithPrefix()} wch := client.Watch(cctx, e.keyPrefix, opts...) for kv == nil { wr, ok := <-wch if !ok || wr.Err() != nil { cancel() return } // only accept puts; a delete will make observe() spin for _, ev := range wr.Events { if ev.Type == mvccpb.Event_PUT { hdr, kv = &wr.Header, ev.Kv // may have multiple revs; hdr.rev = the last rev // set to kv's rev in case batch has multiple Puts hdr.Revision = kv.ModRevision break } } } cancel() } else { hdr, kv = resp.Header, resp.Kvs[0] } select { case ch <- v3.GetResponse{Header: hdr, Kvs: []*mvccpb.KeyValue{kv}}: case <-ctx.Done(): return } cctx, cancel := context.WithCancel(ctx) wch := client.Watch(cctx, string(kv.Key), v3.WithRev(hdr.Revision+1)) keyDeleted := false for !keyDeleted { wr, ok := <-wch if !ok { cancel() return } for _, ev := range wr.Events { if ev.Type == mvccpb.Event_DELETE { keyDeleted = true break } resp.Header = &wr.Header resp.Kvs = []*mvccpb.KeyValue{ev.Kv} select { case ch <- *resp: case <-cctx.Done(): cancel() return } } } cancel() } } // Key returns the leader key if elected, empty string otherwise. func (e *Election) Key() string { return e.leaderKey } // Rev returns the leader key's creation revision, if elected. func (e *Election) Rev() int64 { return e.leaderRev } // Header is the response header from the last successful election proposal. func (e *Election) Header() *pb.ResponseHeader { return e.hdr } ================================================ FILE: client/v3/concurrency/key.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency import ( "context" "errors" "go.etcd.io/etcd/api/v3/mvccpb" v3 "go.etcd.io/etcd/client/v3" ) func waitDelete(ctx context.Context, client *v3.Client, key string, rev int64) error { cctx, cancel := context.WithCancel(ctx) defer cancel() var wr v3.WatchResponse wch := client.Watch(cctx, key, v3.WithRev(rev)) for wr = range wch { for _, ev := range wr.Events { if ev.Type == mvccpb.Event_DELETE { return nil } } } if err := wr.Err(); err != nil { return err } if err := ctx.Err(); err != nil { return err } return errors.New("lost watcher waiting for delete") } // waitDeletes efficiently waits until all keys matching the prefix and no greater // than the create revision are deleted. func waitDeletes(ctx context.Context, client *v3.Client, pfx string, maxCreateRev int64) error { getOpts := append(v3.WithLastCreate(), v3.WithMaxCreateRev(maxCreateRev)) for { resp, err := client.Get(ctx, pfx, getOpts...) if err != nil { return err } if len(resp.Kvs) == 0 { return nil } lastKey := string(resp.Kvs[0].Key) if err = waitDelete(ctx, client, lastKey, resp.Header.Revision); err != nil { return err } } } ================================================ FILE: client/v3/concurrency/main_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency_test import ( "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func exampleEndpoints() []string { return nil } func forUnitTestsRunInMockedContext(mocking func(), _example func()) { mocking() // TODO: Call 'example' when mocking() provides realistic mocking of transport. // The real testing logic of examples gets executed // as part of ./tests/integration/clientv3/concurrency/... } func TestMain(m *testing.M) { testutil.MustTestMainWithLeakDetection(m) } ================================================ FILE: client/v3/concurrency/mutex.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency import ( "context" "errors" "fmt" "strings" "sync" pb "go.etcd.io/etcd/api/v3/etcdserverpb" v3 "go.etcd.io/etcd/client/v3" ) // ErrLocked is returned by TryLock when Mutex is already locked by another session. var ( ErrLocked = errors.New("mutex: Locked by another session") ErrSessionExpired = errors.New("mutex: session is expired") ErrLockReleased = errors.New("mutex: lock has already been released") ) // Mutex implements the sync Locker interface with etcd type Mutex struct { s *Session pfx string myKey string myRev int64 hdr *pb.ResponseHeader } func NewMutex(s *Session, pfx string) *Mutex { return &Mutex{s, pfx + "/", "", -1, nil} } // TryLock locks the mutex if not already locked by another session. // If lock is held by another session, return immediately after attempting necessary cleanup // The ctx argument is used for the sending/receiving Txn RPC. func (m *Mutex) TryLock(ctx context.Context) error { resp, err := m.tryAcquire(ctx) if err != nil { return err } // if no key on prefix / the minimum rev is key, already hold the lock ownerKey := resp.Responses[1].GetResponseRange().Kvs if len(ownerKey) == 0 || ownerKey[0].CreateRevision == m.myRev { m.hdr = resp.Header return nil } client := m.s.Client() // Cannot lock, so delete the key if _, err := client.Delete(ctx, m.myKey); err != nil { return err } m.myKey = "\x00" m.myRev = -1 return ErrLocked } // Lock locks the mutex with a cancelable context. If the context is canceled // while trying to acquire the lock, the mutex tries to clean its stale lock entry. func (m *Mutex) Lock(ctx context.Context) error { resp, err := m.tryAcquire(ctx) if err != nil { return err } // if no key on prefix / the minimum rev is key, already hold the lock ownerKey := resp.Responses[1].GetResponseRange().Kvs if len(ownerKey) == 0 || ownerKey[0].CreateRevision == m.myRev { m.hdr = resp.Header return nil } client := m.s.Client() // wait for deletion revisions prior to myKey // TODO: early termination if the session key is deleted before other session keys with smaller revisions. werr := waitDeletes(ctx, client, m.pfx, m.myRev-1) // release lock key if wait failed if werr != nil { m.Unlock(client.Ctx()) return werr } // make sure the session is not expired, and the owner key still exists. gresp, werr := client.Get(ctx, m.myKey) if werr != nil { m.Unlock(client.Ctx()) return werr } if len(gresp.Kvs) == 0 { // is the session key lost? return ErrSessionExpired } m.hdr = gresp.Header return nil } func (m *Mutex) tryAcquire(ctx context.Context) (*v3.TxnResponse, error) { s := m.s client := m.s.Client() m.myKey = fmt.Sprintf("%s%x", m.pfx, s.Lease()) cmp := v3.Compare(v3.CreateRevision(m.myKey), "=", 0) // put self in lock waiters via myKey; oldest waiter holds lock put := v3.OpPut(m.myKey, "", v3.WithLease(s.Lease())) // reuse key in case this session already holds the lock get := v3.OpGet(m.myKey) // fetch current holder to complete uncontended path with only one RPC getOwner := v3.OpGet(m.pfx, v3.WithFirstCreate()...) resp, err := client.Txn(ctx).If(cmp).Then(put, getOwner).Else(get, getOwner).Commit() if err != nil { return nil, err } m.myRev = resp.Header.Revision if !resp.Succeeded { m.myRev = resp.Responses[0].GetResponseRange().Kvs[0].CreateRevision } return resp, nil } func (m *Mutex) Unlock(ctx context.Context) error { if m.myKey == "" || m.myRev <= 0 || m.myKey == "\x00" { return ErrLockReleased } if !strings.HasPrefix(m.myKey, m.pfx) { return fmt.Errorf("invalid key %q, it should have prefix %q", m.myKey, m.pfx) } client := m.s.Client() if _, err := client.Delete(ctx, m.myKey); err != nil { return err } m.myKey = "\x00" m.myRev = -1 return nil } func (m *Mutex) IsOwner() v3.Cmp { return v3.Compare(v3.CreateRevision(m.myKey), "=", m.myRev) } func (m *Mutex) Key() string { return m.myKey } // Header is the response header received from etcd on acquiring the lock. func (m *Mutex) Header() *pb.ResponseHeader { return m.hdr } type lockerMutex struct{ *Mutex } func (lm *lockerMutex) Lock() { client := lm.s.Client() if err := lm.Mutex.Lock(client.Ctx()); err != nil { panic(err) } } func (lm *lockerMutex) Unlock() { client := lm.s.Client() if err := lm.Mutex.Unlock(client.Ctx()); err != nil { panic(err) } } // NewLocker creates a sync.Locker backed by an etcd mutex. func NewLocker(s *Session, pfx string) sync.Locker { return &lockerMutex{NewMutex(s, pfx)} } ================================================ FILE: client/v3/concurrency/session.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency import ( "context" "time" "go.uber.org/zap" v3 "go.etcd.io/etcd/client/v3" ) const defaultSessionTTL = 60 // Session represents a lease kept alive for the lifetime of a client. // Fault-tolerant applications may use sessions to reason about liveness. type Session struct { client *v3.Client opts *sessionOptions id v3.LeaseID ctx context.Context cancel context.CancelFunc donec <-chan struct{} } // NewSession gets the leased session for a client. func NewSession(client *v3.Client, opts ...SessionOption) (*Session, error) { lg := client.GetLogger() ops := &sessionOptions{ttl: defaultSessionTTL, ctx: client.Ctx()} for _, opt := range opts { opt(ops, lg) } id := ops.leaseID if id == v3.NoLease { resp, err := client.Grant(ops.ctx, int64(ops.ttl)) if err != nil { return nil, err } id = resp.ID } ctx, cancel := context.WithCancel(ops.ctx) keepAlive, err := client.KeepAlive(ctx, id) if err != nil || keepAlive == nil { cancel() return nil, err } donec := make(chan struct{}) s := &Session{client: client, opts: ops, id: id, ctx: ctx, cancel: cancel, donec: donec} // keep the lease alive until client error or cancelled context go func() { defer func() { close(donec) cancel() }() for range keepAlive { // eat messages until keep alive channel closes } }() return s, nil } // Client is the etcd client that is attached to the session. func (s *Session) Client() *v3.Client { return s.client } // Lease is the lease ID for keys bound to the session. func (s *Session) Lease() v3.LeaseID { return s.id } // Ctx is the context attached to the session, it is canceled when the lease is orphaned, expires, or // is otherwise no longer being refreshed. func (s *Session) Ctx() context.Context { return s.ctx } // Done returns a channel that closes when the lease is orphaned, expires, or // is otherwise no longer being refreshed. func (s *Session) Done() <-chan struct{} { return s.donec } // Orphan ends the refresh for the session lease. This is useful // in case the state of the client connection is indeterminate (revoke // would fail) or when transferring lease ownership. func (s *Session) Orphan() { s.cancel() <-s.donec } // Close orphans the session and revokes the session lease. func (s *Session) Close() error { s.Orphan() // if revoke takes longer than the ttl, lease is expired anyway ctx, cancel := context.WithTimeout(s.opts.ctx, time.Duration(s.opts.ttl)*time.Second) _, err := s.client.Revoke(ctx, s.id) cancel() return err } type sessionOptions struct { ttl int leaseID v3.LeaseID ctx context.Context } // SessionOption configures Session. type SessionOption func(*sessionOptions, *zap.Logger) // WithTTL configures the session's TTL in seconds. // If TTL is <= 0, the default 60 seconds TTL will be used. func WithTTL(ttl int) SessionOption { return func(so *sessionOptions, lg *zap.Logger) { if ttl > 0 { so.ttl = ttl } else { lg.Warn("WithTTL(): TTL should be > 0, preserving current TTL", zap.Int64("current-session-ttl", int64(so.ttl))) } } } // WithLease specifies the existing leaseID to be used for the session. // This is useful in process restart scenario, for example, to reclaim // leadership from an election prior to restart. func WithLease(leaseID v3.LeaseID) SessionOption { return func(so *sessionOptions, _ *zap.Logger) { so.leaseID = leaseID } } // WithContext assigns a context to the session instead of defaulting to // using the client context. This is useful for canceling NewSession and // Close operations immediately without having to close the client. If the // context is canceled before Close() completes, the session's lease will be // abandoned and left to expire instead of being revoked. func WithContext(ctx context.Context) SessionOption { return func(so *sessionOptions, _ *zap.Logger) { so.ctx = ctx } } ================================================ FILE: client/v3/concurrency/stm.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency import ( "context" "math" v3 "go.etcd.io/etcd/client/v3" ) // STM is an interface for software transactional memory. type STM interface { // Get returns the value for a key and inserts the key in the txn's read set. // If Get fails, it aborts the transaction with an error, never returning. Get(key ...string) string // Put adds a value for a key to the write set. Put(key, val string, opts ...v3.OpOption) // Rev returns the revision of a key in the read set. Rev(key string) int64 // Del deletes a key. Del(key string) // commit attempts to apply the txn's changes to the server. commit() *v3.TxnResponse reset() } // Isolation is an enumeration of transactional isolation levels which // describes how transactions should interfere and conflict. type Isolation int const ( // SerializableSnapshot provides serializable isolation and also checks // for write conflicts. SerializableSnapshot Isolation = iota // Serializable reads within the same transaction attempt return data // from the revision of the first read. Serializable // RepeatableReads reads within the same transaction attempt always // return the same data. RepeatableReads // ReadCommitted reads keys from any committed revision. ReadCommitted ) // stmError safely passes STM errors through panic to the STM error channel. type stmError struct{ err error } type stmOptions struct { iso Isolation ctx context.Context prefetch []string } type stmOption func(*stmOptions) // WithIsolation specifies the transaction isolation level. func WithIsolation(lvl Isolation) stmOption { return func(so *stmOptions) { so.iso = lvl } } // WithAbortContext specifies the context for permanently aborting the transaction. func WithAbortContext(ctx context.Context) stmOption { return func(so *stmOptions) { so.ctx = ctx } } // WithPrefetch is a hint to prefetch a list of keys before trying to apply. // If an STM transaction will unconditionally fetch a set of keys, prefetching // those keys will save the round-trip cost from requesting each key one by one // with Get(). func WithPrefetch(keys ...string) stmOption { return func(so *stmOptions) { so.prefetch = append(so.prefetch, keys...) } } // NewSTM initiates a new STM instance, using serializable snapshot isolation by default. func NewSTM(c *v3.Client, apply func(STM) error, so ...stmOption) (*v3.TxnResponse, error) { opts := &stmOptions{ctx: c.Ctx()} for _, f := range so { f(opts) } if len(opts.prefetch) != 0 { f := apply apply = func(s STM) error { s.Get(opts.prefetch...) return f(s) } } return runSTM(mkSTM(c, opts), apply) } func mkSTM(c *v3.Client, opts *stmOptions) STM { switch opts.iso { case SerializableSnapshot: s := &stmSerializable{ stm: stm{client: c, ctx: opts.ctx}, prefetch: make(map[string]*v3.GetResponse), } s.conflicts = func() []v3.Cmp { return append(s.rset.cmps(), s.wset.cmps(s.rset.first()+1)...) } return s case Serializable: s := &stmSerializable{ stm: stm{client: c, ctx: opts.ctx}, prefetch: make(map[string]*v3.GetResponse), } s.conflicts = func() []v3.Cmp { return s.rset.cmps() } return s case RepeatableReads: s := &stm{client: c, ctx: opts.ctx, getOpts: []v3.OpOption{v3.WithSerializable()}} s.conflicts = func() []v3.Cmp { return s.rset.cmps() } return s case ReadCommitted: s := &stm{client: c, ctx: opts.ctx, getOpts: []v3.OpOption{v3.WithSerializable()}} s.conflicts = func() []v3.Cmp { return nil } return s default: panic("unsupported stm") } } type stmResponse struct { resp *v3.TxnResponse err error } func runSTM(s STM, apply func(STM) error) (*v3.TxnResponse, error) { outc := make(chan stmResponse, 1) go func() { defer func() { if r := recover(); r != nil { e, ok := r.(stmError) if !ok { // client apply panicked panic(r) } outc <- stmResponse{nil, e.err} } }() var out stmResponse for { s.reset() if out.err = apply(s); out.err != nil { break } if out.resp = s.commit(); out.resp != nil { break } } outc <- out }() r := <-outc return r.resp, r.err } // stm implements repeatable-read software transactional memory over etcd type stm struct { client *v3.Client ctx context.Context // rset holds read key values and revisions rset readSet // wset holds overwritten keys and their values wset writeSet // getOpts are the opts used for gets getOpts []v3.OpOption // conflicts computes the current conflicts on the txn conflicts func() []v3.Cmp } type stmPut struct { val string op v3.Op } type readSet map[string]*v3.GetResponse func (rs readSet) add(keys []string, txnresp *v3.TxnResponse) { for i, resp := range txnresp.Responses { rs[keys[i]] = (*v3.GetResponse)(resp.GetResponseRange()) } } // first returns the store revision from the first fetch func (rs readSet) first() int64 { ret := int64(math.MaxInt64 - 1) for _, resp := range rs { if rev := resp.Header.Revision; rev < ret { ret = rev } } return ret } // cmps guards the txn from updates to read set func (rs readSet) cmps() []v3.Cmp { cmps := make([]v3.Cmp, 0, len(rs)) for k, rk := range rs { cmps = append(cmps, isKeyCurrent(k, rk)) } return cmps } type writeSet map[string]stmPut func (ws writeSet) get(keys ...string) *stmPut { for _, key := range keys { if wv, ok := ws[key]; ok { return &wv } } return nil } // cmps returns a cmp list testing no writes have happened past rev func (ws writeSet) cmps(rev int64) []v3.Cmp { cmps := make([]v3.Cmp, 0, len(ws)) for key := range ws { cmps = append(cmps, v3.Compare(v3.ModRevision(key), "<", rev)) } return cmps } // puts is the list of ops for all pending writes func (ws writeSet) puts() []v3.Op { puts := make([]v3.Op, 0, len(ws)) for _, v := range ws { puts = append(puts, v.op) } return puts } func (s *stm) Get(keys ...string) string { if wv := s.wset.get(keys...); wv != nil { return wv.val } return respToValue(s.fetch(keys...)) } func (s *stm) Put(key, val string, opts ...v3.OpOption) { s.wset[key] = stmPut{val, v3.OpPut(key, val, opts...)} } func (s *stm) Del(key string) { s.wset[key] = stmPut{"", v3.OpDelete(key)} } func (s *stm) Rev(key string) int64 { if resp := s.fetch(key); resp != nil && len(resp.Kvs) != 0 { return resp.Kvs[0].ModRevision } return 0 } func (s *stm) commit() *v3.TxnResponse { txnresp, err := s.client.Txn(s.ctx).If(s.conflicts()...).Then(s.wset.puts()...).Commit() if err != nil { panic(stmError{err}) } if txnresp.Succeeded { return txnresp } return nil } func (s *stm) fetch(keys ...string) *v3.GetResponse { if len(keys) == 0 { return nil } ops := make([]v3.Op, len(keys)) for i, key := range keys { if resp, ok := s.rset[key]; ok { return resp } ops[i] = v3.OpGet(key, s.getOpts...) } txnresp, err := s.client.Txn(s.ctx).Then(ops...).Commit() if err != nil { panic(stmError{err}) } s.rset.add(keys, txnresp) return (*v3.GetResponse)(txnresp.Responses[0].GetResponseRange()) } func (s *stm) reset() { s.rset = make(map[string]*v3.GetResponse) s.wset = make(map[string]stmPut) } type stmSerializable struct { stm prefetch map[string]*v3.GetResponse } func (s *stmSerializable) Get(keys ...string) string { if len(keys) == 0 { return "" } if wv := s.wset.get(keys...); wv != nil { return wv.val } firstRead := len(s.rset) == 0 for _, key := range keys { if resp, ok := s.prefetch[key]; ok { delete(s.prefetch, key) s.rset[key] = resp } } resp := s.stm.fetch(keys...) if firstRead { // txn's base revision is defined by the first read s.getOpts = []v3.OpOption{ v3.WithRev(resp.Header.Revision), v3.WithSerializable(), } } return respToValue(resp) } func (s *stmSerializable) Rev(key string) int64 { s.Get(key) return s.stm.Rev(key) } func (s *stmSerializable) gets() ([]string, []v3.Op) { keys := make([]string, 0, len(s.rset)) ops := make([]v3.Op, 0, len(s.rset)) for k := range s.rset { keys = append(keys, k) ops = append(ops, v3.OpGet(k)) } return keys, ops } func (s *stmSerializable) commit() *v3.TxnResponse { keys, getops := s.gets() txn := s.client.Txn(s.ctx).If(s.conflicts()...).Then(s.wset.puts()...) // use Else to prefetch keys in case of conflict to save a round trip txnresp, err := txn.Else(getops...).Commit() if err != nil { panic(stmError{err}) } if txnresp.Succeeded { return txnresp } // load prefetch with Else data s.rset.add(keys, txnresp) s.prefetch = s.rset s.getOpts = nil return nil } func isKeyCurrent(k string, r *v3.GetResponse) v3.Cmp { if len(r.Kvs) != 0 { return v3.Compare(v3.ModRevision(k), "=", r.Kvs[0].ModRevision) } return v3.Compare(v3.ModRevision(k), "=", 0) } func respToValue(resp *v3.GetResponse) string { if resp == nil || len(resp.Kvs) == 0 { return "" } return string(resp.Kvs[0].Value) } // NewSTMRepeatable is deprecated. func NewSTMRepeatable(ctx context.Context, c *v3.Client, apply func(STM) error) (*v3.TxnResponse, error) { return NewSTM(c, apply, WithAbortContext(ctx), WithIsolation(RepeatableReads)) } // NewSTMSerializable is deprecated. func NewSTMSerializable(ctx context.Context, c *v3.Client, apply func(STM) error) (*v3.TxnResponse, error) { return NewSTM(c, apply, WithAbortContext(ctx), WithIsolation(Serializable)) } // NewSTMReadCommitted is deprecated. func NewSTMReadCommitted(ctx context.Context, c *v3.Client, apply func(STM) error) (*v3.TxnResponse, error) { return NewSTM(c, apply, WithAbortContext(ctx), WithIsolation(ReadCommitted)) } ================================================ FILE: client/v3/concurrency/stm_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency import ( "testing" "github.com/stretchr/testify/assert" ) func TestGet(t *testing.T) { tests := []struct { name string stm *stmSerializable in []string resp string }{ { name: "Empty keys returns empty string", stm: &stmSerializable{}, in: []string{}, resp: "", }, { name: "Nil keys returns empty string", stm: &stmSerializable{}, in: nil, resp: "", }, } for _, test := range tests { test := test t.Run(test.name, func(t *testing.T) { resp := test.stm.Get(test.in...) assert.Equal(t, test.resp, resp) }) } } ================================================ FILE: client/v3/config.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "crypto/tls" "time" "go.uber.org/zap" "google.golang.org/grpc" "go.etcd.io/etcd/client/pkg/v3/transport" ) type Config struct { // Endpoints is a list of URLs. Endpoints []string `json:"endpoints"` // AutoSyncInterval is the interval to update endpoints with its latest members. // 0 disables auto-sync. By default auto-sync is disabled. AutoSyncInterval time.Duration `json:"auto-sync-interval"` // DialTimeout is the timeout for failing to establish a connection. DialTimeout time.Duration `json:"dial-timeout"` // DialKeepAliveTime is the time after which client pings the server to see if // transport is alive. DialKeepAliveTime time.Duration `json:"dial-keep-alive-time"` // DialKeepAliveTimeout is the time that the client waits for a response for the // keep-alive probe. If the response is not received in this time, the connection is closed. DialKeepAliveTimeout time.Duration `json:"dial-keep-alive-timeout"` // MaxCallSendMsgSize is the client-side request send limit in bytes. // If 0, it defaults to 2.0 MiB (2 * 1024 * 1024). // Make sure that "MaxCallSendMsgSize" < server-side default send/recv limit. // ("--max-request-bytes" flag to etcd or "embed.Config.MaxRequestBytes"). MaxCallSendMsgSize int // MaxCallRecvMsgSize is the client-side response receive limit. // If 0, it defaults to "math.MaxInt32", because range response can // easily exceed request send limits. // Make sure that "MaxCallRecvMsgSize" >= server-side default send/recv limit. // ("--max-recv-bytes" flag to etcd). MaxCallRecvMsgSize int // TLS holds the client secure credentials, if any. TLS *tls.Config // Username is a user name for authentication. Username string `json:"username"` // Password is a password for authentication. Password string `json:"password"` // Token is a JWT used for authentication instead of a password. Token string `json:"token"` // RejectOldCluster when set will refuse to create a client against an outdated cluster. RejectOldCluster bool `json:"reject-old-cluster"` // DialOptions is a list of dial options for the grpc client (e.g., for interceptors). // Note that grpc.NewClient ignores options that are specific to grpc.Dial such as // "grpc.WithBlock()". Use DialTimeout to bound client initialization time. DialOptions []grpc.DialOption // Context is the default client context; it can be used to cancel grpc dial out and // other operations that do not have an explicit context. Context context.Context // Logger sets client-side logger. // If nil, fallback to building LogConfig. Logger *zap.Logger // LogConfig configures client-side logger. // If nil, use the default logger. // TODO: configure gRPC logger LogConfig *zap.Config // PermitWithoutStream when set will allow client to send keepalive pings to server without any active streams(RPCs). PermitWithoutStream bool `json:"permit-without-stream"` // MaxUnaryRetries is the maximum number of retries for unary RPCs. MaxUnaryRetries uint `json:"max-unary-retries"` // BackoffWaitBetween is the wait time before retrying an RPC. BackoffWaitBetween time.Duration `json:"backoff-wait-between"` // BackoffJitterFraction is the jitter fraction to randomize backoff wait time. BackoffJitterFraction float64 `json:"backoff-jitter-fraction"` // TODO: support custom balancer picker } // ConfigSpec is the configuration from users, which comes from command-line flags, // environment variables or config file. It is a fully declarative configuration, // and can be serialized & deserialized to/from JSON. type ConfigSpec struct { Endpoints []string `json:"endpoints"` RequestTimeout time.Duration `json:"request-timeout"` DialTimeout time.Duration `json:"dial-timeout"` KeepAliveTime time.Duration `json:"keepalive-time"` KeepAliveTimeout time.Duration `json:"keepalive-timeout"` MaxCallSendMsgSize int `json:"max-request-bytes"` MaxCallRecvMsgSize int `json:"max-recv-bytes"` Secure *SecureConfig `json:"secure"` Auth *AuthConfig `json:"auth"` } type SecureConfig struct { Cert string `json:"cert"` Key string `json:"key"` Cacert string `json:"cacert"` ServerName string `json:"server-name"` InsecureTransport bool `json:"insecure-transport"` InsecureSkipVerify bool `json:"insecure-skip-tls-verify"` } type AuthConfig struct { Username string `json:"username"` Password string `json:"password"` Token string `json:"token"` } func (cs *ConfigSpec) Clone() *ConfigSpec { if cs == nil { return nil } clone := *cs if len(cs.Endpoints) > 0 { clone.Endpoints = make([]string, len(cs.Endpoints)) copy(clone.Endpoints, cs.Endpoints) } if cs.Secure != nil { clone.Secure = &SecureConfig{} *clone.Secure = *cs.Secure } if cs.Auth != nil { clone.Auth = &AuthConfig{} *clone.Auth = *cs.Auth } return &clone } func (cfg AuthConfig) Empty() bool { return cfg.Username == "" && cfg.Password == "" && cfg.Token == "" } // NewClientConfig creates a Config based on the provided ConfigSpec. func NewClientConfig(confSpec *ConfigSpec, lg *zap.Logger) (*Config, error) { tlsCfg, err := newTLSConfig(confSpec.Secure, lg) if err != nil { return nil, err } cfg := &Config{ Endpoints: confSpec.Endpoints, DialTimeout: confSpec.DialTimeout, DialKeepAliveTime: confSpec.KeepAliveTime, DialKeepAliveTimeout: confSpec.KeepAliveTimeout, MaxCallSendMsgSize: confSpec.MaxCallSendMsgSize, MaxCallRecvMsgSize: confSpec.MaxCallRecvMsgSize, TLS: tlsCfg, } if confSpec.Auth != nil { cfg.Username = confSpec.Auth.Username cfg.Password = confSpec.Auth.Password cfg.Token = confSpec.Auth.Token } return cfg, nil } func newTLSConfig(scfg *SecureConfig, lg *zap.Logger) (*tls.Config, error) { var ( tlsCfg *tls.Config err error ) if scfg == nil { return nil, nil } if scfg.Cert != "" || scfg.Key != "" || scfg.Cacert != "" || scfg.ServerName != "" { cfgtls := &transport.TLSInfo{ CertFile: scfg.Cert, KeyFile: scfg.Key, TrustedCAFile: scfg.Cacert, ServerName: scfg.ServerName, Logger: lg, } if tlsCfg, err = cfgtls.ClientConfig(); err != nil { return nil, err } } // If key/cert is not given but user wants secure connection, we // should still setup an empty tls configuration for gRPC to setup // secure connection. if tlsCfg == nil && !scfg.InsecureTransport { tlsCfg = &tls.Config{} } // If the user wants to skip TLS verification then we should set // the InsecureSkipVerify flag in tls configuration. if scfg.InsecureSkipVerify { if tlsCfg == nil { tlsCfg = &tls.Config{} } tlsCfg.InsecureSkipVerify = scfg.InsecureSkipVerify } return tlsCfg, nil } ================================================ FILE: client/v3/config_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "crypto/tls" "encoding/json" "reflect" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/client/pkg/v3/transport" ) func TestNewClientConfig(t *testing.T) { cases := []struct { name string spec ConfigSpec expectedConf Config }{ { name: "only has basic info", spec: ConfigSpec{ Endpoints: []string{"http://192.168.0.10:2379"}, DialTimeout: 2 * time.Second, KeepAliveTime: 3 * time.Second, KeepAliveTimeout: 5 * time.Second, }, expectedConf: Config{ Endpoints: []string{"http://192.168.0.10:2379"}, DialTimeout: 2 * time.Second, DialKeepAliveTime: 3 * time.Second, DialKeepAliveTimeout: 5 * time.Second, }, }, { name: "auth enabled", spec: ConfigSpec{ Endpoints: []string{"http://192.168.0.12:2379"}, DialTimeout: 1 * time.Second, KeepAliveTime: 4 * time.Second, KeepAliveTimeout: 6 * time.Second, Auth: &AuthConfig{ Username: "test", Password: "changeme", }, }, expectedConf: Config{ Endpoints: []string{"http://192.168.0.12:2379"}, DialTimeout: 1 * time.Second, DialKeepAliveTime: 4 * time.Second, DialKeepAliveTimeout: 6 * time.Second, Username: "test", Password: "changeme", }, }, { name: "JWT specified", spec: ConfigSpec{ Endpoints: []string{"http://192.168.0.12:2379"}, DialTimeout: 1 * time.Second, KeepAliveTime: 4 * time.Second, KeepAliveTimeout: 6 * time.Second, Auth: &AuthConfig{ Token: "test", }, }, expectedConf: Config{ Endpoints: []string{"http://192.168.0.12:2379"}, DialTimeout: 1 * time.Second, DialKeepAliveTime: 4 * time.Second, DialKeepAliveTimeout: 6 * time.Second, Token: "test", }, }, { name: "default secure transport", spec: ConfigSpec{ Endpoints: []string{"http://192.168.0.10:2379"}, DialTimeout: 2 * time.Second, KeepAliveTime: 3 * time.Second, KeepAliveTimeout: 5 * time.Second, Secure: &SecureConfig{ InsecureTransport: false, }, }, expectedConf: Config{ Endpoints: []string{"http://192.168.0.10:2379"}, DialTimeout: 2 * time.Second, DialKeepAliveTime: 3 * time.Second, DialKeepAliveTimeout: 5 * time.Second, TLS: &tls.Config{}, }, }, { name: "default secure transport and skip TLS verification", spec: ConfigSpec{ Endpoints: []string{"http://192.168.0.13:2379"}, DialTimeout: 1 * time.Second, KeepAliveTime: 3 * time.Second, KeepAliveTimeout: 5 * time.Second, Secure: &SecureConfig{ InsecureTransport: false, InsecureSkipVerify: true, }, }, expectedConf: Config{ Endpoints: []string{"http://192.168.0.13:2379"}, DialTimeout: 1 * time.Second, DialKeepAliveTime: 3 * time.Second, DialKeepAliveTimeout: 5 * time.Second, TLS: &tls.Config{ InsecureSkipVerify: true, }, }, }, { name: "insecure transport and skip TLS verification", spec: ConfigSpec{ Endpoints: []string{"http://192.168.0.13:2379"}, DialTimeout: 1 * time.Second, KeepAliveTime: 3 * time.Second, KeepAliveTimeout: 5 * time.Second, Secure: &SecureConfig{ InsecureTransport: true, InsecureSkipVerify: true, }, }, expectedConf: Config{ Endpoints: []string{"http://192.168.0.13:2379"}, DialTimeout: 1 * time.Second, DialKeepAliveTime: 3 * time.Second, DialKeepAliveTimeout: 5 * time.Second, TLS: &tls.Config{ InsecureSkipVerify: true, }, }, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { lg, _ := logutil.CreateDefaultZapLogger(zap.InfoLevel) cfg, err := NewClientConfig(&tc.spec, lg) require.NoError(t, err) assert.Equal(t, tc.expectedConf, *cfg) }) } } func TestNewClientConfigWithSecureCfg(t *testing.T) { tls, err := transport.SelfCert(zap.NewNop(), t.TempDir(), []string{"localhost"}, 1) require.NoError(t, err) scfg := &SecureConfig{ Cert: tls.CertFile, Key: tls.KeyFile, Cacert: tls.TrustedCAFile, } cfg, err := NewClientConfig(&ConfigSpec{ Endpoints: []string{"http://192.168.0.13:2379"}, DialTimeout: 2 * time.Second, KeepAliveTime: 3 * time.Second, KeepAliveTimeout: 5 * time.Second, Secure: scfg, }, nil) require.NoErrorf(t, err, "Unexpected result client config") if cfg == nil || cfg.TLS == nil { t.Fatalf("Unexpected result client config: %v", err) } } func TestConfigSpecClone(t *testing.T) { cfgSpec := &ConfigSpec{ Endpoints: []string{"ep1", "ep2", "ep3"}, RequestTimeout: 10 * time.Second, DialTimeout: 2 * time.Second, KeepAliveTime: 5 * time.Second, KeepAliveTimeout: 2 * time.Second, Secure: &SecureConfig{ Cert: "path/2/cert", Key: "path/2/key", Cacert: "path/2/cacert", InsecureTransport: true, InsecureSkipVerify: false, }, Auth: &AuthConfig{ Username: "foo", Password: "changeme", }, } testCases := []struct { name string cs *ConfigSpec newEp []string newSecure *SecureConfig newAuth *AuthConfig expectedEqual bool }{ { name: "normal case", cs: cfgSpec, expectedEqual: true, }, { name: "point to a new slice of endpoint, but with the same data", cs: cfgSpec, newEp: []string{"ep1", "ep2", "ep3"}, expectedEqual: true, }, { name: "update endpoint", cs: cfgSpec, newEp: []string{"ep1", "newep2", "ep3"}, expectedEqual: false, }, { name: "point to a new secureConfig, but with the same data", cs: cfgSpec, newSecure: &SecureConfig{ Cert: "path/2/cert", Key: "path/2/key", Cacert: "path/2/cacert", InsecureTransport: true, InsecureSkipVerify: false, }, expectedEqual: true, }, { name: "update key in secureConfig", cs: cfgSpec, newSecure: &SecureConfig{ Cert: "path/2/cert", Key: "newPath/2/key", Cacert: "path/2/cacert", InsecureTransport: true, InsecureSkipVerify: false, }, expectedEqual: false, }, { name: "update bool values in secureConfig", cs: cfgSpec, newSecure: &SecureConfig{ Cert: "path/2/cert", Key: "path/2/key", Cacert: "path/2/cacert", InsecureTransport: false, InsecureSkipVerify: true, }, expectedEqual: false, }, { name: "point to a new authConfig, but with the same data", cs: cfgSpec, newAuth: &AuthConfig{ Username: "foo", Password: "changeme", }, expectedEqual: true, }, { name: "update authConfig", cs: cfgSpec, newAuth: &AuthConfig{ Username: "newUser", Password: "newPassword", }, expectedEqual: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { dataBeforeTest, err := json.Marshal(tc.cs) require.NoError(t, err) clonedCfgSpec := tc.cs.Clone() if len(tc.newEp) > 0 { clonedCfgSpec.Endpoints = tc.newEp } if tc.newSecure != nil { clonedCfgSpec.Secure = tc.newSecure } if tc.newAuth != nil { clonedCfgSpec.Auth = tc.newAuth } actualEqual := reflect.DeepEqual(tc.cs, clonedCfgSpec) require.Equal(t, tc.expectedEqual, actualEqual) // double-check the original ConfigSpec isn't updated dataAfterTest, err := json.Marshal(tc.cs) require.NoError(t, err) require.True(t, reflect.DeepEqual(dataBeforeTest, dataAfterTest)) }) } } ================================================ FILE: client/v3/credentials/credentials.go ================================================ // Copyright 2019 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package credentials implements gRPC credential interface with etcd specific logic. // e.g., client handshake with custom authority parameter package credentials import ( "context" "crypto/tls" "sync" grpccredentials "google.golang.org/grpc/credentials" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" ) func NewTransportCredential(cfg *tls.Config) grpccredentials.TransportCredentials { return grpccredentials.NewTLS(cfg) } // PerRPCCredentialsBundle defines gRPC credential interface. type PerRPCCredentialsBundle interface { UpdateAuthToken(token string) PerRPCCredentials() grpccredentials.PerRPCCredentials } func NewPerRPCCredentialBundle() PerRPCCredentialsBundle { return &perRPCCredentialBundle{ rc: &perRPCCredential{}, } } // perRPCCredentialBundle implements `PerRPCCredentialsBundle` interface. type perRPCCredentialBundle struct { rc *perRPCCredential } func (b *perRPCCredentialBundle) UpdateAuthToken(token string) { if b.rc == nil { return } b.rc.UpdateAuthToken(token) } func (b *perRPCCredentialBundle) PerRPCCredentials() grpccredentials.PerRPCCredentials { return b.rc } // perRPCCredential implements `grpccredentials.PerRPCCredentials` interface. type perRPCCredential struct { authToken string authTokenMu sync.RWMutex } func (rc *perRPCCredential) RequireTransportSecurity() bool { return false } func (rc *perRPCCredential) GetRequestMetadata(ctx context.Context, s ...string) (map[string]string, error) { rc.authTokenMu.RLock() authToken := rc.authToken rc.authTokenMu.RUnlock() if authToken == "" { return nil, nil } return map[string]string{rpctypes.TokenFieldNameGRPC: authToken}, nil } func (rc *perRPCCredential) UpdateAuthToken(token string) { rc.authTokenMu.Lock() rc.authToken = token rc.authTokenMu.Unlock() } ================================================ FILE: client/v3/credentials/credentials_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package credentials import ( "testing" "github.com/stretchr/testify/assert" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" ) func TestUpdateAuthToken(t *testing.T) { bundle := NewPerRPCCredentialBundle() ctx := t.Context() metadataBeforeUpdate, _ := bundle.PerRPCCredentials().GetRequestMetadata(ctx) assert.Empty(t, metadataBeforeUpdate) bundle.UpdateAuthToken("abcdefg") metadataAfterUpdate, _ := bundle.PerRPCCredentials().GetRequestMetadata(ctx) assert.Equal(t, "abcdefg", metadataAfterUpdate[rpctypes.TokenFieldNameGRPC]) } ================================================ FILE: client/v3/ctx.go ================================================ // Copyright 2020 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "google.golang.org/grpc/metadata" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/api/v3/version" ) // WithRequireLeader requires client requests to only succeed // when the cluster has a leader. func WithRequireLeader(ctx context.Context) context.Context { md, ok := metadata.FromOutgoingContext(ctx) if !ok { // no outgoing metadata ctx key, create one md = metadata.Pairs(rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader) return metadata.NewOutgoingContext(ctx, md) } copied := md.Copy() // avoid racey updates // overwrite/add 'hasleader' key/value copied.Set(rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader) return metadata.NewOutgoingContext(ctx, copied) } // embeds client version func withVersion(ctx context.Context) context.Context { md, ok := metadata.FromOutgoingContext(ctx) if !ok { // no outgoing metadata ctx key, create one md = metadata.Pairs(rpctypes.MetadataClientAPIVersionKey, version.APIVersion) return metadata.NewOutgoingContext(ctx, md) } copied := md.Copy() // avoid racey updates // overwrite/add version key/value copied.Set(rpctypes.MetadataClientAPIVersionKey, version.APIVersion) return metadata.NewOutgoingContext(ctx, copied) } ================================================ FILE: client/v3/ctx_test.go ================================================ // Copyright 2020 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "reflect" "testing" "github.com/stretchr/testify/require" "google.golang.org/grpc/metadata" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/api/v3/version" ) func TestMetadataWithRequireLeader(t *testing.T) { ctx := t.Context() _, ok := metadata.FromOutgoingContext(ctx) require.Falsef(t, ok, "expected no outgoing metadata ctx key") // add a conflicting key with some other value md := metadata.Pairs(rpctypes.MetadataRequireLeaderKey, "invalid") // add a key, and expect not be overwritten md.Set("hello", "1", "2") ctx = metadata.NewOutgoingContext(ctx, md) // expect overwrites but still keep other keys ctx = WithRequireLeader(ctx) md, ok = metadata.FromOutgoingContext(ctx) require.Truef(t, ok, "expected outgoing metadata ctx key") ss := md.Get(rpctypes.MetadataRequireLeaderKey) require.Truef(t, reflect.DeepEqual(ss, []string{rpctypes.MetadataHasLeader}), "unexpected metadata for %q %v", rpctypes.MetadataRequireLeaderKey, ss) ss = md.Get("hello") require.Truef(t, reflect.DeepEqual(ss, []string{"1", "2"}), "unexpected metadata for 'hello' %v", ss) } func TestMetadataWithClientAPIVersion(t *testing.T) { ctx := withVersion(WithRequireLeader(t.Context())) md, ok := metadata.FromOutgoingContext(ctx) require.Truef(t, ok, "expected outgoing metadata ctx key") ss := md.Get(rpctypes.MetadataRequireLeaderKey) require.Truef(t, reflect.DeepEqual(ss, []string{rpctypes.MetadataHasLeader}), "unexpected metadata for %q %v", rpctypes.MetadataRequireLeaderKey, ss) ss = md.Get(rpctypes.MetadataClientAPIVersionKey) require.Truef(t, reflect.DeepEqual(ss, []string{version.APIVersion}), "unexpected metadata for %q %v", rpctypes.MetadataClientAPIVersionKey, ss) } ================================================ FILE: client/v3/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package clientv3 implements the official Go etcd client for v3. // // Create client using `clientv3.New`: // // // expect dial time-out on ipv4 blackhole // _, err := clientv3.New(clientv3.Config{ // Endpoints: []string{"http://254.0.0.1:12345"}, // DialTimeout: 2 * time.Second, // }) // // // etcd clientv3 >= v3.2.10, grpc/grpc-go >= v1.7.3 // if err == context.DeadlineExceeded { // // handle errors // } // // // etcd clientv3 <= v3.2.9, grpc/grpc-go <= v1.2.1 // if err == grpc.ErrClientConnTimeout { // // handle errors // } // // cli, err := clientv3.New(clientv3.Config{ // Endpoints: []string{"localhost:2379", "localhost:22379", "localhost:32379"}, // DialTimeout: 5 * time.Second, // }) // if err != nil { // // handle error! // } // defer cli.Close() // // Make sure to close the client after using it. If the client is not closed, the // connection will have leaky goroutines. // // To specify a client request timeout, wrap the context with context.WithTimeout: // // ctx, cancel := context.WithTimeout(context.Background(), timeout) // defer cancel() // resp, err := kvc.Put(ctx, "sample_key", "sample_value") // if err != nil { // // handle error! // } // // use the response // // The Client has internal state (watchers and leases), so Clients should be reused instead of created as needed. // Clients are safe for concurrent use by multiple goroutines. // // etcd client returns 2 types of errors: // // 1. context error: canceled or deadline exceeded. // 2. gRPC error: e.g. when clock drifts in server-side before client's context deadline exceeded. // See https://github.com/etcd-io/etcd/blob/main/api/v3rpc/rpctypes/error.go // // Here is the example code to handle client errors: // // resp, err := kvc.Put(ctx, "", "") // if err != nil { // if err == context.Canceled { // // ctx is canceled by another routine // } else if err == context.DeadlineExceeded { // // ctx is attached with a deadline and it exceeded // } else if err == rpctypes.ErrEmptyKey { // // client-side error: key is not provided // } else if ev, ok := status.FromError(err); ok { // code := ev.Code() // if code == codes.DeadlineExceeded { // // server-side context might have timed-out first (due to clock skew) // // while original client-side context is not timed-out yet // } // } else { // // bad cluster endpoints, which are not etcd servers // } // } // // go func() { cli.Close() }() // _, err := kvc.Get(ctx, "a") // if err != nil { // // with etcd clientv3 <= v3.3 // if err == context.Canceled { // // grpc balancer calls 'Get' with an inflight client.Close // } else if err == grpc.ErrClientConnClosing { // <= gRCP v1.7.x // // grpc balancer calls 'Get' after client.Close. // } // // with etcd clientv3 >= v3.4 // if clientv3.IsConnCanceled(err) { // // gRPC client connection is closed // } // } // // The grpc load balancer is registered statically and is shared across etcd clients. // To enable detailed load balancer logging, set the ETCD_CLIENT_DEBUG environment // variable. E.g. "ETCD_CLIENT_DEBUG=1". package clientv3 ================================================ FILE: client/v3/experimental/recipes/barrier.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipe import ( "context" "go.etcd.io/etcd/api/v3/mvccpb" v3 "go.etcd.io/etcd/client/v3" ) // Barrier creates a key in etcd to block processes, then deletes the key to // release all blocked processes. type Barrier struct { client *v3.Client ctx context.Context key string } func NewBarrier(client *v3.Client, key string) *Barrier { return &Barrier{client, context.TODO(), key} } // Hold creates the barrier key causing processes to block on Wait. func (b *Barrier) Hold() error { _, err := newKey(b.client, b.key, v3.NoLease) return err } // Release deletes the barrier key to unblock all waiting processes. func (b *Barrier) Release() error { _, err := b.client.Delete(b.ctx, b.key) return err } // Wait blocks on the barrier key until it is deleted. If there is no key, Wait // assumes Release has already been called and returns immediately. func (b *Barrier) Wait() error { resp, err := b.client.Get(b.ctx, b.key) if err != nil { return err } if len(resp.Kvs) == 0 { // key already removed return nil } _, err = WaitEvents( b.client, b.key, resp.Header.Revision+1, []mvccpb.Event_EventType{mvccpb.Event_DELETE}) return err } ================================================ FILE: client/v3/experimental/recipes/client.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipe import ( "context" "errors" spb "go.etcd.io/etcd/api/v3/mvccpb" v3 "go.etcd.io/etcd/client/v3" ) var ( ErrKeyExists = errors.New("key already exists") ErrWaitMismatch = errors.New("unexpected wait result") ErrTooManyClients = errors.New("too many clients") ErrNoWatcher = errors.New("no watcher channel") ) // deleteRevKey deletes a key by revision, returning false if key is missing func deleteRevKey(kv v3.KV, key string, rev int64) (bool, error) { cmp := v3.Compare(v3.ModRevision(key), "=", rev) req := v3.OpDelete(key) txnresp, err := kv.Txn(context.TODO()).If(cmp).Then(req).Commit() if err != nil { return false, err } else if !txnresp.Succeeded { return false, nil } return true, nil } func claimFirstKey(kv v3.KV, kvs []*spb.KeyValue) (*spb.KeyValue, error) { for _, k := range kvs { ok, err := deleteRevKey(kv, string(k.Key), k.ModRevision) if err != nil { return nil, err } else if ok { return k, nil } } return nil, nil } ================================================ FILE: client/v3/experimental/recipes/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package recipe contains experimental client-side distributed // synchronization primitives. package recipe ================================================ FILE: client/v3/experimental/recipes/double_barrier.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipe import ( "context" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" ) // DoubleBarrier blocks processes on Enter until an expected count enters, then // blocks again on Leave until all processes have left. type DoubleBarrier struct { s *concurrency.Session ctx context.Context key string // key for the collective barrier count int myKey *EphemeralKV // current key for this process on the barrier } func NewDoubleBarrier(s *concurrency.Session, key string, count int) *DoubleBarrier { return &DoubleBarrier{ s: s, ctx: context.TODO(), key: key, count: count, } } // Enter waits for "count" processes to enter the barrier then returns func (b *DoubleBarrier) Enter() error { client := b.s.Client() // Check the entered clients before creating the UniqueEphemeralKey, // fail the request if there are already too many clients. if resp1, err := b.enteredClients(client); err != nil { return err } else if len(resp1.Kvs) >= b.count { return ErrTooManyClients } ek, err := newUniqueEphemeralKey(b.s, b.key+"/waiters") if err != nil { return err } b.myKey = ek // Check the entered clients after creating the UniqueEphemeralKey resp2, err := b.enteredClients(client) if err != nil { return err } if len(resp2.Kvs) >= b.count { lastWaiter := resp2.Kvs[b.count-1] if ek.rev > lastWaiter.CreateRevision { // delete itself now, otherwise other processes may need to wait // until these keys are automatically deleted when the related // lease expires. //nolint:staticcheck // SA9003 disable empty branch checker to keep the comment for why we ignore error if err = b.myKey.Delete(); err != nil { // Nothing to do here. We have to wait for the key to be // deleted when the lease expires. } return ErrTooManyClients } if ek.rev == lastWaiter.CreateRevision { // TODO(ahrtr): we might need to compare ek.key and // string(lastWaiter.Key), they should be equal. // unblock all other waiters _, err = client.Put(b.ctx, b.key+"/ready", "") return err } } _, err = WaitEvents( client, b.key+"/ready", ek.Revision(), []mvccpb.Event_EventType{mvccpb.Event_PUT}) return err } // enteredClients gets all the entered clients, which are ordered by the // createRevision in ascending order. func (b *DoubleBarrier) enteredClients(cli *clientv3.Client) (*clientv3.GetResponse, error) { resp, err := cli.Get(b.ctx, b.key+"/waiters", clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByCreateRevision, clientv3.SortAscend)) if err != nil { return nil, err } return resp, nil } // Leave waits for "count" processes to leave the barrier then returns func (b *DoubleBarrier) Leave() error { client := b.s.Client() resp, err := client.Get(b.ctx, b.key+"/waiters", clientv3.WithPrefix()) if err != nil { return err } if len(resp.Kvs) == 0 { return nil } lowest, highest := resp.Kvs[0], resp.Kvs[0] for _, k := range resp.Kvs { if k.ModRevision < lowest.ModRevision { lowest = k } if k.ModRevision > highest.ModRevision { highest = k } } isLowest := string(lowest.Key) == b.myKey.Key() if len(resp.Kvs) == 1 && isLowest { // this is the only node in the barrier; finish up if _, err = client.Delete(b.ctx, b.key+"/ready"); err != nil { return err } return b.myKey.Delete() } // this ensures that if a process fails, the ephemeral lease will be // revoked, its barrier key is removed, and the barrier can resume // lowest process in node => wait on highest process if isLowest { _, err = WaitEvents( client, string(highest.Key), highest.ModRevision, []mvccpb.Event_EventType{mvccpb.Event_DELETE}) if err != nil { return err } return b.Leave() } // delete self and wait on lowest process if err = b.myKey.Delete(); err != nil { return err } key := string(lowest.Key) _, err = WaitEvents( client, key, lowest.ModRevision, []mvccpb.Event_EventType{mvccpb.Event_DELETE}) if err != nil { return err } return b.Leave() } ================================================ FILE: client/v3/experimental/recipes/grpc_gateway/user_add.sh ================================================ #!/bin/bash # Copyright 2018 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. usage () { echo 'Username required: ./user_add.sh $username' exit } if [ "$1" == "" ]; then usage fi newuser=$1 read -r -s -p "Enter password for $newuser" newpass user=root pass=toor host=127.0.0.1 port=2379 api=v3 cacert="path/to/ca.pem" key="path/to/client-key.pem" cert="path/to/client.pem" tokengen() { json=$(printf '{"name": "%s", "password": "%s"}' \ "$(escape "$1")" \ "$(escape "$2")" ) curl -s --cacert $cacert \ --key $key \ --cert $cert \ -X POST \ -d "$json" \ https://${host}:${port}/${api}/auth/authenticate \ | jq -r '.token' } add_user() { json=$(printf '{"name": "%s", "password": "%s"}' \ "$(escape "$1")" \ "$(escape "$2")" ) curl -s --cacert $cacert \ --key $key \ --cert $cert \ -H "Authorization: $3" \ -X POST \ -d "$json" \ https://${host}:${port}/${api}/auth/user/add } escape() { echo "${1//\"/\\\"}" } token=$(tokengen $user $pass) response=$(add_user $newuser $newpass $token) echo -e "\\n$response" ================================================ FILE: client/v3/experimental/recipes/key.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipe import ( "context" "errors" "fmt" "strings" "time" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" ) // RemoteKV is a key/revision pair created by the client and stored on etcd type RemoteKV struct { kv v3.KV key string rev int64 val string } func newKey(kv v3.KV, key string, leaseID v3.LeaseID) (*RemoteKV, error) { return newKV(kv, key, "", leaseID) } func newKV(kv v3.KV, key, val string, leaseID v3.LeaseID) (*RemoteKV, error) { rev, err := putNewKV(kv, key, val, leaseID) if err != nil { return nil, err } return &RemoteKV{kv, key, rev, val}, nil } func newUniqueKV(kv v3.KV, prefix string, val string) (*RemoteKV, error) { for { newKey := fmt.Sprintf("%s/%v", prefix, time.Now().UnixNano()) rev, err := putNewKV(kv, newKey, val, v3.NoLease) if err == nil { return &RemoteKV{kv, newKey, rev, val}, nil } if !errors.Is(err, ErrKeyExists) { return nil, err } } } // putNewKV attempts to create the given key, only succeeding if the key did // not yet exist. func putNewKV(kv v3.KV, key, val string, leaseID v3.LeaseID) (int64, error) { cmp := v3.Compare(v3.Version(key), "=", 0) req := v3.OpPut(key, val, v3.WithLease(leaseID)) txnresp, err := kv.Txn(context.TODO()).If(cmp).Then(req).Commit() if err != nil { return 0, err } if !txnresp.Succeeded { return 0, ErrKeyExists } return txnresp.Header.Revision, nil } // newSequentialKV allocates a new sequential key /nnnnn with a given // prefix and value. Note: a bookkeeping node __ is also allocated. func newSequentialKV(kv v3.KV, prefix, val string) (*RemoteKV, error) { resp, err := kv.Get(context.TODO(), prefix, v3.WithLastKey()...) if err != nil { return nil, err } // add 1 to last key, if any newSeqNum := 0 if len(resp.Kvs) != 0 { fields := strings.Split(string(resp.Kvs[0].Key), "/") _, serr := fmt.Sscanf(fields[len(fields)-1], "%d", &newSeqNum) if serr != nil { return nil, serr } newSeqNum++ } newKey := fmt.Sprintf("%s/%016d", prefix, newSeqNum) // base prefix key must be current (i.e., <=) with the server update; // the base key is important to avoid the following: // N1: LastKey() == 1, start txn. // N2: new Key 2, new Key 3, Delete Key 2 // N1: txn succeeds allocating key 2 when it shouldn't baseKey := "__" + prefix // current revision might contain modification so +1 cmp := v3.Compare(v3.ModRevision(baseKey), "<", resp.Header.Revision+1) reqPrefix := v3.OpPut(baseKey, "") reqnewKey := v3.OpPut(newKey, val) txn := kv.Txn(context.TODO()) txnresp, err := txn.If(cmp).Then(reqPrefix, reqnewKey).Commit() if err != nil { return nil, err } if !txnresp.Succeeded { return newSequentialKV(kv, prefix, val) } return &RemoteKV{kv, newKey, txnresp.Header.Revision, val}, nil } func (rk *RemoteKV) Key() string { return rk.key } func (rk *RemoteKV) Revision() int64 { return rk.rev } func (rk *RemoteKV) Value() string { return rk.val } func (rk *RemoteKV) Delete() error { if rk.kv == nil { return nil } _, err := rk.kv.Delete(context.TODO(), rk.key) rk.kv = nil return err } func (rk *RemoteKV) Put(val string) error { _, err := rk.kv.Put(context.TODO(), rk.key, val) return err } // EphemeralKV is a new key associated with a session lease type EphemeralKV struct{ RemoteKV } // newEphemeralKV creates a new key/value pair associated with a session lease func newEphemeralKV(s *concurrency.Session, key, val string) (*EphemeralKV, error) { k, err := newKV(s.Client(), key, val, s.Lease()) if err != nil { return nil, err } return &EphemeralKV{*k}, nil } // newUniqueEphemeralKey creates a new unique valueless key associated with a session lease func newUniqueEphemeralKey(s *concurrency.Session, prefix string) (*EphemeralKV, error) { return newUniqueEphemeralKV(s, prefix, "") } // newUniqueEphemeralKV creates a new unique key/value pair associated with a session lease func newUniqueEphemeralKV(s *concurrency.Session, prefix, val string) (ek *EphemeralKV, err error) { for { newKey := fmt.Sprintf("%s/%v", prefix, time.Now().UnixNano()) ek, err = newEphemeralKV(s, newKey, val) if err == nil || !errors.Is(err, ErrKeyExists) { break } } return ek, err } ================================================ FILE: client/v3/experimental/recipes/priority_queue.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipe import ( "context" "fmt" "go.etcd.io/etcd/api/v3/mvccpb" v3 "go.etcd.io/etcd/client/v3" ) // PriorityQueue implements a multi-reader, multi-writer distributed queue. type PriorityQueue struct { client *v3.Client ctx context.Context key string } // NewPriorityQueue creates an etcd priority queue. func NewPriorityQueue(client *v3.Client, key string) *PriorityQueue { return &PriorityQueue{client, context.TODO(), key + "/"} } // Enqueue puts a value into a queue with a given priority. func (q *PriorityQueue) Enqueue(val string, pr uint16) error { prefix := fmt.Sprintf("%s%05d", q.key, pr) _, err := newSequentialKV(q.client, prefix, val) return err } // Dequeue returns Enqueue()'d items in FIFO order. If the // queue is empty, Dequeue blocks until items are available. func (q *PriorityQueue) Dequeue() (string, error) { // TODO: fewer round trips by fetching more than one key resp, err := q.client.Get(q.ctx, q.key, v3.WithFirstKey()...) if err != nil { return "", err } kv, err := claimFirstKey(q.client, resp.Kvs) if err != nil { return "", err } else if kv != nil { return string(kv.Value), nil } else if resp.More { // missed some items, retry to read in more return q.Dequeue() } // nothing to dequeue; wait on items ev, err := WaitPrefixEvents( q.client, q.key, resp.Header.Revision, []mvccpb.Event_EventType{mvccpb.Event_PUT}) if err != nil { return "", err } ok, err := deleteRevKey(q.client, string(ev.Kv.Key), ev.Kv.ModRevision) if err != nil { return "", err } else if !ok { return q.Dequeue() } return string(ev.Kv.Value), err } ================================================ FILE: client/v3/experimental/recipes/queue.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipe import ( "context" "go.etcd.io/etcd/api/v3/mvccpb" v3 "go.etcd.io/etcd/client/v3" ) // Queue implements a multi-reader, multi-writer distributed queue. type Queue struct { client *v3.Client ctx context.Context keyPrefix string } func NewQueue(client *v3.Client, keyPrefix string) *Queue { return &Queue{client, context.TODO(), keyPrefix} } func (q *Queue) Enqueue(val string) error { _, err := newUniqueKV(q.client, q.keyPrefix, val) return err } // Dequeue returns Enqueue()'d elements in FIFO order. If the // queue is empty, Dequeue blocks until elements are available. func (q *Queue) Dequeue() (string, error) { // TODO: fewer round trips by fetching more than one key resp, err := q.client.Get(q.ctx, q.keyPrefix, v3.WithFirstRev()...) if err != nil { return "", err } kv, err := claimFirstKey(q.client, resp.Kvs) if err != nil { return "", err } else if kv != nil { return string(kv.Value), nil } else if resp.More { // missed some items, retry to read in more return q.Dequeue() } // nothing yet; wait on elements ev, err := WaitPrefixEvents( q.client, q.keyPrefix, resp.Header.Revision, []mvccpb.Event_EventType{mvccpb.Event_PUT}) if err != nil { return "", err } ok, err := deleteRevKey(q.client, string(ev.Kv.Key), ev.Kv.ModRevision) if err != nil { return "", err } else if !ok { return q.Dequeue() } return string(ev.Kv.Value), err } ================================================ FILE: client/v3/experimental/recipes/rwmutex.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipe import ( "context" "go.etcd.io/etcd/api/v3/mvccpb" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" ) type RWMutex struct { s *concurrency.Session ctx context.Context pfx string myKey *EphemeralKV } func NewRWMutex(s *concurrency.Session, prefix string) *RWMutex { return &RWMutex{s, context.TODO(), prefix + "/", nil} } func (rwm *RWMutex) RLock() error { rk, err := newUniqueEphemeralKey(rwm.s, rwm.pfx+"read") if err != nil { return err } rwm.myKey = rk // wait until nodes with "write-" and a lower revision number than myKey are gone for { if done, werr := rwm.waitOnLastRev(rwm.pfx + "write"); done || werr != nil { return werr } } } func (rwm *RWMutex) Lock() error { rk, err := newUniqueEphemeralKey(rwm.s, rwm.pfx+"write") if err != nil { return err } rwm.myKey = rk // wait until all keys of lower revision than myKey are gone for { if done, werr := rwm.waitOnLastRev(rwm.pfx); done || werr != nil { return werr } // get the new lowest key until this is the only one left } } // waitOnLastRev will wait on the last key with a revision < rwm.myKey.Revision with a // given prefix. If there are no keys left to wait on, return true. func (rwm *RWMutex) waitOnLastRev(pfx string) (bool, error) { client := rwm.s.Client() // get key that's blocking myKey opts := append(v3.WithLastRev(), v3.WithMaxModRev(rwm.myKey.Revision()-1)) lastKey, err := client.Get(rwm.ctx, pfx, opts...) if err != nil { return false, err } if len(lastKey.Kvs) == 0 { return true, nil } // wait for release on blocking key _, err = WaitEvents( client, string(lastKey.Kvs[0].Key), rwm.myKey.Revision(), []mvccpb.Event_EventType{mvccpb.Event_DELETE}) return false, err } func (rwm *RWMutex) RUnlock() error { return rwm.myKey.Delete() } func (rwm *RWMutex) Unlock() error { return rwm.myKey.Delete() } ================================================ FILE: client/v3/experimental/recipes/watch.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipe import ( "context" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" ) // WaitEvents waits on a key until it observes the given events and returns the final one. func WaitEvents(c *clientv3.Client, key string, rev int64, evs []mvccpb.Event_EventType) (*clientv3.Event, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() wc := c.Watch(ctx, key, clientv3.WithRev(rev)) if wc == nil { return nil, ErrNoWatcher } return waitEvents(wc, evs), nil } func WaitPrefixEvents(c *clientv3.Client, prefix string, rev int64, evs []mvccpb.Event_EventType) (*clientv3.Event, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() wc := c.Watch(ctx, prefix, clientv3.WithPrefix(), clientv3.WithRev(rev)) if wc == nil { return nil, ErrNoWatcher } return waitEvents(wc, evs), nil } func waitEvents(wc clientv3.WatchChan, evs []mvccpb.Event_EventType) *clientv3.Event { i := 0 for wresp := range wc { for _, ev := range wresp.Events { if ev.Type == evs[i] { i++ if i == len(evs) { return ev } } } } return nil } ================================================ FILE: client/v3/go.mod ================================================ module go.etcd.io/etcd/client/v3 go 1.26 toolchain go1.26.1 require ( github.com/coreos/go-semver v0.3.1 github.com/dustin/go-humanize v1.0.1 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 github.com/prometheus/client_golang v1.23.2 github.com/stretchr/testify v1.11.1 go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 go.uber.org/zap v1.27.1 google.golang.org/grpc v1.79.2 sigs.k8s.io/yaml v1.6.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.16.1 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect go.opentelemetry.io/otel/sdk v1.42.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace ( go.etcd.io/etcd/api/v3 => ../../api go.etcd.io/etcd/client/pkg/v3 => ../pkg ) ================================================ FILE: client/v3/go.sum ================================================ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= ================================================ FILE: client/v3/internal/endpoint/endpoint.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package endpoint import ( "fmt" "net" "net/url" "path" "strings" ) type CredsRequirement int const ( // CredsRequire - Credentials/certificate required for thi type of connection. CredsRequire CredsRequirement = iota // CredsDrop - Credentials/certificate not needed and should get ignored. CredsDrop // CredsOptional - Credentials/certificate might be used if supplied CredsOptional ) func extractHostFromHostPort(ep string) string { host, _, err := net.SplitHostPort(ep) if err != nil { return ep } return host } // mustSplit2 returns the values from strings.SplitN(s, sep, 2). // If sep is not found, it returns ("", "", false) instead. func mustSplit2(s, sep string) (string, string) { spl := strings.SplitN(s, sep, 2) if len(spl) < 2 { panic(fmt.Errorf("token '%v' expected to have separator sep: `%v`", s, sep)) } return spl[0], spl[1] } func schemeToCredsRequirement(schema string) CredsRequirement { switch schema { case "https", "unixs": return CredsRequire case "http": return CredsDrop case "unix": // Preserving previous behavior from: // https://github.com/etcd-io/etcd/blob/dae29bb719dd69dc119146fc297a0628fcc1ccf8/client/v3/client.go#L212 // that likely was a bug due to missing 'fallthrough'. // At the same time it seems legit to let the users decide whether they // want credential control or not (and 'unixs' schema is not a standard thing). return CredsOptional case "": return CredsOptional default: return CredsOptional } } // This function translates endpoints names supported by etcd server into // endpoints as supported by grpc with additional information // (server_name for cert validation, requireCreds - whether certs are needed). // The main differences: // - etcd supports unixs & https names as opposed to unix & http to // distinguish need to configure certificates. // - etcd support http(s) names as opposed to tcp supported by grpc/dial method. // - etcd supports unix(s)://local-file naming schema // (as opposed to unix:local-file canonical name used by grpc for current dir files). // - Within the unix(s) schemas, the last segment (filename) without 'port' (content after colon) // is considered serverName - to allow local testing of cert-protected communication. // // See more: // - https://github.com/grpc/grpc-go/blob/26c143bd5f59344a4b8a1e491e0f5e18aa97abc7/internal/grpcutil/target.go#L47 // - https://golang.org/pkg/net/#Dial // - https://github.com/grpc/grpc/blob/master/doc/naming.md func translateEndpoint(ep string) (addr string, serverName string, requireCreds CredsRequirement) { if strings.HasPrefix(ep, "unix:") || strings.HasPrefix(ep, "unixs:") { if strings.HasPrefix(ep, "unix:///") || strings.HasPrefix(ep, "unixs:///") { // absolute path case schema, absolutePath := mustSplit2(ep, "://") return "unix://" + absolutePath, path.Base(absolutePath), schemeToCredsRequirement(schema) } if strings.HasPrefix(ep, "unix://") || strings.HasPrefix(ep, "unixs://") { // legacy etcd local path schema, localPath := mustSplit2(ep, "://") return "unix:" + localPath, path.Base(localPath), schemeToCredsRequirement(schema) } schema, localPath := mustSplit2(ep, ":") return "unix:" + localPath, path.Base(localPath), schemeToCredsRequirement(schema) } if strings.Contains(ep, "://") { url, err := url.Parse(ep) if err != nil { return ep, ep, CredsOptional } if url.Scheme == "http" || url.Scheme == "https" { return url.Host, url.Host, schemeToCredsRequirement(url.Scheme) } return ep, url.Host, schemeToCredsRequirement(url.Scheme) } // Handles plain addresses like 10.0.0.44:437. return ep, ep, CredsOptional } // RequiresCredentials returns whether given endpoint requires // credentials/certificates for connection. func RequiresCredentials(ep string) CredsRequirement { _, _, requireCreds := translateEndpoint(ep) return requireCreds } // Interpret endpoint parses an endpoint of the form // (http|https)://*|(unix|unixs)://) // and returns low-level address (supported by 'net') to connect to, // and a server name used for x509 certificate matching. func Interpret(ep string) (address string, serverName string) { addr, serverName, _ := translateEndpoint(ep) return addr, serverName } ================================================ FILE: client/v3/internal/endpoint/endpoint_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package endpoint import ( "testing" ) func Test_interpret(t *testing.T) { tests := []struct { endpoint string wantAddress string wantServerName string wantRequiresCreds CredsRequirement }{ {"127.0.0.1", "127.0.0.1", "127.0.0.1", CredsOptional}, {"localhost", "localhost", "localhost", CredsOptional}, {"localhost:8080", "localhost:8080", "localhost:8080", CredsOptional}, {"unix:127.0.0.1", "unix:127.0.0.1", "127.0.0.1", CredsOptional}, {"unix:127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1:8080", CredsOptional}, {"unix://127.0.0.1", "unix:127.0.0.1", "127.0.0.1", CredsOptional}, {"unix://127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1:8080", CredsOptional}, {"unixs:127.0.0.1", "unix:127.0.0.1", "127.0.0.1", CredsRequire}, {"unixs:127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1:8080", CredsRequire}, {"unixs://127.0.0.1", "unix:127.0.0.1", "127.0.0.1", CredsRequire}, {"unixs://127.0.0.1:8080", "unix:127.0.0.1:8080", "127.0.0.1:8080", CredsRequire}, {"http://127.0.0.1", "127.0.0.1", "127.0.0.1", CredsDrop}, {"http://127.0.0.1:8080", "127.0.0.1:8080", "127.0.0.1:8080", CredsDrop}, {"https://127.0.0.1", "127.0.0.1", "127.0.0.1", CredsRequire}, {"https://127.0.0.1:8080", "127.0.0.1:8080", "127.0.0.1:8080", CredsRequire}, {"https://localhost:20000", "localhost:20000", "localhost:20000", CredsRequire}, {"unix:///tmp/abc", "unix:///tmp/abc", "abc", CredsOptional}, {"unixs:///tmp/abc", "unix:///tmp/abc", "abc", CredsRequire}, {"unix:///tmp/abc:1234", "unix:///tmp/abc:1234", "abc:1234", CredsOptional}, {"unixs:///tmp/abc:1234", "unix:///tmp/abc:1234", "abc:1234", CredsRequire}, {"etcd.io", "etcd.io", "etcd.io", CredsOptional}, {"http://etcd.io/abc", "etcd.io", "etcd.io", CredsDrop}, {"dns://something-other", "dns://something-other", "something-other", CredsOptional}, {"http://[2001:db8:1f70::999:de8:7648:6e8]:100/", "[2001:db8:1f70::999:de8:7648:6e8]:100", "[2001:db8:1f70::999:de8:7648:6e8]:100", CredsDrop}, {"[2001:db8:1f70::999:de8:7648:6e8]:100", "[2001:db8:1f70::999:de8:7648:6e8]:100", "[2001:db8:1f70::999:de8:7648:6e8]:100", CredsOptional}, {"unix:unexpected-file_name#123$456", "unix:unexpected-file_name#123$456", "unexpected-file_name#123$456", CredsOptional}, } for _, tt := range tests { t.Run("Interpret_"+tt.endpoint, func(t *testing.T) { gotAddress, gotServerName := Interpret(tt.endpoint) if gotAddress != tt.wantAddress { t.Errorf("Interpret() gotAddress = %v, want %v", gotAddress, tt.wantAddress) } if gotServerName != tt.wantServerName { t.Errorf("Interpret() gotServerName = %v, want %v", gotServerName, tt.wantServerName) } }) t.Run("RequiresCredentials_"+tt.endpoint, func(t *testing.T) { requiresCreds := RequiresCredentials(tt.endpoint) if requiresCreds != tt.wantRequiresCreds { t.Errorf("RequiresCredentials() got = %v, want %v", requiresCreds, tt.wantRequiresCreds) } }) } } func Test_extractHostFromHostPort(t *testing.T) { tests := []struct { ep string want string }{ {ep: "localhost", want: "localhost"}, {ep: "localhost:8080", want: "localhost"}, {ep: "192.158.7.14:8080", want: "192.158.7.14"}, {ep: "192.158.7.14:8080", want: "192.158.7.14"}, {ep: "[2001:db8:1f70::999:de8:7648:6e8]", want: "[2001:db8:1f70::999:de8:7648:6e8]"}, {ep: "[2001:db8:1f70::999:de8:7648:6e8]:100", want: "2001:db8:1f70::999:de8:7648:6e8"}, } for _, tt := range tests { t.Run(tt.ep, func(t *testing.T) { if got := extractHostFromHostPort(tt.ep); got != tt.want { t.Errorf("extractHostFromHostPort() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: client/v3/internal/resolver/resolver.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package resolver import ( "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" "google.golang.org/grpc/serviceconfig" "go.etcd.io/etcd/client/v3/internal/endpoint" ) const ( Schema = "etcd-endpoints" ) // EtcdManualResolver is a Resolver (and resolver.Builder) that can be updated // using SetEndpoints. type EtcdManualResolver struct { *manual.Resolver endpoints []string serviceConfig *serviceconfig.ParseResult } func New(endpoints ...string) *EtcdManualResolver { r := manual.NewBuilderWithScheme(Schema) return &EtcdManualResolver{Resolver: r, endpoints: endpoints, serviceConfig: nil} } // Build returns itself for Resolver, because it's both a builder and a resolver. func (r *EtcdManualResolver) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { r.serviceConfig = cc.ParseServiceConfig(`{"loadBalancingPolicy": "round_robin"}`) if r.serviceConfig.Err != nil { return nil, r.serviceConfig.Err } res, err := r.Resolver.Build(target, cc, opts) if err != nil { return nil, err } // Populates endpoints stored in r into ClientConn (cc). r.updateState() return res, nil } func (r *EtcdManualResolver) SetEndpoints(endpoints []string) { r.endpoints = endpoints r.updateState() } func (r EtcdManualResolver) updateState() { if getCC(r) != nil { eps := make([]resolver.Endpoint, len(r.endpoints)) for i, ep := range r.endpoints { addr, serverName := endpoint.Interpret(ep) eps[i] = resolver.Endpoint{Addresses: []resolver.Address{ {Addr: addr, ServerName: serverName}, }} } state := resolver.State{ Endpoints: eps, ServiceConfig: r.serviceConfig, } r.UpdateState(state) } } func getCC(r EtcdManualResolver) (cc resolver.ClientConn) { defer func() { if rec := recover(); rec != nil { cc = nil } }() return r.CC() } ================================================ FILE: client/v3/kubernetes/client.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package kubernetes import ( "context" "fmt" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" ) // New creates Client from config. // Caller is responsible to call Close() to clean up client. func New(cfg clientv3.Config) (*Client, error) { c, err := clientv3.New(cfg) if err != nil { return nil, err } kc := &Client{ Client: c, } kc.Kubernetes = kc return kc, nil } type Client struct { *clientv3.Client Kubernetes Interface } var _ Interface = (*Client)(nil) func (k Client) Get(ctx context.Context, key string, opts GetOptions) (resp GetResponse, err error) { rangeResp, err := k.KV.Get(ctx, key, clientv3.WithRev(opts.Revision), clientv3.WithLimit(1)) if err != nil { return resp, err } resp.Revision = rangeResp.Header.Revision if len(rangeResp.Kvs) == 1 { resp.KV = rangeResp.Kvs[0] } return resp, nil } func (k Client) List(ctx context.Context, prefix string, opts ListOptions) (resp ListResponse, err error) { rangeStart := prefix if opts.Continue != "" { rangeStart = opts.Continue } rangeEnd := clientv3.GetPrefixRangeEnd(prefix) rangeResp, err := k.KV.Get(ctx, rangeStart, clientv3.WithRange(rangeEnd), clientv3.WithLimit(opts.Limit), clientv3.WithRev(opts.Revision)) if err != nil { return resp, err } resp.Kvs = rangeResp.Kvs resp.Count = rangeResp.Count resp.Revision = rangeResp.Header.Revision return resp, nil } func (k Client) Count(ctx context.Context, prefix string, _ CountOptions) (int64, error) { resp, err := k.KV.Get(ctx, prefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) if err != nil { return 0, err } return resp.Count, nil } func (k Client) OptimisticPut(ctx context.Context, key string, value []byte, expectedRevision int64, opts PutOptions) (resp PutResponse, err error) { txn := k.KV.Txn(ctx).If( clientv3.Compare(clientv3.ModRevision(key), "=", expectedRevision), ).Then( clientv3.OpPut(key, string(value), clientv3.WithLease(opts.LeaseID)), ) if opts.GetOnFailure { txn = txn.Else(clientv3.OpGet(key)) } txnResp, err := txn.Commit() if err != nil { return resp, err } resp.Succeeded = txnResp.Succeeded resp.Revision = txnResp.Header.Revision if opts.GetOnFailure && !txnResp.Succeeded { if len(txnResp.Responses) == 0 { return resp, fmt.Errorf("invalid OptimisticPut response: %v", txnResp.Responses) } resp.KV = kvFromTxnResponse(txnResp.Responses[0]) } return resp, nil } func (k Client) OptimisticDelete(ctx context.Context, key string, expectedRevision int64, opts DeleteOptions) (resp DeleteResponse, err error) { txn := k.KV.Txn(ctx).If( clientv3.Compare(clientv3.ModRevision(key), "=", expectedRevision), ).Then( clientv3.OpDelete(key), ) if opts.GetOnFailure { txn = txn.Else(clientv3.OpGet(key)) } txnResp, err := txn.Commit() if err != nil { return resp, err } resp.Succeeded = txnResp.Succeeded resp.Revision = txnResp.Header.Revision if opts.GetOnFailure && !txnResp.Succeeded { resp.KV = kvFromTxnResponse(txnResp.Responses[0]) } return resp, nil } func kvFromTxnResponse(resp *pb.ResponseOp) *mvccpb.KeyValue { getResponse := resp.GetResponseRange() if len(getResponse.Kvs) == 1 { return getResponse.Kvs[0] } return nil } ================================================ FILE: client/v3/kubernetes/interface.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package kubernetes import ( "context" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" ) // Interface defines the minimal client-side interface that Kubernetes requires // to interact with etcd. Methods below are standard etcd operations with // semantics adjusted to better suit Kubernetes' needs. type Interface interface { // Get retrieves a single key-value pair from etcd. // // If opts.Revision is set to a non-zero value, the key-value pair is retrieved at the specified revision. // If the required revision has been compacted, the request will fail with ErrCompacted. Get(ctx context.Context, key string, opts GetOptions) (GetResponse, error) // List retrieves key-value pairs with the specified prefix, ordered lexicographically by key. // // If opts.Revision is non-zero, the key-value pairs are retrieved at the specified revision. // If the required revision has been compacted, the request will fail with ErrCompacted. // If opts.Limit is greater than zero, the number of returned key-value pairs is bounded by the limit. // If opts.Continue is not empty, the listing will start from the key // specified by it. When paginating, the Continue value should be set // to the last observed key with "\x00" appended to it. List(ctx context.Context, prefix string, opts ListOptions) (ListResponse, error) // Count returns the number of keys with the specified prefix. // // Currently, there are no options for the Count operation. However, a placeholder options struct (CountOptions) // is provided for future extensibility in case options become necessary. Count(ctx context.Context, prefix string, opts CountOptions) (int64, error) // OptimisticPut creates or updates a key-value pair if the key has not been modified or created // since the revision specified in expectedRevision. // // An OptimisticPut fails if the key has been modified since expectedRevision. OptimisticPut(ctx context.Context, key string, value []byte, expectedRevision int64, opts PutOptions) (PutResponse, error) // OptimisticDelete deletes the key-value pair if it hasn't been modified since the revision // specified in expectedRevision. // // An OptimisticDelete fails if the key has been modified since expectedRevision. OptimisticDelete(ctx context.Context, key string, expectedRevision int64, opts DeleteOptions) (DeleteResponse, error) } type GetOptions struct { // Revision is the point-in-time of the etcd key-value store to use for the Get operation. // If Revision is 0, it gets the latest value. Revision int64 } type ListOptions struct { // Revision is the point-in-time of the etcd key-value store to use for the List operation. // If Revision is 0, it gets the latest values. Revision int64 // Limit is the maximum number of keys to return for a List operation. // 0 means no limitation. Limit int64 // Continue is a key from which to resume the List operation. // It should be set to the last key from a previous ListResponse // with "\x00" appended to it when paginating. Continue string } // CountOptions is a placeholder for potential future options for the Count operation. type CountOptions struct{} type PutOptions struct { // GetOnFailure specifies whether to return the modified key-value pair if the Put operation fails due to a revision mismatch. GetOnFailure bool // LeaseID is the ID of a lease to associate with the key allowing for automatic deletion after lease expires after it's TTL (time to live). // Deprecated: Should be replaced with TTL when Interface starts using one lease per object. LeaseID clientv3.LeaseID } type DeleteOptions struct { // GetOnFailure specifies whether to return the modified key-value pair if the Delete operation fails due to a revision mismatch. GetOnFailure bool } type GetResponse struct { // KV is the key-value pair retrieved from etcd. KV *mvccpb.KeyValue // Revision is the revision of the key-value store at the time of the Get operation. Revision int64 } type ListResponse struct { // Kvs is the list of key-value pairs retrieved from etcd, ordered lexicographically by key. Kvs []*mvccpb.KeyValue // Count is the total number of keys with the specified prefix, even if not all were returned due to a limit. Count int64 // Revision is the revision of the key-value store at the time of the List operation. Revision int64 } type PutResponse struct { // KV is the created or updated key-value pair. If the Put operation failed and GetOnFailure was true, this // will be the modified key-value pair that caused the failure. KV *mvccpb.KeyValue // Succeeded indicates whether the Put operation was successful. Succeeded bool // Revision is the revision of the key-value store after the Put operation. Revision int64 } type DeleteResponse struct { // KV is the deleted key-value pair. If the Delete operation failed and GetOnFailure was true, this // will be the modified key-value pair that caused the failure. KV *mvccpb.KeyValue // Succeeded indicates whether the Delete operation was successful. Succeeded bool // Revision is the revision of the key-value store after the Delete operation. Revision int64 } ================================================ FILE: client/v3/kv.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "google.golang.org/grpc" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" ) type ( CompactResponse pb.CompactionResponse PutResponse pb.PutResponse GetResponse pb.RangeResponse DeleteResponse pb.DeleteRangeResponse TxnResponse pb.TxnResponse ) type KV interface { // Put puts a key-value pair into etcd. // Note that key,value can be plain bytes array and string is // an immutable representation of that bytes array. // To get a string of bytes, do string([]byte{0x10, 0x20}). Put(ctx context.Context, key, val string, opts ...OpOption) (*PutResponse, error) // Get retrieves keys. // By default, Get will return the value for "key", if any. // When passed WithRange(end), Get will return the keys in the range [key, end). // When passed WithFromKey(), Get returns keys greater than or equal to key. // When passed WithRev(rev) with rev > 0, Get retrieves keys at the given revision; // if the required revision is compacted, the request will fail with ErrCompacted . // When passed WithLimit(limit), the number of returned keys is bounded by limit. // When passed WithSort(), the keys will be sorted. Get(ctx context.Context, key string, opts ...OpOption) (*GetResponse, error) // Delete deletes a key, or optionally using WithRange(end), [key, end). Delete(ctx context.Context, key string, opts ...OpOption) (*DeleteResponse, error) // Compact compacts etcd KV history before the given rev. Compact(ctx context.Context, rev int64, opts ...CompactOption) (*CompactResponse, error) // Do applies a single Op on KV without a transaction. // Do is useful when creating arbitrary operations to be issued at a // later time; the user can range over the operations, calling Do to // execute them. Get/Put/Delete, on the other hand, are best suited // for when the operation should be issued at the time of declaration. Do(ctx context.Context, op Op) (OpResponse, error) // Txn creates a transaction. Txn(ctx context.Context) Txn } type OpResponse struct { put *PutResponse get *GetResponse del *DeleteResponse txn *TxnResponse } func (op OpResponse) Put() *PutResponse { return op.put } func (op OpResponse) Get() *GetResponse { return op.get } func (op OpResponse) Del() *DeleteResponse { return op.del } func (op OpResponse) Txn() *TxnResponse { return op.txn } func (resp *PutResponse) OpResponse() OpResponse { return OpResponse{put: resp} } func (resp *GetResponse) OpResponse() OpResponse { return OpResponse{get: resp} } func (resp *DeleteResponse) OpResponse() OpResponse { return OpResponse{del: resp} } func (resp *TxnResponse) OpResponse() OpResponse { return OpResponse{txn: resp} } type kv struct { remote pb.KVClient callOpts []grpc.CallOption } func NewKV(c *Client) KV { api := &kv{remote: RetryKVClient(c)} if c != nil { api.callOpts = c.callOpts } return api } func NewKVFromKVClient(remote pb.KVClient, c *Client) KV { api := &kv{remote: remote} if c != nil { api.callOpts = c.callOpts } return api } func (kv *kv) Put(ctx context.Context, key, val string, opts ...OpOption) (*PutResponse, error) { r, err := kv.Do(ctx, OpPut(key, val, opts...)) return r.put, ContextError(ctx, err) } func (kv *kv) Get(ctx context.Context, key string, opts ...OpOption) (*GetResponse, error) { r, err := kv.Do(ctx, OpGet(key, opts...)) return r.get, ContextError(ctx, err) } func (kv *kv) Delete(ctx context.Context, key string, opts ...OpOption) (*DeleteResponse, error) { r, err := kv.Do(ctx, OpDelete(key, opts...)) return r.del, ContextError(ctx, err) } func (kv *kv) Compact(ctx context.Context, rev int64, opts ...CompactOption) (*CompactResponse, error) { resp, err := kv.remote.Compact(ctx, OpCompact(rev, opts...).toRequest(), kv.callOpts...) if err != nil { return nil, ContextError(ctx, err) } return (*CompactResponse)(resp), nil } func (kv *kv) Txn(ctx context.Context) Txn { return &txn{ kv: kv, ctx: ctx, callOpts: kv.callOpts, } } func (kv *kv) Do(ctx context.Context, op Op) (OpResponse, error) { var err error switch op.t { case tRange: if op.IsSortOptionValid() { var resp *pb.RangeResponse resp, err = kv.remote.Range(ctx, op.toRangeRequest(), kv.callOpts...) if err == nil { return OpResponse{get: (*GetResponse)(resp)}, nil } } else { err = rpctypes.ErrInvalidSortOption } case tPut: var resp *pb.PutResponse r := &pb.PutRequest{Key: op.key, Value: op.val, Lease: int64(op.leaseID), PrevKv: op.prevKV, IgnoreValue: op.ignoreValue, IgnoreLease: op.ignoreLease} resp, err = kv.remote.Put(ctx, r, kv.callOpts...) if err == nil { return OpResponse{put: (*PutResponse)(resp)}, nil } case tDeleteRange: var resp *pb.DeleteRangeResponse r := &pb.DeleteRangeRequest{Key: op.key, RangeEnd: op.end, PrevKv: op.prevKV} resp, err = kv.remote.DeleteRange(ctx, r, kv.callOpts...) if err == nil { return OpResponse{del: (*DeleteResponse)(resp)}, nil } case tTxn: var resp *pb.TxnResponse resp, err = kv.remote.Txn(ctx, op.toTxnRequest(), kv.callOpts...) if err == nil { return OpResponse{txn: (*TxnResponse)(resp)}, nil } default: panic("Unknown op") } return OpResponse{}, ContextError(ctx, err) } ================================================ FILE: client/v3/lease.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "errors" "sync" "time" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/metadata" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" ) type ( LeaseRevokeResponse pb.LeaseRevokeResponse LeaseID int64 ) // LeaseGrantResponse wraps the protobuf message LeaseGrantResponse. type LeaseGrantResponse struct { *pb.ResponseHeader ID LeaseID TTL int64 Error string } // LeaseKeepAliveResponse wraps the protobuf message LeaseKeepAliveResponse. type LeaseKeepAliveResponse struct { *pb.ResponseHeader ID LeaseID TTL int64 } // LeaseTimeToLiveResponse wraps the protobuf message LeaseTimeToLiveResponse. type LeaseTimeToLiveResponse struct { *pb.ResponseHeader ID LeaseID `json:"id"` // TTL is the remaining TTL in seconds for the lease; the lease will expire in under TTL+1 seconds. Expired lease will return -1. TTL int64 `json:"ttl"` // GrantedTTL is the initial granted time in seconds upon lease creation/renewal. GrantedTTL int64 `json:"granted-ttl"` // Keys is the list of keys attached to this lease. Keys [][]byte `json:"keys"` } // LeaseStatus represents a lease status. type LeaseStatus struct { ID LeaseID `json:"id"` // TODO: TTL int64 } // LeaseLeasesResponse wraps the protobuf message LeaseLeasesResponse. type LeaseLeasesResponse struct { *pb.ResponseHeader Leases []LeaseStatus `json:"leases"` } const ( // defaultTTL is the assumed lease TTL used for the first keepalive // deadline before the actual TTL is known to the client. defaultTTL = 5 * time.Second // NoLease is a lease ID for the absence of a lease. NoLease LeaseID = 0 // retryConnWait is how long to wait before retrying request due to an error retryConnWait = 500 * time.Millisecond ) // LeaseResponseChSize is the size of buffer to store unsent lease responses. // WARNING: DO NOT UPDATE. // Only for testing purposes. var LeaseResponseChSize = 16 // ErrKeepAliveHalted is returned if client keep alive loop halts with an unexpected error. // // This usually means that automatic lease renewal via KeepAlive is broken, but KeepAliveOnce will still work as expected. type ErrKeepAliveHalted struct { Reason error } func (e ErrKeepAliveHalted) Error() string { s := "etcdclient: leases keep alive halted" if e.Reason != nil { s += ": " + e.Reason.Error() } return s } type Lease interface { // Grant creates a new lease. Grant(ctx context.Context, ttl int64) (*LeaseGrantResponse, error) // Revoke revokes the given lease. Revoke(ctx context.Context, id LeaseID) (*LeaseRevokeResponse, error) // TimeToLive retrieves the lease information of the given lease ID. TimeToLive(ctx context.Context, id LeaseID, opts ...LeaseOption) (*LeaseTimeToLiveResponse, error) // Leases retrieves all leases. Leases(ctx context.Context) (*LeaseLeasesResponse, error) // KeepAlive attempts to keep the given lease alive forever. If the keepalive responses posted // to the channel are not consumed promptly the channel may become full. When full, the lease // client will continue sending keep alive requests to the etcd server, but will drop responses // until there is capacity on the channel to send more responses. // // If client keep alive loop halts with an unexpected error (e.g. "etcdserver: no leader") or // canceled by the caller (e.g. context.Canceled), KeepAlive returns a ErrKeepAliveHalted error // containing the error reason. // // The returned "LeaseKeepAliveResponse" channel closes if underlying keep // alive stream is interrupted in some way the client cannot handle itself; // given context "ctx" is canceled or timed out. // // TODO(v4.0): post errors to last keep alive message before closing // (see https://github.com/etcd-io/etcd/pull/7866) KeepAlive(ctx context.Context, id LeaseID) (<-chan *LeaseKeepAliveResponse, error) // KeepAliveOnce renews the lease once. The response corresponds to the // first message from calling KeepAlive. If the response has a recoverable // error, KeepAliveOnce will retry the RPC with a new keep alive message. // // In most of the cases, Keepalive should be used instead of KeepAliveOnce. KeepAliveOnce(ctx context.Context, id LeaseID) (*LeaseKeepAliveResponse, error) // Close releases all resources Lease keeps for efficient communication // with the etcd server. Close() error } type lessor struct { mu sync.Mutex // guards all fields // donec is closed and loopErr is set when recvKeepAliveLoop stops donec chan struct{} loopErr error remote pb.LeaseClient stream pb.Lease_LeaseKeepAliveClient streamCancel context.CancelFunc stopCtx context.Context stopCancel context.CancelFunc keepAlives map[LeaseID]*keepAlive // firstKeepAliveTimeout is the timeout for the first keepalive request // before the actual TTL is known to the lease client firstKeepAliveTimeout time.Duration // firstKeepAliveOnce ensures stream starts after first KeepAlive call. firstKeepAliveOnce sync.Once callOpts []grpc.CallOption lg *zap.Logger } // keepAlive multiplexes a keepalive for a lease over multiple channels type keepAlive struct { chs []chan<- *LeaseKeepAliveResponse ctxs []context.Context // deadline is the time the keep alive channels close if no response deadline time.Time // nextKeepAlive is when to send the next keep alive message nextKeepAlive time.Time // donec is closed on lease revoke, expiration, or cancel. donec chan struct{} } func NewLease(c *Client) Lease { return NewLeaseFromLeaseClient(RetryLeaseClient(c), c, c.cfg.DialTimeout+time.Second) } func NewLeaseFromLeaseClient(remote pb.LeaseClient, c *Client, keepAliveTimeout time.Duration) Lease { l := &lessor{ donec: make(chan struct{}), keepAlives: make(map[LeaseID]*keepAlive), remote: remote, firstKeepAliveTimeout: keepAliveTimeout, } if l.firstKeepAliveTimeout == time.Second { l.firstKeepAliveTimeout = defaultTTL } if c != nil { l.lg = c.GetLogger() l.callOpts = c.callOpts } reqLeaderCtx := WithRequireLeader(context.Background()) l.stopCtx, l.stopCancel = context.WithCancel(reqLeaderCtx) return l } func (l *lessor) Grant(ctx context.Context, ttl int64) (*LeaseGrantResponse, error) { r := &pb.LeaseGrantRequest{TTL: ttl} resp, err := l.remote.LeaseGrant(ctx, r, l.callOpts...) if err == nil { gresp := &LeaseGrantResponse{ ResponseHeader: resp.GetHeader(), ID: LeaseID(resp.ID), TTL: resp.TTL, Error: resp.Error, } return gresp, nil } return nil, ContextError(ctx, err) } func (l *lessor) Revoke(ctx context.Context, id LeaseID) (*LeaseRevokeResponse, error) { r := &pb.LeaseRevokeRequest{ID: int64(id)} resp, err := l.remote.LeaseRevoke(ctx, r, l.callOpts...) if err == nil { return (*LeaseRevokeResponse)(resp), nil } return nil, ContextError(ctx, err) } func (l *lessor) TimeToLive(ctx context.Context, id LeaseID, opts ...LeaseOption) (*LeaseTimeToLiveResponse, error) { r := toLeaseTimeToLiveRequest(id, opts...) resp, err := l.remote.LeaseTimeToLive(ctx, r, l.callOpts...) if err != nil { return nil, ContextError(ctx, err) } gresp := &LeaseTimeToLiveResponse{ ResponseHeader: resp.GetHeader(), ID: LeaseID(resp.ID), TTL: resp.TTL, GrantedTTL: resp.GrantedTTL, Keys: resp.Keys, } return gresp, nil } func (l *lessor) Leases(ctx context.Context) (*LeaseLeasesResponse, error) { resp, err := l.remote.LeaseLeases(ctx, &pb.LeaseLeasesRequest{}, l.callOpts...) if err == nil { leases := make([]LeaseStatus, len(resp.Leases)) for i := range resp.Leases { leases[i] = LeaseStatus{ID: LeaseID(resp.Leases[i].ID)} } return &LeaseLeasesResponse{ResponseHeader: resp.GetHeader(), Leases: leases}, nil } return nil, ContextError(ctx, err) } // To identify the context passed to `KeepAlive`, a key/value pair is // attached to the context. The key is a `keepAliveCtxKey` object, and // the value is the pointer to the context object itself, ensuring // uniqueness as each context has a unique memory address. type keepAliveCtxKey struct{} func (l *lessor) KeepAlive(ctx context.Context, id LeaseID) (<-chan *LeaseKeepAliveResponse, error) { ch := make(chan *LeaseKeepAliveResponse, LeaseResponseChSize) l.mu.Lock() // ensure that recvKeepAliveLoop is still running select { case <-l.donec: err := l.loopErr l.mu.Unlock() close(ch) return ch, ErrKeepAliveHalted{Reason: err} default: } ka, ok := l.keepAlives[id] if ctx.Done() != nil { ctx = context.WithValue(ctx, keepAliveCtxKey{}, &ctx) } if !ok { // create fresh keep alive ka = &keepAlive{ chs: []chan<- *LeaseKeepAliveResponse{ch}, ctxs: []context.Context{ctx}, deadline: time.Now().Add(l.firstKeepAliveTimeout), nextKeepAlive: time.Now(), donec: make(chan struct{}), } l.keepAlives[id] = ka } else { // add channel and context to existing keep alive ka.ctxs = append(ka.ctxs, ctx) ka.chs = append(ka.chs, ch) } l.mu.Unlock() if ctx.Done() != nil { go l.keepAliveCtxCloser(ctx, id, ka.donec) } l.firstKeepAliveOnce.Do(func() { go l.recvKeepAliveLoop() go l.deadlineLoop() }) return ch, nil } func (l *lessor) KeepAliveOnce(ctx context.Context, id LeaseID) (*LeaseKeepAliveResponse, error) { for { resp, err := l.keepAliveOnce(ctx, id) if err == nil { if resp.TTL <= 0 { err = rpctypes.ErrLeaseNotFound } return resp, err } if isHaltErr(ctx, err) { return nil, ContextError(ctx, err) } } } func (l *lessor) Close() error { l.stopCancel() // close for synchronous teardown if stream goroutines never launched l.firstKeepAliveOnce.Do(func() { close(l.donec) }) <-l.donec return nil } func (l *lessor) keepAliveCtxCloser(ctx context.Context, id LeaseID, donec <-chan struct{}) { select { case <-donec: return case <-l.donec: return case <-ctx.Done(): } l.mu.Lock() defer l.mu.Unlock() ka, ok := l.keepAlives[id] if !ok { return } // close channel and remove context if still associated with keep alive for i, c := range ka.ctxs { if c.Value(keepAliveCtxKey{}) == ctx.Value(keepAliveCtxKey{}) { close(ka.chs[i]) ka.ctxs = append(ka.ctxs[:i], ka.ctxs[i+1:]...) ka.chs = append(ka.chs[:i], ka.chs[i+1:]...) break } } // remove if no one more listeners if len(ka.chs) == 0 { delete(l.keepAlives, id) } } // closeRequireLeader scans keepAlives for ctxs that have require leader // and closes the associated channels. func (l *lessor) closeRequireLeader() { l.mu.Lock() defer l.mu.Unlock() for _, ka := range l.keepAlives { reqIdxs := 0 // find all required leader channels, close, mark as nil for i, ctx := range ka.ctxs { md, ok := metadata.FromOutgoingContext(ctx) if !ok { continue } ks := md[rpctypes.MetadataRequireLeaderKey] if len(ks) < 1 || ks[0] != rpctypes.MetadataHasLeader { continue } close(ka.chs[i]) ka.chs[i] = nil reqIdxs++ } if reqIdxs == 0 { continue } // remove all channels that required a leader from keepalive newChs := make([]chan<- *LeaseKeepAliveResponse, len(ka.chs)-reqIdxs) newCtxs := make([]context.Context, len(newChs)) newIdx := 0 for i := range ka.chs { if ka.chs[i] == nil { continue } newChs[newIdx], newCtxs[newIdx] = ka.chs[i], ka.ctxs[newIdx] newIdx++ } ka.chs, ka.ctxs = newChs, newCtxs } } func (l *lessor) keepAliveOnce(ctx context.Context, id LeaseID) (karesp *LeaseKeepAliveResponse, ferr error) { cctx, cancel := context.WithCancel(ctx) defer cancel() stream, err := l.remote.LeaseKeepAlive(cctx, l.callOpts...) if err != nil { return nil, ContextError(ctx, err) } defer func() { if cerr := stream.CloseSend(); cerr != nil { if ferr == nil { ferr = ContextError(ctx, cerr) } return } }() err = stream.Send(&pb.LeaseKeepAliveRequest{ID: int64(id)}) if err != nil { return nil, ContextError(ctx, err) } resp, rerr := stream.Recv() if rerr != nil { return nil, ContextError(ctx, rerr) } karesp = &LeaseKeepAliveResponse{ ResponseHeader: resp.GetHeader(), ID: LeaseID(resp.ID), TTL: resp.TTL, } return karesp, nil } func (l *lessor) recvKeepAliveLoop() (gerr error) { defer func() { l.mu.Lock() close(l.donec) l.loopErr = gerr for _, ka := range l.keepAlives { ka.close() } l.keepAlives = make(map[LeaseID]*keepAlive) l.mu.Unlock() }() for { stream, err := l.resetRecv() if err != nil { l.lg.Warn("error occurred during lease keep alive loop", zap.Error(err), ) if canceledByCaller(l.stopCtx, err) { return err } } else { for { resp, err := stream.Recv() if err != nil { if canceledByCaller(l.stopCtx, err) { return err } if errors.Is(ContextError(l.stopCtx, err), rpctypes.ErrNoLeader) { l.closeRequireLeader() } break } l.recvKeepAlive(resp) } } select { case <-time.After(retryConnWait): case <-l.stopCtx.Done(): return l.stopCtx.Err() } } } // resetRecv opens a new lease stream and starts sending keep alive requests. func (l *lessor) resetRecv() (pb.Lease_LeaseKeepAliveClient, error) { sctx, cancel := context.WithCancel(l.stopCtx) stream, err := l.remote.LeaseKeepAlive(sctx, append(l.callOpts, withMax(0))...) if err != nil { cancel() return nil, err } l.mu.Lock() defer l.mu.Unlock() if l.stream != nil && l.streamCancel != nil { l.streamCancel() } l.streamCancel = cancel l.stream = stream go l.sendKeepAliveLoop(stream) return stream, nil } // recvKeepAlive updates a lease based on its LeaseKeepAliveResponse func (l *lessor) recvKeepAlive(resp *pb.LeaseKeepAliveResponse) { karesp := &LeaseKeepAliveResponse{ ResponseHeader: resp.GetHeader(), ID: LeaseID(resp.ID), TTL: resp.TTL, } l.mu.Lock() defer l.mu.Unlock() ka, ok := l.keepAlives[karesp.ID] if !ok { return } if karesp.TTL <= 0 { // lease expired; close all keep alive channels delete(l.keepAlives, karesp.ID) ka.close() return } // send update to all channels nextKeepAlive := time.Now().Add((time.Duration(karesp.TTL) * time.Second) / 3.0) ka.deadline = time.Now().Add(time.Duration(karesp.TTL) * time.Second) for _, ch := range ka.chs { select { case ch <- karesp: default: if l.lg != nil { l.lg.Warn("lease keepalive response queue is full; dropping response send", zap.Int("queue-size", len(ch)), zap.Int("queue-capacity", cap(ch)), ) } } // still advance in order to rate-limit keep-alive sends ka.nextKeepAlive = nextKeepAlive } } // deadlineLoop reaps any keep alive channels that have not received a response // within the lease TTL func (l *lessor) deadlineLoop() { timer := time.NewTimer(time.Second) defer timer.Stop() for { timer.Reset(time.Second) select { case <-timer.C: case <-l.donec: return } now := time.Now() l.mu.Lock() for id, ka := range l.keepAlives { if ka.deadline.Before(now) { // waited too long for response; lease may be expired ka.close() delete(l.keepAlives, id) } } l.mu.Unlock() } } // sendKeepAliveLoop sends keep alive requests for the lifetime of the given stream. func (l *lessor) sendKeepAliveLoop(stream pb.Lease_LeaseKeepAliveClient) { for { var tosend []LeaseID now := time.Now() l.mu.Lock() for id, ka := range l.keepAlives { if ka.nextKeepAlive.Before(now) { tosend = append(tosend, id) } } l.mu.Unlock() for _, id := range tosend { r := &pb.LeaseKeepAliveRequest{ID: int64(id)} if err := stream.Send(r); err != nil { l.lg.Warn("error occurred during lease keep alive request sending", zap.Error(err), ) return } } select { case <-time.After(retryConnWait): case <-stream.Context().Done(): return case <-l.donec: return case <-l.stopCtx.Done(): return } } } func (ka *keepAlive) close() { close(ka.donec) for _, ch := range ka.chs { close(ch) } } ================================================ FILE: client/v3/leasing/cache.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package leasing import ( "context" "strings" "sync" "time" v3pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" v3 "go.etcd.io/etcd/client/v3" ) const revokeBackoff = 2 * time.Second type leaseCache struct { mu sync.RWMutex entries map[string]*leaseKey revokes map[string]time.Time header *v3pb.ResponseHeader } type leaseKey struct { response *v3.GetResponse // rev is the leasing key revision. rev int64 waitc chan struct{} } func (lc *leaseCache) Rev(key string) int64 { lc.mu.RLock() defer lc.mu.RUnlock() if li := lc.entries[key]; li != nil { return li.rev } return 0 } func (lc *leaseCache) Lock(key string) (chan<- struct{}, int64) { lc.mu.Lock() defer lc.mu.Unlock() if li := lc.entries[key]; li != nil { li.waitc = make(chan struct{}) return li.waitc, li.rev } return nil, 0 } func (lc *leaseCache) LockRange(begin, end string) (ret []chan<- struct{}) { lc.mu.Lock() defer lc.mu.Unlock() for k, li := range lc.entries { if inRange(k, begin, end) { li.waitc = make(chan struct{}) ret = append(ret, li.waitc) } } return ret } func inRange(k, begin, end string) bool { if strings.Compare(k, begin) < 0 { return false } if end != "\x00" && strings.Compare(k, end) >= 0 { return false } return true } func (lc *leaseCache) LockWriteOps(ops []v3.Op) (ret []chan<- struct{}) { for _, op := range ops { if op.IsGet() { continue } key := string(op.KeyBytes()) if end := string(op.RangeBytes()); end == "" { if wc, _ := lc.Lock(key); wc != nil { ret = append(ret, wc) } } else { for k := range lc.entries { if !inRange(k, key, end) { continue } if wc, _ := lc.Lock(k); wc != nil { ret = append(ret, wc) } } } } return ret } func (lc *leaseCache) NotifyOps(ops []v3.Op) (wcs []<-chan struct{}) { for _, op := range ops { if op.IsGet() { if _, wc := lc.notify(string(op.KeyBytes())); wc != nil { wcs = append(wcs, wc) } } } return wcs } func (lc *leaseCache) MayAcquire(key string) bool { lc.mu.RLock() lr, ok := lc.revokes[key] lc.mu.RUnlock() return !ok || time.Since(lr) > revokeBackoff } func (lc *leaseCache) Add(key string, resp *v3.GetResponse, op v3.Op) *v3.GetResponse { lk := &leaseKey{resp, resp.Header.Revision, closedCh} lc.mu.Lock() if lc.header == nil || lc.header.Revision < resp.Header.Revision { lc.header = resp.Header } lc.entries[key] = lk ret := lk.get(op) lc.mu.Unlock() return ret } func (lc *leaseCache) Update(key, val []byte, respHeader *v3pb.ResponseHeader) { li := lc.entries[string(key)] if li == nil { return } cacheResp := li.response if len(cacheResp.Kvs) == 0 { kv := &mvccpb.KeyValue{ Key: key, CreateRevision: respHeader.Revision, } cacheResp.Kvs = append(cacheResp.Kvs, kv) cacheResp.Count = 1 } cacheResp.Kvs[0].Version++ if cacheResp.Kvs[0].ModRevision < respHeader.Revision { cacheResp.Header = respHeader cacheResp.Kvs[0].ModRevision = respHeader.Revision cacheResp.Kvs[0].Value = val } } func (lc *leaseCache) Delete(key string, hdr *v3pb.ResponseHeader) { lc.mu.Lock() defer lc.mu.Unlock() lc.delete(key, hdr) } func (lc *leaseCache) delete(key string, hdr *v3pb.ResponseHeader) { if li := lc.entries[key]; li != nil && hdr.Revision >= li.response.Header.Revision { li.response.Kvs = nil li.response.Header = copyHeader(hdr) } } func (lc *leaseCache) Evict(key string) (rev int64) { lc.mu.Lock() defer lc.mu.Unlock() if li := lc.entries[key]; li != nil { rev = li.rev delete(lc.entries, key) lc.revokes[key] = time.Now() } return rev } func (lc *leaseCache) EvictRange(key, end string) { lc.mu.Lock() defer lc.mu.Unlock() for k := range lc.entries { if inRange(k, key, end) { delete(lc.entries, key) lc.revokes[key] = time.Now() } } } func isBadOp(op v3.Op) bool { return op.Rev() > 0 || len(op.RangeBytes()) > 0 } func (lc *leaseCache) Get(ctx context.Context, op v3.Op) (*v3.GetResponse, bool) { if isBadOp(op) { return nil, false } key := string(op.KeyBytes()) li, wc := lc.notify(key) if li == nil { return nil, true } select { case <-wc: case <-ctx.Done(): return nil, true } lc.mu.RLock() lk := *li ret := lk.get(op) lc.mu.RUnlock() return ret, true } func (lk *leaseKey) get(op v3.Op) *v3.GetResponse { ret := *lk.response ret.Header = copyHeader(ret.Header) empty := len(ret.Kvs) == 0 || op.IsCountOnly() empty = empty || (op.MinModRev() > ret.Kvs[0].ModRevision) empty = empty || (op.MaxModRev() != 0 && op.MaxModRev() < ret.Kvs[0].ModRevision) empty = empty || (op.MinCreateRev() > ret.Kvs[0].CreateRevision) empty = empty || (op.MaxCreateRev() != 0 && op.MaxCreateRev() < ret.Kvs[0].CreateRevision) if empty { ret.Kvs = nil } else { kv := *ret.Kvs[0] kv.Key = make([]byte, len(kv.Key)) copy(kv.Key, ret.Kvs[0].Key) if !op.IsKeysOnly() { kv.Value = make([]byte, len(kv.Value)) copy(kv.Value, ret.Kvs[0].Value) } ret.Kvs = []*mvccpb.KeyValue{&kv} } return &ret } func (lc *leaseCache) notify(key string) (*leaseKey, <-chan struct{}) { lc.mu.RLock() defer lc.mu.RUnlock() if li := lc.entries[key]; li != nil { return li, li.waitc } return nil, nil } func (lc *leaseCache) clearOldRevokes(ctx context.Context) { for { select { case <-ctx.Done(): return case <-time.After(time.Second): lc.mu.Lock() for k, lr := range lc.revokes { if time.Since(lr.Add(revokeBackoff)) > 0 { delete(lc.revokes, k) } } lc.mu.Unlock() } } } func (lc *leaseCache) evalCmp(cmps []v3.Cmp) (cmpVal bool, ok bool) { for _, cmp := range cmps { if len(cmp.RangeEnd) > 0 { return false, false } lk := lc.entries[string(cmp.Key)] if lk == nil { return false, false } if !evalCmp(lk.response, cmp) { return false, true } } return true, true } func (lc *leaseCache) evalOps(ops []v3.Op) ([]*v3pb.ResponseOp, bool) { resps := make([]*v3pb.ResponseOp, len(ops)) for i, op := range ops { if !op.IsGet() || isBadOp(op) { // TODO: support read-only Txn return nil, false } lk := lc.entries[string(op.KeyBytes())] if lk == nil { return nil, false } resp := lk.get(op) if resp == nil { return nil, false } resps[i] = &v3pb.ResponseOp{ Response: &v3pb.ResponseOp_ResponseRange{ ResponseRange: (*v3pb.RangeResponse)(resp), }, } } return resps, true } ================================================ FILE: client/v3/leasing/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package leasing serves linearizable reads from a local cache by acquiring // exclusive write access to keys through a client-side leasing protocol. This // leasing layer can either directly wrap the etcd client or it can be exposed // through the etcd grpc proxy server, granting multiple clients write access. // // First, create a leasing KV from a clientv3.Client 'cli': // // lkv, err := leasing.NewKV(cli, "leasing-prefix") // if err != nil { // // handle error // } // // A range request for a key "abc" tries to acquire a leasing key so it can cache the range's // key locally. On the server, the leasing key is stored to "leasing-prefix/abc": // // resp, err := lkv.Get(context.TODO(), "abc") // // Future linearized read requests using 'lkv' will be served locally for the lease's lifetime: // // resp, err = lkv.Get(context.TODO(), "abc") // // If another leasing client writes to a leased key, then the owner relinquishes its exclusive // access, permitting the writer to modify the key: // // lkv2, err := leasing.NewKV(cli, "leasing-prefix") // if err != nil { // // handle error // } // lkv2.Put(context.TODO(), "abc", "456") // resp, err = lkv.Get("abc") package leasing ================================================ FILE: client/v3/leasing/kv.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package leasing import ( "context" "errors" "strings" "sync" "time" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" ) type leasingKV struct { cl *v3.Client kv v3.KV pfx string leases leaseCache ctx context.Context cancel context.CancelFunc wg sync.WaitGroup sessionOpts []concurrency.SessionOption session *concurrency.Session sessionc chan struct{} } var closedCh chan struct{} func init() { closedCh = make(chan struct{}) close(closedCh) } // NewKV wraps a KV instance so that all requests are wired through a leasing protocol. func NewKV(cl *v3.Client, pfx string, opts ...concurrency.SessionOption) (v3.KV, func(), error) { cctx, cancel := context.WithCancel(cl.Ctx()) lkv := &leasingKV{ cl: cl, kv: cl.KV, pfx: pfx, leases: leaseCache{revokes: make(map[string]time.Time)}, ctx: cctx, cancel: cancel, sessionOpts: opts, sessionc: make(chan struct{}), } lkv.wg.Add(2) go func() { defer lkv.wg.Done() lkv.monitorSession() }() go func() { defer lkv.wg.Done() lkv.leases.clearOldRevokes(cctx) }() return lkv, lkv.Close, lkv.waitSession(cctx) } func (lkv *leasingKV) Close() { lkv.cancel() lkv.wg.Wait() } func (lkv *leasingKV) Get(ctx context.Context, key string, opts ...v3.OpOption) (*v3.GetResponse, error) { return lkv.get(ctx, v3.OpGet(key, opts...)) } func (lkv *leasingKV) Put(ctx context.Context, key, val string, opts ...v3.OpOption) (*v3.PutResponse, error) { return lkv.put(ctx, v3.OpPut(key, val, opts...)) } func (lkv *leasingKV) Delete(ctx context.Context, key string, opts ...v3.OpOption) (*v3.DeleteResponse, error) { return lkv.delete(ctx, v3.OpDelete(key, opts...)) } func (lkv *leasingKV) Do(ctx context.Context, op v3.Op) (v3.OpResponse, error) { switch { case op.IsGet(): resp, err := lkv.get(ctx, op) return resp.OpResponse(), err case op.IsPut(): resp, err := lkv.put(ctx, op) return resp.OpResponse(), err case op.IsDelete(): resp, err := lkv.delete(ctx, op) return resp.OpResponse(), err case op.IsTxn(): cmps, thenOps, elseOps := op.Txn() resp, err := lkv.Txn(ctx).If(cmps...).Then(thenOps...).Else(elseOps...).Commit() return resp.OpResponse(), err } return v3.OpResponse{}, nil } func (lkv *leasingKV) Compact(ctx context.Context, rev int64, opts ...v3.CompactOption) (*v3.CompactResponse, error) { return lkv.kv.Compact(ctx, rev, opts...) } func (lkv *leasingKV) Txn(ctx context.Context) v3.Txn { return &txnLeasing{Txn: lkv.kv.Txn(ctx), lkv: lkv, ctx: ctx} } func (lkv *leasingKV) monitorSession() { for lkv.ctx.Err() == nil { if lkv.session != nil { select { case <-lkv.session.Done(): case <-lkv.ctx.Done(): return } } lkv.leases.mu.Lock() select { case <-lkv.sessionc: lkv.sessionc = make(chan struct{}) default: } lkv.leases.entries = make(map[string]*leaseKey) lkv.leases.mu.Unlock() s, err := concurrency.NewSession(lkv.cl, lkv.sessionOpts...) if err != nil { continue } lkv.leases.mu.Lock() lkv.session = s close(lkv.sessionc) lkv.leases.mu.Unlock() } } func (lkv *leasingKV) monitorLease(ctx context.Context, key string, rev int64) { cctx, cancel := context.WithCancel(lkv.ctx) defer cancel() for cctx.Err() == nil { if rev == 0 { resp, err := lkv.kv.Get(ctx, lkv.pfx+key) if err != nil { continue } rev = resp.Header.Revision if len(resp.Kvs) == 0 || string(resp.Kvs[0].Value) == "REVOKE" { lkv.rescind(cctx, key, rev) return } } wch := lkv.cl.Watch(cctx, lkv.pfx+key, v3.WithRev(rev+1)) for resp := range wch { for _, ev := range resp.Events { if string(ev.Kv.Value) != "REVOKE" { continue } if v3.LeaseID(ev.Kv.Lease) == lkv.leaseID() { lkv.rescind(cctx, key, ev.Kv.ModRevision) } return } } rev = 0 } } // rescind releases a lease from this client. func (lkv *leasingKV) rescind(ctx context.Context, key string, rev int64) { if lkv.leases.Evict(key) > rev { return } cmp := v3.Compare(v3.CreateRevision(lkv.pfx+key), "<", rev) op := v3.OpDelete(lkv.pfx + key) for ctx.Err() == nil { if _, err := lkv.kv.Txn(ctx).If(cmp).Then(op).Commit(); err == nil { return } } } func (lkv *leasingKV) waitRescind(ctx context.Context, key string, rev int64) error { cctx, cancel := context.WithCancel(ctx) defer cancel() wch := lkv.cl.Watch(cctx, lkv.pfx+key, v3.WithRev(rev+1)) for resp := range wch { for _, ev := range resp.Events { if ev.Type == v3.EventTypeDelete { return ctx.Err() } } } return ctx.Err() } func (lkv *leasingKV) tryModifyOp(ctx context.Context, op v3.Op) (*v3.TxnResponse, chan<- struct{}, error) { key := string(op.KeyBytes()) wc, rev := lkv.leases.Lock(key) cmp := v3.Compare(v3.CreateRevision(lkv.pfx+key), "<", rev+1) resp, err := lkv.kv.Txn(ctx).If(cmp).Then(op).Commit() switch { case err != nil: lkv.leases.Evict(key) fallthrough case !resp.Succeeded: if wc != nil { close(wc) } return nil, nil, err } return resp, wc, nil } func (lkv *leasingKV) put(ctx context.Context, op v3.Op) (pr *v3.PutResponse, err error) { if err := lkv.waitSession(ctx); err != nil { return nil, err } for ctx.Err() == nil { resp, wc, err := lkv.tryModifyOp(ctx, op) if err != nil || wc == nil { resp, err = lkv.revoke(ctx, string(op.KeyBytes()), op) } if err != nil { return nil, err } if resp.Succeeded { lkv.leases.mu.Lock() lkv.leases.Update(op.KeyBytes(), op.ValueBytes(), resp.Header) lkv.leases.mu.Unlock() pr = (*v3.PutResponse)(resp.Responses[0].GetResponsePut()) pr.Header = resp.Header } if wc != nil { close(wc) } if resp.Succeeded { return pr, nil } } return nil, ctx.Err() } func (lkv *leasingKV) acquire(ctx context.Context, key string, op v3.Op) (*v3.TxnResponse, error) { for ctx.Err() == nil { if err := lkv.waitSession(ctx); err != nil { return nil, err } lcmp := v3.Cmp{Key: []byte(key), Target: pb.Compare_LEASE} resp, err := lkv.kv.Txn(ctx).If( v3.Compare(v3.CreateRevision(lkv.pfx+key), "=", 0), v3.Compare(lcmp, "=", 0)). Then( op, v3.OpPut(lkv.pfx+key, "", v3.WithLease(lkv.leaseID()))). Else( op, v3.OpGet(lkv.pfx+key), ).Commit() if err == nil { if !resp.Succeeded { kvs := resp.Responses[1].GetResponseRange().Kvs // if txn failed since already owner, lease is acquired resp.Succeeded = len(kvs) > 0 && v3.LeaseID(kvs[0].Lease) == lkv.leaseID() } return resp, nil } // retry if transient error var serverErr rpctypes.EtcdError if errors.As(err, &serverErr) { return nil, err } if ev, ok := status.FromError(err); ok && ev.Code() != codes.Unavailable { return nil, err } } return nil, ctx.Err() } func (lkv *leasingKV) get(ctx context.Context, op v3.Op) (*v3.GetResponse, error) { do := func() (*v3.GetResponse, error) { r, err := lkv.kv.Do(ctx, op) return r.Get(), err } if !lkv.readySession() { return do() } if resp, ok := lkv.leases.Get(ctx, op); resp != nil { return resp, nil } else if !ok || op.IsSerializable() { // must be handled by server or can skip linearization return do() } key := string(op.KeyBytes()) if !lkv.leases.MayAcquire(key) { resp, err := lkv.kv.Do(ctx, op) return resp.Get(), err } resp, err := lkv.acquire(ctx, key, v3.OpGet(key)) if err != nil { return nil, err } getResp := (*v3.GetResponse)(resp.Responses[0].GetResponseRange()) getResp.Header = resp.Header if resp.Succeeded { getResp = lkv.leases.Add(key, getResp, op) lkv.wg.Add(1) go func() { defer lkv.wg.Done() lkv.monitorLease(ctx, key, resp.Header.Revision) }() } return getResp, nil } func (lkv *leasingKV) deleteRangeRPC(ctx context.Context, maxLeaseRev int64, key, end string) (*v3.DeleteResponse, error) { lkey, lend := lkv.pfx+key, lkv.pfx+end resp, err := lkv.kv.Txn(ctx).If( v3.Compare(v3.CreateRevision(lkey).WithRange(lend), "<", maxLeaseRev+1), ).Then( v3.OpGet(key, v3.WithRange(end), v3.WithKeysOnly()), v3.OpDelete(key, v3.WithRange(end)), ).Commit() if err != nil { lkv.leases.EvictRange(key, end) return nil, err } if !resp.Succeeded { return nil, nil } for _, kv := range resp.Responses[0].GetResponseRange().Kvs { lkv.leases.Delete(string(kv.Key), resp.Header) } delResp := (*v3.DeleteResponse)(resp.Responses[1].GetResponseDeleteRange()) delResp.Header = resp.Header return delResp, nil } func (lkv *leasingKV) deleteRange(ctx context.Context, op v3.Op) (*v3.DeleteResponse, error) { key, end := string(op.KeyBytes()), string(op.RangeBytes()) for ctx.Err() == nil { maxLeaseRev, err := lkv.revokeRange(ctx, key, end) if err != nil { return nil, err } wcs := lkv.leases.LockRange(key, end) delResp, err := lkv.deleteRangeRPC(ctx, maxLeaseRev, key, end) closeAll(wcs) if err != nil || delResp != nil { return delResp, err } } return nil, ctx.Err() } func (lkv *leasingKV) delete(ctx context.Context, op v3.Op) (dr *v3.DeleteResponse, err error) { if err := lkv.waitSession(ctx); err != nil { return nil, err } if len(op.RangeBytes()) > 0 { return lkv.deleteRange(ctx, op) } key := string(op.KeyBytes()) for ctx.Err() == nil { resp, wc, err := lkv.tryModifyOp(ctx, op) if err != nil || wc == nil { resp, err = lkv.revoke(ctx, key, op) } if err != nil { // don't know if delete was processed lkv.leases.Evict(key) return nil, err } if resp.Succeeded { dr = (*v3.DeleteResponse)(resp.Responses[0].GetResponseDeleteRange()) dr.Header = resp.Header lkv.leases.Delete(key, dr.Header) } if wc != nil { close(wc) } if resp.Succeeded { return dr, nil } } return nil, ctx.Err() } func (lkv *leasingKV) revoke(ctx context.Context, key string, op v3.Op) (*v3.TxnResponse, error) { rev := lkv.leases.Rev(key) txn := lkv.kv.Txn(ctx).If(v3.Compare(v3.CreateRevision(lkv.pfx+key), "<", rev+1)).Then(op) resp, err := txn.Else(v3.OpPut(lkv.pfx+key, "REVOKE", v3.WithIgnoreLease())).Commit() if err != nil || resp.Succeeded { return resp, err } return resp, lkv.waitRescind(ctx, key, resp.Header.Revision) } func (lkv *leasingKV) revokeRange(ctx context.Context, begin, end string) (int64, error) { lkey, lend := lkv.pfx+begin, "" if len(end) > 0 { lend = lkv.pfx + end } leaseKeys, err := lkv.kv.Get(ctx, lkey, v3.WithRange(lend)) if err != nil { return 0, err } return lkv.revokeLeaseKvs(ctx, leaseKeys.Kvs) } func (lkv *leasingKV) revokeLeaseKvs(ctx context.Context, kvs []*mvccpb.KeyValue) (int64, error) { maxLeaseRev := int64(0) for _, kv := range kvs { if rev := kv.CreateRevision; rev > maxLeaseRev { maxLeaseRev = rev } if v3.LeaseID(kv.Lease) == lkv.leaseID() { // don't revoke own keys continue } key := strings.TrimPrefix(string(kv.Key), lkv.pfx) if _, err := lkv.revoke(ctx, key, v3.OpGet(key)); err != nil { return 0, err } } return maxLeaseRev, nil } func (lkv *leasingKV) waitSession(ctx context.Context) error { lkv.leases.mu.RLock() sessionc := lkv.sessionc lkv.leases.mu.RUnlock() select { case <-sessionc: return nil case <-lkv.ctx.Done(): return lkv.ctx.Err() case <-ctx.Done(): return ctx.Err() } } func (lkv *leasingKV) readySession() bool { lkv.leases.mu.RLock() defer lkv.leases.mu.RUnlock() if lkv.session == nil { return false } select { case <-lkv.session.Done(): default: return true } return false } func (lkv *leasingKV) leaseID() v3.LeaseID { lkv.leases.mu.RLock() defer lkv.leases.mu.RUnlock() return lkv.session.Lease() } ================================================ FILE: client/v3/leasing/txn.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package leasing import ( "context" "strings" v3pb "go.etcd.io/etcd/api/v3/etcdserverpb" v3 "go.etcd.io/etcd/client/v3" ) type txnLeasing struct { v3.Txn lkv *leasingKV ctx context.Context cs []v3.Cmp opst []v3.Op opse []v3.Op } func (txn *txnLeasing) If(cs ...v3.Cmp) v3.Txn { txn.cs = append(txn.cs, cs...) txn.Txn = txn.Txn.If(cs...) return txn } func (txn *txnLeasing) Then(ops ...v3.Op) v3.Txn { txn.opst = append(txn.opst, ops...) txn.Txn = txn.Txn.Then(ops...) return txn } func (txn *txnLeasing) Else(ops ...v3.Op) v3.Txn { txn.opse = append(txn.opse, ops...) txn.Txn = txn.Txn.Else(ops...) return txn } func (txn *txnLeasing) Commit() (*v3.TxnResponse, error) { if resp, err := txn.eval(); resp != nil || err != nil { return resp, err } return txn.serverTxn() } func (txn *txnLeasing) eval() (*v3.TxnResponse, error) { // TODO: wait on keys in comparisons thenOps, elseOps := gatherOps(txn.opst), gatherOps(txn.opse) ops := make([]v3.Op, 0, len(thenOps)+len(elseOps)) ops = append(ops, thenOps...) ops = append(ops, elseOps...) for _, ch := range txn.lkv.leases.NotifyOps(ops) { select { case <-ch: case <-txn.ctx.Done(): return nil, txn.ctx.Err() } } txn.lkv.leases.mu.RLock() defer txn.lkv.leases.mu.RUnlock() succeeded, ok := txn.lkv.leases.evalCmp(txn.cs) if !ok || txn.lkv.leases.header == nil { return nil, nil } if ops = txn.opst; !succeeded { ops = txn.opse } resps, ok := txn.lkv.leases.evalOps(ops) if !ok { return nil, nil } return &v3.TxnResponse{Header: copyHeader(txn.lkv.leases.header), Succeeded: succeeded, Responses: resps}, nil } // fallback computes the ops to fetch all possible conflicting // leasing keys for a list of ops. func (txn *txnLeasing) fallback(ops []v3.Op) (fbOps []v3.Op) { for _, op := range ops { if op.IsGet() { continue } lkey, lend := txn.lkv.pfx+string(op.KeyBytes()), "" if len(op.RangeBytes()) > 0 { lend = txn.lkv.pfx + string(op.RangeBytes()) } fbOps = append(fbOps, v3.OpGet(lkey, v3.WithRange(lend))) } return fbOps } func (txn *txnLeasing) guardKeys(ops []v3.Op) (cmps []v3.Cmp) { seen := make(map[string]bool) for _, op := range ops { key := string(op.KeyBytes()) if op.IsGet() || len(op.RangeBytes()) != 0 || seen[key] { continue } rev := txn.lkv.leases.Rev(key) cmps = append(cmps, v3.Compare(v3.CreateRevision(txn.lkv.pfx+key), "<", rev+1)) seen[key] = true } return cmps } func (txn *txnLeasing) guardRanges(ops []v3.Op) (cmps []v3.Cmp, err error) { for _, op := range ops { if op.IsGet() || len(op.RangeBytes()) == 0 { continue } key, end := string(op.KeyBytes()), string(op.RangeBytes()) maxRevLK, err := txn.lkv.revokeRange(txn.ctx, key, end) if err != nil { return nil, err } opts := append(v3.WithLastRev(), v3.WithRange(end)) getResp, err := txn.lkv.kv.Get(txn.ctx, key, opts...) if err != nil { return nil, err } maxModRev := int64(0) if len(getResp.Kvs) > 0 { maxModRev = getResp.Kvs[0].ModRevision } noKeyUpdate := v3.Compare(v3.ModRevision(key).WithRange(end), "<", maxModRev+1) noLeaseUpdate := v3.Compare( v3.CreateRevision(txn.lkv.pfx+key).WithRange(txn.lkv.pfx+end), "<", maxRevLK+1) cmps = append(cmps, noKeyUpdate, noLeaseUpdate) } return cmps, nil } func (txn *txnLeasing) guard(ops []v3.Op) ([]v3.Cmp, error) { cmps := txn.guardKeys(ops) rangeCmps, err := txn.guardRanges(ops) return append(cmps, rangeCmps...), err } func (txn *txnLeasing) commitToCache(txnResp *v3pb.TxnResponse, userTxn v3.Op) { ops := gatherResponseOps(txnResp.Responses, []v3.Op{userTxn}) txn.lkv.leases.mu.Lock() for _, op := range ops { key := string(op.KeyBytes()) if op.IsDelete() && len(op.RangeBytes()) > 0 { end := string(op.RangeBytes()) for k := range txn.lkv.leases.entries { if inRange(k, key, end) { txn.lkv.leases.delete(k, txnResp.Header) } } } else if op.IsDelete() { txn.lkv.leases.delete(key, txnResp.Header) } if op.IsPut() { txn.lkv.leases.Update(op.KeyBytes(), op.ValueBytes(), txnResp.Header) } } txn.lkv.leases.mu.Unlock() } func (txn *txnLeasing) revokeFallback(fbResps []*v3pb.ResponseOp) error { for _, resp := range fbResps { _, err := txn.lkv.revokeLeaseKvs(txn.ctx, resp.GetResponseRange().Kvs) if err != nil { return err } } return nil } func (txn *txnLeasing) serverTxn() (*v3.TxnResponse, error) { if err := txn.lkv.waitSession(txn.ctx); err != nil { return nil, err } userOps := gatherOps(append(txn.opst, txn.opse...)) userTxn := v3.OpTxn(txn.cs, txn.opst, txn.opse) fbOps := txn.fallback(userOps) defer closeAll(txn.lkv.leases.LockWriteOps(userOps)) for { cmps, err := txn.guard(userOps) if err != nil { return nil, err } resp, err := txn.lkv.kv.Txn(txn.ctx).If(cmps...).Then(userTxn).Else(fbOps...).Commit() if err != nil { for _, cmp := range cmps { txn.lkv.leases.Evict(strings.TrimPrefix(string(cmp.Key), txn.lkv.pfx)) } return nil, err } if resp.Succeeded { txn.commitToCache((*v3pb.TxnResponse)(resp), userTxn) userResp := resp.Responses[0].GetResponseTxn() userResp.Header = resp.Header return (*v3.TxnResponse)(userResp), nil } if err := txn.revokeFallback(resp.Responses); err != nil { return nil, err } } } ================================================ FILE: client/v3/leasing/util.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package leasing import ( "bytes" v3pb "go.etcd.io/etcd/api/v3/etcdserverpb" v3 "go.etcd.io/etcd/client/v3" ) func compareInt64(a, b int64) int { switch { case a < b: return -1 case a > b: return 1 default: return 0 } } func evalCmp(resp *v3.GetResponse, tcmp v3.Cmp) bool { var result int if len(resp.Kvs) != 0 { kv := resp.Kvs[0] switch tcmp.Target { case v3pb.Compare_VALUE: if tv, _ := tcmp.TargetUnion.(*v3pb.Compare_Value); tv != nil { result = bytes.Compare(kv.Value, tv.Value) } case v3pb.Compare_CREATE: if tv, _ := tcmp.TargetUnion.(*v3pb.Compare_CreateRevision); tv != nil { result = compareInt64(kv.CreateRevision, tv.CreateRevision) } case v3pb.Compare_MOD: if tv, _ := tcmp.TargetUnion.(*v3pb.Compare_ModRevision); tv != nil { result = compareInt64(kv.ModRevision, tv.ModRevision) } case v3pb.Compare_VERSION: if tv, _ := tcmp.TargetUnion.(*v3pb.Compare_Version); tv != nil { result = compareInt64(kv.Version, tv.Version) } } } switch tcmp.Result { case v3pb.Compare_EQUAL: return result == 0 case v3pb.Compare_NOT_EQUAL: return result != 0 case v3pb.Compare_GREATER: return result > 0 case v3pb.Compare_LESS: return result < 0 } return true } func gatherOps(ops []v3.Op) (ret []v3.Op) { for _, op := range ops { if !op.IsTxn() { ret = append(ret, op) continue } _, thenOps, elseOps := op.Txn() ret = append(ret, gatherOps(append(thenOps, elseOps...))...) } return ret } func gatherResponseOps(resp []*v3pb.ResponseOp, ops []v3.Op) (ret []v3.Op) { for i, op := range ops { if !op.IsTxn() { ret = append(ret, op) continue } _, thenOps, elseOps := op.Txn() if txnResp := resp[i].GetResponseTxn(); txnResp.Succeeded { ret = append(ret, gatherResponseOps(txnResp.Responses, thenOps)...) } else { ret = append(ret, gatherResponseOps(txnResp.Responses, elseOps)...) } } return ret } func copyHeader(hdr *v3pb.ResponseHeader) *v3pb.ResponseHeader { h := *hdr return &h } func closeAll(chs []chan<- struct{}) { for _, ch := range chs { close(ch) } } ================================================ FILE: client/v3/logger.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "log" "os" "go.uber.org/zap/zapcore" "go.uber.org/zap/zapgrpc" "google.golang.org/grpc/grpclog" "go.etcd.io/etcd/client/pkg/v3/logutil" ) func init() { // We override grpc logger only when the environment variable is set // in order to not interfere by default with user's code or other libraries. if os.Getenv("ETCD_CLIENT_DEBUG") != "" { lg, err := logutil.CreateDefaultZapLogger(ClientLogLevel()) if err != nil { panic(err) } lg = lg.Named("etcd-client") grpclog.SetLoggerV2(zapgrpc.NewLogger(lg)) } } // SetLogger sets grpc logger. // // Deprecated: use grpclog.SetLoggerV2 directly or grpc_zap.ReplaceGrpcLoggerV2. func SetLogger(l grpclog.LoggerV2) { grpclog.SetLoggerV2(l) } // ClientLogLevel translates ETCD_CLIENT_DEBUG into zap log level. func ClientLogLevel() zapcore.Level { envLevel := os.Getenv("ETCD_CLIENT_DEBUG") if envLevel == "" || envLevel == "true" { return zapcore.InfoLevel } var l zapcore.Level if err := l.Set(envLevel); err != nil { log.Print("Invalid value for environment variable 'ETCD_CLIENT_DEBUG'. Using default level: 'info'") return zapcore.InfoLevel } return l } ================================================ FILE: client/v3/main_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3_test import ( "testing" "time" "go.etcd.io/etcd/client/pkg/v3/testutil" ) const ( dialTimeout = 5 * time.Second requestTimeout = 10 * time.Second ) func exampleEndpoints() []string { return nil } func forUnitTestsRunInMockedContext(mocking func(), _example func()) { mocking() // TODO: Call 'example' when mocking() provides realistic mocking of transport. // The real testing logic of examples gets executed // as part of ./tests/integration/clientv3/integration/... } func TestMain(m *testing.M) { testutil.MustTestMainWithLeakDetection(m) } ================================================ FILE: client/v3/maintenance.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "errors" "fmt" "io" "go.uber.org/zap" "google.golang.org/grpc" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) type ( DefragmentResponse pb.DefragmentResponse AlarmResponse pb.AlarmResponse AlarmMember pb.AlarmMember StatusResponse pb.StatusResponse HashKVResponse pb.HashKVResponse MoveLeaderResponse pb.MoveLeaderResponse DowngradeResponse pb.DowngradeResponse DowngradeAction pb.DowngradeRequest_DowngradeAction ) const ( DowngradeValidate = DowngradeAction(pb.DowngradeRequest_VALIDATE) DowngradeEnable = DowngradeAction(pb.DowngradeRequest_ENABLE) DowngradeCancel = DowngradeAction(pb.DowngradeRequest_CANCEL) ) type Maintenance interface { // AlarmList gets all active alarms. AlarmList(ctx context.Context) (*AlarmResponse, error) // AlarmDisarm disarms a given alarm. AlarmDisarm(ctx context.Context, m *AlarmMember) (*AlarmResponse, error) // Defragment releases wasted space from internal fragmentation on a given etcd member. // Defragment is only needed when deleting a large number of keys and want to reclaim // the resources. // Defragment is an expensive operation. User should avoid defragmenting multiple members // at the same time. // To defragment multiple members in the cluster, user need to call defragment multiple // times with different endpoints. Defragment(ctx context.Context, endpoint string) (*DefragmentResponse, error) // Status gets the status of the endpoint. Status(ctx context.Context, endpoint string) (*StatusResponse, error) // HashKV returns a hash of the KV state at the time of the RPC. // If revision is zero, the hash is computed on all keys. If the revision // is non-zero, the hash is computed on all keys at or below the given revision. HashKV(ctx context.Context, endpoint string, rev int64) (*HashKVResponse, error) // SnapshotWithVersion returns a reader for a point-in-time snapshot and version of etcd that created it. // If the context "ctx" is canceled or timed out, reading from returned // "io.ReadCloser" would error out (e.g. context.Canceled, context.DeadlineExceeded). SnapshotWithVersion(ctx context.Context) (*SnapshotResponse, error) // Snapshot provides a reader for a point-in-time snapshot of etcd. // If the context "ctx" is canceled or timed out, reading from returned // "io.ReadCloser" would error out (e.g. context.Canceled, context.DeadlineExceeded). // Deprecated: use SnapshotWithVersion instead. Snapshot(ctx context.Context) (io.ReadCloser, error) // MoveLeader requests current leader to transfer its leadership to the transferee. // Request must be made to the leader. MoveLeader(ctx context.Context, transfereeID uint64) (*MoveLeaderResponse, error) // Downgrade requests downgrades, verifies feasibility or cancels downgrade // on the cluster version. // Supported since etcd 3.5. Downgrade(ctx context.Context, action DowngradeAction, version string) (*DowngradeResponse, error) } // SnapshotResponse is aggregated response from the snapshot stream. // Consumer is responsible for closing steam by calling .Snapshot.Close() type SnapshotResponse struct { // Header is the first header in the snapshot stream, has the current key-value store information // and indicates the point in time of the snapshot. Header *pb.ResponseHeader // Snapshot exposes ReaderCloser interface for data stored in the Blob field in the snapshot stream. Snapshot io.ReadCloser // Version is the local version of server that created the snapshot. // In cluster with binaries with different version, each cluster can return different result. // Informs which etcd server version should be used when restoring the snapshot. // Supported on etcd >= v3.6. Version string } type maintenance struct { lg *zap.Logger dial func(endpoint string) (pb.MaintenanceClient, func(), error) remote pb.MaintenanceClient callOpts []grpc.CallOption } func NewMaintenance(c *Client) Maintenance { api := &maintenance{ lg: c.GetLogger(), dial: func(endpoint string) (pb.MaintenanceClient, func(), error) { conn, err := c.Dial(endpoint) if err != nil { return nil, nil, fmt.Errorf("failed to dial endpoint %s with maintenance client: %w", endpoint, err) } cancel := func() { conn.Close() } return RetryMaintenanceClient(c, conn), cancel, nil }, remote: RetryMaintenanceClient(c, c.conn), } if c != nil { api.callOpts = c.callOpts } return api } func NewMaintenanceFromMaintenanceClient(remote pb.MaintenanceClient, c *Client) Maintenance { api := &maintenance{ dial: func(string) (pb.MaintenanceClient, func(), error) { return remote, func() {}, nil }, remote: remote, } if c != nil { api.callOpts = c.callOpts api.lg = c.GetLogger() } return api } func (m *maintenance) AlarmList(ctx context.Context) (*AlarmResponse, error) { req := &pb.AlarmRequest{ Action: pb.AlarmRequest_GET, MemberID: 0, // all Alarm: pb.AlarmType_NONE, // all } resp, err := m.remote.Alarm(ctx, req, m.callOpts...) if err == nil { return (*AlarmResponse)(resp), nil } return nil, ContextError(ctx, err) } func (m *maintenance) AlarmDisarm(ctx context.Context, am *AlarmMember) (*AlarmResponse, error) { req := &pb.AlarmRequest{ Action: pb.AlarmRequest_DEACTIVATE, MemberID: am.MemberID, Alarm: am.Alarm, } if req.MemberID == 0 && req.Alarm == pb.AlarmType_NONE { ar, err := m.AlarmList(ctx) if err != nil { return nil, ContextError(ctx, err) } ret := AlarmResponse{} for _, am := range ar.Alarms { dresp, derr := m.AlarmDisarm(ctx, (*AlarmMember)(am)) if derr != nil { return nil, ContextError(ctx, derr) } ret.Alarms = append(ret.Alarms, dresp.Alarms...) } return &ret, nil } resp, err := m.remote.Alarm(ctx, req, m.callOpts...) if err == nil { return (*AlarmResponse)(resp), nil } return nil, ContextError(ctx, err) } func (m *maintenance) Defragment(ctx context.Context, endpoint string) (*DefragmentResponse, error) { remote, cancel, err := m.dial(endpoint) if err != nil { return nil, ContextError(ctx, err) } defer cancel() resp, err := remote.Defragment(ctx, &pb.DefragmentRequest{}, m.callOpts...) if err != nil { return nil, ContextError(ctx, err) } return (*DefragmentResponse)(resp), nil } func (m *maintenance) Status(ctx context.Context, endpoint string) (*StatusResponse, error) { remote, cancel, err := m.dial(endpoint) if err != nil { return nil, ContextError(ctx, err) } defer cancel() resp, err := remote.Status(ctx, &pb.StatusRequest{}, m.callOpts...) if err != nil { return nil, ContextError(ctx, err) } return (*StatusResponse)(resp), nil } func (m *maintenance) HashKV(ctx context.Context, endpoint string, rev int64) (*HashKVResponse, error) { remote, cancel, err := m.dial(endpoint) if err != nil { return nil, ContextError(ctx, err) } defer cancel() resp, err := remote.HashKV(ctx, &pb.HashKVRequest{Revision: rev}, m.callOpts...) if err != nil { return nil, ContextError(ctx, err) } return (*HashKVResponse)(resp), nil } func (m *maintenance) SnapshotWithVersion(ctx context.Context) (*SnapshotResponse, error) { ss, err := m.remote.Snapshot(ctx, &pb.SnapshotRequest{}, append(m.callOpts, withMax(defaultStreamMaxRetries))...) if err != nil { return nil, ContextError(ctx, err) } m.lg.Info("opened snapshot stream; downloading") pr, pw := io.Pipe() resp, err := ss.Recv() if err != nil { m.logAndCloseWithError(err, pw) return nil, err } go func() { // Saving response is blocking err := m.save(resp, pw) if err != nil { m.logAndCloseWithError(err, pw) return } for { sresp, err := ss.Recv() if err != nil { m.logAndCloseWithError(err, pw) return } err = m.save(sresp, pw) if err != nil { m.logAndCloseWithError(err, pw) return } } }() return &SnapshotResponse{ Header: resp.GetHeader(), Snapshot: &snapshotReadCloser{ctx: ctx, ReadCloser: pr}, Version: resp.GetVersion(), }, nil } func (m *maintenance) Snapshot(ctx context.Context) (io.ReadCloser, error) { ss, err := m.remote.Snapshot(ctx, &pb.SnapshotRequest{}, append(m.callOpts, withMax(defaultStreamMaxRetries))...) if err != nil { return nil, ContextError(ctx, err) } m.lg.Info("opened snapshot stream; downloading") pr, pw := io.Pipe() go func() { for { resp, err := ss.Recv() if err != nil { m.logAndCloseWithError(err, pw) return } err = m.save(resp, pw) if err != nil { m.logAndCloseWithError(err, pw) return } } }() return &snapshotReadCloser{ctx: ctx, ReadCloser: pr}, nil } func (m *maintenance) logAndCloseWithError(err error, pw *io.PipeWriter) { switch { case errors.Is(err, io.EOF): m.lg.Info("completed snapshot read; closing") default: m.lg.Warn("failed to receive from snapshot stream; closing", zap.Error(err)) } pw.CloseWithError(err) } func (m *maintenance) save(resp *pb.SnapshotResponse, pw *io.PipeWriter) error { // can "resp == nil && err == nil" // before we receive snapshot SHA digest? // No, server sends EOF with an empty response // after it sends SHA digest at the end if _, werr := pw.Write(resp.Blob); werr != nil { return werr } return nil } type snapshotReadCloser struct { ctx context.Context io.ReadCloser } func (rc *snapshotReadCloser) Read(p []byte) (n int, err error) { n, err = rc.ReadCloser.Read(p) return n, ContextError(rc.ctx, err) } func (m *maintenance) MoveLeader(ctx context.Context, transfereeID uint64) (*MoveLeaderResponse, error) { resp, err := m.remote.MoveLeader(ctx, &pb.MoveLeaderRequest{TargetID: transfereeID}, m.callOpts...) return (*MoveLeaderResponse)(resp), ContextError(ctx, err) } func (m *maintenance) Downgrade(ctx context.Context, action DowngradeAction, version string) (*DowngradeResponse, error) { var actionType pb.DowngradeRequest_DowngradeAction switch action { case DowngradeValidate: actionType = pb.DowngradeRequest_VALIDATE case DowngradeEnable: actionType = pb.DowngradeRequest_ENABLE case DowngradeCancel: actionType = pb.DowngradeRequest_CANCEL default: return nil, errors.New("etcdclient: unknown downgrade action") } resp, err := m.remote.Downgrade(ctx, &pb.DowngradeRequest{Action: actionType, Version: version}, m.callOpts...) return (*DowngradeResponse)(resp), ContextError(ctx, err) } ================================================ FILE: client/v3/mirror/syncer.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package mirror implements etcd mirroring operations. package mirror import ( "context" clientv3 "go.etcd.io/etcd/client/v3" ) const ( batchLimit = 1000 ) // Syncer syncs with the key-value state of an etcd cluster. type Syncer interface { // SyncBase syncs the base state of the key-value state. // The key-value state are sent through the returned chan. SyncBase(ctx context.Context) (<-chan clientv3.GetResponse, chan error) // SyncUpdates syncs the updates of the key-value state. // The update events are sent through the returned chan. SyncUpdates(ctx context.Context) clientv3.WatchChan } // NewSyncer creates a Syncer. func NewSyncer(c *clientv3.Client, prefix string, rev int64) Syncer { return &syncer{c: c, prefix: prefix, rev: rev} } type syncer struct { c *clientv3.Client rev int64 prefix string } func (s *syncer) SyncBase(ctx context.Context) (<-chan clientv3.GetResponse, chan error) { respchan := make(chan clientv3.GetResponse, 1024) errchan := make(chan error, 1) // if rev is not specified, we will choose the most recent revision. if s.rev == 0 { // If len(s.prefix) == 0, we will check a random key to fetch the most recent // revision (foo), otherwise we use the provided prefix. checkPath := "foo" if len(s.prefix) != 0 { checkPath = s.prefix } resp, err := s.c.Get(ctx, checkPath) if err != nil { errchan <- err close(respchan) close(errchan) return respchan, errchan } s.rev = resp.Header.Revision } go func() { defer close(respchan) defer close(errchan) var key string opts := []clientv3.OpOption{ clientv3.WithLimit(batchLimit), clientv3.WithRev(s.rev), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), } if len(s.prefix) == 0 { // If len(s.prefix) == 0, we will sync the entire key-value space. // We then range from the smallest key (0x00) to the end. opts = append(opts, clientv3.WithFromKey()) key = "\x00" } else { // If len(s.prefix) != 0, we will sync key-value space with given prefix. // We then range from the prefix to the next prefix if exists. Or we will // range from the prefix to the end if the next prefix does not exists. opts = append(opts, clientv3.WithRange(clientv3.GetPrefixRangeEnd(s.prefix))) key = s.prefix } for { resp, err := s.c.Get(ctx, key, opts...) if err != nil { errchan <- err return } respchan <- *resp if !resp.More { return } // move to next key key = string(append(resp.Kvs[len(resp.Kvs)-1].Key, 0)) } }() return respchan, errchan } func (s *syncer) SyncUpdates(ctx context.Context) clientv3.WatchChan { if s.rev == 0 { panic("unexpected revision = 0. Calling SyncUpdates before SyncBase finishes?") } return s.c.Watch(ctx, s.prefix, clientv3.WithPrefix(), clientv3.WithRev(s.rev+1)) } ================================================ FILE: client/v3/mock/mockserver/doc.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package mockserver provides mock implementations for etcdserver's server interface. package mockserver ================================================ FILE: client/v3/mock/mockserver/mockserver.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mockserver import ( "context" "fmt" "net" "os" "sync" "google.golang.org/grpc" "google.golang.org/grpc/resolver" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) // MockServer provides a mocked out grpc server of the etcdserver interface. type MockServer struct { ln net.Listener Network string Address string GRPCServer *grpc.Server } func (ms *MockServer) ResolverAddress() resolver.Address { switch ms.Network { case "unix": return resolver.Address{Addr: fmt.Sprintf("unix://%s", ms.Address)} case "tcp": return resolver.Address{Addr: ms.Address} default: panic("illegal network type: " + ms.Network) } } // MockServers provides a cluster of mocket out gprc servers of the etcdserver interface. type MockServers struct { mu sync.RWMutex Servers []*MockServer wg sync.WaitGroup } // StartMockServers creates the desired count of mock servers // and starts them. func StartMockServers(count int) (ms *MockServers, err error) { return StartMockServersOnNetwork(count, "tcp") } // StartMockServersOnNetwork creates mock servers on either 'tcp' or 'unix' sockets. func StartMockServersOnNetwork(count int, network string) (ms *MockServers, err error) { switch network { case "tcp": return startMockServersTCP(count) case "unix": return startMockServersUnix(count) default: return nil, fmt.Errorf("unsupported network type: %s", network) } } func startMockServersTCP(count int) (ms *MockServers, err error) { addrs := make([]string, 0, count) for i := 0; i < count; i++ { addrs = append(addrs, "localhost:0") } return startMockServers("tcp", addrs) } func startMockServersUnix(count int) (ms *MockServers, err error) { dir := os.TempDir() addrs := make([]string, 0, count) for i := 0; i < count; i++ { f, err := os.CreateTemp(dir, "etcd-unix-so-") if err != nil { return nil, fmt.Errorf("failed to allocate temp file for unix socket: %w", err) } fn := f.Name() err = os.Remove(fn) if err != nil { return nil, fmt.Errorf("failed to remove temp file before creating unix socket: %w", err) } addrs = append(addrs, fn) } return startMockServers("unix", addrs) } func startMockServers(network string, addrs []string) (ms *MockServers, err error) { ms = &MockServers{ Servers: make([]*MockServer, len(addrs)), wg: sync.WaitGroup{}, } defer func() { if err != nil { ms.Stop() } }() for idx, addr := range addrs { ln, err := net.Listen(network, addr) if err != nil { return nil, fmt.Errorf("failed to listen %w", err) } ms.Servers[idx] = &MockServer{ln: ln, Network: network, Address: ln.Addr().String()} ms.StartAt(idx) } return ms, nil } // StartAt restarts mock server at given index. func (ms *MockServers) StartAt(idx int) (err error) { ms.mu.Lock() defer ms.mu.Unlock() if ms.Servers[idx].ln == nil { ms.Servers[idx].ln, err = net.Listen(ms.Servers[idx].Network, ms.Servers[idx].Address) if err != nil { return fmt.Errorf("failed to listen %w", err) } } svr := grpc.NewServer() pb.RegisterKVServer(svr, &mockKVServer{}) pb.RegisterLeaseServer(svr, &mockLeaseServer{}) ms.Servers[idx].GRPCServer = svr ms.wg.Add(1) go func(svr *grpc.Server, l net.Listener) { svr.Serve(l) }(ms.Servers[idx].GRPCServer, ms.Servers[idx].ln) return nil } // StopAt stops mock server at given index. func (ms *MockServers) StopAt(idx int) { ms.mu.Lock() defer ms.mu.Unlock() if ms.Servers[idx].ln == nil { return } ms.Servers[idx].GRPCServer.Stop() ms.Servers[idx].GRPCServer = nil ms.Servers[idx].ln = nil ms.wg.Done() } // Stop stops the mock server, immediately closing all open connections and listeners. func (ms *MockServers) Stop() { for idx := range ms.Servers { ms.StopAt(idx) } ms.wg.Wait() } type mockKVServer struct { // we want compile errors if new methods are added pb.UnsafeKVServer } func (m *mockKVServer) Range(context.Context, *pb.RangeRequest) (*pb.RangeResponse, error) { return &pb.RangeResponse{}, nil } func (m *mockKVServer) Put(context.Context, *pb.PutRequest) (*pb.PutResponse, error) { return &pb.PutResponse{}, nil } func (m *mockKVServer) DeleteRange(context.Context, *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) { return &pb.DeleteRangeResponse{}, nil } func (m *mockKVServer) Txn(context.Context, *pb.TxnRequest) (*pb.TxnResponse, error) { return &pb.TxnResponse{}, nil } func (m *mockKVServer) Compact(context.Context, *pb.CompactionRequest) (*pb.CompactionResponse, error) { return &pb.CompactionResponse{}, nil } func (m *mockKVServer) Lease(context.Context, *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { return &pb.LeaseGrantResponse{}, nil } type mockLeaseServer struct { // we want compile errors if new methods are added pb.UnsafeLeaseServer } func (s mockLeaseServer) LeaseGrant(context.Context, *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { return &pb.LeaseGrantResponse{}, nil } func (s *mockLeaseServer) LeaseRevoke(context.Context, *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) { return &pb.LeaseRevokeResponse{}, nil } func (s *mockLeaseServer) LeaseKeepAlive(pb.Lease_LeaseKeepAliveServer) error { return nil } func (s *mockLeaseServer) LeaseTimeToLive(context.Context, *pb.LeaseTimeToLiveRequest) (*pb.LeaseTimeToLiveResponse, error) { return &pb.LeaseTimeToLiveResponse{}, nil } func (s *mockLeaseServer) LeaseLeases(context.Context, *pb.LeaseLeasesRequest) (*pb.LeaseLeasesResponse, error) { return &pb.LeaseLeasesResponse{}, nil } ================================================ FILE: client/v3/namespace/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package namespace is a clientv3 wrapper that translates all keys to begin // with a given prefix. // // First, create a client: // // cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"localhost:2379"}}) // if err != nil { // // handle error! // } // // Next, override the client interfaces: // // unprefixedKV := cli.KV // cli.KV = namespace.NewKV(cli.KV, "my-prefix/") // cli.Watcher = namespace.NewWatcher(cli.Watcher, "my-prefix/") // cli.Lease = namespace.NewLease(cli.Lease, "my-prefix/") // // Now calls using 'cli' will namespace / prefix all keys with "my-prefix/": // // cli.Put(context.TODO(), "abc", "123") // resp, _ := unprefixedKV.Get(context.TODO(), "my-prefix/abc") // fmt.Printf("%s\n", resp.Kvs[0].Value) // // Output: 123 // unprefixedKV.Put(context.TODO(), "my-prefix/abc", "456") // resp, _ = cli.Get(context.TODO(), "abc") // fmt.Printf("%s\n", resp.Kvs[0].Value) // // Output: 456 package namespace ================================================ FILE: client/v3/namespace/kv.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package namespace import ( "context" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" ) type kvPrefix struct { clientv3.KV pfx string } // NewKV wraps a KV instance so that all requests // are prefixed with a given string. func NewKV(kv clientv3.KV, prefix string) clientv3.KV { return &kvPrefix{kv, prefix} } func (kv *kvPrefix) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { if len(key) == 0 { return nil, rpctypes.ErrEmptyKey } op := kv.prefixOp(clientv3.OpPut(key, val, opts...)) r, err := kv.KV.Do(ctx, op) if err != nil { return nil, err } put := r.Put() kv.unprefixPutResponse(put) return put, nil } func (kv *kvPrefix) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { if len(key) == 0 && !(clientv3.IsOptsWithFromKey(opts) || clientv3.IsOptsWithPrefix(opts)) { return nil, rpctypes.ErrEmptyKey } getOp := clientv3.OpGet(key, opts...) if !getOp.IsSortOptionValid() { return nil, rpctypes.ErrInvalidSortOption } r, err := kv.KV.Do(ctx, kv.prefixOp(getOp)) if err != nil { return nil, err } get := r.Get() kv.unprefixGetResponse(get) return get, nil } func (kv *kvPrefix) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { if len(key) == 0 && !(clientv3.IsOptsWithFromKey(opts) || clientv3.IsOptsWithPrefix(opts)) { return nil, rpctypes.ErrEmptyKey } r, err := kv.KV.Do(ctx, kv.prefixOp(clientv3.OpDelete(key, opts...))) if err != nil { return nil, err } del := r.Del() kv.unprefixDeleteResponse(del) return del, nil } func (kv *kvPrefix) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) { if len(op.KeyBytes()) == 0 && !op.IsTxn() { return clientv3.OpResponse{}, rpctypes.ErrEmptyKey } r, err := kv.KV.Do(ctx, kv.prefixOp(op)) if err != nil { return r, err } switch { case r.Get() != nil: kv.unprefixGetResponse(r.Get()) case r.Put() != nil: kv.unprefixPutResponse(r.Put()) case r.Del() != nil: kv.unprefixDeleteResponse(r.Del()) case r.Txn() != nil: kv.unprefixTxnResponse(r.Txn()) } return r, nil } type txnPrefix struct { clientv3.Txn kv *kvPrefix } func (kv *kvPrefix) Txn(ctx context.Context) clientv3.Txn { return &txnPrefix{kv.KV.Txn(ctx), kv} } func (txn *txnPrefix) If(cs ...clientv3.Cmp) clientv3.Txn { txn.Txn = txn.Txn.If(txn.kv.prefixCmps(cs)...) return txn } func (txn *txnPrefix) Then(ops ...clientv3.Op) clientv3.Txn { txn.Txn = txn.Txn.Then(txn.kv.prefixOps(ops)...) return txn } func (txn *txnPrefix) Else(ops ...clientv3.Op) clientv3.Txn { txn.Txn = txn.Txn.Else(txn.kv.prefixOps(ops)...) return txn } func (txn *txnPrefix) Commit() (*clientv3.TxnResponse, error) { resp, err := txn.Txn.Commit() if err != nil { return nil, err } txn.kv.unprefixTxnResponse(resp) return resp, nil } func (kv *kvPrefix) prefixOp(op clientv3.Op) clientv3.Op { if !op.IsTxn() { begin, end := kv.prefixInterval(op.KeyBytes(), op.RangeBytes()) op.WithKeyBytes(begin) op.WithRangeBytes(end) return op } cmps, thenOps, elseOps := op.Txn() return clientv3.OpTxn(kv.prefixCmps(cmps), kv.prefixOps(thenOps), kv.prefixOps(elseOps)) } func (kv *kvPrefix) unprefixGetResponse(resp *clientv3.GetResponse) { for i := range resp.Kvs { resp.Kvs[i].Key = resp.Kvs[i].Key[len(kv.pfx):] } } func (kv *kvPrefix) unprefixPutResponse(resp *clientv3.PutResponse) { if resp.PrevKv != nil { resp.PrevKv.Key = resp.PrevKv.Key[len(kv.pfx):] } } func (kv *kvPrefix) unprefixDeleteResponse(resp *clientv3.DeleteResponse) { for i := range resp.PrevKvs { resp.PrevKvs[i].Key = resp.PrevKvs[i].Key[len(kv.pfx):] } } func (kv *kvPrefix) unprefixTxnResponse(resp *clientv3.TxnResponse) { for _, r := range resp.Responses { switch tv := r.Response.(type) { case *pb.ResponseOp_ResponseRange: if tv.ResponseRange != nil { kv.unprefixGetResponse((*clientv3.GetResponse)(tv.ResponseRange)) } case *pb.ResponseOp_ResponsePut: if tv.ResponsePut != nil { kv.unprefixPutResponse((*clientv3.PutResponse)(tv.ResponsePut)) } case *pb.ResponseOp_ResponseDeleteRange: if tv.ResponseDeleteRange != nil { kv.unprefixDeleteResponse((*clientv3.DeleteResponse)(tv.ResponseDeleteRange)) } case *pb.ResponseOp_ResponseTxn: if tv.ResponseTxn != nil { kv.unprefixTxnResponse((*clientv3.TxnResponse)(tv.ResponseTxn)) } default: } } } func (kv *kvPrefix) prefixInterval(key, end []byte) (pfxKey []byte, pfxEnd []byte) { return prefixInterval(kv.pfx, key, end) } func (kv *kvPrefix) prefixCmps(cs []clientv3.Cmp) []clientv3.Cmp { newCmps := make([]clientv3.Cmp, len(cs)) for i := range cs { newCmps[i] = cs[i] pfxKey, endKey := kv.prefixInterval(cs[i].KeyBytes(), cs[i].RangeEnd) newCmps[i].WithKeyBytes(pfxKey) if len(cs[i].RangeEnd) != 0 { newCmps[i].RangeEnd = endKey } } return newCmps } func (kv *kvPrefix) prefixOps(ops []clientv3.Op) []clientv3.Op { newOps := make([]clientv3.Op, len(ops)) for i := range ops { newOps[i] = kv.prefixOp(ops[i]) } return newOps } ================================================ FILE: client/v3/namespace/lease.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package namespace import ( "bytes" "context" clientv3 "go.etcd.io/etcd/client/v3" ) type leasePrefix struct { clientv3.Lease pfx []byte } // NewLease wraps a Lease interface to filter for only keys with a prefix // and remove that prefix when fetching attached keys through TimeToLive. func NewLease(l clientv3.Lease, prefix string) clientv3.Lease { return &leasePrefix{l, []byte(prefix)} } func (l *leasePrefix) TimeToLive(ctx context.Context, id clientv3.LeaseID, opts ...clientv3.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) { resp, err := l.Lease.TimeToLive(ctx, id, opts...) if err != nil { return nil, err } if len(resp.Keys) > 0 { var outKeys [][]byte for i := range resp.Keys { if len(resp.Keys[i]) < len(l.pfx) { // too short continue } if !bytes.Equal(resp.Keys[i][:len(l.pfx)], l.pfx) { // doesn't match prefix continue } // strip prefix outKeys = append(outKeys, resp.Keys[i][len(l.pfx):]) } resp.Keys = outKeys } return resp, nil } ================================================ FILE: client/v3/namespace/util.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package namespace func prefixInterval(pfx string, key, end []byte) (pfxKey []byte, pfxEnd []byte) { pfxKey = make([]byte, len(pfx)+len(key)) copy(pfxKey[copy(pfxKey, pfx):], key) if len(end) == 1 && end[0] == 0 { // the edge of the keyspace pfxEnd = make([]byte, len(pfx)) copy(pfxEnd, pfx) ok := false for i := len(pfxEnd) - 1; i >= 0; i-- { if pfxEnd[i]++; pfxEnd[i] != 0 { ok = true break } } if !ok { // 0xff..ff => 0x00 pfxEnd = []byte{0} } } else if len(end) >= 1 { pfxEnd = make([]byte, len(pfx)+len(end)) copy(pfxEnd[copy(pfxEnd, pfx):], end) } return pfxKey, pfxEnd } ================================================ FILE: client/v3/namespace/util_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package namespace import ( "bytes" "testing" ) func TestPrefixInterval(t *testing.T) { tests := []struct { pfx string key []byte end []byte wKey []byte wEnd []byte }{ // single key { pfx: "pfx/", key: []byte("a"), wKey: []byte("pfx/a"), }, // range { pfx: "pfx/", key: []byte("abc"), end: []byte("def"), wKey: []byte("pfx/abc"), wEnd: []byte("pfx/def"), }, // one-sided range { pfx: "pfx/", key: []byte("abc"), end: []byte{0}, wKey: []byte("pfx/abc"), wEnd: []byte("pfx0"), }, // one-sided range, end of keyspace { pfx: "\xff\xff", key: []byte("abc"), end: []byte{0}, wKey: []byte("\xff\xffabc"), wEnd: []byte{0}, }, } for i, tt := range tests { pfxKey, pfxEnd := prefixInterval(tt.pfx, tt.key, tt.end) if !bytes.Equal(pfxKey, tt.wKey) { t.Errorf("#%d: expected key=%q, got key=%q", i, tt.wKey, pfxKey) } if !bytes.Equal(pfxEnd, tt.wEnd) { t.Errorf("#%d: expected end=%q, got end=%q", i, tt.wEnd, pfxEnd) } } } ================================================ FILE: client/v3/namespace/watch.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package namespace import ( "context" "sync" clientv3 "go.etcd.io/etcd/client/v3" ) type watcherPrefix struct { clientv3.Watcher pfx string wg sync.WaitGroup stopc chan struct{} stopOnce sync.Once } // NewWatcher wraps a Watcher instance so that all Watch requests // are prefixed with a given string and all Watch responses have // the prefix removed. func NewWatcher(w clientv3.Watcher, prefix string) clientv3.Watcher { return &watcherPrefix{Watcher: w, pfx: prefix, stopc: make(chan struct{})} } func (w *watcherPrefix) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan { // since OpOption is opaque, determine range for prefixing through an OpGet op := clientv3.OpGet(key, opts...) end := op.RangeBytes() pfxBegin, pfxEnd := prefixInterval(w.pfx, []byte(key), end) if pfxEnd != nil { opts = append(opts, clientv3.WithRange(string(pfxEnd))) } wch := w.Watcher.Watch(ctx, string(pfxBegin), opts...) // translate watch events from prefixed to unprefixed pfxWch := make(chan clientv3.WatchResponse) w.wg.Add(1) go func() { defer func() { close(pfxWch) w.wg.Done() }() for wr := range wch { for i := range wr.Events { wr.Events[i].Kv.Key = wr.Events[i].Kv.Key[len(w.pfx):] if wr.Events[i].PrevKv != nil { wr.Events[i].PrevKv.Key = wr.Events[i].Kv.Key } } select { case pfxWch <- wr: case <-ctx.Done(): return case <-w.stopc: return } } }() return pfxWch } func (w *watcherPrefix) Close() error { err := w.Watcher.Close() w.stopOnce.Do(func() { close(w.stopc) }) w.wg.Wait() return err } ================================================ FILE: client/v3/naming/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package naming provides: // - subpackage endpoints: an abstraction layer to store and read endpoints // information from etcd. // - subpackage resolver: an etcd-backed gRPC resolver for discovering gRPC // services based on the endpoints configuration // // To use, first import the packages: // // import ( // "go.etcd.io/etcd/client/v3" // "go.etcd.io/etcd/client/v3/naming/endpoints" // "go.etcd.io/etcd/client/v3/naming/resolver" // "google.golang.org/grpc" // ) // // First, register new endpoint addresses for a service: // // func etcdAdd(c *clientv3.Client, service, addr string) error { // em := endpoints.NewManager(c, service) // return em.AddEndpoint(c.Ctx(), service+"/"+addr, endpoints.Endpoint{Addr:addr}); // } // // Dial an RPC service using the etcd gRPC resolver and a gRPC Balancer: // // func etcdDial(c *clientv3.Client, service string) (*grpc.ClientConn, error) { // etcdResolver, err := resolver.NewBuilder(c); // if err { return nil, err } // conn, err := grpc.NewClient("etcd:///"+service, grpc.WithResolvers(etcdResolver)) // if err != nil { return nil, err } // return conn, nil // } // // Optionally, force delete an endpoint: // // func etcdDelete(c *clientv3, service, addr string) error { // em := endpoints.NewManager(c, service) // return em.DeleteEndpoint(c.Ctx(), service+"/"+addr) // } // // Or register an expiring endpoint with a lease: // // func etcdAdd(c *clientv3.Client, lid clientv3.LeaseID, service, addr string) error { // em := endpoints.NewManager(c, service) // return em.AddEndpoint(c.Ctx(), service+"/"+addr, endpoints.Endpoint{Addr:addr}, clientv3.WithLease(lid)); // } package naming ================================================ FILE: client/v3/naming/endpoints/endpoints.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package endpoints import ( "context" clientv3 "go.etcd.io/etcd/client/v3" ) // Endpoint represents a single address the connection can be established with. // // Inspired by: https://pkg.go.dev/google.golang.org/grpc/resolver#Address. // Please document etcd version since which version each field is supported. type Endpoint struct { // Addr is the server address on which a connection will be established. // Since etcd 3.1 Addr string // Metadata is the information associated with Addr. // Since etcd 3.1 Metadata any } type Operation uint8 const ( // Add indicates an Endpoint is added. Add Operation = iota // Delete indicates an existing address is deleted. Delete ) // Update describes a single edit action of an Endpoint. type Update struct { // Op - action Add or Delete. Op Operation Key string Endpoint Endpoint } // WatchChannel is used to deliver notifications about endpoints updates. type WatchChannel <-chan []*Update // Key2EndpointMap maps etcd key into struct describing the endpoint. type Key2EndpointMap map[string]Endpoint // UpdateWithOpts describes endpoint update (add or delete) together // with etcd options (e.g. to attach an endpoint to a lease). type UpdateWithOpts struct { Update Opts []clientv3.OpOption } // NewAddUpdateOpts constructs UpdateWithOpts for endpoint registration. func NewAddUpdateOpts(key string, endpoint Endpoint, opts ...clientv3.OpOption) *UpdateWithOpts { return &UpdateWithOpts{Update: Update{Op: Add, Key: key, Endpoint: endpoint}, Opts: opts} } // NewDeleteUpdateOpts constructs UpdateWithOpts for endpoint deletion. func NewDeleteUpdateOpts(key string, opts ...clientv3.OpOption) *UpdateWithOpts { return &UpdateWithOpts{Update: Update{Op: Delete, Key: key}, Opts: opts} } // Manager can be used to add/remove & inspect endpoints stored in etcd for // a particular target. type Manager interface { // Update allows to atomically add/remove a few endpoints from etcd. Update(ctx context.Context, updates []*UpdateWithOpts) error // AddEndpoint registers a single endpoint in etcd. // For more advanced use-cases use the Update method. AddEndpoint(ctx context.Context, key string, endpoint Endpoint, opts ...clientv3.OpOption) error // DeleteEndpoint deletes a single endpoint stored in etcd. // For more advanced use-cases use the Update method. DeleteEndpoint(ctx context.Context, key string, opts ...clientv3.OpOption) error // List returns all the endpoints for the current target as a map. List(ctx context.Context) (Key2EndpointMap, error) // NewWatchChannel creates a channel that populates or endpoint updates. // Cancel the 'ctx' to close the watcher. NewWatchChannel(ctx context.Context) (WatchChannel, error) } ================================================ FILE: client/v3/naming/endpoints/endpoints_impl.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package endpoints // TODO: The API is not yet implemented. import ( "context" "encoding/json" "errors" "strings" "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/naming/endpoints/internal" ) type endpointManager struct { // Client is an initialized etcd client. client *clientv3.Client target string } // NewManager creates an endpoint manager which implements the interface of 'Manager'. func NewManager(client *clientv3.Client, target string) (Manager, error) { if client == nil { return nil, errors.New("invalid etcd client") } if target == "" { return nil, errors.New("invalid target") } em := &endpointManager{ client: client, target: target, } return em, nil } func (m *endpointManager) Update(ctx context.Context, updates []*UpdateWithOpts) (err error) { ops := make([]clientv3.Op, 0, len(updates)) for _, update := range updates { if !strings.HasPrefix(update.Key, m.target+"/") { return status.Errorf(codes.InvalidArgument, "endpoints: endpoint key should be prefixed with '%s/' got: '%s'", m.target, update.Key) } switch update.Op { case Add: internalUpdate := &internal.Update{ Op: internal.Add, Addr: update.Endpoint.Addr, Metadata: update.Endpoint.Metadata, } var v []byte if v, err = json.Marshal(internalUpdate); err != nil { return status.Error(codes.InvalidArgument, err.Error()) } ops = append(ops, clientv3.OpPut(update.Key, string(v), update.Opts...)) case Delete: ops = append(ops, clientv3.OpDelete(update.Key, update.Opts...)) default: return status.Error(codes.InvalidArgument, "endpoints: bad update op") } } _, err = m.client.KV.Txn(ctx).Then(ops...).Commit() return err } func (m *endpointManager) AddEndpoint(ctx context.Context, key string, endpoint Endpoint, opts ...clientv3.OpOption) error { return m.Update(ctx, []*UpdateWithOpts{NewAddUpdateOpts(key, endpoint, opts...)}) } func (m *endpointManager) DeleteEndpoint(ctx context.Context, key string, opts ...clientv3.OpOption) error { return m.Update(ctx, []*UpdateWithOpts{NewDeleteUpdateOpts(key, opts...)}) } func (m *endpointManager) NewWatchChannel(ctx context.Context) (WatchChannel, error) { key := m.target + "/" resp, err := m.client.Get(ctx, key, clientv3.WithPrefix(), clientv3.WithSerializable()) if err != nil { return nil, err } lg := m.client.GetLogger() initUpdates := make([]*Update, 0, len(resp.Kvs)) for _, kv := range resp.Kvs { var iup internal.Update if err := json.Unmarshal(kv.Value, &iup); err != nil { lg.Warn("unmarshal endpoint update failed", zap.String("key", string(kv.Key)), zap.Error(err)) continue } up := &Update{ Op: Add, Key: string(kv.Key), Endpoint: Endpoint{Addr: iup.Addr, Metadata: iup.Metadata}, } initUpdates = append(initUpdates, up) } upch := make(chan []*Update, 1) if len(initUpdates) > 0 { upch <- initUpdates } go m.watch(ctx, resp.Header.Revision+1, upch) return upch, nil } func (m *endpointManager) watch(ctx context.Context, rev int64, upch chan []*Update) { defer close(upch) lg := m.client.GetLogger() opts := []clientv3.OpOption{clientv3.WithRev(rev), clientv3.WithPrefix()} key := m.target + "/" wch := m.client.Watch(ctx, key, opts...) for { select { case <-ctx.Done(): return case wresp, ok := <-wch: if !ok { lg.Warn("watch closed", zap.String("target", m.target)) return } if wresp.Err() != nil { lg.Warn("watch failed", zap.String("target", m.target), zap.Error(wresp.Err())) return } deltaUps := make([]*Update, 0, len(wresp.Events)) for _, e := range wresp.Events { var iup internal.Update var err error var op Operation switch e.Type { case clientv3.EventTypePut: err = json.Unmarshal(e.Kv.Value, &iup) op = Add if err != nil { lg.Warn("unmarshal endpoint update failed", zap.String("key", string(e.Kv.Key)), zap.Error(err)) continue } case clientv3.EventTypeDelete: iup = internal.Update{Op: internal.Delete} op = Delete default: continue } up := &Update{Op: op, Key: string(e.Kv.Key), Endpoint: Endpoint{Addr: iup.Addr, Metadata: iup.Metadata}} deltaUps = append(deltaUps, up) } if len(deltaUps) > 0 { upch <- deltaUps } } } } func (m *endpointManager) List(ctx context.Context) (Key2EndpointMap, error) { key := m.target + "/" resp, err := m.client.Get(ctx, key, clientv3.WithPrefix(), clientv3.WithSerializable()) if err != nil { return nil, err } eps := make(Key2EndpointMap) for _, kv := range resp.Kvs { var iup internal.Update if err := json.Unmarshal(kv.Value, &iup); err != nil { continue } eps[string(kv.Key)] = Endpoint{Addr: iup.Addr, Metadata: iup.Metadata} } return eps, nil } ================================================ FILE: client/v3/naming/endpoints/internal/update.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package internal // Operation describes action performed on endpoint (addition vs deletion). // Must stay JSON-format compatible with: // https://pkg.go.dev/google.golang.org/grpc@v1.29.1/naming#Operation type Operation uint8 const ( // Add indicates a new address is added. Add Operation = iota // Delete indicates an existing address is deleted. Delete ) // Update defines a persistent (JSON marshalled) format representing // endpoint within the etcd storage. // // As the format can be persisted by one version of etcd client library and // read by other the format must be kept backward compatible and // in particular must be superset of the grpc(<=1.29.1) naming.Update structure: // https://pkg.go.dev/google.golang.org/grpc@v1.29.1/naming#Update // // Please document since which version of etcd-client given property is supported. // Please keep the naming consistent with e.g. https://pkg.go.dev/google.golang.org/grpc/resolver#Address. // // Notice that it is not valid having both empty string Addr and nil Metadata in an Update. type Update struct { // Op indicates the operation of the update. // Since etcd 3.1. Op Operation // Addr is the updated address. It is empty string if there is no address update. // Since etcd 3.1. Addr string // Metadata is the updated metadata. It is nil if there is no metadata update. // Metadata is not required for a custom naming implementation. // Since etcd 3.1. Metadata any } ================================================ FILE: client/v3/naming/resolver/resolver.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package resolver import ( "context" "strings" "sync" "google.golang.org/grpc/codes" gresolver "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/naming/endpoints" ) type builder struct { c *clientv3.Client } func (b builder) Build(target gresolver.Target, cc gresolver.ClientConn, opts gresolver.BuildOptions) (gresolver.Resolver, error) { // Refer to https://github.com/grpc/grpc-go/blob/16d3df80f029f57cff5458f1d6da6aedbc23545d/clientconn.go#L1587-L1611 endpoint := target.URL.Path if endpoint == "" { endpoint = target.URL.Opaque } endpoint = strings.TrimPrefix(endpoint, "/") r := &resolver{ c: b.c, target: endpoint, cc: cc, } r.ctx, r.cancel = context.WithCancel(context.Background()) em, err := endpoints.NewManager(r.c, r.target) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "resolver: failed to new endpoint manager: %s", err) } r.wch, err = em.NewWatchChannel(r.ctx) if err != nil { return nil, status.Errorf(codes.Internal, "resolver: failed to new watch channer: %s", err) } r.wg.Add(1) go r.watch() return r, nil } func (b builder) Scheme() string { return "etcd" } // NewBuilder creates a resolver builder. func NewBuilder(client *clientv3.Client) (gresolver.Builder, error) { return builder{c: client}, nil } type resolver struct { c *clientv3.Client target string cc gresolver.ClientConn wch endpoints.WatchChannel ctx context.Context cancel context.CancelFunc wg sync.WaitGroup } func (r *resolver) watch() { defer r.wg.Done() allUps := make(map[string]*endpoints.Update) for { select { case <-r.ctx.Done(): return case ups, ok := <-r.wch: if !ok { return } for _, up := range ups { switch up.Op { case endpoints.Add: allUps[up.Key] = up case endpoints.Delete: delete(allUps, up.Key) } } eps := convertToGRPCEndpoint(allUps) r.cc.UpdateState(gresolver.State{Endpoints: eps}) } } } func convertToGRPCEndpoint(ups map[string]*endpoints.Update) []gresolver.Endpoint { var eps []gresolver.Endpoint for _, up := range ups { ep := gresolver.Endpoint{ Addresses: []gresolver.Address{ { Addr: up.Endpoint.Addr, }, }, } eps = append(eps, ep) } return eps } // ResolveNow is a no-op here. // It's just a hint, resolver can ignore this if it's not necessary. func (r *resolver) ResolveNow(gresolver.ResolveNowOptions) {} func (r *resolver) Close() { r.cancel() r.wg.Wait() } ================================================ FILE: client/v3/op.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import pb "go.etcd.io/etcd/api/v3/etcdserverpb" type opType int const ( // A default Op has opType 0, which is invalid. tRange opType = iota + 1 tPut tDeleteRange tTxn ) var noPrefixEnd = []byte{0} // Op represents an Operation that kv can execute. type Op struct { t opType key []byte end []byte // for range limit int64 sort *SortOption serializable bool keysOnly bool countOnly bool minModRev int64 maxModRev int64 minCreateRev int64 maxCreateRev int64 // for range, watch rev int64 // for watch, put, delete prevKV bool // for watch // fragmentation should be disabled by default // if true, split watch events when total exceeds // "--max-request-bytes" flag value + 512-byte fragment bool // for put ignoreValue bool ignoreLease bool // progressNotify is for progress updates. progressNotify bool // createdNotify is for created event createdNotify bool // filters for watchers filterPut bool filterDelete bool // for put val []byte leaseID LeaseID // txn cmps []Cmp thenOps []Op elseOps []Op isOptsWithFromKey bool isOptsWithPrefix bool } // accessors / mutators // IsTxn returns true if the "Op" type is transaction. func (op Op) IsTxn() bool { return op.t == tTxn } // Txn returns the comparison(if) operations, "then" operations, and "else" operations. func (op Op) Txn() ([]Cmp, []Op, []Op) { return op.cmps, op.thenOps, op.elseOps } // KeyBytes returns the byte slice holding the Op's key. func (op Op) KeyBytes() []byte { return op.key } // WithKeyBytes sets the byte slice for the Op's key. func (op *Op) WithKeyBytes(key []byte) { op.key = key } // RangeBytes returns the byte slice holding with the Op's range end, if any. func (op Op) RangeBytes() []byte { return op.end } // Rev returns the requested revision, if any. func (op Op) Rev() int64 { return op.rev } // Limit returns limit of the result, if any. func (op Op) Limit() int64 { return op.limit } // IsPut returns true iff the operation is a Put. func (op Op) IsPut() bool { return op.t == tPut } // IsGet returns true iff the operation is a Get. func (op Op) IsGet() bool { return op.t == tRange } // IsDelete returns true iff the operation is a Delete. func (op Op) IsDelete() bool { return op.t == tDeleteRange } // IsSerializable returns true if the serializable field is true. func (op Op) IsSerializable() bool { return op.serializable } // IsKeysOnly returns whether keysOnly is set. func (op Op) IsKeysOnly() bool { return op.keysOnly } // IsCountOnly returns whether countOnly is set. func (op Op) IsCountOnly() bool { return op.countOnly } // IsSortSet returns true if WithSort is set. func (op Op) IsSortSet() bool { return op.sort != nil } func (op Op) IsOptsWithFromKey() bool { return op.isOptsWithFromKey } func (op Op) IsOptsWithPrefix() bool { return op.isOptsWithPrefix } // IsPrevKV returns whether WithPrevKV() is set. func (op Op) IsPrevKV() bool { return op.prevKV } // IsFragment returns whether WithFragment() is set. func (op Op) IsFragment() bool { return op.fragment } // IsProgressNotify returns whether WithProgressNotify() is set. func (op Op) IsProgressNotify() bool { return op.progressNotify } // IsCreatedNotify returns whether WithCreatedNotify() is set. func (op Op) IsCreatedNotify() bool { return op.createdNotify } // IsFilterPut returns whether WithFilterPut() is set. func (op Op) IsFilterPut() bool { return op.filterPut } // IsFilterDelete returns whether WithFilterDelete() is set. func (op Op) IsFilterDelete() bool { return op.filterDelete } // MinModRev returns the operation's minimum modify revision. func (op Op) MinModRev() int64 { return op.minModRev } // MaxModRev returns the operation's maximum modify revision. func (op Op) MaxModRev() int64 { return op.maxModRev } // MinCreateRev returns the operation's minimum create revision. func (op Op) MinCreateRev() int64 { return op.minCreateRev } // MaxCreateRev returns the operation's maximum create revision. func (op Op) MaxCreateRev() int64 { return op.maxCreateRev } // WithRangeBytes sets the byte slice for the Op's range end. func (op *Op) WithRangeBytes(end []byte) { op.end = end } // ValueBytes returns the byte slice holding the Op's value, if any. func (op Op) ValueBytes() []byte { return op.val } // WithValueBytes sets the byte slice for the Op's value. func (op *Op) WithValueBytes(v []byte) { op.val = v } func (op Op) toRangeRequest() *pb.RangeRequest { if op.t != tRange { panic("op.t != tRange") } r := &pb.RangeRequest{ Key: op.key, RangeEnd: op.end, Limit: op.limit, Revision: op.rev, Serializable: op.serializable, KeysOnly: op.keysOnly, CountOnly: op.countOnly, MinModRevision: op.minModRev, MaxModRevision: op.maxModRev, MinCreateRevision: op.minCreateRev, MaxCreateRevision: op.maxCreateRev, } if op.sort != nil { r.SortOrder = pb.RangeRequest_SortOrder(op.sort.Order) r.SortTarget = pb.RangeRequest_SortTarget(op.sort.Target) } return r } func (op Op) toTxnRequest() *pb.TxnRequest { thenOps := make([]*pb.RequestOp, len(op.thenOps)) for i, tOp := range op.thenOps { thenOps[i] = tOp.toRequestOp() } elseOps := make([]*pb.RequestOp, len(op.elseOps)) for i, eOp := range op.elseOps { elseOps[i] = eOp.toRequestOp() } cmps := make([]*pb.Compare, len(op.cmps)) for i := range op.cmps { cmps[i] = (*pb.Compare)(&op.cmps[i]) } return &pb.TxnRequest{Compare: cmps, Success: thenOps, Failure: elseOps} } func (op Op) toRequestOp() *pb.RequestOp { switch op.t { case tRange: return &pb.RequestOp{Request: &pb.RequestOp_RequestRange{RequestRange: op.toRangeRequest()}} case tPut: r := &pb.PutRequest{Key: op.key, Value: op.val, Lease: int64(op.leaseID), PrevKv: op.prevKV, IgnoreValue: op.ignoreValue, IgnoreLease: op.ignoreLease} return &pb.RequestOp{Request: &pb.RequestOp_RequestPut{RequestPut: r}} case tDeleteRange: r := &pb.DeleteRangeRequest{Key: op.key, RangeEnd: op.end, PrevKv: op.prevKV} return &pb.RequestOp{Request: &pb.RequestOp_RequestDeleteRange{RequestDeleteRange: r}} case tTxn: return &pb.RequestOp{Request: &pb.RequestOp_RequestTxn{RequestTxn: op.toTxnRequest()}} default: panic("Unknown Op") } } func (op Op) isWrite() bool { if op.t == tTxn { for _, tOp := range op.thenOps { if tOp.isWrite() { return true } } for _, tOp := range op.elseOps { if tOp.isWrite() { return true } } return false } return op.t != tRange } func NewOp() *Op { return &Op{key: []byte("")} } // OpGet returns "get" operation based on given key and operation options. func OpGet(key string, opts ...OpOption) Op { // WithPrefix and WithFromKey are not supported together if IsOptsWithPrefix(opts) && IsOptsWithFromKey(opts) { panic("`WithPrefix` and `WithFromKey` cannot be set at the same time, choose one") } ret := Op{t: tRange, key: []byte(key)} ret.applyOpts(opts) return ret } // OpDelete returns "delete" operation based on given key and operation options. func OpDelete(key string, opts ...OpOption) Op { // WithPrefix and WithFromKey are not supported together if IsOptsWithPrefix(opts) && IsOptsWithFromKey(opts) { panic("`WithPrefix` and `WithFromKey` cannot be set at the same time, choose one") } ret := Op{t: tDeleteRange, key: []byte(key)} ret.applyOpts(opts) switch { case ret.leaseID != 0: panic("unexpected lease in delete") case ret.limit != 0: panic("unexpected limit in delete") case ret.rev != 0: panic("unexpected revision in delete") case ret.sort != nil: panic("unexpected sort in delete") case ret.serializable: panic("unexpected serializable in delete") case ret.countOnly: panic("unexpected countOnly in delete") case ret.minModRev != 0, ret.maxModRev != 0: panic("unexpected mod revision filter in delete") case ret.minCreateRev != 0, ret.maxCreateRev != 0: panic("unexpected create revision filter in delete") case ret.filterDelete, ret.filterPut: panic("unexpected filter in delete") case ret.createdNotify: panic("unexpected createdNotify in delete") } return ret } // OpPut returns "put" operation based on given key-value and operation options. func OpPut(key, val string, opts ...OpOption) Op { ret := Op{t: tPut, key: []byte(key), val: []byte(val)} ret.applyOpts(opts) switch { case ret.end != nil: panic("unexpected range in put") case ret.limit != 0: panic("unexpected limit in put") case ret.rev != 0: panic("unexpected revision in put") case ret.sort != nil: panic("unexpected sort in put") case ret.serializable: panic("unexpected serializable in put") case ret.countOnly: panic("unexpected countOnly in put") case ret.minModRev != 0, ret.maxModRev != 0: panic("unexpected mod revision filter in put") case ret.minCreateRev != 0, ret.maxCreateRev != 0: panic("unexpected create revision filter in put") case ret.filterDelete, ret.filterPut: panic("unexpected filter in put") case ret.createdNotify: panic("unexpected createdNotify in put") } return ret } // OpTxn returns "txn" operation based on given transaction conditions. func OpTxn(cmps []Cmp, thenOps []Op, elseOps []Op) Op { return Op{t: tTxn, cmps: cmps, thenOps: thenOps, elseOps: elseOps} } func OpWatch(key string, opts ...OpOption) Op { ret := Op{t: tRange, key: []byte(key)} ret.applyOpts(opts) switch { case ret.leaseID != 0: panic("unexpected lease in watch") case ret.limit != 0: panic("unexpected limit in watch") case ret.sort != nil: panic("unexpected sort in watch") case ret.serializable: panic("unexpected serializable in watch") case ret.countOnly: panic("unexpected countOnly in watch") case ret.minModRev != 0, ret.maxModRev != 0: panic("unexpected mod revision filter in watch") case ret.minCreateRev != 0, ret.maxCreateRev != 0: panic("unexpected create revision filter in watch") } return ret } func (op *Op) applyOpts(opts []OpOption) { for _, opt := range opts { opt(op) } } // OpOption configures Operations like Get, Put, Delete. type OpOption func(*Op) // WithLease attaches a lease ID to a key in 'Put' request. func WithLease(leaseID LeaseID) OpOption { return func(op *Op) { op.leaseID = leaseID } } // WithLimit limits the number of results to return from 'Get' request. // If WithLimit is given a 0 limit, it is treated as no limit. func WithLimit(n int64) OpOption { return func(op *Op) { op.limit = n } } // WithRev specifies the store revision for 'Get' request. // Or the start revision of 'Watch' request. func WithRev(rev int64) OpOption { return func(op *Op) { op.rev = rev } } // WithSort specifies the ordering in 'Get' request. It requires // 'WithRange' and/or 'WithPrefix' to be specified too. // 'target' specifies the target to sort by: key, version, revisions, value. // 'order' can be either 'SortNone', 'SortAscend', 'SortDescend'. func WithSort(target SortTarget, order SortOrder) OpOption { return func(op *Op) { if target == SortByKey && order == SortAscend { // If order != SortNone, server fetches the entire key-space, // and then applies the sort and limit, if provided. // Since by default the server returns results sorted by keys // in lexicographically ascending order, the client should ignore // SortOrder if the target is SortByKey. order = SortNone } op.sort = &SortOption{target, order} } } // GetPrefixRangeEnd gets the range end of the prefix. // 'Get(foo, WithPrefix())' is equal to 'Get(foo, WithRange(GetPrefixRangeEnd(foo))'. func GetPrefixRangeEnd(prefix string) string { return string(getPrefix([]byte(prefix))) } func getPrefix(key []byte) []byte { end := make([]byte, len(key)) copy(end, key) for i := len(end) - 1; i >= 0; i-- { if end[i] < 0xff { end[i] = end[i] + 1 end = end[:i+1] return end } } // next prefix does not exist (e.g., 0xffff); // default to WithFromKey policy return noPrefixEnd } // WithPrefix enables 'Get', 'Delete', or 'Watch' requests to operate // on the keys with matching prefix. For example, 'Get(foo, WithPrefix())' // can return 'foo1', 'foo2', and so on. func WithPrefix() OpOption { return func(op *Op) { op.isOptsWithPrefix = true if len(op.key) == 0 { op.key, op.end = []byte{0}, []byte{0} return } op.end = getPrefix(op.key) } } // WithRange specifies the range of 'Get', 'Delete', 'Watch' requests. // For example, 'Get' requests with 'WithRange(end)' returns // the keys in the range [key, end). // endKey must be lexicographically greater than start key. func WithRange(endKey string) OpOption { return func(op *Op) { op.end = []byte(endKey) } } // WithFromKey specifies the range of 'Get', 'Delete', 'Watch' requests // to be equal or greater than the key in the argument. func WithFromKey() OpOption { return func(op *Op) { if len(op.key) == 0 { op.key = []byte{0} } op.end = []byte("\x00") op.isOptsWithFromKey = true } } // WithSerializable makes `Get` and `MemberList` requests serializable. // By default, they are linearizable. Serializable requests are better // for lower latency requirement, but users should be aware that they // could get stale data with serializable requests. // // In some situations users may want to use serializable requests. For // example, when adding a new member to a one-node cluster, it's reasonable // and safe to use serializable request before the new added member gets // started. func WithSerializable() OpOption { return func(op *Op) { op.serializable = true } } // WithKeysOnly makes the 'Get' request return only the keys and the corresponding // values will be omitted. func WithKeysOnly() OpOption { return func(op *Op) { op.keysOnly = true } } // WithCountOnly makes the 'Get' request return only the count of keys. func WithCountOnly() OpOption { return func(op *Op) { op.countOnly = true } } // WithMinModRev filters out keys for Get with modification revisions less than the given revision. func WithMinModRev(rev int64) OpOption { return func(op *Op) { op.minModRev = rev } } // WithMaxModRev filters out keys for Get with modification revisions greater than the given revision. func WithMaxModRev(rev int64) OpOption { return func(op *Op) { op.maxModRev = rev } } // WithMinCreateRev filters out keys for Get with creation revisions less than the given revision. func WithMinCreateRev(rev int64) OpOption { return func(op *Op) { op.minCreateRev = rev } } // WithMaxCreateRev filters out keys for Get with creation revisions greater than the given revision. func WithMaxCreateRev(rev int64) OpOption { return func(op *Op) { op.maxCreateRev = rev } } // WithFirstCreate gets the key with the oldest creation revision in the request range. func WithFirstCreate() []OpOption { return withTop(SortByCreateRevision, SortAscend) } // WithLastCreate gets the key with the latest creation revision in the request range. func WithLastCreate() []OpOption { return withTop(SortByCreateRevision, SortDescend) } // WithFirstKey gets the lexically first key in the request range. func WithFirstKey() []OpOption { return withTop(SortByKey, SortAscend) } // WithLastKey gets the lexically last key in the request range. func WithLastKey() []OpOption { return withTop(SortByKey, SortDescend) } // WithFirstRev gets the key with the oldest modification revision in the request range. func WithFirstRev() []OpOption { return withTop(SortByModRevision, SortAscend) } // WithLastRev gets the key with the latest modification revision in the request range. func WithLastRev() []OpOption { return withTop(SortByModRevision, SortDescend) } // withTop gets the first key over the get's prefix given a sort order func withTop(target SortTarget, order SortOrder) []OpOption { return []OpOption{WithPrefix(), WithSort(target, order), WithLimit(1)} } // WithProgressNotify makes watch server send periodic progress updates // every 10 minutes when there is no incoming events. // Progress updates have zero events in WatchResponse. func WithProgressNotify() OpOption { return func(op *Op) { op.progressNotify = true } } // WithCreatedNotify makes watch server sends the created event. func WithCreatedNotify() OpOption { return func(op *Op) { op.createdNotify = true } } // WithFilterPut discards PUT events from the watcher. func WithFilterPut() OpOption { return func(op *Op) { op.filterPut = true } } // WithFilterDelete discards DELETE events from the watcher. func WithFilterDelete() OpOption { return func(op *Op) { op.filterDelete = true } } // WithPrevKV gets the previous key-value pair before the event happens. If the previous KV is already compacted, // nothing will be returned. func WithPrevKV() OpOption { return func(op *Op) { op.prevKV = true } } // WithFragment to receive raw watch response with fragmentation. // Fragmentation is disabled by default. If fragmentation is enabled, // etcd watch server will split watch response before sending to clients // when the total size of watch events exceed server-side request limit. // The default server-side request limit is 1.5 MiB, which can be configured // as "--max-request-bytes" flag value + gRPC-overhead 512 bytes. // See "etcdserver/api/v3rpc/watch.go" for more details. func WithFragment() OpOption { return func(op *Op) { op.fragment = true } } // WithIgnoreValue updates the key using its current value. // This option can not be combined with non-empty values. // Returns an error if the key does not exist. func WithIgnoreValue() OpOption { return func(op *Op) { op.ignoreValue = true } } // WithIgnoreLease updates the key using its current lease. // This option can not be combined with WithLease. // Returns an error if the key does not exist. func WithIgnoreLease() OpOption { return func(op *Op) { op.ignoreLease = true } } // LeaseOp represents an Operation that lease can execute. type LeaseOp struct { id LeaseID // for TimeToLive attachedKeys bool } // LeaseOption configures lease operations. type LeaseOption func(*LeaseOp) func (op *LeaseOp) applyOpts(opts []LeaseOption) { for _, opt := range opts { opt(op) } } // WithAttachedKeys makes TimeToLive list the keys attached to the given lease ID. func WithAttachedKeys() LeaseOption { return func(op *LeaseOp) { op.attachedKeys = true } } func toLeaseTimeToLiveRequest(id LeaseID, opts ...LeaseOption) *pb.LeaseTimeToLiveRequest { ret := &LeaseOp{id: id} ret.applyOpts(opts) return &pb.LeaseTimeToLiveRequest{ID: int64(id), Keys: ret.attachedKeys} } // IsOptsWithPrefix returns true if WithPrefix option is called in the given opts. func IsOptsWithPrefix(opts []OpOption) bool { ret := NewOp() for _, opt := range opts { opt(ret) } return ret.isOptsWithPrefix } // IsOptsWithFromKey returns true if WithFromKey option is called in the given opts. func IsOptsWithFromKey(opts []OpOption) bool { ret := NewOp() for _, opt := range opts { opt(ret) } return ret.isOptsWithFromKey } func (op Op) IsSortOptionValid() bool { if op.sort != nil { sortOrder := int32(op.sort.Order) sortTarget := int32(op.sort.Target) if _, ok := pb.RangeRequest_SortOrder_name[sortOrder]; !ok { return false } if _, ok := pb.RangeRequest_SortTarget_name[sortTarget]; !ok { return false } } return true } ================================================ FILE: client/v3/op_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "reflect" "testing" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) // TestOpWithSort tests if WithSort(ASCEND, KEY) and WithLimit are specified, // RangeRequest ignores the SortOption to avoid unnecessarily fetching // the entire key-space. func TestOpWithSort(t *testing.T) { opReq := OpGet("foo", WithSort(SortByKey, SortAscend), WithLimit(10)).toRequestOp().Request q, ok := opReq.(*pb.RequestOp_RequestRange) if !ok { t.Fatalf("expected range request, got %v", reflect.TypeOf(opReq)) } req := q.RequestRange wreq := &pb.RangeRequest{Key: []byte("foo"), SortOrder: pb.RangeRequest_NONE, Limit: 10} if !reflect.DeepEqual(req, wreq) { t.Fatalf("expected %+v, got %+v", wreq, req) } } func TestIsSortOptionValid(t *testing.T) { rangeReqs := []struct { sortOrder pb.RangeRequest_SortOrder sortTarget pb.RangeRequest_SortTarget expectedValid bool }{ { sortOrder: pb.RangeRequest_ASCEND, sortTarget: pb.RangeRequest_CREATE, expectedValid: true, }, { sortOrder: pb.RangeRequest_ASCEND, sortTarget: 100, expectedValid: false, }, { sortOrder: 200, sortTarget: pb.RangeRequest_MOD, expectedValid: false, }, } for _, req := range rangeReqs { getOp := Op{ sort: &SortOption{ Order: SortOrder(req.sortOrder), Target: SortTarget(req.sortTarget), }, } actualRet := getOp.IsSortOptionValid() if actualRet != req.expectedValid { t.Errorf("expected sortOrder (%d) and sortTarget (%d) to be %t, but got %t", req.sortOrder, req.sortTarget, req.expectedValid, actualRet) } } } func TestIsOptsWithPrefix(t *testing.T) { optswithprefix := []OpOption{WithPrefix()} op := OpGet("key", optswithprefix...) if !IsOptsWithPrefix(optswithprefix) || !op.IsOptsWithPrefix() { t.Errorf("IsOptsWithPrefix = false, expected true") } optswithfromkey := []OpOption{WithFromKey()} op = OpGet("key", optswithfromkey...) if IsOptsWithPrefix(optswithfromkey) || op.IsOptsWithPrefix() { t.Errorf("IsOptsWithPrefix = true, expected false") } } func TestIsOptsWithFromKey(t *testing.T) { optswithfromkey := []OpOption{WithFromKey()} op := OpGet("key", optswithfromkey...) if !IsOptsWithFromKey(optswithfromkey) || !op.IsOptsWithFromKey() { t.Errorf("IsOptsWithFromKey = false, expected true") } optswithprefix := []OpOption{WithPrefix()} op = OpGet("key", optswithprefix...) if IsOptsWithFromKey(optswithprefix) || op.IsOptsWithFromKey() { t.Errorf("IsOptsWithFromKey = true, expected false") } } ================================================ FILE: client/v3/options.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "math" "time" "google.golang.org/grpc" ) var ( // client-side handling retrying of request failures where data was not written to the wire or // where server indicates it did not process the data. gRPC default is "WaitForReady(false)" // but for etcd we default to "WaitForReady(true)" to minimize client request error responses due to // transient failures. defaultWaitForReady = grpc.WaitForReady(true) // client-side request send limit, gRPC default is math.MaxInt32 // Make sure that "client-side send limit < server-side default send/recv limit" // Same value as "embed.DefaultMaxRequestBytes" plus gRPC overhead bytes defaultMaxCallSendMsgSize = grpc.MaxCallSendMsgSize(2 * 1024 * 1024) // client-side response receive limit, gRPC default is 4MB // Make sure that "client-side receive limit >= server-side default send/recv limit" // because range response can easily exceed request send limits // Default to math.MaxInt32; writes exceeding server-side send limit fails anyway defaultMaxCallRecvMsgSize = grpc.MaxCallRecvMsgSize(math.MaxInt32) // client-side non-streaming retry limit, only applied to requests where server responds with // a error code clearly indicating it was unable to process the request such as codes.Unavailable. // If set to 0, retry is disabled. defaultUnaryMaxRetries uint = 100 // client-side streaming retry limit, only applied to requests where server responds with // a error code clearly indicating it was unable to process the request such as codes.Unavailable. // If set to 0, retry is disabled. defaultStreamMaxRetries = ^uint(0) // max uint // client-side retry backoff wait between requests. defaultBackoffWaitBetween = 25 * time.Millisecond // client-side retry backoff default jitter fraction. defaultBackoffJitterFraction = 0.10 ) // defaultCallOpts defines a list of default "gRPC.CallOption". // Some options are exposed to "clientv3.Config". // Defaults will be overridden by the settings in "clientv3.Config". var defaultCallOpts = []grpc.CallOption{ defaultWaitForReady, defaultMaxCallSendMsgSize, defaultMaxCallRecvMsgSize, } // MaxLeaseTTL is the maximum lease TTL value const MaxLeaseTTL = 9000000000 ================================================ FILE: client/v3/ordering/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package ordering is a clientv3 wrapper that caches response header revisions // to detect ordering violations from stale responses. Users may define a // policy on how to handle the ordering violation, but typically the client // should connect to another endpoint and reissue the request. // // The most common situation where an ordering violation happens is a client // reconnects to a partitioned member and issues a serializable read. Since the // partitioned member is likely behind the last member, it may return a Get // response based on a store revision older than the store revision used to // service a prior Get on the former endpoint. // // First, create a client: // // cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"localhost:2379"}}) // if err != nil { // // handle error! // } // // Next, override the client interface with the ordering wrapper: // // vf := func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error { // return fmt.Errorf("ordering: issued %+v, got %+v, expected rev=%v", op, resp, prevRev) // } // cli.KV = ordering.NewKV(cli.KV, vf) // // Now calls using 'cli' will reject order violations with an error. package ordering ================================================ FILE: client/v3/ordering/kv.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ordering import ( "context" "sync" clientv3 "go.etcd.io/etcd/client/v3" ) // kvOrdering ensures that serialized requests do not return // get with revisions less than the previous // returned revision. type kvOrdering struct { clientv3.KV orderViolationFunc OrderViolationFunc prevRev int64 revMu sync.RWMutex } func NewKV(kv clientv3.KV, orderViolationFunc OrderViolationFunc) *kvOrdering { return &kvOrdering{kv, orderViolationFunc, 0, sync.RWMutex{}} } func (kv *kvOrdering) getPrevRev() int64 { kv.revMu.RLock() defer kv.revMu.RUnlock() return kv.prevRev } func (kv *kvOrdering) setPrevRev(currRev int64) { kv.revMu.Lock() defer kv.revMu.Unlock() if currRev > kv.prevRev { kv.prevRev = currRev } } func (kv *kvOrdering) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { // prevRev is stored in a local variable in order to record the prevRev // at the beginning of the Get operation, because concurrent // access to kvOrdering could change the prevRev field in the // middle of the Get operation. prevRev := kv.getPrevRev() op := clientv3.OpGet(key, opts...) for { r, err := kv.KV.Do(ctx, op) if err != nil { return nil, err } resp := r.Get() if resp.Header.Revision == prevRev { return resp, nil } else if resp.Header.Revision > prevRev { kv.setPrevRev(resp.Header.Revision) return resp, nil } err = kv.orderViolationFunc(op, r, prevRev) if err != nil { return nil, err } } } func (kv *kvOrdering) Txn(ctx context.Context) clientv3.Txn { return &txnOrdering{ kv.KV.Txn(ctx), kv, ctx, sync.Mutex{}, []clientv3.Cmp{}, []clientv3.Op{}, []clientv3.Op{}, } } // txnOrdering ensures that serialized requests do not return // txn responses with revisions less than the previous // returned revision. type txnOrdering struct { clientv3.Txn *kvOrdering ctx context.Context mu sync.Mutex cmps []clientv3.Cmp thenOps []clientv3.Op elseOps []clientv3.Op } func (txn *txnOrdering) If(cs ...clientv3.Cmp) clientv3.Txn { txn.mu.Lock() defer txn.mu.Unlock() txn.cmps = cs txn.Txn.If(cs...) return txn } func (txn *txnOrdering) Then(ops ...clientv3.Op) clientv3.Txn { txn.mu.Lock() defer txn.mu.Unlock() txn.thenOps = ops txn.Txn.Then(ops...) return txn } func (txn *txnOrdering) Else(ops ...clientv3.Op) clientv3.Txn { txn.mu.Lock() defer txn.mu.Unlock() txn.elseOps = ops txn.Txn.Else(ops...) return txn } func (txn *txnOrdering) Commit() (*clientv3.TxnResponse, error) { // prevRev is stored in a local variable in order to record the prevRev // at the beginning of the Commit operation, because concurrent // access to txnOrdering could change the prevRev field in the // middle of the Commit operation. prevRev := txn.getPrevRev() opTxn := clientv3.OpTxn(txn.cmps, txn.thenOps, txn.elseOps) for { opResp, err := txn.KV.Do(txn.ctx, opTxn) if err != nil { return nil, err } txnResp := opResp.Txn() if txnResp.Header.Revision >= prevRev { txn.setPrevRev(txnResp.Header.Revision) return txnResp, nil } err = txn.orderViolationFunc(opTxn, opResp, prevRev) if err != nil { return nil, err } } } ================================================ FILE: client/v3/ordering/kv_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ordering import ( "context" "sync" "testing" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" ) type mockKV struct { clientv3.KV response clientv3.OpResponse } func (kv *mockKV) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) { return kv.response, nil } var rangeTests = []struct { prevRev int64 response *clientv3.GetResponse }{ { 5, &clientv3.GetResponse{ Header: &pb.ResponseHeader{ Revision: 5, }, }, }, { 5, &clientv3.GetResponse{ Header: &pb.ResponseHeader{ Revision: 4, }, }, }, { 5, &clientv3.GetResponse{ Header: &pb.ResponseHeader{ Revision: 6, }, }, }, } func TestKvOrdering(t *testing.T) { for i, tt := range rangeTests { mKV := &mockKV{clientv3.NewKVFromKVClient(nil, nil), tt.response.OpResponse()} kv := &kvOrdering{ mKV, func(r *clientv3.GetResponse) OrderViolationFunc { return func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error { r.Header.Revision++ return nil } }(tt.response), tt.prevRev, sync.RWMutex{}, } res, err := kv.Get(t.Context(), "mockKey") if err != nil { t.Errorf("#%d: expected response %+v, got error %+v", i, tt.response, err) } if rev := res.Header.Revision; rev < tt.prevRev { t.Errorf("#%d: expected revision %d, got %d", i, tt.prevRev, rev) } } } var txnTests = []struct { prevRev int64 response *clientv3.TxnResponse }{ { 5, &clientv3.TxnResponse{ Header: &pb.ResponseHeader{ Revision: 5, }, }, }, { 5, &clientv3.TxnResponse{ Header: &pb.ResponseHeader{ Revision: 8, }, }, }, { 5, &clientv3.TxnResponse{ Header: &pb.ResponseHeader{ Revision: 4, }, }, }, } func TestTxnOrdering(t *testing.T) { for i, tt := range txnTests { mKV := &mockKV{clientv3.NewKVFromKVClient(nil, nil), tt.response.OpResponse()} kv := &kvOrdering{ mKV, func(r *clientv3.TxnResponse) OrderViolationFunc { return func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error { r.Header.Revision++ return nil } }(tt.response), tt.prevRev, sync.RWMutex{}, } txn := &txnOrdering{ kv.Txn(t.Context()), kv, t.Context(), sync.Mutex{}, []clientv3.Cmp{}, []clientv3.Op{}, []clientv3.Op{}, } res, err := txn.Commit() if err != nil { t.Errorf("#%d: expected response %+v, got error %+v", i, tt.response, err) } if rev := res.Header.Revision; rev < tt.prevRev { t.Errorf("#%d: expected revision %d, got %d", i, tt.prevRev, rev) } } } ================================================ FILE: client/v3/ordering/util.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ordering import ( "errors" "sync/atomic" clientv3 "go.etcd.io/etcd/client/v3" ) type OrderViolationFunc func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error var ErrNoGreaterRev = errors.New("etcdclient: no cluster members have a revision higher than the previously received revision") func NewOrderViolationSwitchEndpointClosure(c *clientv3.Client) OrderViolationFunc { violationCount := int32(0) return func(_ clientv3.Op, _ clientv3.OpResponse, _ int64) error { // Each request is assigned by round-robin load-balancer's picker to a different // endpoint. If we cycled them 5 times (even with some level of concurrency), // with high probability no endpoint points on a member with fresh data. // TODO: Ideally we should track members (resp.opp.Header) that returned // stale result and explicitly temporarily disable them in 'picker'. if atomic.LoadInt32(&violationCount) > int32(5*len(c.Endpoints())) { return ErrNoGreaterRev } atomic.AddInt32(&violationCount, 1) return nil } } ================================================ FILE: client/v3/retry.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "errors" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" ) type retryPolicy uint8 const ( repeatable retryPolicy = iota nonRepeatable ) func (rp retryPolicy) String() string { switch rp { case repeatable: return "repeatable" case nonRepeatable: return "nonRepeatable" default: return "UNKNOWN" } } // isSafeRetryImmutableRPC returns "true" when an immutable request is safe for retry. // // immutable requests (e.g. Get) should be retried unless it's // an obvious server-side error (e.g. rpctypes.ErrRequestTooLarge). // // Returning "false" means retry should stop, since client cannot // handle itself even with retries. func isSafeRetryImmutableRPC(err error) bool { eErr := rpctypes.Error(err) var serverErr rpctypes.EtcdError if errors.As(eErr, &serverErr) && serverErr.Code() != codes.Unavailable { // interrupted by non-transient server-side or gRPC-side error // client cannot handle itself (e.g. rpctypes.ErrCompacted) return false } // only retry if unavailable ev, ok := status.FromError(err) if !ok { // all errors from RPC is typed "grpc/status.(*statusError)" // (ref. https://github.com/grpc/grpc-go/pull/1782) // // if the error type is not "grpc/status.(*statusError)", // it could be from "Dial" // TODO: do not retry for now // ref. https://github.com/grpc/grpc-go/issues/1581 return false } return ev.Code() == codes.Unavailable } // isSafeRetryMutableRPC returns "true" when a mutable request is safe for retry. // // mutable requests (e.g. Put, Delete, Txn) should only be retried // when the status code is codes.Unavailable when initial connection // has not been established (no endpoint is up). // // Returning "false" means retry should stop, otherwise it violates // write-at-most-once semantics. func isSafeRetryMutableRPC(err error) bool { if ev, ok := status.FromError(err); ok && ev.Code() != codes.Unavailable { // not safe for mutable RPCs // e.g. interrupted by non-transient error that client cannot handle itself, // or transient error while the connection has already been established return false } desc := rpctypes.ErrorDesc(err) return desc == "there is no address available" || desc == "there is no connection available" } type retryKVClient struct { kc pb.KVClient } // RetryKVClient implements a KVClient. func RetryKVClient(c *Client) pb.KVClient { return &retryKVClient{ kc: pb.NewKVClient(c.conn), } } func (rkv *retryKVClient) Range(ctx context.Context, in *pb.RangeRequest, opts ...grpc.CallOption) (resp *pb.RangeResponse, err error) { return rkv.kc.Range(ctx, in, append(opts, withRepeatablePolicy())...) } func (rkv *retryKVClient) Put(ctx context.Context, in *pb.PutRequest, opts ...grpc.CallOption) (resp *pb.PutResponse, err error) { return rkv.kc.Put(ctx, in, opts...) } func (rkv *retryKVClient) DeleteRange(ctx context.Context, in *pb.DeleteRangeRequest, opts ...grpc.CallOption) (resp *pb.DeleteRangeResponse, err error) { return rkv.kc.DeleteRange(ctx, in, opts...) } func (rkv *retryKVClient) Txn(ctx context.Context, in *pb.TxnRequest, opts ...grpc.CallOption) (resp *pb.TxnResponse, err error) { return rkv.kc.Txn(ctx, in, opts...) } func (rkv *retryKVClient) Compact(ctx context.Context, in *pb.CompactionRequest, opts ...grpc.CallOption) (resp *pb.CompactionResponse, err error) { return rkv.kc.Compact(ctx, in, opts...) } type retryLeaseClient struct { lc pb.LeaseClient } // RetryLeaseClient implements a LeaseClient. func RetryLeaseClient(c *Client) pb.LeaseClient { return &retryLeaseClient{ lc: pb.NewLeaseClient(c.conn), } } func (rlc *retryLeaseClient) LeaseTimeToLive(ctx context.Context, in *pb.LeaseTimeToLiveRequest, opts ...grpc.CallOption) (resp *pb.LeaseTimeToLiveResponse, err error) { return rlc.lc.LeaseTimeToLive(ctx, in, append(opts, withRepeatablePolicy())...) } func (rlc *retryLeaseClient) LeaseLeases(ctx context.Context, in *pb.LeaseLeasesRequest, opts ...grpc.CallOption) (resp *pb.LeaseLeasesResponse, err error) { return rlc.lc.LeaseLeases(ctx, in, append(opts, withRepeatablePolicy())...) } func (rlc *retryLeaseClient) LeaseGrant(ctx context.Context, in *pb.LeaseGrantRequest, opts ...grpc.CallOption) (resp *pb.LeaseGrantResponse, err error) { return rlc.lc.LeaseGrant(ctx, in, append(opts, withRepeatablePolicy())...) } func (rlc *retryLeaseClient) LeaseRevoke(ctx context.Context, in *pb.LeaseRevokeRequest, opts ...grpc.CallOption) (resp *pb.LeaseRevokeResponse, err error) { return rlc.lc.LeaseRevoke(ctx, in, append(opts, withRepeatablePolicy())...) } func (rlc *retryLeaseClient) LeaseKeepAlive(ctx context.Context, opts ...grpc.CallOption) (stream pb.Lease_LeaseKeepAliveClient, err error) { return rlc.lc.LeaseKeepAlive(ctx, append(opts, withRepeatablePolicy())...) } type retryClusterClient struct { cc pb.ClusterClient } // RetryClusterClient implements a ClusterClient. func RetryClusterClient(c *Client) pb.ClusterClient { return &retryClusterClient{ cc: pb.NewClusterClient(c.conn), } } func (rcc *retryClusterClient) MemberList(ctx context.Context, in *pb.MemberListRequest, opts ...grpc.CallOption) (resp *pb.MemberListResponse, err error) { return rcc.cc.MemberList(ctx, in, append(opts, withRepeatablePolicy())...) } func (rcc *retryClusterClient) MemberAdd(ctx context.Context, in *pb.MemberAddRequest, opts ...grpc.CallOption) (resp *pb.MemberAddResponse, err error) { return rcc.cc.MemberAdd(ctx, in, opts...) } func (rcc *retryClusterClient) MemberRemove(ctx context.Context, in *pb.MemberRemoveRequest, opts ...grpc.CallOption) (resp *pb.MemberRemoveResponse, err error) { return rcc.cc.MemberRemove(ctx, in, opts...) } func (rcc *retryClusterClient) MemberUpdate(ctx context.Context, in *pb.MemberUpdateRequest, opts ...grpc.CallOption) (resp *pb.MemberUpdateResponse, err error) { return rcc.cc.MemberUpdate(ctx, in, opts...) } func (rcc *retryClusterClient) MemberPromote(ctx context.Context, in *pb.MemberPromoteRequest, opts ...grpc.CallOption) (resp *pb.MemberPromoteResponse, err error) { return rcc.cc.MemberPromote(ctx, in, opts...) } type retryMaintenanceClient struct { mc pb.MaintenanceClient } // RetryMaintenanceClient implements a Maintenance. func RetryMaintenanceClient(c *Client, conn *grpc.ClientConn) pb.MaintenanceClient { return &retryMaintenanceClient{ mc: pb.NewMaintenanceClient(conn), } } func (rmc *retryMaintenanceClient) Alarm(ctx context.Context, in *pb.AlarmRequest, opts ...grpc.CallOption) (resp *pb.AlarmResponse, err error) { return rmc.mc.Alarm(ctx, in, append(opts, withRepeatablePolicy())...) } func (rmc *retryMaintenanceClient) Status(ctx context.Context, in *pb.StatusRequest, opts ...grpc.CallOption) (resp *pb.StatusResponse, err error) { return rmc.mc.Status(ctx, in, append(opts, withRepeatablePolicy())...) } func (rmc *retryMaintenanceClient) Hash(ctx context.Context, in *pb.HashRequest, opts ...grpc.CallOption) (resp *pb.HashResponse, err error) { return rmc.mc.Hash(ctx, in, append(opts, withRepeatablePolicy())...) } func (rmc *retryMaintenanceClient) HashKV(ctx context.Context, in *pb.HashKVRequest, opts ...grpc.CallOption) (resp *pb.HashKVResponse, err error) { return rmc.mc.HashKV(ctx, in, append(opts, withRepeatablePolicy())...) } func (rmc *retryMaintenanceClient) Snapshot(ctx context.Context, in *pb.SnapshotRequest, opts ...grpc.CallOption) (stream pb.Maintenance_SnapshotClient, err error) { return rmc.mc.Snapshot(ctx, in, append(opts, withRepeatablePolicy())...) } func (rmc *retryMaintenanceClient) MoveLeader(ctx context.Context, in *pb.MoveLeaderRequest, opts ...grpc.CallOption) (resp *pb.MoveLeaderResponse, err error) { return rmc.mc.MoveLeader(ctx, in, append(opts, withRepeatablePolicy())...) } func (rmc *retryMaintenanceClient) Defragment(ctx context.Context, in *pb.DefragmentRequest, opts ...grpc.CallOption) (resp *pb.DefragmentResponse, err error) { return rmc.mc.Defragment(ctx, in, opts...) } func (rmc *retryMaintenanceClient) Downgrade(ctx context.Context, in *pb.DowngradeRequest, opts ...grpc.CallOption) (resp *pb.DowngradeResponse, err error) { return rmc.mc.Downgrade(ctx, in, opts...) } type retryAuthClient struct { ac pb.AuthClient } // RetryAuthClient implements a AuthClient. func RetryAuthClient(c *Client) pb.AuthClient { return &retryAuthClient{ ac: pb.NewAuthClient(c.conn), } } func (rac *retryAuthClient) UserList(ctx context.Context, in *pb.AuthUserListRequest, opts ...grpc.CallOption) (resp *pb.AuthUserListResponse, err error) { return rac.ac.UserList(ctx, in, append(opts, withRepeatablePolicy())...) } func (rac *retryAuthClient) UserGet(ctx context.Context, in *pb.AuthUserGetRequest, opts ...grpc.CallOption) (resp *pb.AuthUserGetResponse, err error) { return rac.ac.UserGet(ctx, in, append(opts, withRepeatablePolicy())...) } func (rac *retryAuthClient) RoleGet(ctx context.Context, in *pb.AuthRoleGetRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleGetResponse, err error) { return rac.ac.RoleGet(ctx, in, append(opts, withRepeatablePolicy())...) } func (rac *retryAuthClient) RoleList(ctx context.Context, in *pb.AuthRoleListRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleListResponse, err error) { return rac.ac.RoleList(ctx, in, append(opts, withRepeatablePolicy())...) } func (rac *retryAuthClient) AuthEnable(ctx context.Context, in *pb.AuthEnableRequest, opts ...grpc.CallOption) (resp *pb.AuthEnableResponse, err error) { return rac.ac.AuthEnable(ctx, in, opts...) } func (rac *retryAuthClient) AuthDisable(ctx context.Context, in *pb.AuthDisableRequest, opts ...grpc.CallOption) (resp *pb.AuthDisableResponse, err error) { return rac.ac.AuthDisable(ctx, in, opts...) } func (rac *retryAuthClient) AuthStatus(ctx context.Context, in *pb.AuthStatusRequest, opts ...grpc.CallOption) (resp *pb.AuthStatusResponse, err error) { return rac.ac.AuthStatus(ctx, in, opts...) } func (rac *retryAuthClient) UserAdd(ctx context.Context, in *pb.AuthUserAddRequest, opts ...grpc.CallOption) (resp *pb.AuthUserAddResponse, err error) { return rac.ac.UserAdd(ctx, in, opts...) } func (rac *retryAuthClient) UserDelete(ctx context.Context, in *pb.AuthUserDeleteRequest, opts ...grpc.CallOption) (resp *pb.AuthUserDeleteResponse, err error) { return rac.ac.UserDelete(ctx, in, opts...) } func (rac *retryAuthClient) UserChangePassword(ctx context.Context, in *pb.AuthUserChangePasswordRequest, opts ...grpc.CallOption) (resp *pb.AuthUserChangePasswordResponse, err error) { return rac.ac.UserChangePassword(ctx, in, opts...) } func (rac *retryAuthClient) UserGrantRole(ctx context.Context, in *pb.AuthUserGrantRoleRequest, opts ...grpc.CallOption) (resp *pb.AuthUserGrantRoleResponse, err error) { return rac.ac.UserGrantRole(ctx, in, opts...) } func (rac *retryAuthClient) UserRevokeRole(ctx context.Context, in *pb.AuthUserRevokeRoleRequest, opts ...grpc.CallOption) (resp *pb.AuthUserRevokeRoleResponse, err error) { return rac.ac.UserRevokeRole(ctx, in, opts...) } func (rac *retryAuthClient) RoleAdd(ctx context.Context, in *pb.AuthRoleAddRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleAddResponse, err error) { return rac.ac.RoleAdd(ctx, in, opts...) } func (rac *retryAuthClient) RoleDelete(ctx context.Context, in *pb.AuthRoleDeleteRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleDeleteResponse, err error) { return rac.ac.RoleDelete(ctx, in, opts...) } func (rac *retryAuthClient) RoleGrantPermission(ctx context.Context, in *pb.AuthRoleGrantPermissionRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleGrantPermissionResponse, err error) { return rac.ac.RoleGrantPermission(ctx, in, opts...) } func (rac *retryAuthClient) RoleRevokePermission(ctx context.Context, in *pb.AuthRoleRevokePermissionRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleRevokePermissionResponse, err error) { return rac.ac.RoleRevokePermission(ctx, in, opts...) } func (rac *retryAuthClient) Authenticate(ctx context.Context, in *pb.AuthenticateRequest, opts ...grpc.CallOption) (resp *pb.AuthenticateResponse, err error) { return rac.ac.Authenticate(ctx, in, opts...) } ================================================ FILE: client/v3/retry_interceptor.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Based on github.com/grpc-ecosystem/go-grpc-middleware/retry, but modified to support the more // fine grained error checking required by write-at-most-once retry semantics of etcd. package clientv3 import ( "context" "errors" "io" "sync" "time" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" "google.golang.org/grpc/status" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" ) // unaryClientInterceptor returns a new retrying unary client interceptor. // // The default configuration of the interceptor is to not retry *at all*. This behaviour can be // changed through options (e.g. WithMax) on creation of the interceptor or on call (through grpc.CallOptions). func (c *Client) unaryClientInterceptor(optFuncs ...retryOption) grpc.UnaryClientInterceptor { intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs) return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { ctx = withVersion(ctx) grpcOpts, retryOpts := filterCallOptions(opts) var p peer.Peer grpcOpts = append(grpcOpts, grpc.Peer(&p)) callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts) // short circuit for simplicity, and avoiding allocations. if callOpts.max == 0 { return invoker(ctx, method, req, reply, cc, grpcOpts...) } var lastErr error for attempt := uint(0); attempt < callOpts.max; attempt++ { if err := waitRetryBackoff(ctx, attempt, callOpts); err != nil { return err } c.GetLogger().Debug( "retrying of unary invoker", zap.String("target", cc.Target()), zap.String("method", method), zap.Uint("attempt", attempt), ) lastErr = invoker(ctx, method, req, reply, cc, grpcOpts...) if lastErr == nil { return nil } c.GetLogger().Warn( "retrying of unary invoker failed", zap.String("target", cc.Target()), zap.String("peer", p.String()), zap.String("method", method), zap.Uint("attempt", attempt), zap.Error(lastErr), ) if isContextError(lastErr) { if ctx.Err() != nil { // its the context deadline or cancellation. return lastErr } // its the callCtx deadline or cancellation, in which case try again. continue } if c.shouldRefreshToken(lastErr, callOpts) { gtErr := c.refreshToken(ctx) if gtErr != nil { c.GetLogger().Warn( "retrying of unary invoker failed to fetch new auth token", zap.String("target", cc.Target()), zap.Error(gtErr), ) return gtErr // lastErr must be invalid auth token } continue } if !isSafeRetry(c, lastErr, callOpts) { return lastErr } } return lastErr } } // streamClientInterceptor returns a new retrying stream client interceptor for server side streaming calls. // // The default configuration of the interceptor is to not retry *at all*. This behaviour can be // changed through options (e.g. WithMax) on creation of the interceptor or on call (through grpc.CallOptions). // // Retry logic is available *only for ServerStreams*, i.e. 1:n streams, as the internal logic needs // to buffer the messages sent by the client. If retry is enabled on any other streams (ClientStreams, // BidiStreams), the retry interceptor will fail the call. func (c *Client) streamClientInterceptor(optFuncs ...retryOption) grpc.StreamClientInterceptor { intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs) return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { ctx = withVersion(ctx) // getToken automatically. Otherwise, auth token may be invalid after watch reconnection because the token has expired // (see https://github.com/etcd-io/etcd/issues/11954 for more). err := c.getToken(ctx) if err != nil { c.GetLogger().Error("clientv3/retry_interceptor: getToken failed", zap.Error(err)) return nil, err } grpcOpts, retryOpts := filterCallOptions(opts) callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts) // short circuit for simplicity, and avoiding allocations. if callOpts.max == 0 { return streamer(ctx, desc, cc, method, grpcOpts...) } if desc.ClientStreams { return nil, status.Errorf(codes.Unimplemented, "clientv3/retry_interceptor: cannot retry on ClientStreams, set Disable()") } newStreamer, err := streamer(ctx, desc, cc, method, grpcOpts...) if err != nil { c.GetLogger().Error("streamer failed to create ClientStream", zap.Error(err)) return nil, err // TODO(mwitkow): Maybe dial and transport errors should be retriable? } retryingStreamer := &serverStreamingRetryingStream{ client: c, ClientStream: newStreamer, callOpts: callOpts, ctx: ctx, streamerCall: func(ctx context.Context) (grpc.ClientStream, error) { return streamer(ctx, desc, cc, method, grpcOpts...) }, } return retryingStreamer, nil } } // shouldRefreshToken checks whether there's a need to refresh the token based on the error and callOptions, // and returns a boolean value. func (c *Client) shouldRefreshToken(err error, callOpts *options) bool { if c.Token != "" { // do not try to refresh the token as it is set by user return false } if errors.Is(rpctypes.Error(err), rpctypes.ErrUserEmpty) { // refresh the token when username, password is present but the server returns ErrUserEmpty // which is possible when the client token is cleared somehow return c.authTokenBundle != nil // equal to c.Username != "" && c.Password != "" } return callOpts.retryAuth && (errors.Is(rpctypes.Error(err), rpctypes.ErrInvalidAuthToken) || errors.Is(rpctypes.Error(err), rpctypes.ErrAuthOldRevision)) } func (c *Client) refreshToken(ctx context.Context) error { if c.authTokenBundle == nil { // c.authTokenBundle will be initialized only when // c.Username != "" && c.Password != "". // // When users use the TLS CommonName based authentication, the // authTokenBundle is always nil. But it's possible for the clients // to get `rpctypes.ErrAuthOldRevision` response when the clients // concurrently modify auth data (e.g, addUser, deleteUser etc.). // In this case, there is no need to refresh the token; instead the // clients just need to retry the operations (e.g. Put, Delete etc). return nil } return c.getToken(ctx) } // type serverStreamingRetryingStream is the implementation of grpc.ClientStream that acts as a // proxy to the underlying call. If any of the RecvMsg() calls fail, it will try to reestablish // a new ClientStream according to the retry policy. type serverStreamingRetryingStream struct { grpc.ClientStream client *Client bufferedSends []any // single message that the client can sen receivedGood bool // indicates whether any prior receives were successful wasClosedSend bool // indicates that CloseSend was closed ctx context.Context callOpts *options streamerCall func(ctx context.Context) (grpc.ClientStream, error) mu sync.RWMutex } func (s *serverStreamingRetryingStream) setStream(clientStream grpc.ClientStream) { s.mu.Lock() s.ClientStream = clientStream s.mu.Unlock() } func (s *serverStreamingRetryingStream) getStream() grpc.ClientStream { s.mu.RLock() defer s.mu.RUnlock() return s.ClientStream } func (s *serverStreamingRetryingStream) SendMsg(m any) error { s.mu.Lock() s.bufferedSends = append(s.bufferedSends, m) s.mu.Unlock() return s.getStream().SendMsg(m) } func (s *serverStreamingRetryingStream) CloseSend() error { s.mu.Lock() s.wasClosedSend = true s.mu.Unlock() return s.getStream().CloseSend() } func (s *serverStreamingRetryingStream) Header() (metadata.MD, error) { return s.getStream().Header() } func (s *serverStreamingRetryingStream) Trailer() metadata.MD { return s.getStream().Trailer() } func (s *serverStreamingRetryingStream) RecvMsg(m any) error { attemptRetry, lastErr := s.receiveMsgAndIndicateRetry(m) if !attemptRetry { return lastErr // success or hard failure } // We start off from attempt 1, because zeroth was already made on normal SendMsg(). for attempt := uint(1); attempt < s.callOpts.max; attempt++ { if err := waitRetryBackoff(s.ctx, attempt, s.callOpts); err != nil { return err } newStream, err := s.reestablishStreamAndResendBuffer(s.ctx) if err != nil { s.client.GetLogger().Error("failed reestablishStreamAndResendBuffer", zap.Error(err)) return err // TODO(mwitkow): Maybe dial and transport errors should be retriable? } s.setStream(newStream) s.client.GetLogger().Warn("retrying RecvMsg", zap.Error(lastErr)) attemptRetry, lastErr = s.receiveMsgAndIndicateRetry(m) if !attemptRetry { return lastErr } } return lastErr } func (s *serverStreamingRetryingStream) receiveMsgAndIndicateRetry(m any) (bool, error) { s.mu.RLock() wasGood := s.receivedGood s.mu.RUnlock() err := s.getStream().RecvMsg(m) if err == nil || errors.Is(err, io.EOF) { s.mu.Lock() s.receivedGood = true s.mu.Unlock() return false, err } else if wasGood { // previous RecvMsg in the stream succeeded, no retry logic should interfere return false, err } if isContextError(err) { if s.ctx.Err() != nil { return false, err } // its the callCtx deadline or cancellation, in which case try again. return true, err } if s.client.shouldRefreshToken(err, s.callOpts) { gtErr := s.client.refreshToken(s.ctx) if gtErr != nil { s.client.GetLogger().Warn("retry failed to fetch new auth token", zap.Error(gtErr)) return false, err // return the original error for simplicity } return true, err } return isSafeRetry(s.client, err, s.callOpts), err } func (s *serverStreamingRetryingStream) reestablishStreamAndResendBuffer(callCtx context.Context) (grpc.ClientStream, error) { s.mu.RLock() bufferedSends := s.bufferedSends s.mu.RUnlock() newStream, err := s.streamerCall(callCtx) if err != nil { return nil, err } for _, msg := range bufferedSends { if err := newStream.SendMsg(msg); err != nil { return nil, err } } if err := newStream.CloseSend(); err != nil { return nil, err } return newStream, nil } func waitRetryBackoff(ctx context.Context, attempt uint, callOpts *options) error { waitTime := time.Duration(0) if attempt > 0 { waitTime = callOpts.backoffFunc(attempt) } if waitTime > 0 { timer := time.NewTimer(waitTime) select { case <-ctx.Done(): timer.Stop() return contextErrToGRPCErr(ctx.Err()) case <-timer.C: } } return nil } // isSafeRetry returns "true", if request is safe for retry with the given error. func isSafeRetry(c *Client, err error, callOpts *options) bool { if isContextError(err) { return false } // Situation when learner refuses RPC it is supposed to not serve is from the server // perspective not retryable. // But for backward-compatibility reasons we need to support situation that // customer provides mix of learners (not yet voters) and voters with an // expectation to pick voter in the next attempt. // TODO: Ideally client should be 'aware' which endpoint represents: leader/voter/learner with high probability. if errors.Is(err, rpctypes.ErrGRPCNotSupportedForLearner) && len(c.Endpoints()) > 1 { return true } switch callOpts.retryPolicy { case repeatable: return isSafeRetryImmutableRPC(err) case nonRepeatable: return isSafeRetryMutableRPC(err) default: c.GetLogger().Warn("unrecognized retry policy", zap.String("retryPolicy", callOpts.retryPolicy.String())) return false } } func isContextError(err error) bool { return status.Code(err) == codes.DeadlineExceeded || status.Code(err) == codes.Canceled } func contextErrToGRPCErr(err error) error { switch { case errors.Is(err, context.DeadlineExceeded): return status.Error(codes.DeadlineExceeded, err.Error()) case errors.Is(err, context.Canceled): return status.Error(codes.Canceled, err.Error()) default: return status.Error(codes.Unknown, err.Error()) } } var defaultOptions = &options{ retryPolicy: nonRepeatable, max: 0, // disable backoffFunc: backoffLinearWithJitter(50*time.Millisecond /*jitter*/, 0.10), retryAuth: true, } // backoffFunc denotes a family of functions that control the backoff duration between call retries. // // They are called with an identifier of the attempt, and should return a time the system client should // hold off for. If the time returned is longer than the `context.Context.Deadline` of the request // the deadline of the request takes precedence and the wait will be interrupted before proceeding // with the next iteration. type backoffFunc func(attempt uint) time.Duration // withRepeatablePolicy sets the repeatable policy of this call. func withRepeatablePolicy() retryOption { return retryOption{applyFunc: func(o *options) { o.retryPolicy = repeatable }} } // withMax sets the maximum number of retries on this call, or this interceptor. func withMax(maxRetries uint) retryOption { return retryOption{applyFunc: func(o *options) { o.max = maxRetries }} } // WithBackoff sets the `BackoffFunc` used to control time between retries. func withBackoff(bf backoffFunc) retryOption { return retryOption{applyFunc: func(o *options) { o.backoffFunc = bf }} } type options struct { retryPolicy retryPolicy max uint backoffFunc backoffFunc retryAuth bool } // retryOption is a grpc.CallOption that is local to clientv3's retry interceptor. type retryOption struct { grpc.EmptyCallOption // make sure we implement private after() and before() fields so we don't panic. applyFunc func(opt *options) } func reuseOrNewWithCallOptions(opt *options, retryOptions []retryOption) *options { if len(retryOptions) == 0 { return opt } optCopy := &options{} *optCopy = *opt for _, f := range retryOptions { f.applyFunc(optCopy) } return optCopy } func filterCallOptions(callOptions []grpc.CallOption) (grpcOptions []grpc.CallOption, retryOptions []retryOption) { for _, opt := range callOptions { if co, ok := opt.(retryOption); ok { retryOptions = append(retryOptions, co) } else { grpcOptions = append(grpcOptions, opt) } } return grpcOptions, retryOptions } // BackoffLinearWithJitter waits a set period of time, allowing for jitter (fractional adjustment). // // For example waitBetween=1s and jitter=0.10 can generate waits between 900ms and 1100ms. func backoffLinearWithJitter(waitBetween time.Duration, jitterFraction float64) backoffFunc { return func(attempt uint) time.Duration { return jitterUp(waitBetween, jitterFraction) } } ================================================ FILE: client/v3/retry_interceptor_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "testing" grpccredentials "google.golang.org/grpc/credentials" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/v3/credentials" ) type dummyAuthTokenBundle struct{} func (d dummyAuthTokenBundle) PerRPCCredentials() grpccredentials.PerRPCCredentials { return nil } func (d dummyAuthTokenBundle) UpdateAuthToken(token string) { } func TestClientShouldRefreshToken(t *testing.T) { type fields struct { authTokenBundle credentials.PerRPCCredentialsBundle token string } type args struct { err error callOpts *options } optsWithTrue := &options{ retryAuth: true, } optsWithFalse := &options{ retryAuth: false, } tests := []struct { name string fields fields args args want bool }{ { name: "ErrUserEmpty and non nil authTokenBundle", fields: fields{ authTokenBundle: &dummyAuthTokenBundle{}, }, args: args{rpctypes.ErrGRPCUserEmpty, optsWithTrue}, want: true, }, { name: "ErrUserEmpty and nil authTokenBundle", fields: fields{ authTokenBundle: nil, }, args: args{rpctypes.ErrGRPCUserEmpty, optsWithTrue}, want: false, }, { name: "ErrGRPCInvalidAuthToken and retryAuth", fields: fields{ authTokenBundle: nil, }, args: args{rpctypes.ErrGRPCInvalidAuthToken, optsWithTrue}, want: true, }, { name: "ErrGRPCInvalidAuthToken and !retryAuth", fields: fields{ authTokenBundle: nil, }, args: args{rpctypes.ErrGRPCInvalidAuthToken, optsWithFalse}, want: false, }, { name: "ErrGRPCAuthOldRevision and retryAuth", fields: fields{ authTokenBundle: nil, }, args: args{rpctypes.ErrGRPCAuthOldRevision, optsWithTrue}, want: true, }, { name: "ErrGRPCAuthOldRevision and !retryAuth", fields: fields{ authTokenBundle: nil, }, args: args{rpctypes.ErrGRPCAuthOldRevision, optsWithFalse}, want: false, }, { name: "Other error and retryAuth", fields: fields{ authTokenBundle: nil, }, args: args{rpctypes.ErrGRPCAuthFailed, optsWithTrue}, want: false, }, { name: "Other error and !retryAuth", fields: fields{ authTokenBundle: nil, }, args: args{rpctypes.ErrGRPCAuthFailed, optsWithFalse}, want: false, }, { name: "User provided token, ErrGRPCInvalidAuthToken", fields: fields{ authTokenBundle: nil, token: "user-supplied-token", }, args: args{rpctypes.ErrGRPCInvalidAuthToken, optsWithTrue}, want: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &Client{ authTokenBundle: tt.fields.authTokenBundle, Token: tt.fields.token, } if got := c.shouldRefreshToken(tt.args.err, tt.args.callOpts); got != tt.want { t.Errorf("shouldRefreshToken() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: client/v3/snapshot/doc.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package snapshot implements utilities around etcd snapshot. package snapshot ================================================ FILE: client/v3/snapshot/v3_snapshot.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package snapshot import ( "context" "crypto/sha256" "errors" "fmt" "io" "os" "time" "github.com/dustin/go-humanize" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/fileutil" clientv3 "go.etcd.io/etcd/client/v3" ) // hasChecksum returns "true" if the file size "n" // has appended sha256 hash digest. func hasChecksum(n int64) bool { // 512 is chosen because it's a minimum disk sector size // smaller than (and multiplies to) OS page size in most systems return (n % 512) == sha256.Size } // SaveWithVersion fetches snapshot from remote etcd server, saves data // to target path and returns server version. If the context "ctx" is canceled or timed out, // snapshot save stream will error out (e.g. context.Canceled, // context.DeadlineExceeded). Make sure to specify only one endpoint // in client configuration. Snapshot API must be requested to a // selected node, and saved snapshot is the point-in-time state of // the selected node. // Etcd ", v1), // Compare(Version(k1), "=", 2) // ).Then( // OpPut(k2,v2), OpPut(k3,v3) // ).Else( // OpPut(k4,v4), OpPut(k5,v5) // ).Commit() type Txn interface { // If takes a list of comparison. If all comparisons passed in succeed, // the operations passed into Then() will be executed. Or the operations // passed into Else() will be executed. If(cs ...Cmp) Txn // Then takes a list of operations. The Ops list will be executed, if the // comparisons passed in If() succeed. Then(ops ...Op) Txn // Else takes a list of operations. The Ops list will be executed, if the // comparisons passed in If() fail. Else(ops ...Op) Txn // Commit tries to commit the transaction. Commit() (*TxnResponse, error) } type txn struct { kv *kv ctx context.Context mu sync.Mutex cif bool cthen bool celse bool isWrite bool cmps []*pb.Compare sus []*pb.RequestOp fas []*pb.RequestOp callOpts []grpc.CallOption } func (txn *txn) If(cs ...Cmp) Txn { txn.mu.Lock() defer txn.mu.Unlock() if txn.cif { panic("cannot call If twice!") } if txn.cthen { panic("cannot call If after Then!") } if txn.celse { panic("cannot call If after Else!") } txn.cif = true for i := range cs { txn.cmps = append(txn.cmps, (*pb.Compare)(&cs[i])) } return txn } func (txn *txn) Then(ops ...Op) Txn { txn.mu.Lock() defer txn.mu.Unlock() if txn.cthen { panic("cannot call Then twice!") } if txn.celse { panic("cannot call Then after Else!") } txn.cthen = true for _, op := range ops { txn.isWrite = txn.isWrite || op.isWrite() txn.sus = append(txn.sus, op.toRequestOp()) } return txn } func (txn *txn) Else(ops ...Op) Txn { txn.mu.Lock() defer txn.mu.Unlock() if txn.celse { panic("cannot call Else twice!") } txn.celse = true for _, op := range ops { txn.isWrite = txn.isWrite || op.isWrite() txn.fas = append(txn.fas, op.toRequestOp()) } return txn } func (txn *txn) Commit() (*TxnResponse, error) { txn.mu.Lock() defer txn.mu.Unlock() r := &pb.TxnRequest{Compare: txn.cmps, Success: txn.sus, Failure: txn.fas} var resp *pb.TxnResponse var err error resp, err = txn.kv.remote.Txn(txn.ctx, r, txn.callOpts...) if err != nil { return nil, ContextError(txn.ctx, err) } return (*TxnResponse)(resp), nil } ================================================ FILE: client/v3/txn_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "testing" "time" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func TestTxnPanics(t *testing.T) { testutil.RegisterLeakDetection(t) kv := &kv{} df := func(errc chan string) { if s := recover(); s != nil { errc <- s.(string) } } cmp := Compare(CreateRevision("foo"), "=", 0) op := OpPut("foo", "bar") tests := []struct { f func(chan string) err string }{ { f: func(errc chan string) { defer df(errc) kv.Txn(t.Context()).If(cmp).If(cmp) }, err: "cannot call If twice!", }, { f: func(errc chan string) { defer df(errc) kv.Txn(t.Context()).Then(op).If(cmp) }, err: "cannot call If after Then!", }, { f: func(errc chan string) { defer df(errc) kv.Txn(t.Context()).Else(op).If(cmp) }, err: "cannot call If after Else!", }, { f: func(errc chan string) { defer df(errc) kv.Txn(t.Context()).Then(op).Then(op) }, err: "cannot call Then twice!", }, { f: func(errc chan string) { defer df(errc) kv.Txn(t.Context()).Else(op).Then(op) }, err: "cannot call Then after Else!", }, { f: func(errc chan string) { defer df(errc) kv.Txn(t.Context()).Else(op).Else(op) }, err: "cannot call Else twice!", }, } for i, tt := range tests { errc := make(chan string, 1) go tt.f(errc) select { case err := <-errc: if err != tt.err { t.Errorf("#%d: got %s, wanted %s", i, err, tt.err) } case <-time.After(time.Second): t.Errorf("#%d: did not panic, wanted panic %s", i, tt.err) } } } ================================================ FILE: client/v3/utils.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "math/rand" "time" ) // jitterUp adds random jitter to the duration. // // This adds or subtracts time from the duration within a given jitter fraction. // For example for 10s and jitter 0.1, it will return a time within [9s, 11s]) // // Reference: https://godoc.org/github.com/grpc-ecosystem/go-grpc-middleware/util/backoffutils func jitterUp(duration time.Duration, jitter float64) time.Duration { multiplier := jitter * (rand.Float64()*2 - 1) return time.Duration(float64(duration) * (1 + multiplier)) } ================================================ FILE: client/v3/watch.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "errors" "fmt" "sync" "time" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" v3rpc "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" ) const ( EventTypeDelete = mvccpb.Event_DELETE EventTypePut = mvccpb.Event_PUT closeSendErrTimeout = 250 * time.Millisecond // AutoWatchID is the watcher ID passed in WatchStream.Watch when no // user-provided ID is available. If pass, an ID will automatically be assigned. AutoWatchID = 0 // InvalidWatchID represents an invalid watch ID and prevents duplication with an existing watch. InvalidWatchID = -1 ) type Event mvccpb.Event type WatchChan <-chan WatchResponse type Watcher interface { // Watch watches on a key or prefix. The watched events will be returned // through the returned channel. If revisions waiting to be sent over the // watch are compacted, then the watch will be canceled by the server, the // client will post a compacted error watch response, and the channel will close. // If the requested revision is 0 or unspecified, the returned channel will // return watch events that happen after the server receives the watch request. // If the context "ctx" is canceled or timed out, returned "WatchChan" is closed, // and "WatchResponse" from this closed channel has zero events and nil "Err()". // The context "ctx" MUST be canceled, as soon as watcher is no longer being used, // to release the associated resources. // // If the context is "context.Background/TODO", returned "WatchChan" will // not be closed and block until event is triggered, except when server // returns a non-recoverable error (e.g. ErrCompacted). // For example, when context passed with "WithRequireLeader" and the // connected server has no leader (e.g. due to network partition), // error "etcdserver: no leader" (ErrNoLeader) will be returned, // and then "WatchChan" is closed with non-nil "Err()". // In order to prevent a watch stream being stuck in a partitioned node, // make sure to wrap context with "WithRequireLeader". // // Otherwise, as long as the context has not been canceled or timed out, // watch will retry on other recoverable errors forever until reconnected. // // TODO: explicitly set context error in the last "WatchResponse" message and close channel? // Currently, client contexts are overwritten with "valCtx" that never closes. // TODO(v3.4): configure watch retry policy, limit maximum retry number // (see https://github.com/etcd-io/etcd/issues/8980) Watch(ctx context.Context, key string, opts ...OpOption) WatchChan // RequestProgress requests a progress notify response be sent in all watch channels. RequestProgress(ctx context.Context) error // Close closes the watcher and cancels all watch requests. Close() error } type WatchResponse struct { Header pb.ResponseHeader Events []*Event // CompactRevision is the minimum revision the watcher may receive. CompactRevision int64 // Canceled is used to indicate watch failure. // If the watch failed and the stream was about to close, before the channel is closed, // the channel sends a final response that has Canceled set to true with a non-nil Err(). Canceled bool // Created is used to indicate the creation of the watcher. Created bool closeErr error // CancelReason is a reason of canceling watch CancelReason string } // IsCreate returns true if the event tells that the key is newly created. func (e *Event) IsCreate() bool { return e.Type == EventTypePut && e.Kv.CreateRevision == e.Kv.ModRevision } // IsModify returns true if the event tells that a new value is put on existing key. func (e *Event) IsModify() bool { return e.Type == EventTypePut && e.Kv.CreateRevision != e.Kv.ModRevision } // Err is the error value if this WatchResponse holds an error. func (wr *WatchResponse) Err() error { switch { case wr.closeErr != nil: return v3rpc.Error(wr.closeErr) case wr.CompactRevision != 0: return v3rpc.ErrCompacted case wr.Canceled: if len(wr.CancelReason) != 0 { return v3rpc.Error(status.Error(codes.FailedPrecondition, wr.CancelReason)) } return v3rpc.ErrFutureRev } return nil } // IsProgressNotify returns true if the WatchResponse is progress notification. func (wr *WatchResponse) IsProgressNotify() bool { return len(wr.Events) == 0 && !wr.Canceled && !wr.Created && wr.CompactRevision == 0 && wr.Header.Revision != 0 } // watcher implements the Watcher interface type watcher struct { remote pb.WatchClient callOpts []grpc.CallOption // mu protects the grpc streams map mu sync.Mutex // streams holds all the active grpc streams keyed by ctx value. streams map[string]*watchGRPCStream lg *zap.Logger } // watchGRPCStream tracks all watch resources attached to a single grpc stream. type watchGRPCStream struct { owner *watcher remote pb.WatchClient callOpts []grpc.CallOption // ctx controls internal remote.Watch requests ctx context.Context // ctxKey is the key used when looking up this stream's context ctxKey string cancel context.CancelFunc // substreams holds all active watchers on this grpc stream substreams map[int64]*watcherStream // resuming holds all resuming watchers on this grpc stream resuming []*watcherStream // reqc sends a watch request from Watch() to the main goroutine reqc chan watchStreamRequest // respc receives data from the watch client respc chan *pb.WatchResponse // donec closes to broadcast shutdown donec chan struct{} // errc transmits errors from grpc Recv to the watch stream reconnect logic errc chan error // closingc gets the watcherStream of closing watchers closingc chan *watcherStream // wg is Done when all substream goroutines have exited wg sync.WaitGroup // resumec closes to signal that all substreams should begin resuming resumec chan struct{} // closeErr is the error that closed the watch stream closeErr error lg *zap.Logger } // watchStreamRequest is a union of the supported watch request operation types type watchStreamRequest interface { toPB() *pb.WatchRequest } // watchRequest is issued by the subscriber to start a new watcher type watchRequest struct { ctx context.Context key string end string rev int64 // send created notification event if this field is true createdNotify bool // progressNotify is for progress updates progressNotify bool // fragmentation should be disabled by default // if true, split watch events when total exceeds // "--max-request-bytes" flag value + 512-byte fragment bool // filters is the list of events to filter out filters []pb.WatchCreateRequest_FilterType // get the previous key-value pair before the event happens prevKV bool // retc receives a chan WatchResponse once the watcher is established retc chan chan WatchResponse } // progressRequest is issued by the subscriber to request watch progress type progressRequest struct{} // watcherStream represents a registered watcher type watcherStream struct { // initReq is the request that initiated this request initReq watchRequest // outc publishes watch responses to subscriber outc chan WatchResponse // recvc buffers watch responses before publishing recvc chan *WatchResponse // donec closes when the watcherStream goroutine stops. donec chan struct{} // closing is set to true when stream should be scheduled to shutdown. closing bool // id is the registered watch id on the grpc stream id int64 // buf holds all events received from etcd but not yet consumed by the client buf []*WatchResponse } func NewWatcher(c *Client) Watcher { return NewWatchFromWatchClient(pb.NewWatchClient(c.conn), c) } func NewWatchFromWatchClient(wc pb.WatchClient, c *Client) Watcher { w := &watcher{ remote: wc, streams: make(map[string]*watchGRPCStream), } if c != nil { w.callOpts = c.callOpts w.lg = c.GetLogger() } return w } // never closes var ( valCtxCh = make(chan struct{}) zeroTime = time.Unix(0, 0) ) // ctx with only the values; never Done type valCtx struct{ context.Context } func (vc *valCtx) Deadline() (time.Time, bool) { return zeroTime, false } func (vc *valCtx) Done() <-chan struct{} { return valCtxCh } func (vc *valCtx) Err() error { return nil } func (w *watcher) newWatcherGRPCStream(inctx context.Context) *watchGRPCStream { ctx, cancel := context.WithCancel(&valCtx{inctx}) wgs := &watchGRPCStream{ owner: w, remote: w.remote, callOpts: w.callOpts, ctx: ctx, ctxKey: streamKeyFromCtx(inctx), cancel: cancel, substreams: make(map[int64]*watcherStream), respc: make(chan *pb.WatchResponse), reqc: make(chan watchStreamRequest), donec: make(chan struct{}), errc: make(chan error, 1), closingc: make(chan *watcherStream), resumec: make(chan struct{}), lg: w.lg, } go wgs.run() return wgs } // Watch posts a watch request to run() and waits for a new watcher channel func (w *watcher) Watch(ctx context.Context, key string, opts ...OpOption) WatchChan { ow := OpWatch(key, opts...) var filters []pb.WatchCreateRequest_FilterType if ow.filterPut { filters = append(filters, pb.WatchCreateRequest_NOPUT) } if ow.filterDelete { filters = append(filters, pb.WatchCreateRequest_NODELETE) } wr := &watchRequest{ ctx: ctx, createdNotify: ow.createdNotify, key: string(ow.key), end: string(ow.end), rev: ow.rev, progressNotify: ow.progressNotify, fragment: ow.fragment, filters: filters, prevKV: ow.prevKV, retc: make(chan chan WatchResponse, 1), } ok := false ctxKey := streamKeyFromCtx(ctx) var closeCh chan WatchResponse for { // find or allocate appropriate grpc watch stream w.mu.Lock() if w.streams == nil { // closed w.mu.Unlock() ch := make(chan WatchResponse) close(ch) return ch } wgs := w.streams[ctxKey] if wgs == nil { wgs = w.newWatcherGRPCStream(ctx) w.streams[ctxKey] = wgs } donec := wgs.donec reqc := wgs.reqc w.mu.Unlock() // couldn't create channel; return closed channel if closeCh == nil { closeCh = make(chan WatchResponse, 1) } // submit request select { case reqc <- wr: ok = true case <-wr.ctx.Done(): ok = false case <-donec: ok = false if wgs.closeErr != nil { closeCh <- WatchResponse{Canceled: true, closeErr: wgs.closeErr} break } // retry; may have dropped stream from no ctxs continue } // receive channel if ok { select { case ret := <-wr.retc: return ret case <-ctx.Done(): case <-donec: if wgs.closeErr != nil { closeCh <- WatchResponse{Canceled: true, closeErr: wgs.closeErr} break } // retry; may have dropped stream from no ctxs continue } } break } close(closeCh) return closeCh } func (w *watcher) Close() (err error) { w.mu.Lock() streams := w.streams w.streams = nil w.mu.Unlock() for _, wgs := range streams { if werr := wgs.close(); werr != nil { err = werr } } // Consider context.Canceled as a successful close if errors.Is(err, context.Canceled) { err = nil } return err } // RequestProgress requests a progress notify response be sent in all watch channels. func (w *watcher) RequestProgress(ctx context.Context) (err error) { ctxKey := streamKeyFromCtx(ctx) w.mu.Lock() if w.streams == nil { w.mu.Unlock() return errors.New("no stream found for context") } wgs := w.streams[ctxKey] if wgs == nil { wgs = w.newWatcherGRPCStream(ctx) w.streams[ctxKey] = wgs } donec := wgs.donec reqc := wgs.reqc w.mu.Unlock() pr := &progressRequest{} select { case reqc <- pr: return nil case <-ctx.Done(): return ctx.Err() case <-donec: if wgs.closeErr != nil { return wgs.closeErr } // retry; may have dropped stream from no ctxs return w.RequestProgress(ctx) } } func (w *watchGRPCStream) close() (err error) { w.cancel() <-w.donec select { case err = <-w.errc: default: } return ContextError(w.ctx, err) } func (w *watcher) closeStream(wgs *watchGRPCStream) { w.mu.Lock() close(wgs.donec) wgs.cancel() if w.streams != nil { delete(w.streams, wgs.ctxKey) } w.mu.Unlock() } func (w *watchGRPCStream) addSubstream(resp *pb.WatchResponse, ws *watcherStream) { // check watch ID for backward compatibility (<= v3.3) if resp.WatchId == InvalidWatchID || (resp.Canceled && resp.CancelReason != "") { w.closeErr = v3rpc.Error(errors.New(resp.CancelReason)) // failed; no channel close(ws.recvc) return } ws.id = resp.WatchId w.substreams[ws.id] = ws } func (w *watchGRPCStream) sendCloseSubstream(ws *watcherStream, resp *WatchResponse) { select { case ws.outc <- *resp: case <-ws.initReq.ctx.Done(): case <-time.After(closeSendErrTimeout): } close(ws.outc) } func (w *watchGRPCStream) closeSubstream(ws *watcherStream) { // send channel response in case stream was never established select { case ws.initReq.retc <- ws.outc: default: } // close subscriber's channel if closeErr := w.closeErr; closeErr != nil && ws.initReq.ctx.Err() == nil { go w.sendCloseSubstream(ws, &WatchResponse{Canceled: true, closeErr: w.closeErr}) } else if ws.outc != nil { close(ws.outc) } if ws.id != InvalidWatchID { delete(w.substreams, ws.id) return } for i := range w.resuming { if w.resuming[i] == ws { w.resuming[i] = nil return } } } // run is the root of the goroutines for managing a watcher client func (w *watchGRPCStream) run() { var wc pb.Watch_WatchClient var closeErr error // substreams marked to close but goroutine still running; needed for // avoiding double-closing recvc on grpc stream teardown closing := make(map[*watcherStream]struct{}) defer func() { w.closeErr = closeErr // shutdown substreams and resuming substreams for _, ws := range w.substreams { if _, ok := closing[ws]; !ok { close(ws.recvc) closing[ws] = struct{}{} } } for _, ws := range w.resuming { if _, ok := closing[ws]; ws != nil && !ok { close(ws.recvc) closing[ws] = struct{}{} } } w.joinSubstreams() for range closing { w.closeSubstream(<-w.closingc) } w.wg.Wait() w.owner.closeStream(w) }() // start a stream with the etcd grpc server if wc, closeErr = w.newWatchClient(); closeErr != nil { return } cancelSet := make(map[int64]struct{}) var cur *pb.WatchResponse backoff := time.Millisecond for { select { // Watch() requested case req := <-w.reqc: switch wreq := req.(type) { case *watchRequest: outc := make(chan WatchResponse, 1) // TODO: pass custom watch ID? ws := &watcherStream{ initReq: *wreq, id: InvalidWatchID, outc: outc, // unbuffered so resumes won't cause repeat events recvc: make(chan *WatchResponse), } ws.donec = make(chan struct{}) w.wg.Add(1) go w.serveSubstream(ws, w.resumec) // queue up for watcher creation/resume w.resuming = append(w.resuming, ws) if len(w.resuming) == 1 { // head of resume queue, can register a new watcher if err := wc.Send(ws.initReq.toPB()); err != nil { w.lg.Debug("error when sending request", zap.Error(err)) } } case *progressRequest: if err := wc.Send(wreq.toPB()); err != nil { w.lg.Debug("error when sending request", zap.Error(err)) } } // new events from the watch client case pbresp := <-w.respc: if cur == nil || pbresp.Created || pbresp.Canceled { cur = pbresp } else if cur.WatchId == pbresp.WatchId { // merge new events cur.Events = append(cur.Events, pbresp.Events...) // update "Fragment" field; last response with "Fragment" == false cur.Fragment = pbresp.Fragment } switch { case pbresp.Created: // response to head of queue creation if len(w.resuming) != 0 { if ws := w.resuming[0]; ws != nil { w.addSubstream(pbresp, ws) w.dispatchEvent(pbresp) w.resuming[0] = nil } } if ws := w.nextResume(); ws != nil { if err := wc.Send(ws.initReq.toPB()); err != nil { w.lg.Debug("error when sending request", zap.Error(err)) } } // reset for next iteration cur = nil case pbresp.Canceled && pbresp.CompactRevision == 0: delete(cancelSet, pbresp.WatchId) if ws, ok := w.substreams[pbresp.WatchId]; ok { // signal to stream goroutine to update closingc close(ws.recvc) closing[ws] = struct{}{} } // reset for next iteration cur = nil case cur.Fragment: // watch response events are still fragmented // continue to fetch next fragmented event arrival continue default: // dispatch to appropriate watch stream ok := w.dispatchEvent(cur) // reset for next iteration cur = nil if ok { break } // watch response on unexpected watch id; cancel id if _, ok := cancelSet[pbresp.WatchId]; ok { break } cancelSet[pbresp.WatchId] = struct{}{} cr := &pb.WatchRequest_CancelRequest{ CancelRequest: &pb.WatchCancelRequest{ WatchId: pbresp.WatchId, }, } req := &pb.WatchRequest{RequestUnion: cr} w.lg.Debug("sending watch cancel request for failed dispatch", zap.Int64("watch-id", pbresp.WatchId)) if err := wc.Send(req); err != nil { w.lg.Debug("failed to send watch cancel request", zap.Int64("watch-id", pbresp.WatchId), zap.Error(err)) } } // watch client failed on Recv; spawn another if possible case err := <-w.errc: if isHaltErr(w.ctx, err) || errors.Is(ContextError(w.ctx, err), v3rpc.ErrNoLeader) { closeErr = err return } backoff = w.backoffIfUnavailable(backoff, err) if wc, closeErr = w.newWatchClient(); closeErr != nil { return } if ws := w.nextResume(); ws != nil { if err := wc.Send(ws.initReq.toPB()); err != nil { w.lg.Debug("error when sending request", zap.Error(err)) } } cancelSet = make(map[int64]struct{}) case <-w.ctx.Done(): return case ws := <-w.closingc: w.closeSubstream(ws) delete(closing, ws) // no more watchers on this stream, shutdown, skip cancellation if len(w.substreams)+len(w.resuming) == 0 { return } if ws.id != InvalidWatchID { // client is closing an established watch; close it on the server proactively instead of waiting // to close when the next message arrives cancelSet[ws.id] = struct{}{} cr := &pb.WatchRequest_CancelRequest{ CancelRequest: &pb.WatchCancelRequest{ WatchId: ws.id, }, } req := &pb.WatchRequest{RequestUnion: cr} w.lg.Debug("sending watch cancel request for closed watcher", zap.Int64("watch-id", ws.id)) if err := wc.Send(req); err != nil { w.lg.Debug("failed to send watch cancel request", zap.Int64("watch-id", ws.id), zap.Error(err)) } } } } } // nextResume chooses the next resuming to register with the grpc stream. Abandoned // streams are marked as nil in the queue since the head must wait for its inflight registration. func (w *watchGRPCStream) nextResume() *watcherStream { for len(w.resuming) != 0 { if w.resuming[0] != nil { return w.resuming[0] } w.resuming = w.resuming[1:len(w.resuming)] } return nil } // dispatchEvent sends a WatchResponse to the appropriate watcher stream func (w *watchGRPCStream) dispatchEvent(pbresp *pb.WatchResponse) bool { events := make([]*Event, len(pbresp.Events)) for i, ev := range pbresp.Events { events[i] = (*Event)(ev) } // TODO: return watch ID? wr := &WatchResponse{ Header: *pbresp.Header, Events: events, CompactRevision: pbresp.CompactRevision, Created: pbresp.Created, Canceled: pbresp.Canceled, CancelReason: pbresp.CancelReason, } // watch IDs are zero indexed, so request notify watch responses are assigned a watch ID of InvalidWatchID to // indicate they should be broadcast. if wr.IsProgressNotify() && pbresp.WatchId == InvalidWatchID { return w.broadcastResponse(wr) } return w.unicastResponse(wr, pbresp.WatchId) } // broadcastResponse send a watch response to all watch substreams. func (w *watchGRPCStream) broadcastResponse(wr *WatchResponse) bool { for _, ws := range w.substreams { select { case ws.recvc <- wr: case <-ws.donec: } } return true } // unicastResponse sends a watch response to a specific watch substream. func (w *watchGRPCStream) unicastResponse(wr *WatchResponse, watchID int64) bool { ws, ok := w.substreams[watchID] if !ok { return false } select { case ws.recvc <- wr: case <-ws.donec: return false } return true } // serveWatchClient forwards messages from the grpc stream to run() func (w *watchGRPCStream) serveWatchClient(wc pb.Watch_WatchClient) { for { resp, err := wc.Recv() if err != nil { select { case w.errc <- err: case <-w.donec: } return } select { case w.respc <- resp: case <-w.donec: return } } } // serveSubstream forwards watch responses from run() to the subscriber func (w *watchGRPCStream) serveSubstream(ws *watcherStream, resumec chan struct{}) { if ws.closing { panic("created substream goroutine but substream is closing") } // nextRev is the minimum expected next revision nextRev := ws.initReq.rev resuming := false defer func() { if !resuming { ws.closing = true } close(ws.donec) if !resuming { w.closingc <- ws } w.wg.Done() }() emptyWr := &WatchResponse{} for { curWr := emptyWr outc := ws.outc if len(ws.buf) > 0 { curWr = ws.buf[0] } else { outc = nil } select { case outc <- *curWr: if ws.buf[0].Err() != nil { return } ws.buf[0] = nil ws.buf = ws.buf[1:] case wr, ok := <-ws.recvc: if !ok { // shutdown from closeSubstream return } if wr.Created { if ws.initReq.retc != nil { ws.initReq.retc <- ws.outc // to prevent next write from taking the slot in buffered channel // and posting duplicate create events ws.initReq.retc = nil // send first creation event only if requested if ws.initReq.createdNotify { ws.outc <- *wr } // once the watch channel is returned, a current revision // watch must resume at the store revision. This is necessary // for the following case to work as expected: // wch := m1.Watch("a") // m2.Put("a", "b") // <-wch // If the revision is only bound on the first observed event, // if wch is disconnected before the Put is issued, then reconnects // after it is committed, it'll miss the Put. if ws.initReq.rev == 0 { nextRev = wr.Header.Revision } } } else { // current progress of watch; <= store revision nextRev = wr.Header.Revision + 1 } if len(wr.Events) > 0 { nextRev = wr.Events[len(wr.Events)-1].Kv.ModRevision + 1 } ws.initReq.rev = nextRev // created event is already sent above, // watcher should not post duplicate events if wr.Created { continue } // TODO pause channel if buffer gets too large ws.buf = append(ws.buf, wr) case <-w.ctx.Done(): return case <-ws.initReq.ctx.Done(): return case <-resumec: resuming = true return } } // lazily send cancel message if events on missing id } func (w *watchGRPCStream) newWatchClient() (pb.Watch_WatchClient, error) { // mark all substreams as resuming close(w.resumec) w.resumec = make(chan struct{}) w.joinSubstreams() for _, ws := range w.substreams { ws.id = InvalidWatchID w.resuming = append(w.resuming, ws) } // strip out nils, if any var resuming []*watcherStream for _, ws := range w.resuming { if ws != nil { resuming = append(resuming, ws) } } w.resuming = resuming w.substreams = make(map[int64]*watcherStream) // connect to grpc stream while accepting watcher cancellation stopc := make(chan struct{}) donec := w.waitCancelSubstreams(stopc) wc, err := w.openWatchClient() close(stopc) <-donec // serve all non-closing streams, even if there's a client error // so that the teardown path can shutdown the streams as expected. for _, ws := range w.resuming { if ws.closing { continue } ws.donec = make(chan struct{}) w.wg.Add(1) go w.serveSubstream(ws, w.resumec) } if err != nil { return nil, v3rpc.Error(err) } // receive data from new grpc stream go w.serveWatchClient(wc) return wc, nil } func (w *watchGRPCStream) waitCancelSubstreams(stopc <-chan struct{}) <-chan struct{} { var wg sync.WaitGroup wg.Add(len(w.resuming)) donec := make(chan struct{}) for i := range w.resuming { go func(ws *watcherStream) { defer wg.Done() if ws.closing { if ws.initReq.ctx.Err() != nil && ws.outc != nil { close(ws.outc) ws.outc = nil } return } select { case <-ws.initReq.ctx.Done(): // closed ws will be removed from resuming ws.closing = true close(ws.outc) ws.outc = nil w.wg.Add(1) go func() { defer w.wg.Done() w.closingc <- ws }() case <-stopc: } }(w.resuming[i]) } go func() { defer close(donec) wg.Wait() }() return donec } // joinSubstreams waits for all substream goroutines to complete. func (w *watchGRPCStream) joinSubstreams() { for _, ws := range w.substreams { <-ws.donec } for _, ws := range w.resuming { if ws != nil { <-ws.donec } } } var maxBackoff = 100 * time.Millisecond func (w *watchGRPCStream) backoffIfUnavailable(backoff time.Duration, err error) time.Duration { if isUnavailableErr(w.ctx, err) { // retry, but backoff if backoff < maxBackoff { // 25% backoff factor backoff = backoff + backoff/4 if backoff > maxBackoff { backoff = maxBackoff } } time.Sleep(backoff) } return backoff } // openWatchClient retries opening a watch client until success or halt. // manually retry in case "ws==nil && err==nil" // TODO: remove FailFast=false func (w *watchGRPCStream) openWatchClient() (ws pb.Watch_WatchClient, err error) { backoff := time.Millisecond for { select { case <-w.ctx.Done(): if err == nil { return nil, w.ctx.Err() } return nil, err default: } if ws, err = w.remote.Watch(w.ctx, w.callOpts...); ws != nil && err == nil { break } if isHaltErr(w.ctx, err) { return nil, v3rpc.Error(err) } backoff = w.backoffIfUnavailable(backoff, err) } return ws, nil } // toPB converts an internal watch request structure to its protobuf WatchRequest structure. func (wr *watchRequest) toPB() *pb.WatchRequest { req := &pb.WatchCreateRequest{ StartRevision: wr.rev, Key: []byte(wr.key), RangeEnd: []byte(wr.end), ProgressNotify: wr.progressNotify, Filters: wr.filters, PrevKv: wr.prevKV, Fragment: wr.fragment, } cr := &pb.WatchRequest_CreateRequest{CreateRequest: req} return &pb.WatchRequest{RequestUnion: cr} } // toPB converts an internal progress request structure to its protobuf WatchRequest structure. func (pr *progressRequest) toPB() *pb.WatchRequest { req := &pb.WatchProgressRequest{} cr := &pb.WatchRequest_ProgressRequest{ProgressRequest: req} return &pb.WatchRequest{RequestUnion: cr} } func streamKeyFromCtx(ctx context.Context) string { if md, ok := metadata.FromOutgoingContext(ctx); ok { return fmt.Sprintf("%+v", map[string][]string(md)) } return "" } ================================================ FILE: client/v3/watch_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3 import ( "context" "testing" "google.golang.org/grpc/metadata" "go.etcd.io/etcd/api/v3/mvccpb" ) func TestEvent(t *testing.T) { tests := []struct { ev *Event isCreate bool isModify bool }{{ ev: &Event{ Type: EventTypePut, Kv: &mvccpb.KeyValue{ CreateRevision: 3, ModRevision: 3, }, }, isCreate: true, }, { ev: &Event{ Type: EventTypePut, Kv: &mvccpb.KeyValue{ CreateRevision: 3, ModRevision: 4, }, }, isModify: true, }} for i, tt := range tests { if tt.isCreate && !tt.ev.IsCreate() { t.Errorf("#%d: event should be Create event", i) } if tt.isModify && !tt.ev.IsModify() { t.Errorf("#%d: event should be Modify event", i) } } } // TestStreamKeyFromCtx tests the streamKeyFromCtx function to ensure it correctly // formats metadata as a map[string][]string when extracting metadata from the context. // // The fmt package in Go guarantees that maps are printed in a consistent order, // sorted by the keys. This test verifies that the streamKeyFromCtx function // produces the expected formatted string representation of metadata maps when called with // various context scenarios. func TestStreamKeyFromCtx(t *testing.T) { tests := []struct { name string ctx context.Context expected string }{ { name: "multiple keys", ctx: metadata.NewOutgoingContext(t.Context(), metadata.MD{ "key1": []string{"value1"}, "key2": []string{"value2a", "value2b"}, }), expected: "map[key1:[value1] key2:[value2a value2b]]", }, { name: "no keys", ctx: metadata.NewOutgoingContext(t.Context(), metadata.MD{}), expected: "map[]", }, { name: "only one key", ctx: metadata.NewOutgoingContext(t.Context(), metadata.MD{ "key1": []string{"value1", "value1a"}, }), expected: "map[key1:[value1 value1a]]", }, { name: "no metadata", ctx: t.Context(), expected: "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { actual := streamKeyFromCtx(tt.ctx) if actual != tt.expected { t.Errorf("streamKeyFromCtx() = %v, expected %v", actual, tt.expected) } }) } } ================================================ FILE: client/v3/yaml/config.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package yaml handles yaml-formatted clientv3 configuration data. package yaml import ( "crypto/tls" "crypto/x509" "os" "sigs.k8s.io/yaml" "go.etcd.io/etcd/client/pkg/v3/tlsutil" clientv3 "go.etcd.io/etcd/client/v3" ) type yamlConfig struct { clientv3.Config InsecureTransport bool `json:"insecure-transport"` InsecureSkipTLSVerify bool `json:"insecure-skip-tls-verify"` Certfile string `json:"cert-file"` Keyfile string `json:"key-file"` TrustedCAfile string `json:"trusted-ca-file"` // CAfile is being deprecated. Use 'TrustedCAfile' instead. // TODO: deprecate this in v4 CAfile string `json:"ca-file"` } // NewConfig creates a new clientv3.Config from a yaml file. func NewConfig(fpath string) (*clientv3.Config, error) { b, err := os.ReadFile(fpath) if err != nil { return nil, err } yc := &yamlConfig{} err = yaml.Unmarshal(b, yc) if err != nil { return nil, err } if yc.InsecureTransport { return &yc.Config, nil } var ( cert *tls.Certificate cp *x509.CertPool ) if yc.Certfile != "" && yc.Keyfile != "" { cert, err = tlsutil.NewCert(yc.Certfile, yc.Keyfile, nil) if err != nil { return nil, err } } if yc.TrustedCAfile != "" { cp, err = tlsutil.NewCertPool([]string{yc.TrustedCAfile}) if err != nil { return nil, err } } tlscfg := &tls.Config{ MinVersion: tls.VersionTLS12, InsecureSkipVerify: yc.InsecureSkipTLSVerify, RootCAs: cp, } if cert != nil { tlscfg.Certificates = []tls.Certificate{*cert} } yc.Config.TLS = tlscfg return &yc.Config, nil } ================================================ FILE: client/v3/yaml/config_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package yaml import ( "log" "os" "reflect" "testing" "github.com/stretchr/testify/require" "sigs.k8s.io/yaml" ) var ( certPath = "../../../tests/fixtures/server.crt" privateKeyPath = "../../../tests/fixtures/server.key.insecure" caPath = "../../../tests/fixtures/ca.crt" ) func TestConfigFromFile(t *testing.T) { tests := []struct { ym *yamlConfig werr bool }{ { &yamlConfig{}, false, }, { &yamlConfig{ InsecureTransport: true, }, false, }, { &yamlConfig{ Keyfile: privateKeyPath, Certfile: certPath, TrustedCAfile: caPath, InsecureSkipTLSVerify: true, }, false, }, { &yamlConfig{ Keyfile: "bad", Certfile: "bad", }, true, }, { &yamlConfig{ Keyfile: privateKeyPath, Certfile: certPath, TrustedCAfile: "bad", }, true, }, } for i, tt := range tests { tmpfile, err := os.CreateTemp(t.TempDir(), "clientcfg") if err != nil { log.Fatal(err) } b, err := yaml.Marshal(tt.ym) require.NoError(t, err) _, err = tmpfile.Write(b) require.NoError(t, err) require.NoError(t, tmpfile.Close()) cfg, cerr := NewConfig(tmpfile.Name()) if cerr != nil && !tt.werr { t.Errorf("#%d: err = %v, want %v", i, cerr, tt.werr) continue } if cerr != nil { os.Remove(tmpfile.Name()) continue } if !reflect.DeepEqual(cfg.Endpoints, tt.ym.Endpoints) { t.Errorf("#%d: endpoint = %v, want %v", i, cfg.Endpoints, tt.ym.Endpoints) } if tt.ym.InsecureTransport != (cfg.TLS == nil) { t.Errorf("#%d: insecureTransport = %v, want %v", i, cfg.TLS == nil, tt.ym.InsecureTransport) } if !tt.ym.InsecureTransport { if tt.ym.Certfile != "" && len(cfg.TLS.Certificates) == 0 { t.Errorf("#%d: failed to load in cert", i) } if tt.ym.TrustedCAfile != "" && cfg.TLS.RootCAs == nil { t.Errorf("#%d: failed to load in ca cert", i) } if cfg.TLS.InsecureSkipVerify != tt.ym.InsecureSkipTLSVerify { t.Errorf("#%d: skipTLSVeify = %v, want %v", i, cfg.TLS.InsecureSkipVerify, tt.ym.InsecureSkipTLSVerify) } } os.Remove(tmpfile.Name()) } } ================================================ FILE: code-of-conduct.md ================================================ ## etcd Community Code of Conduct etcd follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). ================================================ FILE: codecov.yml ================================================ --- # https://docs.codecov.com/docs/codecovyml-reference codecov: token: 6040de41-c073-4d6f-bbf8-d89256ef31e1 disable_default_path_fixes: true require_ci_to_pass: false notify: wait_for_ci: false fixes: - go.etcd.io/etcd/api/v3/::api/ - go.etcd.io/etcd/client/v3/::client/v3/ - go.etcd.io/etcd/etcdctl/v3/::etcdctl/ - go.etcd.io/etcd/etcdutl/v3/::etcdutl/ - go.etcd.io/etcd/pkg/v3/::pkg/ - go.etcd.io/etcd/server/v3/::server/ ignore: - '**/*.pb.go' - '**/*.pb.gw.go' - tests/**/* - go.etcd.io/etcd/tests/**/* coverage: range: 60..80 round: down precision: 2 status: project: default: target: auto # allow some coverage reductions within a threshold # this allows a 1% drop from the previous base commit coverage threshold: 1% patch: default: target: auto threshold: 80% comment: layout: "header, files, diff, footer" behavior: default # default: update, if exists. Otherwise post new; new: delete old and post new require_changes: false # if true: only post the comment if coverage changes require_base: false # [true :: must have a base report to post] require_head: true # [true :: must have a head report to post] hide_project_coverage: false # [true :: only show coverage on the git diff] ================================================ FILE: contrib/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/contrib ================================================ FILE: contrib/README.md ================================================ ## Contrib Scripts and files which may be useful but aren't part of the core etcd project. * [lock](lock) - example addressing the expired lease problem of distributed locking with etcd * [mixin](mixin) - customisable set of Grafana dashboard and Prometheus alerts for etcd * [raftexample](raftexample) - an example distributed key-value store using raft * [systemd](systemd) - an example unit file for deploying etcd on systemd-based distributions * [systemd/etcd3-multinode](systemd/etcd3-multinode) - multi-node cluster setup with systemd ================================================ FILE: contrib/lock/README.md ================================================ # What is this? This directory provides an executable example of the scenarios described in [the article by Martin Kleppmann][fencing]. Generally speaking, a lease-based lock service cannot provide mutual exclusion to processes. This is because such a lease mechanism depends on the physical clock of both the lock service and client processes. Many factors (e.g. stop-the-world GC pause of a language runtime) can cause false expiration of a granted lease as depicted in the below figure: ![unsafe lock][unsafe-lock] As discussed in [notes on the usage of lock and lease][why], such a problem can be solved with a technique called version number validation or fencing tokens. With this technique a shared resource (storage in the figures) needs to validate requests from clients based on their tokens like this: ![fencing tokens][fencing-tokens] This directory contains two programs: `client` and `storage`. With `etcd`, you can reproduce the expired lease problem of distributed locking and a simple example solution of the validation technique which can avoid incorrect access from a client with an expired lease. `storage` works as a very simple key value in-memory store which is accessible through HTTP and a custom JSON protocol. `client` works as client processes which tries to write a key/value to `storage` with coordination of etcd locking. ## How to build For building `client` and `storage`, just execute `go build` in each directory. ## How to try At first, you need to start an etcd cluster, which works as lock service in the figures. On top of the etcd source directory, execute commands like below: ``` $ make # build etcd $ bin/etcd # start etcd ``` Then run `storage` command in `storage` directory: ``` $ ./storage ``` Now client processes ("Client 1" and "Client 2" in the figures) can be started. At first, execute below command for starting a client process which corresponds to "Client 1": ``` $ ./client 1 ``` It will show an output like this: ``` client 1 starts created etcd client and session acquired lock, version: 694d82254d5fa305 please manually revoke the lease using 'etcdctl lease revoke 694d82254d5fa305' or wait for it to expire, then start executing client 2 and hit any key... ``` Verify the lease was created using: ``` $ bin/etcdctl lease list found 1 leases 694d82254d5fa305 ``` Then proceed to manually revoke the lease using: ``` $ bin/etcdctl lease revoke 694d82254d5fa305 lease 694d82254d5fa305 revoked ``` Now another client process can be started like this: ``` $ ./client 2 client 2 starts created etcd client and session acquired lock, version: 694d82254e18770a this is client 2, continuing ``` If things go well the second client process invoked as `./client 2` finishes soon. It successfully writes a key to `storage` process. After checking this, please hit any key for `./client 1` and resume the process. It will show an output like below: ``` resuming client 1 expected fail to write to storage with old lease version: error: given version (694d82254d5fa305) is different from the existing version (694d82254e18770a) ``` [fencing]: https://martin.kleppmann.com/2016/02/08/how-to-do-distributed-locking.html [fencing-tokens]: https://martin.kleppmann.com/2016/02/fencing-tokens.png [unsafe-lock]: https://martin.kleppmann.com/2016/02/unsafe-lock.png [why]: https://etcd.io/docs/next/learning/why/#notes-on-the-usage-of-lock-and-lease ================================================ FILE: contrib/mixin/.gitignore ================================================ vendor ================================================ FILE: contrib/mixin/.lint ================================================ --- exclusions: template-instance-rule: reason: The mixin only uses `instance` for alerts, and `cluster` for dashboard queries template-job-rule: reason: The dashboards use 'cluster' label as selector, rather than 'job' target-job-rule: reason: The mixin uses 'cluster' instead of 'job' target-instance-rule: reason: The mixin only uses `instance` for alerts, and `cluster` for dashboard queries alert-name-camelcase: reason: etcd is spelled all lowercase, meaning all alert name start with a lowercase alert-summary-style: reason: etcd is spelled all lowercase, meaning summaries starting with 'etcd' are still valid panel-units-rule: reason: Stat panels have no unit, and some panels use custom unit or text panel-title-description-rule: reason: Suppress noisy linting rule until we can address minor tech debt like this ================================================ FILE: contrib/mixin/Makefile ================================================ .PHONY: tools manifests test clean jb_install OS := linux ARCH ?= amd64 PROMETHEUS_VERSION := 2.33.1 tools: go install github.com/google/go-jsonnet/cmd/jsonnet@latest go install github.com/brancz/gojsontoyaml@latest go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest wget -qO- "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.${OS}-${ARCH}.tar.gz" |\ tar xvz --strip-components=1 -C "$$(go env GOPATH)/bin" prometheus-${PROMETHEUS_VERSION}.${OS}-${ARCH}/promtool manifests: manifests/etcd-prometheusRules.yaml manifests/etcd-prometheusRules.yaml: mkdir -p manifests jsonnet -e '(import "mixin.libsonnet").prometheusAlerts' | gojsontoyaml > manifests/etcd-prometheusRules.yaml test: manifests/etcd-prometheusRules.yaml promtool test rules test.yaml jb_install: jb install clean: rm -rf manifests/*.yaml ================================================ FILE: contrib/mixin/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/observability ================================================ FILE: contrib/mixin/README.md ================================================ # Prometheus Monitoring Mixin for etcd > NOTE: This project is *alpha* stage. Flags, configuration, behaviour and design may change significantly in following releases. A customisable set of Grafana dashboard and Prometheus alerts for etcd. Instructions for use are the same as the [kubernetes-mixin](https://github.com/kubernetes-monitoring/kubernetes-mixin). ## Grafana 7.x support By default, this mixin generates the dashboard compatible with Grafana 8.x or newer. To generate dashboard for Grafana 7.x, set in the config.libsonnet: ``` // set to true if dashboards should be compatible with Grafana 7x or earlier grafana7x: true, ``` ## Background * For more information about monitoring mixins, see this [design doc](https://docs.google.com/document/d/1A9xvzwqnFVSOZ5fD3blKODXfsat5fg6ZhnKu9LK3lB4/edit#). ## Testing alerts Make sure to have [jsonnet](https://jsonnet.org/) and [gojsontoyaml](https://github.com/brancz/gojsontoyaml) installed. You can fetch it via ``` make tools ``` First compile the mixin to a YAML file, which the promtool will read: ``` make manifests ``` Then run the unit test: ``` promtool test rules test.yaml ``` ================================================ FILE: contrib/mixin/alerts/alerts.libsonnet ================================================ { prometheusAlerts+:: { groups+: [ { name: 'etcd', rules: [ { alert: 'etcdMembersDown', expr: ||| max without (endpoint) ( sum without (%(etcd_instance_labels)s) (up{%(etcd_selector)s} == bool 0) or count without (To) ( sum without (%(etcd_instance_labels)s) (rate(etcd_network_peer_sent_failures_total{%(etcd_selector)s}[%(network_failure_range)ss])) > 0.01 ) ) > 0 ||| % { etcd_instance_labels: $._config.etcd_instance_labels, etcd_selector: $._config.etcd_selector, network_failure_range: $._config.scrape_interval_seconds * 4 }, 'for': '20m', labels: { severity: 'warning', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": members are down ({{ $value }}).' % $._config.clusterLabel, summary: 'etcd cluster members are down.', }, }, { alert: 'etcdInsufficientMembers', expr: ||| sum(up{%(etcd_selector)s} == bool 1) without (%(etcd_instance_labels)s) < ((count(up{%(etcd_selector)s}) without (%(etcd_instance_labels)s) + 1) / 2) ||| % $._config, 'for': '3m', labels: { severity: 'critical', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": insufficient members ({{ $value }}).' % $._config.clusterLabel, summary: 'etcd cluster has insufficient number of members.', }, }, { alert: 'etcdNoLeader', expr: ||| etcd_server_has_leader{%(etcd_selector)s} == 0 ||| % $._config, 'for': '1m', labels: { severity: 'critical', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": member {{ $labels.instance }} has no leader.' % $._config.clusterLabel, summary: 'etcd cluster has no leader.', }, }, { alert: 'etcdHighNumberOfLeaderChanges', expr: ||| increase((max without (%(etcd_instance_labels)s) (etcd_server_leader_changes_seen_total{%(etcd_selector)s}) or 0*absent(etcd_server_leader_changes_seen_total{%(etcd_selector)s}))[15m:1m]) >= 4 ||| % $._config, 'for': '5m', labels: { severity: 'warning', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": {{ $value }} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' % $._config.clusterLabel, summary: 'etcd cluster has high number of leader changes.', }, }, { alert: 'etcdHighNumberOfFailedGRPCRequests', expr: ||| 100 * sum(rate(grpc_server_handled_total{%(etcd_selector)s, grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) / sum(rate(grpc_server_handled_total{%(etcd_selector)s}[5m])) without (grpc_type, grpc_code) > 1 ||| % $._config, 'for': '10m', labels: { severity: 'warning', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": {{ $value }}%% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.' % $._config.clusterLabel, summary: 'etcd cluster has high number of failed grpc requests.', }, }, { alert: 'etcdHighNumberOfFailedGRPCRequests', expr: ||| 100 * sum(rate(grpc_server_handled_total{%(etcd_selector)s, grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code) / sum(rate(grpc_server_handled_total{%(etcd_selector)s}[5m])) without (grpc_type, grpc_code) > 5 ||| % $._config, 'for': '5m', labels: { severity: 'critical', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": {{ $value }}%% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.' % $._config.clusterLabel, summary: 'etcd cluster has high number of failed grpc requests.', }, }, { alert: 'etcdGRPCRequestsSlow', expr: ||| histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{%(etcd_selector)s, grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) > 0.15 ||| % $._config, 'for': '10m', labels: { severity: 'critical', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.' % $._config.clusterLabel, summary: 'etcd grpc requests are slow', }, }, { alert: 'etcdMemberCommunicationSlow', expr: ||| histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{%(etcd_selector)s}[5m])) > 0.15 ||| % $._config, 'for': '10m', labels: { severity: 'warning', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.' % $._config.clusterLabel, summary: 'etcd cluster member communication is slow.', }, }, { alert: 'etcdHighNumberOfFailedProposals', expr: ||| rate(etcd_server_proposals_failed_total{%(etcd_selector)s}[15m]) > 5 ||| % $._config, 'for': '15m', labels: { severity: 'warning', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": {{ $value }} proposal failures within the last 30 minutes on etcd instance {{ $labels.instance }}.' % $._config.clusterLabel, summary: 'etcd cluster has high number of proposal failures.', }, }, { alert: 'etcdHighFsyncDurations', expr: ||| histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{%(etcd_selector)s}[5m])) > 0.5 ||| % $._config, 'for': '10m', labels: { severity: 'warning', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.' % $._config.clusterLabel, summary: 'etcd cluster 99th percentile fsync durations are too high.', }, }, { alert: 'etcdHighFsyncDurations', expr: ||| histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{%(etcd_selector)s}[5m])) > 1 ||| % $._config, 'for': '10m', labels: { severity: 'critical', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.' % $._config.clusterLabel, summary: 'etcd cluster 99th percentile fsync durations are too high.', }, }, { alert: 'etcdHighCommitDurations', expr: ||| histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{%(etcd_selector)s}[5m])) > 0.25 ||| % $._config, 'for': '10m', labels: { severity: 'warning', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.' % $._config.clusterLabel, summary: 'etcd cluster 99th percentile commit durations are too high.', }, }, { alert: 'etcdDatabaseQuotaLowSpace', expr: ||| (last_over_time(etcd_mvcc_db_total_size_in_bytes{%(etcd_selector)s}[5m]) / last_over_time(etcd_server_quota_backend_bytes{%(etcd_selector)s}[5m]))*100 > 95 ||| % $._config, 'for': '10m', labels: { severity: 'critical', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' % $._config.clusterLabel, summary: 'etcd cluster database is running full.', }, }, { alert: 'etcdExcessiveDatabaseGrowth', expr: ||| predict_linear(etcd_mvcc_db_total_size_in_bytes{%(etcd_selector)s}[4h], 4*60*60) > etcd_server_quota_backend_bytes{%(etcd_selector)s} ||| % $._config, 'for': '10m', labels: { severity: 'warning', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive.' % $._config.clusterLabel, summary: 'etcd cluster database growing very fast.', }, }, { alert: 'etcdDatabaseHighFragmentationRatio', expr: ||| (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{%(etcd_selector)s}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{%(etcd_selector)s}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{%(etcd_selector)s} > 104857600 ||| % $._config, 'for': '10m', labels: { severity: 'warning', }, annotations: { description: 'etcd cluster "{{ $labels.%s }}": database size in use on instance {{ $labels.instance }} is {{ $value | humanizePercentage }} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.' % $._config.clusterLabel, summary: 'etcd database size in use is less than 50% of the actual allocated storage.', runbook_url: 'https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation', }, }, ], }, ], }, } ================================================ FILE: contrib/mixin/config.libsonnet ================================================ { _config+:: { // set to true if dashboards should be compatible with Grafana 7x or earlier grafana7x: false, etcd_selector: 'job=~".*etcd.*"', // etcd_instance_labels are the label names that are uniquely // identifying an instance and need to be aggreated away for alerts // that are about an etcd cluster as a whole. For example, if etcd // instances are deployed on K8s, you will likely want to change // this to 'instance, pod'. etcd_instance_labels: 'instance', // scrape_interval_seconds is the global scrape interval which can be // used to dynamically adjust rate windows as a function of the interval. scrape_interval_seconds: 30, // Dashboard variable refresh option on Grafana (https://grafana.com/docs/grafana/latest/datasources/prometheus/). // 0 : Never (Will never refresh the Dashboard variables values) // 1 : On Dashboard Load (Will refresh Dashboards variables when dashboard are loaded) // 2 : On Time Range Change (Will refresh Dashboards variables when time range will be changed) dashboard_var_refresh: 2, // clusterLabel is used to identify a cluster. clusterLabel: 'job', }, } ================================================ FILE: contrib/mixin/dashboards/dashboards.libsonnet ================================================ (import "etcd.libsonnet") + (import "etcd-grafana7x.libsonnet") ================================================ FILE: contrib/mixin/dashboards/etcd-grafana7x.libsonnet ================================================ { grafanaDashboards+:: if $._config.grafana7x then { 'etcd.json': { uid: std.md5('etcd.json'), title: 'etcd', description: 'etcd sample Grafana dashboard with Prometheus', tags: ['etcd-mixin'], style: 'dark', timezone: 'browser', editable: true, hideControls: false, sharedCrosshair: false, rows: [ { collapse: false, editable: true, height: '250px', panels: [ { cacheTimeout: null, colorBackground: false, colorValue: false, colors: [ 'rgba(245, 54, 54, 0.9)', 'rgba(237, 129, 40, 0.89)', 'rgba(50, 172, 45, 0.97)', ], datasource: '$datasource', editable: true, 'error': false, format: 'none', gauge: { maxValue: 100, minValue: 0, show: false, thresholdLabels: false, thresholdMarkers: true, }, id: 28, interval: null, isNew: true, links: [], mappingType: 1, mappingTypes: [ { name: 'value to text', value: 1, }, { name: 'range to text', value: 2, }, ], maxDataPoints: 100, nullPointMode: 'connected', nullText: null, postfix: '', postfixFontSize: '50%', prefix: '', prefixFontSize: '50%', rangeMaps: [{ from: 'null', text: 'N/A', to: 'null', }], span: 3, sparkline: { fillColor: 'rgba(31, 118, 189, 0.18)', full: false, lineColor: 'rgb(31, 120, 193)', show: false, }, targets: [{ expr: 'sum(etcd_server_has_leader{%s, %s="$cluster"})' % [$._config.etcd_selector, $._config.clusterLabel], intervalFactor: 2, legendFormat: '', metric: 'etcd_server_has_leader', refId: 'A', step: 20, }], thresholds: '', title: 'Up', type: 'singlestat', valueFontSize: '200%', valueMaps: [{ op: '=', text: 'N/A', value: 'null', }], valueName: 'avg', }, { aliasColors: {}, bars: false, datasource: '$datasource', editable: true, 'error': false, fill: 0, id: 23, isNew: true, legend: { avg: false, current: false, max: false, min: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', percentage: false, pointradius: 5, points: false, renderer: 'flot', seriesOverrides: [], span: 5, stack: false, steppedLine: false, targets: [ { expr: 'sum(rate(grpc_server_started_total{%s, %s="$cluster",grpc_type="unary"}[$__rate_interval]))' % [$._config.etcd_selector, $._config.clusterLabel], format: 'time_series', intervalFactor: 2, legendFormat: 'RPC Rate', metric: 'grpc_server_started_total', refId: 'A', step: 2, }, { expr: 'sum(rate(grpc_server_handled_total{%s, %s="$cluster",grpc_type="unary",grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[$__rate_interval]))' % [$._config.etcd_selector, $._config.clusterLabel], format: 'time_series', intervalFactor: 2, legendFormat: 'RPC Failed Rate', metric: 'grpc_server_handled_total', refId: 'B', step: 2, }, ], thresholds: [], timeFrom: null, timeShift: null, title: 'RPC Rate', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'individual', }, type: 'graph', xaxis: { mode: 'time', name: null, show: true, values: [], }, yaxes: [ { format: 'ops', label: null, logBase: 1, max: null, min: null, show: true, }, { format: 'short', label: null, logBase: 1, max: null, min: null, show: true, }, ], }, { aliasColors: {}, bars: false, datasource: '$datasource', editable: true, 'error': false, fill: 0, id: 41, isNew: true, legend: { avg: false, current: false, max: false, min: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', percentage: false, pointradius: 5, points: false, renderer: 'flot', seriesOverrides: [], span: 4, stack: true, steppedLine: false, targets: [ { expr: 'sum(grpc_server_started_total{%(etcd_selector)s,%(clusterLabel)s="$cluster",grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{%(clusterLabel)s="$cluster",grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"})' % $._config, intervalFactor: 2, legendFormat: 'Watch Streams', metric: 'grpc_server_handled_total', refId: 'A', step: 4, }, { expr: 'sum(grpc_server_started_total{%(etcd_selector)s,%(clusterLabel)s="$cluster",grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{%(clusterLabel)s="$cluster",grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"})' % $._config, intervalFactor: 2, legendFormat: 'Lease Streams', metric: 'grpc_server_handled_total', refId: 'B', step: 4, }, ], thresholds: [], timeFrom: null, timeShift: null, title: 'Active Streams', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'individual', }, type: 'graph', xaxis: { mode: 'time', name: null, show: true, values: [], }, yaxes: [ { format: 'short', label: '', logBase: 1, max: null, min: null, show: true, }, { format: 'short', label: null, logBase: 1, max: null, min: null, show: true, }, ], }, ], showTitle: false, title: 'Row', }, { collapse: false, editable: true, height: '250px', panels: [ { aliasColors: {}, bars: false, datasource: '$datasource', decimals: null, editable: true, 'error': false, fill: 0, grid: {}, id: 1, legend: { avg: false, current: false, max: false, min: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', percentage: false, pointradius: 5, points: false, renderer: 'flot', seriesOverrides: [], span: 4, stack: false, steppedLine: false, targets: [{ expr: 'etcd_mvcc_db_total_size_in_bytes{%s, %s="$cluster"}' % [$._config.etcd_selector, $._config.clusterLabel], hide: false, interval: '', intervalFactor: 2, legendFormat: '{{instance}} DB Size', metric: '', refId: 'A', step: 4, }], thresholds: [], timeFrom: null, timeShift: null, title: 'DB Size', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'cumulative', }, type: 'graph', xaxis: { mode: 'time', name: null, show: true, values: [], }, yaxes: [ { format: 'bytes', logBase: 1, max: null, min: null, show: true, }, { format: 'short', logBase: 1, max: null, min: null, show: false, }, ], }, { aliasColors: {}, bars: false, datasource: '$datasource', editable: true, 'error': false, fill: 0, grid: {}, id: 3, legend: { avg: false, current: false, max: false, min: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', percentage: false, pointradius: 1, points: false, renderer: 'flot', seriesOverrides: [], span: 4, stack: false, steppedLine: true, targets: [ { expr: 'histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{%s, %s="$cluster"}[$__rate_interval])) by (instance, le))' % [$._config.etcd_selector, $._config.clusterLabel], hide: false, intervalFactor: 2, legendFormat: '{{instance}} WAL fsync', metric: 'etcd_disk_wal_fsync_duration_seconds_bucket', refId: 'A', step: 4, }, { expr: 'histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{%s, %s="$cluster"}[$__rate_interval])) by (instance, le))' % [$._config.etcd_selector, $._config.clusterLabel], intervalFactor: 2, legendFormat: '{{instance}} DB fsync', metric: 'etcd_disk_backend_commit_duration_seconds_bucket', refId: 'B', step: 4, }, ], thresholds: [], timeFrom: null, timeShift: null, title: 'Disk Sync Duration', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'cumulative', }, type: 'graph', xaxis: { mode: 'time', name: null, show: true, values: [], }, yaxes: [ { format: 's', logBase: 1, max: null, min: null, show: true, }, { format: 'short', logBase: 1, max: null, min: null, show: false, }, ], }, { aliasColors: {}, bars: false, datasource: '$datasource', editable: true, 'error': false, fill: 0, id: 29, isNew: true, legend: { avg: false, current: false, max: false, min: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', percentage: false, pointradius: 5, points: false, renderer: 'flot', seriesOverrides: [], span: 4, stack: false, steppedLine: false, targets: [{ expr: 'process_resident_memory_bytes{%s, %s="$cluster"}' % [$._config.etcd_selector, $._config.clusterLabel], intervalFactor: 2, legendFormat: '{{instance}} Resident Memory', metric: 'process_resident_memory_bytes', refId: 'A', step: 4, }], thresholds: [], timeFrom: null, timeShift: null, title: 'Memory', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'individual', }, type: 'graph', xaxis: { mode: 'time', name: null, show: true, values: [], }, yaxes: [ { format: 'bytes', label: null, logBase: 1, max: null, min: null, show: true, }, { format: 'short', label: null, logBase: 1, max: null, min: null, show: true, }, ], }, ], title: 'New row', }, { collapse: false, editable: true, height: '250px', panels: [ { aliasColors: {}, bars: false, datasource: '$datasource', editable: true, 'error': false, fill: 5, id: 22, isNew: true, legend: { avg: false, current: false, max: false, min: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', percentage: false, pointradius: 5, points: false, renderer: 'flot', seriesOverrides: [], span: 3, stack: true, steppedLine: false, targets: [{ expr: 'rate(etcd_network_client_grpc_received_bytes_total{%s, %s="$cluster"}[$__rate_interval])' % [$._config.etcd_selector, $._config.clusterLabel], intervalFactor: 2, legendFormat: '{{instance}} Client Traffic In', metric: 'etcd_network_client_grpc_received_bytes_total', refId: 'A', step: 4, }], thresholds: [], timeFrom: null, timeShift: null, title: 'Client Traffic In', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'individual', }, type: 'graph', xaxis: { mode: 'time', name: null, show: true, values: [], }, yaxes: [ { format: 'Bps', label: null, logBase: 1, max: null, min: null, show: true, }, { format: 'short', label: null, logBase: 1, max: null, min: null, show: true, }, ], }, { aliasColors: {}, bars: false, datasource: '$datasource', editable: true, 'error': false, fill: 5, id: 21, isNew: true, legend: { avg: false, current: false, max: false, min: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', percentage: false, pointradius: 5, points: false, renderer: 'flot', seriesOverrides: [], span: 3, stack: true, steppedLine: false, targets: [{ expr: 'rate(etcd_network_client_grpc_sent_bytes_total{%s, %s="$cluster"}[$__rate_interval])' % [$._config.etcd_selector, $._config.clusterLabel], intervalFactor: 2, legendFormat: '{{instance}} Client Traffic Out', metric: 'etcd_network_client_grpc_sent_bytes_total', refId: 'A', step: 4, }], thresholds: [], timeFrom: null, timeShift: null, title: 'Client Traffic Out', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'individual', }, type: 'graph', xaxis: { mode: 'time', name: null, show: true, values: [], }, yaxes: [ { format: 'Bps', label: null, logBase: 1, max: null, min: null, show: true, }, { format: 'short', label: null, logBase: 1, max: null, min: null, show: true, }, ], }, { aliasColors: {}, bars: false, datasource: '$datasource', editable: true, 'error': false, fill: 0, id: 20, isNew: true, legend: { avg: false, current: false, max: false, min: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', percentage: false, pointradius: 5, points: false, renderer: 'flot', seriesOverrides: [], span: 3, stack: false, steppedLine: false, targets: [{ expr: 'sum(rate(etcd_network_peer_received_bytes_total{%s, %s="$cluster"}[$__rate_interval])) by (instance)' % [$._config.etcd_selector, $._config.clusterLabel], intervalFactor: 2, legendFormat: '{{instance}} Peer Traffic In', metric: 'etcd_network_peer_received_bytes_total', refId: 'A', step: 4, }], thresholds: [], timeFrom: null, timeShift: null, title: 'Peer Traffic In', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'individual', }, type: 'graph', xaxis: { mode: 'time', name: null, show: true, values: [], }, yaxes: [ { format: 'Bps', label: null, logBase: 1, max: null, min: null, show: true, }, { format: 'short', label: null, logBase: 1, max: null, min: null, show: true, }, ], }, { aliasColors: {}, bars: false, datasource: '$datasource', decimals: null, editable: true, 'error': false, fill: 0, grid: {}, id: 16, legend: { avg: false, current: false, max: false, min: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', percentage: false, pointradius: 5, points: false, renderer: 'flot', seriesOverrides: [], span: 3, stack: false, steppedLine: false, targets: [{ expr: 'sum(rate(etcd_network_peer_sent_bytes_total{%s, %s="$cluster"}[$__rate_interval])) by (instance)' % [$._config.etcd_selector, $._config.clusterLabel], hide: false, interval: '', intervalFactor: 2, legendFormat: '{{instance}} Peer Traffic Out', metric: 'etcd_network_peer_sent_bytes_total', refId: 'A', step: 4, }], thresholds: [], timeFrom: null, timeShift: null, title: 'Peer Traffic Out', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'cumulative', }, type: 'graph', xaxis: { mode: 'time', name: null, show: true, values: [], }, yaxes: [ { format: 'Bps', logBase: 1, max: null, min: null, show: true, }, { format: 'short', logBase: 1, max: null, min: null, show: true, }, ], }, ], title: 'New row', }, { collapse: false, editable: true, height: '250px', panels: [ { aliasColors: {}, bars: false, datasource: '$datasource', editable: true, 'error': false, fill: 0, id: 40, isNew: true, legend: { avg: false, current: false, max: false, min: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', percentage: false, pointradius: 5, points: false, renderer: 'flot', seriesOverrides: [], span: 6, stack: false, steppedLine: false, targets: [ { expr: 'sum(rate(etcd_server_proposals_failed_total{%s, %s="$cluster"}[$__rate_interval]))' % [$._config.etcd_selector, $._config.clusterLabel], intervalFactor: 2, legendFormat: 'Proposal Failure Rate', metric: 'etcd_server_proposals_failed_total', refId: 'A', step: 2, }, { expr: 'sum(etcd_server_proposals_pending{%s, %s="$cluster"})' % [$._config.etcd_selector, $._config.clusterLabel], intervalFactor: 2, legendFormat: 'Proposal Pending Total', metric: 'etcd_server_proposals_pending', refId: 'B', step: 2, }, { expr: 'sum(rate(etcd_server_proposals_committed_total{%s, %s="$cluster"}[$__rate_interval]))' % [$._config.etcd_selector, $._config.clusterLabel], intervalFactor: 2, legendFormat: 'Proposal Commit Rate', metric: 'etcd_server_proposals_committed_total', refId: 'C', step: 2, }, { expr: 'sum(rate(etcd_server_proposals_applied_total{%s, %s="$cluster"}[$__rate_interval]))' % [$._config.etcd_selector, $._config.clusterLabel], intervalFactor: 2, legendFormat: 'Proposal Apply Rate', refId: 'D', step: 2, }, ], thresholds: [], timeFrom: null, timeShift: null, title: 'Raft Proposals', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'individual', }, type: 'graph', xaxis: { mode: 'time', name: null, show: true, values: [], }, yaxes: [ { format: 'short', label: '', logBase: 1, max: null, min: null, show: true, }, { format: 'short', label: null, logBase: 1, max: null, min: null, show: true, }, ], }, { aliasColors: {}, bars: false, datasource: '$datasource', decimals: 0, editable: true, 'error': false, fill: 0, id: 19, isNew: true, legend: { alignAsTable: false, avg: false, current: false, max: false, min: false, rightSide: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', percentage: false, pointradius: 5, points: false, renderer: 'flot', seriesOverrides: [], span: 6, stack: false, steppedLine: false, targets: [{ expr: 'changes(etcd_server_leader_changes_seen_total{%s, %s="$cluster"}[1d])' % [$._config.etcd_selector, $._config.clusterLabel], intervalFactor: 2, legendFormat: '{{instance}} Total Leader Elections Per Day', metric: 'etcd_server_leader_changes_seen_total', refId: 'A', step: 2, }], thresholds: [], timeFrom: null, timeShift: null, title: 'Total Leader Elections Per Day', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'individual', }, type: 'graph', xaxis: { mode: 'time', name: null, show: true, values: [], }, yaxes: [ { format: 'short', label: null, logBase: 1, max: null, min: null, show: true, }, { format: 'short', label: null, logBase: 1, max: null, min: null, show: true, }, ], }, { aliasColors: {}, bars: false, dashLength: 10, dashes: false, datasource: '$datasource', decimals: 0, editable: true, 'error': false, fieldConfig: { defaults: { custom: {}, }, overrides: [], }, fill: 0, fillGradient: 0, gridPos: { h: 7, w: 12, x: 0, y: 28, }, hiddenSeries: false, id: 42, isNew: true, legend: { alignAsTable: false, avg: false, current: false, max: false, min: false, rightSide: false, show: false, total: false, values: false, }, lines: true, linewidth: 2, links: [], nullPointMode: 'connected', options: { alertThreshold: true, }, percentage: false, pluginVersion: '7.4.3', pointradius: 5, points: false, renderer: 'flot', seriesOverrides: [], spaceLength: 10, stack: false, steppedLine: false, targets: [ { expr: 'histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{%s, %s="$cluster"}[$__rate_interval])))' % [$._config.etcd_selector, $._config.clusterLabel], interval: '', intervalFactor: 2, legendFormat: '{{instance}} Peer round trip time', metric: 'etcd_network_peer_round_trip_time_seconds_bucket', refId: 'A', step: 2, }, ], thresholds: [], timeFrom: null, timeRegions: [], timeShift: null, title: 'Peer round trip time', tooltip: { msResolution: false, shared: true, sort: 0, value_type: 'individual', }, type: 'graph', xaxis: { buckets: null, mode: 'time', name: null, show: true, values: [], }, yaxes: [ { '$$hashKey': 'object:925', decimals: null, format: 's', label: null, logBase: 1, max: null, min: null, show: true, }, { '$$hashKey': 'object:926', format: 'short', label: null, logBase: 1, max: null, min: null, show: true, }, ], yaxis: { align: false, alignLevel: null, }, }, ], title: 'New row', }, ], time: { from: 'now-15m', to: 'now', }, timepicker: { now: true, refresh_intervals: [ '5s', '10s', '30s', '1m', '5m', '15m', '30m', '1h', '2h', '1d', ], time_options: [ '5m', '15m', '1h', '6h', '12h', '24h', '2d', '7d', '30d', ], }, templating: { list: [ { current: { text: 'Prometheus', value: 'Prometheus', }, hide: 0, label: 'Data Source', name: 'datasource', options: [], query: 'prometheus', refresh: 1, regex: '', type: 'datasource', }, { allValue: null, current: { text: 'prod', value: 'prod', }, datasource: '$datasource', hide: 0, includeAll: false, label: 'cluster', multi: false, name: 'cluster', options: [], query: 'label_values(etcd_server_has_leader{%s}, %s)' % [$._config.etcd_selector, $._config.clusterLabel], refresh: $._config.dashboard_var_refresh, regex: '', sort: 2, tagValuesQuery: '', tags: [], tagsQuery: '', type: 'query', useTags: false, }, ], }, annotations: { list: [], }, refresh: '10s', schemaVersion: 13, version: 215, links: [], gnetId: null, }, } else {}, } ================================================ FILE: contrib/mixin/dashboards/etcd.libsonnet ================================================ { grafanaDashboards+:: if !$._config.grafana7x then { local g = import './g.libsonnet', local panels = import './panels.libsonnet', local variables = import './variables.libsonnet', local targets = import './targets.libsonnet', local v = variables($._config), local t = targets(v, $._config), 'etcd.json': g.dashboard.new('etcd') + g.dashboard.withUid(std.md5('etcd.json')) + g.dashboard.withRefresh('10s') + g.dashboard.time.withFrom('now-15m') + g.dashboard.time.withTo('now') + g.dashboard.withDescription('etcd sample Grafana dashboard with Prometheus') + g.dashboard.withTags(['etcd-mixin']) + g.dashboard.withVariables([ v.datasource, v.cluster, ]) + g.dashboard.withPanels( [ panels.stat.up('Up', t.up) { gridPos: { x: 0, h: 7, w: 6, y: 0 } }, panels.timeSeries.rpcRate('RPC rate', [t.rpcRate, t.rpcFailedRate]) { gridPos: { x: 6, h: 7, w: 10, y: 0 } }, panels.timeSeries.activeStreams('Active streams', [t.watchStreams, t.leaseStreams]) { gridPos: { x: 16, h: 7, w: 8, y: 0 } }, panels.timeSeries.dbSize('DB size', [t.dbSize]) { gridPos: { x: 0, h: 7, w: 8, y: 25 } }, panels.timeSeries.diskSync('Disk sync duration', [t.walFsync, t.dbFsync]) { gridPos: { x: 8, h: 7, w: 8, y: 25 } }, panels.timeSeries.memory('Memory', [t.memory]) { gridPos: { x: 16, h: 7, w: 8, y: 25 } }, panels.timeSeries.traffic('Client traffic in', [t.clientTrafficIn]) { gridPos: { x: 0, h: 7, w: 6, y: 50 } }, panels.timeSeries.traffic('Client traffic out', [t.clientTrafficOut]) { gridPos: { x: 6, h: 7, w: 6, y: 50 } }, panels.timeSeries.traffic('Peer traffic in', [t.peerTrafficIn]) { gridPos: { x: 12, h: 7, w: 6, y: 50 } }, panels.timeSeries.traffic('Peer traffic out', [t.peerTrafficOut]) { gridPos: { x: 18, h: 7, w: 6, y: 50 } }, panels.timeSeries.raftProposals('Raft proposals', [t.raftProposals]) { gridPos: { x: 0, h: 7, w: 8, y: 75 } }, panels.timeSeries.leaderElections('Total leader elections per day', [t.leaderElections]) { gridPos: { x: 8, h: 7, w: 8, y: 75 } }, panels.timeSeries.peerRtt('Peer round trip time', [t.peerRtt]) { gridPos: { x: 16, h: 7, w: 8, y: 75 } }, ] ), } else {}, } ================================================ FILE: contrib/mixin/dashboards/g.libsonnet ================================================ import 'github.com/grafana/grafonnet/gen/grafonnet-v10.0.0/main.libsonnet' ================================================ FILE: contrib/mixin/dashboards/panels.libsonnet ================================================ local g = import 'g.libsonnet'; { stat: { local stat = g.panel.stat, base(title, targets): stat.new(title) + stat.queryOptions.withTargets(targets) + stat.queryOptions.withInterval('1m'), up(title, targets): self.base(title, targets) + stat.options.withColorMode('none') + stat.options.withGraphMode('none') + stat.options.reduceOptions.withCalcs([ 'lastNotNull', ]), }, timeSeries: { local timeSeries = g.panel.timeSeries, local fieldOverride = g.panel.timeSeries.fieldOverride, local custom = timeSeries.fieldConfig.defaults.custom, local defaults = timeSeries.fieldConfig.defaults, local options = timeSeries.options, base(title, targets): timeSeries.new(title) + timeSeries.queryOptions.withTargets(targets) + timeSeries.queryOptions.withInterval('1m') + custom.withLineWidth(2) + custom.withFillOpacity(0) + custom.withShowPoints('never'), rpcRate(title, targets): self.base(title, targets) + timeSeries.standardOptions.withUnit('ops'), activeStreams(title, targets): self.base(title, targets), dbSize(title, targets): self.base(title, targets) + timeSeries.standardOptions.withUnit('bytes'), diskSync(title, targets): self.base(title, targets) + timeSeries.standardOptions.withUnit('s'), memory(title, targets): self.base(title, targets) + timeSeries.standardOptions.withUnit('bytes'), traffic(title, targets): self.base(title, targets) + timeSeries.standardOptions.withUnit('Bps'), raftProposals(title, targets): self.base(title, targets), leaderElections(title, targets): self.base(title, targets), peerRtt(title, targets): self.base(title, targets) + timeSeries.standardOptions.withUnit('s'), }, } ================================================ FILE: contrib/mixin/dashboards/targets.libsonnet ================================================ local g = import './g.libsonnet'; local prometheusQuery = g.query.prometheus; function(variables, config) { up: prometheusQuery.new( '$' + variables.datasource.name, 'sum(etcd_server_has_leader{%s, %s="$cluster"})' % [config.etcd_selector, config.clusterLabel] ) + prometheusQuery.withLegendFormat(||| {{cluster}} - {{namespace}} |||), rpcRate: prometheusQuery.new( '$' + variables.datasource.name, 'sum(rate(grpc_server_started_total{%s, %s="$cluster",grpc_type="unary"}[$__rate_interval]))' % [config.etcd_selector, config.clusterLabel] ) + prometheusQuery.withLegendFormat('RPC rate'), rpcFailedRate: prometheusQuery.new( '$' + variables.datasource.name, 'sum(rate(grpc_server_handled_total{%s, %s="$cluster",grpc_type="unary",grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[$__rate_interval]))' % [config.etcd_selector, config.clusterLabel] ) + prometheusQuery.withLegendFormat('RPC failed rate'), watchStreams: prometheusQuery.new( '$' + variables.datasource.name, 'sum(grpc_server_started_total{%(etcd_selector)s,%(clusterLabel)s="$cluster",grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{%(clusterLabel)s="$cluster",grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"})' % config ) + prometheusQuery.withLegendFormat('Watch streams'), leaseStreams: prometheusQuery.new( '$' + variables.datasource.name, 'sum(grpc_server_started_total{%(etcd_selector)s,%(clusterLabel)s="$cluster",grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{%(clusterLabel)s="$cluster",grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"})' % config ) + prometheusQuery.withLegendFormat('Lease streams'), dbSize: prometheusQuery.new( '$' + variables.datasource.name, 'etcd_mvcc_db_total_size_in_bytes{%s, %s="$cluster"}' % [config.etcd_selector, config.clusterLabel], ) + prometheusQuery.withLegendFormat('{{instance}} DB size'), walFsync: prometheusQuery.new( '$' + variables.datasource.name, 'histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{%s, %s="$cluster"}[$__rate_interval])) by (instance, le))' % [config.etcd_selector, config.clusterLabel], ) + prometheusQuery.withLegendFormat('{{instance}} WAL fsync'), dbFsync: prometheusQuery.new( '$' + variables.datasource.name, 'histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{%s, %s="$cluster"}[$__rate_interval])) by (instance, le))' % [config.etcd_selector, config.clusterLabel], ) + prometheusQuery.withLegendFormat('{{instance}} DB fsync'), memory: prometheusQuery.new( '$' + variables.datasource.name, 'process_resident_memory_bytes{%s, %s="$cluster"}' % [config.etcd_selector, config.clusterLabel], ) + prometheusQuery.withLegendFormat('{{instance}} resident memory'), clientTrafficIn: prometheusQuery.new( '$' + variables.datasource.name, 'rate(etcd_network_client_grpc_received_bytes_total{%s, %s="$cluster"}[$__rate_interval])' % [config.etcd_selector, config.clusterLabel], ) + prometheusQuery.withLegendFormat('{{instance}} client traffic in'), clientTrafficOut: prometheusQuery.new( '$' + variables.datasource.name, 'rate(etcd_network_client_grpc_sent_bytes_total{%s, %s="$cluster"}[$__rate_interval])' % [config.etcd_selector, config.clusterLabel], ) + prometheusQuery.withLegendFormat('{{instance}} client traffic out'), peerTrafficIn: prometheusQuery.new( '$' + variables.datasource.name, 'sum(rate(etcd_network_peer_received_bytes_total{%s, %s="$cluster"}[$__rate_interval])) by (instance)' % [config.etcd_selector, config.clusterLabel], ) + prometheusQuery.withLegendFormat('{{instance}} peer traffic in'), peerTrafficOut: prometheusQuery.new( '$' + variables.datasource.name, 'sum(rate(etcd_network_peer_sent_bytes_total{%s, %s="$cluster"}[$__rate_interval])) by (instance)' % [config.etcd_selector, config.clusterLabel], ) + prometheusQuery.withLegendFormat('{{instance}} peer traffic out'), raftProposals: prometheusQuery.new( '$' + variables.datasource.name, 'changes(etcd_server_leader_changes_seen_total{%s, %s="$cluster"}[1d])' % [config.etcd_selector, config.clusterLabel], ) + prometheusQuery.withLegendFormat('{{instance}} total leader elections per day'), leaderElections: prometheusQuery.new( '$' + variables.datasource.name, 'changes(etcd_server_leader_changes_seen_total{%s, %s="$cluster"}[1d])' % [config.etcd_selector, config.clusterLabel], ) + prometheusQuery.withLegendFormat('{{instance}} total leader elections per day'), peerRtt: prometheusQuery.new( '$' + variables.datasource.name, 'histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{%s, %s="$cluster"}[$__rate_interval])))' % [config.etcd_selector, config.clusterLabel], ) + prometheusQuery.withLegendFormat('{{instance}} peer round trip time'), } ================================================ FILE: contrib/mixin/dashboards/variables.libsonnet ================================================ // variables.libsonnet local g = import './g.libsonnet'; local var = g.dashboard.variable; function(config) { datasource: var.datasource.new('datasource', 'prometheus') + var.datasource.generalOptions.withLabel('Data Source'), cluster: var.query.new('cluster') + var.query.generalOptions.withLabel('cluster') + var.query.withDatasourceFromVariable(self.datasource) + { refresh: config.dashboard_var_refresh } + var.query.queryTypes.withLabelValues( config.clusterLabel, 'etcd_server_has_leader{%s}' % [config.etcd_selector] ), } ================================================ FILE: contrib/mixin/jsonnetfile.json ================================================ { "version": 1, "dependencies": [ { "source": { "git": { "remote": "https://github.com/grafana/grafonnet.git", "subdir": "gen/grafonnet-v10.0.0" } }, "version": "main" } ], "legacyImports": true } ================================================ FILE: contrib/mixin/jsonnetfile.lock.json ================================================ { "version": 1, "dependencies": [ { "source": { "git": { "remote": "https://github.com/grafana/grafonnet.git", "subdir": "gen/grafonnet-v10.0.0" } }, "version": "e85299323fd8808187d30865cc5c7a38a347399a", "sum": "uJCTMGtY/7c5HSLQ7UQD38TOPmuSYrIKLIKmdSF/Htk=" }, { "source": { "git": { "remote": "https://github.com/jsonnet-libs/docsonnet.git", "subdir": "doc-util" } }, "version": "fd8de9039b3c06da77d635a3a8289809a5bfb542", "sum": "mFebrE9fhyAKW4zbnidcjVFupziN5LPA/Z7ii94uCzs=" }, { "source": { "git": { "remote": "https://github.com/jsonnet-libs/xtd.git", "subdir": "" } }, "version": "0256a910ac71f0f842696d7bca0bf01ea77eb654", "sum": "zBOpb1oTNvXdq9RF6yzTHill5r1YTJLBBoqyx4JYtAg=" } ], "legacyImports": false } ================================================ FILE: contrib/mixin/mixin.libsonnet ================================================ (import './config.libsonnet') + (import './dashboards/dashboards.libsonnet') + (import './alerts/alerts.libsonnet') ================================================ FILE: contrib/mixin/test.yaml ================================================ --- rule_files: [manifests/etcd-prometheusRules.yaml] evaluation_interval: 1m tests: - interval: 1m input_series: - series: up{job="etcd",instance="10.10.10.0"} values: 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - series: up{job="etcd",instance="10.10.10.1"} values: 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - series: up{job="etcd",instance="10.10.10.2"} values: 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 alert_rule_test: - eval_time: 3m alertname: etcdInsufficientMembers - eval_time: 5m alertname: etcdInsufficientMembers - eval_time: 22m alertname: etcdMembersDown - eval_time: 24m alertname: etcdMembersDown exp_alerts: - exp_labels: job: etcd severity: warning exp_annotations: description: 'etcd cluster "etcd": members are down (3).' summary: etcd cluster members are down. - eval_time: 7m alertname: etcdInsufficientMembers - eval_time: 11m alertname: etcdInsufficientMembers exp_alerts: - exp_labels: job: etcd severity: critical exp_annotations: description: 'etcd cluster "etcd": insufficient members (1).' summary: etcd cluster has insufficient number of members. - eval_time: 15m alertname: etcdInsufficientMembers exp_alerts: - exp_labels: job: etcd severity: critical exp_annotations: description: 'etcd cluster "etcd": insufficient members (0).' summary: etcd cluster has insufficient number of members. - interval: 1m input_series: - series: up{job="etcd",instance="10.10.10.0"} values: 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 - series: up{job="etcd",instance="10.10.10.1"} values: 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 - series: up{job="etcd",instance="10.10.10.2"} values: 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 alert_rule_test: - eval_time: 24m alertname: etcdMembersDown exp_alerts: - exp_labels: job: etcd severity: warning exp_annotations: description: 'etcd cluster "etcd": members are down (3).' summary: etcd cluster members are down. - interval: 1m input_series: - series: up{job="etcd",instance="10.10.10.0"} values: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 - series: up{job="etcd",instance="10.10.10.1"} values: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 - series: etcd_network_peer_sent_failures_total{To="member-1",job="etcd",endpoint="test"} values: 0 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 alert_rule_test: - eval_time: 23m alertname: etcdMembersDown exp_alerts: - exp_labels: job: etcd severity: warning exp_annotations: description: 'etcd cluster "etcd": members are down (1).' summary: etcd cluster members are down. - interval: 1m input_series: - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.0"} values: 0 0 2 0 0 1 0 0 0 0 0 0 0 0 0 0 - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.1"} values: 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.2"} values: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 alert_rule_test: - eval_time: 10m alertname: etcdHighNumberOfLeaderChanges exp_alerts: - exp_labels: job: etcd severity: warning exp_annotations: description: 'etcd cluster "etcd": 4 leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' summary: etcd cluster has high number of leader changes. - interval: 1m input_series: - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.0"} values: 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.1"} values: 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 - series: etcd_server_leader_changes_seen_total{job="etcd",instance="10.10.10.2"} values: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 alert_rule_test: - eval_time: 10m alertname: etcdHighNumberOfLeaderChanges exp_alerts: - interval: 1m input_series: - series: etcd_mvcc_db_total_size_in_bytes{job="etcd",instance="10.10.10.0"} values: 0+8192x240 - series: etcd_server_quota_backend_bytes{job="etcd",instance="10.10.10.0"} values: 524288+0x240 - series: etcd_mvcc_db_total_size_in_bytes{job="etcd",instance="10.10.10.1"} values: 0+1024x240 - series: etcd_server_quota_backend_bytes{job="etcd",instance="10.10.10.1"} values: 524288+0x240 alert_rule_test: - eval_time: 11m alertname: etcdExcessiveDatabaseGrowth exp_alerts: - exp_labels: instance: 10.10.10.0 job: etcd severity: warning exp_annotations: description: 'etcd cluster "etcd": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance 10.10.10.0, please check as it might be disruptive.' summary: etcd cluster database growing very fast. - interval: 1m input_series: - series: etcd_mvcc_db_total_size_in_use_in_bytes{job="etcd",instance="10.10.10.0"} values: 300000000+0x10 - series: etcd_mvcc_db_total_size_in_bytes{job="etcd",instance="10.10.10.0"} values: 1000000000+0x10 - series: etcd_mvcc_db_total_size_in_use_in_bytes{job="etcd",instance="10.10.10.1"} values: 700000000+0x10 - series: etcd_mvcc_db_total_size_in_bytes{job="etcd",instance="10.10.10.1"} values: 1000000000+0x10 alert_rule_test: - eval_time: 11m alertname: etcdDatabaseHighFragmentationRatio exp_alerts: - exp_labels: instance: 10.10.10.0 job: etcd severity: warning exp_annotations: description: 'etcd cluster "etcd": database size in use on instance 10.10.10.0 is 30% of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.' runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation summary: etcd database size in use is less than 50% of the actual allocated storage. ================================================ FILE: contrib/raftexample/Procfile ================================================ # Use goreman to run `go install github.com/mattn/goreman@latest` raftexample1: ./raftexample --id 1 --cluster http://127.0.0.1:12379,http://127.0.0.1:22379,http://127.0.0.1:32379 --port 12380 raftexample2: ./raftexample --id 2 --cluster http://127.0.0.1:12379,http://127.0.0.1:22379,http://127.0.0.1:32379 --port 22380 raftexample3: ./raftexample --id 3 --cluster http://127.0.0.1:12379,http://127.0.0.1:22379,http://127.0.0.1:32379 --port 32380 ================================================ FILE: contrib/raftexample/README.md ================================================ # raftexample raftexample is an example usage of etcd's [raft library](https://github.com/etcd-io/raft). It provides a simple REST API for a key-value store cluster backed by the [Raft][raft] consensus algorithm. [raft]: http://raftconsensus.github.io/ ## Getting Started ### Building raftexample Clone `etcd` to `/src/go.etcd.io/etcd` ```sh export GOPATH= cd /src/go.etcd.io/etcd/contrib/raftexample go build -o raftexample ``` ### Running single node raftexample First start a single-member cluster of raftexample: ```sh raftexample --id 1 --cluster http://127.0.0.1:12379 --port 12380 ``` Each raftexample process maintains a single raft instance and a key-value server. The process's list of comma separated peers (--cluster), its raft ID index into the peer list (--id), and http key-value server port (--port) are passed through the command line. Next, store a value ("hello") to a key ("my-key"): ``` curl -L http://127.0.0.1:12380/my-key -XPUT -d hello ``` Finally, retrieve the stored key: ``` curl -L http://127.0.0.1:12380/my-key ``` ### Running a local cluster First install [goreman](https://github.com/mattn/goreman), which manages Procfile-based applications. The [Procfile script](./Procfile) will set up a local example cluster. Start it with: ```sh goreman start ``` This will bring up three raftexample instances. Now it's possible to write a key-value pair to any member of the cluster and likewise retrieve it from any member. ### Fault Tolerance To test cluster recovery, first start a cluster and write a value "foo": ```sh goreman start curl -L http://127.0.0.1:12380/my-key -XPUT -d foo ``` Next, remove a node and replace the value with "bar" to check cluster availability: ```sh goreman run stop raftexample2 curl -L http://127.0.0.1:12380/my-key -XPUT -d bar curl -L http://127.0.0.1:32380/my-key ``` Finally, bring the node back up and verify it recovers with the updated value "bar": ```sh goreman run start raftexample2 curl -L http://127.0.0.1:22380/my-key ``` ### Dynamic cluster reconfiguration Nodes can be added to or removed from a running cluster using requests to the REST API. For example, suppose we have a 3-node cluster that was started with the commands: ```sh raftexample --id 1 --cluster http://127.0.0.1:12379,http://127.0.0.1:22379,http://127.0.0.1:32379 --port 12380 raftexample --id 2 --cluster http://127.0.0.1:12379,http://127.0.0.1:22379,http://127.0.0.1:32379 --port 22380 raftexample --id 3 --cluster http://127.0.0.1:12379,http://127.0.0.1:22379,http://127.0.0.1:32379 --port 32380 ``` A fourth node with ID 4 can be added by issuing a POST: ```sh curl -L http://127.0.0.1:12380/4 -XPOST -d http://127.0.0.1:42379 ``` Then the new node can be started as the others were, using the --join option: ```sh raftexample --id 4 --cluster http://127.0.0.1:12379,http://127.0.0.1:22379,http://127.0.0.1:32379,http://127.0.0.1:42379 --port 42380 --join ``` The new node should join the cluster and be able to service key/value requests. We can remove a node using a DELETE request: ```sh curl -L http://127.0.0.1:12380/3 -XDELETE ``` Node 3 should shut itself down once the cluster has processed this request. ## Design The raftexample consists of three components: a raft-backed key-value store, a REST API server, and a raft consensus server based on etcd's raft implementation. The raft-backed key-value store is a key-value map that holds all committed key-values. The store bridges communication between the raft server and the REST server. Key-value updates are issued through the store to the raft server. The store updates its map once raft reports the updates are committed. The REST server exposes the current raft consensus by accessing the raft-backed key-value store. A GET command looks up a key in the store and returns the value, if any. A key-value PUT command issues an update proposal to the store. The raft server participates in consensus with its cluster peers. When the REST server submits a proposal, the raft server transmits the proposal to its peers. When raft reaches a consensus, the server publishes all committed updates over a commit channel. For raftexample, this commit channel is consumed by the key-value store. ================================================ FILE: contrib/raftexample/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // raftexample is a simple KV store using the raft and rafthttp libraries. package main ================================================ FILE: contrib/raftexample/httpapi.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "io" "log" "net/http" "strconv" "go.etcd.io/raft/v3/raftpb" ) // Handler for a http based key-value store backed by raft type httpKVAPI struct { store *kvstore confChangeC chan<- raftpb.ConfChange } func (h *httpKVAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) { key := r.RequestURI defer r.Body.Close() switch r.Method { case http.MethodPut: v, err := io.ReadAll(r.Body) if err != nil { log.Printf("Failed to read on PUT (%v)\n", err) http.Error(w, "Failed on PUT", http.StatusBadRequest) return } h.store.Propose(key, string(v)) // Optimistic-- no waiting for ack from raft. Value is not yet // committed so a subsequent GET on the key may return old value w.WriteHeader(http.StatusNoContent) case http.MethodGet: if v, ok := h.store.Lookup(key); ok { w.Write([]byte(v)) } else { http.Error(w, "Failed to GET", http.StatusNotFound) } case http.MethodPost: url, err := io.ReadAll(r.Body) if err != nil { log.Printf("Failed to read on POST (%v)\n", err) http.Error(w, "Failed on POST", http.StatusBadRequest) return } nodeID, err := strconv.ParseUint(key[1:], 0, 64) if err != nil { log.Printf("Failed to convert ID for conf change (%v)\n", err) http.Error(w, "Failed on POST", http.StatusBadRequest) return } cc := raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: nodeID, Context: url, } h.confChangeC <- cc // As above, optimistic that raft will apply the conf change w.WriteHeader(http.StatusNoContent) case http.MethodDelete: nodeID, err := strconv.ParseUint(key[1:], 0, 64) if err != nil { log.Printf("Failed to convert ID for conf change (%v)\n", err) http.Error(w, "Failed on DELETE", http.StatusBadRequest) return } cc := raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: nodeID, } h.confChangeC <- cc // As above, optimistic that raft will apply the conf change w.WriteHeader(http.StatusNoContent) default: w.Header().Set("Allow", http.MethodPut) w.Header().Add("Allow", http.MethodGet) w.Header().Add("Allow", http.MethodPost) w.Header().Add("Allow", http.MethodDelete) http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) } } // serveHTTPKVAPI starts a key-value server with a GET/PUT API and listens. func serveHTTPKVAPI(kv *kvstore, port int, confChangeC chan<- raftpb.ConfChange, errorC <-chan error) { srv := http.Server{ Addr: ":" + strconv.Itoa(port), Handler: &httpKVAPI{ store: kv, confChangeC: confChangeC, }, } go func() { if err := srv.ListenAndServe(); err != nil { log.Fatal(err) } }() // exit when raft goes down if err, ok := <-errorC; ok { log.Fatal(err) } } ================================================ FILE: contrib/raftexample/kvstore.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "bytes" "encoding/gob" "encoding/json" "errors" "log" "strings" "sync" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/raft/v3/raftpb" ) // a key-value store backed by raft type kvstore struct { proposeC chan<- string // channel for proposing updates mu sync.RWMutex kvStore map[string]string // current committed key-value pairs snapshotter *snap.Snapshotter } type kv struct { Key string Val string } func newKVStore(snapshotter *snap.Snapshotter, proposeC chan<- string, commitC <-chan *commit, errorC <-chan error) *kvstore { s := &kvstore{proposeC: proposeC, kvStore: make(map[string]string), snapshotter: snapshotter} snapshot, err := s.loadSnapshot() if err != nil { log.Panic(err) } if snapshot != nil { log.Printf("loading snapshot at term %d and index %d", snapshot.Metadata.Term, snapshot.Metadata.Index) if err := s.recoverFromSnapshot(snapshot.Data); err != nil { log.Panic(err) } } // read commits from raft into kvStore map until error go s.readCommits(commitC, errorC) return s } func (s *kvstore) Lookup(key string) (string, bool) { s.mu.RLock() defer s.mu.RUnlock() v, ok := s.kvStore[key] return v, ok } func (s *kvstore) Propose(k string, v string) { var buf strings.Builder if err := gob.NewEncoder(&buf).Encode(kv{k, v}); err != nil { log.Fatal(err) } s.proposeC <- buf.String() } func (s *kvstore) readCommits(commitC <-chan *commit, errorC <-chan error) { for commit := range commitC { if commit == nil { // signaled to load snapshot snapshot, err := s.loadSnapshot() if err != nil { log.Panic(err) } if snapshot != nil { log.Printf("loading snapshot at term %d and index %d", snapshot.Metadata.Term, snapshot.Metadata.Index) if err := s.recoverFromSnapshot(snapshot.Data); err != nil { log.Panic(err) } } continue } for _, data := range commit.data { var dataKv kv dec := gob.NewDecoder(bytes.NewBufferString(data)) if err := dec.Decode(&dataKv); err != nil { log.Fatalf("raftexample: could not decode message (%v)", err) } s.mu.Lock() s.kvStore[dataKv.Key] = dataKv.Val s.mu.Unlock() } close(commit.applyDoneC) } if err, ok := <-errorC; ok { log.Fatal(err) } } func (s *kvstore) getSnapshot() ([]byte, error) { s.mu.RLock() defer s.mu.RUnlock() return json.Marshal(s.kvStore) } func (s *kvstore) loadSnapshot() (*raftpb.Snapshot, error) { snapshot, err := s.snapshotter.Load() if errors.Is(err, snap.ErrNoSnapshot) { return nil, nil } if err != nil { return nil, err } return snapshot, nil } func (s *kvstore) recoverFromSnapshot(snapshot []byte) error { var store map[string]string if err := json.Unmarshal(snapshot, &store); err != nil { return err } s.mu.Lock() defer s.mu.Unlock() s.kvStore = store return nil } ================================================ FILE: contrib/raftexample/kvstore_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "reflect" "testing" "github.com/stretchr/testify/require" ) func Test_kvstore_snapshot(t *testing.T) { tm := map[string]string{"foo": "bar"} s := &kvstore{kvStore: tm} v, _ := s.Lookup("foo") require.Equalf(t, "bar", v, "foo has unexpected value, got %s", v) data, err := s.getSnapshot() require.NoError(t, err) s.kvStore = nil err = s.recoverFromSnapshot(data) require.NoError(t, err) v, _ = s.Lookup("foo") require.Equalf(t, "bar", v, "foo has unexpected value, got %s", v) require.Truef(t, reflect.DeepEqual(s.kvStore, tm), "store expected %+v, got %+v", tm, s.kvStore) } ================================================ FILE: contrib/raftexample/listener.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "errors" "net" "time" ) // stoppableListener sets TCP keep-alive timeouts on accepted // connections and waits on stopc message type stoppableListener struct { *net.TCPListener stopc <-chan struct{} } func newStoppableListener(addr string, stopc <-chan struct{}) (*stoppableListener, error) { ln, err := net.Listen("tcp", addr) if err != nil { return nil, err } return &stoppableListener{ln.(*net.TCPListener), stopc}, nil } func (ln stoppableListener) Accept() (c net.Conn, err error) { connc := make(chan *net.TCPConn, 1) errc := make(chan error, 1) go func() { tc, err := ln.AcceptTCP() if err != nil { errc <- err return } connc <- tc }() select { case <-ln.stopc: return nil, errors.New("server stopped") case err := <-errc: return nil, err case tc := <-connc: tc.SetKeepAlive(true) tc.SetKeepAlivePeriod(3 * time.Minute) return tc, nil } } ================================================ FILE: contrib/raftexample/main.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "flag" "strings" "go.etcd.io/raft/v3/raftpb" ) func main() { cluster := flag.String("cluster", "http://127.0.0.1:9021", "comma separated cluster peers") id := flag.Int("id", 1, "node ID") kvport := flag.Int("port", 9121, "key-value server port") join := flag.Bool("join", false, "join an existing cluster") flag.Parse() proposeC := make(chan string) defer close(proposeC) confChangeC := make(chan raftpb.ConfChange) defer close(confChangeC) // raft provides a commit stream for the proposals from the http api var kvs *kvstore getSnapshot := func() ([]byte, error) { return kvs.getSnapshot() } commitC, errorC, snapshotterReady := newRaftNode(*id, strings.Split(*cluster, ","), *join, getSnapshot, proposeC, confChangeC) kvs = newKVStore(<-snapshotterReady, proposeC, commitC, errorC) // the key-value http handler will propose updates to raft serveHTTPKVAPI(kvs, *kvport, confChangeC, errorC) } ================================================ FILE: contrib/raftexample/raft.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "context" "errors" "fmt" "log" "net/http" "net/url" "os" "strconv" "time" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) type commit struct { data []string applyDoneC chan<- struct{} } // A key-value stream backed by raft type raftNode struct { proposeC <-chan string // proposed messages (k,v) confChangeC <-chan raftpb.ConfChange // proposed cluster config changes commitC chan<- *commit // entries committed to log (k,v) errorC chan<- error // errors from raft session id int // client ID for raft session peers []string // raft peer URLs join bool // node is joining an existing cluster waldir string // path to WAL directory snapdir string // path to snapshot directory getSnapshot func() ([]byte, error) confState raftpb.ConfState snapshotIndex uint64 appliedIndex uint64 // raft backing for the commit/error channel node raft.Node raftStorage *raft.MemoryStorage wal *wal.WAL snapshotter *snap.Snapshotter snapshotterReady chan *snap.Snapshotter // signals when snapshotter is ready snapCount uint64 transport *rafthttp.Transport stopc chan struct{} // signals proposal channel closed httpstopc chan struct{} // signals http server to shutdown httpdonec chan struct{} // signals http server shutdown complete logger *zap.Logger } var defaultSnapshotCount uint64 = 10000 // newRaftNode initiates a raft instance and returns a committed log entry // channel and error channel. Proposals for log updates are sent over the // provided the proposal channel. All log entries are replayed over the // commit channel, followed by a nil message (to indicate the channel is // current), then new log entries. To shutdown, close proposeC and read errorC. func newRaftNode(id int, peers []string, join bool, getSnapshot func() ([]byte, error), proposeC <-chan string, confChangeC <-chan raftpb.ConfChange, ) (<-chan *commit, <-chan error, <-chan *snap.Snapshotter) { commitC := make(chan *commit) errorC := make(chan error) rc := &raftNode{ proposeC: proposeC, confChangeC: confChangeC, commitC: commitC, errorC: errorC, id: id, peers: peers, join: join, waldir: fmt.Sprintf("raftexample-%d", id), snapdir: fmt.Sprintf("raftexample-%d-snap", id), getSnapshot: getSnapshot, snapCount: defaultSnapshotCount, stopc: make(chan struct{}), httpstopc: make(chan struct{}), httpdonec: make(chan struct{}), logger: zap.NewExample(), snapshotterReady: make(chan *snap.Snapshotter, 1), // rest of structure populated after WAL replay } go rc.startRaft() return commitC, errorC, rc.snapshotterReady } func (rc *raftNode) saveSnap(snap raftpb.Snapshot) error { walSnap := walpb.Snapshot{ Index: new(snap.Metadata.Index), Term: new(snap.Metadata.Term), ConfState: &snap.Metadata.ConfState, } // save the snapshot file before writing the snapshot to the wal. // This makes it possible for the snapshot file to become orphaned, but prevents // a WAL snapshot entry from having no corresponding snapshot file. if err := rc.snapshotter.SaveSnap(snap); err != nil { return err } if err := rc.wal.SaveSnapshot(walSnap); err != nil { return err } return rc.wal.ReleaseLockTo(snap.Metadata.Index) } func (rc *raftNode) entriesToApply(ents []raftpb.Entry) (nents []raftpb.Entry) { if len(ents) == 0 { return ents } firstIdx := ents[0].Index if firstIdx > rc.appliedIndex+1 { log.Fatalf("first index of committed entry[%d] should <= progress.appliedIndex[%d]+1", firstIdx, rc.appliedIndex) } if rc.appliedIndex-firstIdx+1 < uint64(len(ents)) { nents = ents[rc.appliedIndex-firstIdx+1:] } return nents } // publishEntries writes committed log entries to commit channel and returns // whether all entries could be published. func (rc *raftNode) publishEntries(ents []raftpb.Entry) (<-chan struct{}, bool) { if len(ents) == 0 { return nil, true } data := make([]string, 0, len(ents)) for i := range ents { switch ents[i].Type { case raftpb.EntryNormal: if len(ents[i].Data) == 0 { // ignore empty messages break } s := string(ents[i].Data) data = append(data, s) case raftpb.EntryConfChange: var cc raftpb.ConfChange cc.Unmarshal(ents[i].Data) rc.confState = *rc.node.ApplyConfChange(cc) switch cc.Type { case raftpb.ConfChangeAddNode: if len(cc.Context) > 0 { rc.transport.AddPeer(types.ID(cc.NodeID), []string{string(cc.Context)}) } case raftpb.ConfChangeRemoveNode: if cc.NodeID == uint64(rc.id) { log.Println("I've been removed from the cluster! Shutting down.") return nil, false } rc.transport.RemovePeer(types.ID(cc.NodeID)) } } } var applyDoneC chan struct{} if len(data) > 0 { applyDoneC = make(chan struct{}, 1) select { case rc.commitC <- &commit{data, applyDoneC}: case <-rc.stopc: return nil, false } } // after commit, update appliedIndex rc.appliedIndex = ents[len(ents)-1].Index return applyDoneC, true } func (rc *raftNode) loadSnapshot() *raftpb.Snapshot { if wal.Exist(rc.waldir) { walSnaps, err := wal.ValidSnapshotEntries(rc.logger, rc.waldir) if err != nil { log.Fatalf("raftexample: error listing snapshots (%v)", err) } snapshot, err := rc.snapshotter.LoadNewestAvailable(walSnaps) if err != nil && !errors.Is(err, snap.ErrNoSnapshot) { log.Fatalf("raftexample: error loading snapshot (%v)", err) } return snapshot } return &raftpb.Snapshot{} } // openWAL returns a WAL ready for reading. func (rc *raftNode) openWAL(snapshot *raftpb.Snapshot) *wal.WAL { if !wal.Exist(rc.waldir) { if err := os.Mkdir(rc.waldir, 0o750); err != nil { log.Fatalf("raftexample: cannot create dir for wal (%v)", err) } w, err := wal.Create(zap.NewExample(), rc.waldir, nil) if err != nil { log.Fatalf("raftexample: create wal error (%v)", err) } w.Close() } walsnap := walpb.Snapshot{} if snapshot != nil { walsnap.Index, walsnap.Term = new(snapshot.Metadata.Index), new(snapshot.Metadata.Term) } log.Printf("loading WAL at term %d and index %d", walsnap.GetTerm(), walsnap.GetIndex()) w, err := wal.Open(zap.NewExample(), rc.waldir, walsnap) if err != nil { log.Fatalf("raftexample: error loading wal (%v)", err) } return w } // replayWAL replays WAL entries into the raft instance. func (rc *raftNode) replayWAL() *wal.WAL { log.Printf("replaying WAL of member %d", rc.id) snapshot := rc.loadSnapshot() w := rc.openWAL(snapshot) _, st, ents, err := w.ReadAll() if err != nil { log.Fatalf("raftexample: failed to read WAL (%v)", err) } rc.raftStorage = raft.NewMemoryStorage() if snapshot != nil { rc.raftStorage.ApplySnapshot(*snapshot) } rc.raftStorage.SetHardState(st) // append to storage so raft starts at the right place in log rc.raftStorage.Append(ents) return w } func (rc *raftNode) writeError(err error) { rc.stopHTTP() close(rc.commitC) rc.errorC <- err close(rc.errorC) rc.node.Stop() } func (rc *raftNode) startRaft() { if !fileutil.Exist(rc.snapdir) { if err := os.Mkdir(rc.snapdir, 0o750); err != nil { log.Fatalf("raftexample: cannot create dir for snapshot (%v)", err) } } rc.snapshotter = snap.New(zap.NewExample(), rc.snapdir) oldwal := wal.Exist(rc.waldir) rc.wal = rc.replayWAL() // signal replay has finished rc.snapshotterReady <- rc.snapshotter rpeers := make([]raft.Peer, len(rc.peers)) for i := range rpeers { rpeers[i] = raft.Peer{ID: uint64(i + 1)} } c := &raft.Config{ ID: uint64(rc.id), ElectionTick: 10, HeartbeatTick: 1, Storage: rc.raftStorage, MaxSizePerMsg: 1024 * 1024, MaxInflightMsgs: 256, MaxUncommittedEntriesSize: 1 << 30, } if oldwal || rc.join { rc.node = raft.RestartNode(c) } else { rc.node = raft.StartNode(c, rpeers) } rc.transport = &rafthttp.Transport{ Logger: rc.logger, ID: types.ID(rc.id), ClusterID: 0x1000, Raft: rc, ServerStats: stats.NewServerStats("", ""), LeaderStats: stats.NewLeaderStats(zap.NewExample(), strconv.Itoa(rc.id)), ErrorC: make(chan error), } rc.transport.Start() for i := range rc.peers { if i+1 != rc.id { rc.transport.AddPeer(types.ID(i+1), []string{rc.peers[i]}) } } go rc.serveRaft() go rc.serveChannels() } // stop closes http, closes all channels, and stops raft. func (rc *raftNode) stop() { rc.stopHTTP() close(rc.commitC) close(rc.errorC) rc.node.Stop() } func (rc *raftNode) stopHTTP() { rc.transport.Stop() close(rc.httpstopc) <-rc.httpdonec } func (rc *raftNode) publishSnapshot(snapshotToSave raftpb.Snapshot) { if raft.IsEmptySnap(snapshotToSave) { return } log.Printf("publishing snapshot at index %d", rc.snapshotIndex) defer log.Printf("finished publishing snapshot at index %d", rc.snapshotIndex) if snapshotToSave.Metadata.Index <= rc.appliedIndex { log.Fatalf("snapshot index [%d] should > progress.appliedIndex [%d]", snapshotToSave.Metadata.Index, rc.appliedIndex) } rc.commitC <- nil // trigger kvstore to load snapshot rc.confState = snapshotToSave.Metadata.ConfState rc.snapshotIndex = snapshotToSave.Metadata.Index rc.appliedIndex = snapshotToSave.Metadata.Index } var snapshotCatchUpEntriesN uint64 = 10000 func (rc *raftNode) maybeTriggerSnapshot(applyDoneC <-chan struct{}) { if rc.appliedIndex-rc.snapshotIndex <= rc.snapCount { return } // wait until all committed entries are applied (or server is closed) if applyDoneC != nil { select { case <-applyDoneC: case <-rc.stopc: return } } log.Printf("start snapshot [applied index: %d | last snapshot index: %d]", rc.appliedIndex, rc.snapshotIndex) data, err := rc.getSnapshot() if err != nil { log.Panic(err) } snap, err := rc.raftStorage.CreateSnapshot(rc.appliedIndex, &rc.confState, data) if err != nil { panic(err) } if err := rc.saveSnap(snap); err != nil { panic(err) } compactIndex := uint64(1) if rc.appliedIndex > snapshotCatchUpEntriesN { compactIndex = rc.appliedIndex - snapshotCatchUpEntriesN } if err := rc.raftStorage.Compact(compactIndex); err != nil { if !errors.Is(err, raft.ErrCompacted) { panic(err) } } else { log.Printf("compacted log at index %d", compactIndex) } rc.snapshotIndex = rc.appliedIndex } func (rc *raftNode) serveChannels() { snap, err := rc.raftStorage.Snapshot() if err != nil { panic(err) } rc.confState = snap.Metadata.ConfState rc.snapshotIndex = snap.Metadata.Index rc.appliedIndex = snap.Metadata.Index defer rc.wal.Close() ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() // send proposals over raft go func() { confChangeCount := uint64(0) for rc.proposeC != nil && rc.confChangeC != nil { select { case prop, ok := <-rc.proposeC: if !ok { rc.proposeC = nil } else { // blocks until accepted by raft state machine rc.node.Propose(context.TODO(), []byte(prop)) } case cc, ok := <-rc.confChangeC: if !ok { rc.confChangeC = nil } else { confChangeCount++ cc.ID = confChangeCount rc.node.ProposeConfChange(context.TODO(), cc) } } } // client closed channel; shutdown raft if not already close(rc.stopc) }() // event loop on raft state machine updates for { select { case <-ticker.C: rc.node.Tick() // store raft entries to wal, then publish over commit channel case rd := <-rc.node.Ready(): // Must save the snapshot file and WAL snapshot entry before saving any other entries // or hardstate to ensure that recovery after a snapshot restore is possible. if !raft.IsEmptySnap(rd.Snapshot) { rc.saveSnap(rd.Snapshot) } rc.wal.Save(rd.HardState, rd.Entries) if !raft.IsEmptySnap(rd.Snapshot) { rc.raftStorage.ApplySnapshot(rd.Snapshot) rc.publishSnapshot(rd.Snapshot) } rc.raftStorage.Append(rd.Entries) rc.transport.Send(rc.processMessages(rd.Messages)) applyDoneC, ok := rc.publishEntries(rc.entriesToApply(rd.CommittedEntries)) if !ok { rc.stop() return } rc.maybeTriggerSnapshot(applyDoneC) rc.node.Advance() case err := <-rc.transport.ErrorC: rc.writeError(err) return case <-rc.stopc: rc.stop() return } } } // When there is a `raftpb.EntryConfChange` after creating the snapshot, // then the confState included in the snapshot is out of date. so We need // to update the confState before sending a snapshot to a follower. func (rc *raftNode) processMessages(ms []raftpb.Message) []raftpb.Message { for i := 0; i < len(ms); i++ { if ms[i].Type == raftpb.MsgSnap { ms[i].Snapshot.Metadata.ConfState = rc.confState } } return ms } func (rc *raftNode) serveRaft() { url, err := url.Parse(rc.peers[rc.id-1]) if err != nil { log.Fatalf("raftexample: Failed parsing URL (%v)", err) } ln, err := newStoppableListener(url.Host, rc.httpstopc) if err != nil { log.Fatalf("raftexample: Failed to listen rafthttp (%v)", err) } err = (&http.Server{Handler: rc.transport.Handler()}).Serve(ln) select { case <-rc.httpstopc: default: log.Fatalf("raftexample: Failed to serve rafthttp (%v)", err) } close(rc.httpdonec) } func (rc *raftNode) Process(ctx context.Context, m raftpb.Message) error { return rc.node.Step(ctx, m) } func (rc *raftNode) IsIDRemoved(_ uint64) bool { return false } func (rc *raftNode) ReportUnreachable(id uint64) { rc.node.ReportUnreachable(id) } func (rc *raftNode) ReportSnapshot(id uint64, status raft.SnapshotStatus) { rc.node.ReportSnapshot(id, status) } ================================================ FILE: contrib/raftexample/raft_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "reflect" "testing" "github.com/stretchr/testify/require" "go.etcd.io/raft/v3/raftpb" ) func TestProcessMessages(t *testing.T) { cases := []struct { name string confState raftpb.ConfState InputMessages []raftpb.Message ExpectedMessages []raftpb.Message }{ { name: "only one snapshot message", confState: raftpb.ConfState{ Voters: []uint64{2, 6, 8, 10}, }, InputMessages: []raftpb.Message{ { Type: raftpb.MsgSnap, To: 8, Snapshot: &raftpb.Snapshot{ Metadata: raftpb.SnapshotMetadata{ Index: 100, Term: 3, ConfState: raftpb.ConfState{ Voters: []uint64{2, 6, 8}, AutoLeave: true, }, }, }, }, }, ExpectedMessages: []raftpb.Message{ { Type: raftpb.MsgSnap, To: 8, Snapshot: &raftpb.Snapshot{ Metadata: raftpb.SnapshotMetadata{ Index: 100, Term: 3, ConfState: raftpb.ConfState{ Voters: []uint64{2, 6, 8, 10}, }, }, }, }, }, }, { name: "one snapshot message and one other message", confState: raftpb.ConfState{ Voters: []uint64{2, 7, 8, 12}, }, InputMessages: []raftpb.Message{ { Type: raftpb.MsgSnap, To: 8, Snapshot: &raftpb.Snapshot{ Metadata: raftpb.SnapshotMetadata{ Index: 100, Term: 3, ConfState: raftpb.ConfState{ Voters: []uint64{2, 6, 8}, AutoLeave: true, }, }, }, }, { Type: raftpb.MsgApp, From: 6, To: 8, }, }, ExpectedMessages: []raftpb.Message{ { Type: raftpb.MsgSnap, To: 8, Snapshot: &raftpb.Snapshot{ Metadata: raftpb.SnapshotMetadata{ Index: 100, Term: 3, ConfState: raftpb.ConfState{ Voters: []uint64{2, 7, 8, 12}, }, }, }, }, { Type: raftpb.MsgApp, From: 6, To: 8, }, }, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { rn := &raftNode{ confState: tc.confState, } outputMessages := rn.processMessages(tc.InputMessages) require.Truef(t, reflect.DeepEqual(outputMessages, tc.ExpectedMessages), "Unexpected messages, expected: %v, got %v", tc.ExpectedMessages, outputMessages) }) } } ================================================ FILE: contrib/raftexample/raftexample_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "bytes" "fmt" "io" "net/http" "net/http/httptest" "os" "sync" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/raft/v3/raftpb" ) func getSnapshotFn() (func() ([]byte, error), <-chan struct{}) { snapshotTriggeredC := make(chan struct{}) return func() ([]byte, error) { snapshotTriggeredC <- struct{}{} return nil, nil }, snapshotTriggeredC } type cluster struct { peers []string commitC []<-chan *commit errorC []<-chan error proposeC []chan string confChangeC []chan raftpb.ConfChange snapshotTriggeredC []<-chan struct{} } // newCluster creates a cluster of n nodes func newCluster(n int) *cluster { peers := make([]string, n) for i := range peers { peers[i] = fmt.Sprintf("http://127.0.0.1:%d", 10000+i) } clus := &cluster{ peers: peers, commitC: make([]<-chan *commit, len(peers)), errorC: make([]<-chan error, len(peers)), proposeC: make([]chan string, len(peers)), confChangeC: make([]chan raftpb.ConfChange, len(peers)), snapshotTriggeredC: make([]<-chan struct{}, len(peers)), } for i := range clus.peers { os.RemoveAll(fmt.Sprintf("raftexample-%d", i+1)) os.RemoveAll(fmt.Sprintf("raftexample-%d-snap", i+1)) clus.proposeC[i] = make(chan string, 1) clus.confChangeC[i] = make(chan raftpb.ConfChange, 1) fn, snapshotTriggeredC := getSnapshotFn() clus.snapshotTriggeredC[i] = snapshotTriggeredC clus.commitC[i], clus.errorC[i], _ = newRaftNode(i+1, clus.peers, false, fn, clus.proposeC[i], clus.confChangeC[i]) } return clus } // Close closes all cluster nodes and returns an error if any failed. func (clus *cluster) Close() (err error) { for i := range clus.peers { go func(i int) { for range clus.commitC[i] { //revive:disable-line:empty-block // drain pending commits } }(i) close(clus.proposeC[i]) // wait for channel to close if erri := <-clus.errorC[i]; erri != nil { err = erri } // clean intermediates os.RemoveAll(fmt.Sprintf("raftexample-%d", i+1)) os.RemoveAll(fmt.Sprintf("raftexample-%d-snap", i+1)) } return err } func (clus *cluster) closeNoErrors(t *testing.T) { t.Log("closing cluster...") err := clus.Close() require.NoError(t, err) t.Log("closing cluster [done]") } // TestProposeOnCommit starts three nodes and feeds commits back into the proposal // channel. The intent is to ensure blocking on a proposal won't block raft progress. func TestProposeOnCommit(t *testing.T) { clus := newCluster(3) defer clus.closeNoErrors(t) donec := make(chan struct{}) for i := range clus.peers { // feedback for "n" committed entries, then update donec go func(pC chan<- string, cC <-chan *commit, eC <-chan error) { for n := 0; n < 100; n++ { c, ok := <-cC if !ok { pC = nil } select { case pC <- c.data[0]: continue case err := <-eC: t.Errorf("eC message (%v)", err) } } donec <- struct{}{} for range cC { //revive:disable-line:empty-block // acknowledge the commits from other nodes so // raft continues to make progress } }(clus.proposeC[i], clus.commitC[i], clus.errorC[i]) // one message feedback per node go func(i int) { clus.proposeC[i] <- "foo" }(i) } for range clus.peers { <-donec } } // TestCloseProposerBeforeReplay tests closing the producer before raft starts. func TestCloseProposerBeforeReplay(t *testing.T) { clus := newCluster(1) // close before replay so raft never starts defer clus.closeNoErrors(t) } // TestCloseProposerInflight tests closing the producer while // committed messages are being published to the client. func TestCloseProposerInflight(t *testing.T) { clus := newCluster(1) defer clus.closeNoErrors(t) var wg sync.WaitGroup // some inflight ops wg.Go(func() { clus.proposeC[0] <- "foo" clus.proposeC[0] <- "bar" }) // wait for one message if c, ok := <-clus.commitC[0]; !ok || c.data[0] != "foo" { t.Fatalf("Commit failed") } wg.Wait() } func TestPutAndGetKeyValue(t *testing.T) { clusters := []string{"http://127.0.0.1:9021"} proposeC := make(chan string) defer close(proposeC) confChangeC := make(chan raftpb.ConfChange) defer close(confChangeC) var kvs *kvstore getSnapshot := func() ([]byte, error) { return kvs.getSnapshot() } commitC, errorC, snapshotterReady := newRaftNode(1, clusters, false, getSnapshot, proposeC, confChangeC) kvs = newKVStore(<-snapshotterReady, proposeC, commitC, errorC) srv := httptest.NewServer(&httpKVAPI{ store: kvs, confChangeC: confChangeC, }) defer srv.Close() // wait server started <-time.After(time.Second * 3) wantKey, wantValue := "test-key", "test-value" url := fmt.Sprintf("%s/%s", srv.URL, wantKey) body := bytes.NewBufferString(wantValue) cli := srv.Client() req, err := http.NewRequest(http.MethodPut, url, body) require.NoError(t, err) req.Header.Set("Content-Type", "text/html; charset=utf-8") _, err = cli.Do(req) require.NoError(t, err) // wait for a moment for processing message, otherwise get would be failed. <-time.After(time.Second) resp, err := cli.Get(url) require.NoError(t, err) data, err := io.ReadAll(resp.Body) require.NoError(t, err) defer resp.Body.Close() gotValue := string(data) require.Equalf(t, wantValue, gotValue, "expect %s, got %s", wantValue, gotValue) } // TestAddNewNode tests adding new node to the existing cluster. func TestAddNewNode(t *testing.T) { clus := newCluster(3) defer clus.closeNoErrors(t) os.RemoveAll("raftexample-4") os.RemoveAll("raftexample-4-snap") defer func() { os.RemoveAll("raftexample-4") os.RemoveAll("raftexample-4-snap") }() newNodeURL := "http://127.0.0.1:10004" clus.confChangeC[0] <- raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: 4, Context: []byte(newNodeURL), } proposeC := make(chan string) defer close(proposeC) confChangeC := make(chan raftpb.ConfChange) defer close(confChangeC) newRaftNode(4, append(clus.peers, newNodeURL), true, nil, proposeC, confChangeC) go func() { proposeC <- "foo" }() if c, ok := <-clus.commitC[0]; !ok || c.data[0] != "foo" { t.Fatalf("Commit failed") } } func TestSnapshot(t *testing.T) { prevDefaultSnapshotCount := defaultSnapshotCount prevSnapshotCatchUpEntriesN := snapshotCatchUpEntriesN defaultSnapshotCount = 4 snapshotCatchUpEntriesN = 4 defer func() { defaultSnapshotCount = prevDefaultSnapshotCount snapshotCatchUpEntriesN = prevSnapshotCatchUpEntriesN }() clus := newCluster(3) defer clus.closeNoErrors(t) go func() { clus.proposeC[0] <- "foo" }() c := <-clus.commitC[0] select { case <-clus.snapshotTriggeredC[0]: t.Fatalf("snapshot triggered before applying done") default: } close(c.applyDoneC) <-clus.snapshotTriggeredC[0] } ================================================ FILE: contrib/systemd/etcd.service ================================================ [Unit] Description=etcd key-value store Documentation=https://github.com/etcd-io/etcd After=network-online.target local-fs.target remote-fs.target time-sync.target Wants=network-online.target local-fs.target remote-fs.target time-sync.target [Service] User=etcd Type=notify Environment=ETCD_DATA_DIR=/var/lib/etcd Environment=ETCD_NAME=%m ExecStart=/usr/bin/etcd Restart=always RestartSec=10s LimitNOFILE=40000 [Install] WantedBy=multi-user.target ================================================ FILE: contrib/systemd/etcd3-multinode/README.md ================================================ # etcd3 multi-node cluster Here's how to deploy etcd cluster with systemd. ## Set up data directory etcd needs data directory on host machine. Configure the data directory accessible to systemd as: ``` sudo mkdir -p /var/lib/etcd sudo chown -R root:$(whoami) /var/lib/etcd sudo chmod -R a+rw /var/lib/etcd ``` ## Write systemd service file In each machine, write etcd systemd service files: ``` cat > /tmp/my-etcd-1.service < /tmp/my-etcd-2.service < /tmp/my-etcd-3.service < etcdctl ======== `etcdctl` is a command line client for [etcd][etcd]. The v3 API is used by default on main branch. For the v2 API, make sure to set environment variable `ETCDCTL_API=2`. See also [READMEv2][READMEv2]. If using released versions earlier than v3.4, set `ETCDCTL_API=3` to use v3 API. Global flags (e.g., `dial-timeout`, `--cacert`, `--cert`, `--key`) can be set with environment variables: ``` ETCDCTL_DIAL_TIMEOUT=3s ETCDCTL_CACERT=/tmp/ca.pem ETCDCTL_CERT=/tmp/cert.pem ETCDCTL_KEY=/tmp/key.pem ``` Prefix flag strings with `ETCDCTL_`, convert all letters to upper-case, and replace dash(`-`) with underscore(`_`). Note that the environment variables with the prefix `ETCDCTL_` can only be used with the etcdctl global flags. Also, the environment variable `ETCDCTL_API` is a special case variable for etcdctl internal use only. ## Key-value commands ### PUT [options] \ \ PUT assigns the specified value with the specified key. If key already holds a value, it is overwritten. RPC: Put #### Options - lease -- lease ID (in hexadecimal) to attach to the key. - prev-kv -- return the previous key-value pair before modification. - ignore-value -- updates the key using its current value. - ignore-lease -- updates the key using its current lease. #### Output `OK` #### Examples ```bash ./etcdctl put foo bar --lease=1234abcd # OK ./etcdctl get foo # foo # bar ./etcdctl put foo --ignore-value # to detache lease # OK ``` ```bash ./etcdctl put foo bar --lease=1234abcd # OK ./etcdctl put foo bar1 --ignore-lease # to use existing lease 1234abcd # OK ./etcdctl get foo # foo # bar1 ``` ```bash ./etcdctl put foo bar1 --prev-kv # OK # foo # bar ./etcdctl get foo # foo # bar1 ``` #### Remarks If \ isn't given as command line argument, this command tries to read the value from standard input. When \ begins with '-', \ is interpreted as a flag. Insert '--' for workaround: ```bash ./etcdctl put -- ./etcdctl put -- ``` Providing \ in a new line after using `carriage return` is not supported and etcdctl may hang in that case. For example, following case is not supported: ```bash ./etcdctl put \r ``` A \ can have multiple lines or spaces but it must be provided with a double-quote as demonstrated below: ```bash ./etcdctl put foo "bar1 2 3" ``` ### GET [options] \ [range_end] GET gets the key or a range of keys [key, range_end) if range_end is given. RPC: Range #### Options - hex -- print out key and value as hex encode string - limit -- maximum number of results - prefix -- get keys by matching prefix - order -- order of results; ASCEND or DESCEND - sort-by -- sort target; CREATE, KEY, MODIFY, VALUE, or VERSION - rev -- specify the kv revision - print-value-only -- print only value when used with write-out=simple - consistency -- Linearizable(l) or Serializable(s), defaults to Linearizable(l). - from-key -- Get keys that are greater than or equal to the given key using byte compare - keys-only -- Get only the keys - max-create-revision -- restrict results to kvs with create revision lower or equal than the supplied revision - min-create-revision -- restrict results to kvs with create revision greater or equal than the supplied revision - max-mod-revision -- restrict results to kvs with modified revision lower or equal than the supplied revision - min-mod-revision -- restrict results to kvs with modified revision greater or equal than the supplied revision #### Output Prints the data in format below, ``` \\n\\n\\n\... ``` Note serializable requests are better for lower latency requirement, but stale data might be returned if serializable option (`--consistency=s`) is specified. #### Examples First, populate etcd with some keys: ```bash ./etcdctl put foo bar # OK ./etcdctl put foo1 bar1 # OK ./etcdctl put foo2 bar2 # OK ./etcdctl put foo3 bar3 # OK ``` Get the key named `foo`: ```bash ./etcdctl get foo # foo # bar ``` Get all keys: ```bash ./etcdctl get --from-key '' # foo # bar # foo1 # bar1 # foo2 # foo2 # foo3 # bar3 ``` Get all keys with names greater than or equal to `foo1`: ```bash ./etcdctl get --from-key foo1 # foo1 # bar1 # foo2 # bar2 # foo3 # bar3 ``` Get keys with names greater than or equal to `foo1` and less than `foo3`: ```bash ./etcdctl get foo1 foo3 # foo1 # bar1 # foo2 # bar2 ``` #### Remarks If any key or value contains non-printable characters or control characters, simple formatted output can be ambiguous due to new lines. To resolve this issue, set `--hex` to hex encode all strings. ### DEL [options] \ [range_end] Removes the specified key or range of keys [key, range_end) if range_end is given. RPC: DeleteRange #### Options - prefix -- delete keys by matching prefix - prev-kv -- return deleted key-value pairs - from-key -- delete keys that are greater than or equal to the given key using byte compare #### Output Prints the number of keys that were removed in decimal if DEL succeeded. #### Examples ```bash ./etcdctl put foo bar # OK ./etcdctl del foo # 1 ./etcdctl get foo ``` ```bash ./etcdctl put key val # OK ./etcdctl del --prev-kv key # 1 # key # val ./etcdctl get key ``` ```bash ./etcdctl put a 123 # OK ./etcdctl put b 456 # OK ./etcdctl put z 789 # OK ./etcdctl del --from-key a # 3 ./etcdctl get --from-key a ``` ```bash ./etcdctl put zoo val # OK ./etcdctl put zoo1 val1 # OK ./etcdctl put zoo2 val2 # OK ./etcdctl del --prefix zoo # 3 ./etcdctl get zoo2 ``` ### TXN [options] TXN reads multiple etcd requests from standard input and applies them as a single atomic transaction. A transaction consists of list of conditions, a list of requests to apply if all the conditions are true, and a list of requests to apply if any condition is false. RPC: Txn #### Options - hex -- print out keys and values as hex encoded strings. - interactive -- input transaction with interactive prompting. #### Input Format ```ebnf ::= * "\n" "\n" "\n" ::= (||||) "\n" ::= "<" | "=" | ">" := ("c"|"create")"("")" ::= ("m"|"mod")"("")" ::= ("val"|"value")"("")" ::= ("ver"|"version")"("")" ::= "lease("")" ::= * ::= * ::= ((see put, get, del etcdctl command syntax)) "\n" ::= (%q formatted string) ::= (%q formatted string) ::= "\""[0-9]+"\"" ::= "\""[0-9]+"\"" ::= "\""[0-9]+\"" ``` #### Output `SUCCESS` if etcd processed the transaction success list, `FAILURE` if etcd processed the transaction failure list. Prints the output for each command in the executed request list, each separated by a blank line. #### Examples txn in interactive mode: ```bash ./etcdctl txn -i # compares: mod("key1") > "0" # success requests (get, put, delete): put key1 "overwrote-key1" # failure requests (get, put, delete): put key1 "created-key1" put key2 "some extra key" # FAILURE # OK # OK ``` txn in non-interactive mode: ```bash ./etcdctl txn <<<'mod("key1") > "0" put key1 "overwrote-key1" put key1 "created-key1" put key2 "some extra key" ' # FAILURE # OK # OK ``` #### Remarks When using multi-line values within a TXN command, newlines must be represented as `\n`. Literal newlines will cause parsing failures. This differs from other commands (such as PUT) where the shell will convert literal newlines for us. For example: ```bash ./etcdctl txn <<<'mod("key1") > "0" put key1 "overwrote-key1" put key1 "created-key1" put key2 "this is\na multi-line\nvalue" ' # FAILURE # OK # OK ``` ### COMPACTION [options] \ COMPACTION discards all etcd event history prior to a given revision. Since etcd uses a multiversion concurrency control model, it preserves all key updates as event history. When the event history up to some revision is no longer needed, all superseded keys may be compacted away to reclaim storage space in the etcd backend database. RPC: Compact #### Options - physical -- 'true' to wait for compaction to physically remove all old revisions #### Output Prints the compacted revision. #### Example ```bash ./etcdctl compaction 1234 # compacted revision 1234 ``` ### WATCH [options] [key or prefix] [range_end] [--] [exec-command arg1 arg2 ...] Watch watches events stream on keys or prefixes, [key or prefix, range_end) if range_end is given. The watch command runs until it encounters an error or is terminated by the user. If range_end is given, it must be lexicographically greater than key or "\x00". RPC: Watch #### Options - hex -- print out key and value as hex encode string - interactive -- begins an interactive watch session - prefix -- watch on a prefix if prefix is set. - prev-kv -- get the previous key-value pair before the event happens. - rev -- the revision to start watching. Specifying a revision is useful for observing past events. #### Input format Input is only accepted for interactive mode. ``` watch [options] \n ``` #### Output \[\n\\n\]\n\\n\\n\\n\\n\\n... #### Examples ##### Non-interactive ```bash ./etcdctl watch foo # PUT # foo # bar ``` ```bash ETCDCTL_WATCH_KEY=foo ./etcdctl watch # PUT # foo # bar ``` Receive events and execute `echo watch event received`: ```bash ./etcdctl watch foo -- echo watch event received # PUT # foo # bar # watch event received ``` Watch response is set via `ETCD_WATCH_*` environmental variables: ```bash ./etcdctl watch foo -- sh -c "env | grep ETCD_WATCH_" # PUT # foo # bar # ETCD_WATCH_REVISION=11 # ETCD_WATCH_KEY="foo" # ETCD_WATCH_EVENT_TYPE="PUT" # ETCD_WATCH_VALUE="bar" ``` Watch with environmental variables and execute `echo watch event received`: ```bash export ETCDCTL_WATCH_KEY=foo ./etcdctl watch -- echo watch event received # PUT # foo # bar # watch event received ``` ```bash export ETCDCTL_WATCH_KEY=foo export ETCDCTL_WATCH_RANGE_END=foox ./etcdctl watch -- echo watch event received # PUT # fob # bar # watch event received ``` ##### Interactive ```bash ./etcdctl watch -i watch foo watch foo # PUT # foo # bar # PUT # foo # bar ``` Receive events and execute `echo watch event received`: ```bash ./etcdctl watch -i watch foo -- echo watch event received # PUT # foo # bar # watch event received ``` Watch with environmental variables and execute `echo watch event received`: ```bash export ETCDCTL_WATCH_KEY=foo ./etcdctl watch -i watch -- echo watch event received # PUT # foo # bar # watch event received ``` ```bash export ETCDCTL_WATCH_KEY=foo export ETCDCTL_WATCH_RANGE_END=foox ./etcdctl watch -i watch -- echo watch event received # PUT # fob # bar # watch event received ``` ### LEASE \ LEASE provides commands for key lease management. ### LEASE GRANT \ LEASE GRANT creates a fresh lease with a server-selected time-to-live in seconds greater than or equal to the requested TTL value. RPC: LeaseGrant #### Output Prints a message with the granted lease ID. #### Example ```bash ./etcdctl lease grant 60 # lease 32695410dcc0ca06 granted with TTL(60s) ``` ### LEASE REVOKE \ LEASE REVOKE destroys a given lease, deleting all attached keys. RPC: LeaseRevoke #### Output Prints a message indicating the lease is revoked. #### Example ```bash ./etcdctl lease revoke 32695410dcc0ca06 # lease 32695410dcc0ca06 revoked ``` ### LEASE TIMETOLIVE \ [options] LEASE TIMETOLIVE retrieves the lease information with the given lease ID. RPC: LeaseTimeToLive #### Options - keys -- Get keys attached to this lease #### Output Prints lease information. #### Example ```bash ./etcdctl lease grant 500 # lease 2d8257079fa1bc0c granted with TTL(500s) ./etcdctl put foo1 bar --lease=2d8257079fa1bc0c # OK ./etcdctl put foo2 bar --lease=2d8257079fa1bc0c # OK ./etcdctl lease timetolive 2d8257079fa1bc0c # lease 2d8257079fa1bc0c granted with TTL(500s), remaining(481s) ./etcdctl lease timetolive 2d8257079fa1bc0c --keys # lease 2d8257079fa1bc0c granted with TTL(500s), remaining(472s), attached keys([foo2 foo1]) ./etcdctl lease timetolive 2d8257079fa1bc0c --write-out=json # {"cluster_id":17186838941855831277,"member_id":4845372305070271874,"revision":3,"raft_term":2,"id":3279279168933706764,"ttl":465,"granted-ttl":500,"keys":null} ./etcdctl lease timetolive 2d8257079fa1bc0c --write-out=json --keys # {"cluster_id":17186838941855831277,"member_id":4845372305070271874,"revision":3,"raft_term":2,"id":3279279168933706764,"ttl":459,"granted-ttl":500,"keys":["Zm9vMQ==","Zm9vMg=="]} ./etcdctl lease timetolive 2d8257079fa1bc0c # lease 2d8257079fa1bc0c already expired ``` ### LEASE LIST LEASE LIST lists all active leases. RPC: LeaseLeases #### Output Prints a message with a list of active leases. #### Example ```bash ./etcdctl lease grant 60 # lease 32695410dcc0ca06 granted with TTL(60s) ./etcdctl lease list 32695410dcc0ca06 ``` ### LEASE KEEP-ALIVE \ LEASE KEEP-ALIVE periodically refreshes a lease so it does not expire. RPC: LeaseKeepAlive #### Output Prints a message for every keep alive sent or prints a message indicating the lease is gone. #### Example ```bash ./etcdctl lease keep-alive 32695410dcc0ca0 # lease 32695410dcc0ca0 keepalived with TTL(100) # lease 32695410dcc0ca0 keepalived with TTL(100) # lease 32695410dcc0ca0 keepalived with TTL(100) ... ``` ## Cluster maintenance commands ### MEMBER \ MEMBER provides commands for managing etcd cluster membership. ### MEMBER ADD \ [options] MEMBER ADD introduces a new member into the etcd cluster as a new peer. RPC: MemberAdd #### Options - peer-urls -- comma separated list of URLs to associate with the new member. #### Output Prints the member ID of the new member and the cluster ID. #### Example ```bash ./etcdctl member add newMember --peer-urls=https://127.0.0.1:12345 Member ced000fda4d05edf added to cluster 8c4281cc65c7b112 ETCD_NAME="newMember" ETCD_INITIAL_CLUSTER="newMember=https://127.0.0.1:12345,default=http://10.0.0.30:2380" ETCD_INITIAL_CLUSTER_STATE="existing" ``` ### MEMBER UPDATE \ [options] MEMBER UPDATE sets the peer URLs for an existing member in the etcd cluster. RPC: MemberUpdate #### Options - peer-urls -- comma separated list of URLs to associate with the updated member. #### Output Prints the member ID of the updated member and the cluster ID. #### Example ```bash ./etcdctl member update 2be1eb8f84b7f63e --peer-urls=https://127.0.0.1:11112 # Member 2be1eb8f84b7f63e updated in cluster ef37ad9dc622a7c4 ``` ### MEMBER REMOVE \ MEMBER REMOVE removes a member of an etcd cluster from participating in cluster consensus. RPC: MemberRemove #### Output Prints the member ID of the removed member and the cluster ID. #### Example ```bash ./etcdctl member remove 2be1eb8f84b7f63e # Member 2be1eb8f84b7f63e removed from cluster ef37ad9dc622a7c4 ``` ### MEMBER LIST MEMBER LIST prints the member details for all members associated with an etcd cluster. RPC: MemberList #### Options - consistency -- Linearizable(l) or Serializable(s), defaults to Linearizable(l). #### Output Prints a humanized table of the member IDs, statuses, names, peer addresses, and client addresses. Note serializable requests are better for lower latency requirement, but stale member list might be returned if serializable option (`--consistency=s`) is specified. In some situations users may want to use serializable requests. For example, when adding a new member to a one-node cluster, it's reasonable and safe to use serializable request before the new added member gets started. #### Examples ```bash ./etcdctl member list # 8211f1d0f64f3269, started, infra1, http://127.0.0.1:12380, http://127.0.0.1:2379 # 91bc3c398fb3c146, started, infra2, http://127.0.0.1:22380, http://127.0.0.1:22379 # fd422379fda50e48, started, infra3, http://127.0.0.1:32380, http://127.0.0.1:32379 ``` ```bash ./etcdctl -w json member list # {"header":{"cluster_id":17237436991929493444,"member_id":9372538179322589801,"raft_term":2},"members":[{"ID":9372538179322589801,"name":"infra1","peerURLs":["http://127.0.0.1:12380"],"clientURLs":["http://127.0.0.1:2379"]},{"ID":10501334649042878790,"name":"infra2","peerURLs":["http://127.0.0.1:22380"],"clientURLs":["http://127.0.0.1:22379"]},{"ID":18249187646912138824,"name":"infra3","peerURLs":["http://127.0.0.1:32380"],"clientURLs":["http://127.0.0.1:32379"]}]} ``` ```bash ./etcdctl -w table member list +------------------+---------+--------+------------------------+------------------------+ | ID | STATUS | NAME | PEER ADDRS | CLIENT ADDRS | +------------------+---------+--------+------------------------+------------------------+ | 8211f1d0f64f3269 | started | infra1 | http://127.0.0.1:12380 | http://127.0.0.1:2379 | | 91bc3c398fb3c146 | started | infra2 | http://127.0.0.1:22380 | http://127.0.0.1:22379 | | fd422379fda50e48 | started | infra3 | http://127.0.0.1:32380 | http://127.0.0.1:32379 | +------------------+---------+--------+------------------------+------------------------+ ``` ### ENDPOINT \ ENDPOINT provides commands for querying individual endpoints. #### Options - cluster -- fetch and use all endpoints from the etcd cluster member list ### ENDPOINT HEALTH ENDPOINT HEALTH checks the health of the list of endpoints with respect to cluster. An endpoint is unhealthy when it cannot participate in consensus with the rest of the cluster. #### Output If an endpoint can participate in consensus, prints a message indicating the endpoint is healthy. If an endpoint fails to participate in consensus, prints a message indicating the endpoint is unhealthy. #### Example Check the default endpoint's health: ```bash ./etcdctl endpoint health # 127.0.0.1:2379 is healthy: successfully committed proposal: took = 2.095242ms ``` Check all endpoints for the cluster associated with the default endpoint: ```bash ./etcdctl endpoint --cluster health # http://127.0.0.1:2379 is healthy: successfully committed proposal: took = 1.060091ms # http://127.0.0.1:22379 is healthy: successfully committed proposal: took = 903.138µs # http://127.0.0.1:32379 is healthy: successfully committed proposal: took = 1.113848ms ``` ### ENDPOINT STATUS ENDPOINT STATUS queries the status of each endpoint in the given endpoint list. #### Output ##### Simple format Prints a humanized table of each endpoint URL, ID, version, database size, leadership status, raft term, and raft status. ##### JSON format Prints a line of JSON encoding each endpoint URL, ID, version, database size, leadership status, raft term, and raft status. #### Examples Get the status for the default endpoint: ```bash ./etcdctl endpoint status # 127.0.0.1:2379, 8211f1d0f64f3269, 3.0.0, 25 kB, false, 2, 63 ``` Get the status for the default endpoint as JSON: ```bash ./etcdctl -w json endpoint status # [{"Endpoint":"127.0.0.1:2379","Status":{"header":{"cluster_id":17237436991929493444,"member_id":9372538179322589801,"revision":2,"raft_term":2},"version":"3.0.0","dbSize":24576,"leader":18249187646912138824,"raftIndex":32623,"raftTerm":2}}] ``` Get the status for all endpoints in the cluster associated with the default endpoint: ```bash ./etcdctl -w table endpoint --cluster status +------------------------+------------------+---------------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+ | ENDPOINT | ID | VERSION | STORAGE VERSION | DB SIZE | DB SIZE IN USE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | +------------------------+------------------+---------------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+ | http://127.0.0.1:2379 | 8211f1d0f64f3269 | 3.6.0-alpha.0 | 3.6.0 | 25 kB | 25 kB | false | false | 2 | 8 | 8 | | | http://127.0.0.1:22379 | 91bc3c398fb3c146 | 3.6.0-alpha.0 | 3.6.0 | 25 kB | 25 kB | true | false | 2 | 8 | 8 | | | http://127.0.0.1:32379 | fd422379fda50e48 | 3.6.0-alpha.0 | 3.6.0 | 25 kB | 25 kB | false | false | 2 | 8 | 8 | | +------------------------+------------------+---------------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+ ``` ### ENDPOINT HASHKV ENDPOINT HASHKV fetches the hash of the key-value store of an endpoint. #### Output ##### Simple format Prints a humanized table of each endpoint URL and KV history hash. ##### JSON format Prints a line of JSON encoding each endpoint URL and KV history hash. #### Examples Get the hash for the default endpoint: ```bash ./etcdctl endpoint hashkv --cluster http://127.0.0.1:2379, 2064120424, 13 http://127.0.0.1:22379, 2064120424, 13 http://127.0.0.1:32379, 2064120424, 13 ``` Get the status for the default endpoint as JSON: ```bash ./etcdctl endpoint hash --cluster -w json | jq [ { "Endpoint": "http://127.0.0.1:2379", "HashKV": { "header": { "cluster_id": 17237436991929494000, "member_id": 9372538179322590000, "revision": 13, "raft_term": 2 }, "hash": 2064120424, "compact_revision": -1, "hash_revision": 13 } }, { "Endpoint": "http://127.0.0.1:22379", "HashKV": { "header": { "cluster_id": 17237436991929494000, "member_id": 10501334649042878000, "revision": 13, "raft_term": 2 }, "hash": 2064120424, "compact_revision": -1, "hash_revision": 13 } }, { "Endpoint": "http://127.0.0.1:32379", "HashKV": { "header": { "cluster_id": 17237436991929494000, "member_id": 18249187646912140000, "revision": 13, "raft_term": 2 }, "hash": 2064120424, "compact_revision": -1, "hash_revision": 13 } } ] ``` Get the status for all endpoints in the cluster associated with the default endpoint: ```bash $ ./etcdctl endpoint hash --cluster -w table +------------------------+-----------+---------------+ | ENDPOINT | HASH | HASH REVISION | +------------------------+-----------+---------------+ | http://127.0.0.1:2379 | 784522900 | 16 | | http://127.0.0.1:22379 | 784522900 | 16 | | http://127.0.0.1:32379 | 784522900 | 16 | +------------------------+-----------+---------------+ ``` ### ALARM \ Provides alarm related commands ### ALARM DISARM `alarm disarm` Disarms all alarms RPC: Alarm #### Output `alarm:` if alarm is present and disarmed. #### Examples ```bash ./etcdctl alarm disarm ``` If NOSPACE alarm is present: ```bash ./etcdctl alarm disarm # alarm:NOSPACE ``` ### ALARM LIST `alarm list` lists all alarms. RPC: Alarm #### Output `alarm:` if alarm is present, empty string if no alarms present. #### Examples ```bash ./etcdctl alarm list ``` If NOSPACE alarm is present: ```bash ./etcdctl alarm list # alarm:NOSPACE ``` ### DEFRAG [options] DEFRAG defragments the backend database file for a set of given endpoints while etcd is running. When an etcd member reclaims storage space from deleted and compacted keys, the space is kept in a free list and the database file remains the same size. By defragmenting the database, the etcd member releases this free space back to the file system. **Note: to defragment offline (`--data-dir` flag), use: `etcutl defrag` instead** **Note that defragmentation to a live member blocks the system from reading and writing data while rebuilding its states.** **Note that defragmentation request does not get replicated over cluster. That is, the request is only applied to the local node. Specify all members in `--endpoints` flag or `--cluster` flag to automatically find all cluster members.** #### Output For each endpoints, prints a message indicating whether the endpoint was successfully defragmented. #### Example ```bash ./etcdctl --endpoints=localhost:2379,badendpoint:2379 defrag # Finished defragmenting etcd member[localhost:2379] # Failed to defragment etcd member[badendpoint:2379] (grpc: timed out trying to connect) ``` Run defragment operations for all endpoints in the cluster associated with the default endpoint: ```bash ./etcdctl defrag --cluster Finished defragmenting etcd member[http://127.0.0.1:2379] Finished defragmenting etcd member[http://127.0.0.1:22379] Finished defragmenting etcd member[http://127.0.0.1:32379] ``` #### Remarks DEFRAG returns a zero exit code only if it succeeded defragmenting all given endpoints. ### SNAPSHOT \ SNAPSHOT provides commands to restore a snapshot of a running etcd server into a fresh cluster. ### SNAPSHOT SAVE \ SNAPSHOT SAVE writes a point-in-time snapshot of the etcd backend database to a file. #### Output The backend snapshot is written to the given file path. #### Example Save a snapshot to "snapshot.db": ``` ./etcdctl snapshot save snapshot.db ``` ### SNAPSHOT RESTORE [options] \ Removed in v3.6. Use `etcdutl snapshot restore` instead. ### SNAPSHOT STATUS \ Removed in v3.6. Use `etcdutl snapshot status` instead. ### MOVE-LEADER \ MOVE-LEADER transfers leadership from the leader to another member in the cluster. #### Example ```bash # to choose transferee transferee_id=$(./etcdctl \ --endpoints localhost:2379,localhost:22379,localhost:32379 \ endpoint status | grep -m 1 "false" | awk -F', ' '{print $2}') echo ${transferee_id} # c89feb932daef420 # endpoints should include leader node ./etcdctl --endpoints ${transferee_ep} move-leader ${transferee_id} # Error: no leader endpoint given at [localhost:22379 localhost:32379] # request to leader with target node ID ./etcdctl --endpoints ${leader_ep} move-leader ${transferee_id} # Leadership transferred from 45ddc0e800e20b93 to c89feb932daef420 ``` ### DOWNGRADE \ NOTICE: Downgrades is an experimental feature in v3.6 and is not recommended for production clusters. Downgrade provides commands to downgrade cluster. Normally etcd members cannot be downgraded due to cluster version mechanism. After initial bootstrap, cluster members agree on the cluster version. Every 5 seconds, leader checks versions of all members and picks lowers minor version. New members will refuse joining cluster with cluster version newer than theirs, thus preventing cluster from downgrading. Downgrade commands allow cluster administrator to force cluster version to be lowered to previous minor version, thus allowing to downgrade the cluster. Downgrade should be executed in stages: 1. Verify that cluster is ready to be downgraded by running `etcdctl downgrade validate ` 2. Start the downgrade process by running `etcdctl downgrade enable ` 3. For each cluster member: 1. Ensure that member is ready for downgrade by confirming that it wrote `The server is ready to downgrade` log. 2. Replace member binary with one with older version. 3. Confirm that member has correctly started and joined the cluster. 4. Ensure that downgrade process has succeeded by checking leader log for `the cluster has been downgraded` Downgrade can be canceled by running `etcdctl downgrade cancel` command. In case of downgrade being canceled, cluster version will return to its normal behavior (pick the lowest member minor version). If no members were downgraded, cluster version will return to original value. If at least one member was downgraded, cluster version will stay at the `` until downgraded members are upgraded back. ### DOWNGRADE VALIDATE \ DOWNGRADE VALIDATE validate downgrade capability before starting downgrade. #### Example ```bash ./etcdctl downgrade validate 3.5 Downgrade validate success, cluster version 3.6 ./etcdctl downgrade validate 3.4 Error: etcdserver: invalid downgrade target version ``` ### DOWNGRADE ENABLE \ DOWNGRADE ENABLE starts a downgrade action to cluster. #### Example ```bash ./etcdctl downgrade enable 3.5 Downgrade enable success, cluster version 3.6 ``` ### DOWNGRADE CANCEL DOWNGRADE CANCEL cancels the ongoing downgrade action to cluster. #### Example ```bash ./etcdctl downgrade cancel Downgrade cancel success, cluster version 3.5 ``` ### DIAGNOSIS `etcdctl diagnosis [flags]` - Collects and analyzes troubleshooting data from a running etcd cluster. The `diagnosis` command gathers a concise set of diagnostic details from each cluster member by performing several checks, including: * **Membership checks**: Verifies the cluster membership information. * **Endpoint status**: Retrieves the status of each endpoint. * **Serializable and linearizable reads**: Performs read operations to validate data consistency. * **Metrics snapshot**: Collects a small snapshot of key metrics. #### Flags - `--cluster`: use all endpoints discovered from the cluster member list. - `--etcd-storage-quota-bytes`: expected etcd storage quota in bytes (value passed to etcd with `--quota-backend-bytes`). - `-o, --output`: optional file path to write the JSON report; by default the report is written to stdout. Logs are written to stderr. Global flags (like `--endpoints`, TLS, auth, and timeouts) are shared with other `etcdctl` commands. See `etcdctl options` for the full list. #### Examples To perform analysis of a running etcd cluster, you can use the following command. This will collect and analyze data from all specified endpoints. ```bash etcdctl diagnosis --endpoints=https://10.0.1.10:2379,https://10.0.1.11:2379,https://10.0.1.12:2379 \ --cacert ./ca.crt --key ./etcd-diagnosis.key --cert ./etcd-diagnosis.crt # Use cluster-discovered endpoints etcdctl diagnosis --cluster # Write report to a file (logs still go to stderr) etcdctl diagnosis -o report.json ``` Example output: see [ctlv3/command/diagnosis/examples/etcd_diagnosis_report.json](ctlv3/command/diagnosis/examples/etcd_diagnosis_report.json) ## Concurrency commands ### LOCK [options] \ [command arg1 arg2 ...] LOCK acquires a distributed mutex with a given name. Once the lock is acquired, it will be held until etcdctl is terminated. #### Options - ttl - time out in seconds of lock session. #### Output Once the lock is acquired but no command is given, the result for the GET on the unique lock holder key is displayed. If a command is given, it will be executed with environment variables `ETCD_LOCK_KEY` and `ETCD_LOCK_REV` set to the lock's holder key and revision. #### Example Acquire lock with standard output display: ```bash ./etcdctl lock mylock # mylock/1234534535445 ``` Acquire lock and execute `echo lock acquired`: ```bash ./etcdctl lock mylock echo lock acquired # lock acquired ``` Acquire lock and execute `etcdctl put` command ```bash ./etcdctl lock mylock ./etcdctl put foo bar # OK ``` #### Remarks LOCK returns a zero exit code only if it is terminated by a signal and releases the lock. If LOCK is abnormally terminated or fails to contact the cluster to release the lock, the lock will remain held until the lease expires. Progress may be delayed by up to the default lease length of 60 seconds. ### ELECT [options] \ [proposal] ELECT participates on a named election. A node announces its candidacy in the election by providing a proposal value. If a node wishes to observe the election, ELECT listens for new leaders values. Whenever a leader is elected, its proposal is given as output. #### Options - listen -- observe the election. #### Output - If a candidate, ELECT displays the GET on the leader key once the node is elected election. - If observing, ELECT streams the result for a GET on the leader key for the current election and all future elections. #### Example ```bash ./etcdctl elect myelection foo # myelection/1456952310051373265 # foo ``` #### Remarks ELECT returns a zero exit code only if it is terminated by a signal and can revoke its candidacy or leadership, if any. If a candidate is abnormally terminated, election progress may be delayed by up to the default lease length of 60 seconds. ## Authentication commands ### AUTH \ `auth enable` activates authentication on an etcd cluster and `auth disable` deactivates. When authentication is enabled, etcd checks all requests for appropriate authorization. RPC: AuthEnable/AuthDisable #### Output `Authentication Enabled`. #### Examples ```bash ./etcdctl user add root # Password of root:#type password for root # Type password of root again for confirmation:#re-type password for root # User root created ./etcdctl user grant-role root root # Role root is granted to user root ./etcdctl user get root # User: root # Roles: root ./etcdctl role add root # Role root created ./etcdctl role get root # Role root # KV Read: # KV Write: ./etcdctl auth enable # Authentication Enabled ``` ### ROLE \ ROLE is used to specify different roles which can be assigned to etcd user(s). ### ROLE ADD \ `role add` creates a role. RPC: RoleAdd #### Output `Role created`. #### Examples ```bash ./etcdctl --user=root:123 role add myrole # Role myrole created ``` ### ROLE GET \ `role get` lists detailed role information. RPC: RoleGet #### Output Detailed role information. #### Examples ```bash ./etcdctl --user=root:123 role get myrole # Role myrole # KV Read: # foo # KV Write: # foo ``` ### ROLE DELETE \ `role delete` deletes a role. RPC: RoleDelete #### Output `Role deleted`. #### Examples ```bash ./etcdctl --user=root:123 role delete myrole # Role myrole deleted ``` ### ROLE LIST \ `role list` lists all roles in etcd. RPC: RoleList #### Output A role per line. #### Examples ```bash ./etcdctl --user=root:123 role list # roleA # roleB # myrole ``` ### ROLE GRANT-PERMISSION [options] \ \ \ [endkey] `role grant-permission` grants a key to a role. RPC: RoleGrantPermission #### Options - from-key -- grant a permission of keys that are greater than or equal to the given key using byte compare - prefix -- grant a prefix permission #### Output `Role updated`. #### Examples Grant read and write permission on the key `foo` to role `myrole`: ```bash ./etcdctl --user=root:123 role grant-permission myrole readwrite foo # Role myrole updated ``` Grant read permission on the wildcard key pattern `foo/*` to role `myrole`: ```bash ./etcdctl --user=root:123 role grant-permission --prefix myrole readwrite foo/ # Role myrole updated ``` ### ROLE REVOKE-PERMISSION \ \ \ [endkey] `role revoke-permission` revokes a key from a role. RPC: RoleRevokePermission #### Options - from-key -- revoke a permission of keys that are greater than or equal to the given key using byte compare - prefix -- revoke a prefix permission #### Output `Permission of key is revoked from role ` for single key. `Permission of range [, ) is revoked from role ` for a key range. Exit code is zero. #### Examples ```bash ./etcdctl --user=root:123 role revoke-permission myrole foo # Permission of key foo is revoked from role myrole ``` ### USER \ USER provides commands for managing users of etcd. ### USER ADD \ [options] `user add` creates a user. RPC: UserAdd #### Options - interactive -- Read password from stdin instead of interactive terminal #### Output `User created`. #### Examples ```bash ./etcdctl --user=root:123 user add myuser # Password of myuser: #type password for my user # Type password of myuser again for confirmation:#re-type password for my user # User myuser created ``` ### USER GET \ [options] `user get` lists detailed user information. RPC: UserGet #### Options - detail -- Show permissions of roles granted to the user #### Output Detailed user information. #### Examples ```bash ./etcdctl --user=root:123 user get myuser # User: myuser # Roles: ``` ### USER DELETE \ `user delete` deletes a user. RPC: UserDelete #### Output `User deleted`. #### Examples ```bash ./etcdctl --user=root:123 user delete myuser # User myuser deleted ``` ### USER LIST `user list` lists detailed user information. RPC: UserList #### Output - List of users, one per line. #### Examples ```bash ./etcdctl --user=root:123 user list # user1 # user2 # myuser ``` ### USER PASSWD \ [options] `user passwd` changes a user's password. RPC: UserChangePassword #### Options - interactive -- if true, read password in interactive terminal #### Output `Password updated`. #### Examples ```bash ./etcdctl --user=root:123 user passwd myuser # Password of myuser: #type new password for my user # Type password of myuser again for confirmation: #re-type the new password for my user # Password updated ``` ### USER GRANT-ROLE \ \ `user grant-role` grants a role to a user RPC: UserGrantRole #### Output `Role is granted to user `. #### Examples ```bash ./etcdctl --user=root:123 user grant-role userA roleA # Role roleA is granted to user userA ``` ### USER REVOKE-ROLE \ \ `user revoke-role` revokes a role from a user RPC: UserRevokeRole #### Output `Role is revoked from user `. #### Examples ```bash ./etcdctl --user=root:123 user revoke-role userA roleA # Role roleA is revoked from user userA ``` ## Utility commands ### MAKE-MIRROR [options] \ [make-mirror][mirror] mirrors a key prefix in an etcd cluster to a destination etcd cluster. #### Options - dest-cacert -- TLS certificate authority file for destination cluster - dest-cert -- TLS certificate file for destination cluster - dest-key -- TLS key file for destination cluster - prefix -- The key-value prefix to mirror - dest-prefix -- The destination prefix to mirror a prefix to a different prefix in the destination cluster - no-dest-prefix -- Mirror key-values to the root of the destination cluster - dest-insecure-transport -- Disable transport security for client connections - max-txn-ops -- Maximum number of operations permitted in a transaction during syncing updates #### Output The approximate total number of keys transferred to the destination cluster, updated every 30 seconds. #### Examples ``` ./etcdctl make-mirror mirror.example.com:2379 # 10 # 18 ``` [mirror]: ./doc/mirror_maker.md ### VERSION Prints the version of etcdctl. #### Output Prints etcd version and API version. #### Examples ```bash ./etcdctl version # etcdctl version: 3.1.0-alpha.0+git # API version: 3.1 ``` ### CHECK \ CHECK provides commands for checking properties of the etcd cluster. ### CHECK PERF [options] CHECK PERF checks the performance of the etcd cluster for 60 seconds. Running the `check perf` often can create a large keyspace history which can be auto compacted and defragmented using the `--auto-compact` and `--auto-defrag` options as described below. Notice that different workload models use different configurations in terms of number of clients and throughput. Here is the configuration for each load: | Load | Number of clients | Number of put requests (requests/sec) | |---------|------|---------| | Small | 50 | 10000 | | Medium | 200 | 100000 | | Large | 500 | 1000000 | | xLarge | 1000 | 3000000 | The test checks for the following conditions: - The throughput should be at least 90% of the issued request - All the requests should be done in less than 500 ms - The standard deviation of the requests should be less than 100 ms Hence, a workload model may work while another one might fail. RPC: CheckPerf #### Options - load -- the performance check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge) - prefix -- the prefix for writing the performance check's keys. - auto-compact -- if true, compact storage with last revision after test is finished. - auto-defrag -- if true, defragment storage after test is finished. #### Output Prints the result of performance check on different criteria like throughput. Also prints an overall status of the check as pass or fail. #### Examples Shows examples of both, pass and fail, status. The failure is due to the fact that a large workload was tried on a single node etcd cluster running on a laptop environment created for development and testing purpose. ```bash ./etcdctl check perf --load="s" # 60 / 60 Booooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo! 100.00%1m0s # PASS: Throughput is 150 writes/s # PASS: Slowest request took 0.087509s # PASS: Stddev is 0.011084s # PASS ./etcdctl check perf --load="l" # 60 / 60 Booooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo! 100.00%1m0s # FAIL: Throughput too low: 6808 writes/s # PASS: Slowest request took 0.228191s # PASS: Stddev is 0.033547s # FAIL ``` ### CHECK DATASCALE [options] CHECK DATASCALE checks the memory usage of holding data for different workloads on a given server endpoint. Running the `check datascale` often can create a large keyspace history which can be auto compacted and defragmented using the `--auto-compact` and `--auto-defrag` options as described below. RPC: CheckDatascale #### Options - load -- the datascale check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge) - prefix -- the prefix for writing the datascale check's keys. - auto-compact -- if true, compact storage with last revision after test is finished. - auto-defrag -- if true, defragment storage after test is finished. #### Output Prints the system memory usage for a given workload. Also prints status of compact and defragment if related options are passed. #### Examples ```bash ./etcdctl check datascale --load="s" --auto-compact=true --auto-defrag=true # Start data scale check for work load [10000 key-value pairs, 1024 bytes per key-value, 50 concurrent clients]. # Compacting with revision 18346204 # Compacted with revision 18346204 # Defragmenting "127.0.0.1:2379" # Defragmented "127.0.0.1:2379" # PASS: Approximate system memory used : 64.30 MB. ``` ## Exit codes For all commands, a successful execution return a zero exit code. All failures will return non-zero exit codes. ## Output formats All commands accept an output format by setting `-w` or `--write-out`. All commands default to the "simple" output format, which is meant to be human-readable. The simple format is listed in each command's `Output` description since it is customized for each command. If a command has a corresponding RPC, it will respect all output formats. If a command fails, returning a non-zero exit code, an error string will be written to standard error regardless of output format. ### Simple A format meant to be easy to parse and human-readable. Specific to each command. ### JSON The JSON encoding of the command's [RPC response][etcdrpc]. Since etcd's RPCs use byte strings, the JSON output will encode keys and values in base64. Some commands without an RPC also support JSON; see the command's `Output` description. ### Protobuf The protobuf encoding of the command's [RPC response][etcdrpc]. If an RPC is streaming, the stream messages will be concetenated. If an RPC is not given for a command, the protobuf output is not defined. ### Fields An output format similar to JSON but meant to parse with coreutils. For an integer field named `Field`, it writes a line in the format `"Field" : %d` where `%d` is go's integer formatting. For byte array fields, it writes `"Field" : %q` where `%q` is go's quoted string formatting (e.g., `[]byte{'a', '\n'}` is written as `"a\n"`). ## Compatibility Support etcdctl is still in its early stage. We try out best to ensure fully compatible releases, however we might break compatibility to fix bugs or improve commands. If we intend to release a version of etcdctl with backward incompatibilities, we will provide notice prior to release and have instructions on how to upgrade. ### Input Compatibility Input includes the command name, its flags, and its arguments. We ensure backward compatibility of the input of normal commands in non-interactive mode. ### Output Compatibility Output includes output from etcdctl and its exit code. etcdctl provides `simple` output format by default. We ensure compatibility for the `simple` output format of normal commands in non-interactive mode. Currently, we do not ensure backward compatibility for `JSON` format and the format in non-interactive mode. Currently, we do not ensure backward compatibility of utility commands. ### TODO: compatibility with etcd server [etcd]: https://github.com/coreos/etcd [READMEv2]: READMEv2.md [etcdrpc]: ../api/etcdserverpb/rpc.proto ================================================ FILE: etcdctl/READMEv2.md ================================================ etcdctl ======== `etcdctl` is a command line client for [etcd][etcd]. It can be used in scripts or for administrators to explore an etcd cluster. ## Getting etcdctl The latest release is available as a binary at [Github][github-release] along with etcd. etcdctl can also be built from source using the build script found in the parent directory. ## Configuration ### --debug + output cURL commands which can be used to reproduce the request ### --no-sync + don't synchronize cluster information before sending request + Use this to access non-published client endpoints + Without this flag, values from `--endpoint` flag will be overwritten by etcd cluster when it does internal sync. ### --output, -o + output response in the given format (`simple`, `extended` or `json`) + default: `"simple"` ### --discovery-srv, -D + domain name to query for SRV records describing cluster endpoints + default: none + env variable: ETCDCTL_DISCOVERY_SRV ### --peers + a comma-delimited list of machine addresses in the cluster + default: `"http://127.0.0.1:2379"` + env variable: ETCDCTL_PEERS ### --endpoint + a comma-delimited list of machine addresses in the cluster + default: `"http://127.0.0.1:2379"` + env variable: ETCDCTL_ENDPOINT + Without `--no-sync` flag, this will be overwritten by etcd cluster when it does internal sync. ### --cert-file + identify HTTPS client using this SSL certificate file + default: none + env variable: ETCDCTL_CERT_FILE ### --key-file + identify HTTPS client using this SSL key file + default: none + env variable: ETCDCTL_KEY_FILE ### --ca-file + verify certificates of HTTPS-enabled servers using this CA bundle + default: none + env variable: ETCDCTL_CA_FILE ### --username, -u + provide username[:password] and prompt if password is not supplied + default: none + env variable: ETCDCTL_USERNAME ### --timeout + connection timeout per request + default: `"1s"` ### --total-timeout + timeout for the command execution (except watch) + default: `"5s"` ## Usage ### Setting Key Values Set a value on the `/foo/bar` key: ```sh $ etcdctl set /foo/bar "Hello world" Hello world ``` Set a value on the `/foo/bar` key with a value that expires in 60 seconds: ```sh $ etcdctl set /foo/bar "Hello world" --ttl 60 Hello world ``` Conditionally set a value on `/foo/bar` if the previous value was "Hello world": ```sh $ etcdctl set /foo/bar "Goodbye world" --swap-with-value "Hello world" Goodbye world ``` Conditionally set a value on `/foo/bar` if the previous etcd index was 12: ```sh $ etcdctl set /foo/bar "Goodbye world" --swap-with-index 12 Goodbye world ``` Create a new key `/foo/bar`, only if the key did not previously exist: ```sh $ etcdctl mk /foo/new_bar "Hello world" Hello world ``` Create a new in-order key under dir `/fooDir`: ```sh $ etcdctl mk --in-order /fooDir "Hello world" ``` Create a new dir `/fooDir`, only if the key did not previously exist: ```sh $ etcdctl mkdir /fooDir ``` Update an existing key `/foo/bar`, only if the key already existed: ```sh $ etcdctl update /foo/bar "Hola mundo" Hola mundo ``` Create or update a directory called `/mydir`: ```sh $ etcdctl setdir /mydir ``` ### Retrieving a key value Get the current value for a single key in the local etcd node: ```sh $ etcdctl get /foo/bar Hello world ``` Get the value of a key with additional metadata in a parseable format: ```sh $ etcdctl -o extended get /foo/bar Key: /foo/bar Modified-Index: 72 TTL: 0 Etcd-Index: 72 Raft-Index: 5611 Raft-Term: 1 Hello World ``` ### Listing a directory Explore the keyspace using the `ls` command ```sh $ etcdctl ls /akey /adir $ etcdctl ls /adir /adir/key1 /adir/key2 ``` Add `--recursive` to recursively list subdirectories encountered. ```sh $ etcdctl ls --recursive /akey /adir /adir/key1 /adir/key2 ``` Directories can also have a trailing `/` added to output using `-p`. ```sh $ etcdctl ls -p /akey /adir/ ``` ### Deleting a key Delete a key: ```sh $ etcdctl rm /foo/bar ``` Delete an empty directory or a key-value pair ```sh $ etcdctl rmdir /path/to/dir ``` or ```sh $ etcdctl rm /path/to/dir --dir ``` Recursively delete a key and all child keys: ```sh $ etcdctl rm /path/to/dir --recursive ``` Conditionally delete `/foo/bar` if the previous value was "Hello world": ```sh $ etcdctl rm /foo/bar --with-value "Hello world" ``` Conditionally delete `/foo/bar` if the previous etcd index was 12: ```sh $ etcdctl rm /foo/bar --with-index 12 ``` ### Watching for changes Watch for only the next change on a key: ```sh $ etcdctl watch /foo/bar Hello world ``` Continuously watch a key: ```sh $ etcdctl watch /foo/bar --forever Hello world .... client hangs forever until ctrl+C printing values as key change ``` Continuously watch a key, starting with a given etcd index: ```sh $ etcdctl watch /foo/bar --forever --index 12 Hello world .... client hangs forever until ctrl+C printing values as key change ``` Continuously watch a key and exec a program: ```sh $ etcdctl exec-watch /foo/bar -- sh -c "env | grep ETCD" ETCD_WATCH_ACTION=set ETCD_WATCH_VALUE=My configuration stuff ETCD_WATCH_MODIFIED_INDEX=1999 ETCD_WATCH_KEY=/foo/bar ETCD_WATCH_ACTION=set ETCD_WATCH_VALUE=My new configuration stuff ETCD_WATCH_MODIFIED_INDEX=2000 ETCD_WATCH_KEY=/foo/bar ``` Continuously and recursively watch a key and exec a program: ```sh $ etcdctl exec-watch --recursive /foo -- sh -c "env | grep ETCD" ETCD_WATCH_ACTION=set ETCD_WATCH_VALUE=My configuration stuff ETCD_WATCH_MODIFIED_INDEX=1999 ETCD_WATCH_KEY=/foo/bar ETCD_WATCH_ACTION=set ETCD_WATCH_VALUE=My new configuration stuff ETCD_WATCH_MODIFIED_INDEX=2000 ETCD_WATCH_KEY=/foo/barbar ``` ## Return Codes The following exit codes can be returned from etcdctl: ``` 0 Success 1 Malformed etcdctl arguments 2 Failed to connect to host 3 Failed to auth (client cert rejected, ca validation failure, etc) 4 400 error from etcd 5 500 error from etcd ``` ## Endpoint If the etcd cluster isn't available on `http://127.0.0.1:2379`, specify a `--endpoint` flag or `ETCDCTL_ENDPOINT` environment variable. One endpoint or a comma-separated list of endpoints can be listed. This option is ignored if the `--discovery-srv` option is provided. ```sh ETCDCTL_ENDPOINT="http://10.0.28.1:4002" etcdctl set my-key to-a-value ETCDCTL_ENDPOINT="http://10.0.28.1:4002,http://10.0.28.2:4002,http://10.0.28.3:4002" etcdctl set my-key to-a-value etcdctl --endpoint http://10.0.28.1:4002 my-key to-a-value etcdctl --endpoint http://10.0.28.1:4002,http://10.0.28.2:4002,http://10.0.28.3:4002 etcdctl set my-key to-a-value ``` ## Username and Password If the etcd cluster is protected by [authentication][authentication], specify username and password using the [`--username`][username-flag] or `ETCDCTL_USERNAME` environment variable. When `--username` flag or `ETCDCTL_USERNAME` environment variable doesn't contain password, etcdctl will prompt password in interactive mode. ```sh ETCDCTL_USERNAME="root:password" etcdctl set my-key to-a-value ``` ## DNS Discovery To discover the etcd cluster through domain SRV records, specify a `--discovery-srv` flag or `ETCDCTL_DISCOVERY_SRV` environment variable. This option takes precedence over the `--endpoint` flag. ```sh ETCDCTL_DISCOVERY_SRV="some-domain" etcdctl set my-key to-a-value etcdctl --discovery-srv some-domain set my-key to-a-value ``` ## Project Details ### Versioning etcdctl uses [semantic versioning][semver]. Releases will follow lockstep with the etcd release cycle. ### License etcdctl is under the Apache 2.0 license. See the [LICENSE][license] file for details. [authentication]: https://github.com/etcd-io/website/blob/main/content/docs/v2/authentication.md [etcd]: https://github.com/coreos/etcd [github-release]: https://github.com/coreos/etcd/releases/ [license]: ../LICENSE [semver]: http://semver.org/ [username-flag]: #--username--u ================================================ FILE: etcdctl/ctlv3/command/alarm_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "github.com/spf13/cobra" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) // NewAlarmCommand returns the cobra command for "alarm". func NewAlarmCommand() *cobra.Command { ac := &cobra.Command{ Use: "alarm ", Short: "Alarm related commands. Use `etcdctl alarm --help` to see subcommands", Long: "Alarm related commands", GroupID: groupClusterMaintenanceID, } ac.AddCommand(NewAlarmDisarmCommand()) ac.AddCommand(NewAlarmListCommand()) return ac } func NewAlarmDisarmCommand() *cobra.Command { cmd := cobra.Command{ Use: "disarm", Short: "Disarms all alarms", Run: alarmDisarmCommandFunc, } return &cmd } // alarmDisarmCommandFunc executes the "alarm disarm" command. func alarmDisarmCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("alarm disarm command accepts no arguments")) } ctx, cancel := commandCtx(cmd) resp, err := mustClientFromCmd(cmd).AlarmDisarm(ctx, &v3.AlarmMember{}) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.Alarm(*resp) } func NewAlarmListCommand() *cobra.Command { cmd := cobra.Command{ Use: "list", Short: "Lists all alarms", Run: alarmListCommandFunc, } return &cmd } // alarmListCommandFunc executes the "alarm list" command. func alarmListCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("alarm list command accepts no arguments")) } ctx, cancel := commandCtx(cmd) resp, err := mustClientFromCmd(cmd).AlarmList(ctx) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.Alarm(*resp) } ================================================ FILE: etcdctl/ctlv3/command/auth_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "errors" "fmt" "github.com/spf13/cobra" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/pkg/v3/cobrautl" ) // NewAuthCommand returns the cobra command for "auth". func NewAuthCommand() *cobra.Command { ac := &cobra.Command{ Use: "auth ", Short: "Enable or disable authentication. Use `etcdctl auth --help` to see subcommands", Long: "Enable or disable authentication", GroupID: groupAuthenticationID, } ac.AddCommand(newAuthEnableCommand()) ac.AddCommand(newAuthDisableCommand()) ac.AddCommand(newAuthStatusCommand()) return ac } func newAuthStatusCommand() *cobra.Command { return &cobra.Command{ Use: "status", Short: "Returns authentication status", Run: authStatusCommandFunc, } } // authStatusCommandFunc executes the "auth status" command. func authStatusCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("auth status command does not accept any arguments")) } ctx, cancel := commandCtx(cmd) result, err := mustClientFromCmd(cmd).Auth.AuthStatus(ctx) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.AuthStatus(*result) } func newAuthEnableCommand() *cobra.Command { return &cobra.Command{ Use: "enable", Short: "Enables authentication", Run: authEnableCommandFunc, } } // authEnableCommandFunc executes the "auth enable" command. func authEnableCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("auth enable command does not accept any arguments")) } ctx, cancel := commandCtx(cmd) cli := mustClientFromCmd(cmd) var err error for err == nil { if _, err = cli.AuthEnable(ctx); err == nil { break } if errors.Is(err, rpctypes.ErrRootRoleNotExist) { if _, err = cli.RoleAdd(ctx, "root"); err != nil { break } if _, err = cli.UserGrantRole(ctx, "root", "root"); err != nil { break } } } cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } fmt.Println("Authentication Enabled") } func newAuthDisableCommand() *cobra.Command { return &cobra.Command{ Use: "disable", Short: "Disables authentication", Run: authDisableCommandFunc, } } // authDisableCommandFunc executes the "auth disable" command. func authDisableCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("auth disable command does not accept any arguments")) } ctx, cancel := commandCtx(cmd) _, err := mustClientFromCmd(cmd).Auth.AuthDisable(ctx) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } fmt.Println("Authentication Disabled") } ================================================ FILE: etcdctl/ctlv3/command/check.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "context" "encoding/binary" "fmt" "math" "math/rand" "os" "os/signal" "strconv" "sync" "time" "github.com/cheggaaa/pb/v3" "github.com/spf13/cobra" "golang.org/x/time/rate" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" "go.etcd.io/etcd/pkg/v3/report" ) var ( checkPerfLoad string checkPerfPrefix string checkDatascaleLoad string checkDatascalePrefix string autoCompact bool autoDefrag bool ) type checkPerfCfg struct { limit int clients int duration int } var checkPerfCfgMap = map[string]checkPerfCfg{ // TODO: support read limit "s": { limit: 150, clients: 50, duration: 60, }, "m": { limit: 1000, clients: 200, duration: 60, }, "l": { limit: 8000, clients: 500, duration: 60, }, "xl": { limit: 15000, clients: 1000, duration: 60, }, } type checkDatascaleCfg struct { limit int kvSize int clients int } var checkDatascaleCfgMap = map[string]checkDatascaleCfg{ "s": { limit: 10000, kvSize: 1024, clients: 50, }, "m": { limit: 100000, kvSize: 1024, clients: 200, }, "l": { limit: 1000000, kvSize: 1024, clients: 500, }, "xl": { // xl tries to hit the upper bound aggressively which is 3 versions of 1M objects (3M in total) limit: 3000000, kvSize: 1024, clients: 1000, }, } // NewCheckCommand returns the cobra command for "check". func NewCheckCommand() *cobra.Command { cc := &cobra.Command{ Use: "check ", Short: "commands for checking properties of the etcd cluster. Use `etcdctl check --help` to see subcommands", Long: "commands for checking properties of the etcd cluster", GroupID: groupUtilityID, } cc.AddCommand(NewCheckPerfCommand()) cc.AddCommand(NewCheckDatascaleCommand()) return cc } // NewCheckPerfCommand returns the cobra command for "check perf". func NewCheckPerfCommand() *cobra.Command { cmd := &cobra.Command{ Use: "perf [options]", Short: "Check the performance of the etcd cluster", Run: newCheckPerfCommand, } // TODO: support customized configuration cmd.Flags().StringVar(&checkPerfLoad, "load", "s", "The performance check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge). Different workload models use different configurations in terms of number of clients and expected throughput.") cmd.Flags().StringVar(&checkPerfPrefix, "prefix", "/etcdctl-check-perf/", "The prefix for writing the performance check's keys.") cmd.Flags().BoolVar(&autoCompact, "auto-compact", false, "Compact storage with last revision after test is finished.") cmd.Flags().BoolVar(&autoDefrag, "auto-defrag", false, "Defragment storage after test is finished.") cmd.RegisterFlagCompletionFunc("load", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { return []string{"small", "medium", "large", "xLarge"}, cobra.ShellCompDirectiveDefault }) return cmd } // newCheckPerfCommand executes the "check perf" command. func newCheckPerfCommand(cmd *cobra.Command, args []string) { checkPerfAlias := map[string]string{ "s": "s", "small": "s", "m": "m", "medium": "m", "l": "l", "large": "l", "xl": "xl", "xLarge": "xl", } model, ok := checkPerfAlias[checkPerfLoad] if !ok { cobrautl.ExitWithError(cobrautl.ExitBadFeature, fmt.Errorf("unknown load option %v", checkPerfLoad)) } cfg := checkPerfCfgMap[model] requests := make(chan v3.Op, cfg.clients) limit := rate.NewLimiter(rate.Limit(cfg.limit), 1) cc := clientConfigFromCmd(cmd) clients := make([]*v3.Client, cfg.clients) for i := 0; i < cfg.clients; i++ { clients[i] = mustClient(cc) } ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.duration)*time.Second) defer cancel() ctx, icancel := interruptableContext(ctx, func() { attemptCleanup(clients[0], false) }) defer icancel() gctx, gcancel := context.WithCancel(ctx) resp, err := clients[0].Get(gctx, checkPerfPrefix, v3.WithPrefix(), v3.WithLimit(1)) gcancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } if len(resp.Kvs) > 0 { cobrautl.ExitWithError(cobrautl.ExitInvalidInput, fmt.Errorf("prefix %q has keys. Delete with 'etcdctl del --prefix %s' first", checkPerfPrefix, checkPerfPrefix)) } ksize, vsize := 256, 1024 k, v := make([]byte, ksize), string(make([]byte, vsize)) bar := pb.New(cfg.duration) bar.Start() r := report.NewReport("%4.4f", "", false) var wg sync.WaitGroup wg.Add(len(clients)) for i := range clients { go func(c *v3.Client) { defer wg.Done() for op := range requests { st := time.Now() _, derr := c.Do(context.Background(), op) r.Results() <- report.Result{Err: derr, Start: st, End: time.Now()} } }(clients[i]) } go func() { cctx, ccancel := context.WithCancel(ctx) defer ccancel() for limit.Wait(cctx) == nil { binary.PutVarint(k, rand.Int63n(math.MaxInt64)) requests <- v3.OpPut(checkPerfPrefix+string(k), v) } close(requests) }() go func() { for i := 0; i < cfg.duration; i++ { time.Sleep(time.Second) bar.Add(1) } bar.Finish() }() sc := r.Stats() wg.Wait() close(r.Results()) s := <-sc attemptCleanup(clients[0], autoCompact) if autoDefrag { for _, ep := range clients[0].Endpoints() { defrag(clients[0], ep) } } ok = true if len(s.ErrorDist) != 0 { fmt.Println("FAIL: too many errors") for k, v := range s.ErrorDist { fmt.Printf("FAIL: ERROR(%v) -> %d\n", k, v) } ok = false } if s.RPS/float64(cfg.limit) <= 0.9 { fmt.Printf("FAIL: Throughput too low: %d writes/s\n", int(s.RPS)+1) ok = false } else { fmt.Printf("PASS: Throughput is %d writes/s\n", int(s.RPS)+1) } if s.Slowest > 0.5 { // slowest request > 500ms fmt.Printf("Slowest request took too long: %fs\n", s.Slowest) ok = false } else { fmt.Printf("PASS: Slowest request took %fs\n", s.Slowest) } if s.Stddev > 0.1 { // stddev > 100ms fmt.Printf("Stddev too high: %fs\n", s.Stddev) ok = false } else { fmt.Printf("PASS: Stddev is %fs\n", s.Stddev) } if !ok { fmt.Println("FAIL") os.Exit(cobrautl.ExitError) } fmt.Println("PASS") } func attemptCleanup(client *v3.Client, autoCompact bool) { dctx, dcancel := context.WithTimeout(context.Background(), 30*time.Second) defer dcancel() dresp, err := client.Delete(dctx, checkPerfPrefix, v3.WithPrefix()) if err != nil { fmt.Printf("FAIL: Cleanup failed during key deletion: ERROR(%v)\n", err) return } if autoCompact { compact(client, dresp.Header.Revision) } } func interruptableContext(ctx context.Context, attemptCleanup func()) (context.Context, func()) { ctx, cancel := context.WithCancel(ctx) signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, os.Interrupt) go func() { defer signal.Stop(signalChan) select { case <-signalChan: cancel() attemptCleanup() } }() return ctx, cancel } // NewCheckDatascaleCommand returns the cobra command for "check datascale". func NewCheckDatascaleCommand() *cobra.Command { cmd := &cobra.Command{ Use: "datascale [options]", Short: "Check the memory usage of holding data for different workloads on a given server endpoint.", Long: "If no endpoint is provided, localhost will be used. If multiple endpoints are provided, first endpoint will be used.", Run: newCheckDatascaleCommand, } cmd.Flags().StringVar(&checkDatascaleLoad, "load", "s", "The datascale check's workload model. Accepted workloads: s(small), m(medium), l(large), xl(xLarge)") cmd.Flags().StringVar(&checkDatascalePrefix, "prefix", "/etcdctl-check-datascale/", "The prefix for writing the datascale check's keys.") cmd.Flags().BoolVar(&autoCompact, "auto-compact", false, "Compact storage with last revision after test is finished.") cmd.Flags().BoolVar(&autoDefrag, "auto-defrag", false, "Defragment storage after test is finished.") return cmd } // newCheckDatascaleCommand executes the "check datascale" command. func newCheckDatascaleCommand(cmd *cobra.Command, args []string) { checkDatascaleAlias := map[string]string{ "s": "s", "small": "s", "m": "m", "medium": "m", "l": "l", "large": "l", "xl": "xl", "xLarge": "xl", } model, ok := checkDatascaleAlias[checkDatascaleLoad] if !ok { cobrautl.ExitWithError(cobrautl.ExitBadFeature, fmt.Errorf("unknown load option %v", checkDatascaleLoad)) } cfg := checkDatascaleCfgMap[model] requests := make(chan v3.Op, cfg.clients) cc := clientConfigFromCmd(cmd) clients := make([]*v3.Client, cfg.clients) for i := 0; i < cfg.clients; i++ { clients[i] = mustClient(cc) } // get endpoints eps, errEndpoints := endpointsFromCmd(cmd) if errEndpoints != nil { cobrautl.ExitWithError(cobrautl.ExitError, errEndpoints) } sec := secureCfgFromCmd(cmd) ctx, cancel := context.WithCancel(context.Background()) resp, err := clients[0].Get(ctx, checkDatascalePrefix, v3.WithPrefix(), v3.WithLimit(1)) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } if len(resp.Kvs) > 0 { cobrautl.ExitWithError(cobrautl.ExitInvalidInput, fmt.Errorf("prefix %q has keys. Delete with etcdctl del --prefix %s first", checkDatascalePrefix, checkDatascalePrefix)) } ksize, vsize := 512, 512 k, v := make([]byte, ksize), string(make([]byte, vsize)) r := report.NewReport("%4.4f", "", false) var wg sync.WaitGroup wg.Add(len(clients)) // get the process_resident_memory_bytes and process_virtual_memory_bytes before the put operations bytesBefore := endpointMemoryMetrics(eps[0], sec) if bytesBefore == 0 { fmt.Println("FAIL: Could not read process_resident_memory_bytes before the put operations.") os.Exit(cobrautl.ExitError) } fmt.Printf("Start data scale check for work load [%v key-value pairs, %v bytes per key-value, %v concurrent clients].\n", cfg.limit, cfg.kvSize, cfg.clients) bar := pb.New(cfg.limit) bar.Start() for i := range clients { go func(c *v3.Client) { defer wg.Done() for op := range requests { st := time.Now() _, derr := c.Do(context.Background(), op) r.Results() <- report.Result{Err: derr, Start: st, End: time.Now()} bar.Increment() } }(clients[i]) } go func() { for i := 0; i < cfg.limit; i++ { binary.PutVarint(k, rand.Int63n(math.MaxInt64)) requests <- v3.OpPut(checkDatascalePrefix+string(k), v) } close(requests) }() sc := r.Stats() wg.Wait() close(r.Results()) bar.Finish() s := <-sc // get the process_resident_memory_bytes after the put operations bytesAfter := endpointMemoryMetrics(eps[0], sec) if bytesAfter == 0 { fmt.Println("FAIL: Could not read process_resident_memory_bytes after the put operations.") os.Exit(cobrautl.ExitError) } // delete the created kv pairs ctx, cancel = context.WithCancel(context.Background()) dresp, derr := clients[0].Delete(ctx, checkDatascalePrefix, v3.WithPrefix()) defer cancel() if derr != nil { cobrautl.ExitWithError(cobrautl.ExitError, derr) } if autoCompact { compact(clients[0], dresp.Header.Revision) } if autoDefrag { for _, ep := range clients[0].Endpoints() { defrag(clients[0], ep) } } if bytesAfter == 0 { fmt.Println("FAIL: Could not read process_resident_memory_bytes after the put operations.") os.Exit(cobrautl.ExitError) } bytesUsed := bytesAfter - bytesBefore mbUsed := bytesUsed / (1024 * 1024) if len(s.ErrorDist) != 0 { fmt.Println("FAIL: too many errors") for k, v := range s.ErrorDist { fmt.Printf("FAIL: ERROR(%v) -> %d\n", k, v) } os.Exit(cobrautl.ExitError) } fmt.Printf("PASS: Approximate system memory used : %v MB.\n", strconv.FormatFloat(mbUsed, 'f', 2, 64)) } ================================================ FILE: etcdctl/ctlv3/command/compaction_command.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "strconv" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var compactPhysical bool // NewCompactionCommand returns the cobra command for "compaction". func NewCompactionCommand() *cobra.Command { cmd := &cobra.Command{ Use: "compaction [options] ", Short: "Compacts the event history in etcd", Run: compactionCommandFunc, GroupID: groupKVID, } cmd.Flags().BoolVar(&compactPhysical, "physical", false, "'true' to wait for compaction to physically remove all old revisions") return cmd } // compactionCommandFunc executes the "compaction" command. func compactionCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("compaction command needs 1 argument")) } rev, err := strconv.ParseInt(args[0], 10, 64) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } var opts []clientv3.CompactOption if compactPhysical { opts = append(opts, clientv3.WithCompactPhysical()) } c := mustClientFromCmd(cmd) ctx, cancel := commandCtx(cmd) _, cerr := c.Compact(ctx, rev, opts...) cancel() if cerr != nil { cobrautl.ExitWithError(cobrautl.ExitError, cerr) } fmt.Println("compacted revision", rev) } ================================================ FILE: etcdctl/ctlv3/command/completion_command.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "os" "github.com/spf13/cobra" ) func NewCompletionCommand() *cobra.Command { cmd := &cobra.Command{ Use: "completion [bash|zsh|fish|powershell]", Short: "Generate completion script", Long: `To load completions: Bash: $ source <(etcdctl completion bash) # To load completions for each session, execute once: # Linux: $ etcdctl completion bash > /etc/bash_completion.d/etcdctl # macOS: $ etcdctl completion bash > /usr/local/etc/bash_completion.d/etcdctl Zsh: # If shell completion is not already enabled in your environment, # you will need to enable it. You can execute the following once: $ echo "autoload -U compinit; compinit" >> ~/.zshrc # To load completions for each session, execute once: $ etcdctl completion zsh > "${fpath[1]}/_etcdctl" # You will need to start a new shell for this setup to take effect. fish: $ etcdctl completion fish | source # To load completions for each session, execute once: $ etcdctl completion fish > ~/.config/fish/completions/etcdctl.fish PowerShell: PS> etcdctl completion powershell | Out-String | Invoke-Expression # To load completions for every new session, run: PS> etcdctl completion powershell > etcdctl.ps1 # and source this file from your PowerShell profile. `, DisableFlagsInUseLine: true, ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), Run: func(cmd *cobra.Command, args []string) { switch args[0] { case "bash": cmd.Root().GenBashCompletion(os.Stdout) case "zsh": cmd.Root().GenZshCompletion(os.Stdout) case "fish": cmd.Root().GenFishCompletion(os.Stdout, true) case "powershell": cmd.Root().GenPowerShellCompletionWithDesc(os.Stdout) } }, GroupID: groupUtilityID, } return cmd } ================================================ FILE: etcdctl/ctlv3/command/defrag_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "os" "time" "github.com/spf13/cobra" "go.etcd.io/etcd/pkg/v3/cobrautl" ) // NewDefragCommand returns the cobra command for "Defrag". func NewDefragCommand() *cobra.Command { cmd := &cobra.Command{ Use: "defrag", Short: "Defragments the storage of the etcd members with given endpoints", Run: defragCommandFunc, GroupID: groupClusterMaintenanceID, } cmd.PersistentFlags().BoolVar(&epClusterEndpoints, "cluster", false, "use all endpoints from the cluster member list") return cmd } func defragCommandFunc(cmd *cobra.Command, args []string) { failures := 0 cfg := clientConfigFromCmd(cmd) for _, ep := range endpointsFromCluster(cmd) { cfg.Endpoints = []string{ep} c := mustClient(cfg) ctx, cancel := commandCtx(cmd) start := time.Now() _, err := c.Defragment(ctx, ep) d := time.Since(start) cancel() if err != nil { fmt.Fprintf(os.Stderr, "Failed to defragment etcd member[%s]. took %s. (%v)\n", ep, d.String(), err) failures++ } else { fmt.Printf("Finished defragmenting etcd member[%s]. took %s\n", ep, d.String()) } c.Close() } if failures != 0 { os.Exit(cobrautl.ExitError) } } ================================================ FILE: etcdctl/ctlv3/command/del_command.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "os" "time" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var ( delPrefix bool delPrevKV bool delFromKey bool delRange bool ) // NewDelCommand returns the cobra command for "del". func NewDelCommand() *cobra.Command { cmd := &cobra.Command{ Use: "del [options] [range_end]", Short: "Removes the specified key or range of keys [key, range_end)", Run: delCommandFunc, GroupID: groupKVID, } cmd.Flags().BoolVar(&delPrefix, "prefix", false, "delete keys with matching prefix") cmd.Flags().BoolVar(&delPrevKV, "prev-kv", false, "return deleted key-value pairs") cmd.Flags().BoolVar(&delFromKey, "from-key", false, "delete keys that are greater than or equal to the given key using byte compare") cmd.Flags().BoolVar(&delRange, "range", false, "delete range of keys") return cmd } // delCommandFunc executes the "del" command. func delCommandFunc(cmd *cobra.Command, args []string) { key, opts := getDelOp(args) ctx, cancel := commandCtx(cmd) resp, err := mustClientFromCmd(cmd).Delete(ctx, key, opts...) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.Del(*resp) } func getDelOp(args []string) (string, []clientv3.OpOption) { if len(args) == 0 || len(args) > 2 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("del command needs one argument as key and an optional argument as range_end")) } if delPrefix && delFromKey { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("`--prefix` and `--from-key` cannot be set at the same time, choose one")) } var opts []clientv3.OpOption key := args[0] if len(args) > 1 { if delPrefix || delFromKey { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("too many arguments, only accept one argument when `--prefix` or `--from-key` is set")) } opts = append(opts, clientv3.WithRange(args[1])) if !delRange { fmt.Fprintf(os.Stderr, "Warning: Keys between %q and %q will be deleted. Please interrupt the command within next 2 seconds to cancel. "+ "You can provide `--range` flag to avoid the delay.\n", args[0], args[1]) time.Sleep(2 * time.Second) } } if delPrefix { if len(key) == 0 { key = "\x00" opts = append(opts, clientv3.WithFromKey()) } else { opts = append(opts, clientv3.WithPrefix()) } } if delPrevKV { opts = append(opts, clientv3.WithPrevKV()) } if delFromKey { if len(key) == 0 { key = "\x00" } opts = append(opts, clientv3.WithFromKey()) } return key, opts } ================================================ FILE: etcdctl/ctlv3/command/diagnosis/engine/diagnosis.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package engine import ( "encoding/json" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/engine/intf" ) type report struct { Input any `json:"input,omitempty"` Results []any `json:"results,omitempty"` } // Diagnose runs all provided plugins and returns a JSON report. // It logs plugin progress and individual results to stderr. func Diagnose(input any, plugins []intf.Plugin) ([]byte, error) { rp := report{ Input: input, } for _, plugin := range plugins { result := plugin.Diagnose() rp.Results = append(rp.Results, result) } return json.MarshalIndent(rp, "", "\t") } ================================================ FILE: etcdctl/ctlv3/command/diagnosis/engine/intf/plugin.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package intf type Plugin interface { // Name returns the name of the plugin Name() string // Diagnose performs diagnosis and returns the result. If it fails // to do the diagnosis for any reason, it gets the detailed reason // included in the diagnosis result. Diagnose() any } // FailedResult is the result returned by a plugin if it fails to // perform the diagnosis for any reason. type FailedResult struct { Name string `json:"name"` Reason string `json:"reason"` } ================================================ FILE: etcdctl/ctlv3/command/diagnosis/examples/etcd_diagnosis_report.json ================================================ { "input": { "endpoints": [ "http://127.0.0.1:2379" ], "useClusterEndpoints": true, "dial-timeout": 2000000000, "command-timeout": 5000000000, "keep-alive-time": 2000000000, "keep-alive-timeout": 5000000000, "insecure": true, "insecure-discovery": true, "db-quota-bytes": 2147483648 }, "results": [ { "name": "membershipChecker", "memberList": { "header": { "cluster_id": 17237436991929493444, "member_id": 9372538179322589801, "raft_term": 2 }, "members": [ { "ID": 9372538179322589801, "name": "infra1", "peerURLs": [ "http://127.0.0.1:12380" ], "clientURLs": [ "http://127.0.0.1:2379" ] }, { "ID": 10501334649042878790, "name": "infra2", "peerURLs": [ "http://127.0.0.1:22380" ], "clientURLs": [ "http://127.0.0.1:22379" ] }, { "ID": 18249187646912138824, "name": "infra3", "peerURLs": [ "http://127.0.0.1:32380" ], "clientURLs": [ "http://127.0.0.1:32379" ] } ] } }, { "name": "epStatusChecker", "summary": [ "Successful" ], "epStatusList": [ { "endpoint": "http://127.0.0.1:2379", "epStatus": { "header": { "cluster_id": 17237436991929493444, "member_id": 9372538179322589801, "revision": 1, "raft_term": 2 }, "version": "3.5.9", "dbSize": 98304, "leader": 18249187646912138824, "raftIndex": 8, "raftTerm": 2, "raftAppliedIndex": 8, "dbSizeInUse": 98304 } }, { "endpoint": "http://127.0.0.1:22379", "epStatus": { "header": { "cluster_id": 17237436991929493444, "member_id": 10501334649042878790, "revision": 1, "raft_term": 2 }, "version": "3.5.9", "dbSize": 98304, "leader": 18249187646912138824, "raftIndex": 8, "raftTerm": 2, "raftAppliedIndex": 8, "dbSizeInUse": 98304 } }, { "endpoint": "http://127.0.0.1:32379", "epStatus": { "header": { "cluster_id": 17237436991929493444, "member_id": 18249187646912138824, "revision": 1, "raft_term": 2 }, "version": "3.5.9", "dbSize": 98304, "leader": 18249187646912138824, "raftIndex": 8, "raftTerm": 2, "raftAppliedIndex": 8, "dbSizeInUse": 98304 } } ] }, { "name": "serializableReadChecker", "summary": "Successful", "readResponses": [ { "endpoint": "http://127.0.0.1:2379", "took": "686.5µs" }, { "endpoint": "http://127.0.0.1:22379", "took": "1.129291ms" }, { "endpoint": "http://127.0.0.1:32379", "took": "1.034625ms" } ] }, { "name": "linearizableReadChecker", "summary": "Successful", "readResponses": [ { "endpoint": "http://127.0.0.1:2379", "took": "1.286333ms" }, { "endpoint": "http://127.0.0.1:22379", "took": "890.417µs" }, { "endpoint": "http://127.0.0.1:32379", "took": "1.257791ms" } ] }, { "name": "metricsChecker", "summary": [ "Successful" ], "epMetricsList": [ { "endpoint": "http://127.0.0.1:2379", "took": "3.752625ms", "epMetrics": { "etcd_disk_backend_commit_duration_seconds_bucket": [ "etcd_disk_backend_commit_duration_seconds_bucket{le=\"0.001\"} 0" ], "etcd_disk_wal_fsync_duration_seconds_bucket": [ "etcd_disk_wal_fsync_duration_seconds_bucket{le=\"0.001\"} 0" ], "etcd_network_peer_round_trip_time_seconds_bucket": [ "etcd_network_peer_round_trip_time_seconds_bucket{To=\"91bc3c398fb3c146\",le=\"0.0001\"} 2" ], "process_resident_memory_bytes": null } } ] } ] } ================================================ FILE: etcdctl/ctlv3/command/diagnosis/plugins/common/checker.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "time" clientv3 "go.etcd.io/etcd/client/v3" ) // Checker carries shared configuration for diagnosis plugins. // It embeds generic options such as the etcd client configuration, // resolved endpoints, and command timeout. type Checker struct { Cfg *clientv3.ConfigSpec Endpoints []string CommandTimeout time.Duration DbQuotaBytes int64 Name string } ================================================ FILE: etcdctl/ctlv3/command/diagnosis/plugins/common/client.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/logutil" clientv3 "go.etcd.io/etcd/client/v3" ) // NewClient creates an etcd client from the given configuration spec. func NewClient(cfg *clientv3.ConfigSpec) (*clientv3.Client, error) { lg, _ := logutil.CreateDefaultZapLogger(zap.InfoLevel) cliCfg, err := clientv3.NewClientConfig(cfg, lg) if err != nil { return nil, err } return clientv3.New(*cliCfg) } // ConfigWithEndpoint returns a shallow copy of cfg with Endpoints set to the // provided single endpoint. func ConfigWithEndpoint(cfg *clientv3.ConfigSpec, ep string) *clientv3.ConfigSpec { c := *cfg c.Endpoints = []string{ep} return &c } ================================================ FILE: etcdctl/ctlv3/command/diagnosis/plugins/epstatus/plugin.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package epstatus import ( "context" "fmt" "log" "time" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/engine/intf" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/plugins/common" ) type epStatusChecker struct { common.Checker } type epStatus struct { Endpoint string `json:"endpoint,omitempty"` EpStatus *clientv3.StatusResponse `json:"epStatus,omitempty"` } type checkResult struct { Name string `json:"name,omitempty"` Summary []string `json:"summary,omitempty"` EpStatusList []epStatus `json:"epStatusList,omitempty"` } func NewPlugin(cfg *clientv3.ConfigSpec, eps []string, timeout time.Duration, dbQuota int64) intf.Plugin { return &epStatusChecker{ Checker: common.Checker{ Cfg: cfg, Endpoints: eps, CommandTimeout: timeout, DbQuotaBytes: dbQuota, Name: "epStatusChecker", }, } } func (ck *epStatusChecker) Name() string { return ck.Checker.Name } func (ck *epStatusChecker) Diagnose() (result any) { var err error eps := ck.Endpoints defer func() { if err != nil { result = &intf.FailedResult{ Name: ck.Name(), Reason: err.Error(), } } }() var ( maxRetries = 3 retries = 0 shouldRetry = true chkResult = initCheckResult(ck.Name(), len(eps)) ) for { for i, ep := range eps { chkResult.EpStatusList[i].Endpoint = ep cfg := common.ConfigWithEndpoint(ck.Cfg, ep) c, err := common.NewClient(cfg) if err != nil { appendSummary(&chkResult, "Failed to create client for %q: %v", ep, err) continue } ctx, cancel := context.WithTimeout(context.Background(), ck.CommandTimeout) chkResult.EpStatusList[i].EpStatus, err = c.Status(ctx, ep) cancel() c.Close() if err != nil { appendSummary(&chkResult, "Failed to get endpoint status from %q: %v", ep, err) continue } if len(chkResult.EpStatusList[i].EpStatus.Errors) > 0 { appendSummary(&chkResult, "Detected errors in endpoint %q: %v\n", ep, chkResult.EpStatusList[i].EpStatus.Errors) shouldRetry = false continue } if i > 0 { if !compareHardInfo(chkResult.EpStatusList[0].EpStatus, chkResult.EpStatusList[i].EpStatus) { appendSummary(&chkResult, "Detected inconsistent hard endpoint info between %q and %q\n", eps[0], eps[i]) shouldRetry = false } if !shouldRetry { continue } if !compareSoftInfo(chkResult.EpStatusList[0].EpStatus, chkResult.EpStatusList[i].EpStatus) { appendSummary(&chkResult, "Detected inconsistent soft endpoint info between %q and %q\n", eps[0], eps[i]) } } } retries++ if len(chkResult.Summary) == 0 || !shouldRetry || retries >= maxRetries { break } chkResult = initCheckResult(ck.Name(), len(eps)) log.Printf("Retrying checking endpoint status: %d/%d\n", retries, maxRetries) time.Sleep(time.Second) } checkDBSize(&chkResult, ck.DbQuotaBytes) if len(chkResult.Summary) == 0 { chkResult.Summary = []string{"Successful"} } result = chkResult return result } func initCheckResult(name string, epCount int) checkResult { return checkResult{ Name: name, Summary: []string{}, EpStatusList: make([]epStatus, epCount), } } func appendSummary(chkResult *checkResult, format string, v ...any) { errMsg := fmt.Sprintf(format, v...) log.Println(errMsg) chkResult.Summary = append(chkResult.Summary, errMsg) } func compareHardInfo(s1, s2 *clientv3.StatusResponse) bool { if s1 == nil || s2 == nil { return false } return s1.Header.ClusterId == s2.Header.ClusterId && s1.Version == s2.Version && s1.StorageVersion == s2.StorageVersion } func compareSoftInfo(s1, s2 *clientv3.StatusResponse) bool { if s1 == nil || s2 == nil { return false } return s1.Header.Revision == s2.Header.Revision && s1.RaftTerm == s2.RaftTerm && s1.RaftIndex == s2.RaftIndex && s1.RaftAppliedIndex == s2.RaftAppliedIndex && s1.Leader == s2.Leader } func checkDBSize(chkResult *checkResult, dbQuota int64) { for _, sts := range chkResult.EpStatusList { if sts.EpStatus == nil { continue } freeSize := sts.EpStatus.DbSize - sts.EpStatus.DbSizeInUse if freeSize > sts.EpStatus.DbSizeInUse && freeSize > 1_000_000_000 /* about 1GB */ || sts.EpStatus.DbSize >= dbQuota*80/100 { appendSummary(chkResult, "Detected large amount of db [free] space for endpoint %q, dbQuota: %d, dbSize: %d, dbSizeInUse: %d, dbSizeFree: %d", sts.Endpoint, dbQuota, sts.EpStatus.DbSize, sts.EpStatus.DbSizeInUse, freeSize) } } } ================================================ FILE: etcdctl/ctlv3/command/diagnosis/plugins/membership/plugin.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership import ( "context" "log" "reflect" "time" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/engine/intf" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/plugins/common" ) type membershipChecker struct { common.Checker } type checkResult struct { Name string `json:"name,omitempty"` Summary string `json:"summary,omitempty"` MemberList *clientv3.MemberListResponse `json:"memberList,omitempty"` AllMemberLists []*clientv3.MemberListResponse `json:"allMemberLists,omitempty"` } func NewPlugin(cfg *clientv3.ConfigSpec, eps []string, timeout time.Duration) intf.Plugin { return &membershipChecker{ Checker: common.Checker{ Cfg: cfg, Endpoints: eps, CommandTimeout: timeout, Name: "membershipChecker", }, } } func (ck *membershipChecker) Name() string { return ck.Checker.Name } func (ck *membershipChecker) Diagnose() (result any) { var err error eps := ck.Endpoints defer func() { if err != nil { result = &intf.FailedResult{ Name: ck.Name(), Reason: err.Error(), } } }() memberLists := make([]*clientv3.MemberListResponse, len(eps)) detectedInconsistency := false for i, ep := range eps { cfg := common.ConfigWithEndpoint(ck.Cfg, ep) c, err := common.NewClient(cfg) if err != nil { detectedInconsistency = true log.Printf("Failed to create client for %q: %v\n", ep, err) continue } ctx, cancel := context.WithTimeout(context.Background(), ck.CommandTimeout) memberLists[i], err = c.MemberList(ctx, clientv3.WithSerializable()) cancel() c.Close() if err != nil { detectedInconsistency = true log.Printf("Failed to get member list from %q: %v\n", ep, err) continue } if i > 0 { if !compareMembers(memberLists[0], memberLists[i]) { detectedInconsistency = true log.Printf("Detected inconsistent member list between %q and %q\n", eps[0], eps[i]) } } } if detectedInconsistency { result = checkResult{ Name: ck.Name(), Summary: "Detected inconsistent member list between different members", AllMemberLists: memberLists, } } else { result = checkResult{ Name: ck.Name(), Summary: "Successful", MemberList: memberLists[0], } } return result } func compareMembers(m1, m2 *clientv3.MemberListResponse) bool { if m1 == nil || m2 == nil { return false } return m1.Header.ClusterId == m2.Header.ClusterId && reflect.DeepEqual(m1.Members, m2.Members) } ================================================ FILE: etcdctl/ctlv3/command/diagnosis/plugins/metrics/plugin.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package metrics import ( "crypto/tls" "crypto/x509" "fmt" "io" "log" "net/http" "net/url" "os" "strings" "time" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/engine/intf" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/plugins/common" ) var metricsNames = []string{ "etcd_disk_wal_fsync_duration_seconds_bucket", "etcd_disk_backend_commit_duration_seconds_bucket", "etcd_network_peer_round_trip_time_seconds_bucket", "process_resident_memory_bytes", //"process_cpu_seconds_total", } type metricsChecker struct { common.Checker } type epMetrics struct { Endpoint string `json:"endpoint,omitempty"` Took string `json:"took,omitempty"` EpMetrics map[string][]string `json:"epMetrics,omitempty"` } type checkResult struct { Name string `json:"name,omitempty"` Summary []string `json:"summary,omitempty"` EpMetricsList []epMetrics `json:"epMetricsList,omitempty"` } func NewPlugin(cfg *clientv3.ConfigSpec, eps []string, timeout time.Duration) intf.Plugin { return &metricsChecker{ Checker: common.Checker{ Cfg: cfg, Endpoints: eps, CommandTimeout: timeout, Name: "metricsChecker", }, } } func (ck *metricsChecker) Name() string { return ck.Checker.Name } func (ck *metricsChecker) Diagnose() (result any) { var err error eps := ck.Endpoints defer func() { if err != nil { result = &intf.FailedResult{ Name: ck.Name(), Reason: err.Error(), } } }() chkResult := checkResult{ Name: ck.Name(), Summary: []string{}, EpMetricsList: make([]epMetrics, len(eps)), } for i, ep := range eps { chkResult.EpMetricsList[i].Endpoint = ep startTs := time.Now() allMetrics, err := fetchMetrics(ck.Cfg, ep, ck.CommandTimeout) chkResult.EpMetricsList[i].Took = time.Since(startTs).String() if err != nil { appendSummary(&chkResult, "Failed to get endpoint metrics from %q: %v", ep, err) continue } metricsMap := map[string][]string{} for _, prefix := range metricsNames { ret := metrics(allMetrics, prefix) metricsMap[prefix] = ret } chkResult.EpMetricsList[i].EpMetrics = metricsMap } if len(chkResult.Summary) == 0 { chkResult.Summary = []string{"Successful"} } result = chkResult return result } func metrics(lines []string, prefix string) []string { var ret []string for _, line := range lines { if strings.HasPrefix(line, prefix) { ret = append(ret, line) } } return ret } func appendSummary(chkResult *checkResult, format string, v ...any) { errMsg := fmt.Sprintf(format, v...) log.Println(errMsg) chkResult.Summary = append(chkResult.Summary, errMsg) } func fetchMetrics(cfg *clientv3.ConfigSpec, ep string, timeout time.Duration) ([]string, error) { if !strings.HasPrefix(ep, "http://") && !strings.HasPrefix(ep, "https://") { ep = "http://" + ep } urlPath, err := url.JoinPath(ep, "metrics") if err != nil { return nil, fmt.Errorf("failed to join metrics url path: %w", err) } client := &http.Client{Timeout: timeout} if strings.HasPrefix(urlPath, "https://") && cfg.Secure != nil { cert, certErr := tls.LoadX509KeyPair(cfg.Secure.Cert, cfg.Secure.Key) if certErr != nil { return nil, fmt.Errorf("failed to load certificate: %w", certErr) } caCert, caErr := os.ReadFile(cfg.Secure.Cacert) if caErr != nil { return nil, fmt.Errorf("failed to load CA: %w", caErr) } caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) tr := &http.Transport{ TLSClientConfig: &tls.Config{ Certificates: []tls.Certificate{cert}, RootCAs: caCertPool, InsecureSkipVerify: cfg.Secure.InsecureSkipVerify, }, } client.Transport = tr } resp, err := client.Get(urlPath) if err != nil { return nil, fmt.Errorf("http get failed: %w", err) } defer resp.Body.Close() data, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read metrics response: %w", err) } return strings.Split(string(data), "\n"), nil } ================================================ FILE: etcdctl/ctlv3/command/diagnosis/plugins/read/plugin.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package read import ( "context" "errors" "log" "time" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/engine/intf" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/plugins/common" ) type readChecker struct { common.Checker linearizable bool } type readResponse struct { Endpoint string `json:"endpoint,omitempty"` Took string `json:"took,omitempty"` Error string `json:"error,omitempty"` } type checkResult struct { Name string `json:"name,omitempty"` Summary string `json:"summary,omitempty"` ReadResponses []readResponse `json:"readResponses,omitempty"` } func NewPlugin(cfg *clientv3.ConfigSpec, eps []string, timeout time.Duration, linearizable bool) intf.Plugin { return &readChecker{ Checker: common.Checker{ Cfg: cfg, Endpoints: eps, CommandTimeout: timeout, Name: generateName(linearizable), }, linearizable: linearizable, } } func (ck *readChecker) Name() string { return ck.Checker.Name } func generateName(linearizable bool) string { if linearizable { return "linearizableReadChecker" } return "serializableReadChecker" } func (ck *readChecker) Diagnose() (result any) { var err error eps := ck.Endpoints defer func() { if err != nil { result = &intf.FailedResult{ Name: ck.Name(), Reason: err.Error(), } } }() var ( maxRetries = 3 retries = 0 chkResult = initCheckResult(ck.Name(), len(eps)) ) for { shouldRetry := false for i, ep := range eps { chkResult.ReadResponses[i].Endpoint = ep startTs := time.Now() cfg := common.ConfigWithEndpoint(ck.Cfg, ep) c, err := common.NewClient(cfg) if err != nil { chkResult.ReadResponses[i].Error = err.Error() shouldRetry = true continue } ctx, cancel := context.WithTimeout(context.Background(), ck.CommandTimeout) if ck.linearizable { _, err = c.Get(ctx, "health") } else { _, err = c.Get(ctx, "health", clientv3.WithSerializable()) } cancel() c.Close() if err != nil && !errors.Is(err, rpctypes.ErrPermissionDenied) { chkResult.ReadResponses[i].Error = err.Error() shouldRetry = true } chkResult.ReadResponses[i].Took = time.Since(startTs).String() } retries++ if !shouldRetry || retries >= maxRetries { break } chkResult = initCheckResult(ck.Name(), len(eps)) log.Printf("Retrying checking read: %d/%d\n", retries, maxRetries) time.Sleep(time.Second) } chkResult.Summary = "Successful" for _, resp := range chkResult.ReadResponses { if len(resp.Error) > 0 { chkResult.Summary = "Unsuccessful" break } } result = chkResult return result } func initCheckResult(name string, epCount int) checkResult { return checkResult{ Name: name, Summary: "", ReadResponses: make([]readResponse, epCount), } } ================================================ FILE: etcdctl/ctlv3/command/diagnosis_command.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "os" "github.com/spf13/cobra" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/engine" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/engine/intf" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/plugins/epstatus" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/plugins/membership" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/plugins/metrics" readplugin "go.etcd.io/etcd/etcdctl/v3/ctlv3/command/diagnosis/plugins/read" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var ( useCluster bool dbQuotaBytes int64 outputFile string ) // NewDiagnosisCommand returns the cobra command for "diagnosis". func NewDiagnosisCommand() *cobra.Command { cmd := &cobra.Command{ Use: "diagnosis", Short: "One-stop etcd diagnosis tool", Run: runDiagnosis, GroupID: groupClusterMaintenanceID, } cmd.Flags().BoolVar(&useCluster, "cluster", false, "use all endpoints from the cluster member list") cmd.Flags().Int64Var(&dbQuotaBytes, "etcd-storage-quota-bytes", 2*1024*1024*1024, "etcd storage quota in bytes (the value passed to etcd instance by flag --quota-backend-bytes)") cmd.Flags().StringVarP(&outputFile, "output", "o", "", "write report to file instead of stdout") return cmd } func runDiagnosis(cmd *cobra.Command, args []string) { cfg := clientConfigFromCmd(cmd) cli := mustClientFromCmd(cmd) defer cli.Close() eps := cfg.Endpoints if useCluster { ctx, cancel := commandCtx(cmd) members, err := cli.MemberList(ctx) cancel() if err != nil { fmt.Fprintf(os.Stderr, "failed to fetch member list: %v\n", err) os.Exit(cobrautl.ExitError) } var clusterEps []string for _, m := range members.Members { clusterEps = append(clusterEps, m.ClientURLs...) } eps = clusterEps cfg.Endpoints = eps } timeout, err := cmd.Flags().GetDuration("command-timeout") if err != nil { fmt.Fprintf(os.Stderr, "failed to get command-timeout: %v\n", err) os.Exit(cobrautl.ExitError) } plugins := []intf.Plugin{ membership.NewPlugin(cfg, eps, timeout), epstatus.NewPlugin(cfg, eps, timeout, dbQuotaBytes), readplugin.NewPlugin(cfg, eps, timeout, false), readplugin.NewPlugin(cfg, eps, timeout, true), metrics.NewPlugin(cfg, eps, timeout), } report, err := engine.Diagnose(cfg, plugins) if err != nil { fmt.Fprintf(os.Stderr, "diagnosis failed: %v\n", err) os.Exit(cobrautl.ExitError) } if outputFile != "" { if err := os.WriteFile(outputFile, report, 0o644); err != nil { fmt.Fprintf(os.Stderr, "failed to write report: %v\n", err) os.Exit(cobrautl.ExitError) } return } fmt.Fprintln(os.Stdout, string(report)) } ================================================ FILE: etcdctl/ctlv3/command/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package command is a set of libraries for etcd v3 commands. package command ================================================ FILE: etcdctl/ctlv3/command/downgrade_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "errors" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) // NewDowngradeCommand returns the cobra command for "downgrade". func NewDowngradeCommand() *cobra.Command { dc := &cobra.Command{ Use: "downgrade ", Short: "Downgrade related commands. Use `etcdctl downgrade --help` to see subcommands", Long: "Downgrade related commands", GroupID: groupClusterMaintenanceID, } dc.AddCommand(NewDowngradeValidateCommand()) dc.AddCommand(NewDowngradeEnableCommand()) dc.AddCommand(NewDowngradeCancelCommand()) return dc } // NewDowngradeValidateCommand returns the cobra command for "downgrade validate". func NewDowngradeValidateCommand() *cobra.Command { cc := &cobra.Command{ Use: "validate ", Short: "Validate downgrade capability before starting downgrade", Run: downgradeValidateCommandFunc, } return cc } // NewDowngradeEnableCommand returns the cobra command for "downgrade enable". func NewDowngradeEnableCommand() *cobra.Command { cc := &cobra.Command{ Use: "enable ", Short: "Start a downgrade action to cluster", Run: downgradeEnableCommandFunc, } return cc } // NewDowngradeCancelCommand returns the cobra command for "downgrade cancel". func NewDowngradeCancelCommand() *cobra.Command { cc := &cobra.Command{ Use: "cancel", Short: "Cancel the ongoing downgrade action to cluster", Run: downgradeCancelCommandFunc, } return cc } // downgradeValidateCommandFunc executes the "downgrade validate" command. func downgradeValidateCommandFunc(cmd *cobra.Command, args []string) { if len(args) < 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("TARGET_VERSION not provided")) } if len(args) > 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("too many arguments")) } targetVersion := args[0] if len(targetVersion) == 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("target version not provided")) } ctx, cancel := commandCtx(cmd) cli := mustClientFromCmd(cmd) resp, err := cli.Downgrade(ctx, clientv3.DowngradeValidate, targetVersion) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.DowngradeValidate(*resp) } // downgradeEnableCommandFunc executes the "downgrade enable" command. func downgradeEnableCommandFunc(cmd *cobra.Command, args []string) { if len(args) < 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("TARGET_VERSION not provided")) } if len(args) > 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("too many arguments")) } targetVersion := args[0] if len(targetVersion) == 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("target version not provided")) } ctx, cancel := commandCtx(cmd) cli := mustClientFromCmd(cmd) resp, err := cli.Downgrade(ctx, clientv3.DowngradeEnable, targetVersion) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.DowngradeEnable(*resp) } // downgradeCancelCommandFunc executes the "downgrade cancel" command. func downgradeCancelCommandFunc(cmd *cobra.Command, args []string) { ctx, cancel := commandCtx(cmd) cli := mustClientFromCmd(cmd) resp, err := cli.Downgrade(ctx, clientv3.DowngradeCancel, "") cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.DowngradeCancel(*resp) } ================================================ FILE: etcdctl/ctlv3/command/elect_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "context" "errors" "os" "os/signal" "syscall" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var electListen bool // NewElectCommand returns the cobra command for "elect". func NewElectCommand() *cobra.Command { cmd := &cobra.Command{ Use: "elect [proposal]", Short: "Observes and participates in leader election", Run: electCommandFunc, GroupID: groupConcurrencyID, } cmd.Flags().BoolVarP(&electListen, "listen", "l", false, "observation mode") return cmd } func electCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 && len(args) != 2 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("elect takes one election name argument and an optional proposal argument")) } c := mustClientFromCmd(cmd) var err error if len(args) == 1 { if !electListen { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("no proposal argument but -l not set")) } err = observe(c, args[0]) } else { if electListen { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("proposal given but -l is set")) } err = campaign(c, args[0], args[1]) } if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } } func observe(c *clientv3.Client, election string) error { s, err := concurrency.NewSession(c) if err != nil { return err } e := concurrency.NewElection(s, election) ctx, cancel := context.WithCancel(context.TODO()) donec := make(chan struct{}) sigc := make(chan os.Signal, 1) signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) go func() { <-sigc cancel() }() go func() { for resp := range e.Observe(ctx) { display.Get(resp) } close(donec) }() <-donec select { case <-ctx.Done(): default: return errors.New("elect: observer lost") } return nil } func campaign(c *clientv3.Client, election string, prop string) error { s, err := concurrency.NewSession(c) if err != nil { return err } e := concurrency.NewElection(s, election) ctx, cancel := context.WithCancel(context.TODO()) donec := make(chan struct{}) sigc := make(chan os.Signal, 1) signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) go func() { <-sigc cancel() close(donec) }() if err = e.Campaign(ctx, prop); err != nil { return err } // print key since elected resp, err := c.Get(ctx, e.Key()) if err != nil { return err } display.Get(*resp) select { case <-donec: case <-s.Done(): return errors.New("elect: session expired") } return e.Resign(context.TODO()) } ================================================ FILE: etcdctl/ctlv3/command/ep_command.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "errors" "fmt" "os" "sync" "time" "github.com/spf13/cobra" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/logutil" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var ( epClusterEndpoints bool epHashKVRev int64 ) // NewEndpointCommand returns the cobra command for "endpoint". func NewEndpointCommand() *cobra.Command { ec := &cobra.Command{ Use: "endpoint ", Short: "Endpoint related commands. Use `etcdctl endpoint --help` to see subcommands", Long: "Endpoint related commands", GroupID: groupClusterMaintenanceID, } ec.PersistentFlags().BoolVar(&epClusterEndpoints, "cluster", false, "use all endpoints from the cluster member list") ec.AddCommand(newEpHealthCommand()) ec.AddCommand(newEpStatusCommand()) ec.AddCommand(newEpHashKVCommand()) return ec } func newEpHealthCommand() *cobra.Command { cmd := &cobra.Command{ Use: "health", Short: "Checks the healthiness of endpoints specified in `--endpoints` flag", Run: epHealthCommandFunc, } return cmd } func newEpStatusCommand() *cobra.Command { return &cobra.Command{ Use: "status", Short: "Prints out the status of endpoints specified in `--endpoints` flag", Long: `When --write-out is set to simple, this command prints out comma-separated status lists for each endpoint. The items in the lists are endpoint, ID, version, db size, is leader, is learner, raft term, raft index, raft applied index, errors. `, Run: epStatusCommandFunc, } } func newEpHashKVCommand() *cobra.Command { hc := &cobra.Command{ Use: "hashkv", Short: "Prints the KV history hash for each endpoint in --endpoints", Run: epHashKVCommandFunc, } hc.PersistentFlags().Int64Var(&epHashKVRev, "rev", 0, "maximum revision to hash (default: latest revision)") return hc } type epHealth struct { Ep string `json:"endpoint"` Health bool `json:"health"` Took string `json:"took"` Error string `json:"error,omitempty"` } // epHealthCommandFunc executes the "endpoint-health" command. func epHealthCommandFunc(cmd *cobra.Command, args []string) { lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } cfgSpec := clientConfigFromCmd(cmd) var cfgs []*clientv3.Config for _, ep := range endpointsFromCluster(cmd) { cloneCfgSpec := cfgSpec.Clone() cloneCfgSpec.Endpoints = []string{ep} cfg, err := clientv3.NewClientConfig(cloneCfgSpec, lg) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } cfgs = append(cfgs, cfg) } var wg sync.WaitGroup hch := make(chan epHealth, len(cfgs)) for _, cfg := range cfgs { wg.Add(1) go func(cfg *clientv3.Config) { defer wg.Done() ep := cfg.Endpoints[0] cfg.Logger = lg.Named("client") cli, err := clientv3.New(*cfg) if err != nil { hch <- epHealth{Ep: ep, Health: false, Error: err.Error()} return } st := time.Now() // get a random key. As long as we can get the response without an error, the // endpoint is health. ctx, cancel := commandCtx(cmd) _, err = cli.Get(ctx, "health") eh := epHealth{Ep: ep, Health: false, Took: time.Since(st).String()} // permission denied is OK since proposal goes through consensus to get it if err == nil || errors.Is(err, rpctypes.ErrPermissionDenied) { eh.Health = true } else { eh.Error = err.Error() } if eh.Health { resp, err := cli.AlarmList(ctx) if err == nil && len(resp.Alarms) > 0 { eh.Health = false eh.Error = "Active Alarm(s): " for _, v := range resp.Alarms { switch v.Alarm { case etcdserverpb.AlarmType_NOSPACE: eh.Error = eh.Error + "NOSPACE " case etcdserverpb.AlarmType_CORRUPT: eh.Error = eh.Error + "CORRUPT " default: eh.Error = eh.Error + "UNKNOWN " } } } else if err != nil { eh.Health = false eh.Error = "Unable to fetch the alarm list" } } cancel() hch <- eh }(cfg) } wg.Wait() close(hch) errs := false var healthList []epHealth for h := range hch { healthList = append(healthList, h) if h.Error != "" { errs = true } } display.EndpointHealth(healthList) if errs { cobrautl.ExitWithError(cobrautl.ExitError, fmt.Errorf("unhealthy cluster")) } } type epStatus struct { Ep string `json:"Endpoint"` Resp *clientv3.StatusResponse `json:"Status"` } func epStatusCommandFunc(cmd *cobra.Command, args []string) { cfg := clientConfigFromCmd(cmd) var statusList []epStatus var err error for _, ep := range endpointsFromCluster(cmd) { cfg.Endpoints = []string{ep} c := mustClient(cfg) ctx, cancel := commandCtx(cmd) resp, serr := c.Status(ctx, ep) cancel() c.Close() if serr != nil { err = serr fmt.Fprintf(os.Stderr, "Failed to get the status of endpoint %s (%v)\n", ep, serr) continue } statusList = append(statusList, epStatus{Ep: ep, Resp: resp}) } display.EndpointStatus(statusList) if err != nil { os.Exit(cobrautl.ExitError) } } type epHashKV struct { Ep string `json:"Endpoint"` Resp *clientv3.HashKVResponse `json:"HashKV"` } func epHashKVCommandFunc(cmd *cobra.Command, args []string) { cfg := clientConfigFromCmd(cmd) var hashList []epHashKV var err error for _, ep := range endpointsFromCluster(cmd) { cfg.Endpoints = []string{ep} c := mustClient(cfg) ctx, cancel := commandCtx(cmd) resp, serr := c.HashKV(ctx, ep, epHashKVRev) cancel() c.Close() if serr != nil { err = serr fmt.Fprintf(os.Stderr, "Failed to get the hash of endpoint %s (%v)\n", ep, serr) continue } hashList = append(hashList, epHashKV{Ep: ep, Resp: resp}) } display.EndpointHashKV(hashList) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } } func endpointsFromCluster(cmd *cobra.Command) []string { if !epClusterEndpoints { endpoints, err := cmd.Flags().GetStringSlice("endpoints") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } return endpoints } sec := secureCfgFromCmd(cmd) dt := dialTimeoutFromCmd(cmd) ka := keepAliveTimeFromCmd(cmd) kat := keepAliveTimeoutFromCmd(cmd) eps, err := endpointsFromCmd(cmd) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } // exclude auth for not asking needless password (MemberList() doesn't need authentication) lg, _ := logutil.CreateDefaultZapLogger(zap.InfoLevel) cfg, err := clientv3.NewClientConfig(&clientv3.ConfigSpec{ Endpoints: eps, DialTimeout: dt, KeepAliveTime: ka, KeepAliveTimeout: kat, Secure: sec, }, lg) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } c, err := clientv3.New(*cfg) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } ctx, cancel := commandCtx(cmd) defer func() { c.Close() cancel() }() membs, err := c.MemberList(ctx) if err != nil { err = fmt.Errorf("failed to fetch endpoints from etcd cluster member list: %w", err) cobrautl.ExitWithError(cobrautl.ExitError, err) } var ret []string for _, m := range membs.Members { ret = append(ret, m.ClientURLs...) } return ret } ================================================ FILE: etcdctl/ctlv3/command/get_command.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "strings" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var ( getConsistency string getLimit int64 getSortOrder string getSortTarget string getPrefix bool getFromKey bool getRev int64 getKeysOnly bool getCountOnly bool printValueOnly bool getMinCreateRev int64 getMaxCreateRev int64 getMinModRev int64 getMaxModRev int64 ) // NewGetCommand returns the cobra command for "get". func NewGetCommand() *cobra.Command { cmd := &cobra.Command{ Use: "get [options] [range_end]", Short: "Gets the key or a range of keys", Run: getCommandFunc, GroupID: groupKVID, } cmd.Flags().StringVar(&getConsistency, "consistency", "l", "Linearizable(l) or Serializable(s)") cmd.Flags().StringVar(&getSortOrder, "order", "", "Order of results; ASCEND or DESCEND (ASCEND by default)") cmd.Flags().StringVar(&getSortTarget, "sort-by", "", "Sort target; CREATE, KEY, MODIFY, VALUE, or VERSION") cmd.Flags().Int64Var(&getLimit, "limit", 0, "Maximum number of results") cmd.Flags().BoolVar(&getPrefix, "prefix", false, "Get keys with matching prefix") cmd.Flags().BoolVar(&getFromKey, "from-key", false, "Get keys that are greater than or equal to the given key using byte compare") cmd.Flags().Int64Var(&getRev, "rev", 0, "Specify the kv revision") cmd.Flags().BoolVar(&getKeysOnly, "keys-only", false, "Get only the keys") cmd.Flags().BoolVar(&getCountOnly, "count-only", false, "Get only the count") cmd.Flags().BoolVar(&printValueOnly, "print-value-only", false, `Only write values when using the "simple" output format`) cmd.Flags().Int64Var(&getMinCreateRev, "min-create-rev", 0, "Minimum create revision") cmd.Flags().Int64Var(&getMaxCreateRev, "max-create-rev", 0, "Maximum create revision") cmd.Flags().Int64Var(&getMinModRev, "min-mod-rev", 0, "Minimum modification revision") cmd.Flags().Int64Var(&getMaxModRev, "max-mod-rev", 0, "Maximum modification revision") cmd.RegisterFlagCompletionFunc("consistency", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { return []string{"l", "s"}, cobra.ShellCompDirectiveDefault }) cmd.RegisterFlagCompletionFunc("order", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { return []string{"ASCEND", "DESCEND"}, cobra.ShellCompDirectiveDefault }) cmd.RegisterFlagCompletionFunc("sort-by", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { return []string{"CREATE", "KEY", "MODIFY", "VALUE", "VERSION"}, cobra.ShellCompDirectiveDefault }) return cmd } // getCommandFunc executes the "get" command. func getCommandFunc(cmd *cobra.Command, args []string) { key, opts := getGetOp(args) ctx, cancel := commandCtx(cmd) resp, err := mustClientFromCmd(cmd).Get(ctx, key, opts...) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } if getCountOnly { if _, fields := display.(*fieldsPrinter); !fields { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("--count-only is only for `--write-out=fields`")) } } if printValueOnly { dp, simple := (display).(*simplePrinter) if !simple { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("print-value-only is only for `--write-out=simple`")) } dp.valueOnly = true } display.Get(*resp) } func getGetOp(args []string) (string, []clientv3.OpOption) { if len(args) == 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("get command needs one argument as key and an optional argument as range_end")) } if getPrefix && getFromKey { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("`--prefix` and `--from-key` cannot be set at the same time, choose one")) } if getKeysOnly && getCountOnly { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("`--keys-only` and `--count-only` cannot be set at the same time, choose one")) } var opts []clientv3.OpOption if IsSerializable(getConsistency) { opts = append(opts, clientv3.WithSerializable()) } key := args[0] if len(args) > 1 { if getPrefix || getFromKey { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("too many arguments, only accept one argument when `--prefix` or `--from-key` is set")) } opts = append(opts, clientv3.WithRange(args[1])) } opts = append(opts, clientv3.WithLimit(getLimit)) if getRev > 0 { opts = append(opts, clientv3.WithRev(getRev)) } sortByOrder := clientv3.SortNone sortOrder := strings.ToUpper(getSortOrder) switch { case sortOrder == "ASCEND": sortByOrder = clientv3.SortAscend case sortOrder == "DESCEND": sortByOrder = clientv3.SortDescend case sortOrder == "": // nothing default: cobrautl.ExitWithError(cobrautl.ExitBadFeature, fmt.Errorf("bad sort order %v", getSortOrder)) } sortByTarget := clientv3.SortByKey sortTarget := strings.ToUpper(getSortTarget) switch { case sortTarget == "CREATE": sortByTarget = clientv3.SortByCreateRevision case sortTarget == "KEY": sortByTarget = clientv3.SortByKey case sortTarget == "MODIFY": sortByTarget = clientv3.SortByModRevision case sortTarget == "VALUE": sortByTarget = clientv3.SortByValue case sortTarget == "VERSION": sortByTarget = clientv3.SortByVersion case sortTarget == "": // nothing default: cobrautl.ExitWithError(cobrautl.ExitBadFeature, fmt.Errorf("bad sort target %v", getSortTarget)) } opts = append(opts, clientv3.WithSort(sortByTarget, sortByOrder)) if getPrefix { if len(key) == 0 { key = "\x00" opts = append(opts, clientv3.WithFromKey()) } else { opts = append(opts, clientv3.WithPrefix()) } } if getFromKey { if len(key) == 0 { key = "\x00" } opts = append(opts, clientv3.WithFromKey()) } if getKeysOnly { opts = append(opts, clientv3.WithKeysOnly()) } if getCountOnly { opts = append(opts, clientv3.WithCountOnly()) } if getMinCreateRev > 0 { opts = append(opts, clientv3.WithMinCreateRev(getMinCreateRev)) } if getMaxCreateRev > 0 { if getMinCreateRev > getMaxCreateRev { cobrautl.ExitWithError(cobrautl.ExitBadFeature, fmt.Errorf("getMinCreateRev(=%v) > getMaxCreateRev(=%v)", getMinCreateRev, getMaxCreateRev)) } opts = append(opts, clientv3.WithMaxCreateRev(getMaxCreateRev)) } if getMinModRev > 0 { opts = append(opts, clientv3.WithMinModRev(getMinModRev)) } if getMaxModRev > 0 { if getMinModRev > getMaxModRev { cobrautl.ExitWithError(cobrautl.ExitBadFeature, fmt.Errorf("getMinModRev(=%v) > getMaxModRev(=%v)", getMinModRev, getMaxModRev)) } opts = append(opts, clientv3.WithMaxModRev(getMaxModRev)) } return key, opts } ================================================ FILE: etcdctl/ctlv3/command/global.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "errors" "fmt" "io" "os" "strings" "time" "github.com/bgentry/speakeasy" "github.com/spf13/cobra" "github.com/spf13/pflag" "go.uber.org/zap" "google.golang.org/grpc/grpclog" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/client/pkg/v3/srv" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" "go.etcd.io/etcd/pkg/v3/flags" ) // GlobalFlags are flags that defined globally // and are inherited to all sub-commands. type GlobalFlags struct { Insecure bool InsecureSkipVerify bool InsecureDiscovery bool Endpoints []string DialTimeout time.Duration CommandTimeOut time.Duration KeepAliveTime time.Duration KeepAliveTimeout time.Duration MaxCallSendMsgSize int MaxCallRecvMsgSize int DNSClusterServiceName string TLS transport.TLSInfo OutputFormat string IsHex bool User string Password string Token string Debug bool } type discoveryCfg struct { domain string insecure bool serviceName string } var display printer = &simplePrinter{} func initDisplayFromCmd(cmd *cobra.Command) { isHex, err := cmd.Flags().GetBool("hex") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } outputType, err := cmd.Flags().GetString("write-out") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } if display = NewPrinter(outputType, isHex); display == nil { cobrautl.ExitWithError(cobrautl.ExitBadFeature, errors.New("unsupported output format")) } } type discardValue struct{} func (*discardValue) String() string { return "" } func (*discardValue) Set(string) error { return nil } func (*discardValue) Type() string { return "" } func clientConfigFromCmd(cmd *cobra.Command) *clientv3.ConfigSpec { lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } fs := cmd.InheritedFlags() if strings.HasPrefix(cmd.Use, "watch") { // silence "pkg/flags: unrecognized environment variable ETCDCTL_WATCH_KEY=foo" warnings // silence "pkg/flags: unrecognized environment variable ETCDCTL_WATCH_RANGE_END=bar" warnings fs.AddFlag(&pflag.Flag{Name: "watch-key", Value: &discardValue{}}) fs.AddFlag(&pflag.Flag{Name: "watch-range-end", Value: &discardValue{}}) } flags.SetPflagsFromEnv(lg, "ETCDCTL", fs) debug, err := cmd.Flags().GetBool("debug") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } if debug { grpclog.SetLoggerV2(grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 4)) fs.VisitAll(func(f *pflag.Flag) { fmt.Fprintf(os.Stderr, "%s=%v\n", flags.FlagToEnv("ETCDCTL", f.Name), f.Value) }) } else { // WARNING logs contain important information like TLS misconfirugation, but spams // too many routine connection disconnects to turn on by default. // // See https://github.com/etcd-io/etcd/pull/9623 for background grpclog.SetLoggerV2(grpclog.NewLoggerV2(io.Discard, io.Discard, os.Stderr)) } cfg := &clientv3.ConfigSpec{} cfg.Endpoints, err = endpointsFromCmd(cmd) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } cfg.DialTimeout = dialTimeoutFromCmd(cmd) cfg.KeepAliveTime = keepAliveTimeFromCmd(cmd) cfg.KeepAliveTimeout = keepAliveTimeoutFromCmd(cmd) cfg.MaxCallSendMsgSize = maxCallSendMsgSizeFromCmd(cmd) cfg.MaxCallRecvMsgSize = maxCallRecvMsgSizeFromCmd(cmd) cfg.Secure = secureCfgFromCmd(cmd) cfg.Auth = authCfgFromCmd(cmd) initDisplayFromCmd(cmd) return cfg } func mustClientCfgFromCmd(cmd *cobra.Command) *clientv3.Config { cc := clientConfigFromCmd(cmd) lg, _ := logutil.CreateDefaultZapLogger(zap.InfoLevel) cfg, err := clientv3.NewClientConfig(cc, lg) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } return cfg } func mustClientFromCmd(cmd *cobra.Command) *clientv3.Client { cfg := clientConfigFromCmd(cmd) return mustClient(cfg) } func mustClient(cc *clientv3.ConfigSpec) *clientv3.Client { lg, _ := logutil.CreateDefaultZapLogger(zap.InfoLevel) cfg, err := clientv3.NewClientConfig(cc, lg) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } client, err := clientv3.New(*cfg) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadConnection, err) } return client } func argOrStdin(args []string, stdin io.Reader, i int) (string, error) { if i < len(args) { return args[i], nil } bytes, err := io.ReadAll(stdin) if string(bytes) == "" || err != nil { return "", errors.New("no available argument and stdin") } return string(bytes), nil } func dialTimeoutFromCmd(cmd *cobra.Command) time.Duration { dialTimeout, err := cmd.Flags().GetDuration("dial-timeout") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } return dialTimeout } func keepAliveTimeFromCmd(cmd *cobra.Command) time.Duration { keepAliveTime, err := cmd.Flags().GetDuration("keepalive-time") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } return keepAliveTime } func keepAliveTimeoutFromCmd(cmd *cobra.Command) time.Duration { keepAliveTimeout, err := cmd.Flags().GetDuration("keepalive-timeout") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } return keepAliveTimeout } func maxCallSendMsgSizeFromCmd(cmd *cobra.Command) int { maxRequestBytes, err := cmd.Flags().GetInt("max-request-bytes") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } return maxRequestBytes } func maxCallRecvMsgSizeFromCmd(cmd *cobra.Command) int { maxReceiveBytes, err := cmd.Flags().GetInt("max-recv-bytes") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } return maxReceiveBytes } func secureCfgFromCmd(cmd *cobra.Command) *clientv3.SecureConfig { cert, key, cacert := keyAndCertFromCmd(cmd) insecureTr := insecureTransportFromCmd(cmd) skipVerify := insecureSkipVerifyFromCmd(cmd) discoveryCfg := discoveryCfgFromCmd(cmd) if discoveryCfg.insecure { discoveryCfg.domain = "" } return &clientv3.SecureConfig{ Cert: cert, Key: key, Cacert: cacert, ServerName: discoveryCfg.domain, InsecureTransport: insecureTr, InsecureSkipVerify: skipVerify, } } func insecureTransportFromCmd(cmd *cobra.Command) bool { insecureTr, err := cmd.Flags().GetBool("insecure-transport") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } return insecureTr } func insecureSkipVerifyFromCmd(cmd *cobra.Command) bool { skipVerify, err := cmd.Flags().GetBool("insecure-skip-tls-verify") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } return skipVerify } func keyAndCertFromCmd(cmd *cobra.Command) (cert, key, cacert string) { var err error if cert, err = cmd.Flags().GetString("cert"); err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } else if cert == "" && cmd.Flags().Changed("cert") { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("empty string is passed to --cert option")) } if key, err = cmd.Flags().GetString("key"); err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } else if key == "" && cmd.Flags().Changed("key") { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("empty string is passed to --key option")) } if cacert, err = cmd.Flags().GetString("cacert"); err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } else if cacert == "" && cmd.Flags().Changed("cacert") { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("empty string is passed to --cacert option")) } return cert, key, cacert } func authCfgFromCmd(cmd *cobra.Command) *clientv3.AuthConfig { userFlag, err := cmd.Flags().GetString("user") if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } passwordFlag, err := cmd.Flags().GetString("password") if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } tokenFlag, err := cmd.Flags().GetString("auth-jwt-token") if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } if userFlag == "" && tokenFlag == "" { return nil } var cfg clientv3.AuthConfig if tokenFlag != "" { cfg.Token = tokenFlag return &cfg } if passwordFlag == "" { splitted := strings.SplitN(userFlag, ":", 2) if len(splitted) < 2 { cfg.Username = userFlag cfg.Password, err = speakeasy.Ask("Password: ") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } } else { cfg.Username = splitted[0] cfg.Password = splitted[1] } } else { cfg.Username = userFlag cfg.Password = passwordFlag } return &cfg } func insecureDiscoveryFromCmd(cmd *cobra.Command) bool { discovery, err := cmd.Flags().GetBool("insecure-discovery") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } return discovery } func discoverySrvFromCmd(cmd *cobra.Command) string { domainStr, err := cmd.Flags().GetString("discovery-srv") if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } return domainStr } func discoveryDNSClusterServiceNameFromCmd(cmd *cobra.Command) string { serviceNameStr, err := cmd.Flags().GetString("discovery-srv-name") if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } return serviceNameStr } func discoveryCfgFromCmd(cmd *cobra.Command) *discoveryCfg { return &discoveryCfg{ domain: discoverySrvFromCmd(cmd), insecure: insecureDiscoveryFromCmd(cmd), serviceName: discoveryDNSClusterServiceNameFromCmd(cmd), } } func endpointsFromCmd(cmd *cobra.Command) ([]string, error) { eps, err := endpointsFromFlagValue(cmd) if err != nil { return nil, err } // If domain discovery returns no endpoints, check endpoints flag if len(eps) == 0 { eps, err = cmd.Flags().GetStringSlice("endpoints") if err == nil { for i, ip := range eps { eps[i] = strings.TrimSpace(ip) } } } return eps, err } func endpointsFromFlagValue(cmd *cobra.Command) ([]string, error) { discoveryCfg := discoveryCfgFromCmd(cmd) // If we still don't have domain discovery, return nothing if discoveryCfg.domain == "" { return []string{}, nil } srvs, err := srv.GetClient("etcd-client", discoveryCfg.domain, discoveryCfg.serviceName) if err != nil { return nil, err } eps := srvs.Endpoints if discoveryCfg.insecure { return eps, err } // strip insecure connections var ret []string for _, ep := range eps { if strings.HasPrefix(ep, "http://") { fmt.Fprintf(os.Stderr, "ignoring discovered insecure endpoint %q\n", ep) continue } ret = append(ret, ep) } return ret, err } ================================================ FILE: etcdctl/ctlv3/command/groups.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import "github.com/spf13/cobra" const ( groupKVID = "kv" groupClusterMaintenanceID = "cluster maintenance" groupConcurrencyID = "concurrency" groupAuthenticationID = "authentication" groupUtilityID = "utility" ) func NewKVGroup() *cobra.Group { return &cobra.Group{ ID: groupKVID, Title: "Key-value commands", } } func NewClusterMaintenanceGroup() *cobra.Group { return &cobra.Group{ ID: groupClusterMaintenanceID, Title: "Cluster maintenance commands", } } func NewConcurrencyGroup() *cobra.Group { return &cobra.Group{ ID: groupConcurrencyID, Title: "Concurrency commands", } } func NewAuthenticationGroup() *cobra.Group { return &cobra.Group{ ID: groupAuthenticationID, Title: "Authentication commands", } } func NewUtilityGroup() *cobra.Group { return &cobra.Group{ ID: groupUtilityID, Title: "Utility commands", } } ================================================ FILE: etcdctl/ctlv3/command/help_command.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import "github.com/spf13/cobra" func SetHelpCmdGroup(rootCmd *cobra.Command) { rootCmd.SetHelpCommandGroupID(groupUtilityID) } ================================================ FILE: etcdctl/ctlv3/command/lease_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "context" "fmt" "strconv" "github.com/spf13/cobra" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) // NewLeaseCommand returns the cobra command for "lease". func NewLeaseCommand() *cobra.Command { lc := &cobra.Command{ Use: "lease ", Short: "Lease related commands. Use `etcdctl lease --help` to see subcommands", Long: "Lease related commands", GroupID: groupKVID, } lc.AddCommand(NewLeaseGrantCommand()) lc.AddCommand(NewLeaseRevokeCommand()) lc.AddCommand(NewLeaseTimeToLiveCommand()) lc.AddCommand(NewLeaseListCommand()) lc.AddCommand(NewLeaseKeepAliveCommand()) return lc } // NewLeaseGrantCommand returns the cobra command for "lease grant". func NewLeaseGrantCommand() *cobra.Command { lc := &cobra.Command{ Use: "grant ", Short: "Creates leases", Run: leaseGrantCommandFunc, } return lc } // leaseGrantCommandFunc executes the "lease grant" command. func leaseGrantCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("lease grant command needs TTL argument")) } ttl, err := strconv.ParseInt(args[0], 10, 64) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("bad TTL (%w)", err)) } ctx, cancel := commandCtx(cmd) resp, err := mustClientFromCmd(cmd).Grant(ctx, ttl) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, fmt.Errorf("failed to grant lease (%w)", err)) } display.Grant(*resp) } // NewLeaseRevokeCommand returns the cobra command for "lease revoke". func NewLeaseRevokeCommand() *cobra.Command { lc := &cobra.Command{ Use: "revoke ", Short: "Revokes leases", Run: leaseRevokeCommandFunc, } return lc } // leaseRevokeCommandFunc executes the "lease grant" command. func leaseRevokeCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("lease revoke command needs 1 argument")) } id := leaseFromArgs(args[0]) ctx, cancel := commandCtx(cmd) resp, err := mustClientFromCmd(cmd).Revoke(ctx, id) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, fmt.Errorf("failed to revoke lease (%w)", err)) } display.Revoke(id, *resp) } var timeToLiveKeys bool // NewLeaseTimeToLiveCommand returns the cobra command for "lease timetolive". func NewLeaseTimeToLiveCommand() *cobra.Command { lc := &cobra.Command{ Use: "timetolive [options]", Short: "Get lease information", Run: leaseTimeToLiveCommandFunc, } lc.Flags().BoolVar(&timeToLiveKeys, "keys", false, "Get keys attached to this lease") return lc } // leaseTimeToLiveCommandFunc executes the "lease timetolive" command. func leaseTimeToLiveCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("lease timetolive command needs lease ID as argument")) } var opts []v3.LeaseOption if timeToLiveKeys { opts = append(opts, v3.WithAttachedKeys()) } resp, rerr := mustClientFromCmd(cmd).TimeToLive(context.TODO(), leaseFromArgs(args[0]), opts...) if rerr != nil { cobrautl.ExitWithError(cobrautl.ExitBadConnection, rerr) } display.TimeToLive(*resp, timeToLiveKeys) } // NewLeaseListCommand returns the cobra command for "lease list". func NewLeaseListCommand() *cobra.Command { lc := &cobra.Command{ Use: "list", Short: "List all active leases", Run: leaseListCommandFunc, } return lc } // leaseListCommandFunc executes the "lease list" command. func leaseListCommandFunc(cmd *cobra.Command, args []string) { resp, rerr := mustClientFromCmd(cmd).Leases(context.TODO()) if rerr != nil { cobrautl.ExitWithError(cobrautl.ExitBadConnection, rerr) } display.Leases(*resp) } var leaseKeepAliveOnce bool // NewLeaseKeepAliveCommand returns the cobra command for "lease keep-alive". func NewLeaseKeepAliveCommand() *cobra.Command { lc := &cobra.Command{ Use: "keep-alive [options] ", Short: "Keeps leases alive (renew)", Run: leaseKeepAliveCommandFunc, } lc.Flags().BoolVar(&leaseKeepAliveOnce, "once", false, "Resets the keep-alive time to its original value and cobrautl.Exits immediately") return lc } // leaseKeepAliveCommandFunc executes the "lease keep-alive" command. func leaseKeepAliveCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("lease keep-alive command needs lease ID as argument")) } id := leaseFromArgs(args[0]) if leaseKeepAliveOnce { respc, kerr := mustClientFromCmd(cmd).KeepAliveOnce(context.TODO(), id) if kerr != nil { cobrautl.ExitWithError(cobrautl.ExitBadConnection, kerr) } display.KeepAlive(*respc) return } respc, kerr := mustClientFromCmd(cmd).KeepAlive(context.TODO(), id) if kerr != nil { cobrautl.ExitWithError(cobrautl.ExitBadConnection, kerr) } for resp := range respc { display.KeepAlive(*resp) } if _, ok := (display).(*simplePrinter); ok { fmt.Printf("lease %016x expired or revoked.\n", id) } } func leaseFromArgs(arg string) v3.LeaseID { id, err := strconv.ParseInt(arg, 16, 64) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("bad lease ID arg (%w), expecting ID in Hex", err)) } return v3.LeaseID(id) } ================================================ FILE: etcdctl/ctlv3/command/lock_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "context" "errors" "fmt" "os" "os/exec" "os/signal" "syscall" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var lockTTL = 10 // NewLockCommand returns the cobra command for "lock". func NewLockCommand() *cobra.Command { c := &cobra.Command{ Use: "lock [exec-command arg1 arg2 ...]", Short: "Acquires a named lock", Run: lockCommandFunc, GroupID: groupConcurrencyID, } c.Flags().IntVarP(&lockTTL, "ttl", "", lockTTL, "timeout for session") return c } func lockCommandFunc(cmd *cobra.Command, args []string) { if len(args) == 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("lock takes a lock name argument and an optional command to execute")) } c := mustClientFromCmd(cmd) if err := lockUntilSignal(c, args[0], args[1:]); err != nil { code := getExitCodeFromError(err) cobrautl.ExitWithError(code, err) } } func getExitCodeFromError(err error) int { if err == nil { return cobrautl.ExitSuccess } var exitErr *exec.ExitError if errors.As(err, &exitErr) { if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { return status.ExitStatus() } } return cobrautl.ExitError } func lockUntilSignal(c *clientv3.Client, lockname string, cmdArgs []string) error { s, err := concurrency.NewSession(c, concurrency.WithTTL(lockTTL)) if err != nil { return err } m := concurrency.NewMutex(s, lockname) ctx, cancel := context.WithCancel(context.TODO()) // unlock in case of ordinary shutdown donec := make(chan struct{}) sigc := make(chan os.Signal, 1) signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) go func() { <-sigc cancel() close(donec) }() if err := m.Lock(ctx); err != nil { return err } if len(cmdArgs) > 0 { cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) cmd.Env = append(environLockResponse(m), os.Environ()...) cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr err := cmd.Run() unlockErr := m.Unlock(context.TODO()) if err != nil { return err } return unlockErr } k, kerr := c.Get(ctx, m.Key()) if kerr != nil { return kerr } if len(k.Kvs) == 0 { return errors.New("lock lost on init") } display.Get(*k) select { case <-donec: return m.Unlock(context.TODO()) case <-s.Done(): } return errors.New("session expired") } func environLockResponse(m *concurrency.Mutex) []string { return []string{ "ETCD_LOCK_KEY=" + m.Key(), fmt.Sprintf("ETCD_LOCK_REV=%d", m.Header().Revision), } } ================================================ FILE: etcdctl/ctlv3/command/make_mirror_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "context" "errors" "fmt" "strings" "sync/atomic" "time" "github.com/bgentry/speakeasy" "github.com/spf13/cobra" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/mirror" "go.etcd.io/etcd/pkg/v3/cobrautl" ) const ( defaultMaxTxnOps = uint(128) ) var ( mminsecureTr bool mmcert string mmkey string mmcacert string mmprefix string mmdestprefix string mmuser string mmpassword string mmnodestprefix bool mmrev int64 mmmaxTxnOps uint ) // NewMakeMirrorCommand returns the cobra command for "makeMirror". func NewMakeMirrorCommand() *cobra.Command { c := &cobra.Command{ Use: "make-mirror [options] ", Short: "Makes a mirror at the destination etcd cluster", Run: makeMirrorCommandFunc, GroupID: groupUtilityID, } c.Flags().StringVar(&mmprefix, "prefix", "", "Key-value prefix to mirror") c.Flags().Int64Var(&mmrev, "rev", 0, "Specify the kv revision to start to mirror") c.Flags().UintVar(&mmmaxTxnOps, "max-txn-ops", defaultMaxTxnOps, "Maximum number of operations permitted in a transaction during syncing updates.") c.Flags().StringVar(&mmdestprefix, "dest-prefix", "", "destination prefix to mirror a prefix to a different prefix in the destination cluster") c.Flags().BoolVar(&mmnodestprefix, "no-dest-prefix", false, "mirror key-values to the root of the destination cluster") c.Flags().StringVar(&mmcert, "dest-cert", "", "Identify secure client using this TLS certificate file for the destination cluster") c.Flags().StringVar(&mmkey, "dest-key", "", "Identify secure client using this TLS key file") c.Flags().StringVar(&mmcacert, "dest-cacert", "", "Verify certificates of TLS enabled secure servers using this CA bundle") // TODO: secure by default when etcd enables secure gRPC by default. c.Flags().BoolVar(&mminsecureTr, "dest-insecure-transport", true, "Disable transport security for client connections") c.Flags().StringVar(&mmuser, "dest-user", "", "Destination username[:password] for authentication (prompt if password is not supplied)") c.Flags().StringVar(&mmpassword, "dest-password", "", "Destination password for authentication (if this option is used, --user option shouldn't include password)") return c } func authDestCfg() *clientv3.AuthConfig { if mmuser == "" { return nil } var cfg clientv3.AuthConfig if mmpassword == "" { splitted := strings.SplitN(mmuser, ":", 2) if len(splitted) < 2 { var err error cfg.Username = mmuser cfg.Password, err = speakeasy.Ask("Destination Password: ") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } } else { cfg.Username = splitted[0] cfg.Password = splitted[1] } } else { cfg.Username = mmuser cfg.Password = mmpassword } return &cfg } func makeMirrorCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("make-mirror takes one destination argument")) } dialTimeout := dialTimeoutFromCmd(cmd) keepAliveTime := keepAliveTimeFromCmd(cmd) keepAliveTimeout := keepAliveTimeoutFromCmd(cmd) maxCallSendMsgSize := maxCallSendMsgSizeFromCmd(cmd) maxCallRecvMsgSize := maxCallRecvMsgSizeFromCmd(cmd) sec := &clientv3.SecureConfig{ Cert: mmcert, Key: mmkey, Cacert: mmcacert, InsecureTransport: mminsecureTr, } auth := authDestCfg() cc := &clientv3.ConfigSpec{ Endpoints: []string{args[0]}, DialTimeout: dialTimeout, KeepAliveTime: keepAliveTime, KeepAliveTimeout: keepAliveTimeout, MaxCallSendMsgSize: maxCallSendMsgSize, MaxCallRecvMsgSize: maxCallRecvMsgSize, Secure: sec, Auth: auth, } dc := mustClient(cc) c := mustClientFromCmd(cmd) err := makeMirror(context.TODO(), c, dc) cobrautl.ExitWithError(cobrautl.ExitError, err) } func makeMirror(ctx context.Context, c *clientv3.Client, dc *clientv3.Client) error { total := int64(0) // if destination prefix is specified and remove destination prefix is true return error if mmnodestprefix && len(mmdestprefix) > 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("`--dest-prefix` and `--no-dest-prefix` cannot be set at the same time, choose one")) } go func() { for { time.Sleep(30 * time.Second) fmt.Println(atomic.LoadInt64(&total)) } }() startRev := mmrev - 1 if startRev < 0 { startRev = 0 } s := mirror.NewSyncer(c, mmprefix, startRev) // If a rev is provided, then do not sync the whole key space. // Instead, just start watching the key space starting from the rev if startRev == 0 { rc, errc := s.SyncBase(ctx) // if remove destination prefix is false and destination prefix is empty set the value of destination prefix same as prefix if !mmnodestprefix && len(mmdestprefix) == 0 { mmdestprefix = mmprefix } for r := range rc { for _, kv := range r.Kvs { _, err := dc.Put(ctx, modifyPrefix(string(kv.Key)), string(kv.Value)) if err != nil { return err } atomic.AddInt64(&total, 1) } } err := <-errc if err != nil { return err } } wc := s.SyncUpdates(ctx) for wr := range wc { if wr.CompactRevision != 0 { return rpctypes.ErrCompacted } var lastRev int64 var ops []clientv3.Op for _, ev := range wr.Events { nextRev := ev.Kv.ModRevision if lastRev != 0 && nextRev > lastRev { _, err := dc.Txn(ctx).Then(ops...).Commit() if err != nil { return err } ops = []clientv3.Op{} } lastRev = nextRev if len(ops) == int(mmmaxTxnOps) { _, err := dc.Txn(ctx).Then(ops...).Commit() if err != nil { return err } ops = []clientv3.Op{} } switch ev.Type { case mvccpb.Event_PUT: ops = append(ops, clientv3.OpPut(modifyPrefix(string(ev.Kv.Key)), string(ev.Kv.Value))) atomic.AddInt64(&total, 1) case mvccpb.Event_DELETE: ops = append(ops, clientv3.OpDelete(modifyPrefix(string(ev.Kv.Key)))) atomic.AddInt64(&total, 1) default: panic("unexpected event type") } } if len(ops) != 0 { _, err := dc.Txn(ctx).Then(ops...).Commit() if err != nil { return err } } } return nil } func modifyPrefix(key string) string { return strings.Replace(key, mmprefix, mmdestprefix, 1) } ================================================ FILE: etcdctl/ctlv3/command/member_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "errors" "fmt" "strconv" "strings" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var ( memberPeerURLs string isLearner bool memberConsistency string ) // NewMemberCommand returns the cobra command for "member". func NewMemberCommand() *cobra.Command { mc := &cobra.Command{ Use: "member ", Short: "Membership related commands. Use `etcdctl member --help` to see subcommands", Long: "Membership related commands", GroupID: groupClusterMaintenanceID, } mc.AddCommand(NewMemberAddCommand()) mc.AddCommand(NewMemberRemoveCommand()) mc.AddCommand(NewMemberUpdateCommand()) mc.AddCommand(NewMemberListCommand()) mc.AddCommand(NewMemberPromoteCommand()) return mc } // NewMemberAddCommand returns the cobra command for "member add". func NewMemberAddCommand() *cobra.Command { cc := &cobra.Command{ Use: "add [options]", Short: "Adds a member into the cluster", Run: memberAddCommandFunc, } cc.Flags().StringVar(&memberPeerURLs, "peer-urls", "", "comma separated peer URLs for the new member.") cc.Flags().BoolVar(&isLearner, "learner", false, "indicates if the new member is raft learner") return cc } // NewMemberRemoveCommand returns the cobra command for "member remove". func NewMemberRemoveCommand() *cobra.Command { cc := &cobra.Command{ Use: "remove ", Short: "Removes a member from the cluster", Run: memberRemoveCommandFunc, } return cc } // NewMemberUpdateCommand returns the cobra command for "member update". func NewMemberUpdateCommand() *cobra.Command { cc := &cobra.Command{ Use: "update [options]", Short: "Updates a member in the cluster", Run: memberUpdateCommandFunc, } cc.Flags().StringVar(&memberPeerURLs, "peer-urls", "", "comma separated peer URLs for the updated member.") return cc } // NewMemberListCommand returns the cobra command for "member list". func NewMemberListCommand() *cobra.Command { cc := &cobra.Command{ Use: "list", Short: "Lists all members in the cluster", Long: `When --write-out is set to simple, this command prints out comma-separated member lists for each endpoint. The items in the lists are ID, Status, Name, Peer Addrs, Client Addrs, Is Learner. `, Run: memberListCommandFunc, } cc.Flags().StringVar(&memberConsistency, "consistency", "l", "Linearizable(l) or Serializable(s)") return cc } // NewMemberPromoteCommand returns the cobra command for "member promote". func NewMemberPromoteCommand() *cobra.Command { cc := &cobra.Command{ Use: "promote ", Short: "Promotes a non-voting member in the cluster", Long: `Promotes a non-voting learner member to a voting one in the cluster. `, Run: memberPromoteCommandFunc, } return cc } // memberAddCommandFunc executes the "member add" command. func memberAddCommandFunc(cmd *cobra.Command, args []string) { if len(args) < 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("member name not provided")) } if len(args) > 1 { ev := "too many arguments" for _, s := range args { if strings.HasPrefix(strings.ToLower(s), "http") { ev += fmt.Sprintf(`, did you mean --peer-urls=%s`, s) } } cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New(ev)) } newMemberName := args[0] if len(memberPeerURLs) == 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("member peer urls not provided")) } urls := strings.Split(memberPeerURLs, ",") ctx, cancel := commandCtx(cmd) cli := mustClientFromCmd(cmd) var ( resp *clientv3.MemberAddResponse err error ) if isLearner { resp, err = cli.MemberAddAsLearner(ctx, urls) } else { resp, err = cli.MemberAdd(ctx, urls) } cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } newID := resp.Member.ID display.MemberAdd(*resp) if _, ok := (display).(*simplePrinter); ok { var conf []string for _, memb := range resp.Members { for _, u := range memb.PeerURLs { n := memb.Name if memb.ID == newID { n = newMemberName } conf = append(conf, fmt.Sprintf("%s=%s", n, u)) } } fmt.Print("\n") fmt.Printf("ETCD_NAME=%q\n", newMemberName) fmt.Printf("ETCD_INITIAL_CLUSTER=%q\n", strings.Join(conf, ",")) fmt.Printf("ETCD_INITIAL_ADVERTISE_PEER_URLS=%q\n", memberPeerURLs) fmt.Print("ETCD_INITIAL_CLUSTER_STATE=\"existing\"\n") } } // memberRemoveCommandFunc executes the "member remove" command. func memberRemoveCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("member ID is not provided")) } id, err := strconv.ParseUint(args[0], 16, 64) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("bad member ID arg (%w), expecting ID in Hex", err)) } ctx, cancel := commandCtx(cmd) resp, err := mustClientFromCmd(cmd).MemberRemove(ctx, id) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.MemberRemove(id, *resp) } // memberUpdateCommandFunc executes the "member update" command. func memberUpdateCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("member ID is not provided")) } id, err := strconv.ParseUint(args[0], 16, 64) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("bad member ID arg (%w), expecting ID in Hex", err)) } if len(memberPeerURLs) == 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("member peer urls not provided")) } urls := strings.Split(memberPeerURLs, ",") ctx, cancel := commandCtx(cmd) resp, err := mustClientFromCmd(cmd).MemberUpdate(ctx, id, urls) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.MemberUpdate(id, *resp) } // memberListCommandFunc executes the "member list" command. func memberListCommandFunc(cmd *cobra.Command, args []string) { var opts []clientv3.OpOption if IsSerializable(memberConsistency) { opts = append(opts, clientv3.WithSerializable()) } ctx, cancel := commandCtx(cmd) resp, err := mustClientFromCmd(cmd).MemberList(ctx, opts...) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.MemberList(*resp) } // memberPromoteCommandFunc executes the "member promote" command. func memberPromoteCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("member ID is not provided")) } id, err := strconv.ParseUint(args[0], 16, 64) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("bad member ID arg (%w), expecting ID in Hex", err)) } ctx, cancel := commandCtx(cmd) resp, err := mustClientFromCmd(cmd).MemberPromote(ctx, id) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.MemberPromote(id, *resp) } ================================================ FILE: etcdctl/ctlv3/command/move_leader_command.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "strconv" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) // NewMoveLeaderCommand returns the cobra command for "move-leader". func NewMoveLeaderCommand() *cobra.Command { cmd := &cobra.Command{ Use: "move-leader ", Short: "Transfers leadership to another etcd cluster member.", Run: transferLeadershipCommandFunc, GroupID: groupClusterMaintenanceID, } return cmd } // transferLeadershipCommandFunc executes the "compaction" command. func transferLeadershipCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("move-leader command needs 1 argument")) } target, err := strconv.ParseUint(args[0], 16, 64) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } cfg := clientConfigFromCmd(cmd) cli := mustClient(cfg) eps := cli.Endpoints() cli.Close() ctx, cancel := commandCtx(cmd) // find current leader var leaderCli *clientv3.Client var leaderID uint64 for _, ep := range eps { cfg.Endpoints = []string{ep} cli := mustClient(cfg) resp, serr := cli.Status(ctx, ep) if serr != nil { cobrautl.ExitWithError(cobrautl.ExitError, serr) } if resp.Header.GetMemberId() == resp.Leader { leaderCli = cli leaderID = resp.Leader break } cli.Close() } if leaderCli == nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("no leader endpoint given at %v", eps)) } var resp *clientv3.MoveLeaderResponse resp, err = leaderCli.MoveLeader(ctx, target) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.MoveLeader(leaderID, target, *resp) } ================================================ FILE: etcdctl/ctlv3/command/options_command.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "github.com/spf13/cobra" "github.com/spf13/pflag" ) func NewOptionsCommand(rootCmd *cobra.Command) *cobra.Command { cmd := &cobra.Command{ Use: "options", Short: "Show the global command-line flags", Run: func(cmd *cobra.Command, args []string) { fs := unhideCopy(rootCmd.PersistentFlags()) fmt.Fprintf(cmd.OutOrStdout(), "The following options can be passed to any command:\n\n") fmt.Fprint(cmd.OutOrStdout(), fs.FlagUsages()) }, GroupID: groupUtilityID, } return cmd } func unhideCopy(src *pflag.FlagSet) *pflag.FlagSet { out := pflag.NewFlagSet("global", pflag.ContinueOnError) src.VisitAll(func(f *pflag.Flag) { nf := *f nf.Hidden = false out.AddFlag(&nf) }) return out } ================================================ FILE: etcdctl/ctlv3/command/printer.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "errors" "fmt" "strconv" "strings" "github.com/dustin/go-humanize" pb "go.etcd.io/etcd/api/v3/etcdserverpb" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) type printer interface { Del(v3.DeleteResponse) Get(v3.GetResponse) Put(v3.PutResponse) Txn(v3.TxnResponse) Watch(v3.WatchResponse) Grant(r v3.LeaseGrantResponse) Revoke(id v3.LeaseID, r v3.LeaseRevokeResponse) KeepAlive(r v3.LeaseKeepAliveResponse) TimeToLive(r v3.LeaseTimeToLiveResponse, keys bool) Leases(r v3.LeaseLeasesResponse) MemberAdd(v3.MemberAddResponse) MemberRemove(id uint64, r v3.MemberRemoveResponse) MemberUpdate(id uint64, r v3.MemberUpdateResponse) MemberPromote(id uint64, r v3.MemberPromoteResponse) MemberList(v3.MemberListResponse) EndpointHealth([]epHealth) EndpointStatus([]epStatus) EndpointHashKV([]epHashKV) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) DowngradeValidate(r v3.DowngradeResponse) DowngradeEnable(r v3.DowngradeResponse) DowngradeCancel(r v3.DowngradeResponse) Alarm(v3.AlarmResponse) RoleAdd(role string, r v3.AuthRoleAddResponse) RoleGet(role string, r v3.AuthRoleGetResponse) RoleDelete(role string, r v3.AuthRoleDeleteResponse) RoleList(v3.AuthRoleListResponse) RoleGrantPermission(role string, r v3.AuthRoleGrantPermissionResponse) RoleRevokePermission(role string, key string, end string, r v3.AuthRoleRevokePermissionResponse) UserAdd(user string, r v3.AuthUserAddResponse) UserGet(user string, r v3.AuthUserGetResponse) UserList(r v3.AuthUserListResponse) UserChangePassword(v3.AuthUserChangePasswordResponse) UserGrantRole(user string, role string, r v3.AuthUserGrantRoleResponse) UserRevokeRole(user string, role string, r v3.AuthUserRevokeRoleResponse) UserDelete(user string, r v3.AuthUserDeleteResponse) AuthStatus(r v3.AuthStatusResponse) } func NewPrinter(printerType string, isHex bool) printer { switch printerType { case "simple": return &simplePrinter{isHex: isHex} case "fields": return &fieldsPrinter{printer: newPrinterUnsupported("fields"), isHex: isHex} case "json": return newJSONPrinter(isHex) case "protobuf": return newPBPrinter() case "table": return &tablePrinter{newPrinterUnsupported("table")} } return nil } type printerRPC struct { printer p func(any) } func (p *printerRPC) Del(r v3.DeleteResponse) { p.p((*pb.DeleteRangeResponse)(&r)) } func (p *printerRPC) Get(r v3.GetResponse) { p.p((*pb.RangeResponse)(&r)) } func (p *printerRPC) Put(r v3.PutResponse) { p.p((*pb.PutResponse)(&r)) } func (p *printerRPC) Txn(r v3.TxnResponse) { p.p((*pb.TxnResponse)(&r)) } func (p *printerRPC) Watch(r v3.WatchResponse) { p.p(&r) } func (p *printerRPC) Grant(r v3.LeaseGrantResponse) { p.p(r) } func (p *printerRPC) Revoke(id v3.LeaseID, r v3.LeaseRevokeResponse) { p.p(r) } func (p *printerRPC) KeepAlive(r v3.LeaseKeepAliveResponse) { p.p(r) } func (p *printerRPC) TimeToLive(r v3.LeaseTimeToLiveResponse, keys bool) { p.p(&r) } func (p *printerRPC) Leases(r v3.LeaseLeasesResponse) { p.p(&r) } func (p *printerRPC) MemberAdd(r v3.MemberAddResponse) { p.p((*pb.MemberAddResponse)(&r)) } func (p *printerRPC) MemberRemove(id uint64, r v3.MemberRemoveResponse) { p.p((*pb.MemberRemoveResponse)(&r)) } func (p *printerRPC) MemberUpdate(id uint64, r v3.MemberUpdateResponse) { p.p((*pb.MemberUpdateResponse)(&r)) } func (p *printerRPC) MemberPromote(id uint64, r v3.MemberPromoteResponse) { p.p((*pb.MemberPromoteResponse)(&r)) } func (p *printerRPC) MemberList(r v3.MemberListResponse) { p.p((*pb.MemberListResponse)(&r)) } func (p *printerRPC) Alarm(r v3.AlarmResponse) { p.p((*pb.AlarmResponse)(&r)) } func (p *printerRPC) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) { p.p((*pb.MoveLeaderResponse)(&r)) } func (p *printerRPC) DowngradeValidate(r v3.DowngradeResponse) { p.p((*pb.DowngradeResponse)(&r)) } func (p *printerRPC) DowngradeEnable(r v3.DowngradeResponse) { p.p((*pb.DowngradeResponse)(&r)) } func (p *printerRPC) DowngradeCancel(r v3.DowngradeResponse) { p.p((*pb.DowngradeResponse)(&r)) } func (p *printerRPC) RoleAdd(_ string, r v3.AuthRoleAddResponse) { p.p((*pb.AuthRoleAddResponse)(&r)) } func (p *printerRPC) RoleGet(_ string, r v3.AuthRoleGetResponse) { p.p((*pb.AuthRoleGetResponse)(&r)) } func (p *printerRPC) RoleDelete(_ string, r v3.AuthRoleDeleteResponse) { p.p((*pb.AuthRoleDeleteResponse)(&r)) } func (p *printerRPC) RoleList(r v3.AuthRoleListResponse) { p.p((*pb.AuthRoleListResponse)(&r)) } func (p *printerRPC) RoleGrantPermission(_ string, r v3.AuthRoleGrantPermissionResponse) { p.p((*pb.AuthRoleGrantPermissionResponse)(&r)) } func (p *printerRPC) RoleRevokePermission(_ string, _ string, _ string, r v3.AuthRoleRevokePermissionResponse) { p.p((*pb.AuthRoleRevokePermissionResponse)(&r)) } func (p *printerRPC) UserAdd(_ string, r v3.AuthUserAddResponse) { p.p((*pb.AuthUserAddResponse)(&r)) } func (p *printerRPC) UserGet(_ string, r v3.AuthUserGetResponse) { p.p((*pb.AuthUserGetResponse)(&r)) } func (p *printerRPC) UserList(r v3.AuthUserListResponse) { p.p((*pb.AuthUserListResponse)(&r)) } func (p *printerRPC) UserChangePassword(r v3.AuthUserChangePasswordResponse) { p.p((*pb.AuthUserChangePasswordResponse)(&r)) } func (p *printerRPC) UserGrantRole(_ string, _ string, r v3.AuthUserGrantRoleResponse) { p.p((*pb.AuthUserGrantRoleResponse)(&r)) } func (p *printerRPC) UserRevokeRole(_ string, _ string, r v3.AuthUserRevokeRoleResponse) { p.p((*pb.AuthUserRevokeRoleResponse)(&r)) } func (p *printerRPC) UserDelete(_ string, r v3.AuthUserDeleteResponse) { p.p((*pb.AuthUserDeleteResponse)(&r)) } func (p *printerRPC) AuthStatus(r v3.AuthStatusResponse) { p.p((*pb.AuthStatusResponse)(&r)) } type printerUnsupported struct{ printerRPC } func newPrinterUnsupported(n string) printer { f := func(any) { cobrautl.ExitWithError(cobrautl.ExitBadFeature, errors.New(n+" not supported as output format")) } return &printerUnsupported{printerRPC{nil, f}} } func (p *printerUnsupported) EndpointHealth([]epHealth) { p.p(nil) } func (p *printerUnsupported) EndpointStatus([]epStatus) { p.p(nil) } func (p *printerUnsupported) EndpointHashKV([]epHashKV) { p.p(nil) } func (p *printerUnsupported) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) { p.p(nil) } func (p *printerUnsupported) DowngradeValidate(r v3.DowngradeResponse) { p.p(nil) } func (p *printerUnsupported) DowngradeEnable(r v3.DowngradeResponse) { p.p(nil) } func (p *printerUnsupported) DowngradeCancel(r v3.DowngradeResponse) { p.p(nil) } func makeMemberListTable(r v3.MemberListResponse) (hdr []string, rows [][]string) { hdr = []string{"ID", "Status", "Name", "Peer Addrs", "Client Addrs", "Is Learner"} for _, m := range r.Members { status := "started" if len(m.Name) == 0 { status = "unstarted" } isLearner := "false" if m.IsLearner { isLearner = "true" } rows = append(rows, []string{ fmt.Sprintf("%x", m.ID), status, m.Name, strings.Join(m.PeerURLs, ","), strings.Join(m.ClientURLs, ","), isLearner, }) } return hdr, rows } func makeEndpointHealthTable(healthList []epHealth) (hdr []string, rows [][]string) { hdr = []string{"endpoint", "health", "took", "error"} for _, h := range healthList { rows = append(rows, []string{ h.Ep, fmt.Sprintf("%v", h.Health), h.Took, h.Error, }) } return hdr, rows } func makeEndpointStatusTable(statusList []epStatus) (hdr []string, rows [][]string) { hdr = []string{ "endpoint", "ID", "version", "storage version", "db size", "in use", "percentage not in use", "quota", "is leader", "is learner", "raft term", "raft index", "raft applied index", "errors", "downgrade target version", "downgrade enabled", } for _, status := range statusList { rows = append(rows, []string{ status.Ep, fmt.Sprintf("%x", status.Resp.Header.MemberId), status.Resp.Version, status.Resp.StorageVersion, humanize.Bytes(uint64(status.Resp.DbSize)), humanize.Bytes(uint64(status.Resp.DbSizeInUse)), fmt.Sprintf("%d%%", int(float64(100-(status.Resp.DbSizeInUse*100/status.Resp.DbSize)))), humanize.Bytes(uint64(status.Resp.DbSizeQuota)), fmt.Sprint(status.Resp.Leader == status.Resp.Header.MemberId), fmt.Sprint(status.Resp.IsLearner), fmt.Sprint(status.Resp.RaftTerm), fmt.Sprint(status.Resp.RaftIndex), fmt.Sprint(status.Resp.RaftAppliedIndex), fmt.Sprint(strings.Join(status.Resp.Errors, ", ")), status.Resp.DowngradeInfo.GetTargetVersion(), strconv.FormatBool(status.Resp.DowngradeInfo.GetEnabled()), }) } return hdr, rows } func makeEndpointHashKVTable(hashList []epHashKV) (hdr []string, rows [][]string) { hdr = []string{"endpoint", "hash", "hash_revision"} for _, h := range hashList { rows = append(rows, []string{ h.Ep, fmt.Sprint(h.Resp.Hash), fmt.Sprint(h.Resp.HashRevision), }) } return hdr, rows } ================================================ FILE: etcdctl/ctlv3/command/printer_fields.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" pb "go.etcd.io/etcd/api/v3/etcdserverpb" spb "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/client/pkg/v3/types" v3 "go.etcd.io/etcd/client/v3" ) type fieldsPrinter struct { printer isHex bool } func (p *fieldsPrinter) kv(pfx string, kv *spb.KeyValue) { fmt.Printf("\"%sKey\" : %q\n", pfx, string(kv.Key)) fmt.Printf("\"%sCreateRevision\" : %d\n", pfx, kv.CreateRevision) fmt.Printf("\"%sModRevision\" : %d\n", pfx, kv.ModRevision) fmt.Printf("\"%sVersion\" : %d\n", pfx, kv.Version) fmt.Printf("\"%sValue\" : %q\n", pfx, string(kv.Value)) if p.isHex { fmt.Printf("\"%sLease\" : %016x\n", pfx, kv.Lease) } else { fmt.Printf("\"%sLease\" : %d\n", pfx, kv.Lease) } } func (p *fieldsPrinter) hdr(h *pb.ResponseHeader) { if p.isHex { fmt.Println(`"ClusterID" :`, types.ID(h.ClusterId)) fmt.Println(`"MemberID" :`, types.ID(h.MemberId)) } else { fmt.Println(`"ClusterID" :`, h.ClusterId) fmt.Println(`"MemberID" :`, h.MemberId) } // Revision only makes sense for k/v responses. For other kinds of // responses, i.e. MemberList, usually the revision isn't populated // at all; so it would be better to hide this field in these cases. if h.Revision > 0 { fmt.Println(`"Revision" :`, h.Revision) } fmt.Println(`"RaftTerm" :`, h.RaftTerm) } func (p *fieldsPrinter) Del(r v3.DeleteResponse) { p.hdr(r.Header) fmt.Println(`"Deleted" :`, r.Deleted) for _, kv := range r.PrevKvs { p.kv("Prev", kv) } } func (p *fieldsPrinter) Get(r v3.GetResponse) { p.hdr(r.Header) for _, kv := range r.Kvs { p.kv("", kv) } fmt.Println(`"More" :`, r.More) fmt.Println(`"Count" :`, r.Count) } func (p *fieldsPrinter) Put(r v3.PutResponse) { p.hdr(r.Header) if r.PrevKv != nil { p.kv("Prev", r.PrevKv) } } func (p *fieldsPrinter) Txn(r v3.TxnResponse) { p.hdr(r.Header) fmt.Println(`"Succeeded" :`, r.Succeeded) for _, resp := range r.Responses { switch v := resp.Response.(type) { case *pb.ResponseOp_ResponseDeleteRange: p.Del((v3.DeleteResponse)(*v.ResponseDeleteRange)) case *pb.ResponseOp_ResponsePut: p.Put((v3.PutResponse)(*v.ResponsePut)) case *pb.ResponseOp_ResponseRange: p.Get((v3.GetResponse)(*v.ResponseRange)) default: fmt.Printf("\"Unknown\" : %q\n", fmt.Sprintf("%+v", v)) } } } func (p *fieldsPrinter) Watch(resp v3.WatchResponse) { p.hdr(&resp.Header) for _, e := range resp.Events { fmt.Println(`"Type" :`, e.Type) if e.PrevKv != nil { p.kv("Prev", e.PrevKv) } p.kv("", e.Kv) } } func (p *fieldsPrinter) Grant(r v3.LeaseGrantResponse) { p.hdr(r.ResponseHeader) if p.isHex { fmt.Printf("\"ID\" : %016x\n", r.ID) } else { fmt.Println(`"ID" :`, r.ID) } fmt.Println(`"TTL" :`, r.TTL) } func (p *fieldsPrinter) Revoke(id v3.LeaseID, r v3.LeaseRevokeResponse) { p.hdr(r.Header) } func (p *fieldsPrinter) KeepAlive(r v3.LeaseKeepAliveResponse) { p.hdr(r.ResponseHeader) if p.isHex { fmt.Printf("\"ID\" : %016x\n", r.ID) } else { fmt.Println(`"ID" :`, r.ID) } fmt.Println(`"TTL" :`, r.TTL) } func (p *fieldsPrinter) TimeToLive(r v3.LeaseTimeToLiveResponse, keys bool) { p.hdr(r.ResponseHeader) if p.isHex { fmt.Printf("\"ID\" : %016x\n", r.ID) } else { fmt.Println(`"ID" :`, r.ID) } fmt.Println(`"TTL" :`, r.TTL) fmt.Println(`"GrantedTTL" :`, r.GrantedTTL) for _, k := range r.Keys { fmt.Printf("\"Key\" : %q\n", string(k)) } } func (p *fieldsPrinter) Leases(r v3.LeaseLeasesResponse) { p.hdr(r.ResponseHeader) for _, item := range r.Leases { if p.isHex { fmt.Printf("\"ID\" : %016x\n", item.ID) } else { fmt.Println(`"ID" :`, item.ID) } } } func (p *fieldsPrinter) MemberList(r v3.MemberListResponse) { p.hdr(r.Header) for _, m := range r.Members { if p.isHex { fmt.Println(`"ID" :`, types.ID(m.ID)) } else { fmt.Println(`"ID" :`, m.ID) } fmt.Printf("\"Name\" : %q\n", m.Name) for _, u := range m.PeerURLs { fmt.Printf("\"PeerURL\" : %q\n", u) } for _, u := range m.ClientURLs { fmt.Printf("\"ClientURL\" : %q\n", u) } fmt.Println(`"IsLearner" :`, m.IsLearner) fmt.Println() } } func (p *fieldsPrinter) EndpointHealth(hs []epHealth) { for _, h := range hs { fmt.Printf("\"Endpoint\" : %q\n", h.Ep) fmt.Println(`"Health" :`, h.Health) fmt.Println(`"Took" :`, h.Took) fmt.Println(`"Error" :`, h.Error) fmt.Println() } } func (p *fieldsPrinter) EndpointStatus(eps []epStatus) { for _, ep := range eps { p.hdr(ep.Resp.Header) fmt.Printf("\"Version\" : %q\n", ep.Resp.Version) fmt.Printf("\"StorageVersion\" : %q\n", ep.Resp.StorageVersion) fmt.Println(`"DBSize" :`, ep.Resp.DbSize) fmt.Println(`"DBSizeInUse" :`, ep.Resp.DbSizeInUse) fmt.Println(`"DBSizeQuota" :`, ep.Resp.DbSizeQuota) fmt.Println(`"Leader" :`, ep.Resp.Leader) fmt.Println(`"IsLearner" :`, ep.Resp.IsLearner) fmt.Println(`"RaftIndex" :`, ep.Resp.RaftIndex) fmt.Println(`"RaftTerm" :`, ep.Resp.RaftTerm) fmt.Println(`"RaftAppliedIndex" :`, ep.Resp.RaftAppliedIndex) fmt.Println(`"Errors" :`, ep.Resp.Errors) fmt.Printf("\"Endpoint\" : %q\n", ep.Ep) fmt.Printf("\"DowngradeTargetVersion\" : %q\n", ep.Resp.DowngradeInfo.GetTargetVersion()) fmt.Println(`"DowngradeEnabled" :`, ep.Resp.DowngradeInfo.GetEnabled()) fmt.Println() } } func (p *fieldsPrinter) EndpointHashKV(hs []epHashKV) { for _, h := range hs { p.hdr(h.Resp.Header) fmt.Printf("\"Endpoint\" : %q\n", h.Ep) fmt.Println(`"Hash" :`, h.Resp.Hash) fmt.Println(`"HashRevision" :`, h.Resp.HashRevision) fmt.Println() } } func (p *fieldsPrinter) Alarm(r v3.AlarmResponse) { p.hdr(r.Header) for _, a := range r.Alarms { if p.isHex { fmt.Println(`"MemberID" :`, types.ID(a.MemberID)) } else { fmt.Println(`"MemberID" :`, a.MemberID) } fmt.Println(`"AlarmType" :`, a.Alarm) fmt.Println() } } func (p *fieldsPrinter) RoleAdd(role string, r v3.AuthRoleAddResponse) { p.hdr(r.Header) } func (p *fieldsPrinter) RoleGet(role string, r v3.AuthRoleGetResponse) { p.hdr(r.Header) for _, p := range r.Perm { fmt.Println(`"PermType" : `, p.PermType.String()) fmt.Printf("\"Key\" : %q\n", string(p.Key)) fmt.Printf("\"RangeEnd\" : %q\n", string(p.RangeEnd)) } } func (p *fieldsPrinter) RoleDelete(role string, r v3.AuthRoleDeleteResponse) { p.hdr(r.Header) } func (p *fieldsPrinter) RoleList(r v3.AuthRoleListResponse) { p.hdr(r.Header) fmt.Print(`"Roles" :`) for _, r := range r.Roles { fmt.Printf(" %q", r) } fmt.Println() } func (p *fieldsPrinter) RoleGrantPermission(role string, r v3.AuthRoleGrantPermissionResponse) { p.hdr(r.Header) } func (p *fieldsPrinter) RoleRevokePermission(role string, key string, end string, r v3.AuthRoleRevokePermissionResponse) { p.hdr(r.Header) } func (p *fieldsPrinter) UserAdd(user string, r v3.AuthUserAddResponse) { p.hdr(r.Header) } func (p *fieldsPrinter) UserChangePassword(r v3.AuthUserChangePasswordResponse) { p.hdr(r.Header) } func (p *fieldsPrinter) UserGrantRole(user string, role string, r v3.AuthUserGrantRoleResponse) { p.hdr(r.Header) } func (p *fieldsPrinter) UserRevokeRole(user string, role string, r v3.AuthUserRevokeRoleResponse) { p.hdr(r.Header) } func (p *fieldsPrinter) UserDelete(user string, r v3.AuthUserDeleteResponse) { p.hdr(r.Header) } ================================================ FILE: etcdctl/ctlv3/command/printer_json.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "encoding/json" "fmt" "io" "os" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" ) type jsonPrinter struct { writer io.Writer isHex bool printer } type ( HexResponseHeader pb.ResponseHeader HexMember pb.Member ) func (h *HexResponseHeader) MarshalJSON() ([]byte, error) { type Alias pb.ResponseHeader return json.Marshal(&struct { ClusterID string `json:"cluster_id"` MemberID string `json:"member_id"` Alias }{ ClusterID: fmt.Sprintf("%x", h.ClusterId), MemberID: fmt.Sprintf("%x", h.MemberId), Alias: (Alias)(*h), }) } func (m *HexMember) MarshalJSON() ([]byte, error) { type Alias pb.Member return json.Marshal(&struct { ID string `json:"ID"` Alias }{ ID: fmt.Sprintf("%x", m.ID), Alias: (Alias)(*m), }) } func newJSONPrinter(isHex bool) printer { return &jsonPrinter{ writer: os.Stdout, isHex: isHex, printer: &printerRPC{newPrinterUnsupported("json"), printJSON}, } } func (p *jsonPrinter) EndpointHealth(r []epHealth) { printJSON(r) } func (p *jsonPrinter) EndpointStatus(r []epStatus) { printJSON(r) } func (p *jsonPrinter) EndpointHashKV(r []epHashKV) { printJSON(r) } func (p *jsonPrinter) MemberAdd(r clientv3.MemberAddResponse) { p.printJSON(r) } func (p *jsonPrinter) MemberRemove(_ uint64, r clientv3.MemberRemoveResponse) { p.printJSON(r) } func (p *jsonPrinter) MemberUpdate(_ uint64, r clientv3.MemberUpdateResponse) { p.printJSON(r) } func (p *jsonPrinter) MemberPromote(_ uint64, r clientv3.MemberPromoteResponse) { p.printJSON(r) } func (p *jsonPrinter) MemberList(r clientv3.MemberListResponse) { p.printJSON(r) } func printJSONTo(w io.Writer, v any) { b, err := json.Marshal(v) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) return } fmt.Fprintln(w, string(b)) } func printJSON(v any) { printJSONTo(os.Stdout, v) } func (p *jsonPrinter) printJSON(v any) { var data any if !p.isHex { printJSONTo(p.writer, v) return } switch r := v.(type) { case clientv3.MemberAddResponse: type Alias clientv3.MemberAddResponse data = &struct { Header *HexResponseHeader `json:"header"` Member *HexMember `json:"member"` Members []*HexMember `json:"members"` *Alias }{ Header: (*HexResponseHeader)(r.Header), Member: (*HexMember)(r.Member), Members: toHexMembers(r.Members), Alias: (*Alias)(&r), } case clientv3.MemberRemoveResponse: type Alias clientv3.MemberRemoveResponse data = &struct { Header *HexResponseHeader `json:"header"` Members []*HexMember `json:"members"` *Alias }{ Header: (*HexResponseHeader)(r.Header), Members: toHexMembers(r.Members), Alias: (*Alias)(&r), } case clientv3.MemberUpdateResponse: type Alias clientv3.MemberUpdateResponse data = &struct { Header *HexResponseHeader `json:"header"` Members []*HexMember `json:"members"` *Alias }{ Header: (*HexResponseHeader)(r.Header), Members: toHexMembers(r.Members), Alias: (*Alias)(&r), } case clientv3.MemberPromoteResponse: type Alias clientv3.MemberPromoteResponse data = &struct { Header *HexResponseHeader `json:"header"` Members []*HexMember `json:"members"` *Alias }{ Header: (*HexResponseHeader)(r.Header), Members: toHexMembers(r.Members), Alias: (*Alias)(&r), } case clientv3.MemberListResponse: type Alias clientv3.MemberListResponse data = &struct { Header *HexResponseHeader `json:"header"` Members []*HexMember `json:"members"` *Alias }{ Header: (*HexResponseHeader)(r.Header), Members: toHexMembers(r.Members), Alias: (*Alias)(&r), } default: data = v } printJSONTo(p.writer, data) } func toHexMembers(members []*pb.Member) []*HexMember { hexMembers := make([]*HexMember, len(members)) for i, member := range members { hexMembers[i] = (*HexMember)(member) } return hexMembers } ================================================ FILE: etcdctl/ctlv3/command/printer_json_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "bytes" "encoding/json" "fmt" "math" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" ) const ( keyHeader = "header" keyMember = "member" keyMembers = "members" keyClusterID = "cluster_id" keyMemberID = "member_id" keyRaftTerm = "raft_term" keyRevision = "revision" keyID = "ID" ) func assertNumericFieldEqual(t *testing.T, obj map[string]any, key string, want int64) { raw, ok := obj[key] require.Truef(t, ok, "missing key %q in map %v", key, obj) n, ok := raw.(json.Number) require.Truef(t, ok, "field %q is not json.Number: %v", key, raw) val, err := n.Int64() require.NoErrorf(t, err, "failed to convert field %q to int64: %v", key, n) assert.Equalf(t, want, val, "unexpected value for field %q", key) } func assertHexFieldEqual(t *testing.T, obj map[string]any, key string, want string) { raw, ok := obj[key] require.Truef(t, ok, "missing key %q in map %v", key, obj) str, ok := raw.(string) require.Truef(t, ok, "field %q is not a string: %v", key, str) assert.Equalf(t, want, str, "unexpected value for hex field %q", key) } func assertHeader(t *testing.T, testGroup *testScenario, tt *testCase, got map[string]any) { rawHeader, ok := got[keyHeader] require.Truef(t, ok, "output does not contain %q field: %v", keyHeader, got) header, ok := rawHeader.(map[string]any) require.Truef(t, ok, "field %q is not map[string]any: %v", keyHeader, rawHeader) if testGroup.isHex { assertHexFieldEqual(t, header, keyClusterID, tt.wantHexString) assertHexFieldEqual(t, header, keyMemberID, tt.wantHexString) } else { assertNumericFieldEqual(t, header, keyClusterID, tt.wantDecimalNumber) assertNumericFieldEqual(t, header, keyMemberID, tt.wantDecimalNumber) } assertNumericFieldEqual(t, header, keyRaftTerm, tt.wantDecimalNumber) assertNumericFieldEqual(t, header, keyRevision, tt.wantDecimalNumber) } func assertMember(t *testing.T, testGroup *testScenario, tt *testCase, rawMember any) { member, ok := rawMember.(map[string]any) require.Truef(t, ok, "field %q is not map[string]any: %v", keyMember, rawMember) if testGroup.isHex { assertHexFieldEqual(t, member, keyID, tt.wantHexString) } else { assertNumericFieldEqual(t, member, keyID, tt.wantDecimalNumber) } } func assertMembers(t *testing.T, testGroup *testScenario, tt *testCase, got map[string]any) { rawMembers, ok := got[keyMembers] require.Truef(t, ok, "output does not contain %q field: %v", keyMembers, got) members, ok := rawMembers.([]any) require.Truef(t, ok, "field %q is not []any: %v", keyMembers, rawMembers) for _, rawMember := range members { assertMember(t, testGroup, tt, rawMember) } } type testCase struct { number uint64 wantHexString string wantDecimalNumber int64 } type testScenario struct { name string isHex bool cases []testCase } var testCases = []testCase{ {1, "1", 1}, {100, "64", 100}, {1234567890, "499602d2", 1234567890}, {math.MaxInt64, "7fffffffffffffff", math.MaxInt64}, } func TestMemberAdd(t *testing.T) { tests := []testScenario{ {name: "decimal", isHex: false, cases: testCases}, {name: "hex", isHex: true, cases: testCases}, } for _, testGroup := range tests { t.Run(testGroup.name, func(t *testing.T) { var buffer bytes.Buffer p := &jsonPrinter{writer: &buffer, isHex: testGroup.isHex} for _, tt := range testGroup.cases { t.Run(fmt.Sprintf("number=%d", tt.number), func(t *testing.T) { buffer.Reset() decoder := json.NewDecoder(&buffer) decoder.UseNumber() response := clientv3.MemberAddResponse{ Header: &pb.ResponseHeader{ ClusterId: tt.number, MemberId: tt.number, Revision: int64(tt.number), RaftTerm: tt.number, }, Member: &pb.Member{ID: tt.number}, Members: []*pb.Member{{ID: tt.number}}, } p.MemberAdd(response) var got map[string]any err := decoder.Decode(&got) require.NoErrorf(t, err, "failed to decode JSON") assertHeader(t, &testGroup, &tt, got) rawMember, ok := got[keyMember] require.Truef(t, ok, "output does not contain %q field: %v", keyMember, got) assertMember(t, &testGroup, &tt, rawMember) assertMembers(t, &testGroup, &tt, got) }) } }) } } func TestMemberRemove(t *testing.T) { tests := []testScenario{ {name: "decimal", isHex: false, cases: testCases}, {name: "hex", isHex: true, cases: testCases}, } for _, testGroup := range tests { t.Run(testGroup.name, func(t *testing.T) { var buffer bytes.Buffer p := &jsonPrinter{writer: &buffer, isHex: testGroup.isHex} for _, tt := range testGroup.cases { t.Run(fmt.Sprintf("number=%d", tt.number), func(t *testing.T) { buffer.Reset() decoder := json.NewDecoder(&buffer) decoder.UseNumber() response := clientv3.MemberRemoveResponse{ Header: &pb.ResponseHeader{ ClusterId: tt.number, MemberId: tt.number, Revision: int64(tt.number), RaftTerm: tt.number, }, Members: []*pb.Member{{ID: tt.number}}, } p.MemberRemove(0, response) var got map[string]any err := decoder.Decode(&got) require.NoErrorf(t, err, "failed to decode JSON") assertHeader(t, &testGroup, &tt, got) assertMembers(t, &testGroup, &tt, got) }) } }) } } func TestMemberUpdate(t *testing.T) { tests := []testScenario{ {name: "decimal", isHex: false, cases: testCases}, {name: "hex", isHex: true, cases: testCases}, } for _, testGroup := range tests { t.Run(testGroup.name, func(t *testing.T) { var buffer bytes.Buffer p := &jsonPrinter{writer: &buffer, isHex: testGroup.isHex} for _, tt := range testGroup.cases { t.Run(fmt.Sprintf("number=%d", tt.number), func(t *testing.T) { buffer.Reset() decoder := json.NewDecoder(&buffer) decoder.UseNumber() response := clientv3.MemberUpdateResponse{ Header: &pb.ResponseHeader{ ClusterId: tt.number, MemberId: tt.number, Revision: int64(tt.number), RaftTerm: tt.number, }, Members: []*pb.Member{{ID: tt.number}}, } p.MemberUpdate(0, response) var got map[string]any err := decoder.Decode(&got) require.NoErrorf(t, err, "failed to decode JSON") assertHeader(t, &testGroup, &tt, got) assertMembers(t, &testGroup, &tt, got) }) } }) } } func TestMemberPromote(t *testing.T) { tests := []testScenario{ {name: "decimal", isHex: false, cases: testCases}, {name: "hex", isHex: true, cases: testCases}, } for _, testGroup := range tests { t.Run(testGroup.name, func(t *testing.T) { var buffer bytes.Buffer p := &jsonPrinter{writer: &buffer, isHex: testGroup.isHex} for _, tt := range testGroup.cases { t.Run(fmt.Sprintf("number=%d", tt.number), func(t *testing.T) { buffer.Reset() decoder := json.NewDecoder(&buffer) decoder.UseNumber() response := clientv3.MemberPromoteResponse{ Header: &pb.ResponseHeader{ ClusterId: tt.number, MemberId: tt.number, Revision: int64(tt.number), RaftTerm: tt.number, }, Members: []*pb.Member{{ID: tt.number}}, } p.MemberPromote(0, response) var got map[string]any err := decoder.Decode(&got) require.NoErrorf(t, err, "failed to decode JSON") assertHeader(t, &testGroup, &tt, got) assertMembers(t, &testGroup, &tt, got) }) } }) } } func TestMemberList(t *testing.T) { tests := []testScenario{ {name: "decimal", isHex: false, cases: testCases}, {name: "hex", isHex: true, cases: testCases}, } for _, testGroup := range tests { t.Run(testGroup.name, func(t *testing.T) { var buffer bytes.Buffer p := &jsonPrinter{writer: &buffer, isHex: testGroup.isHex} for _, tt := range testGroup.cases { t.Run(fmt.Sprintf("number=%d", tt.number), func(t *testing.T) { buffer.Reset() decoder := json.NewDecoder(&buffer) decoder.UseNumber() response := clientv3.MemberListResponse{ Header: &pb.ResponseHeader{ ClusterId: tt.number, MemberId: tt.number, Revision: int64(tt.number), RaftTerm: tt.number, }, Members: []*pb.Member{{ID: tt.number}}, } p.MemberList(response) var got map[string]any err := decoder.Decode(&got) require.NoErrorf(t, err, "failed to decode JSON") assertHeader(t, &testGroup, &tt, got) assertMembers(t, &testGroup, &tt, got) }) } }) } } ================================================ FILE: etcdctl/ctlv3/command/printer_protobuf.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "os" pb "go.etcd.io/etcd/api/v3/etcdserverpb" mvccpb "go.etcd.io/etcd/api/v3/mvccpb" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) type pbPrinter struct{ printer } type pbMarshal interface { Marshal() ([]byte, error) } func newPBPrinter() printer { return &pbPrinter{ &printerRPC{newPrinterUnsupported("protobuf"), printPB}, } } func (p *pbPrinter) Watch(r v3.WatchResponse) { evs := make([]*mvccpb.Event, len(r.Events)) for i, ev := range r.Events { evs[i] = (*mvccpb.Event)(ev) } wr := pb.WatchResponse{ Header: &r.Header, Events: evs, CompactRevision: r.CompactRevision, Canceled: r.Canceled, Created: r.Created, } printPB(&wr) } func printPB(v any) { m, ok := v.(pbMarshal) if !ok { cobrautl.ExitWithError(cobrautl.ExitBadFeature, fmt.Errorf("marshal unsupported for type %T (%v)", v, v)) } b, err := m.Marshal() if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) return } fmt.Print(string(b)) } ================================================ FILE: etcdctl/ctlv3/command/printer_simple.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "os" "strings" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/types" v3 "go.etcd.io/etcd/client/v3" ) const rootRole = "root" type simplePrinter struct { isHex bool valueOnly bool } func (s *simplePrinter) Del(resp v3.DeleteResponse) { fmt.Println(resp.Deleted) for _, kv := range resp.PrevKvs { printKV(s.isHex, s.valueOnly, kv) } } func (s *simplePrinter) Get(resp v3.GetResponse) { for _, kv := range resp.Kvs { printKV(s.isHex, s.valueOnly, kv) } } func (s *simplePrinter) Put(r v3.PutResponse) { fmt.Println("OK") if r.PrevKv != nil { printKV(s.isHex, s.valueOnly, r.PrevKv) } } func (s *simplePrinter) Txn(resp v3.TxnResponse) { if resp.Succeeded { fmt.Println("SUCCESS") } else { fmt.Println("FAILURE") } for _, r := range resp.Responses { fmt.Println("") switch v := r.Response.(type) { case *pb.ResponseOp_ResponseDeleteRange: s.Del((v3.DeleteResponse)(*v.ResponseDeleteRange)) case *pb.ResponseOp_ResponsePut: s.Put((v3.PutResponse)(*v.ResponsePut)) case *pb.ResponseOp_ResponseRange: s.Get(((v3.GetResponse)(*v.ResponseRange))) default: fmt.Printf("unexpected response %+v\n", r) } } } func (s *simplePrinter) Watch(resp v3.WatchResponse) { for _, e := range resp.Events { fmt.Println(e.Type) if e.PrevKv != nil { printKV(s.isHex, s.valueOnly, e.PrevKv) } printKV(s.isHex, s.valueOnly, e.Kv) } } func (s *simplePrinter) Grant(resp v3.LeaseGrantResponse) { fmt.Printf("lease %016x granted with TTL(%ds)\n", resp.ID, resp.TTL) } func (s *simplePrinter) Revoke(id v3.LeaseID, r v3.LeaseRevokeResponse) { fmt.Printf("lease %016x revoked\n", id) } func (s *simplePrinter) KeepAlive(resp v3.LeaseKeepAliveResponse) { fmt.Printf("lease %016x keepalived with TTL(%d)\n", resp.ID, resp.TTL) } func (s *simplePrinter) TimeToLive(resp v3.LeaseTimeToLiveResponse, keys bool) { if resp.GrantedTTL == 0 && resp.TTL == -1 { fmt.Printf("lease %016x already expired\n", resp.ID) return } txt := fmt.Sprintf("lease %016x granted with TTL(%ds), remaining(%ds)", resp.ID, resp.GrantedTTL, resp.TTL) if keys { ks := make([]string, len(resp.Keys)) for i := range resp.Keys { ks[i] = string(resp.Keys[i]) } txt += fmt.Sprintf(", attached keys(%v)", ks) } fmt.Println(txt) } func (s *simplePrinter) Leases(resp v3.LeaseLeasesResponse) { fmt.Printf("found %d leases\n", len(resp.Leases)) for _, item := range resp.Leases { fmt.Printf("%016x\n", item.ID) } } func (s *simplePrinter) Alarm(resp v3.AlarmResponse) { for _, e := range resp.Alarms { fmt.Printf("%+v\n", e) } } func (s *simplePrinter) MemberAdd(r v3.MemberAddResponse) { asLearner := " " if r.Member.IsLearner { asLearner = " as learner " } fmt.Printf("Member %16x added%sto cluster %16x\n", r.Member.ID, asLearner, r.Header.ClusterId) } func (s *simplePrinter) MemberRemove(id uint64, r v3.MemberRemoveResponse) { fmt.Printf("Member %16x removed from cluster %16x\n", id, r.Header.ClusterId) } func (s *simplePrinter) MemberUpdate(id uint64, r v3.MemberUpdateResponse) { fmt.Printf("Member %16x updated in cluster %16x\n", id, r.Header.ClusterId) } func (s *simplePrinter) MemberPromote(id uint64, r v3.MemberPromoteResponse) { fmt.Printf("Member %16x promoted in cluster %16x\n", id, r.Header.ClusterId) } func (s *simplePrinter) MemberList(resp v3.MemberListResponse) { _, rows := makeMemberListTable(resp) for _, row := range rows { fmt.Println(strings.Join(row, ", ")) } } func (s *simplePrinter) EndpointHealth(hs []epHealth) { for _, h := range hs { if h.Error == "" { fmt.Printf("%s is healthy: successfully committed proposal: took = %v\n", h.Ep, h.Took) } else { fmt.Fprintf(os.Stderr, "%s is unhealthy: failed to commit proposal: %v\n", h.Ep, h.Error) } } } func (s *simplePrinter) EndpointStatus(statusList []epStatus) { _, rows := makeEndpointStatusTable(statusList) for _, row := range rows { fmt.Println(strings.Join(row, ", ")) } } func (s *simplePrinter) EndpointHashKV(hashList []epHashKV) { _, rows := makeEndpointHashKVTable(hashList) for _, row := range rows { fmt.Println(strings.Join(row, ", ")) } } func (s *simplePrinter) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) { fmt.Printf("Leadership transferred from %s to %s\n", types.ID(leader), types.ID(target)) } func (s *simplePrinter) DowngradeValidate(r v3.DowngradeResponse) { fmt.Printf("Downgrade validate success, cluster version %s\n", r.Version) } func (s *simplePrinter) DowngradeEnable(r v3.DowngradeResponse) { fmt.Printf("Downgrade enable success, cluster version %s\n", r.Version) } func (s *simplePrinter) DowngradeCancel(r v3.DowngradeResponse) { fmt.Printf("Downgrade cancel success, cluster version %s\n", r.Version) } func (s *simplePrinter) RoleAdd(role string, r v3.AuthRoleAddResponse) { fmt.Printf("Role %s created\n", role) } func (s *simplePrinter) RoleGet(role string, r v3.AuthRoleGetResponse) { fmt.Printf("Role %s\n", role) if rootRole == role && r.Perm == nil { fmt.Println("KV Read:") fmt.Println("\t[, ") fmt.Println("KV Write:") fmt.Println("\t[, ") return } fmt.Println("KV Read:") printRange := func(perm *v3.Permission) { sKey := string(perm.Key) sRangeEnd := string(perm.RangeEnd) if sRangeEnd != "\x00" { fmt.Printf("\t[%s, %s)", sKey, sRangeEnd) } else { fmt.Printf("\t[%s, ", sKey) } if v3.GetPrefixRangeEnd(sKey) == sRangeEnd && len(sKey) > 0 { fmt.Printf(" (prefix %s)", sKey) } fmt.Print("\n") } for _, perm := range r.Perm { if perm.PermType == v3.PermRead || perm.PermType == v3.PermReadWrite { if len(perm.RangeEnd) == 0 { fmt.Printf("\t%s\n", perm.Key) } else { printRange((*v3.Permission)(perm)) } } } fmt.Println("KV Write:") for _, perm := range r.Perm { if perm.PermType == v3.PermWrite || perm.PermType == v3.PermReadWrite { if len(perm.RangeEnd) == 0 { fmt.Printf("\t%s\n", perm.Key) } else { printRange((*v3.Permission)(perm)) } } } } func (s *simplePrinter) RoleList(r v3.AuthRoleListResponse) { for _, role := range r.Roles { fmt.Printf("%s\n", role) } } func (s *simplePrinter) RoleDelete(role string, r v3.AuthRoleDeleteResponse) { fmt.Printf("Role %s deleted\n", role) } func (s *simplePrinter) RoleGrantPermission(role string, r v3.AuthRoleGrantPermissionResponse) { fmt.Printf("Role %s updated\n", role) } func (s *simplePrinter) RoleRevokePermission(role string, key string, end string, r v3.AuthRoleRevokePermissionResponse) { if len(end) == 0 { fmt.Printf("Permission of key %s is revoked from role %s\n", key, role) return } if end != "\x00" { fmt.Printf("Permission of range [%s, %s) is revoked from role %s\n", key, end, role) } else { fmt.Printf("Permission of range [%s, is revoked from role %s\n", key, role) } } func (s *simplePrinter) UserAdd(name string, r v3.AuthUserAddResponse) { fmt.Printf("User %s created\n", name) } func (s *simplePrinter) UserGet(name string, r v3.AuthUserGetResponse) { fmt.Printf("User: %s\n", name) fmt.Print("Roles:") for _, role := range r.Roles { fmt.Printf(" %s", role) } fmt.Print("\n") } func (s *simplePrinter) UserChangePassword(v3.AuthUserChangePasswordResponse) { fmt.Println("Password updated") } func (s *simplePrinter) UserGrantRole(user string, role string, r v3.AuthUserGrantRoleResponse) { fmt.Printf("Role %s is granted to user %s\n", role, user) } func (s *simplePrinter) UserRevokeRole(user string, role string, r v3.AuthUserRevokeRoleResponse) { fmt.Printf("Role %s is revoked from user %s\n", role, user) } func (s *simplePrinter) UserDelete(user string, r v3.AuthUserDeleteResponse) { fmt.Printf("User %s deleted\n", user) } func (s *simplePrinter) UserList(r v3.AuthUserListResponse) { for _, user := range r.Users { fmt.Printf("%s\n", user) } } func (s *simplePrinter) AuthStatus(r v3.AuthStatusResponse) { fmt.Println("Authentication Status:", r.Enabled) fmt.Println("AuthRevision:", r.AuthRevision) } ================================================ FILE: etcdctl/ctlv3/command/printer_table.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "os" "github.com/olekukonko/tablewriter" "github.com/olekukonko/tablewriter/tw" v3 "go.etcd.io/etcd/client/v3" ) type tablePrinter struct{ printer } func (tp *tablePrinter) MemberList(r v3.MemberListResponse) { hdr, rows := makeMemberListTable(r) cfgBuilder := tablewriter.NewConfigBuilder().WithRowAlignment(tw.AlignRight) table := tablewriter.NewTable(os.Stdout, tablewriter.WithConfig(cfgBuilder.Build())) table.Header(hdr) for _, row := range rows { table.Append(row) } table.Render() } func (tp *tablePrinter) EndpointHealth(r []epHealth) { hdr, rows := makeEndpointHealthTable(r) cfgBuilder := tablewriter.NewConfigBuilder().WithRowAlignment(tw.AlignRight) table := tablewriter.NewTable(os.Stdout, tablewriter.WithConfig(cfgBuilder.Build())) table.Header(hdr) for _, row := range rows { table.Append(row) } table.Render() } func (tp *tablePrinter) EndpointStatus(r []epStatus) { hdr, rows := makeEndpointStatusTable(r) cfgBuilder := tablewriter.NewConfigBuilder().WithRowAlignment(tw.AlignRight) table := tablewriter.NewTable(os.Stdout, tablewriter.WithConfig(cfgBuilder.Build())) table.Header(hdr) for _, row := range rows { table.Append(row) } table.Render() } func (tp *tablePrinter) EndpointHashKV(r []epHashKV) { hdr, rows := makeEndpointHashKVTable(r) cfgBuilder := tablewriter.NewConfigBuilder().WithRowAlignment(tw.AlignRight) table := tablewriter.NewTable(os.Stdout, tablewriter.WithConfig(cfgBuilder.Build())) table.Header(hdr) for _, row := range rows { table.Append(row) } table.Render() } ================================================ FILE: etcdctl/ctlv3/command/put_command.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "os" "strconv" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var ( leaseStr string putPrevKV bool putIgnoreVal bool putIgnoreLease bool ) // NewPutCommand returns the cobra command for "put". func NewPutCommand() *cobra.Command { cmd := &cobra.Command{ Use: "put [options] ( can also be given from stdin)", Short: "Puts the given key into the store", Long: ` Puts the given key into the store. When begins with '-', is interpreted as a flag. Insert '--' for workaround: $ put -- $ put -- If isn't given as a command line argument and '--ignore-value' is not specified, this command tries to read the value from standard input. If isn't given as a command line argument and '--ignore-lease' is not specified, this command tries to read the value from standard input. For example, $ cat file | put will store the content of the file to . `, Run: putCommandFunc, GroupID: groupKVID, } cmd.Flags().StringVar(&leaseStr, "lease", "0", "lease ID (in hexadecimal) to attach to the key") cmd.Flags().BoolVar(&putPrevKV, "prev-kv", false, "return the previous key-value pair before modification") cmd.Flags().BoolVar(&putIgnoreVal, "ignore-value", false, "updates the key using its current value") cmd.Flags().BoolVar(&putIgnoreLease, "ignore-lease", false, "updates the key using its current lease") return cmd } // putCommandFunc executes the "put" command. func putCommandFunc(cmd *cobra.Command, args []string) { key, value, opts := getPutOp(args) ctx, cancel := commandCtx(cmd) resp, err := mustClientFromCmd(cmd).Put(ctx, key, value, opts...) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.Put(*resp) } func getPutOp(args []string) (string, string, []clientv3.OpOption) { if len(args) == 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("put command needs 1 argument and input from stdin or 2 arguments")) } key := args[0] if putIgnoreVal && len(args) > 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("put command needs only 1 argument when 'ignore-value' is set")) } var value string var err error if !putIgnoreVal { value, err = argOrStdin(args, os.Stdin, 1) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("put command needs 1 argument and input from stdin or 2 arguments")) } } id, err := strconv.ParseInt(leaseStr, 16, 64) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("bad lease ID (%w), expecting ID in Hex", err)) } var opts []clientv3.OpOption if id != 0 { opts = append(opts, clientv3.WithLease(clientv3.LeaseID(id))) } if putPrevKV { opts = append(opts, clientv3.WithPrevKV()) } if putIgnoreVal { opts = append(opts, clientv3.WithIgnoreValue()) } if putIgnoreLease { opts = append(opts, clientv3.WithIgnoreLease()) } return key, value, opts } ================================================ FILE: etcdctl/ctlv3/command/role_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "context" "fmt" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var ( rolePermPrefix bool rolePermFromKey bool ) // NewRoleCommand returns the cobra command for "role". func NewRoleCommand() *cobra.Command { ac := &cobra.Command{ Use: "role ", Short: "Role related commands. Use `etcdctl role --help` to see subcommands", Long: "Role related commands", GroupID: groupAuthenticationID, } ac.AddCommand(newRoleAddCommand()) ac.AddCommand(newRoleDeleteCommand()) ac.AddCommand(newRoleGetCommand()) ac.AddCommand(newRoleListCommand()) ac.AddCommand(newRoleGrantPermissionCommand()) ac.AddCommand(newRoleRevokePermissionCommand()) return ac } func newRoleAddCommand() *cobra.Command { return &cobra.Command{ Use: "add ", Short: "Adds a new role", Run: roleAddCommandFunc, } } func newRoleDeleteCommand() *cobra.Command { return &cobra.Command{ Use: "delete ", Short: "Deletes a role", Run: roleDeleteCommandFunc, } } func newRoleGetCommand() *cobra.Command { return &cobra.Command{ Use: "get ", Short: "Gets detailed information of a role", Run: roleGetCommandFunc, } } func newRoleListCommand() *cobra.Command { return &cobra.Command{ Use: "list", Short: "Lists all roles", Run: roleListCommandFunc, } } func newRoleGrantPermissionCommand() *cobra.Command { cmd := &cobra.Command{ Use: "grant-permission [options] [endkey]", Short: "Grants a key to a role", Run: roleGrantPermissionCommandFunc, } cmd.Flags().BoolVar(&rolePermPrefix, "prefix", false, "grant a prefix permission") cmd.Flags().BoolVar(&rolePermFromKey, "from-key", false, "grant a permission of keys that are greater than or equal to the given key using byte compare") return cmd } func newRoleRevokePermissionCommand() *cobra.Command { cmd := &cobra.Command{ Use: "revoke-permission [endkey]", Short: "Revokes a key from a role", Run: roleRevokePermissionCommandFunc, } cmd.Flags().BoolVar(&rolePermPrefix, "prefix", false, "revoke a prefix permission") cmd.Flags().BoolVar(&rolePermFromKey, "from-key", false, "revoke a permission of keys that are greater than or equal to the given key using byte compare") return cmd } // roleAddCommandFunc executes the "role add" command. func roleAddCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("role add command requires role name as its argument")) } resp, err := mustClientFromCmd(cmd).Auth.RoleAdd(context.TODO(), args[0]) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.RoleAdd(args[0], *resp) } // roleDeleteCommandFunc executes the "role delete" command. func roleDeleteCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("role delete command requires role name as its argument")) } resp, err := mustClientFromCmd(cmd).Auth.RoleDelete(context.TODO(), args[0]) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.RoleDelete(args[0], *resp) } // roleGetCommandFunc executes the "role get" command. func roleGetCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("role get command requires role name as its argument")) } name := args[0] resp, err := mustClientFromCmd(cmd).Auth.RoleGet(context.TODO(), name) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.RoleGet(name, *resp) } // roleListCommandFunc executes the "role list" command. func roleListCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("role list command requires no arguments")) } resp, err := mustClientFromCmd(cmd).Auth.RoleList(context.TODO()) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.RoleList(*resp) } // roleGrantPermissionCommandFunc executes the "role grant-permission" command. func roleGrantPermissionCommandFunc(cmd *cobra.Command, args []string) { if len(args) < 3 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("role grant command requires role name, permission type, and key [endkey] as its argument")) } perm, err := clientv3.StrToPermissionType(args[1]) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } key, rangeEnd := permRange(args[2:]) resp, err := mustClientFromCmd(cmd).Auth.RoleGrantPermission(context.TODO(), args[0], key, rangeEnd, perm) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.RoleGrantPermission(args[0], *resp) } // roleRevokePermissionCommandFunc executes the "role revoke-permission" command. func roleRevokePermissionCommandFunc(cmd *cobra.Command, args []string) { if len(args) < 2 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("role revoke-permission command requires role name and key [endkey] as its argument")) } key, rangeEnd := permRange(args[1:]) resp, err := mustClientFromCmd(cmd).Auth.RoleRevokePermission(context.TODO(), args[0], key, rangeEnd) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.RoleRevokePermission(args[0], args[1], rangeEnd, *resp) } func permRange(args []string) (string, string) { key := args[0] var rangeEnd string if len(key) == 0 { if rolePermPrefix && rolePermFromKey { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("--from-key and --prefix flags are mutually exclusive")) } // Range permission is expressed as adt.BytesAffineInterval, // so the empty prefix which should be matched with every key must be like this ["\x00", ). key = "\x00" if rolePermPrefix || rolePermFromKey { // For the both cases of prefix and from-key, a permission with an empty key // should allow access to the entire key space. // 0x00 will be treated as open ended in server side. rangeEnd = "\x00" } } else { var err error rangeEnd, err = rangeEndFromPermFlags(args[0:]) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } } return key, rangeEnd } func rangeEndFromPermFlags(args []string) (string, error) { if len(args) == 1 { if rolePermPrefix { if rolePermFromKey { return "", fmt.Errorf("--from-key and --prefix flags are mutually exclusive") } return clientv3.GetPrefixRangeEnd(args[0]), nil } if rolePermFromKey { return "\x00", nil } // single key case return "", nil } if rolePermPrefix { return "", fmt.Errorf("unexpected endkey argument with --prefix flag") } if rolePermFromKey { return "", fmt.Errorf("unexpected endkey argument with --from-key flag") } return args[1], nil } ================================================ FILE: etcdctl/ctlv3/command/snapshot_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "context" "fmt" "github.com/spf13/cobra" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/logutil" snapshot "go.etcd.io/etcd/client/v3/snapshot" "go.etcd.io/etcd/etcdctl/v3/util" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var snapshotExample = util.Normalize(` # Save snapshot to a given file etcdctl snapshot save /backup/etcd-snapshot.db # Get snapshot from given address and save it to file etcdctl snapshot save --endpoints=127.0.0.1:3000 /backup/etcd-snapshot.db # Get snapshot from given address with certificates etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/etcd/ca.crt --cert=/etc/etcd/etcd.crt --key=/etc/etcd/etcd.key snapshot save /backup/etcd-snapshot.db # Get snapshot with certain user and password etcdctl --user=root --password=password123 snapshot save /backup/etcd-snapshot.db # Get snapshot from given address with timeout etcdctl --endpoints=https://127.0.0.1:2379 --dial-timeout=20s snapshot save /backup/etcd-snapshot.db # Save snapshot with desirable time format etcdctl snapshot save /mnt/backup/etcd/backup_$(date +%Y%m%d_%H%M%S).db`) // NewSnapshotCommand returns the cobra command for "snapshot". func NewSnapshotCommand() *cobra.Command { cmd := &cobra.Command{ Use: "snapshot ", Short: "Manages etcd node snapshots", Example: snapshotExample, GroupID: groupClusterMaintenanceID, } cmd.AddCommand(NewSnapshotSaveCommand()) return cmd } func NewSnapshotSaveCommand() *cobra.Command { return &cobra.Command{ Use: "save ", Short: "Stores an etcd node backend snapshot to a given file", Run: snapshotSaveCommandFunc, Example: snapshotExample, } } func snapshotSaveCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { err := fmt.Errorf("snapshot save expects one argument ") cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } cfg := mustClientCfgFromCmd(cmd) // if user does not specify "--command-timeout" flag, there will be no timeout for snapshot save command ctx, cancel := context.WithCancel(context.Background()) if isCommandTimeoutFlagSet(cmd) { ctx, cancel = commandCtx(cmd) } defer cancel() path := args[0] version, err := snapshot.SaveWithVersion(ctx, lg, *cfg, path) if err != nil { cobrautl.ExitWithError(cobrautl.ExitInterrupted, err) } fmt.Printf("Snapshot saved at %s\n", path) if version != "" { fmt.Printf("Server version %s\n", version) } } ================================================ FILE: etcdctl/ctlv3/command/txn_command.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "bufio" "context" "fmt" "os" "strconv" "strings" "github.com/spf13/cobra" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var txnInteractive bool // NewTxnCommand returns the cobra command for "txn". func NewTxnCommand() *cobra.Command { cmd := &cobra.Command{ Use: "txn [options]", Short: "Txn processes all the requests in one transaction", Long: `Txn reads multiple etcd requests from standard input and applies them as a single atomic transaction. A transaction consists of three components: 1) a list of conditions, 2) a list of requests to apply if all the conditions are true, 3) a list of requests to apply if any condition is false. Example interactive stdin usage: --- etcdctl txn -i # compares: mod("key1") > "0" # success requests (get, put, delete): put key1 "overwrote-key1" # failure requests (get, put, delete): put key1 "created-key1" put key2 "some extra key" --- Refer to https://github.com/etcd-io/etcd/blob/main/etcdctl/README.md#txn-options.`, Run: txnCommandFunc, GroupID: groupKVID, } cmd.Flags().BoolVarP(&txnInteractive, "interactive", "i", false, "Input transaction in interactive mode") return cmd } // txnCommandFunc executes the "txn" command. func txnCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("txn command does not accept argument")) } reader := bufio.NewReader(os.Stdin) txn := mustClientFromCmd(cmd).Txn(context.Background()) promptInteractive("compares:") txn.If(readCompares(reader)...) promptInteractive("success requests (get, put, del):") txn.Then(readOps(reader)...) promptInteractive("failure requests (get, put, del):") txn.Else(readOps(reader)...) resp, err := txn.Commit() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.Txn(*resp) } func promptInteractive(s string) { if txnInteractive { fmt.Println(s) } } func readCompares(r *bufio.Reader) (cmps []clientv3.Cmp) { for { line, err := r.ReadString('\n') if err != nil { cobrautl.ExitWithError(cobrautl.ExitInvalidInput, err) } // remove space from the line line = strings.TrimSpace(line) if len(line) == 0 { break } cmp, err := ParseCompare(line) if err != nil { cobrautl.ExitWithError(cobrautl.ExitInvalidInput, err) } cmps = append(cmps, *cmp) } return cmps } func readOps(r *bufio.Reader) (ops []clientv3.Op) { for { line, err := r.ReadString('\n') if err != nil { cobrautl.ExitWithError(cobrautl.ExitInvalidInput, err) } // remove space from the line line = strings.TrimSpace(line) if len(line) == 0 { break } op, err := parseRequestUnion(line) if err != nil { cobrautl.ExitWithError(cobrautl.ExitInvalidInput, err) } ops = append(ops, *op) } return ops } func parseRequestUnion(line string) (*clientv3.Op, error) { args := Argify(line) if len(args) < 2 { return nil, fmt.Errorf("invalid txn compare request: %s", line) } opc := make(chan clientv3.Op, 1) put := NewPutCommand() put.GroupID = "" put.Run = func(cmd *cobra.Command, args []string) { key, value, opts := getPutOp(args) opc <- clientv3.OpPut(key, value, opts...) } get := NewGetCommand() get.GroupID = "" get.Run = func(cmd *cobra.Command, args []string) { key, opts := getGetOp(args) opc <- clientv3.OpGet(key, opts...) } del := NewDelCommand() del.GroupID = "" del.Run = func(cmd *cobra.Command, args []string) { key, opts := getDelOp(args) opc <- clientv3.OpDelete(key, opts...) } cmds := &cobra.Command{SilenceErrors: true} cmds.AddCommand(put, get, del) cmds.SetArgs(args) if err := cmds.Execute(); err != nil { return nil, fmt.Errorf("invalid txn request: %s", line) } op := <-opc return &op, nil } func ParseCompare(line string) (*clientv3.Cmp, error) { var ( key string op string val string ) lparenSplit := strings.SplitN(line, "(", 2) if len(lparenSplit) != 2 { return nil, fmt.Errorf("malformed comparison: %s", line) } target := lparenSplit[0] n, serr := fmt.Sscanf(lparenSplit[1], "%q) %s %q", &key, &op, &val) if n != 3 { return nil, fmt.Errorf("malformed comparison: %s; got %s(%q) %s %q", line, target, key, op, val) } if serr != nil { return nil, fmt.Errorf("malformed comparison: %s (%w)", line, serr) } var ( v int64 err error cmp clientv3.Cmp ) switch target { case "ver", "version": if v, err = strconv.ParseInt(val, 10, 64); err == nil { cmp = clientv3.Compare(clientv3.Version(key), op, v) } case "c", "create": if v, err = strconv.ParseInt(val, 10, 64); err == nil { cmp = clientv3.Compare(clientv3.CreateRevision(key), op, v) } case "m", "mod": if v, err = strconv.ParseInt(val, 10, 64); err == nil { cmp = clientv3.Compare(clientv3.ModRevision(key), op, v) } case "val", "value": cmp = clientv3.Compare(clientv3.Value(key), op, val) case "lease": cmp = clientv3.Compare(clientv3.Cmp{Target: pb.Compare_LEASE}, op, val) default: return nil, fmt.Errorf("malformed comparison: %s (unknown target %s)", line, target) } if err != nil { return nil, fmt.Errorf("invalid txn compare request: %s", line) } return &cmp, nil } ================================================ FILE: etcdctl/ctlv3/command/user_command.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "context" "fmt" "strings" "github.com/bgentry/speakeasy" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var userShowDetail bool // NewUserCommand returns the cobra command for "user". func NewUserCommand() *cobra.Command { ac := &cobra.Command{ Use: "user ", Short: "User related commands. Use `etcdctl user --help` to see subcommands", Long: "User related commands", GroupID: groupAuthenticationID, } ac.AddCommand(newUserAddCommand()) ac.AddCommand(newUserDeleteCommand()) ac.AddCommand(newUserGetCommand()) ac.AddCommand(newUserListCommand()) ac.AddCommand(newUserChangePasswordCommand()) ac.AddCommand(newUserGrantRoleCommand()) ac.AddCommand(newUserRevokeRoleCommand()) return ac } var ( passwordInteractive bool passwordFromFlag string noPassword bool ) func newUserAddCommand() *cobra.Command { cmd := cobra.Command{ Use: "add [options]", Short: "Adds a new user", Run: userAddCommandFunc, } cmd.Flags().BoolVar(&passwordInteractive, "interactive", true, "Read password from stdin instead of interactive terminal") cmd.Flags().StringVar(&passwordFromFlag, "new-user-password", "", "Supply password from the command line flag") cmd.Flags().BoolVar(&noPassword, "no-password", false, "Create a user without password (CN based auth only)") return &cmd } func newUserDeleteCommand() *cobra.Command { return &cobra.Command{ Use: "delete ", Short: "Deletes a user", Run: userDeleteCommandFunc, } } func newUserGetCommand() *cobra.Command { cmd := cobra.Command{ Use: "get [options]", Short: "Gets detailed information of a user", Run: userGetCommandFunc, } cmd.Flags().BoolVar(&userShowDetail, "detail", false, "Show permissions of roles granted to the user") return &cmd } func newUserListCommand() *cobra.Command { return &cobra.Command{ Use: "list", Short: "Lists all users", Run: userListCommandFunc, } } func newUserChangePasswordCommand() *cobra.Command { cmd := cobra.Command{ Use: "passwd [options]", Short: "Changes password of user", Run: userChangePasswordCommandFunc, } cmd.Flags().BoolVar(&passwordInteractive, "interactive", true, "If true, read password from stdin instead of interactive terminal") return &cmd } func newUserGrantRoleCommand() *cobra.Command { return &cobra.Command{ Use: "grant-role ", Short: "Grants a role to a user", Run: userGrantRoleCommandFunc, } } func newUserRevokeRoleCommand() *cobra.Command { return &cobra.Command{ Use: "revoke-role ", Short: "Revokes a role from a user", Run: userRevokeRoleCommandFunc, } } // userAddCommandFunc executes the "user add" command. func userAddCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("user add command requires user name as its argument")) } var password string var user string options := &clientv3.UserAddOptions{ NoPassword: false, } if !noPassword { if passwordFromFlag != "" { user = args[0] password = passwordFromFlag } else { splitted := strings.SplitN(args[0], ":", 2) if len(splitted) < 2 { user = args[0] if !passwordInteractive { fmt.Scanf("%s", &password) } else { password = readPasswordInteractive(args[0]) } } else { user = splitted[0] password = splitted[1] if len(user) == 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("empty user name is not allowed")) } } } } else { user = args[0] options.NoPassword = true } resp, err := mustClientFromCmd(cmd).Auth.UserAddWithOptions(context.TODO(), user, password, options) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.UserAdd(user, *resp) } // userDeleteCommandFunc executes the "user delete" command. func userDeleteCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("user delete command requires user name as its argument")) } resp, err := mustClientFromCmd(cmd).Auth.UserDelete(context.TODO(), args[0]) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.UserDelete(args[0], *resp) } // userGetCommandFunc executes the "user get" command. func userGetCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("user get command requires user name as its argument")) } name := args[0] client := mustClientFromCmd(cmd) resp, err := client.Auth.UserGet(context.TODO(), name) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } if userShowDetail { fmt.Printf("User: %s\n", name) for _, role := range resp.Roles { fmt.Print("\n") roleResp, err := client.Auth.RoleGet(context.TODO(), role) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.RoleGet(role, *roleResp) } } else { display.UserGet(name, *resp) } } // userListCommandFunc executes the "user list" command. func userListCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("user list command requires no arguments")) } resp, err := mustClientFromCmd(cmd).Auth.UserList(context.TODO()) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.UserList(*resp) } // userChangePasswordCommandFunc executes the "user passwd" command. func userChangePasswordCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("user passwd command requires user name as its argument")) } var password string if !passwordInteractive { fmt.Scanf("%s", &password) } else { password = readPasswordInteractive(args[0]) } resp, err := mustClientFromCmd(cmd).Auth.UserChangePassword(context.TODO(), args[0], password) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.UserChangePassword(*resp) } // userGrantRoleCommandFunc executes the "user grant-role" command. func userGrantRoleCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 2 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("user grant command requires user name and role name as its argument")) } resp, err := mustClientFromCmd(cmd).Auth.UserGrantRole(context.TODO(), args[0], args[1]) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.UserGrantRole(args[0], args[1], *resp) } // userRevokeRoleCommandFunc executes the "user revoke-role" command. func userRevokeRoleCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 2 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("user revoke-role requires user name and role name as its argument")) } resp, err := mustClientFromCmd(cmd).Auth.UserRevokeRole(context.TODO(), args[0], args[1]) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } display.UserRevokeRole(args[0], args[1], *resp) } func readPasswordInteractive(name string) string { prompt1 := fmt.Sprintf("Password of %s: ", name) password1, err1 := speakeasy.Ask(prompt1) if err1 != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("failed to ask password: %w", err1)) } if len(password1) == 0 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("empty password")) } prompt2 := fmt.Sprintf("Type password of %s again for confirmation: ", name) password2, err2 := speakeasy.Ask(prompt2) if err2 != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("failed to ask password: %w", err2)) } if password1 != password2 { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("given passwords are different")) } return password1 } ================================================ FILE: etcdctl/ctlv3/command/util.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "context" "crypto/tls" "encoding/hex" "fmt" "io" "net/http" "regexp" "strconv" "strings" "time" "github.com/spf13/cobra" pb "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) func printKV(isHex bool, valueOnly bool, kv *pb.KeyValue) { k, v := string(kv.Key), string(kv.Value) if isHex { k = addHexPrefix(hex.EncodeToString(kv.Key)) v = addHexPrefix(hex.EncodeToString(kv.Value)) } if !valueOnly { fmt.Println(k) } fmt.Println(v) } func addHexPrefix(s string) string { ns := make([]byte, len(s)*2) for i := 0; i < len(s); i += 2 { ns[i*2] = '\\' ns[i*2+1] = 'x' ns[i*2+2] = s[i] ns[i*2+3] = s[i+1] } return string(ns) } var argsRegexp = regexp.MustCompile(`"(?:[^"\\]|\\.)*"|'[^']*'|[^'"\s]\S*[^'"\s]?`) func Argify(s string) []string { args := argsRegexp.FindAllString(s, -1) for i := range args { if len(args[i]) == 0 { continue } if args[i][0] == '\'' { // 'single-quoted string' args[i] = args[i][1 : len(args[i])-1] } else if args[i][0] == '"' { // "double quoted string" if _, err := fmt.Sscanf(args[i], "%q", &args[i]); err != nil { cobrautl.ExitWithError(cobrautl.ExitInvalidInput, err) } } } return args } func commandCtx(cmd *cobra.Command) (context.Context, context.CancelFunc) { timeOut, err := cmd.Flags().GetDuration("command-timeout") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } return context.WithTimeout(context.Background(), timeOut) } func isCommandTimeoutFlagSet(cmd *cobra.Command) bool { commandTimeoutFlag := cmd.Flags().Lookup("command-timeout") if commandTimeoutFlag == nil { panic("expect command-timeout flag to exist") } return commandTimeoutFlag.Changed } // get the process_resident_memory_bytes from /metrics func endpointMemoryMetrics(host string, scfg *clientv3.SecureConfig) float64 { residentMemoryKey := "process_resident_memory_bytes" var residentMemoryValue string if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") { host = "http://" + host } url := host + "/metrics" if strings.HasPrefix(host, "https://") { // load client certificate cert, err := tls.LoadX509KeyPair(scfg.Cert, scfg.Key) if err != nil { fmt.Printf("client certificate error: %v\n", err) return 0.0 } http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: scfg.InsecureSkipVerify, } } resp, err := http.Get(url) if err != nil { fmt.Printf("fetch error: %v\n", err) return 0.0 } byts, readerr := io.ReadAll(resp.Body) resp.Body.Close() if readerr != nil { fmt.Printf("fetch error: reading %s: %v\n", url, readerr) return 0.0 } for _, line := range strings.Split(string(byts), "\n") { if strings.HasPrefix(line, residentMemoryKey) { residentMemoryValue = strings.TrimSpace(strings.TrimPrefix(line, residentMemoryKey)) break } } if residentMemoryValue == "" { fmt.Printf("could not find: %v\n", residentMemoryKey) return 0.0 } residentMemoryBytes, parseErr := strconv.ParseFloat(residentMemoryValue, 64) if parseErr != nil { fmt.Printf("parse error: %v\n", parseErr) return 0.0 } return residentMemoryBytes } // compact keyspace history to a provided revision func compact(c *clientv3.Client, rev int64) { fmt.Printf("Compacting with revision %d\n", rev) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) _, err := c.Compact(ctx, rev, clientv3.WithCompactPhysical()) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } fmt.Printf("Compacted with revision %d\n", rev) } // defrag a given endpoint func defrag(c *clientv3.Client, ep string) { fmt.Printf("Defragmenting %q\n", ep) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) _, err := c.Defragment(ctx, ep) cancel() if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } fmt.Printf("Defragmented %q\n", ep) } func IsSerializable(option string) bool { switch option { case "s": return true case "l": default: cobrautl.ExitWithError(cobrautl.ExitBadFeature, fmt.Errorf("unknown consistency flag %q", getConsistency)) } return false } ================================================ FILE: etcdctl/ctlv3/command/util_test.go ================================================ // Copyright 2026 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "reflect" "testing" ) func TestArgify(t *testing.T) { tests := []struct { name string input string expected []string }{ { name: "empty string", input: "", expected: nil, }, { name: "simple args", input: "foo bar baz", expected: []string{"foo", "bar", "baz"}, }, { name: "single-quoted string", input: "'hello world'", expected: []string{"hello world"}, }, { name: "double-quoted string", input: `"hello world"`, expected: []string{"hello world"}, }, { name: "mixed args", input: "put 'my key' 'my value'", expected: []string{"put", "my key", "my value"}, }, { name: "empty single-quoted string", input: "''", expected: []string{""}, }, { name: "empty double-quoted string", input: `""`, expected: []string{""}, }, { name: "double-quoted with escape", input: `"hello\"world"`, expected: []string{`hello"world`}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := Argify(tt.input) if !reflect.DeepEqual(result, tt.expected) { t.Errorf("Argify(%q) = %v, want %v", tt.input, result, tt.expected) } }) } } ================================================ FILE: etcdctl/ctlv3/command/version_command.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "github.com/spf13/cobra" "go.etcd.io/etcd/api/v3/version" ) // NewVersionCommand prints out the version of etcd. func NewVersionCommand() *cobra.Command { return &cobra.Command{ Use: "version", Short: "Prints the version of etcdctl", Run: versionCommandFunc, GroupID: groupUtilityID, } } func versionCommandFunc(cmd *cobra.Command, args []string) { fmt.Println("etcdctl version:", version.Version) fmt.Println("API version:", version.APIVersion) } ================================================ FILE: etcdctl/ctlv3/command/watch_command.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "bufio" "context" "errors" "fmt" "os" "os/exec" "strings" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var ( errBadArgsNum = errors.New("bad number of arguments") errBadArgsNumConflictEnv = errors.New("bad number of arguments (found conflicting environment key)") errBadArgsNumSeparator = errors.New("bad number of arguments (found separator --, but no commands)") errBadArgsInteractiveWatch = errors.New("args[0] must be 'watch' for interactive calls") ) var ( watchRev int64 watchPrefix bool watchInteractive bool watchPrevKey bool progressNotify bool ) // NewWatchCommand returns the cobra command for "watch". func NewWatchCommand() *cobra.Command { cmd := &cobra.Command{ Use: "watch [options] [key or prefix] [range_end] [--] [exec-command arg1 arg2 ...]", Short: "Watches events stream on keys or prefixes", Run: watchCommandFunc, GroupID: groupKVID, } cmd.Flags().BoolVarP(&watchInteractive, "interactive", "i", false, "Interactive mode") cmd.Flags().BoolVar(&watchPrefix, "prefix", false, "Watch on a prefix if prefix is set") cmd.Flags().Int64Var(&watchRev, "rev", 0, "Revision to start watching") cmd.Flags().BoolVar(&watchPrevKey, "prev-kv", false, "get the previous key-value pair before the event happens") cmd.Flags().BoolVar(&progressNotify, "progress-notify", false, "get periodic watch progress notification from server") return cmd } // watchCommandFunc executes the "watch" command. func watchCommandFunc(cmd *cobra.Command, args []string) { envKey, envRange := os.Getenv("ETCDCTL_WATCH_KEY"), os.Getenv("ETCDCTL_WATCH_RANGE_END") if envKey == "" && envRange != "" { cobrautl.ExitWithError(cobrautl.ExitBadArgs, fmt.Errorf("ETCDCTL_WATCH_KEY is empty but got ETCDCTL_WATCH_RANGE_END=%q", envRange)) } if watchInteractive { watchInteractiveFunc(cmd, os.Args, envKey, envRange) return } watchArgs, execArgs, err := parseWatchArgs(os.Args, args, envKey, envRange, false) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } c := mustClientFromCmd(cmd) wc, err := getWatchChan(c, watchArgs) if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } printWatchCh(c, wc, execArgs) if err = c.Close(); err != nil { cobrautl.ExitWithError(cobrautl.ExitBadConnection, err) } cobrautl.ExitWithError(cobrautl.ExitInterrupted, fmt.Errorf("watch is canceled by the server")) } func watchInteractiveFunc(cmd *cobra.Command, osArgs []string, envKey, envRange string) { c := mustClientFromCmd(cmd) reader := bufio.NewReader(os.Stdin) for { l, err := reader.ReadString('\n') if err != nil { cobrautl.ExitWithError(cobrautl.ExitInvalidInput, fmt.Errorf("error reading watch request line: %w", err)) } l = strings.TrimSuffix(l, "\n") args := Argify(l) if len(args) < 1 { fmt.Fprintf(os.Stderr, "Invalid command: %s (watch and progress supported)\n", l) continue } switch args[0] { case "watch": if len(args) < 2 && envKey == "" { fmt.Fprintf(os.Stderr, "Invalid command %s (command type or key is not provided)\n", l) continue } watchArgs, execArgs, perr := parseWatchArgs(osArgs, args, envKey, envRange, true) if perr != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, perr) } ch, err := getWatchChan(c, watchArgs) if err != nil { fmt.Fprintf(os.Stderr, "Invalid command %s (%v)\n", l, err) continue } go printWatchCh(c, ch, execArgs) case "progress": err := c.RequestProgress(clientv3.WithRequireLeader(context.Background())) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } default: fmt.Fprintf(os.Stderr, "Invalid command %s (only support watch)\n", l) continue } } } func getWatchChan(c *clientv3.Client, args []string) (clientv3.WatchChan, error) { if len(args) < 1 { return nil, errBadArgsNum } key := args[0] opts := []clientv3.OpOption{clientv3.WithRev(watchRev)} if len(args) == 2 { if watchPrefix { return nil, fmt.Errorf("`range_end` and `--prefix` are mutually exclusive") } opts = append(opts, clientv3.WithRange(args[1])) } if watchPrefix { opts = append(opts, clientv3.WithPrefix()) } if watchPrevKey { opts = append(opts, clientv3.WithPrevKV()) } if progressNotify { opts = append(opts, clientv3.WithProgressNotify()) } return c.Watch(clientv3.WithRequireLeader(context.Background()), key, opts...), nil } func printWatchCh(c *clientv3.Client, ch clientv3.WatchChan, execArgs []string) { for resp := range ch { if resp.Canceled { fmt.Fprintf(os.Stderr, "watch was canceled (%v)\n", resp.Err()) } if resp.IsProgressNotify() { fmt.Fprintf(os.Stdout, "progress notify: %d\n", resp.Header.Revision) } display.Watch(resp) if len(execArgs) > 0 { for _, ev := range resp.Events { cmd := exec.CommandContext(c.Ctx(), execArgs[0], execArgs[1:]...) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf("ETCD_WATCH_REVISION=%d", resp.Header.Revision)) cmd.Env = append(cmd.Env, fmt.Sprintf("ETCD_WATCH_EVENT_TYPE=%q", ev.Type)) cmd.Env = append(cmd.Env, fmt.Sprintf("ETCD_WATCH_KEY=%q", ev.Kv.Key)) cmd.Env = append(cmd.Env, fmt.Sprintf("ETCD_WATCH_VALUE=%q", ev.Kv.Value)) cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr if err := cmd.Run(); err != nil { fmt.Fprintf(os.Stderr, "command %q error (%v)\n", execArgs, err) os.Exit(1) } } } } } // "commandArgs" is the command arguments after "spf13/cobra" parses // all "watch" command flags, strips out special characters (e.g. "--"). // "orArgs" is the raw arguments passed to "watch" command // (e.g. ./bin/etcdctl watch foo --rev 1 bar). // "--" characters are invalid arguments for "spf13/cobra" library, // so no need to handle such cases. func parseWatchArgs(osArgs, commandArgs []string, envKey, envRange string, interactive bool) (watchArgs []string, execArgs []string, err error) { rawArgs := make([]string, len(osArgs)) copy(rawArgs, osArgs) watchArgs = make([]string, len(commandArgs)) copy(watchArgs, commandArgs) // remove preceding commands (e.g. ./bin/etcdctl watch) // handle "./bin/etcdctl watch foo -- echo watch event" for idx := range rawArgs { if rawArgs[idx] == "watch" { rawArgs = rawArgs[idx+1:] break } } // remove preceding commands (e.g. "watch foo bar" in interactive mode) // handle "./bin/etcdctl watch foo -- echo watch event" if interactive { if watchArgs[0] != "watch" { // "watch" not found watchPrefix, watchRev, watchPrevKey = false, 0, false return nil, nil, errBadArgsInteractiveWatch } watchArgs = watchArgs[1:] } execIdx, execExist := 0, false if !interactive { for execIdx = range rawArgs { if rawArgs[execIdx] == "--" { execExist = true break } } if execExist && execIdx == len(rawArgs)-1 { // "watch foo bar --" should error return nil, nil, errBadArgsNumSeparator } // "watch" with no argument should error if !execExist && len(rawArgs) < 1 && envKey == "" { return nil, nil, errBadArgsNum } if execExist && envKey != "" { // "ETCDCTL_WATCH_KEY=foo watch foo -- echo 1" should error // (watchArgs==["foo","echo","1"]) widx, ridx := len(watchArgs)-1, len(rawArgs)-1 for ; widx >= 0; widx-- { if watchArgs[widx] == rawArgs[ridx] { ridx-- continue } // watchArgs has extra: // ETCDCTL_WATCH_KEY=foo watch foo -- echo 1 // watchArgs: foo echo 1 if ridx == execIdx { return nil, nil, errBadArgsNumConflictEnv } } } // check conflicting arguments // e.g. "watch --rev 1 -- echo Hello World" has no conflict if !execExist && len(watchArgs) > 0 && envKey != "" { // "ETCDCTL_WATCH_KEY=foo watch foo" should error // (watchArgs==["foo"]) return nil, nil, errBadArgsNumConflictEnv } } else { for execIdx = range watchArgs { if watchArgs[execIdx] == "--" { execExist = true break } } if execExist && execIdx == len(watchArgs)-1 { // "watch foo bar --" should error watchPrefix, watchRev, watchPrevKey = false, 0, false return nil, nil, errBadArgsNumSeparator } flagset := NewWatchCommand().Flags() if perr := flagset.Parse(watchArgs); perr != nil { watchPrefix, watchRev, watchPrevKey = false, 0, false return nil, nil, perr } pArgs := flagset.Args() // "watch" with no argument should error if !execExist && envKey == "" && len(pArgs) < 1 { watchPrefix, watchRev, watchPrevKey = false, 0, false return nil, nil, errBadArgsNum } // check conflicting arguments // e.g. "watch --rev 1 -- echo Hello World" has no conflict if !execExist && len(pArgs) > 0 && envKey != "" { // "ETCDCTL_WATCH_KEY=foo watch foo" should error // (watchArgs==["foo"]) watchPrefix, watchRev, watchPrevKey = false, 0, false return nil, nil, errBadArgsNumConflictEnv } } argsWithSep := rawArgs if interactive { // interactive mode directly passes "--" to the command args argsWithSep = watchArgs } idx, foundSep := 0, false for idx = range argsWithSep { if argsWithSep[idx] == "--" { foundSep = true break } } if foundSep { execArgs = argsWithSep[idx+1:] } if interactive { flagset := NewWatchCommand().Flags() if perr := flagset.Parse(argsWithSep); perr != nil { return nil, nil, perr } watchArgs = flagset.Args() watchPrefix, err = flagset.GetBool("prefix") if err != nil { return nil, nil, err } watchRev, err = flagset.GetInt64("rev") if err != nil { return nil, nil, err } watchPrevKey, err = flagset.GetBool("prev-kv") if err != nil { return nil, nil, err } } // "ETCDCTL_WATCH_KEY=foo watch -- echo hello" // should translate "watch foo -- echo hello" // (watchArgs=["echo","hello"] should be ["foo","echo","hello"]) if envKey != "" { ranges := []string{envKey} if envRange != "" { ranges = append(ranges, envRange) } watchArgs = append(ranges, watchArgs...) } if !foundSep { return watchArgs, nil, nil } // "watch foo bar --rev 1 -- echo hello" or "watch foo --rev 1 bar -- echo hello", // then "watchArgs" is "foo bar echo hello" // so need ignore args after "argsWithSep[idx]", which is "--" endIdx := 0 for endIdx = len(watchArgs) - 1; endIdx >= 0; endIdx-- { if watchArgs[endIdx] == argsWithSep[idx+1] { break } } watchArgs = watchArgs[:endIdx] return watchArgs, execArgs, nil } ================================================ FILE: etcdctl/ctlv3/command/watch_command_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "reflect" "testing" "github.com/stretchr/testify/require" ) func Test_parseWatchArgs(t *testing.T) { tt := []struct { osArgs []string // raw arguments to "watch" command commandArgs []string // arguments after "spf13/cobra" preprocessing envKey, envRange string interactive bool interactiveWatchPrefix bool interactiveWatchRev int64 interactiveWatchPrevKey bool watchArgs []string execArgs []string err error }{ { osArgs: []string{"./bin/etcdctl", "watch", "foo", "bar"}, commandArgs: []string{"foo", "bar"}, interactive: false, watchArgs: []string{"foo", "bar"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "bar", "--"}, commandArgs: []string{"foo", "bar"}, interactive: false, watchArgs: nil, execArgs: nil, err: errBadArgsNumSeparator, }, { osArgs: []string{"./bin/etcdctl", "watch"}, commandArgs: nil, envKey: "foo", envRange: "bar", interactive: false, watchArgs: []string{"foo", "bar"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo"}, commandArgs: []string{"foo"}, envKey: "foo", envRange: "", interactive: false, watchArgs: nil, execArgs: nil, err: errBadArgsNumConflictEnv, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "bar"}, commandArgs: []string{"foo", "bar"}, envKey: "foo", envRange: "", interactive: false, watchArgs: nil, execArgs: nil, err: errBadArgsNumConflictEnv, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "bar"}, commandArgs: []string{"foo", "bar"}, envKey: "foo", envRange: "bar", interactive: false, watchArgs: nil, execArgs: nil, err: errBadArgsNumConflictEnv, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo"}, commandArgs: []string{"foo"}, interactive: false, watchArgs: []string{"foo"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch"}, commandArgs: nil, envKey: "foo", interactive: false, watchArgs: []string{"foo"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "--rev", "1", "foo"}, commandArgs: []string{"foo"}, interactive: false, watchArgs: []string{"foo"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "--rev", "1", "foo"}, commandArgs: []string{"foo"}, envKey: "foo", interactive: false, watchArgs: nil, execArgs: nil, err: errBadArgsNumConflictEnv, }, { osArgs: []string{"./bin/etcdctl", "watch", "--rev", "1"}, commandArgs: nil, envKey: "foo", interactive: false, watchArgs: []string{"foo"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "--rev", "1"}, commandArgs: []string{"foo"}, interactive: false, watchArgs: []string{"foo"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "--", "echo", "Hello", "World"}, commandArgs: []string{"foo", "echo", "Hello", "World"}, interactive: false, watchArgs: []string{"foo"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "--", "echo", "watch", "event", "received"}, commandArgs: []string{"foo", "echo", "watch", "event", "received"}, interactive: false, watchArgs: []string{"foo"}, execArgs: []string{"echo", "watch", "event", "received"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "--rev", "1", "--", "echo", "Hello", "World"}, commandArgs: []string{"foo", "echo", "Hello", "World"}, interactive: false, watchArgs: []string{"foo"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "--rev", "1", "--", "echo", "watch", "event", "received"}, commandArgs: []string{"foo", "echo", "watch", "event", "received"}, interactive: false, watchArgs: []string{"foo"}, execArgs: []string{"echo", "watch", "event", "received"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "--rev", "1", "foo", "--", "echo", "watch", "event", "received"}, commandArgs: []string{"foo", "echo", "watch", "event", "received"}, interactive: false, watchArgs: []string{"foo"}, execArgs: []string{"echo", "watch", "event", "received"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "bar", "--", "echo", "Hello", "World"}, commandArgs: []string{"foo", "bar", "echo", "Hello", "World"}, interactive: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "--rev", "1", "foo", "bar", "--", "echo", "Hello", "World"}, commandArgs: []string{"foo", "bar", "echo", "Hello", "World"}, interactive: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "--rev", "1", "bar", "--", "echo", "Hello", "World"}, commandArgs: []string{"foo", "bar", "echo", "Hello", "World"}, interactive: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "bar", "--rev", "1", "--", "echo", "Hello", "World"}, commandArgs: []string{"foo", "bar", "echo", "Hello", "World"}, interactive: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "bar", "--rev", "1", "--", "echo", "watch", "event", "received"}, commandArgs: []string{"foo", "bar", "echo", "watch", "event", "received"}, interactive: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "watch", "event", "received"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "--rev", "1", "bar", "--", "echo", "Hello", "World"}, commandArgs: []string{"foo", "bar", "echo", "Hello", "World"}, interactive: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "--rev", "1", "foo", "bar", "--", "echo", "Hello", "World"}, commandArgs: []string{"foo", "bar", "echo", "Hello", "World"}, interactive: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "--rev", "1", "--", "echo", "Hello", "World"}, commandArgs: []string{"echo", "Hello", "World"}, envKey: "foo", envRange: "", interactive: false, watchArgs: []string{"foo"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "--rev", "1", "--", "echo", "Hello", "World"}, commandArgs: []string{"echo", "Hello", "World"}, envKey: "foo", envRange: "bar", interactive: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "foo", "bar", "--rev", "1", "--", "echo", "Hello", "World"}, commandArgs: []string{"foo", "bar", "echo", "Hello", "World"}, envKey: "foo", interactive: false, watchArgs: nil, execArgs: nil, err: errBadArgsNumConflictEnv, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"foo", "bar", "--", "echo", "Hello", "World"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 0, interactiveWatchPrevKey: false, watchArgs: nil, execArgs: nil, err: errBadArgsInteractiveWatch, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "foo"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 0, interactiveWatchPrevKey: false, watchArgs: []string{"foo"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "foo", "bar"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 0, interactiveWatchPrevKey: false, watchArgs: []string{"foo", "bar"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch"}, envKey: "foo", envRange: "bar", interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 0, interactiveWatchPrevKey: false, watchArgs: []string{"foo", "bar"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch"}, envKey: "hello world!", envRange: "bar", interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 0, interactiveWatchPrevKey: false, watchArgs: []string{"hello world!", "bar"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "foo", "--rev", "1"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 1, interactiveWatchPrevKey: false, watchArgs: []string{"foo"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "foo", "--rev", "1", "--", "echo", "Hello", "World"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 1, interactiveWatchPrevKey: false, watchArgs: []string{"foo"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "--rev", "1", "foo", "--", "echo", "Hello", "World"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 1, interactiveWatchPrevKey: false, watchArgs: []string{"foo"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "--rev", "5", "--prev-kv", "foo", "--", "echo", "Hello", "World"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 5, interactiveWatchPrevKey: true, watchArgs: []string{"foo"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "--rev", "1"}, envKey: "foo", interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 1, interactiveWatchPrevKey: false, watchArgs: []string{"foo"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "--rev", "1"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 0, interactiveWatchPrevKey: false, watchArgs: nil, execArgs: nil, err: errBadArgsNum, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "--rev", "1", "--prefix"}, envKey: "foo", interactive: true, interactiveWatchPrefix: true, interactiveWatchRev: 1, interactiveWatchPrevKey: false, watchArgs: []string{"foo"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "--rev", "100", "--prefix", "--prev-kv"}, envKey: "foo", interactive: true, interactiveWatchPrefix: true, interactiveWatchRev: 100, interactiveWatchPrevKey: true, watchArgs: []string{"foo"}, execArgs: nil, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "--rev", "1", "--prefix"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 0, interactiveWatchPrevKey: false, watchArgs: nil, execArgs: nil, err: errBadArgsNum, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "--", "echo", "Hello", "World"}, envKey: "foo", interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 0, interactiveWatchPrevKey: false, watchArgs: []string{"foo"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "--", "echo", "Hello", "World"}, envKey: "foo", envRange: "bar", interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 0, interactiveWatchPrevKey: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "foo", "bar", "--", "echo", "Hello", "World"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 0, interactiveWatchPrevKey: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "--rev", "1", "foo", "bar", "--", "echo", "Hello", "World"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 1, interactiveWatchPrevKey: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "--rev", "1", "--", "echo", "Hello", "World"}, envKey: "foo", envRange: "bar", interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 1, interactiveWatchPrevKey: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "foo", "--rev", "1", "bar", "--", "echo", "Hello", "World"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 1, interactiveWatchPrevKey: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "foo", "bar", "--rev", "1", "--", "echo", "Hello", "World"}, interactive: true, interactiveWatchPrefix: false, interactiveWatchRev: 1, interactiveWatchPrevKey: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "foo", "bar", "--rev", "7", "--prefix", "--", "echo", "Hello", "World"}, interactive: true, interactiveWatchPrefix: true, interactiveWatchRev: 7, interactiveWatchPrevKey: false, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, { osArgs: []string{"./bin/etcdctl", "watch", "-i"}, commandArgs: []string{"watch", "foo", "bar", "--rev", "7", "--prefix", "--prev-kv", "--", "echo", "Hello", "World"}, interactive: true, interactiveWatchPrefix: true, interactiveWatchRev: 7, interactiveWatchPrevKey: true, watchArgs: []string{"foo", "bar"}, execArgs: []string{"echo", "Hello", "World"}, err: nil, }, } for i, ts := range tt { watchArgs, execArgs, err := parseWatchArgs(ts.osArgs, ts.commandArgs, ts.envKey, ts.envRange, ts.interactive) require.ErrorIsf(t, err, ts.err, "#%d: error expected %v, got %v", i, ts.err, err) require.Truef(t, reflect.DeepEqual(watchArgs, ts.watchArgs), "#%d: watchArgs expected %q, got %v", i, ts.watchArgs, watchArgs) require.Truef(t, reflect.DeepEqual(execArgs, ts.execArgs), "#%d: execArgs expected %q, got %v", i, ts.execArgs, execArgs) if ts.interactive { require.Equalf(t, ts.interactiveWatchPrefix, watchPrefix, "#%d: interactive watchPrefix expected %v, got %v", i, ts.interactiveWatchPrefix, watchPrefix) require.Equalf(t, ts.interactiveWatchRev, watchRev, "#%d: interactive watchRev expected %d, got %d", i, ts.interactiveWatchRev, watchRev) require.Equalf(t, ts.interactiveWatchPrevKey, watchPrevKey, "#%d: interactive watchPrevKey expected %v, got %v", i, ts.interactiveWatchPrevKey, watchPrevKey) } } } ================================================ FILE: etcdctl/ctlv3/ctl.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package ctlv3 contains the main entry point for the etcdctl for v3 API. package ctlv3 import ( "fmt" "os" "time" "github.com/spf13/cobra" "github.com/spf13/pflag" "go.etcd.io/etcd/etcdctl/v3/ctlv3/command" "go.etcd.io/etcd/pkg/v3/cobrautl" ) const ( cliName = "etcdctl" cliDescription = "A simple command line client for etcd3." defaultDialTimeout = 2 * time.Second defaultCommandTimeOut = 5 * time.Second defaultKeepAliveTime = 2 * time.Second defaultKeepAliveTimeOut = 6 * time.Second ) var ( globalFlags = command.GlobalFlags{} rootCmd = &cobra.Command{ Use: cliName, Short: cliDescription, SuggestFor: []string{"etcdctl"}, } ) func init() { rootCmd.PersistentFlags().StringSliceVar(&globalFlags.Endpoints, "endpoints", []string{"127.0.0.1:2379"}, "gRPC endpoints") rootCmd.PersistentFlags().BoolVar(&globalFlags.Debug, "debug", false, "enable client-side debug logging") rootCmd.PersistentFlags().StringVarP(&globalFlags.OutputFormat, "write-out", "w", "simple", "set the output format (fields, json, protobuf, simple, table)") rootCmd.PersistentFlags().BoolVar(&globalFlags.IsHex, "hex", false, "print byte strings as hex encoded strings") rootCmd.RegisterFlagCompletionFunc("write-out", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { return []string{"fields", "json", "protobuf", "simple", "table"}, cobra.ShellCompDirectiveDefault }) rootCmd.PersistentFlags().DurationVar(&globalFlags.DialTimeout, "dial-timeout", defaultDialTimeout, "dial timeout for client connections") rootCmd.PersistentFlags().DurationVar(&globalFlags.CommandTimeOut, "command-timeout", defaultCommandTimeOut, "timeout for short running command (excluding dial timeout)") rootCmd.PersistentFlags().DurationVar(&globalFlags.KeepAliveTime, "keepalive-time", defaultKeepAliveTime, "keepalive time for client connections") rootCmd.PersistentFlags().DurationVar(&globalFlags.KeepAliveTimeout, "keepalive-timeout", defaultKeepAliveTimeOut, "keepalive timeout for client connections") rootCmd.PersistentFlags().IntVar(&globalFlags.MaxCallSendMsgSize, "max-request-bytes", 0, "client-side request send limit in bytes (if 0, it defaults to 2.0 MiB (2 * 1024 * 1024).)") rootCmd.PersistentFlags().IntVar(&globalFlags.MaxCallRecvMsgSize, "max-recv-bytes", 0, "client-side response receive limit in bytes (if 0, it defaults to \"math.MaxInt32\")") // TODO: secure by default when etcd enables secure gRPC by default. rootCmd.PersistentFlags().BoolVar(&globalFlags.Insecure, "insecure-transport", true, "disable transport security for client connections") rootCmd.PersistentFlags().BoolVar(&globalFlags.InsecureDiscovery, "insecure-discovery", true, "accept insecure SRV records describing cluster endpoints") rootCmd.PersistentFlags().BoolVar(&globalFlags.InsecureSkipVerify, "insecure-skip-tls-verify", false, "skip server certificate verification (CAUTION: this option should be enabled only for testing purposes)") rootCmd.PersistentFlags().StringVar(&globalFlags.TLS.CertFile, "cert", "", "identify secure client using this TLS certificate file") rootCmd.PersistentFlags().StringVar(&globalFlags.TLS.KeyFile, "key", "", "identify secure client using this TLS key file") rootCmd.PersistentFlags().StringVar(&globalFlags.TLS.TrustedCAFile, "cacert", "", "verify certificates of TLS-enabled secure servers using this CA bundle") rootCmd.PersistentFlags().StringVar(&globalFlags.Token, "auth-jwt-token", "", "JWT token used for authentication (if this option is used, --user and --password should not be set)") rootCmd.PersistentFlags().StringVar(&globalFlags.User, "user", "", "username[:password] for authentication (prompt if password is not supplied)") rootCmd.PersistentFlags().StringVar(&globalFlags.Password, "password", "", "password for authentication (if this option is used, --user option shouldn't include password)") rootCmd.PersistentFlags().StringVarP(&globalFlags.TLS.ServerName, "discovery-srv", "d", "", "domain name to query for SRV records describing cluster endpoints") rootCmd.PersistentFlags().StringVarP(&globalFlags.DNSClusterServiceName, "discovery-srv-name", "", "", "service name to query when using DNS discovery") rootCmd.AddGroup( command.NewKVGroup(), command.NewClusterMaintenanceGroup(), command.NewConcurrencyGroup(), command.NewAuthenticationGroup(), command.NewUtilityGroup(), ) rootCmd.AddCommand( command.NewGetCommand(), command.NewPutCommand(), command.NewDelCommand(), command.NewTxnCommand(), command.NewCompactionCommand(), command.NewAlarmCommand(), command.NewDefragCommand(), command.NewEndpointCommand(), command.NewMoveLeaderCommand(), command.NewWatchCommand(), command.NewVersionCommand(), command.NewLeaseCommand(), command.NewMemberCommand(), command.NewSnapshotCommand(), command.NewMakeMirrorCommand(), command.NewLockCommand(), command.NewElectCommand(), command.NewAuthCommand(), command.NewUserCommand(), command.NewRoleCommand(), command.NewCheckCommand(), command.NewDiagnosisCommand(), command.NewCompletionCommand(), command.NewDowngradeCommand(), command.NewOptionsCommand(rootCmd), ) command.SetHelpCmdGroup(rootCmd) hideAllGlobalFlags() hideHelpFlag() addOptionsPrompt() } func Start() error { return rootCmd.Execute() } func MustStart() { if err := Start(); err != nil { if rootCmd.SilenceErrors { cobrautl.ExitWithError(cobrautl.ExitError, err) } os.Exit(cobrautl.ExitError) } } func hideAllGlobalFlags() { rootCmd.PersistentFlags().VisitAll(func(f *pflag.Flag) { rootCmd.PersistentFlags().MarkHidden(f.Name) }) } func hideHelpFlag() { if rootCmd.Flags().Lookup("help") == nil { rootCmd.Flags().BoolP("help", "h", false, "help for "+rootCmd.Name()) } rootCmd.Flags().MarkHidden("help") } func addOptionsPrompt() { defaultHelpFunc := rootCmd.HelpFunc() rootCmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { defaultHelpFunc(cmd, args) fmt.Fprintln(cmd.OutOrStdout(), `Use "etcdctl options" for a list of global command-line options (applies to all commands).`) }) } func init() { cobra.EnablePrefixMatching = true } ================================================ FILE: etcdctl/doc/mirror_maker.md ================================================ ## Mirror Maker Mirror maker mirrors a prefix in the key-value space of an etcd cluster into another prefix in another cluster. Mirroring is designed for copying configuration to various clusters distributed around the world. Mirroring usually has very low latency once it completes synchronizing with the initial state. Mirror maker utilizes the etcd watcher facility to immediately inform the mirror of any key modifications. Based on our experiments, the network latency between the mirror maker and the two clusters accounts for most of the latency. If the network is healthy, copying configuration held in etcd to the mirror should take under one second even for a world-wide deployment. If the mirror maker fails to connect to one of the clusters, the mirroring will pause. Mirroring can be resumed automatically once connectivity is reestablished. The mirroring mechanism is unidirectional. Changing the value on the mirrored cluster won't reflect the value back to the origin cluster. The mirror maker only mirrors key-value pairs; metadata, such as version number or modification revision, is discarded. However, mirror maker still attempts to preserve update ordering during normal operation, but there is no ordering guarantee during initial sync nor during failure recovery following network interruption. As a rule of thumb, the ordering of the updates on the mirror should not be considered reliable. ``` +-------------+ | | | source | +-----------+ | cluster +----> | mirror | | | | maker | +-------------+ +---+-------+ | v +-------------+ | | | mirror | | cluster | | | +-------------+ ``` Mirror-maker is a built-in feature of [etcdctl][etcdctl]. [etcdctl]: ../README.md ================================================ FILE: etcdctl/go.mod ================================================ module go.etcd.io/etcd/etcdctl/v3 go 1.26 toolchain go1.26.1 require ( github.com/bgentry/speakeasy v0.2.0 github.com/cheggaaa/pb/v3 v3.1.7 github.com/dustin/go-humanize v1.0.1 github.com/olekukonko/tablewriter v1.1.3 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/v3 v3.6.0-alpha.0 go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0 go.uber.org/zap v1.27.1 golang.org/x/time v0.14.0 google.golang.org/grpc v1.79.2 ) require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/clipperhouse/displaywidth v0.6.2 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fatih/color v1.18.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/errors v1.1.0 // indirect github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace ( go.etcd.io/etcd/api/v3 => ../api go.etcd.io/etcd/client/pkg/v3 => ../client/pkg go.etcd.io/etcd/client/v3 => ../client/v3 go.etcd.io/etcd/pkg/v3 => ../pkg ) ================================================ FILE: etcdctl/go.sum ================================================ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.2.0 h1:tgObeVOf8WAvtuAX6DhJ4xks4CFNwPDZiqzGqIHE51E= github.com/bgentry/speakeasy v0.2.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb/v3 v3.1.7 h1:2FsIW307kt7A/rz/ZI2lvPO+v3wKazzE4K/0LtTWsOI= github.com/cheggaaa/pb/v3 v3.1.7/go.mod h1:/Ji89zfVPeC/u5j8ukD0MBPHt2bzTYp74lQ7KlgFWTQ= github.com/clipperhouse/displaywidth v0.6.2 h1:ZDpTkFfpHOKte4RG5O/BOyf3ysnvFswpyYrV7z2uAKo= github.com/clipperhouse/displaywidth v0.6.2/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM= github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0 h1:jrYnow5+hy3WRDCBypUFvVKNSPPCdqgSXIE9eJDD8LM= github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew= github.com/olekukonko/tablewriter v1.1.3 h1:VSHhghXxrP0JHl+0NnKid7WoEmd9/urKRJLysb70nnA= github.com/olekukonko/tablewriter v1.1.3/go.mod h1:9VU0knjhmMkXjnMKrZ3+L2JhhtsQ/L38BbL3CRNE8tM= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: etcdctl/main.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // etcdctl is a command line application that controls etcd. package main import ( "go.etcd.io/etcd/etcdctl/v3/ctlv3" ) func main() { ctlv3.MustStart() } ================================================ FILE: etcdctl/util/normalizer.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package util import "strings" const indentation = " " // Normalize normalizes a string: // 1. trim the leading and trailing space // 2. add an indentation before each line func Normalize(s string) string { if len(s) == 0 { return s } return normalizer{s}.trim().indent().string } type normalizer struct { string } func (n normalizer) trim() normalizer { n.string = strings.TrimSpace(n.string) return n } func (n normalizer) indent() normalizer { indentedLines := []string{} for _, line := range strings.Split(n.string, "\n") { trimmed := strings.TrimSpace(line) indented := indentation + trimmed indentedLines = append(indentedLines, indented) } n.string = strings.Join(indentedLines, "\n") return n } ================================================ FILE: etcdutl/.gomodguard.yaml ================================================ --- blocked: modules: - go.etcd.io/etcd: reason: "Forbidden dependency" - go.etcd.io/etcd/tests/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/v3: reason: "Forbidden dependency" ================================================ FILE: etcdutl/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: etcdutl/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/etcdutl ================================================ FILE: etcdutl/README.md ================================================ # etcdutl `etcdutl` is a command line administration utility for [etcd][etcd]. It's designed to operate directly on etcd data files. For operations over a network, please use `etcdctl`. ### DEFRAG [options] DEFRAG directly defragments an etcd data directory while etcd is not running. When an etcd member reclaims storage space from deleted and compacted keys, the space is kept in a free list and the database file remains the same size. By defragmenting the database, the etcd member releases this free space back to the file system. In order to defrag a live etcd instances over the network, please use `etcdctl defrag` instead. #### Options - data-dir -- Optional. If present, defragments a data directory not in use by etcd. #### Output Exit status '0' when the process was successful. #### Example To defragment a data directory directly, use the `--data-dir` flag: ``` bash # Defragment while etcd is not running ./etcdutl defrag --data-dir default.etcd # success (exit status 0) # Error: cannot open database at default.etcd/member/snap/db ``` #### Remarks DEFRAG returns a zero exit code only if it succeeded in defragmenting all given endpoints. ### SNAPSHOT RESTORE [options] \ SNAPSHOT RESTORE creates an etcd data directory for an etcd cluster member from a backend database snapshot and a new cluster configuration. Restoring the snapshot into each member for a new cluster configuration will initialize a new etcd cluster preloaded by the snapshot data. #### Options The snapshot restore options closely resemble to those used in the `etcd` command for defining a cluster. - data-dir -- Path to the data directory. Uses \.etcd if none given. - wal-dir -- Path to the WAL directory. Uses data directory if none given. - initial-cluster -- The initial cluster configuration for the restored etcd cluster. - initial-cluster-token -- Initial cluster token for the restored etcd cluster. - initial-advertise-peer-urls -- List of peer URLs for the member being restored. - name -- Human-readable name for the etcd cluster member being restored. - skip-hash-check -- Ignore snapshot integrity hash value (required if copied from data directory) - bump-revision -- How much to increase the latest revision after restore - mark-compacted -- Mark the latest revision after restore as the point of scheduled compaction (required if --bump-revision > 0, disallowed otherwise) #### Output A new etcd data directory initialized with the snapshot. #### Example Save a snapshot, restore into a new 3 node cluster, and start the cluster: ``` # save snapshot ./etcdctl snapshot save snapshot.db # restore members ./etcdutl snapshot restore snapshot.db --initial-cluster-token etcd-cluster-1 --initial-advertise-peer-urls http://127.0.0.1:12380 --name sshot1 --initial-cluster 'sshot1=http://127.0.0.1:12380,sshot2=http://127.0.0.1:22380,sshot3=http://127.0.0.1:32380' ./etcdutl snapshot restore snapshot.db --initial-cluster-token etcd-cluster-1 --initial-advertise-peer-urls http://127.0.0.1:22380 --name sshot2 --initial-cluster 'sshot1=http://127.0.0.1:12380,sshot2=http://127.0.0.1:22380,sshot3=http://127.0.0.1:32380' ./etcdutl snapshot restore snapshot.db --initial-cluster-token etcd-cluster-1 --initial-advertise-peer-urls http://127.0.0.1:32380 --name sshot3 --initial-cluster 'sshot1=http://127.0.0.1:12380,sshot2=http://127.0.0.1:22380,sshot3=http://127.0.0.1:32380' # launch members ./etcd --name sshot1 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:2379 --listen-peer-urls http://127.0.0.1:12380 & ./etcd --name sshot2 --listen-client-urls http://127.0.0.1:22379 --advertise-client-urls http://127.0.0.1:22379 --listen-peer-urls http://127.0.0.1:22380 & ./etcd --name sshot3 --listen-client-urls http://127.0.0.1:32379 --advertise-client-urls http://127.0.0.1:32379 --listen-peer-urls http://127.0.0.1:32380 & ``` ### SNAPSHOT STATUS \ SNAPSHOT STATUS lists information about a given backend database snapshot file. #### Output ##### Simple format Prints a humanized table of the database hash, revision, total keys, and size. ##### JSON format Prints a line of JSON encoding the database hash, revision, total keys, and size. #### Examples ```bash ./etcdutl snapshot status file.db # cf1550fb, 3, 3, 25 kB ``` ```bash ./etcdutl --write-out=json snapshot status file.db # {"hash":3474280699,"revision":3,"totalKey":3,"totalSize":24576} ``` ```bash ./etcdutl --write-out=table snapshot status file.db +----------+----------+------------+------------+ | HASH | REVISION | TOTAL KEYS | TOTAL SIZE | +----------+----------+------------+------------+ | cf1550fb | 3 | 3 | 25 kB | +----------+----------+------------+------------+ ``` ### HASHKV [options] \ HASHKV prints hash of keys and values up to given revision. #### Options - rev -- Revision number. Default is 0 which means the latest revision. #### Output ##### Simple format Prints a humanized table of the KV hash, hash revision and compact revision. ##### JSON format Prints a line of JSON encoding the KV hash, hash revision and compact revision. #### Examples ```bash ./etcdutl hashkv file.db # 35c86e9b, 214, 150 ``` ```bash ./etcdutl --write-out=json hashkv file.db # {"hash":902327963,"hashRevision":214,"compactRevision":150} ``` ```bash ./etcdutl --write-out=table hashkv file.db +----------+---------------+------------------+ | HASH | HASH REVISION | COMPACT REVISION | +----------+---------------+------------------+ | 35c86e9b | 214 | 150 | +----------+---------------+------------------+ ``` ### VERSION Prints the version of etcdutl. #### Output Prints etcd version and API version. #### Examples ```bash ./etcdutl version # etcdutl version: 3.5.0 # API version: 3.1 ``` ### LIST-BUCKET [options] \ `list-bucket` prints all bucket names. #### Flags - timeout -- Time to wait to obtain a file lock on db file, 0 to block indefinitely. ##### Examples for LIST-BUCKET ```bash $ ./etcdutl list-bucket ~/tmp/etcd/default.etcd/member/snap/db alarm auth authRoles authUsers cluster key lease members members_removed meta ``` ### ITERATE-BUCKET [options] \ \ `iterate-bucket` lists key-value pairs in a given bucket in reverse order. #### Flags for ITERATE-BUCKET - timeout -- Time to wait to obtain a file lock on db file, 0 to block indefinitely. - limit -- Max number of key-value pairs to iterate (0 to iterate all). - decode -- true to decode Protocol Buffer encoded data. ##### Examples for ITERATE-BUCKET ```bash # with `--decode` option $ ./etcdutl iterate-bucket ~/tmp/etcd/default.etcd/member/snap/db key --decode rev={Revision:{Main:4 Sub:0} tombstone:false}, value=[key "k1" | val "v3" | created 2 | mod 4 | ver 3] rev={Revision:{Main:3 Sub:0} tombstone:false}, value=[key "k1" | val "v2" | created 2 | mod 3 | ver 2] rev={Revision:{Main:2 Sub:0} tombstone:false}, value=[key "k1" | val "v1" | created 2 | mod 2 | ver 1] # without `--decode` option $ ./etcdutl iterate-bucket ~/tmp/etcd/default.etcd/member/snap/db key key="\x00\x00\x00\x00\x00\x00\x00\x04_\x00\x00\x00\x00\x00\x00\x00\x00", value="\n\x02k1\x10\x02\x18\x04 \x03*\x02v3" key="\x00\x00\x00\x00\x00\x00\x00\x03_\x00\x00\x00\x00\x00\x00\x00\x00", value="\n\x02k1\x10\x02\x18\x03 \x02*\x02v2" key="\x00\x00\x00\x00\x00\x00\x00\x02_\x00\x00\x00\x00\x00\x00\x00\x00", value="\n\x02k1\x10\x02\x18\x02 \x01*\x02v1" ``` ### HASH [options] \ `hash` prints the hash of the db file. #### Flags for HASH - timeout -- Time to wait to obtain a file lock on db file, 0 to block indefinitely. ##### Examples for HASH ```bash $ ./etcdutl hash ~/tmp/etcd/default.etcd/member/snap/db db path: /Users/wachao/tmp/etcd/default.etcd/member/snap/db Hash: 4031086527 ``` ## Exit codes For all commands, a successful execution returns a zero exit code. All failures will return non-zero exit codes. ## Output formats All commands accept an output format by setting `-w` or `--write-out`. All commands default to the "simple" output format, which is meant to be human-readable. The simple format is listed in each command's `Output` description since it is customized for each command. If a command has a corresponding RPC, it will respect all output formats. If a command fails, returning a non-zero exit code, an error string will be written to standard error regardless of output format. ### Simple A format meant to be easy to parse and human-readable. Specific to each command. ### JSON The JSON encoding of the command's [RPC response][etcdrpc]. Since etcd's RPCs use byte strings, the JSON output will encode keys and values in base64. Some commands without an RPC also support JSON; see the command's `Output` description. ### Protobuf The protobuf encoding of the command's [RPC response][etcdrpc]. If an RPC is streaming, the stream messages will be concatenated. If an RPC is not given for a command, the protobuf output is not defined. ### Fields An output format similar to JSON but meant to parse with coreutils. For an integer field named `Field`, it writes a line in the format `"Field" : %d` where `%d` is go's integer formatting. For byte array fields, it writes `"Field" : %q` where `%q` is go's quoted string formatting (e.g., `[]byte{'a', '\n'}` is written as `"a\n"`). ## Compatibility Support etcdutl is still in its early stage. We try out best to ensure fully compatible releases, however we might break compatibility to fix bugs or improve commands. If we intend to release a version of etcdutl with backward incompatibilities, we will provide notice prior to release and have instructions on how to upgrade. ### Input Compatibility Input includes the command name, its flags, and its arguments. We ensure backward compatibility of the input of normal commands in non-interactive mode. ### Output Compatibility Currently, we do not ensure backward compatibility of utility commands. ### TODO: compatibility with etcd server [etcd]: https://github.com/coreos/etcd [READMEv2]: READMEv2.md [v2key]: ../store/node_extern.go#L28-L37 [v3key]: ../api/mvccpb/kv.proto#L12-L29 [etcdrpc]: ../api/etcdserverpb/rpc.proto [storagerpc]: ../api/mvccpb/kv.proto ================================================ FILE: etcdutl/ctl.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package etcdutl contains the main entry point for the etcdutl. package main import ( "time" "github.com/spf13/cobra" "go.etcd.io/etcd/etcdutl/v3/etcdutl" ) const ( cliName = "etcdutl" cliDescription = "An administrative command line tool for etcd3." ) var rootCmd = &cobra.Command{ Use: cliName, Short: cliDescription, SuggestFor: []string{"etcdutl"}, } func init() { rootCmd.PersistentFlags().DurationVar(&etcdutl.FlockTimeout, "timeout", 10*time.Second, "time to wait to obtain a file lock on db file, 0 to block indefinitely") rootCmd.PersistentFlags().StringVarP(&etcdutl.OutputFormat, "write-out", "w", "simple", "set the output format (fields, json, protobuf, simple, table)") rootCmd.RegisterFlagCompletionFunc("write-out", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { return []string{"fields", "json", "protobuf", "simple", "table"}, cobra.ShellCompDirectiveDefault }) rootCmd.AddCommand( etcdutl.NewDefragCommand(), etcdutl.NewSnapshotCommand(), etcdutl.NewHashKVCommand(), etcdutl.NewVersionCommand(), etcdutl.NewCompletionCommand(), etcdutl.NewMigrateCommand(), etcdutl.NewListBucketCommand(), etcdutl.NewIterateBucketCommand(), etcdutl.NewHashCommand(), ) } func Start() error { // Make help just show the usage rootCmd.SetHelpTemplate(`{{.UsageString}}`) return rootCmd.Execute() } func init() { cobra.EnablePrefixMatching = true } ================================================ FILE: etcdutl/etcdutl/bucket_command.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "encoding/binary" "fmt" "path/filepath" "strings" "github.com/spf13/cobra" "go.uber.org/zap" bolt "go.etcd.io/bbolt" "go.etcd.io/etcd/api/v3/authpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/server/v3/lease/leasepb" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" ) var ( iterateBucketLimit uint64 iterateBucketDecode bool ) func NewListBucketCommand() *cobra.Command { cmd := &cobra.Command{ Use: "list-bucket [data dir or db file path]", Short: "bucket lists all buckets.", Args: cobra.ExactArgs(1), Run: listBucketCommandFunc, } return cmd } func NewIterateBucketCommand() *cobra.Command { cmd := &cobra.Command{ Use: "iterate-bucket [data dir or db file path] [bucket name]", Short: "iterate-bucket lists key-value pairs in reverse order.", Args: cobra.ExactArgs(2), Run: iterateBucketCommandFunc, } cmd.PersistentFlags().Uint64Var(&iterateBucketLimit, "limit", 0, "max number of key-value pairs to iterate (0 to iterate all)") cmd.PersistentFlags().BoolVar(&iterateBucketDecode, "decode", false, "true to decode Protocol Buffer encoded data") return cmd } func NewHashCommand() *cobra.Command { cmd := &cobra.Command{ Use: "hash [data dir or db file path]", Short: "hash computes the hash of db file.", Args: cobra.ExactArgs(1), Run: getHashCommandFunc, } return cmd } func listBucketCommandFunc(_ *cobra.Command, args []string) { lg := GetLogger() dp := args[0] if !strings.HasSuffix(dp, "db") { dp = filepath.Join(datadir.ToSnapDir(dp), "db") } if !fileutil.Exist(dp) { lg.Fatal("db file not exist", zap.String("path", dp)) } bts, err := getBuckets(dp) if err != nil { lg.Fatal("Failed to get buckets", zap.Error(err)) } for _, b := range bts { fmt.Println(b) } } func getBuckets(dbPath string) (buckets []string, err error) { db, derr := bolt.Open(dbPath, 0o600, &bolt.Options{Timeout: FlockTimeout}) if derr != nil { return nil, fmt.Errorf("failed to open bolt DB %w", derr) } defer db.Close() err = db.View(func(tx *bolt.Tx) error { return tx.ForEach(func(b []byte, _ *bolt.Bucket) error { buckets = append(buckets, string(b)) return nil }) }) return buckets, err } func iterateBucketCommandFunc(_ *cobra.Command, args []string) { lg := GetLogger() dp := args[0] if !strings.HasSuffix(dp, "db") { dp = filepath.Join(datadir.ToSnapDir(dp), "db") } if !fileutil.Exist(dp) { lg.Fatal("db file not exist", zap.String("path", dp)) } bucket := args[1] err := iterateBucket(dp, bucket, iterateBucketLimit, iterateBucketDecode) if err != nil { lg.Fatal("Failed to iterate bucket", zap.Error(err)) } } type decoder func(k, v []byte) // key is the bucket name, and value is the function to decode K/V in the bucket. var decoders = map[string]decoder{ "key": keyDecoder, "lease": leaseDecoder, "auth": authDecoder, "authRoles": authRolesDecoder, "authUsers": authUsersDecoder, "meta": metaDecoder, } func defaultDecoder(k, v []byte) { fmt.Printf("key=%q, value=%q\n", k, v) } func keyDecoder(k, v []byte) { rev := mvcc.BytesToBucketKey(k) var kv mvccpb.KeyValue if err := kv.Unmarshal(v); err != nil { panic(err) } fmt.Printf("rev=%+v, value=[key %q | val %q | created %d | mod %d | ver %d]\n", rev, string(kv.Key), string(kv.Value), kv.CreateRevision, kv.ModRevision, kv.Version) } func bytesToLeaseID(bytes []byte) int64 { if len(bytes) != 8 { panic(fmt.Errorf("lease ID must be 8-byte")) } return int64(binary.BigEndian.Uint64(bytes)) } func leaseDecoder(k, v []byte) { leaseID := bytesToLeaseID(k) var lpb leasepb.Lease if err := lpb.Unmarshal(v); err != nil { panic(err) } fmt.Printf("lease ID=%016x, TTL=%ds, remaining TTL=%ds\n", leaseID, lpb.TTL, lpb.RemainingTTL) } func authDecoder(k, v []byte) { if string(k) == "authRevision" { rev := binary.BigEndian.Uint64(v) fmt.Printf("key=%q, value=%v\n", k, rev) } else { fmt.Printf("key=%q, value=%v\n", k, v) } } func authRolesDecoder(_, v []byte) { role := &authpb.Role{} err := role.Unmarshal(v) if err != nil { panic(err) } fmt.Printf("role=%q, keyPermission=%v\n", string(role.Name), role.KeyPermission) } func authUsersDecoder(_, v []byte) { user := &authpb.User{} err := user.Unmarshal(v) if err != nil { panic(err) } fmt.Printf("user=%q, roles=%q, option=%v\n", user.Name, user.Roles, user.Options) } func metaDecoder(k, v []byte) { if string(k) == string(schema.MetaConsistentIndexKeyName) || string(k) == string(schema.MetaTermKeyName) { fmt.Printf("key=%q, value=%v\n", k, binary.BigEndian.Uint64(v)) } else if string(k) == string(schema.ScheduledCompactKeyName) || string(k) == string(schema.FinishedCompactKeyName) { rev := mvcc.BytesToRev(v) fmt.Printf("key=%q, value=%v\n", k, rev) } else { defaultDecoder(k, v) } } func iterateBucket(dbPath, bucket string, limit uint64, decode bool) (err error) { db, err := bolt.Open(dbPath, 0o600, &bolt.Options{Timeout: FlockTimeout}) if err != nil { return fmt.Errorf("failed to open bolt DB %w", err) } defer db.Close() err = db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(bucket)) if b == nil { return fmt.Errorf("got nil bucket for %s", bucket) } c := b.Cursor() // iterate in reverse order (use First() and Next() for ascending order) for k, v := c.Last(); k != nil; k, v = c.Prev() { // TODO: remove sensitive information // (https://github.com/etcd-io/etcd/issues/7620) if dec, ok := decoders[bucket]; decode && ok { dec(k, v) } else { defaultDecoder(k, v) } limit-- if limit == 0 { break } } return nil }) return err } func getHashCommandFunc(_ *cobra.Command, args []string) { lg := GetLogger() dp := args[0] if !strings.HasSuffix(dp, "db") { dp = filepath.Join(datadir.ToSnapDir(dp), "db") } if !fileutil.Exist(dp) { lg.Fatal("db file not exist", zap.String("path", dp)) } hash, err := getHash(dp) if err != nil { lg.Fatal("failed to get hash", zap.Error(err)) } fmt.Printf("db path: %s\nHash: %d\n", dp, hash) } func getHash(dbPath string) (hash uint32, err error) { b := backend.NewDefaultBackend(zap.NewNop(), dbPath, backend.WithTimeout(FlockTimeout)) return b.Hash(schema.DefaultIgnores) } ================================================ FILE: etcdutl/etcdutl/common.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/pkg/v3/cobrautl" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" ) // FlockTimeout is the duration to wait to obtain a file lock on db file. var FlockTimeout time.Duration func GetLogger() *zap.Logger { config := logutil.DefaultZapLoggerConfig config.Encoding = "console" config.EncoderConfig.EncodeTime = zapcore.RFC3339TimeEncoder lg, err := config.Build() if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } return lg } func getLatestWALSnap(lg *zap.Logger, dataDir string) (walpb.Snapshot, error) { walPath := datadir.ToWALDir(dataDir) walSnaps, err := wal.ValidSnapshotEntries(lg, walPath) if err != nil { return walpb.Snapshot{}, err } if len(walSnaps) > 0 { lastIdx := len(walSnaps) - 1 return walSnaps[lastIdx], nil } return walpb.Snapshot{}, nil } // SimpleLessor is a simplified implementation of Lessor interface. // Used by etcdutl tools to simulate Lessor behavior without full lease management type SimpleLessor struct { LeaseSet map[lease.LeaseID]struct{} } var _ lease.Lessor = (*SimpleLessor)(nil) func (sl *SimpleLessor) SetRangeDeleter(dr lease.RangeDeleter) {} func (sl *SimpleLessor) SetCheckpointer(cp lease.Checkpointer) {} func (sl *SimpleLessor) Grant(id lease.LeaseID, ttl int64) (*lease.Lease, error) { sl.LeaseSet[id] = struct{}{} return nil, nil } func (sl *SimpleLessor) Revoke(id lease.LeaseID) error { return nil } func (sl *SimpleLessor) Checkpoint(id lease.LeaseID, remainingTTL int64) error { return nil } func (sl *SimpleLessor) Attach(id lease.LeaseID, items []lease.LeaseItem) error { return nil } func (sl *SimpleLessor) GetLease(item lease.LeaseItem) lease.LeaseID { return 0 } func (sl *SimpleLessor) Detach(id lease.LeaseID, items []lease.LeaseItem) error { return nil } func (sl *SimpleLessor) Promote(extend time.Duration) {} func (sl *SimpleLessor) Demote() {} func (sl *SimpleLessor) Renew(id lease.LeaseID) (int64, error) { return 10, nil } func (sl *SimpleLessor) Lookup(id lease.LeaseID) *lease.Lease { if _, ok := sl.LeaseSet[id]; ok { return &lease.Lease{ID: id} } return nil } func (sl *SimpleLessor) Leases() []*lease.Lease { return nil } func (sl *SimpleLessor) ExpiredLeasesC() <-chan []*lease.Lease { return nil } func (sl *SimpleLessor) Recover(b backend.Backend, rd lease.RangeDeleter) {} func (sl *SimpleLessor) Stop() {} ================================================ FILE: etcdutl/etcdutl/common_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "testing" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) func TestGetLatestWalSnap(t *testing.T) { testCases := []struct { name string walSnaps []walpb.Snapshot snapshots []raftpb.Snapshot expectedLatestWALSnap walpb.Snapshot }{ { name: "wal snapshot records match the snapshot files", walSnaps: []walpb.Snapshot{ {Index: new(uint64(10)), Term: new(uint64(2))}, {Index: new(uint64(20)), Term: new(uint64(3))}, {Index: new(uint64(30)), Term: new(uint64(5))}, }, snapshots: []raftpb.Snapshot{ {Metadata: raftpb.SnapshotMetadata{Index: 10, Term: 2}}, {Metadata: raftpb.SnapshotMetadata{Index: 20, Term: 3}}, {Metadata: raftpb.SnapshotMetadata{Index: 30, Term: 5}}, }, expectedLatestWALSnap: walpb.Snapshot{Index: new(uint64(30)), Term: new(uint64(5))}, }, { name: "there are orphan snapshot files", walSnaps: []walpb.Snapshot{ {Index: new(uint64(10)), Term: new(uint64(2))}, {Index: new(uint64(20)), Term: new(uint64(3))}, {Index: new(uint64(35)), Term: new(uint64(5))}, }, snapshots: []raftpb.Snapshot{ {Metadata: raftpb.SnapshotMetadata{Index: 10, Term: 2}}, {Metadata: raftpb.SnapshotMetadata{Index: 20, Term: 3}}, {Metadata: raftpb.SnapshotMetadata{Index: 35, Term: 5}}, {Metadata: raftpb.SnapshotMetadata{Index: 40, Term: 6}}, {Metadata: raftpb.SnapshotMetadata{Index: 50, Term: 7}}, }, expectedLatestWALSnap: walpb.Snapshot{Index: new(uint64(35)), Term: new(uint64(5))}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { dataDir := t.TempDir() lg := zap.NewNop() require.NoError(t, fileutil.TouchDirAll(lg, datadir.ToMemberDir(dataDir))) require.NoError(t, fileutil.TouchDirAll(lg, datadir.ToWALDir(dataDir))) require.NoError(t, fileutil.TouchDirAll(lg, datadir.ToSnapDir(dataDir))) // populate wal file w, err := wal.Create(lg, datadir.ToWALDir(dataDir), pbutil.MustMarshal( &etcdserverpb.Metadata{ NodeID: new(uint64(1)), ClusterID: new(uint64(2)), }, )) require.NoError(t, err) for _, walSnap := range tc.walSnaps { walSnap.ConfState = &raftpb.ConfState{Voters: []uint64{1}} walErr := w.SaveSnapshot(walSnap) require.NoError(t, walErr) walErr = w.Save(raftpb.HardState{Term: walSnap.GetTerm(), Commit: walSnap.GetIndex(), Vote: 1}, nil) require.NoError(t, walErr) } err = w.Close() require.NoError(t, err) // generate snapshot files ss := snap.New(lg, datadir.ToSnapDir(dataDir)) for _, snap := range tc.snapshots { snap.Metadata.ConfState = raftpb.ConfState{Voters: []uint64{1}} snapErr := ss.SaveSnap(snap) require.NoError(t, snapErr) } walSnap, err := getLatestWALSnap(lg, dataDir) require.NoError(t, err) require.Equal(t, tc.expectedLatestWALSnap.Term, walSnap.Term) require.Equal(t, tc.expectedLatestWALSnap.Index, walSnap.Index) }) } } ================================================ FILE: etcdutl/etcdutl/completion_commmand.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "os" "github.com/spf13/cobra" ) func NewCompletionCommand() *cobra.Command { cmd := &cobra.Command{ Use: "completion [bash|zsh|fish|powershell]", Short: "Generate completion script", Long: `To load completions: Bash: $ source <(etcdutl completion bash) # To load completions for each session, execute once: # Linux: $ etcdutl completion bash > /etc/bash_completion.d/etcdutl # macOS: $ etcdutl completion bash > /usr/local/etc/bash_completion.d/etcdutl Zsh: # If shell completion is not already enabled in your environment, # you will need to enable it. You can execute the following once: $ echo "autoload -U compinit; compinit" >> ~/.zshrc # To load completions for each session, execute once: $ etcdutl completion zsh > "${fpath[1]}/_etcdutl" # You will need to start a new shell for this setup to take effect. fish: $ etcdutl completion fish | source # To load completions for each session, execute once: $ etcdutl completion fish > ~/.config/fish/completions/etcdutl.fish PowerShell: PS> etcdutl completion powershell | Out-String | Invoke-Expression # To load completions for every new session, run: PS> etcdutl completion powershell > etcdutl.ps1 # and source this file from your PowerShell profile. `, DisableFlagsInUseLine: true, ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), Run: func(cmd *cobra.Command, args []string) { switch args[0] { case "bash": cmd.Root().GenBashCompletion(os.Stdout) case "zsh": cmd.Root().GenZshCompletion(os.Stdout) case "fish": cmd.Root().GenFishCompletion(os.Stdout, true) case "powershell": cmd.Root().GenPowerShellCompletionWithDesc(os.Stdout) } }, } return cmd } ================================================ FILE: etcdutl/etcdutl/defrag_command.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "fmt" "github.com/spf13/cobra" "go.etcd.io/etcd/pkg/v3/cobrautl" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/datadir" ) var defragDataDir string // NewDefragCommand returns the cobra command for "Defrag". func NewDefragCommand() *cobra.Command { cmd := &cobra.Command{ Use: "defrag", Short: "Defragments the storage of the etcd", Run: defragCommandFunc, } cmd.Flags().StringVar(&defragDataDir, "data-dir", "", "Required. Defragments a data directory not in use by etcd.") cmd.MarkFlagRequired("data-dir") cmd.MarkFlagDirname("data-dir") return cmd } func defragCommandFunc(cmd *cobra.Command, args []string) { err := DefragData(defragDataDir) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, fmt.Errorf("Failed to defragment etcd data[%s] (%w)", defragDataDir, err)) } } func DefragData(dataDir string) error { b := backend.NewDefaultBackend( GetLogger(), datadir.ToBackendFileName(dataDir), backend.WithTimeout(FlockTimeout)) return b.Defrag() } ================================================ FILE: etcdutl/etcdutl/hashkv_command.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "github.com/spf13/cobra" "go.uber.org/zap" "go.etcd.io/etcd/pkg/v3/cobrautl" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" ) var hashKVRevision int64 // NewHashKVCommand returns the cobra command for "hashkv". func NewHashKVCommand() *cobra.Command { cmd := &cobra.Command{ Use: "hashkv ", Short: "Prints the KV history hash of a given file", Args: cobra.ExactArgs(1), Run: hashKVCommandFunc, } cmd.Flags().Int64Var(&hashKVRevision, "rev", 0, "maximum revision to hash (default: latest revision)") return cmd } func hashKVCommandFunc(cmd *cobra.Command, args []string) { printer := initPrinterFromCmd(cmd) ds, err := calculateHashKV(args[0], hashKVRevision) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } printer.DBHashKV(ds) } type HashKV struct { Hash uint32 `json:"hash"` HashRevision int64 `json:"hashRevision"` CompactRevision int64 `json:"compactRevision"` } func calculateHashKV(dbPath string, rev int64) (HashKV, error) { b := backend.NewDefaultBackend(zap.NewNop(), dbPath, backend.WithTimeout(FlockTimeout)) // Since `etcdutl hashkv` only hashes the keyspace and ignores leases, we use a simple lessor to simplify the implementation. st := mvcc.NewStore(zap.NewNop(), b, &SimpleLessor{}, mvcc.StoreConfig{}) hst := mvcc.NewHashStorage(zap.NewNop(), st) h, _, err := hst.HashByRev(rev) if err != nil { return HashKV{}, err } return HashKV{ Hash: h.Hash, HashRevision: h.Revision, CompactRevision: h.CompactRevision, }, nil } ================================================ FILE: etcdutl/etcdutl/hashkv_command_test.go ================================================ // Copyright 2026 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "path/filepath" "testing" "go.uber.org/zap" "gotest.tools/v3/assert" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" ) func TestCalculateHashKV(t *testing.T) { type testCase struct { name string setupFunc func(t *testing.T) (dbPath string, cleanup func()) revision int64 expectedHash uint32 expectedRev int64 expectedCompact int64 expectError bool errorContains string } testCases := []testCase{ { name: "non-existent file", setupFunc: func(t *testing.T) (string, func()) { return "/nonexistent/path/to/db", func() {} }, expectError: true, }, { name: "empty directory path", setupFunc: func(t *testing.T) (string, func()) { return "", func() {} }, expectError: true, }, { name: "empty database", setupFunc: func(t *testing.T) (string, func()) { tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test.db") b := backend.NewDefaultBackend(zap.NewNop(), dbPath) st := mvcc.NewStore(zap.NewNop(), b, &lease.FakeLessor{}, mvcc.StoreConfig{}) _ = st b.Close() return dbPath, func() {} }, revision: 0, expectedHash: 1084519789, expectedRev: 1, expectedCompact: -1, expectError: false, }, { name: "database with data", setupFunc: func(t *testing.T) (string, func()) { tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test_with_data.db") b := backend.NewDefaultBackend(zap.NewNop(), dbPath) st := mvcc.NewStore(zap.NewNop(), b, &lease.FakeLessor{}, mvcc.StoreConfig{}) st.Put([]byte("test-key"), []byte("test-value"), 1) st.Close() b.Close() return dbPath, func() {} }, revision: 0, expectedHash: 645561629, expectedRev: 2, expectedCompact: -1, expectError: false, }, { name: "invalid revision", setupFunc: func(t *testing.T) (string, func()) { tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test_invalid_rev.db") b := backend.NewDefaultBackend(zap.NewNop(), dbPath) st := mvcc.NewStore(zap.NewNop(), b, &lease.FakeLessor{}, mvcc.StoreConfig{}) st.Put([]byte("key"), []byte("value"), 1) st.Close() b.Close() return dbPath, func() {} }, revision: 999, expectError: true, errorContains: "required revision is a future revision", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { defer func() { if r := recover(); r != nil { t.Logf("Recovered from panic: %v", r) } }() dbPath, cleanup := tc.setupFunc(t) defer cleanup() result, err := calculateHashKV(dbPath, tc.revision) if tc.expectError { assert.Assert(t, err != nil) if tc.errorContains != "" { assert.ErrorContains(t, err, tc.errorContains) } return } assert.NilError(t, err) assert.Equal(t, tc.expectedHash, result.Hash) assert.Equal(t, tc.expectedRev, result.HashRevision) assert.Equal(t, tc.expectedCompact, result.CompactRevision) t.Logf("Test %s - Hash: %d, HashRevision: %d, CompactRevision: %d", tc.name, result.Hash, result.HashRevision, result.CompactRevision) }) } } ================================================ FILE: etcdutl/etcdutl/migrate_command.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "fmt" "strings" "github.com/coreos/go-semver/semver" "github.com/spf13/cobra" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/pkg/v3/cobrautl" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/etcd/server/v3/storage/wal" ) // NewMigrateCommand prints out the version of etcd. func NewMigrateCommand() *cobra.Command { o := newMigrateOptions() cmd := &cobra.Command{ Use: "migrate", Short: "Migrates schema of etcd data dir files to make them compatible with different etcd version", Run: func(cmd *cobra.Command, args []string) { cfg, err := o.Config() if err != nil { cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } err = migrateCommandFunc(cfg) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } }, } o.AddFlags(cmd) return cmd } type migrateOptions struct { dataDir string targetVersion string force bool } func newMigrateOptions() *migrateOptions { return &migrateOptions{} } func (o *migrateOptions) AddFlags(cmd *cobra.Command) { cmd.Flags().StringVar(&o.dataDir, "data-dir", o.dataDir, "Path to the etcd data dir") cmd.MarkFlagRequired("data-dir") cmd.MarkFlagDirname("data-dir") cmd.Flags().StringVar(&o.targetVersion, "target-version", o.targetVersion, `Target etcd version to migrate contents of data dir. Minimal value 3.5. Format "X.Y" for example 3.6.`) cmd.MarkFlagRequired("target-version") cmd.Flags().BoolVar(&o.force, "force", o.force, "Ignore migration failure and forcefully override storage version. Not recommended.") } func (o *migrateOptions) Config() (*migrateConfig, error) { c := &migrateConfig{ force: o.force, dataDir: o.dataDir, lg: GetLogger(), } var err error dotCount := strings.Count(o.targetVersion, ".") if dotCount != 1 { return nil, fmt.Errorf(`wrong target version format, expected "X.Y", got %q`, o.targetVersion) } c.targetVersion, err = semver.NewVersion(o.targetVersion + ".0") if err != nil { return nil, fmt.Errorf("failed to parse target version: %w", err) } if c.targetVersion.LessThan(version.V3_5) { return nil, fmt.Errorf(`target version %q not supported. Minimal "3.5"`, storageVersionToString(c.targetVersion)) } return c, nil } type migrateConfig struct { lg *zap.Logger targetVersion *semver.Version walVersion wal.Version dataDir string force bool } func (c *migrateConfig) finalize() error { walPath := datadir.ToWALDir(c.dataDir) walSnap, err := getLatestWALSnap(c.lg, c.dataDir) if err != nil { return fmt.Errorf("failed to get the lastest snapshot: %w", err) } w, err := wal.OpenForRead(c.lg, walPath, walSnap) if err != nil { return fmt.Errorf(`failed to open wal: %w`, err) } defer w.Close() c.walVersion, err = wal.ReadWALVersion(w) if err != nil { return fmt.Errorf(`failed to read wal: %w`, err) } return nil } func migrateCommandFunc(c *migrateConfig) error { dbPath := datadir.ToBackendFileName(c.dataDir) be := backend.NewDefaultBackend(GetLogger(), dbPath, backend.WithTimeout(FlockTimeout)) defer be.Close() tx := be.BatchTx() current, err := schema.DetectSchemaVersion(c.lg, be.ReadTx()) if err != nil { c.lg.Error("failed to detect storage version. Please make sure you are using data dir from etcd v3.5 and older") return err } if current == *c.targetVersion { c.lg.Info("storage version up-to-date", zap.String("storage-version", storageVersionToString(¤t))) return nil } if err = c.finalize(); err != nil { c.lg.Error("Failed to finalize config", zap.Error(err)) return err } err = schema.Migrate(c.lg, tx, c.walVersion, *c.targetVersion) if err != nil { if !c.force { return err } c.lg.Info("normal migrate failed, trying with force", zap.Error(err)) migrateForce(c.lg, tx, c.targetVersion) } be.ForceCommit() return nil } func migrateForce(lg *zap.Logger, tx backend.BatchTx, target *semver.Version) { tx.LockOutsideApply() defer tx.Unlock() // Storage version is only supported since v3.6 if target.LessThan(version.V3_6) { schema.UnsafeClearStorageVersion(tx) lg.Warn("forcefully cleared storage version") } else { schema.UnsafeSetStorageVersion(tx, target) lg.Warn("forcefully set storage version", zap.String("storage-version", storageVersionToString(target))) } } func storageVersionToString(ver *semver.Version) string { return fmt.Sprintf("%d.%d", ver.Major, ver.Minor) } ================================================ FILE: etcdutl/etcdutl/printer.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "errors" "fmt" "github.com/dustin/go-humanize" "github.com/spf13/cobra" "go.etcd.io/etcd/etcdutl/v3/snapshot" "go.etcd.io/etcd/pkg/v3/cobrautl" ) var OutputFormat string type printer interface { DBStatus(snapshot.Status) DBHashKV(HashKV) } func NewPrinter(printerType string) printer { switch printerType { case "simple": return &simplePrinter{} case "fields": return &fieldsPrinter{newPrinterUnsupported("fields")} case "json": return newJSONPrinter() case "protobuf": return newPBPrinter() case "table": return &tablePrinter{newPrinterUnsupported("table")} } return nil } type printerRPC struct { printer p func(any) } type printerUnsupported struct{ printerRPC } func newPrinterUnsupported(n string) printer { f := func(any) { cobrautl.ExitWithError(cobrautl.ExitBadFeature, errors.New(n+" not supported as output format")) } return &printerUnsupported{printerRPC{nil, f}} } func (p *printerUnsupported) DBStatus(snapshot.Status) { p.p(nil) } func (p *printerUnsupported) DBHashKV(HashKV) { p.p(nil) } func makeDBStatusTable(ds snapshot.Status) (hdr []string, rows [][]string) { hdr = []string{"hash", "revision", "total keys", "total size", "version"} rows = append(rows, []string{ fmt.Sprintf("%x", ds.Hash), fmt.Sprint(ds.Revision), fmt.Sprint(ds.TotalKey), humanize.Bytes(uint64(ds.TotalSize)), ds.Version, }) return hdr, rows } func makeDBHashKVTable(ds HashKV) (hdr []string, rows [][]string) { hdr = []string{"hash", "hash revision", "compact revision"} rows = append(rows, []string{ fmt.Sprint(ds.Hash), fmt.Sprint(ds.HashRevision), fmt.Sprint(ds.CompactRevision), }) return hdr, rows } func initPrinterFromCmd(cmd *cobra.Command) (p printer) { outputType, err := cmd.Flags().GetString("write-out") if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } if p = NewPrinter(outputType); p == nil { cobrautl.ExitWithError(cobrautl.ExitBadFeature, errors.New("unsupported output format")) } return p } ================================================ FILE: etcdutl/etcdutl/printer_fields.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "fmt" "go.etcd.io/etcd/etcdutl/v3/snapshot" ) type fieldsPrinter struct{ printer } func (p *fieldsPrinter) DBStatus(r snapshot.Status) { fmt.Println(`"Hash" :`, r.Hash) fmt.Println(`"Revision" :`, r.Revision) fmt.Println(`"Keys" :`, r.TotalKey) fmt.Println(`"Size" :`, r.TotalSize) fmt.Println(`"Version" :`, r.Version) } func (p *fieldsPrinter) DBHashKV(r HashKV) { fmt.Println(`"Hash" :`, r.Hash) fmt.Println(`"Hash revision" :`, r.HashRevision) fmt.Println(`"Compact revision" :`, r.CompactRevision) } ================================================ FILE: etcdutl/etcdutl/printer_json.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "encoding/json" "fmt" "os" "go.etcd.io/etcd/etcdutl/v3/snapshot" ) type jsonPrinter struct { printer } func newJSONPrinter() printer { return &jsonPrinter{ printer: &printerRPC{newPrinterUnsupported("json"), printJSON}, } } func (p *jsonPrinter) DBStatus(r snapshot.Status) { printJSON(r) } func (p *jsonPrinter) DBHashKV(r HashKV) { printJSON(r) } // !!! Share ?? func printJSON(v any) { b, err := json.Marshal(v) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) return } fmt.Println(string(b)) } ================================================ FILE: etcdutl/etcdutl/printer_protobuf.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "fmt" "os" "go.etcd.io/etcd/pkg/v3/cobrautl" ) type pbPrinter struct{ printer } type pbMarshal interface { Marshal() ([]byte, error) } func newPBPrinter() printer { return &pbPrinter{ &printerRPC{newPrinterUnsupported("protobuf"), printPB}, } } func printPB(v any) { m, ok := v.(pbMarshal) if !ok { cobrautl.ExitWithError(cobrautl.ExitBadFeature, fmt.Errorf("marshal unsupported for type %T (%v)", v, v)) } b, err := m.Marshal() if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) return } fmt.Print(string(b)) } ================================================ FILE: etcdutl/etcdutl/printer_simple.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "fmt" "strings" "go.etcd.io/etcd/etcdutl/v3/snapshot" ) type simplePrinter struct{} func (s *simplePrinter) DBStatus(ds snapshot.Status) { _, rows := makeDBStatusTable(ds) for _, row := range rows { fmt.Println(strings.Join(row, ", ")) } } func (s *simplePrinter) DBHashKV(ds HashKV) { _, rows := makeDBHashKVTable(ds) for _, row := range rows { fmt.Println(strings.Join(row, ", ")) } } ================================================ FILE: etcdutl/etcdutl/printer_table.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "os" "github.com/olekukonko/tablewriter" "github.com/olekukonko/tablewriter/tw" "go.etcd.io/etcd/etcdutl/v3/snapshot" ) type tablePrinter struct{ printer } func (tp *tablePrinter) DBStatus(r snapshot.Status) { hdr, rows := makeDBStatusTable(r) cfgBuilder := tablewriter.NewConfigBuilder().WithRowAlignment(tw.AlignRight) table := tablewriter.NewTable(os.Stdout, tablewriter.WithConfig(cfgBuilder.Build())) table.Header(hdr) for _, row := range rows { table.Append(row) } table.Render() } func (tp *tablePrinter) DBHashKV(r HashKV) { hdr, rows := makeDBHashKVTable(r) cfgBuilder := tablewriter.NewConfigBuilder().WithRowAlignment(tw.AlignRight) table := tablewriter.NewTable(os.Stdout, tablewriter.WithConfig(cfgBuilder.Build())) table.Header(hdr) for _, row := range rows { table.Append(row) } table.Render() } ================================================ FILE: etcdutl/etcdutl/snapshot_command.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "fmt" "strings" "github.com/spf13/cobra" "go.etcd.io/etcd/etcdutl/v3/snapshot" "go.etcd.io/etcd/pkg/v3/cobrautl" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/datadir" ) const ( defaultName = "default" defaultInitialAdvertisePeerURLs = "http://localhost:2380" ) var ( restoreCluster string restoreClusterToken string restoreDataDir string restoreWALDir string restorePeerURLs string restoreName string skipHashCheck bool initialMmapSize = backend.InitialMmapSize markCompacted bool revisionBump uint64 ) // NewSnapshotCommand returns the cobra command for "snapshot". func NewSnapshotCommand() *cobra.Command { cmd := &cobra.Command{ Use: "snapshot ", Short: "Manages etcd node snapshots", } cmd.AddCommand(NewSnapshotRestoreCommand()) cmd.AddCommand(newSnapshotStatusCommand()) return cmd } func newSnapshotStatusCommand() *cobra.Command { return &cobra.Command{ Use: "status ", Short: "Gets backend snapshot status of a given file", Long: `When --write-out is set to simple, this command prints out comma-separated status lists for each endpoint. The items in the lists are hash, revision, total keys, total size. `, Run: SnapshotStatusCommandFunc, } } func NewSnapshotRestoreCommand() *cobra.Command { cmd := &cobra.Command{ Use: "restore --data-dir {output dir} [options]", Short: "Restores an etcd member snapshot to an etcd directory", Run: snapshotRestoreCommandFunc, } cmd.Flags().StringVar(&restoreDataDir, "data-dir", "", "Path to the output data directory") cmd.Flags().StringVar(&restoreWALDir, "wal-dir", "", "Path to the WAL directory (use --data-dir if none given)") cmd.Flags().StringVar(&restoreCluster, "initial-cluster", initialClusterFromName(defaultName), "Initial cluster configuration for restore bootstrap") cmd.Flags().StringVar(&restoreClusterToken, "initial-cluster-token", "etcd-cluster", "Initial cluster token for the etcd cluster during restore bootstrap") cmd.Flags().StringVar(&restorePeerURLs, "initial-advertise-peer-urls", defaultInitialAdvertisePeerURLs, "List of this member's peer URLs to advertise to the rest of the cluster") cmd.Flags().StringVar(&restoreName, "name", defaultName, "Human-readable name for this member") cmd.Flags().BoolVar(&skipHashCheck, "skip-hash-check", false, "Ignore snapshot integrity hash value (required if copied from data directory)") cmd.Flags().Uint64Var(&initialMmapSize, "initial-memory-map-size", initialMmapSize, "Initial memory map size of the database in bytes. It uses the default value if not defined or defined to 0") cmd.Flags().Uint64Var(&revisionBump, "bump-revision", 0, "How much to increase the latest revision after restore") cmd.Flags().BoolVar(&markCompacted, "mark-compacted", false, "Mark the latest revision after restore as the point of scheduled compaction (required if --bump-revision > 0, disallowed otherwise)") cmd.MarkFlagDirname("data-dir") cmd.MarkFlagDirname("wal-dir") return cmd } func SnapshotStatusCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { err := fmt.Errorf("snapshot status requires exactly one argument") cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } printer := initPrinterFromCmd(cmd) lg := GetLogger() sp := snapshot.NewV3(lg) ds, err := sp.Status(args[0]) if err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } printer.DBStatus(ds) } func snapshotRestoreCommandFunc(_ *cobra.Command, args []string) { SnapshotRestoreCommandFunc(restoreCluster, restoreClusterToken, restoreDataDir, restoreWALDir, restorePeerURLs, restoreName, skipHashCheck, initialMmapSize, revisionBump, markCompacted, args) } func SnapshotRestoreCommandFunc(restoreCluster string, restoreClusterToken string, restoreDataDir string, restoreWALDir string, restorePeerURLs string, restoreName string, skipHashCheck bool, initialMmapSize uint64, revisionBump uint64, markCompacted bool, args []string, ) { if len(args) != 1 { err := fmt.Errorf("snapshot restore requires exactly one argument") cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } if (revisionBump == 0 && markCompacted) || (revisionBump > 0 && !markCompacted) { err := fmt.Errorf("--mark-compacted required if --revision-bump > 0") cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) } dataDir := restoreDataDir if dataDir == "" { dataDir = restoreName + ".etcd" } walDir := restoreWALDir if walDir == "" { walDir = datadir.ToWALDir(dataDir) } lg := GetLogger() sp := snapshot.NewV3(lg) if err := sp.Restore(snapshot.RestoreConfig{ SnapshotPath: args[0], Name: restoreName, OutputDataDir: dataDir, OutputWALDir: walDir, PeerURLs: strings.Split(restorePeerURLs, ","), InitialCluster: restoreCluster, InitialClusterToken: restoreClusterToken, SkipHashCheck: skipHashCheck, InitialMmapSize: initialMmapSize, RevisionBump: revisionBump, MarkCompacted: markCompacted, }); err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } } func initialClusterFromName(name string) string { n := name if name == "" { n = defaultName } return fmt.Sprintf("%s=http://localhost:2380", n) } ================================================ FILE: etcdutl/etcdutl/version_command.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdutl import ( "fmt" "github.com/spf13/cobra" "go.etcd.io/etcd/api/v3/version" ) // NewVersionCommand prints out the version of etcd. func NewVersionCommand() *cobra.Command { return &cobra.Command{ Use: "version", Short: "Prints the version of etcdutl", Run: versionCommandFunc, } } func versionCommandFunc(cmd *cobra.Command, args []string) { fmt.Println("etcdutl version:", version.Version) fmt.Println("API version:", version.APIVersion) } ================================================ FILE: etcdutl/go.mod ================================================ module go.etcd.io/etcd/etcdutl/v3 go 1.26 toolchain go1.26.1 replace ( go.etcd.io/etcd/api/v3 => ../api go.etcd.io/etcd/client/pkg/v3 => ../client/pkg go.etcd.io/etcd/client/v3 => ../client/v3 go.etcd.io/etcd/pkg/v3 => ../pkg go.etcd.io/etcd/server/v3 => ../server ) require ( github.com/coreos/go-semver v0.3.1 github.com/dustin/go-humanize v1.0.1 github.com/olekukonko/tablewriter v1.1.3 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 go.etcd.io/bbolt v1.4.3 go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/v3 v3.6.0-alpha.0 go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0 go.etcd.io/etcd/server/v3 v3.6.0-alpha.0 go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee go.uber.org/zap v1.27.1 gotest.tools/v3 v3.5.2 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/displaywidth v0.6.2 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fatih/color v1.18.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/errors v1.1.0 // indirect github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect go.opentelemetry.io/otel/sdk v1.42.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/grpc v1.79.2 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/utils v0.0.0-20260108192941-914a6e750570 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) ================================================ FILE: etcdutl/go.sum ================================================ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clipperhouse/displaywidth v0.6.2 h1:ZDpTkFfpHOKte4RG5O/BOyf3ysnvFswpyYrV7z2uAKo= github.com/clipperhouse/displaywidth v0.6.2/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM= github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0 h1:jrYnow5+hy3WRDCBypUFvVKNSPPCdqgSXIE9eJDD8LM= github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew= github.com/olekukonko/tablewriter v1.1.3 h1:VSHhghXxrP0JHl+0NnKid7WoEmd9/urKRJLysb70nnA= github.com/olekukonko/tablewriter v1.1.3/go.mod h1:9VU0knjhmMkXjnMKrZ3+L2JhhtsQ/L38BbL3CRNE8tM= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee h1:9s5V0M58uCy51LgP6SUjROx7Ofqf8lGmeD/cCLaoagI= go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee/go.mod h1:VteWcRz3UV3TOpfex1x8jgPKAyjRXLKw3j8RdK3UAps= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 h1:XmiuHzgJt067+a6kwyAzkhXooYVv3/TOw9cM2VfJgUM= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= k8s.io/utils v0.0.0-20260108192941-914a6e750570 h1:JT4W8lsdrGENg9W+YwwdLJxklIuKWdRm+BC+xt33FOY= k8s.io/utils v0.0.0-20260108192941-914a6e750570/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= ================================================ FILE: etcdutl/main.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // etcdutl is a command line application that operates on etcd files. package main import ( "go.etcd.io/etcd/pkg/v3/cobrautl" ) func main() { if err := Start(); err != nil { cobrautl.ExitWithError(cobrautl.ExitError, err) } } ================================================ FILE: etcdutl/snapshot/doc.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package snapshot implements utilities around etcd snapshot. package snapshot ================================================ FILE: etcdutl/snapshot/v3_snapshot.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package snapshot import ( "bytes" "context" "crypto/sha256" "encoding/json" "fmt" "hash/crc32" "io" "os" "path/filepath" "reflect" "strings" "go.uber.org/zap" bolt "go.etcd.io/bbolt" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/types" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/snapshot" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/etcd/server/v3/etcdserver/cindex" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/etcd/server/v3/verify" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) // Manager defines snapshot methods. type Manager interface { // Save fetches snapshot from remote etcd server, saves data // to target path and returns server version. If the context "ctx" is canceled or timed out, // snapshot save stream will error out (e.g. context.Canceled, // context.DeadlineExceeded). Make sure to specify only one endpoint // in client configuration. Snapshot API must be requested to a // selected node, and saved snapshot is the point-in-time state of // the selected node. Save(ctx context.Context, cfg clientv3.Config, dbPath string) (version string, err error) // Status returns the snapshot file information. Status(dbPath string) (Status, error) // Restore restores a new etcd data directory from given snapshot // file. It returns an error if specified data directory already // exists, to prevent unintended data directory overwrites. Restore(cfg RestoreConfig) error } // NewV3 returns a new snapshot Manager for v3.x snapshot. func NewV3(lg *zap.Logger) Manager { return &v3Manager{lg: lg} } type v3Manager struct { lg *zap.Logger name string srcDbPath string walDir string snapDir string cl *membership.RaftCluster skipHashCheck bool initialMmapSize uint64 } // hasChecksum returns "true" if the file size "n" // has appended sha256 hash digest. func hasChecksum(n int64) bool { // 512 is chosen because it's a minimum disk sector size // smaller than (and multiplies to) OS page size in most systems return (n % 512) == sha256.Size } // Save fetches snapshot from remote etcd server and saves data to target path. func (s *v3Manager) Save(ctx context.Context, cfg clientv3.Config, dbPath string) (version string, err error) { return snapshot.SaveWithVersion(ctx, s.lg, cfg, dbPath) } // Status is the snapshot file status. type Status struct { Hash uint32 `json:"hash"` Revision int64 `json:"revision"` TotalKey int `json:"totalKey"` TotalSize int64 `json:"totalSize"` // Version is equal to storageVersion of the snapshot // Empty if server does not supports versioned snapshots ( 0 { return fmt.Errorf("snapshot file integrity check failed. %d errors found.\n"+strings.Join(dbErrStrings, "\n"), len(dbErrStrings)) } ds.TotalSize = tx.Size() v := schema.ReadStorageVersionFromSnapshot(tx) if v != nil { ds.Version = v.String() } c := tx.Cursor() for next, _ := c.First(); next != nil; next, _ = c.Next() { b := tx.Bucket(next) if b == nil { return fmt.Errorf("nil bucket: %q", string(next)) } _, err = h.Write(next) if err != nil { return fmt.Errorf("cannot hash bucket name: %q err: %w", string(next), err) } iskeyb := (bytes.Equal(next, schema.Key.Name())) if err = b.ForEach(func(k, v []byte) error { _, err = h.Write(k) if err != nil { return fmt.Errorf("cannot hash bucket key: %q err: %w", k, err) } _, err = h.Write(v) if err != nil { return fmt.Errorf("cannot hash bucket key: %q value: %q err: %w", k, v, err) } if iskeyb { var rev mvcc.Revision rev, err = bytesToRev(k) if err != nil { return fmt.Errorf("cannot parse revision key: %q err: %w", k, err) } ds.Revision = rev.Main var kv mvccpb.KeyValue err = kv.Unmarshal(v) if err != nil { return fmt.Errorf("cannot unmarshal value, key: %q value: %q err: %w", k, v, err) } key := string(kv.Key) // refer to https://etcd.io/docs/v3.5/learning/data_model/ if !mvcc.IsTombstone(k) { seenKeys[key] = struct{}{} } else { delete(seenKeys, key) } } return nil }); err != nil { return fmt.Errorf("error during bucket key iteration, name: %q err: %w", string(next), err) } } return nil }); err != nil { return ds, err } ds.TotalKey = len(seenKeys) ds.Hash = h.Sum32() return ds, nil } func bytesToRev(b []byte) (rev mvcc.Revision, err error) { defer func() { if r := recover(); r != nil { err = fmt.Errorf("%s", r) } }() return mvcc.BytesToRev(b), err } // RestoreConfig configures snapshot restore operation. type RestoreConfig struct { // SnapshotPath is the path of snapshot file to restore from. SnapshotPath string // Name is the human-readable name of this member. Name string // OutputDataDir is the target data directory to save restored data. // OutputDataDir should not conflict with existing etcd data directory. // If OutputDataDir already exists, it will return an error to prevent // unintended data directory overwrites. // If empty, defaults to "[Name].etcd" if not given. OutputDataDir string // OutputWALDir is the target WAL data directory. // If empty, defaults to "[OutputDataDir]/member/wal" if not given. OutputWALDir string // PeerURLs is a list of member's peer URLs to advertise to the rest of the cluster. PeerURLs []string // InitialCluster is the initial cluster configuration for restore bootstrap. InitialCluster string // InitialClusterToken is the initial cluster token for etcd cluster during restore bootstrap. InitialClusterToken string // SkipHashCheck is "true" to ignore snapshot integrity hash value // (required if copied from data directory). SkipHashCheck bool // InitialMmapSize is the database initial memory map size. InitialMmapSize uint64 // RevisionBump is the amount to increase the latest revision after restore, // to allow administrators to trick clients into thinking that revision never decreased. // If 0, revision bumping is skipped. // (required if MarkCompacted == true) RevisionBump uint64 // MarkCompacted is "true" to mark the latest revision as compacted. // (required if RevisionBump > 0) MarkCompacted bool } // Restore restores a new etcd data directory from given snapshot file. func (s *v3Manager) Restore(cfg RestoreConfig) error { pURLs, err := types.NewURLs(cfg.PeerURLs) if err != nil { return err } var ics types.URLsMap ics, err = types.NewURLsMap(cfg.InitialCluster) if err != nil { return err } srv := config.ServerConfig{ Logger: s.lg, Name: cfg.Name, PeerURLs: pURLs, InitialPeerURLsMap: ics, InitialClusterToken: cfg.InitialClusterToken, } if err = srv.VerifyBootstrap(); err != nil { return err } s.cl, err = membership.NewClusterFromURLsMap(s.lg, cfg.InitialClusterToken, ics) if err != nil { return err } dataDir := cfg.OutputDataDir if dataDir == "" { dataDir = cfg.Name + ".etcd" } if fileutil.Exist(dataDir) && !fileutil.DirEmpty(dataDir) { return fmt.Errorf("data-dir %q not empty or could not be read", dataDir) } walDir := cfg.OutputWALDir if walDir == "" { walDir = filepath.Join(dataDir, "member", "wal") } else if fileutil.Exist(walDir) { return fmt.Errorf("wal-dir %q exists", walDir) } s.name = cfg.Name s.srcDbPath = cfg.SnapshotPath s.walDir = walDir s.snapDir = filepath.Join(dataDir, "member", "snap") s.skipHashCheck = cfg.SkipHashCheck s.initialMmapSize = cfg.InitialMmapSize s.lg.Info( "restoring snapshot", zap.String("path", s.srcDbPath), zap.String("wal-dir", s.walDir), zap.String("data-dir", dataDir), zap.String("snap-dir", s.snapDir), zap.Uint64("initial-memory-map-size", s.initialMmapSize), ) if err = s.saveDB(); err != nil { return err } if cfg.MarkCompacted && cfg.RevisionBump > 0 { if err = s.modifyLatestRevision(cfg.RevisionBump); err != nil { return err } } hardstate, err := s.saveWALAndSnap() if err != nil { return err } if err := s.updateCIndex(hardstate.Commit, hardstate.Term); err != nil { return err } s.lg.Info( "restored snapshot", zap.String("path", s.srcDbPath), zap.String("wal-dir", s.walDir), zap.String("data-dir", dataDir), zap.String("snap-dir", s.snapDir), zap.Uint64("initial-memory-map-size", s.initialMmapSize), ) return verify.VerifyIfEnabled(verify.Config{ ExactIndex: true, Logger: s.lg, DataDir: dataDir, }) } func (s *v3Manager) outDbPath() string { return filepath.Join(s.snapDir, "db") } // saveDB copies the database snapshot to the snapshot directory func (s *v3Manager) saveDB() error { err := s.copyAndVerifyDB() if err != nil { return err } be := backend.NewDefaultBackend(s.lg, s.outDbPath(), backend.WithMmapSize(s.initialMmapSize)) defer be.Close() err = schema.NewMembershipBackend(s.lg, be).TrimMembershipFromBackend() if err != nil { return err } return nil } // modifyLatestRevision can increase the latest revision by the given amount and sets the scheduled compaction // to that revision so that the server will consider this revision compacted. func (s *v3Manager) modifyLatestRevision(bumpAmount uint64) error { be := backend.NewDefaultBackend(s.lg, s.outDbPath()) defer func() { be.ForceCommit() be.Close() }() tx := be.BatchTx() tx.LockOutsideApply() defer tx.Unlock() latest, err := s.unsafeGetLatestRevision(tx) if err != nil { return err } latest = s.unsafeBumpBucketsRevision(tx, latest, int64(bumpAmount)) s.unsafeMarkRevisionCompacted(tx, latest) return nil } func (s *v3Manager) unsafeBumpBucketsRevision(tx backend.UnsafeWriter, latest mvcc.Revision, amount int64) mvcc.Revision { s.lg.Info( "bumping latest revision", zap.Int64("latest-revision", latest.Main), zap.Int64("bump-amount", amount), zap.Int64("new-latest-revision", latest.Main+amount), ) latest.Main += amount latest.Sub = 0 k := mvcc.NewRevBytes() k = mvcc.RevToBytes(latest, k) tx.UnsafePut(schema.Key, k, []byte{}) return latest } func (s *v3Manager) unsafeMarkRevisionCompacted(tx backend.UnsafeWriter, latest mvcc.Revision) { s.lg.Info( "marking revision compacted", zap.Int64("revision", latest.Main), ) mvcc.UnsafeSetScheduledCompact(tx, latest.Main) } func (s *v3Manager) unsafeGetLatestRevision(tx backend.UnsafeReader) (mvcc.Revision, error) { var latest mvcc.Revision err := tx.UnsafeForEach(schema.Key, func(k, _ []byte) (err error) { rev := mvcc.BytesToRev(k) if rev.GreaterThan(latest) { latest = rev } return nil }) return latest, err } func (s *v3Manager) copyAndVerifyDB() error { srcf, ferr := os.Open(s.srcDbPath) if ferr != nil { return ferr } defer srcf.Close() // get snapshot integrity hash if _, err := srcf.Seek(-sha256.Size, io.SeekEnd); err != nil { return err } sha := make([]byte, sha256.Size) if _, err := srcf.Read(sha); err != nil { return err } if _, err := srcf.Seek(0, io.SeekStart); err != nil { return err } if err := fileutil.CreateDirAll(s.lg, s.snapDir); err != nil { return err } outDbPath := s.outDbPath() db, dberr := os.OpenFile(outDbPath, os.O_RDWR|os.O_CREATE, 0o600) if dberr != nil { return dberr } defer db.Close() if _, err := io.Copy(db, srcf); err != nil { return err } // truncate away integrity hash, if any. off, serr := db.Seek(0, io.SeekEnd) if serr != nil { return serr } hasHash := hasChecksum(off) if hasHash { if err := db.Truncate(off - sha256.Size); err != nil { return err } } if !hasHash && !s.skipHashCheck { return fmt.Errorf("snapshot missing hash but --skip-hash-check=false") } if hasHash && !s.skipHashCheck { // check for match if _, err := db.Seek(0, io.SeekStart); err != nil { return err } h := sha256.New() if _, err := io.Copy(h, db); err != nil { return err } dbsha := h.Sum(nil) if !reflect.DeepEqual(sha, dbsha) { return fmt.Errorf("expected sha256 %v, got %v", sha, dbsha) } } // db hash is OK, can now modify DB so it can be part of a new cluster return nil } // saveWALAndSnap creates a WAL for the initial cluster // // TODO: This code ignores learners !!! func (s *v3Manager) saveWALAndSnap() (*raftpb.HardState, error) { if err := fileutil.CreateDirAll(s.lg, s.walDir); err != nil { return nil, err } // add members again to persist them to the backend we create. be := backend.NewDefaultBackend(s.lg, s.outDbPath(), backend.WithMmapSize(s.initialMmapSize)) defer be.Close() s.cl.SetBackend(schema.NewMembershipBackend(s.lg, be)) for _, m := range s.cl.Members() { s.cl.AddMember(m, true) } m := s.cl.MemberByName(s.name) //nolint:staticcheck // See https://github.com/dominikh/go-tools/issues/1698 md := &etcdserverpb.Metadata{NodeID: new(uint64(m.ID)), ClusterID: new(uint64(s.cl.ID()))} metadata, merr := md.Marshal() if merr != nil { return nil, merr } w, walerr := wal.Create(s.lg, s.walDir, metadata) if walerr != nil { return nil, walerr } defer w.Close() peers := make([]raft.Peer, len(s.cl.MemberIDs())) for i, id := range s.cl.MemberIDs() { ctx, err := json.Marshal((*s.cl).Member(id)) if err != nil { return nil, err } peers[i] = raft.Peer{ID: uint64(id), Context: ctx} } ents := make([]raftpb.Entry, len(peers)) nodeIDs := make([]uint64, len(peers)) for i, p := range peers { nodeIDs[i] = p.ID cc := raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: p.ID, Context: p.Context, } d, err := cc.Marshal() if err != nil { return nil, err } ents[i] = raftpb.Entry{ Type: raftpb.EntryConfChange, Term: 1, Index: uint64(i + 1), Data: d, } } commit, term := uint64(len(ents)), uint64(1) hardState := raftpb.HardState{ Term: term, Vote: peers[0].ID, Commit: commit, } if err := w.Save(hardState, ents); err != nil { return nil, err } confState := raftpb.ConfState{ Voters: nodeIDs, } raftSnap := raftpb.Snapshot{ Data: etcdserver.GetMembershipInfoInV2Format(s.lg, s.cl), Metadata: raftpb.SnapshotMetadata{ Index: commit, Term: term, ConfState: confState, }, } sn := snap.New(s.lg, s.snapDir) if err := sn.SaveSnap(raftSnap); err != nil { return nil, err } snapshot := walpb.Snapshot{Index: &commit, Term: &term, ConfState: &confState} return &hardState, w.SaveSnapshot(snapshot) } func (s *v3Manager) updateCIndex(commit uint64, term uint64) error { be := backend.NewDefaultBackend(s.lg, s.outDbPath(), backend.WithMmapSize(s.initialMmapSize)) defer be.Close() cindex.UpdateConsistentIndexForce(be.BatchTx(), commit, term) return nil } ================================================ FILE: etcdutl/snapshot/v3_snapshot_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package snapshot import ( "errors" "path/filepath" "strconv" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.etcd.io/bbolt" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" ) // TestSnapshotStatus is the happy case. // It inserts pre-defined number of keys and asserts the output hash of status command. // The expected hash value must not be changed. // If it changes, there must be some backwards incompatible change introduced. func TestSnapshotStatus(t *testing.T) { dbpath := createDB(t, insertKeys(t, 10, 100)) status, err := NewV3(zap.NewNop()).Status(dbpath) require.NoError(t, err) assert.Equal(t, uint32(0xe7a6e44b), status.Hash) assert.Equal(t, int64(11), status.Revision) } // TestSnapshotStatusCorruptRevision tests if snapshot status command fails when there is an unexpected revision in "key" bucket. func TestSnapshotStatusCorruptRevision(t *testing.T) { dbpath := createDB(t, insertKeys(t, 1, 0)) db, err := bbolt.Open(dbpath, 0o600, nil) require.NoError(t, err) defer db.Close() err = db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte("key")) if b == nil { return errors.New("key bucket not found") } return b.Put([]byte("0"), []byte{}) }) require.NoError(t, err) db.Close() _, err = NewV3(zap.NewNop()).Status(dbpath) require.ErrorContains(t, err, "invalid revision length") } // TestSnapshotStatusNegativeRevisionMain tests if snapshot status command fails when main revision number is negative. func TestSnapshotStatusNegativeRevisionMain(t *testing.T) { dbpath := createDB(t, insertKeys(t, 1, 0)) db, err := bbolt.Open(dbpath, 0o666, nil) require.NoError(t, err) defer db.Close() err = db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket(schema.Key.Name()) if b == nil { return errors.New("key bucket not found") } bytes := mvcc.NewRevBytes() mvcc.RevToBytes(mvcc.Revision{Main: -1}, bytes) return b.Put(bytes, []byte{}) }) require.NoError(t, err) db.Close() _, err = NewV3(zap.NewNop()).Status(dbpath) require.ErrorContains(t, err, "negative revision") } // TestSnapshotStatusNegativeRevisionSub tests if snapshot status command fails when sub revision number is negative. func TestSnapshotStatusNegativeRevisionSub(t *testing.T) { dbpath := createDB(t, insertKeys(t, 1, 0)) db, err := bbolt.Open(dbpath, 0o666, nil) require.NoError(t, err) defer db.Close() err = db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte("key")) if b == nil { return errors.New("key bucket not found") } bytes := mvcc.NewRevBytes() mvcc.RevToBytes(mvcc.Revision{Sub: -1}, bytes) return b.Put(bytes, []byte{}) }) require.NoError(t, err) db.Close() _, err = NewV3(zap.NewNop()).Status(dbpath) require.ErrorContains(t, err, "negative revision") } // TestSnapshotStatusTotalKey tests if snapshot status command correctly reports total number of valid keys. func TestSnapshotStatusTotalKey(t *testing.T) { cases := []struct { name string prepare func(srv *etcdserver.EtcdServer) expected int }{ { name: "duplicate keys", prepare: func(srv *etcdserver.EtcdServer) { keys := []string{"key1", "key2", "key1"} val := make([]byte, len(keys)) for _, key := range keys { for i := 0; i < 3; i++ { req := etcdserverpb.PutRequest{ Key: []byte(key), Value: val, } _, err := srv.Put(t.Context(), &req) require.NoError(t, err) } } }, expected: 2, }, { name: "mixed revisions", prepare: func(srv *etcdserver.EtcdServer) { // key1: create -> put -> delete key := []byte("key1") for i := 0; i < 3; i++ { if i < 2 { _, err := srv.Put(t.Context(), &etcdserverpb.PutRequest{Key: key, Value: []byte(strconv.Itoa(i))}) require.NoError(t, err) } else { _, err := srv.DeleteRange(t.Context(), &etcdserverpb.DeleteRangeRequest{Key: key}) require.NoError(t, err) } } }, expected: 0, }, { name: "ignored tombstones", prepare: func(srv *etcdserver.EtcdServer) { // key1: create -> delete -> re-create -> delete key := []byte("key1") for i := 0; i < 2; i++ { _, err := srv.Put(t.Context(), &etcdserverpb.PutRequest{Key: key, Value: make([]byte, 1)}) require.NoError(t, err) _, err = srv.DeleteRange(t.Context(), &etcdserverpb.DeleteRangeRequest{Key: key}) require.NoError(t, err) } }, expected: 0, }, { name: "restored keys", prepare: func(srv *etcdserver.EtcdServer) { // key1: create -> delete -> re-create -> delete -> re-create key := []byte("key1") for i := 0; i < 5; i++ { if i%2 == 0 { _, err := srv.Put(t.Context(), &etcdserverpb.PutRequest{Key: key, Value: make([]byte, 1)}) require.NoError(t, err) } else { _, err := srv.DeleteRange(t.Context(), &etcdserverpb.DeleteRangeRequest{Key: key}) require.NoError(t, err) } } }, expected: 1, }, { name: "mixed deletions", prepare: func(srv *etcdserver.EtcdServer) { // Put("key1") -> Put("key2")-> Delete("key1") _, err := srv.Put(t.Context(), &etcdserverpb.PutRequest{Key: []byte("key1"), Value: make([]byte, 1)}) require.NoError(t, err) _, err = srv.Put(t.Context(), &etcdserverpb.PutRequest{Key: []byte("key2"), Value: make([]byte, 1)}) require.NoError(t, err) _, err = srv.DeleteRange(t.Context(), &etcdserverpb.DeleteRangeRequest{Key: []byte("key1")}) require.NoError(t, err) }, expected: 1, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { dbpath := createDB(t, tc.prepare) status, err := NewV3(zap.NewNop()).Status(dbpath) require.NoError(t, err) assert.Equal(t, tc.expected, status.TotalKey) }) } } // insertKeys insert `numKeys` number of keys of `valueSize` size into a running etcd server. func insertKeys(t *testing.T, numKeys, valueSize int) func(*etcdserver.EtcdServer) { t.Helper() return func(srv *etcdserver.EtcdServer) { val := make([]byte, valueSize) for i := 0; i < numKeys; i++ { req := etcdserverpb.PutRequest{ Key: []byte(strconv.Itoa(i)), Value: val, } _, err := srv.Put(t.Context(), &req) require.NoError(t, err) } } } // createDB creates a bbolt database file by running an embedded etcd server. // While the server is running, `generateContent` function is called to insert values. // It returns the path of bbolt database. func createDB(t *testing.T, generateContent func(*etcdserver.EtcdServer)) string { t.Helper() cfg := embed.NewConfig() cfg.BackendBatchLimit = 1 cfg.LogLevel = "fatal" cfg.Dir = t.TempDir() etcd, err := embed.StartEtcd(cfg) require.NoError(t, err) defer etcd.Close() select { case <-etcd.Server.ReadyNotify(): case <-time.After(10 * time.Second): t.FailNow() } generateContent(etcd.Server) return filepath.Join(cfg.Dir, "member", "snap", "db") } ================================================ FILE: go.mod ================================================ module go.etcd.io/etcd/v3 go 1.26 toolchain go1.26.1 replace ( go.etcd.io/etcd/api/v3 => ./api go.etcd.io/etcd/cache/v3 => ./cache go.etcd.io/etcd/client/pkg/v3 => ./client/pkg go.etcd.io/etcd/client/v3 => ./client/v3 go.etcd.io/etcd/etcdctl/v3 => ./etcdctl go.etcd.io/etcd/etcdutl/v3 => ./etcdutl go.etcd.io/etcd/pkg/v3 => ./pkg go.etcd.io/etcd/server/v3 => ./server go.etcd.io/etcd/tests/v3 => ./tests ) require ( github.com/bgentry/speakeasy v0.2.0 github.com/cheggaaa/pb/v3 v3.1.7 github.com/coreos/go-semver v0.3.1 github.com/dustin/go-humanize v1.0.1 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 go.etcd.io/bbolt v1.4.3 go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/v3 v3.6.0-alpha.0 go.etcd.io/etcd/etcdctl/v3 v3.6.0-alpha.0 go.etcd.io/etcd/etcdutl/v3 v3.6.0-alpha.0 go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0 go.etcd.io/etcd/server/v3 v3.6.0-alpha.0 go.etcd.io/etcd/tests/v3 v3.0.0-00010101000000-000000000000 go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee go.uber.org/zap v1.27.1 golang.org/x/time v0.14.0 golang.org/x/tools v0.42.0 google.golang.org/grpc v1.79.2 google.golang.org/protobuf v1.36.11 ) require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/displaywidth v0.6.2 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fatih/color v1.18.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/errors v1.1.0 // indirect github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0 // indirect github.com/olekukonko/tablewriter v1.1.3 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/gofail v0.2.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect go.opentelemetry.io/otel/sdk v1.42.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/utils v0.0.0-20260108192941-914a6e750570 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) ================================================ FILE: go.sum ================================================ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.2.0 h1:tgObeVOf8WAvtuAX6DhJ4xks4CFNwPDZiqzGqIHE51E= github.com/bgentry/speakeasy v0.2.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb/v3 v3.1.7 h1:2FsIW307kt7A/rz/ZI2lvPO+v3wKazzE4K/0LtTWsOI= github.com/cheggaaa/pb/v3 v3.1.7/go.mod h1:/Ji89zfVPeC/u5j8ukD0MBPHt2bzTYp74lQ7KlgFWTQ= github.com/clipperhouse/displaywidth v0.6.2 h1:ZDpTkFfpHOKte4RG5O/BOyf3ysnvFswpyYrV7z2uAKo= github.com/clipperhouse/displaywidth v0.6.2/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM= github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0 h1:jrYnow5+hy3WRDCBypUFvVKNSPPCdqgSXIE9eJDD8LM= github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew= github.com/olekukonko/tablewriter v1.1.3 h1:VSHhghXxrP0JHl+0NnKid7WoEmd9/urKRJLysb70nnA= github.com/olekukonko/tablewriter v1.1.3/go.mod h1:9VU0knjhmMkXjnMKrZ3+L2JhhtsQ/L38BbL3CRNE8tM= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.etcd.io/gofail v0.2.0 h1:p19drv16FKK345a09a1iubchlw/vmRuksmRzgBIGjcA= go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o= go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee h1:9s5V0M58uCy51LgP6SUjROx7Ofqf8lGmeD/cCLaoagI= go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee/go.mod h1:VteWcRz3UV3TOpfex1x8jgPKAyjRXLKw3j8RdK3UAps= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 h1:XmiuHzgJt067+a6kwyAzkhXooYVv3/TOw9cM2VfJgUM= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= k8s.io/utils v0.0.0-20260108192941-914a6e750570 h1:JT4W8lsdrGENg9W+YwwdLJxklIuKWdRm+BC+xt33FOY= k8s.io/utils v0.0.0-20260108192941-914a6e750570/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= ================================================ FILE: go.work ================================================ // This is a generated file. Do not edit directly. go 1.26 toolchain go1.26.1 use ( . ./api ./cache ./client/pkg ./client/v3 ./etcdctl ./etcdutl ./pkg ./server ./tests ./tools/mod ./tools/rw-heatmaps ./tools/testgrid-analysis ) ================================================ FILE: go.work.sum ================================================ bitbucket.org/creachadair/shell v0.0.8 h1:3yM6JcAfaGWzjzcCamTblzSIWXm/YSs0PFGIzBm2HTo= bitbucket.org/creachadair/shell v0.0.8/go.mod h1:vINzudofoUXZSJ5tREgpy+Etyjsag3ait5WOWImEVZ0= bitbucket.org/creachadair/stringset v0.0.11 h1:6Sv4CCv14Wm+OipW4f3tWOb0SQVpBDLW0knnJqUnmZ8= bitbucket.org/liamstask/goose v0.0.0-20150115234039-8488cc47d90c h1:bkb2NMGo3/Du52wvYj9Whth5KZfMV6d3O0Vbr3nz/UE= bitbucket.org/liamstask/goose v0.0.0-20150115234039-8488cc47d90c/go.mod h1:hSVuE3qU7grINVSwrmzHfpg9k87ALBk+XaualNyUzI4= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1 h1:YhMSc48s25kr7kv31Z8vf7sPUIq5YJva9z1mn/hAt0M= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1/go.mod h1:avRlCjnFzl98VPaeCtJ24RrV/wwHFzB8sWXhj26+n/U= buf.build/go/protovalidate v0.12.0 h1:4GKJotbspQjRCcqZMGVSuC8SjwZ/FmgtSuKDpKUTZew= buf.build/go/protovalidate v0.12.0/go.mod h1:q3PFfbzI05LeqxSwq+begW2syjy2Z6hLxZSkP1OH/D0= cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.121.2 h1:v2qQpN6Dx9x2NmwrqlesOt3Ys4ol5/lFZ6Mg1B7OJCg= cloud.google.com/go v0.121.2/go.mod h1:nRFlrHq39MNVWu+zESP2PosMWA0ryJw8KUBZ2iZpxbw= cloud.google.com/go/accessapproval v1.7.1 h1:/5YjNhR6lzCvmJZAnByYkfEgWjfAKwYP6nkuTk6nKFE= cloud.google.com/go/accesscontextmanager v1.8.1 h1:WIAt9lW9AXtqw/bnvrEUaE8VG/7bAAeMzRCBGMkc4+w= cloud.google.com/go/aiplatform v1.48.0 h1:M5davZWCTzE043rJCn+ZLW6hSxfG1KAx4vJTtas2/ec= cloud.google.com/go/analytics v0.21.3 h1:TFBC1ZAqX9/jL56GEXdLrVe5vT3I22bDVWyDwZX4IEg= cloud.google.com/go/apigateway v1.6.1 h1:aBSwCQPcp9rZ0zVEUeJbR623palnqtvxJlUyvzsKGQc= cloud.google.com/go/apigeeconnect v1.6.1 h1:6u/jj0P2c3Mcm+H9qLsXI7gYcTiG9ueyQL3n6vCmFJM= cloud.google.com/go/apigeeregistry v0.7.1 h1:hgq0ANLDx7t2FDZDJQrCMtCtddR/pjCqVuvQWGrQbXw= cloud.google.com/go/apikeys v0.6.0 h1:B9CdHFZTFjVti89tmyXXrO+7vSNo2jvZuHG8zD5trdQ= cloud.google.com/go/appengine v1.8.1 h1:J+aaUZ6IbTpBegXbmEsh8qZZy864ZVnOoWyfa1XSNbI= cloud.google.com/go/area120 v0.8.1 h1:wiOq3KDpdqXmaHzvZwKdpoM+3lDcqsI2Lwhyac7stss= cloud.google.com/go/artifactregistry v1.14.1 h1:k6hNqab2CubhWlGcSzunJ7kfxC7UzpAfQ1UPb9PDCKI= cloud.google.com/go/asset v1.14.1 h1:vlHdznX70eYW4V1y1PxocvF6tEwxJTTarwIGwOhFF3U= cloud.google.com/go/assuredworkloads v1.11.1 h1:yaO0kwS+SnhVSTF7BqTyVGt3DTocI6Jqo+S3hHmCwNk= cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI= cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ= cloud.google.com/go/automl v1.13.1 h1:iP9iQurb0qbz+YOOMfKSEjhONA/WcoOIjt6/m+6pIgo= cloud.google.com/go/baremetalsolution v1.1.1 h1:0Ge9PQAy6cZ1tRrkc44UVgYV15nw2TVnzJzYsMHXF+E= cloud.google.com/go/batch v1.3.1 h1:uE0Q//W7FOGPjf7nuPiP0zoE8wOT3ngoIO2HIet0ilY= cloud.google.com/go/beyondcorp v1.0.0 h1:VPg+fZXULQjs8LiMeWdLaB5oe8G9sEoZ0I0j6IMiG1Q= cloud.google.com/go/bigquery v1.53.0 h1:K3wLbjbnSlxhuG5q4pntHv5AEbQM1QqHKGYgwFIqOTg= cloud.google.com/go/billing v1.16.0 h1:1iktEAIZ2uA6KpebC235zi/rCXDdDYQ0bTXTNetSL80= cloud.google.com/go/binaryauthorization v1.6.1 h1:cAkOhf1ic92zEN4U1zRoSupTmwmxHfklcp1X7CCBKvE= cloud.google.com/go/certificatemanager v1.7.1 h1:uKsohpE0hiobx1Eak9jNcPCznwfB6gvyQCcS28Ah9E8= cloud.google.com/go/channel v1.16.0 h1:dqRkK2k7Ll/HHeYGxv18RrfhozNxuTJRkspW0iaFZoY= cloud.google.com/go/cloudbuild v1.13.0 h1:YBbAWcvE4x6xPWTyS+OU4eiUpz5rCS3VCM/aqmfddPA= cloud.google.com/go/clouddms v1.6.1 h1:rjR1nV6oVf2aNNB7B5uz1PDIlBjlOiBgR+q5n7bbB7M= cloud.google.com/go/cloudtasks v1.12.1 h1:cMh9Q6dkvh+Ry5LAPbD/U2aw6KAqdiU6FttwhbTo69w= cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/contactcenterinsights v1.10.0 h1:YR2aPedGVQPpFBZXJnPkqRj8M//8veIZZH5ZvICoXnI= cloud.google.com/go/container v1.24.0 h1:N51t/cgQJFqDD/W7Mb+IvmAPHrf8AbPx7Bb7aF4lROE= cloud.google.com/go/containeranalysis v0.10.1 h1:SM/ibWHWp4TYyJMwrILtcBtYKObyupwOVeceI9pNblw= cloud.google.com/go/datacatalog v1.16.0 h1:qVeQcw1Cz93/cGu2E7TYUPh8Lz5dn5Ws2siIuQ17Vng= cloud.google.com/go/dataflow v0.9.1 h1:VzG2tqsk/HbmOtq/XSfdF4cBvUWRK+S+oL9k4eWkENQ= cloud.google.com/go/dataform v0.8.1 h1:xcWso0hKOoxeW72AjBSIp/UfkvpqHNzzS0/oygHlcqY= cloud.google.com/go/datafusion v1.7.1 h1:eX9CZoyhKQW6g1Xj7+RONeDj1mV8KQDKEB9KLELX9/8= cloud.google.com/go/datalabeling v0.8.1 h1:zxsCD/BLKXhNuRssen8lVXChUj8VxF3ofN06JfdWOXw= cloud.google.com/go/dataplex v1.9.0 h1:yoBWuuUZklYp7nx26evIhzq8+i/nvKYuZr1jka9EqLs= cloud.google.com/go/dataproc v1.12.0 h1:W47qHL3W4BPkAIbk4SWmIERwsWBaNnWm0P2sdx3YgGU= cloud.google.com/go/dataproc/v2 v2.0.1 h1:4OpSiPMMGV3XmtPqskBU/RwYpj3yMFjtMLj/exi425Q= cloud.google.com/go/dataqna v0.8.1 h1:ITpUJep04hC9V7C+gcK390HO++xesQFSUJ7S4nSnF3U= cloud.google.com/go/datastore v1.13.0 h1:ktbC66bOQB3HJPQe8qNI1/aiQ77PMu7hD4mzE6uxe3w= cloud.google.com/go/datastream v1.10.0 h1:ra/+jMv36zTAGPfi8TRne1hXme+UsKtdcK4j6bnqQiw= cloud.google.com/go/deploy v1.13.0 h1:A+w/xpWgz99EYzB6e31gMGAI/P5jTZ2UO7veQK5jQ8o= cloud.google.com/go/dialogflow v1.40.0 h1:sCJbaXt6ogSbxWQnERKAzos57f02PP6WkGbOZvXUdwc= cloud.google.com/go/dlp v1.10.1 h1:tF3wsJ2QulRhRLWPzWVkeDz3FkOGVoMl6cmDUHtfYxw= cloud.google.com/go/documentai v1.22.0 h1:dW8ex9yb3oT9s1yD2+yLcU8Zq15AquRZ+wd0U+TkxFw= cloud.google.com/go/domains v0.9.1 h1:rqz6KY7mEg7Zs/69U6m6LMbB7PxFDWmT3QWNXIqhHm0= cloud.google.com/go/edgecontainer v1.1.1 h1:zhHWnLzg6AqzE+I3gzJqiIwHfjEBhWctNQEzqb+FaRo= cloud.google.com/go/errorreporting v0.3.0 h1:kj1XEWMu8P0qlLhm3FwcaFsUvXChV/OraZwA70trRR0= cloud.google.com/go/essentialcontacts v1.6.2 h1:OEJ0MLXXCW/tX1fkxzEZOsv/wRfyFsvDVNaHWBAvoV0= cloud.google.com/go/eventarc v1.13.0 h1:xIP3XZi0Xawx8DEfh++mE2lrIi5kQmCr/KcWhJ1q0J4= cloud.google.com/go/filestore v1.7.1 h1:Eiz8xZzMJc5ppBWkuaod/PUdUZGCFR8ku0uS+Ah2fRw= cloud.google.com/go/firestore v1.11.0 h1:PPgtwcYUOXV2jFe1bV3nda3RCrOa8cvBjTOn2MQVfW8= cloud.google.com/go/functions v1.15.1 h1:LtAyqvO1TFmNLcROzHZhV0agEJfBi+zfMZsF4RT/a7U= cloud.google.com/go/gaming v1.10.1 h1:5qZmZEWzMf8GEFgm9NeC3bjFRpt7x4S6U7oLbxaf7N8= cloud.google.com/go/gkebackup v1.3.0 h1:lgyrpdhtJKV7l1GM15YFt+OCyHMxsQZuSydyNmS0Pxo= cloud.google.com/go/gkeconnect v0.8.1 h1:a1ckRvVznnuvDWESM2zZDzSVFvggeBaVY5+BVB8tbT0= cloud.google.com/go/gkehub v0.14.1 h1:2BLSb8i+Co1P05IYCKATXy5yaaIw/ZqGvVSBTLdzCQo= cloud.google.com/go/gkemulticloud v1.0.0 h1:MluqhtPVZReoriP5+adGIw+ij/RIeRik8KApCW2WMTw= cloud.google.com/go/grafeas v0.3.0 h1:oyTL/KjiUeBs9eYLw/40cpSZglUC+0F7X4iu/8t7NWs= cloud.google.com/go/gsuiteaddons v1.6.1 h1:mi9jxZpzVjLQibTS/XfPZvl+Jr6D5Bs8pGqUjllRb00= cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= cloud.google.com/go/iap v1.8.1 h1:X1tcp+EoJ/LGX6cUPt3W2D4H2Kbqq0pLAsldnsCjLlE= cloud.google.com/go/ids v1.4.1 h1:khXYmSoDDhWGEVxHl4c4IgbwSRR+qE/L4hzP3vaU9Hc= cloud.google.com/go/iot v1.7.1 h1:yrH0OSmicD5bqGBoMlWG8UltzdLkYzNUwNVUVz7OT54= cloud.google.com/go/kms v1.15.0 h1:xYl5WEaSekKYN5gGRyhjvZKM22GVBBCzegGNVPy+aIs= cloud.google.com/go/language v1.10.1 h1:3MXeGEv8AlX+O2LyV4pO4NGpodanc26AmXwOuipEym0= cloud.google.com/go/lifesciences v0.9.1 h1:axkANGx1wiBXHiPcJZAE+TDjjYoJRIDzbHC/WYllCBU= cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= cloud.google.com/go/managedidentities v1.6.1 h1:2/qZuOeLgUHorSdxSQGtnOu9xQkBn37+j+oZQv/KHJY= cloud.google.com/go/maps v1.4.0 h1:PdfgpBLhAoSzZrQXP+/zBc78fIPLZSJp5y8+qSMn2UU= cloud.google.com/go/mediatranslation v0.8.1 h1:50cF7c1l3BanfKrpnTCaTvhf+Fo6kdF21DG0byG7gYU= cloud.google.com/go/memcache v1.10.1 h1:7lkLsF0QF+Mre0O/NvkD9Q5utUNwtzvIYjrOLOs0HO0= cloud.google.com/go/metastore v1.12.0 h1:+9DsxUOHvsqvC0ylrRc/JwzbXJaaBpfIK3tX0Lx8Tcc= cloud.google.com/go/monitoring v1.17.0 h1:blrdvF0MkPPivSO041ihul7rFMhXdVp8Uq7F59DKXTU= cloud.google.com/go/monitoring v1.17.0/go.mod h1:KwSsX5+8PnXv5NJnICZzW2R8pWTis8ypC4zmdRD63Tw= cloud.google.com/go/networkconnectivity v1.12.1 h1:LnrYM6lBEeTq+9f2lR4DjBhv31EROSAQi/P5W4Q0AEc= cloud.google.com/go/networkmanagement v1.8.0 h1:/3xP37eMxnyvkfLrsm1nv1b2FbMMSAEAOlECTvoeCq4= cloud.google.com/go/networksecurity v0.9.1 h1:TBLEkMp3AE+6IV/wbIGRNTxnqLXHCTEQWoxRVC18TzY= cloud.google.com/go/notebooks v1.9.1 h1:CUqMNEtv4EHFnbogV+yGHQH5iAQLmijOx191innpOcs= cloud.google.com/go/optimization v1.4.1 h1:pEwOAmO00mxdbesCRSsfj8Sd4rKY9kBrYW7Vd3Pq7cA= cloud.google.com/go/orchestration v1.8.1 h1:KmN18kE/xa1n91cM5jhCh7s1/UfIguSCisw7nTMUzgE= cloud.google.com/go/orgpolicy v1.11.1 h1:I/7dHICQkNwym9erHqmlb50LRU588NPCvkfIY0Bx9jI= cloud.google.com/go/osconfig v1.12.1 h1:dgyEHdfqML6cUW6/MkihNdTVc0INQst0qSE8Ou1ub9c= cloud.google.com/go/oslogin v1.10.1 h1:LdSuG3xBYu2Sgr3jTUULL1XCl5QBx6xwzGqzoDUw1j0= cloud.google.com/go/phishingprotection v0.8.1 h1:aK/lNmSd1vtbft/vLe2g7edXK72sIQbqr2QyrZN/iME= cloud.google.com/go/policytroubleshooter v1.8.0 h1:XTMHy31yFmXgQg57CB3w9YQX8US7irxDX0Fl0VwlZyY= cloud.google.com/go/privatecatalog v0.9.1 h1:B/18xGo+E0EMS9LOEQ0zXz7F2asMgmVgTYGSI89MHOA= cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= cloud.google.com/go/pubsublite v1.8.1 h1:pX+idpWMIH30/K7c0epN6V703xpIcMXWRjKJsz0tYGY= cloud.google.com/go/recaptchaenterprise v1.3.1 h1:u6EznTGzIdsyOsvm+Xkw0aSuKFXQlyjGE9a4exk6iNQ= cloud.google.com/go/recaptchaenterprise/v2 v2.7.2 h1:IGkbudobsTXAwmkEYOzPCQPApUCsN4Gbq3ndGVhHQpI= cloud.google.com/go/recommendationengine v0.8.1 h1:nMr1OEVHuDambRn+/y4RmNAmnR/pXCuHtH0Y4tCgGRQ= cloud.google.com/go/recommender v1.10.1 h1:UKp94UH5/Lv2WXSQe9+FttqV07x/2p1hFTMMYVFtilg= cloud.google.com/go/redis v1.13.1 h1:YrjQnCC7ydk+k30op7DSjSHw1yAYhqYXFcOq1bSXRYA= cloud.google.com/go/resourcemanager v1.9.1 h1:QIAMfndPOHR6yTmMUB0ZN+HSeRmPjR/21Smq5/xwghI= cloud.google.com/go/resourcesettings v1.6.1 h1:Fdyq418U69LhvNPFdlEO29w+DRRjwDA4/pFamm4ksAg= cloud.google.com/go/retail v1.14.1 h1:gYBrb9u/Hc5s5lUTFXX1Vsbc/9BEvgtioY6ZKaK0DK8= cloud.google.com/go/run v1.2.0 h1:kHeIG8q+N6Zv0nDkBjSOYfK2eWqa5FnaiDPH/7/HirE= cloud.google.com/go/scheduler v1.10.1 h1:yoZbZR8880KgPGLmACOMCiY2tPk+iX4V/dkxqTirlz8= cloud.google.com/go/secretmanager v1.11.1 h1:cLTCwAjFh9fKvU6F13Y4L9vPcx9yiWPyWXE4+zkuEQs= cloud.google.com/go/security v1.15.1 h1:jR3itwycg/TgGA0uIgTItcVhA55hKWiNJxaNNpQJaZE= cloud.google.com/go/securitycenter v1.23.0 h1:XOGJ9OpnDtqg8izd7gYk/XUhj8ytjIalyjjsR6oyG0M= cloud.google.com/go/servicecontrol v1.11.1 h1:d0uV7Qegtfaa7Z2ClDzr9HJmnbJW7jn0WhZ7wOX6hLE= cloud.google.com/go/servicedirectory v1.11.0 h1:pBWpjCFVGWkzVTkqN3TBBIqNSoSHY86/6RL0soSQ4z8= cloud.google.com/go/servicemanagement v1.8.0 h1:fopAQI/IAzlxnVeiKn/8WiV6zKndjFkvi+gzu+NjywY= cloud.google.com/go/serviceusage v1.6.0 h1:rXyq+0+RSIm3HFypctp7WoXxIA563rn206CfMWdqXX4= cloud.google.com/go/shell v1.7.1 h1:aHbwH9LSqs4r2rbay9f6fKEls61TAjT63jSyglsw7sI= cloud.google.com/go/spanner v1.47.0 h1:aqiMP8dhsEXgn9K5EZBWxPG7dxIiyM2VaikqeU4iteg= cloud.google.com/go/speech v1.19.0 h1:MCagaq8ObV2tr1kZJcJYgXYbIn8Ai5rp42tyGYw9rls= cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI= cloud.google.com/go/storagetransfer v1.10.0 h1:+ZLkeXx0K0Pk5XdDmG0MnUVqIR18lllsihU/yq39I8Q= cloud.google.com/go/talent v1.6.2 h1:j46ZgD6N2YdpFPux9mc7OAf4YK3tiBCsbLKc8rQx+bU= cloud.google.com/go/texttospeech v1.7.1 h1:S/pR/GZT9p15R7Y2dk2OXD/3AufTct/NSxT4a7nxByw= cloud.google.com/go/tpu v1.6.1 h1:kQf1jgPY04UJBYYjNUO+3GrZtIb57MfGAW2bwgLbR3A= cloud.google.com/go/trace v1.10.4 h1:2qOAuAzNezwW3QN+t41BtkDJOG42HywL73q8x/f6fnM= cloud.google.com/go/trace v1.10.4/go.mod h1:Nso99EDIK8Mj5/zmB+iGr9dosS/bzWCJ8wGmE6TXNWY= cloud.google.com/go/translate v1.8.2 h1:PQHamiOzlehqLBJMnM72lXk/OsMQewZB12BKJ8zXrU0= cloud.google.com/go/video v1.19.0 h1:BRyyS+wU+Do6VOXnb8WfPr42ZXti9hzmLKLUCkggeK4= cloud.google.com/go/videointelligence v1.11.1 h1:MBMWnkQ78GQnRz5lfdTAbBq/8QMCF3wahgtHh3s/J+k= cloud.google.com/go/vision v1.2.0 h1:/CsSTkbmO9HC8iQpxbK8ATms3OQaX3YQUeTMGCxlaK4= cloud.google.com/go/vision/v2 v2.7.2 h1:ccK6/YgPfGHR/CyESz1mvIbsht5Y2xRsWCPqmTNydEw= cloud.google.com/go/vmmigration v1.7.1 h1:gnjIclgqbEMc+cF5IJuPxp53wjBIlqZ8h9hE8Rkwp7A= cloud.google.com/go/vmwareengine v1.0.0 h1:qsJ0CPlOQu/3MFBGklu752v3AkD+Pdu091UmXJ+EjTA= cloud.google.com/go/vpcaccess v1.7.1 h1:ram0GzjNWElmbxXMIzeOZUkQ9J8ZAahD6V8ilPGqX0Y= cloud.google.com/go/webrisk v1.9.1 h1:Ssy3MkOMOnyRV5H2bkMQ13Umv7CwB/kugo3qkAX83Fk= cloud.google.com/go/websecurityscanner v1.6.1 h1:CfEF/vZ+xXyAR3zC9iaC/QRdf1MEgS20r5UR17Q4gOg= cloud.google.com/go/workflows v1.11.1 h1:2akeQ/PgtRhrNuD/n1WvJd5zb7YyuDZrlOanBj2ihPg= codeberg.org/go-fonts/stix v0.3.0 h1:vHI1LmLWEcAdcf+5aRMtA1eYKJJ9ZjetVstBD/dRe1Q= codeberg.org/go-fonts/stix v0.3.0/go.mod h1:1OSJSnA/PoHqbW2tjkkqTmNPp5xTtJQN2GRXJjO/+WA= contrib.go.opencensus.io/exporter/stackdriver v0.13.14 h1:zBakwHardp9Jcb8sQHcHpXy/0+JIb1M8KjigCJzx7+4= contrib.go.opencensus.io/exporter/stackdriver v0.13.14/go.mod h1:5pSSGY0Bhuk7waTHuDf4aQ8D2DrhgETRo9fy6k3Xlzc= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= gioui.org v0.2.0 h1:RbzDn1h/pCVf/q44ImQSa/J3MIFpY3OWphzT/Tyei+w= gioui.org v0.2.0/go.mod h1:1H72sKEk/fNFV+l0JNeM2Dt3co3Y4uaQcD+I+/GQ0e4= gioui.org/cpu v0.0.0-20220412190645-f1e9e8c3b1f7 h1:tNJdnP5CgM39PRc+KWmBRRYX/zJ+rd5XaYxY5d5veqA= gioui.org/cpu v0.0.0-20220412190645-f1e9e8c3b1f7/go.mod h1:A8M0Cn5o+vY5LTMlnRoK3O5kG+rH0kWfJjeKd9QpBmQ= gioui.org/shader v1.0.6 h1:cvZmU+eODFR2545X+/8XucgZdTtEjR3QWW6W65b0q5Y= gioui.org/shader v1.0.6/go.mod h1:mWdiME581d/kV7/iEhLmUgUK5iZ09XR5XpduXzbePVM= gioui.org/x v0.2.0 h1:/MbdjKH19F16auv19UiQxli2n6BYPw7eyh9XBOTgmEw= gioui.org/x v0.2.0/go.mod h1:rCGN2nZ8ZHqrtseJoQxCMZpt2xrZUrdZ2WuMRLBJmYs= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9 h1:7kQgkwGRoLzC9K0oyXdJo7nve/bynv/KwUsxbiTlzAM= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19 h1:iXUgAaqDcIUGbRoy2TdeofRG/j1zpGRSEmNK05T+bi8= github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/stroke v0.0.0-20221221101821-bd29b49d73f0 h1:uF5Q/hWnDU1XZeT6CsrRSxHLroUSEYYO3kgES+yd+So= github.com/andybalholm/stroke v0.0.0-20221221101821-bd29b49d73f0/go.mod h1:ccdDYaY5+gO+cbnQdFxEXqfy0RkoV25H3jLXUDNM3wg= github.com/anthropics/anthropic-sdk-go v1.26.0 h1:oUTzFaUpAevfuELAP1sjL6CQJ9HHAfT7CoSYSac11PY= github.com/anthropics/anthropic-sdk-go v1.26.0/go.mod h1:qUKmaW+uuPB64iy1l+4kOSvaLqPXnHTTBKH6RVZ7q5Q= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/apache/arrow/go/v10 v10.0.1 h1:n9dERvixoC/1JjDmBcs9FPaEryoANa2sCgVFo6ez9cI= github.com/apache/arrow/go/v11 v11.0.0 h1:hqauxvFQxww+0mEU/2XHG6LT7eZternCZq+A5Yly2uM= github.com/apache/arrow/go/v12 v12.0.0 h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc= github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/aws/aws-sdk-go v1.46.4 h1:48tKgtm9VMPkb6y7HuYlsfhQmoIRAsTEXTsWLVlty4M= github.com/aws/aws-sdk-go v1.46.4/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/boombuler/barcode v1.0.1 h1:NDBbPmhS+EqABEs5Kg3n/5ZNjy73Pz7SIV+KCeqyXcs= github.com/bufbuild/protocompile v0.6.0 h1:Uu7WiSQ6Yj9DbkdnOe7U4mNKp58y9WDMKDn28/ZlunY= github.com/bufbuild/protocompile v0.6.0/go.mod h1:YNP35qEYoYGme7QMtz5SBCoN4kL4g12jTtjuzRNdjpE= github.com/bwesterb/go-ristretto v1.2.0 h1:xxWOVbN5m8NNKiSDZXE1jtZvZnC6JSJ9cYFADiZcWtw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg= github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/cloudflare/backoff v0.0.0-20161212185259-647f3cdfc87a h1:8d1CEOF1xldesKds5tRG3tExBsMOgWYownMHNCsev54= github.com/cloudflare/backoff v0.0.0-20161212185259-647f3cdfc87a/go.mod h1:rzgs2ZOiguV6/NpiDgADjRLPNyZlApIWxKpkT+X8SdY= github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/redoctober v0.0.0-20211013234631-6a74ccc611f6 h1:QKzett0dn5FhjcIHNKSClEilabfhWCnsdijq3ftm9Ms= github.com/cloudflare/redoctober v0.0.0-20211013234631-6a74ccc611f6/go.mod h1:Ikt4Wfpln1YOrak+auA8BNxgiilj0Y2y7nO+aN2eMzk= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk= github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= github.com/cristalhq/acmd v0.12.0 h1:RdlKnxjN+txbQosg8p/TRNZ+J1Rdne43MVQZ1zDhGWk= github.com/cristalhq/acmd v0.12.0/go.mod h1:LG5oa43pE/BbxtfMoImHCQN++0Su7dzipdgBjMCBVDQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/expr-lang/expr v1.17.7 h1:Q0xY/e/2aCIp8g9s/LGvMDCC5PxYlvHgDZRQ4y16JX8= github.com/expr-lang/expr v1.17.7/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fullstorydev/grpcurl v1.8.9 h1:JMvZXK8lHDGyLmTQ0ZdGDnVVGuwjbpaumf8p42z0d+c= github.com/fullstorydev/grpcurl v1.8.9/go.mod h1:PNNKevV5VNAV2loscyLISrEnWQI61eqR0F8l3bVadAA= github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw= github.com/getsentry/sentry-go v0.11.0 h1:qro8uttJGvNAMr5CLcFI9CHR0aDzXl0Vs3Pmw/oTPg8= github.com/getsentry/sentry-go v0.11.0/go.mod h1:KBQIxiZAetw62Cj8Ri964vAEWVdgfaUCn30Q3bCvANo= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= github.com/go-fonts/dejavu v0.1.0 h1:JSajPXURYqpr+Cu8U9bt8K+XcACIHWqWrvWCKyeFmVQ= github.com/go-fonts/latin-modern v0.2.0 h1:5/Tv1Ek/QCr20C6ZOz15vw3g7GELYL98KWr8Hgo+3vk= github.com/go-fonts/liberation v0.2.0 h1:jAkAWJP4S+OsrPLZM4/eC9iW7CtHy+HBXrEwZXWo5VM= github.com/go-fonts/stix v0.1.0 h1:UlZlgrvvmT/58o573ot7NFw0vZasZ5I6bcIft/oMdgg= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/log v0.1.0 h1:DGJh0Sm43HbOeYDNnVZFl8BvcYVvjD5bqYJvp0REbwQ= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81 h1:6zl3BbBhdnMkpSj2YY30qV3gDcVBGtFgVsV3+/i+mKQ= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-pdf/fpdf v0.6.0 h1:MlgtGIfsdMEEQJr2le6b/HNr1ZlQwxyWr77r2aj2U/8= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-text/typesetting v0.0.0-20230803102845-24e03d8b5372 h1:FQivqchis6bE2/9uF70M2gmmLpe82esEm2QadL0TEJo= github.com/go-text/typesetting v0.0.0-20230803102845-24e03d8b5372/go.mod h1:evDBbvNR/KaVFZ2ZlDSOWWXIUKq0wCOEtzLxRM8SG3k= github.com/goccmack/gocc v0.0.0-20230228185258-2292f9e40198 h1:FSii2UQeSLngl3jFoR4tUKZLprO7qUlh/TKKticc0BM= github.com/goccmack/gocc v0.0.0-20230228185258-2292f9e40198/go.mod h1:DTh/Y2+NbnOVVoypCCQrovMPDKUGp4yZpSbWg5D0XIM= github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cel-go v0.25.0 h1:jsFw9Fhn+3y2kBbltZR4VEz5xKkcIFRPDnuEzAGv5GY= github.com/google/cel-go v0.25.0/go.mod h1:hjEb6r5SuOSlhCHmFoLzu8HGCERvIsDAbxDAyNU/MmI= github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/go-github/v50 v50.2.0 h1:j2FyongEHlO9nxXLc+LP3wuBSVU9mVxfpdYUexMpIfk= github.com/google/go-pkcs11 v0.2.0 h1:5meDPB26aJ98f+K9G21f0AqZwo/S5BJMJh8nuhMbdsI= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/trillian v1.6.0 h1:jMBeDBIkINFvS2n6oV5maDqfRlxREAc6CW9QYWQ0qT4= github.com/google/trillian v1.6.0/go.mod h1:Yu3nIMITzNhhMJEHjAtp6xKiu+H/iHu2Oq5FjV2mCWI= github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 h1:tlyzajkF3030q6M8SvmJSemC9DTHL/xaMa18b65+JM4= github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA= github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jhump/protoreflect v1.15.3 h1:6SFRuqU45u9hIZPJAoZ8c28T3nK64BNdp9w6jFonzls= github.com/jhump/protoreflect v1.15.3/go.mod h1:4ORHmSBmlCW8fh3xHmJMGyul1zNqZK4Elxc8qKP+p1k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kylelemons/go-gypsy v1.0.0 h1:7/wQ7A3UL1bnqRMnZ6T8cwCOArfZCxFmb1iTxaOOo1s= github.com/kylelemons/go-gypsy v1.0.0/go.mod h1:chkXM0zjdpXOiqkCW1XcCHDfjfk14PH2KKkQWxfJUcU= github.com/letsencrypt/pkcs11key/v4 v4.0.0 h1:qLc/OznH7xMr5ARJgkZCCWk+EomQkiNTOoOF5LAgagc= github.com/letsencrypt/pkcs11key/v4 v4.0.0/go.mod h1:EFUvBDay26dErnNb70Nd0/VW3tJiIbETBPTl9ATXQag= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-star v0.6.1 h1:erE0rdztuaDq3bpGifD95wfoPrSZc95nGA6tbiNYh6M= github.com/lyft/protoc-gen-star/v2 v2.0.1 h1:keaAo8hRuAT0O3DfJ/wM3rufbAjGeJ1lAtWZHDjKGB0= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/mgechev/dots v1.0.0 h1:o+4OJ3OjWzgQHGJXKfJ8rbH4dqDugu5BiEy84nxg0k4= github.com/mgechev/dots v1.0.0/go.mod h1:rykuMydC9t3wfkM+ccYH3U3ss03vZGg6h3hmOznXLH0= github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/mozilla/tls-observatory v0.0.0-20250923143331-eef96233227e h1:gOlpekCwR+xjqedQsHo1c7aUSixaQUIe3sAcEeDCMLc= github.com/mozilla/tls-observatory v0.0.0-20250923143331-eef96233227e/go.mod h1:FUqVoUPHSEdDR0MnFM3Dh8AU0pZHLXUD127SAJGER/s= github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d h1:tLWCMSjfL8XyZwpu1RzI2UpJSPbZCOZ6DVHQFnlpL7A= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/olekukonko/ts v0.0.0-20171002115256-78ecb04241c0 h1:LiZB1h0GIcudcDci2bxbqI6DXV8bF8POAnArqvRrIyw= github.com/olekukonko/ts v0.0.0-20171002115256-78ecb04241c0/go.mod h1:F/7q8/HZz+TXjlsoZQQKVYvXTZaFH4QRa3y+j1p7MS0= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/openai/openai-go/v3 v3.23.0 h1:FRFwTcB4FoWFtIunTY/8fgHvzSHgqbfWjiCwOMVrsvw= github.com/openai/openai-go/v3 v3.23.0/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo= github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI= github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc= github.com/phpdave11/gofpdf v1.4.2 h1:KPKiIbfwbvC/wOncwhrpRdXVj2CZTCFlw4wnoyjtHfQ= github.com/phpdave11/gofpdi v1.0.13 h1:o61duiW8M9sMlkVXWlvP92sZJtGKENvW3VExs6dZukQ= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/prometheus v0.47.2 h1:jWcnuQHz1o1Wu3MZ6nMJDuTI0kU5yJp9pkxh8XEkNvI= github.com/prometheus/prometheus v0.47.2/go.mod h1:J/bmOSjgH7lFxz2gZhrWEZs2i64vMS+HIuZfmYNhJ/M= github.com/quasilyte/go-ruleguard/rules v0.0.0-20211022131956-028d6511ab71 h1:CNooiryw5aisadVfzneSZPswRWvnVW8hF1bS/vo8ReI= github.com/quasilyte/go-ruleguard/rules v0.0.0-20211022131956-028d6511ab71/go.mod h1:4cgAphtvu7Ftv7vOT2ZOYhC6CvBxZixcasr8qIOTA50= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245 h1:K1Xf3bKttbF+koVGaX5xngRIZ5bVjbmPnaxE/dR08uY= github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI= github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/quicktemplate v1.8.0 h1:zU0tjbIqTRgKQzFY1L42zq0qR3eh4WoQQdIdqCysW5k= github.com/valyala/quicktemplate v1.8.0/go.mod h1:qIqW8/igXt8fdrUln5kOSb+KWMaJ4Y8QUsfd1k6L2jM= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= github.com/zmap/zcertificate v0.0.1 h1:2X15TRx4Fr6qzKItfwUdww294OeRSmHILLa+Xn2Uv+s= go.etcd.io/etcd/client/v2 v2.305.12 h1:0m4ovXYo1CHaA/Mp3X/Fak5sRNIWf01wk/X1/G3sGKI= go.etcd.io/etcd/client/v2 v2.305.12/go.mod h1:aQ/yhsxMu+Oht1FOupSr60oBvcS9cKXHrzBpDsPTf9E= go.etcd.io/etcd/raft/v3 v3.5.12 h1:7r22RufdDsq2z3STjoR7Msz6fYH8tmbkdheGfwJNRmU= go.etcd.io/etcd/raft/v3 v3.5.12/go.mod h1:ERQuZVe79PI6vcC3DlKBukDCLja/L7YMu29B74Iwj4U= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE= go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/exp/shiny v0.0.0-20241009180824-f66d83c29e7c h1:jTMrjjZRcSH/BDxWhXCP6OWsfVgmnwI7J+F4/nyVXaU= golang.org/x/exp/shiny v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:3F+MieQB7dRYLTmnncoFbb1crS5lfQoTfDgQy6K4N0o= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= google.golang.org/api v0.155.0 h1:vBmGhCYs0djJttDNynWo44zosHlPvHmA0XiN2zP2DtA= google.golang.org/api v0.155.0/go.mod h1:GI5qK5f40kCpHfPn6+YzGAByIKWv8ujFnmoWm7Igduk= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genai v1.47.0 h1:iWCS7gEdO6rctOqfCYLOrZGKu2D+N42aTnCEcBvB1jo= google.golang.org/genai v1.47.0/go.mod h1:A3kkl0nyBjyFlNjgxIwKq70julKbIxpSxqKO5gw/gmk= google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230720185612-659f7aaaa771 h1:gm8vsVR64Jx1GxHY8M+p8YA2bxU/H/lymcutB2l7l9s= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= k8s.io/api v0.27.4 h1:0pCo/AN9hONazBKlNUdhQymmnfLRbSZjd5H5H3f0bSs= k8s.io/apimachinery v0.27.4 h1:CdxflD4AF61yewuid0fLl6bM4a3q04jWel0IlP+aYjs= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c h1:GohjlNKauSai7gN4wsJkeZ3WAJx4Sh+oT/b5IYn5suA= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/sqlite v1.18.2 h1:S2uFiaNPd/vTAP/4EmyY8Qe2Quzu26A2L1e25xRNTio= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/tcl v1.13.2 h1:5PQgL/29XkQ9wsEmmNPjzKs+7iPCaYqUJAhzPvQbjDA= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= ================================================ FILE: hack/README.md ================================================ Various hacks that are used by developers. ================================================ FILE: hack/benchmark/README.md ================================================ ## Usage Benchmark 3-member etcd cluster to get its read and write performance. ## Instructions 1. Start 3-member etcd cluster on 3 machines 2. Update `$leader` and `$servers` in the script 3. Run the script in a separate machine ## Caveat 1. Set environment variable `GOMAXPROCS` as the number of available cores to maximize CPU resources for both etcd member and bench process. 2. Set the number of open files per process as 10000 for amounts of client connections for both etcd member and benchmark process. ================================================ FILE: hack/benchmark/bench.sh ================================================ #!/bin/bash -e leader=http://localhost:2379 # assume three servers servers=( http://localhost:2379 http://localhost:22379 http://localhost:32379 ) keyarray=( 64 256 ) for keysize in ${keyarray[@]}; do echo write, 1 client, $keysize key size, to leader ./hey -m PUT -n 10 -d value=`head -c $keysize < /dev/zero | tr '\0' '\141'` -c 1 -T application/x-www-form-urlencoded $leader/v2/keys/foo | grep -e "Requests/sec" -e "Latency" -e "90%" | tr "\n" "\t" | xargs echo echo write, 64 client, $keysize key size, to leader ./hey -m PUT -n 640 -d value=`head -c $keysize < /dev/zero | tr '\0' '\141'` -c 64 -T application/x-www-form-urlencoded $leader/v2/keys/foo | grep -e "Requests/sec" -e "Latency" -e "90%" | tr "\n" "\t" | xargs echo echo write, 256 client, $keysize key size, to leader ./hey -m PUT -n 2560 -d value=`head -c $keysize < /dev/zero | tr '\0' '\141'` -c 256 -T application/x-www-form-urlencoded $leader/v2/keys/foo | grep -e "Requests/sec" -e "Latency" -e "90%" | tr "\n" "\t" | xargs echo echo write, 64 client, $keysize key size, to all servers for i in ${servers[@]}; do ./hey -m PUT -n 210 -d value=`head -c $keysize < /dev/zero | tr '\0' '\141'` -c 21 -T application/x-www-form-urlencoded $i/v2/keys/foo | grep -e "Requests/sec" -e "Latency" -e "90%" | tr "\n" "\t" | xargs echo & done # wait for all heys to start running sleep 3 # wait for all heys to finish for pid in $(pgrep 'hey'); do while kill -0 "$pid" 2> /dev/null; do sleep 3 done done echo write, 256 client, $keysize key size, to all servers for i in ${servers[@]}; do ./hey -m PUT -n 850 -d value=`head -c $keysize < /dev/zero | tr '\0' '\141'` -c 85 -T application/x-www-form-urlencoded $i/v2/keys/foo | grep -e "Requests/sec" -e "Latency" -e "90%" | tr "\n" "\t" | xargs echo & done sleep 3 for pid in $(pgrep 'hey'); do while kill -0 "$pid" 2> /dev/null; do sleep 3 done done echo read, 1 client, $keysize key size, to leader ./hey -n 100 -c 1 $leader/v2/keys/foo | grep -e "Requests/sec" -e "Latency" -e "90%" | tr "\n" "\t" | xargs echo echo read, 64 client, $keysize key size, to leader ./hey -n 6400 -c 64 $leader/v2/keys/foo | grep -e "Requests/sec" -e "Latency" -e "90%" | tr "\n" "\t" | xargs echo echo read, 256 client, $keysize key size, to leader ./hey -n 25600 -c 256 $leader/v2/keys/foo | grep -e "Requests/sec" -e "Latency" -e "90%" | tr "\n" "\t" | xargs echo echo read, 64 client, $keysize key size, to all servers # bench servers one by one, so it doesn't overload this benchmark machine # It doesn't impact correctness because read request doesn't involve peer interaction. for i in ${servers[@]}; do ./hey -n 21000 -c 21 $i/v2/keys/foo | grep -e "Requests/sec" -e "Latency" -e "90%" | tr "\n" "\t" | xargs echo done echo read, 256 client, $keysize key size, to all servers for i in ${servers[@]}; do ./hey -n 85000 -c 85 $i/v2/keys/foo | grep -e "Requests/sec" -e "Latency" -e "90%" | tr "\n" "\t" | xargs echo done done ================================================ FILE: hack/insta-discovery/Procfile ================================================ # Use goreman to run `go get github.com/mattn/goreman` # One of the four etcd members falls back to a proxy etcd1: ../../bin/etcd --name infra1 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:2379 --listen-peer-urls http://127.0.0.1:2380 --initial-advertise-peer-urls http://127.0.0.1:2380 etcd2: ../../bin/etcd --name infra2 --listen-client-urls http://127.0.0.1:12379 --advertise-client-urls http://127.0.0.1:12379 --listen-peer-urls http://127.0.0.1:12380 --initial-advertise-peer-urls http://127.0.0.1:12380 etcd3: ../../bin/etcd --name infra3 --listen-client-urls http://127.0.0.1:22379 --advertise-client-urls http://127.0.0.1:22379 --listen-peer-urls http://127.0.0.1:22380 --initial-advertise-peer-urls http://127.0.0.1:22380 etcd4: ../../bin/etcd --name infra4 --listen-client-urls http://127.0.0.1:32379 --advertise-client-urls http://127.0.0.1:32379 --listen-peer-urls http://127.0.0.1:32380 --initial-advertise-peer-urls http://127.0.0.1:32380 ================================================ FILE: hack/insta-discovery/README.md ================================================ Starts a cluster via the discovery service locally. Useful for testing. ================================================ FILE: hack/insta-discovery/discovery ================================================ #!/bin/sh rm -rf infra*.etcd disc=$(curl -s https://discovery.etcd.io/new?size=3) echo ETCD_DISCOVERY=${disc} > .env echo "setup discovery start your cluster" cat .env goreman start ================================================ FILE: hack/kubernetes-deploy/README.md ================================================ # etcd on Kubernetes This is an example setting up etcd as a set of pods and services running on top of kubernetes. Using: ``` $ kubectl create -f etcd.yml services/etcd-client pods/etcd0 services/etcd0 pods/etcd1 services/etcd1 pods/etcd2 services/etcd2 $ # now deploy a service that consumes etcd, such as vulcand $ kubectl create -f vulcand.yml ``` TODO: - create a replication controller like service that knows how to add and remove nodes from the cluster correctly - use kubernetes secrets API to configure TLS for etcd clients and peers ================================================ FILE: hack/kubernetes-deploy/etcd.yml ================================================ --- apiVersion: v1 kind: Service metadata: name: etcd-client spec: ports: - name: etcd-client-port port: 2379 protocol: TCP targetPort: 2379 selector: app: etcd --- apiVersion: v1 kind: Pod metadata: labels: app: etcd etcd_node: etcd0 name: etcd0 spec: containers: - command: - /usr/local/bin/etcd - --name - etcd0 - --initial-advertise-peer-urls - http://etcd0:2380 - --listen-peer-urls - http://0.0.0.0:2380 - --listen-client-urls - http://0.0.0.0:2379 - --advertise-client-urls - http://etcd0:2379 - --initial-cluster - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380 - --initial-cluster-state - new image: quay.io/coreos/etcd:latest name: etcd0 ports: - containerPort: 2379 name: client protocol: TCP - containerPort: 2380 name: server protocol: TCP restartPolicy: Always --- apiVersion: v1 kind: Service metadata: labels: etcd_node: etcd0 name: etcd0 spec: ports: - name: client port: 2379 protocol: TCP targetPort: 2379 - name: server port: 2380 protocol: TCP targetPort: 2380 selector: etcd_node: etcd0 --- apiVersion: v1 kind: Pod metadata: labels: app: etcd etcd_node: etcd1 name: etcd1 spec: containers: - command: - /usr/local/bin/etcd - --name - etcd1 - --initial-advertise-peer-urls - http://etcd1:2380 - --listen-peer-urls - http://0.0.0.0:2380 - --listen-client-urls - http://0.0.0.0:2379 - --advertise-client-urls - http://etcd1:2379 - --initial-cluster - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380 - --initial-cluster-state - new image: quay.io/coreos/etcd:latest name: etcd1 ports: - containerPort: 2379 name: client protocol: TCP - containerPort: 2380 name: server protocol: TCP restartPolicy: Always --- apiVersion: v1 kind: Service metadata: labels: etcd_node: etcd1 name: etcd1 spec: ports: - name: client port: 2379 protocol: TCP targetPort: 2379 - name: server port: 2380 protocol: TCP targetPort: 2380 selector: etcd_node: etcd1 --- apiVersion: v1 kind: Pod metadata: labels: app: etcd etcd_node: etcd2 name: etcd2 spec: containers: - command: - /usr/local/bin/etcd - --name - etcd2 - --initial-advertise-peer-urls - http://etcd2:2380 - --listen-peer-urls - http://0.0.0.0:2380 - --listen-client-urls - http://0.0.0.0:2379 - --advertise-client-urls - http://etcd2:2379 - --initial-cluster - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380 - --initial-cluster-state - new image: quay.io/coreos/etcd:latest name: etcd2 ports: - containerPort: 2379 name: client protocol: TCP - containerPort: 2380 name: server protocol: TCP restartPolicy: Always --- apiVersion: v1 kind: Service metadata: labels: etcd_node: etcd2 name: etcd2 spec: ports: - name: client port: 2379 protocol: TCP targetPort: 2379 - name: server port: 2380 protocol: TCP targetPort: 2380 selector: etcd_node: etcd2 ================================================ FILE: hack/kubernetes-deploy/vulcand.yml ================================================ --- apiVersion: v1 kind: Pod metadata: labels: app: vulcand name: vulcand spec: containers: - command: - /go/bin/vulcand - -apiInterface=0.0.0.0 - --etcd=http://etcd-client:2379 image: mailgun/vulcand:v0.8.0-beta.2 name: vulcand ports: - containerPort: 8081 name: api protocol: TCP - containerPort: 8082 name: server protocol: TCP restartPolicy: Always ================================================ FILE: hack/patch/README.md ================================================ # ./hack/patch/cherrypick.sh Handles cherry-picks of PR(s) from etcd main to a stable etcd release branch automatically. ## Setup Set the `UPSTREAM_REMOTE` and `FORK_REMOTE` environment variables. `UPSTREAM_REMOTE` should be set to git remote name of `github.com/etcd-io/etcd`, and `FORK_REMOTE` should be set to the git remote name of the forked etcd repo (`github.com/${github-username}/etcd`). Use `git remote -v` to look up the git remote names. If etcd has not been forked, create one on github.com and register it locally with `git remote add ...`. ``` export UPSTREAM_REMOTE=upstream export FORK_REMOTE=origin export GITHUB_USER=${github-username} ``` Next, install hub from https://github.com/github/hub ## Usage To cherry pick PR 12345 onto release-3.2 and propose is as a PR, run: ```sh ./hack/patch/cherrypick.sh ${UPSTREAM_REMOTE}/release-3.2 12345 ``` To cherry pick 12345 then 56789 and propose them togther as a single PR, run: ``` ./hack/patch/cherrypick.sh ${UPSTREAM_REMOTE}/release-3.2 12345 56789 ``` ================================================ FILE: hack/patch/cherrypick.sh ================================================ #!/usr/bin/env bash # Based on github.com/kubernetes/kubernetes/blob/v1.8.2/hack/cherry_pick_pull.sh # Checkout a PR from GitHub. (Yes, this is sitting in a Git tree. How # meta.) Assumes you care about pulls from remote "upstream" and # checks thems out to a branch named: # automated-cherry-pick-of--- set -o errexit set -o nounset set -o pipefail declare -r ETCD_ROOT="$(dirname "${BASH_SOURCE}")/../.." cd "${ETCD_ROOT}" declare -r STARTINGBRANCH=$(git symbolic-ref --short HEAD) declare -r REBASEMAGIC="${ETCD_ROOT}/.git/rebase-apply" DRY_RUN=${DRY_RUN:-""} REGENERATE_DOCS=${REGENERATE_DOCS:-""} UPSTREAM_REMOTE=${UPSTREAM_REMOTE:-upstream} FORK_REMOTE=${FORK_REMOTE:-origin} if [[ -z ${GITHUB_USER:-} ]]; then echo "Please export GITHUB_USER= (or GH organization, if that's where your fork lives)" exit 1 fi if ! command -v hub > /dev/null; then echo "Can't find 'hub' tool in PATH, please install from https://github.com/github/hub" exit 1 fi if [[ "$#" -lt 2 ]]; then echo "${0} ...: cherry pick one or more onto and leave instructions for proposing pull request" echo echo " Checks out and handles the cherry-pick of (possibly multiple) for you." echo " Examples:" echo " $0 upstream/release-3.14 12345 # Cherry-picks PR 12345 onto upstream/release-3.14 and proposes that as a PR." echo " $0 upstream/release-3.14 12345 56789 # Cherry-picks PR 12345, then 56789 and proposes the combination as a single PR." echo echo " Set the DRY_RUN environment var to skip git push and creating PR." echo " This is useful for creating patches to a release branch without making a PR." echo " When DRY_RUN is set the script will leave you in a branch containing the commits you cherry-picked." echo echo " Set the REGENERATE_DOCS environment var to regenerate documentation for the target branch after picking the specified commits." echo " This is useful when picking commits containing changes to API documentation." echo echo " Set UPSTREAM_REMOTE (default: upstream) and FORK_REMOTE (default: origin)" echo " To override the default remote names to what you have locally." exit 2 fi if git_status=$(git status --porcelain --untracked=no 2>/dev/null) && [[ -n "${git_status}" ]]; then echo "!!! Dirty tree. Clean up and try again." exit 1 fi if [[ -e "${REBASEMAGIC}" ]]; then echo "!!! 'git rebase' or 'git am' in progress. Clean up and try again." exit 1 fi declare -r BRANCH="$1" shift 1 declare -r PULLS=( "$@" ) function join { local IFS="$1"; shift; echo "$*"; } declare -r PULLDASH=$(join - "${PULLS[@]/#/#}") # Generates something like "#12345-#56789" declare -r PULLSUBJ=$(join " " "${PULLS[@]/#/#}") # Generates something like "#12345 #56789" echo "+++ Updating remotes..." git remote update "${UPSTREAM_REMOTE}" "${FORK_REMOTE}" if ! git log -n1 --format=%H "${BRANCH}" >/dev/null 2>&1; then echo "!!! '${BRANCH}' not found. The second argument should be something like ${UPSTREAM_REMOTE}/release-0.21." echo " (In particular, it needs to be a valid, existing remote branch that I can 'git checkout'.)" exit 1 fi declare -r NEWBRANCHREQ="automated-cherry-pick-of-${PULLDASH}" # "Required" portion for tools. declare -r NEWBRANCH="$(echo "${NEWBRANCHREQ}-${BRANCH}" | sed 's/\//-/g')" declare -r NEWBRANCHUNIQ="${NEWBRANCH}-$(date +%s)" echo "+++ Creating local branch ${NEWBRANCHUNIQ}" cleanbranch="" prtext="" gitamcleanup=false function return_to_kansas { if [[ "${gitamcleanup}" == "true" ]]; then echo echo "+++ Aborting in-progress git am." git am --abort >/dev/null 2>&1 || true fi # return to the starting branch and delete the PR text file if [[ -z "${DRY_RUN}" ]]; then echo echo "+++ Returning you to the ${STARTINGBRANCH} branch and cleaning up." git checkout -f "${STARTINGBRANCH}" >/dev/null 2>&1 || true if [[ -n "${cleanbranch}" ]]; then git branch -D "${cleanbranch}" >/dev/null 2>&1 || true fi if [[ -n "${prtext}" ]]; then rm "${prtext}" fi fi } trap return_to_kansas EXIT SUBJECTS=() function make-a-pr() { local rel="$(basename "${BRANCH}")" echo echo "+++ Creating a pull request on GitHub at ${GITHUB_USER}:${NEWBRANCH}" # This looks like an unnecessary use of a tmpfile, but it avoids # https://github.com/github/hub/issues/976 Otherwise stdin is stolen # when we shove the heredoc at hub directly, tickling the ioctl # crash. prtext="$(mktemp -t prtext.XXXX)" # cleaned in return_to_kansas cat >"${prtext}" <&2 exit 1 fi done if [[ "${conflicts}" != "true" ]]; then echo "!!! git am failed, likely because of an in-progress 'git am' or 'git rebase'" exit 1 fi } # set the subject subject=$(grep -m 1 "^Subject" "/tmp/${pull}.patch" | sed -e 's/Subject: \[PATCH//g' | sed 's/.*] //') SUBJECTS+=("#${pull}: ${subject}") # remove the patch file from /tmp rm -f "/tmp/${pull}.patch" done gitamcleanup=false # Re-generate docs (if needed) if [[ -n "${REGENERATE_DOCS}" ]]; then echo echo "Regenerating docs..." if ! hack/generate-docs.sh; then echo echo "hack/generate-docs.sh FAILED to complete." exit 1 fi fi if [[ -n "${DRY_RUN}" ]]; then echo "!!! Skipping git push and PR creation because you set DRY_RUN." echo "To return to the branch you were in when you invoked this script:" echo echo " git checkout ${STARTINGBRANCH}" echo echo "To delete this branch:" echo echo " git branch -D ${NEWBRANCHUNIQ}" exit 0 fi if git remote -v | grep ^${FORK_REMOTE} | grep etcd/etcd.git; then echo "!!! You have ${FORK_REMOTE} configured as your etcd/etcd.git" echo "This isn't normal. Leaving you with push instructions:" echo echo "+++ First manually push the branch this script created:" echo echo " git push REMOTE ${NEWBRANCHUNIQ}:${NEWBRANCH}" echo echo "where REMOTE is your personal fork (maybe ${UPSTREAM_REMOTE}? Consider swapping those.)." echo "OR consider setting UPSTREAM_REMOTE and FORK_REMOTE to different values." echo make-a-pr cleanbranch="" exit 0 fi echo echo "+++ I'm about to do the following to push to GitHub (and I'm assuming ${FORK_REMOTE} is your personal fork):" echo echo " git push ${FORK_REMOTE} ${NEWBRANCHUNIQ}:${NEWBRANCH}" echo read -p "+++ Proceed (anything but 'y' aborts the cherry-pick)? [y/n] " -r if ! [[ "${REPLY}" =~ ^[yY]$ ]]; then echo "Aborting." >&2 exit 1 fi git push "${FORK_REMOTE}" -f "${NEWBRANCHUNIQ}:${NEWBRANCH}" make-a-pr ================================================ FILE: hack/tls-setup/Makefile ================================================ .PHONY: cfssl ca req clean CFSSL = @env PATH=$(GOPATH)/bin:$(PATH) cfssl JSON = env PATH=$(GOPATH)/bin:$(PATH) cfssljson all: ca req cfssl: HTTPS_PROXY=127.0.0.1:12639 go get -u -tags nopkcs11 github.com/cloudflare/cfssl/cmd/cfssl HTTPS_PROXY=127.0.0.1:12639 go get -u github.com/cloudflare/cfssl/cmd/cfssljson HTTPS_PROXY=127.0.0.1:12639 go get -u github.com/mattn/goreman ca: mkdir -p certs $(CFSSL) gencert -initca config/ca-csr.json | $(JSON) -bare certs/ca req: $(CFSSL) gencert \ -ca certs/ca.pem \ -ca-key certs/ca-key.pem \ -config config/ca-config.json \ config/req-csr.json | $(JSON) -bare certs/${infra0} $(CFSSL) gencert \ -ca certs/ca.pem \ -ca-key certs/ca-key.pem \ -config config/ca-config.json \ config/req-csr.json | $(JSON) -bare certs/${infra1} $(CFSSL) gencert \ -ca certs/ca.pem \ -ca-key certs/ca-key.pem \ -config config/ca-config.json \ config/req-csr.json | $(JSON) -bare certs/${infra2} $(CFSSL) gencert \ -ca certs/ca.pem \ -ca-key certs/ca-key.pem \ -config config/ca-config.json \ config/req-csr.json | $(JSON) -bare certs/peer-${infra0} $(CFSSL) gencert \ -ca certs/ca.pem \ -ca-key certs/ca-key.pem \ -config config/ca-config.json \ config/req-csr.json | $(JSON) -bare certs/peer-${infra1} $(CFSSL) gencert \ -ca certs/ca.pem \ -ca-key certs/ca-key.pem \ -config config/ca-config.json \ config/req-csr.json | $(JSON) -bare certs/peer-${infra2} clean: rm -rf certs ================================================ FILE: hack/tls-setup/Procfile ================================================ # Use goreman to run `go get github.com/mattn/goreman` etcd1: ../../bin/etcd --name infra1 --listen-client-urls https://localhost:2379 --advertise-client-urls https://localhost:2379 --listen-peer-urls https://localhost:2380 --initial-advertise-peer-urls https://localhost:2380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=https://localhost:2380,infra2=https://localhost:12380,infra3=https://localhost:22380' --initial-cluster-state new --cert-file=certs/etcd1.pem --key-file=certs/etcd1-key.pem --peer-cert-file=certs/etcd1.pem --peer-key-file=certs/etcd1-key.pem --peer-client-cert-auth --peer-trusted-ca-file=certs/ca.pem etcd2: ../../bin/etcd --name infra2 --listen-client-urls https://localhost:12379 --advertise-client-urls https://localhost:12379 --listen-peer-urls https://localhost:12380 --initial-advertise-peer-urls https://localhost:12380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=https://localhost:2380,infra2=https://localhost:12380,infra3=https://localhost:22380' --initial-cluster-state new --cert-file=certs/etcd2.pem --key-file=certs/etcd2-key.pem --peer-cert-file=certs/etcd2.pem --peer-key-file=certs/etcd2-key.pem --peer-client-cert-auth --peer-trusted-ca-file=certs/ca.pem etcd3: ../../bin/etcd --name infra3 --listen-client-urls https://localhost:22379 --advertise-client-urls https://localhost:22379 --listen-peer-urls https://localhost:22380 --initial-advertise-peer-urls https://localhost:22380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=https://localhost:2380,infra2=https://localhost:12380,infra3=https://localhost:22380' --initial-cluster-state new --cert-file=certs/etcd3.pem --key-file=certs/etcd3-key.pem --peer-cert-file=certs/etcd3.pem --peer-key-file=certs/etcd3-key.pem --peer-client-cert-auth --peer-trusted-ca-file=certs/ca.pem proxy: ../../bin/etcd --name proxy1 --proxy=on --listen-client-urls https://localhost:8080 --initial-cluster 'infra1=https://localhost:2380,infra2=https://localhost:12380,infra3=https://localhost:22380' --cert-file=certs/proxy1.pem --key-file=certs/proxy1-key.pem --trusted-ca-file=certs/ca.pem --peer-cert-file=certs/proxy1.pem --peer-key-file=certs/proxy1-key.pem --peer-client-cert-auth --peer-trusted-ca-file=certs/ca.pem ================================================ FILE: hack/tls-setup/README.md ================================================ This demonstrates using Cloudflare's [cfssl](https://github.com/cloudflare/cfssl) to easily generate certificates for an etcd cluster. Defaults generate an ECDSA-384 root and leaf certificates for `localhost`. etcd nodes will use the same certificates for both sides of mutual authentication, but won't require client certs for non-peer clients. **Instructions** 1. Install git, go, and make 2. Amend https://github.com/etcd-io/etcd/blob/main/hack/tls-setup/config/req-csr.json - IP's currently in the config should be replaced/added with IP addresses of each cluster node, please note 127.0.0.1 is always required for loopback purposes: ```json Example: { "CN": "etcd", "hosts": [ "3.8.121.201", "46.4.19.20", "127.0.0.1" ], "key": { "algo": "ecdsa", "size": 384 }, "names": [ { "O": "autogenerated", "OU": "etcd cluster", "L": "the internet" } ] } ``` 3. Set the following environment variables subsituting your IP address: ```bash export infra0={IP-0} export infra1={IP-1} export infra2={IP-2} ``` 4. Run `make` to generate the certs ================================================ FILE: hack/tls-setup/config/ca-config.json ================================================ { "signing": { "default": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "876000h" } } } ================================================ FILE: hack/tls-setup/config/ca-csr.json ================================================ { "CN": "Autogenerated CA", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "Honest Achmed's Used Certificates", "OU": "Hastily-Generated Values Divison", "L": "San Francisco", "ST": "California", "C": "US" } ], "ca": { "expiry": "876000h" } } ================================================ FILE: hack/tls-setup/config/req-csr.json ================================================ { "CN": "etcd", "hosts": [ "localhost", "127.0.0.1", "9.145.89.120", "9.145.89.173", "9.145.89.225" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "autogenerated", "OU": "etcd cluster", "L": "the internet" } ] } ================================================ FILE: pkg/.gomodguard.yaml ================================================ --- blocked: modules: - go.etcd.io/etcd: reason: "Forbidden dependency" - go.etcd.io/etcd/api/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/server/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/tests/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/v3: reason: "Forbidden dependency" ================================================ FILE: pkg/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: pkg/README.md ================================================ pkg/ is a collection of utility packages used by etcd without being specific to etcd itself. A package belongs here only if it could possibly be moved out into its own repository in the future. ================================================ FILE: pkg/adt/README.md ================================================ ## Red-Black Tree *"Introduction to Algorithms" (Cormen et al, 3rd ed.), Chapter 13* 1. Every node is either red or black. 2. The root is black. 3. Every leaf (NIL) is black. 4. If a node is red, then both its children are black. 5. For each node, all simple paths from the node to descendant leaves contain the same number of black nodes. For example, ```go import ( "fmt" "go.etcd.io/etcd/pkg/v3/adt" ) func main() { ivt := adt.NewIntervalTree() ivt.Insert(NewInt64Interval(510, 511), 0) ivt.Insert(NewInt64Interval(82, 83), 0) ivt.Insert(NewInt64Interval(830, 831), 0) ... ``` After inserting the values `510`, `82`, `830`, `11`, `383`, `647`, `899`, `261`, `410`, `514`, `815`, `888`, `972`, `238`, `292`, `953`. ![red-black-tree-01-insertion.png](img/red-black-tree-01-insertion.png) Deleting the node `514` should not trigger any rebalancing: ![red-black-tree-02-delete-514.png](img/red-black-tree-02-delete-514.png) Deleting the node `11` triggers multiple rotates for rebalancing: ![red-black-tree-03-delete-11.png](img/red-black-tree-03-delete-11.png) ![red-black-tree-04-delete-11.png](img/red-black-tree-04-delete-11.png) ![red-black-tree-05-delete-11.png](img/red-black-tree-05-delete-11.png) ![red-black-tree-06-delete-11.png](img/red-black-tree-06-delete-11.png) ![red-black-tree-07-delete-11.png](img/red-black-tree-07-delete-11.png) ![red-black-tree-08-delete-11.png](img/red-black-tree-08-delete-11.png) ![red-black-tree-09-delete-11.png](img/red-black-tree-09-delete-11.png) Try yourself at https://www.cs.usfca.edu/~galles/visualization/RedBlack.html. ================================================ FILE: pkg/adt/adt.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package adt implements useful abstract data types. package adt ================================================ FILE: pkg/adt/example_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adt_test import ( "fmt" "go.etcd.io/etcd/pkg/v3/adt" ) func Example() { ivt := adt.NewIntervalTree() ivt.Insert(adt.NewInt64Interval(1, 3), 123) ivt.Insert(adt.NewInt64Interval(9, 13), 456) ivt.Insert(adt.NewInt64Interval(7, 20), 789) rs := ivt.Stab(adt.NewInt64Point(10)) for _, v := range rs { fmt.Printf("Overlapping range: %+v\n", v) } // output: // Overlapping range: &{Ivl:{Begin:7 End:20} Val:789} // Overlapping range: &{Ivl:{Begin:9 End:13} Val:456} } ================================================ FILE: pkg/adt/interval_tree.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adt import ( "bytes" "fmt" "math" "strings" ) // Comparable is an interface for trichotomic comparisons. type Comparable interface { // Compare gives the result of a 3-way comparison // a.Compare(b) = 1 => a > b // a.Compare(b) = 0 => a == b // a.Compare(b) = -1 => a < b Compare(c Comparable) int } type rbcolor int const ( black rbcolor = iota red ) func (c rbcolor) String() string { switch c { case black: return "black" case red: return "red" default: panic(fmt.Errorf("unknown color %d", c)) } } // Interval implements a Comparable interval [begin, end) // TODO: support different sorts of intervals: (a,b), [a,b], (a, b] type Interval struct { Begin Comparable End Comparable } // Compare on an interval gives == if the interval overlaps. func (ivl *Interval) Compare(c Comparable) int { ivl2 := c.(*Interval) ivbCmpBegin := ivl.Begin.Compare(ivl2.Begin) ivbCmpEnd := ivl.Begin.Compare(ivl2.End) iveCmpBegin := ivl.End.Compare(ivl2.Begin) // ivl is left of ivl2 if ivbCmpBegin < 0 && iveCmpBegin <= 0 { return -1 } // iv is right of iv2 if ivbCmpEnd >= 0 { return 1 } return 0 } type intervalNode struct { // iv is the interval-value pair entry. iv IntervalValue // max endpoint of all descendent nodes. max Comparable // left and right are sorted by low endpoint of key interval left, right *intervalNode // parent is the direct ancestor of the node parent *intervalNode c rbcolor } func (x *intervalNode) color(sentinel *intervalNode) rbcolor { if x == sentinel { return black } return x.c } func (x *intervalNode) height(sentinel *intervalNode) int { if x == sentinel { return 0 } ld := x.left.height(sentinel) rd := x.right.height(sentinel) if ld < rd { return rd + 1 } return ld + 1 } func (x *intervalNode) min(sentinel *intervalNode) *intervalNode { for x.left != sentinel { x = x.left } return x } // successor is the next in-order node in the tree func (x *intervalNode) successor(sentinel *intervalNode) *intervalNode { if x.right != sentinel { return x.right.min(sentinel) } y := x.parent for y != sentinel && x == y.right { x = y y = y.parent } return y } // updateMax updates the maximum values for a node and its ancestors func (x *intervalNode) updateMax(sentinel *intervalNode) { for x != sentinel { oldmax := x.max max := x.iv.Ivl.End if x.left != sentinel && x.left.max.Compare(max) > 0 { max = x.left.max } if x.right != sentinel && x.right.max.Compare(max) > 0 { max = x.right.max } if oldmax.Compare(max) == 0 { break } x.max = max x = x.parent } } type nodeVisitor func(n *intervalNode) bool // visit will call a node visitor on each node that overlaps the given interval func (x *intervalNode) visit(iv *Interval, sentinel *intervalNode, nv nodeVisitor) bool { if x == sentinel { return true } v := iv.Compare(&x.iv.Ivl) switch { case v < 0: if !x.left.visit(iv, sentinel, nv) { return false } case v > 0: maxiv := Interval{x.iv.Ivl.Begin, x.max} if maxiv.Compare(iv) == 0 { if !x.left.visit(iv, sentinel, nv) || !x.right.visit(iv, sentinel, nv) { return false } } default: if !x.left.visit(iv, sentinel, nv) || !nv(x) || !x.right.visit(iv, sentinel, nv) { return false } } return true } // IntervalValue represents a range tree node that contains a range and a value. type IntervalValue struct { Ivl Interval Val any } // IntervalTree represents a (mostly) textbook implementation of the // "Introduction to Algorithms" (Cormen et al, 3rd ed.) chapter 13 red-black tree // and chapter 14.3 interval tree with search supporting "stabbing queries". type IntervalTree interface { // Insert adds a node with the given interval into the tree. Insert(ivl Interval, val any) // Delete removes the node with the given interval from the tree, returning // true if a node is in fact removed. Delete(ivl Interval) bool // Len gives the number of elements in the tree. Len() int // Height is the number of levels in the tree; one node has height 1. Height() int // MaxHeight is the expected maximum tree height given the number of nodes. MaxHeight() int // Visit calls a visitor function on every tree node intersecting the given interval. // It will visit each interval [x, y) in ascending order sorted on x. Visit(ivl Interval, ivv IntervalVisitor) // Find gets the IntervalValue for the node matching the given interval Find(ivl Interval) *IntervalValue // Intersects returns true if there is some tree node intersecting the given interval. Intersects(iv Interval) bool // Contains returns true if the interval tree's keys cover the entire given interval. Contains(ivl Interval) bool // Stab returns a slice with all elements in the tree intersecting the interval. Stab(iv Interval) []*IntervalValue // Union merges a given interval tree into the receiver. Union(inIvt IntervalTree, ivl Interval) } // NewIntervalTree returns a new interval tree. func NewIntervalTree() IntervalTree { sentinel := &intervalNode{ iv: IntervalValue{}, max: nil, left: nil, right: nil, parent: nil, c: black, } return &intervalTree{ root: sentinel, count: 0, sentinel: sentinel, } } type intervalTree struct { root *intervalNode count int // red-black NIL node // use 'sentinel' as a dummy object to simplify boundary conditions // use the sentinel to treat a nil child of a node x as an ordinary node whose parent is x // use one shared sentinel to represent all nil leaves and the root's parent sentinel *intervalNode } // TODO: make this consistent with textbook implementation // // "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.4, p324 // // RB-DELETE(T, z) // // y = z // y-original-color = y.color // // if z.left == T.nil // x = z.right // RB-TRANSPLANT(T, z, z.right) // else if z.right == T.nil // x = z.left // RB-TRANSPLANT(T, z, z.left) // else // y = TREE-MINIMUM(z.right) // y-original-color = y.color // x = y.right // if y.p == z // x.p = y // else // RB-TRANSPLANT(T, y, y.right) // y.right = z.right // y.right.p = y // RB-TRANSPLANT(T, z, y) // y.left = z.left // y.left.p = y // y.color = z.color // // if y-original-color == BLACK // RB-DELETE-FIXUP(T, x) // Delete removes the node with the given interval from the tree, returning // true if a node is in fact removed. func (ivt *intervalTree) Delete(ivl Interval) bool { z := ivt.find(ivl) if z == ivt.sentinel { return false } y := z if z.left != ivt.sentinel && z.right != ivt.sentinel { y = z.successor(ivt.sentinel) } x := ivt.sentinel if y.left != ivt.sentinel { x = y.left } else if y.right != ivt.sentinel { x = y.right } x.parent = y.parent if y.parent == ivt.sentinel { ivt.root = x } else { if y == y.parent.left { y.parent.left = x } else { y.parent.right = x } y.parent.updateMax(ivt.sentinel) } if y != z { z.iv = y.iv z.updateMax(ivt.sentinel) } if y.color(ivt.sentinel) == black { ivt.deleteFixup(x) } ivt.count-- return true } // "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.4, p326 // // RB-DELETE-FIXUP(T, z) // // while x ≠ T.root and x.color == BLACK // if x == x.p.left // w = x.p.right // if w.color == RED // w.color = BLACK // x.p.color = RED // LEFT-ROTATE(T, x, p) // if w.left.color == BLACK and w.right.color == BLACK // w.color = RED // x = x.p // else if w.right.color == BLACK // w.left.color = BLACK // w.color = RED // RIGHT-ROTATE(T, w) // w = w.p.right // w.color = x.p.color // x.p.color = BLACK // LEFT-ROTATE(T, w.p) // x = T.root // else // w = x.p.left // if w.color == RED // w.color = BLACK // x.p.color = RED // RIGHT-ROTATE(T, x, p) // if w.right.color == BLACK and w.left.color == BLACK // w.color = RED // x = x.p // else if w.left.color == BLACK // w.right.color = BLACK // w.color = RED // LEFT-ROTATE(T, w) // w = w.p.left // w.color = x.p.color // x.p.color = BLACK // RIGHT-ROTATE(T, w.p) // x = T.root // // x.color = BLACK func (ivt *intervalTree) deleteFixup(x *intervalNode) { for x != ivt.root && x.color(ivt.sentinel) == black { if x == x.parent.left { // line 3-20 w := x.parent.right if w.color(ivt.sentinel) == red { w.c = black x.parent.c = red ivt.rotateLeft(x.parent) w = x.parent.right } if w == nil { break } if w.left.color(ivt.sentinel) == black && w.right.color(ivt.sentinel) == black { w.c = red x = x.parent } else { if w.right.color(ivt.sentinel) == black { w.left.c = black w.c = red ivt.rotateRight(w) w = x.parent.right } w.c = x.parent.color(ivt.sentinel) x.parent.c = black w.right.c = black ivt.rotateLeft(x.parent) x = ivt.root } } else { // line 22-38 // same as above but with left and right exchanged w := x.parent.left if w.color(ivt.sentinel) == red { w.c = black x.parent.c = red ivt.rotateRight(x.parent) w = x.parent.left } if w == nil { break } if w.left.color(ivt.sentinel) == black && w.right.color(ivt.sentinel) == black { w.c = red x = x.parent } else { if w.left.color(ivt.sentinel) == black { w.right.c = black w.c = red ivt.rotateLeft(w) w = x.parent.left } w.c = x.parent.color(ivt.sentinel) x.parent.c = black w.left.c = black ivt.rotateRight(x.parent) x = ivt.root } } } if x != nil { x.c = black } } func (ivt *intervalTree) createIntervalNode(ivl Interval, val any) *intervalNode { return &intervalNode{ iv: IntervalValue{ivl, val}, max: ivl.End, c: red, left: ivt.sentinel, right: ivt.sentinel, parent: ivt.sentinel, } } // Insert adds a node with the given interval into the tree. // // Cormen "Introduction to Algorithms", Chapter 14 Exercise 14.3.5. // The algorithm follows Cormen "Introduction to Algorithms", Chapter 14 Exercise 14.3.5. // for modifying an interval tree structure to support exact interval matching. func (ivt *intervalTree) Insert(ivl Interval, val any) { y := ivt.sentinel z := ivt.createIntervalNode(ivl, val) x := ivt.root for x != ivt.sentinel { y = x // Split on left endpoint. If left endpoints match, instead split on right endpoint. beginCompare := z.iv.Ivl.Begin.Compare(x.iv.Ivl.Begin) if beginCompare < 0 { x = x.left } else if beginCompare == 0 { if z.iv.Ivl.End.Compare(x.iv.Ivl.End) < 0 { x = x.left } else { x = x.right } } else { x = x.right } } z.parent = y if y == ivt.sentinel { ivt.root = z } else { beginCompare := z.iv.Ivl.Begin.Compare(y.iv.Ivl.Begin) if beginCompare < 0 { y.left = z } else if beginCompare == 0 { if z.iv.Ivl.End.Compare(y.iv.Ivl.End) < 0 { y.left = z } else { y.right = z } } else { y.right = z } y.updateMax(ivt.sentinel) } z.c = red ivt.insertFixup(z) ivt.count++ } // "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.3, p316 // // RB-INSERT-FIXUP(T, z) // // while z.p.color == RED // if z.p == z.p.p.left // y = z.p.p.right // if y.color == RED // z.p.color = BLACK // y.color = BLACK // z.p.p.color = RED // z = z.p.p // else if z == z.p.right // z = z.p // LEFT-ROTATE(T, z) // z.p.color = BLACK // z.p.p.color = RED // RIGHT-ROTATE(T, z.p.p) // else // y = z.p.p.left // if y.color == RED // z.p.color = BLACK // y.color = BLACK // z.p.p.color = RED // z = z.p.p // else if z == z.p.right // z = z.p // RIGHT-ROTATE(T, z) // z.p.color = BLACK // z.p.p.color = RED // LEFT-ROTATE(T, z.p.p) // // T.root.color = BLACK func (ivt *intervalTree) insertFixup(z *intervalNode) { for z.parent.color(ivt.sentinel) == red { if z.parent == z.parent.parent.left { // line 3-15 y := z.parent.parent.right if y.color(ivt.sentinel) == red { y.c = black z.parent.c = black z.parent.parent.c = red z = z.parent.parent } else { if z == z.parent.right { z = z.parent ivt.rotateLeft(z) } z.parent.c = black z.parent.parent.c = red ivt.rotateRight(z.parent.parent) } } else { // line 16-28 // same as then with left/right exchanged y := z.parent.parent.left if y.color(ivt.sentinel) == red { y.c = black z.parent.c = black z.parent.parent.c = red z = z.parent.parent } else { if z == z.parent.left { z = z.parent ivt.rotateRight(z) } z.parent.c = black z.parent.parent.c = red ivt.rotateLeft(z.parent.parent) } } } // line 30 ivt.root.c = black } // rotateLeft moves x so it is left of its right child // // "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.2, p313 // // LEFT-ROTATE(T, x) // // y = x.right // x.right = y.left // // if y.left ≠ T.nil // y.left.p = x // // y.p = x.p // // if x.p == T.nil // T.root = y // else if x == x.p.left // x.p.left = y // else // x.p.right = y // // y.left = x // x.p = y func (ivt *intervalTree) rotateLeft(x *intervalNode) { // rotateLeft x must have right child if x.right == ivt.sentinel { return } // line 2-3 y := x.right x.right = y.left // line 5-6 if y.left != ivt.sentinel { y.left.parent = x } x.updateMax(ivt.sentinel) // line 10-15, 18 ivt.replaceParent(x, y) // line 17 y.left = x y.updateMax(ivt.sentinel) } // rotateRight moves x so it is right of its left child // // RIGHT-ROTATE(T, x) // // y = x.left // x.left = y.right // // if y.right ≠ T.nil // y.right.p = x // // y.p = x.p // // if x.p == T.nil // T.root = y // else if x == x.p.right // x.p.right = y // else // x.p.left = y // // y.right = x // x.p = y func (ivt *intervalTree) rotateRight(x *intervalNode) { // rotateRight x must have left child if x.left == ivt.sentinel { return } // line 2-3 y := x.left x.left = y.right // line 5-6 if y.right != ivt.sentinel { y.right.parent = x } x.updateMax(ivt.sentinel) // line 10-15, 18 ivt.replaceParent(x, y) // line 17 y.right = x y.updateMax(ivt.sentinel) } // replaceParent replaces x's parent with y func (ivt *intervalTree) replaceParent(x *intervalNode, y *intervalNode) { y.parent = x.parent if x.parent == ivt.sentinel { ivt.root = y } else { if x == x.parent.left { x.parent.left = y } else { x.parent.right = y } x.parent.updateMax(ivt.sentinel) } x.parent = y } // Len gives the number of elements in the tree func (ivt *intervalTree) Len() int { return ivt.count } // Height is the number of levels in the tree; one node has height 1. func (ivt *intervalTree) Height() int { return ivt.root.height(ivt.sentinel) } // MaxHeight is the expected maximum tree height given the number of nodes func (ivt *intervalTree) MaxHeight() int { return int((2 * math.Log2(float64(ivt.Len()+1))) + 0.5) } // IntervalVisitor is used on tree searches; return false to stop searching. type IntervalVisitor func(n *IntervalValue) bool // Visit calls a visitor function on every tree node intersecting the given interval. // It will visit each interval [x, y) in ascending order sorted on x. func (ivt *intervalTree) Visit(ivl Interval, ivv IntervalVisitor) { ivt.root.visit(&ivl, ivt.sentinel, func(n *intervalNode) bool { return ivv(&n.iv) }) } // find the exact node for a given interval. The implementation follows // Cormen "Introduction to Algorithms", Chapter 14 Exercise 14.3.5. for // exact interval matching. The search runs in O(log n) time on an n-node // interval tree. func (ivt *intervalTree) find(ivl Interval) *intervalNode { x := ivt.root // Search until hit sentinel or exact match. for x != ivt.sentinel { beginCompare := ivl.Begin.Compare(x.iv.Ivl.Begin) endCompare := ivl.End.Compare(x.iv.Ivl.End) if beginCompare == 0 && endCompare == 0 { return x } // Split on left endpoint. If left endpoints match, // instead split on right endpoints. if beginCompare < 0 { x = x.left } else if beginCompare == 0 { if endCompare < 0 { x = x.left } else { x = x.right } } else { x = x.right } } return x } // Find gets the IntervalValue for the node matching the given interval func (ivt *intervalTree) Find(ivl Interval) (ret *IntervalValue) { n := ivt.find(ivl) if n == ivt.sentinel { return nil } return &n.iv } // Intersects returns true if there is some tree node intersecting the given interval. func (ivt *intervalTree) Intersects(iv Interval) bool { x := ivt.root for x != ivt.sentinel && iv.Compare(&x.iv.Ivl) != 0 { if x.left != ivt.sentinel && x.left.max.Compare(iv.Begin) > 0 { x = x.left } else { x = x.right } } return x != ivt.sentinel } // Contains returns true if the interval tree's keys cover the entire given interval. func (ivt *intervalTree) Contains(ivl Interval) bool { var maxEnd, minBegin Comparable isContiguous := true ivt.Visit(ivl, func(n *IntervalValue) bool { if minBegin == nil { minBegin = n.Ivl.Begin maxEnd = n.Ivl.End return true } if maxEnd.Compare(n.Ivl.Begin) < 0 { isContiguous = false return false } if n.Ivl.End.Compare(maxEnd) > 0 { maxEnd = n.Ivl.End } return true }) return isContiguous && minBegin != nil && maxEnd.Compare(ivl.End) >= 0 && minBegin.Compare(ivl.Begin) <= 0 } // Stab returns a slice with all elements in the tree intersecting the interval. func (ivt *intervalTree) Stab(iv Interval) (ivs []*IntervalValue) { if ivt.count == 0 { return nil } f := func(n *IntervalValue) bool { ivs = append(ivs, n); return true } ivt.Visit(iv, f) return ivs } // Union merges a given interval tree into the receiver. func (ivt *intervalTree) Union(inIvt IntervalTree, ivl Interval) { f := func(n *IntervalValue) bool { ivt.Insert(n.Ivl, n.Val) return true } inIvt.Visit(ivl, f) } type visitedInterval struct { root Interval left Interval right Interval color rbcolor depth int } func (vi visitedInterval) String() string { bd := new(strings.Builder) bd.WriteString(fmt.Sprintf("root [%v,%v,%v], left [%v,%v], right [%v,%v], depth %d", vi.root.Begin, vi.root.End, vi.color, vi.left.Begin, vi.left.End, vi.right.Begin, vi.right.End, vi.depth, )) return bd.String() } // visitLevel traverses tree in level order. // used for testing func (ivt *intervalTree) visitLevel() []visitedInterval { if ivt.root == ivt.sentinel { return nil } rs := make([]visitedInterval, 0, ivt.Len()) type pair struct { node *intervalNode depth int } queue := []pair{{ivt.root, 0}} for len(queue) > 0 { f := queue[0] queue = queue[1:] vi := visitedInterval{ root: f.node.iv.Ivl, color: f.node.color(ivt.sentinel), depth: f.depth, } if f.node.left != ivt.sentinel { vi.left = f.node.left.iv.Ivl queue = append(queue, pair{f.node.left, f.depth + 1}) } if f.node.right != ivt.sentinel { vi.right = f.node.right.iv.Ivl queue = append(queue, pair{f.node.right, f.depth + 1}) } rs = append(rs, vi) } return rs } type StringComparable string func (s StringComparable) Compare(c Comparable) int { sc := c.(StringComparable) if s < sc { return -1 } if s > sc { return 1 } return 0 } func NewStringInterval(begin, end string) Interval { return Interval{StringComparable(begin), StringComparable(end)} } func NewStringPoint(s string) Interval { return Interval{StringComparable(s), StringComparable(s + "\x00")} } // StringAffineComparable treats "" as > all other strings type StringAffineComparable string func (s StringAffineComparable) Compare(c Comparable) int { sc := c.(StringAffineComparable) if len(s) == 0 { if len(sc) == 0 { return 0 } return 1 } if len(sc) == 0 { return -1 } if s < sc { return -1 } if s > sc { return 1 } return 0 } func NewStringAffineInterval(begin, end string) Interval { return Interval{StringAffineComparable(begin), StringAffineComparable(end)} } func NewStringAffinePoint(s string) Interval { return NewStringAffineInterval(s, s+"\x00") } func NewInt64Interval(a int64, b int64) Interval { return Interval{Int64Comparable(a), Int64Comparable(b)} } func newInt64EmptyInterval() Interval { return Interval{Begin: nil, End: nil} } func NewInt64Point(a int64) Interval { return Interval{Int64Comparable(a), Int64Comparable(a + 1)} } type Int64Comparable int64 func (v Int64Comparable) Compare(c Comparable) int { vc := c.(Int64Comparable) cmp := v - vc if cmp < 0 { return -1 } if cmp > 0 { return 1 } return 0 } // BytesAffineComparable treats empty byte arrays as > all other byte arrays type BytesAffineComparable []byte func (b BytesAffineComparable) Compare(c Comparable) int { bc := c.(BytesAffineComparable) if len(b) == 0 { if len(bc) == 0 { return 0 } return 1 } if len(bc) == 0 { return -1 } return bytes.Compare(b, bc) } func NewBytesAffineInterval(begin, end []byte) Interval { return Interval{BytesAffineComparable(begin), BytesAffineComparable(end)} } func NewBytesAffinePoint(b []byte) Interval { be := make([]byte, len(b)+1) copy(be, b) be[len(b)] = 0 return NewBytesAffineInterval(b, be) } ================================================ FILE: pkg/adt/interval_tree_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adt import ( "math/rand" "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestIntervalTreeInsert tests interval tree insertion. func TestIntervalTreeInsert(t *testing.T) { // "Introduction to Algorithms" (Cormen et al, 3rd ed.) chapter 14, Figure 14.4 ivt := NewIntervalTree() ivt.Insert(NewInt64Interval(16, 21), 30) ivt.Insert(NewInt64Interval(8, 9), 23) ivt.Insert(NewInt64Interval(0, 3), 3) ivt.Insert(NewInt64Interval(5, 8), 10) ivt.Insert(NewInt64Interval(6, 10), 10) ivt.Insert(NewInt64Interval(15, 23), 23) ivt.Insert(NewInt64Interval(17, 19), 20) ivt.Insert(NewInt64Interval(25, 30), 30) ivt.Insert(NewInt64Interval(26, 26), 26) ivt.Insert(NewInt64Interval(19, 20), 20) expected := []visitedInterval{ {root: NewInt64Interval(16, 21), color: black, left: NewInt64Interval(8, 9), right: NewInt64Interval(25, 30), depth: 0}, {root: NewInt64Interval(8, 9), color: red, left: NewInt64Interval(5, 8), right: NewInt64Interval(15, 23), depth: 1}, {root: NewInt64Interval(25, 30), color: red, left: NewInt64Interval(17, 19), right: NewInt64Interval(26, 26), depth: 1}, {root: NewInt64Interval(5, 8), color: black, left: NewInt64Interval(0, 3), right: NewInt64Interval(6, 10), depth: 2}, {root: NewInt64Interval(15, 23), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 2}, {root: NewInt64Interval(17, 19), color: black, left: newInt64EmptyInterval(), right: NewInt64Interval(19, 20), depth: 2}, {root: NewInt64Interval(26, 26), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 2}, {root: NewInt64Interval(0, 3), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(6, 10), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(19, 20), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, } tr := ivt.(*intervalTree) visits := tr.visitLevel() require.Truef(t, reflect.DeepEqual(expected, visits), "level order expected %v, got %v", expected, visits) } // TestIntervalTreeSelfBalanced ensures range tree is self-balanced after inserting ranges to the tree. // Use https://www.cs.usfca.edu/~galles/visualization/RedBlack.html for test case creation. // // Regular Binary Search Tree // // [0,1] // \ // [1,2] // \ // [3,4] // \ // [5,6] // \ // [7,8] // \ // [8,9] // // Self-Balancing Binary Search Tree // // [1,2] // / \ // [0,1] [5,6] // / \ // [3,4] [7,8] // \ // [8,9] func TestIntervalTreeSelfBalanced(t *testing.T) { ivt := NewIntervalTree() ivt.Insert(NewInt64Interval(0, 1), 0) ivt.Insert(NewInt64Interval(1, 2), 0) ivt.Insert(NewInt64Interval(3, 4), 0) ivt.Insert(NewInt64Interval(5, 6), 0) ivt.Insert(NewInt64Interval(7, 8), 0) ivt.Insert(NewInt64Interval(8, 9), 0) expected := []visitedInterval{ {root: NewInt64Interval(1, 2), color: black, left: NewInt64Interval(0, 1), right: NewInt64Interval(5, 6), depth: 0}, {root: NewInt64Interval(0, 1), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 1}, {root: NewInt64Interval(5, 6), color: red, left: NewInt64Interval(3, 4), right: NewInt64Interval(7, 8), depth: 1}, {root: NewInt64Interval(3, 4), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 2}, {root: NewInt64Interval(7, 8), color: black, left: newInt64EmptyInterval(), right: NewInt64Interval(8, 9), depth: 2}, {root: NewInt64Interval(8, 9), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, } tr := ivt.(*intervalTree) visits := tr.visitLevel() require.Truef(t, reflect.DeepEqual(expected, visits), "level order expected %v, got %v", expected, visits) require.Equalf(t, 3, visits[len(visits)-1].depth, "expected self-balanced tree with last level 3, but last level got %d", visits[len(visits)-1].depth) } // TestIntervalTreeDelete ensures delete operation maintains red-black tree properties. // Use https://www.cs.usfca.edu/~galles/visualization/RedBlack.html for test case creation. // See https://github.com/etcd-io/etcd/issues/10877 for more detail. // // After insertion: // // [510,511] // / \ // ---------- ----------------------- // / \ // [82,83] [830,831] // / \ / \ // / \ / \ // [11,12] [383,384](red) [647,648] [899,900](red) // / \ / \ / \ // / \ / \ / \ // [261,262] [410,411] [514,515](red) [815,816](red) [888,889] [972,973] // / \ / // / \ / // [238,239](red) [292,293](red) [953,954](red) // // After deleting 514 (no rebalance): // // [510,511] // / \ // ---------- ----------------------- // / \ // [82,83] [830,831] // / \ / \ // / \ / \ // [11,12] [383,384](red) [647,648] [899,900](red) // / \ \ / \ // / \ \ / \ // [261,262] [410,411] [815,816](red) [888,889] [972,973] // / \ / // / \ / // [238,239](red) [292,293](red) [953,954](red) // // After deleting 11 (requires rebalancing): // // [510,511] // / \ // ---------- -------------------------- // / \ // [383,384] [830,831] // / \ / \ // / \ / \ // [261,262](red) [410,411] [647,648] [899,900](red) // / \ \ / \ // / \ \ / \ // [82,83] [292,293] [815,816](red) [888,889] [972,973] // \ / // \ / // [238,239](red) [953,954](red) func TestIntervalTreeDelete(t *testing.T) { ivt := NewIntervalTree() ivt.Insert(NewInt64Interval(510, 511), 0) ivt.Insert(NewInt64Interval(82, 83), 0) ivt.Insert(NewInt64Interval(830, 831), 0) ivt.Insert(NewInt64Interval(11, 12), 0) ivt.Insert(NewInt64Interval(383, 384), 0) ivt.Insert(NewInt64Interval(647, 648), 0) ivt.Insert(NewInt64Interval(899, 900), 0) ivt.Insert(NewInt64Interval(261, 262), 0) ivt.Insert(NewInt64Interval(410, 411), 0) ivt.Insert(NewInt64Interval(514, 515), 0) ivt.Insert(NewInt64Interval(815, 816), 0) ivt.Insert(NewInt64Interval(888, 889), 0) ivt.Insert(NewInt64Interval(972, 973), 0) ivt.Insert(NewInt64Interval(238, 239), 0) ivt.Insert(NewInt64Interval(292, 293), 0) ivt.Insert(NewInt64Interval(953, 954), 0) tr := ivt.(*intervalTree) expectedBeforeDelete := []visitedInterval{ {root: NewInt64Interval(510, 511), color: black, left: NewInt64Interval(82, 83), right: NewInt64Interval(830, 831), depth: 0}, {root: NewInt64Interval(82, 83), color: black, left: NewInt64Interval(11, 12), right: NewInt64Interval(383, 384), depth: 1}, {root: NewInt64Interval(830, 831), color: black, left: NewInt64Interval(647, 648), right: NewInt64Interval(899, 900), depth: 1}, {root: NewInt64Interval(11, 12), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 2}, {root: NewInt64Interval(383, 384), color: red, left: NewInt64Interval(261, 262), right: NewInt64Interval(410, 411), depth: 2}, {root: NewInt64Interval(647, 648), color: black, left: NewInt64Interval(514, 515), right: NewInt64Interval(815, 816), depth: 2}, {root: NewInt64Interval(899, 900), color: red, left: NewInt64Interval(888, 889), right: NewInt64Interval(972, 973), depth: 2}, {root: NewInt64Interval(261, 262), color: black, left: NewInt64Interval(238, 239), right: NewInt64Interval(292, 293), depth: 3}, {root: NewInt64Interval(410, 411), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(514, 515), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(815, 816), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(888, 889), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(972, 973), color: black, left: NewInt64Interval(953, 954), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(238, 239), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 4}, {root: NewInt64Interval(292, 293), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 4}, {root: NewInt64Interval(953, 954), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 4}, } visitsBeforeDelete := tr.visitLevel() require.Truef(t, reflect.DeepEqual(expectedBeforeDelete, visitsBeforeDelete), "level order after insertion expected %v, got %v", expectedBeforeDelete, visitsBeforeDelete) // delete the node "514" range514 := NewInt64Interval(514, 515) require.Truef(t, tr.Delete(NewInt64Interval(514, 515)), "range %v not deleted", range514) expectedAfterDelete514 := []visitedInterval{ {root: NewInt64Interval(510, 511), color: black, left: NewInt64Interval(82, 83), right: NewInt64Interval(830, 831), depth: 0}, {root: NewInt64Interval(82, 83), color: black, left: NewInt64Interval(11, 12), right: NewInt64Interval(383, 384), depth: 1}, {root: NewInt64Interval(830, 831), color: black, left: NewInt64Interval(647, 648), right: NewInt64Interval(899, 900), depth: 1}, {root: NewInt64Interval(11, 12), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 2}, {root: NewInt64Interval(383, 384), color: red, left: NewInt64Interval(261, 262), right: NewInt64Interval(410, 411), depth: 2}, {root: NewInt64Interval(647, 648), color: black, left: newInt64EmptyInterval(), right: NewInt64Interval(815, 816), depth: 2}, {root: NewInt64Interval(899, 900), color: red, left: NewInt64Interval(888, 889), right: NewInt64Interval(972, 973), depth: 2}, {root: NewInt64Interval(261, 262), color: black, left: NewInt64Interval(238, 239), right: NewInt64Interval(292, 293), depth: 3}, {root: NewInt64Interval(410, 411), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(815, 816), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(888, 889), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(972, 973), color: black, left: NewInt64Interval(953, 954), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(238, 239), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 4}, {root: NewInt64Interval(292, 293), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 4}, {root: NewInt64Interval(953, 954), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 4}, } visitsAfterDelete514 := tr.visitLevel() require.Truef(t, reflect.DeepEqual(expectedAfterDelete514, visitsAfterDelete514), "level order after deleting '514' expected %v, got %v", expectedAfterDelete514, visitsAfterDelete514) // delete the node "11" range11 := NewInt64Interval(11, 12) require.Truef(t, tr.Delete(NewInt64Interval(11, 12)), "range %v not deleted", range11) expectedAfterDelete11 := []visitedInterval{ {root: NewInt64Interval(510, 511), color: black, left: NewInt64Interval(383, 384), right: NewInt64Interval(830, 831), depth: 0}, {root: NewInt64Interval(383, 384), color: black, left: NewInt64Interval(261, 262), right: NewInt64Interval(410, 411), depth: 1}, {root: NewInt64Interval(830, 831), color: black, left: NewInt64Interval(647, 648), right: NewInt64Interval(899, 900), depth: 1}, {root: NewInt64Interval(261, 262), color: red, left: NewInt64Interval(82, 83), right: NewInt64Interval(292, 293), depth: 2}, {root: NewInt64Interval(410, 411), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 2}, {root: NewInt64Interval(647, 648), color: black, left: newInt64EmptyInterval(), right: NewInt64Interval(815, 816), depth: 2}, {root: NewInt64Interval(899, 900), color: red, left: NewInt64Interval(888, 889), right: NewInt64Interval(972, 973), depth: 2}, {root: NewInt64Interval(82, 83), color: black, left: newInt64EmptyInterval(), right: NewInt64Interval(238, 239), depth: 3}, {root: NewInt64Interval(292, 293), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(815, 816), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(888, 889), color: black, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(972, 973), color: black, left: NewInt64Interval(953, 954), right: newInt64EmptyInterval(), depth: 3}, {root: NewInt64Interval(238, 239), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 4}, {root: NewInt64Interval(953, 954), color: red, left: newInt64EmptyInterval(), right: newInt64EmptyInterval(), depth: 4}, } visitsAfterDelete11 := tr.visitLevel() require.Truef(t, reflect.DeepEqual(expectedAfterDelete11, visitsAfterDelete11), "level order after deleting '11' expected %v, got %v", expectedAfterDelete11, visitsAfterDelete11) } func TestIntervalTreeFind(t *testing.T) { ivt := NewIntervalTree() ivl1 := NewInt64Interval(3, 6) val := 123 assert.Nilf(t, ivt.Find(ivl1), "find for %v expected nil on empty tree", ivl1) // insert interval [3, 6) into tree ivt.Insert(ivl1, val) // check cases of expected find matches and non-matches assert.NotNilf(t, ivt.Find(ivl1), "find expected not-nil on exact-matched interval %v", ivl1) assert.Equalf(t, ivl1, ivt.Find(ivl1).Ivl, "find expected to return exact-matched interval %v", ivl1) ivl2 := NewInt64Interval(3, 7) assert.Nilf(t, ivt.Find(ivl2), "find expected nil on matched start, different end %v", ivl2) ivl3 := NewInt64Interval(2, 6) assert.Nilf(t, ivt.Find(ivl3), "find expected nil on different start, matched end %v", ivl3) ivl4 := NewInt64Interval(10, 20) assert.Nilf(t, ivt.Find(ivl4), "find expected nil on different start, different end %v", ivl4) // insert the additional intervals into the tree, and check they can each be found. ivls := []Interval{ivl2, ivl3, ivl4} for _, ivl := range ivls { ivt.Insert(ivl, val) assert.NotNilf(t, ivt.Find(ivl), "find expected not-nil on exact-matched interval %v", ivl) assert.Equalf(t, ivl, ivt.Find(ivl).Ivl, "find expected to return exact-matched interval %v", ivl) } // check additional intervals no longer found after deletion for _, ivl := range ivls { assert.Truef(t, ivt.Delete(ivl), "expected successful delete on %v", ivl) assert.Nilf(t, ivt.Find(ivl), "find expected nil after deleted interval %v", ivl) } } func TestIntervalTreeIntersects(t *testing.T) { ivt := NewIntervalTree() ivt.Insert(NewStringInterval("1", "3"), 123) assert.Falsef(t, ivt.Intersects(NewStringPoint("0")), "contains 0") assert.Truef(t, ivt.Intersects(NewStringPoint("1")), "missing 1") assert.Truef(t, ivt.Intersects(NewStringPoint("11")), "missing 11") assert.Truef(t, ivt.Intersects(NewStringPoint("2")), "missing 2") assert.Falsef(t, ivt.Intersects(NewStringPoint("3")), "contains 3") } func TestIntervalTreeStringAffine(t *testing.T) { ivt := NewIntervalTree() ivt.Insert(NewStringAffineInterval("8", ""), 123) assert.Truef(t, ivt.Intersects(NewStringAffinePoint("9")), "missing 9") assert.Falsef(t, ivt.Intersects(NewStringAffinePoint("7")), "contains 7") } func TestIntervalTreeStab(t *testing.T) { ivt := NewIntervalTree() ivt.Insert(NewStringInterval("0", "1"), 123) ivt.Insert(NewStringInterval("0", "2"), 456) ivt.Insert(NewStringInterval("5", "6"), 789) ivt.Insert(NewStringInterval("6", "8"), 999) ivt.Insert(NewStringInterval("0", "3"), 0) tr := ivt.(*intervalTree) require.Equalf(t, 0, tr.root.max.Compare(StringComparable("8")), "wrong root max got %v, expected 8", tr.root.max) assert.Len(t, ivt.Stab(NewStringPoint("0")), 3) assert.Len(t, ivt.Stab(NewStringPoint("1")), 2) assert.Len(t, ivt.Stab(NewStringPoint("2")), 1) assert.Empty(t, ivt.Stab(NewStringPoint("3"))) assert.Len(t, ivt.Stab(NewStringPoint("5")), 1) assert.Len(t, ivt.Stab(NewStringPoint("55")), 1) assert.Len(t, ivt.Stab(NewStringPoint("6")), 1) } type xy struct { x int64 y int64 } func TestIntervalTreeRandom(t *testing.T) { // generate unique intervals ivs := make(map[xy]struct{}) ivt := NewIntervalTree() maxv := 128 for i := rand.Intn(maxv) + 1; i != 0; i-- { x, y := int64(rand.Intn(maxv)), int64(rand.Intn(maxv)) if x > y { t := x x = y y = t } else if x == y { y++ } iv := xy{x, y} if _, ok := ivs[iv]; ok { // don't double insert continue } ivt.Insert(NewInt64Interval(x, y), 123) ivs[iv] = struct{}{} } for ab := range ivs { for xy := range ivs { v := xy.x + int64(rand.Intn(int(xy.y-xy.x))) require.NotEmptyf(t, ivt.Stab(NewInt64Point(v)), "expected %v stab non-zero for [%+v)", v, xy) require.Truef(t, ivt.Intersects(NewInt64Point(v)), "did not get %d as expected for [%+v)", v, xy) } ivl := NewInt64Interval(ab.x, ab.y) iv := ivt.Find(ivl) assert.NotNilf(t, iv, "expected find non-nil on %v", ab) assert.Equalf(t, ivl, iv.Ivl, "find did not get matched interval %v", ab) assert.Truef(t, ivt.Delete(ivl), "did not delete %v as expected", ab) delete(ivs, ab) ivAfterDel := ivt.Find(ivl) assert.Nilf(t, ivAfterDel, "expected find nil after deletion on %v", ab) } assert.Equalf(t, 0, ivt.Len(), "got ivt.Len() = %v, expected 0", ivt.Len()) } // TestIntervalTreeSortedVisit tests that intervals are visited in sorted order. func TestIntervalTreeSortedVisit(t *testing.T) { tests := []struct { ivls []Interval visitRange Interval }{ { ivls: []Interval{NewInt64Interval(1, 10), NewInt64Interval(2, 5), NewInt64Interval(3, 6)}, visitRange: NewInt64Interval(0, 100), }, { ivls: []Interval{NewInt64Interval(1, 10), NewInt64Interval(10, 12), NewInt64Interval(3, 6)}, visitRange: NewInt64Interval(0, 100), }, { ivls: []Interval{NewInt64Interval(2, 3), NewInt64Interval(3, 4), NewInt64Interval(6, 7), NewInt64Interval(5, 6)}, visitRange: NewInt64Interval(0, 100), }, { ivls: []Interval{ NewInt64Interval(2, 3), NewInt64Interval(2, 4), NewInt64Interval(3, 7), NewInt64Interval(2, 5), NewInt64Interval(3, 8), NewInt64Interval(3, 5), }, visitRange: NewInt64Interval(0, 100), }, } for i, tt := range tests { ivt := NewIntervalTree() for _, ivl := range tt.ivls { ivt.Insert(ivl, struct{}{}) } last := tt.ivls[0].Begin count := 0 chk := func(iv *IntervalValue) bool { assert.LessOrEqualf(t, last.Compare(iv.Ivl.Begin), 0, "#%d: expected less than %d, got interval %+v", i, last, iv.Ivl) last = iv.Ivl.Begin count++ return true } ivt.Visit(tt.visitRange, chk) assert.Lenf(t, tt.ivls, count, "#%d: did not cover all intervals. expected %d, got %d", i, len(tt.ivls), count) } } // TestIntervalTreeVisitExit tests that visiting can be stopped. func TestIntervalTreeVisitExit(t *testing.T) { ivls := []Interval{NewInt64Interval(1, 10), NewInt64Interval(2, 5), NewInt64Interval(3, 6), NewInt64Interval(4, 8)} ivlRange := NewInt64Interval(0, 100) tests := []struct { f IntervalVisitor wcount int }{ { f: func(n *IntervalValue) bool { return false }, wcount: 1, }, { f: func(n *IntervalValue) bool { return n.Ivl.Begin.Compare(ivls[0].Begin) <= 0 }, wcount: 2, }, { f: func(n *IntervalValue) bool { return n.Ivl.Begin.Compare(ivls[2].Begin) < 0 }, wcount: 3, }, { f: func(n *IntervalValue) bool { return true }, wcount: 4, }, } for i, tt := range tests { ivt := NewIntervalTree() for _, ivl := range ivls { ivt.Insert(ivl, struct{}{}) } count := 0 ivt.Visit(ivlRange, func(n *IntervalValue) bool { count++ return tt.f(n) }) assert.Equalf(t, count, tt.wcount, "#%d: expected count %d, got %d", i, tt.wcount, count) } } // TestIntervalTreeContains tests that contains returns true iff the ivt maps the entire interval. func TestIntervalTreeContains(t *testing.T) { tests := []struct { ivls []Interval chkIvl Interval wContains bool }{ { ivls: []Interval{NewInt64Interval(1, 10)}, chkIvl: NewInt64Interval(0, 100), wContains: false, }, { ivls: []Interval{NewInt64Interval(1, 10)}, chkIvl: NewInt64Interval(1, 10), wContains: true, }, { ivls: []Interval{NewInt64Interval(1, 10)}, chkIvl: NewInt64Interval(2, 8), wContains: true, }, { ivls: []Interval{NewInt64Interval(1, 5), NewInt64Interval(6, 10)}, chkIvl: NewInt64Interval(1, 10), wContains: false, }, { ivls: []Interval{NewInt64Interval(1, 5), NewInt64Interval(3, 10)}, chkIvl: NewInt64Interval(1, 10), wContains: true, }, { ivls: []Interval{NewInt64Interval(1, 4), NewInt64Interval(4, 7), NewInt64Interval(3, 10)}, chkIvl: NewInt64Interval(1, 10), wContains: true, }, { ivls: []Interval{}, chkIvl: NewInt64Interval(1, 10), wContains: false, }, } for i, tt := range tests { ivt := NewIntervalTree() for _, ivl := range tt.ivls { ivt.Insert(ivl, struct{}{}) } v := ivt.Contains(tt.chkIvl) assert.Equalf(t, v, tt.wContains, "#%d: ivt.Contains got %v, expected %v", i, v, tt.wContains) } } ================================================ FILE: pkg/cobrautl/error.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cobrautl import ( "fmt" "os" ) const ( // http://tldp.org/LDP/abs/html/exitcodes.html ExitSuccess = iota ExitError ExitBadConnection ExitInvalidInput // for txn, watch command ExitBadFeature // provided a valid flag with an unsupported value ExitInterrupted ExitIO ExitBadArgs = 128 ExitServerError = 4 ExitClusterNotHealthy = 5 ) func ExitWithError(code int, err error) { fmt.Fprintln(os.Stderr, "Error:", err) os.Exit(code) } ================================================ FILE: pkg/cobrautl/help.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // copied from https://github.com/rkt/rkt/blob/master/rkt/help.go package cobrautl import ( "fmt" "io" "os" "strings" "text/tabwriter" "text/template" "github.com/spf13/cobra" "github.com/spf13/pflag" ) var ( commandUsageTemplate *template.Template templFuncs = template.FuncMap{ "descToLines": func(s string) []string { // trim leading/trailing whitespace and split into slice of lines return strings.Split(strings.Trim(s, "\n\t "), "\n") }, "cmdName": func(cmd *cobra.Command, startCmd *cobra.Command) string { parts := []string{cmd.Name()} for cmd.HasParent() && cmd.Parent().Name() != startCmd.Name() { cmd = cmd.Parent() parts = append([]string{cmd.Name()}, parts...) } return strings.Join(parts, " ") }, "indent": func(s string) string { pad := strings.Repeat(" ", 2) return pad + strings.Replace(s, "\n", "\n"+pad, -1) }, } ) func init() { commandUsage := ` {{ $cmd := .Cmd }}\ {{ $cmdname := cmdName .Cmd .Cmd.Root }}\ NAME: {{if not .Cmd.HasParent}}\ {{printf "%s - %s" .Cmd.Name .Cmd.Short | indent}} {{else}}\ {{printf "%s - %s" $cmdname .Cmd.Short | indent}} {{end}}\ USAGE: {{printf "%s" .Cmd.UseLine | indent}} {{ if not .Cmd.HasParent }}\ VERSION: {{printf "%s" .Version | indent}} {{end}}\ {{if .Cmd.HasSubCommands}}\ API VERSION: {{.APIVersion | indent}} {{end}}\ {{if .Cmd.HasExample}}\ Examples: {{.Cmd.Example}} {{end}}\ {{if .Cmd.HasSubCommands}}\ COMMANDS: {{range .SubCommands}}\ {{ $cmdname := cmdName . $cmd }}\ {{ if .Runnable }}\ {{printf "%s\t%s" $cmdname .Short | indent}} {{end}}\ {{end}}\ {{end}}\ {{ if .Cmd.Long }}\ DESCRIPTION: {{range $line := descToLines .Cmd.Long}}{{printf "%s" $line | indent}} {{end}}\ {{end}}\ {{if .Cmd.HasLocalFlags}}\ OPTIONS: {{.LocalFlags}}\ {{end}}\ {{if .Cmd.HasInheritedFlags}}\ GLOBAL OPTIONS: {{.GlobalFlags}}\ {{end}} `[1:] commandUsageTemplate = template.Must(template.New("command_usage").Funcs(templFuncs).Parse(strings.ReplaceAll(commandUsage, "\\\n", ""))) } func etcdFlagUsages(flagSet *pflag.FlagSet) string { x := new(strings.Builder) flagSet.VisitAll(func(flag *pflag.Flag) { if len(flag.Deprecated) > 0 { return } var format string if len(flag.Shorthand) > 0 { format = " -%s, --%s" } else { format = " %s --%s" } if len(flag.NoOptDefVal) > 0 { format = format + "[" } if flag.Value.Type() == "string" { // put quotes on the value format = format + "=%q" } else { format = format + "=%s" } if len(flag.NoOptDefVal) > 0 { format = format + "]" } format = format + "\t%s\n" shorthand := flag.Shorthand fmt.Fprintf(x, format, shorthand, flag.Name, flag.DefValue, flag.Usage) }) return x.String() } func getSubCommands(cmd *cobra.Command) []*cobra.Command { var subCommands []*cobra.Command for _, subCmd := range cmd.Commands() { subCommands = append(subCommands, subCmd) subCommands = append(subCommands, getSubCommands(subCmd)...) } return subCommands } // UsageFunc is the usage function for the cobra command. // Deprecated: Please use go.etcd.io/etcd/etcdctl/v3/util instead. func UsageFunc(cmd *cobra.Command, version, APIVersion string) error { subCommands := getSubCommands(cmd) tabOut := getTabOutWithWriter(os.Stdout) commandUsageTemplate.Execute(tabOut, struct { Cmd *cobra.Command LocalFlags string GlobalFlags string SubCommands []*cobra.Command Version string APIVersion string }{ cmd, etcdFlagUsages(cmd.LocalFlags()), etcdFlagUsages(cmd.InheritedFlags()), subCommands, version, APIVersion, }) tabOut.Flush() return nil } func getTabOutWithWriter(writer io.Writer) *tabwriter.Writer { aTabOut := new(tabwriter.Writer) aTabOut.Init(writer, 0, 8, 1, '\t', 0) return aTabOut } ================================================ FILE: pkg/contention/contention.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package contention import ( "sync" "time" ) // TimeoutDetector detects routine starvations by // observing the actual time duration to finish an action // or between two events that should happen in a fixed // interval. If the observed duration is longer than // the expectation, the detector will report the result. type TimeoutDetector struct { mu sync.Mutex // protects all maxDuration time.Duration // map from event to last seen time of event. records map[uint64]time.Time } // NewTimeoutDetector creates the TimeoutDetector. func NewTimeoutDetector(maxDuration time.Duration) *TimeoutDetector { return &TimeoutDetector{ maxDuration: maxDuration, records: make(map[uint64]time.Time), } } // Reset resets the TimeoutDetector. func (td *TimeoutDetector) Reset() { td.mu.Lock() defer td.mu.Unlock() td.records = make(map[uint64]time.Time) } // Observe observes an event of given id. It computes // the time elapsed between successive events of given id. // It returns whether this time elapsed exceeds the expectation, // and the amount by which it exceeds the expectation. func (td *TimeoutDetector) Observe(id uint64) (bool, time.Duration) { td.mu.Lock() defer td.mu.Unlock() ok := true now := time.Now() exceed := time.Duration(0) if pt, found := td.records[id]; found { exceed = now.Sub(pt) - td.maxDuration if exceed > 0 { ok = false } } td.records[id] = now return ok, exceed } ================================================ FILE: pkg/contention/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package contention provides facilities for detecting system contention. package contention ================================================ FILE: pkg/cpuutil/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package cpuutil provides facilities for detecting cpu-specific features. package cpuutil ================================================ FILE: pkg/cpuutil/endian.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cpuutil import ( "encoding/binary" "golang.org/x/sys/cpu" ) // ByteOrder returns the byte order for the CPU's native endianness. func ByteOrder() binary.ByteOrder { if cpu.IsBigEndian { return binary.BigEndian } return binary.LittleEndian } ================================================ FILE: pkg/crc/crc.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package crc provides utility function for cyclic redundancy check // algorithms. package crc import ( "hash" "hash/crc32" ) // The size of a CRC-32 checksum in bytes. const Size = 4 type digest struct { crc uint32 tab *crc32.Table } // New creates a new hash.Hash32 computing the CRC-32 checksum // using the polynomial represented by the Table. // Modified by xiangli to take a prevcrc. func New(prev uint32, tab *crc32.Table) hash.Hash32 { return &digest{prev, tab} } func (d *digest) Size() int { return Size } func (d *digest) BlockSize() int { return 1 } func (d *digest) Reset() { d.crc = 0 } func (d *digest) Write(p []byte) (n int, err error) { d.crc = crc32.Update(d.crc, d.tab, p) return len(p), nil } func (d *digest) Sum32() uint32 { return d.crc } func (d *digest) Sum(in []byte) []byte { s := d.Sum32() return append(in, byte(s>>24), byte(s>>16), byte(s>>8), byte(s)) } ================================================ FILE: pkg/crc/crc_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package crc import ( "hash/crc32" "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // TestHash32 tests that Hash32 provided by this package can take an initial // crc and behaves exactly the same as the standard one in the following calls. func TestHash32(t *testing.T) { stdhash := crc32.New(crc32.IEEETable) _, err := stdhash.Write([]byte("test data")) require.NoErrorf(t, err, "unexpected write error: %v", err) // create a new hash with stdhash.Sum32() as initial crc hash := New(stdhash.Sum32(), crc32.IEEETable) assert.Equalf(t, hash.Size(), stdhash.Size(), "size") assert.Equalf(t, hash.BlockSize(), stdhash.BlockSize(), "block size") assert.Equalf(t, hash.Sum32(), stdhash.Sum32(), "Sum32") wsum := stdhash.Sum(make([]byte, 32)) g := hash.Sum(make([]byte, 32)) assert.Truef(t, reflect.DeepEqual(g, wsum), "sum") // write something _, err = stdhash.Write([]byte("test data")) require.NoErrorf(t, err, "unexpected write error: %v", err) _, err = hash.Write([]byte("test data")) require.NoErrorf(t, err, "unexpected write error: %v", err) assert.Equalf(t, hash.Sum32(), stdhash.Sum32(), "Sum32 after write") // reset stdhash.Reset() hash.Reset() assert.Equalf(t, hash.Sum32(), stdhash.Sum32(), "Sum32 after reset") } ================================================ FILE: pkg/debugutil/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package debugutil includes utility functions for debugging. package debugutil ================================================ FILE: pkg/debugutil/pprof.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package debugutil import ( "net/http" "net/http/pprof" "runtime" ) const HTTPPrefixPProf = "/debug/pprof" // PProfHandlers returns a map of pprof handlers keyed by the HTTP path. func PProfHandlers() map[string]http.Handler { // set only when there's no existing setting if runtime.SetMutexProfileFraction(-1) == 0 { // 1 out of 5 mutex events are reported, on average runtime.SetMutexProfileFraction(5) } m := make(map[string]http.Handler) m[HTTPPrefixPProf+"/"] = http.HandlerFunc(pprof.Index) m[HTTPPrefixPProf+"/profile"] = http.HandlerFunc(pprof.Profile) m[HTTPPrefixPProf+"/symbol"] = http.HandlerFunc(pprof.Symbol) m[HTTPPrefixPProf+"/cmdline"] = http.HandlerFunc(pprof.Cmdline) m[HTTPPrefixPProf+"/trace"] = http.HandlerFunc(pprof.Trace) m[HTTPPrefixPProf+"/heap"] = pprof.Handler("heap") m[HTTPPrefixPProf+"/goroutine"] = pprof.Handler("goroutine") m[HTTPPrefixPProf+"/threadcreate"] = pprof.Handler("threadcreate") m[HTTPPrefixPProf+"/block"] = pprof.Handler("block") m[HTTPPrefixPProf+"/mutex"] = pprof.Handler("mutex") return m } ================================================ FILE: pkg/expect/expect.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package expect implements a small expect-style interface package expect import ( "bufio" "context" "errors" "fmt" "io" "os" "os/exec" "regexp" "strings" "sync" "syscall" "time" "github.com/creack/pty" ) const debugLinesTail = 40 var ErrProcessRunning = fmt.Errorf("process is still running") type ExpectedResponse struct { Value string IsRegularExpr bool } type ExpectProcess struct { cfg expectConfig cmd *exec.Cmd fpty *os.File wg sync.WaitGroup readCloseCh chan struct{} // close it if async read goroutine exits mu sync.Mutex // protects lines, count, cur, exitErr and exitCode lines []string count int // increment whenever new line gets added cur int // current read position exitErr error // process exit error exitCode int } // NewExpect creates a new process for expect testing. func NewExpect(name string, arg ...string) (ep *ExpectProcess, err error) { // if env[] is nil, use current system env and the default command as name return NewExpectWithEnv(name, arg, nil, name) } // NewExpectWithEnv creates a new process with user defined env variables for expect testing. func NewExpectWithEnv(name string, args []string, env []string, serverProcessConfigName string) (ep *ExpectProcess, err error) { ep = &ExpectProcess{ cfg: expectConfig{ name: serverProcessConfigName, cmd: name, args: args, env: env, }, readCloseCh: make(chan struct{}), } ep.cmd = commandFromConfig(ep.cfg) if ep.fpty, err = pty.Start(ep.cmd); err != nil { return nil, err } ep.wg.Add(2) go ep.read() go ep.waitSaveExitErr() return ep, nil } type expectConfig struct { name string cmd string args []string env []string } func commandFromConfig(config expectConfig) *exec.Cmd { cmd := exec.Command(config.cmd, config.args...) cmd.Env = config.env cmd.Stderr = cmd.Stdout cmd.Stdin = nil return cmd } func (ep *ExpectProcess) Pid() int { return ep.cmd.Process.Pid } func (ep *ExpectProcess) read() { defer func() { ep.wg.Done() close(ep.readCloseCh) }() defer func(fpty *os.File) { err := fpty.Close() if err != nil { // we deliberately only log the error here, closing the PTY should mostly be (expected) broken pipes fmt.Printf("error while closing fpty: %v", err) } }(ep.fpty) r := bufio.NewReader(ep.fpty) for { err := ep.tryReadNextLine(r) if err != nil { break } } } func (ep *ExpectProcess) tryReadNextLine(r *bufio.Reader) error { printDebugLines := os.Getenv("EXPECT_DEBUG") != "" l, err := r.ReadString('\n') ep.mu.Lock() defer ep.mu.Unlock() if l != "" { if printDebugLines { fmt.Printf("%s (%s) (%d): %s", ep.cmd.Path, ep.cfg.name, ep.cmd.Process.Pid, l) } ep.lines = append(ep.lines, l) ep.count++ } // we're checking the error here at the bottom to ensure any leftover reads are still taken into account return err } func (ep *ExpectProcess) waitSaveExitErr() { defer ep.wg.Done() err := ep.waitProcess() ep.mu.Lock() defer ep.mu.Unlock() if err != nil { ep.exitErr = err } } // ExpectFunc returns the first line satisfying the function f. func (ep *ExpectProcess) ExpectFunc(ctx context.Context, f func(string) bool) (string, error) { i := 0 for { line, errsFound := func() (string, bool) { ep.mu.Lock() defer ep.mu.Unlock() // check if this expect has been already closed if ep.cmd == nil { return "", true } for i < len(ep.lines) { line := ep.lines[i] i++ if f(line) { return line, false } } return "", ep.exitErr != nil }() if line != "" { return line, nil } if errsFound { break } select { case <-ctx.Done(): return "", fmt.Errorf("context done before matching log found: %w", ctx.Err()) case <-time.After(time.Millisecond * 10): // continue loop } } select { // NOTE: we wait readCloseCh for ep.read() to complete draining the log before acquiring the lock. case <-ep.readCloseCh: case <-ctx.Done(): return "", fmt.Errorf("context done before to found matching log") } ep.mu.Lock() defer ep.mu.Unlock() // retry it since we get all the log data for i < len(ep.lines) { line := ep.lines[i] i++ if f(line) { return line, nil } } lastLinesIndex := len(ep.lines) - debugLinesTail if lastLinesIndex < 0 { lastLinesIndex = 0 } lastLines := strings.Join(ep.lines[lastLinesIndex:], "") return "", fmt.Errorf("match not found. "+ " Set EXPECT_DEBUG for more info Errs: [%v], last lines:\n%s", ep.exitErr, lastLines) } // ExpectWithContext returns the first line containing the given string. func (ep *ExpectProcess) ExpectWithContext(ctx context.Context, s ExpectedResponse) (string, error) { var ( expr *regexp.Regexp err error ) if s.IsRegularExpr { expr, err = regexp.Compile(s.Value) if err != nil { return "", err } } return ep.ExpectFunc(ctx, func(txt string) bool { if expr != nil { return expr.MatchString(txt) } return strings.Contains(txt, s.Value) }) } // Expect returns the first line containing the given string. // Deprecated: please use ExpectWithContext instead. func (ep *ExpectProcess) Expect(s string) (string, error) { return ep.ExpectWithContext(context.Background(), ExpectedResponse{Value: s}) } // LineCount returns the number of recorded lines since // the beginning of the process. func (ep *ExpectProcess) LineCount() int { ep.mu.Lock() defer ep.mu.Unlock() return ep.count } // ExitCode returns the exit code of this process. // If the process is still running, it returns exit code 0 and ErrProcessRunning. func (ep *ExpectProcess) ExitCode() (int, error) { ep.mu.Lock() defer ep.mu.Unlock() if ep.cmd == nil { return ep.exitCode, nil } if ep.exitErr != nil { // If the child process panics or is killed, for instance, the // goFailpoint triggers the exit event, the ep.cmd isn't nil and // the exitCode will describe the case. if ep.exitCode != 0 { return ep.exitCode, nil } // If the wait4(2) in waitProcess returns error, the child // process might be reaped if the process handles the SIGCHILD // in other goroutine. It's unlikely in this repo. But we // should return the error for log even if the child process // is still running. return 0, ep.exitErr } return 0, ErrProcessRunning } // ExitError returns the exit error of this process (if any). // If the process is still running, it returns ErrProcessRunning instead. func (ep *ExpectProcess) ExitError() error { ep.mu.Lock() defer ep.mu.Unlock() if ep.cmd == nil { return ep.exitErr } return ErrProcessRunning } // Stop signals the process to terminate via SIGTERM func (ep *ExpectProcess) Stop() error { err := ep.Signal(syscall.SIGTERM) if err != nil && errors.Is(err, os.ErrProcessDone) { return nil } return err } // Signal sends a signal to the expect process func (ep *ExpectProcess) Signal(sig os.Signal) error { ep.mu.Lock() defer ep.mu.Unlock() if ep.cmd == nil { return errors.New("expect process already closed") } return ep.cmd.Process.Signal(sig) } func (ep *ExpectProcess) waitProcess() error { state, err := ep.cmd.Process.Wait() if err != nil { return err } ep.mu.Lock() defer ep.mu.Unlock() ep.exitCode = exitCode(state) if !state.Success() { return fmt.Errorf("unexpected exit code [%d] after running [%s]", ep.exitCode, ep.cmd.String()) } return nil } // exitCode returns correct exit code for a process based on signaled or exited. func exitCode(state *os.ProcessState) int { status := state.Sys().(syscall.WaitStatus) if status.Signaled() { return 128 + int(status.Signal()) } return status.ExitStatus() } // Wait waits for the process to finish. func (ep *ExpectProcess) Wait() { ep.wg.Wait() } // Close waits for the expect process to exit and return its error. func (ep *ExpectProcess) Close() error { ep.wg.Wait() ep.mu.Lock() defer ep.mu.Unlock() // this signals to other funcs that the process has finished ep.cmd = nil return ep.exitErr } func (ep *ExpectProcess) Send(command string) error { _, err := io.WriteString(ep.fpty, command) return err } func (ep *ExpectProcess) Lines() []string { ep.mu.Lock() defer ep.mu.Unlock() return ep.lines } // ReadLine returns line by line. func (ep *ExpectProcess) ReadLine() string { ep.mu.Lock() defer ep.mu.Unlock() if ep.count > ep.cur { line := ep.lines[ep.cur] ep.cur++ return line } return "" } ================================================ FILE: pkg/expect/expect_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // build !windows package expect import ( "context" "os" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestExpectFunc(t *testing.T) { ep, err := NewExpect("echo", "hello world") require.NoError(t, err) wstr := "hello world\r\n" l, eerr := ep.ExpectFunc(t.Context(), func(a string) bool { return len(a) > 10 }) require.NoError(t, eerr) require.Equalf(t, l, wstr, `got "%v", expected "%v"`, l, wstr) require.NoError(t, ep.Close()) } func TestExpectFuncTimeout(t *testing.T) { ep, err := NewExpect("tail", "-f", "/dev/null") require.NoError(t, err) go func() { // It's enough to have "talkative" process to stuck in the infinite loop of reading for { if serr := ep.Send("new line\n"); serr != nil { return } } }() ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond) defer cancel() _, err = ep.ExpectFunc(ctx, func(a string) bool { return false }) require.ErrorIs(t, err, context.DeadlineExceeded) require.NoError(t, ep.Stop()) require.ErrorContains(t, ep.Close(), "unexpected exit code [143]") require.Equal(t, 143, ep.exitCode) } func TestExpectFuncExitFailure(t *testing.T) { // tail -x should not exist and return a non-zero exit code ep, err := NewExpect("tail", "-x") require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond) defer cancel() _, err = ep.ExpectFunc(ctx, func(s string) bool { return strings.Contains(s, "something entirely unexpected") }) require.ErrorContains(t, err, "unexpected exit code [1]") require.Equal(t, 1, ep.exitCode) } func TestExpectFuncExitFailureStop(t *testing.T) { // tail -x should not exist and return a non-zero exit code ep, err := NewExpect("tail", "-x") require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond) defer cancel() _, err = ep.ExpectFunc(ctx, func(s string) bool { return strings.Contains(s, "something entirely unexpected") }) require.ErrorContains(t, err, "unexpected exit code [1]") exitCode, err := ep.ExitCode() require.Equal(t, 1, exitCode) require.NoError(t, err) require.NoError(t, ep.Stop()) require.ErrorContains(t, ep.Close(), "unexpected exit code [1]") exitCode, err = ep.ExitCode() require.Equal(t, 1, exitCode) require.NoError(t, err) } func TestEcho(t *testing.T) { ep, err := NewExpect("echo", "hello world") require.NoError(t, err) ctx := t.Context() l, eerr := ep.ExpectWithContext(ctx, ExpectedResponse{Value: "world"}) require.NoError(t, eerr) wstr := "hello world" require.Equalf(t, l[:len(wstr)], wstr, `got "%v", expected "%v"`, l, wstr) require.NoError(t, ep.Close()) _, eerr = ep.ExpectWithContext(ctx, ExpectedResponse{Value: "..."}) require.Errorf(t, eerr, "expected error on closed expect process") } func TestLineCount(t *testing.T) { ep, err := NewExpect("printf", "1\n2\n3") require.NoError(t, err) wstr := "3" l, eerr := ep.ExpectWithContext(t.Context(), ExpectedResponse{Value: wstr}) require.NoError(t, eerr) require.Equalf(t, l, wstr, `got "%v", expected "%v"`, l, wstr) require.Equalf(t, 3, ep.LineCount(), "got %d, expected 3", ep.LineCount()) require.NoError(t, ep.Close()) } func TestSend(t *testing.T) { ep, err := NewExpect("tr", "a", "b") require.NoError(t, err) err = ep.Send("a\r") require.NoError(t, err) _, err = ep.ExpectWithContext(t.Context(), ExpectedResponse{Value: "b"}) require.NoError(t, err) require.NoError(t, ep.Stop()) } func TestSignal(t *testing.T) { ep, err := NewExpect("sleep", "100") require.NoError(t, err) ep.Signal(os.Interrupt) donec := make(chan struct{}) go func() { defer close(donec) err = ep.Close() assert.ErrorContains(t, err, "unexpected exit code [130]") assert.ErrorContains(t, err, "sleep 100") }() select { case <-time.After(5 * time.Second): t.Fatalf("signal test timed out") case <-donec: } } func TestExitCodeAfterKill(t *testing.T) { ep, err := NewExpect("sleep", "100") require.NoError(t, err) ep.Signal(os.Kill) ep.Wait() code, err := ep.ExitCode() assert.Equal(t, 137, code) assert.NoError(t, err) } func TestExpectForFailFastCommand(t *testing.T) { ep, err := NewExpect("sh", "-c", `echo "curl: (59) failed setting cipher list"; exit 59`) require.NoError(t, err) _, err = ep.Expect("failed setting cipher list") require.NoError(t, err) } func TestResponseMatchRegularExpr(t *testing.T) { testCases := []struct { name string mockOutput string expectedResp ExpectedResponse expectMatch bool }{ { name: "exact match", mockOutput: "hello world", expectedResp: ExpectedResponse{Value: "hello world"}, expectMatch: true, }, { name: "not exact match", mockOutput: "hello world", expectedResp: ExpectedResponse{Value: "hello wld"}, expectMatch: false, }, { name: "match regular expression", mockOutput: "hello world", expectedResp: ExpectedResponse{Value: `.*llo\sworld`, IsRegularExpr: true}, expectMatch: true, }, { name: "not match regular expression", mockOutput: "hello world", expectedResp: ExpectedResponse{Value: `.*llo wrld`, IsRegularExpr: true}, expectMatch: false, }, } for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { ep, err := NewExpect("echo", "-n", tc.mockOutput) require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() l, err := ep.ExpectWithContext(ctx, tc.expectedResp) if tc.expectMatch { require.Equal(t, tc.mockOutput, l) } else { require.Error(t, err) } require.NoError(t, ep.Close()) }) } } ================================================ FILE: pkg/featuregate/feature_gate.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package featuregate is copied from k8s.io/component-base@v0.30.1 to avoid any potential circular dependency between k8s and etcd. package featuregate import ( "flag" "fmt" "maps" "sort" "strconv" "strings" "sync" "sync/atomic" "github.com/spf13/pflag" "go.uber.org/zap" ) type Feature string const ( defaultFlagName = "feature-gates" // allAlphaGate is a global toggle for alpha features. Per-feature key // values override the default set by allAlphaGate. Examples: // AllAlpha=false,NewFeature=true will result in newFeature=true // AllAlpha=true,NewFeature=false will result in newFeature=false allAlphaGate Feature = "AllAlpha" // allBetaGate is a global toggle for beta features. Per-feature key // values override the default set by allBetaGate. Examples: // AllBeta=false,NewFeature=true will result in NewFeature=true // AllBeta=true,NewFeature=false will result in NewFeature=false allBetaGate Feature = "AllBeta" ) var ( // The generic features. defaultFeatures = map[Feature]FeatureSpec{ allAlphaGate: {Default: false, PreRelease: Alpha}, allBetaGate: {Default: false, PreRelease: Beta}, } // Special handling for a few gates. specialFeatures = map[Feature]func(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool){ allAlphaGate: setUnsetAlphaGates, allBetaGate: setUnsetBetaGates, } ) type FeatureSpec struct { // Default is the default enablement state for the feature Default bool // LockToDefault indicates that the feature is locked to its default and cannot be changed LockToDefault bool // PreRelease indicates the maturity level of the feature PreRelease prerelease } type prerelease string const ( // Values for PreRelease. Alpha = prerelease("ALPHA") Beta = prerelease("BETA") GA = prerelease("") // Deprecated Deprecated = prerelease("DEPRECATED") ) // FeatureGate indicates whether a given feature is enabled or not type FeatureGate interface { // Enabled returns true if the key is enabled. Enabled(key Feature) bool // KnownFeatures returns a slice of strings describing the FeatureGate's known features. KnownFeatures() []string // DeepCopy returns a deep copy of the FeatureGate object, such that gates can be // set on the copy without mutating the original. This is useful for validating // config against potential feature gate changes before committing those changes. DeepCopy() MutableFeatureGate // String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...". String() string } // MutableFeatureGate parses and stores flag gates for known features from // a string like feature1=true,feature2=false,... type MutableFeatureGate interface { FeatureGate // AddFlag adds a flag for setting global feature gates to the specified FlagSet. AddFlag(fs *flag.FlagSet, flagName string) // Set parses and stores flag gates for known features // from a string like feature1=true,feature2=false,... Set(value string) error // SetFromMap stores flag gates for known features from a map[string]bool or returns an error SetFromMap(m map[string]bool) error // Add adds features to the featureGate. Add(features map[Feature]FeatureSpec) error // GetAll returns a copy of the map of known feature names to feature specs. GetAll() map[Feature]FeatureSpec // OverrideDefault sets a local override for the registered default value of a named // feature. If the feature has not been previously registered (e.g. by a call to Add), has a // locked default, or if the gate has already registered itself with a FlagSet, a non-nil // error is returned. // // When two or more components consume a common feature, one component can override its // default at runtime in order to adopt new defaults before or after the other // components. For example, a new feature can be evaluated with a limited blast radius by // overriding its default to true for a limited number of components without simultaneously // changing its default for all consuming components. OverrideDefault(name Feature, override bool) error } // featureGate implements FeatureGate as well as pflag.Value for flag parsing. type featureGate struct { lg *zap.Logger featureGateName string special map[Feature]func(map[Feature]FeatureSpec, map[Feature]bool, bool) // lock guards writes to known, enabled, and reads/writes of closed lock sync.Mutex // known holds a map[Feature]FeatureSpec known atomic.Value // enabled holds a map[Feature]bool enabled atomic.Value // closed is set to true when AddFlag is called, and prevents subsequent calls to Add closed bool } func setUnsetAlphaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) { for k, v := range known { if v.PreRelease == Alpha { if _, found := enabled[k]; !found { enabled[k] = val } } } } func setUnsetBetaGates(known map[Feature]FeatureSpec, enabled map[Feature]bool, val bool) { for k, v := range known { if v.PreRelease == Beta { if _, found := enabled[k]; !found { enabled[k] = val } } } } // Set, String, and Type implement pflag.Value var _ pflag.Value = &featureGate{} func New(name string, lg *zap.Logger) MutableFeatureGate { if lg == nil { lg = zap.NewNop() } known := maps.Clone(defaultFeatures) f := &featureGate{ lg: lg, featureGateName: name, special: specialFeatures, } f.known.Store(known) f.enabled.Store(map[Feature]bool{}) return f } // Set parses a string of the form "key1=value1,key2=value2,..." into a // map[string]bool of known keys or returns an error. func (f *featureGate) Set(value string) error { m := make(map[string]bool) for _, s := range strings.Split(value, ",") { if len(s) == 0 { continue } arr := strings.SplitN(s, "=", 2) k := strings.TrimSpace(arr[0]) if len(arr) != 2 { return fmt.Errorf("missing bool value for %s", k) } v := strings.TrimSpace(arr[1]) boolValue, err := strconv.ParseBool(v) if err != nil { return fmt.Errorf("invalid value of %s=%s, err: %w", k, v, err) } m[k] = boolValue } return f.SetFromMap(m) } // SetFromMap stores flag gates for known features from a map[string]bool or returns an error func (f *featureGate) SetFromMap(m map[string]bool) error { f.lock.Lock() defer f.lock.Unlock() // Copy existing state known := map[Feature]FeatureSpec{} maps.Copy(known, f.known.Load().(map[Feature]FeatureSpec)) enabled := map[Feature]bool{} maps.Copy(enabled, f.enabled.Load().(map[Feature]bool)) for k, v := range m { k := Feature(k) featureSpec, ok := known[k] if !ok { return fmt.Errorf("unrecognized feature gate: %s", k) } if featureSpec.LockToDefault && featureSpec.Default != v { return fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, featureSpec.Default) } enabled[k] = v // Handle "special" features like "all alpha gates" if fn, found := f.special[k]; found { fn(known, enabled, v) } if featureSpec.PreRelease == Deprecated { f.lg.Warn(fmt.Sprintf("Setting deprecated feature gate %s=%t. It will be removed in a future release.", k, v)) } else if featureSpec.PreRelease == GA { f.lg.Warn(fmt.Sprintf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v)) } } // Persist changes f.known.Store(known) f.enabled.Store(enabled) f.lg.Info(fmt.Sprintf("feature gates: %v", f.enabled)) return nil } // String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...". func (f *featureGate) String() string { pairs := []string{} for k, v := range f.enabled.Load().(map[Feature]bool) { pairs = append(pairs, fmt.Sprintf("%s=%t", k, v)) } sort.Strings(pairs) return strings.Join(pairs, ",") } func (f *featureGate) Type() string { return "mapStringBool" } // Add adds features to the featureGate. func (f *featureGate) Add(features map[Feature]FeatureSpec) error { f.lock.Lock() defer f.lock.Unlock() if f.closed { return fmt.Errorf("cannot add a feature gate after adding it to the flag set") } // Copy existing state known := map[Feature]FeatureSpec{} maps.Copy(known, f.known.Load().(map[Feature]FeatureSpec)) for name, spec := range features { if existingSpec, found := known[name]; found { if existingSpec == spec { continue } return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec) } known[name] = spec } // Persist updated state f.known.Store(known) return nil } func (f *featureGate) OverrideDefault(name Feature, override bool) error { f.lock.Lock() defer f.lock.Unlock() if f.closed { return fmt.Errorf("cannot override default for feature %q: gates already added to a flag set", name) } known := map[Feature]FeatureSpec{} for name, spec := range f.known.Load().(map[Feature]FeatureSpec) { known[name] = spec } spec, ok := known[name] switch { case !ok: return fmt.Errorf("cannot override default: feature %q is not registered", name) case spec.LockToDefault: return fmt.Errorf("cannot override default: feature %q default is locked to %t", name, spec.Default) case spec.PreRelease == Deprecated: f.lg.Warn(fmt.Sprintf("Overriding default of deprecated feature gate %s=%t. It will be removed in a future release.", name, override)) case spec.PreRelease == GA: f.lg.Warn(fmt.Sprintf("Overriding default of GA feature gate %s=%t. It will be removed in a future release.", name, override)) } spec.Default = override known[name] = spec f.known.Store(known) return nil } // GetAll returns a copy of the map of known feature names to feature specs. func (f *featureGate) GetAll() map[Feature]FeatureSpec { retval := map[Feature]FeatureSpec{} maps.Copy(retval, f.known.Load().(map[Feature]FeatureSpec)) return retval } // Enabled returns true if the key is enabled. If the key is not known, this call will panic. func (f *featureGate) Enabled(key Feature) bool { if v, ok := f.enabled.Load().(map[Feature]bool)[key]; ok { return v } if v, ok := f.known.Load().(map[Feature]FeatureSpec)[key]; ok { return v.Default } panic(fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName)) } // AddFlag adds a flag for setting global feature gates to the specified FlagSet. func (f *featureGate) AddFlag(fs *flag.FlagSet, flagName string) { if flagName == "" { flagName = defaultFlagName } f.lock.Lock() // TODO(mtaufen): Shouldn't we just close it on the first Set/SetFromMap instead? // Not all components expose a feature gates flag using this AddFlag method, and // in the future, all components will completely stop exposing a feature gates flag, // in favor of componentconfig. f.closed = true f.lock.Unlock() known := f.KnownFeatures() fs.Var(f, flagName, ""+ "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ "Options are:\n"+strings.Join(known, "\n")) } // KnownFeatures returns a slice of strings describing the FeatureGate's known features. // Deprecated and GA features are hidden from the list. func (f *featureGate) KnownFeatures() []string { var known []string for k, v := range f.known.Load().(map[Feature]FeatureSpec) { if v.PreRelease == GA || v.PreRelease == Deprecated { continue } known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v.PreRelease, v.Default)) } sort.Strings(known) return known } // DeepCopy returns a deep copy of the FeatureGate object, such that gates can be // set on the copy without mutating the original. This is useful for validating // config against potential feature gate changes before committing those changes. func (f *featureGate) DeepCopy() MutableFeatureGate { // Copy existing state. known := map[Feature]FeatureSpec{} maps.Copy(known, f.known.Load().(map[Feature]FeatureSpec)) enabled := map[Feature]bool{} maps.Copy(enabled, f.enabled.Load().(map[Feature]bool)) // Construct a new featureGate around the copied state. // Note that specialFeatures is treated as immutable by convention, // and we maintain the value of f.closed across the copy. fg := &featureGate{ special: specialFeatures, closed: f.closed, } fg.known.Store(known) fg.enabled.Store(enabled) return fg } ================================================ FILE: pkg/featuregate/feature_gate_test.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package featuregate import ( "flag" "fmt" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" ) func TestFeatureGateFlag(t *testing.T) { // gates for testing const testAlphaGate Feature = "TestAlpha" const testBetaGate Feature = "TestBeta" tests := []struct { arg string expect map[Feature]bool parseError string }{ { arg: "", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: false, testBetaGate: false, }, }, { arg: "fooBarBaz=true", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: false, testBetaGate: false, }, parseError: "unrecognized feature gate: fooBarBaz", }, { arg: "AllAlpha=false", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: false, testBetaGate: false, }, }, { arg: "AllAlpha=true", expect: map[Feature]bool{ allAlphaGate: true, allBetaGate: false, testAlphaGate: true, testBetaGate: false, }, }, { arg: "AllAlpha=banana", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: false, testBetaGate: false, }, parseError: "invalid value of AllAlpha", }, { arg: "AllAlpha=false,TestAlpha=true", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: true, testBetaGate: false, }, }, { arg: "TestAlpha=true,AllAlpha=false", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: true, testBetaGate: false, }, }, { arg: "AllAlpha=true,TestAlpha=false", expect: map[Feature]bool{ allAlphaGate: true, allBetaGate: false, testAlphaGate: false, testBetaGate: false, }, }, { arg: "TestAlpha=false,AllAlpha=true", expect: map[Feature]bool{ allAlphaGate: true, allBetaGate: false, testAlphaGate: false, testBetaGate: false, }, }, { arg: "TestBeta=true,AllAlpha=false", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: false, testBetaGate: true, }, }, { arg: "AllBeta=false", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: false, testBetaGate: false, }, }, { arg: "AllBeta=true", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: true, testAlphaGate: false, testBetaGate: true, }, }, { arg: "AllBeta=banana", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: false, testBetaGate: false, }, parseError: "invalid value of AllBeta", }, { arg: "AllBeta=false,TestBeta=true", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: false, testBetaGate: true, }, }, { arg: "TestBeta=true,AllBeta=false", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: false, testBetaGate: true, }, }, { arg: "AllBeta=true,TestBeta=false", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: true, testAlphaGate: false, testBetaGate: false, }, }, { arg: "TestBeta=false,AllBeta=true", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: true, testAlphaGate: false, testBetaGate: false, }, }, { arg: "TestAlpha=true,AllBeta=false", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testAlphaGate: true, testBetaGate: false, }, }, } for i, test := range tests { t.Run(test.arg, func(t *testing.T) { fs := flag.NewFlagSet("testfeaturegateflag", flag.ContinueOnError) f := New("test", zaptest.NewLogger(t)) f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: false, PreRelease: Beta}, }) f.AddFlag(fs, defaultFlagName) err := fs.Parse([]string{fmt.Sprintf("--%s=%s", defaultFlagName, test.arg)}) if test.parseError != "" { assert.Containsf(t, err.Error(), test.parseError, "%d: Parse() Expected %v, Got %v", i, test.parseError, err) } else if err != nil { t.Errorf("%d: Parse() Expected nil, Got %v", i, err) } for k, v := range test.expect { actual := f.Enabled(k) assert.Equalf(t, actual, v, "%d: expected %s=%v, Got %v", i, k, v, actual) } }) } } func TestFeatureGateOverride(t *testing.T) { const testAlphaGate Feature = "TestAlpha" const testBetaGate Feature = "TestBeta" // Don't parse the flag, assert defaults are used. f := New("test", zaptest.NewLogger(t)) f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: false, PreRelease: Beta}, }) f.Set("TestAlpha=true,TestBeta=true") assert.Truef(t, f.Enabled(testAlphaGate), "Expected true") assert.Truef(t, f.Enabled(testBetaGate), "Expected true") f.Set("TestAlpha=false") assert.Falsef(t, f.Enabled(testAlphaGate), "Expected false") assert.Truef(t, f.Enabled(testBetaGate), "Expected true") } func TestFeatureGateFlagDefaults(t *testing.T) { // gates for testing const testAlphaGate Feature = "TestAlpha" const testBetaGate Feature = "TestBeta" // Don't parse the flag, assert defaults are used. f := New("test", zaptest.NewLogger(t)) f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: true, PreRelease: Beta}, }) assert.Falsef(t, f.Enabled(testAlphaGate), "Expected false") assert.Truef(t, f.Enabled(testBetaGate), "Expected true") } func TestFeatureGateKnownFeatures(t *testing.T) { // gates for testing const ( testAlphaGate Feature = "TestAlpha" testBetaGate Feature = "TestBeta" testGAGate Feature = "TestGA" testDeprecatedGate Feature = "TestDeprecated" ) // Don't parse the flag, assert defaults are used. f := New("test", zaptest.NewLogger(t)) f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: true, PreRelease: Beta}, testGAGate: {Default: true, PreRelease: GA}, testDeprecatedGate: {Default: false, PreRelease: Deprecated}, }) known := strings.Join(f.KnownFeatures(), " ") assert.Contains(t, known, testAlphaGate) assert.Contains(t, known, testBetaGate) assert.NotContains(t, known, testGAGate) assert.NotContains(t, known, testDeprecatedGate) } func TestFeatureGateSetFromMap(t *testing.T) { // gates for testing const testAlphaGate Feature = "TestAlpha" const testBetaGate Feature = "TestBeta" const testLockedTrueGate Feature = "TestLockedTrue" const testLockedFalseGate Feature = "TestLockedFalse" tests := []struct { name string setmap map[string]bool expect map[Feature]bool setmapError string }{ { name: "set TestAlpha and TestBeta true", setmap: map[string]bool{ "TestAlpha": true, "TestBeta": true, }, expect: map[Feature]bool{ testAlphaGate: true, testBetaGate: true, }, }, { name: "set TestBeta true", setmap: map[string]bool{ "TestBeta": true, }, expect: map[Feature]bool{ testAlphaGate: false, testBetaGate: true, }, }, { name: "set TestAlpha false", setmap: map[string]bool{ "TestAlpha": false, }, expect: map[Feature]bool{ testAlphaGate: false, testBetaGate: false, }, }, { name: "set TestInvaild true", setmap: map[string]bool{ "TestInvaild": true, }, expect: map[Feature]bool{ testAlphaGate: false, testBetaGate: false, }, setmapError: "unrecognized feature gate:", }, { name: "set locked gates", setmap: map[string]bool{ "TestLockedTrue": true, "TestLockedFalse": false, }, expect: map[Feature]bool{ testAlphaGate: false, testBetaGate: false, }, }, { name: "set locked gates", setmap: map[string]bool{ "TestLockedTrue": false, }, expect: map[Feature]bool{ testAlphaGate: false, testBetaGate: false, }, setmapError: "cannot set feature gate TestLockedTrue to false, feature is locked to true", }, { name: "set locked gates", setmap: map[string]bool{ "TestLockedFalse": true, }, expect: map[Feature]bool{ testAlphaGate: false, testBetaGate: false, }, setmapError: "cannot set feature gate TestLockedFalse to true, feature is locked to false", }, } for i, test := range tests { t.Run(fmt.Sprintf("SetFromMap %s", test.name), func(t *testing.T) { f := New("test", zaptest.NewLogger(t)) f.Add(map[Feature]FeatureSpec{ testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: false, PreRelease: Beta}, testLockedTrueGate: {Default: true, PreRelease: GA, LockToDefault: true}, testLockedFalseGate: {Default: false, PreRelease: GA, LockToDefault: true}, }) err := f.SetFromMap(test.setmap) if test.setmapError != "" { if err == nil { t.Errorf("expected error, got none") } else if !strings.Contains(err.Error(), test.setmapError) { t.Errorf("%d: SetFromMap(%#v) Expected err:%v, Got err:%v", i, test.setmap, test.setmapError, err) } } else if err != nil { t.Errorf("%d: SetFromMap(%#v) Expected success, Got err:%v", i, test.setmap, err) } for k, v := range test.expect { actual := f.Enabled(k) assert.Equalf(t, actual, v, "%d: SetFromMap(%#v) Expected %s=%v, Got %s=%v", i, test.setmap, k, v, k, actual) } }) } } func TestFeatureGateMetrics(t *testing.T) { // TODO(henrybear327): Add tests once feature gate metrics are added. } func TestFeatureGateString(t *testing.T) { // gates for testing const testAlphaGate Feature = "TestAlpha" const testBetaGate Feature = "TestBeta" const testGAGate Feature = "TestGA" featuremap := map[Feature]FeatureSpec{ testGAGate: {Default: true, PreRelease: GA}, testAlphaGate: {Default: false, PreRelease: Alpha}, testBetaGate: {Default: true, PreRelease: Beta}, } tests := []struct { setmap map[string]bool expect string }{ { setmap: map[string]bool{ "TestAlpha": false, }, expect: "TestAlpha=false", }, { setmap: map[string]bool{ "TestAlpha": false, "TestBeta": true, }, expect: "TestAlpha=false,TestBeta=true", }, { setmap: map[string]bool{ "TestGA": true, "TestAlpha": false, "TestBeta": true, }, expect: "TestAlpha=false,TestBeta=true,TestGA=true", }, } for i, test := range tests { t.Run(fmt.Sprintf("SetFromMap %s", test.expect), func(t *testing.T) { f := New("test", zaptest.NewLogger(t)) f.Add(featuremap) f.SetFromMap(test.setmap) result := f.String() assert.Equalf(t, result, test.expect, "%d: SetFromMap(%#v) Expected %s, Got %s", i, test.setmap, test.expect, result) }) } } func TestFeatureGateOverrideDefault(t *testing.T) { t.Run("overrides take effect", func(t *testing.T) { f := New("test", zaptest.NewLogger(t)) err := f.Add(map[Feature]FeatureSpec{ "TestFeature1": {Default: true}, "TestFeature2": {Default: false}, }) require.NoError(t, err) require.NoError(t, f.OverrideDefault("TestFeature1", false)) require.NoError(t, f.OverrideDefault("TestFeature2", true)) assert.Falsef(t, f.Enabled("TestFeature1"), "expected TestFeature1 to have effective default of false") assert.Truef(t, f.Enabled("TestFeature2"), "expected TestFeature2 to have effective default of true") }) t.Run("overrides are preserved across deep copies", func(t *testing.T) { f := New("test", zaptest.NewLogger(t)) err := f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: false}}) require.NoError(t, err) require.NoError(t, f.OverrideDefault("TestFeature", true)) fcopy := f.DeepCopy() assert.Truef(t, fcopy.Enabled("TestFeature"), "default override was not preserved by deep copy") }) t.Run("reflected in known features", func(t *testing.T) { f := New("test", zaptest.NewLogger(t)) err := f.Add(map[Feature]FeatureSpec{"TestFeature": { Default: false, PreRelease: Alpha, }}) require.NoError(t, err) require.NoError(t, f.OverrideDefault("TestFeature", true)) var found bool for _, s := range f.KnownFeatures() { if !strings.Contains(s, "TestFeature") { continue } found = true assert.Containsf(t, s, "default=true", "expected override of default to be reflected in known feature description %q", s) } assert.Truef(t, found, "found no entry for TestFeature in known features") }) t.Run("may not change default for specs with locked defaults", func(t *testing.T) { f := New("test", zaptest.NewLogger(t)) err := f.Add(map[Feature]FeatureSpec{ "LockedFeature": { Default: true, LockToDefault: true, }, }) require.NoError(t, err) require.Errorf(t, f.OverrideDefault("LockedFeature", false), "expected error when attempting to override the default for a feature with a locked default") assert.Errorf(t, f.OverrideDefault("LockedFeature", true), "expected error when attempting to override the default for a feature with a locked default") }) t.Run("does not supersede explicitly-set value", func(t *testing.T) { f := New("test", zaptest.NewLogger(t)) err := f.Add(map[Feature]FeatureSpec{"TestFeature": {Default: true}}) require.NoError(t, err) require.NoError(t, f.OverrideDefault("TestFeature", false)) require.NoError(t, f.SetFromMap(map[string]bool{"TestFeature": true})) assert.Truef(t, f.Enabled("TestFeature"), "expected feature to be effectively enabled despite default override") }) t.Run("prevents re-registration of feature spec after overriding default", func(t *testing.T) { f := New("test", zaptest.NewLogger(t)) err := f.Add(map[Feature]FeatureSpec{ "TestFeature": { Default: true, PreRelease: Alpha, }, }) require.NoError(t, err) require.NoError(t, f.OverrideDefault("TestFeature", false)) err = f.Add(map[Feature]FeatureSpec{ "TestFeature": { Default: true, PreRelease: Alpha, }, }) assert.Errorf(t, err, "expected re-registration to return a non-nil error after overriding its default") }) t.Run("does not allow override for an unknown feature", func(t *testing.T) { f := New("test", zaptest.NewLogger(t)) err := f.OverrideDefault("TestFeature", true) assert.Errorf(t, err, "expected an error to be returned in attempt to override default for unregistered feature") }) t.Run("returns error if already added to flag set", func(t *testing.T) { f := New("test", zaptest.NewLogger(t)) fs := flag.NewFlagSet("test", flag.ContinueOnError) f.AddFlag(fs, defaultFlagName) err := f.OverrideDefault("TestFeature", true) assert.Errorf(t, err, "expected a non-nil error to be returned") }) } ================================================ FILE: pkg/flags/flag.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package flags implements command-line flag parsing. package flags import ( "flag" "fmt" "os" "strconv" "strings" "github.com/spf13/pflag" "go.uber.org/zap" ) // SetFlagsFromEnv parses all registered flags in the given flagset, // and if they are not already set it attempts to set their values from // environment variables. Environment variables take the name of the flag but // are UPPERCASE, have the given prefix and any dashes are replaced by // underscores - for example: some-flag => ETCD_SOME_FLAG func SetFlagsFromEnv(lg *zap.Logger, prefix string, fs *flag.FlagSet) error { var err error alreadySet := make(map[string]bool) fs.Visit(func(f *flag.Flag) { alreadySet[FlagToEnv(prefix, f.Name)] = true }) usedEnvKey := make(map[string]bool) fs.VisitAll(func(f *flag.Flag) { if serr := setFlagFromEnv(lg, fs, prefix, f.Name, usedEnvKey, alreadySet, true); serr != nil { err = serr } }) verifyEnv(lg, prefix, usedEnvKey, alreadySet) return err } // SetPflagsFromEnv is similar to SetFlagsFromEnv. However, the accepted flagset type is pflag.FlagSet // and it does not do any logging. func SetPflagsFromEnv(lg *zap.Logger, prefix string, fs *pflag.FlagSet) error { var err error alreadySet := make(map[string]bool) usedEnvKey := make(map[string]bool) fs.VisitAll(func(f *pflag.Flag) { if f.Changed { alreadySet[FlagToEnv(prefix, f.Name)] = true } if serr := setFlagFromEnv(lg, fs, prefix, f.Name, usedEnvKey, alreadySet, false); serr != nil { err = serr } }) verifyEnv(lg, prefix, usedEnvKey, alreadySet) return err } // FlagToEnv converts flag string to upper-case environment variable key string. func FlagToEnv(prefix, name string) string { return prefix + "_" + strings.ToUpper(strings.ReplaceAll(name, "-", "_")) } func verifyEnv(lg *zap.Logger, prefix string, usedEnvKey, alreadySet map[string]bool) { for _, env := range os.Environ() { kv := strings.SplitN(env, "=", 2) if len(kv) != 2 { if lg != nil { lg.Warn("found invalid environment variable", zap.String("environment-variable", env)) } } if usedEnvKey[kv[0]] { continue } if alreadySet[kv[0]] { if lg != nil { lg.Fatal( "conflicting environment variable is shadowed by corresponding command-line flag (either unset environment variable or disable flag))", zap.String("environment-variable", kv[0]), ) } } if strings.HasPrefix(env, prefix+"_") { if lg != nil { lg.Warn("unrecognized environment variable", zap.String("environment-variable", env)) } } } } type flagSetter interface { Set(fk string, fv string) error } func setFlagFromEnv(lg *zap.Logger, fs flagSetter, prefix, fname string, usedEnvKey, alreadySet map[string]bool, log bool) error { key := FlagToEnv(prefix, fname) if !alreadySet[key] { val := os.Getenv(key) if val != "" { usedEnvKey[key] = true if serr := fs.Set(fname, val); serr != nil { return fmt.Errorf("invalid value %q for %s: %w", val, key, serr) } if log && lg != nil { lg.Info( "recognized and used environment variable", zap.String("variable-name", key), zap.String("variable-value", val), ) } } } return nil } func IsSet(fs *flag.FlagSet, name string) bool { set := false fs.Visit(func(f *flag.Flag) { if f.Name == name { set = true } }) return set } // GetBoolFlagVal returns the value of the a given bool flag if it is explicitly set // in the cmd line arguments, otherwise returns nil. func GetBoolFlagVal(fs *flag.FlagSet, flagName string) (*bool, error) { if !IsSet(fs, flagName) { return nil, nil } flagVal, parseErr := strconv.ParseBool(fs.Lookup(flagName).Value.String()) if parseErr != nil { return nil, parseErr } return &flagVal, nil } ================================================ FILE: pkg/flags/flag_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "flag" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" ) func TestSetFlagsFromEnv(t *testing.T) { fs := flag.NewFlagSet("testing", flag.ExitOnError) fs.String("a", "", "") fs.String("b", "", "") fs.String("c", "", "") fs.Parse([]string{}) // flags should be settable using env vars t.Setenv("ETCD_A", "foo") // and command-line flags require.NoError(t, fs.Set("b", "bar")) // first verify that flags are as expected before reading the env for f, want := range map[string]string{ "a": "", "b": "bar", } { got := fs.Lookup(f).Value.String() require.Equalf(t, want, got, "flag %q=%q, want %q", f, got, want) } // now read the env and verify flags were updated as expected require.NoError(t, SetFlagsFromEnv(zaptest.NewLogger(t), "ETCD", fs)) for f, want := range map[string]string{ "a": "foo", "b": "bar", } { got := fs.Lookup(f).Value.String() assert.Equalf(t, want, got, "flag %q=%q, want %q", f, got, want) } } func TestSetFlagsFromEnvBad(t *testing.T) { // now verify that an error is propagated fs := flag.NewFlagSet("testing", flag.ExitOnError) fs.Int("x", 0, "") t.Setenv("ETCD_X", "not_a_number") assert.Error(t, SetFlagsFromEnv(zaptest.NewLogger(t), "ETCD", fs)) } func TestSetFlagsFromEnvParsingError(t *testing.T) { fs := flag.NewFlagSet("etcd", flag.ContinueOnError) var tickMs uint fs.UintVar(&tickMs, "heartbeat-interval", 0, "Time (in milliseconds) of a heartbeat interval.") t.Setenv("ETCD_HEARTBEAT_INTERVAL", "100 # ms") err := SetFlagsFromEnv(zaptest.NewLogger(t), "ETCD", fs) for _, v := range []string{"invalid syntax", "parse error"} { if strings.Contains(err.Error(), v) { err = nil break } } require.NoErrorf(t, err, "unexpected error %v", err) } ================================================ FILE: pkg/flags/ignored.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import "go.uber.org/zap" // IgnoredFlag encapsulates a flag that may have been previously valid but is // now ignored. If an IgnoredFlag is set, a warning is printed and // operation continues. type IgnoredFlag struct { lg *zap.Logger Name string } // IsBoolFlag is defined to allow the flag to be defined without an argument func (f *IgnoredFlag) IsBoolFlag() bool { return true } func (f *IgnoredFlag) Set(s string) error { if f.lg != nil { f.lg.Warn("flag is no longer supported - ignoring", zap.String("flag-name", f.Name)) } return nil } func (f *IgnoredFlag) String() string { return "" } ================================================ FILE: pkg/flags/selective_string.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "errors" "fmt" "sort" "strings" ) // SelectiveStringValue implements the flag.Value interface. type SelectiveStringValue struct { v string valids map[string]struct{} } // Set verifies the argument to be a valid member of the allowed values // before setting the underlying flag value. func (ss *SelectiveStringValue) Set(s string) error { if _, ok := ss.valids[s]; ok { ss.v = s return nil } return errors.New("invalid value") } // String returns the set value (if any) of the SelectiveStringValue func (ss *SelectiveStringValue) String() string { return ss.v } // Valids returns the list of valid strings. func (ss *SelectiveStringValue) Valids() []string { s := make([]string, 0, len(ss.valids)) for k := range ss.valids { s = append(s, k) } sort.Strings(s) return s } // NewSelectiveStringValue creates a new string flag // for which any one of the given strings is a valid value, // and any other value is an error. // // valids[0] will be default value. Caller must be sure // len(valids) != 0 or it will panic. func NewSelectiveStringValue(valids ...string) *SelectiveStringValue { vm := make(map[string]struct{}, len(valids)) for _, v := range valids { vm[v] = struct{}{} } return &SelectiveStringValue{valids: vm, v: valids[0]} } // SelectiveStringsValue implements the flag.Value interface. type SelectiveStringsValue struct { vs []string valids map[string]struct{} } // Set verifies the argument to be a valid member of the allowed values // before setting the underlying flag value. func (ss *SelectiveStringsValue) Set(s string) error { vs := strings.Split(s, ",") for i := range vs { if _, ok := ss.valids[vs[i]]; !ok { return fmt.Errorf("invalid value %q", vs[i]) } ss.vs = append(ss.vs, vs[i]) } sort.Strings(ss.vs) return nil } // String returns the set value (if any) of the SelectiveStringsValue. func (ss *SelectiveStringsValue) String() string { return strings.Join(ss.vs, ",") } // Valids returns the list of valid strings. func (ss *SelectiveStringsValue) Valids() []string { s := make([]string, 0, len(ss.valids)) for k := range ss.valids { s = append(s, k) } sort.Strings(s) return s } // NewSelectiveStringsValue creates a new string slice flag // for which any one of the given strings is a valid value, // and any other value is an error. func NewSelectiveStringsValue(valids ...string) *SelectiveStringsValue { vm := make(map[string]struct{}, len(valids)) for _, v := range valids { vm[v] = struct{}{} } return &SelectiveStringsValue{valids: vm, vs: []string{}} } ================================================ FILE: pkg/flags/selective_string_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "testing" "github.com/stretchr/testify/assert" ) func TestSelectiveStringValue(t *testing.T) { tests := []struct { vals []string val string pass bool }{ // known values {[]string{"abc", "def"}, "abc", true}, {[]string{"on", "off", "false"}, "on", true}, // unrecognized values {[]string{"abc", "def"}, "ghi", false}, {[]string{"on", "off"}, "", false}, } for i, tt := range tests { sf := NewSelectiveStringValue(tt.vals...) assert.Equalf(t, sf.v, tt.vals[0], "#%d: want default val=%v,but got %v", i, tt.vals[0], sf.v) err := sf.Set(tt.val) assert.Equalf(t, tt.pass, (err == nil), "#%d: want pass=%t, but got err=%v", i, tt.pass, err) } } func TestSelectiveStringsValue(t *testing.T) { tests := []struct { vals []string val string pass bool }{ {[]string{"abc", "def"}, "abc", true}, {[]string{"abc", "def"}, "abc,def", true}, {[]string{"abc", "def"}, "abc, def", false}, {[]string{"on", "off", "false"}, "on,false", true}, {[]string{"abc", "def"}, "ghi", false}, {[]string{"on", "off"}, "", false}, {[]string{"a", "b", "c", "d", "e"}, "a,c,e", true}, } for i, tt := range tests { sf := NewSelectiveStringsValue(tt.vals...) err := sf.Set(tt.val) assert.Equalf(t, tt.pass, (err == nil), "#%d: want pass=%t, but got err=%v", i, tt.pass, err) } } ================================================ FILE: pkg/flags/strings.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "flag" "fmt" "sort" "strings" ) // StringsValue wraps "sort.StringSlice". type StringsValue sort.StringSlice // Set parses a command line set of strings, separated by comma. // Implements "flag.Value" interface. func (ss *StringsValue) Set(s string) error { *ss = strings.Split(s, ",") return nil } // String implements "flag.Value" interface. func (ss *StringsValue) String() string { return strings.Join(*ss, ",") } // NewStringsValue implements string slice as "flag.Value" interface. // Given value is to be separated by comma. func NewStringsValue(s string) (ss *StringsValue) { if s == "" { return &StringsValue{} } ss = new(StringsValue) if err := ss.Set(s); err != nil { panic(fmt.Sprintf("new StringsValue should never fail: %v", err)) } return ss } // StringsFromFlag returns a string slice from the flag. func StringsFromFlag(fs *flag.FlagSet, flagName string) []string { return *fs.Lookup(flagName).Value.(*StringsValue) } ================================================ FILE: pkg/flags/strings_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "reflect" "testing" "github.com/stretchr/testify/require" ) func TestStringsValue(t *testing.T) { tests := []struct { s string exp []string }{ {s: "a,b,c", exp: []string{"a", "b", "c"}}, {s: "a, b,c", exp: []string{"a", " b", "c"}}, {s: "", exp: []string{}}, } for i := range tests { ss := []string(*NewStringsValue(tests[i].s)) require.Truef(t, reflect.DeepEqual(tests[i].exp, ss), "#%d: expected %q, got %q", i, tests[i].exp, ss) } } ================================================ FILE: pkg/flags/uint32.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "flag" "strconv" ) type uint32Value uint32 // NewUint32Value creates an uint32 instance with the provided value. func NewUint32Value(v uint32) *uint32Value { val := new(uint32Value) *val = uint32Value(v) return val } // Set parses a command line uint32 value. // Implements "flag.Value" interface. func (i *uint32Value) Set(s string) error { v, err := strconv.ParseUint(s, 0, 32) *i = uint32Value(v) return err } func (i *uint32Value) String() string { return strconv.FormatUint(uint64(*i), 10) } // Uint32FromFlag return the uint32 value of a flag with the given name func Uint32FromFlag(fs *flag.FlagSet, name string) uint32 { val := *fs.Lookup(name).Value.(*uint32Value) return uint32(val) } ================================================ FILE: pkg/flags/uint32_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "flag" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestUint32Value(t *testing.T) { cases := []struct { name string s string expectedVal uint32 expectError bool }{ { name: "normal uint32 value", s: "200", expectedVal: 200, }, { name: "zero value", s: "0", expectedVal: 0, }, { name: "negative int value", s: "-200", expectError: true, }, { name: "invalid integer value", s: "invalid", expectError: true, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { var val uint32Value err := val.Set(tc.s) if tc.expectError { assert.Errorf(t, err, "Expected failure on parsing uint32 value from %s", tc.s) } else { require.NoErrorf(t, err, "Unexpected error when parsing %s: %v", tc.s, err) assert.Equal(t, tc.expectedVal, uint32(val)) } }) } } func TestUint32FromFlag(t *testing.T) { const flagName = "max-concurrent-streams" cases := []struct { name string defaultVal uint32 arguments []string expectedVal uint32 }{ { name: "only default value", defaultVal: 15, arguments: []string{}, expectedVal: 15, }, { name: "argument has different value from the default one", defaultVal: 16, arguments: []string{"--max-concurrent-streams", "200"}, expectedVal: 200, }, { name: "argument has the same value from the default one", defaultVal: 105, arguments: []string{"--max-concurrent-streams", "105"}, expectedVal: 105, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { fs := flag.NewFlagSet("etcd", flag.ContinueOnError) fs.Var(NewUint32Value(tc.defaultVal), flagName, "Maximum concurrent streams that each client can open at a time.") require.NoError(t, fs.Parse(tc.arguments)) actualMaxStream := Uint32FromFlag(fs, flagName) assert.Equal(t, tc.expectedVal, actualMaxStream) }) } } ================================================ FILE: pkg/flags/unique_strings.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "flag" "fmt" "sort" "strings" ) // UniqueStringsValue wraps a list of unique strings. // The values are set in order. type UniqueStringsValue struct { Values map[string]struct{} } // Set parses a command line set of strings, separated by comma. // Implements "flag.Value" interface. // The values are set in order. func (us *UniqueStringsValue) Set(s string) error { values := strings.Split(s, ",") us.Values = make(map[string]struct{}, len(values)) for _, v := range values { us.Values[v] = struct{}{} } return nil } // String implements "flag.Value" interface. func (us *UniqueStringsValue) String() string { return strings.Join(us.stringSlice(), ",") } func (us *UniqueStringsValue) stringSlice() []string { ss := make([]string, 0, len(us.Values)) for v := range us.Values { ss = append(ss, v) } sort.Strings(ss) return ss } // NewUniqueStringsValue implements string slice as "flag.Value" interface. // Given value is to be separated by comma. // The values are set in order. func NewUniqueStringsValue(s string) (us *UniqueStringsValue) { us = &UniqueStringsValue{Values: make(map[string]struct{})} if s == "" { return us } if err := us.Set(s); err != nil { panic(fmt.Sprintf("new UniqueStringsValue should never fail: %v", err)) } return us } // UniqueStringsFromFlag returns a string slice from the flag. func UniqueStringsFromFlag(fs *flag.FlagSet, flagName string) []string { return (*fs.Lookup(flagName).Value.(*UniqueStringsValue)).stringSlice() } // UniqueStringsMapFromFlag returns a map of strings from the flag. func UniqueStringsMapFromFlag(fs *flag.FlagSet, flagName string) map[string]struct{} { return (*fs.Lookup(flagName).Value.(*UniqueStringsValue)).Values } ================================================ FILE: pkg/flags/unique_strings_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "reflect" "testing" "github.com/stretchr/testify/require" ) func TestNewUniqueStrings(t *testing.T) { tests := []struct { s string exp map[string]struct{} rs string }{ { // non-URL but allowed by exception s: "*", exp: map[string]struct{}{"*": {}}, rs: "*", }, { s: "", exp: map[string]struct{}{}, rs: "", }, { s: "example.com", exp: map[string]struct{}{"example.com": {}}, rs: "example.com", }, { s: "localhost,localhost", exp: map[string]struct{}{"localhost": {}}, rs: "localhost", }, { s: "b.com,a.com", exp: map[string]struct{}{"a.com": {}, "b.com": {}}, rs: "a.com,b.com", }, { s: "c.com,b.com", exp: map[string]struct{}{"b.com": {}, "c.com": {}}, rs: "b.com,c.com", }, } for i := range tests { uv := NewUniqueStringsValue(tests[i].s) require.Truef(t, reflect.DeepEqual(tests[i].exp, uv.Values), "#%d: expected %+v, got %+v", i, tests[i].exp, uv.Values) require.Equalf(t, uv.String(), tests[i].rs, "#%d: expected %q, got %q", i, tests[i].rs, uv.String()) } } ================================================ FILE: pkg/flags/unique_urls.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "flag" "fmt" "net/url" "sort" "strings" "go.etcd.io/etcd/client/pkg/v3/types" ) // UniqueURLs contains unique URLs // with non-URL exceptions. type UniqueURLs struct { Values map[string]struct{} uss []url.URL Allowed map[string]struct{} } // Set parses a command line set of URLs formatted like: // http://127.0.0.1:2380,http://10.1.1.2:80 // Implements "flag.Value" interface. func (us *UniqueURLs) Set(s string) error { if _, ok := us.Values[s]; ok { return nil } if _, ok := us.Allowed[s]; ok { us.Values[s] = struct{}{} return nil } ss, err := types.NewURLs(strings.Split(s, ",")) if err != nil { return err } us.Values = make(map[string]struct{}) us.uss = make([]url.URL, 0) for _, v := range ss { x := v.String() if _, exists := us.Values[x]; exists { continue } us.Values[x] = struct{}{} us.uss = append(us.uss, v) } return nil } // String implements "flag.Value" interface. func (us *UniqueURLs) String() string { all := make([]string, 0, len(us.Values)) for u := range us.Values { all = append(all, u) } sort.Strings(all) return strings.Join(all, ",") } // NewUniqueURLsWithExceptions implements "url.URL" slice as flag.Value interface. // Given value is to be separated by comma. func NewUniqueURLsWithExceptions(s string, exceptions ...string) *UniqueURLs { us := &UniqueURLs{Values: make(map[string]struct{}), Allowed: make(map[string]struct{})} for _, v := range exceptions { us.Allowed[v] = struct{}{} } if s == "" { return us } if err := us.Set(s); err != nil { panic(fmt.Sprintf("new UniqueURLs should never fail: %v", err)) } return us } // UniqueURLsFromFlag returns a slice from urls got from the flag. func UniqueURLsFromFlag(fs *flag.FlagSet, urlsFlagName string) []url.URL { return (*fs.Lookup(urlsFlagName).Value.(*UniqueURLs)).uss } // UniqueURLsMapFromFlag returns a map from url strings got from the flag. func UniqueURLsMapFromFlag(fs *flag.FlagSet, urlsFlagName string) map[string]struct{} { return (*fs.Lookup(urlsFlagName).Value.(*UniqueURLs)).Values } ================================================ FILE: pkg/flags/unique_urls_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "flag" "strings" "testing" "github.com/stretchr/testify/require" ) func TestNewUniqueURLsWithExceptions(t *testing.T) { tests := []struct { s string exp map[string]struct{} rs string exception string }{ { // non-URL but allowed by exception s: "*", exp: map[string]struct{}{"*": {}}, rs: "*", exception: "*", }, { s: "", exp: map[string]struct{}{}, rs: "", exception: "*", }, { s: "https://1.2.3.4:8080", exp: map[string]struct{}{"https://1.2.3.4:8080": {}}, rs: "https://1.2.3.4:8080", exception: "*", }, { s: "https://1.2.3.4:8080,https://1.2.3.4:8080", exp: map[string]struct{}{"https://1.2.3.4:8080": {}}, rs: "https://1.2.3.4:8080", exception: "*", }, { s: "http://10.1.1.1:80", exp: map[string]struct{}{"http://10.1.1.1:80": {}}, rs: "http://10.1.1.1:80", exception: "*", }, { s: "http://localhost:80", exp: map[string]struct{}{"http://localhost:80": {}}, rs: "http://localhost:80", exception: "*", }, { s: "http://:80", exp: map[string]struct{}{"http://:80": {}}, rs: "http://:80", exception: "*", }, { s: "https://localhost:5,https://localhost:3", exp: map[string]struct{}{"https://localhost:3": {}, "https://localhost:5": {}}, rs: "https://localhost:3,https://localhost:5", exception: "*", }, { s: "http://localhost:5,https://localhost:3", exp: map[string]struct{}{"https://localhost:3": {}, "http://localhost:5": {}}, rs: "http://localhost:5,https://localhost:3", exception: "*", }, } for i := range tests { uv := NewUniqueURLsWithExceptions(tests[i].s, tests[i].exception) require.Equal(t, tests[i].exp, uv.Values) require.Equal(t, tests[i].rs, uv.String()) } } func TestUniqueURLsFromFlag(t *testing.T) { const name = "test" urls := []string{ "https://1.2.3.4:1", "https://1.2.3.4:2", "https://1.2.3.4:3", "https://1.2.3.4:1", } fs := flag.NewFlagSet(name, flag.ExitOnError) u := NewUniqueURLsWithExceptions(strings.Join(urls, ",")) fs.Var(u, name, "usage") uss := UniqueURLsFromFlag(fs, name) require.Len(t, uss, len(u.Values)) um := make(map[string]struct{}) for _, x := range uss { um[x.String()] = struct{}{} } require.Equal(t, u.Values, um) } ================================================ FILE: pkg/flags/urls.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "flag" "fmt" "net/url" "strings" "go.etcd.io/etcd/client/pkg/v3/types" ) // URLsValue wraps "types.URLs". type URLsValue types.URLs // Set parses a command line set of URLs formatted like: // http://127.0.0.1:2380,http://10.1.1.2:80 // Implements "flag.Value" interface. func (us *URLsValue) Set(s string) error { ss, err := types.NewURLs(strings.Split(s, ",")) if err != nil { return err } *us = URLsValue(ss) return nil } // String implements "flag.Value" interface. func (us *URLsValue) String() string { all := make([]string, len(*us)) for i, u := range *us { all[i] = u.String() } return strings.Join(all, ",") } // NewURLsValue implements "url.URL" slice as flag.Value interface. // Given value is to be separated by comma. func NewURLsValue(s string) *URLsValue { if s == "" { return &URLsValue{} } v := &URLsValue{} if err := v.Set(s); err != nil { panic(fmt.Sprintf("new URLsValue should never fail: %v", err)) } return v } // URLsFromFlag returns a slices from url got from the flag. func URLsFromFlag(fs *flag.FlagSet, urlsFlagName string) []url.URL { return *fs.Lookup(urlsFlagName).Value.(*URLsValue) } ================================================ FILE: pkg/flags/urls_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package flags import ( "net/url" "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestValidateURLsValueBad(t *testing.T) { tests := []string{ // bad IP specification ":2379", "127.0:8080", "123:456", // bad port specification "127.0.0.1:foo", "127.0.0.1:", // bad strings "somewhere", "234#$", "file://foo/bar", "http://hello/asdf", "http://10.1.1.1", } for i, in := range tests { u := URLsValue{} assert.Errorf(t, u.Set(in), `#%d: unexpected nil error for in=%q`, i, in) } } func TestNewURLsValue(t *testing.T) { tests := []struct { s string exp []url.URL }{ {s: "https://1.2.3.4:8080", exp: []url.URL{{Scheme: "https", Host: "1.2.3.4:8080"}}}, {s: "http://10.1.1.1:80", exp: []url.URL{{Scheme: "http", Host: "10.1.1.1:80"}}}, {s: "http://localhost:80", exp: []url.URL{{Scheme: "http", Host: "localhost:80"}}}, {s: "http://:80", exp: []url.URL{{Scheme: "http", Host: ":80"}}}, {s: "unix://tmp/etcd.sock", exp: []url.URL{{Scheme: "unix", Host: "tmp", Path: "/etcd.sock"}}}, {s: "unix:///tmp/127.27.84.4:23432", exp: []url.URL{{Scheme: "unix", Path: "/tmp/127.27.84.4:23432"}}}, {s: "unix://127.0.0.5:1456", exp: []url.URL{{Scheme: "unix", Host: "127.0.0.5:1456"}}}, { s: "http://localhost:1,https://localhost:2", exp: []url.URL{ {Scheme: "http", Host: "localhost:1"}, {Scheme: "https", Host: "localhost:2"}, }, }, } for i := range tests { uu := []url.URL(*NewURLsValue(tests[i].s)) require.Truef(t, reflect.DeepEqual(tests[i].exp, uu), "#%d: expected %+v, got %+v", i, tests[i].exp, uu) } } ================================================ FILE: pkg/go.mod ================================================ module go.etcd.io/etcd/pkg/v3 go 1.26 toolchain go1.26.1 require ( github.com/creack/pty v1.1.18 github.com/dustin/go-humanize v1.0.1 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 go.opentelemetry.io/otel/trace v1.42.0 go.uber.org/zap v1.27.1 golang.org/x/sys v0.41.0 google.golang.org/grpc v1.79.2 ) require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace go.etcd.io/etcd/client/pkg/v3 => ../client/pkg ================================================ FILE: pkg/go.sum ================================================ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: pkg/grpctesting/recorder.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpctesting import ( "context" "sync" "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) type GRPCRecorder struct { mux sync.RWMutex requests []RequestInfo } type RequestInfo struct { FullMethod string Authority string } func (ri *GRPCRecorder) UnaryInterceptor() grpc.UnaryServerInterceptor { return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { ri.record(toRequestInfo(ctx, info)) resp, err := handler(ctx, req) return resp, err } } func (ri *GRPCRecorder) RecordedRequests() []RequestInfo { ri.mux.RLock() defer ri.mux.RUnlock() reqs := make([]RequestInfo, len(ri.requests)) copy(reqs, ri.requests) return reqs } func toRequestInfo(ctx context.Context, info *grpc.UnaryServerInfo) RequestInfo { req := RequestInfo{ FullMethod: info.FullMethod, } md, ok := metadata.FromIncomingContext(ctx) if ok { as := md.Get(":authority") if len(as) != 0 { req.Authority = as[0] } } return req } func (ri *GRPCRecorder) record(r RequestInfo) { ri.mux.Lock() defer ri.mux.Unlock() ri.requests = append(ri.requests, r) } ================================================ FILE: pkg/grpctesting/stub_server.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpctesting import ( "context" "fmt" "net" "strconv" "sync/atomic" "google.golang.org/grpc" testpb "google.golang.org/grpc/interop/grpc_testing" ) // StubServer is borrowed from the internal package of grpc-go. // See https://github.com/grpc/grpc-go/blob/master/internal/stubserver/stubserver.go // Since it cannot be imported directly, we have to copy and paste it here, // and useless code for our testing is removed. // StubServer is a server that is easy to customize within individual test // cases. type StubServer struct { testService testpb.TestServiceServer // Network and Address are parameters for Listen. Defaults will be used if these are empty before Start. Network string Address string s *grpc.Server cleanups []func() // Lambdas executed in Stop(); populated by Start(). started chan struct{} } func New(testService testpb.TestServiceServer) *StubServer { return &StubServer{ testService: testService, started: make(chan struct{}), } } // Start starts the server and creates a client connected to it. func (ss *StubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) error { if ss.Network == "" { ss.Network = "tcp" } if ss.Address == "" { ss.Address = "localhost:0" } lis, err := net.Listen(ss.Network, ss.Address) if err != nil { return fmt.Errorf("net.Listen(%q, %q) = %w", ss.Network, ss.Address, err) } ss.Address = lis.Addr().String() ss.cleanups = append(ss.cleanups, func() { lis.Close() }) s := grpc.NewServer(sopts...) testpb.RegisterTestServiceServer(s, ss.testService) go func() { close(ss.started) s.Serve(lis) }() ss.cleanups = append(ss.cleanups, s.Stop) ss.s = s return nil } // Stop stops ss and cleans up all resources it consumed. func (ss *StubServer) Stop() { <-ss.started for i := len(ss.cleanups) - 1; i >= 0; i-- { ss.cleanups[i]() } } // Addr gets the address the server listening on. func (ss *StubServer) Addr() string { return ss.Address } type dummyStubServer struct { testpb.UnimplementedTestServiceServer counter uint64 } func (d *dummyStubServer) UnaryCall(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { newCount := atomic.AddUint64(&d.counter, 1) return &testpb.SimpleResponse{ Payload: &testpb.Payload{ Type: testpb.PayloadType_COMPRESSABLE, Body: []byte(strconv.FormatUint(newCount, 10)), }, }, nil } // NewDummyStubServer creates a simple test server that serves Unary calls with // responses with the given payload. func NewDummyStubServer(body []byte) *StubServer { return New(&dummyStubServer{}) } ================================================ FILE: pkg/httputil/httputil.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package httputil provides HTTP utility functions. package httputil import ( "io" "net" "net/http" ) // GracefulClose drains http.Response.Body until it hits EOF // and closes it. This prevents TCP/TLS connections from closing, // therefore available for reuse. // Borrowed from golang/net/context/ctxhttp/cancelreq.go. func GracefulClose(resp *http.Response) { io.Copy(io.Discard, resp.Body) resp.Body.Close() } // GetHostname returns the hostname from request Host field. // It returns empty string, if Host field contains invalid // value (e.g. "localhost:::" with too many colons). func GetHostname(req *http.Request) string { if req == nil { return "" } h, _, err := net.SplitHostPort(req.Host) if err != nil { return req.Host } return h } ================================================ FILE: pkg/httputil/httputil_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package httputil import ( "net/http" "testing" "github.com/stretchr/testify/assert" ) func TestGetHostname(t *testing.T) { tt := []struct { req *http.Request host string }{ {&http.Request{Host: "localhost"}, "localhost"}, {&http.Request{Host: "localhost:2379"}, "localhost"}, {&http.Request{Host: "localhost."}, "localhost."}, {&http.Request{Host: "localhost.:2379"}, "localhost."}, {&http.Request{Host: "127.0.0.1"}, "127.0.0.1"}, {&http.Request{Host: "127.0.0.1:2379"}, "127.0.0.1"}, {&http.Request{Host: "localhos"}, "localhos"}, {&http.Request{Host: "localhos:2379"}, "localhos"}, {&http.Request{Host: "localhos."}, "localhos."}, {&http.Request{Host: "localhos.:2379"}, "localhos."}, {&http.Request{Host: "1.2.3.4"}, "1.2.3.4"}, {&http.Request{Host: "1.2.3.4:2379"}, "1.2.3.4"}, // too many colons in address {&http.Request{Host: "localhost:::::"}, "localhost:::::"}, } for i := range tt { hv := GetHostname(tt[i].req) assert.Equalf(t, hv, tt[i].host, "#%d: %q expected host %q, got '%v'", i, tt[i].req.Host, tt[i].host, hv) } } ================================================ FILE: pkg/idutil/id.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package idutil implements utility functions for generating unique, // randomized ids. package idutil import ( "math" "sync/atomic" "time" ) const ( tsLen = 5 * 8 cntLen = 8 suffixLen = tsLen + cntLen ) // Generator generates unique identifiers based on counters, timestamps, and // a node member ID. // // The initial id is in this format: // High order 2 bytes are from memberID, next 5 bytes are from timestamp, // and low order one byte is a counter. // | prefix | suffix | // | 2 bytes | 5 bytes | 1 byte | // | memberID | timestamp | cnt | // // The timestamp 5 bytes is different when the machine is restart // after 1 ms and before 35 years. // // It increases suffix to generate the next id. // The count field may overflow to timestamp field, which is intentional. // It helps to extend the event window to 2^48. This doesn't break that // id generated after restart is unique because etcd throughput is << // 256req/ms(250k reqs/second). type Generator struct { // high order 2 bytes prefix uint64 // low order 6 bytes suffix uint64 } func NewGenerator(memberID uint16, now time.Time) *Generator { prefix := uint64(memberID) << suffixLen unixMilli := uint64(now.UnixNano()) / uint64(time.Millisecond/time.Nanosecond) suffix := lowbit(unixMilli, tsLen) << cntLen return &Generator{ prefix: prefix, suffix: suffix, } } // Next generates a id that is unique. func (g *Generator) Next() uint64 { suffix := atomic.AddUint64(&g.suffix, 1) id := g.prefix | lowbit(suffix, suffixLen) return id } func lowbit(x uint64, n uint) uint64 { return x & (math.MaxUint64 >> (64 - n)) } ================================================ FILE: pkg/idutil/id_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package idutil import ( "testing" "time" "github.com/stretchr/testify/assert" ) func TestNewGenerator(t *testing.T) { g := NewGenerator(0x12, time.Unix(0, 0).Add(0x3456*time.Millisecond)) id := g.Next() wid := uint64(0x12000000345601) assert.Equalf(t, id, wid, "id = %x, want %x", id, wid) } func TestNewGeneratorUnique(t *testing.T) { g := NewGenerator(0, time.Time{}) id := g.Next() // different server generates different ID assert.NotEqualf(t, id, NewGenerator(1, time.Time{}).Next(), "generate the same id %x using different server ID", id) // restarted server generates different ID assert.NotEqualf(t, id, NewGenerator(0, time.Now()).Next(), "generate the same id %x after restart", id) } func TestNext(t *testing.T) { g := NewGenerator(0x12, time.Unix(0, 0).Add(0x3456*time.Millisecond)) wid := uint64(0x12000000345601) for i := 0; i < 1000; i++ { id := g.Next() assert.Equalf(t, id, wid+uint64(i), "id = %x, want %x", id, wid+uint64(i)) } } func BenchmarkNext(b *testing.B) { g := NewGenerator(0x12, time.Now()) b.ResetTimer() for i := 0; i < b.N; i++ { g.Next() } } ================================================ FILE: pkg/ioutil/pagewriter.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ioutil import ( "io" "go.etcd.io/etcd/client/pkg/v3/verify" ) var defaultBufferBytes = 128 * 1024 // PageWriter implements the io.Writer interface so that writes will // either be in page chunks or from flushing. type PageWriter struct { w io.Writer // pageOffset tracks the page offset of the base of the buffer pageOffset int // pageBytes is the number of bytes per page pageBytes int // bufferedBytes counts the number of bytes pending for write in the buffer bufferedBytes int // buf holds the write buffer buf []byte // bufWatermarkBytes is the number of bytes the buffer can hold before it needs // to be flushed. It is less than len(buf) so there is space for slack writes // to bring the writer to page alignment. bufWatermarkBytes int } // NewPageWriter creates a new PageWriter. pageBytes is the number of bytes // to write per page. pageOffset is the starting offset of io.Writer. func NewPageWriter(w io.Writer, pageBytes, pageOffset int) *PageWriter { verify.Assert(pageBytes > 0, "invalid pageBytes (%d) value, it must be greater than 0", pageBytes) return &PageWriter{ w: w, pageOffset: pageOffset, pageBytes: pageBytes, buf: make([]byte, defaultBufferBytes+pageBytes), bufWatermarkBytes: defaultBufferBytes, } } func (pw *PageWriter) Write(p []byte) (n int, err error) { if len(p)+pw.bufferedBytes <= pw.bufWatermarkBytes { // no overflow copy(pw.buf[pw.bufferedBytes:], p) pw.bufferedBytes += len(p) return len(p), nil } // complete the slack page in the buffer if unaligned slack := pw.pageBytes - ((pw.pageOffset + pw.bufferedBytes) % pw.pageBytes) if slack != pw.pageBytes { partial := slack > len(p) if partial { // not enough data to complete the slack page slack = len(p) } // special case: writing to slack page in buffer copy(pw.buf[pw.bufferedBytes:], p[:slack]) pw.bufferedBytes += slack n = slack p = p[slack:] if partial { // avoid forcing an unaligned flush return n, nil } } // buffer contents are now page-aligned; clear out if err = pw.Flush(); err != nil { return n, err } // directly write all complete pages without copying if len(p) > pw.pageBytes { pages := len(p) / pw.pageBytes c, werr := pw.w.Write(p[:pages*pw.pageBytes]) n += c if werr != nil { return n, werr } p = p[pages*pw.pageBytes:] } // write remaining tail to buffer c, werr := pw.Write(p) n += c return n, werr } // Flush flushes buffered data. func (pw *PageWriter) Flush() error { _, err := pw.flush() return err } func (pw *PageWriter) flush() (int, error) { if pw.bufferedBytes == 0 { return 0, nil } n, err := pw.w.Write(pw.buf[:pw.bufferedBytes]) pw.pageOffset = (pw.pageOffset + pw.bufferedBytes) % pw.pageBytes pw.bufferedBytes = 0 return n, err } ================================================ FILE: pkg/ioutil/pagewriter_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ioutil import ( "math/rand" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestPageWriterRandom(t *testing.T) { // smaller buffer for stress testing defaultBufferBytes = 8 * 1024 pageBytes := 128 buf := make([]byte, 4*defaultBufferBytes) cw := &checkPageWriter{pageBytes: pageBytes, t: t} w := NewPageWriter(cw, pageBytes, 0) n := 0 for i := 0; i < 4096; i++ { c, err := w.Write(buf[:rand.Intn(len(buf))]) require.NoError(t, err) n += c } require.LessOrEqualf(t, cw.writeBytes, n, "wrote %d bytes to io.Writer, but only wrote %d bytes", cw.writeBytes, n) maxPendingBytes := pageBytes + defaultBufferBytes require.LessOrEqualf(t, n-cw.writeBytes, maxPendingBytes, "got %d bytes pending, expected less than %d bytes", n-cw.writeBytes, maxPendingBytes) t.Logf("total writes: %d", cw.writes) t.Logf("total write bytes: %d (of %d)", cw.writeBytes, n) } // TestPageWriterPartialSlack tests the case where a write overflows the buffer // but there is not enough data to complete the slack write. func TestPageWriterPartialSlack(t *testing.T) { defaultBufferBytes = 1024 pageBytes := 128 buf := make([]byte, defaultBufferBytes) cw := &checkPageWriter{pageBytes: 64, t: t} w := NewPageWriter(cw, pageBytes, 0) // put writer in non-zero page offset _, err := w.Write(buf[:64]) require.NoError(t, err) require.NoError(t, w.Flush()) require.Equalf(t, 1, cw.writes, "got %d writes, expected 1", cw.writes) // nearly fill buffer _, err = w.Write(buf[:1022]) require.NoError(t, err) // overflow buffer, but without enough to write as aligned _, err = w.Write(buf[:8]) require.NoError(t, err) require.Equalf(t, 1, cw.writes, "got %d writes, expected 1", cw.writes) // finish writing slack space _, err = w.Write(buf[:128]) require.NoError(t, err) require.Equalf(t, 2, cw.writes, "got %d writes, expected 2", cw.writes) } // TestPageWriterOffset tests if page writer correctly repositions when offset is given. func TestPageWriterOffset(t *testing.T) { defaultBufferBytes = 1024 pageBytes := 128 buf := make([]byte, defaultBufferBytes) cw := &checkPageWriter{pageBytes: 64, t: t} w := NewPageWriter(cw, pageBytes, 0) _, err := w.Write(buf[:64]) require.NoError(t, err) require.NoError(t, w.Flush()) require.Equalf(t, 64, w.pageOffset, "w.pageOffset expected 64, got %d", w.pageOffset) w = NewPageWriter(cw, w.pageOffset, pageBytes) _, err = w.Write(buf[:64]) require.NoError(t, err) require.NoError(t, w.Flush()) require.Equalf(t, 0, w.pageOffset, "w.pageOffset expected 0, got %d", w.pageOffset) } func TestPageWriterPageBytes(t *testing.T) { cases := []struct { name string pageBytes int expectPanic bool }{ { name: "normal page bytes", pageBytes: 4096, expectPanic: false, }, { name: "negative page bytes", pageBytes: -1, expectPanic: true, }, { name: "zero page bytes", pageBytes: 0, expectPanic: true, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { defaultBufferBytes = 1024 cw := &checkPageWriter{pageBytes: tc.pageBytes, t: t} if tc.expectPanic { assert.Panicsf(t, func() { NewPageWriter(cw, tc.pageBytes, 0) }, "expected panic when pageBytes is %d", tc.pageBytes) } else { pw := NewPageWriter(cw, tc.pageBytes, 0) assert.NotNil(t, pw) } }) } } // checkPageWriter implements an io.Writer that fails a test on unaligned writes. type checkPageWriter struct { pageBytes int writes int writeBytes int t *testing.T } func (cw *checkPageWriter) Write(p []byte) (int, error) { require.Equalf(cw.t, 0, len(p)%cw.pageBytes, "got write len(p) = %d, expected len(p) == k*cw.pageBytes", len(p)) cw.writes++ cw.writeBytes += len(p) return len(p), nil } ================================================ FILE: pkg/ioutil/readcloser.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ioutil import ( "fmt" "io" ) // ReaderAndCloser implements io.ReadCloser interface by combining // reader and closer together. type ReaderAndCloser struct { io.Reader io.Closer } var ( ErrShortRead = fmt.Errorf("ioutil: short read") ErrExpectEOF = fmt.Errorf("ioutil: expect EOF") ) // NewExactReadCloser returns a ReadCloser that returns errors if the underlying // reader does not read back exactly the requested number of bytes. func NewExactReadCloser(rc io.ReadCloser, totalBytes int64) io.ReadCloser { return &exactReadCloser{rc: rc, totalBytes: totalBytes} } type exactReadCloser struct { rc io.ReadCloser br int64 totalBytes int64 } func (e *exactReadCloser) Read(p []byte) (int, error) { n, err := e.rc.Read(p) e.br += int64(n) if e.br > e.totalBytes { return 0, ErrExpectEOF } if e.br < e.totalBytes && n == 0 { return 0, ErrShortRead } return n, err } func (e *exactReadCloser) Close() error { if err := e.rc.Close(); err != nil { return err } if e.br < e.totalBytes { return ErrShortRead } return nil } ================================================ FILE: pkg/ioutil/readcloser_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ioutil import ( "bytes" "io" "testing" "github.com/stretchr/testify/require" ) type readerNilCloser struct{ io.Reader } func (rc *readerNilCloser) Close() error { return nil } // TestExactReadCloserExpectEOF expects an eof when reading too much. func TestExactReadCloserExpectEOF(t *testing.T) { buf := bytes.NewBuffer(make([]byte, 10)) rc := NewExactReadCloser(&readerNilCloser{buf}, 1) _, err := rc.Read(make([]byte, 10)) require.ErrorIsf(t, err, ErrExpectEOF, "expected %v, got %v", ErrExpectEOF, err) } // TestExactReadCloserShort expects an eof when reading too little func TestExactReadCloserShort(t *testing.T) { buf := bytes.NewBuffer(make([]byte, 5)) rc := NewExactReadCloser(&readerNilCloser{buf}, 10) _, err := rc.Read(make([]byte, 10)) require.NoErrorf(t, err, "Read expected nil err, got %v", err) require.ErrorIs(t, rc.Close(), ErrShortRead) } ================================================ FILE: pkg/ioutil/reader.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package ioutil implements I/O utility functions. package ioutil import "io" // NewLimitedBufferReader returns a reader that reads from the given reader // but limits the amount of data returned to at most n bytes. func NewLimitedBufferReader(r io.Reader, n int) io.Reader { return &limitedBufferReader{ r: r, n: n, } } type limitedBufferReader struct { r io.Reader n int } func (r *limitedBufferReader) Read(p []byte) (n int, err error) { np := p if len(np) > r.n { np = np[:r.n] } return r.r.Read(np) } ================================================ FILE: pkg/ioutil/reader_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ioutil import ( "bytes" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestLimitedBufferReaderRead(t *testing.T) { buf := bytes.NewBuffer(make([]byte, 10)) ln := 1 lr := NewLimitedBufferReader(buf, ln) n, err := lr.Read(make([]byte, 10)) require.NoErrorf(t, err, "unexpected read error: %v", err) assert.Equalf(t, n, ln, "len(data read) = %d, want %d", n, ln) } ================================================ FILE: pkg/ioutil/util.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package ioutil import ( "io" "os" "go.etcd.io/etcd/client/pkg/v3/fileutil" ) // WriteAndSyncFile behaves just like ioutil.WriteFile in the standard library, // but calls Sync before closing the file. WriteAndSyncFile guarantees the data // is synced if there is no error returned. func WriteAndSyncFile(filename string, data []byte, perm os.FileMode) error { f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) if err != nil { return err } n, err := f.Write(data) if err == nil && n < len(data) { err = io.ErrShortWrite } if err == nil { err = fileutil.Fsync(f) } if err1 := f.Close(); err == nil { err = err1 } return err } ================================================ FILE: pkg/netutil/doc.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package netutil implements network-related utility functions. package netutil ================================================ FILE: pkg/netutil/host_normalize.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package netutil import ( "net" "net/url" "strings" ) // urlsHostNormalizedEqual compares two URLs for scheme, normalized host (including IPv6), and path equality. func urlsHostNormalizedEqual(a, b url.URL) bool { return a.Scheme == b.Scheme && normalizeHost(a.Host) == normalizeHost(b.Host) && a.Path == b.Path } // normalizeHost returns the canonical string for the host and normalizes IPv6 and IPv4 addresses. func normalizeHost(host string) string { hostOnly, port, err := net.SplitHostPort(host) if err != nil { hostOnly = host port = "" } // Check if hostOnly is an IPv6 address. It could be with or without brackets. ipStr := strings.Trim(hostOnly, "[]") if ip := net.ParseIP(ipStr); ip != nil { if ip.To4() == nil { // For IPv6 address, always use brackets when there is a port. return "[" + ip.String() + "]" + normalizePort(port) } // IPv4 address return ip.String() + normalizePort(port) } return host } func normalizePort(port string) string { if port == "" { return "" } return ":" + port } ================================================ FILE: pkg/netutil/host_normalize_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package netutil import ( "testing" "github.com/stretchr/testify/assert" "go.uber.org/zap/zaptest" ) func TestIPv6AddressNormalization(t *testing.T) { testCases := []struct { name string urlsA []string urlsB []string expected bool }{ { name: "IPv6 with leading zeros vs without should match", urlsA: []string{"https://[c262:266f:fa53:0ee6:966e:e3f0:d68f:b046]:2380"}, urlsB: []string{"https://[c262:266f:fa53:ee6:966e:e3f0:d68f:b046]:2380"}, expected: true, }, { name: "IPv6 different (lower/upper) case should match", urlsA: []string{"https://[2001:DB8::1]:2380"}, urlsB: []string{"https://[2001:db8::1]:2380"}, expected: true, }, { name: "IPv4 address normalization should still work", urlsA: []string{"http://192.168.1.1:2380"}, urlsB: []string{"http://192.168.1.1:2380"}, expected: true, }, { name: "Different IPv6 addresses should not match", urlsA: []string{"https://[2001:db8::1]:2380"}, urlsB: []string{"https://[2001:db8::2]:2380"}, expected: false, }, { name: "IPv6 without port should match", urlsA: []string{"https://[2001:db8::1]"}, urlsB: []string{"https://[2001:0db8:0:00:000:0000:0:01]"}, expected: true, }, { name: "IPv4 without port should match", urlsA: []string{"http://192.168.1.1"}, urlsB: []string{"http://192.168.1.1"}, expected: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result, err := URLStringsEqual(t.Context(), zaptest.NewLogger(t), tc.urlsA, tc.urlsB) if tc.expected { assert.True(t, result) } else { if err != nil { t.Logf("Got expected error for non-matching URLs: %v", err) } else { assert.False(t, result) } } }) } } func TestNormalizeHostFunction(t *testing.T) { testCases := []struct { name string input string expected string }{ { name: "IPv6 with leading zeros should be normalized", input: "[c262:266f:fa53:0ee6:966e:e3f0:d68f:b046]:2380", expected: "[c262:266f:fa53:ee6:966e:e3f0:d68f:b046]:2380", }, { name: "Compressed IPv6 should remain compressed", input: "[2001:db8::1]:2380", expected: "[2001:db8::1]:2380", }, { name: "IPv6 case should be normalized to lowercase", input: "[2001:DB8::1]:2380", expected: "[2001:db8::1]:2380", }, { name: "IPv4 should remain unchanged", input: "192.168.1.1:2380", expected: "192.168.1.1:2380", }, { name: "Hostname should remain unchanged", input: "example.com:2380", expected: "example.com:2380", }, { name: "IPv6 without port", input: "[2025:db8::1]", expected: "[2025:db8::1]", }, { name: "IPv4 without port", input: "192.168.1.1", expected: "192.168.1.1", }, { name: "Hostname without port", input: "example.com", expected: "example.com", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := normalizeHost(tc.input) assert.Equal(t, tc.expected, result) }) } } ================================================ FILE: pkg/netutil/netutil.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package netutil import ( "context" "errors" "fmt" "net" "net/url" "sort" "time" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/types" ) // indirection for testing var resolveTCPAddr = resolveTCPAddrDefault const retryInterval = time.Second // taken from go's ResolveTCP code but uses configurable ctx func resolveTCPAddrDefault(ctx context.Context, addr string) (*net.TCPAddr, error) { host, port, serr := net.SplitHostPort(addr) if serr != nil { return nil, serr } portnum, perr := net.DefaultResolver.LookupPort(ctx, "tcp", port) if perr != nil { return nil, perr } var ips []net.IPAddr if ip := net.ParseIP(host); ip != nil { ips = []net.IPAddr{{IP: ip}} } else { // Try as a DNS name. ipss, err := net.DefaultResolver.LookupIPAddr(ctx, host) if err != nil { return nil, err } ips = ipss } // randomize? ip := ips[0] return &net.TCPAddr{IP: ip.IP, Port: portnum, Zone: ip.Zone}, nil } // resolveTCPAddrs is a convenience wrapper for net.ResolveTCPAddr. // resolveTCPAddrs return a new set of url.URLs, in which all DNS hostnames // are resolved. func resolveTCPAddrs(ctx context.Context, lg *zap.Logger, urls [][]url.URL) ([][]url.URL, error) { newurls := make([][]url.URL, 0) for _, us := range urls { nus := make([]url.URL, len(us)) for i, u := range us { nu, err := url.Parse(u.String()) if err != nil { return nil, fmt.Errorf("failed to parse %q (%w)", u.String(), err) } nus[i] = *nu } for i, u := range nus { h, err := resolveURL(ctx, lg, u) if err != nil { return nil, fmt.Errorf("failed to resolve %q (%w)", u.String(), err) } if h != "" { nus[i].Host = h } } newurls = append(newurls, nus) } return newurls, nil } func resolveURL(ctx context.Context, lg *zap.Logger, u url.URL) (string, error) { if u.Scheme == "unix" || u.Scheme == "unixs" { // unix sockets don't resolve over TCP return "", nil } host, _, err := net.SplitHostPort(u.Host) if err != nil { lg.Warn( "failed to parse URL Host while resolving URL", zap.String("url", u.String()), zap.String("host", u.Host), zap.Error(err), ) return "", err } if host == "localhost" { return "", nil } for ctx.Err() == nil { tcpAddr, err := resolveTCPAddr(ctx, u.Host) if err == nil { lg.Info( "resolved URL Host", zap.String("url", u.String()), zap.String("host", u.Host), zap.String("resolved-addr", tcpAddr.String()), ) return tcpAddr.String(), nil } lg.Warn( "failed to resolve URL Host", zap.String("url", u.String()), zap.String("host", u.Host), zap.Duration("retry-interval", retryInterval), zap.Error(err), ) select { case <-ctx.Done(): lg.Warn( "failed to resolve URL Host; returning", zap.String("url", u.String()), zap.String("host", u.Host), zap.Duration("retry-interval", retryInterval), zap.Error(err), ) return "", err case <-time.After(retryInterval): } } return "", ctx.Err() } // urlsEqual checks equality of url.URLS between two arrays. // This check pass even if an URL is in hostname and opposite is in IP address. func urlsEqual(ctx context.Context, lg *zap.Logger, a []url.URL, b []url.URL) (bool, error) { if len(a) != len(b) { return false, fmt.Errorf("len(%q) != len(%q)", urlsToStrings(a), urlsToStrings(b)) } sort.Sort(types.URLs(a)) sort.Sort(types.URLs(b)) var needResolve bool for i := range a { if !urlsHostNormalizedEqual(a[i], b[i]) { needResolve = true break } } if !needResolve { return true, nil } // If URLs are not equal, try to resolve it and compare again. urls, err := resolveTCPAddrs(ctx, lg, [][]url.URL{a, b}) if err != nil { return false, err } a, b = urls[0], urls[1] sort.Sort(types.URLs(a)) sort.Sort(types.URLs(b)) for i := range a { if !urlsHostNormalizedEqual(a[i], b[i]) { return false, fmt.Errorf("resolved urls: %q != %q", a[i].String(), b[i].String()) } } return true, nil } // URLStringsEqual returns "true" if given URLs are valid // and resolved to same IP addresses. Otherwise, return "false" // and error, if any. func URLStringsEqual(ctx context.Context, lg *zap.Logger, a []string, b []string) (bool, error) { if len(a) != len(b) { return false, fmt.Errorf("len(%q) != len(%q)", a, b) } urlsA, err := stringsToURLs(a) if err != nil { return false, err } urlsB, err := stringsToURLs(b) if err != nil { return false, err } return urlsEqual(ctx, lg, urlsA, urlsB) } func urlsToStrings(us []url.URL) []string { rs := make([]string, len(us)) for i := range us { rs[i] = us[i].String() } return rs } func stringsToURLs(us []string) ([]url.URL, error) { urls := make([]url.URL, 0, len(us)) for _, str := range us { u, err := url.Parse(str) if err != nil { return nil, fmt.Errorf("failed to parse string to URL: %q", str) } urls = append(urls, *u) } return urls, nil } func IsNetworkTimeoutError(err error) bool { var nerr net.Error return errors.As(err, &nerr) && nerr.Timeout() } ================================================ FILE: pkg/netutil/netutil_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package netutil import ( "context" "errors" "fmt" "net" "net/url" "reflect" "strconv" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" ) func TestResolveTCPAddrs(t *testing.T) { defer func() { resolveTCPAddr = resolveTCPAddrDefault }() tests := []struct { urls [][]url.URL expected [][]url.URL hostMap map[string]string hasError bool }{ { urls: [][]url.URL{ { {Scheme: "http", Host: "127.0.0.1:4001"}, {Scheme: "http", Host: "127.0.0.1:2379"}, }, { {Scheme: "http", Host: "127.0.0.1:7001"}, {Scheme: "http", Host: "127.0.0.1:2380"}, }, }, expected: [][]url.URL{ { {Scheme: "http", Host: "127.0.0.1:4001"}, {Scheme: "http", Host: "127.0.0.1:2379"}, }, { {Scheme: "http", Host: "127.0.0.1:7001"}, {Scheme: "http", Host: "127.0.0.1:2380"}, }, }, }, { urls: [][]url.URL{ { {Scheme: "http", Host: "infra0.example.com:4001"}, {Scheme: "http", Host: "infra0.example.com:2379"}, }, { {Scheme: "http", Host: "infra0.example.com:7001"}, {Scheme: "http", Host: "infra0.example.com:2380"}, }, }, expected: [][]url.URL{ { {Scheme: "http", Host: "10.0.1.10:4001"}, {Scheme: "http", Host: "10.0.1.10:2379"}, }, { {Scheme: "http", Host: "10.0.1.10:7001"}, {Scheme: "http", Host: "10.0.1.10:2380"}, }, }, hostMap: map[string]string{ "infra0.example.com": "10.0.1.10", }, hasError: false, }, { urls: [][]url.URL{ { {Scheme: "http", Host: "infra0.example.com:4001"}, {Scheme: "http", Host: "infra0.example.com:2379"}, }, { {Scheme: "http", Host: "infra0.example.com:7001"}, {Scheme: "http", Host: "infra0.example.com:2380"}, }, }, hostMap: map[string]string{ "infra0.example.com": "", }, hasError: true, }, { urls: [][]url.URL{ { {Scheme: "http", Host: "ssh://infra0.example.com:4001"}, {Scheme: "http", Host: "ssh://infra0.example.com:2379"}, }, { {Scheme: "http", Host: "ssh://infra0.example.com:7001"}, {Scheme: "http", Host: "ssh://infra0.example.com:2380"}, }, }, hasError: true, }, } for _, tt := range tests { resolveTCPAddr = func(ctx context.Context, addr string) (*net.TCPAddr, error) { host, port, err := net.SplitHostPort(addr) if err != nil { return nil, err } i, err := strconv.Atoi(port) if err != nil { return nil, err } if ip := net.ParseIP(host); ip != nil { return &net.TCPAddr{IP: ip, Port: i, Zone: ""}, nil } if tt.hostMap[host] == "" { return nil, errors.New("cannot resolve host") } return &net.TCPAddr{IP: net.ParseIP(tt.hostMap[host]), Port: i, Zone: ""}, nil } ctx, cancel := context.WithTimeout(t.Context(), time.Second) urls, err := resolveTCPAddrs(ctx, zaptest.NewLogger(t), tt.urls) cancel() if tt.hasError { require.Errorf(t, err, "expected error") continue } assert.Truef(t, reflect.DeepEqual(urls, tt.expected), "expected: %v, got %v", tt.expected, urls) } } func TestURLsEqual(t *testing.T) { defer func() { resolveTCPAddr = resolveTCPAddrDefault }() hostm := map[string]string{ "example.com": "10.0.10.1", "first.com": "10.0.11.1", "second.com": "10.0.11.2", } resolveTCPAddr = func(ctx context.Context, addr string) (*net.TCPAddr, error) { host, port, err := net.SplitHostPort(addr) if err != nil { return nil, err } i, err := strconv.Atoi(port) if err != nil { return nil, err } if ip := net.ParseIP(host); ip != nil { return &net.TCPAddr{IP: ip, Port: i, Zone: ""}, nil } if hostm[host] == "" { return nil, errors.New("cannot resolve host") } return &net.TCPAddr{IP: net.ParseIP(hostm[host]), Port: i, Zone: ""}, nil } tests := []struct { n int a []url.URL b []url.URL expect bool err error }{ { n: 0, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}}, b: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}}, expect: true, }, { n: 1, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.10.1:2379"}}, expect: true, }, { n: 2, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}}, b: []url.URL{{Scheme: "https", Host: "10.0.10.1:2379"}}, expect: false, err: errors.New(`resolved urls: "http://10.0.10.1:2379" != "https://10.0.10.1:2379"`), }, { n: 3, a: []url.URL{{Scheme: "https", Host: "example.com:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.10.1:2379"}}, expect: false, err: errors.New(`resolved urls: "https://10.0.10.1:2379" != "http://10.0.10.1:2379"`), }, { n: 4, a: []url.URL{{Scheme: "unix", Host: "abc:2379"}}, b: []url.URL{{Scheme: "unix", Host: "abc:2379"}}, expect: true, }, { n: 5, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: true, }, { n: 6, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: true, }, { n: 7, a: []url.URL{{Scheme: "http", Host: "10.0.10.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: true, }, { n: 8, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}}, b: []url.URL{{Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, err: errors.New(`resolved urls: "http://127.0.0.1:2379" != "http://127.0.0.1:2380"`), }, { n: 9, a: []url.URL{{Scheme: "http", Host: "example.com:2380"}}, b: []url.URL{{Scheme: "http", Host: "10.0.10.1:2379"}}, expect: false, err: errors.New(`resolved urls: "http://10.0.10.1:2380" != "http://10.0.10.1:2379"`), }, { n: 10, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}}, expect: false, err: errors.New(`resolved urls: "http://127.0.0.1:2379" != "http://10.0.0.1:2379"`), }, { n: 11, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}}, expect: false, err: errors.New(`resolved urls: "http://10.0.10.1:2379" != "http://10.0.0.1:2379"`), }, { n: 12, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "127.0.0.1:2380"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, err: errors.New(`resolved urls: "http://127.0.0.1:2379" != "http://127.0.0.1:2380"`), }, { n: 13, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "127.0.0.1:2380"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, err: errors.New(`resolved urls: "http://10.0.10.1:2379" != "http://127.0.0.1:2380"`), }, { n: 14, a: []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, err: errors.New(`resolved urls: "http://127.0.0.1:2379" != "http://10.0.0.1:2379"`), }, { n: 15, a: []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, b: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, err: errors.New(`resolved urls: "http://10.0.10.1:2379" != "http://10.0.0.1:2379"`), }, { n: 16, a: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}}, expect: false, err: errors.New(`len(["http://10.0.0.1:2379"]) != len(["http://10.0.0.1:2379" "http://127.0.0.1:2380"])`), }, { n: 17, a: []url.URL{{Scheme: "http", Host: "first.com:2379"}, {Scheme: "http", Host: "second.com:2380"}}, b: []url.URL{{Scheme: "http", Host: "10.0.11.1:2379"}, {Scheme: "http", Host: "10.0.11.2:2380"}}, expect: true, }, { n: 18, a: []url.URL{{Scheme: "http", Host: "second.com:2380"}, {Scheme: "http", Host: "first.com:2379"}}, b: []url.URL{{Scheme: "http", Host: "10.0.11.1:2379"}, {Scheme: "http", Host: "10.0.11.2:2380"}}, expect: true, }, } for i, test := range tests { result, err := urlsEqual(t.Context(), zaptest.NewLogger(t), test.a, test.b) assert.Equalf(t, result, test.expect, "idx=%d #%d: a:%v b:%v, expected %v but %v", i, test.n, test.a, test.b, test.expect, result) if test.err != nil { if err.Error() != test.err.Error() { t.Errorf("idx=%d #%d: err expected %v but %v", i, test.n, test.err, err) } } } } func TestURLStringsEqual(t *testing.T) { defer func() { resolveTCPAddr = resolveTCPAddrDefault }() errOnResolve := func(ctx context.Context, addr string) (*net.TCPAddr, error) { return nil, fmt.Errorf("unexpected attempt to resolve: %q", addr) } cases := []struct { urlsA []string urlsB []string resolver func(ctx context.Context, addr string) (*net.TCPAddr, error) }{ {[]string{"http://127.0.0.1:8080"}, []string{"http://127.0.0.1:8080"}, resolveTCPAddrDefault}, {[]string{ "http://host1:8080", "http://host2:8080", }, []string{ "http://host1:8080", "http://host2:8080", }, errOnResolve}, { urlsA: []string{"https://[c262:266f:fa53:0ee6:966e:e3f0:d68f:b046]:2380"}, urlsB: []string{"https://[c262:266f:fa53:ee6:966e:e3f0:d68f:b046]:2380"}, resolver: resolveTCPAddrDefault, }, } for idx, c := range cases { t.Logf("TestURLStringsEqual, case #%d", idx) resolveTCPAddr = c.resolver result, err := URLStringsEqual(t.Context(), zaptest.NewLogger(t), c.urlsA, c.urlsB) assert.Truef(t, result, "unexpected result %v", result) assert.NoErrorf(t, err, "unexpected error %v", err) } } ================================================ FILE: pkg/netutil/routes.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !linux package netutil import ( "fmt" "runtime" ) // GetDefaultHost fetches the a resolvable name that corresponds // to the machine's default routable interface func GetDefaultHost() (string, error) { return "", fmt.Errorf("default host not supported on %s_%s", runtime.GOOS, runtime.GOARCH) } // GetDefaultInterfaces fetches the device name of default routable interface. func GetDefaultInterfaces() (map[string]uint8, error) { return nil, fmt.Errorf("default host not supported on %s_%s", runtime.GOOS, runtime.GOARCH) } ================================================ FILE: pkg/netutil/routes_linux.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build linux package netutil import ( "bytes" "encoding/binary" "fmt" "net" "slices" "syscall" "go.etcd.io/etcd/pkg/v3/cpuutil" ) var ( errNoDefaultRoute = fmt.Errorf("could not find default route") errNoDefaultHost = fmt.Errorf("could not find default host") errNoDefaultInterface = fmt.Errorf("could not find default interface") ) // GetDefaultHost obtains the first IP address of machine from the routing table and returns the IP address as string. // An IPv4 address is preferred to an IPv6 address for backward compatibility. func GetDefaultHost() (string, error) { rmsgs, rerr := getDefaultRoutes() if rerr != nil { return "", rerr } // prioritize IPv4 if rmsg, ok := rmsgs[syscall.AF_INET]; ok { if host, err := chooseHost(syscall.AF_INET, rmsg); host != "" || err != nil { return host, err } delete(rmsgs, syscall.AF_INET) } // sort so choice is deterministic var families []uint8 for family := range rmsgs { families = append(families, family) } slices.Sort(families) for _, family := range families { if host, err := chooseHost(family, rmsgs[family]); host != "" || err != nil { return host, err } } return "", errNoDefaultHost } func chooseHost(family uint8, rmsg *syscall.NetlinkMessage) (string, error) { host, oif, err := parsePREFSRC(rmsg) if host != "" || err != nil { return host, err } // prefsrc not detected, fall back to getting address from iface ifmsg, ierr := getIfaceAddr(oif, family) if ierr != nil { return "", ierr } attrs, aerr := syscall.ParseNetlinkRouteAttr(ifmsg) if aerr != nil { return "", aerr } for _, attr := range attrs { // search for RTA_DST because ipv6 doesn't have RTA_SRC if attr.Attr.Type == syscall.RTA_DST { return net.IP(attr.Value).String(), nil } } return "", nil } func getDefaultRoutes() (map[uint8]*syscall.NetlinkMessage, error) { dat, err := syscall.NetlinkRIB(syscall.RTM_GETROUTE, syscall.AF_UNSPEC) if err != nil { return nil, err } msgs, msgErr := syscall.ParseNetlinkMessage(dat) if msgErr != nil { return nil, msgErr } routes := make(map[uint8]*syscall.NetlinkMessage) rtmsg := syscall.RtMsg{} for _, m := range msgs { if m.Header.Type != syscall.RTM_NEWROUTE { continue } buf := bytes.NewBuffer(m.Data[:syscall.SizeofRtMsg]) if rerr := binary.Read(buf, cpuutil.ByteOrder(), &rtmsg); rerr != nil { continue } if rtmsg.Dst_len == 0 && rtmsg.Table == syscall.RT_TABLE_MAIN { // zero-length Dst_len implies default route msg := m routes[rtmsg.Family] = &msg } } if len(routes) > 0 { return routes, nil } return nil, errNoDefaultRoute } // Used to get an address of interface. func getIfaceAddr(idx uint32, family uint8) (*syscall.NetlinkMessage, error) { dat, err := syscall.NetlinkRIB(syscall.RTM_GETADDR, int(family)) if err != nil { return nil, err } msgs, msgErr := syscall.ParseNetlinkMessage(dat) if msgErr != nil { return nil, msgErr } ifaddrmsg := syscall.IfAddrmsg{} for _, m := range msgs { if m.Header.Type != syscall.RTM_NEWADDR { continue } buf := bytes.NewBuffer(m.Data[:syscall.SizeofIfAddrmsg]) if rerr := binary.Read(buf, cpuutil.ByteOrder(), &ifaddrmsg); rerr != nil { continue } if ifaddrmsg.Index == idx { return &m, nil } } return nil, fmt.Errorf("could not find address for interface index %v", idx) } // Used to get a name of interface. func getIfaceLink(idx uint32) (*syscall.NetlinkMessage, error) { dat, err := syscall.NetlinkRIB(syscall.RTM_GETLINK, syscall.AF_UNSPEC) if err != nil { return nil, err } msgs, msgErr := syscall.ParseNetlinkMessage(dat) if msgErr != nil { return nil, msgErr } ifinfomsg := syscall.IfInfomsg{} for _, m := range msgs { if m.Header.Type != syscall.RTM_NEWLINK { continue } buf := bytes.NewBuffer(m.Data[:syscall.SizeofIfInfomsg]) if rerr := binary.Read(buf, cpuutil.ByteOrder(), &ifinfomsg); rerr != nil { continue } if ifinfomsg.Index == int32(idx) { return &m, nil } } return nil, fmt.Errorf("could not find link for interface index %v", idx) } // GetDefaultInterfaces gets names of interfaces and returns a map[interface]families. func GetDefaultInterfaces() (map[string]uint8, error) { interfaces := make(map[string]uint8) rmsgs, rerr := getDefaultRoutes() if rerr != nil { return interfaces, rerr } for family, rmsg := range rmsgs { _, oif, err := parsePREFSRC(rmsg) if err != nil { return interfaces, err } ifmsg, ierr := getIfaceLink(oif) if ierr != nil { return interfaces, ierr } attrs, aerr := syscall.ParseNetlinkRouteAttr(ifmsg) if aerr != nil { return interfaces, aerr } for _, attr := range attrs { if attr.Attr.Type == syscall.IFLA_IFNAME { // key is an interface name // possible values: 2 - AF_INET, 10 - AF_INET6, 12 - dualstack interfaces[string(attr.Value[:len(attr.Value)-1])] += family } } } if len(interfaces) > 0 { return interfaces, nil } return interfaces, errNoDefaultInterface } // parsePREFSRC returns preferred source address and output interface index (RTA_OIF). func parsePREFSRC(m *syscall.NetlinkMessage) (host string, oif uint32, err error) { var attrs []syscall.NetlinkRouteAttr attrs, err = syscall.ParseNetlinkRouteAttr(m) if err != nil { return "", 0, err } for _, attr := range attrs { if attr.Attr.Type == syscall.RTA_PREFSRC { host = net.IP(attr.Value).String() } if attr.Attr.Type == syscall.RTA_OIF { oif = cpuutil.ByteOrder().Uint32(attr.Value) } if host != "" && oif != uint32(0) { break } } if oif == 0 { err = errNoDefaultRoute } return host, oif, err } ================================================ FILE: pkg/netutil/routes_linux_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build linux package netutil import ( "testing" "github.com/stretchr/testify/require" ) func TestGetDefaultInterface(t *testing.T) { ifc, err := GetDefaultInterfaces() require.NoError(t, err) t.Logf("default network interfaces: %+v\n", ifc) } func TestGetDefaultHost(t *testing.T) { ip, err := GetDefaultHost() require.NoError(t, err) t.Logf("default ip: %v", ip) } ================================================ FILE: pkg/notify/notify.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package notify import ( "sync" ) // Notifier is a thread safe struct that can be used to send notification about // some event to multiple consumers. type Notifier struct { mu sync.RWMutex channel chan struct{} } // NewNotifier returns new notifier func NewNotifier() *Notifier { return &Notifier{ channel: make(chan struct{}), } } // Receive returns channel that can be used to wait for notification. // Consumers will be informed by closing the channel. func (n *Notifier) Receive() <-chan struct{} { n.mu.RLock() defer n.mu.RUnlock() return n.channel } // Notify closes the channel passed to consumers and creates new channel to used // for next notification. func (n *Notifier) Notify() { newChannel := make(chan struct{}) n.mu.Lock() channelToClose := n.channel n.channel = newChannel n.mu.Unlock() close(channelToClose) } ================================================ FILE: pkg/osutil/interrupt_unix.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows && !plan9 package osutil import ( "os" "os/signal" "sync" "syscall" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/verify" ) // InterruptHandler is a function that is called on receiving a // SIGTERM or SIGINT signal. type InterruptHandler func() var ( interruptRegisterMu, interruptExitMu sync.Mutex // interruptHandlers holds all registered InterruptHandlers in order // they will be executed. interruptHandlers []InterruptHandler ) // RegisterInterruptHandler registers a new InterruptHandler. Handlers registered // after interrupt handing was initiated will not be executed. func RegisterInterruptHandler(h InterruptHandler) { interruptRegisterMu.Lock() defer interruptRegisterMu.Unlock() interruptHandlers = append(interruptHandlers, h) } // HandleInterrupts calls the handler functions on receiving a SIGINT or SIGTERM. func HandleInterrupts(lg *zap.Logger) { verify.Assert(lg != nil, "the logger should not be nil") notifier := make(chan os.Signal, 1) signal.Notify(notifier, syscall.SIGINT, syscall.SIGTERM) go func() { sig := <-notifier interruptRegisterMu.Lock() ihs := make([]InterruptHandler, len(interruptHandlers)) copy(ihs, interruptHandlers) interruptRegisterMu.Unlock() interruptExitMu.Lock() lg.Info("received signal; shutting down", zap.String("signal", sig.String())) for _, h := range ihs { h() } signal.Stop(notifier) pid := syscall.Getpid() // exit directly if it is the "init" process, since the kernel will not help to kill pid 1. if pid == 1 { os.Exit(0) } setDflSignal(sig.(syscall.Signal)) syscall.Kill(pid, sig.(syscall.Signal)) }() } // Exit relays to os.Exit if no interrupt handlers are running, blocks otherwise. func Exit(code int) { interruptExitMu.Lock() os.Exit(code) } ================================================ FILE: pkg/osutil/interrupt_windows.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows package osutil import ( "os" "go.uber.org/zap" ) type InterruptHandler func() // RegisterInterruptHandler is a no-op on windows func RegisterInterruptHandler(h InterruptHandler) {} // HandleInterrupts is a no-op on windows func HandleInterrupts(*zap.Logger) {} // Exit calls os.Exit func Exit(code int) { os.Exit(code) } ================================================ FILE: pkg/osutil/osutil.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package osutil implements operating system-related utility functions. package osutil // support to override setting SIG_DFL so tests don't terminate early var setDflSignal = dflSignal ================================================ FILE: pkg/osutil/osutil_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package osutil import ( "os" "os/signal" "syscall" "testing" "time" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" ) func init() { setDflSignal = func(syscall.Signal) {} } func waitSig(t *testing.T, c <-chan os.Signal, sig os.Signal) { select { case s := <-c: require.Equalf(t, s, sig, "signal was %v, want %v", s, sig) case <-time.After(1 * time.Second): t.Fatalf("timeout waiting for %v", sig) } } func TestHandleInterrupts(t *testing.T) { for _, sig := range []syscall.Signal{syscall.SIGINT, syscall.SIGTERM} { n := 1 RegisterInterruptHandler(func() { n++ }) RegisterInterruptHandler(func() { n *= 2 }) c := make(chan os.Signal, 2) signal.Notify(c, sig) HandleInterrupts(zaptest.NewLogger(t)) syscall.Kill(syscall.Getpid(), sig) // we should receive the signal once from our own kill and // a second time from HandleInterrupts waitSig(t, c, sig) waitSig(t, c, sig) require.NotEqualf(t, 3, n, "interrupt handlers were called in wrong order") require.Equalf(t, 4, n, "interrupt handlers were not called properly") // reset interrupt handlers interruptHandlers = interruptHandlers[:0] interruptExitMu.Unlock() } } ================================================ FILE: pkg/osutil/signal.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !linux package osutil import "syscall" func dflSignal(sig syscall.Signal) { /* nop */ } ================================================ FILE: pkg/osutil/signal_linux.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build linux package osutil import ( "syscall" "unsafe" ) // dflSignal sets the given signal to SIG_DFL func dflSignal(sig syscall.Signal) { // clearing out the sigact sets the signal to SIG_DFL var sigactBuf [32]uint64 ptr := unsafe.Pointer(&sigactBuf) syscall.Syscall6(uintptr(syscall.SYS_RT_SIGACTION), uintptr(sig), uintptr(ptr), 0, 8, 0, 0) } ================================================ FILE: pkg/pbutil/pbutil.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package pbutil defines interfaces for handling Protocol Buffer objects. package pbutil import "fmt" type Marshaler interface { Marshal() (data []byte, err error) } type Unmarshaler interface { Unmarshal(data []byte) error } func MustMarshal(m Marshaler) []byte { d, err := m.Marshal() if err != nil { panic(fmt.Sprintf("marshal should never fail (%v)", err)) } return d } func MustUnmarshal(um Unmarshaler, data []byte) { if err := um.Unmarshal(data); err != nil { panic(fmt.Sprintf("unmarshal should never fail (%v)", err)) } } func MaybeUnmarshal(um Unmarshaler, data []byte) bool { if err := um.Unmarshal(data); err != nil { return false } return true } func GetBool(v *bool) (vv bool, set bool) { if v == nil { return false, false } return *v, true } func Boolp(b bool) *bool { return &b } ================================================ FILE: pkg/pbutil/pbutil_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pbutil import ( "errors" "reflect" "testing" "github.com/stretchr/testify/assert" ) func TestMarshaler(t *testing.T) { data := []byte("test data") m := &fakeMarshaler{data: data} g := MustMarshal(m) assert.Truef(t, reflect.DeepEqual(g, data), "data = %s, want %s", g, m.data) } func TestMarshalerPanic(t *testing.T) { defer func() { assert.NotNilf(t, recover(), "recover = nil, want error") }() m := &fakeMarshaler{err: errors.New("blah")} MustMarshal(m) } func TestUnmarshaler(t *testing.T) { data := []byte("test data") m := &fakeUnmarshaler{} MustUnmarshal(m, data) assert.Truef(t, reflect.DeepEqual(m.data, data), "data = %s, want %s", m.data, data) } func TestUnmarshalerPanic(t *testing.T) { defer func() { assert.NotNilf(t, recover(), "recover = nil, want error") }() m := &fakeUnmarshaler{err: errors.New("blah")} MustUnmarshal(m, nil) } func TestGetBool(t *testing.T) { tests := []struct { b *bool wb bool wset bool }{ {nil, false, false}, {Boolp(true), true, true}, {Boolp(false), false, true}, } for i, tt := range tests { b, set := GetBool(tt.b) assert.Equalf(t, b, tt.wb, "#%d: value = %v, want %v", i, b, tt.wb) assert.Equalf(t, set, tt.wset, "#%d: set = %v, want %v", i, set, tt.wset) } } type fakeMarshaler struct { data []byte err error } func (m *fakeMarshaler) Marshal() ([]byte, error) { return m.data, m.err } type fakeUnmarshaler struct { data []byte err error } func (m *fakeUnmarshaler) Unmarshal(data []byte) error { m.data = data return m.err } ================================================ FILE: pkg/proxy/doc.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package proxy implements proxy servers for network fault testing. package proxy ================================================ FILE: pkg/proxy/fixtures/ca-csr.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "ca", "ca": { "expiry": "87600h" } } ================================================ FILE: pkg/proxy/fixtures/ca.crt ================================================ -----BEGIN CERTIFICATE----- MIIDsTCCApmgAwIBAgIUZzOo4zcHY/nEXY1PD8A7povXlWUwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMDIxNjQxMDBaFw0yNzEyMzExNjQx MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQDD4Ys48LDWGyojj3Rcr6fnESY+UycaaGoTXADWLPmm+sQR3KcsJxF4054S d2G+NBfJHZvTHhVqOeqZxNtoqgje4paY2A5TbWBdV+xoGfbakwwngiX1yeF1I54k KH19zb8rBKAm7xixO60hE2CIYzMuw9lDkwoHpI6/PJdy7jwtytbo2Oac512JiO9Y dHp9dr3mrCzoKEBRtL1asRKfzp6gBC5rIw5T4jrq37feerV4pDEJX7fvexxVocVm tT4bmMq3Ap6OFFAzmE/ITI8pXvFaOd9lyebNXQmrreKJLUfEIZa6JulLCYxfkJ8z +CcNLyn6ZXNMaIZ8G9Hm6VRdRi8/AgMBAAGjRTBDMA4GA1UdDwEB/wQEAwIBBjAS BgNVHRMBAf8ECDAGAQH/AgECMB0GA1UdDgQWBBRDLNYEX8XI7nM53k1rUR+mpTjQ NTANBgkqhkiG9w0BAQsFAAOCAQEACDe3Fa1KE/rvVtyCLW/IBfKV01NShFTsb6x8 GrPEQ6NJLZQ2MzdyJgAF2a/nZ9KVgrhGXoyoZBCKP9Dd/JDzSSZcBztfNK8dRv2A XHBBF6tZ19I+XY9c7/CfhJ2CEYJpeN9r3GKSqV+njkmg8n/On2BTlFsij88plK8H ORyemc1nQI+ARPSu2r3rJbYa4yI2U6w4L4BTCVImg3bX50GImmXGlwvnJMFik1FX +0hdfetRxxMZ1pm2Uy6099KkULnSKabZGwRiBUHQJYh0EeuAOQ4a6MG5DRkURWNs dInjPOLY9/7S5DQKwz/NtqXA8EEymZosHxpiRp+zzKB4XaV9Ig== -----END CERTIFICATE----- ================================================ FILE: pkg/proxy/fixtures/gencert.json ================================================ { "signing": { "default": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "87600h" } } } ================================================ FILE: pkg/proxy/fixtures/gencerts.sh ================================================ #!/bin/bash set -euo pipefail if ! [[ "$0" =~ "./gencerts.sh" ]]; then echo "must be run from 'fixtures'" exit 255 fi if ! command -v cfssl; then echo "cfssl is not installed" echo 'use: bash -c "cd ../../../tools/mod; go install github.com/cloudflare/cfssl/cmd/cfssl"' exit 255 fi if ! command -v cfssljson; then echo "cfssljson is not installed" echo 'use: bash -c "cd ../../../tools/mod; go install github.com/cloudflare/cfssl/cmd/cfssljson"' exit 255 fi cfssl gencert --initca=true ./ca-csr.json | cfssljson --bare ./ca mv ca.pem ca.crt openssl x509 -in ca.crt -noout -text # generate DNS: localhost, IP: 127.0.0.1, CN: example.com certificates cfssl gencert \ --ca ./ca.crt \ --ca-key ./ca-key.pem \ --config ./gencert.json \ ./server-ca-csr.json | cfssljson --bare ./server mv server.pem server.crt mv server-key.pem server.key.insecure rm -f *.csr *.pem *.stderr *.txt ================================================ FILE: pkg/proxy/fixtures/server-ca-csr.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "example.com", "hosts": [ "127.0.0.1", "localhost" ] } ================================================ FILE: pkg/proxy/fixtures/server.crt ================================================ -----BEGIN CERTIFICATE----- MIIEEjCCAvqgAwIBAgIUIYc+vmysep1pDc2ua/VQEeMFQVAwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMDIxNjQxMDBaFw0yNzEyMzExNjQx MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQDEq7aT2BQZfmJ2xpUm8xWJlN0c3cOLVZRH9mIrEutIHmip BYq3ZIq3q52w+T3sMcaJNMGjCteE8Lu+G9YSmtfZMAWnkaM02KOjVMkkQcK7Z4vM lOUjlO+dsvhfmw3CPghqSs6M1K2CTqhuEiXdOBofuEMmwKNRgkV/jT92PUs0h8kq loc/I3/H+hx/ZJ1i0S0xkZKpaImc0oZ9ZDo07biMrsUIzjwbN69mEs+CtVkah4sy k6UyRoU2k21lyRTK0LxNjWc9ylzDNUuf6DwduU7lPZsqTaJrFNAAPpOlI4k2EcjL 3zD8amKkJGDm+PQz97PbTA381ec4ZAtB8volxCebAgMBAAGjgZwwgZkwDgYDVR0P AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB Af8EAjAAMB0GA1UdDgQWBBTTZQnMn5tuUgVE+8c9W0hmbghGoDAfBgNVHSMEGDAW gBRDLNYEX8XI7nM53k1rUR+mpTjQNTAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A AAEwDQYJKoZIhvcNAQELBQADggEBAKUQVj0YDuxg4tinlOZhp4ge7tCA+gL7vV+Q iDrkWfOlGjDgwYqWMYDXMHWKIW9ea8LzyI/bVEcaHlnBmNOYuS7g47EWNiU7WUA5 iTkm3CKA5zHFFPcXHW0GQeCQrX9y3SepKS3cP8TAyZFfC/FvV24Kn1oQhJbEe0ZV In/vPHssW7jlVe0FGVUn7FutRQgiA1pTAtS6AP4LeZ9O41DTWkPqV4nBgcxlvkgD KjEoXXSb5C0LoR5zwAo9zB3RtmqnmvkHAOv3G92YctdS2VbCmd8CNLj9H7gMmQiH ThsStVOhb2uo6Ni4PgzUIYKGTd4ZjUXCYxFKck//ajDyCHlL8v4= -----END CERTIFICATE----- ================================================ FILE: pkg/proxy/fixtures/server.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEogIBAAKCAQEAxKu2k9gUGX5idsaVJvMViZTdHN3Di1WUR/ZiKxLrSB5oqQWK t2SKt6udsPk97DHGiTTBowrXhPC7vhvWEprX2TAFp5GjNNijo1TJJEHCu2eLzJTl I5TvnbL4X5sNwj4IakrOjNStgk6obhIl3TgaH7hDJsCjUYJFf40/dj1LNIfJKpaH PyN/x/ocf2SdYtEtMZGSqWiJnNKGfWQ6NO24jK7FCM48GzevZhLPgrVZGoeLMpOl MkaFNpNtZckUytC8TY1nPcpcwzVLn+g8HblO5T2bKk2iaxTQAD6TpSOJNhHIy98w /GpipCRg5vj0M/ez20wN/NXnOGQLQfL6JcQnmwIDAQABAoIBAGTx1eaQk9B6BEP+ rXOudTGGzO8SDFop9M/y8HQ3Y7hCk2mdxJNY8bJQTcIWS+g9rC+kencbC3/aqCJt 2zT1cTCy61QU9nYbc/JThGIttqvF/AVnryzSNyL0R3Oa/Dbk7CDSgK3cQ6qMgPru Ka0gLJh3VVBAtBMUEGPltdsUntM4sHTh5FAabP0ioBJ1QLG6Aak7LOQikjBEFJoc Tea4uRsE7IreP5Mn7UW92nkt1ey5UGzBtNNtpHbVaHmfQojwlwkLtnV35sumbvK6 6KTMNREZv6xSIMwkYxm1zRE3Cus/1jGIc8MZF0BxgcCR+G37l+BKwL8CSymHPxhH dvGxoPECgYEA3STp52CbI/KyVfvjxK2OIex/NV1jKh85wQsLtkaRv3/a/EEg7MV7 54dEvo5KKOZXfeOd9r9G9h1RffjSD9MhxfPhyGwuOcqa8IE1zNwlY/v7KL7HtDIf 2mrXWF5Klafh8aXYcaRH0ZSLnl/nXUXYht4/0NRGiXnttUgqs6hvY70CgYEA46tO J5QkgF3YVY0gx10wRCAnnKLkAaHdtxtteXOJh79xsGXQ4LLngc+mz1hLt+TNJza+ BZhoWwY/ZgyiTH0pebGr/U0QUMoUHlGgjgj3Aa/XFpOhtyLU+IU/PYl0BUz9dqsN TDtv6p/HQhfd98vUNsbACQda+YAo+oRdO5kLQjcCgYB3OAZNcXxRte5EgoY5KqN8 UGYH2++w7qKRGqZWvtamGYRyB557Zr+0gu0hmc4LHJrASGyJcHcOCaI8Ol7snxMP B7qJ9SA6kapTzCS361rQ+zBct/UrhPY9JuovPq4Q3i/luVXldf4t01otqGAvnY7s rnZS242nYa8v0tcKgdyDNQKBgB3Z60BzQyn1pBTrkT2ysU5tbOQz03OHVrvYg80l 4gWDi5OWdgHQU1yI7pVHPX5aKLAYlGfFaQFuW0e1Jl6jFpoXOrbWsOn25RZom4Wk FUcKWEhkiRKrJYOEbRtTd3vucVlq6i5xqKX51zWKTZddCXE5NBq69Sm7rSPT0Sms UnaXAoGAXYAE5slvjcylJpMV4lxTBmNtA9+pw1T7I379mIyqZ0OS25nmpskHU7FR SQDSRHw7hHuyjEHyhMoHEGLfUMIltQoi+pcrieVQelJdSuX7VInzHPAR5RppUVFl jOZZKlIiqs+UfCoOgsIblXuw7a/ATnAnXakutSFgHU1lN1gN02U= -----END RSA PRIVATE KEY----- ================================================ FILE: pkg/proxy/server.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package proxy import ( "context" "errors" "fmt" "io" mrand "math/rand" "net" "net/http" "net/url" "strconv" "strings" "sync" "time" humanize "github.com/dustin/go-humanize" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/transport" ) var ( defaultDialTimeout = 3 * time.Second defaultBufferSize = 48 * 1024 defaultRetryInterval = 10 * time.Millisecond ) // Server defines proxy server layer that simulates common network faults: // latency spikes and packet drop or corruption. The proxy overhead is very // small overhead (<500μs per request). Please run tests to compute actual // overhead. type Server interface { // From returns proxy source address in "scheme://host:port" format. From() string // To returns proxy destination address in "scheme://host:port" format. To() string // Ready returns when proxy is ready to serve. Ready() <-chan struct{} // Done returns when proxy has been closed. Done() <-chan struct{} // Error sends errors while serving proxy. Error() <-chan error // Close closes listener and transport. Close() error // PauseAccept stops accepting new connections. PauseAccept() // UnpauseAccept removes pause operation on accepting new connections. UnpauseAccept() // DelayAccept adds latency ± random variable to accepting // new incoming connections. DelayAccept(latency, rv time.Duration) // UndelayAccept removes sending latencies. UndelayAccept() // LatencyAccept returns current latency on accepting // new incoming connections. LatencyAccept() time.Duration // DelayTx adds latency ± random variable for "outgoing" traffic // in "sending" layer. DelayTx(latency, rv time.Duration) // UndelayTx removes sending latencies. UndelayTx() // LatencyTx returns current send latency. LatencyTx() time.Duration // DelayRx adds latency ± random variable for "incoming" traffic // in "receiving" layer. DelayRx(latency, rv time.Duration) // UndelayRx removes "receiving" latencies. UndelayRx() // LatencyRx returns current receive latency. LatencyRx() time.Duration // ModifyTx alters/corrupts/drops "outgoing" packets from the listener // with the given edit function. ModifyTx(f func(data []byte) []byte) // UnmodifyTx removes modify operation on "forwarding". UnmodifyTx() // ModifyRx alters/corrupts/drops "incoming" packets to client // with the given edit function. ModifyRx(f func(data []byte) []byte) // UnmodifyRx removes modify operation on "receiving". UnmodifyRx() // BlackholeTx drops all "outgoing" packets before "forwarding". // "BlackholeTx" operation is a wrapper around "ModifyTx" with // a function that returns empty bytes. BlackholeTx() // UnblackholeTx removes blackhole operation on "sending". UnblackholeTx() // BlackholeRx drops all "incoming" packets to client. // "BlackholeRx" operation is a wrapper around "ModifyRx" with // a function that returns empty bytes. BlackholeRx() // UnblackholeRx removes blackhole operation on "receiving". UnblackholeRx() // PauseTx stops "forwarding" packets; "outgoing" traffic blocks. PauseTx() // UnpauseTx removes "forwarding" pause operation. UnpauseTx() // PauseRx stops "receiving" packets; "incoming" traffic blocks. PauseRx() // UnpauseRx removes "receiving" pause operation. UnpauseRx() // ResetListener closes and restarts listener. ResetListener() error } // ServerConfig defines proxy server configuration. type ServerConfig struct { Logger *zap.Logger From url.URL To url.URL TLSInfo transport.TLSInfo DialTimeout time.Duration BufferSize int RetryInterval time.Duration } type server struct { lg *zap.Logger from url.URL fromPort int to url.URL toPort int tlsInfo transport.TLSInfo dialTimeout time.Duration bufferSize int retryInterval time.Duration readyc chan struct{} donec chan struct{} errc chan error closeOnce sync.Once closeWg sync.WaitGroup listenerMu sync.RWMutex listener net.Listener pauseAcceptMu sync.Mutex pauseAcceptc chan struct{} latencyAcceptMu sync.RWMutex latencyAccept time.Duration modifyTxMu sync.RWMutex modifyTx func(data []byte) []byte modifyRxMu sync.RWMutex modifyRx func(data []byte) []byte pauseTxMu sync.Mutex pauseTxc chan struct{} pauseRxMu sync.Mutex pauseRxc chan struct{} latencyTxMu sync.RWMutex latencyTx time.Duration latencyRxMu sync.RWMutex latencyRx time.Duration } // NewServer returns a proxy implementation with no iptables/tc dependencies. // The proxy layer overhead is <1ms. func NewServer(cfg ServerConfig) Server { s := &server{ lg: cfg.Logger, from: cfg.From, to: cfg.To, tlsInfo: cfg.TLSInfo, dialTimeout: cfg.DialTimeout, bufferSize: cfg.BufferSize, retryInterval: cfg.RetryInterval, readyc: make(chan struct{}), donec: make(chan struct{}), errc: make(chan error, 16), pauseAcceptc: make(chan struct{}), pauseTxc: make(chan struct{}), pauseRxc: make(chan struct{}), } _, fromPort, err := net.SplitHostPort(cfg.From.Host) if err == nil { s.fromPort, _ = strconv.Atoi(fromPort) } var toPort string _, toPort, err = net.SplitHostPort(cfg.To.Host) if err == nil { s.toPort, _ = strconv.Atoi(toPort) } if s.dialTimeout == 0 { s.dialTimeout = defaultDialTimeout } if s.bufferSize == 0 { s.bufferSize = defaultBufferSize } if s.retryInterval == 0 { s.retryInterval = defaultRetryInterval } close(s.pauseAcceptc) close(s.pauseTxc) close(s.pauseRxc) if strings.HasPrefix(s.from.Scheme, "http") { s.from.Scheme = "tcp" } if strings.HasPrefix(s.to.Scheme, "http") { s.to.Scheme = "tcp" } addr := fmt.Sprintf(":%d", s.fromPort) if s.fromPort == 0 { // unix addr = s.from.Host } var ln net.Listener if !s.tlsInfo.Empty() { ln, err = transport.NewListener(addr, s.from.Scheme, &s.tlsInfo) } else { ln, err = net.Listen(s.from.Scheme, addr) } if err != nil { s.errc <- err s.Close() return s } s.listener = ln s.closeWg.Add(1) go s.listenAndServe() s.lg.Info("started proxying", zap.String("from", s.From()), zap.String("to", s.To())) return s } func (s *server) From() string { return fmt.Sprintf("%s://%s", s.from.Scheme, s.from.Host) } func (s *server) To() string { return fmt.Sprintf("%s://%s", s.to.Scheme, s.to.Host) } // TODO: implement packet reordering from multiple TCP connections // buffer packets per connection for awhile, reorder before transmit // - https://github.com/etcd-io/etcd/issues/5614 // - https://github.com/etcd-io/etcd/pull/6918#issuecomment-264093034 func (s *server) listenAndServe() { defer s.closeWg.Done() ctx := context.Background() s.lg.Info("proxy is listening on", zap.String("from", s.From())) close(s.readyc) for { s.pauseAcceptMu.Lock() pausec := s.pauseAcceptc s.pauseAcceptMu.Unlock() select { case <-pausec: case <-s.donec: return } s.latencyAcceptMu.RLock() lat := s.latencyAccept s.latencyAcceptMu.RUnlock() if lat > 0 { select { case <-time.After(lat): case <-s.donec: return } } s.listenerMu.RLock() ln := s.listener s.listenerMu.RUnlock() in, err := ln.Accept() if err != nil { select { case s.errc <- err: select { case <-s.donec: return default: } case <-s.donec: return } s.lg.Debug("listener accept error", zap.Error(err)) if strings.HasSuffix(err.Error(), "use of closed network connection") { select { case <-time.After(s.retryInterval): case <-s.donec: return } s.lg.Debug("listener is closed; retry listening on", zap.String("from", s.From())) if err = s.ResetListener(); err != nil { select { case s.errc <- err: select { case <-s.donec: return default: } case <-s.donec: return } s.lg.Warn("failed to reset listener", zap.Error(err)) } } continue } var out net.Conn if !s.tlsInfo.Empty() { var tp *http.Transport tp, err = transport.NewTransport(s.tlsInfo, s.dialTimeout) if err != nil { select { case s.errc <- err: select { case <-s.donec: return default: } case <-s.donec: return } continue } out, err = tp.DialContext(ctx, s.to.Scheme, s.to.Host) } else { out, err = net.Dial(s.to.Scheme, s.to.Host) } if err != nil { select { case s.errc <- err: select { case <-s.donec: return default: } case <-s.donec: return } s.lg.Debug("failed to dial", zap.Error(err)) continue } s.closeWg.Add(2) go func() { defer s.closeWg.Done() // read incoming bytes from listener, dispatch to outgoing connection s.transmit(out, in) out.Close() in.Close() }() go func() { defer s.closeWg.Done() // read response from outgoing connection, write back to listener s.receive(in, out) in.Close() out.Close() }() } } func (s *server) transmit(dst io.Writer, src io.Reader) { s.ioCopy(dst, src, proxyTx) } func (s *server) receive(dst io.Writer, src io.Reader) { s.ioCopy(dst, src, proxyRx) } type proxyType uint8 const ( proxyTx proxyType = iota proxyRx ) func (s *server) ioCopy(dst io.Writer, src io.Reader, ptype proxyType) { buf := make([]byte, s.bufferSize) for { nr1, err := src.Read(buf) if err != nil { if errors.Is(err, io.EOF) { return } // connection already closed if strings.HasSuffix(err.Error(), "read: connection reset by peer") { return } if strings.HasSuffix(err.Error(), "use of closed network connection") { return } select { case s.errc <- err: select { case <-s.donec: return default: } case <-s.donec: return } s.lg.Debug("failed to read", zap.Error(err)) return } if nr1 == 0 { return } data := buf[:nr1] // alters/corrupts/drops data switch ptype { case proxyTx: s.modifyTxMu.RLock() if s.modifyTx != nil { data = s.modifyTx(data) } s.modifyTxMu.RUnlock() case proxyRx: s.modifyRxMu.RLock() if s.modifyRx != nil { data = s.modifyRx(data) } s.modifyRxMu.RUnlock() default: panic("unknown proxy type") } nr2 := len(data) switch ptype { case proxyTx: s.lg.Debug( "modified tx", zap.String("data-received", humanize.Bytes(uint64(nr1))), zap.String("data-modified", humanize.Bytes(uint64(nr2))), zap.String("from", s.From()), zap.String("to", s.To()), ) case proxyRx: s.lg.Debug( "modified rx", zap.String("data-received", humanize.Bytes(uint64(nr1))), zap.String("data-modified", humanize.Bytes(uint64(nr2))), zap.String("from", s.To()), zap.String("to", s.From()), ) default: panic("unknown proxy type") } // pause before packet dropping, blocking, and forwarding var pausec chan struct{} switch ptype { case proxyTx: s.pauseTxMu.Lock() pausec = s.pauseTxc s.pauseTxMu.Unlock() case proxyRx: s.pauseRxMu.Lock() pausec = s.pauseRxc s.pauseRxMu.Unlock() default: panic("unknown proxy type") } select { case <-pausec: case <-s.donec: return } // pause first, and then drop packets if nr2 == 0 { continue } // block before forwarding var lat time.Duration switch ptype { case proxyTx: s.latencyTxMu.RLock() lat = s.latencyTx s.latencyTxMu.RUnlock() case proxyRx: s.latencyRxMu.RLock() lat = s.latencyRx s.latencyRxMu.RUnlock() default: panic("unknown proxy type") } if lat > 0 { select { case <-time.After(lat): case <-s.donec: return } } // now forward packets to target var nw int nw, err = dst.Write(data) if err != nil { if errors.Is(err, io.EOF) { return } select { case s.errc <- err: select { case <-s.donec: return default: } case <-s.donec: return } switch ptype { case proxyTx: s.lg.Debug("write fail on tx", zap.Error(err)) case proxyRx: s.lg.Debug("write fail on rx", zap.Error(err)) default: panic("unknown proxy type") } return } if nr2 != nw { select { case s.errc <- io.ErrShortWrite: select { case <-s.donec: return default: } case <-s.donec: return } switch ptype { case proxyTx: s.lg.Debug( "write fail on tx; read/write bytes are different", zap.Int("read-bytes", nr1), zap.Int("write-bytes", nw), zap.Error(io.ErrShortWrite), ) case proxyRx: s.lg.Debug( "write fail on rx; read/write bytes are different", zap.Int("read-bytes", nr1), zap.Int("write-bytes", nw), zap.Error(io.ErrShortWrite), ) default: panic("unknown proxy type") } return } switch ptype { case proxyTx: s.lg.Debug( "transmitted", zap.String("data-size", humanize.Bytes(uint64(nr1))), zap.String("from", s.From()), zap.String("to", s.To()), ) case proxyRx: s.lg.Debug( "received", zap.String("data-size", humanize.Bytes(uint64(nr1))), zap.String("from", s.To()), zap.String("to", s.From()), ) default: panic("unknown proxy type") } } } func (s *server) Ready() <-chan struct{} { return s.readyc } func (s *server) Done() <-chan struct{} { return s.donec } func (s *server) Error() <-chan error { return s.errc } func (s *server) Close() (err error) { s.closeOnce.Do(func() { close(s.donec) s.listenerMu.Lock() if s.listener != nil { err = s.listener.Close() s.lg.Info( "closed proxy listener", zap.String("from", s.From()), zap.String("to", s.To()), ) } s.lg.Sync() s.listenerMu.Unlock() }) s.closeWg.Wait() return err } func (s *server) PauseAccept() { s.pauseAcceptMu.Lock() s.pauseAcceptc = make(chan struct{}) s.pauseAcceptMu.Unlock() s.lg.Info( "paused accept", zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) UnpauseAccept() { s.pauseAcceptMu.Lock() select { case <-s.pauseAcceptc: // already unpaused case <-s.donec: s.pauseAcceptMu.Unlock() return default: close(s.pauseAcceptc) } s.pauseAcceptMu.Unlock() s.lg.Info( "unpaused accept", zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) DelayAccept(latency, rv time.Duration) { if latency <= 0 { return } d := computeLatency(latency, rv) s.latencyAcceptMu.Lock() s.latencyAccept = d s.latencyAcceptMu.Unlock() s.lg.Info( "set accept latency", zap.Duration("latency", d), zap.Duration("given-latency", latency), zap.Duration("given-latency-random-variable", rv), zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) UndelayAccept() { s.latencyAcceptMu.Lock() d := s.latencyAccept s.latencyAccept = 0 s.latencyAcceptMu.Unlock() s.lg.Info( "removed accept latency", zap.Duration("latency", d), zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) LatencyAccept() time.Duration { s.latencyAcceptMu.RLock() d := s.latencyAccept s.latencyAcceptMu.RUnlock() return d } func (s *server) DelayTx(latency, rv time.Duration) { if latency <= 0 { return } d := computeLatency(latency, rv) s.latencyTxMu.Lock() s.latencyTx = d s.latencyTxMu.Unlock() s.lg.Info( "set transmit latency", zap.Duration("latency", d), zap.Duration("given-latency", latency), zap.Duration("given-latency-random-variable", rv), zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) UndelayTx() { s.latencyTxMu.Lock() d := s.latencyTx s.latencyTx = 0 s.latencyTxMu.Unlock() s.lg.Info( "removed transmit latency", zap.Duration("latency", d), zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) LatencyTx() time.Duration { s.latencyTxMu.RLock() d := s.latencyTx s.latencyTxMu.RUnlock() return d } func (s *server) DelayRx(latency, rv time.Duration) { if latency <= 0 { return } d := computeLatency(latency, rv) s.latencyRxMu.Lock() s.latencyRx = d s.latencyRxMu.Unlock() s.lg.Info( "set receive latency", zap.Duration("latency", d), zap.Duration("given-latency", latency), zap.Duration("given-latency-random-variable", rv), zap.String("from", s.To()), zap.String("to", s.From()), ) } func (s *server) UndelayRx() { s.latencyRxMu.Lock() d := s.latencyRx s.latencyRx = 0 s.latencyRxMu.Unlock() s.lg.Info( "removed receive latency", zap.Duration("latency", d), zap.String("from", s.To()), zap.String("to", s.From()), ) } func (s *server) LatencyRx() time.Duration { s.latencyRxMu.RLock() d := s.latencyRx s.latencyRxMu.RUnlock() return d } func computeLatency(lat, rv time.Duration) time.Duration { if rv == 0 { return lat } if rv < 0 { rv *= -1 } if rv > lat { rv = lat / 10 } now := time.Now() sign := 1 if now.Second()%2 == 0 { sign = -1 } return lat + time.Duration(int64(sign)*mrand.Int63n(rv.Nanoseconds())) } func (s *server) ModifyTx(f func([]byte) []byte) { s.modifyTxMu.Lock() s.modifyTx = f s.modifyTxMu.Unlock() s.lg.Info( "modifying tx", zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) UnmodifyTx() { s.modifyTxMu.Lock() s.modifyTx = nil s.modifyTxMu.Unlock() s.lg.Info( "unmodifyed tx", zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) ModifyRx(f func([]byte) []byte) { s.modifyRxMu.Lock() s.modifyRx = f s.modifyRxMu.Unlock() s.lg.Info( "modifying rx", zap.String("from", s.To()), zap.String("to", s.From()), ) } func (s *server) UnmodifyRx() { s.modifyRxMu.Lock() s.modifyRx = nil s.modifyRxMu.Unlock() s.lg.Info( "unmodifyed rx", zap.String("from", s.To()), zap.String("to", s.From()), ) } func (s *server) BlackholeTx() { s.ModifyTx(func([]byte) []byte { return nil }) s.lg.Info( "blackholed tx", zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) UnblackholeTx() { s.UnmodifyTx() s.lg.Info( "unblackholed tx", zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) BlackholeRx() { s.ModifyRx(func([]byte) []byte { return nil }) s.lg.Info( "blackholed rx", zap.String("from", s.To()), zap.String("to", s.From()), ) } func (s *server) UnblackholeRx() { s.UnmodifyRx() s.lg.Info( "unblackholed rx", zap.String("from", s.To()), zap.String("to", s.From()), ) } func (s *server) PauseTx() { s.pauseTxMu.Lock() s.pauseTxc = make(chan struct{}) s.pauseTxMu.Unlock() s.lg.Info( "paused tx", zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) UnpauseTx() { s.pauseTxMu.Lock() select { case <-s.pauseTxc: // already unpaused case <-s.donec: s.pauseTxMu.Unlock() return default: close(s.pauseTxc) } s.pauseTxMu.Unlock() s.lg.Info( "unpaused tx", zap.String("from", s.From()), zap.String("to", s.To()), ) } func (s *server) PauseRx() { s.pauseRxMu.Lock() s.pauseRxc = make(chan struct{}) s.pauseRxMu.Unlock() s.lg.Info( "paused rx", zap.String("from", s.To()), zap.String("to", s.From()), ) } func (s *server) UnpauseRx() { s.pauseRxMu.Lock() select { case <-s.pauseRxc: // already unpaused case <-s.donec: s.pauseRxMu.Unlock() return default: close(s.pauseRxc) } s.pauseRxMu.Unlock() s.lg.Info( "unpaused rx", zap.String("from", s.To()), zap.String("to", s.From()), ) } func (s *server) ResetListener() error { s.listenerMu.Lock() defer s.listenerMu.Unlock() if err := s.listener.Close(); err != nil { // already closed if !strings.HasSuffix(err.Error(), "use of closed network connection") { return err } } var ln net.Listener var err error if !s.tlsInfo.Empty() { ln, err = transport.NewListener(s.from.Host, s.from.Scheme, &s.tlsInfo) } else { ln, err = net.Listen(s.from.Scheme, s.from.Host) } if err != nil { return err } s.listener = ln s.lg.Info( "reset listener on", zap.String("from", s.From()), ) return nil } ================================================ FILE: pkg/proxy/server_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package proxy import ( "bytes" "crypto/tls" "fmt" "io" "log" "math/rand" "net" "net/http" "net/url" "os" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/transport" ) func TestServer_Unix_Insecure(t *testing.T) { testServer(t, "unix", false, false) } func TestServer_TCP_Insecure(t *testing.T) { testServer(t, "tcp", false, false) } func TestServer_Unix_Secure(t *testing.T) { testServer(t, "unix", true, false) } func TestServer_TCP_Secure(t *testing.T) { testServer(t, "tcp", true, false) } func TestServer_Unix_Insecure_DelayTx(t *testing.T) { testServer(t, "unix", false, true) } func TestServer_TCP_Insecure_DelayTx(t *testing.T) { testServer(t, "tcp", false, true) } func TestServer_Unix_Secure_DelayTx(t *testing.T) { testServer(t, "unix", true, true) } func TestServer_TCP_Secure_DelayTx(t *testing.T) { testServer(t, "tcp", true, true) } func testServer(t *testing.T, scheme string, secure bool, delayTx bool) { lg := zaptest.NewLogger(t) srcAddr, dstAddr := newUnixAddr(), newUnixAddr() if scheme == "tcp" { ln1, ln2 := listen(t, "tcp", "localhost:0", transport.TLSInfo{}), listen(t, "tcp", "localhost:0", transport.TLSInfo{}) srcAddr, dstAddr = ln1.Addr().String(), ln2.Addr().String() ln1.Close() ln2.Close() } else { defer func() { os.RemoveAll(srcAddr) os.RemoveAll(dstAddr) }() } tlsInfo := createTLSInfo(lg, secure) ln := listen(t, scheme, dstAddr, tlsInfo) defer ln.Close() cfg := ServerConfig{ Logger: lg, From: url.URL{Scheme: scheme, Host: srcAddr}, To: url.URL{Scheme: scheme, Host: dstAddr}, } if secure { cfg.TLSInfo = tlsInfo } p := NewServer(cfg) waitForServer(t, p) defer p.Close() data1 := []byte("Hello World!") donec, writec := make(chan struct{}), make(chan []byte) go func() { defer close(donec) for data := range writec { send(t, data, scheme, srcAddr, tlsInfo) //nolint:testifylint //FIXME } }() recvc := make(chan []byte, 1) go func() { for i := 0; i < 2; i++ { recvc <- receive(t, ln) //nolint:testifylint //FIXME } }() writec <- data1 now := time.Now() if d := <-recvc; !bytes.Equal(data1, d) { close(writec) t.Fatalf("expected %q, got %q", string(data1), string(d)) } took1 := time.Since(now) t.Logf("took %v with no latency", took1) lat, rv := 50*time.Millisecond, 5*time.Millisecond if delayTx { p.DelayTx(lat, rv) } data2 := []byte("new data") writec <- data2 now = time.Now() if d := <-recvc; !bytes.Equal(data2, d) { close(writec) t.Fatalf("expected %q, got %q", string(data2), string(d)) } took2 := time.Since(now) if delayTx { t.Logf("took %v with latency %v+-%v", took2, lat, rv) } else { t.Logf("took %v with no latency", took2) } if delayTx { p.UndelayTx() if took2 < lat-rv { close(writec) t.Fatalf("expected took2 %v (with latency) > delay: %v", took2, lat-rv) } } close(writec) select { case <-donec: case <-time.After(3 * time.Second): t.Fatal("took too long to write") } select { case <-p.Done(): t.Fatal("unexpected done") case err := <-p.Error(): t.Fatal(err) default: } require.NoError(t, p.Close()) select { case <-p.Done(): case err := <-p.Error(): if !strings.HasPrefix(err.Error(), "accept ") && !strings.HasSuffix(err.Error(), "use of closed network connection") { t.Fatal(err) } case <-time.After(3 * time.Second): t.Fatal("took too long to close") } } func createTLSInfo(lg *zap.Logger, secure bool) transport.TLSInfo { if secure { return transport.TLSInfo{ KeyFile: "../../tests/fixtures/server.key.insecure", CertFile: "../../tests/fixtures/server.crt", TrustedCAFile: "../../tests/fixtures/ca.crt", ClientCertAuth: true, Logger: lg, } } return transport.TLSInfo{Logger: lg} } func TestServer_Unix_Insecure_DelayAccept(t *testing.T) { testServerDelayAccept(t, false) } func TestServer_Unix_Secure_DelayAccept(t *testing.T) { testServerDelayAccept(t, true) } func testServerDelayAccept(t *testing.T, secure bool) { lg := zaptest.NewLogger(t) srcAddr, dstAddr := newUnixAddr(), newUnixAddr() defer func() { os.RemoveAll(srcAddr) os.RemoveAll(dstAddr) }() tlsInfo := createTLSInfo(lg, secure) scheme := "unix" ln := listen(t, scheme, dstAddr, tlsInfo) defer ln.Close() cfg := ServerConfig{ Logger: lg, From: url.URL{Scheme: scheme, Host: srcAddr}, To: url.URL{Scheme: scheme, Host: dstAddr}, } if secure { cfg.TLSInfo = tlsInfo } p := NewServer(cfg) waitForServer(t, p) defer p.Close() data := []byte("Hello World!") now := time.Now() send(t, data, scheme, srcAddr, tlsInfo) d := receive(t, ln) require.Truef(t, bytes.Equal(data, d), "expected %q, got %q", string(data), string(d)) took1 := time.Since(now) t.Logf("took %v with no latency", took1) lat, rv := 700*time.Millisecond, 10*time.Millisecond p.DelayAccept(lat, rv) defer p.UndelayAccept() require.NoError(t, p.ResetListener()) time.Sleep(200 * time.Millisecond) now = time.Now() send(t, data, scheme, srcAddr, tlsInfo) d = receive(t, ln) require.Truef(t, bytes.Equal(data, d), "expected %q, got %q", string(data), string(d)) took2 := time.Since(now) t.Logf("took %v with latency %v±%v", took2, lat, rv) require.Lessf(t, took1, took2, "expected took1 %v < took2 %v", took1, took2) } func TestServer_PauseTx(t *testing.T) { lg := zaptest.NewLogger(t) scheme := "unix" srcAddr, dstAddr := newUnixAddr(), newUnixAddr() defer func() { os.RemoveAll(srcAddr) os.RemoveAll(dstAddr) }() ln := listen(t, scheme, dstAddr, transport.TLSInfo{}) defer ln.Close() p := NewServer(ServerConfig{ Logger: lg, From: url.URL{Scheme: scheme, Host: srcAddr}, To: url.URL{Scheme: scheme, Host: dstAddr}, }) waitForServer(t, p) defer p.Close() p.PauseTx() data := []byte("Hello World!") send(t, data, scheme, srcAddr, transport.TLSInfo{}) recvc := make(chan []byte, 1) go func() { recvc <- receive(t, ln) //nolint:testifylint //FIXME }() select { case d := <-recvc: t.Fatalf("received unexpected data %q during pause", string(d)) case <-time.After(200 * time.Millisecond): } p.UnpauseTx() select { case d := <-recvc: require.Truef(t, bytes.Equal(data, d), "expected %q, got %q", string(data), string(d)) case <-time.After(2 * time.Second): t.Fatal("took too long to receive after unpause") } } func TestServer_ModifyTx_corrupt(t *testing.T) { lg := zaptest.NewLogger(t) scheme := "unix" srcAddr, dstAddr := newUnixAddr(), newUnixAddr() defer func() { os.RemoveAll(srcAddr) os.RemoveAll(dstAddr) }() ln := listen(t, scheme, dstAddr, transport.TLSInfo{}) defer ln.Close() p := NewServer(ServerConfig{ Logger: lg, From: url.URL{Scheme: scheme, Host: srcAddr}, To: url.URL{Scheme: scheme, Host: dstAddr}, }) waitForServer(t, p) defer p.Close() p.ModifyTx(func(d []byte) []byte { d[len(d)/2]++ return d }) data := []byte("Hello World!") send(t, data, scheme, srcAddr, transport.TLSInfo{}) d := receive(t, ln) require.Falsef(t, bytes.Equal(d, data), "expected corrupted data, got %q", string(d)) p.UnmodifyTx() send(t, data, scheme, srcAddr, transport.TLSInfo{}) d = receive(t, ln) require.Truef(t, bytes.Equal(d, data), "expected uncorrupted data, got %q", string(d)) } func TestServer_ModifyTx_packet_loss(t *testing.T) { lg := zaptest.NewLogger(t) scheme := "unix" srcAddr, dstAddr := newUnixAddr(), newUnixAddr() defer func() { os.RemoveAll(srcAddr) os.RemoveAll(dstAddr) }() ln := listen(t, scheme, dstAddr, transport.TLSInfo{}) defer ln.Close() p := NewServer(ServerConfig{ Logger: lg, From: url.URL{Scheme: scheme, Host: srcAddr}, To: url.URL{Scheme: scheme, Host: dstAddr}, }) waitForServer(t, p) defer p.Close() // 50% packet loss p.ModifyTx(func(d []byte) []byte { half := len(d) / 2 return d[:half:half] }) data := []byte("Hello World!") send(t, data, scheme, srcAddr, transport.TLSInfo{}) d := receive(t, ln) require.Falsef(t, bytes.Equal(d, data), "expected corrupted data, got %q", string(d)) p.UnmodifyTx() send(t, data, scheme, srcAddr, transport.TLSInfo{}) d = receive(t, ln) require.Truef(t, bytes.Equal(d, data), "expected uncorrupted data, got %q", string(d)) } func TestServer_BlackholeTx(t *testing.T) { lg := zaptest.NewLogger(t) scheme := "unix" srcAddr, dstAddr := newUnixAddr(), newUnixAddr() defer func() { os.RemoveAll(srcAddr) os.RemoveAll(dstAddr) }() ln := listen(t, scheme, dstAddr, transport.TLSInfo{}) defer ln.Close() p := NewServer(ServerConfig{ Logger: lg, From: url.URL{Scheme: scheme, Host: srcAddr}, To: url.URL{Scheme: scheme, Host: dstAddr}, }) waitForServer(t, p) defer p.Close() p.BlackholeTx() data := []byte("Hello World!") send(t, data, scheme, srcAddr, transport.TLSInfo{}) recvc := make(chan []byte, 1) go func() { recvc <- receive(t, ln) //nolint:testifylint //FIXME }() select { case d := <-recvc: t.Fatalf("unexpected data receive %q during blackhole", string(d)) case <-time.After(200 * time.Millisecond): } p.UnblackholeTx() // expect different data, old data dropped data[0]++ send(t, data, scheme, srcAddr, transport.TLSInfo{}) select { case d := <-recvc: require.Truef(t, bytes.Equal(data, d), "expected %q, got %q", string(data), string(d)) case <-time.After(2 * time.Second): t.Fatal("took too long to receive after unblackhole") } } func TestServer_Shutdown(t *testing.T) { lg := zaptest.NewLogger(t) scheme := "unix" srcAddr, dstAddr := newUnixAddr(), newUnixAddr() defer func() { os.RemoveAll(srcAddr) os.RemoveAll(dstAddr) }() ln := listen(t, scheme, dstAddr, transport.TLSInfo{}) defer ln.Close() p := NewServer(ServerConfig{ Logger: lg, From: url.URL{Scheme: scheme, Host: srcAddr}, To: url.URL{Scheme: scheme, Host: dstAddr}, }) waitForServer(t, p) defer p.Close() s, _ := p.(*server) s.listener.Close() time.Sleep(200 * time.Millisecond) data := []byte("Hello World!") send(t, data, scheme, srcAddr, transport.TLSInfo{}) d := receive(t, ln) require.Truef(t, bytes.Equal(d, data), "expected %q, got %q", string(data), string(d)) } func TestServer_ShutdownListener(t *testing.T) { lg := zaptest.NewLogger(t) scheme := "unix" srcAddr, dstAddr := newUnixAddr(), newUnixAddr() defer func() { os.RemoveAll(srcAddr) os.RemoveAll(dstAddr) }() ln := listen(t, scheme, dstAddr, transport.TLSInfo{}) defer ln.Close() p := NewServer(ServerConfig{ Logger: lg, From: url.URL{Scheme: scheme, Host: srcAddr}, To: url.URL{Scheme: scheme, Host: dstAddr}, }) waitForServer(t, p) defer p.Close() // shut down destination ln.Close() time.Sleep(200 * time.Millisecond) ln = listen(t, scheme, dstAddr, transport.TLSInfo{}) defer ln.Close() data := []byte("Hello World!") send(t, data, scheme, srcAddr, transport.TLSInfo{}) d := receive(t, ln) require.Truef(t, bytes.Equal(d, data), "expected %q, got %q", string(data), string(d)) } func TestServerHTTP_Insecure_DelayTx(t *testing.T) { testServerHTTP(t, false, true) } func TestServerHTTP_Secure_DelayTx(t *testing.T) { testServerHTTP(t, true, true) } func TestServerHTTP_Insecure_DelayRx(t *testing.T) { testServerHTTP(t, false, false) } func TestServerHTTP_Secure_DelayRx(t *testing.T) { testServerHTTP(t, true, false) } func testServerHTTP(t *testing.T, secure, delayTx bool) { lg := zaptest.NewLogger(t) scheme := "tcp" ln1, ln2 := listen(t, scheme, "localhost:0", transport.TLSInfo{}), listen(t, scheme, "localhost:0", transport.TLSInfo{}) srcAddr, dstAddr := ln1.Addr().String(), ln2.Addr().String() ln1.Close() ln2.Close() mux := http.NewServeMux() mux.HandleFunc("/hello", func(w http.ResponseWriter, req *http.Request) { d, err := io.ReadAll(req.Body) req.Body.Close() require.NoError(t, err) //nolint:testifylint //FIXME _, err = w.Write([]byte(fmt.Sprintf("%q(confirmed)", string(d)))) require.NoError(t, err) //nolint:testifylint //FIXME }) tlsInfo := createTLSInfo(lg, secure) var tlsConfig *tls.Config if secure { _, err := tlsInfo.ServerConfig() require.NoError(t, err) } srv := &http.Server{ Addr: dstAddr, Handler: mux, TLSConfig: tlsConfig, ErrorLog: log.New(io.Discard, "net/http", 0), } donec := make(chan struct{}) defer func() { srv.Close() <-donec }() go func() { if !secure { srv.ListenAndServe() } else { srv.ListenAndServeTLS(tlsInfo.CertFile, tlsInfo.KeyFile) } defer close(donec) }() time.Sleep(200 * time.Millisecond) cfg := ServerConfig{ Logger: lg, From: url.URL{Scheme: scheme, Host: srcAddr}, To: url.URL{Scheme: scheme, Host: dstAddr}, } if secure { cfg.TLSInfo = tlsInfo } p := NewServer(cfg) waitForServer(t, p) defer func() { lg.Info("closing Proxy server...") p.Close() lg.Info("closed Proxy server.") }() data := "Hello World!" var resp *http.Response var err error now := time.Now() if secure { tp, terr := transport.NewTransport(tlsInfo, 3*time.Second) require.NoError(t, terr) cli := &http.Client{Transport: tp} resp, err = cli.Post("https://"+srcAddr+"/hello", "", strings.NewReader(data)) defer cli.CloseIdleConnections() defer tp.CloseIdleConnections() } else { resp, err = http.Post("http://"+srcAddr+"/hello", "", strings.NewReader(data)) defer http.DefaultClient.CloseIdleConnections() } require.NoError(t, err) d, err := io.ReadAll(resp.Body) require.NoError(t, err) resp.Body.Close() took1 := time.Since(now) t.Logf("took %v with no latency", took1) rs1 := string(d) exp := fmt.Sprintf("%q(confirmed)", data) require.Equalf(t, exp, rs1, "got %q, expected %q", rs1, exp) lat, rv := 100*time.Millisecond, 10*time.Millisecond if delayTx { p.DelayTx(lat, rv) defer p.UndelayTx() } else { p.DelayRx(lat, rv) defer p.UndelayRx() } now = time.Now() if secure { tp, terr := transport.NewTransport(tlsInfo, 3*time.Second) require.NoError(t, terr) cli := &http.Client{Transport: tp} resp, err = cli.Post("https://"+srcAddr+"/hello", "", strings.NewReader(data)) defer cli.CloseIdleConnections() defer tp.CloseIdleConnections() } else { resp, err = http.Post("http://"+srcAddr+"/hello", "", strings.NewReader(data)) defer http.DefaultClient.CloseIdleConnections() } require.NoError(t, err) d, err = io.ReadAll(resp.Body) require.NoError(t, err) resp.Body.Close() took2 := time.Since(now) t.Logf("took %v with latency %v±%v", took2, lat, rv) rs2 := string(d) require.Equalf(t, exp, rs2, "got %q, expected %q", rs2, exp) require.LessOrEqualf(t, took1, took2, "expected took1 %v < took2 %v", took1, took2) } func newUnixAddr() string { now := time.Now().UnixNano() addr := fmt.Sprintf("%X%X.unix-conn", now, rand.Intn(35000)) os.RemoveAll(addr) return addr } func listen(t *testing.T, scheme, addr string, tlsInfo transport.TLSInfo) (ln net.Listener) { var err error if !tlsInfo.Empty() { ln, err = transport.NewListener(addr, scheme, &tlsInfo) } else { ln, err = net.Listen(scheme, addr) } require.NoError(t, err) return ln } func send(t *testing.T, data []byte, scheme, addr string, tlsInfo transport.TLSInfo) { var out net.Conn var err error if !tlsInfo.Empty() { tp, terr := transport.NewTransport(tlsInfo, 3*time.Second) require.NoError(t, terr) out, err = tp.DialContext(t.Context(), scheme, addr) } else { out, err = net.Dial(scheme, addr) } require.NoError(t, err) _, err = out.Write(data) require.NoError(t, err) require.NoError(t, out.Close()) } func receive(t *testing.T, ln net.Listener) (data []byte) { buf := bytes.NewBuffer(make([]byte, 0, 1024)) for { in, err := ln.Accept() require.NoError(t, err) var n int64 n, err = buf.ReadFrom(in) require.NoError(t, err) if n > 0 { break } } return buf.Bytes() } // Waits until a proxy is ready to serve. // Aborts test on proxy start-up error. func waitForServer(t *testing.T, s Server) { select { case <-s.Ready(): case err := <-s.Error(): t.Fatal(err) } } ================================================ FILE: pkg/report/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package report generates human-readable benchmark reports. package report ================================================ FILE: pkg/report/perfdash.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package report import ( "encoding/json" "fmt" "math" "os" "path/filepath" "strings" "time" ) type Metrics struct { Perc50 float64 `json:"Perc50"` Perc90 float64 `json:"Perc90"` Perc99 float64 `json:"Perc99"` } type Labels struct { Operation string `json:"Operation"` } type DataItem struct { Data Metrics `json:"data"` Labels Labels `json:"labels"` Unit string `json:"unit"` } type perfdashFormattedReport struct { Version string `json:"version"` DataItems []DataItem `json:"dataItems"` } func (r *report) writePerfDashReport(benchmarkOp string) { pcls, data := Percentiles(r.stats.Lats) pclsData := make(map[float64]float64) for i := 0; i < len(pcls); i++ { pclsData[pcls[i]] = data[i] * 1000 // Since the reported data is in seconds, convert to ms. } report := perfdashFormattedReport{ Version: "v1", DataItems: []DataItem{ { Data: Metrics{ Perc50: math.Round(pclsData[50]*10000) / 10000, Perc90: math.Round(pclsData[90]*10000) / 10000, Perc99: math.Round(pclsData[99]*10000) / 10000, }, Unit: "ms", Labels: Labels{ Operation: strings.ToUpper(benchmarkOp), }, }, }, } reportB, _ := json.MarshalIndent(report, "", " ") artifactsDir := os.Getenv("ARTIFACTS") if artifactsDir == "" { artifactsDir = "./_artifacts" } fileName := fmt.Sprintf("EtcdAPI_benchmark_%s_%s.json", benchmarkOp, time.Now().UTC().Format(time.RFC3339)) err := os.MkdirAll(artifactsDir, 0o755) if err != nil { fmt.Println("Error creating artifacts directory:", err) } destPath := filepath.Join(artifactsDir, fileName) err = os.WriteFile(destPath, reportB, 0o644) if err != nil { fmt.Println("Error writing to file:", err) } fmt.Println("Successfully created a JSON perf report at", destPath) } ================================================ FILE: pkg/report/report.go ================================================ // Copyright 2014 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // the file is borrowed from github.com/rakyll/boom/boomer/print.go package report import ( "fmt" "maps" "math" "slices" "sort" "strings" "time" ) const ( barChar = "∎" ) // Result describes the timings for an operation. type Result struct { Start time.Time End time.Time Err error Weight float64 } func (res *Result) Duration() time.Duration { return res.End.Sub(res.Start) } type report struct { generatePerfReport bool benchmarkOp string precision string results chan Result stats Stats sps *secondPoints } // Stats exposes results raw data. type Stats struct { AvgTotal float64 Fastest float64 Slowest float64 Average float64 Stddev float64 RPS float64 Total time.Duration ErrorDist map[string]int Lats []float64 TimeSeries TimeSeries } func (s *Stats) copy() Stats { ss := *s ss.ErrorDist = copyMap(ss.ErrorDist) ss.Lats = slices.Clone(ss.Lats) return ss } // Report processes a result stream until it is closed, then produces a // string with information about the consumed result data. type Report interface { Results() chan<- Result // Run returns results in print-friendly format. Run() <-chan string // Stats returns results in raw data. Stats() <-chan Stats } func NewReport(precision, benchmarkOp string, generatePerfReport bool) Report { return newReport(precision, benchmarkOp, generatePerfReport) } func newReport(precision, benchmarkOp string, generatePerfReport bool) *report { r := &report{ results: make(chan Result, 16), precision: precision, generatePerfReport: generatePerfReport, benchmarkOp: benchmarkOp, } r.stats.ErrorDist = make(map[string]int) return r } func NewReportSample(precision, benchmarkOp string, generatePerfReport bool) Report { r := NewReport(precision, benchmarkOp, generatePerfReport).(*report) r.sps = newSecondPoints() return r } func (r *report) Results() chan<- Result { return r.results } func (r *report) Run() <-chan string { donec := make(chan string, 1) go func() { defer close(donec) r.processResults() if r.generatePerfReport { r.writePerfDashReport(r.benchmarkOp) } donec <- r.String() }() return donec } func (r *report) Stats() <-chan Stats { donec := make(chan Stats, 1) go func() { defer close(donec) r.processResults() s := r.stats.copy() if r.sps != nil { s.TimeSeries = r.sps.getTimeSeries() } donec <- s }() return donec } func copyMap(m map[string]int) (c map[string]int) { c = make(map[string]int, len(m)) maps.Copy(c, m) return c } func (r *report) String() (s string) { if len(r.stats.Lats) > 0 { s += "\nSummary:\n" s += fmt.Sprintf(" Total:\t%s.\n", r.sec2str(r.stats.Total.Seconds())) s += fmt.Sprintf(" Slowest:\t%s.\n", r.sec2str(r.stats.Slowest)) s += fmt.Sprintf(" Fastest:\t%s.\n", r.sec2str(r.stats.Fastest)) s += fmt.Sprintf(" Average:\t%s.\n", r.sec2str(r.stats.Average)) s += fmt.Sprintf(" Stddev:\t%s.\n", r.sec2str(r.stats.Stddev)) s += fmt.Sprintf(" Requests/sec:\t"+r.precision+"\n", r.stats.RPS) s += r.histogram() s += r.sprintLatencies() if r.sps != nil { s += fmt.Sprintf("%v\n", r.sps.getTimeSeries()) } } if len(r.stats.ErrorDist) > 0 { s += r.errors() } return s } func (r *report) sec2str(sec float64) string { return fmt.Sprintf(r.precision+" secs", sec) } type reportRate struct{ *report } func NewReportRate(precision, benchmarkOp string, generatePerfReport bool) Report { return &reportRate{NewReport(precision, benchmarkOp, generatePerfReport).(*report)} } func (r *reportRate) String() string { return fmt.Sprintf(" Requests/sec:\t"+r.precision+"\n", r.stats.RPS) } func (r *report) processResult(res *Result) { if res.Err != nil { r.stats.ErrorDist[res.Err.Error()]++ return } dur := res.Duration() r.stats.Lats = append(r.stats.Lats, dur.Seconds()) r.stats.AvgTotal += dur.Seconds() if r.sps != nil { r.sps.Add(res.Start, dur) } } func (r *report) processResults() { st := time.Now() for res := range r.results { r.processResult(&res) } r.stats.Total = time.Since(st) r.stats.RPS = float64(len(r.stats.Lats)) / r.stats.Total.Seconds() r.stats.Average = r.stats.AvgTotal / float64(len(r.stats.Lats)) for i := range r.stats.Lats { dev := r.stats.Lats[i] - r.stats.Average r.stats.Stddev += dev * dev } r.stats.Stddev = math.Sqrt(r.stats.Stddev / float64(len(r.stats.Lats))) sort.Float64s(r.stats.Lats) if len(r.stats.Lats) > 0 { r.stats.Fastest = r.stats.Lats[0] r.stats.Slowest = r.stats.Lats[len(r.stats.Lats)-1] } } var pctls = []float64{10, 25, 50, 75, 90, 95, 99, 99.9} // Percentiles returns percentile distribution of float64 slice. func Percentiles(nums []float64) (pcs []float64, data []float64) { return pctls, percentiles(nums) } func percentiles(nums []float64) (data []float64) { data = make([]float64, len(pctls)) j := 0 n := len(nums) for i := 0; i < n && j < len(pctls); i++ { current := float64(i) * 100.0 / float64(n) if current >= pctls[j] { data[j] = nums[i] j++ } } return data } func (r *report) sprintLatencies() string { data := percentiles(r.stats.Lats) s := "\nLatency distribution:\n" for i := 0; i < len(pctls); i++ { if data[i] > 0 { s += fmt.Sprintf(" %v%% in %s.\n", pctls[i], r.sec2str(data[i])) } } return s } func (r *report) histogram() string { bc := 10 buckets := make([]float64, bc+1) counts := make([]int, bc+1) bs := (r.stats.Slowest - r.stats.Fastest) / float64(bc) for i := 0; i < bc; i++ { buckets[i] = r.stats.Fastest + bs*float64(i) } buckets[bc] = r.stats.Slowest var bi int var max int for i := 0; i < len(r.stats.Lats); { if r.stats.Lats[i] <= buckets[bi] { i++ counts[bi]++ if max < counts[bi] { max = counts[bi] } } else if bi < len(buckets)-1 { bi++ } } s := "\nResponse time histogram:\n" for i := 0; i < len(buckets); i++ { // Normalize bar lengths. var barLen int if max > 0 { barLen = counts[i] * 40 / max } s += fmt.Sprintf(" "+r.precision+" [%v]\t|%v\n", buckets[i], counts[i], strings.Repeat(barChar, barLen)) } return s } func (r *report) errors() string { s := "\nError distribution:\n" for err, num := range r.stats.ErrorDist { s += fmt.Sprintf(" [%d]\t%s\n", num, err) } return s } ================================================ FILE: pkg/report/report_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package report import ( "fmt" "reflect" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestPercentiles(t *testing.T) { nums := make([]float64, 100) nums[99] = 1 // 99-percentile (1 out of 100) data := percentiles(nums) require.InDeltaf(t, 1, data[len(pctls)-2], 0.0, "99-percentile expected 1, got %f", data[len(pctls)-2]) nums = make([]float64, 1000) nums[999] = 1 // 99.9-percentile (1 out of 1000) data = percentiles(nums) require.InDeltaf(t, 1, data[len(pctls)-1], 0.0, "99.9-percentile expected 1, got %f", data[len(pctls)-1]) } func TestReport(t *testing.T) { r := NewReportSample("%f", "", false) go func() { start := time.Now() for i := 0; i < 5; i++ { end := start.Add(time.Second) r.Results() <- Result{Start: start, End: end} start = end } r.Results() <- Result{Start: start, End: start.Add(time.Second), Err: fmt.Errorf("oops")} close(r.Results()) }() stats := <-r.Stats() stats.TimeSeries = nil // ignore timeseries since it uses wall clock wStats := Stats{ AvgTotal: 5.0, Fastest: 1.0, Slowest: 1.0, Average: 1.0, Stddev: 0.0, Total: stats.Total, RPS: 5.0 / stats.Total.Seconds(), ErrorDist: map[string]int{"oops": 1}, Lats: []float64{1.0, 1.0, 1.0, 1.0, 1.0}, } require.Truef(t, reflect.DeepEqual(stats, wStats), "got %+v, want %+v", stats, wStats) wstrs := []string{ "Stddev:\t0", "Average:\t1.0", "Slowest:\t1.0", "Fastest:\t1.0", } ss := <-r.Run() for i, ws := range wstrs { assert.Containsf(t, ss, ws, "#%d: stats string missing %s", i, ws) } } func TestWeightedReport(t *testing.T) { r := NewWeightedReport(NewReport("%f", "", false), "%f", "", false) go func() { start := time.Now() for i := 0; i < 5; i++ { end := start.Add(time.Second) r.Results() <- Result{Start: start, End: end, Weight: 2.0} start = end } r.Results() <- Result{Start: start, End: start.Add(time.Second), Err: fmt.Errorf("oops")} close(r.Results()) }() stats := <-r.Stats() stats.TimeSeries = nil // ignore timeseries since it uses wall clock wStats := Stats{ AvgTotal: 10.0, Fastest: 0.5, Slowest: 0.5, Average: 0.5, Stddev: 0.0, Total: stats.Total, RPS: 10.0 / stats.Total.Seconds(), ErrorDist: map[string]int{"oops": 1}, Lats: []float64{0.5, 0.5, 0.5, 0.5, 0.5}, } require.Truef(t, reflect.DeepEqual(stats, wStats), "got %+v, want %+v", stats, wStats) } ================================================ FILE: pkg/report/timeseries.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package report import ( "encoding/csv" "fmt" "log" "math" "sort" "strings" "sync" "time" ) type DataPoint struct { Timestamp int64 MinLatency time.Duration AvgLatency time.Duration MaxLatency time.Duration ThroughPut int64 } type TimeSeries []DataPoint func (t TimeSeries) Swap(i, j int) { t[i], t[j] = t[j], t[i] } func (t TimeSeries) Len() int { return len(t) } func (t TimeSeries) Less(i, j int) bool { return t[i].Timestamp < t[j].Timestamp } type secondPoint struct { minLatency time.Duration maxLatency time.Duration totalLatency time.Duration count int64 } type secondPoints struct { mu sync.Mutex tm map[int64]secondPoint } func newSecondPoints() *secondPoints { return &secondPoints{tm: make(map[int64]secondPoint)} } func (sp *secondPoints) Add(ts time.Time, lat time.Duration) { sp.mu.Lock() defer sp.mu.Unlock() tk := ts.Unix() if v, ok := sp.tm[tk]; !ok { sp.tm[tk] = secondPoint{minLatency: lat, maxLatency: lat, totalLatency: lat, count: 1} } else { if lat != time.Duration(0) { v.minLatency = min(v.minLatency, lat) } v.maxLatency = max(v.maxLatency, lat) v.totalLatency += lat v.count++ sp.tm[tk] = v } } func (sp *secondPoints) getTimeSeries() TimeSeries { sp.mu.Lock() defer sp.mu.Unlock() var ( minTs int64 = math.MaxInt64 maxTs int64 = -1 ) for k := range sp.tm { if minTs > k { minTs = k } if maxTs < k { maxTs = k } } for ti := minTs; ti < maxTs; ti++ { if _, ok := sp.tm[ti]; !ok { // fill-in empties sp.tm[ti] = secondPoint{totalLatency: 0, count: 0} } } var ( tslice = make(TimeSeries, len(sp.tm)) i int ) for k, v := range sp.tm { var lat time.Duration if v.count > 0 { lat = v.totalLatency / time.Duration(v.count) } tslice[i] = DataPoint{ Timestamp: k, MinLatency: v.minLatency, AvgLatency: lat, MaxLatency: v.maxLatency, ThroughPut: v.count, } i++ } sort.Sort(tslice) return tslice } func (t TimeSeries) String() string { buf := new(strings.Builder) wr := csv.NewWriter(buf) if err := wr.Write([]string{"UNIX-SECOND", "MIN-LATENCY-MS", "AVG-LATENCY-MS", "MAX-LATENCY-MS", "AVG-THROUGHPUT"}); err != nil { log.Fatal(err) } var rows [][]string for i := range t { row := []string{ fmt.Sprintf("%d", t[i].Timestamp), t[i].MinLatency.String(), t[i].AvgLatency.String(), t[i].MaxLatency.String(), fmt.Sprintf("%d", t[i].ThroughPut), } rows = append(rows, row) } if err := wr.WriteAll(rows); err != nil { log.Fatal(err) } wr.Flush() if err := wr.Error(); err != nil { log.Fatal(err) } return fmt.Sprintf("\nSample in one second (unix latency throughput):\n%s", buf.String()) } ================================================ FILE: pkg/report/timeseries_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package report import ( "testing" "time" "github.com/stretchr/testify/require" ) func TestGetTimeseries(t *testing.T) { sp := newSecondPoints() now := time.Now() sp.Add(now, time.Second) sp.Add(now.Add(5*time.Second), time.Second) n := sp.getTimeSeries().Len() require.GreaterOrEqualf(t, n, 3, "expected at 6 points of time series, got %s", sp.getTimeSeries()) // add a point with duplicate timestamp sp.Add(now, 3*time.Second) ts := sp.getTimeSeries() require.Equalf(t, time.Second, ts[0].MinLatency, "ts[0] min latency expected %v, got %s", time.Second, ts[0].MinLatency) require.Equalf(t, 2*time.Second, ts[0].AvgLatency, "ts[0] average latency expected %v, got %s", 2*time.Second, ts[0].AvgLatency) require.Equalf(t, 3*time.Second, ts[0].MaxLatency, "ts[0] max latency expected %v, got %s", 3*time.Second, ts[0].MaxLatency) } ================================================ FILE: pkg/report/weighted.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // the file is borrowed from github.com/rakyll/boom/boomer/print.go package report import ( "time" ) type weightedReport struct { baseReport Report report *report results chan Result weightTotal float64 } // NewWeightedReport returns a report that includes // both weighted and unweighted statistics. func NewWeightedReport(r Report, precision, benchmarkOp string, generatePerfReport bool) Report { return &weightedReport{ baseReport: r, report: newReport(precision, benchmarkOp, generatePerfReport), results: make(chan Result, 16), } } func (wr *weightedReport) Results() chan<- Result { return wr.results } func (wr *weightedReport) Run() <-chan string { donec := make(chan string, 2) go func() { defer close(donec) basec, rc := make(chan string, 1), make(chan Stats, 1) go func() { basec <- (<-wr.baseReport.Run()) }() go func() { rc <- (<-wr.report.Stats()) }() go wr.processResults() wr.report.stats = wr.reweighStat(<-rc) donec <- wr.report.String() donec <- (<-basec) }() return donec } func (wr *weightedReport) Stats() <-chan Stats { donec := make(chan Stats, 2) go func() { defer close(donec) basec, rc := make(chan Stats, 1), make(chan Stats, 1) go func() { basec <- (<-wr.baseReport.Stats()) }() go func() { rc <- (<-wr.report.Stats()) }() go wr.processResults() donec <- wr.reweighStat(<-rc) donec <- (<-basec) }() return donec } func (wr *weightedReport) processResults() { defer close(wr.report.results) defer close(wr.baseReport.Results()) for res := range wr.results { wr.processResult(res) wr.baseReport.Results() <- res } } func (wr *weightedReport) processResult(res Result) { if res.Err != nil { wr.report.results <- res return } if res.Weight == 0 { res.Weight = 1.0 } wr.weightTotal += res.Weight res.End = res.Start.Add(time.Duration(float64(res.End.Sub(res.Start)) / res.Weight)) res.Weight = 1.0 wr.report.results <- res } func (wr *weightedReport) reweighStat(s Stats) Stats { weightCoef := wr.weightTotal / float64(len(s.Lats)) // weight > 1 => processing more than one request s.RPS *= weightCoef s.AvgTotal *= weightCoef * weightCoef return s } ================================================ FILE: pkg/runtime/fds_linux.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package runtime implements utility functions for runtime systems. package runtime import ( "os" "syscall" ) func FDLimit() (uint64, error) { var rlimit syscall.Rlimit if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil { return 0, err } return rlimit.Cur, nil } func FDUsage() (uint64, error) { return countFiles("/proc/self/fd") } // countFiles reads the directory named by dirname and returns the count. func countFiles(dirname string) (uint64, error) { f, err := os.Open(dirname) if err != nil { return 0, err } list, err := f.Readdirnames(-1) f.Close() if err != nil { return 0, err } return uint64(len(list)), nil } ================================================ FILE: pkg/runtime/fds_other.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !linux package runtime import ( "fmt" "runtime" ) func FDLimit() (uint64, error) { return 0, fmt.Errorf("cannot get FDLimit on %s", runtime.GOOS) } func FDUsage() (uint64, error) { return 0, fmt.Errorf("cannot get FDUsage on %s", runtime.GOOS) } ================================================ FILE: pkg/schedule/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package schedule provides mechanisms and policies for scheduling units of work. package schedule ================================================ FILE: pkg/schedule/schedule.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schedule import ( "context" "sync" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/verify" ) type Job interface { Name() string Do(context.Context) } type job struct { name string do func(context.Context) } func (j job) Name() string { return j.name } func (j job) Do(ctx context.Context) { j.do(ctx) } func NewJob(name string, do func(ctx context.Context)) Job { return job{ name: name, do: do, } } // Scheduler can schedule jobs. type Scheduler interface { // Schedule asks the scheduler to schedule a job defined by the given func. // Schedule to a stopped scheduler might panic. Schedule(j Job) // Pending returns number of pending jobs Pending() int // Scheduled returns the number of scheduled jobs (excluding pending jobs) Scheduled() int // Finished returns the number of finished jobs Finished() int // WaitFinish waits until at least n job are finished and all pending jobs are finished. WaitFinish(n int) // Stop stops the scheduler. Stop() } type fifo struct { mu sync.Mutex resume chan struct{} scheduled int finished int pendings []Job ctx context.Context cancel context.CancelFunc finishCond *sync.Cond donec chan struct{} lg *zap.Logger } // NewFIFOScheduler returns a Scheduler that schedules jobs in FIFO // order sequentially func NewFIFOScheduler(lg *zap.Logger) Scheduler { verify.Assert(lg != nil, "the logger should not be nil") f := &fifo{ resume: make(chan struct{}, 1), donec: make(chan struct{}, 1), lg: lg, } f.finishCond = sync.NewCond(&f.mu) f.ctx, f.cancel = context.WithCancel(context.Background()) go f.run() return f } // Schedule schedules a job that will be ran in FIFO order sequentially. func (f *fifo) Schedule(j Job) { f.mu.Lock() defer f.mu.Unlock() if f.cancel == nil { panic("schedule: schedule to stopped scheduler") } if len(f.pendings) == 0 { select { case f.resume <- struct{}{}: default: } } f.pendings = append(f.pendings, j) } func (f *fifo) Pending() int { f.mu.Lock() defer f.mu.Unlock() return len(f.pendings) } func (f *fifo) Scheduled() int { f.mu.Lock() defer f.mu.Unlock() return f.scheduled } func (f *fifo) Finished() int { f.finishCond.L.Lock() defer f.finishCond.L.Unlock() return f.finished } func (f *fifo) WaitFinish(n int) { f.finishCond.L.Lock() for f.finished < n || len(f.pendings) != 0 { f.finishCond.Wait() } f.finishCond.L.Unlock() } // Stop stops the scheduler and cancels all pending jobs. func (f *fifo) Stop() { f.mu.Lock() f.cancel() f.cancel = nil f.mu.Unlock() <-f.donec } func (f *fifo) run() { defer func() { close(f.donec) close(f.resume) }() for { var todo Job f.mu.Lock() if len(f.pendings) != 0 { f.scheduled++ todo = f.pendings[0] } f.mu.Unlock() if todo == nil { select { case <-f.resume: case <-f.ctx.Done(): f.mu.Lock() pendings := f.pendings f.pendings = nil f.mu.Unlock() // clean up pending jobs for _, todo := range pendings { f.executeJob(todo, true) } return } } else { f.executeJob(todo, false) } } } func (f *fifo) executeJob(todo Job, updatedFinishedStats bool) { defer func() { if !updatedFinishedStats { f.finishCond.L.Lock() f.finished++ f.pendings = f.pendings[1:] f.finishCond.Broadcast() f.finishCond.L.Unlock() } if err := recover(); err != nil { f.lg.Panic("execute job failed", zap.String("job", todo.Name()), zap.Any("panic", err)) } }() todo.Do(f.ctx) } ================================================ FILE: pkg/schedule/schedule_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schedule import ( "context" "fmt" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" ) func TestFIFOSchedule(t *testing.T) { s := NewFIFOScheduler(zaptest.NewLogger(t)) defer s.Stop() next := 0 jobCreator := func(i int) Job { return NewJob(fmt.Sprintf("i_%d_increse", i), func(ctx context.Context) { defer func() { if err := recover(); err != nil { fmt.Println("err: ", err) } }() require.Equalf(t, next, i, "job#%d: got %d, want %d", i, next, i) next = i + 1 if next%3 == 0 { panic("fifo panic") } }) } var jobs []Job for i := 0; i < 100; i++ { jobs = append(jobs, jobCreator(i)) } for _, j := range jobs { s.Schedule(j) } s.WaitFinish(100) assert.Equalf(t, 100, s.Finished(), "finished = %d, want %d", s.Finished(), 100) } ================================================ FILE: pkg/stringutil/doc.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package stringutil exports string utility functions. package stringutil ================================================ FILE: pkg/stringutil/rand.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stringutil import ( "math/rand" ) // UniqueStrings returns a slice of randomly generated unique strings. func UniqueStrings(slen uint, n int) (ss []string) { exist := make(map[string]struct{}) ss = make([]string, 0, n) for len(ss) < n { s := RandString(slen) if _, ok := exist[s]; !ok { ss = append(ss, s) exist[s] = struct{}{} } } return ss } // RandomStrings returns a slice of randomly generated strings. func RandomStrings(slen uint, n int) (ss []string) { ss = make([]string, 0, n) for i := 0; i < n; i++ { ss = append(ss, RandString(slen)) } return ss } const chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" func RandString(l uint) string { s := make([]byte, l) for i := 0; i < int(l); i++ { s[i] = chars[rand.Intn(len(chars))] } return string(s) } ================================================ FILE: pkg/stringutil/rand_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stringutil import ( "sort" "testing" "github.com/stretchr/testify/require" ) func TestUniqueStrings(t *testing.T) { ss := UniqueStrings(10, 50) sort.Strings(ss) for i := 1; i < len(ss); i++ { require.NotEqualf(t, ss[i-1], ss[i], "ss[i-1] %q == ss[i] %q", ss[i-1], ss[i]) } } ================================================ FILE: pkg/traceutil/trace.go ================================================ // Copyright 2019 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package traceutil implements tracing utilities using "context". package traceutil import ( "context" "fmt" "math/rand" "strings" "time" "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" "go.uber.org/zap" ) const instrumentationScope = "go.etcd.io/etcd" var Tracer trace.Tracer = noop.NewTracerProvider().Tracer(instrumentationScope) func Init(tp trace.TracerProvider) { Tracer = tp.Tracer(instrumentationScope) } // TraceKey is used as a key of context for Trace. type TraceKey struct{} // StartTimeKey is used as a key of context for start time of operation. type StartTimeKey struct{} // Field is a kv pair to record additional details of the trace. type Field struct { Key string Value any } func (f *Field) format() string { return fmt.Sprintf("%s:%v; ", f.Key, f.Value) } func writeFields(fields []Field) string { if len(fields) == 0 { return "" } var buf strings.Builder buf.WriteString("{") for _, f := range fields { buf.WriteString(f.format()) } buf.WriteString("}") return buf.String() } type Trace struct { operation string lg *zap.Logger fields []Field startTime time.Time steps []step stepDisabled bool isEmpty bool } type step struct { time time.Time msg string fields []Field isSubTraceStart bool isSubTraceEnd bool } func newTrace(op string, lg *zap.Logger, fields ...Field) *Trace { return &Trace{operation: op, lg: lg, startTime: time.Now(), fields: fields} } // TODO returns a non-nil, empty Trace func TODO() *Trace { return &Trace{isEmpty: true} } func Get(ctx context.Context) *Trace { if trace, ok := ctx.Value(TraceKey{}).(*Trace); ok && trace != nil { return trace } return TODO() } // EnsureTrace creates a new trace if needed and adds it to the context. func EnsureTrace(ctx context.Context, lg *zap.Logger, operation string, fields ...Field) (context.Context, *Trace) { trace := Get(ctx) if trace.IsEmpty() { trace = newTrace(operation, lg, fields..., ) ctx = context.WithValue(ctx, TraceKey{}, trace) } return ctx, trace } func (t *Trace) GetStartTime() time.Time { return t.startTime } func (t *Trace) SetStartTime(time time.Time) { t.startTime = time } func (t *Trace) InsertStep(at int, time time.Time, msg string, fields ...Field) { newStep := step{time: time, msg: msg, fields: fields} if at < len(t.steps) { t.steps = append(t.steps[:at+1], t.steps[at:]...) t.steps[at] = newStep } else { t.steps = append(t.steps, newStep) } } // StartSubTrace adds step to trace as a start sign of sublevel trace // All steps in the subtrace will log out the input fields of this function func (t *Trace) StartSubTrace(fields ...Field) { t.steps = append(t.steps, step{fields: fields, isSubTraceStart: true}) } // StopSubTrace adds step to trace as a end sign of sublevel trace // All steps in the subtrace will log out the input fields of this function func (t *Trace) StopSubTrace(fields ...Field) { t.steps = append(t.steps, step{fields: fields, isSubTraceEnd: true}) } // Step adds step to trace func (t *Trace) Step(msg string, fields ...Field) { if !t.stepDisabled { t.steps = append(t.steps, step{time: time.Now(), msg: msg, fields: fields}) } } // StepWithFunction will measure the input function as a single step func (t *Trace) StepWithFunction(f func(), msg string, fields ...Field) { t.disableStep() f() t.enableStep() t.Step(msg, fields...) } func (t *Trace) AddField(fields ...Field) { for _, f := range fields { if !t.updateFieldIfExist(f) { t.fields = append(t.fields, f) } } } func (t *Trace) IsEmpty() bool { return t.isEmpty } // Log dumps all steps in the Trace func (t *Trace) Log() { t.LogWithStepThreshold(0) } // LogIfLong dumps logs if the duration is longer than threshold func (t *Trace) LogIfLong(threshold time.Duration) { if time.Since(t.startTime) > threshold { stepThreshold := threshold / time.Duration(len(t.steps)+1) t.LogWithStepThreshold(stepThreshold) } } // LogAllStepsIfLong dumps all logs if the duration is longer than threshold func (t *Trace) LogAllStepsIfLong(threshold time.Duration) { if time.Since(t.startTime) > threshold { t.LogWithStepThreshold(0) } } // LogWithStepThreshold only dumps step whose duration is longer than step threshold func (t *Trace) LogWithStepThreshold(threshold time.Duration) { msg, fs := t.logInfo(threshold) if t.lg != nil { t.lg.Info(msg, fs...) } } func (t *Trace) logInfo(threshold time.Duration) (string, []zap.Field) { endTime := time.Now() totalDuration := endTime.Sub(t.startTime) traceNum := rand.Int31() msg := fmt.Sprintf("trace[%d] %s", traceNum, t.operation) var steps []string lastStepTime := t.startTime for i := 0; i < len(t.steps); i++ { tstep := t.steps[i] // add subtrace common fields which defined at the beginning to each sub-steps if tstep.isSubTraceStart { for j := i + 1; j < len(t.steps) && !t.steps[j].isSubTraceEnd; j++ { t.steps[j].fields = append(tstep.fields, t.steps[j].fields...) } continue } // add subtrace common fields which defined at the end to each sub-steps if tstep.isSubTraceEnd { for j := i - 1; j >= 0 && !t.steps[j].isSubTraceStart; j-- { t.steps[j].fields = append(tstep.fields, t.steps[j].fields...) } continue } } for i := 0; i < len(t.steps); i++ { tstep := t.steps[i] if tstep.isSubTraceStart || tstep.isSubTraceEnd { continue } stepDuration := tstep.time.Sub(lastStepTime) if stepDuration > threshold { steps = append(steps, fmt.Sprintf("trace[%d] '%v' %s (duration: %v)", traceNum, tstep.msg, writeFields(tstep.fields), stepDuration)) } lastStepTime = tstep.time } fs := []zap.Field{ zap.String("detail", writeFields(t.fields)), zap.Duration("duration", totalDuration), zap.Time("start", t.startTime), zap.Time("end", endTime), zap.Strings("steps", steps), zap.Int("step_count", len(steps)), } return msg, fs } func (t *Trace) updateFieldIfExist(f Field) bool { for i, v := range t.fields { if v.Key == f.Key { t.fields[i].Value = f.Value return true } } return false } // disableStep sets the flag to prevent the trace from adding steps func (t *Trace) disableStep() { t.stepDisabled = true } // enableStep re-enable the trace to add steps func (t *Trace) enableStep() { t.stepDisabled = false } ================================================ FILE: pkg/traceutil/trace_test.go ================================================ // Copyright 2019 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package traceutil import ( "bytes" "context" "fmt" "os" "path/filepath" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/logutil" ) func TestGet(t *testing.T) { traceForTest := &Trace{operation: "Test"} tests := []struct { name string inputCtx context.Context outputTrace *Trace }{ { name: "When the context does not have trace", inputCtx: t.Context(), outputTrace: TODO(), }, { name: "When the context has trace", inputCtx: context.WithValue(t.Context(), TraceKey{}, traceForTest), outputTrace: traceForTest, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { trace := Get(tt.inputCtx) assert.NotNilf(t, trace, "Expected %v; Got nil", tt.outputTrace) if tt.outputTrace == nil || trace.operation != tt.outputTrace.operation { t.Errorf("Expected %v; Got %v", tt.outputTrace, trace) } }) } } func TestCreate(t *testing.T) { var ( op = "Test" steps = []string{"Step1, Step2"} fields = []Field{ {"traceKey1", "traceValue1"}, {"traceKey2", "traceValue2"}, } stepFields = []Field{ {"stepKey1", "stepValue2"}, {"stepKey2", "stepValue2"}, } ) _, trace := EnsureTrace(t.Context(), nil, op, fields[0], fields[1]) assert.Equalf(t, trace.operation, op, "Expected %v; Got %v", op, trace.operation) for i, f := range trace.fields { assert.Equalf(t, f.Key, fields[i].Key, "Expected %v; Got %v", fields[i].Key, f.Key) assert.Equalf(t, f.Value, fields[i].Value, "Expected %v; Got %v", fields[i].Value, f.Value) } for i, v := range steps { trace.Step(v, stepFields[i]) } for i, v := range trace.steps { assert.Equalf(t, steps[i], v.msg, "Expected %v; Got %v", steps[i], v.msg) assert.Equalf(t, stepFields[i].Key, v.fields[0].Key, "Expected %v; Got %v", stepFields[i].Key, v.fields[0].Key) assert.Equalf(t, stepFields[i].Value, v.fields[0].Value, "Expected %v; Got %v", stepFields[i].Value, v.fields[0].Value) } } func TestLog(t *testing.T) { tests := []struct { name string trace *Trace fields []Field expectedMsg []string }{ { name: "When dump all logs", trace: &Trace{ operation: "Test", startTime: time.Now().Add(-100 * time.Millisecond), steps: []step{ {time: time.Now().Add(-80 * time.Millisecond), msg: "msg1"}, {time: time.Now().Add(-50 * time.Millisecond), msg: "msg2"}, }, }, expectedMsg: []string{ "msg1", "msg2", }, }, { name: "When trace has fields", trace: &Trace{ operation: "Test", startTime: time.Now().Add(-100 * time.Millisecond), steps: []step{ { time: time.Now().Add(-80 * time.Millisecond), msg: "msg1", fields: []Field{{"stepKey1", "stepValue1"}}, }, { time: time.Now().Add(-50 * time.Millisecond), msg: "msg2", fields: []Field{{"stepKey2", "stepValue2"}}, }, }, }, fields: []Field{ {"traceKey1", "traceValue1"}, {"count", 1}, }, expectedMsg: []string{ "Test", "msg1", "msg2", "traceKey1:traceValue1", "count:1", "stepKey1:stepValue1", "stepKey2:stepValue2", "\"step_count\":2", }, }, { name: "When trace has subtrace", trace: &Trace{ operation: "Test", startTime: time.Now().Add(-100 * time.Millisecond), steps: []step{ { time: time.Now().Add(-80 * time.Millisecond), msg: "msg1", fields: []Field{{"stepKey1", "stepValue1"}}, }, { fields: []Field{{"beginSubTrace", "true"}}, isSubTraceStart: true, }, { time: time.Now().Add(-50 * time.Millisecond), msg: "submsg", fields: []Field{{"subStepKey", "subStepValue"}}, }, { fields: []Field{{"endSubTrace", "true"}}, isSubTraceEnd: true, }, { time: time.Now().Add(-30 * time.Millisecond), msg: "msg2", fields: []Field{{"stepKey2", "stepValue2"}}, }, }, }, fields: []Field{ {"traceKey1", "traceValue1"}, {"count", 1}, }, expectedMsg: []string{ "Test", "msg1", "msg2", "submsg", "traceKey1:traceValue1", "count:1", "stepKey1:stepValue1", "stepKey2:stepValue2", "subStepKey:subStepValue", "beginSubTrace:true", "endSubTrace:true", "\"step_count\":3", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { logPath := filepath.Join(os.TempDir(), fmt.Sprintf("test-log-%d", time.Now().UnixNano())) defer os.RemoveAll(logPath) lcfg := logutil.DefaultZapLoggerConfig lcfg.OutputPaths = []string{logPath} lcfg.ErrorOutputPaths = []string{logPath} lg, _ := lcfg.Build() for _, f := range tt.fields { tt.trace.AddField(f) } tt.trace.lg = lg tt.trace.Log() data, err := os.ReadFile(logPath) require.NoError(t, err) for _, msg := range tt.expectedMsg { assert.Truef(t, bytes.Contains(data, []byte(msg)), "Expected to find %v in log", msg) } }) } } func TestLogIfLong(t *testing.T) { tests := []struct { name string threshold time.Duration trace *Trace expectedMsg []string }{ { name: "When the duration is smaller than threshold", threshold: 200 * time.Millisecond, trace: &Trace{ operation: "Test", startTime: time.Now().Add(-100 * time.Millisecond), steps: []step{ {time: time.Now().Add(-50 * time.Millisecond), msg: "msg1"}, {time: time.Now(), msg: "msg2"}, }, }, expectedMsg: []string{}, }, { name: "When the duration is longer than threshold", threshold: 50 * time.Millisecond, trace: &Trace{ operation: "Test", startTime: time.Now().Add(-100 * time.Millisecond), steps: []step{ {time: time.Now().Add(-50 * time.Millisecond), msg: "msg1"}, {time: time.Now(), msg: "msg2"}, }, }, expectedMsg: []string{ "msg1", "msg2", }, }, { name: "When not all steps are longer than step threshold", threshold: 50 * time.Millisecond, trace: &Trace{ operation: "Test", startTime: time.Now().Add(-100 * time.Millisecond), steps: []step{ {time: time.Now(), msg: "msg1"}, {time: time.Now(), msg: "msg2"}, }, }, expectedMsg: []string{ "msg1", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { logPath := filepath.Join(os.TempDir(), fmt.Sprintf("test-log-%d", time.Now().UnixNano())) defer os.RemoveAll(logPath) lcfg := logutil.DefaultZapLoggerConfig lcfg.OutputPaths = []string{logPath} lcfg.ErrorOutputPaths = []string{logPath} lg, _ := lcfg.Build() tt.trace.lg = lg tt.trace.LogIfLong(tt.threshold) data, err := os.ReadFile(logPath) require.NoError(t, err) for _, msg := range tt.expectedMsg { assert.Truef(t, bytes.Contains(data, []byte(msg)), "Expected to find %v in log", msg) } }) } } ================================================ FILE: pkg/wait/wait.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package wait provides utility functions for polling, listening using Go // channel. package wait import ( "log" "sync" ) const ( // To avoid lock contention we use an array of list struct (rw mutex & map) // for the id argument, we apply mod operation and uses its remainder to // index into the array and find the corresponding element. defaultListElementLength = 64 ) // Wait is an interface that provides the ability to wait and trigger events that // are associated with IDs. type Wait interface { // Register waits returns a chan that waits on the given ID. // The chan will be triggered when Trigger is called with // the same ID. Register(id uint64) <-chan any // Trigger triggers the waiting chans with the given ID. Trigger(id uint64, x any) IsRegistered(id uint64) bool } type list struct { e []listElement } type listElement struct { l sync.RWMutex m map[uint64]chan any } // New creates a Wait. func New() Wait { res := list{ e: make([]listElement, defaultListElementLength), } for i := 0; i < len(res.e); i++ { res.e[i].m = make(map[uint64]chan any) } return &res } func (w *list) Register(id uint64) <-chan any { idx := id % defaultListElementLength newCh := make(chan any, 1) w.e[idx].l.Lock() defer w.e[idx].l.Unlock() if _, ok := w.e[idx].m[id]; !ok { w.e[idx].m[id] = newCh } else { log.Panicf("dup id %x", id) } return newCh } func (w *list) Trigger(id uint64, x any) { idx := id % defaultListElementLength w.e[idx].l.Lock() ch := w.e[idx].m[id] delete(w.e[idx].m, id) w.e[idx].l.Unlock() if ch != nil { ch <- x close(ch) } } func (w *list) IsRegistered(id uint64) bool { idx := id % defaultListElementLength w.e[idx].l.RLock() defer w.e[idx].l.RUnlock() _, ok := w.e[idx].m[id] return ok } type waitWithResponse struct { ch <-chan any } func NewWithResponse(ch <-chan any) Wait { return &waitWithResponse{ch: ch} } func (w *waitWithResponse) Register(id uint64) <-chan any { return w.ch } func (w *waitWithResponse) Trigger(id uint64, x any) {} func (w *waitWithResponse) IsRegistered(id uint64) bool { panic("waitWithResponse.IsRegistered() shouldn't be called") } ================================================ FILE: pkg/wait/wait_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wait import ( "fmt" "testing" "time" "github.com/stretchr/testify/assert" ) func TestWait(t *testing.T) { const eid = 1 wt := New() ch := wt.Register(eid) wt.Trigger(eid, "foo") v := <-ch g, w := fmt.Sprintf("%v (%T)", v, v), "foo (string)" assert.Equalf(t, g, w, "<-ch = %v, want %v", g, w) if g := <-ch; g != nil { t.Errorf("unexpected non-nil value: %v (%T)", g, g) } } func TestRegisterDupPanic(t *testing.T) { const eid = 1 wt := New() ch1 := wt.Register(eid) panicC := make(chan struct{}, 1) func() { defer func() { if r := recover(); r != nil { panicC <- struct{}{} } }() wt.Register(eid) }() select { case <-panicC: case <-time.After(1 * time.Second): t.Errorf("failed to receive panic") } wt.Trigger(eid, "foo") <-ch1 } func TestTriggerDupSuppression(t *testing.T) { const eid = 1 wt := New() ch := wt.Register(eid) wt.Trigger(eid, "foo") wt.Trigger(eid, "bar") v := <-ch g, w := fmt.Sprintf("%v (%T)", v, v), "foo (string)" assert.Equalf(t, g, w, "<-ch = %v, want %v", g, w) if g := <-ch; g != nil { t.Errorf("unexpected non-nil value: %v (%T)", g, g) } } func TestIsRegistered(t *testing.T) { wt := New() wt.Register(0) wt.Register(1) wt.Register(2) for i := uint64(0); i < 3; i++ { assert.Truef(t, wt.IsRegistered(i), "event ID %d isn't registered", i) } assert.Falsef(t, wt.IsRegistered(4), "event ID 4 shouldn't be registered") wt.Trigger(0, "foo") assert.Falsef(t, wt.IsRegistered(0), "event ID 0 is already triggered, shouldn't be registered") } ================================================ FILE: pkg/wait/wait_time.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wait import "sync" type WaitTime interface { // Wait returns a chan that waits on the given logical deadline. // The chan will be triggered when Trigger is called with a // deadline that is later than or equal to the one it is waiting for. Wait(deadline uint64) <-chan struct{} // Trigger triggers all the waiting chans with an equal or earlier logical deadline. Trigger(deadline uint64) } var closec chan struct{} func init() { closec = make(chan struct{}); close(closec) } type timeList struct { l sync.Mutex lastTriggerDeadline uint64 m map[uint64]chan struct{} } func NewTimeList() *timeList { return &timeList{m: make(map[uint64]chan struct{})} } func (tl *timeList) Wait(deadline uint64) <-chan struct{} { tl.l.Lock() defer tl.l.Unlock() if tl.lastTriggerDeadline >= deadline { return closec } ch := tl.m[deadline] if ch == nil { ch = make(chan struct{}) tl.m[deadline] = ch } return ch } func (tl *timeList) Trigger(deadline uint64) { tl.l.Lock() defer tl.l.Unlock() tl.lastTriggerDeadline = deadline for t, ch := range tl.m { if t <= deadline { delete(tl.m, t) close(ch) } } } ================================================ FILE: pkg/wait/wait_time_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wait import ( "testing" "time" ) func TestWaitTime(t *testing.T) { wt := NewTimeList() ch1 := wt.Wait(1) wt.Trigger(2) select { case <-ch1: default: t.Fatalf("cannot receive from ch as expected") } ch2 := wt.Wait(4) wt.Trigger(3) select { case <-ch2: t.Fatalf("unexpected to receive from ch2") default: } wt.Trigger(4) select { case <-ch2: default: t.Fatalf("cannot receive from ch2 as expected") } select { // wait on a triggered deadline case <-wt.Wait(4): default: t.Fatalf("unexpected blocking when wait on triggered deadline") } } func TestWaitTestStress(t *testing.T) { chs := make([]<-chan struct{}, 0) wt := NewTimeList() for i := 0; i <= 10000; i++ { chs = append(chs, wt.Wait(uint64(i))) } wt.Trigger(10000) for _, ch := range chs { select { case <-ch: case <-time.After(time.Second): t.Fatalf("cannot receive from ch as expected") } } } func BenchmarkWaitTime(b *testing.B) { wt := NewTimeList() for i := 0; i < b.N; i++ { wt.Wait(1) } } func BenchmarkTriggerAnd10KWaitTime(b *testing.B) { for i := 0; i < b.N; i++ { wt := NewTimeList() for j := 0; j <= 10000; j++ { wt.Wait(uint64(j)) } wt.Trigger(10000) } } ================================================ FILE: scripts/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners approvers: - ivanvc # Ivan Valdes ================================================ FILE: scripts/README ================================================ scripts for etcd development ================================================ FILE: scripts/benchmark_test.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This script runs a benchmark on a locally started etcd server set -euo pipefail source ./scripts/test_lib.sh COMMON_BENCHMARK_FLAGS="--report-perfdash" if [[ $# -lt 1 ]]; then echo "Usage: $0 [tester args...]" exit 1 fi BENCHMARK_NAME="$1" ARGS="${*:2}" echo "Starting the etcd server..." # Create a directory for etcd data under /tmp/etcd mkdir -p /tmp/etcd DATA_DIR=$(mktemp -d /tmp/etcd/data-XXXXXX) ./bin/etcd --data-dir="$DATA_DIR" > /tmp/etcd.log 2>&1 & etcd_pid=$! trap 'log_warning -e "Stopping etcd server - PID $etcd_pid"; kill $etcd_pid 2>/dev/null; rm -rf $DATA_DIR; log_success "Deleted the contents from $DATA_DIR related to benchmark test"' EXIT # Wait until etcd becomes healthy for retry in {1..10}; do if ./bin/etcdctl endpoint health --cluster> /dev/null 2>&1; then log_success -e "\\netcd is healthy" break fi log_warning -e "\\nWaiting for etcd to be healthy..." sleep 1 if [[ $retry -eq 10 ]]; then log_error -e "\\nFailed to confirm etcd health after $retry attempts. Check /tmp/etcd.log for more information" exit 1 fi done log_success -e "etcd process is running with PID $etcd_pid" log_callout -e "\\nPerforming benchmark $BENCHMARK_NAME with arguments: $ARGS" read -r -a TESTER_OPTIONS <<< "$ARGS" log_callout "Running: benchmark $BENCHMARK_NAME ${TESTER_OPTIONS[*]} $COMMON_BENCHMARK_FLAGS" benchmark "$BENCHMARK_NAME" "${TESTER_OPTIONS[@]}" $COMMON_BENCHMARK_FLAGS log_callout "Completed: benchmark $BENCHMARK_NAME ${TESTER_OPTIONS[*]} $COMMON_BENCHMARK_FLAGS" ================================================ FILE: scripts/build-binary.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail source ./scripts/test_lib.sh VER=${1:-} REPOSITORY="${REPOSITORY:-git@github.com:etcd-io/etcd.git}" if [ -z "$VER" ]; then echo "Usage: ${0} VERSION" >> /dev/stderr exit 255 fi function setup_env { local ver=${1} local proj=${2} if [ ! -d "${proj}" ]; then run git clone "${REPOSITORY}" fi pushd "${proj}" >/dev/null run git fetch --all run git checkout "${ver}" popd >/dev/null } function package { local target=${1} local srcdir="${2}/bin" local ccdir="${srcdir}/${GOOS}_${GOARCH}" if [ -d "${ccdir}" ]; then srcdir="${ccdir}" fi local ext="" if [ "${GOOS}" == "windows" ]; then ext=".exe" fi for bin in etcd etcdctl etcdutl; do cp "${srcdir}/${bin}" "${target}/${bin}${ext}" done cp etcd/README.md "${target}"/README.md cp etcd/etcdctl/README.md "${target}"/README-etcdctl.md cp etcd/etcdctl/READMEv2.md "${target}"/READMEv2-etcdctl.md cp etcd/etcdutl/README.md "${target}"/README-etcdutl.md cp -R etcd/Documentation "${target}"/Documentation } function main { local proj="etcd" mkdir -p release cd release setup_env "${VER}" "${proj}" local tarcmd=tar if [[ $(go env GOOS) == "darwin" ]]; then echo "Please use linux machine for release builds." exit 1 fi for os in darwin windows linux; do export GOOS=${os} TARGET_ARCHS=("amd64") if [ ${GOOS} == "linux" ]; then TARGET_ARCHS+=("arm64") TARGET_ARCHS+=("ppc64le") TARGET_ARCHS+=("s390x") fi if [ ${GOOS} == "darwin" ]; then TARGET_ARCHS+=("arm64") fi for TARGET_ARCH in "${TARGET_ARCHS[@]}"; do export GOARCH=${TARGET_ARCH} pushd etcd >/dev/null GO_LDFLAGS="-s -w" ./scripts/build.sh popd >/dev/null TARGET="etcd-${VER}-${GOOS}-${GOARCH}" mkdir "${TARGET}" package "${TARGET}" "${proj}" if [ ${GOOS} == "linux" ]; then ${tarcmd} cfz "${TARGET}.tar.gz" "${TARGET}" echo "Wrote release/${TARGET}.tar.gz" else zip -qr "${TARGET}.zip" "${TARGET}" echo "Wrote release/${TARGET}.zip" fi done done } main ================================================ FILE: scripts/build-docker.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail if [ "$#" -ne 1 ]; then echo "Usage: $0 VERSION" >&2 exit 1 fi VERSION=${1} if [ -z "$VERSION" ]; then echo "Usage: ${0} VERSION" >&2 exit 1 fi ARCH=$(go env GOARCH) VERSION="${VERSION}-${ARCH}" DOCKERFILE="Dockerfile" if [ -z "${BINARYDIR:-}" ]; then RELEASE="etcd-${1}"-$(go env GOOS)-${ARCH} BINARYDIR="${RELEASE}" TARFILE="${RELEASE}.tar.gz" TARURL="https://github.com/etcd-io/etcd/releases/download/${1}/${TARFILE}" if ! curl -f -L -o "${TARFILE}" "${TARURL}" ; then echo "Failed to download ${TARURL}." exit 1 fi tar -zvxf "${TARFILE}" fi BINARYDIR=${BINARYDIR:-.} BUILDDIR=${BUILDDIR:-.} IMAGEDIR=${BUILDDIR}/image-docker mkdir -p "${IMAGEDIR}"/var/etcd mkdir -p "${IMAGEDIR}"/var/lib/etcd cp "${BINARYDIR}"/etcd "${BINARYDIR}"/etcdctl "${BINARYDIR}"/etcdutl "${IMAGEDIR}" cat ./"${DOCKERFILE}" > "${IMAGEDIR}"/Dockerfile if [ -z "${TAG:-}" ]; then # Fix incorrect image "Architecture" using buildkit # From https://stackoverflow.com/q/72144329/ DOCKER_BUILDKIT=1 docker build --build-arg="ARCH=${ARCH}" -t "gcr.io/etcd-development/etcd:${VERSION}" "${IMAGEDIR}" DOCKER_BUILDKIT=1 docker build --build-arg="ARCH=${ARCH}" -t "quay.io/coreos/etcd:${VERSION}" "${IMAGEDIR}" else docker build -t "${TAG}:${VERSION}" "${IMAGEDIR}" fi ================================================ FILE: scripts/build-release.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Build all release binaries and images to directory ./release. # Run from repository root. # set -euo pipefail source ./scripts/test_lib.sh VERSION=${1:-} if [ -z "${VERSION}" ]; then echo "Usage: ${0} VERSION" >> /dev/stderr exit 255 fi if ! command -v docker >/dev/null; then echo "cannot find docker" exit 1 fi ETCD_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. pushd "${ETCD_ROOT}" >/dev/null log_callout "Building etcd binary..." ./scripts/build-binary.sh "${VERSION}" for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do log_callout "Building ${TARGET_ARCH} docker image..." GOOS=linux GOARCH=${TARGET_ARCH} BINARYDIR=release/etcd-${VERSION}-linux-${TARGET_ARCH} BUILDDIR=release ./scripts/build-docker.sh "${VERSION}" done popd >/dev/null ================================================ FILE: scripts/build.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This scripts build the etcd binaries # To build the tools, run `build_tools.sh` set -euo pipefail source ./scripts/test_lib.sh source ./scripts/build_lib.sh # only build when called directly, not sourced if echo "$0" | grep -E "build(.sh)?$" >/dev/null; then run_build etcd_build fi ================================================ FILE: scripts/build_lib.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail source ./scripts/test_lib.sh GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound") VERSION_SYMBOL="${ROOT_MODULE}/api/v3/version.GitSHA" # use go env if noset GOOS=${GOOS:-$(go env GOOS)} GOARCH=${GOARCH:-$(go env GOARCH)} GO_BUILD_FLAGS=${GO_BUILD_FLAGS:-} CGO_ENABLED="${CGO_ENABLED:-0}" # Set GO_LDFLAGS="-s" for building without symbols for debugging. # shellcheck disable=SC2206 GO_LDFLAGS=(${GO_LDFLAGS:-} "-X=${VERSION_SYMBOL}=${GIT_SHA}") GO_GCFLAGS=${GO_GCFLAGS:-} GO_BUILD_ENV=("CGO_ENABLED=${CGO_ENABLED}" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS}" "GOOS=${GOOS}" "GOARCH=${GOARCH}") etcd_build() { out="bin" if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi run rm -f "${out}/etcd" ( cd ./server # Static compilation is useful when etcd is run in a container. $GO_BUILD_FLAGS is OK # shellcheck disable=SC2086 run env "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \ -trimpath \ -installsuffix=cgo \ "-ldflags=${GO_LDFLAGS[*]}" \ -gcflags="${GO_GCFLAGS}" \ -o="../${out}/etcd" . || return 2 ) || return 2 run rm -f "${out}/etcdutl" # shellcheck disable=SC2086 ( cd ./etcdutl run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \ -trimpath \ -installsuffix=cgo \ "-ldflags=${GO_LDFLAGS[*]}" \ -gcflags="${GO_GCFLAGS}" \ -o="../${out}/etcdutl" . || return 2 ) || return 2 run rm -f "${out}/etcdctl" # shellcheck disable=SC2086 ( cd ./etcdctl run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" "${GO_BUILD_ENV[@]}" go build $GO_BUILD_FLAGS \ -trimpath \ -installsuffix=cgo \ "-ldflags=${GO_LDFLAGS[*]}" \ -gcflags="${GO_GCFLAGS}" \ -o="../${out}/etcdctl" . || return 2 ) || return 2 # Verify whether symbol we overwrote exists # For cross-compiling we cannot run: ${out}/etcd --version | grep -q "Git SHA: ${GIT_SHA}" # We need symbols to do this check: if [[ "${GO_LDFLAGS[*]}" != *"-s"* ]]; then go tool nm "${out}/etcd" | grep "${VERSION_SYMBOL}" > /dev/null if [[ "${PIPESTATUS[*]}" != "0 0" ]]; then log_error "FAIL: Symbol ${VERSION_SYMBOL} not found in binary: ${out}/etcd" return 2 fi fi } tools_build() { out="bin" if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi tools_path="tools/benchmark tools/etcd-dump-db tools/etcd-dump-logs tools/local-tester/bridge" for tool in ${tools_path} do echo "Building" "'${tool}'"... run rm -f "${out}/${tool}" # shellcheck disable=SC2086 run env GO_BUILD_FLAGS="${GO_BUILD_FLAGS}" CGO_ENABLED=${CGO_ENABLED} go build ${GO_BUILD_FLAGS} \ -trimpath \ -installsuffix=cgo \ "-ldflags=${GO_LDFLAGS[*]}" \ -o="${out}/${tool}" "./${tool}" || return 2 done } run_build() { echo Running "$1" if $1; then log_success "SUCCESS: $1 (GOARCH=${GOARCH})" else log_error "FAIL: $1 (GOARCH=${GOARCH})" exit 2 fi } ================================================ FILE: scripts/build_tools.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail source ./scripts/test_lib.sh source ./scripts/build_lib.sh run_build tools_build ================================================ FILE: scripts/codecov_upload.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Script used to collect and upload test coverage. set -o pipefail # We try to upload whatever we have: mkdir -p bin curl -sf -o ./bin/codecov.sh https://codecov.io/bash bash ./bin/codecov.sh -f "${COVERDIR}/all.coverprofile" \ -cF all \ -C "${PULL_PULL_SHA:-${PULL_BASE_SHA}}" \ -r "${REPO_OWNER}/${REPO_NAME}" \ -P "${PULL_NUMBER}" \ -b "${BUILD_ID}" \ -B "${PULL_BASE_REF}" \ -N "${PULL_BASE_SHA}" ================================================ FILE: scripts/etcd_version_annotations.txt ================================================ authpb.Permission: "" authpb.Permission.READ: "" authpb.Permission.READWRITE: "" authpb.Permission.Type: "" authpb.Permission.WRITE: "" authpb.Permission.key: "" authpb.Permission.permType: "" authpb.Permission.range_end: "" authpb.Role: "" authpb.Role.keyPermission: "" authpb.Role.name: "" authpb.User: "" authpb.User.name: "" authpb.User.options: "" authpb.User.password: "" authpb.User.roles: "" authpb.UserAddOptions: "" authpb.UserAddOptions.no_password: "" etcdserverpb.AlarmMember: "3.0" etcdserverpb.AlarmMember.alarm: "" etcdserverpb.AlarmMember.memberID: "" etcdserverpb.AlarmRequest: "3.0" etcdserverpb.AlarmRequest.ACTIVATE: "" etcdserverpb.AlarmRequest.AlarmAction: "3.0" etcdserverpb.AlarmRequest.DEACTIVATE: "" etcdserverpb.AlarmRequest.GET: "" etcdserverpb.AlarmRequest.action: "" etcdserverpb.AlarmRequest.alarm: "" etcdserverpb.AlarmRequest.memberID: "" etcdserverpb.AlarmResponse: "3.0" etcdserverpb.AlarmResponse.alarms: "" etcdserverpb.AlarmResponse.header: "" etcdserverpb.AlarmType: "3.0" etcdserverpb.AuthDisableRequest: "3.0" etcdserverpb.AuthDisableResponse: "3.0" etcdserverpb.AuthDisableResponse.header: "" etcdserverpb.AuthEnableRequest: "3.0" etcdserverpb.AuthEnableResponse: "3.0" etcdserverpb.AuthEnableResponse.header: "" etcdserverpb.AuthRoleAddRequest: "3.0" etcdserverpb.AuthRoleAddRequest.name: "" etcdserverpb.AuthRoleAddResponse: "3.0" etcdserverpb.AuthRoleAddResponse.header: "" etcdserverpb.AuthRoleDeleteRequest: "3.0" etcdserverpb.AuthRoleDeleteRequest.role: "" etcdserverpb.AuthRoleDeleteResponse: "3.0" etcdserverpb.AuthRoleDeleteResponse.header: "" etcdserverpb.AuthRoleGetRequest: "3.0" etcdserverpb.AuthRoleGetRequest.role: "" etcdserverpb.AuthRoleGetResponse: "" etcdserverpb.AuthRoleGetResponse.header: "3.0" etcdserverpb.AuthRoleGetResponse.perm: "3.0" etcdserverpb.AuthRoleGrantPermissionRequest: "3.0" etcdserverpb.AuthRoleGrantPermissionRequest.name: "" etcdserverpb.AuthRoleGrantPermissionRequest.perm: "" etcdserverpb.AuthRoleGrantPermissionResponse: "3.0" etcdserverpb.AuthRoleGrantPermissionResponse.header: "" etcdserverpb.AuthRoleListRequest: "3.0" etcdserverpb.AuthRoleListResponse: "3.0" etcdserverpb.AuthRoleListResponse.header: "" etcdserverpb.AuthRoleListResponse.roles: "" etcdserverpb.AuthRoleRevokePermissionRequest: "3.0" etcdserverpb.AuthRoleRevokePermissionRequest.key: "" etcdserverpb.AuthRoleRevokePermissionRequest.range_end: "" etcdserverpb.AuthRoleRevokePermissionRequest.role: "" etcdserverpb.AuthRoleRevokePermissionResponse: "3.0" etcdserverpb.AuthRoleRevokePermissionResponse.header: "" etcdserverpb.AuthStatusRequest: "3.5" etcdserverpb.AuthStatusResponse: "3.5" etcdserverpb.AuthStatusResponse.authRevision: "" etcdserverpb.AuthStatusResponse.enabled: "" etcdserverpb.AuthStatusResponse.header: "" etcdserverpb.AuthUserAddRequest: "3.0" etcdserverpb.AuthUserAddRequest.hashedPassword: "3.5" etcdserverpb.AuthUserAddRequest.name: "" etcdserverpb.AuthUserAddRequest.options: "3.4" etcdserverpb.AuthUserAddRequest.password: "" etcdserverpb.AuthUserAddResponse: "3.0" etcdserverpb.AuthUserAddResponse.header: "" etcdserverpb.AuthUserChangePasswordRequest: "3.0" etcdserverpb.AuthUserChangePasswordRequest.hashedPassword: "3.5" etcdserverpb.AuthUserChangePasswordRequest.name: "" etcdserverpb.AuthUserChangePasswordRequest.password: "" etcdserverpb.AuthUserChangePasswordResponse: "3.0" etcdserverpb.AuthUserChangePasswordResponse.header: "" etcdserverpb.AuthUserDeleteRequest: "3.0" etcdserverpb.AuthUserDeleteRequest.name: "" etcdserverpb.AuthUserDeleteResponse: "3.0" etcdserverpb.AuthUserDeleteResponse.header: "" etcdserverpb.AuthUserGetRequest: "3.0" etcdserverpb.AuthUserGetRequest.name: "" etcdserverpb.AuthUserGetResponse: "3.0" etcdserverpb.AuthUserGetResponse.header: "" etcdserverpb.AuthUserGetResponse.roles: "" etcdserverpb.AuthUserGrantRoleRequest: "3.0" etcdserverpb.AuthUserGrantRoleRequest.role: "" etcdserverpb.AuthUserGrantRoleRequest.user: "" etcdserverpb.AuthUserGrantRoleResponse: "3.0" etcdserverpb.AuthUserGrantRoleResponse.header: "" etcdserverpb.AuthUserListRequest: "3.0" etcdserverpb.AuthUserListResponse: "3.0" etcdserverpb.AuthUserListResponse.header: "" etcdserverpb.AuthUserListResponse.users: "" etcdserverpb.AuthUserRevokeRoleRequest: "3.0" etcdserverpb.AuthUserRevokeRoleRequest.name: "" etcdserverpb.AuthUserRevokeRoleRequest.role: "" etcdserverpb.AuthUserRevokeRoleResponse: "3.0" etcdserverpb.AuthUserRevokeRoleResponse.header: "" etcdserverpb.AuthenticateRequest: "3.0" etcdserverpb.AuthenticateRequest.name: "" etcdserverpb.AuthenticateRequest.password: "" etcdserverpb.AuthenticateResponse: "3.0" etcdserverpb.AuthenticateResponse.header: "" etcdserverpb.AuthenticateResponse.token: "" etcdserverpb.CORRUPT: "3.3" etcdserverpb.CompactionRequest: "3.0" etcdserverpb.CompactionRequest.physical: "" etcdserverpb.CompactionRequest.revision: "" etcdserverpb.CompactionResponse: "3.0" etcdserverpb.CompactionResponse.header: "" etcdserverpb.Compare: "3.0" etcdserverpb.Compare.CREATE: "" etcdserverpb.Compare.CompareResult: "3.0" etcdserverpb.Compare.CompareTarget: "3.0" etcdserverpb.Compare.EQUAL: "" etcdserverpb.Compare.GREATER: "" etcdserverpb.Compare.LEASE: "3.3" etcdserverpb.Compare.LESS: "" etcdserverpb.Compare.MOD: "" etcdserverpb.Compare.NOT_EQUAL: "3.1" etcdserverpb.Compare.VALUE: "" etcdserverpb.Compare.VERSION: "" etcdserverpb.Compare.create_revision: "" etcdserverpb.Compare.key: "" etcdserverpb.Compare.lease: "3.3" etcdserverpb.Compare.mod_revision: "" etcdserverpb.Compare.range_end: "3.3" etcdserverpb.Compare.result: "" etcdserverpb.Compare.target: "" etcdserverpb.Compare.value: "" etcdserverpb.Compare.version: "" etcdserverpb.DefragmentRequest: "3.0" etcdserverpb.DefragmentResponse: "3.0" etcdserverpb.DefragmentResponse.header: "" etcdserverpb.DeleteRangeRequest: "3.0" etcdserverpb.DeleteRangeRequest.key: "" etcdserverpb.DeleteRangeRequest.prev_kv: "3.1" etcdserverpb.DeleteRangeRequest.range_end: "" etcdserverpb.DeleteRangeResponse: "3.0" etcdserverpb.DeleteRangeResponse.deleted: "" etcdserverpb.DeleteRangeResponse.header: "" etcdserverpb.DeleteRangeResponse.prev_kvs: "3.1" etcdserverpb.DowngradeInfo: "" etcdserverpb.DowngradeInfo.enabled: "" etcdserverpb.DowngradeInfo.targetVersion: "" etcdserverpb.DowngradeRequest: "3.5" etcdserverpb.DowngradeRequest.CANCEL: "" etcdserverpb.DowngradeRequest.DowngradeAction: "3.5" etcdserverpb.DowngradeRequest.ENABLE: "" etcdserverpb.DowngradeRequest.VALIDATE: "" etcdserverpb.DowngradeRequest.action: "" etcdserverpb.DowngradeRequest.version: "" etcdserverpb.DowngradeResponse: "3.5" etcdserverpb.DowngradeResponse.header: "" etcdserverpb.DowngradeResponse.version: "" etcdserverpb.DowngradeVersionTestRequest: "3.6" etcdserverpb.DowngradeVersionTestRequest.ver: "" etcdserverpb.EmptyResponse: "" etcdserverpb.HashKVRequest: "3.3" etcdserverpb.HashKVRequest.revision: "" etcdserverpb.HashKVResponse: "3.3" etcdserverpb.HashKVResponse.compact_revision: "" etcdserverpb.HashKVResponse.hash: "" etcdserverpb.HashKVResponse.hash_revision: "3.6" etcdserverpb.HashKVResponse.header: "" etcdserverpb.HashRequest: "3.0" etcdserverpb.HashResponse: "3.0" etcdserverpb.HashResponse.hash: "" etcdserverpb.HashResponse.header: "" etcdserverpb.InternalAuthenticateRequest: "3.0" etcdserverpb.InternalAuthenticateRequest.name: "" etcdserverpb.InternalAuthenticateRequest.password: "" etcdserverpb.InternalAuthenticateRequest.simple_token: "" etcdserverpb.InternalRaftRequest: "3.0" etcdserverpb.InternalRaftRequest.ID: "" etcdserverpb.InternalRaftRequest.alarm: "" etcdserverpb.InternalRaftRequest.auth_disable: "" etcdserverpb.InternalRaftRequest.auth_enable: "" etcdserverpb.InternalRaftRequest.auth_role_add: "" etcdserverpb.InternalRaftRequest.auth_role_delete: "" etcdserverpb.InternalRaftRequest.auth_role_get: "" etcdserverpb.InternalRaftRequest.auth_role_grant_permission: "" etcdserverpb.InternalRaftRequest.auth_role_list: "" etcdserverpb.InternalRaftRequest.auth_role_revoke_permission: "" etcdserverpb.InternalRaftRequest.auth_status: "3.5" etcdserverpb.InternalRaftRequest.auth_user_add: "" etcdserverpb.InternalRaftRequest.auth_user_change_password: "" etcdserverpb.InternalRaftRequest.auth_user_delete: "" etcdserverpb.InternalRaftRequest.auth_user_get: "" etcdserverpb.InternalRaftRequest.auth_user_grant_role: "" etcdserverpb.InternalRaftRequest.auth_user_list: "" etcdserverpb.InternalRaftRequest.auth_user_revoke_role: "" etcdserverpb.InternalRaftRequest.authenticate: "" etcdserverpb.InternalRaftRequest.cluster_member_attr_set: "3.5" etcdserverpb.InternalRaftRequest.cluster_version_set: "3.5" etcdserverpb.InternalRaftRequest.compaction: "" etcdserverpb.InternalRaftRequest.delete_range: "" etcdserverpb.InternalRaftRequest.downgrade_info_set: "3.5" etcdserverpb.InternalRaftRequest.downgrade_version_test: "3.6" etcdserverpb.InternalRaftRequest.header: "" etcdserverpb.InternalRaftRequest.lease_checkpoint: "3.4" etcdserverpb.InternalRaftRequest.lease_grant: "" etcdserverpb.InternalRaftRequest.lease_revoke: "" etcdserverpb.InternalRaftRequest.put: "" etcdserverpb.InternalRaftRequest.range: "" etcdserverpb.InternalRaftRequest.txn: "" etcdserverpb.LeaseCheckpoint: "3.4" etcdserverpb.LeaseCheckpoint.ID: "" etcdserverpb.LeaseCheckpoint.remaining_TTL: "" etcdserverpb.LeaseCheckpointRequest: "3.4" etcdserverpb.LeaseCheckpointRequest.checkpoints: "" etcdserverpb.LeaseCheckpointResponse: "3.4" etcdserverpb.LeaseCheckpointResponse.header: "" etcdserverpb.LeaseGrantRequest: "3.0" etcdserverpb.LeaseGrantRequest.ID: "" etcdserverpb.LeaseGrantRequest.TTL: "" etcdserverpb.LeaseGrantResponse: "3.0" etcdserverpb.LeaseGrantResponse.ID: "" etcdserverpb.LeaseGrantResponse.TTL: "" etcdserverpb.LeaseGrantResponse.error: "" etcdserverpb.LeaseGrantResponse.header: "" etcdserverpb.LeaseKeepAliveRequest: "3.0" etcdserverpb.LeaseKeepAliveRequest.ID: "" etcdserverpb.LeaseKeepAliveResponse: "3.0" etcdserverpb.LeaseKeepAliveResponse.ID: "" etcdserverpb.LeaseKeepAliveResponse.TTL: "" etcdserverpb.LeaseKeepAliveResponse.header: "" etcdserverpb.LeaseLeasesRequest: "3.3" etcdserverpb.LeaseLeasesResponse: "3.3" etcdserverpb.LeaseLeasesResponse.header: "" etcdserverpb.LeaseLeasesResponse.leases: "" etcdserverpb.LeaseRevokeRequest: "3.0" etcdserverpb.LeaseRevokeRequest.ID: "" etcdserverpb.LeaseRevokeResponse: "3.0" etcdserverpb.LeaseRevokeResponse.header: "" etcdserverpb.LeaseStatus: "3.3" etcdserverpb.LeaseStatus.ID: "" etcdserverpb.LeaseTimeToLiveRequest: "3.1" etcdserverpb.LeaseTimeToLiveRequest.ID: "" etcdserverpb.LeaseTimeToLiveRequest.keys: "" etcdserverpb.LeaseTimeToLiveResponse: "3.1" etcdserverpb.LeaseTimeToLiveResponse.ID: "" etcdserverpb.LeaseTimeToLiveResponse.TTL: "" etcdserverpb.LeaseTimeToLiveResponse.grantedTTL: "" etcdserverpb.LeaseTimeToLiveResponse.header: "" etcdserverpb.LeaseTimeToLiveResponse.keys: "" etcdserverpb.Member: "3.0" etcdserverpb.Member.ID: "" etcdserverpb.Member.clientURLs: "" etcdserverpb.Member.isLearner: "3.4" etcdserverpb.Member.name: "" etcdserverpb.Member.peerURLs: "" etcdserverpb.MemberAddRequest: "3.0" etcdserverpb.MemberAddRequest.isLearner: "3.4" etcdserverpb.MemberAddRequest.peerURLs: "" etcdserverpb.MemberAddResponse: "3.0" etcdserverpb.MemberAddResponse.header: "" etcdserverpb.MemberAddResponse.member: "" etcdserverpb.MemberAddResponse.members: "" etcdserverpb.MemberListRequest: "3.0" etcdserverpb.MemberListRequest.linearizable: "3.5" etcdserverpb.MemberListResponse: "3.0" etcdserverpb.MemberListResponse.header: "" etcdserverpb.MemberListResponse.members: "" etcdserverpb.MemberPromoteRequest: "3.4" etcdserverpb.MemberPromoteRequest.ID: "" etcdserverpb.MemberPromoteResponse: "3.4" etcdserverpb.MemberPromoteResponse.header: "" etcdserverpb.MemberPromoteResponse.members: "" etcdserverpb.MemberRemoveRequest: "3.0" etcdserverpb.MemberRemoveRequest.ID: "" etcdserverpb.MemberRemoveResponse: "3.0" etcdserverpb.MemberRemoveResponse.header: "" etcdserverpb.MemberRemoveResponse.members: "" etcdserverpb.MemberUpdateRequest: "3.0" etcdserverpb.MemberUpdateRequest.ID: "" etcdserverpb.MemberUpdateRequest.peerURLs: "" etcdserverpb.MemberUpdateResponse: "3.0" etcdserverpb.MemberUpdateResponse.header: "" etcdserverpb.MemberUpdateResponse.members: "3.1" etcdserverpb.Metadata: "" etcdserverpb.Metadata.ClusterID: "" etcdserverpb.Metadata.NodeID: "" etcdserverpb.MoveLeaderRequest: "3.3" etcdserverpb.MoveLeaderRequest.targetID: "" etcdserverpb.MoveLeaderResponse: "3.3" etcdserverpb.MoveLeaderResponse.header: "" etcdserverpb.NONE: "" etcdserverpb.NOSPACE: "" etcdserverpb.PutRequest: "3.0" etcdserverpb.PutRequest.ignore_lease: "3.2" etcdserverpb.PutRequest.ignore_value: "3.2" etcdserverpb.PutRequest.key: "" etcdserverpb.PutRequest.lease: "" etcdserverpb.PutRequest.prev_kv: "3.1" etcdserverpb.PutRequest.value: "" etcdserverpb.PutResponse: "3.0" etcdserverpb.PutResponse.header: "" etcdserverpb.PutResponse.prev_kv: "3.1" etcdserverpb.RangeRequest: "3.0" etcdserverpb.RangeRequest.ASCEND: "" etcdserverpb.RangeRequest.CREATE: "" etcdserverpb.RangeRequest.DESCEND: "" etcdserverpb.RangeRequest.KEY: "" etcdserverpb.RangeRequest.MOD: "" etcdserverpb.RangeRequest.NONE: "" etcdserverpb.RangeRequest.SortOrder: "3.0" etcdserverpb.RangeRequest.SortTarget: "3.0" etcdserverpb.RangeRequest.VALUE: "" etcdserverpb.RangeRequest.VERSION: "" etcdserverpb.RangeRequest.count_only: "" etcdserverpb.RangeRequest.key: "" etcdserverpb.RangeRequest.keys_only: "" etcdserverpb.RangeRequest.limit: "" etcdserverpb.RangeRequest.max_create_revision: "3.1" etcdserverpb.RangeRequest.max_mod_revision: "3.1" etcdserverpb.RangeRequest.min_create_revision: "3.1" etcdserverpb.RangeRequest.min_mod_revision: "3.1" etcdserverpb.RangeRequest.range_end: "" etcdserverpb.RangeRequest.revision: "" etcdserverpb.RangeRequest.serializable: "" etcdserverpb.RangeRequest.sort_order: "" etcdserverpb.RangeRequest.sort_target: "" etcdserverpb.RangeResponse: "3.0" etcdserverpb.RangeResponse.count: "" etcdserverpb.RangeResponse.header: "" etcdserverpb.RangeResponse.kvs: "" etcdserverpb.RangeResponse.more: "" etcdserverpb.RequestHeader: "3.0" etcdserverpb.RequestHeader.ID: "" etcdserverpb.RequestHeader.auth_revision: "3.1" etcdserverpb.RequestHeader.username: "" etcdserverpb.RequestOp: "3.0" etcdserverpb.RequestOp.request_delete_range: "" etcdserverpb.RequestOp.request_put: "" etcdserverpb.RequestOp.request_range: "" etcdserverpb.RequestOp.request_txn: "3.3" etcdserverpb.ResponseHeader: "3.0" etcdserverpb.ResponseHeader.cluster_id: "" etcdserverpb.ResponseHeader.member_id: "" etcdserverpb.ResponseHeader.raft_term: "" etcdserverpb.ResponseHeader.revision: "" etcdserverpb.ResponseOp: "3.0" etcdserverpb.ResponseOp.response_delete_range: "" etcdserverpb.ResponseOp.response_put: "" etcdserverpb.ResponseOp.response_range: "" etcdserverpb.ResponseOp.response_txn: "3.3" etcdserverpb.SnapshotRequest: "3.3" etcdserverpb.SnapshotResponse: "3.3" etcdserverpb.SnapshotResponse.blob: "" etcdserverpb.SnapshotResponse.header: "" etcdserverpb.SnapshotResponse.remaining_bytes: "" etcdserverpb.SnapshotResponse.version: "3.6" etcdserverpb.StatusRequest: "3.0" etcdserverpb.StatusResponse: "3.0" etcdserverpb.StatusResponse.dbSize: "" etcdserverpb.StatusResponse.dbSizeInUse: "3.4" etcdserverpb.StatusResponse.dbSizeQuota: "3.6" etcdserverpb.StatusResponse.downgradeInfo: "3.6" etcdserverpb.StatusResponse.errors: "3.4" etcdserverpb.StatusResponse.header: "" etcdserverpb.StatusResponse.isLearner: "3.4" etcdserverpb.StatusResponse.leader: "" etcdserverpb.StatusResponse.raftAppliedIndex: "3.4" etcdserverpb.StatusResponse.raftIndex: "" etcdserverpb.StatusResponse.raftTerm: "" etcdserverpb.StatusResponse.storageVersion: "3.6" etcdserverpb.StatusResponse.version: "" etcdserverpb.TxnRequest: "3.0" etcdserverpb.TxnRequest.compare: "" etcdserverpb.TxnRequest.failure: "" etcdserverpb.TxnRequest.success: "" etcdserverpb.TxnResponse: "3.0" etcdserverpb.TxnResponse.header: "" etcdserverpb.TxnResponse.responses: "" etcdserverpb.TxnResponse.succeeded: "" etcdserverpb.WatchCancelRequest: "3.1" etcdserverpb.WatchCancelRequest.watch_id: "3.1" etcdserverpb.WatchCreateRequest: "3.0" etcdserverpb.WatchCreateRequest.FilterType: "3.1" etcdserverpb.WatchCreateRequest.NODELETE: "" etcdserverpb.WatchCreateRequest.NOPUT: "" etcdserverpb.WatchCreateRequest.filters: "3.1" etcdserverpb.WatchCreateRequest.fragment: "3.4" etcdserverpb.WatchCreateRequest.key: "" etcdserverpb.WatchCreateRequest.prev_kv: "3.1" etcdserverpb.WatchCreateRequest.progress_notify: "" etcdserverpb.WatchCreateRequest.range_end: "" etcdserverpb.WatchCreateRequest.start_revision: "" etcdserverpb.WatchCreateRequest.watch_id: "3.4" etcdserverpb.WatchProgressRequest: "3.4" etcdserverpb.WatchRequest: "3.0" etcdserverpb.WatchRequest.cancel_request: "" etcdserverpb.WatchRequest.create_request: "" etcdserverpb.WatchRequest.progress_request: "3.4" etcdserverpb.WatchResponse: "3.0" etcdserverpb.WatchResponse.cancel_reason: "3.4" etcdserverpb.WatchResponse.canceled: "" etcdserverpb.WatchResponse.compact_revision: "" etcdserverpb.WatchResponse.created: "" etcdserverpb.WatchResponse.events: "" etcdserverpb.WatchResponse.fragment: "3.4" etcdserverpb.WatchResponse.header: "" etcdserverpb.WatchResponse.watch_id: "" membershippb.Attributes: "3.5" membershippb.Attributes.client_urls: "" membershippb.Attributes.name: "" membershippb.ClusterMemberAttrSetRequest: "3.5" membershippb.ClusterMemberAttrSetRequest.member_ID: "" membershippb.ClusterMemberAttrSetRequest.member_attributes: "" membershippb.ClusterVersionSetRequest: "3.5" membershippb.ClusterVersionSetRequest.ver: "" membershippb.DowngradeInfoSetRequest: "3.5" membershippb.DowngradeInfoSetRequest.enabled: "" membershippb.DowngradeInfoSetRequest.ver: "" membershippb.Member: "3.5" membershippb.Member.ID: "" membershippb.Member.member_attributes: "" membershippb.Member.raft_attributes: "" membershippb.RaftAttributes: "3.5" membershippb.RaftAttributes.is_learner: "" membershippb.RaftAttributes.peer_urls: "" mvccpb.Event: "" mvccpb.Event.DELETE: "" mvccpb.Event.EventType: "" mvccpb.Event.PUT: "" mvccpb.Event.kv: "" mvccpb.Event.prev_kv: "" mvccpb.Event.type: "" mvccpb.KeyValue: "" mvccpb.KeyValue.create_revision: "" mvccpb.KeyValue.key: "" mvccpb.KeyValue.lease: "" mvccpb.KeyValue.mod_revision: "" mvccpb.KeyValue.value: "" mvccpb.KeyValue.version: "" pb.GoFeatures: "" pb.GoFeatures.APILevel: "" pb.GoFeatures.API_HYBRID: "" pb.GoFeatures.API_LEVEL_UNSPECIFIED: "" pb.GoFeatures.API_OPAQUE: "" pb.GoFeatures.API_OPEN: "" pb.GoFeatures.STRIP_ENUM_PREFIX_GENERATE_BOTH: "" pb.GoFeatures.STRIP_ENUM_PREFIX_KEEP: "" pb.GoFeatures.STRIP_ENUM_PREFIX_STRIP: "" pb.GoFeatures.STRIP_ENUM_PREFIX_UNSPECIFIED: "" pb.GoFeatures.StripEnumPrefix: "" pb.GoFeatures.api_level: "" pb.GoFeatures.legacy_unmarshal_json_enum: "" pb.GoFeatures.strip_enum_prefix: "" walpb.Record: "" walpb.Record.crc: "" walpb.Record.data: "" walpb.Record.type: "" walpb.Snapshot: "" walpb.Snapshot.conf_state: "" walpb.Snapshot.index: "" walpb.Snapshot.term: "" ================================================ FILE: scripts/fix/bom.sh ================================================ #!/usr/bin/env bash # Copyright 2026 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ETCD_ROOT_DIR=${ETCD_ROOT_DIR:-$(git rev-parse --show-toplevel)} source "${ETCD_ROOT_DIR}/scripts/test_lib.sh" log_callout "Generating bill of materials..." _bom_modules=() load_workspace_relative_modules_for_bom _bom_modules # Internally license-bill-of-materials tends to modify go.sum run cp go.sum go.sum.tmp || exit 2 run cp go.mod go.mod.tmp || exit 2 # Intentionally run the command once first, so it fetches dependencies. The exit code on the first # run in a just cloned repository is always dirty. GOOS=linux run_go_tool github.com/appscodelabs/license-bill-of-materials \ --override-file ./bill-of-materials.override.json "${_bom_modules[@]}" &>/dev/null || true # BOM file should be generated for linux. Otherwise running this command on other operating systems such as OSX # results in certain dependencies being excluded from the BOM file, such as procfs. # For more info, https://github.com/etcd-io/etcd/issues/19665 output=$(GOOS=linux run_go_tool github.com/appscodelabs/license-bill-of-materials \ --override-file ./bill-of-materials.override.json \ "${_bom_modules[@]}") code="$?" run cp go.sum.tmp go.sum || exit 2 run cp go.mod.tmp go.mod || exit 2 if [ "${code}" -ne 0 ]; then log_error -e "license-bill-of-materials (code: ${code}) failed with:\\n${output}" exit 255 fi echo "${output}" > bill-of-materials.json log_success "bill-of-materials.json generated" ================================================ FILE: scripts/fix/mod-tidy.sh ================================================ #!/usr/bin/env bash # Copyright 2026 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail ETCD_ROOT_DIR=${ETCD_ROOT_DIR:-$(git rev-parse --show-toplevel)} source "${ETCD_ROOT_DIR}/scripts/test_lib.sh" log_callout "Tidying go.mod files" run_for_workspace_modules run sh -c "rm -f ./go.sum && go mod tidy" log_success "go.mod files tidied" ================================================ FILE: scripts/fix/shell_ws.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Fixes whitespaces in bash scripts. set -euo pipefail ETCD_ROOT_DIR=${ETCD_ROOT_DIR:-$(git rev-parse --show-toplevel)} source "${ETCD_ROOT_DIR}/scripts/test_lib.sh" function main { local TAB=$'\t' log_callout "Fixing whitespaces in bash scripts" # Makes sure all bash scripts do use ' ' (double space) for indention. run find "${ETCD_ROOT_DIR}" -name '*.sh' -exec sed -i.bak "s|${TAB}| |g" {} \; run find "${ETCD_ROOT_DIR}" -name '*.sh.bak' -delete } # only run when called directly, not sourced if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then main fi ================================================ FILE: scripts/fix/yamllint.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Fixes linter issues in YAML files. set -euo pipefail ETCD_ROOT_DIR=${ETCD_ROOT_DIR:-$(git rev-parse --show-toplevel)} source "${ETCD_ROOT_DIR}/scripts/test_lib.sh" function main { run_go_tool github.com/google/yamlfmt/cmd/yamlfmt \ -conf "${ETCD_ROOT_DIR}/tools/.yamlfmt" . } # only run when called directly, not sourced if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then main fi ================================================ FILE: scripts/fuzzing.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail source ./scripts/test_lib.sh GO_CMD="go" fuzz_time=${FUZZ_TIME:-"300s"} target_path=${TARGET_PATH:-"./server/etcdserver/api/v3rpc"} TARGETS="FuzzTxnRangeRequest FuzzTxnPutRequest FuzzTxnDeleteRangeRequest" for target in ${TARGETS}; do log_callout -e "\\nExecuting fuzzing with target ${target} in $target_path with a timeout of $fuzz_time\\n" run pushd "${target_path}" $GO_CMD test -fuzz "${target}" -fuzztime "${fuzz_time}" run popd log_success -e "\\COMPLETED: fuzzing with target $target in $target_path \\n" done ================================================ FILE: scripts/genproto.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Generate all etcd protobuf bindings. # Run from repository root directory named etcd. # set -euo pipefail shopt -s globstar if ! [[ "$0" =~ scripts/genproto.sh ]]; then echo "must be run from repository root" exit 255 fi if [ -z "${OS:-}" ]; then OS=$(uname -s | tr '[:upper:]' '[:lower:]') fi # Set SED variable if LANG=C sed --help 2>&1 | grep -q GNU; then SED="sed" elif command -v gsed &>/dev/null; then SED="gsed" elif [ "$OS" == "darwin" ]; then echo "You are on Mac, running: brew install gnu-sed" brew install gnu-sed SED="/opt/homebrew/opt/gnu-sed/libexec/gnubin/sed" else echo "Failed to find GNU sed as sed or gsed." >&2 exit 1 fi source ./scripts/test_lib.sh PATH=$(pwd)/bin:$(go env GOPATH)/bin:$PATH export PATH if [[ $(protoc --version | cut -f2 -d' ') != "3.20.3" ]]; then echo "Could not find protoc 3.20.3, installing now..." arch=$(go env GOARCH) case ${arch} in "amd64") file="x86_64" ;; "arm64") file="aarch_64" ;; *) echo "Unsupported architecture: ${arch}" exit 255 ;; esac protoc_download_file="protoc-3.20.3-linux-${file}.zip" if [ "$OS" == "darwin" ]; then # protoc-3.20.3 does not have pre-built binaries for darwin_arm64. Thanks to Rosetta, we could use x86_64 binary. protoc_download_file="protoc-3.20.3-osx-x86_64.zip" fi download_url="https://github.com/protocolbuffers/protobuf/releases/download/v3.20.3/${protoc_download_file}" echo "Running on ${OS} ${arch}. Downloading ${protoc_download_file}" mkdir -p bin wget ${download_url} && unzip -p ${protoc_download_file} bin/protoc > tmpFile && mv tmpFile bin/protoc rm ${protoc_download_file} chmod +x bin/protoc echo "Now running: $(protoc --version)" fi GOFAST_BIN=$(tool_get_bin github.com/gogo/protobuf/protoc-gen-gofast) GOGEN_BIN=$(tool_get_bin google.golang.org/protobuf/cmd/protoc-gen-go) GOGENGRPC_BIN=$(tool_get_bin google.golang.org/grpc/cmd/protoc-gen-go-grpc) GRPC_GATEWAY_BIN=$(tool_get_bin github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway) OPENAPIV2_BIN=$(tool_get_bin github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2) GOGOPROTO_ROOT="$(tool_pkg_dir github.com/gogo/protobuf/proto)/.." GRPC_GATEWAY_ROOT="$(tool_pkg_dir github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway)/.." RAFT_ROOT="$(tool_pkg_dir go.etcd.io/raft/v3/raftpb)/.." GOOGLEAPI_ROOT=$(mktemp -d -t 'googleapi.XXXXX') module_mapping_list=( Mraftpb/raft.proto=go.etcd.io/raft/v3/raftpb Mgoogle/protobuf/descriptor.proto=google.golang.org/protobuf/types/descriptorpb Mgoogle/protobuf/struct.proto=google.golang.org/protobuf/types/known/structpb ) module_mappings=$(IFS=$','; echo "${module_mapping_list[*]}" ) readonly googleapi_commit=0adf469dcd7822bf5bc058a7b0217f5558a75643 function cleanup_googleapi() { rm -rf "${GOOGLEAPI_ROOT}" } trap cleanup_googleapi EXIT # TODO(ahrtr): use buf (https://github.com/bufbuild/buf) to manage the protobuf dependencies? function download_googleapi() { run pushd "${GOOGLEAPI_ROOT}" run git init run git remote add upstream https://github.com/googleapis/googleapis.git run git fetch upstream "${googleapi_commit}" run git reset --hard FETCH_HEAD run popd } download_googleapi echo echo "Resolved binary and packages versions:" echo " - protoc-gen-gofast: ${GOFAST_BIN}" echo " - protoc-gen-go: ${GOGEN_BIN}" echo " - protoc-gen-go-grpc: ${GOGENGRPC_BIN}" echo " - protoc-gen-grpc-gateway: ${GRPC_GATEWAY_BIN}" echo " - openapiv2: ${OPENAPIV2_BIN}" echo " - gogoproto-root: ${GOGOPROTO_ROOT}" echo " - grpc-gateway-root: ${GRPC_GATEWAY_ROOT}" echo " - raft-root: ${RAFT_ROOT}" GOGOPROTO_PATH="${GOGOPROTO_ROOT}:${GOGOPROTO_ROOT}/protobuf" # directories containing protos to be built DIRS="./server/storage/wal/walpb ./api/etcdserverpb ./server/etcdserver/api/snap/snappb ./api/mvccpb ./server/lease/leasepb ./api/authpb ./server/etcdserver/api/v3lock/v3lockpb ./server/etcdserver/api/v3election/v3electionpb ./api/membershippb ./api/versionpb" log_callout -e "\\nRunning gofast (gogo) proto generation..." for dir in ${DIRS}; do run pushd "${dir}" run protoc --gofast_out=. -I=".:${GOGOPROTO_PATH}:${ETCD_ROOT_DIR}/..:${RAFT_ROOT}:${ETCD_ROOT_DIR}:${GOOGLEAPI_ROOT}" \ "--gofast_opt=paths=source_relative,${module_mappings}" \ --go-grpc_out=. \ "--go-grpc_opt=paths=source_relative,${module_mappings}" \ -I"${GRPC_GATEWAY_ROOT}" \ --plugin="${GOFAST_BIN}" ./**/*.proto run gofmt -s -w ./**/*.pb.go run_go_tool "golang.org/x/tools/cmd/goimports" -w ./**/*.pb.go run popd done log_callout -e "\\nRunning swagger & grpc_gateway proto generation..." # remove old swagger files so it's obvious whether the files fail to generate rm -rf Documentation/dev-guide/apispec/swagger/*json for pb in api/etcdserverpb/rpc server/etcdserver/api/v3lock/v3lockpb/v3lock server/etcdserver/api/v3election/v3electionpb/v3election; do log_callout "grpc & swagger for: ${pb}.proto" run protoc -I. \ -I"${GOOGLEAPI_ROOT}" \ -I"${GRPC_GATEWAY_ROOT}" \ -I"${GOGOPROTO_PATH}" \ -I"${ETCD_ROOT_DIR}/.." \ -I"${RAFT_ROOT}" \ --grpc-gateway_out=logtostderr=true,paths=source_relative:. \ "--grpc-gateway_opt=${module_mappings}" \ --openapiv2_out=json_names_for_fields=false,logtostderr=true:./Documentation/dev-guide/apispec/swagger/. \ "--openapiv2_opt=${module_mappings}:." \ --plugin="${OPENAPIV2_BIN}" \ --plugin="${GRPC_GATEWAY_BIN}" \ ${pb}.proto # hack to move gw files around so client won't include them pkgpath=$(dirname "${pb}") pkg=$(basename "${pkgpath}") gwfile="${pb}.pb.gw.go" run ${SED?} -i -E "s#package $pkg#package gw#g" "${gwfile}" run ${SED?} -i -E "s#import \\(#import \\(\"go.etcd.io/etcd/${pkgpath}\"#g" "${gwfile}" run ${SED?} -i -E "s#([ (])([a-zA-Z0-9_]*(Client|Server|Request)([^(]|$))#\\1${pkg}.\\2#g" "${gwfile}" run ${SED?} -i -E "s# (New[a-zA-Z0-9_]*Client\\()# ${pkg}.\\1#g" "${gwfile}" run ${SED?} -i -E "s|go.etcd.io/etcd|go.etcd.io/etcd/v3|g" "${gwfile}" run ${SED?} -i -E "s|go.etcd.io/etcd/v3/api|go.etcd.io/etcd/api/v3|g" "${gwfile}" run ${SED?} -i -E "s|go.etcd.io/etcd/v3/server|go.etcd.io/etcd/server/v3|g" "${gwfile}" run go fmt "${gwfile}" gwdir="${pkgpath}/gw/" run mkdir -p "${gwdir}" run mv "${gwfile}" "${gwdir}" swaggerName=$(basename ${pb}) run mv Documentation/dev-guide/apispec/swagger/${pb}.swagger.json \ Documentation/dev-guide/apispec/swagger/"${swaggerName}".swagger.json done # We only upgraded grpc-gateway from v1 to v2, but keep gogo/protobuf as it's for now. # So we have to convert v1 message to v2 message. Once we get rid of gogo/protobuf, and # start to depend on protobuf v2, then we can remove this patch. # # TODO(https://github.com/etcd-io/etcd/issues/14533): Remove the patch below after removal of gogo/protobuf for pb in api/etcdserverpb/rpc server/etcdserver/api/v3lock/v3lockpb/v3lock server/etcdserver/api/v3election/v3electionpb/v3election; do gwfile="$(dirname ${pb})/gw/$(basename ${pb}).pb.gw.go" # Changes something like below, # import ( # + protov1 "github.com/golang/protobuf/proto" # + run ${SED?} -i -E "s|import \(|import \(\n\tprotov1 \"github.com/golang/protobuf/proto\"\n|g" "${gwfile}" # Changes something like below, # - return msg, metadata, err # + return protov1.MessageV2(msg), metadata, err run ${SED?} -i -E "s|return msg, metadata, err|return protov1.MessageV2\(msg\), metadata, err|g" "${gwfile}" # Changes something like below, # - if err := marshaler.NewDecoder(newReader()).Decode(&protoReq); err != nil && err != io.EOF { # + if err := marshaler.NewDecoder(newReader()).Decode(protov1.MessageV2(&protoReq)); err != nil && err != io.EOF { run ${SED?} -i -E "s|Decode\(\&protoReq\)|Decode\(protov1\.MessageV2\(\&protoReq\)\)|g" "${gwfile}" # Changes something like below, # - forward_Lease_LeaseKeepAlive_0(annotatedContext, mux, outboundMarshaler, w, req, func() (proto.Message, error) { return resp.Recv() }, mux.GetForwardResponseOptions()...) # + forward_Lease_LeaseKeepAlive_0(annotatedContext, mux, outboundMarshaler, w, req, func() (proto.Message, error) { # + m1, err := resp.Recv() # + return protov1.MessageV2(m1), err # + }, mux.GetForwardResponseOptions()...) run ${SED?} -i -E "s|return resp.Recv\(\)|\n\t\t\tm1, err := resp.Recv\(\)\n\t\t\treturn protov1.MessageV2\(m1\), err\n\t\t|g" "${gwfile}" run go fmt "${gwfile}" done if [ "${1:-}" != "--skip-protodoc" ]; then log_callout "protodoc is auto-generating grpc API reference documentation..." # API reference API_REFERENCE_FILE="Documentation/dev-guide/api_reference_v3.md" run rm -rf ${API_REFERENCE_FILE} run_go_tool go.etcd.io/protodoc --directories="api/etcdserverpb=service_message,api/mvccpb=service_message,server/lease/leasepb=service_message,api/authpb=service_message" \ --output="${API_REFERENCE_FILE}" \ --message-only-from-this-file="api/etcdserverpb/rpc.proto" \ --disclaimer="--- title: API reference --- This API reference is autogenerated from the named \`.proto\` files." || exit 2 # remove the first 3 lines of the doc as an empty --title adds '### ' to the top of the file. run ${SED?} -i -e 1,3d ${API_REFERENCE_FILE} # API reference: concurrency API_REFERENCE_CONCURRENCY_FILE="Documentation/dev-guide/api_concurrency_reference_v3.md" run rm -rf ${API_REFERENCE_CONCURRENCY_FILE} run_go_tool go.etcd.io/protodoc --directories="server/etcdserver/api/v3lock/v3lockpb=service_message,server/etcdserver/api/v3election/v3electionpb=service_message,api/mvccpb=service_message" \ --output="${API_REFERENCE_CONCURRENCY_FILE}" \ --disclaimer="--- title: \"API reference: concurrency\" --- This API reference is autogenerated from the named \`.proto\` files." || exit 2 # remove the first 3 lines of the doc as an empty --title adds '### ' to the top of the file. run ${SED?} -i -e 1,3d ${API_REFERENCE_CONCURRENCY_FILE} log_success "protodoc is finished." log_warning -e "\\nThe API references have NOT been automatically published on the website." log_success -e "\\nTo publish the API references, copy the following files" log_success " - ${API_REFERENCE_FILE}" log_success " - ${API_REFERENCE_CONCURRENCY_FILE}" log_success "to the etcd-io/website repo under the /content/en/docs/next/dev-guide/ folder." log_success "(https://github.com/etcd-io/website/tree/main/content/en/docs/next/dev-guide)" else log_warning "skipping grpc API reference document auto-generation..." fi log_success -e "\\n./genproto SUCCESS" ================================================ FILE: scripts/markdown_diff_lint.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # This script runs markdownlint-cli2 on changed files. # Usage: ./markdown_lint.sh ETCD_ROOT_DIR=$(git rev-parse --show-toplevel) # We source ./scripts/test_utils.sh, it sets the log functions and color variables. source ./scripts/test_utils.sh # When we source ./scripts/test_utils.sh, it has the line set -u which treats unset variables as errors. # We need to unset the variable to avoid the error. set +u -eo pipefail if ! command markdownlint-cli2 dummy.md &>/dev/null; then log_error "markdownlint-cli2 needs to be installed." log_error "Please refer to https://github.com/DavidAnson/markdownlint-cli2?tab=readme-ov-file#install for installation instructions." exit 1 fi if [ -z "${PULL_BASE_SHA}" ]; then echo "Empty base reference (\$PULL_BASE_SHA), assuming: main" PULL_BASE_SHA=main fi if [ -z "${PULL_PULL_SHA}" ]; then PULL_PULL_SHA="$(git rev-parse HEAD)" echo "Empty pull reference (\$PULL_PULL_SHA), assuming: ${PULL_PULL_SHA}" fi MD_LINT_URL_PREFIX="https://github.com/DavidAnson/markdownlint/blob/main/doc/" mapfile -t changed_files < <(git diff "${PULL_BASE_SHA}" --name-only) declare -A files_with_failures start_ranges end_ranges for file in "${changed_files[@]}"; do if ! [[ "$file" =~ .md$ ]]; then continue fi # Find start and end ranges from changed files. start_ranges=() end_ranges=() # From https://github.com/paleite/eslint-plugin-diff/blob/46c5bcf296e9928db19333288457bf2805aad3b9/src/git.ts#L8-L27 ranges=$(git diff "${PULL_BASE_SHA}" \ --diff-algorithm=histogram \ --diff-filter=ACM \ --find-renames=100% \ --no-ext-diff \ --relative \ --unified=0 -- "${file}" | \ gawk 'match($0, /^@@\s-[0-9,]+\s\+([0-9]+)(,([0-9]+))?/, m) { \ print m[1] ":" m[1] + ((m[3] == "") ? "0" : m[3]) }') i=0 for range in ${ranges}; do start_ranges["${i}"]=$(echo "${range}" | awk -F: '{print $1}') end_ranges["${i}"]=$(echo "${range}" | awk -F: '{print $2}') i=$((1 + i)) done if [ -z "${ranges}" ]; then start_ranges[0]=0 end_ranges[0]=0 fi i=0 # Run markdownlint-cli2 with the changed file and print only the summary (stdout). markdownlint-cli2 "${file}" --config "${ETCD_ROOT_DIR}/tools/.markdownlint.jsonc" 2>/dev/null || true while IFS= read -r line; do line_number=$(echo "${line}" | awk -F: '{print $2}' | awk '{print $1}') while [ "${i}" -lt "${#end_ranges[@]}" ] && [ "${line_number}" -gt "${end_ranges["${i}"]}" ]; do i=$((1 + i)) done rule=$(echo "${line}" | gawk 'match($2, /([^\/]+)/, m) {print tolower(m[1])}') lint_error="${line} (${MD_LINT_URL_PREFIX}${rule}.md)" if [ "${i}" -lt "${#start_ranges[@]}" ] && [ "${line_number}" -ge "${start_ranges["${i}"]}" ] && [ "${line_number}" -le "${end_ranges["${i}"]}" ]; then # Inside range with changes, raise an error. log_error "${lint_error}" files_with_failures["${file}"]=1 else # Outside of range, raise a warning. log_warning "${lint_error}" fi done < <(markdownlint-cli2 "${file}" --config "${ETCD_ROOT_DIR}/tools/.markdownlint.jsonc" 2>&1 >/dev/null || true) done echo "Finished linting" for file in "${!files_with_failures[@]}"; do log_error "${file} has linting issues" done if [ "${#files_with_failures[@]}" -gt "0" ]; then exit 1 fi ================================================ FILE: scripts/measure-testgrid-flakiness.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Measures test flakiness and create issues for flaky tests set -euo pipefail if [[ -z ${GITHUB_TOKEN:-} ]] then echo "Please set the \$GITHUB_TOKEN environment variable for the script to work" exit 1 fi pushd ./tools/testgrid-analysis # ci-etcd-e2e-amd64 and ci-etcd-unit-test-amd64 runs 6 times a day. Keeping a rolling window of 14 days. go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64 --max-days=14 go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64 --max-days=14 # do not create issues for presubmit tests go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-e2e-amd64 go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-unit-test popd ================================================ FILE: scripts/release.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail source ./scripts/test_lib.sh source ./scripts/release_mod.sh DRY_RUN=${DRY_RUN:-true} # Following preparation steps help with the release process: # If you use password-protected gpg key, make sure the password is managed # by agent: # # % gpg-connect-agent reloadagent /bye # % gpg -s --default-key [git-email]@google.com -o /dev/null -s /dev/null # # Refresh your google credentials: # % gcloud auth login # or # % gcloud auth activate-service-account --key-file=gcp-key-etcd-development.json # # Make sure gcloud-docker plugin is configured: # % gcloud auth configure-docker help() { echo "$(basename "$0") [version]" echo "Release etcd using the same approach as the etcd-release-runbook (https://goo.gl/Gxwysq)" echo "" echo "WARNING: This does not perform the 'Add API capabilities', 'Performance testing' " echo " or 'Documentation' steps. These steps must be performed manually BEFORE running this tool." echo "" echo "WARNING: This script does not send announcement emails. This step must be performed manually AFTER running this tool." echo "" echo " args:" echo " version: version of etcd to release, e.g. 'v3.2.18'" echo " flags:" echo " --in-place: build binaries using current branch." echo " --no-docker-push: skip docker image pushes." echo " --no-gh-release: skip creating the GitHub release using gh." echo " --no-upload: skip gs://etcd binary artifact uploads." echo "" echo "One can perform a (dry-run) test release from any (uncommitted) branch using:" echo " DRY_RUN=true REPOSITORY=\`pwd\` BRANCH='local-branch-name' ./scripts/release 3.5.0-foobar.2" } main() { # Allow to receive the version with the "v" prefix, i.e. v3.6.0. VERSION=${1#v} if [[ ! "${VERSION}" =~ ^[0-9]+.[0-9]+.[0-9]+ ]]; then log_error "Expected 'version' param of the form '..' but got '${VERSION}'" exit 1 fi RELEASE_VERSION="v${VERSION}" MINOR_VERSION=$(echo "${VERSION}" | cut -d. -f 1-2) if [ "${IN_PLACE}" == 1 ]; then # Trigger release in current branch REPOSITORY=$(pwd) BRANCH=$(git rev-parse --abbrev-ref HEAD) else REPOSITORY=${REPOSITORY:-"git@github.com:etcd-io/etcd.git"} BRANCH=${BRANCH:-"release-${MINOR_VERSION}"} fi log_warning "DRY_RUN=${DRY_RUN}" log_callout "RELEASE_VERSION=${RELEASE_VERSION}" log_callout "MINOR_VERSION=${MINOR_VERSION}" log_callout "BRANCH=${BRANCH}" log_callout "REPOSITORY=${REPOSITORY}" log_callout "" # Required to enable 'docker manifest ...' export DOCKER_CLI_EXPERIMENTAL=enabled if ! command -v docker >/dev/null; then log_error "cannot find docker" exit 1 fi # Expected umask for etcd release artifacts umask 022 # Set up release directory. local reldir="/tmp/etcd-release-${VERSION}" log_callout "Preparing temporary directory: ${reldir}" if [ "${IN_PLACE}" == 0 ]; then if [ ! -d "${reldir}/etcd" ]; then mkdir -p "${reldir}" cd "${reldir}" run git clone "${REPOSITORY}" --branch "${BRANCH}" --depth 1 fi run cd "${reldir}/etcd" || exit 2 run git checkout "${BRANCH}" || exit 2 run git pull origin git_assert_branch_in_sync || exit 2 fi # mark local directory as root for test_lib scripts executions set_root_dir # If a release version tag already exists, use it. local remote_tag_exists remote_tag_exists=$(run git ls-remote origin "refs/tags/${RELEASE_VERSION}" | grep -c "${RELEASE_VERSION}" || true) if [ "${remote_tag_exists}" -gt 0 ]; then log_callout "Release version tag exists on remote. Checking out refs/tags/${RELEASE_VERSION}" git checkout -q "tags/${RELEASE_VERSION}" fi # Check go version. log_callout "Check go version" local go_version current_go_version go_version="go$(cat .go-version)" current_go_version=$(go version | awk '{ print $3 }') if [[ "${current_go_version}" != "${go_version}" ]]; then log_error "Current go version is ${current_go_version}, but etcd ${RELEASE_VERSION} requires ${go_version} (see .go-version)." exit 1 fi if [ "${NO_GH_RELEASE}" == 1 ]; then log_callout "Skipping gh verification, --no-gh-release is set" else # Check that gh is installed and logged in. log_callout "Check gh installation" if ! command -v gh >/dev/null; then log_error "Cannot find gh. Please follow the installation instructions at https://github.com/cli/cli#installation" exit 1 fi if ! gh auth status &>/dev/null; then log_error "GitHub authentication failed for gh. Please run gh auth login." exit 1 fi fi # If the release tag does not already exist remotely, create it. log_callout "Create tag if not present" if [ "${remote_tag_exists}" -eq 0 ]; then # Bump version/version.go to release version. local source_version source_version=$(grep -E "\s+Version\s*=" api/version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g") if [[ "${source_version}" != "${VERSION}" ]]; then source_minor_version=$(echo "${source_version}" | cut -d. -f 1-2) if [[ "${source_minor_version}" != "${MINOR_VERSION}" ]]; then log_error "Wrong etcd minor version in api/version/version.go. Expected ${MINOR_VERSION} but got ${source_minor_version}. Aborting." exit 1 fi log_callout "Updating modules definitions" TARGET_VERSION="v${VERSION}" update_versions_cmd log_callout "Updating version from ${source_version} to ${VERSION} in api/version/version.go" sed -i "s/${source_version}/${VERSION}/g" api/version/version.go fi log_callout "Building etcd and checking --version output" run ./scripts/build.sh local etcd_version etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }') if [[ "${etcd_version}" != "${VERSION}" ]]; then log_error "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting." exit 1 fi if [[ -n $(git status -s) ]]; then log_callout "Committing mods & api/version/version.go update." run git add api/version/version.go # shellcheck disable=SC2038,SC2046 run git add $(find . -name go.mod ! -path './release/*'| xargs) run git diff --staged | cat run git commit --signoff --message "version: bump up to ${VERSION}" run git diff --staged | cat fi # Push the version change if it's not already been pushed. if [ "${DRY_RUN}" != "true" ] && [ "$(git rev-list --count "origin/${BRANCH}..${BRANCH}")" -gt 0 ]; then read -p "Push version bump up to ${VERSION} to '$(git remote get-url origin)' [y/N]? " -r confirm [[ "${confirm,,}" == "y" ]] || exit 1 maybe_run git push fi # Tag release. if git tag --list | grep --quiet "^${RELEASE_VERSION}$"; then log_callout "Skipping tag step. git tag ${RELEASE_VERSION} already exists." else log_callout "Tagging release..." REMOTE_REPO="origin" push_mod_tags_cmd fi if [ "${IN_PLACE}" == 0 ]; then # Tried with `local branch=$(git branch -a --contains tags/"${RELEASE_VERSION}")` # so as to work with both current branch and main/release-3.X. # But got error below on current branch mode, # Error: Git tag v3.6.99 should be on branch '* (HEAD detached at pull/14860/merge)' but is on '* (HEAD detached from pull/14860/merge)' # # Verify the version tag is on the right branch # shellcheck disable=SC2155 local branch=$(git for-each-ref --contains "${RELEASE_VERSION}" --format="%(refname)" 'refs/heads' | cut -d '/' -f 3) if [ "${branch}" != "${BRANCH}" ]; then log_error "Error: Git tag ${RELEASE_VERSION} should be on branch '${BRANCH}' but is on '${branch}'" exit 1 fi fi fi log_callout "Verify the latest commit has the version tag" # Verify the latest commit has the version tag # shellcheck disable=SC2155 local tag="$(git describe --exact-match HEAD)" if [ "${tag}" != "${RELEASE_VERSION}" ]; then log_error "Error: Expected HEAD to be tagged with ${RELEASE_VERSION}, but 'git describe --exact-match HEAD' reported: ${tag}" exit 1 fi log_callout "Verify the work space is clean" # Verify the clean working tree # shellcheck disable=SC2155 local diff="$(git diff HEAD --stat)" if [[ "${diff}" != '' ]]; then log_error "Error: Expected clean working tree, but 'git diff --stat' reported: ${diff}" exit 1 fi # Build release. # TODO: check the release directory for all required build artifacts. if [ -d release ]; then log_warning "Skipping release build step. /release directory already exists." else log_callout "Building release..." REPOSITORY=$(pwd) ./scripts/build-release.sh "${RELEASE_VERSION}" fi # Sanity checks. "./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcd" --version | grep -q "etcd Version: ${VERSION}" || true "./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcdctl" version | grep -q "etcdctl version: ${VERSION}" || true "./release/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcdutl" version | grep -q "etcdutl version: ${VERSION}" || true # Generate SHA256SUMS log_callout "Generating sha256sums of release artifacts." pushd ./release # shellcheck disable=SC2010 ls . | grep -E '\.tar.gz$|\.zip$' | xargs shasum -a 256 > ./SHA256SUMS popd if [ -s ./release/SHA256SUMS ]; then cat ./release/SHA256SUMS else log_error "sha256sums is not valid. Aborting." exit 1 fi # Upload artifacts. if [ "${DRY_RUN}" == "true" ] || [ "${NO_UPLOAD}" == 1 ]; then log_callout "Skipping artifact upload to gs://etcd. --no-upload flag is set." else read -p "Upload etcd ${RELEASE_VERSION} release artifacts to gs://etcd [y/N]? " -r confirm [[ "${confirm,,}" == "y" ]] || exit 1 maybe_run gsutil -m cp ./release/SHA256SUMS "gs://etcd/${RELEASE_VERSION}/" maybe_run gsutil -m cp ./release/*.zip "gs://etcd/${RELEASE_VERSION}/" maybe_run gsutil -m cp ./release/*.tar.gz "gs://etcd/${RELEASE_VERSION}/" maybe_run gsutil -m acl ch -u allUsers:R -r "gs://etcd/${RELEASE_VERSION}/" fi # Push images. if [ "${DRY_RUN}" == "true" ] || [ "${NO_DOCKER_PUSH}" == 1 ]; then log_callout "Skipping docker push. --no-docker-push flag is set." else read -p "Publish etcd ${RELEASE_VERSION} docker images to quay.io [y/N]? " -r confirm [[ "${confirm,,}" == "y" ]] || exit 1 # shellcheck disable=SC2034 for i in {1..5}; do docker login quay.io && break log_warning "login failed, retrying" done for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do log_callout "Pushing container images to quay.io ${RELEASE_VERSION}-${TARGET_ARCH}" maybe_run docker push "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" log_callout "Pushing container images to gcr.io ${RELEASE_VERSION}-${TARGET_ARCH}" maybe_run docker push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" done log_callout "Creating manifest-list (multi-image)..." for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do maybe_run docker manifest create --amend "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" maybe_run docker manifest annotate "quay.io/coreos/etcd:${RELEASE_VERSION}" "quay.io/coreos/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}" maybe_run docker manifest create --amend "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" maybe_run docker manifest annotate "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}-${TARGET_ARCH}" --arch "${TARGET_ARCH}" done log_callout "Pushing container manifest list to quay.io ${RELEASE_VERSION}" maybe_run docker manifest push "quay.io/coreos/etcd:${RELEASE_VERSION}" log_callout "Pushing container manifest list to gcr.io ${RELEASE_VERSION}" maybe_run docker manifest push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}" fi ### Release validation mkdir -p downloads # Check image versions for IMAGE in "quay.io/coreos/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}"; do if [ "${DRY_RUN}" == "true" ] || [ "${NO_DOCKER_PUSH}" == 1 ]; then IMAGE="${IMAGE}-amd64" fi # shellcheck disable=SC2155 local image_version=$(docker run --rm "${IMAGE}" etcd --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]') if [ "${image_version}" != "${VERSION}" ]; then log_error "Check failed: etcd --version output for ${IMAGE} is incorrect: ${image_version}" exit 1 fi done # Check gsutil binary versions # shellcheck disable=SC2155 local BINARY_TGZ="etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64.tar.gz" if [ "${DRY_RUN}" == "true" ] || [ "${NO_UPLOAD}" == 1 ]; then cp "./release/${BINARY_TGZ}" downloads else gsutil cp "gs://etcd/${RELEASE_VERSION}/${BINARY_TGZ}" downloads fi tar -zx -C downloads -f "downloads/${BINARY_TGZ}" # shellcheck disable=SC2155 local binary_version=$("./downloads/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcd" --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]') if [ "${binary_version}" != "${VERSION}" ]; then log_error "Check failed: etcd --version output for ${BINARY_TGZ} from gs://etcd/${RELEASE_VERSION} is incorrect: ${binary_version}" exit 1 fi if [ "${DRY_RUN}" == "true" ] || [ "${NO_GH_RELEASE}" == 1 ]; then log_warning "" log_warning "WARNING: Skipping creating GitHub release, --no-gh-release is set." log_warning "WARNING: If not running on DRY_MODE, please do the GitHub release manually." log_warning "" else local gh_repo local release_notes_temp_file local release_url local gh_release_args=() # For the main branch (v3.6), we should mark the release as a prerelease. # The release-3.5 (v3.5) branch, should be marked as latest. And release-3.4 (v3.4) # should be left without any additional mark (therefore, it doesn't need a special argument). if [ "${BRANCH}" = "main" ]; then gh_release_args=(--prerelease) elif [ "${BRANCH}" = "release-3.5" ]; then gh_release_args=(--latest) fi if [ "${REPOSITORY}" = "$(pwd)" ]; then gh_repo=$(git remote get-url origin) else gh_repo="${REPOSITORY}" fi gh_repo=$(echo "${gh_repo}" | sed 's/^[^@]\+@//' | sed 's/https\?:\/\///' | sed 's/\.git$//' | tr ':' '/') log_callout "Creating GitHub release for ${RELEASE_VERSION} on ${gh_repo}" release_notes_temp_file=$(mktemp) local release_version=${RELEASE_VERSION#v} # Remove the v prefix from the release version (i.e., v3.6.1 -> 3.6.1) local release_version_major_minor release_version_major_minor=$(echo "${release_version}" | cut -d. -f1-2) # Remove the patch from the version (i.e., 3.6) local release_version_major=${release_version_major_minor%.*} # Extract the major (i.e., 3) local release_version_minor=${release_version_major_minor/*./} # Extract the minor (i.e., 6) # Disable sellcheck SC2016, the single quoted syntax for sed is intentional. # shellcheck disable=SC2016 sed 's/${RELEASE_VERSION}/'"${RELEASE_VERSION}"'/g' ./scripts/release_notes.tpl.txt | sed 's/${RELEASE_VERSION_MAJOR_MINOR}/'"${release_version_major_minor}"'/g' | sed 's/${RELEASE_VERSION_MAJOR}/'"${release_version_major}"'/g' | sed 's/${RELEASE_VERSION_MINOR}/'"${release_version_minor}"'/g' > "${release_notes_temp_file}" if ! gh --repo "${gh_repo}" release view "${RELEASE_VERSION}" &>/dev/null; then maybe_run gh release create "${RELEASE_VERSION}" \ --repo "${gh_repo}" \ --draft \ --title "${RELEASE_VERSION}" \ --notes-file "${release_notes_temp_file}" \ "${gh_release_args[@]}" fi # Upload files one by one, as gh doesn't support passing globs as input. maybe_run find ./release '(' -name '*.tar.gz' -o -name '*.zip' ')' -exec \ gh --repo "${gh_repo}" release upload "${RELEASE_VERSION}" {} --clobber \; maybe_run gh --repo "${gh_repo}" release upload "${RELEASE_VERSION}" ./release/SHA256SUMS --clobber release_url=$(gh --repo "${gh_repo}" release view "${RELEASE_VERSION}" --json url --jq '.url') log_warning "" log_warning "WARNING: The GitHub release for ${RELEASE_VERSION} has been created as a draft, please go to ${release_url} and release it." log_warning "" fi log_success "Success." exit 0 } POSITIONAL=() NO_UPLOAD=0 NO_DOCKER_PUSH=0 IN_PLACE=0 NO_GH_RELEASE=0 while test $# -gt 0; do case "$1" in -h|--help) shift help exit 0 ;; --in-place) IN_PLACE=1 shift ;; --no-upload) NO_UPLOAD=1 shift ;; --no-docker-push) NO_DOCKER_PUSH=1 shift ;; --no-gh-release) NO_GH_RELEASE=1 shift ;; *) POSITIONAL+=("$1") # save it in an array for later shift # past argument ;; esac done set -- "${POSITIONAL[@]}" # restore positional parameters if [[ ! $# -eq 1 ]]; then help exit 1 fi # Note that we shouldn't upload artifacts in --in-place mode, so it # must be called with DRY_RUN=true if [ "${DRY_RUN}" != "true" ] && [ "${IN_PLACE}" == 1 ]; then log_error "--in-place should only be called with DRY_RUN=true" exit 1 fi main "$1" ================================================ FILE: scripts/release_mod.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Examples: # Edit go.mod files such that all etcd modules are pointing on given version: # # % DRY_RUN=false TARGET_VERSION="v3.5.13" ./scripts/release_mod.sh update_versions # Tag latest commit with current version number for all the modules and push upstream: # # % DRY_RUN=false REMOTE_REPO="origin" ./scripts/release_mod.sh push_mod_tags set -euo pipefail source ./scripts/test_lib.sh DRY_RUN=${DRY_RUN:-true} # _cmd prints help message function _cmd() { log_error "Command required: ${0} [cmd]" log_info "Available commands:" log_info " - update_versions - Updates all cross-module versions to \${TARGET_VERSION} in the local client." log_info " - push_mod_tags - Tags HEAD with all modules versions tags and pushes it to \${REMOTE_REPO}." } # update_module_version [v2version] [v3version] # Updates versions of cross-references in all internal references in current module. function update_module_version() { local v3version="${1}" local v2version="${2}" local modules run go mod tidy modules=$(go mod edit -json | jq -r '.Require[] | select(.Indirect | not) | .Path') v3deps=$(echo "${modules}" | grep -E "${ROOT_MODULE}/.*/v3") for dep in ${v3deps}; do run go mod edit -require "${dep}@${v3version}" done v2deps=$(echo "${modules}" | grep -E "${ROOT_MODULE}/.*/v2") for dep in ${v2deps}; do run go mod edit -require "${dep}@${v2version}" done run go mod tidy } function mod_tidy_fix { run rm ./go.sum run go mod tidy || return 2 } # Updates all cross-module versions to ${TARGET_VERSION} in local client. function update_versions_cmd() { assert_no_git_modifications || return 2 if [ -z "${TARGET_VERSION:-}" ]; then log_error "TARGET_VERSION environment variable not set. Set it to e.g. v3.5.10-alpha.0" return 2 fi local v3version="${TARGET_VERSION}" local v2version # converts e.g. v3.5.0-alpha.0 --> v2.305.0-alpha.0 # shellcheck disable=SC2001 v2version="$(echo "${TARGET_VERSION}" | sed 's|^v3.\([0-9]*\).|v2.30\1.|g')" log_info "DRY_RUN : ${DRY_RUN}" log_info "TARGET_VERSION: ${TARGET_VERSION}" log_info "" log_info "v3version: ${v3version}" log_info "v2version: ${v2version}" run_for_modules update_module_version "${v3version}" "${v2version}" run_for_modules mod_tidy_fix || exit 2 } function get_gpg_key { gitemail=$(git config --get user.email) keyid=$(run gpg --list-keys --with-colons "${gitemail}" | awk -F: '/^pub:/ { print $5 }') if [[ -z "${keyid}" ]]; then log_error "Failed to load gpg key. Is gpg set up correctly for etcd releases?" return 2 fi echo "$keyid" } function push_mod_tags_cmd { assert_no_git_modifications || return 2 if [ -z "${REMOTE_REPO:-}" ]; then log_error "REMOTE_REPO environment variable not set" return 2 fi log_info "REMOTE_REPO: ${REMOTE_REPO}" # Any module ccan be used for this local main_version main_version=$(go mod edit -json | jq -r '.Require[] | select(.Path == "'"${ROOT_MODULE}"'/api/v3") | .Version') local tags=() keyid=$(get_gpg_key) || return 2 for module in $(modules); do local version version=$(go mod edit -json | jq -r '.Require[] | select(.Path == "'"${module}"'") | .Version') local path path=$(go mod edit -json | jq -r '.Require[] | select(.Path == "'"${module}"'") | .Path') local subdir="${path//${ROOT_MODULE}\//}" local tag if [ -z "${version}" ]; then tag="${main_version}" version="${main_version}" else tag="${subdir///v[23]/}/${version}" fi log_info "Tags for: ${module} version:${version} tag:${tag}" # The sleep is ugly hack that guarantees that 'git describe' will # consider main-module's tag as the latest. run sleep 2 run git tag --local-user "${keyid}" --sign "${tag}" --message "${version}" tags+=("${tag}") done maybe_run git push -f "${REMOTE_REPO}" "${tags[@]}" } # only release_mod when called directly, not sourced if echo "$0" | grep -E "release_mod.sh$" >/dev/null; then "${1}_cmd" if "${DRY_RUN}"; then log_info log_warning "WARNING: It was a DRY_RUN. No files were modified." fi fi ================================================ FILE: scripts/release_notes.tpl.txt ================================================ Please check out [CHANGELOG](https://github.com/etcd-io/etcd/blob/main/CHANGELOG/CHANGELOG-${RELEASE_VERSION_MAJOR_MINOR}.md) for a full list of changes. And make sure to read [upgrade guide](https://etcd.io/docs/v${RELEASE_VERSION_MAJOR_MINOR}/upgrades/upgrade_${RELEASE_VERSION_MAJOR}_${RELEASE_VERSION_MINOR}/) before upgrading etcd (there may be breaking changes). For installation guides, please check out [play.etcd.io](http://play.etcd.io) and [operating etcd](https://etcd.io/docs/v${RELEASE_VERSION_MAJOR_MINOR}/op-guide/). Latest support status for common architectures and operating systems can be found at [supported platforms](https://etcd.io/docs/v${RELEASE_VERSION_MAJOR_MINOR}/op-guide/supported-platform/). ###### Linux ```sh ETCD_VER=${RELEASE_VERSION} # choose either URL GOOGLE_URL=https://storage.googleapis.com/etcd GITHUB_URL=https://github.com/etcd-io/etcd/releases/download DOWNLOAD_URL=${GOOGLE_URL} rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1 --no-same-owner rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz /tmp/etcd-download-test/etcd --version /tmp/etcd-download-test/etcdctl version /tmp/etcd-download-test/etcdutl version # start a local etcd server /tmp/etcd-download-test/etcd # write,read to etcd /tmp/etcd-download-test/etcdctl --endpoints=localhost:2379 put foo bar /tmp/etcd-download-test/etcdctl --endpoints=localhost:2379 get foo ``` ###### macOS (Darwin) ```sh ETCD_VER=${RELEASE_VERSION} # choose either URL GOOGLE_URL=https://storage.googleapis.com/etcd GITHUB_URL=https://github.com/etcd-io/etcd/releases/download DOWNLOAD_URL=${GOOGLE_URL} rm -f /tmp/etcd-${ETCD_VER}-darwin-amd64.zip rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-darwin-amd64.zip -o /tmp/etcd-${ETCD_VER}-darwin-amd64.zip unzip /tmp/etcd-${ETCD_VER}-darwin-amd64.zip -d /tmp && rm -f /tmp/etcd-${ETCD_VER}-darwin-amd64.zip mv /tmp/etcd-${ETCD_VER}-darwin-amd64/* /tmp/etcd-download-test && rm -rf mv /tmp/etcd-${ETCD_VER}-darwin-amd64 /tmp/etcd-download-test/etcd --version /tmp/etcd-download-test/etcdctl version /tmp/etcd-download-test/etcdutl version ``` ###### Docker etcd uses [`gcr.io/etcd-development/etcd`](https://gcr.io/etcd-development/etcd) as a primary container registry, and [`quay.io/coreos/etcd`](https://quay.io/coreos/etcd) as secondary. ```sh ETCD_VER=${RELEASE_VERSION} rm -rf /tmp/etcd-data.tmp && mkdir -p /tmp/etcd-data.tmp && \ docker rmi gcr.io/etcd-development/etcd:${ETCD_VER} || true && \ docker run \ -p 2379:2379 \ -p 2380:2380 \ --mount type=bind,source=/tmp/etcd-data.tmp,destination=/etcd-data \ --name etcd-gcr-${ETCD_VER} \ gcr.io/etcd-development/etcd:${ETCD_VER} \ /usr/local/bin/etcd \ --name s1 \ --data-dir /etcd-data \ --listen-client-urls http://0.0.0.0:2379 \ --advertise-client-urls http://0.0.0.0:2379 \ --listen-peer-urls http://0.0.0.0:2380 \ --initial-advertise-peer-urls http://0.0.0.0:2380 \ --initial-cluster s1=http://0.0.0.0:2380 \ --initial-cluster-token tkn \ --initial-cluster-state new \ --log-level info \ --logger zap \ --log-outputs stderr docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcd --version docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl version docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdutl version docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl endpoint health docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl put foo bar docker exec etcd-gcr-${ETCD_VER} /usr/local/bin/etcdctl get foo ``` ================================================ FILE: scripts/sync_go_toolchain_directive.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This script looks at the version present in the .go-version file and treats # that to be the value of the toolchain directive that go should use. It then # updates the toolchain directives of all go.mod files to reflect this version. # # We do this to ensure that .go-version acts as the source of truth for go versions. set -euo pipefail source ./scripts/test_lib.sh TARGET_GO_VERSION="${TARGET_GO_VERSION:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" find . -name 'go.mod' -exec go mod edit -toolchain=go"${TARGET_GO_VERSION}" {} \; go work edit -toolchain=go"${TARGET_GO_VERSION}" ================================================ FILE: scripts/test.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Run all etcd tests # ./scripts/test.sh # ./scripts/test.sh -v # # # Run specified test pass # # $ PASSES=unit ./scripts/test.sh # $ PASSES=integration ./scripts/test.sh # # # Run tests for one package # Each pass has different default timeout, if you just run tests in one package or 1 test case then you can set TIMEOUT # flag for different expectation # # $ PASSES=unit PKG=./wal TIMEOUT=1m ./scripts/test.sh # $ PASSES=integration PKG=./clientv3 TIMEOUT=1m ./scripts/test.sh # # Run specified unit tests in one package # To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew "; # to run only "TestNew", set "TESTCASE="\bTestNew\b"" # # $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./scripts/test.sh # $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./scripts/test.sh # $ PASSES=integration PKG=./client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./scripts/test.sh # # KEEP_GOING_SUITE must be set to true to keep going with the next suite execution, passed to PASSES variable when there is a failure # in a particular suite. # KEEP_GOING_MODULE must be set to true to keep going with execution when there is failure in any module. # # Run code coverage # COVERDIR must either be a absolute path or a relative path to the etcd root # $ COVERDIR=coverage PASSES="build cov" ./scripts/test.sh # $ go tool cover -html ./coverage/cover.out set -e # Consider command as failed when any component of the pipe fails: # https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash set -o pipefail set -o nounset # The test script is not supposed to make any changes to the files # e.g. add/update missing dependencies. Such divergences should be # detected and trigger a failure that needs explicit developer's action. export GOFLAGS=-mod=readonly export ETCD_VERIFY=all source ./scripts/test_lib.sh source ./scripts/build_lib.sh OUTPUT_FILE=${OUTPUT_FILE:-""} if [ -n "${OUTPUT_FILE}" ]; then log_callout "Dumping output to: ${OUTPUT_FILE}" exec > >(tee -a "${OUTPUT_FILE}") 2>&1 fi PASSES=${PASSES:-"bom dep build unit"} KEEP_GOING_SUITE=${KEEP_GOING_SUITE:-false} PKG=${PKG:-} SHELLCHECK_VERSION=${SHELLCHECK_VERSION:-"v0.10.0"} MARKDOWN_MARKER_VERSION=${MARKDOWN_MARKER_VERSION:="v0.10.0"} if [ -z "${GOARCH:-}" ]; then GOARCH=$(go env GOARCH); fi if [ -z "${OS:-}" ]; then OS=$(uname -s | tr '[:upper:]' '[:lower:]') fi if [ -z "${ARCH:-}" ]; then ARCH=$(uname -m) if [ "$ARCH" = "arm64" ]; then ARCH="aarch64" fi fi # determine whether target supports race detection if [ -z "${RACE:-}" ] ; then if [ "$GOARCH" == "amd64" ] || [ "$GOARCH" == "arm64" ]; then RACE="--race" else RACE="--race=false" fi else RACE="--race=${RACE:-true}" fi # This options make sense for cases where SUT (System Under Test) is compiled by test. COMMON_TEST_FLAGS=("${RACE}") if [[ -n "${CPU:-}" ]]; then COMMON_TEST_FLAGS+=("--cpu=${CPU}") fi log_callout "Running with ${COMMON_TEST_FLAGS[*]}" RUN_ARG=() if [ -n "${TESTCASE:-}" ]; then RUN_ARG=("-run=${TESTCASE}") fi function build_pass { log_callout "Building etcd" run_for_modules run go build "${@}" || return 2 GO_BUILD_FLAGS="-v" etcd_build "${@}" GO_BUILD_FLAGS="-v" tools_build "${@}" } ################# REGULAR TESTS ################################################ function unit_pass { run_for_all_workspace_modules \ run_go_tests -short \ -failfast \ -timeout="${TIMEOUT:-3m}" \ "${COMMON_TEST_FLAGS[@]}" \ "${RUN_ARG[@]}" \ "$@" } function integration_extra { if [ -z "${PKG}" ] ; then run_go_tests_expanding_packages ./tests/integration/v2store/... \ -timeout="${TIMEOUT:-5m}" \ "${COMMON_TEST_FLAGS[@]}" \ "${RUN_ARG[@]}" \ "$@" else log_warning "integration_extra ignored when PKG is specified" fi } function integration_pass { run_go_tests ./tests/integration/... \ -p=2 \ -failfast \ -timeout="${TIMEOUT:-15m}" \ "${COMMON_TEST_FLAGS[@]}" \ "${RUN_ARG[@]}" \ "$@" || return 2 run_go_tests ./tests/common/... \ -p=2 \ -failfast \ -tags=integration \ -timeout="${TIMEOUT:-15m}" \ "${COMMON_TEST_FLAGS[@]}" \ "${RUN_ARG[@]}" \ "$@" || return 2 integration_extra "$@" } function e2e_pass { # e2e tests are running pre-build binary. Settings like --race,-cover,-cpu do not have any impact. run_go_tests_expanding_packages ./tests/e2e/... \ -timeout="${TIMEOUT:-30m}" \ "${RUN_ARG[@]}" \ "$@" || return 2 run_go_tests_expanding_packages ./tests/common/... \ -tags=e2e \ -timeout="${TIMEOUT:-30m}" \ "${RUN_ARG[@]}" \ "$@" } function robustness_pass { # e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact. run_go_tests ./tests/robustness \ -timeout="${TIMEOUT:-30m}" \ "${RUN_ARG[@]}" \ "$@" } function integration_e2e_pass { run_pass "integration" "${@}" || return 2 run_pass "e2e" "${@}" } # generic_checker [cmd...] # executes given command in the current module, and clearly fails if it # failed or returned output. function generic_checker { local cmd=("$@") if ! output=$("${cmd[@]}"); then echo "${output}" log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)" return 255 fi if [ -n "${output}" ]; then echo "${output}" log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)" return 255 fi } function grpcproxy_pass { run_pass "grpcproxy_integration" "${@}" || return 2 run_pass "grpcproxy_e2e" "${@}" } function grpcproxy_integration_pass { run_go_tests_expanding_packages ./tests/integration/... \ -tags=cluster_proxy \ -timeout="${TIMEOUT:-30m}" \ "${COMMON_TEST_FLAGS[@]}" \ "${RUN_ARG[@]}" \ "$@" } function grpcproxy_e2e_pass { run_go_tests_expanding_packages ./tests/e2e/... \ -tags=cluster_proxy \ -timeout="${TIMEOUT:-30m}" \ "${COMMON_TEST_FLAGS[@]}" \ "${RUN_ARG[@]}" \ "$@" } ################# COVERAGE ##################################################### # pkg_to_coverflag [prefix] [pkgs] # produces name of .coverprofile file to be used for tests of this package function pkg_to_coverprofileflag { local prefix="${1}" local pkgs="${2}" local pkgs_normalized prefix_normalized=$(echo "${prefix}" | tr "./ " "__+") if [ "${pkgs}" == "./..." ]; then pkgs_normalized="all" else pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+") fi mkdir -p "${coverdir}/${prefix_normalized}" echo -n "-coverprofile=${coverdir}/${prefix_normalized}/${pkgs_normalized}.coverprofile" } function not_test_packages { for m in $(modules); do if [[ $m =~ .*/etcd/tests/v3 ]]; then continue; fi if [[ $m =~ .*/etcd/v3 ]]; then continue; fi echo "${m}/..." done } # split_dir [dir] [num] function split_dir { local d="${1}" local num="${2}" local i=0 for f in "${d}/"*; do local g=$(( i % num )) mkdir -p "${d}_${g}" mv "${f}" "${d}_${g}/" (( i++ )) done } function split_dir_pass { split_dir ./covdir/integration 4 } # merge_cov_files [coverdir] [outfile] # merges all coverprofile files into a single file in the given directory. function merge_cov_files { local coverdir="${1}" local cover_out_file="${2}" log_callout "Merging coverage results in: ${coverdir}" # gocovmerge requires not-empty test to start with: echo "mode: set" > "${cover_out_file}" local i=0 local count count=$(find "${coverdir}"/*.coverprofile | wc -l) for f in "${coverdir}"/*.coverprofile; do # print once per 20 files if ! (( "${i}" % 20 )); then log_callout "${i} of ${count}: Merging file: ${f}" fi run_go_tool "github.com/alexfalkowski/gocovmerge" "${f}" "${cover_out_file}" > "${coverdir}/cover.tmp" 2>/dev/null if [ -s "${coverdir}"/cover.tmp ]; then mv "${coverdir}/cover.tmp" "${cover_out_file}" fi (( i++ )) done } # merge_cov [coverdir] function merge_cov { log_callout "[$(date)] Merging coverage files ..." coverdir="${1}" for d in "${coverdir}"/*/; do d=${d%*/} # remove the trailing "/" merge_cov_files "${d}" "${d}.coverprofile" & done wait merge_cov_files "${coverdir}" "${coverdir}/all.coverprofile" } # https://docs.codecov.com/docs/unexpected-coverage-changes#reasons-for-indirect-changes function cov_pass { # shellcheck disable=SC2153 if [ -z "${COVERDIR:-}" ]; then log_error "COVERDIR undeclared" return 255 fi local coverdir coverdir=$(readlink -f "${COVERDIR}") mkdir -p "${coverdir}" find "${coverdir}" -print0 -name '*.coverprofile' | xargs -0 rm local covpkgs covpkgs=$(not_test_packages) local coverpkg_comma coverpkg_comma=$(echo "${covpkgs[@]}" | xargs | tr ' ' ',') local gocov_build_flags=("-covermode=set" "-coverpkg=$coverpkg_comma") local failed="" log_callout "[$(date)] Collecting coverage from unit tests ..." for m in $(module_dirs); do run_for_module "${m}" go_test "./..." "parallel" "pkg_to_coverprofileflag unit_${m}" -short -timeout=30m \ "${gocov_build_flags[@]}" "$@" || failed="$failed unit" done log_callout "[$(date)] Collecting coverage from integration tests ..." run_for_module "tests" go_test "./integration/..." "parallel" "pkg_to_coverprofileflag integration" \ -timeout=30m "${gocov_build_flags[@]}" "$@" || failed="$failed integration" # integration-store-v2 run_for_module "tests" go_test "./integration/v2store/..." "keep_going" "pkg_to_coverprofileflag store_v2" \ -timeout=5m "${gocov_build_flags[@]}" "$@" || failed="$failed integration_v2" # integration_cluster_proxy run_for_module "tests" go_test "./integration/..." "parallel" "pkg_to_coverprofileflag integration_cluster_proxy" \ -tags cluster_proxy -timeout=30m "${gocov_build_flags[@]}" || failed="$failed integration_cluster_proxy" local cover_out_file="${coverdir}/all.coverprofile" merge_cov "${coverdir}" # strip out generated files (using GNU-style sed) sed --in-place -E "/[.]pb[.](gw[.])?go/d" "${cover_out_file}" || true sed --in-place -E "s|go.etcd.io/etcd/api/v3/|api/|g" "${cover_out_file}" || true sed --in-place -E "s|go.etcd.io/etcd/client/v3/|client/v3/|g" "${cover_out_file}" || true sed --in-place -E "s|go.etcd.io/etcd/client/pkg/v3|client/pkg/v3/|g" "${cover_out_file}" || true sed --in-place -E "s|go.etcd.io/etcd/etcdctl/v3/|etcdctl/|g" "${cover_out_file}" || true sed --in-place -E "s|go.etcd.io/etcd/etcdutl/v3/|etcdutl/|g" "${cover_out_file}" || true sed --in-place -E "s|go.etcd.io/etcd/pkg/v3/|pkg/|g" "${cover_out_file}" || true sed --in-place -E "s|go.etcd.io/etcd/server/v3/|server/|g" "${cover_out_file}" || true # held failures to generate the full coverage file, now fail if [ -n "$failed" ]; then for f in $failed; do log_error "--- FAIL:" "$f" done log_warning "Despite failures, you can see partial report:" log_warning " go tool cover -html ${cover_out_file}" return 255 fi log_success "done :) [see report: go tool cover -html ${cover_out_file}]" } ######### Code formatting checkers ############################################# function shellcheck_pass { SHELLCHECK=shellcheck if ! tool_exists "shellcheck" "https://github.com/koalaman/shellcheck#installing"; then log_callout "Installing shellcheck $SHELLCHECK_VERSION" wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.${OS}.${ARCH}.tar.xz" | tar -xJv -C /tmp/ --strip-components=1 mkdir -p ./bin mv /tmp/shellcheck ./bin/ SHELLCHECK=./bin/shellcheck fi generic_checker run ${SHELLCHECK} -fgcc scripts/*.sh } function shellws_pass { log_callout "Ensuring no tab-based indention in shell scripts" local files if files=$(find . -name '*.sh' -print0 | xargs -0 grep -E -n $'^\s*\t'); then log_error "FAIL: found tab-based indention in the following bash scripts. Use ' ' (double space):" log_error "${files}" log_warning "Suggestion: run \"make fix\" to address the issue." return 255 fi log_success "SUCCESS: no tabulators found." } function markdown_marker_pass { local marker="marker" # TODO: check other markdown files when marker handles headers with '[]' if ! tool_exists "$marker" "https://crates.io/crates/marker"; then log_callout "Installing markdown marker $MARKDOWN_MARKER_VERSION" MARKER_OS=$OS if [ "$OS" = "darwin" ]; then MARKER_OS="apple-darwin" elif [ "$OS" = "linux" ]; then MARKER_OS="unknown-linux-musl" fi wget -qO- "https://github.com/crawford/marker/releases/download/${MARKDOWN_MARKER_VERSION}/marker-${MARKDOWN_MARKER_VERSION}-${ARCH}-${MARKER_OS}.tar.gz" | tar -xzv -C /tmp/ --strip-components=1 >/dev/null mkdir -p ./bin mv /tmp/marker ./bin/ marker=./bin/marker fi generic_checker run "${marker}" --skip-http --allow-absolute-paths --root "${ETCD_ROOT_DIR}" -e ./CHANGELOG -e ./etcdctl -e etcdutl -e ./tools 2>&1 } function govuln_pass { run go install golang.org/x/vuln/cmd/govulncheck@latest run_for_modules run govulncheck -show verbose } function lint_pass { run_for_all_workspace_modules golangci-lint run --config "${ETCD_ROOT_DIR}/tools/.golangci.yaml" } function lint_fix_pass { run_for_all_workspace_modules golangci-lint run --config "${ETCD_ROOT_DIR}/tools/.golangci.yaml" --fix } function bom_pass { log_callout "Checking bill of materials..." local _bom_modules=() load_workspace_relative_modules_for_bom _bom_modules # Internally license-bill-of-materials tends to modify go.sum run cp go.sum go.sum.tmp || return 2 run cp go.mod go.mod.tmp || return 2 # Intentionally run the command once first, so it fetches dependencies. The exit code on the first # run in a just cloned repository is always dirty. GOOS=linux run_go_tool github.com/appscodelabs/license-bill-of-materials \ --override-file ./bill-of-materials.override.json "${_bom_modules[@]}" &>/dev/null # BOM file should be generated for linux. Otherwise running this command on other operating systems such as OSX # results in certain dependencies being excluded from the BOM file, such as procfs. # For more info, https://github.com/etcd-io/etcd/issues/19665 output=$(GOOS=linux run_go_tool github.com/appscodelabs/license-bill-of-materials \ --override-file ./bill-of-materials.override.json \ "${_bom_modules[@]}") local code="$?" run cp go.sum.tmp go.sum || return 2 run cp go.mod.tmp go.mod || return 2 if [ "${code}" -ne 0 ] ; then log_error -e "license-bill-of-materials (code: ${code}) failed with:\\n${output}" return 255 else echo "${output}" > "bom-now.json.tmp" fi if ! diff ./bill-of-materials.json bom-now.json.tmp; then log_error "modularized licenses do not match given bill of materials" return 255 fi rm bom-now.json.tmp } function module_gomodguard { if [ ! -f .gomodguard.yaml ]; then # Nothing to validate, return. return fi local tool_bin="$1" run "${tool_bin}" } function gomodguard_pass { local tool_bin tool_bin=$(tool_get_bin github.com/ryancurrah/gomodguard/cmd/gomodguard) run_for_workspace_modules module_gomodguard "${tool_bin}" } ######## VARIOUS CHECKERS ###################################################### function dump_module_deps() { local json_mod json_mod=$(run go mod edit -json) local module if ! module=$(echo "${json_mod}" | jq -r .Module.Path); then return 255 fi local require require=$(echo "${json_mod}" | jq -r '.Require') if [ "$require" == "null" ]; then return 0 fi echo "$require" | jq -r '.[] | .Path+","+.Version+","+if .Indirect then " (indirect)" else "" end+",'"${module}"'"' } # Checks whether dependencies are consistent across modules function dep_pass { local all_dependencies all_dependencies=$(run_for_workspace_modules dump_module_deps | sort) || return 2 local duplicates duplicates=$(echo "${all_dependencies}" | cut -d ',' -f 1,2 | sort | uniq | cut -d ',' -f 1 | sort | uniq -d) || return 2 if [[ -n "${duplicates}" ]]; then for dup in ${duplicates}; do log_error "FAIL: inconsistent versions for dependency: ${dup}" echo "${all_dependencies}" | grep "${dup}," | sed 's|\([^,]*\),\([^,]*\),\([^,]*\),\([^,]*\)| - \1@\2\3 from: \4|g' done log_error "FAIL: inconsistent dependencies" return 2 fi log_success "SUCCESS: dependencies are consistent across modules" } function release_pass { rm -f ./bin/etcd-last-release # Work out the previous release based on the version reported by etcd binary binary_version=$(./bin/etcd --version | grep --only-matching --perl-regexp '(?<=etcd Version: )\d+\.\d+') binary_major=$(echo "${binary_version}" | cut -d '.' -f 1) binary_minor=$(echo "${binary_version}" | cut -d '.' -f 2) previous_minor=$((binary_minor - 1)) # Handle the edge case where we go to a new major version # When this happens we obtain latest minor release of previous major if [ "${binary_minor}" -eq 0 ]; then binary_major=$((binary_major - 1)) previous_minor=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \ | grep --only-matching --perl-regexp "(?<=v)${binary_major}.\d.[\d]+?(?=[\^])" \ | sort --numeric-sort --key 1.3 | tail -1 | cut -d '.' -f 2) fi # This gets a list of all remote tags for the release branch in regex # Sort key is used to sort numerically by patch version # Latest version is then stored for use below UPGRADE_VER=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \ | grep --only-matching --perl-regexp "(?<=v)${binary_major}.${previous_minor}.[\d]+?(?=[\^])" \ | sort --numeric-sort --key 1.5 | tail -1 | sed 's/^/v/') log_callout "Found previous minor version (v${binary_major}.${previous_minor}) latest release: ${UPGRADE_VER}." if [ -n "${MANUAL_VER:-}" ]; then # in case, we need to test against different version UPGRADE_VER=$MANUAL_VER fi if [[ -z ${UPGRADE_VER} ]]; then UPGRADE_VER="v3.5.0" log_warning "fallback to" ${UPGRADE_VER} fi local file if [[ "$(uname -s)" == 'Darwin' ]]; then file="etcd-$UPGRADE_VER-darwin-$GOARCH.zip" else file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz" fi log_callout "Downloading $file" set +e curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file" local result=$? set -e case $result in 0) ;; *) log_error "--- FAIL:" ${result} return $result ;; esac tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1 --no-same-owner mkdir -p ./bin mv /tmp/etcd ./bin/etcd-last-release } function release_tests_pass { if [ -z "${VERSION:-}" ]; then VERSION=$(go list -m go.etcd.io/etcd/api/v3 2>/dev/null | \ awk '{split(substr($2,2), a, "."); print a[1]"."a[2]".99"}') fi if [ -n "${CI:-}" ]; then git config user.email "prow@etcd.io" git config user.name "Prow" gpg --batch --gen-key </dev/null; then log_error "Error: Cannot find docker. Please follow the installation instructions at: https://docs.docker.com/get-docker/" exit 1 fi # Start a container with the given Docker image. function start_container { local container_name=$1 local image=$2 # run docker in the background docker run --detach --rm --name "${container_name}" "${image}" # wait for etcd daemon to bootstrap local attempts=0 while ! docker exec "${container_name}" /usr/local/bin/etcdctl endpoint health --command-timeout=1s; do sleep 1 attempts=$((attempts + 1)) if [ "${attempts}" -gt 10 ]; then log_error "Error: etcd daemon failed to start." exit 1 fi done } # Run a version check for the given Docker image. function run_version_check { local output local found_version local image=$1 shift local expected_version=$1 shift output=$(docker run --rm "${image}" "${@}") found_version=$(echo "${output}" | head -1 | rev | cut -d" " -f 1 | rev) if [[ "${found_version}" != "${expected_version}" ]]; then log_error "Error: Invalid Version." log_error "Got ${found_version}, expected ${expected_version}." log_error "Output: ${output}." exit 1 fi } # Put a key-value pair in the etcd container and check if it can be retrieved, # and has the expected value. function put_get_check { local container_name=$1 local key="foo" local value="bar" local result result=$(docker exec "${container_name}" /usr/local/bin/etcdctl put "${key}" "${value}") if [ "${result}" != "OK" ]; then log_error "Error: Storing key failed. Result: ${result}." exit 1 fi result=$(docker exec "${container_name}" /usr/local/bin/etcdctl get "${key}" --print-value-only) if [ "${result}" != "${value}" ]; then log_error "Error: Problem with getting key. Got: ${result}, expected: ${value}." exit 1 fi } # Verify that the images have the correct architecture. function verify_images_architecture { local repository=$1 local version=$2 local target_arch local arch_tag local img_arch for target_arch in "amd64" "arm64" "ppc64le" "s390x"; do arch_tag="v${version}-${target_arch}" img_arch=$(docker inspect --format '{{.Architecture}}' "${repository}:${arch_tag}") if [ "${img_arch}" != "${target_arch}" ];then log_error "Error: Incorrect Docker image architecture. Got ${img_arch}, expected: ${arch_tag}." exit 1 fi log_success "Correct architecture for ${arch_tag}." done } function main { local version="$1" local repository=${REPOSITORY:-"gcr.io/etcd-development/etcd"} local arch arch=$(go env GOARCH) local tag="v${version}-${arch}" local image="${TEST_IMAGE:-"${repository}:${tag}"}" local container_name="test_etcd" if [[ "$(docker images -q "${image}" 2> /dev/null)" == "" ]]; then log_error "Error: ${image} not present locally." exit 1 fi log_callout "Running version check." run_version_check "${image}" "${version}" "/usr/local/bin/etcd" "--version" run_version_check "${image}" "${version}" "/usr/local/bin/etcdctl" "version" run_version_check "${image}" "${version}" "/usr/local/bin/etcdutl" "version" log_success "Successfully ran version check." log_callout "Running sanity check in Docker image." start_container "${container_name}" "${image}" # stop container trap 'docker stop '"${container_name}" EXIT put_get_check "${container_name}" log_success "Successfully tested etcd local image ${tag}." log_callout "Verifying images architecture." verify_images_architecture "${repository}" "${version}" log_success "Successfully tested images architecture." } if [ -z "$VERSION" ]; then log_error "Error: VERSION not supplied." exit 1 fi main "${VERSION}" ================================================ FILE: scripts/test_lib.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail source ./scripts/test_utils.sh ROOT_MODULE="go.etcd.io/etcd" if [[ "$(go list)" != "${ROOT_MODULE}/v3" ]]; then echo "must be run from '${ROOT_MODULE}/v3' module directory" exit 255 fi function set_root_dir { ETCD_ROOT_DIR=$(go list -f '{{.Dir}}' "${ROOT_MODULE}/v3") } set_root_dir #### Discovery of files/packages within a go module ##### # pkgs_in_module [optional:package_pattern] # returns list of all packages in the current (dir) module. # if the package_pattern is given, its being resolved. function pkgs_in_module { go list -mod=mod "${1:-./...}"; } # Prints subdirectory (from the repo root) for the current module. function module_subdir { relativePath "${ETCD_ROOT_DIR}" "${PWD}" } #### Running actions against multiple modules #### # run [command...] - runs given command, printing it first and # again if it failed (in RED). Use to wrap important test commands # that user might want to re-execute to shorten the feedback loop when fixing # the test. function run { local rpath local command rpath=$(module_subdir) # Quoting all components as the commands are fully copy-parsable: command=("${@}") command=("${command[@]@Q}") if [[ "${rpath}" != "." && "${rpath}" != "" ]]; then repro="(cd ${rpath} && ${command[*]})" else repro="${command[*]}" fi log_cmd "% ${repro}" "${@}" 2> >(while read -r line; do echo -e "${COLOR_NONE}stderr: ${COLOR_MAGENTA}${line}${COLOR_NONE}">&2; done) local error_code=$? if [ ${error_code} -ne 0 ]; then log_error -e "FAIL: (code:${error_code}):\\n % ${repro}" return ${error_code} fi } # run_for_module [module] [cmd] # executes given command in the given module for given pkgs. # module_name - "." (in future: tests, client, server) # cmd - cmd to be executed - that takes package as last argument function run_for_module { local module=${1:-"."} shift 1 ( cd "${ETCD_ROOT_DIR}/${module}" && "$@" ) } function module_dirs() { echo "api pkg client/pkg client/v3 server etcdutl etcdctl tests tools/mod tools/rw-heatmaps tools/testgrid-analysis cache ." } # maybe_run [cmd...] runs given command depending on the DRY_RUN flag. function maybe_run() { if ${DRY_RUN}; then log_warning -e "# DRY_RUN:\\n % ${*}" else run "${@}" fi } # modules # returns the list of all modules in the project, not including the tools, # as they are not considered to be added to the bill for materials. function modules() { modules=( "${ROOT_MODULE}/api/v3" "${ROOT_MODULE}/pkg/v3" "${ROOT_MODULE}/client/pkg/v3" "${ROOT_MODULE}/client/v3" "${ROOT_MODULE}/server/v3" "${ROOT_MODULE}/etcdutl/v3" "${ROOT_MODULE}/etcdctl/v3" "${ROOT_MODULE}/tests/v3" "${ROOT_MODULE}/v3") echo "${modules[@]}" } # Receives a reference to an array variable, and returns the workspace relative modules. function load_workspace_relative_modules() { local -n _relative_modules=$1 while IFS= read -r line; do _relative_modules+=("$line"); done < <( go work edit -json | jq -r '.Use[].DiskPath + "/..."' ) } # Receives a reference to an array variable, and returns the workspace relative modules, not # including the tools, as they are not considered to be added to the bill for materials. function load_workspace_relative_modules_for_bom() { local -n relative_modules_for_bom=$1 local modules=() load_workspace_relative_modules modules for module in "${modules[@]}"; do if [[ ! "${module}" =~ ^./tools ]]; then relative_modules_for_bom+=("${module}") fi done } # run_for_all_workspace_modules [cmd] # run given command across all workspace modules # (unless the set is limited using ${PKG} or / ${USERMOD}) function run_for_all_workspace_modules { local pkg="${PKG:-./...}" if [ -z "${USERMOD:-}" ]; then local _modules=() load_workspace_relative_modules _modules run "$@" "${_modules[@]}" else run_for_module "${USERMOD}" "$@" "${pkg}" || return "$?" fi } # run_for_workspace_modules [cmd] # run given command in each individual workspace module # (unless the set is limited using ${PKG} or / ${USERMOD}) function run_for_workspace_modules { local keep_going_module=${KEEP_GOING_MODULE:-false} local fail_mod=false local pkg="${PKG:-./...}" if [ -z "${USERMOD:-}" ]; then local _modules=() load_workspace_relative_modules _modules for module in "${_modules[@]}"; do if ! run_for_module "${module%...}" "$@"; then if [ "$keep_going_module" = false ]; then log_error "There was a Failure in module ${module}, aborting..." return 1 fi log_error "There was a Failure in module ${module}, keep going..." fail_mod=true fi done if [ "$fail_mod" = true ]; then return 1 fi else run_for_module "${USERMOD}" "$@" "${pkg}" || return "$?" fi } # run_for_modules [cmd] # run given command across all modules and packages # (unless the set is limited using ${PKG} or / ${USERMOD}) function run_for_modules { KEEP_GOING_MODULE=${KEEP_GOING_MODULE:-false} local pkg="${PKG:-./...}" local fail_mod=false if [ -z "${USERMOD:-}" ]; then for m in $(module_dirs); do if run_for_module "${m}" "$@" "${pkg}"; then continue else if [ "$KEEP_GOING_MODULE" = false ]; then log_error "There was a Failure in module ${m}, aborting..." return 1 fi log_error "There was a Failure in module ${m}, keep going..." fail_mod=true fi done if [ "$fail_mod" = true ]; then return 1 fi else run_for_module "${USERMOD}" "$@" "${pkg}" || return "$?" fi } function get_junit_filename_prefix { local junit_report_dir="$1" if [[ -z "${junit_report_dir}" ]]; then echo "" return fi mkdir -p "${junit_report_dir}" mktemp --dry-run "${junit_report_dir}/junit_XXXXXXXXXX" } junitFilenamePrefix() { if [[ -z "${JUNIT_REPORT_DIR:-}" ]]; then echo "" return fi mkdir -p "${JUNIT_REPORT_DIR}" DATE=$( date +%s | base64 | head -c 15 ) echo "${JUNIT_REPORT_DIR}/junit_$DATE" } function produce_junit_xmlreport { local -r junit_filename_prefix=${1:-} if [[ -z "${junit_filename_prefix}" ]]; then return fi local junit_xml_filename junit_xml_filename="${junit_filename_prefix}.xml" # Ensure that gotestsum is run without cross-compiling run_go_tool gotest.tools/gotestsum --junitfile "${junit_xml_filename}" --raw-command cat "${junit_filename_prefix}"*.stdout || exit 1 if [ "${VERBOSE:-}" != "1" ]; then rm "${junit_filename_prefix}"*.stdout fi log_callout "Saved JUnit XML test report to ${junit_xml_filename}" } #### Running go test ######## # go_test [packages] [mode] [flags_for_package_func] [$@] # [mode] supports 3 states: # - "parallel": fastest as concurrently processes multiple packages, but silent # till the last package. See: https://github.com/golang/go/issues/2731 # - "keep_going" : executes tests package by package, but postpones reporting error to the last # - "fail_fast" : executes tests packages 1 by 1, exits on the first failure. # # [flags_for_package_func] is a name of function that takes list of packages as parameter # and computes additional flags to the go_test commands. # Use 'true' or ':' if you dont need additional arguments. # # depends on the VERBOSE top-level variable. # # Example: # go_test "./..." "keep_going" ":" --short # # The function returns != 0 code in case of test failure. function go_test { local packages="${1}" local mode="${2}" local flags_for_package_func="${3}" local junit_filename_prefix shift 3 local goTestFlags="" local goTestEnv="" ##### Create a junit-style XML test report in this directory if set. ##### JUNIT_REPORT_DIR=${JUNIT_REPORT_DIR:-} # If JUNIT_REPORT_DIR is unset, and ARTIFACTS is set, then have them match. if [[ -z "${JUNIT_REPORT_DIR:-}" && -n "${ARTIFACTS:-}" ]]; then export JUNIT_REPORT_DIR="${ARTIFACTS}" fi # Used to filter verbose test output. go_test_grep_pattern=".*" if [[ -n "${JUNIT_REPORT_DIR}" ]] ; then goTestFlags+="-v " goTestFlags+="-json " # Show only summary lines by matching lines like "status package/test" go_test_grep_pattern="^[^[:space:]]\+[[:space:]]\+[^[:space:]]\+/[^[[:space:]]\+" fi junit_filename_prefix=$(junitFilenamePrefix) if [ "${VERBOSE:-}" == "1" ]; then goTestFlags="-v " goTestFlags+="-json " fi # Expanding patterns (like ./...) into list of packages local unpacked_packages=("${packages}") if [ "${mode}" != "parallel" ]; then # shellcheck disable=SC2207 # shellcheck disable=SC2086 if ! unpacked_packages=($(go list ${packages})); then log_error "Cannot resolve packages: ${packages}" return 255 fi fi if [ "${mode}" == "fail_fast" ]; then goTestFlags+="-failfast " fi local failures="" # execution of tests against packages: for pkg in "${unpacked_packages[@]}"; do local additional_flags # shellcheck disable=SC2086 additional_flags=$(${flags_for_package_func} ${pkg}) # shellcheck disable=SC2206 local cmd=( go test ${goTestFlags} ${additional_flags} ${pkg} "$@" ) # shellcheck disable=SC2086 if ! run env ${goTestEnv} ETCD_VERIFY="${ETCD_VERIFY}" "${cmd[@]}" | tee ${junit_filename_prefix:+"${junit_filename_prefix}.stdout"} | grep --binary-files=text "${go_test_grep_pattern}" ; then if [ "${mode}" != "keep_going" ]; then produce_junit_xmlreport "${junit_filename_prefix}" return 2 else failures=("${failures[@]}" "${pkg}") fi fi produce_junit_xmlreport "${junit_filename_prefix}" done if [ -n "${failures[*]}" ] ; then log_error -e "ERROR: Tests for following packages failed:\\n ${failures[*]}" return 2 fi } # run_go_tests_expanding_packages [arguments to pass to go test] # Expands the packages in the list of arguments, i.e. ./... into a list of # packages for that given module. Then, it calls run_go_tests with the expanded # packages. Implements the legacy modes for non-parallel testing. function run_go_tests_expanding_packages { local packages=() local args=() for arg in "$@"; do if [[ "${arg}" =~ ^\./ || "${arg}" =~ ^go\.etcd\.io/etcd ]]; then packages+=("${arg}") else args+=("${arg}") fi done # Expanding patterns (like ./...) into list of packages local unpacked_packages=() while IFS='' read -r line; do unpacked_packages+=("$line"); done < <( go list "${packages[@]}" ) run_go_tests "${unpacked_packages[@]}" "${args[@]}" } # run_go_test [arguments to pass to go test] # The following environment variables affect how the tests run: # - JUNIT_REPORT_DIR/ARTIFACTS: Enables collecting JUnit XML reports. # - VERBOSE: Sets a verbose output. # # Example: # KEEP_GOING_TESTS=true run_go_tests "./..." --short # # The function returns != 0 code in case of test failure. function run_go_tests { local go_test_flags=() # If JUNIT_REPORT_DIR is unset, and ARTIFACTS is set, then have them match. local junit_report_dir=${JUNIT_REPORT_DIR:-${ARTIFACTS:-}} local go_test_grep_pattern=".*" if [[ -n "${junit_report_dir}" ]]; then # Show only summary lines by matching lines like "status package/test" go_test_grep_pattern="^[^[:space:]]\+[[:space:]]\+[^[:space:]]\+/[^[[:space:]]\+" fi if [[ -n "${junit_report_dir}" || "${VERBOSE:-}" == "1" ]]; then go_test_flags+=("-v" "-json") fi local cmd=(go test "${go_test_flags[@]}" "$@") local junit_filename_prefix junit_filename_prefix=$(get_junit_filename_prefix "${junit_report_dir}") if ! run env ETCD_VERIFY="${ETCD_VERIFY}" "${cmd[@]}" | tee ${junit_filename_prefix:+"${junit_filename_prefix}.stdout"} | grep --binary-files=text "${go_test_grep_pattern}" ; then produce_junit_xmlreport "${junit_filename_prefix}" return 2 fi produce_junit_xmlreport "${junit_filename_prefix}" } #### Other #### # tool_exists [tool] [instruction] # Checks whether given [tool] is installed. In case of failure, # prints a warning with installation [instruction] and returns !=0 code. # # WARNING: This depend on "any" version of the 'binary' that might be tricky # from hermetic build perspective. For go binaries prefer 'tool_go_run' function tool_exists { local tool="${1}" local instruction="${2}" if ! command -v "${tool}" >/dev/null; then log_warning "Tool: '${tool}' not found on PATH. ${instruction}" return 255 fi } # tool_get_bin [tool] - returns absolute path to a tool binary (or returns error). # This function is only used to run commands that are managed by tools/mod. function tool_get_bin { local tool="$1" local pkg_part="$1" if [[ "$tool" == *"@"* ]]; then pkg_part=$(echo "${tool}" | cut -d'@' -f1) # shellcheck disable=SC2086 run go install ${GOBINARGS:-} "${tool}" || return 2 else # shellcheck disable=SC2086 run_for_module ./tools/mod run go install ${GOBINARGS:-} "${tool}" || return 2 fi # remove the version suffix, such as removing "/v3" from "go.etcd.io/etcd/v3". local cmd_base_name cmd_base_name=$(basename "${pkg_part}") if [[ ${cmd_base_name} =~ ^v[0-9]*$ ]]; then pkg_part=$(dirname "${pkg_part}") fi run_for_module ./tools/mod go list -f '{{.Target}}' "${pkg_part}" } # tool_pkg_dir [pkg] - returns absolute path to a directory that stores given pkg. # The pkg versions must be defined in ./tools/mod directory. function tool_pkg_dir { run_for_module ./tools/mod run go list -f '{{.Dir}}' "${1}" } # tool_get_bin [tool] function run_go_tool { local cmdbin if ! cmdbin=$(GOARCH="" GOOS="" tool_get_bin "${1}"); then log_warning "Failed to install tool '${1}'" return 2 fi shift 1 GOARCH="" run "${cmdbin}" "$@" || return 2 } # assert_no_git_modifications fails if there are any uncommitted changes. function assert_no_git_modifications { log_callout "Making sure everything is committed." if ! git diff --cached --exit-code; then log_error "Found staged by uncommitted changes. Do commit/stash your changes first." return 2 fi if ! git diff --exit-code; then log_error "Found unstaged and uncommitted changes. Do commit/stash your changes first." return 2 fi } # makes sure that the current branch is in sync with the origin branch: # - no uncommitted nor unstaged changes # - no differencing commits in relation to the origin/$branch function git_assert_branch_in_sync { local branch # TODO: When git 2.22 popular, change to: # branch=$(git branch --show-current) branch=$(run git rev-parse --abbrev-ref HEAD) log_callout "Verify the current branch '${branch}' is clean" if [[ $(run git status --porcelain --untracked-files=no) ]]; then log_error "The workspace in '$(pwd)' for branch: ${branch} has uncommitted changes" log_error "Consider cleaning up / renaming this directory or (cd $(pwd) && git reset --hard)" return 2 fi log_callout "Verify the current branch '${branch}' is in sync with the 'origin/${branch}'" if [ -n "${branch}" ]; then ref_local=$(run git rev-parse "${branch}") ref_origin=$(run git rev-parse "origin/${branch}") if [ "x${ref_local}" != "x${ref_origin}" ]; then log_error "In workspace '$(pwd)' the branch: ${branch} diverges from the origin." log_error "Consider cleaning up / renaming this directory or (cd $(pwd) && git reset --hard origin/${branch})" return 2 fi else log_warning "Cannot verify consistency with the origin, as git is on detached branch." fi } # The version present in the .go-verion is the default version that test and build scripts will use. # However, it is possible to control the version that should be used with the help of env vars: # - FORCE_HOST_GO: if set to a non-empty value, use the version of go installed in system's $PATH. # - GO_VERSION: desired version of go to be used, might differ from what is present in .go-version. # If empty, the value defaults to the version in .go-version. function determine_go_version { # Borrowing from how Kubernetes does this: # https://github.com/kubernetes/kubernetes/blob/17854f0e0a153b06f9d0db096e2cd8ab2fa89c11/hack/lib/golang.sh#L510-L520 # # default GO_VERSION to content of .go-version GO_VERSION="${GO_VERSION:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" if [ "${GOTOOLCHAIN:-auto}" != 'auto' ]; then # no-op, just respect GOTOOLCHAIN : elif [ -n "${FORCE_HOST_GO:-}" ]; then export GOTOOLCHAIN='local' else GOTOOLCHAIN="go${GO_VERSION}" export GOTOOLCHAIN fi } determine_go_version ================================================ FILE: scripts/test_utils.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -euo pipefail #### Convenient IO methods ##### export COLOR_RED='\033[0;31m' export COLOR_ORANGE='\033[0;33m' export COLOR_GREEN='\033[0;32m' export COLOR_LIGHTCYAN='\033[0;36m' export COLOR_BLUE='\033[0;94m' export COLOR_BOLD='\033[1m' export COLOR_MAGENTA='\033[95m' export COLOR_NONE='\033[0m' # No Color function log_error { >&2 echo -n -e "${COLOR_BOLD}${COLOR_RED}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } function log_warning { >&2 echo -n -e "${COLOR_ORANGE}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } function log_callout { >&2 echo -n -e "${COLOR_LIGHTCYAN}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } function log_cmd { >&2 echo -n -e "${COLOR_BLUE}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } function log_success { >&2 echo -n -e "${COLOR_GREEN}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } function log_info { >&2 echo -n -e "${COLOR_NONE}" >&2 echo "$@" >&2 echo -n -e "${COLOR_NONE}" } # From http://stackoverflow.com/a/12498485 function relativePath { # both $1 and $2 are absolute paths beginning with / # returns relative path to $2 from $1 local source=$1 local target=$2 local commonPart=$source local result="" while [[ "${target#"$commonPart"}" == "${target}" ]]; do # no match, means that candidate common part is not correct # go up one level (reduce common part) commonPart="$(dirname "$commonPart")" # and record that we went back, with correct / handling if [[ -z $result ]]; then result=".." else result="../$result" fi done if [[ $commonPart == "/" ]]; then # special case for root (no common path) result="$result/" fi # since we now have identified the common part, # compute the non-common part local forwardPart="${target#"$commonPart"}" # and now stick all parts together if [[ -n $result ]] && [[ -n $forwardPart ]]; then result="$result$forwardPart" elif [[ -n $forwardPart ]]; then # extra slash removal result="${forwardPart:1}" fi echo "$result" } ================================================ FILE: scripts/update_dep.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Usage: # ./scripts/update_dep.sh module version # or ./scripts/update_dep.sh module (to update to the latest version) # e.g. # ./scripts/update_dep.sh github.com/golang/groupcache # ./scripts/update_dep.sh github.com/soheilhy/cmux v0.1.5 # # Updates version of given dependency in all the modules that depend on the mod. set -euo pipefail source ./scripts/test_lib.sh if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then log_error "Illegal number of parameters. Usage: $0 module [version]" exit 1 fi mod="$1" ver="${2:-}" function print_current_dep_version { log_info "${mod} version in all go.mod files:" find . -name go.mod -exec grep -H "^\s*${mod}\s" {} + | sed 's|:|\t|' || true printf "\n" } function is_fully_indirect { local result result=$(find . -name go.mod -print0 | xargs -0 -I{} /bin/sh -c "cd \$(dirname {}); go list -f \"{{if eq .Path \\\"${mod}\\\"}}{{.Indirect}}{{end}}\" -m all" | sort | uniq) [ "$result" = "true" ] } function update_module { local subdir subdir=$(module_subdir) # The `go get` command is most effective on dependencies that are explicitly # listed as direct requirements in the go.mod file. When updating a purely # indirect dependency, `go get` might not update it as expected. # # To work around this, we temporarily promote the indirect dependency to a # direct one in the go.mod file using `go mod edit`. This ensures that # `go get` will see and correctly update the module. Subsequent cleanup # commands (like `go mod tidy`) will automatically move it back # to an indirect dependency, but at the designated updated version. # # Note: `go mod edit` requires a specific version (e.g., v1.2.3), so we only # use it when a version is explicitly provided. For "latest", we skip this # step and let `go get -u` handle it directly. if [ -n "${ver}" ]; then run go mod edit -require "${mod}@${ver}" || true fi # Check if the module is a dependency. if go list -m all | grep -q -E "^\s*${mod}\s"; then log_info " Updating in ${subdir}..." if [ -z "${ver}" ]; then run go get -u "${mod}" else run go get "${mod}@${ver}" fi fi } print_current_dep_version if is_fully_indirect; then read -p "Module ${mod} is a purely indirect dependency. Are you sure you want to update it? [y/N] " -r confirm [[ "$confirm" == [Yy] ]] || exit # Default is No fi log_info "Updating '${mod}' to ${ver:-latest} across all modules..." run_for_modules update_module make fix-mod-tidy fix-bom update-go-workspace verify-dep print_current_dep_version ================================================ FILE: scripts/update_go_workspace.sh ================================================ #!/usr/bin/env bash # # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Based on k/k scripts/update-go-workspace.sh: # https://github.com/kubernetes/kubernetes/blob/e2b96b25661849775dedf441b2f5c555392caa84/hack/update-go-workspace.sh # This script generates go.work so that it includes all Go packages # in this repo, with a few exceptions. set -euo pipefail source ./scripts/test_lib.sh # Detect sed variant (BSD vs GNU) for in-place editing # Source: https://stackoverflow.com/a/22084103 (CC BY-SA 4.0) case "$OSTYPE" in darwin*|bsd*) sed_no_backup=( -i '' ) ;; *) sed_no_backup=( -i ) ;; esac # Avoid issues and remove the workspace files. rm -f go.work go.work.sum # Generate the workspace. go work init # Prepend comment header (portable sed syntax with literal newline after backslash) sed "${sed_no_backup[@]}" '1i\ // This is a generated file. Do not edit directly.\ ' go.work # Include all submodules from the repository. # Use while-read loop for portability (dirname -z is GNU-specific, not available on macOS) git ls-files -z ':(glob)**/go.mod' | while IFS= read -r -d '' modfile; do go work edit -use "$(dirname "$modfile")" done go work edit -toolchain "go$(cat .go-version)" go work edit -go "$(go mod edit -json | jq -r .Go)" # generate go.work.sum go mod download ================================================ FILE: scripts/update_proto_annotations.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Updates etcd_version_annotations.txt based on state of annotations in proto files. # Developers can run this script to avoid manually updating etcd_version_annotations.txt. # Before running this script please ensure that fields/messages that you added are annotated with next etcd version. set -o errexit set -o nounset set -o pipefail tmpfile=$(mktemp) go run ./tools/proto-annotations/main.go --annotation etcd_version > "${tmpfile}" mv "${tmpfile}" ./scripts/etcd_version_annotations.txt ================================================ FILE: scripts/verify_genproto.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # This scripts is automatically run by CI to prevent pull requests missing running genproto.sh # after changing *.proto file. set -o errexit set -o nounset set -o pipefail tmpWorkDir=$(mktemp -d -t 'twd.XXXXXX') mkdir "$tmpWorkDir/etcd" tmpWorkDir="$tmpWorkDir/etcd" cp -r . "$tmpWorkDir" pushd "$tmpWorkDir" git add -A git commit -m init || true # maybe fail because nothing to commit ./scripts/genproto.sh diff=$(git diff --numstat | awk '{print $3}') popd if [ -z "$diff" ]; then echo "PASSED genproto-verification!" exit 0 fi echo "Failed genproto-verification!" >&2 printf "* Found changed files:\n%s\n" "$diff" >&2 echo "* Please rerun genproto.sh after changing *.proto file" >&2 echo "* Run ./scripts/genproto.sh" >&2 exit 1 ================================================ FILE: scripts/verify_go_versions.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # This script verifies that the value of the toolchain directive in the # go.mod files always match that of the .go-version file to ensure that # we accidentally don't test and release with differing versions of Go. set -euo pipefail source ./scripts/test_lib.sh target_go_version="${target_go_version:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" log_info "expected go toolchain directive: go${target_go_version}" log_info toolchain_out_of_sync="false" go_line_violation="false" # verify_go_versions takes a go.mod filepath as an argument # and checks if: # (1) go directive <= version in .go-version # (2) toolchain directive == version in .go-version function verify_go_versions() { # shellcheck disable=SC2086 toolchain_version="$(go mod edit -json $1 | jq -r .Toolchain)" # shellcheck disable=SC2086 go_line_version="$(go mod edit -json $1 | jq -r .Go)" if [[ "go${target_go_version}" != "${toolchain_version}" ]]; then log_error "go toolchain directive out of sync for $1, got: ${toolchain_version}" toolchain_out_of_sync="true" fi if ! printf '%s\n' "${go_line_version}" "${target_go_version}" | sort --check=silent --version-sort; then log_error "go directive in $1 is greater than maximum allowed: go${target_go_version}" go_line_violation="true" fi } # Workaround to get go.work's toolchain, as go work edit -json doesn't return # the toolchain as of Go 1.24. When this is fixed, we can replace these two # checks with verify_go_versions go.work toolchain_version="$(grep toolchain go.work | cut -d' ' -f2)" if [[ "go${target_go_version}" != "${toolchain_version}" ]]; then log_error "go toolchain directive out of sync for go.work, got: ${toolchain_version}" toolchain_out_of_sync="true" fi go_line_version="$(go work edit -json | jq -r .Go)" if ! printf '%s\n' "${go_line_version}" "${target_go_version}" | sort --check=silent --version-sort; then log_error "go directive in go.work is greater than maximum allowed: go${target_go_version}" go_line_violation="true" fi while read -r mod; do verify_go_versions "${mod}"; done < <(find . -name 'go.mod') if [[ "${toolchain_out_of_sync}" == "true" ]]; then log_error log_error "Please run scripts/sync_go_toolchain_directive.sh or update .go-version to rectify this error" fi if [[ "${go_line_violation}" == "true" ]]; then log_error log_error "Please update .go-version to rectify this error, any go directive should be <= .go-version" fi if [[ "${go_line_violation}" == "true" ]] || [[ "${toolchain_out_of_sync}" == "true" ]]; then exit 1 fi log_success "SUCCESS: Go toolchain directive in sync" ================================================ FILE: scripts/verify_golangci-lint_version.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. function install_golangci_lint() { echo "Installing golangci-lint ${GOLANGCI_LINT_VERSION}" curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" "${GOLANGCI_LINT_VERSION}" } GOLANGCI_LINT_VERSION=$(cd tools/mod && go list -m -f '{{.Version}}' github.com/golangci/golangci-lint/v2) echo "golangci-lint version: $GOLANGCI_LINT_VERSION" GOLANGCI_LINT_PRESENT=$(which golangci-lint) if [ -z "$GOLANGCI_LINT_PRESENT" ]; then echo "golangci-lint is not available" install_golangci_lint exit 0 fi GOLANGCI_LINT_INSTALLED=v$(golangci-lint version | grep -Eo 'version [0-9.]+' | grep -Eo '[0-9.]+') if [ "$GOLANGCI_LINT_VERSION" != "$GOLANGCI_LINT_INSTALLED" ]; then echo "different golangci-lint version installed: $GOLANGCI_LINT_INSTALLED" install_golangci_lint echo "golangci-lint version: $GOLANGCI_LINT_VERSION" fi ================================================ FILE: scripts/verify_grpc_experimental.sh ================================================ #!/usr/bin/env bash set -e # Ensure we are at the root of the repo ROOT_DIR=$(git rev-parse --show-toplevel) cd "${ROOT_DIR}" source ./scripts/test_lib.sh TOOL_SRC="${ETCD_ROOT_DIR}/tools/check-grpc-experimental" ALLOWLIST="${TOOL_SRC}/allowlist.txt" FAILURES=0 for MOD_DIR in $(module_dirs); do echo "------------------------------------------------" echo "Checking module: ${MOD_DIR}" pushd "${MOD_DIR}" > /dev/null if ! go run "${TOOL_SRC}" -allow-list="${ALLOWLIST}" ./...; then echo "ERROR: Experimental usage found in ${MOD_DIR}" FAILURES=$((FAILURES+1)) fi popd > /dev/null done echo "------------------------------------------------" if [ "$FAILURES" -eq 0 ]; then echo "SUCCESS: No experimental gRPC APIs found in any module." exit 0 else echo "FAILURE: Found experimental gRPC API usage in ${FAILURES} module(s)." exit 1 fi ================================================ FILE: scripts/verify_proto_annotations.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Verifies proto annotations to ensure all new proto fields and messages are annotated by comparing it with etcd_version_annotations.txt file. # This scripts is automatically run by CI to prevent pull requests missing adding a proto annotation. set -o errexit set -o nounset set -o pipefail tmpfile=$(mktemp) go run ./tools/proto-annotations/main.go --annotation=etcd_version > "${tmpfile}" if diff -u ./scripts/etcd_version_annotations.txt "${tmpfile}"; then echo "PASSED proto-annotations verification!" exit 0 fi echo "Failed proto-annotations-verification!" >&2 echo "If you are adding new proto fields/messages that will be included in raft log:" >&2 echo "* Please add etcd_version annotation in *.proto file with next etcd version" >&2 echo "* Run ./scripts/genproto.sh" >&2 echo "* Run ./scripts/update_proto_annotations.sh" >&2 exit 1 ================================================ FILE: security/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/security ================================================ FILE: security/README.md ================================================ ## Security Announcements Join the [etcd-dev](https://groups.google.com/g/etcd-dev) group for emails about security and major announcements. ## Report a Vulnerability We’re extremely grateful for security researchers and users that report vulnerabilities to the etcd Open Source Community. All reports are thoroughly investigated by a dedicated committee of community volunteers called [Product Security Committee](security-release-process.md#product-security-committee). To make a report, please email the private [security@etcd.io](mailto:security@etcd.io) list with the security details and the details expected for [all etcd bug reports](https://github.com/etcd-io/etcd/blob/main/Documentation/contributor-guide/reporting_bugs.md). ### When Should I Report a Vulnerability? - When discovered a potential security vulnerability in etcd - When unsure how a vulnerability affects etcd - When discovered a vulnerability in another project that etcd depends on ### When Should I NOT Report a Vulnerability? - Need help tuning etcd for security - Need help applying security related updates - When an issue is not security related ## Security Vulnerability Response Each report is acknowledged and analyzed by Product Security Committee members within 3 working days. This will set off the [Security Release Process](security-release-process.md). Any vulnerability information shared with Product Security Committee stays within etcd project and will not be disseminated to other projects unless it is necessary to get the issue fixed. As the security issue moves from triage, to identified fix, to release planning we will keep the reporter updated. ## Public Disclosure Timing A public disclosure date is negotiated by the etcd Product Security Committee and the bug reporter. We prefer to fully disclose the bug as soon as possible once user mitigation is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for vendor coordination. The timeframe for disclosure is from immediate (especially if it's already publicly known) to a few weeks. As a basic default, we expect report date to disclosure date to be on the order of 7 days. The etcd Product Security Committee holds the final say when setting a disclosure date. ## Security Audit A third party security audit was performed by Trail of Bits, find the full report [here](SECURITY_AUDIT.pdf). A third party fuzzing audit was performed by Ada Logics, find the full report [here](FUZZING_AUDIT_2022.PDF). ================================================ FILE: security/email-templates.md ================================================ # etcd Security Process Email Templates This is a collection of email templates to handle various situations the security team encounters. ## Upcoming security release ``` Subject: Upcoming security release of etcd $VERSION To: etcd-dev@googlegroups.com Cc: security@etcd-io Cc: etcd-maintainers@googlegroups.com Hello etcd Community, The etcd Product Security Committee and maintainers would like to announce the forthcoming release of etcd $VERSION. This release will be made available on the $ORDINALDAY of $MONTH $YEAR at $PDTHOUR PDT ($GMTHOUR GMT). This release will fix $NUMDEFECTS security defect(s). The highest rated security defect is considered $SEVERITY severity. No further details or patches will be made available in advance of the release. **Thanks** Thanks to $REPORTER, $DEVELOPERS, and the $RELEASELEADS for the coordination is making this release. Thanks, $PERSON on behalf of the etcd Product Security Committee and maintainers ``` ## Security Fix Announcement ``` Subject: Security release of etcd $VERSION is now available To: etcd-dev@googlegroups.com Cc: security@etcd-io Cc: etcd-maintainers@googlegroups.com Hello etcd Community, The Product Security Committee and maintainers would like to announce the availability of etcd $VERSION. This addresses the following CVE(s): * CVE-YEAR-ABCDEF (CVSS score $CVSS): $CVESUMMARY ... Upgrading to $VERSION is encouraged to fix these issues. **Am I vulnerable?** Run `etcd --version` and if it indicates a base version of $OLDVERSION or older that means it is a vulnerable version. **How do I mitigate the vulnerability?** **How do I upgrade?** Follow the upgrade instructions at https://etcd.io/docs **Vulnerability Details** ***CVE-YEAR-ABCDEF*** $CVESUMMARY This issue is filed as $CVE. We have rated it as [$CVSSSTRING]($CVSSURL) ($CVSS, $SEVERITY) [See the GitHub issue for more details]($GITHUBISSUEURL) **Thanks** Thanks to $REPORTER, $DEVELOPERS, and the $RELEASELEADS for the coordination in making this release. Thanks, $PERSON on behalf of the etcd Product Security Committee and maintainers ``` ================================================ FILE: security/security-release-process.md ================================================ # Security Release Process etcd is a growing community of volunteers, users, and vendors. The etcd community has adopted this security disclosures and response policy to ensure we responsibly handle critical issues. ## Product Security Committee (PSC) Security vulnerabilities should be handled quickly and sometimes privately. The primary goal of this process is to reduce the total time users are vulnerable to publicly known exploits. The PSC is responsible for organizing the entire response including internal communication and external disclosure but will need help from relevant developers and release leads to successfully run this process. The PSC consists of the following: - Maintainers - Volunteer members as described in the [Product Security Committee Membership](#Product-Security-Committee-Membership) The PSC members will share various tasks as listed below: - Triage: make sure the people who should be in "the know" (aka notified) are notified, also responds to issues that are not actually issues and let the etcd maintainers know that. This person is the escalation path for a bug if it is one. - Infra: make sure we can test the fixes appropriately. - Disclosure: handles public messaging around the bug. Documentation on how to upgrade. Changelog. Explaining to public the severity. notifications of bugs sent to mailing lists etc. Requests CVEs. - Release: Create new release addressing a security fix. ### Contacting the Product Security Committee Contact the team by sending email to [security@etcd.io](mailto:security@etcd.io). ### Product Security Committee Membership #### Joining New potential members to the PSC can express their interest to the PSC members. These individuals can be nominated by PSC members or etcd maintainers. If representation changes due to job shifts then PSC members are encouraged to grow the team or replace themselves through mentoring new members. ##### Product Security Committee Lazy Consensus Selection Selection of new members will be done by lazy consensus amongst members for adding new people with fallback on majority vote. #### Stepping Down Members may step down at any time and propose a replacement from existing active contributors of etcd. #### Responsibilities - Members must remain active and responsive. - Members taking an extended leave of two weeks or more should coordinate with other members to ensure the role is adequately staffed during the leave. - Members going on leave for 1-3 months may identify a temporary replacement. - Members of a role should remove any other members that have not communicated a leave of absence and either cannot be reached for more than 1 month or are not fulfilling their documented responsibilities for more than 1 month. This may be done through a super-majority vote of members. ## Disclosures ### Private Disclosure Processes The etcd Community asks that all suspected vulnerabilities be privately and responsibly disclosed as explained in the [README](README.md). ### Public Disclosure Processes If anyone knows of a publicly disclosed security vulnerability please IMMEDIATELY email [security@etcd.io](mailto:security@etcd.io) to inform the PSC about the vulnerability so they may start the patch, release, and communication process. If possible the PSC will ask the person making the public report if the issue can be handled via a private disclosure process. If the reporter denies the PSC will move swiftly with the fix and release process. In extreme cases GitHub can be asked to delete the issue but this generally isn't necessary and is unlikely to make a public disclosure less damaging. ## Patch, Release, and Public Communication For each vulnerability, the PSC members will coordinate to create the fix and release, and sending email to the rest of the community. All of the timelines below are suggestions and assume a Private Disclosure. The PSC drives the schedule using their best judgment based on severity, development time, and release work. If the PSC is dealing with a Public Disclosure all timelines become ASAP. If the fix relies on another upstream project's disclosure timeline, that will adjust the process as well. We will work with the upstream project to fit their timeline and best protect etcd users. ### Fix Team Organization These steps should be completed within the first 24 hours of Disclosure. - The PSC will work quickly to identify relevant engineers from the affected projects and packages and CC those engineers into the disclosure thread. These selected developers are the Fix Team. A best guess is to invite all maintainers. ### Fix Development Process These steps should be completed within the 1-7 days of Disclosure. - The PSC and the Fix Team will create a [CVSS](https://www.first.org/cvss/specification-document) using the [CVSS Calculator](https://www.first.org/cvss/calculator/3.0) to determine the effect and severity of the bug. The PSC makes the final call on the calculated risk; it is better to move quickly than make the perfect assessment. - The PSC will request a [CVE](https://cveform.mitre.org/). - The Fix Team will notify the PSC that work on the fix branch is complete once there are LGTMs on all commits from one or more maintainers. If the CVSS score is under ~4.0 ([a low severity score](https://www.first.org/cvss/specification-document#i5)) or the assessed risk is low the Fix Team can decide to slow the release process down in the face of holidays, developer bandwidth, etc. Note: CVSS is convenient but imperfect. Ultimately, the PSC has discretion on classifying the severity of a vulnerability. The severity of the bug and related handling decisions must be discussed on the [security@etcd.io](mailto:security@etcd.io) mailing list. ### Fix Disclosure Process With the Fix Development underway, the PSC needs to come up with an overall communication plan for the wider community. This Disclosure process should begin after the Fix Team has developed a Fix or mitigation so that a realistic timeline can be communicated to users. **Fix Release Day** (Completed within 1-21 days of Disclosure) - The PSC will cherry-pick the patches onto the main branch and all relevant release branches. The Fix Team will `lgtm` and `approve`. - The etcd maintainers will merge these PRs as quickly as possible. - The PSC will ensure all the binaries are built, publicly available, and functional. - The PSC will announce the new releases, the CVE number, severity, and impact, and the location of the binaries to get wide distribution and user action. As much as possible this announcement should be actionable, and include any mitigating steps users can take prior to upgrading to a fixed version. The recommended target time is 4pm UTC on a non-Friday weekday. This means the announcement will be seen morning Pacific, early evening Europe, and late evening Asia. The announcement will be sent via the following channels: - etcd-dev@googlegroups.com - [Kubernetes announcement slack channel](https://kubernetes.slack.com/messages/C9T0QMNG4) - [sig-etcd slack channel](https://kubernetes.slack.com/archives/C3HD8ARJ5) ## Retrospective These steps should be completed 1-3 days after the Release Date. The retrospective process [should be blameless](https://landing.google.com/sre/book/chapters/postmortem-culture.html). - The PSC will send a retrospective of the process to etcd-dev@googlegroups.com including details on everyone involved, the timeline of the process, links to relevant PRs that introduced the issue, if relevant, and any critiques of the response and release process. - The PSC and Fix Team are also encouraged to send their own feedback on the process to etcd-dev@googlegroups.com. Honest critique is the only way we are going to get good at this as a community. ================================================ FILE: server/.gomodguard.yaml ================================================ --- blocked: modules: - go.etcd.io/etcd: reason: "Forbidden dependency" - go.etcd.io/etcd/tests/v3: reason: "Forbidden dependency" - go.etcd.io/etcd/v3: reason: "Forbidden dependency" ================================================ FILE: server/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: server/auth/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package auth provides client role authentication for accessing keys in etcd. package auth ================================================ FILE: server/auth/jwt.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import ( "context" "crypto/ecdsa" "crypto/ed25519" "crypto/rsa" "errors" "time" "github.com/golang-jwt/jwt/v5" "go.uber.org/zap" ) type tokenJWT struct { lg *zap.Logger signMethod jwt.SigningMethod key any ttl time.Duration verifyOnly bool } func (t *tokenJWT) enable() {} func (t *tokenJWT) disable() {} func (t *tokenJWT) invalidateUser(string) {} func (t *tokenJWT) genTokenPrefix() (string, error) { return "", nil } func (t *tokenJWT) info(ctx context.Context, token string, rev uint64) (*AuthInfo, bool) { // rev isn't used in JWT, it is only used in simple token var ( username string revision float64 ) parsed, err := jwt.Parse(token, func(token *jwt.Token) (any, error) { if token.Method.Alg() != t.signMethod.Alg() { return nil, errors.New("invalid signing method") } switch k := t.key.(type) { case *rsa.PrivateKey: return &k.PublicKey, nil case *ecdsa.PrivateKey: return &k.PublicKey, nil case ed25519.PrivateKey: return k.Public(), nil default: return t.key, nil } }) if err != nil { t.lg.Warn( "failed to parse a JWT token", zap.Error(err), ) return nil, false } claims, ok := parsed.Claims.(jwt.MapClaims) if !parsed.Valid || !ok { t.lg.Warn("failed to obtain claims from a JWT token") return nil, false } username, ok = claims["username"].(string) if !ok { t.lg.Warn("failed to obtain user claims from jwt token") return nil, false } revision, ok = claims["revision"].(float64) if !ok { t.lg.Warn("failed to obtain revision claims from jwt token") return nil, false } return &AuthInfo{Username: username, Revision: uint64(revision)}, true } func (t *tokenJWT) assign(ctx context.Context, username string, revision uint64) (string, error) { if t.verifyOnly { return "", ErrVerifyOnly } // Future work: let a jwt token include permission information would be useful for // permission checking in proxy side. tk := jwt.NewWithClaims(t.signMethod, jwt.MapClaims{ "username": username, "revision": revision, "exp": time.Now().Add(t.ttl).Unix(), }) token, err := tk.SignedString(t.key) if err != nil { t.lg.Debug( "failed to sign a JWT token", zap.String("user-name", username), zap.Uint64("revision", revision), zap.Error(err), ) return "", err } if ce := t.lg.Check(zap.DebugLevel, "created/assigned a new JWT token"); ce != nil { tokenFingerprint := redactToken(token) ce.Write(zap.String("user-name", username), zap.Uint64("revision", revision), zap.String("token-fingerprint", tokenFingerprint)) } return token, nil } func newTokenProviderJWT(lg *zap.Logger, optMap map[string]string) (*tokenJWT, error) { if lg == nil { lg = zap.NewNop() } var err error var opts jwtOptions err = opts.ParseWithDefaults(optMap) if err != nil { lg.Error("problem loading JWT options", zap.Error(err)) return nil, ErrInvalidAuthOpts } keys := make([]string, 0, len(optMap)) for k := range optMap { if !knownOptions[k] { keys = append(keys, k) } } if len(keys) > 0 { lg.Warn("unknown JWT options", zap.Strings("keys", keys)) } key, err := opts.Key() if err != nil { return nil, err } t := &tokenJWT{ lg: lg, ttl: opts.TTL, signMethod: opts.SignMethod, key: key, } switch t.signMethod.(type) { case *jwt.SigningMethodECDSA: if _, ok := t.key.(*ecdsa.PublicKey); ok { t.verifyOnly = true } case *jwt.SigningMethodEd25519: if _, ok := t.key.(ed25519.PublicKey); ok { t.verifyOnly = true } case *jwt.SigningMethodRSA, *jwt.SigningMethodRSAPSS: if _, ok := t.key.(*rsa.PublicKey); ok { t.verifyOnly = true } } return t, nil } ================================================ FILE: server/auth/jwt_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import ( "fmt" "maps" "testing" "time" "github.com/golang-jwt/jwt/v5" "github.com/stretchr/testify/require" "go.uber.org/zap" ) const ( jwtRSAPubKey = "../../tests/fixtures/server.crt" jwtRSAPrivKey = "../../tests/fixtures/server.key.insecure" jwtECPubKey = "../../tests/fixtures/server-ecdsa.crt" jwtECPrivKey = "../../tests/fixtures/server-ecdsa.key.insecure" jwtEdPubKey = "../../tests/fixtures/ed25519-public-key.pem" jwtEdPrivKey = "../../tests/fixtures/ed25519-private-key.pem" ) func TestJWTInfo(t *testing.T) { optsMap := map[string]map[string]string{ "RSA-priv": { "priv-key": jwtRSAPrivKey, "sign-method": "RS256", "ttl": "1h", }, "RSA": { "pub-key": jwtRSAPubKey, "priv-key": jwtRSAPrivKey, "sign-method": "RS256", }, "RSAPSS-priv": { "priv-key": jwtRSAPrivKey, "sign-method": "PS256", }, "RSAPSS": { "pub-key": jwtRSAPubKey, "priv-key": jwtRSAPrivKey, "sign-method": "PS256", }, "ECDSA-priv": { "priv-key": jwtECPrivKey, "sign-method": "ES256", }, "ECDSA": { "pub-key": jwtECPubKey, "priv-key": jwtECPrivKey, "sign-method": "ES256", }, "Ed25519-priv": { "priv-key": jwtEdPrivKey, "sign-method": "EdDSA", }, "Ed25519": { "pub-key": jwtEdPubKey, "priv-key": jwtEdPrivKey, "sign-method": "EdDSA", }, "HMAC": { "priv-key": jwtECPrivKey, // any file, raw bytes used as shared secret "sign-method": "HS256", }, } for k, opts := range optsMap { t.Run(k, func(tt *testing.T) { testJWTInfo(tt, opts) }) } } func testJWTInfo(t *testing.T, opts map[string]string) { lg := zap.NewNop() jwt, err := newTokenProviderJWT(lg, opts) if err != nil { t.Fatal(err) } ctx := t.Context() token, aerr := jwt.assign(ctx, "abc", 123) if aerr != nil { t.Fatalf("%#v", aerr) } ai, ok := jwt.info(ctx, token, 123) require.Truef(t, ok, "failed to authenticate with token %s", token) require.Equalf(t, uint64(123), ai.Revision, "expected revision 123, got %d", ai.Revision) ai, ok = jwt.info(ctx, "aaa", 120) if ok || ai != nil { t.Fatalf("expected aaa to fail to authenticate, got %+v", ai) } // test verify-only provider if opts["pub-key"] != "" && opts["priv-key"] != "" { t.Run("verify-only", func(t *testing.T) { newOpts := make(map[string]string, len(opts)) maps.Copy(newOpts, opts) delete(newOpts, "priv-key") verify, err := newTokenProviderJWT(lg, newOpts) if err != nil { t.Fatal(err) } ai, ok := verify.info(ctx, token, 123) require.Truef(t, ok, "failed to authenticate with token %s", token) require.Equalf(t, uint64(123), ai.Revision, "expected revision 123, got %d", ai.Revision) ai, ok = verify.info(ctx, "aaa", 120) if ok || ai != nil { t.Fatalf("expected aaa to fail to authenticate, got %+v", ai) } _, aerr := verify.assign(ctx, "abc", 123) require.ErrorIsf(t, aerr, ErrVerifyOnly, "unexpected error when attempting to sign with public key: %v", aerr) }) } } func TestJWTTokenWithMissingFields(t *testing.T) { testCases := []struct { name string username string // An empty string means not present revision uint64 // 0 means not present expectValid bool }{ { name: "valid token", username: "hello", revision: 100, expectValid: true, }, { name: "no username", username: "", revision: 100, expectValid: false, }, { name: "no revision", username: "hello", revision: 0, expectValid: false, }, } for _, tc := range testCases { tc := tc optsMap := map[string]string{ "priv-key": jwtRSAPrivKey, "sign-method": "RS256", "ttl": "1h", } t.Run(tc.name, func(t *testing.T) { // prepare claims claims := jwt.MapClaims{ "exp": time.Now().Add(time.Hour).Unix(), } if tc.username != "" { claims["username"] = tc.username } if tc.revision != 0 { claims["revision"] = tc.revision } // generate a JWT token with the given claims var opts jwtOptions err := opts.ParseWithDefaults(optsMap) require.NoError(t, err) key, err := opts.Key() require.NoError(t, err) tk := jwt.NewWithClaims(opts.SignMethod, claims) token, err := tk.SignedString(key) require.NoError(t, err) // verify the token jwtProvider, err := newTokenProviderJWT(zap.NewNop(), optsMap) require.NoError(t, err) ai, ok := jwtProvider.info(t.Context(), token, 123) require.Equal(t, tc.expectValid, ok) if ok { require.Equal(t, tc.username, ai.Username) require.Equal(t, tc.revision, ai.Revision) } }) } } func TestJWTBad(t *testing.T) { badCases := map[string]map[string]string{ "no options": {}, "invalid method": { "sign-method": "invalid", }, "rsa no key": { "sign-method": "RS256", }, "invalid ttl": { "sign-method": "RS256", "ttl": "forever", }, "rsa invalid public key": { "sign-method": "RS256", "pub-key": jwtRSAPrivKey, "priv-key": jwtRSAPrivKey, }, "rsa invalid private key": { "sign-method": "RS256", "pub-key": jwtRSAPubKey, "priv-key": jwtRSAPubKey, }, "hmac no key": { "sign-method": "HS256", }, "hmac pub key": { "sign-method": "HS256", "pub-key": jwtRSAPubKey, }, "missing public key file": { "sign-method": "HS256", "pub-key": "missing-file", }, "missing private key file": { "sign-method": "HS256", "priv-key": "missing-file", }, "ecdsa no key": { "sign-method": "ES256", }, "ecdsa invalid public key": { "sign-method": "ES256", "pub-key": jwtECPrivKey, "priv-key": jwtECPrivKey, }, "ecdsa invalid private key": { "sign-method": "ES256", "pub-key": jwtECPubKey, "priv-key": jwtECPubKey, }, } lg := zap.NewNop() for k, v := range badCases { t.Run(k, func(t *testing.T) { _, err := newTokenProviderJWT(lg, v) if err == nil { t.Errorf("expected error for options %v", v) } }) } } // testJWTOpts is useful for passing to NewTokenProvider which requires a string. func testJWTOpts() string { return fmt.Sprintf("%s,pub-key=%s,priv-key=%s,sign-method=RS256", tokenTypeJWT, jwtRSAPubKey, jwtRSAPrivKey) } ================================================ FILE: server/auth/main_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import ( "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func TestMain(m *testing.M) { testutil.MustTestMainWithLeakDetection(m) } ================================================ FILE: server/auth/metrics.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import ( "sync" "github.com/prometheus/client_golang/prometheus" ) var ( currentAuthRevision = prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "auth", Name: "revision", Help: "The current revision of auth store.", }, func() float64 { reportCurrentAuthRevMu.RLock() defer reportCurrentAuthRevMu.RUnlock() return reportCurrentAuthRev() }, ) // overridden by auth store initialization reportCurrentAuthRevMu sync.RWMutex reportCurrentAuthRev = func() float64 { return 0 } ) func init() { prometheus.MustRegister(currentAuthRevision) } ================================================ FILE: server/auth/nop.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import ( "context" ) type tokenNop struct{} func (t *tokenNop) enable() {} func (t *tokenNop) disable() {} func (t *tokenNop) invalidateUser(string) {} func (t *tokenNop) genTokenPrefix() (string, error) { return "", nil } func (t *tokenNop) info(ctx context.Context, token string, rev uint64) (*AuthInfo, bool) { return nil, false } func (t *tokenNop) assign(ctx context.Context, username string, revision uint64) (string, error) { return "", ErrAuthFailed } func newTokenProviderNop() (*tokenNop, error) { return &tokenNop{}, nil } ================================================ FILE: server/auth/options.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import ( "crypto" "crypto/ecdsa" "crypto/ed25519" "crypto/rsa" "fmt" "os" "time" "github.com/golang-jwt/jwt/v5" ) const ( optSignMethod = "sign-method" optPublicKey = "pub-key" optPrivateKey = "priv-key" optTTL = "ttl" ) var knownOptions = map[string]bool{ optSignMethod: true, optPublicKey: true, optPrivateKey: true, optTTL: true, } // DefaultTTL will be used when a 'ttl' is not specified var DefaultTTL = 5 * time.Minute type jwtOptions struct { SignMethod jwt.SigningMethod PublicKey []byte PrivateKey []byte TTL time.Duration } // ParseWithDefaults will load options from the specified map or set defaults where appropriate func (opts *jwtOptions) ParseWithDefaults(optMap map[string]string) error { if opts.TTL == 0 && optMap[optTTL] == "" { opts.TTL = DefaultTTL } return opts.Parse(optMap) } // Parse will load options from the specified map func (opts *jwtOptions) Parse(optMap map[string]string) error { var err error if ttl := optMap[optTTL]; ttl != "" { opts.TTL, err = time.ParseDuration(ttl) if err != nil { return err } } if file := optMap[optPublicKey]; file != "" { opts.PublicKey, err = os.ReadFile(file) if err != nil { return err } } if file := optMap[optPrivateKey]; file != "" { opts.PrivateKey, err = os.ReadFile(file) if err != nil { return err } } // signing method is a required field method := optMap[optSignMethod] opts.SignMethod = jwt.GetSigningMethod(method) if opts.SignMethod == nil { return ErrInvalidAuthMethod } return nil } // Key will parse and return the appropriately typed key for the selected signature method func (opts *jwtOptions) Key() (any, error) { switch opts.SignMethod.(type) { case *jwt.SigningMethodRSA, *jwt.SigningMethodRSAPSS: return opts.rsaKey() case *jwt.SigningMethodECDSA: return opts.ecKey() case *jwt.SigningMethodEd25519: return opts.edKey() case *jwt.SigningMethodHMAC: return opts.hmacKey() default: return nil, fmt.Errorf("unsupported signing method: %T", opts.SignMethod) } } func (opts *jwtOptions) hmacKey() (any, error) { if len(opts.PrivateKey) == 0 { return nil, ErrMissingKey } return opts.PrivateKey, nil } func (opts *jwtOptions) rsaKey() (any, error) { var ( priv *rsa.PrivateKey pub *rsa.PublicKey err error ) if len(opts.PrivateKey) > 0 { priv, err = jwt.ParseRSAPrivateKeyFromPEM(opts.PrivateKey) if err != nil { return nil, err } } if len(opts.PublicKey) > 0 { pub, err = jwt.ParseRSAPublicKeyFromPEM(opts.PublicKey) if err != nil { return nil, err } } if priv == nil { if pub == nil { // Neither key given return nil, ErrMissingKey } // Public key only, can verify tokens return pub, nil } // both keys provided, make sure they match if pub != nil && !pub.Equal(priv.Public()) { return nil, ErrKeyMismatch } return priv, nil } func (opts *jwtOptions) ecKey() (any, error) { var ( priv *ecdsa.PrivateKey pub *ecdsa.PublicKey err error ) if len(opts.PrivateKey) > 0 { priv, err = jwt.ParseECPrivateKeyFromPEM(opts.PrivateKey) if err != nil { return nil, err } } if len(opts.PublicKey) > 0 { pub, err = jwt.ParseECPublicKeyFromPEM(opts.PublicKey) if err != nil { return nil, err } } if priv == nil { if pub == nil { // Neither key given return nil, ErrMissingKey } // Public key only, can verify tokens return pub, nil } // both keys provided, make sure they match if pub != nil && !pub.Equal(priv.Public()) { return nil, ErrKeyMismatch } return priv, nil } func (opts *jwtOptions) edKey() (any, error) { var ( priv ed25519.PrivateKey pub ed25519.PublicKey err error ) if len(opts.PrivateKey) > 0 { var privKey crypto.PrivateKey privKey, err = jwt.ParseEdPrivateKeyFromPEM(opts.PrivateKey) if err != nil { return nil, err } priv = privKey.(ed25519.PrivateKey) } if len(opts.PublicKey) > 0 { var pubKey crypto.PublicKey pubKey, err = jwt.ParseEdPublicKeyFromPEM(opts.PublicKey) if err != nil { return nil, err } pub = pubKey.(ed25519.PublicKey) } if priv == nil { if pub == nil { // Neither key given return nil, ErrMissingKey } // Public key only, can verify tokens return pub, nil } // both keys provided, make sure they match if pub != nil && !pub.Equal(priv.Public()) { return nil, ErrKeyMismatch } return priv, nil } ================================================ FILE: server/auth/range_perm_cache.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import ( "go.uber.org/zap" "go.etcd.io/etcd/api/v3/authpb" "go.etcd.io/etcd/pkg/v3/adt" ) func getMergedPerms(tx UnsafeAuthReader, userName string) *unifiedRangePermissions { user := tx.UnsafeGetUser(userName) if user == nil { return nil } readPerms := adt.NewIntervalTree() writePerms := adt.NewIntervalTree() for _, roleName := range user.Roles { role := tx.UnsafeGetRole(roleName) if role == nil { continue } for _, perm := range role.KeyPermission { var ivl adt.Interval var rangeEnd []byte if len(perm.RangeEnd) != 1 || perm.RangeEnd[0] != 0 { rangeEnd = perm.RangeEnd } if len(perm.RangeEnd) != 0 { ivl = adt.NewBytesAffineInterval(perm.Key, rangeEnd) } else { ivl = adt.NewBytesAffinePoint(perm.Key) } switch perm.PermType { case authpb.Permission_READWRITE: readPerms.Insert(ivl, struct{}{}) writePerms.Insert(ivl, struct{}{}) case authpb.Permission_READ: readPerms.Insert(ivl, struct{}{}) case authpb.Permission_WRITE: writePerms.Insert(ivl, struct{}{}) } } } return &unifiedRangePermissions{ readPerms: readPerms, writePerms: writePerms, } } func checkKeyInterval( lg *zap.Logger, cachedPerms *unifiedRangePermissions, key, rangeEnd []byte, permtyp authpb.Permission_Type, ) bool { if isOpenEnded(rangeEnd) { rangeEnd = nil // nil rangeEnd will be converetd to []byte{}, the largest element of BytesAffineComparable, // in NewBytesAffineInterval(). } ivl := adt.NewBytesAffineInterval(key, rangeEnd) switch permtyp { case authpb.Permission_READ: return cachedPerms.readPerms.Contains(ivl) case authpb.Permission_WRITE: return cachedPerms.writePerms.Contains(ivl) default: lg.Panic("unknown auth type", zap.String("auth-type", permtyp.String())) } return false } func checkKeyPoint(lg *zap.Logger, cachedPerms *unifiedRangePermissions, key []byte, permtyp authpb.Permission_Type) bool { pt := adt.NewBytesAffinePoint(key) switch permtyp { case authpb.Permission_READ: return cachedPerms.readPerms.Intersects(pt) case authpb.Permission_WRITE: return cachedPerms.writePerms.Intersects(pt) default: lg.Panic("unknown auth type", zap.String("auth-type", permtyp.String())) } return false } func (as *authStore) isRangeOpPermitted(userName string, key, rangeEnd []byte, permtyp authpb.Permission_Type) bool { // assumption: tx is Lock()ed as.rangePermCacheMu.RLock() defer as.rangePermCacheMu.RUnlock() rangePerm, ok := as.rangePermCache[userName] if !ok { as.lg.Error( "user doesn't exist", zap.String("user-name", userName), ) return false } if len(rangeEnd) == 0 { return checkKeyPoint(as.lg, rangePerm, key, permtyp) } return checkKeyInterval(as.lg, rangePerm, key, rangeEnd, permtyp) } func (as *authStore) refreshRangePermCache(tx UnsafeAuthReader) { // Note that every authentication configuration update calls this method and it invalidates the entire // rangePermCache and reconstruct it based on information of users and roles stored in the backend. // This can be a costly operation. as.rangePermCacheMu.Lock() defer as.rangePermCacheMu.Unlock() as.lg.Debug("Refreshing rangePermCache") as.rangePermCache = make(map[string]*unifiedRangePermissions) users := tx.UnsafeGetAllUsers() for _, user := range users { userName := string(user.Name) perms := getMergedPerms(tx, userName) if perms == nil { as.lg.Error( "failed to create a merged permission", zap.String("user-name", userName), ) continue } as.rangePermCache[userName] = perms } } type unifiedRangePermissions struct { readPerms adt.IntervalTree writePerms adt.IntervalTree } // Constraints related to key range // Assumptions: // a1. key must be non-nil // a2. []byte{} (in the case of string, "") is not a valid key of etcd // For representing an open-ended range, BytesAffineComparable uses []byte{} as the largest element. // a3. []byte{0x00} is the minimum valid etcd key // // Based on the above assumptions, key and rangeEnd must follow below rules: // b1. for representing a single key point, rangeEnd should be nil or zero length byte array (in the case of string, "") // Rule a2 guarantees that (X, []byte{}) for any X is not a valid range. So such ranges can be used for representing // a single key permission. // // b2. key range with upper limit, like (X, Y), larger or equal to X and smaller than Y // // b3. key range with open-ended, like (X, ), is represented like (X, []byte{0x00}) // Because of rule a3, if we have (X, []byte{0x00}), such a range represents an empty range and makes no sense to have // such a permission. So we use []byte{0x00} for representing an open-ended permission. // Note that rangeEnd with []byte{0x00} will be converted into []byte{} before inserted into the interval tree // (rule a2 ensures that this is the largest element). // Special range like key = []byte{0x00} and rangeEnd = []byte{0x00} is treated as a range which matches with all keys. // // Treating a range whose rangeEnd with []byte{0x00} as an open-ended comes from the rules of Range() and Watch() API. func isOpenEnded(rangeEnd []byte) bool { // check rule b3 return len(rangeEnd) == 1 && rangeEnd[0] == 0 } func isValidPermissionRange(key, rangeEnd []byte) bool { if len(key) == 0 { return false } if len(rangeEnd) == 0 { // ensure rule b1 return true } begin := adt.BytesAffineComparable(key) end := adt.BytesAffineComparable(rangeEnd) if begin.Compare(end) == -1 { // rule b2 return true } return isOpenEnded(rangeEnd) } ================================================ FILE: server/auth/range_perm_cache_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import ( "testing" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/authpb" "go.etcd.io/etcd/pkg/v3/adt" ) func TestRangePermission(t *testing.T) { tests := []struct { perms []adt.Interval begin []byte end []byte want bool }{ { []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("c")), adt.NewBytesAffineInterval([]byte("x"), []byte("z"))}, []byte("a"), []byte("z"), false, }, { []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("f")), adt.NewBytesAffineInterval([]byte("c"), []byte("d")), adt.NewBytesAffineInterval([]byte("f"), []byte("z"))}, []byte("a"), []byte("z"), true, }, { []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte("c"), []byte("f"))}, []byte("a"), []byte("f"), true, }, { []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte("c"), []byte("f"))}, []byte("a"), []byte{}, false, }, { []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte{})}, []byte("a"), []byte{}, true, }, { []adt.Interval{adt.NewBytesAffineInterval([]byte{0x00}, []byte{})}, []byte("a"), []byte{}, true, }, { []adt.Interval{adt.NewBytesAffineInterval([]byte{0x00}, []byte{})}, []byte{0x00}, []byte{}, true, }, } for i, tt := range tests { readPerms := adt.NewIntervalTree() for _, p := range tt.perms { readPerms.Insert(p, struct{}{}) } result := checkKeyInterval(zaptest.NewLogger(t), &unifiedRangePermissions{readPerms: readPerms}, tt.begin, tt.end, authpb.Permission_READ) if result != tt.want { t.Errorf("#%d: result=%t, want=%t", i, result, tt.want) } } } func TestKeyPermission(t *testing.T) { tests := []struct { perms []adt.Interval key []byte want bool }{ { []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("c")), adt.NewBytesAffineInterval([]byte("x"), []byte("z"))}, []byte("f"), false, }, { []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("f")), adt.NewBytesAffineInterval([]byte("c"), []byte("d")), adt.NewBytesAffineInterval([]byte("f"), []byte("z"))}, []byte("b"), true, }, { []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte("c"), []byte("f"))}, []byte("d"), true, }, { []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte("c"), []byte("f"))}, []byte("f"), false, }, { []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte("c"), []byte{})}, []byte("f"), true, }, { []adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte{0x00}, []byte{})}, []byte("f"), true, }, } for i, tt := range tests { readPerms := adt.NewIntervalTree() for _, p := range tt.perms { readPerms.Insert(p, struct{}{}) } result := checkKeyPoint(zaptest.NewLogger(t), &unifiedRangePermissions{readPerms: readPerms}, tt.key, authpb.Permission_READ) if result != tt.want { t.Errorf("#%d: result=%t, want=%t", i, result, tt.want) } } } func TestRangeCheck(t *testing.T) { tests := []struct { name string key []byte rangeEnd []byte want bool }{ { name: "valid single key", key: []byte("a"), rangeEnd: []byte(""), want: true, }, { name: "valid single key", key: []byte("a"), rangeEnd: nil, want: true, }, { name: "valid key range, key < rangeEnd", key: []byte("a"), rangeEnd: []byte("b"), want: true, }, { name: "invalid empty key range, key == rangeEnd", key: []byte("a"), rangeEnd: []byte("a"), want: false, }, { name: "invalid empty key range, key > rangeEnd", key: []byte("b"), rangeEnd: []byte("a"), want: false, }, { name: "invalid key, key must not be \"\"", key: []byte(""), rangeEnd: []byte("a"), want: false, }, { name: "invalid key range, key must not be \"\"", key: []byte(""), rangeEnd: []byte(""), want: false, }, { name: "invalid key range, key must not be \"\"", key: []byte(""), rangeEnd: []byte("\x00"), want: false, }, { name: "valid single key (not useful in practice)", key: []byte("\x00"), rangeEnd: []byte(""), want: true, }, { name: "valid key range, larger or equals to \"a\"", key: []byte("a"), rangeEnd: []byte("\x00"), want: true, }, { name: "valid key range, which includes all keys", key: []byte("\x00"), rangeEnd: []byte("\x00"), want: true, }, } for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := isValidPermissionRange(tt.key, tt.rangeEnd) if result != tt.want { t.Errorf("#%d: result=%t, want=%t", i, result, tt.want) } }) } } ================================================ FILE: server/auth/simple_token.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth // CAUTION: This random number based token mechanism is only for testing purpose. // JWT based mechanism will be added in the near future. import ( "context" "crypto/rand" "errors" "fmt" "math/big" "strconv" "strings" "sync" "time" "go.uber.org/zap" ) const ( letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" defaultSimpleTokenLength = 16 ) // var for testing purposes // TODO: Remove this mutable global state - as it's race-prone. var ( simpleTokenTTLDefault = 300 * time.Second simpleTokenTTLResolution = 1 * time.Second ) type simpleTokenTTLKeeper struct { tokens map[string]time.Time donec chan struct{} stopc chan struct{} deleteTokenFunc func(string) mu *sync.Mutex simpleTokenTTL time.Duration } func (tm *simpleTokenTTLKeeper) stop() { select { case tm.stopc <- struct{}{}: case <-tm.donec: } <-tm.donec } func (tm *simpleTokenTTLKeeper) addSimpleToken(token string) { tm.tokens[token] = time.Now().Add(tm.simpleTokenTTL) } func (tm *simpleTokenTTLKeeper) resetSimpleToken(token string) { if _, ok := tm.tokens[token]; ok { tm.tokens[token] = time.Now().Add(tm.simpleTokenTTL) } } func (tm *simpleTokenTTLKeeper) deleteSimpleToken(token string) { delete(tm.tokens, token) } func (tm *simpleTokenTTLKeeper) run() { tokenTicker := time.NewTicker(simpleTokenTTLResolution) defer func() { tokenTicker.Stop() close(tm.donec) }() for { select { case <-tokenTicker.C: nowtime := time.Now() tm.mu.Lock() for t, tokenendtime := range tm.tokens { if nowtime.After(tokenendtime) { tm.deleteTokenFunc(t) delete(tm.tokens, t) } } tm.mu.Unlock() case <-tm.stopc: return } } } type tokenSimple struct { lg *zap.Logger indexWaiter func(uint64) <-chan struct{} simpleTokenKeeper *simpleTokenTTLKeeper simpleTokensMu sync.Mutex simpleTokens map[string]string // token -> username simpleTokenTTL time.Duration } func (t *tokenSimple) genTokenPrefix() (string, error) { ret := make([]byte, defaultSimpleTokenLength) for i := 0; i < defaultSimpleTokenLength; i++ { bInt, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters)))) if err != nil { return "", err } ret[i] = letters[bInt.Int64()] } return string(ret), nil } func (t *tokenSimple) assignSimpleTokenToUser(username, token string) { t.simpleTokensMu.Lock() defer t.simpleTokensMu.Unlock() if t.simpleTokenKeeper == nil { return } _, ok := t.simpleTokens[token] if ok { tokenFingerprint := redactToken(token) t.lg.Panic( "failed to assign already-used simple token to a user", zap.String("user-name", username), zap.String("token-fingerprint", tokenFingerprint), ) } t.simpleTokens[token] = username t.simpleTokenKeeper.addSimpleToken(token) } func (t *tokenSimple) invalidateUser(username string) { if t.simpleTokenKeeper == nil { return } t.simpleTokensMu.Lock() for token, name := range t.simpleTokens { if name == username { delete(t.simpleTokens, token) t.simpleTokenKeeper.deleteSimpleToken(token) } } t.simpleTokensMu.Unlock() } func (t *tokenSimple) enable() { t.simpleTokensMu.Lock() defer t.simpleTokensMu.Unlock() if t.simpleTokenKeeper != nil { // already enabled return } if t.simpleTokenTTL <= 0 { t.simpleTokenTTL = simpleTokenTTLDefault } delf := func(tk string) { if username, ok := t.simpleTokens[tk]; ok { t.lg.Debug( "deleted a simple token", zap.String("user-name", username), zap.String("token", tk), ) delete(t.simpleTokens, tk) } } t.simpleTokenKeeper = &simpleTokenTTLKeeper{ tokens: make(map[string]time.Time), donec: make(chan struct{}), stopc: make(chan struct{}), deleteTokenFunc: delf, mu: &t.simpleTokensMu, simpleTokenTTL: t.simpleTokenTTL, } go t.simpleTokenKeeper.run() } func (t *tokenSimple) disable() { t.simpleTokensMu.Lock() tk := t.simpleTokenKeeper t.simpleTokenKeeper = nil t.simpleTokens = make(map[string]string) // invalidate all tokens t.simpleTokensMu.Unlock() if tk != nil { tk.stop() } } func (t *tokenSimple) info(ctx context.Context, token string, revision uint64) (*AuthInfo, bool) { if !t.isValidSimpleToken(ctx, token) { return nil, false } t.simpleTokensMu.Lock() username, ok := t.simpleTokens[token] if ok && t.simpleTokenKeeper != nil { t.simpleTokenKeeper.resetSimpleToken(token) } t.simpleTokensMu.Unlock() return &AuthInfo{Username: username, Revision: revision}, ok } func (t *tokenSimple) assign(ctx context.Context, username string, rev uint64) (string, error) { // rev isn't used in simple token, it is only used in JWT var index uint64 var ok bool if index, ok = ctx.Value(AuthenticateParamIndex{}).(uint64); !ok { return "", errors.New("failed to assign") } simpleTokenPrefix := ctx.Value(AuthenticateParamSimpleTokenPrefix{}).(string) token := fmt.Sprintf("%s.%d", simpleTokenPrefix, index) t.assignSimpleTokenToUser(username, token) return token, nil } func (t *tokenSimple) isValidSimpleToken(ctx context.Context, token string) bool { splitted := strings.Split(token, ".") if len(splitted) != 2 { return false } index, err := strconv.ParseUint(splitted[1], 10, 0) if err != nil { return false } select { case <-t.indexWaiter(index): return true case <-ctx.Done(): } return false } func newTokenProviderSimple(lg *zap.Logger, indexWaiter func(uint64) <-chan struct{}, TokenTTL time.Duration) *tokenSimple { if lg == nil { lg = zap.NewNop() } return &tokenSimple{ lg: lg, simpleTokens: make(map[string]string), indexWaiter: indexWaiter, simpleTokenTTL: TokenTTL, } } ================================================ FILE: server/auth/simple_token_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import ( "context" "testing" "go.uber.org/zap/zaptest" ) // TestSimpleTokenDisabled ensures that TokenProviderSimple behaves correctly when // disabled. func TestSimpleTokenDisabled(t *testing.T) { initialState := newTokenProviderSimple(zaptest.NewLogger(t), dummyIndexWaiter, simpleTokenTTLDefault) explicitlyDisabled := newTokenProviderSimple(zaptest.NewLogger(t), dummyIndexWaiter, simpleTokenTTLDefault) explicitlyDisabled.enable() explicitlyDisabled.disable() for _, tp := range []*tokenSimple{initialState, explicitlyDisabled} { ctx := context.WithValue(context.WithValue(t.Context(), AuthenticateParamIndex{}, uint64(1)), AuthenticateParamSimpleTokenPrefix{}, "dummy") token, err := tp.assign(ctx, "user1", 0) if err != nil { t.Fatal(err) } authInfo, ok := tp.info(ctx, token, 0) if ok { t.Errorf("expected (true, \"user1\") got (%t, %s)", ok, authInfo.Username) } tp.invalidateUser("user1") // should be no-op } } // TestSimpleTokenAssign ensures that TokenProviderSimple can correctly assign a // token, look it up with info, and invalidate it by user. func TestSimpleTokenAssign(t *testing.T) { tp := newTokenProviderSimple(zaptest.NewLogger(t), dummyIndexWaiter, simpleTokenTTLDefault) tp.enable() defer tp.disable() ctx := context.WithValue(context.WithValue(t.Context(), AuthenticateParamIndex{}, uint64(1)), AuthenticateParamSimpleTokenPrefix{}, "dummy") token, err := tp.assign(ctx, "user1", 0) if err != nil { t.Fatal(err) } authInfo, ok := tp.info(ctx, token, 0) if !ok || authInfo.Username != "user1" { t.Errorf("expected (true, \"token2\") got (%t, %s)", ok, authInfo.Username) } tp.invalidateUser("user1") _, ok = tp.info(t.Context(), token, 0) if ok { t.Errorf("expected ok == false after user is invalidated") } } ================================================ FILE: server/auth/store.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import ( "bytes" "context" "crypto/sha256" "encoding/base64" "encoding/hex" "errors" "sort" "strings" "sync" "sync/atomic" "time" "go.uber.org/zap" "golang.org/x/crypto/bcrypt" "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" "go.etcd.io/etcd/api/v3/authpb" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" ) var _ AuthStore = (*authStore)(nil) var ( rootPerm = authpb.Permission{PermType: authpb.Permission_READWRITE, Key: []byte{}, RangeEnd: []byte{0}} ErrRootUserNotExist = errors.New("auth: root user does not exist") ErrRootRoleNotExist = errors.New("auth: root user does not have root role") ErrUserAlreadyExist = errors.New("auth: user already exists") ErrUserEmpty = errors.New("auth: user name is empty") ErrUserNotFound = errors.New("auth: user not found") ErrRoleAlreadyExist = errors.New("auth: role already exists") ErrRoleNotFound = errors.New("auth: role not found") ErrRoleEmpty = errors.New("auth: role name is empty") ErrPermissionNotGiven = errors.New("auth: permission not given") ErrAuthFailed = errors.New("auth: authentication failed, invalid user ID or password") ErrNoPasswordUser = errors.New("auth: authentication failed, password was given for no password user") ErrPermissionDenied = errors.New("auth: permission denied") ErrRoleNotGranted = errors.New("auth: role is not granted to the user") ErrPermissionNotGranted = errors.New("auth: permission is not granted to the role") ErrAuthNotEnabled = errors.New("auth: authentication is not enabled") ErrAuthOldRevision = errors.New("auth: revision in header is old") ErrInvalidAuthToken = errors.New("auth: invalid auth token") ErrInvalidAuthOpts = errors.New("auth: invalid auth options") ErrInvalidAuthMgmt = errors.New("auth: invalid auth management") ErrInvalidAuthMethod = errors.New("auth: invalid auth signature method") ErrMissingKey = errors.New("auth: missing key data") ErrKeyMismatch = errors.New("auth: public and private keys don't match") ErrVerifyOnly = errors.New("auth: token signing attempted with verify-only key") ) const ( rootUser = "root" rootRole = "root" tokenTypeSimple = "simple" tokenTypeJWT = "jwt" ) type AuthInfo struct { Username string Revision uint64 } // AuthenticateParamIndex is used for a key of context in the parameters of Authenticate() type AuthenticateParamIndex struct{} // AuthenticateParamSimpleTokenPrefix is used for a key of context in the parameters of Authenticate() type AuthenticateParamSimpleTokenPrefix struct{} // AuthStore defines auth storage interface. type AuthStore interface { // AuthEnable turns on the authentication feature AuthEnable() error // AuthDisable turns off the authentication feature AuthDisable() // IsAuthEnabled returns true if the authentication feature is enabled. IsAuthEnabled() bool // Authenticate does authentication based on given user name and password Authenticate(ctx context.Context, username, password string) (*pb.AuthenticateResponse, error) // Recover recovers the state of auth store from the given backend Recover(be AuthBackend) // UserAdd adds a new user UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, error) // UserDelete deletes a user UserDelete(r *pb.AuthUserDeleteRequest) (*pb.AuthUserDeleteResponse, error) // UserChangePassword changes a password of a user UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*pb.AuthUserChangePasswordResponse, error) // UserGrantRole grants a role to the user UserGrantRole(r *pb.AuthUserGrantRoleRequest) (*pb.AuthUserGrantRoleResponse, error) // UserGet gets the detailed information of a users UserGet(r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) // UserRevokeRole revokes a role of a user UserRevokeRole(r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUserRevokeRoleResponse, error) // RoleAdd adds a new role RoleAdd(r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, error) // RoleGrantPermission grants a permission to a role RoleGrantPermission(r *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) // RoleGet gets the detailed information of a role RoleGet(r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) // RoleRevokePermission gets the detailed information of a role RoleRevokePermission(r *pb.AuthRoleRevokePermissionRequest) (*pb.AuthRoleRevokePermissionResponse, error) // RoleDelete gets the detailed information of a role RoleDelete(r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDeleteResponse, error) // UserList gets a list of all users UserList(r *pb.AuthUserListRequest) (*pb.AuthUserListResponse, error) // RoleList gets a list of all roles RoleList(r *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error) // IsPutPermitted checks put permission of the user IsPutPermitted(authInfo *AuthInfo, key []byte) error // IsRangePermitted checks range permission of the user IsRangePermitted(authInfo *AuthInfo, key, rangeEnd []byte) error // IsDeleteRangePermitted checks delete-range permission of the user IsDeleteRangePermitted(authInfo *AuthInfo, key, rangeEnd []byte) error // IsAdminPermitted checks admin permission of the user IsAdminPermitted(authInfo *AuthInfo) error // GenTokenPrefix produces a random string in a case of simple token // in a case of JWT, it produces an empty string GenTokenPrefix() (string, error) // Revision gets current revision of authStore Revision() uint64 // CheckPassword checks a given pair of username and password is correct CheckPassword(username, password string) (uint64, error) // Close does cleanup of AuthStore Close() error // AuthInfoFromCtx gets AuthInfo from gRPC's context AuthInfoFromCtx(ctx context.Context) (*AuthInfo, error) // AuthInfoFromTLS gets AuthInfo from TLS info of gRPC's context AuthInfoFromTLS(ctx context.Context) *AuthInfo // WithRoot generates and installs a token that can be used as a root credential WithRoot(ctx context.Context) context.Context // HasRole checks that user has role HasRole(user, role string) bool // BcryptCost gets strength of hashing bcrypted auth password BcryptCost() int } type TokenProvider interface { info(ctx context.Context, token string, revision uint64) (*AuthInfo, bool) assign(ctx context.Context, username string, revision uint64) (string, error) enable() disable() invalidateUser(string) genTokenPrefix() (string, error) } type AuthBackend interface { CreateAuthBuckets() ForceCommit() ReadTx() AuthReadTx BatchTx() AuthBatchTx GetUser(string) *authpb.User GetAllUsers() []*authpb.User GetRole(string) *authpb.Role GetAllRoles() []*authpb.Role } type AuthReadTx interface { RLock() RUnlock() UnsafeAuthReader } type UnsafeAuthReader interface { UnsafeReadAuthEnabled() bool UnsafeReadAuthRevision() uint64 UnsafeGetUser(string) *authpb.User UnsafeGetRole(string) *authpb.Role UnsafeGetAllUsers() []*authpb.User UnsafeGetAllRoles() []*authpb.Role } type AuthBatchTx interface { Lock() Unlock() UnsafeAuthReadWriter } type UnsafeAuthReadWriter interface { UnsafeAuthReader UnsafeAuthWriter } type UnsafeAuthWriter interface { UnsafeSaveAuthEnabled(enabled bool) UnsafeSaveAuthRevision(rev uint64) UnsafePutUser(*authpb.User) UnsafeDeleteUser(string) UnsafePutRole(*authpb.Role) UnsafeDeleteRole(string) } type authStore struct { // atomic operations; need 64-bit align, or 32-bit tests will crash revision uint64 lg *zap.Logger be AuthBackend enabled bool enabledMu sync.RWMutex // rangePermCache needs to be protected by rangePermCacheMu // rangePermCacheMu needs to be write locked only in initialization phase or configuration changes // Hot paths like Range(), needs to acquire read lock for improving performance // // Note that BatchTx and ReadTx cannot be a mutex for rangePermCache because they are independent resources // see also: https://github.com/etcd-io/etcd/pull/13920#discussion_r849114855 rangePermCache map[string]*unifiedRangePermissions // username -> unifiedRangePermissions rangePermCacheMu sync.RWMutex tokenProvider TokenProvider bcryptCost int // the algorithm cost / strength for hashing auth passwords } func (as *authStore) AuthEnable() error { as.enabledMu.Lock() defer as.enabledMu.Unlock() if as.enabled { as.lg.Info("authentication is already enabled; ignored auth enable request") return nil } tx := as.be.BatchTx() tx.Lock() defer func() { tx.Unlock() as.be.ForceCommit() }() u := tx.UnsafeGetUser(rootUser) if u == nil { return ErrRootUserNotExist } if !hasRootRole(u) { return ErrRootRoleNotExist } tx.UnsafeSaveAuthEnabled(true) as.enabled = true as.tokenProvider.enable() as.refreshRangePermCache(tx) as.setRevision(tx.UnsafeReadAuthRevision()) as.lg.Info("enabled authentication") return nil } func (as *authStore) AuthDisable() { as.enabledMu.Lock() defer as.enabledMu.Unlock() if !as.enabled { return } b := as.be tx := b.BatchTx() tx.Lock() tx.UnsafeSaveAuthEnabled(false) as.commitRevision(tx) tx.Unlock() b.ForceCommit() as.enabled = false as.tokenProvider.disable() as.lg.Info("disabled authentication") } func (as *authStore) Close() error { as.enabledMu.Lock() defer as.enabledMu.Unlock() if !as.enabled { return nil } as.tokenProvider.disable() return nil } func (as *authStore) Authenticate(ctx context.Context, username, password string) (*pb.AuthenticateResponse, error) { if !as.IsAuthEnabled() { return nil, ErrAuthNotEnabled } user := as.be.GetUser(username) if user == nil { return nil, ErrAuthFailed } if user.Options != nil && user.Options.NoPassword { return nil, ErrAuthFailed } // Password checking is already performed in the API layer, so we don't need to check for now. // Staleness of password can be detected with OCC in the API layer, too. token, err := as.tokenProvider.assign(ctx, username, as.Revision()) if err != nil { return nil, err } if ce := as.lg.Check(zap.DebugLevel, "authenticated a user"); ce != nil { tokenFingerprint := redactToken(token) ce.Write(zap.String("user-name", username), zap.String("token-fingerprint", tokenFingerprint)) } return &pb.AuthenticateResponse{Token: token}, nil } func (as *authStore) CheckPassword(username, password string) (uint64, error) { if !as.IsAuthEnabled() { return 0, ErrAuthNotEnabled } var user *authpb.User // CompareHashAndPassword is very expensive, so we use closures // to avoid putting it in the critical section of the tx lock. revision, err := func() (uint64, error) { tx := as.be.ReadTx() tx.RLock() defer tx.RUnlock() user = tx.UnsafeGetUser(username) if user == nil { return 0, ErrAuthFailed } if user.Options != nil && user.Options.NoPassword { return 0, ErrNoPasswordUser } return tx.UnsafeReadAuthRevision(), nil }() if err != nil { return 0, err } if bcrypt.CompareHashAndPassword(user.Password, []byte(password)) != nil { as.lg.Info("invalid password", zap.String("user-name", username)) return 0, ErrAuthFailed } return revision, nil } func (as *authStore) Recover(be AuthBackend) { as.be = be tx := be.ReadTx() tx.RLock() enabled := tx.UnsafeReadAuthEnabled() as.setRevision(tx.UnsafeReadAuthRevision()) as.refreshRangePermCache(tx) tx.RUnlock() as.enabledMu.Lock() as.enabled = enabled if enabled { as.tokenProvider.enable() } as.enabledMu.Unlock() } func (as *authStore) selectPassword(password string, hashedPassword string) ([]byte, error) { if password != "" && hashedPassword == "" { // This path is for processing log entries created by etcd whose version is older than 3.5 return bcrypt.GenerateFromPassword([]byte(password), as.bcryptCost) } return base64.StdEncoding.DecodeString(hashedPassword) } func (as *authStore) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, error) { if len(r.Name) == 0 { return nil, ErrUserEmpty } tx := as.be.BatchTx() tx.Lock() defer tx.Unlock() user := tx.UnsafeGetUser(r.Name) if user != nil { return nil, ErrUserAlreadyExist } options := r.Options if options == nil { options = &authpb.UserAddOptions{ NoPassword: false, } } var password []byte var err error if !options.NoPassword { password, err = as.selectPassword(r.Password, r.HashedPassword) if err != nil { return nil, ErrNoPasswordUser } } newUser := &authpb.User{ Name: []byte(r.Name), Password: password, Options: options, } tx.UnsafePutUser(newUser) as.commitRevision(tx) as.refreshRangePermCache(tx) as.lg.Info("added a user", zap.String("user-name", r.Name)) return &pb.AuthUserAddResponse{}, nil } func (as *authStore) UserDelete(r *pb.AuthUserDeleteRequest) (*pb.AuthUserDeleteResponse, error) { if as.enabled && r.Name == rootUser { as.lg.Error("cannot delete 'root' user", zap.String("user-name", r.Name)) return nil, ErrInvalidAuthMgmt } tx := as.be.BatchTx() tx.Lock() defer tx.Unlock() user := tx.UnsafeGetUser(r.Name) if user == nil { return nil, ErrUserNotFound } tx.UnsafeDeleteUser(r.Name) as.commitRevision(tx) as.refreshRangePermCache(tx) as.tokenProvider.invalidateUser(r.Name) as.lg.Info( "deleted a user", zap.String("user-name", r.Name), zap.Strings("user-roles", user.Roles), ) return &pb.AuthUserDeleteResponse{}, nil } func (as *authStore) UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*pb.AuthUserChangePasswordResponse, error) { tx := as.be.BatchTx() tx.Lock() defer tx.Unlock() user := tx.UnsafeGetUser(r.Name) if user == nil { return nil, ErrUserNotFound } var password []byte var err error // Backward compatible with old versions of etcd, user options is nil if user.Options == nil || !user.Options.NoPassword { password, err = as.selectPassword(r.Password, r.HashedPassword) if err != nil { return nil, ErrNoPasswordUser } } updatedUser := &authpb.User{ Name: []byte(r.Name), Roles: user.Roles, Password: password, Options: user.Options, } tx.UnsafePutUser(updatedUser) as.commitRevision(tx) as.refreshRangePermCache(tx) as.tokenProvider.invalidateUser(r.Name) as.lg.Info( "changed a password of a user", zap.String("user-name", r.Name), zap.Strings("user-roles", user.Roles), ) return &pb.AuthUserChangePasswordResponse{}, nil } func (as *authStore) UserGrantRole(r *pb.AuthUserGrantRoleRequest) (*pb.AuthUserGrantRoleResponse, error) { tx := as.be.BatchTx() tx.Lock() defer tx.Unlock() user := tx.UnsafeGetUser(r.User) if user == nil { return nil, ErrUserNotFound } if r.Role != rootRole { role := tx.UnsafeGetRole(r.Role) if role == nil { return nil, ErrRoleNotFound } } idx := sort.SearchStrings(user.Roles, r.Role) if idx < len(user.Roles) && user.Roles[idx] == r.Role { as.lg.Warn( "ignored grant role request to a user", zap.String("user-name", r.User), zap.Strings("user-roles", user.Roles), zap.String("duplicate-role-name", r.Role), ) return &pb.AuthUserGrantRoleResponse{}, nil } user.Roles = append(user.Roles, r.Role) sort.Strings(user.Roles) tx.UnsafePutUser(user) as.commitRevision(tx) as.refreshRangePermCache(tx) as.lg.Info( "granted a role to a user", zap.String("user-name", r.User), zap.Strings("user-roles", user.Roles), zap.String("added-role-name", r.Role), ) return &pb.AuthUserGrantRoleResponse{}, nil } func (as *authStore) UserGet(r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) { user := as.be.GetUser(r.Name) if user == nil { return nil, ErrUserNotFound } var resp pb.AuthUserGetResponse resp.Roles = append(resp.Roles, user.Roles...) return &resp, nil } func (as *authStore) UserList(r *pb.AuthUserListRequest) (*pb.AuthUserListResponse, error) { users := as.be.GetAllUsers() resp := &pb.AuthUserListResponse{Users: make([]string, len(users))} for i := range users { resp.Users[i] = string(users[i].Name) } return resp, nil } func (as *authStore) UserRevokeRole(r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUserRevokeRoleResponse, error) { if as.enabled && r.Name == rootUser && r.Role == rootRole { as.lg.Error( "'root' user cannot revoke 'root' role", zap.String("user-name", r.Name), zap.String("role-name", r.Role), ) return nil, ErrInvalidAuthMgmt } tx := as.be.BatchTx() tx.Lock() defer tx.Unlock() user := tx.UnsafeGetUser(r.Name) if user == nil { return nil, ErrUserNotFound } updatedUser := &authpb.User{ Name: user.Name, Password: user.Password, Options: user.Options, } for _, role := range user.Roles { if role != r.Role { updatedUser.Roles = append(updatedUser.Roles, role) } } if len(updatedUser.Roles) == len(user.Roles) { return nil, ErrRoleNotGranted } tx.UnsafePutUser(updatedUser) as.commitRevision(tx) as.refreshRangePermCache(tx) as.lg.Info( "revoked a role from a user", zap.String("user-name", r.Name), zap.Strings("old-user-roles", user.Roles), zap.Strings("new-user-roles", updatedUser.Roles), zap.String("revoked-role-name", r.Role), ) return &pb.AuthUserRevokeRoleResponse{}, nil } func (as *authStore) RoleGet(r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) { var resp pb.AuthRoleGetResponse role := as.be.GetRole(r.Role) if role == nil { return nil, ErrRoleNotFound } if rootRole == string(role.Name) { resp.Perm = append(resp.Perm, &rootPerm) } else { resp.Perm = append(resp.Perm, role.KeyPermission...) } return &resp, nil } func (as *authStore) RoleList(r *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error) { roles := as.be.GetAllRoles() resp := &pb.AuthRoleListResponse{Roles: make([]string, len(roles))} for i := range roles { resp.Roles[i] = string(roles[i].Name) } return resp, nil } func (as *authStore) RoleRevokePermission(r *pb.AuthRoleRevokePermissionRequest) (*pb.AuthRoleRevokePermissionResponse, error) { tx := as.be.BatchTx() tx.Lock() defer tx.Unlock() role := tx.UnsafeGetRole(r.Role) if role == nil { return nil, ErrRoleNotFound } updatedRole := &authpb.Role{ Name: role.Name, } for _, perm := range role.KeyPermission { if !bytes.Equal(perm.Key, r.Key) || !bytes.Equal(perm.RangeEnd, r.RangeEnd) { updatedRole.KeyPermission = append(updatedRole.KeyPermission, perm) } } if len(role.KeyPermission) == len(updatedRole.KeyPermission) { return nil, ErrPermissionNotGranted } tx.UnsafePutRole(updatedRole) as.commitRevision(tx) as.refreshRangePermCache(tx) as.lg.Info( "revoked a permission on range", zap.String("role-name", r.Role), zap.String("key", string(r.Key)), zap.String("range-end", string(r.RangeEnd)), ) return &pb.AuthRoleRevokePermissionResponse{}, nil } func (as *authStore) RoleDelete(r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDeleteResponse, error) { if as.enabled && r.Role == rootRole { as.lg.Error("cannot delete 'root' role", zap.String("role-name", r.Role)) return nil, ErrInvalidAuthMgmt } tx := as.be.BatchTx() tx.Lock() defer tx.Unlock() role := tx.UnsafeGetRole(r.Role) if role == nil { return nil, ErrRoleNotFound } tx.UnsafeDeleteRole(r.Role) users := tx.UnsafeGetAllUsers() for _, user := range users { updatedUser := &authpb.User{ Name: user.Name, Password: user.Password, Options: user.Options, } for _, role := range user.Roles { if role != r.Role { updatedUser.Roles = append(updatedUser.Roles, role) } } if len(updatedUser.Roles) == len(user.Roles) { continue } tx.UnsafePutUser(updatedUser) } as.commitRevision(tx) as.refreshRangePermCache(tx) as.lg.Info("deleted a role", zap.String("role-name", r.Role)) return &pb.AuthRoleDeleteResponse{}, nil } func (as *authStore) RoleAdd(r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, error) { if len(r.Name) == 0 { return nil, ErrRoleEmpty } tx := as.be.BatchTx() tx.Lock() defer tx.Unlock() role := tx.UnsafeGetRole(r.Name) if role != nil { return nil, ErrRoleAlreadyExist } newRole := &authpb.Role{ Name: []byte(r.Name), } tx.UnsafePutRole(newRole) as.commitRevision(tx) as.lg.Info("created a role", zap.String("role-name", r.Name)) return &pb.AuthRoleAddResponse{}, nil } func (as *authStore) authInfoFromToken(ctx context.Context, token string) (*AuthInfo, bool) { return as.tokenProvider.info(ctx, token, as.Revision()) } type permSlice []*authpb.Permission func (perms permSlice) Len() int { return len(perms) } func (perms permSlice) Less(i, j int) bool { return bytes.Compare(perms[i].Key, perms[j].Key) < 0 } func (perms permSlice) Swap(i, j int) { perms[i], perms[j] = perms[j], perms[i] } func (as *authStore) RoleGrantPermission(r *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) { if r.Perm == nil { return nil, ErrPermissionNotGiven } if !isValidPermissionRange(r.Perm.Key, r.Perm.RangeEnd) { return nil, ErrInvalidAuthMgmt } tx := as.be.BatchTx() tx.Lock() defer tx.Unlock() role := tx.UnsafeGetRole(r.Name) if role == nil { return nil, ErrRoleNotFound } idx := sort.Search(len(role.KeyPermission), func(i int) bool { return bytes.Compare(role.KeyPermission[i].Key, r.Perm.Key) >= 0 }) if idx < len(role.KeyPermission) && bytes.Equal(role.KeyPermission[idx].Key, r.Perm.Key) && bytes.Equal(role.KeyPermission[idx].RangeEnd, r.Perm.RangeEnd) { // update existing permission role.KeyPermission[idx].PermType = r.Perm.PermType } else { // append new permission to the role newPerm := &authpb.Permission{ Key: r.Perm.Key, RangeEnd: r.Perm.RangeEnd, PermType: r.Perm.PermType, } role.KeyPermission = append(role.KeyPermission, newPerm) sort.Sort(permSlice(role.KeyPermission)) } tx.UnsafePutRole(role) as.commitRevision(tx) as.refreshRangePermCache(tx) as.lg.Info( "granted/updated a permission to a user", zap.String("user-name", r.Name), zap.String("permission-name", authpb.Permission_Type_name[int32(r.Perm.PermType)]), zap.ByteString("key", r.Perm.Key), zap.ByteString("range-end", r.Perm.RangeEnd), ) return &pb.AuthRoleGrantPermissionResponse{}, nil } func (as *authStore) isOpPermitted(userName string, revision uint64, key, rangeEnd []byte, permTyp authpb.Permission_Type) error { // TODO(mitake): this function would be costly so we need a caching mechanism if !as.IsAuthEnabled() { return nil } // only gets rev == 0 when passed AuthInfo{}; no user given if revision == 0 { return ErrUserEmpty } rev := as.Revision() if revision < rev { as.lg.Warn("request auth revision is less than current node auth revision", zap.Uint64("current node auth revision", rev), zap.Uint64("request auth revision", revision), zap.ByteString("request key", key), zap.Error(ErrAuthOldRevision)) return ErrAuthOldRevision } tx := as.be.ReadTx() tx.RLock() defer tx.RUnlock() user := tx.UnsafeGetUser(userName) if user == nil { as.lg.Error("cannot find a user for permission check", zap.String("user-name", userName)) return ErrPermissionDenied } // root role should have permission on all ranges if hasRootRole(user) { return nil } if as.isRangeOpPermitted(userName, key, rangeEnd, permTyp) { return nil } return ErrPermissionDenied } func (as *authStore) IsPutPermitted(authInfo *AuthInfo, key []byte) error { return as.isOpPermitted(authInfo.Username, authInfo.Revision, key, nil, authpb.Permission_WRITE) } func (as *authStore) IsRangePermitted(authInfo *AuthInfo, key, rangeEnd []byte) error { return as.isOpPermitted(authInfo.Username, authInfo.Revision, key, rangeEnd, authpb.Permission_READ) } func (as *authStore) IsDeleteRangePermitted(authInfo *AuthInfo, key, rangeEnd []byte) error { return as.isOpPermitted(authInfo.Username, authInfo.Revision, key, rangeEnd, authpb.Permission_WRITE) } func (as *authStore) IsAdminPermitted(authInfo *AuthInfo) error { if !as.IsAuthEnabled() { return nil } if authInfo == nil || authInfo.Username == "" { return ErrUserEmpty } tx := as.be.ReadTx() tx.RLock() defer tx.RUnlock() u := tx.UnsafeGetUser(authInfo.Username) if u == nil { return ErrUserNotFound } if !hasRootRole(u) { return ErrPermissionDenied } return nil } func (as *authStore) IsAuthEnabled() bool { as.enabledMu.RLock() defer as.enabledMu.RUnlock() return as.enabled } // NewAuthStore creates a new AuthStore. func NewAuthStore(lg *zap.Logger, be AuthBackend, tp TokenProvider, bcryptCost int) AuthStore { if lg == nil { lg = zap.NewNop() } if bcryptCost < bcrypt.MinCost || bcryptCost > bcrypt.MaxCost { lg.Warn( "use default bcrypt cost instead of the invalid given cost", zap.Int("min-cost", bcrypt.MinCost), zap.Int("max-cost", bcrypt.MaxCost), zap.Int("default-cost", bcrypt.DefaultCost), zap.Int("given-cost", bcryptCost), ) bcryptCost = bcrypt.DefaultCost } be.CreateAuthBuckets() tx := be.BatchTx() // We should call LockOutsideApply here, but the txPostLockHoos isn't set // to EtcdServer yet, so it's OK. tx.Lock() enabled := tx.UnsafeReadAuthEnabled() as := &authStore{ revision: tx.UnsafeReadAuthRevision(), lg: lg, be: be, enabled: enabled, rangePermCache: make(map[string]*unifiedRangePermissions), tokenProvider: tp, bcryptCost: bcryptCost, } if enabled { as.tokenProvider.enable() } if as.Revision() == 0 { as.commitRevision(tx) } as.setupMetricsReporter() as.refreshRangePermCache(tx) tx.Unlock() be.ForceCommit() return as } func hasRootRole(u *authpb.User) bool { // u.Roles is sorted in UserGrantRole(), so we can use binary search. idx := sort.SearchStrings(u.Roles, rootRole) return idx != len(u.Roles) && u.Roles[idx] == rootRole } func (as *authStore) commitRevision(tx UnsafeAuthWriter) { atomic.AddUint64(&as.revision, 1) tx.UnsafeSaveAuthRevision(as.Revision()) } func (as *authStore) setRevision(rev uint64) { atomic.StoreUint64(&as.revision, rev) } func (as *authStore) Revision() uint64 { return atomic.LoadUint64(&as.revision) } func (as *authStore) AuthInfoFromTLS(ctx context.Context) (ai *AuthInfo) { peer, ok := peer.FromContext(ctx) if !ok || peer == nil || peer.AuthInfo == nil { return nil } tlsInfo := peer.AuthInfo.(credentials.TLSInfo) for _, chains := range tlsInfo.State.VerifiedChains { if len(chains) < 1 { continue } ai = &AuthInfo{ Username: chains[0].Subject.CommonName, Revision: as.Revision(), } md, ok := metadata.FromIncomingContext(ctx) if !ok { return nil } // gRPC-gateway proxy request to etcd server includes Grpcgateway-Accept // header. The proxy uses etcd client server certificate. If the certificate // has a CommonName we should never use this for authentication. if gw := md["grpcgateway-accept"]; len(gw) > 0 { as.lg.Warn( "ignoring common name in gRPC-gateway proxy request", zap.String("common-name", ai.Username), zap.String("user-name", ai.Username), zap.Uint64("revision", ai.Revision), ) return nil } as.lg.Debug( "found command name", zap.String("common-name", ai.Username), zap.String("user-name", ai.Username), zap.Uint64("revision", ai.Revision), ) break } return ai } func (as *authStore) AuthInfoFromCtx(ctx context.Context) (*AuthInfo, error) { if !as.IsAuthEnabled() { return nil, nil } md, ok := metadata.FromIncomingContext(ctx) if !ok { return nil, nil } // TODO(mitake|hexfusion) review unifying key names ts, ok := md[rpctypes.TokenFieldNameGRPC] if !ok { ts, ok = md[rpctypes.TokenFieldNameSwagger] } if !ok { return nil, nil } token := ts[0] authInfo, uok := as.authInfoFromToken(ctx, token) if !uok { tokenFingerprint := redactToken(token) as.lg.Warn("invalid auth token", zap.String("token-fingerprint", tokenFingerprint)) return nil, ErrInvalidAuthToken } return authInfo, nil } func (as *authStore) GenTokenPrefix() (string, error) { return as.tokenProvider.genTokenPrefix() } func decomposeOpts(lg *zap.Logger, optstr string) (string, map[string]string, error) { opts := strings.Split(optstr, ",") tokenType := opts[0] typeSpecificOpts := make(map[string]string) for i := 1; i < len(opts); i++ { pair := strings.Split(opts[i], "=") if len(pair) != 2 { if lg != nil { lg.Error("invalid token option", zap.String("option", optstr)) } return "", nil, ErrInvalidAuthOpts } if _, ok := typeSpecificOpts[pair[0]]; ok { if lg != nil { lg.Error( "invalid token option", zap.String("option", optstr), zap.String("duplicate-parameter", pair[0]), ) } return "", nil, ErrInvalidAuthOpts } typeSpecificOpts[pair[0]] = pair[1] } return tokenType, typeSpecificOpts, nil } // NewTokenProvider creates a new token provider. func NewTokenProvider( lg *zap.Logger, tokenOpts string, indexWaiter func(uint64) <-chan struct{}, TokenTTL time.Duration, ) (TokenProvider, error) { tokenType, typeSpecificOpts, err := decomposeOpts(lg, tokenOpts) if err != nil { return nil, ErrInvalidAuthOpts } switch tokenType { case tokenTypeSimple: if lg != nil { lg.Warn("simple token is not cryptographically signed") } return newTokenProviderSimple(lg, indexWaiter, TokenTTL), nil case tokenTypeJWT: return newTokenProviderJWT(lg, typeSpecificOpts) case "": return newTokenProviderNop() default: if lg != nil { lg.Warn( "unknown token type", zap.String("type", tokenType), zap.Error(ErrInvalidAuthOpts), ) } return nil, ErrInvalidAuthOpts } } func (as *authStore) WithRoot(ctx context.Context) context.Context { if !as.IsAuthEnabled() { return ctx } var ctxForAssign context.Context if ts, ok := as.tokenProvider.(*tokenSimple); ok && ts != nil { ctx1 := context.WithValue(ctx, AuthenticateParamIndex{}, uint64(0)) prefix, err := ts.genTokenPrefix() if err != nil { as.lg.Error( "failed to generate prefix of internally used token", zap.Error(err), ) return ctx } ctxForAssign = context.WithValue(ctx1, AuthenticateParamSimpleTokenPrefix{}, prefix) } else { ctxForAssign = ctx } token, err := as.tokenProvider.assign(ctxForAssign, "root", as.Revision()) if err != nil { // this must not happen as.lg.Error( "failed to assign token for lease revoking", zap.Error(err), ) return ctx } mdMap := map[string]string{ rpctypes.TokenFieldNameGRPC: token, } tokenMD := metadata.New(mdMap) // use "mdIncomingKey{}" since it's called from local etcdserver return metadata.NewIncomingContext(ctx, tokenMD) } func (as *authStore) HasRole(user, role string) bool { tx := as.be.BatchTx() tx.Lock() u := tx.UnsafeGetUser(user) tx.Unlock() if u == nil { as.lg.Warn( "'has-role' requested for non-existing user", zap.String("user-name", user), zap.String("role-name", role), ) return false } for _, r := range u.Roles { if role == r { return true } } return false } func (as *authStore) BcryptCost() int { return as.bcryptCost } func (as *authStore) setupMetricsReporter() { reportCurrentAuthRevMu.Lock() reportCurrentAuthRev = func() float64 { return float64(as.Revision()) } reportCurrentAuthRevMu.Unlock() } func redactToken(token string) string { sum := sha256.Sum256([]byte(token)) return hex.EncodeToString(sum[:])[:12] } ================================================ FILE: server/auth/store_mock_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import "go.etcd.io/etcd/api/v3/authpb" type backendMock struct { users map[string]*authpb.User roles map[string]*authpb.Role enabled bool revision uint64 } func newBackendMock() *backendMock { return &backendMock{ users: make(map[string]*authpb.User), roles: make(map[string]*authpb.Role), } } func (b *backendMock) CreateAuthBuckets() { } func (b *backendMock) ForceCommit() { } func (b *backendMock) ReadTx() AuthReadTx { return &txMock{be: b} } func (b *backendMock) BatchTx() AuthBatchTx { return &txMock{be: b} } func (b *backendMock) GetUser(s string) *authpb.User { return b.users[s] } func (b *backendMock) GetAllUsers() []*authpb.User { return b.BatchTx().UnsafeGetAllUsers() } func (b *backendMock) GetRole(s string) *authpb.Role { return b.roles[s] } func (b *backendMock) GetAllRoles() []*authpb.Role { return b.BatchTx().UnsafeGetAllRoles() } var _ AuthBackend = (*backendMock)(nil) type txMock struct { be *backendMock } var _ AuthBatchTx = (*txMock)(nil) func (t txMock) UnsafeReadAuthEnabled() bool { return t.be.enabled } func (t txMock) UnsafeReadAuthRevision() uint64 { return t.be.revision } func (t txMock) UnsafeGetUser(s string) *authpb.User { return t.be.users[s] } func (t txMock) UnsafeGetRole(s string) *authpb.Role { return t.be.roles[s] } func (t txMock) UnsafeGetAllUsers() []*authpb.User { var users []*authpb.User for _, u := range t.be.users { users = append(users, u) } return users } func (t txMock) UnsafeGetAllRoles() []*authpb.Role { var roles []*authpb.Role for _, r := range t.be.roles { roles = append(roles, r) } return roles } func (t txMock) Lock() { } func (t txMock) Unlock() { } func (t txMock) RLock() { } func (t txMock) RUnlock() { } func (t txMock) UnsafeSaveAuthEnabled(enabled bool) { t.be.enabled = enabled } func (t txMock) UnsafeSaveAuthRevision(rev uint64) { t.be.revision = rev } func (t txMock) UnsafePutUser(user *authpb.User) { t.be.users[string(user.Name)] = user } func (t txMock) UnsafeDeleteUser(s string) { delete(t.be.users, s) } func (t txMock) UnsafePutRole(role *authpb.Role) { t.be.roles[string(role.Name)] = role } func (t txMock) UnsafeDeleteRole(s string) { delete(t.be.roles, s) } ================================================ FILE: server/auth/store_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package auth import ( "context" "encoding/base64" "errors" "fmt" "strings" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "golang.org/x/crypto/bcrypt" "google.golang.org/grpc/metadata" "go.etcd.io/etcd/api/v3/authpb" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/pkg/v3/adt" ) func dummyIndexWaiter(index uint64) <-chan struct{} { ch := make(chan struct{}, 1) go func() { ch <- struct{}{} }() return ch } // TestNewAuthStoreRevision ensures newly auth store // keeps the old revision when there are no changes. func TestNewAuthStoreRevision(t *testing.T) { tp, err := NewTokenProvider(zaptest.NewLogger(t), tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault) if err != nil { t.Fatal(err) } be := newBackendMock() as := NewAuthStore(zaptest.NewLogger(t), be, tp, bcrypt.MinCost) err = enableAuthAndCreateRoot(as) if err != nil { t.Fatal(err) } old := as.Revision() as.Close() // no changes to commit as = NewAuthStore(zaptest.NewLogger(t), be, tp, bcrypt.MinCost) defer as.Close() new := as.Revision() require.Equalf(t, old, new, "expected revision %d, got %d", old, new) } // TestNewAuthStoreBcryptCost ensures that NewAuthStore uses default when given bcrypt-cost is invalid func TestNewAuthStoreBcryptCost(t *testing.T) { tp, err := NewTokenProvider(zaptest.NewLogger(t), tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault) if err != nil { t.Fatal(err) } invalidCosts := [2]int{bcrypt.MinCost - 1, bcrypt.MaxCost + 1} for _, invalidCost := range invalidCosts { as := NewAuthStore(zaptest.NewLogger(t), newBackendMock(), tp, invalidCost) defer as.Close() require.Equalf(t, bcrypt.DefaultCost, as.BcryptCost(), "expected DefaultCost when bcryptcost is invalid") } } func encodePassword(s string) string { hashedPassword, _ := bcrypt.GenerateFromPassword([]byte(s), bcrypt.MinCost) return base64.StdEncoding.EncodeToString(hashedPassword) } func setupAuthStore(t *testing.T) (store *authStore, teardownfunc func(t *testing.T)) { tp, err := NewTokenProvider(zaptest.NewLogger(t), tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault) if err != nil { t.Fatal(err) } as := NewAuthStore(zaptest.NewLogger(t), newBackendMock(), tp, bcrypt.MinCost) err = enableAuthAndCreateRoot(as) if err != nil { t.Fatal(err) } // adds a new role _, err = as.RoleAdd(&pb.AuthRoleAddRequest{Name: "role-test"}) if err != nil { t.Fatal(err) } ua := &pb.AuthUserAddRequest{Name: "foo", HashedPassword: encodePassword("bar"), Options: &authpb.UserAddOptions{NoPassword: false}} _, err = as.UserAdd(ua) // add a non-existing user if err != nil { t.Fatal(err) } // The UserAdd function cannot generate old etcd version user data (user's option is nil) // add special users through the underlying interface asImpl, ok := as.(*authStore) require.Truef(t, ok, "addUserWithNoOption: needs an AuthStore implementation") addUserWithNoOption(asImpl) tearDown := func(_ *testing.T) { as.Close() } return asImpl, tearDown } func addUserWithNoOption(as *authStore) { tx := as.be.BatchTx() tx.Lock() defer tx.Unlock() tx.UnsafePutUser(&authpb.User{ Name: []byte("foo-no-user-options"), Password: []byte("bar"), }) as.commitRevision(tx) as.refreshRangePermCache(tx) } func enableAuthAndCreateRoot(as AuthStore) error { _, err := as.UserAdd(&pb.AuthUserAddRequest{Name: "root", HashedPassword: encodePassword("root"), Options: &authpb.UserAddOptions{NoPassword: false}}) if err != nil { return err } _, err = as.RoleAdd(&pb.AuthRoleAddRequest{Name: "root"}) if err != nil { return err } _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "root", Role: "root"}) if err != nil { return err } return as.AuthEnable() } func TestUserAdd(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) const userName = "foo" ua := &pb.AuthUserAddRequest{Name: userName, Options: &authpb.UserAddOptions{NoPassword: false}} _, err := as.UserAdd(ua) // add an existing user require.Errorf(t, err, "expected %v, got %v", ErrUserAlreadyExist, err) require.ErrorIsf(t, err, ErrUserAlreadyExist, "expected %v, got %v", ErrUserAlreadyExist, err) ua = &pb.AuthUserAddRequest{Name: "", Options: &authpb.UserAddOptions{NoPassword: false}} _, err = as.UserAdd(ua) // add a user with empty name if !errors.Is(err, ErrUserEmpty) { t.Fatal(err) } _, ok := as.rangePermCache[userName] require.Truef(t, ok, "user %s should be added but it doesn't exist in rangePermCache", userName) } func TestRecover(t *testing.T) { as, tearDown := setupAuthStore(t) defer as.Close() defer tearDown(t) as.enabled = false as.Recover(as.be) require.Truef(t, as.IsAuthEnabled(), "expected auth enabled got disabled") } func TestRecoverWithEmptyRangePermCache(t *testing.T) { as, tearDown := setupAuthStore(t) defer as.Close() defer tearDown(t) as.enabled = false as.rangePermCache = map[string]*unifiedRangePermissions{} as.Recover(as.be) require.Truef(t, as.IsAuthEnabled(), "expected auth enabled got disabled") require.Lenf(t, as.rangePermCache, 3, "rangePermCache should have permission information for 3 users (\"root\" and \"foo\",\"foo-no-user-options\"), but has %d information", len(as.rangePermCache)) _, ok := as.rangePermCache["root"] require.Truef(t, ok, "user \"root\" should be created by setupAuthStore() but doesn't exist in rangePermCache") _, ok = as.rangePermCache["foo"] require.Truef(t, ok, "user \"foo\" should be created by setupAuthStore() but doesn't exist in rangePermCache") } func TestCheckPassword(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) // auth a non-existing user _, err := as.CheckPassword("foo-test", "bar") require.Errorf(t, err, "expected %v, got %v", ErrAuthFailed, err) require.ErrorIsf(t, err, ErrAuthFailed, "expected %v, got %v", ErrAuthFailed, err) // auth an existing user with correct password _, err = as.CheckPassword("foo", "bar") if err != nil { t.Fatal(err) } // auth an existing user but with wrong password _, err = as.CheckPassword("foo", "") require.Errorf(t, err, "expected %v, got %v", ErrAuthFailed, err) require.ErrorIsf(t, err, ErrAuthFailed, "expected %v, got %v", ErrAuthFailed, err) } func TestUserDelete(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) // delete an existing user const userName = "foo" ud := &pb.AuthUserDeleteRequest{Name: userName} _, err := as.UserDelete(ud) if err != nil { t.Fatal(err) } // delete a non-existing user _, err = as.UserDelete(ud) require.Errorf(t, err, "expected %v, got %v", ErrUserNotFound, err) require.ErrorIsf(t, err, ErrUserNotFound, "expected %v, got %v", ErrUserNotFound, err) _, ok := as.rangePermCache[userName] require.Falsef(t, ok, "user %s should be deleted but it exists in rangePermCache", userName) } func TestUserDeleteAndPermCache(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) // delete an existing user const deletedUserName = "foo" ud := &pb.AuthUserDeleteRequest{Name: deletedUserName} _, err := as.UserDelete(ud) if err != nil { t.Fatal(err) } // delete a non-existing user _, err = as.UserDelete(ud) require.ErrorIsf(t, err, ErrUserNotFound, "expected %v, got %v", ErrUserNotFound, err) _, ok := as.rangePermCache[deletedUserName] require.Falsef(t, ok, "user %s should be deleted but it exists in rangePermCache", deletedUserName) // add a new user const newUser = "bar" ua := &pb.AuthUserAddRequest{Name: newUser, HashedPassword: encodePassword("pwd1"), Options: &authpb.UserAddOptions{NoPassword: false}} _, err = as.UserAdd(ua) if err != nil { t.Fatal(err) } _, ok = as.rangePermCache[newUser] require.Truef(t, ok, "user %s should exist but it doesn't exist in rangePermCache", deletedUserName) } func TestUserChangePassword(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) ctx1 := context.WithValue(context.WithValue(t.Context(), AuthenticateParamIndex{}, uint64(1)), AuthenticateParamSimpleTokenPrefix{}, "dummy") _, err := as.Authenticate(ctx1, "foo", "bar") if err != nil { t.Fatal(err) } _, err = as.UserChangePassword(&pb.AuthUserChangePasswordRequest{Name: "foo", HashedPassword: encodePassword("baz")}) if err != nil { t.Fatal(err) } ctx2 := context.WithValue(context.WithValue(t.Context(), AuthenticateParamIndex{}, uint64(2)), AuthenticateParamSimpleTokenPrefix{}, "dummy") _, err = as.Authenticate(ctx2, "foo", "baz") if err != nil { t.Fatal(err) } // change a non-existing user _, err = as.UserChangePassword(&pb.AuthUserChangePasswordRequest{Name: "foo-test", HashedPassword: encodePassword("bar")}) require.Errorf(t, err, "expected %v, got %v", ErrUserNotFound, err) require.ErrorIsf(t, err, ErrUserNotFound, "expected %v, got %v", ErrUserNotFound, err) // change a user(user option is nil) password _, err = as.UserChangePassword(&pb.AuthUserChangePasswordRequest{Name: "foo-no-user-options", HashedPassword: encodePassword("bar")}) if err != nil { t.Fatal(err) } } func TestRoleAdd(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) // adds a new role _, err := as.RoleAdd(&pb.AuthRoleAddRequest{Name: "role-test-1"}) if err != nil { t.Fatal(err) } // add a role with empty name _, err = as.RoleAdd(&pb.AuthRoleAddRequest{Name: ""}) if !errors.Is(err, ErrRoleEmpty) { t.Fatal(err) } } func TestUserGrant(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) // grants a role to the user _, err := as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "foo", Role: "role-test"}) if err != nil { t.Fatal(err) } // grants a role to a non-existing user _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "foo-test", Role: "role-test"}) if err == nil { t.Errorf("expected %v, got %v", ErrUserNotFound, err) } if !errors.Is(err, ErrUserNotFound) { t.Errorf("expected %v, got %v", ErrUserNotFound, err) } } func TestHasRole(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) // grants a role to the user _, err := as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "foo", Role: "role-test"}) if err != nil { t.Fatal(err) } // checks role reflects correctly hr := as.HasRole("foo", "role-test") require.Truef(t, hr, "expected role granted, got false") // checks non existent role hr = as.HasRole("foo", "non-existent-role") require.Falsef(t, hr, "expected role not found, got true") // checks non existent user hr = as.HasRole("nouser", "role-test") require.Falsef(t, hr, "expected user not found got true") } func TestIsOpPermitted(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) // add new role _, err := as.RoleAdd(&pb.AuthRoleAddRequest{Name: "role-test-1"}) if err != nil { t.Fatal(err) } perm := &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("Keys"), RangeEnd: []byte("RangeEnd"), } _, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ Name: "role-test-1", Perm: perm, }) if err != nil { t.Fatal(err) } // grants a role to the user _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "foo", Role: "role-test-1"}) if err != nil { t.Fatal(err) } // check permission reflected to user err = as.isOpPermitted("foo", as.Revision(), perm.Key, perm.RangeEnd, perm.PermType) if err != nil { t.Fatal(err) } // Drop the user's permission from cache and expect a permission denied // error. as.rangePermCacheMu.Lock() delete(as.rangePermCache, "foo") as.rangePermCacheMu.Unlock() if err := as.isOpPermitted("foo", as.Revision(), perm.Key, perm.RangeEnd, perm.PermType); !errors.Is(err, ErrPermissionDenied) { t.Fatal(err) } } func TestGetUser(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) _, err := as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "foo", Role: "role-test"}) if err != nil { t.Fatal(err) } u, err := as.UserGet(&pb.AuthUserGetRequest{Name: "foo"}) if err != nil { t.Fatal(err) } require.NotNilf(t, u, "expect user not nil, got nil") expected := []string{"role-test"} assert.Equal(t, expected, u.Roles) // check non existent user _, err = as.UserGet(&pb.AuthUserGetRequest{Name: "nouser"}) if err == nil { t.Errorf("expected %v, got %v", ErrUserNotFound, err) } } func TestListUsers(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) ua := &pb.AuthUserAddRequest{Name: "user1", HashedPassword: encodePassword("pwd1"), Options: &authpb.UserAddOptions{NoPassword: false}} _, err := as.UserAdd(ua) // add a non-existing user if err != nil { t.Fatal(err) } ul, err := as.UserList(&pb.AuthUserListRequest{}) if err != nil { t.Fatal(err) } if !contains(ul.Users, "root") { t.Errorf("expected %v in %v", "root", ul.Users) } if !contains(ul.Users, "user1") { t.Errorf("expected %v in %v", "user1", ul.Users) } } func TestRoleGrantPermission(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) _, err := as.RoleAdd(&pb.AuthRoleAddRequest{Name: "role-test-1"}) if err != nil { t.Fatal(err) } perm := &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("Keys"), RangeEnd: []byte("RangeEnd"), } _, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ Name: "role-test-1", Perm: perm, }) if err != nil { t.Error(err) } r, err := as.RoleGet(&pb.AuthRoleGetRequest{Role: "role-test-1"}) if err != nil { t.Fatal(err) } assert.Equal(t, perm, r.Perm[0]) // trying to grant nil permissions returns an error (and doesn't change the actual permissions!) _, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ Name: "role-test-1", }) if !errors.Is(err, ErrPermissionNotGiven) { t.Error(err) } r, err = as.RoleGet(&pb.AuthRoleGetRequest{Role: "role-test-1"}) if err != nil { t.Fatal(err) } assert.Equal(t, perm, r.Perm[0]) } func TestRoleGrantInvalidPermission(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) _, err := as.RoleAdd(&pb.AuthRoleAddRequest{Name: "role-test-1"}) if err != nil { t.Fatal(err) } tests := []struct { name string perm *authpb.Permission want error }{ { name: "valid range", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("Keys"), RangeEnd: []byte("RangeEnd"), }, want: nil, }, { name: "invalid range: nil key", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: nil, RangeEnd: []byte("RangeEnd"), }, want: ErrInvalidAuthMgmt, }, { name: "valid range: single key", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("Keys"), RangeEnd: nil, }, want: nil, }, { name: "valid range: single key", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("Keys"), RangeEnd: []byte{}, }, want: nil, }, { name: "invalid range: empty (Key == RangeEnd)", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("a"), RangeEnd: []byte("a"), }, want: ErrInvalidAuthMgmt, }, { name: "invalid range: empty (Key > RangeEnd)", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("b"), RangeEnd: []byte("a"), }, want: ErrInvalidAuthMgmt, }, { name: "invalid range: length of key is 0", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte(""), RangeEnd: []byte("a"), }, want: ErrInvalidAuthMgmt, }, { name: "invalid range: length of key is 0", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte(""), RangeEnd: []byte(""), }, want: ErrInvalidAuthMgmt, }, { name: "invalid range: length of key is 0", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte(""), RangeEnd: []byte{0x00}, }, want: ErrInvalidAuthMgmt, }, { name: "valid range: single key permission for []byte{0x00}", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte{0x00}, RangeEnd: []byte(""), }, want: nil, }, { name: "valid range: \"a\" or larger keys", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("a"), RangeEnd: []byte{0x00}, }, want: nil, }, { name: "valid range: the entire keys", perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte{0x00}, RangeEnd: []byte{0x00}, }, want: nil, }, } for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ Name: "role-test-1", Perm: tt.perm, }) if !errors.Is(err, tt.want) { t.Errorf("#%d: result=%t, want=%t", i, err, tt.want) } }) } } func TestRootRoleGrantPermission(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) perm := &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("Keys"), RangeEnd: []byte("RangeEnd"), } _, err := as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ Name: "root", Perm: perm, }) if err != nil { t.Error(err) } r, err := as.RoleGet(&pb.AuthRoleGetRequest{Role: "root"}) if err != nil { t.Fatal(err) } // whatever grant permission to root, it always return root permission. expectPerm := &authpb.Permission{ PermType: authpb.Permission_READWRITE, Key: []byte{}, RangeEnd: []byte{0}, } assert.Equal(t, expectPerm, r.Perm[0]) } func TestRoleRevokePermission(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) _, err := as.RoleAdd(&pb.AuthRoleAddRequest{Name: "role-test-1"}) if err != nil { t.Fatal(err) } perm := &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("Keys"), RangeEnd: []byte("RangeEnd"), } _, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ Name: "role-test-1", Perm: perm, }) if err != nil { t.Fatal(err) } _, err = as.RoleGet(&pb.AuthRoleGetRequest{Role: "role-test-1"}) if err != nil { t.Fatal(err) } _, err = as.RoleRevokePermission(&pb.AuthRoleRevokePermissionRequest{ Role: "role-test-1", Key: []byte("Keys"), RangeEnd: []byte("RangeEnd"), }) if err != nil { t.Fatal(err) } var r *pb.AuthRoleGetResponse r, err = as.RoleGet(&pb.AuthRoleGetRequest{Role: "role-test-1"}) if err != nil { t.Fatal(err) } if len(r.Perm) != 0 { t.Errorf("expected %v, got %v", 0, len(r.Perm)) } } func TestUserRevokePermission(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) _, err := as.RoleAdd(&pb.AuthRoleAddRequest{Name: "role-test-1"}) if err != nil { t.Fatal(err) } const userName = "foo" _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: userName, Role: "role-test"}) if err != nil { t.Fatal(err) } _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: userName, Role: "role-test-1"}) if err != nil { t.Fatal(err) } perm := &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("WriteKeyBegin"), RangeEnd: []byte("WriteKeyEnd"), } _, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ Name: "role-test-1", Perm: perm, }) if err != nil { t.Fatal(err) } _, ok := as.rangePermCache[userName] require.Truef(t, ok, "User %s should have its entry in rangePermCache", userName) unifiedPerm := as.rangePermCache[userName] pt1 := adt.NewBytesAffinePoint([]byte("WriteKeyBegin")) require.Truef(t, unifiedPerm.writePerms.Contains(pt1), "rangePermCache should contain WriteKeyBegin") pt2 := adt.NewBytesAffinePoint([]byte("OutOfRange")) require.Falsef(t, unifiedPerm.writePerms.Contains(pt2), "rangePermCache should not contain OutOfRange") u, err := as.UserGet(&pb.AuthUserGetRequest{Name: userName}) if err != nil { t.Fatal(err) } expected := []string{"role-test", "role-test-1"} assert.Equal(t, expected, u.Roles) _, err = as.UserRevokeRole(&pb.AuthUserRevokeRoleRequest{Name: userName, Role: "role-test-1"}) if err != nil { t.Fatal(err) } u, err = as.UserGet(&pb.AuthUserGetRequest{Name: userName}) if err != nil { t.Fatal(err) } expected = []string{"role-test"} assert.Equal(t, expected, u.Roles) } func TestRoleDelete(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) _, err := as.RoleDelete(&pb.AuthRoleDeleteRequest{Role: "role-test"}) if err != nil { t.Fatal(err) } rl, err := as.RoleList(&pb.AuthRoleListRequest{}) if err != nil { t.Fatal(err) } expected := []string{"root"} assert.Equal(t, expected, rl.Roles) } func TestAuthInfoFromCtx(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) ctx := t.Context() ai, err := as.AuthInfoFromCtx(ctx) if err != nil && ai != nil { t.Errorf("expected (nil, nil), got (%v, %v)", ai, err) } // as if it came from RPC ctx = metadata.NewIncomingContext(t.Context(), metadata.New(map[string]string{"tokens": "dummy"})) ai, err = as.AuthInfoFromCtx(ctx) if err != nil && ai != nil { t.Errorf("expected (nil, nil), got (%v, %v)", ai, err) } ctx = context.WithValue(context.WithValue(t.Context(), AuthenticateParamIndex{}, uint64(1)), AuthenticateParamSimpleTokenPrefix{}, "dummy") resp, err := as.Authenticate(ctx, "foo", "bar") if err != nil { t.Error(err) } ctx = metadata.NewIncomingContext(t.Context(), metadata.New(map[string]string{rpctypes.TokenFieldNameGRPC: "Invalid Token"})) _, err = as.AuthInfoFromCtx(ctx) if !errors.Is(err, ErrInvalidAuthToken) { t.Errorf("expected %v, got %v", ErrInvalidAuthToken, err) } ctx = metadata.NewIncomingContext(t.Context(), metadata.New(map[string]string{rpctypes.TokenFieldNameGRPC: "Invalid.Token"})) _, err = as.AuthInfoFromCtx(ctx) if !errors.Is(err, ErrInvalidAuthToken) { t.Errorf("expected %v, got %v", ErrInvalidAuthToken, err) } ctx = metadata.NewIncomingContext(t.Context(), metadata.New(map[string]string{rpctypes.TokenFieldNameGRPC: resp.Token})) ai, err = as.AuthInfoFromCtx(ctx) if err != nil { t.Error(err) } if ai.Username != "foo" { t.Errorf("expected %v, got %v", "foo", ai.Username) } } func TestAuthDisable(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) as.AuthDisable() ctx := context.WithValue(context.WithValue(t.Context(), AuthenticateParamIndex{}, uint64(2)), AuthenticateParamSimpleTokenPrefix{}, "dummy") _, err := as.Authenticate(ctx, "foo", "bar") if !errors.Is(err, ErrAuthNotEnabled) { t.Errorf("expected %v, got %v", ErrAuthNotEnabled, err) } // Disabling disabled auth to make sure it can return safely if store is already disabled. as.AuthDisable() _, err = as.Authenticate(ctx, "foo", "bar") if !errors.Is(err, ErrAuthNotEnabled) { t.Errorf("expected %v, got %v", ErrAuthNotEnabled, err) } } func TestIsAuthEnabled(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) // enable authentication to test the first possible condition as.AuthEnable() status := as.IsAuthEnabled() ctx := context.WithValue(context.WithValue(t.Context(), AuthenticateParamIndex{}, uint64(2)), AuthenticateParamSimpleTokenPrefix{}, "dummy") _, _ = as.Authenticate(ctx, "foo", "bar") if status != true { t.Errorf("expected %v, got %v", true, false) } // Disabling disabled auth to test the other condition that can be return as.AuthDisable() status = as.IsAuthEnabled() _, _ = as.Authenticate(ctx, "foo", "bar") if status != false { t.Errorf("expected %v, got %v", false, true) } } // TestAuthInfoFromCtxRace ensures that access to authStore.revision is thread-safe. func TestAuthInfoFromCtxRace(t *testing.T) { tp, err := NewTokenProvider(zaptest.NewLogger(t), tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault) if err != nil { t.Fatal(err) } as := NewAuthStore(zaptest.NewLogger(t), newBackendMock(), tp, bcrypt.MinCost) defer as.Close() donec := make(chan struct{}) go func() { defer close(donec) ctx := metadata.NewIncomingContext(t.Context(), metadata.New(map[string]string{rpctypes.TokenFieldNameGRPC: "test"})) as.AuthInfoFromCtx(ctx) }() as.UserAdd(&pb.AuthUserAddRequest{Name: "test", Options: &authpb.UserAddOptions{NoPassword: false}}) <-donec } func TestIsAdminPermitted(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) err := as.IsAdminPermitted(&AuthInfo{Username: "root", Revision: 1}) if err != nil { t.Errorf("expected nil, got %v", err) } // invalid user err = as.IsAdminPermitted(&AuthInfo{Username: "rooti", Revision: 1}) if !errors.Is(err, ErrUserNotFound) { t.Errorf("expected %v, got %v", ErrUserNotFound, err) } // empty user err = as.IsAdminPermitted(&AuthInfo{Username: "", Revision: 1}) if !errors.Is(err, ErrUserEmpty) { t.Errorf("expected %v, got %v", ErrUserEmpty, err) } // non-admin user err = as.IsAdminPermitted(&AuthInfo{Username: "foo", Revision: 1}) if !errors.Is(err, ErrPermissionDenied) { t.Errorf("expected %v, got %v", ErrPermissionDenied, err) } // disabled auth should return nil as.AuthDisable() err = as.IsAdminPermitted(&AuthInfo{Username: "root", Revision: 1}) if err != nil { t.Errorf("expected nil, got %v", err) } } func TestRecoverFromSnapshot(t *testing.T) { as, teardown := setupAuthStore(t) defer teardown(t) ua := &pb.AuthUserAddRequest{Name: "foo", Options: &authpb.UserAddOptions{NoPassword: false}} _, err := as.UserAdd(ua) // add an existing user require.Errorf(t, err, "expected %v, got %v", ErrUserAlreadyExist, err) require.ErrorIsf(t, err, ErrUserAlreadyExist, "expected %v, got %v", ErrUserAlreadyExist, err) ua = &pb.AuthUserAddRequest{Name: "", Options: &authpb.UserAddOptions{NoPassword: false}} _, err = as.UserAdd(ua) // add a user with empty name if !errors.Is(err, ErrUserEmpty) { t.Fatal(err) } as.Close() tp, err := NewTokenProvider(zaptest.NewLogger(t), tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault) if err != nil { t.Fatal(err) } as2 := NewAuthStore(zaptest.NewLogger(t), as.be, tp, bcrypt.MinCost) defer as2.Close() require.Truef(t, as2.IsAuthEnabled(), "recovering authStore from existing backend failed") ul, err := as.UserList(&pb.AuthUserListRequest{}) if err != nil { t.Fatal(err) } if !contains(ul.Users, "root") { t.Errorf("expected %v in %v", "root", ul.Users) } } func contains(array []string, str string) bool { for _, s := range array { if s == str { return true } } return false } func TestHammerSimpleAuthenticate(t *testing.T) { // set TTL values low to try to trigger races oldTTL, oldTTLRes := simpleTokenTTLDefault, simpleTokenTTLResolution defer func() { simpleTokenTTLDefault = oldTTL simpleTokenTTLResolution = oldTTLRes }() simpleTokenTTLDefault = 10 * time.Millisecond simpleTokenTTLResolution = simpleTokenTTLDefault users := make(map[string]struct{}) as, tearDown := setupAuthStore(t) defer tearDown(t) // create lots of users for i := 0; i < 50; i++ { u := fmt.Sprintf("user-%d", i) ua := &pb.AuthUserAddRequest{Name: u, HashedPassword: encodePassword("123"), Options: &authpb.UserAddOptions{NoPassword: false}} if _, err := as.UserAdd(ua); err != nil { t.Fatal(err) } users[u] = struct{}{} } // hammer on authenticate with lots of users for i := 0; i < 10; i++ { var wg sync.WaitGroup wg.Add(len(users)) for u := range users { go func(user string) { defer wg.Done() token := fmt.Sprintf("%s(%d)", user, i) ctx := context.WithValue(context.WithValue(t.Context(), AuthenticateParamIndex{}, uint64(1)), AuthenticateParamSimpleTokenPrefix{}, token) if _, err := as.Authenticate(ctx, user, "123"); err != nil { t.Error(err) } if _, err := as.AuthInfoFromCtx(ctx); err != nil { t.Error(err) } }(u) } time.Sleep(time.Millisecond) wg.Wait() } } // TestRolesOrder tests authpb.User.Roles is sorted func TestRolesOrder(t *testing.T) { tp, err := NewTokenProvider(zaptest.NewLogger(t), tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault) defer tp.disable() if err != nil { t.Fatal(err) } as := NewAuthStore(zaptest.NewLogger(t), newBackendMock(), tp, bcrypt.MinCost) defer as.Close() err = enableAuthAndCreateRoot(as) if err != nil { t.Fatal(err) } username := "user" _, err = as.UserAdd(&pb.AuthUserAddRequest{Name: username, HashedPassword: encodePassword("pass"), Options: &authpb.UserAddOptions{NoPassword: false}}) if err != nil { t.Fatal(err) } roles := []string{"role1", "role2", "abc", "xyz", "role3"} for _, role := range roles { _, err = as.RoleAdd(&pb.AuthRoleAddRequest{Name: role}) if err != nil { t.Fatal(err) } _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: username, Role: role}) if err != nil { t.Fatal(err) } } user, err := as.UserGet(&pb.AuthUserGetRequest{Name: username}) if err != nil { t.Fatal(err) } for i := 1; i < len(user.Roles); i++ { if strings.Compare(user.Roles[i-1], user.Roles[i]) != -1 { t.Errorf("User.Roles isn't sorted (%s vs %s)", user.Roles[i-1], user.Roles[i]) } } } func TestAuthInfoFromCtxWithRootSimple(t *testing.T) { testAuthInfoFromCtxWithRoot(t, tokenTypeSimple) } func TestAuthInfoFromCtxWithRootJWT(t *testing.T) { opts := testJWTOpts() testAuthInfoFromCtxWithRoot(t, opts) } // testAuthInfoFromCtxWithRoot ensures "WithRoot" properly embeds token in the context. func testAuthInfoFromCtxWithRoot(t *testing.T, opts string) { tp, err := NewTokenProvider(zaptest.NewLogger(t), opts, dummyIndexWaiter, simpleTokenTTLDefault) if err != nil { t.Fatal(err) } as := NewAuthStore(zaptest.NewLogger(t), newBackendMock(), tp, bcrypt.MinCost) defer as.Close() if err = enableAuthAndCreateRoot(as); err != nil { t.Fatal(err) } ctx := t.Context() ctx = as.WithRoot(ctx) ai, aerr := as.AuthInfoFromCtx(ctx) if aerr != nil { t.Fatal(err) } require.NotNilf(t, ai, "expected non-nil *AuthInfo") if ai.Username != "root" { t.Errorf("expected user name 'root', got %+v", ai) } } func TestUserNoPasswordAdd(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) username := "usernopass" ua := &pb.AuthUserAddRequest{Name: username, Options: &authpb.UserAddOptions{NoPassword: true}} _, err := as.UserAdd(ua) if err != nil { t.Fatal(err) } ctx := context.WithValue(context.WithValue(t.Context(), AuthenticateParamIndex{}, uint64(1)), AuthenticateParamSimpleTokenPrefix{}, "dummy") _, err = as.Authenticate(ctx, username, "") require.ErrorIsf(t, err, ErrAuthFailed, "expected %v, got %v", ErrAuthFailed, err) } func TestUserAddWithOldLog(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) ua := &pb.AuthUserAddRequest{Name: "bar", Password: "baz", Options: &authpb.UserAddOptions{NoPassword: false}} _, err := as.UserAdd(ua) if err != nil { t.Fatal(err) } } func TestUserChangePasswordWithOldLog(t *testing.T) { as, tearDown := setupAuthStore(t) defer tearDown(t) ctx1 := context.WithValue(context.WithValue(t.Context(), AuthenticateParamIndex{}, uint64(1)), AuthenticateParamSimpleTokenPrefix{}, "dummy") _, err := as.Authenticate(ctx1, "foo", "bar") if err != nil { t.Fatal(err) } _, err = as.UserChangePassword(&pb.AuthUserChangePasswordRequest{Name: "foo", Password: "baz"}) if err != nil { t.Fatal(err) } ctx2 := context.WithValue(context.WithValue(t.Context(), AuthenticateParamIndex{}, uint64(2)), AuthenticateParamSimpleTokenPrefix{}, "dummy") _, err = as.Authenticate(ctx2, "foo", "baz") if err != nil { t.Fatal(err) } // change a non-existing user _, err = as.UserChangePassword(&pb.AuthUserChangePasswordRequest{Name: "foo-test", HashedPassword: encodePassword("bar")}) require.Errorf(t, err, "expected %v, got %v", ErrUserNotFound, err) require.ErrorIsf(t, err, ErrUserNotFound, "expected %v, got %v", ErrUserNotFound, err) } ================================================ FILE: server/config/config.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "context" "fmt" "path/filepath" "sort" "strings" "time" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.uber.org/zap" bolt "go.etcd.io/bbolt" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/featuregate" "go.etcd.io/etcd/pkg/v3/netutil" "go.etcd.io/etcd/server/v3/etcdserver/api/v3discovery" "go.etcd.io/etcd/server/v3/storage/datadir" ) const ( grpcOverheadBytes = 512 * 1024 ) // ServerConfig holds the configuration of etcd as taken from the command line or discovery. type ServerConfig struct { Name string DiscoveryCfg v3discovery.DiscoveryConfig ClientURLs types.URLs PeerURLs types.URLs DataDir string // DedicatedWALDir config will make the etcd to write the WAL to the WALDir // rather than the dataDir/member/wal. DedicatedWALDir string SnapshotCount uint64 // SnapshotCatchUpEntries is the number of entries for a slow follower // to catch-up after compacting the raft storage entries. // We expect the follower has a millisecond level latency with the leader. // The max throughput is around 10K. Keep a 5K entries is enough for helping // follower to catch up. SnapshotCatchUpEntries uint64 MaxSnapFiles uint MaxWALFiles uint // BackendBatchInterval is the maximum time before commit the backend transaction. BackendBatchInterval time.Duration // BackendBatchLimit is the maximum operations before commit the backend transaction. BackendBatchLimit int // BackendFreelistType is the type of the backend boltdb freelist. BackendFreelistType bolt.FreelistType InitialPeerURLsMap types.URLsMap InitialClusterToken string NewCluster bool PeerTLSInfo transport.TLSInfo CORS map[string]struct{} // HostWhitelist lists acceptable hostnames from client requests. // If server is insecure (no TLS), server only accepts requests // whose Host header value exists in this white list. HostWhitelist map[string]struct{} TickMs uint ElectionTicks int // InitialElectionTickAdvance is true, then local member fast-forwards // election ticks to speed up "initial" leader election trigger. This // benefits the case of larger election ticks. For instance, cross // datacenter deployment may require longer election timeout of 10-second. // If true, local node does not need wait up to 10-second. Instead, // forwards its election ticks to 8-second, and have only 2-second left // before leader election. // // Major assumptions are that: // - cluster has no active leader thus advancing ticks enables faster // leader election, or // - cluster already has an established leader, and rejoining follower // is likely to receive heartbeats from the leader after tick advance // and before election timeout. // // However, when network from leader to rejoining follower is congested, // and the follower does not receive leader heartbeat within left election // ticks, disruptive election has to happen thus affecting cluster // availabilities. // // Disabling this would slow down initial bootstrap process for cross // datacenter deployments. Make your own tradeoffs by configuring // --initial-election-tick-advance at the cost of slow initial bootstrap. // // If single-node, it advances ticks regardless. // // See https://github.com/etcd-io/etcd/issues/9333 for more detail. InitialElectionTickAdvance bool BootstrapTimeout time.Duration AutoCompactionRetention time.Duration AutoCompactionMode string CompactionBatchLimit int CompactionSleepInterval time.Duration QuotaBackendBytes int64 MaxTxnOps uint // MaxRequestBytes is the maximum request size to send over raft. MaxRequestBytes uint // MaxConcurrentStreams specifies the maximum number of concurrent // streams that each client can open at a time. MaxConcurrentStreams uint32 WarningApplyDuration time.Duration WarningUnaryRequestDuration time.Duration StrictReconfigCheck bool // ClientCertAuthEnabled is true when cert has been signed by the client CA. ClientCertAuthEnabled bool AuthToken string BcryptCost uint TokenTTL uint // InitialCorruptCheck is true to check data corruption on boot // before serving any peer/client traffic. InitialCorruptCheck bool CorruptCheckTime time.Duration CompactHashCheckTime time.Duration // PreVote is true to enable Raft Pre-Vote. PreVote bool // SocketOpts are socket options passed to listener config. SocketOpts transport.SocketOpts // Logger logs server-side operations. Logger *zap.Logger ForceNewCluster bool // LeaseCheckpointInterval time.Duration is the wait duration between lease checkpoints. LeaseCheckpointInterval time.Duration EnableGRPCGateway bool // EnableDistributedTracing enables distributed tracing using OpenTelemetry protocol. EnableDistributedTracing bool // TracerOptions are options for OpenTelemetry gRPC interceptor. TracerOptions []otelgrpc.Option WatchProgressNotifyInterval time.Duration // UnsafeNoFsync disables all uses of fsync. // Setting this is unsafe and will cause data loss. UnsafeNoFsync bool `json:"unsafe-no-fsync"` DowngradeCheckTime time.Duration // MemoryMlock enables mlocking of etcd owned memory pages. // The setting improves etcd tail latency in environments were: // - memory pressure might lead to swapping pages to disk // - disk latency might be unstable // Currently all etcd memory gets mlocked, but in future the flag can // be refined to mlock in-use area of bbolt only. MemoryMlock bool `json:"memory-mlock"` // BootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to // consider running defrag during bootstrap. Needs to be set to non-zero value to take effect. BootstrapDefragThresholdMegabytes uint `json:"bootstrap-defrag-threshold-megabytes"` // MaxLearners sets a limit to the number of learner members that can exist in the cluster membership. MaxLearners int `json:"max-learners"` // V2Deprecation defines a phase of v2store deprecation process. V2Deprecation V2DeprecationEnum `json:"v2-deprecation"` // LocalAddress is the local IP address to use when communicating with a peer. LocalAddress string `json:"local-address"` // ServerFeatureGate is a server level feature gate ServerFeatureGate featuregate.FeatureGate // Metrics types of metrics - should be either 'basic' or 'extensive' Metrics string } // VerifyBootstrap sanity-checks the initial config for bootstrap case // and returns an error for things that should never happen. func (c *ServerConfig) VerifyBootstrap() error { if err := c.hasLocalMember(); err != nil { return err } if err := c.advertiseMatchesCluster(); err != nil { return err } if CheckDuplicateURL(c.InitialPeerURLsMap) { return fmt.Errorf("initial cluster %s has duplicate url", c.InitialPeerURLsMap) } if c.InitialPeerURLsMap.String() == "" && !c.ShouldDiscover() { return fmt.Errorf("initial cluster unset and no discovery endpoints found") } return nil } // VerifyJoinExisting sanity-checks the initial config for join existing cluster // case and returns an error for things that should never happen. func (c *ServerConfig) VerifyJoinExisting() error { // The member has announced its peer urls to the cluster before starting; no need to // set the configuration again. if err := c.hasLocalMember(); err != nil { return err } if CheckDuplicateURL(c.InitialPeerURLsMap) { return fmt.Errorf("initial cluster %s has duplicate url", c.InitialPeerURLsMap) } if c.ShouldDiscover() { return fmt.Errorf("discovery URL should not be set when joining existing initial cluster") } return nil } // hasLocalMember checks that the cluster at least contains the local server. func (c *ServerConfig) hasLocalMember() error { if urls := c.InitialPeerURLsMap[c.Name]; urls == nil { return fmt.Errorf("couldn't find local name %q in the initial cluster configuration", c.Name) } return nil } // advertiseMatchesCluster confirms peer URLs match those in the cluster peer list. func (c *ServerConfig) advertiseMatchesCluster() error { urls, apurls := c.InitialPeerURLsMap[c.Name], c.PeerURLs.StringSlice() urls.Sort() sort.Strings(apurls) ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) defer cancel() ok, err := netutil.URLStringsEqual(ctx, c.Logger, apurls, urls.StringSlice()) if ok { return nil } initMap, apMap := make(map[string]struct{}), make(map[string]struct{}) for _, url := range c.PeerURLs { apMap[url.String()] = struct{}{} } for _, url := range c.InitialPeerURLsMap[c.Name] { initMap[url.String()] = struct{}{} } var missing []string for url := range initMap { if _, ok := apMap[url]; !ok { missing = append(missing, url) } } if len(missing) > 0 { for i := range missing { missing[i] = c.Name + "=" + missing[i] } mstr := strings.Join(missing, ",") apStr := strings.Join(apurls, ",") return fmt.Errorf("--initial-cluster has %s but missing from --initial-advertise-peer-urls=%s (%w)", mstr, apStr, err) } for url := range apMap { if _, ok := initMap[url]; !ok { missing = append(missing, url) } } if len(missing) > 0 { mstr := strings.Join(missing, ",") umap := types.URLsMap(map[string]types.URLs{c.Name: c.PeerURLs}) return fmt.Errorf("--initial-advertise-peer-urls has %s but missing from --initial-cluster=%s", mstr, umap.String()) } // resolved URLs from "--initial-advertise-peer-urls" and "--initial-cluster" did not match or failed apStr := strings.Join(apurls, ",") umap := types.URLsMap(map[string]types.URLs{c.Name: c.PeerURLs}) return fmt.Errorf("failed to resolve %s to match --initial-cluster=%s (%w)", apStr, umap.String(), err) } func (c *ServerConfig) MemberDir() string { return datadir.ToMemberDir(c.DataDir) } func (c *ServerConfig) WALDir() string { if c.DedicatedWALDir != "" { return c.DedicatedWALDir } return datadir.ToWALDir(c.DataDir) } func (c *ServerConfig) SnapDir() string { return filepath.Join(c.MemberDir(), "snap") } func (c *ServerConfig) ShouldDiscover() bool { return len(c.DiscoveryCfg.Endpoints) > 0 } // ReqTimeout returns timeout for request to finish. func (c *ServerConfig) ReqTimeout() time.Duration { // 5s for queue waiting, computation and disk IO delay // + 2 * election timeout for possible leader election return 5*time.Second + 2*time.Duration(c.ElectionTicks*int(c.TickMs))*time.Millisecond } func (c *ServerConfig) ElectionTimeout() time.Duration { return time.Duration(c.ElectionTicks*int(c.TickMs)) * time.Millisecond } func (c *ServerConfig) PeerDialTimeout() time.Duration { // 1s for queue wait and election timeout return time.Second + time.Duration(c.ElectionTicks*int(c.TickMs))*time.Millisecond } func CheckDuplicateURL(urlsmap types.URLsMap) bool { um := make(map[string]bool) for _, urls := range urlsmap { for _, url := range urls { u := url.String() if um[u] { return true } um[u] = true } } return false } func (c *ServerConfig) BootstrapTimeoutEffective() time.Duration { if c.BootstrapTimeout != 0 { return c.BootstrapTimeout } return time.Second } func (c *ServerConfig) BackendPath() string { return datadir.ToBackendFileName(c.DataDir) } func (c *ServerConfig) MaxRequestBytesWithOverhead() uint { return c.MaxRequestBytes + grpcOverheadBytes } ================================================ FILE: server/config/config_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "net/url" "testing" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/types" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver/api/v3discovery" ) func mustNewURLs(t *testing.T, urls []string) []url.URL { if len(urls) == 0 { return nil } u, err := types.NewURLs(urls) require.NoErrorf(t, err, "error creating new URLs from %q: %v", urls, err) return u } func TestConfigVerifyBootstrapWithoutClusterFail(t *testing.T) { c := &ServerConfig{ Name: "node1", DiscoveryCfg: v3discovery.DiscoveryConfig{ ConfigSpec: clientv3.ConfigSpec{ Endpoints: []string{}, }, }, InitialPeerURLsMap: types.URLsMap{}, Logger: zaptest.NewLogger(t), } if err := c.VerifyBootstrap(); err == nil { t.Errorf("err = nil, want not nil") } } func TestConfigVerifyExistingWithDiscoveryURLFail(t *testing.T) { cluster, err := types.NewURLsMap("node1=http://127.0.0.1:2380") require.NoErrorf(t, err, "NewCluster error: %v", err) c := &ServerConfig{ Name: "node1", DiscoveryCfg: v3discovery.DiscoveryConfig{ ConfigSpec: clientv3.ConfigSpec{ Endpoints: []string{"http://192.168.0.100:2379"}, }, }, PeerURLs: mustNewURLs(t, []string{"http://127.0.0.1:2380"}), InitialPeerURLsMap: cluster, NewCluster: false, Logger: zaptest.NewLogger(t), } if err := c.VerifyJoinExisting(); err == nil { t.Errorf("err = nil, want not nil") } } func TestConfigVerifyLocalMember(t *testing.T) { tests := []struct { clusterSetting string apurls []string strict bool shouldError bool }{ { // Node must exist in cluster "", nil, true, true, }, { // Initial cluster set "node1=http://localhost:7001,node2=http://localhost:7002", []string{"http://localhost:7001"}, true, false, }, { // Default initial cluster "node1=http://localhost:2380,node1=http://localhost:7001", []string{"http://localhost:2380", "http://localhost:7001"}, true, false, }, { // Advertised peer URLs must match those in cluster-state "node1=http://localhost:7001", []string{"http://localhost:12345"}, true, true, }, { // Advertised peer URLs must match those in cluster-state "node1=http://localhost:2380,node1=http://localhost:12345", []string{"http://localhost:12345"}, true, true, }, { // Advertised peer URLs must match those in cluster-state "node1=http://localhost:12345", []string{"http://localhost:2380", "http://localhost:12345"}, true, true, }, { // Advertised peer URLs must match those in cluster-state "node1=http://localhost:2380", []string{}, true, true, }, { // do not care about the urls if strict is not set "node1=http://localhost:2380", []string{}, false, false, }, } for i, tt := range tests { cluster, err := types.NewURLsMap(tt.clusterSetting) require.NoErrorf(t, err, "#%d: Got unexpected error: %v", i, err) cfg := ServerConfig{ Name: "node1", InitialPeerURLsMap: cluster, Logger: zaptest.NewLogger(t), } if tt.apurls != nil { cfg.PeerURLs = mustNewURLs(t, tt.apurls) } if err = cfg.hasLocalMember(); err == nil && tt.strict { err = cfg.advertiseMatchesCluster() } if (err == nil) && tt.shouldError { t.Errorf("#%d: Got no error where one was expected", i) } if (err != nil) && !tt.shouldError { t.Errorf("#%d: Got unexpected error: %v", i, err) } } } func TestSnapDir(t *testing.T) { tests := map[string]string{ "/": "/member/snap", "/var/lib/etc": "/var/lib/etc/member/snap", } for dd, w := range tests { cfg := ServerConfig{ DataDir: dd, Logger: zaptest.NewLogger(t), } if g := cfg.SnapDir(); g != w { t.Errorf("DataDir=%q: SnapDir()=%q, want=%q", dd, g, w) } } } func TestWALDir(t *testing.T) { tests := map[string]string{ "/": "/member/wal", "/var/lib/etc": "/var/lib/etc/member/wal", } for dd, w := range tests { cfg := ServerConfig{ DataDir: dd, Logger: zaptest.NewLogger(t), } if g := cfg.WALDir(); g != w { t.Errorf("DataDir=%q: WALDir()=%q, want=%q", dd, g, w) } } } func TestShouldDiscover(t *testing.T) { tests := map[string]bool{ "": false, "foo": true, "http://discovery.etcd.io/asdf": true, } for durl, w := range tests { var eps []string if durl != "" { eps = append(eps, durl) } cfg := ServerConfig{ DiscoveryCfg: v3discovery.DiscoveryConfig{ ConfigSpec: clientv3.ConfigSpec{ Endpoints: eps, }, }, Logger: zaptest.NewLogger(t), } if g := cfg.ShouldDiscover(); g != w { t.Errorf("durl=%q: ShouldDiscover()=%t, want=%t", durl, g, w) } } } ================================================ FILE: server/config/v2_deprecation.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config type V2DeprecationEnum string const ( // V2Depr0NotYet means v2store isn't deprecated yet. // Default in v3.5, and no longer supported in v3.6. V2Depr0NotYet = V2DeprecationEnum("not-yet") // Deprecated: to be decommissioned in 3.7. Please use V2Depr0NotYet. // TODO: remove in 3.7 //revive:disable-next-line:var-naming V2_DEPR_0_NOT_YET = V2Depr0NotYet // V2Depr1WriteOnly means only writing v2store is allowed. // Default in v3.6. Meaningful v2 state is not allowed. // The V2 files are maintained for v3.5 rollback. V2Depr1WriteOnly = V2DeprecationEnum("write-only") // Deprecated: to be decommissioned in 3.7. Please use V2Depr1WriteOnly. // TODO: remove in 3.7 //revive:disable-next-line:var-naming V2_DEPR_1_WRITE_ONLY = V2Depr1WriteOnly // V2Depr1WriteOnlyDrop means v2store is WIPED if found !!! // Will be default in 3.7. V2Depr1WriteOnlyDrop = V2DeprecationEnum("write-only-drop-data") // Deprecated: to be decommissioned in 3.7. Pleae use V2Depr1WriteOnlyDrop. // TODO: remove in 3.7 //revive:disable-next-line:var-naming V2_DEPR_1_WRITE_ONLY_DROP = V2Depr1WriteOnlyDrop // V2Depr2Gone means v2store is completely gone. The v2store is // neither written nor read. Anything related to v2store will be // cleaned up in v3.8. Usage of this configuration is blocking // ability to rollback to etcd v3.5. V2Depr2Gone = V2DeprecationEnum("gone") // Deprecated: to be decommissioned in 3.7. Please use V2Depr2Gone. // TODO: remove in 3.7 //revive:disable-next-line:var-naming V2_DEPR_2_GONE = V2Depr2Gone // V2DeprDefault is the default deprecation level. V2DeprDefault = V2Depr1WriteOnly // Deprecated: to be decommissioned in 3.7. Please use V2DeprDefault. // TODO: remove in 3.7 //revive:disable-next-line:var-naming V2_DEPR_DEFAULT = V2DeprDefault ) func (e V2DeprecationEnum) IsAtLeast(v2d V2DeprecationEnum) bool { return e.level() >= v2d.level() } func (e V2DeprecationEnum) level() int { switch e { case V2Depr0NotYet: return 0 case V2Depr1WriteOnly: return 1 case V2Depr1WriteOnlyDrop: return 2 case V2Depr2Gone: return 3 } panic("Unknown V2DeprecationEnum: " + e) } ================================================ FILE: server/config/v2_deprecation_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import "testing" func TestV2DeprecationEnum_IsAtLeast(t *testing.T) { tests := []struct { e V2DeprecationEnum v2d V2DeprecationEnum want bool }{ {V2Depr0NotYet, V2Depr0NotYet, true}, {V2Depr0NotYet, V2Depr1WriteOnlyDrop, false}, {V2Depr0NotYet, V2Depr2Gone, false}, {V2Depr2Gone, V2Depr1WriteOnlyDrop, true}, {V2Depr2Gone, V2Depr0NotYet, true}, {V2Depr2Gone, V2Depr2Gone, true}, {V2Depr1WriteOnly, V2Depr1WriteOnlyDrop, false}, {V2Depr1WriteOnlyDrop, V2Depr1WriteOnly, true}, } for _, tt := range tests { t.Run(string(tt.e)+" >= "+string(tt.v2d), func(t *testing.T) { if got := tt.e.IsAtLeast(tt.v2d); got != tt.want { t.Errorf("IsAtLeast() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: server/embed/auth_test.go ================================================ // Copyright 2020 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package embed import ( "testing" "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" ) func TestEnableAuth(t *testing.T) { tdir := t.TempDir() cfg := NewConfig() cfg.Dir = tdir e, err := StartEtcd(cfg) if err != nil { t.Fatal(err) } defer e.Close() client := v3client.New(e.Server) defer client.Close() _, err = client.RoleAdd(t.Context(), "root") if err != nil { t.Fatal(err) } _, err = client.UserAdd(t.Context(), "root", "root") if err != nil { t.Fatal(err) } _, err = client.UserGrantRole(t.Context(), "root", "root") if err != nil { t.Fatal(err) } _, err = client.AuthEnable(t.Context()) if err != nil { t.Fatal(err) } } ================================================ FILE: server/embed/config.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package embed import ( "crypto/tls" "errors" "flag" "fmt" "math" "net" "net/http" "net/netip" "net/url" "os" "path/filepath" "strings" "sync" "time" "go.uber.org/zap" "golang.org/x/crypto/bcrypt" "google.golang.org/grpc" "sigs.k8s.io/yaml" bolt "go.etcd.io/bbolt" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/client/pkg/v3/srv" "go.etcd.io/etcd/client/pkg/v3/tlsutil" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/featuregate" "go.etcd.io/etcd/pkg/v3/flags" "go.etcd.io/etcd/pkg/v3/netutil" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/etcdserver/api/v3compactor" "go.etcd.io/etcd/server/v3/etcdserver/api/v3discovery" "go.etcd.io/etcd/server/v3/features" ) const ( ClusterStateFlagNew = "new" ClusterStateFlagExisting = "existing" DefaultName = "default" DefaultMaxSnapshots = 5 DefaultMaxWALs = 5 DefaultMaxTxnOps = uint(128) DefaultWarningApplyDuration = 100 * time.Millisecond DefaultWarningUnaryRequestDuration = 300 * time.Millisecond DefaultMaxRequestBytes = 1.5 * 1024 * 1024 DefaultMaxConcurrentStreams = math.MaxUint32 DefaultGRPCKeepAliveMinTime = 5 * time.Second DefaultGRPCKeepAliveInterval = 2 * time.Hour DefaultGRPCKeepAliveTimeout = 20 * time.Second DefaultDowngradeCheckTime = 5 * time.Second DefaultAutoCompactionMode = "periodic" DefaultAutoCompactionRetention = "0" DefaultAuthToken = "simple" DefaultCompactHashCheckTime = time.Minute DefaultLoggingFormat = "json" DefaultDiscoveryDialTimeout = 2 * time.Second DefaultDiscoveryRequestTimeOut = 5 * time.Second DefaultDiscoveryKeepAliveTime = 2 * time.Second DefaultDiscoveryKeepAliveTimeOut = 6 * time.Second DefaultDiscoveryInsecureTransport = true DefaultSelfSignedCertValidity = 1 DefaultTLSMinVersion = string(tlsutil.TLSVersion12) DefaultListenPeerURLs = "http://localhost:2380" DefaultListenClientURLs = "http://localhost:2379" DefaultLogOutput = "default" JournalLogOutput = "systemd/journal" StdErrLogOutput = "stderr" StdOutLogOutput = "stdout" // DefaultLogRotationConfig is the default configuration used for log rotation. // Log rotation is disabled by default. // MaxSize = 100 // MB // MaxAge = 0 // days (no limit) // MaxBackups = 0 // no limit // LocalTime = false // use computers local time, UTC by default // Compress = false // compress the rotated log in gzip format DefaultLogRotationConfig = `{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}` // DefaultDistributedTracingAddress is the default collector address. DefaultDistributedTracingAddress = "localhost:4317" // DefaultDistributedTracingServiceName is the default etcd service name. DefaultDistributedTracingServiceName = "etcd" // DefaultStrictReconfigCheck is the default value for "--strict-reconfig-check" flag. // It's enabled by default. DefaultStrictReconfigCheck = true // maxElectionMs specifies the maximum value of election timeout. // More details are listed on etcd.io/docs > version > tuning/#time-parameters maxElectionMs = 50000 // backend freelist map type freelistArrayType = "array" ServerFeatureGateFlagName = "feature-gates" ) var ( ErrConflictBootstrapFlags = fmt.Errorf("multiple discovery or bootstrap flags are set. " + "Choose one of \"initial-cluster\", \"discovery-endpoints\" or \"discovery-srv\"") ErrUnsetAdvertiseClientURLsFlag = fmt.Errorf("--advertise-client-urls is required when --listen-client-urls is set explicitly") ErrLogRotationInvalidLogOutput = fmt.Errorf("--log-outputs requires a single file path when --log-rotate-config-json is defined") DefaultInitialAdvertisePeerURLs = "http://localhost:2380" DefaultAdvertiseClientURLs = "http://localhost:2379" defaultHostname string defaultHostStatus error // indirection for testing getCluster = srv.GetCluster ) var ( // CompactorModePeriodic is periodic compaction mode // for "Config.AutoCompactionMode" field. // If "AutoCompactionMode" is CompactorModePeriodic and // "AutoCompactionRetention" is "1h", it automatically compacts // compacts storage every hour. CompactorModePeriodic = v3compactor.ModePeriodic // CompactorModeRevision is revision-based compaction mode // for "Config.AutoCompactionMode" field. // If "AutoCompactionMode" is CompactorModeRevision and // "AutoCompactionRetention" is "1000", it compacts log on // revision 5000 when the current revision is 6000. // This runs every 5-minute if enough of logs have proceeded. CompactorModeRevision = v3compactor.ModeRevision ) func init() { defaultHostname, defaultHostStatus = netutil.GetDefaultHost() } // Config holds the arguments for configuring an etcd server. type Config struct { Name string `json:"name"` Dir string `json:"data-dir"` //revive:disable-next-line:var-naming WalDir string `json:"wal-dir"` // SnapshotCount is the number of committed transactions that trigger a snapshot. SnapshotCount uint64 `json:"snapshot-count"` // SnapshotCatchUpEntries is the number of entires for a slow follower // to catch-up after compacting the raft storage entries. // We expect the follower has a millisecond level latency with the leader. // The max throughput is around 10K. Keep a 5K entries is enough for helping // follower to catch up. SnapshotCatchUpEntries uint64 `json:"snapshot-catchup-entries"` // MaxSnapFiles is the maximum number of snapshot files. // TODO: remove it in 3.8. // Deprecated: Will be removed in v3.8. MaxSnapFiles uint `json:"max-snapshots"` //revive:disable-next-line:var-naming MaxWalFiles uint `json:"max-wals"` // TickMs is the number of milliseconds between heartbeat ticks. // TODO: decouple tickMs and heartbeat tick (current heartbeat tick = 1). // make ticks a cluster wide configuration. TickMs uint `json:"heartbeat-interval"` ElectionMs uint `json:"election-timeout"` // InitialElectionTickAdvance is true, then local member fast-forwards // election ticks to speed up "initial" leader election trigger. This // benefits the case of larger election ticks. For instance, cross // datacenter deployment may require longer election timeout of 10-second. // If true, local node does not need wait up to 10-second. Instead, // forwards its election ticks to 8-second, and have only 2-second left // before leader election. // // Major assumptions are that: // - cluster has no active leader thus advancing ticks enables faster // leader election, or // - cluster already has an established leader, and rejoining follower // is likely to receive heartbeats from the leader after tick advance // and before election timeout. // // However, when network from leader to rejoining follower is congested, // and the follower does not receive leader heartbeat within left election // ticks, disruptive election has to happen thus affecting cluster // availabilities. // // Disabling this would slow down initial bootstrap process for cross // datacenter deployments. Make your own tradeoffs by configuring // --initial-election-tick-advance at the cost of slow initial bootstrap. // // If single-node, it advances ticks regardless. // // See https://github.com/etcd-io/etcd/issues/9333 for more detail. InitialElectionTickAdvance bool `json:"initial-election-tick-advance"` // BackendBatchInterval is the maximum time before commit the backend transaction. BackendBatchInterval time.Duration `json:"backend-batch-interval"` // BackendBatchLimit is the maximum operations before commit the backend transaction. BackendBatchLimit int `json:"backend-batch-limit"` // BackendFreelistType specifies the type of freelist that boltdb backend uses (array and map are supported types). BackendFreelistType string `json:"backend-bbolt-freelist-type"` QuotaBackendBytes int64 `json:"quota-backend-bytes"` MaxTxnOps uint `json:"max-txn-ops"` MaxRequestBytes uint `json:"max-request-bytes"` // MaxConcurrentStreams specifies the maximum number of concurrent // streams that each client can open at a time. MaxConcurrentStreams uint32 `json:"max-concurrent-streams"` //revive:disable:var-naming ListenPeerUrls, ListenClientUrls, ListenClientHttpUrls []url.URL AdvertisePeerUrls, AdvertiseClientUrls []url.URL //revive:enable:var-naming ClientTLSInfo transport.TLSInfo ClientAutoTLS bool PeerTLSInfo transport.TLSInfo PeerAutoTLS bool // SelfSignedCertValidity specifies the validity period of the client and peer certificates // that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS, // the unit is year, and the default is 1 SelfSignedCertValidity uint `json:"self-signed-cert-validity"` // CipherSuites is a list of supported TLS cipher suites between // client/server and peers. If empty, Go auto-populates the list. // Note that cipher suites are prioritized in the given order. CipherSuites []string `json:"cipher-suites"` // TlsMinVersion is the minimum accepted TLS version between client/server and peers. //revive:disable-next-line:var-naming TlsMinVersion string `json:"tls-min-version"` // TlsMaxVersion is the maximum accepted TLS version between client/server and peers. //revive:disable-next-line:var-naming TlsMaxVersion string `json:"tls-max-version"` ClusterState string `json:"initial-cluster-state"` DNSCluster string `json:"discovery-srv"` DNSClusterServiceName string `json:"discovery-srv-name"` DiscoveryCfg v3discovery.DiscoveryConfig `json:"discovery-config"` InitialCluster string `json:"initial-cluster"` InitialClusterToken string `json:"initial-cluster-token"` StrictReconfigCheck bool `json:"strict-reconfig-check"` // AutoCompactionMode is either 'periodic' or 'revision'. AutoCompactionMode string `json:"auto-compaction-mode"` // AutoCompactionRetention is either duration string with time unit // (e.g. '5m' for 5-minute), or revision unit (e.g. '5000'). // If no time unit is provided and compaction mode is 'periodic', // the unit defaults to hour. For example, '5' translates into 5-hour. AutoCompactionRetention string `json:"auto-compaction-retention"` // GRPCKeepAliveMinTime is the minimum interval that a client should // wait before pinging server. When client pings "too fast", server // sends goaway and closes the connection (errors: too_many_pings, // http2.ErrCodeEnhanceYourCalm). When too slow, nothing happens. // Server expects client pings only when there is any active streams // (PermitWithoutStream is set false). GRPCKeepAliveMinTime time.Duration `json:"grpc-keepalive-min-time"` // GRPCKeepAliveInterval is the frequency of server-to-client ping // to check if a connection is alive. Close a non-responsive connection // after an additional duration of Timeout. 0 to disable. GRPCKeepAliveInterval time.Duration `json:"grpc-keepalive-interval"` // GRPCKeepAliveTimeout is the additional duration of wait // before closing a non-responsive connection. 0 to disable. GRPCKeepAliveTimeout time.Duration `json:"grpc-keepalive-timeout"` // GRPCAdditionalServerOptions is the additional server option hook // for changing the default internal gRPC configuration. Note these // additional configurations take precedence over the existing individual // configurations if present. Please refer to // https://github.com/etcd-io/etcd/pull/14066#issuecomment-1248682996 GRPCAdditionalServerOptions []grpc.ServerOption `json:"grpc-additional-server-options"` // SocketOpts are socket options passed to listener config. SocketOpts transport.SocketOpts `json:"socket-options"` // PreVote is true to enable Raft Pre-Vote. // If enabled, Raft runs an additional election phase // to check whether it would get enough votes to win // an election, thus minimizing disruptions. PreVote bool `json:"pre-vote"` CORS map[string]struct{} // HostWhitelist lists acceptable hostnames from HTTP client requests. // Client origin policy protects against "DNS Rebinding" attacks // to insecure etcd servers. That is, any website can simply create // an authorized DNS name, and direct DNS to "localhost" (or any // other address). Then, all HTTP endpoints of etcd server listening // on "localhost" becomes accessible, thus vulnerable to DNS rebinding // attacks. See "CVE-2018-5702" for more detail. // // 1. If client connection is secure via HTTPS, allow any hostnames. // 2. If client connection is not secure and "HostWhitelist" is not empty, // only allow HTTP requests whose Host field is listed in whitelist. // // Note that the client origin policy is enforced whether authentication // is enabled or not, for tighter controls. // // By default, "HostWhitelist" is "*", which allows any hostnames. // Note that when specifying hostnames, loopback addresses are not added // automatically. To allow loopback interfaces, leave it empty or set it "*", // or add them to whitelist manually (e.g. "localhost", "127.0.0.1", etc.). // // CVE-2018-5702 reference: // - https://bugs.chromium.org/p/project-zero/issues/detail?id=1447#c2 // - https://github.com/transmission/transmission/pull/468 // - https://github.com/etcd-io/etcd/issues/9353 HostWhitelist map[string]struct{} // UserHandlers is for registering users handlers and only used for // embedding etcd into other applications. // The map key is the route path for the handler, and // you must ensure it can't be conflicted with etcd's. UserHandlers map[string]http.Handler `json:"-"` // ServiceRegister is for registering users' gRPC services. A simple usage example: // cfg := embed.NewConfig() // cfg.ServiceRegister = func(s *grpc.Server) { // pb.RegisterFooServer(s, &fooServer{}) // pb.RegisterBarServer(s, &barServer{}) // } // embed.StartEtcd(cfg) ServiceRegister func(*grpc.Server) `json:"-"` AuthToken string `json:"auth-token"` BcryptCost uint `json:"bcrypt-cost"` // AuthTokenTTL in seconds of the simple token AuthTokenTTL uint `json:"auth-token-ttl"` // CorruptCheckTime is the duration of time between cluster corruption check passes. CorruptCheckTime time.Duration `json:"corrupt-check-time"` // CompactHashCheckTime is the duration of time between leader checks followers compaction hashes. CompactHashCheckTime time.Duration `json:"compact-hash-check-time"` // CompactionBatchLimit Sets the maximum revisions deleted in each compaction batch. CompactionBatchLimit int `json:"compaction-batch-limit"` // CompactionSleepInterval is the sleep interval between every etcd compaction loop. CompactionSleepInterval time.Duration `json:"compaction-sleep-interval"` // WatchProgressNotifyInterval is the time duration of periodic watch progress notifications. WatchProgressNotifyInterval time.Duration `json:"watch-progress-notify-interval"` // WarningApplyDuration is the time duration after which a warning is generated if applying request WarningApplyDuration time.Duration `json:"warning-apply-duration"` // BootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to BootstrapDefragThresholdMegabytes uint `json:"bootstrap-defrag-threshold-megabytes"` // WarningUnaryRequestDuration is the time duration after which a warning is generated if applying // unary request takes more time than this value. WarningUnaryRequestDuration time.Duration `json:"warning-unary-request-duration"` // MaxLearners sets a limit to the number of learner members that can exist in the cluster membership. MaxLearners int `json:"max-learners"` // ForceNewCluster starts a new cluster even if previously started; unsafe. ForceNewCluster bool `json:"force-new-cluster"` EnablePprof bool `json:"enable-pprof"` Metrics string `json:"metrics"` ListenMetricsUrls []url.URL ListenMetricsUrlsJSON string `json:"listen-metrics-urls"` // EnableDistributedTracing indicates if tracing using OpenTelemetry is enabled. EnableDistributedTracing bool `json:"enable-distributed-tracing"` // DistributedTracingAddress is the address of the OpenTelemetry Collector. // Can only be set if EnableDistributedTracing is true. DistributedTracingAddress string `json:"distributed-tracing-address"` // DistributedTracingServiceName is the name of the service. // Can only be used if EnableDistributedTracing is true. DistributedTracingServiceName string `json:"distributed-tracing-service-name"` // DistributedTracingServiceInstanceID is the ID key of the service. // This ID must be unique, as helps to distinguish instances of the same service // that exist at the same time. // Can only be used if EnableDistributedTracing is true. DistributedTracingServiceInstanceID string `json:"distributed-tracing-instance-id"` // DistributedTracingSamplingRatePerMillion is the number of samples to collect per million spans. // Defaults to 0. DistributedTracingSamplingRatePerMillion int `json:"distributed-tracing-sampling-rate"` // Logger is logger options: currently only supports "zap". // "capnslog" is removed in v3.5. Logger string `json:"logger"` // LogLevel configures log level. Only supports debug, info, warn, error, panic, or fatal. Default 'info'. LogLevel string `json:"log-level"` // LogFormat set log encoding. Only supports json, console. Default is 'json'. LogFormat string `json:"log-format"` // LogOutputs is either: // - "default" as os.Stderr, // - "stderr" as os.Stderr, // - "stdout" as os.Stdout, // - file path to append server logs to. // It can be multiple when "Logger" is zap. LogOutputs []string `json:"log-outputs"` // EnableLogRotation enables log rotation of a single LogOutputs file target. EnableLogRotation bool `json:"enable-log-rotation"` // LogRotationConfigJSON is a passthrough allowing a log rotation JSON config to be passed directly. LogRotationConfigJSON string `json:"log-rotation-config-json"` // ZapLoggerBuilder is used to build the zap logger. ZapLoggerBuilder func(*Config) error // logger logs server-side operations. The default is nil, // and "setupLogging" must be called before starting server. // Do not set logger directly. loggerMu *sync.RWMutex logger *zap.Logger // EnableGRPCGateway enables grpc gateway. // The gateway translates a RESTful HTTP API into gRPC. EnableGRPCGateway bool `json:"enable-grpc-gateway"` // UnsafeNoFsync disables all uses of fsync. // Setting this is unsafe and will cause data loss. UnsafeNoFsync bool `json:"unsafe-no-fsync"` // DowngradeCheckTime is the duration between two downgrade status checks (in seconds). DowngradeCheckTime time.Duration `json:"downgrade-check-time"` // MemoryMlock enables mlocking of etcd owned memory pages. // The setting improves etcd tail latency in environments were: // - memory pressure might lead to swapping pages to disk // - disk latency might be unstable // Currently all etcd memory gets mlocked, but in future the flag can // be refined to mlock in-use area of bbolt only. MemoryMlock bool `json:"memory-mlock"` // V2Deprecation describes phase of API & Storage V2 support. // Do not set this field for embedded use cases, as it has no effect. However, setting it will not cause any harm. // TODO: Delete in v3.8 // Deprecated: The default value is enforced, to be removed in v3.8. V2Deprecation config.V2DeprecationEnum `json:"v2-deprecation"` // ServerFeatureGate is a server level feature gate ServerFeatureGate featuregate.FeatureGate // FlagsExplicitlySet stores if a flag is explicitly set from the cmd line or config file. FlagsExplicitlySet map[string]bool } // configYAML holds the config suitable for yaml parsing type configYAML struct { Config configJSON } // configJSON has file options that are translated into Config options type configJSON struct { ListenPeerURLs string `json:"listen-peer-urls"` ListenClientURLs string `json:"listen-client-urls"` ListenClientHTTPURLs string `json:"listen-client-http-urls"` AdvertisePeerURLs string `json:"initial-advertise-peer-urls"` AdvertiseClientURLs string `json:"advertise-client-urls"` CORSJSON string `json:"cors"` HostWhitelistJSON string `json:"host-whitelist"` ClientSecurityJSON securityConfig `json:"client-transport-security"` PeerSecurityJSON securityConfig `json:"peer-transport-security"` ServerFeatureGatesJSON string `json:"feature-gates"` } type securityConfig struct { CertFile string `json:"cert-file"` KeyFile string `json:"key-file"` ClientCertFile string `json:"client-cert-file"` ClientKeyFile string `json:"client-key-file"` CertAuth bool `json:"client-cert-auth"` TrustedCAFile string `json:"trusted-ca-file"` AutoTLS bool `json:"auto-tls"` AllowedCNs []string `json:"allowed-cn"` AllowedHostnames []string `json:"allowed-hostname"` SkipClientSANVerify bool `json:"skip-client-san-verification,omitempty"` } // NewConfig creates a new Config populated with default values. func NewConfig() *Config { lpurl, _ := url.Parse(DefaultListenPeerURLs) apurl, _ := url.Parse(DefaultInitialAdvertisePeerURLs) lcurl, _ := url.Parse(DefaultListenClientURLs) acurl, _ := url.Parse(DefaultAdvertiseClientURLs) cfg := &Config{ MaxSnapFiles: DefaultMaxSnapshots, MaxWalFiles: DefaultMaxWALs, Name: DefaultName, SnapshotCount: etcdserver.DefaultSnapshotCount, SnapshotCatchUpEntries: etcdserver.DefaultSnapshotCatchUpEntries, MaxTxnOps: DefaultMaxTxnOps, MaxRequestBytes: DefaultMaxRequestBytes, MaxConcurrentStreams: DefaultMaxConcurrentStreams, WarningApplyDuration: DefaultWarningApplyDuration, GRPCKeepAliveMinTime: DefaultGRPCKeepAliveMinTime, GRPCKeepAliveInterval: DefaultGRPCKeepAliveInterval, GRPCKeepAliveTimeout: DefaultGRPCKeepAliveTimeout, SocketOpts: transport.SocketOpts{ ReusePort: false, ReuseAddress: false, }, TickMs: 100, ElectionMs: 1000, InitialElectionTickAdvance: true, ListenPeerUrls: []url.URL{*lpurl}, ListenClientUrls: []url.URL{*lcurl}, AdvertisePeerUrls: []url.URL{*apurl}, AdvertiseClientUrls: []url.URL{*acurl}, ClusterState: ClusterStateFlagNew, InitialClusterToken: "etcd-cluster", StrictReconfigCheck: DefaultStrictReconfigCheck, Metrics: "basic", CORS: map[string]struct{}{"*": {}}, HostWhitelist: map[string]struct{}{"*": {}}, AuthToken: DefaultAuthToken, BcryptCost: uint(bcrypt.DefaultCost), AuthTokenTTL: 300, SelfSignedCertValidity: DefaultSelfSignedCertValidity, TlsMinVersion: DefaultTLSMinVersion, PreVote: true, loggerMu: new(sync.RWMutex), logger: nil, Logger: "zap", LogFormat: DefaultLoggingFormat, LogOutputs: []string{DefaultLogOutput}, LogLevel: logutil.DefaultLogLevel, EnableLogRotation: false, LogRotationConfigJSON: DefaultLogRotationConfig, EnableGRPCGateway: true, DowngradeCheckTime: DefaultDowngradeCheckTime, MemoryMlock: false, MaxLearners: membership.DefaultMaxLearners, DistributedTracingAddress: DefaultDistributedTracingAddress, DistributedTracingServiceName: DefaultDistributedTracingServiceName, CompactHashCheckTime: DefaultCompactHashCheckTime, V2Deprecation: config.V2DeprDefault, DiscoveryCfg: v3discovery.DiscoveryConfig{ ConfigSpec: clientv3.ConfigSpec{ DialTimeout: DefaultDiscoveryDialTimeout, RequestTimeout: DefaultDiscoveryRequestTimeOut, KeepAliveTime: DefaultDiscoveryKeepAliveTime, KeepAliveTimeout: DefaultDiscoveryKeepAliveTimeOut, Secure: &clientv3.SecureConfig{ InsecureTransport: true, }, Auth: &clientv3.AuthConfig{}, }, }, AutoCompactionMode: DefaultAutoCompactionMode, AutoCompactionRetention: DefaultAutoCompactionRetention, ServerFeatureGate: features.NewDefaultServerFeatureGate(DefaultName, nil), FlagsExplicitlySet: map[string]bool{}, } cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) return cfg } func (cfg *Config) AddFlags(fs *flag.FlagSet) { // member fs.StringVar(&cfg.Dir, "data-dir", cfg.Dir, "Path to the data directory.") fs.StringVar(&cfg.WalDir, "wal-dir", cfg.WalDir, "Path to the dedicated wal directory.") fs.Var( flags.NewUniqueURLsWithExceptions(DefaultListenPeerURLs, ""), "listen-peer-urls", "List of URLs to listen on for peer traffic.", ) fs.Var( flags.NewUniqueURLsWithExceptions(DefaultListenClientURLs, ""), "listen-client-urls", "List of URLs to listen on for client grpc traffic and http as long as --listen-client-http-urls is not specified.", ) fs.Var( flags.NewUniqueURLsWithExceptions("", ""), "listen-client-http-urls", "List of URLs to listen on for http only client traffic. Enabling this flag removes http services from --listen-client-urls.", ) fs.Var( flags.NewUniqueURLsWithExceptions("", ""), "listen-metrics-urls", "List of URLs to listen on for the metrics and health endpoints.", ) fs.UintVar(&cfg.MaxSnapFiles, "max-snapshots", cfg.MaxSnapFiles, "Maximum number of snapshot files to retain (0 is unlimited). Deprecated in v3.6 and will be decommissioned in v3.8.") fs.UintVar(&cfg.MaxWalFiles, "max-wals", cfg.MaxWalFiles, "Maximum number of wal files to retain (0 is unlimited).") fs.StringVar(&cfg.Name, "name", cfg.Name, "Human-readable name for this member.") fs.Uint64Var(&cfg.SnapshotCount, "snapshot-count", cfg.SnapshotCount, "Number of committed transactions to trigger a snapshot.") fs.UintVar(&cfg.TickMs, "heartbeat-interval", cfg.TickMs, "Time (in milliseconds) of a heartbeat interval.") fs.UintVar(&cfg.ElectionMs, "election-timeout", cfg.ElectionMs, "Time (in milliseconds) for an election to timeout.") fs.BoolVar(&cfg.InitialElectionTickAdvance, "initial-election-tick-advance", cfg.InitialElectionTickAdvance, "Whether to fast-forward initial election ticks on boot for faster election.") fs.Int64Var(&cfg.QuotaBackendBytes, "quota-backend-bytes", cfg.QuotaBackendBytes, "Sets the maximum size (in bytes) that the etcd backend database may consume. Exceeding this triggers an alarm and puts etcd in read-only mode. Set to 0 to use the default 2GiB limit.") fs.StringVar(&cfg.BackendFreelistType, "backend-bbolt-freelist-type", cfg.BackendFreelistType, "BackendFreelistType specifies the type of freelist that boltdb backend uses(array and map are supported types)") fs.DurationVar(&cfg.BackendBatchInterval, "backend-batch-interval", cfg.BackendBatchInterval, "BackendBatchInterval is the maximum time before commit the backend transaction.") fs.IntVar(&cfg.BackendBatchLimit, "backend-batch-limit", cfg.BackendBatchLimit, "BackendBatchLimit is the maximum operations before commit the backend transaction.") fs.UintVar(&cfg.MaxTxnOps, "max-txn-ops", cfg.MaxTxnOps, "Maximum number of operations permitted in a transaction.") fs.UintVar(&cfg.MaxRequestBytes, "max-request-bytes", cfg.MaxRequestBytes, "Maximum client request size in bytes the server will accept.") fs.DurationVar(&cfg.GRPCKeepAliveMinTime, "grpc-keepalive-min-time", cfg.GRPCKeepAliveMinTime, "Minimum interval duration that a client should wait before pinging server.") fs.DurationVar(&cfg.GRPCKeepAliveInterval, "grpc-keepalive-interval", cfg.GRPCKeepAliveInterval, "Frequency duration of server-to-client ping to check if a connection is alive (0 to disable).") fs.DurationVar(&cfg.GRPCKeepAliveTimeout, "grpc-keepalive-timeout", cfg.GRPCKeepAliveTimeout, "Additional duration of wait before closing a non-responsive connection (0 to disable).") fs.BoolVar(&cfg.SocketOpts.ReusePort, "socket-reuse-port", cfg.SocketOpts.ReusePort, "Enable to set socket option SO_REUSEPORT on listeners allowing rebinding of a port already in use.") fs.BoolVar(&cfg.SocketOpts.ReuseAddress, "socket-reuse-address", cfg.SocketOpts.ReuseAddress, "Enable to set socket option SO_REUSEADDR on listeners allowing binding to an address in `TIME_WAIT` state.") fs.Var(flags.NewUint32Value(cfg.MaxConcurrentStreams), "max-concurrent-streams", "Maximum concurrent streams that each client can open at a time.") // raft connection timeouts fs.DurationVar(&rafthttp.ConnReadTimeout, "raft-read-timeout", rafthttp.DefaultConnReadTimeout, "Read timeout set on each rafthttp connection") fs.DurationVar(&rafthttp.ConnWriteTimeout, "raft-write-timeout", rafthttp.DefaultConnWriteTimeout, "Write timeout set on each rafthttp connection") // clustering fs.Var( flags.NewUniqueURLsWithExceptions(DefaultInitialAdvertisePeerURLs, ""), "initial-advertise-peer-urls", "List of this member's peer URLs to advertise to the rest of the cluster.", ) fs.Var( flags.NewUniqueURLsWithExceptions(DefaultAdvertiseClientURLs, ""), "advertise-client-urls", "List of this member's client URLs to advertise to the public.", ) fs.Var( flags.NewUniqueStringsValue(""), "discovery-endpoints", "V3 discovery: List of gRPC endpoints of the discovery service.", ) fs.StringVar(&cfg.DiscoveryCfg.Token, "discovery-token", "", "V3 discovery: discovery token for the etcd cluster to be bootstrapped.") fs.DurationVar(&cfg.DiscoveryCfg.DialTimeout, "discovery-dial-timeout", cfg.DiscoveryCfg.DialTimeout, "V3 discovery: dial timeout for client connections.") fs.DurationVar(&cfg.DiscoveryCfg.RequestTimeout, "discovery-request-timeout", cfg.DiscoveryCfg.RequestTimeout, "V3 discovery: timeout for discovery requests (excluding dial timeout).") fs.DurationVar(&cfg.DiscoveryCfg.KeepAliveTime, "discovery-keepalive-time", cfg.DiscoveryCfg.KeepAliveTime, "V3 discovery: keepalive time for client connections.") fs.DurationVar(&cfg.DiscoveryCfg.KeepAliveTimeout, "discovery-keepalive-timeout", cfg.DiscoveryCfg.KeepAliveTimeout, "V3 discovery: keepalive timeout for client connections.") fs.BoolVar(&cfg.DiscoveryCfg.Secure.InsecureTransport, "discovery-insecure-transport", true, "V3 discovery: disable transport security for client connections.") fs.BoolVar(&cfg.DiscoveryCfg.Secure.InsecureSkipVerify, "discovery-insecure-skip-tls-verify", false, "V3 discovery: skip server certificate verification (CAUTION: this option should be enabled only for testing purposes).") fs.StringVar(&cfg.DiscoveryCfg.Secure.Cert, "discovery-cert", "", "V3 discovery: identify secure client using this TLS certificate file.") fs.StringVar(&cfg.DiscoveryCfg.Secure.Key, "discovery-key", "", "V3 discovery: identify secure client using this TLS key file.") fs.StringVar(&cfg.DiscoveryCfg.Secure.Cacert, "discovery-cacert", "", "V3 discovery: verify certificates of TLS-enabled secure servers using this CA bundle.") fs.StringVar(&cfg.DiscoveryCfg.Auth.Username, "discovery-user", "", "V3 discovery: username[:password] for authentication (prompt if password is not supplied).") fs.StringVar(&cfg.DiscoveryCfg.Auth.Password, "discovery-password", "", "V3 discovery: password for authentication (if this option is used, --user option shouldn't include password).") fs.StringVar(&cfg.DNSCluster, "discovery-srv", cfg.DNSCluster, "DNS domain used to bootstrap initial cluster.") fs.StringVar(&cfg.DNSClusterServiceName, "discovery-srv-name", cfg.DNSClusterServiceName, "Service name to query when using DNS discovery.") fs.StringVar(&cfg.InitialCluster, "initial-cluster", cfg.InitialCluster, "Initial cluster configuration for bootstrapping.") fs.StringVar(&cfg.InitialClusterToken, "initial-cluster-token", cfg.InitialClusterToken, "Initial cluster token for the etcd cluster during bootstrap.") fs.BoolVar(&cfg.StrictReconfigCheck, "strict-reconfig-check", cfg.StrictReconfigCheck, "Reject reconfiguration requests that would cause quorum loss.") fs.BoolVar(&cfg.PreVote, "pre-vote", cfg.PreVote, "Enable the raft Pre-Vote algorithm to prevent disruption when a node that has been partitioned away rejoins the cluster.") // security fs.StringVar(&cfg.ClientTLSInfo.CertFile, "cert-file", "", "Path to the client server TLS cert file.") fs.StringVar(&cfg.ClientTLSInfo.KeyFile, "key-file", "", "Path to the client server TLS key file.") fs.StringVar(&cfg.ClientTLSInfo.ClientCertFile, "client-cert-file", "", "Path to an explicit peer client TLS cert file otherwise cert file will be used when client auth is required.") fs.StringVar(&cfg.ClientTLSInfo.ClientKeyFile, "client-key-file", "", "Path to an explicit peer client TLS key file otherwise key file will be used when client auth is required.") fs.BoolVar(&cfg.ClientTLSInfo.ClientCertAuth, "client-cert-auth", false, "Enable client cert authentication.") fs.StringVar(&cfg.ClientTLSInfo.CRLFile, "client-crl-file", "", "Path to the client certificate revocation list file.") fs.Var(flags.NewStringsValue(""), "client-cert-allowed-hostname", "Comma-separated list of allowed SAN hostnames for client cert authentication.") fs.StringVar(&cfg.ClientTLSInfo.TrustedCAFile, "trusted-ca-file", "", "Path to the client server TLS trusted CA cert file.") fs.BoolVar(&cfg.ClientAutoTLS, "auto-tls", false, "Client TLS using generated certificates") fs.StringVar(&cfg.PeerTLSInfo.CertFile, "peer-cert-file", "", "Path to the peer server TLS cert file.") fs.StringVar(&cfg.PeerTLSInfo.KeyFile, "peer-key-file", "", "Path to the peer server TLS key file.") fs.StringVar(&cfg.PeerTLSInfo.ClientCertFile, "peer-client-cert-file", "", "Path to an explicit peer client TLS cert file otherwise peer cert file will be used when client auth is required.") fs.StringVar(&cfg.PeerTLSInfo.ClientKeyFile, "peer-client-key-file", "", "Path to an explicit peer client TLS key file otherwise peer key file will be used when client auth is required.") fs.BoolVar(&cfg.PeerTLSInfo.ClientCertAuth, "peer-client-cert-auth", false, "Enable peer client cert authentication.") fs.StringVar(&cfg.PeerTLSInfo.TrustedCAFile, "peer-trusted-ca-file", "", "Path to the peer server TLS trusted CA file.") fs.BoolVar(&cfg.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates") fs.UintVar(&cfg.SelfSignedCertValidity, "self-signed-cert-validity", 1, "The validity period of the client and peer certificates, unit is year") fs.StringVar(&cfg.PeerTLSInfo.CRLFile, "peer-crl-file", "", "Path to the peer certificate revocation list file.") fs.Var(flags.NewStringsValue(""), "peer-cert-allowed-cn", "Comma-separated list of allowed CNs for inter-peer TLS authentication.") fs.Var(flags.NewStringsValue(""), "peer-cert-allowed-hostname", "Comma-separated list of allowed SAN hostnames for inter-peer TLS authentication.") fs.Var(flags.NewStringsValue(""), "cipher-suites", "Comma-separated list of supported TLS cipher suites between client/server and peers (empty will be auto-populated by Go).") fs.BoolVar(&cfg.PeerTLSInfo.SkipClientSANVerify, "peer-skip-client-san-verification", false, "Skip verification of SAN field in client certificate for peer connections.") fs.StringVar(&cfg.TlsMinVersion, "tls-min-version", string(tlsutil.TLSVersion12), "Minimum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3.") fs.StringVar(&cfg.TlsMaxVersion, "tls-max-version", string(tlsutil.TLSVersionDefault), "Maximum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3 (empty defers to Go).") fs.Var( flags.NewUniqueURLsWithExceptions("*", "*"), "cors", "Comma-separated white list of origins for CORS, or cross-origin resource sharing, (empty or * means allow all)", ) fs.Var(flags.NewUniqueStringsValue("*"), "host-whitelist", "Comma-separated acceptable hostnames from HTTP client requests, if server is not secure (empty means allow all).") // logging fs.StringVar(&cfg.Logger, "logger", "zap", "Currently only supports 'zap' for structured logging.") fs.Var(flags.NewUniqueStringsValue(DefaultLogOutput), "log-outputs", "Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd, or list of comma separated output targets.") fs.StringVar(&cfg.LogLevel, "log-level", logutil.DefaultLogLevel, "Configures log level. Only supports debug, info, warn, error, panic, or fatal. Default 'info'.") fs.StringVar(&cfg.LogFormat, "log-format", logutil.DefaultLogFormat, "Configures log format. Only supports json, console. Default is 'json'.") fs.BoolVar(&cfg.EnableLogRotation, "enable-log-rotation", false, "Enable log rotation of a single log-outputs file target.") fs.StringVar(&cfg.LogRotationConfigJSON, "log-rotation-config-json", DefaultLogRotationConfig, "Configures log rotation if enabled with a JSON logger config. Default: MaxSize=100(MB), MaxAge=0(days,no limit), MaxBackups=0(no limit), LocalTime=false(UTC), Compress=false(gzip)") fs.StringVar(&cfg.AutoCompactionRetention, "auto-compaction-retention", "0", "Auto compaction retention for mvcc key value store. 0 means disable auto compaction.") fs.StringVar(&cfg.AutoCompactionMode, "auto-compaction-mode", "periodic", "interpret 'auto-compaction-retention' one of: periodic|revision. 'periodic' for duration based retention, defaulting to hours if no time unit is provided (e.g. '5m'). 'revision' for revision number based retention.") // pprof profiler via HTTP fs.BoolVar(&cfg.EnablePprof, "enable-pprof", false, "Enable runtime profiling data via HTTP server. Address is at client URL + \"/debug/pprof/\"") // additional metrics fs.StringVar(&cfg.Metrics, "metrics", cfg.Metrics, "Set level of detail for exported metrics, specify 'extensive' to include server side grpc histogram metrics") fs.BoolVar(&cfg.EnableDistributedTracing, "enable-distributed-tracing", false, "Enable distributed tracing using OpenTelemetry Tracing.") fs.StringVar(&cfg.DistributedTracingAddress, "distributed-tracing-address", cfg.DistributedTracingAddress, "Address for distributed tracing used for OpenTelemetry Tracing (if enabled with enable-distributed-tracing flag).") fs.StringVar(&cfg.DistributedTracingServiceName, "distributed-tracing-service-name", cfg.DistributedTracingServiceName, "Configures service name for distributed tracing to be used to define service name for OpenTelemetry Tracing (if enabled with enable-distributed-tracing flag). 'etcd' is the default service name. Use the same service name for all instances of etcd.") fs.StringVar(&cfg.DistributedTracingServiceInstanceID, "distributed-tracing-instance-id", "", "Configures service instance ID for distributed tracing to be used to define service instance ID key for OpenTelemetry Tracing (if enabled with enable-distributed-tracing flag). There is no default value set. This ID must be unique per etcd instance.") fs.IntVar(&cfg.DistributedTracingSamplingRatePerMillion, "distributed-tracing-sampling-rate", 0, "Number of samples to collect per million spans for OpenTelemetry Tracing (if enabled with enable-distributed-tracing flag).") // auth fs.StringVar(&cfg.AuthToken, "auth-token", cfg.AuthToken, "Specify auth token specific options.") fs.UintVar(&cfg.BcryptCost, "bcrypt-cost", cfg.BcryptCost, "Specify bcrypt algorithm cost factor for auth password hashing.") fs.UintVar(&cfg.AuthTokenTTL, "auth-token-ttl", cfg.AuthTokenTTL, "The lifetime in seconds of the auth token.") // gateway fs.BoolVar(&cfg.EnableGRPCGateway, "enable-grpc-gateway", cfg.EnableGRPCGateway, "Enable GRPC gateway.") fs.DurationVar(&cfg.CorruptCheckTime, "corrupt-check-time", cfg.CorruptCheckTime, "Duration of time between cluster corruption check passes.") fs.DurationVar(&cfg.CompactHashCheckTime, "compact-hash-check-time", cfg.CompactHashCheckTime, "Duration of time between leader checks followers compaction hashes.") fs.IntVar(&cfg.CompactionBatchLimit, "compaction-batch-limit", cfg.CompactionBatchLimit, "Sets the maximum revisions deleted in each compaction batch.") fs.DurationVar(&cfg.CompactionSleepInterval, "compaction-sleep-interval", cfg.CompactionSleepInterval, "Sets the sleep interval between each compaction batch.") fs.DurationVar(&cfg.WatchProgressNotifyInterval, "watch-progress-notify-interval", cfg.WatchProgressNotifyInterval, "Duration of periodic watch progress notifications.") fs.DurationVar(&cfg.DowngradeCheckTime, "downgrade-check-time", cfg.DowngradeCheckTime, "Duration of time between two downgrade status checks.") fs.DurationVar(&cfg.WarningApplyDuration, "warning-apply-duration", cfg.WarningApplyDuration, "Time duration after which a warning is generated if watch progress takes more time.") fs.DurationVar(&cfg.WarningUnaryRequestDuration, "warning-unary-request-duration", cfg.WarningUnaryRequestDuration, "Time duration after which a warning is generated if a unary request takes more time.") fs.BoolVar(&cfg.MemoryMlock, "memory-mlock", cfg.MemoryMlock, "Enable to enforce etcd pages (in particular bbolt) to stay in RAM.") fs.UintVar(&cfg.BootstrapDefragThresholdMegabytes, "bootstrap-defrag-threshold-megabytes", 0, "Enable the defrag during etcd server bootstrap on condition that it will free at least the provided threshold of disk space. Needs to be set to non-zero value to take effect.") fs.IntVar(&cfg.MaxLearners, "max-learners", membership.DefaultMaxLearners, "Sets the maximum number of learners that can be available in the cluster membership.") fs.Uint64Var(&cfg.SnapshotCatchUpEntries, "snapshot-catchup-entries", cfg.SnapshotCatchUpEntries, "Number of entries for a slow follower to catch up after compacting the raft storage entries.") // unsafe fs.BoolVar(&cfg.UnsafeNoFsync, "unsafe-no-fsync", false, "Disables fsync, unsafe, will cause data loss.") fs.BoolVar(&cfg.ForceNewCluster, "force-new-cluster", false, "Force to create a new one member cluster.") // featuregate cfg.ServerFeatureGate.(featuregate.MutableFeatureGate).AddFlag(fs, ServerFeatureGateFlagName) } func ConfigFromFile(path string) (*Config, error) { cfg := &configYAML{Config: *NewConfig()} if err := cfg.configFromFile(path); err != nil { return nil, err } return &cfg.Config, nil } func (cfg *configYAML) configFromFile(path string) error { b, err := os.ReadFile(path) if err != nil { return err } defaultInitialCluster := cfg.InitialCluster err = yaml.Unmarshal(b, cfg) if err != nil { return err } if cfg.configJSON.ServerFeatureGatesJSON != "" { err = cfg.Config.ServerFeatureGate.(featuregate.MutableFeatureGate).Set(cfg.configJSON.ServerFeatureGatesJSON) if err != nil { return err } } // parses the yaml bytes to raw map first, then getBoolFlagVal can get the top level bool flag value. var cfgMap map[string]any err = yaml.Unmarshal(b, &cfgMap) if err != nil { return err } for flg := range cfgMap { cfg.FlagsExplicitlySet[flg] = true } if peerTransportSecurity, ok := cfgMap["peer-transport-security"]; ok { peerTransportSecurityMap, isMap := peerTransportSecurity.(map[string]any) if !isMap { return fmt.Errorf("invalid peer-transport-security") } for k := range peerTransportSecurityMap { cfg.FlagsExplicitlySet[fmt.Sprintf("peer-%s", k)] = true } } if cfg.configJSON.ListenPeerURLs != "" { u, err := types.NewURLs(strings.Split(cfg.configJSON.ListenPeerURLs, ",")) if err != nil { fmt.Fprintf(os.Stderr, "unexpected error setting up listen-peer-urls: %v\n", err) os.Exit(1) } cfg.Config.ListenPeerUrls = u } if cfg.configJSON.ListenClientURLs != "" { u, err := types.NewURLs(strings.Split(cfg.configJSON.ListenClientURLs, ",")) if err != nil { fmt.Fprintf(os.Stderr, "unexpected error setting up listen-client-urls: %v\n", err) os.Exit(1) } cfg.Config.ListenClientUrls = u } if cfg.configJSON.ListenClientHTTPURLs != "" { u, err := types.NewURLs(strings.Split(cfg.configJSON.ListenClientHTTPURLs, ",")) if err != nil { fmt.Fprintf(os.Stderr, "unexpected error setting up listen-client-http-urls: %v\n", err) os.Exit(1) } cfg.Config.ListenClientHttpUrls = u } if cfg.configJSON.AdvertisePeerURLs != "" { u, err := types.NewURLs(strings.Split(cfg.configJSON.AdvertisePeerURLs, ",")) if err != nil { fmt.Fprintf(os.Stderr, "unexpected error setting up initial-advertise-peer-urls: %v\n", err) os.Exit(1) } cfg.Config.AdvertisePeerUrls = u } if cfg.configJSON.AdvertiseClientURLs != "" { u, err := types.NewURLs(strings.Split(cfg.configJSON.AdvertiseClientURLs, ",")) if err != nil { fmt.Fprintf(os.Stderr, "unexpected error setting up advertise-peer-urls: %v\n", err) os.Exit(1) } cfg.Config.AdvertiseClientUrls = u } if cfg.ListenMetricsUrlsJSON != "" { u, err := types.NewURLs(strings.Split(cfg.ListenMetricsUrlsJSON, ",")) if err != nil { fmt.Fprintf(os.Stderr, "unexpected error setting up listen-metrics-urls: %v\n", err) os.Exit(1) } cfg.ListenMetricsUrls = u } if cfg.CORSJSON != "" { uv := flags.NewUniqueURLsWithExceptions(cfg.CORSJSON, "*") cfg.CORS = uv.Values } if cfg.HostWhitelistJSON != "" { uv := flags.NewUniqueStringsValue(cfg.HostWhitelistJSON) cfg.HostWhitelist = uv.Values } // If a discovery or discovery-endpoints flag is set, clear default initial cluster set by InitialClusterFromName if (cfg.DNSCluster != "" || len(cfg.DiscoveryCfg.Endpoints) > 0) && cfg.InitialCluster == defaultInitialCluster { cfg.InitialCluster = "" } if cfg.ClusterState == "" { cfg.ClusterState = ClusterStateFlagNew } copySecurityDetails := func(tls *transport.TLSInfo, ysc *securityConfig) { tls.CertFile = ysc.CertFile tls.KeyFile = ysc.KeyFile tls.ClientCertFile = ysc.ClientCertFile tls.ClientKeyFile = ysc.ClientKeyFile tls.ClientCertAuth = ysc.CertAuth tls.TrustedCAFile = ysc.TrustedCAFile tls.AllowedCNs = ysc.AllowedCNs tls.AllowedHostnames = ysc.AllowedHostnames tls.SkipClientSANVerify = ysc.SkipClientSANVerify } copySecurityDetails(&cfg.ClientTLSInfo, &cfg.ClientSecurityJSON) copySecurityDetails(&cfg.PeerTLSInfo, &cfg.PeerSecurityJSON) cfg.ClientAutoTLS = cfg.ClientSecurityJSON.AutoTLS cfg.PeerAutoTLS = cfg.PeerSecurityJSON.AutoTLS if cfg.SelfSignedCertValidity == 0 { cfg.SelfSignedCertValidity = 1 } return cfg.Validate() } func updateCipherSuites(tls *transport.TLSInfo, ss []string) error { if len(tls.CipherSuites) > 0 && len(ss) > 0 { return fmt.Errorf("TLSInfo.CipherSuites is already specified (given %v)", ss) } if len(ss) > 0 { cs, err := tlsutil.GetCipherSuites(ss) if err != nil { return err } tls.CipherSuites = cs } return nil } func updateMinMaxVersions(info *transport.TLSInfo, min, max string) { // Validate() has been called to check the user input, so it should never fail. var err error if info.MinVersion, err = tlsutil.GetTLSVersion(min); err != nil { panic(err) } if info.MaxVersion, err = tlsutil.GetTLSVersion(max); err != nil { panic(err) } } // Validate ensures that '*embed.Config' fields are properly configured. func (cfg *Config) Validate() error { if err := cfg.setupLogging(); err != nil { return err } if err := checkBindURLs(cfg.ListenPeerUrls); err != nil { return err } if err := checkBindURLs(cfg.ListenClientUrls); err != nil { return err } if err := checkBindURLs(cfg.ListenClientHttpUrls); err != nil { return err } if len(cfg.ListenClientHttpUrls) == 0 { cfg.logger.Warn("Running http and grpc server on single port. This is not recommended for production.") } if err := checkBindURLs(cfg.ListenMetricsUrls); err != nil { return err } if err := checkHostURLs(cfg.AdvertisePeerUrls); err != nil { addrs := cfg.getAdvertisePeerURLs() return fmt.Errorf(`--initial-advertise-peer-urls %q must be "host:port" (%w)`, strings.Join(addrs, ","), err) } if err := checkHostURLs(cfg.AdvertiseClientUrls); err != nil { addrs := cfg.getAdvertiseClientURLs() return fmt.Errorf(`--advertise-client-urls %q must be in the format "host:port", "unix:/path/to/socket" or "unixs:/path/to/socket" (%w)`, strings.Join(addrs, ","), err) } // Check if conflicting flags are passed. nSet := 0 for _, v := range []bool{cfg.InitialCluster != "", cfg.DNSCluster != "", len(cfg.DiscoveryCfg.Endpoints) > 0} { if v { nSet++ } } if cfg.ClusterState != ClusterStateFlagNew && cfg.ClusterState != ClusterStateFlagExisting { return fmt.Errorf("unexpected clusterState %q", cfg.ClusterState) } if nSet > 1 { return ErrConflictBootstrapFlags } // If one of `discovery-token` and `discovery-endpoints` is provided, // then the other one must be provided as well. if (cfg.DiscoveryCfg.Token != "") != (len(cfg.DiscoveryCfg.Endpoints) > 0) { return errors.New("both --discovery-token and --discovery-endpoints must be set") } for _, ep := range cfg.DiscoveryCfg.Endpoints { if strings.TrimSpace(ep) == "" { return errors.New("--discovery-endpoints must not contain empty endpoints") } } if cfg.TickMs == 0 { return fmt.Errorf("--heartbeat-interval must be >0 (set to %dms)", cfg.TickMs) } if cfg.ElectionMs == 0 { return fmt.Errorf("--election-timeout must be >0 (set to %dms)", cfg.ElectionMs) } if 5*cfg.TickMs > cfg.ElectionMs { return fmt.Errorf("--election-timeout[%vms] should be at least as 5 times as --heartbeat-interval[%vms]", cfg.ElectionMs, cfg.TickMs) } if cfg.ElectionMs > maxElectionMs { return fmt.Errorf("--election-timeout[%vms] is too long, and should be set less than %vms", cfg.ElectionMs, maxElectionMs) } // check this last since proxying in etcdmain may make this OK if cfg.ListenClientUrls != nil && cfg.AdvertiseClientUrls == nil { return ErrUnsetAdvertiseClientURLsFlag } switch cfg.AutoCompactionMode { case CompactorModeRevision, CompactorModePeriodic: case "": return errors.New("undefined auto-compaction-mode") default: return fmt.Errorf("unknown auto-compaction-mode %q", cfg.AutoCompactionMode) } // Validate distributed tracing configuration but only if enabled. if cfg.EnableDistributedTracing { if err := validateTracingConfig(cfg.DistributedTracingSamplingRatePerMillion); err != nil { return fmt.Errorf("distributed tracing configurition is not valid: (%w)", err) } } if !cfg.ServerFeatureGate.Enabled(features.LeaseCheckpointPersist) && cfg.ServerFeatureGate.Enabled(features.LeaseCheckpoint) { cfg.logger.Warn("Detected that checkpointing is enabled without persistence. Consider enabling feature gate LeaseCheckpointPersist") } if cfg.ServerFeatureGate.Enabled(features.LeaseCheckpointPersist) && !cfg.ServerFeatureGate.Enabled(features.LeaseCheckpoint) { return fmt.Errorf("enabling feature gate LeaseCheckpointPersist requires enabling feature gate LeaseCheckpoint") } if cfg.CompactHashCheckTime <= 0 { return fmt.Errorf("--compact-hash-check-time must be >0 (set to %v)", cfg.CompactHashCheckTime) } // If `--name` isn't configured, then multiple members may have the same "default" name. // When adding a new member with the "default" name as well, etcd may regards its peerURL // as one additional peerURL of the existing member which has the same "default" name, // because each member can have multiple client or peer URLs. // Please refer to https://github.com/etcd-io/etcd/issues/13757 if cfg.Name == DefaultName { cfg.logger.Warn( "it isn't recommended to use default name, please set a value for --name. "+ "Note that etcd might run into issue when multiple members have the same default name", zap.String("name", cfg.Name)) } minVersion, err := tlsutil.GetTLSVersion(cfg.TlsMinVersion) if err != nil { return err } maxVersion, err := tlsutil.GetTLSVersion(cfg.TlsMaxVersion) if err != nil { return err } // maxVersion == 0 means that Go selects the highest available version. if maxVersion != 0 && minVersion > maxVersion { return fmt.Errorf("min version (%s) is greater than max version (%s)", cfg.TlsMinVersion, cfg.TlsMaxVersion) } // Check if user attempted to configure ciphers for TLS1.3 only: Go does not support that currently. if minVersion == tls.VersionTLS13 && len(cfg.CipherSuites) > 0 { return fmt.Errorf("cipher suites cannot be configured when only TLS1.3 is enabled") } return nil } // PeerURLsMapAndToken sets up an initial peer URLsMap and cluster token for bootstrap or discovery. func (cfg *Config) PeerURLsMapAndToken(which string) (urlsmap types.URLsMap, token string, err error) { token = cfg.InitialClusterToken switch { case len(cfg.DiscoveryCfg.Endpoints) > 0: urlsmap = types.URLsMap{} // If using v3 discovery, generate a temporary cluster based on // self's advertised peer URLs urlsmap[cfg.Name] = cfg.AdvertisePeerUrls token = cfg.DiscoveryCfg.Token case cfg.DNSCluster != "": clusterStrs, cerr := cfg.GetDNSClusterNames() lg := cfg.logger if cerr != nil { lg.Warn("failed to resolve during SRV discovery", zap.Error(cerr)) } if len(clusterStrs) == 0 { return nil, "", cerr } for _, s := range clusterStrs { lg.Info("got bootstrap from DNS for etcd-server", zap.String("node", s)) } clusterStr := strings.Join(clusterStrs, ",") if strings.Contains(clusterStr, "https://") && cfg.PeerTLSInfo.TrustedCAFile == "" { cfg.PeerTLSInfo.ServerName = cfg.DNSCluster } urlsmap, err = types.NewURLsMap(clusterStr) // only etcd member must belong to the discovered cluster. // proxy does not need to belong to the discovered cluster. if which == "etcd" { if _, ok := urlsmap[cfg.Name]; !ok { return nil, "", fmt.Errorf("cannot find local etcd member %q in SRV records", cfg.Name) } } default: // We're statically configured, and cluster has appropriately been set. urlsmap, err = types.NewURLsMap(cfg.InitialCluster) } return urlsmap, token, err } // GetDNSClusterNames uses DNS SRV records to get a list of initial nodes for cluster bootstrapping. // This function will return a list of one or more nodes, as well as any errors encountered while // performing service discovery. // Note: Because this checks multiple sets of SRV records, discovery should only be considered to have // failed if the returned node list is empty. func (cfg *Config) GetDNSClusterNames() ([]string, error) { var ( clusterStrs []string cerr error serviceNameSuffix string ) if cfg.DNSClusterServiceName != "" { serviceNameSuffix = "-" + cfg.DNSClusterServiceName } lg := cfg.GetLogger() // Use both etcd-server-ssl and etcd-server for discovery. // Combine the results if both are available. clusterStrs, cerr = getCluster("https", "etcd-server-ssl"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.AdvertisePeerUrls) if cerr != nil { clusterStrs = make([]string, 0) } lg.Info( "get cluster for etcd-server-ssl SRV", zap.String("service-scheme", "https"), zap.String("service-name", "etcd-server-ssl"+serviceNameSuffix), zap.String("server-name", cfg.Name), zap.String("discovery-srv", cfg.DNSCluster), zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()), zap.Strings("found-cluster", clusterStrs), zap.Error(cerr), ) defaultHTTPClusterStrs, httpCerr := getCluster("http", "etcd-server"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.AdvertisePeerUrls) if httpCerr == nil { clusterStrs = append(clusterStrs, defaultHTTPClusterStrs...) } lg.Info( "get cluster for etcd-server SRV", zap.String("service-scheme", "http"), zap.String("service-name", "etcd-server"+serviceNameSuffix), zap.String("server-name", cfg.Name), zap.String("discovery-srv", cfg.DNSCluster), zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()), zap.Strings("found-cluster", clusterStrs), zap.Error(httpCerr), ) return clusterStrs, errors.Join(cerr, httpCerr) } func (cfg *Config) InitialClusterFromName(name string) (ret string) { if len(cfg.AdvertisePeerUrls) == 0 { return "" } n := name if name == "" { n = DefaultName } for i := range cfg.AdvertisePeerUrls { ret = ret + "," + n + "=" + cfg.AdvertisePeerUrls[i].String() } return ret[1:] } // InferLocalAddr tries to determine the LocalAddr used when communicating with // an etcd peer. If SetMemberLocalAddr is true, then it will try to get the host // from AdvertisePeerUrls by searching for the first URL with a specified // non-loopback address. Otherwise, it defaults to empty string and the // LocalAddr used will be the default for the Golang HTTP client. func (cfg *Config) InferLocalAddr() string { if !cfg.ServerFeatureGate.Enabled(features.SetMemberLocalAddr) { return "" } lg := cfg.GetLogger() lg.Info( "searching for a suitable member local address in AdvertisePeerURLs", zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()), ) for _, peerURL := range cfg.AdvertisePeerUrls { if addr, err := netip.ParseAddr(peerURL.Hostname()); err == nil { if addr.IsLoopback() || addr.IsUnspecified() { continue } lg.Info( "setting member local address", zap.String("LocalAddr", addr.String()), ) return addr.String() } } lg.Warn( "unable to set a member local address due to lack of suitable local addresses", zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()), ) return "" } func (cfg *Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew } func (cfg *Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) } func (cfg *Config) V2DeprecationEffective() config.V2DeprecationEnum { if cfg.V2Deprecation == "" { return config.V2DeprDefault } return cfg.V2Deprecation } func (cfg *Config) defaultPeerHost() bool { return len(cfg.AdvertisePeerUrls) == 1 && cfg.AdvertisePeerUrls[0].String() == DefaultInitialAdvertisePeerURLs } func (cfg *Config) defaultClientHost() bool { return len(cfg.AdvertiseClientUrls) == 1 && cfg.AdvertiseClientUrls[0].String() == DefaultAdvertiseClientURLs } func (cfg *Config) ClientSelfCert() (err error) { if !cfg.ClientAutoTLS { return nil } if !cfg.ClientTLSInfo.Empty() { cfg.logger.Warn("ignoring client auto TLS since certs given") return nil } chosts := make([]string, 0, len(cfg.ListenClientUrls)+len(cfg.ListenClientHttpUrls)) for _, u := range cfg.ListenClientUrls { chosts = append(chosts, u.Host) } for _, u := range cfg.ListenClientHttpUrls { chosts = append(chosts, u.Host) } cfg.ClientTLSInfo, err = transport.SelfCert(cfg.logger, filepath.Join(cfg.Dir, "fixtures", "client"), chosts, cfg.SelfSignedCertValidity) if err != nil { return err } return updateCipherSuites(&cfg.ClientTLSInfo, cfg.CipherSuites) } func (cfg *Config) PeerSelfCert() (err error) { if !cfg.PeerAutoTLS { return nil } if !cfg.PeerTLSInfo.Empty() { cfg.logger.Warn("ignoring peer auto TLS since certs given") return nil } phosts := make([]string, len(cfg.ListenPeerUrls)) for i, u := range cfg.ListenPeerUrls { phosts[i] = u.Host } cfg.PeerTLSInfo, err = transport.SelfCert(cfg.logger, filepath.Join(cfg.Dir, "fixtures", "peer"), phosts, cfg.SelfSignedCertValidity) if err != nil { return err } return updateCipherSuites(&cfg.PeerTLSInfo, cfg.CipherSuites) } // UpdateDefaultClusterFromName updates cluster advertise URLs with, if available, default host, // if advertise URLs are default values(localhost:2379,2380) AND if listen URL is 0.0.0.0. // e.g. advertise peer URL localhost:2380 or listen peer URL 0.0.0.0:2380 // then the advertise peer host would be updated with machine's default host, // while keeping the listen URL's port. // User can work around this by explicitly setting URL with 127.0.0.1. // It returns the default hostname, if used, and the error, if any, from getting the machine's default host. // TODO: check whether fields are set instead of whether fields have default value func (cfg *Config) UpdateDefaultClusterFromName(defaultInitialCluster string) (string, error) { if defaultHostname == "" || defaultHostStatus != nil { // update 'initial-cluster' when only the name is specified (e.g. 'etcd --name=abc') if cfg.Name != DefaultName && cfg.InitialCluster == defaultInitialCluster { cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) } return "", defaultHostStatus } used := false pip, pport := cfg.ListenPeerUrls[0].Hostname(), cfg.ListenPeerUrls[0].Port() if cfg.defaultPeerHost() && pip == "0.0.0.0" { cfg.AdvertisePeerUrls[0] = url.URL{Scheme: cfg.AdvertisePeerUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, pport)} used = true } // update 'initial-cluster' when only the name is specified (e.g. 'etcd --name=abc') if cfg.Name != DefaultName && cfg.InitialCluster == defaultInitialCluster { cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) } cip, cport := cfg.ListenClientUrls[0].Hostname(), cfg.ListenClientUrls[0].Port() if cfg.defaultClientHost() && cip == "0.0.0.0" { cfg.AdvertiseClientUrls[0] = url.URL{Scheme: cfg.AdvertiseClientUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, cport)} used = true } dhost := defaultHostname if !used { dhost = "" } return dhost, defaultHostStatus } // checkBindURLs returns an error if any URL uses a domain name. func checkBindURLs(urls []url.URL) error { for _, url := range urls { if url.Scheme == "unix" || url.Scheme == "unixs" { continue } host, _, err := net.SplitHostPort(url.Host) if err != nil { return err } if host == "localhost" { // special case for local address // TODO: support /etc/hosts ? continue } if net.ParseIP(host) == nil { return fmt.Errorf("expected IP in URL for binding (%s)", url.String()) } } return nil } func checkHostURLs(urls []url.URL) error { for _, url := range urls { if url.Scheme == "unix" || url.Scheme == "unixs" { continue } host, _, err := net.SplitHostPort(url.Host) if err != nil { return err } if host == "" { return fmt.Errorf("unexpected empty host (%s)", url.String()) } } return nil } func (cfg *Config) getAdvertisePeerURLs() (ss []string) { ss = make([]string, len(cfg.AdvertisePeerUrls)) for i := range cfg.AdvertisePeerUrls { ss[i] = cfg.AdvertisePeerUrls[i].String() } return ss } func (cfg *Config) getListenPeerURLs() (ss []string) { ss = make([]string, len(cfg.ListenPeerUrls)) for i := range cfg.ListenPeerUrls { ss[i] = cfg.ListenPeerUrls[i].String() } return ss } func (cfg *Config) getAdvertiseClientURLs() (ss []string) { ss = make([]string, len(cfg.AdvertiseClientUrls)) for i := range cfg.AdvertiseClientUrls { ss[i] = cfg.AdvertiseClientUrls[i].String() } return ss } func (cfg *Config) getListenClientURLs() (ss []string) { ss = make([]string, len(cfg.ListenClientUrls)) for i := range cfg.ListenClientUrls { ss[i] = cfg.ListenClientUrls[i].String() } return ss } func (cfg *Config) getMetricsURLs() (ss []string) { ss = make([]string, len(cfg.ListenMetricsUrls)) for i := range cfg.ListenMetricsUrls { ss[i] = cfg.ListenMetricsUrls[i].String() } return ss } func parseBackendFreelistType(freelistType string) bolt.FreelistType { if freelistType == freelistArrayType { return bolt.FreelistArrayType } return bolt.FreelistMapType } ================================================ FILE: server/embed/config_logging.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package embed import ( "crypto/tls" "encoding/json" "errors" "fmt" "io" "net/url" "os" "go.uber.org/zap" "go.uber.org/zap/zapcore" "go.uber.org/zap/zapgrpc" "google.golang.org/grpc" "google.golang.org/grpc/grpclog" "gopkg.in/natefinch/lumberjack.v2" "go.etcd.io/etcd/client/pkg/v3/logutil" ) // GetLogger returns the logger. func (cfg *Config) GetLogger() *zap.Logger { cfg.loggerMu.RLock() l := cfg.logger cfg.loggerMu.RUnlock() return l } // setupLogging initializes etcd logging. // Must be called after flag parsing or finishing configuring embed.Config. func (cfg *Config) setupLogging() error { switch cfg.Logger { case "capnslog": // removed in v3.5 return fmt.Errorf("--logger=capnslog is removed in v3.5") case "zap": if len(cfg.LogOutputs) == 0 { cfg.LogOutputs = []string{DefaultLogOutput} } if len(cfg.LogOutputs) > 1 { for _, v := range cfg.LogOutputs { if v == DefaultLogOutput { return fmt.Errorf("multi logoutput for %q is not supported yet", DefaultLogOutput) } } } if cfg.EnableLogRotation { if err := setupLogRotation(cfg.LogOutputs, cfg.LogRotationConfigJSON); err != nil { return err } } outputPaths, errOutputPaths := make([]string, 0), make([]string, 0) isJournal := false for _, v := range cfg.LogOutputs { switch v { case DefaultLogOutput: outputPaths = append(outputPaths, StdErrLogOutput) errOutputPaths = append(errOutputPaths, StdErrLogOutput) case JournalLogOutput: isJournal = true case StdErrLogOutput: outputPaths = append(outputPaths, StdErrLogOutput) errOutputPaths = append(errOutputPaths, StdErrLogOutput) case StdOutLogOutput: outputPaths = append(outputPaths, StdOutLogOutput) errOutputPaths = append(errOutputPaths, StdOutLogOutput) default: var path string if cfg.EnableLogRotation { // append rotate scheme to logs managed by lumberjack log rotation if v[0:1] == "/" { path = fmt.Sprintf("rotate:/%%2F%s", v[1:]) } else { path = fmt.Sprintf("rotate:/%s", v) } } else { path = v } outputPaths = append(outputPaths, path) errOutputPaths = append(errOutputPaths, path) } } if !isJournal { copied := logutil.DefaultZapLoggerConfig copied.OutputPaths = outputPaths copied.ErrorOutputPaths = errOutputPaths copied = logutil.MergeOutputPaths(copied) copied.Level = zap.NewAtomicLevelAt(logutil.ConvertToZapLevel(cfg.LogLevel)) encoding, err := logutil.ConvertToZapFormat(cfg.LogFormat) if err != nil { return err } copied.Encoding = encoding if cfg.ZapLoggerBuilder == nil { lg, err := copied.Build() if err != nil { return err } cfg.ZapLoggerBuilder = NewZapLoggerBuilder(lg) } } else { if len(cfg.LogOutputs) > 1 { for _, v := range cfg.LogOutputs { if v != DefaultLogOutput { return fmt.Errorf("running with systemd/journal but other '--log-outputs' values (%q) are configured with 'default'; override 'default' value with something else", cfg.LogOutputs) } } } // use stderr as fallback syncer, lerr := getJournalWriteSyncer() if lerr != nil { return lerr } lvl := zap.NewAtomicLevelAt(logutil.ConvertToZapLevel(cfg.LogLevel)) var encoder zapcore.Encoder encoding, err := logutil.ConvertToZapFormat(cfg.LogFormat) if err != nil { return err } if encoding == logutil.ConsoleLogFormat { encoder = zapcore.NewConsoleEncoder(logutil.DefaultZapLoggerConfig.EncoderConfig) } else { encoder = zapcore.NewJSONEncoder(logutil.DefaultZapLoggerConfig.EncoderConfig) } // WARN: do not change field names in encoder config // journald logging writer assumes field names of "level" and "caller" cr := zapcore.NewCore( encoder, syncer, lvl, ) if cfg.ZapLoggerBuilder == nil { cfg.ZapLoggerBuilder = NewZapLoggerBuilder(zap.New(cr, zap.AddCaller(), zap.ErrorOutput(syncer))) } } err := cfg.ZapLoggerBuilder(cfg) if err != nil { return err } logTLSHandshakeFailureFunc := func(msg string) func(conn *tls.Conn, err error) { return func(conn *tls.Conn, err error) { // Log EOF errors on DEBUG not to spam logs too much. logFunc := cfg.logger.Warn if errors.Is(err, io.EOF) { logFunc = cfg.logger.Debug } state := conn.ConnectionState() remoteAddr := conn.RemoteAddr().String() serverName := state.ServerName if len(state.PeerCertificates) > 0 { cert := state.PeerCertificates[0] ips := make([]string, len(cert.IPAddresses)) for i := range cert.IPAddresses { ips[i] = cert.IPAddresses[i].String() } logFunc( msg, zap.String("remote-addr", remoteAddr), zap.String("server-name", serverName), zap.Strings("ip-addresses", ips), zap.Strings("dns-names", cert.DNSNames), zap.Error(err), ) } else { logFunc( msg, zap.String("remote-addr", remoteAddr), zap.String("server-name", serverName), zap.Error(err), ) } } } cfg.ClientTLSInfo.HandshakeFailure = logTLSHandshakeFailureFunc("rejected connection on client endpoint") cfg.PeerTLSInfo.HandshakeFailure = logTLSHandshakeFailureFunc("rejected connection on peer endpoint") default: return fmt.Errorf("unknown logger option %q", cfg.Logger) } return nil } // NewZapLoggerBuilder generates a zap logger builder that sets given logger // for embedded etcd. func NewZapLoggerBuilder(lg *zap.Logger) func(*Config) error { return func(cfg *Config) error { cfg.loggerMu.Lock() defer cfg.loggerMu.Unlock() cfg.logger = lg return nil } } // SetupGlobalLoggers configures 'global' loggers (grpc, zapGlobal) based on the cfg. // // The method is not executed by embed server by default (since 3.5) to // enable setups where grpc/zap.Global logging is configured independently // or spans separate lifecycle (like in tests). func (cfg *Config) SetupGlobalLoggers() { lg := cfg.GetLogger() if lg != nil { if cfg.LogLevel == "debug" { grpc.EnableTracing = true grpclog.SetLoggerV2(zapgrpc.NewLogger(lg)) } else { grpclog.SetLoggerV2(grpclog.NewLoggerV2(io.Discard, os.Stderr, os.Stderr)) } zap.ReplaceGlobals(lg) } } type logRotationConfig struct { *lumberjack.Logger } // Sync implements zap.Sink func (logRotationConfig) Sync() error { return nil } // setupLogRotation initializes log rotation for a single file path target. func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error { var logRotationCfg logRotationConfig outputFilePaths := 0 for _, v := range logOutputs { switch v { case DefaultLogOutput, StdErrLogOutput, StdOutLogOutput: continue default: outputFilePaths++ } } // log rotation requires file target if len(logOutputs) == 1 && outputFilePaths == 0 { return ErrLogRotationInvalidLogOutput } // support max 1 file target for log rotation if outputFilePaths > 1 { return ErrLogRotationInvalidLogOutput } if err := json.Unmarshal([]byte(logRotateConfigJSON), &logRotationCfg); err != nil { var unmarshalTypeError *json.UnmarshalTypeError var syntaxError *json.SyntaxError switch { case errors.As(err, &syntaxError): return fmt.Errorf("improperly formatted log rotation config: %w", err) case errors.As(err, &unmarshalTypeError): return fmt.Errorf("invalid log rotation config: %w", err) default: return fmt.Errorf("fail to unmarshal log rotation config: %w", err) } } zap.RegisterSink("rotate", func(u *url.URL) (zap.Sink, error) { logRotationCfg.Filename = u.Path[1:] return &logRotationCfg, nil }) return nil } ================================================ FILE: server/embed/config_logging_journal_unix.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !windows package embed import ( "fmt" "os" "go.uber.org/zap/zapcore" "go.etcd.io/etcd/client/pkg/v3/logutil" ) // use stderr as fallback func getJournalWriteSyncer() (zapcore.WriteSyncer, error) { jw, err := logutil.NewJournalWriter(os.Stderr) if err != nil { return nil, fmt.Errorf("can't find journal (%w)", err) } return zapcore.AddSync(jw), nil } ================================================ FILE: server/embed/config_logging_journal_windows.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows package embed import ( "os" "go.uber.org/zap/zapcore" ) func getJournalWriteSyncer() (zapcore.WriteSyncer, error) { return zapcore.AddSync(os.Stderr), nil } ================================================ FILE: server/embed/config_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package embed import ( "crypto/tls" "errors" "flag" "fmt" "net" "net/url" "os" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sigs.k8s.io/yaml" "go.etcd.io/etcd/client/pkg/v3/srv" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/featuregate" "go.etcd.io/etcd/server/v3/etcdserver/api/v3discovery" "go.etcd.io/etcd/server/v3/features" ) func notFoundErr(service, domain string) error { name := fmt.Sprintf("_%s._tcp.%s", service, domain) return &net.DNSError{Err: "no such host", Name: name, Server: "10.0.0.53:53", IsTimeout: false, IsTemporary: false, IsNotFound: true} } func TestConfigFileOtherFields(t *testing.T) { ctls := securityConfig{TrustedCAFile: "cca", CertFile: "ccert", KeyFile: "ckey"} // Note AllowedCN and AllowedHostname are mutually exclusive, this test is just to verify the fields can be correctly marshalled & unmarshalled. ptls := securityConfig{TrustedCAFile: "pca", CertFile: "pcert", KeyFile: "pkey", AllowedCNs: []string{"etcd"}, AllowedHostnames: []string{"whatever.example.com"}} yc := struct { ClientSecurityCfgFile securityConfig `json:"client-transport-security"` PeerSecurityCfgFile securityConfig `json:"peer-transport-security"` ForceNewCluster bool `json:"force-new-cluster"` Logger string `json:"logger"` LogOutputs []string `json:"log-outputs"` Debug bool `json:"debug"` SocketOpts transport.SocketOpts `json:"socket-options"` }{ ctls, ptls, true, "zap", []string{"/dev/null"}, false, transport.SocketOpts{ ReusePort: true, }, } b, err := yaml.Marshal(&yc) if err != nil { t.Fatal(err) } tmpfile := mustCreateCfgFile(t, b) defer os.Remove(tmpfile.Name()) cfg, err := ConfigFromFile(tmpfile.Name()) if err != nil { t.Fatal(err) } if !ctls.equals(&cfg.ClientTLSInfo) { t.Errorf("ClientTLS = %v, want %v", cfg.ClientTLSInfo, ctls) } if !ptls.equals(&cfg.PeerTLSInfo) { t.Errorf("PeerTLS = %v, want %v", cfg.PeerTLSInfo, ptls) } assert.Truef(t, cfg.ForceNewCluster, "ForceNewCluster does not match") assert.Truef(t, cfg.SocketOpts.ReusePort, "ReusePort does not match") assert.Falsef(t, cfg.SocketOpts.ReuseAddress, "ReuseAddress does not match") } func TestConfigFileFeatureGates(t *testing.T) { testCases := []struct { name string serverFeatureGatesJSON string expectErr bool expectedFeatures map[featuregate.Feature]bool }{ { name: "default", expectedFeatures: map[featuregate.Feature]bool{ features.StopGRPCServiceOnDefrag: false, features.InitialCorruptCheck: false, features.TxnModeWriteWithSharedBuffer: true, features.LeaseCheckpoint: false, features.LeaseCheckpointPersist: false, features.FastLeaseKeepAlive: true, }, }, { name: "can set feature gate StopGRPCServiceOnDefrag to true from feature gate flag", serverFeatureGatesJSON: "StopGRPCServiceOnDefrag=true", expectedFeatures: map[featuregate.Feature]bool{ features.StopGRPCServiceOnDefrag: true, features.TxnModeWriteWithSharedBuffer: true, features.FastLeaseKeepAlive: true, }, }, { name: "can set feature gate InitialCorruptCheck to true from feature gate flag", serverFeatureGatesJSON: "InitialCorruptCheck=true", expectedFeatures: map[featuregate.Feature]bool{ features.InitialCorruptCheck: true, features.TxnModeWriteWithSharedBuffer: true, features.FastLeaseKeepAlive: true, }, }, { name: "can set feature gate StopGRPCServiceOnDefrag to false from feature gate flag", serverFeatureGatesJSON: "StopGRPCServiceOnDefrag=false", expectedFeatures: map[featuregate.Feature]bool{ features.StopGRPCServiceOnDefrag: false, features.TxnModeWriteWithSharedBuffer: true, features.FastLeaseKeepAlive: true, }, }, { name: "can set feature gate TxnModeWriteWithSharedBuffer to true from feature gate flag", serverFeatureGatesJSON: "TxnModeWriteWithSharedBuffer=true", expectedFeatures: map[featuregate.Feature]bool{ features.TxnModeWriteWithSharedBuffer: true, features.FastLeaseKeepAlive: true, }, }, { name: "can set feature gate TxnModeWriteWithSharedBuffer to false from feature gate flag", serverFeatureGatesJSON: "TxnModeWriteWithSharedBuffer=false", expectedFeatures: map[featuregate.Feature]bool{ features.TxnModeWriteWithSharedBuffer: false, features.FastLeaseKeepAlive: true, }, }, { name: "can set feature gate CompactHashCheck to true from feature gate flag", serverFeatureGatesJSON: "CompactHashCheck=true", expectedFeatures: map[featuregate.Feature]bool{ features.CompactHashCheck: true, features.TxnModeWriteWithSharedBuffer: true, features.FastLeaseKeepAlive: true, }, }, { name: "can set feature gate LeaseCheckpoint and LeaseCheckpointPersist to true from feature gate flag", serverFeatureGatesJSON: "LeaseCheckpointPersist=true,LeaseCheckpoint=true", expectedFeatures: map[featuregate.Feature]bool{ features.TxnModeWriteWithSharedBuffer: true, features.LeaseCheckpoint: true, features.LeaseCheckpointPersist: true, features.FastLeaseKeepAlive: true, }, }, { name: "can set feature gate FastLeaseKeepAlive to true from feature gate flag", serverFeatureGatesJSON: "FastLeaseKeepAlive=false", expectedFeatures: map[featuregate.Feature]bool{ features.TxnModeWriteWithSharedBuffer: true, features.FastLeaseKeepAlive: false, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { yc := struct { ServerFeatureGatesJSON string `json:"feature-gates"` }{ ServerFeatureGatesJSON: tc.serverFeatureGatesJSON, } b, err := yaml.Marshal(&yc) if err != nil { t.Fatal(err) } tmpfile := mustCreateCfgFile(t, b) defer os.Remove(tmpfile.Name()) cfg, err := ConfigFromFile(tmpfile.Name()) if tc.expectErr { require.Errorf(t, err, "expect parse error") return } if err != nil { t.Fatal(err) } for f := range features.DefaultEtcdServerFeatureGates { if tc.expectedFeatures[f] != cfg.ServerFeatureGate.Enabled(f) { t.Errorf("expected feature gate %s=%v, got %v", f, tc.expectedFeatures[f], cfg.ServerFeatureGate.Enabled(f)) } } }) } } // TestUpdateDefaultClusterFromName ensures that etcd can start with 'etcd --name=abc'. func TestUpdateDefaultClusterFromName(t *testing.T) { cfg := NewConfig() defaultInitialCluster := cfg.InitialCluster oldscheme := cfg.AdvertisePeerUrls[0].Scheme origpeer := cfg.AdvertisePeerUrls[0].String() origadvc := cfg.AdvertiseClientUrls[0].String() cfg.Name = "abc" lpport := cfg.ListenPeerUrls[0].Port() // in case of 'etcd --name=abc' exp := fmt.Sprintf("%s=%s://localhost:%s", cfg.Name, oldscheme, lpport) _, _ = cfg.UpdateDefaultClusterFromName(defaultInitialCluster) require.Equalf(t, exp, cfg.InitialCluster, "initial-cluster expected %q, got %q", exp, cfg.InitialCluster) // advertise peer URL should not be affected require.Equalf(t, origpeer, cfg.AdvertisePeerUrls[0].String(), "advertise peer url expected %q, got %q", origadvc, cfg.AdvertisePeerUrls[0].String()) // advertise client URL should not be affected require.Equalf(t, origadvc, cfg.AdvertiseClientUrls[0].String(), "advertise client url expected %q, got %q", origadvc, cfg.AdvertiseClientUrls[0].String()) } // TestUpdateDefaultClusterFromNameOverwrite ensures that machine's default host is only used // if advertise URLs are default values(localhost:2379,2380) AND if listen URL is 0.0.0.0. func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) { if defaultHostname == "" { t.Skip("machine's default host not found") } cfg := NewConfig() defaultInitialCluster := cfg.InitialCluster oldscheme := cfg.AdvertisePeerUrls[0].Scheme origadvc := cfg.AdvertiseClientUrls[0].String() cfg.Name = "abc" lpport := cfg.ListenPeerUrls[0].Port() cfg.ListenPeerUrls[0] = url.URL{Scheme: cfg.ListenPeerUrls[0].Scheme, Host: fmt.Sprintf("0.0.0.0:%s", lpport)} dhost, _ := cfg.UpdateDefaultClusterFromName(defaultInitialCluster) require.Equalf(t, dhost, defaultHostname, "expected default host %q, got %q", defaultHostname, dhost) aphost, apport := cfg.AdvertisePeerUrls[0].Hostname(), cfg.AdvertisePeerUrls[0].Port() require.Equalf(t, apport, lpport, "advertise peer url got different port %s, expected %s", apport, lpport) require.Equalf(t, aphost, defaultHostname, "advertise peer url expected machine default host %q, got %q", defaultHostname, aphost) expected := fmt.Sprintf("%s=%s://%s:%s", cfg.Name, oldscheme, defaultHostname, lpport) require.Equalf(t, expected, cfg.InitialCluster, "initial-cluster expected %q, got %q", expected, cfg.InitialCluster) // advertise client URL should not be affected require.Equalf(t, origadvc, cfg.AdvertiseClientUrls[0].String(), "advertise-client-url expected %q, got %q", origadvc, cfg.AdvertiseClientUrls[0].String()) } func TestInferLocalAddr(t *testing.T) { tests := []struct { name string advertisePeerURLs []string serverFeatureGates string expectedLocalAddr string }{ { "defaults, SetMemberLocalAddr=false ", []string{DefaultInitialAdvertisePeerURLs}, "SetMemberLocalAddr=false", "", }, { "IPv4 address, SetMemberLocalAddr=false ", []string{"https://192.168.100.110:2380"}, "SetMemberLocalAddr=false", "", }, { "defaults, SetMemberLocalAddr=true", []string{DefaultInitialAdvertisePeerURLs}, "SetMemberLocalAddr=true", "", }, { "IPv4 unspecified address, SetMemberLocalAddr=true", []string{"https://0.0.0.0:2380"}, "SetMemberLocalAddr=true", "", }, { "IPv6 unspecified address, SetMemberLocalAddr=true", []string{"https://[::]:2380"}, "SetMemberLocalAddr=true", "", }, { "IPv4 loopback address, SetMemberLocalAddr=true", []string{"https://127.0.0.1:2380"}, "SetMemberLocalAddr=true", "", }, { "IPv6 loopback address, SetMemberLocalAddr=true", []string{"https://[::1]:2380"}, "SetMemberLocalAddr=true", "", }, { "IPv4 address, SetMemberLocalAddr=true", []string{"https://192.168.100.110:2380"}, "SetMemberLocalAddr=true", "192.168.100.110", }, { "Hostname only, SetMemberLocalAddr=true", []string{"https://123-host-3.corp.internal:2380"}, "SetMemberLocalAddr=true", "", }, { "Hostname and IPv4 address, SetMemberLocalAddr=true", []string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380"}, "SetMemberLocalAddr=true", "192.168.100.110", }, { "IPv4 address and Hostname, SetMemberLocalAddr=true", []string{"https://192.168.100.110:2380", "https://123-host-3.corp.internal:2380"}, "SetMemberLocalAddr=true", "192.168.100.110", }, { "IPv4 and IPv6 addresses, SetMemberLocalAddr=true", []string{"https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"}, "SetMemberLocalAddr=true", "192.168.100.110", }, { "IPv6 and IPv4 addresses, SetMemberLocalAddr=true", // IPv4 addresses will always sort before IPv6 ones anyway []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"}, "SetMemberLocalAddr=true", "192.168.100.110", }, { "Hostname, IPv4 and IPv6 addresses, SetMemberLocalAddr=true", []string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"}, "SetMemberLocalAddr=true", "192.168.100.110", }, { "Hostname, IPv6 and IPv4 addresses, SetMemberLocalAddr=true", // IPv4 addresses will always sort before IPv6 ones anyway []string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"}, "SetMemberLocalAddr=true", "192.168.100.110", }, { "IPv6 address, SetMemberLocalAddr=true", []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380"}, "SetMemberLocalAddr=true", "2001:db8:85a3::8a2e:370:7334", }, { "Hostname and IPv6 address, SetMemberLocalAddr=true", []string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"}, "SetMemberLocalAddr=true", "2001:db8:85a3::8a2e:370:7334", }, { "IPv6 address and Hostname, SetMemberLocalAddr=true", []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://123-host-3.corp.internal:2380"}, "SetMemberLocalAddr=true", "2001:db8:85a3::8a2e:370:7334", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := NewConfig() cfg.AdvertisePeerUrls = types.MustNewURLs(tt.advertisePeerURLs) cfg.ServerFeatureGate.(featuregate.MutableFeatureGate).Set(tt.serverFeatureGates) require.NoError(t, cfg.Validate()) require.Equal(t, tt.expectedLocalAddr, cfg.InferLocalAddr()) }) } } func TestSetMemberLocalAddrValidate(t *testing.T) { tcs := []struct { name string serverFeatureGates string }{ { name: "Default config should pass", }, { name: "Enabling SetMemberLocalAddr should pass", serverFeatureGates: "SetMemberLocalAddr=true", }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { cfg := *NewConfig() cfg.ServerFeatureGate.(featuregate.MutableFeatureGate).Set(tc.serverFeatureGates) err := cfg.Validate() require.NoError(t, err) }) } } func (s *securityConfig) equals(t *transport.TLSInfo) bool { return s.CertFile == t.CertFile && s.CertAuth == t.ClientCertAuth && s.TrustedCAFile == t.TrustedCAFile && s.ClientCertFile == t.ClientCertFile && s.ClientKeyFile == t.ClientKeyFile && s.KeyFile == t.KeyFile && compareSlices(s.AllowedCNs, t.AllowedCNs) && compareSlices(s.AllowedHostnames, t.AllowedHostnames) } func compareSlices(slice1, slice2 []string) bool { if len(slice1) != len(slice2) { return false } for i, v := range slice1 { if v != slice2[i] { return false } } return true } func mustCreateCfgFile(t *testing.T, b []byte) *os.File { tmpfile, err := os.CreateTemp(t.TempDir(), "servercfg") if err != nil { t.Fatal(err) } if _, err = tmpfile.Write(b); err != nil { t.Fatal(err) } if err = tmpfile.Close(); err != nil { t.Fatal(err) } return tmpfile } func TestAutoCompactionModeInvalid(t *testing.T) { cfg := NewConfig() cfg.Logger = "zap" cfg.LogOutputs = []string{"/dev/null"} cfg.AutoCompactionMode = "period" err := cfg.Validate() if err == nil { t.Errorf("expected non-nil error, got %v", err) } } func TestAutoCompactionModeParse(t *testing.T) { tests := []struct { mode string retention string werr bool wdur time.Duration }{ // revision {"revision", "1", false, 1}, {"revision", "1h", false, time.Hour}, {"revision", "a", true, 0}, {"revision", "-1", true, 0}, // periodic {"periodic", "1", false, time.Hour}, {"periodic", "a", true, 0}, {"revision", "-1", true, 0}, // err mode {"errmode", "1", false, 0}, {"errmode", "1h", false, time.Hour}, // empty mode {"", "1", true, 0}, {"", "1h", false, time.Hour}, {"", "a", true, 0}, {"", "-1", true, 0}, } hasErr := func(err error) bool { return err != nil } for i, tt := range tests { dur, err := parseCompactionRetention(tt.mode, tt.retention) if hasErr(err) != tt.werr { t.Errorf("#%d: err = %v, want %v", i, err, tt.werr) } if dur != tt.wdur { t.Errorf("#%d: duration = %s, want %s", i, dur, tt.wdur) } } } func TestPeerURLsMapAndTokenFromSRV(t *testing.T) { defer func() { getCluster = srv.GetCluster }() tests := []struct { withSSL []string withoutSSL []string apurls []string wurls string werr bool }{ { []string{}, []string{}, []string{"http://localhost:2380"}, "", true, }, { []string{"1.example.com=https://1.example.com:2380", "0=https://2.example.com:2380", "1=https://3.example.com:2380"}, []string{}, []string{"https://1.example.com:2380"}, "0=https://2.example.com:2380,1.example.com=https://1.example.com:2380,1=https://3.example.com:2380", false, }, { []string{"1.example.com=https://1.example.com:2380"}, []string{"0=http://2.example.com:2380", "1=http://3.example.com:2380"}, []string{"https://1.example.com:2380"}, "0=http://2.example.com:2380,1.example.com=https://1.example.com:2380,1=http://3.example.com:2380", false, }, { []string{}, []string{"1.example.com=http://1.example.com:2380", "0=http://2.example.com:2380", "1=http://3.example.com:2380"}, []string{"http://1.example.com:2380"}, "0=http://2.example.com:2380,1.example.com=http://1.example.com:2380,1=http://3.example.com:2380", false, }, } hasErr := func(err error) bool { return err != nil } for i, tt := range tests { getCluster = func(serviceScheme string, service string, name string, dns string, apurls types.URLs) ([]string, error) { var urls []string if serviceScheme == "https" && service == "etcd-server-ssl" { urls = tt.withSSL } else if serviceScheme == "http" && service == "etcd-server" { urls = tt.withoutSSL } if len(urls) > 0 { return urls, nil } return urls, notFoundErr(service, dns) } cfg := NewConfig() cfg.Name = "1.example.com" cfg.InitialCluster = "" cfg.InitialClusterToken = "" cfg.DNSCluster = "example.com" cfg.AdvertisePeerUrls = types.MustNewURLs(tt.apurls) if err := cfg.Validate(); err != nil { t.Errorf("#%d: failed to validate test Config: %v", i, err) continue } urlsmap, _, err := cfg.PeerURLsMapAndToken("etcd") if urlsmap.String() != tt.wurls { t.Errorf("#%d: urlsmap = %s, want = %s", i, urlsmap.String(), tt.wurls) } if hasErr(err) != tt.werr { t.Errorf("#%d: err = %v, want = %v", i, err, tt.werr) } } } func TestLeaseCheckpointValidate(t *testing.T) { tcs := []struct { name string serverFeatureGates string expectError bool }{ { name: "Default config should pass", }, { name: "Enabling checkpoint leases should pass", serverFeatureGates: "LeaseCheckpoint=true", }, { name: "Enabling checkpoint leases and persist should pass", serverFeatureGates: "LeaseCheckpointPersist=true,LeaseCheckpoint=true", }, { name: "Enabling checkpoint leases persist without checkpointing itself should fail", serverFeatureGates: "LeaseCheckpointPersist=true", expectError: true, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { cfg := *NewConfig() cfg.ServerFeatureGate.(featuregate.MutableFeatureGate).Set(tc.serverFeatureGates) err := cfg.Validate() if (err != nil) != tc.expectError { t.Errorf("config.Validate() = %q, expected error: %v", err, tc.expectError) } }) } } func TestLogRotation(t *testing.T) { tests := []struct { name string logOutputs []string logRotationConfig string wantErr bool wantErrMsg error }{ { name: "mixed log output targets", logOutputs: []string{"stderr", "/tmp/path"}, logRotationConfig: `{"maxsize": 1}`, }, { name: "log output relative path", logOutputs: []string{"stderr", "tmp/path"}, logRotationConfig: `{"maxsize": 1}`, }, { name: "no file targets", logOutputs: []string{"stderr"}, logRotationConfig: `{"maxsize": 1}`, wantErr: true, wantErrMsg: ErrLogRotationInvalidLogOutput, }, { name: "multiple file targets", logOutputs: []string{"/tmp/path1", "/tmp/path2"}, logRotationConfig: DefaultLogRotationConfig, wantErr: true, wantErrMsg: ErrLogRotationInvalidLogOutput, }, { name: "default output", logRotationConfig: `{"maxsize": 1}`, wantErr: true, wantErrMsg: ErrLogRotationInvalidLogOutput, }, { name: "default log rotation config", logOutputs: []string{"/tmp/path"}, logRotationConfig: DefaultLogRotationConfig, }, { name: "invalid logger config", logOutputs: []string{"/tmp/path"}, logRotationConfig: `{"maxsize": true}`, wantErr: true, wantErrMsg: errors.New("invalid log rotation config: json: cannot unmarshal bool into Go struct field logRotationConfig.Logger.maxsize of type int"), }, { name: "improperly formatted logger config", logOutputs: []string{"/tmp/path"}, logRotationConfig: `{"maxsize": true`, wantErr: true, wantErrMsg: errors.New("improperly formatted log rotation config: unexpected end of JSON input"), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := NewConfig() cfg.Logger = "zap" cfg.LogOutputs = tt.logOutputs cfg.EnableLogRotation = true cfg.LogRotationConfigJSON = tt.logRotationConfig err := cfg.Validate() if err != nil && !tt.wantErr { t.Errorf("test %q, unexpected error %v", tt.name, err) } if err != nil && tt.wantErr && tt.wantErrMsg.Error() != err.Error() { t.Errorf("test %q, expected error: %+v, got: %+v", tt.name, tt.wantErrMsg, err) } if err == nil && tt.wantErr { t.Errorf("test %q, expected error, got nil", tt.name) } if err == nil { cfg.GetLogger().Info("test log") } }) } } func TestTLSVersionMinMax(t *testing.T) { tests := []struct { name string givenTLSMinVersion string givenTLSMaxVersion string givenCipherSuites []string expectError bool expectedMinTLSVersion uint16 expectedMaxTLSVersion uint16 }{ { name: "Minimum TLS version is set", givenTLSMinVersion: "TLS1.3", expectedMinTLSVersion: tls.VersionTLS13, expectedMaxTLSVersion: 0, }, { name: "Maximum TLS version is set", givenTLSMaxVersion: "TLS1.2", expectedMinTLSVersion: 0, expectedMaxTLSVersion: tls.VersionTLS12, }, { name: "Minimum and Maximum TLS versions are set", givenTLSMinVersion: "TLS1.3", givenTLSMaxVersion: "TLS1.3", expectedMinTLSVersion: tls.VersionTLS13, expectedMaxTLSVersion: tls.VersionTLS13, }, { name: "Minimum and Maximum TLS versions are set in reverse order", givenTLSMinVersion: "TLS1.3", givenTLSMaxVersion: "TLS1.2", expectError: true, }, { name: "Invalid minimum TLS version", givenTLSMinVersion: "invalid version", expectError: true, }, { name: "Invalid maximum TLS version", givenTLSMaxVersion: "invalid version", expectError: true, }, { name: "Cipher suites configured for TLS 1.3", givenTLSMinVersion: "TLS1.3", givenCipherSuites: []string{"TLS_AES_128_GCM_SHA256"}, expectError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := NewConfig() cfg.TlsMinVersion = tt.givenTLSMinVersion cfg.TlsMaxVersion = tt.givenTLSMaxVersion cfg.CipherSuites = tt.givenCipherSuites err := cfg.Validate() if err != nil { assert.Truef(t, tt.expectError, "Validate() returned error while expecting success: %v", err) return } updateMinMaxVersions(&cfg.PeerTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion) updateMinMaxVersions(&cfg.ClientTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion) assert.Equal(t, tt.expectedMinTLSVersion, cfg.PeerTLSInfo.MinVersion) assert.Equal(t, tt.expectedMaxTLSVersion, cfg.PeerTLSInfo.MaxVersion) assert.Equal(t, tt.expectedMinTLSVersion, cfg.ClientTLSInfo.MinVersion) assert.Equal(t, tt.expectedMaxTLSVersion, cfg.ClientTLSInfo.MaxVersion) }) } } func TestUndefinedAutoCompactionModeValidate(t *testing.T) { cfg := *NewConfig() cfg.AutoCompactionMode = "" err := cfg.Validate() require.Error(t, err) } func TestMatchNewConfigAddFlags(t *testing.T) { cfg := NewConfig() fs := flag.NewFlagSet("etcd", flag.ContinueOnError) cfg.AddFlags(fs) require.NoError(t, fs.Parse(nil)) // TODO: Reduce number of unexported fields set in config if diff := cmp.Diff(NewConfig(), cfg, cmpopts.IgnoreUnexported(transport.TLSInfo{}, Config{}), cmp.Comparer(func(a, b featuregate.FeatureGate) bool { return a.String() == b.String() })); diff != "" { t.Errorf("Diff: %s", diff) } } func TestCheckHostURLs(t *testing.T) { tests := []struct { name string urls []url.URL wantErr bool }{ { name: "valid HTTP URLs", urls: []url.URL{ {Scheme: "http", Host: "127.0.0.1:2379"}, {Scheme: "http", Host: "localhost:2379"}, }, wantErr: false, }, { name: "valid HTTPS URLs", urls: []url.URL{ {Scheme: "https", Host: "127.0.0.1:2379"}, {Scheme: "https", Host: "localhost:2379"}, }, wantErr: false, }, { name: "valid Unix socket URLs", urls: []url.URL{ {Scheme: "unix", Host: "", Path: "/tmp/etcd.sock"}, {Scheme: "unixs", Host: "", Path: "/tmp/etcd-secure.sock"}, }, wantErr: false, }, { name: "empty host in URL", urls: []url.URL{ {Scheme: "http", Host: ""}, }, wantErr: true, }, { name: "invalid host format", urls: []url.URL{ {Scheme: "http", Host: "invalid_host"}, }, wantErr: true, }, { name: "missing port in host", urls: []url.URL{ {Scheme: "http", Host: "127.0.0.1"}, }, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := checkHostURLs(tt.urls) if (err != nil) != tt.wantErr { t.Errorf("checkHostURLs() error = %v, wantErr %v", err, tt.wantErr) } }) } } func TestDiscoveryCfg(t *testing.T) { testCases := []struct { name string discoveryCfg v3discovery.DiscoveryConfig wantErr bool }{ { name: "Valid discovery config", discoveryCfg: v3discovery.DiscoveryConfig{ ConfigSpec: clientv3.ConfigSpec{ Endpoints: []string{"http://10.0.0.100:2379", "http://10.0.0.101:2379"}, }, }, wantErr: false, }, { name: "Partial empty discovery endpoints", discoveryCfg: v3discovery.DiscoveryConfig{ ConfigSpec: clientv3.ConfigSpec{ Endpoints: []string{"http://10.0.0.100:2379", ""}, }, }, wantErr: true, }, { name: "Empty discovery endpoint", discoveryCfg: v3discovery.DiscoveryConfig{ ConfigSpec: clientv3.ConfigSpec{ Endpoints: []string{"", ""}, }, }, wantErr: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { cfg := NewConfig() cfg.InitialCluster = "" cfg.DiscoveryCfg = tc.discoveryCfg cfg.DiscoveryCfg.Token = "foo" err := cfg.Validate() require.Equal(t, tc.wantErr, err != nil) }) } } func TestFastLeaseKeepAliveValidate(t *testing.T) { tcs := []struct { name string serverFeatureGates string expectEnabled bool }{ { name: "Default config should pass", expectEnabled: true, }, { name: "Enabling FastLeaseKeepAlive should pass", serverFeatureGates: "FastLeaseKeepAlive=true", expectEnabled: true, }, { name: "Disabling FastLeaseKeepAlive should pass", serverFeatureGates: "FastLeaseKeepAlive=false", expectEnabled: false, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { cfg := *NewConfig() cfg.ServerFeatureGate.(featuregate.MutableFeatureGate).Set(tc.serverFeatureGates) require.NoError(t, cfg.Validate()) require.Equal(t, tc.expectEnabled, cfg.ServerFeatureGate.Enabled(features.FastLeaseKeepAlive)) }) } } ================================================ FILE: server/embed/config_tracing.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package embed import ( "context" "fmt" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" tracesdk "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.17.0" "go.uber.org/zap" "go.etcd.io/etcd/pkg/v3/traceutil" ) const maxSamplingRatePerMillion = 1000000 func validateTracingConfig(samplingRate int) error { if samplingRate < 0 { return fmt.Errorf("tracing sampling rate must be positive") } if samplingRate > maxSamplingRatePerMillion { return fmt.Errorf("tracing sampling rate must be less than %d", maxSamplingRatePerMillion) } return nil } type tracingExporter struct { exporter tracesdk.SpanExporter opts []otelgrpc.Option provider *tracesdk.TracerProvider } func newTracingExporter(ctx context.Context, cfg *Config) (*tracingExporter, error) { exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure(), otlptracegrpc.WithEndpoint(cfg.DistributedTracingAddress), ) if err != nil { return nil, err } res, err := resource.New(ctx, resource.WithAttributes( semconv.ServiceNameKey.String(cfg.DistributedTracingServiceName), ), ) if err != nil { return nil, err } if resWithIDKey := determineResourceWithIDKey(cfg.DistributedTracingServiceInstanceID); resWithIDKey != nil { // Merge resources into a new // resource in case of duplicates. res, err = resource.Merge(res, resWithIDKey) if err != nil { return nil, err } } traceProvider := tracesdk.NewTracerProvider( tracesdk.WithBatcher(exporter), tracesdk.WithResource(res), tracesdk.WithSampler( tracesdk.ParentBased(determineSampler(cfg.DistributedTracingSamplingRatePerMillion)), ), ) traceutil.Init(traceProvider) options := []otelgrpc.Option{ otelgrpc.WithPropagators( propagation.NewCompositeTextMapPropagator( propagation.TraceContext{}, propagation.Baggage{}, ), ), otelgrpc.WithTracerProvider( traceProvider, ), } cfg.logger.Debug( "distributed tracing enabled", zap.String("address", cfg.DistributedTracingAddress), zap.String("service-name", cfg.DistributedTracingServiceName), zap.String("service-instance-id", cfg.DistributedTracingServiceInstanceID), zap.Int("sampling-rate", cfg.DistributedTracingSamplingRatePerMillion), ) return &tracingExporter{ exporter: exporter, opts: options, provider: traceProvider, }, nil } func (te *tracingExporter) Close(ctx context.Context) { if te.provider != nil { te.provider.Shutdown(ctx) } if te.exporter != nil { te.exporter.Shutdown(ctx) } } func determineSampler(samplingRate int) tracesdk.Sampler { sampler := tracesdk.NeverSample() if samplingRate == 0 { return sampler } return tracesdk.TraceIDRatioBased(float64(samplingRate) / float64(maxSamplingRatePerMillion)) } // As Tracing service Instance ID must be unique, it should // never use the empty default string value, it's set if // if it's a non empty string. func determineResourceWithIDKey(serviceInstanceID string) *resource.Resource { if serviceInstanceID != "" { return resource.NewSchemaless( (semconv.ServiceInstanceIDKey.String(serviceInstanceID)), ) } return nil } ================================================ FILE: server/embed/config_tracing_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package embed import ( "testing" ) const neverSampleDescription = "AlwaysOffSampler" func TestDetermineSampler(t *testing.T) { tests := []struct { name string sampleRate int wantSamplerDescription string }{ { name: "sample rate is disabled", sampleRate: 0, wantSamplerDescription: neverSampleDescription, }, { name: "sample rate is 100", sampleRate: 100, wantSamplerDescription: "TraceIDRatioBased{0.0001}", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { sampler := determineSampler(tc.sampleRate) if tc.wantSamplerDescription != sampler.Description() { t.Errorf("tracing sampler was not as expected; expected sampler: %#+v, got sampler: %#+v", tc.wantSamplerDescription, sampler.Description()) } }) } } func TestTracingConfig(t *testing.T) { tests := []struct { name string sampleRate int wantErr bool }{ { name: "invalid - sample rate is less than 0", sampleRate: -1, wantErr: true, }, { name: "invalid - sample rate is more than allowed value", sampleRate: maxSamplingRatePerMillion + 1, wantErr: true, }, { name: "valid - sample rate is 100", sampleRate: 100, wantErr: false, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { err := validateTracingConfig(tc.sampleRate) if err == nil && tc.wantErr { t.Errorf("expected error got (%v) error", err) } if err != nil && !tc.wantErr { t.Errorf("expected no errors, got error: (%v)", err) } }) } } ================================================ FILE: server/embed/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /* Package embed provides bindings for embedding an etcd server in a program. Launch an embedded etcd server using the configuration defaults: import ( "log" "time" "go.etcd.io/etcd/server/v3/embed" ) func main() { cfg := embed.NewConfig() cfg.Dir = "default.etcd" e, err := embed.StartEtcd(cfg) if err != nil { log.Fatal(err) } defer e.Close() select { case <-e.Server.ReadyNotify(): log.Printf("Server is ready!") case <-time.After(60 * time.Second): e.Server.Stop() // trigger a shutdown log.Printf("Server took too long to start!") } log.Fatal(<-e.Err()) } */ package embed ================================================ FILE: server/embed/etcd.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package embed import ( "context" "errors" "fmt" "io" defaultLog "log" "math" "net" "net/http" "net/url" "runtime" "sort" "strconv" "strings" "sync" "time" "github.com/soheilhy/cmux" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/keepalive" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/client/v3/credentials" "go.etcd.io/etcd/pkg/v3/debugutil" runtimeutil "go.etcd.io/etcd/pkg/v3/runtime" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/features" "go.etcd.io/etcd/server/v3/storage" "go.etcd.io/etcd/server/v3/verify" ) const ( // internal fd usage includes disk usage and transport usage. // To read/write snapshot, snap pkg needs 1. In normal case, wal pkg needs // at most 2 to read/lock/write WALs. One case that it needs to 2 is to // read all logs after some snapshot index, which locates at the end of // the second last and the head of the last. For purging, it needs to read // directory, so it needs 1. For fd monitor, it needs 1. // For transport, rafthttp builds two long-polling connections and at most // four temporary connections with each member. There are at most 9 members // in a cluster, so it should reserve 96. // For the safety, we set the total reserved number to 150. reservedInternalFDNum = 150 ) // Etcd contains a running etcd server and its listeners. type Etcd struct { Peers []*peerListener Clients []net.Listener // a map of contexts for the servers that serves client requests. sctxs map[string]*serveCtx metricsListeners []net.Listener tracingExporterShutdown func() Server *etcdserver.EtcdServer cfg Config // closeOnce is to ensure `stopc` is closed only once, no matter // how many times the Close() method is called. closeOnce sync.Once // stopc is used to notify the sub goroutines not to send // any errors to `errc`. stopc chan struct{} // errc is used to receive error from sub goroutines (including // client handler, peer handler and metrics handler). It's closed // after all these sub goroutines exit (checked via `wg`). Writers // should avoid writing after `stopc` is closed by selecting on // reading from `stopc`. errc chan error // wg is used to track the lifecycle of all sub goroutines which // need to send error back to the `errc`. wg sync.WaitGroup } type peerListener struct { net.Listener serve func() error close func(context.Context) error } // StartEtcd launches the etcd server and HTTP handlers for client/server communication. // The returned Etcd.Server is not guaranteed to have joined the cluster. Wait // on the Etcd.Server.ReadyNotify() channel to know when it completes and is ready for use. func StartEtcd(inCfg *Config) (e *Etcd, err error) { if err = inCfg.Validate(); err != nil { return nil, err } serving := false e = &Etcd{cfg: *inCfg, stopc: make(chan struct{})} cfg := &e.cfg defer func() { if e == nil || err == nil { return } if !serving { // errored before starting gRPC server for serveCtx.serversC for _, sctx := range e.sctxs { sctx.close() } } e.Close() e = nil }() if !cfg.SocketOpts.Empty() { cfg.logger.Info( "configuring socket options", zap.Bool("reuse-address", cfg.SocketOpts.ReuseAddress), zap.Bool("reuse-port", cfg.SocketOpts.ReusePort), ) } e.cfg.logger.Info( "configuring peer listeners", zap.Strings("listen-peer-urls", e.cfg.getListenPeerURLs()), ) if e.Peers, err = configurePeerListeners(cfg); err != nil { return e, err } e.cfg.logger.Info( "configuring client listeners", zap.Strings("listen-client-urls", e.cfg.getListenClientURLs()), ) if e.sctxs, err = configureClientListeners(cfg); err != nil { return e, err } for _, sctx := range e.sctxs { e.Clients = append(e.Clients, sctx.l) } var ( urlsmap types.URLsMap token string ) memberInitialized := true if !isMemberInitialized(cfg) { memberInitialized = false urlsmap, token, err = cfg.PeerURLsMapAndToken("etcd") if err != nil { return e, fmt.Errorf("error setting up initial cluster: %w", err) } } // AutoCompactionRetention defaults to "0" if not set. if len(cfg.AutoCompactionRetention) == 0 { cfg.AutoCompactionRetention = "0" } autoCompactionRetention, err := parseCompactionRetention(cfg.AutoCompactionMode, cfg.AutoCompactionRetention) if err != nil { return e, err } backendFreelistType := parseBackendFreelistType(cfg.BackendFreelistType) srvcfg := config.ServerConfig{ Name: cfg.Name, ClientURLs: cfg.AdvertiseClientUrls, PeerURLs: cfg.AdvertisePeerUrls, DataDir: cfg.Dir, DedicatedWALDir: cfg.WalDir, SnapshotCount: cfg.SnapshotCount, SnapshotCatchUpEntries: cfg.SnapshotCatchUpEntries, MaxSnapFiles: cfg.MaxSnapFiles, MaxWALFiles: cfg.MaxWalFiles, InitialPeerURLsMap: urlsmap, InitialClusterToken: token, DiscoveryCfg: cfg.DiscoveryCfg, NewCluster: cfg.IsNewCluster(), PeerTLSInfo: cfg.PeerTLSInfo, TickMs: cfg.TickMs, ElectionTicks: cfg.ElectionTicks(), InitialElectionTickAdvance: cfg.InitialElectionTickAdvance, AutoCompactionRetention: autoCompactionRetention, AutoCompactionMode: cfg.AutoCompactionMode, QuotaBackendBytes: cfg.QuotaBackendBytes, BackendBatchLimit: cfg.BackendBatchLimit, BackendFreelistType: backendFreelistType, BackendBatchInterval: cfg.BackendBatchInterval, MaxTxnOps: cfg.MaxTxnOps, MaxRequestBytes: cfg.MaxRequestBytes, MaxConcurrentStreams: cfg.MaxConcurrentStreams, SocketOpts: cfg.SocketOpts, StrictReconfigCheck: cfg.StrictReconfigCheck, ClientCertAuthEnabled: cfg.ClientTLSInfo.ClientCertAuth, AuthToken: cfg.AuthToken, BcryptCost: cfg.BcryptCost, TokenTTL: cfg.AuthTokenTTL, CORS: cfg.CORS, HostWhitelist: cfg.HostWhitelist, CorruptCheckTime: cfg.CorruptCheckTime, CompactHashCheckTime: cfg.CompactHashCheckTime, PreVote: cfg.PreVote, Logger: cfg.logger, ForceNewCluster: cfg.ForceNewCluster, EnableGRPCGateway: cfg.EnableGRPCGateway, EnableDistributedTracing: cfg.EnableDistributedTracing, UnsafeNoFsync: cfg.UnsafeNoFsync, CompactionBatchLimit: cfg.CompactionBatchLimit, CompactionSleepInterval: cfg.CompactionSleepInterval, WatchProgressNotifyInterval: cfg.WatchProgressNotifyInterval, DowngradeCheckTime: cfg.DowngradeCheckTime, WarningApplyDuration: cfg.WarningApplyDuration, WarningUnaryRequestDuration: cfg.WarningUnaryRequestDuration, MemoryMlock: cfg.MemoryMlock, BootstrapDefragThresholdMegabytes: cfg.BootstrapDefragThresholdMegabytes, MaxLearners: cfg.MaxLearners, V2Deprecation: cfg.V2DeprecationEffective(), LocalAddress: cfg.InferLocalAddr(), ServerFeatureGate: cfg.ServerFeatureGate, Metrics: cfg.Metrics, } if srvcfg.EnableDistributedTracing { tctx := context.Background() tracingExporter, terr := newTracingExporter(tctx, cfg) if terr != nil { return e, terr } e.tracingExporterShutdown = func() { tracingExporter.Close(tctx) } srvcfg.TracerOptions = tracingExporter.opts e.cfg.logger.Info( "distributed tracing setup enabled", ) } srvcfg.PeerTLSInfo.LocalAddr = srvcfg.LocalAddress print(e.cfg.logger, *cfg, srvcfg, memberInitialized) if e.Server, err = etcdserver.NewServer(srvcfg); err != nil { return e, err } // buffer channel so goroutines on closed connections won't wait forever e.errc = make(chan error, len(e.Peers)+len(e.Clients)+2*len(e.sctxs)) // newly started member ("memberInitialized==false") // does not need corruption check if memberInitialized && srvcfg.ServerFeatureGate.Enabled(features.InitialCorruptCheck) { if err = e.Server.CorruptionChecker().InitialCheck(); err != nil { // set "EtcdServer" to nil, so that it does not block on "EtcdServer.Close()" // (nothing to close since rafthttp transports have not been started) e.cfg.logger.Error("checkInitialHashKV failed", zap.Error(err)) e.Server.Cleanup() e.Server = nil return e, err } } e.Server.Start() e.servePeers() e.serveClients() if err = e.serveMetrics(); err != nil { return e, err } e.cfg.logger.Info( "now serving peer/client/metrics", zap.String("local-member-id", e.Server.MemberID().String()), zap.Strings("initial-advertise-peer-urls", e.cfg.getAdvertisePeerURLs()), zap.Strings("listen-peer-urls", e.cfg.getListenPeerURLs()), zap.Strings("advertise-client-urls", e.cfg.getAdvertiseClientURLs()), zap.Strings("listen-client-urls", e.cfg.getListenClientURLs()), zap.Strings("listen-metrics-urls", e.cfg.getMetricsURLs()), ) serving = true return e, nil } func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized bool) { cors := make([]string, 0, len(ec.CORS)) for v := range ec.CORS { cors = append(cors, v) } sort.Strings(cors) hss := make([]string, 0, len(ec.HostWhitelist)) for v := range ec.HostWhitelist { hss = append(hss, v) } sort.Strings(hss) quota := ec.QuotaBackendBytes if quota == 0 { quota = storage.DefaultQuotaBytes } lg.Info( "starting an etcd server", zap.String("etcd-version", version.Version), zap.String("git-sha", version.GitSHA), zap.String("go-version", runtime.Version()), zap.String("go-os", runtime.GOOS), zap.String("go-arch", runtime.GOARCH), zap.Int("max-cpu-set", runtime.GOMAXPROCS(0)), zap.Int("max-cpu-available", runtime.NumCPU()), zap.Bool("member-initialized", memberInitialized), zap.String("name", sc.Name), zap.String("data-dir", sc.DataDir), zap.String("wal-dir", ec.WalDir), zap.String("wal-dir-dedicated", sc.DedicatedWALDir), zap.String("member-dir", sc.MemberDir()), zap.Bool("force-new-cluster", sc.ForceNewCluster), zap.String("heartbeat-interval", fmt.Sprintf("%v", time.Duration(sc.TickMs)*time.Millisecond)), zap.String("election-timeout", fmt.Sprintf("%v", time.Duration(sc.ElectionTicks*int(sc.TickMs))*time.Millisecond)), zap.Bool("initial-election-tick-advance", sc.InitialElectionTickAdvance), zap.Uint64("snapshot-count", sc.SnapshotCount), zap.Uint("max-wals", sc.MaxWALFiles), zap.Uint("max-snapshots", sc.MaxSnapFiles), zap.Uint64("snapshot-catchup-entries", sc.SnapshotCatchUpEntries), zap.Strings("initial-advertise-peer-urls", ec.getAdvertisePeerURLs()), zap.Strings("listen-peer-urls", ec.getListenPeerURLs()), zap.Strings("advertise-client-urls", ec.getAdvertiseClientURLs()), zap.Strings("listen-client-urls", ec.getListenClientURLs()), zap.Strings("listen-metrics-urls", ec.getMetricsURLs()), zap.String("local-address", sc.LocalAddress), zap.Strings("cors", cors), zap.Strings("host-whitelist", hss), zap.String("initial-cluster", sc.InitialPeerURLsMap.String()), zap.String("initial-cluster-state", ec.ClusterState), zap.String("initial-cluster-token", sc.InitialClusterToken), zap.Int64("quota-backend-bytes", quota), zap.Uint("max-request-bytes", sc.MaxRequestBytes), zap.Uint32("max-concurrent-streams", sc.MaxConcurrentStreams), zap.Bool("pre-vote", sc.PreVote), zap.String(ServerFeatureGateFlagName, sc.ServerFeatureGate.String()), zap.Bool("initial-corrupt-check", sc.InitialCorruptCheck), zap.String("corrupt-check-time-interval", sc.CorruptCheckTime.String()), zap.Duration("compact-check-time-interval", sc.CompactHashCheckTime), zap.String("auto-compaction-mode", sc.AutoCompactionMode), zap.Duration("auto-compaction-retention", sc.AutoCompactionRetention), zap.String("auto-compaction-interval", sc.AutoCompactionRetention.String()), zap.String("discovery-token", sc.DiscoveryCfg.Token), zap.String("discovery-endpoints", strings.Join(sc.DiscoveryCfg.Endpoints, ",")), zap.String("discovery-dial-timeout", sc.DiscoveryCfg.DialTimeout.String()), zap.String("discovery-request-timeout", sc.DiscoveryCfg.RequestTimeout.String()), zap.String("discovery-keepalive-time", sc.DiscoveryCfg.KeepAliveTime.String()), zap.String("discovery-keepalive-timeout", sc.DiscoveryCfg.KeepAliveTimeout.String()), zap.Bool("discovery-insecure-transport", sc.DiscoveryCfg.Secure.InsecureTransport), zap.Bool("discovery-insecure-skip-tls-verify", sc.DiscoveryCfg.Secure.InsecureSkipVerify), zap.String("discovery-cert", sc.DiscoveryCfg.Secure.Cert), zap.String("discovery-key", sc.DiscoveryCfg.Secure.Key), zap.String("discovery-cacert", sc.DiscoveryCfg.Secure.Cacert), zap.String("discovery-user", sc.DiscoveryCfg.Auth.Username), zap.String("downgrade-check-interval", sc.DowngradeCheckTime.String()), zap.Int("max-learners", sc.MaxLearners), zap.String("v2-deprecation", string(ec.V2Deprecation)), ) } // Config returns the current configuration. func (e *Etcd) Config() Config { return e.cfg } // Close gracefully shuts down all servers/listeners. // Client requests will be terminated with request timeout. // After timeout, enforce remaning requests be closed immediately. // // The rough workflow to shut down etcd: // 1. close the `stopc` channel, so that all error handlers (child // goroutines) won't send back any errors anymore; // 2. stop the http and grpc servers gracefully, within request timeout; // 3. close all client and metrics listeners, so that etcd server // stops receiving any new connection; // 4. call the cancel function to close the gateway context, so that // all gateway connections are closed. // 5. stop etcd server gracefully, and ensure the main raft loop // goroutine is stopped; // 6. stop all peer listeners, so that it stops receiving peer connections // and messages (wait up to 1-second); // 7. wait for all child goroutines (i.e. client handlers, peer handlers // and metrics handlers) to exit; // 8. close the `errc` channel to release the resource. Note that it's only // safe to close the `errc` after step 7 above is done, otherwise the // child goroutines may send errors back to already closed `errc` channel. func (e *Etcd) Close() { fields := []zap.Field{ zap.String("name", e.cfg.Name), zap.String("data-dir", e.cfg.Dir), zap.Strings("advertise-peer-urls", e.cfg.getAdvertisePeerURLs()), zap.Strings("advertise-client-urls", e.cfg.getAdvertiseClientURLs()), } lg := e.GetLogger() lg.Info("closing etcd server", fields...) defer func() { lg.Info("closed etcd server", fields...) verify.MustVerifyIfEnabled(verify.Config{ Logger: lg, DataDir: e.cfg.Dir, ExactIndex: false, }) lg.Sync() }() e.closeOnce.Do(func() { close(e.stopc) }) // close client requests with request timeout timeout := 2 * time.Second if e.Server != nil { timeout = e.Server.Cfg.ReqTimeout() } for _, sctx := range e.sctxs { for ss := range sctx.serversC { ctx, cancel := context.WithTimeout(context.Background(), timeout) stopServers(ctx, ss) cancel() } } for _, sctx := range e.sctxs { sctx.cancel() } for i := range e.Clients { if e.Clients[i] != nil { e.Clients[i].Close() } } for i := range e.metricsListeners { e.metricsListeners[i].Close() } // shutdown tracing exporter if e.tracingExporterShutdown != nil { e.tracingExporterShutdown() } // close rafthttp transports if e.Server != nil { e.Server.Stop() } // close all idle connections in peer handler (wait up to 1-second) for i := range e.Peers { if e.Peers[i] != nil && e.Peers[i].close != nil { ctx, cancel := context.WithTimeout(context.Background(), time.Second) e.Peers[i].close(ctx) cancel() } } if e.errc != nil { e.wg.Wait() close(e.errc) } } func stopServers(ctx context.Context, ss *servers) { // first, close the http.Server if ss.http != nil { ss.http.Shutdown(ctx) } if ss.grpc == nil { return } // do not grpc.Server.GracefulStop when grpc runs under http server // See https://github.com/grpc/grpc-go/issues/1384#issuecomment-317124531 // and https://github.com/etcd-io/etcd/issues/8916 if ss.secure && ss.http != nil { ss.grpc.Stop() return } ch := make(chan struct{}) go func() { defer close(ch) // close listeners to stop accepting new connections, // will block on any existing transports ss.grpc.GracefulStop() }() // wait until all pending RPCs are finished select { case <-ch: case <-ctx.Done(): // took too long, manually close open transports // e.g. watch streams ss.grpc.Stop() // concurrent GracefulStop should be interrupted <-ch } } // Err - return channel used to report errors during etcd run/shutdown. // Since etcd 3.5 the channel is being closed when the etcd is over. func (e *Etcd) Err() <-chan error { return e.errc } func configurePeerListeners(cfg *Config) (peers []*peerListener, err error) { if err = updateCipherSuites(&cfg.PeerTLSInfo, cfg.CipherSuites); err != nil { return nil, err } if err = cfg.PeerSelfCert(); err != nil { cfg.logger.Fatal("failed to get peer self-signed certs", zap.Error(err)) } updateMinMaxVersions(&cfg.PeerTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion) if !cfg.PeerTLSInfo.Empty() { cfg.logger.Info( "starting with peer TLS", zap.String("tls-info", fmt.Sprintf("%+v", cfg.PeerTLSInfo)), zap.Strings("cipher-suites", cfg.CipherSuites), ) } peers = make([]*peerListener, len(cfg.ListenPeerUrls)) defer func() { if err == nil { return } for i := range peers { if peers[i] != nil && peers[i].close != nil { cfg.logger.Warn( "closing peer listener", zap.String("address", cfg.ListenPeerUrls[i].String()), zap.Error(err), ) ctx, cancel := context.WithTimeout(context.Background(), time.Second) peers[i].close(ctx) cancel() } } }() for i, u := range cfg.ListenPeerUrls { if u.Scheme == "http" { if !cfg.PeerTLSInfo.Empty() { cfg.logger.Warn("scheme is HTTP while key and cert files are present; ignoring key and cert files", zap.String("peer-url", u.String())) } if cfg.PeerTLSInfo.ClientCertAuth { cfg.logger.Warn("scheme is HTTP while --peer-client-cert-auth is enabled; ignoring client cert auth for this URL", zap.String("peer-url", u.String())) } } peers[i] = &peerListener{close: func(context.Context) error { return nil }} peers[i].Listener, err = transport.NewListenerWithOpts(u.Host, u.Scheme, transport.WithTLSInfo(&cfg.PeerTLSInfo), transport.WithSocketOpts(&cfg.SocketOpts), transport.WithTimeout(rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout), ) if err != nil { cfg.logger.Error("creating peer listener failed", zap.Error(err)) return nil, err } // once serve, overwrite with 'http.Server.Shutdown' peers[i].close = func(context.Context) error { return peers[i].Listener.Close() } } return peers, nil } // configure peer handlers after rafthttp.Transport started func (e *Etcd) servePeers() { ph := etcdhttp.NewPeerHandler(e.GetLogger(), e.Server) for _, p := range e.Peers { u := p.Listener.Addr().String() m := cmux.New(p.Listener) srv := &http.Server{ Handler: ph, ReadTimeout: 5 * time.Minute, ErrorLog: defaultLog.New(io.Discard, "", 0), // do not log user error } go srv.Serve(m.Match(cmux.Any())) p.serve = func() error { e.cfg.logger.Info( "cmux::serve", zap.String("address", u), ) return m.Serve() } p.close = func(ctx context.Context) error { // gracefully shutdown http.Server // close open listeners, idle connections // until context cancel or time-out e.cfg.logger.Info( "stopping serving peer traffic", zap.String("address", u), ) srv.Shutdown(ctx) e.cfg.logger.Info( "stopped serving peer traffic", zap.String("address", u), ) m.Close() return nil } } // start peer servers in a goroutine for _, pl := range e.Peers { l := pl e.startHandler(func() error { u := l.Addr().String() e.cfg.logger.Info( "serving peer traffic", zap.String("address", u), ) return l.serve() }) } } func configureClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err error) { if err = updateCipherSuites(&cfg.ClientTLSInfo, cfg.CipherSuites); err != nil { return nil, err } if err = cfg.ClientSelfCert(); err != nil { cfg.logger.Fatal("failed to get client self-signed certs", zap.Error(err)) } updateMinMaxVersions(&cfg.ClientTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion) if cfg.EnablePprof { cfg.logger.Info("pprof is enabled", zap.String("path", debugutil.HTTPPrefixPProf)) } sctxs = make(map[string]*serveCtx) for _, u := range append(cfg.ListenClientUrls, cfg.ListenClientHttpUrls...) { if u.Scheme == "http" || u.Scheme == "unix" { if !cfg.ClientTLSInfo.Empty() { cfg.logger.Warn("scheme is http or unix while key and cert files are present; ignoring key and cert files", zap.String("client-url", u.String())) } if cfg.ClientTLSInfo.ClientCertAuth { cfg.logger.Warn("scheme is http or unix while --client-cert-auth is enabled; ignoring client cert auth for this URL", zap.String("client-url", u.String())) } } if (u.Scheme == "https" || u.Scheme == "unixs") && cfg.ClientTLSInfo.Empty() { return nil, fmt.Errorf("TLS key/cert (--cert-file, --key-file) must be provided for client url %s with HTTPS scheme", u.String()) } } for _, u := range cfg.ListenClientUrls { addr, secure, network := resolveURL(u) sctx := sctxs[addr] if sctx == nil { sctx = newServeCtx(cfg.logger) sctxs[addr] = sctx } sctx.secure = sctx.secure || secure sctx.insecure = sctx.insecure || !secure sctx.scheme = u.Scheme sctx.addr = addr sctx.network = network } for _, u := range cfg.ListenClientHttpUrls { addr, secure, network := resolveURL(u) sctx := sctxs[addr] if sctx == nil { sctx = newServeCtx(cfg.logger) sctxs[addr] = sctx } else if !sctx.httpOnly { return nil, fmt.Errorf("cannot bind both --listen-client-urls and --listen-client-http-urls on the same url %s", u.String()) } sctx.secure = sctx.secure || secure sctx.insecure = sctx.insecure || !secure sctx.scheme = u.Scheme sctx.addr = addr sctx.network = network sctx.httpOnly = true } for _, sctx := range sctxs { if sctx.l, err = transport.NewListenerWithOpts(sctx.addr, sctx.scheme, transport.WithSocketOpts(&cfg.SocketOpts), transport.WithSkipTLSInfoCheck(true), ); err != nil { return nil, err } // net.Listener will rewrite ipv4 0.0.0.0 to ipv6 [::], breaking // hosts that disable ipv6. So, use the address given by the user. if fdLimit, fderr := runtimeutil.FDLimit(); fderr == nil { if fdLimit <= reservedInternalFDNum { cfg.logger.Fatal( "file descriptor limit of etcd process is too low; please set higher", zap.Uint64("limit", fdLimit), zap.Int("recommended-limit", reservedInternalFDNum), ) } sctx.l = transport.LimitListener(sctx.l, int(fdLimit-reservedInternalFDNum)) } defer func(sctx *serveCtx) { if err == nil || sctx.l == nil { return } sctx.l.Close() cfg.logger.Warn( "closing peer listener", zap.String("address", sctx.addr), zap.Error(err), ) }(sctx) for k := range cfg.UserHandlers { sctx.userHandlers[k] = cfg.UserHandlers[k] } sctx.serviceRegister = cfg.ServiceRegister if cfg.EnablePprof || cfg.LogLevel == "debug" { sctx.registerPprof() } if cfg.LogLevel == "debug" { sctx.registerTrace() } } return sctxs, nil } func resolveURL(u url.URL) (addr string, secure bool, network string) { addr = u.Host network = "tcp" if u.Scheme == "unix" || u.Scheme == "unixs" { addr = u.Host + u.Path network = "unix" } secure = u.Scheme == "https" || u.Scheme == "unixs" return addr, secure, network } func (e *Etcd) serveClients() { if !e.cfg.ClientTLSInfo.Empty() { e.cfg.logger.Info( "starting with client TLS", zap.String("tls-info", fmt.Sprintf("%+v", e.cfg.ClientTLSInfo)), zap.Strings("cipher-suites", e.cfg.CipherSuites), ) } // Start a client server goroutine for each listen address mux := http.NewServeMux() etcdhttp.HandleDebug(mux) etcdhttp.HandleVersion(mux, e.Server) etcdhttp.HandleMetrics(mux) etcdhttp.HandleHealth(e.cfg.logger, mux, e.Server) var gopts []grpc.ServerOption if e.cfg.GRPCKeepAliveMinTime > time.Duration(0) { gopts = append(gopts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ MinTime: e.cfg.GRPCKeepAliveMinTime, PermitWithoutStream: false, })) } if e.cfg.GRPCKeepAliveInterval > time.Duration(0) && e.cfg.GRPCKeepAliveTimeout > time.Duration(0) { gopts = append(gopts, grpc.KeepaliveParams(keepalive.ServerParameters{ Time: e.cfg.GRPCKeepAliveInterval, Timeout: e.cfg.GRPCKeepAliveTimeout, })) } gopts = append(gopts, e.cfg.GRPCAdditionalServerOptions...) splitHTTP := false for _, sctx := range e.sctxs { if sctx.httpOnly { splitHTTP = true } } // start client servers in each goroutine for _, sctx := range e.sctxs { s := sctx e.startHandler(func() error { return s.serve(e.Server, &e.cfg.ClientTLSInfo, mux, e.errHandler, e.grpcGatewayDial(splitHTTP), splitHTTP, gopts...) }) } } func (e *Etcd) grpcGatewayDial(splitHTTP bool) (grpcDial func(ctx context.Context) (*grpc.ClientConn, error)) { if !e.cfg.EnableGRPCGateway { return nil } sctx := e.pickGRPCGatewayServeContext(splitHTTP) addr := sctx.addr if network := sctx.network; network == "unix" { // explicitly define unix network for gRPC socket support addr = fmt.Sprintf("%s:%s", network, addr) } opts := []grpc.DialOption{grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32))} if sctx.secure { tlscfg, tlsErr := e.cfg.ClientTLSInfo.ServerConfig() if tlsErr != nil { return func(ctx context.Context) (*grpc.ClientConn, error) { return nil, tlsErr } } dtls := tlscfg.Clone() // trust local server dtls.InsecureSkipVerify = true opts = append(opts, grpc.WithTransportCredentials(credentials.NewTransportCredential(dtls))) } else { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } return func(_ context.Context) (*grpc.ClientConn, error) { conn, err := grpc.NewClient(addr, opts...) if err != nil { sctx.lg.Error("failed to setup grpc-gateway client", zap.String("addr", addr), zap.Error(err)) return nil, err } return conn, nil } } func (e *Etcd) pickGRPCGatewayServeContext(splitHTTP bool) *serveCtx { for _, sctx := range e.sctxs { if !splitHTTP || !sctx.httpOnly { return sctx } } panic("Expect at least one context able to serve grpc") } var ErrMissingClientTLSInfoForMetricsURL = errors.New("client TLS key/cert (--cert-file, --key-file) must be provided for metrics secure url") func (e *Etcd) createMetricsListener(murl url.URL) (net.Listener, error) { tlsInfo := &e.cfg.ClientTLSInfo switch murl.Scheme { case "http": tlsInfo = nil case "https", "unixs": if e.cfg.ClientTLSInfo.Empty() { return nil, ErrMissingClientTLSInfoForMetricsURL } } return transport.NewListenerWithOpts(murl.Host, murl.Scheme, transport.WithTLSInfo(tlsInfo), transport.WithSocketOpts(&e.cfg.SocketOpts), ) } func (e *Etcd) serveMetrics() (err error) { if len(e.cfg.ListenMetricsUrls) > 0 { metricsMux := http.NewServeMux() etcdhttp.HandleMetrics(metricsMux) etcdhttp.HandleHealth(e.cfg.logger, metricsMux, e.Server) for _, murl := range e.cfg.ListenMetricsUrls { u := murl ml, err := e.createMetricsListener(murl) if err != nil { return err } e.metricsListeners = append(e.metricsListeners, ml) e.startHandler(func() error { e.cfg.logger.Info( "serving metrics", zap.String("address", u.String()), ) return http.Serve(ml, metricsMux) }) } } return nil } func (e *Etcd) startHandler(handler func() error) { // start each handler in a separate goroutine e.wg.Add(1) go func() { defer e.wg.Done() e.errHandler(handler()) }() } func (e *Etcd) errHandler(err error) { if err != nil { e.GetLogger().Error("setting up serving from embedded etcd failed.", zap.Error(err)) } select { case <-e.stopc: return default: } select { case <-e.stopc: case e.errc <- err: } } // GetLogger returns the logger. func (e *Etcd) GetLogger() *zap.Logger { e.cfg.loggerMu.RLock() l := e.cfg.logger e.cfg.loggerMu.RUnlock() return l } func parseCompactionRetention(mode, retention string) (ret time.Duration, err error) { h, err := strconv.Atoi(retention) if err == nil && h >= 0 { switch mode { case CompactorModeRevision: ret = time.Duration(int64(h)) case CompactorModePeriodic: ret = time.Duration(int64(h)) * time.Hour case "": return 0, errors.New("--auto-compaction-mode is undefined") } } else { // periodic compaction ret, err = time.ParseDuration(retention) if err != nil { return 0, fmt.Errorf("error parsing CompactionRetention: %w", err) } } return ret, nil } ================================================ FILE: server/embed/etcd_test.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package embed import ( "net/url" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/transport" ) func TestEmptyClientTLSInfo_createMetricsListener(t *testing.T) { e := &Etcd{ cfg: Config{ ClientTLSInfo: transport.TLSInfo{}, }, } murl := url.URL{ Scheme: "https", Host: "localhost:8080", } _, err := e.createMetricsListener(murl) require.ErrorIsf(t, err, ErrMissingClientTLSInfoForMetricsURL, "expected error %v, got %v", ErrMissingClientTLSInfoForMetricsURL, err) } ================================================ FILE: server/embed/serve.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package embed import ( "context" "errors" "fmt" "io" defaultLog "log" "net" "net/http" "strings" "sync" gw "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/soheilhy/cmux" "github.com/tmc/grpc-websocket-proxy/wsproxy" "go.uber.org/zap" "golang.org/x/net/http2" "golang.org/x/net/trace" "google.golang.org/grpc" "google.golang.org/protobuf/encoding/protojson" etcdservergw "go.etcd.io/etcd/api/v3/etcdserverpb/gw" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/pkg/v3/debugutil" "go.etcd.io/etcd/pkg/v3/httputil" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" "go.etcd.io/etcd/server/v3/etcdserver/api/v3election" "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb" v3electiongw "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb/gw" "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock" "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb" v3lockgw "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb/gw" "go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc" ) type serveCtx struct { lg *zap.Logger l net.Listener scheme string addr string network string secure bool insecure bool httpOnly bool // ctx is used to control the grpc gateway. Terminate the grpc gateway // by calling `cancel` when shutting down the etcd. ctx context.Context cancel context.CancelFunc userHandlers map[string]http.Handler serviceRegister func(*grpc.Server) // serversC is used to receive the http and grpc server objects (created // in `serve`), both of which will be closed when shutting down the etcd. // Close it when `serve` returns or when etcd fails to bootstrap. serversC chan *servers // closeOnce is to ensure `serversC` is closed only once. closeOnce sync.Once // wg is used to track the lifecycle of all sub goroutines created by `serve`. wg sync.WaitGroup } func (sctx *serveCtx) startHandler(errHandler func(error), handler func() error) { // start each handler in a separate goroutine sctx.wg.Add(1) go func() { defer sctx.wg.Done() err := handler() if errHandler != nil { errHandler(err) } }() } type servers struct { secure bool grpc *grpc.Server http *http.Server } func newServeCtx(lg *zap.Logger) *serveCtx { ctx, cancel := context.WithCancel(context.Background()) if lg == nil { lg = zap.NewNop() } return &serveCtx{ lg: lg, ctx: ctx, cancel: cancel, userHandlers: make(map[string]http.Handler), serversC: make(chan *servers, 2), // in case sctx.insecure,sctx.secure true } } // serve accepts incoming connections on the listener l, // creating a new service goroutine for each. The service goroutines // read requests and then call handler to reply to them. func (sctx *serveCtx) serve( s *etcdserver.EtcdServer, tlsinfo *transport.TLSInfo, handler http.Handler, errHandler func(error), grpcDialForRestGatewayBackends func(ctx context.Context) (*grpc.ClientConn, error), splitHTTP bool, gopts ...grpc.ServerOption, ) (err error) { logger := defaultLog.New(io.Discard, "etcdhttp", 0) // Make sure serversC is closed even if we prematurely exit the function. defer sctx.close() select { case <-s.StoppingNotify(): return errors.New("server is stopping") case <-s.ReadyNotify(): } sctx.lg.Info("ready to serve client requests") m := cmux.New(sctx.l) var server func() error onlyGRPC := splitHTTP && !sctx.httpOnly onlyHTTP := splitHTTP && sctx.httpOnly grpcEnabled := !onlyHTTP httpEnabled := !onlyGRPC v3c := v3client.New(s) servElection := v3election.NewElectionServer(v3c) servLock := v3lock.NewLockServer(v3c) var gwmux *gw.ServeMux if s.Cfg.EnableGRPCGateway { // GRPC gateway connects to grpc server via connection provided by grpc dial. gwmux, err = sctx.registerGateway(grpcDialForRestGatewayBackends) if err != nil { sctx.lg.Error("registerGateway failed", zap.Error(err)) return err } } var traffic string switch { case onlyGRPC: traffic = "grpc" case onlyHTTP: traffic = "http" default: traffic = "grpc+http" } if sctx.insecure { var gs *grpc.Server var srv *http.Server if httpEnabled { httpmux := sctx.createMux(gwmux, handler) srv = &http.Server{ Handler: createAccessController(sctx.lg, s, httpmux), ErrorLog: logger, // do not log user error } if err = configureHTTPServer(srv, s.Cfg); err != nil { sctx.lg.Error("Configure http server failed", zap.Error(err)) return err } } if grpcEnabled { gs = v3rpc.Server(s, nil, nil, gopts...) v3electionpb.RegisterElectionServer(gs, servElection) v3lockpb.RegisterLockServer(gs, servLock) if sctx.serviceRegister != nil { sctx.serviceRegister(gs) } defer func(gs *grpc.Server) { if err != nil { sctx.lg.Warn("stopping insecure grpc server due to error", zap.Error(err)) gs.Stop() sctx.lg.Warn("stopped insecure grpc server due to error", zap.Error(err)) } }(gs) } if onlyGRPC { server = func() error { return gs.Serve(sctx.l) } } else { server = m.Serve httpl := m.Match(cmux.HTTP1()) sctx.startHandler(errHandler, func() error { return srv.Serve(httpl) }) if grpcEnabled { grpcl := m.Match(cmux.HTTP2()) sctx.startHandler(errHandler, func() error { return gs.Serve(grpcl) }) } } sctx.serversC <- &servers{grpc: gs, http: srv} sctx.lg.Info( "serving client traffic insecurely; this is strongly discouraged!", zap.String("traffic", traffic), zap.String("address", sctx.l.Addr().String()), ) } if sctx.secure { var gs *grpc.Server var srv *http.Server tlscfg, tlsErr := tlsinfo.ServerConfig() if tlsErr != nil { return tlsErr } if grpcEnabled { gs = v3rpc.Server(s, tlscfg, nil, gopts...) v3electionpb.RegisterElectionServer(gs, servElection) v3lockpb.RegisterLockServer(gs, servLock) if sctx.serviceRegister != nil { sctx.serviceRegister(gs) } defer func(gs *grpc.Server) { if err != nil { sctx.lg.Warn("stopping secure grpc server due to error", zap.Error(err)) gs.Stop() sctx.lg.Warn("stopped secure grpc server due to error", zap.Error(err)) } }(gs) } if httpEnabled { if grpcEnabled { handler = grpcHandlerFunc(gs, handler) } httpmux := sctx.createMux(gwmux, handler) srv = &http.Server{ Handler: createAccessController(sctx.lg, s, httpmux), TLSConfig: tlscfg, ErrorLog: logger, // do not log user error } if err = configureHTTPServer(srv, s.Cfg); err != nil { sctx.lg.Error("Configure https server failed", zap.Error(err)) return err } } if onlyGRPC { server = func() error { return gs.Serve(sctx.l) } } else { server = m.Serve tlsl, tlsErr := transport.NewTLSListener(m.Match(cmux.Any()), tlsinfo) if tlsErr != nil { return tlsErr } sctx.startHandler(errHandler, func() error { return srv.Serve(tlsl) }) } sctx.serversC <- &servers{secure: true, grpc: gs, http: srv} sctx.lg.Info( "serving client traffic securely", zap.String("traffic", traffic), zap.String("address", sctx.l.Addr().String()), ) } err = server() sctx.close() sctx.wg.Wait() return err } func configureHTTPServer(srv *http.Server, cfg config.ServerConfig) error { // todo (ahrtr): should we support configuring other parameters in the future as well? return http2.ConfigureServer(srv, &http2.Server{ MaxConcurrentStreams: cfg.MaxConcurrentStreams, }) } // grpcHandlerFunc returns an http.Handler that delegates to grpcServer on incoming gRPC // connections or otherHandler otherwise. Given in gRPC docs. func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Handler { if otherHandler == nil { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { grpcServer.ServeHTTP(w, r) }) } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.ProtoMajor == 2 && strings.Contains(r.Header.Get("Content-Type"), "application/grpc") { grpcServer.ServeHTTP(w, r) } else { otherHandler.ServeHTTP(w, r) } }) } type registerHandlerFunc func(context.Context, *gw.ServeMux, *grpc.ClientConn) error func (sctx *serveCtx) registerGateway(dial func(ctx context.Context) (*grpc.ClientConn, error)) (*gw.ServeMux, error) { ctx := sctx.ctx conn, err := dial(ctx) if err != nil { return nil, err } // Refer to https://grpc-ecosystem.github.io/grpc-gateway/docs/mapping/customizing_your_gateway/ gwmux := gw.NewServeMux( gw.WithMarshalerOption(gw.MIMEWildcard, &gw.HTTPBodyMarshaler{ Marshaler: &gw.JSONPb{ MarshalOptions: protojson.MarshalOptions{ UseProtoNames: true, EmitUnpopulated: false, }, UnmarshalOptions: protojson.UnmarshalOptions{ DiscardUnknown: true, }, }, }, ), ) handlers := []registerHandlerFunc{ etcdservergw.RegisterKVHandler, etcdservergw.RegisterWatchHandler, etcdservergw.RegisterLeaseHandler, etcdservergw.RegisterClusterHandler, etcdservergw.RegisterMaintenanceHandler, etcdservergw.RegisterAuthHandler, v3lockgw.RegisterLockHandler, v3electiongw.RegisterElectionHandler, } for _, h := range handlers { if err := h(ctx, gwmux, conn); err != nil { return nil, err } } sctx.startHandler(nil, func() error { <-ctx.Done() if cerr := conn.Close(); cerr != nil { sctx.lg.Warn( "failed to close connection", zap.String("address", sctx.l.Addr().String()), zap.Error(cerr), ) } return nil }) return gwmux, nil } type wsProxyZapLogger struct { *zap.Logger } func (w wsProxyZapLogger) Warnln(i ...any) { w.Warn(fmt.Sprint(i...)) } func (w wsProxyZapLogger) Debugln(i ...any) { w.Debug(fmt.Sprint(i...)) } func (sctx *serveCtx) createMux(gwmux *gw.ServeMux, handler http.Handler) *http.ServeMux { httpmux := http.NewServeMux() for path, h := range sctx.userHandlers { httpmux.Handle(path, h) } if gwmux != nil { httpmux.Handle( "/v3/", wsproxy.WebsocketProxy( gwmux, wsproxy.WithRequestMutator( // Default to the POST method for streams func(_ *http.Request, outgoing *http.Request) *http.Request { outgoing.Method = "POST" return outgoing }, ), wsproxy.WithMaxRespBodyBufferSize(0x7fffffff), wsproxy.WithLogger(wsProxyZapLogger{sctx.lg}), ), ) } if handler != nil { httpmux.Handle("/", handler) } return httpmux } // createAccessController wraps HTTP multiplexer: // - mutate gRPC gateway request paths // - check hostname whitelist // client HTTP requests goes here first func createAccessController(lg *zap.Logger, s *etcdserver.EtcdServer, mux *http.ServeMux) http.Handler { if lg == nil { lg = zap.NewNop() } return &accessController{lg: lg, s: s, mux: mux} } type accessController struct { lg *zap.Logger s *etcdserver.EtcdServer mux *http.ServeMux } func (ac *accessController) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if req == nil { http.Error(rw, "Request is nil", http.StatusBadRequest) return } // redirect for backward compatibilities if req.URL != nil && strings.HasPrefix(req.URL.Path, "/v3beta/") { req.URL.Path = strings.Replace(req.URL.Path, "/v3beta/", "/v3/", 1) } if req.TLS == nil { // check origin if client connection is not secure host := httputil.GetHostname(req) if !ac.s.AccessController.IsHostWhitelisted(host) { ac.lg.Warn( "rejecting HTTP request to prevent DNS rebinding attacks", zap.String("host", host), ) http.Error(rw, errCVE20185702(host), http.StatusMisdirectedRequest) return } } else if ac.s.Cfg.ClientCertAuthEnabled && ac.s.Cfg.EnableGRPCGateway && ac.s.AuthStore().IsAuthEnabled() && strings.HasPrefix(req.URL.Path, "/v3/") { for _, chains := range req.TLS.VerifiedChains { if len(chains) < 1 { continue } if len(chains[0].Subject.CommonName) != 0 { http.Error(rw, "CommonName of client sending a request against gateway will be ignored and not used as expected", http.StatusBadRequest) return } } } // Write CORS header. if ac.s.AccessController.OriginAllowed("*") { addCORSHeader(rw, "*") } else if origin := req.Header.Get("Origin"); ac.s.OriginAllowed(origin) { addCORSHeader(rw, origin) } if req.Method == http.MethodOptions { rw.WriteHeader(http.StatusOK) return } ac.mux.ServeHTTP(rw, req) } // addCORSHeader adds the correct cors headers given an origin func addCORSHeader(w http.ResponseWriter, origin string) { w.Header().Add("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE") w.Header().Add("Access-Control-Allow-Origin", origin) w.Header().Add("Access-Control-Allow-Headers", "accept, content-type, authorization") } // https://github.com/transmission/transmission/pull/468 func errCVE20185702(host string) string { return fmt.Sprintf(` etcd received your request, but the Host header was unrecognized. To fix this, choose one of the following options: - Enable TLS, then any HTTPS request will be allowed. - Add the hostname you want to use to the whitelist in settings. - e.g. etcd --host-whitelist %q This requirement has been added to help prevent "DNS Rebinding" attacks (CVE-2018-5702). `, host) } // WrapCORS wraps existing handler with CORS. // TODO: deprecate this after v2 proxy deprecate func WrapCORS(cors map[string]struct{}, h http.Handler) http.Handler { return &corsHandler{ ac: &etcdserver.AccessController{CORS: cors}, h: h, } } type corsHandler struct { ac *etcdserver.AccessController h http.Handler } func (ch *corsHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if ch.ac.OriginAllowed("*") { addCORSHeader(rw, "*") } else if origin := req.Header.Get("Origin"); ch.ac.OriginAllowed(origin) { addCORSHeader(rw, origin) } if req.Method == http.MethodOptions { rw.WriteHeader(http.StatusOK) return } ch.h.ServeHTTP(rw, req) } func (sctx *serveCtx) registerUserHandler(s string, h http.Handler) { if sctx.userHandlers[s] != nil { sctx.lg.Warn("path is already registered by user handler", zap.String("path", s)) return } sctx.userHandlers[s] = h } func (sctx *serveCtx) registerPprof() { for p, h := range debugutil.PProfHandlers() { sctx.registerUserHandler(p, h) } } func (sctx *serveCtx) registerTrace() { reqf := func(w http.ResponseWriter, r *http.Request) { trace.Render(w, r, true) } sctx.registerUserHandler("/debug/requests", http.HandlerFunc(reqf)) evf := func(w http.ResponseWriter, r *http.Request) { trace.RenderEvents(w, r, true) } sctx.registerUserHandler("/debug/events", http.HandlerFunc(evf)) } func (sctx *serveCtx) close() { sctx.closeOnce.Do(func() { close(sctx.serversC) }) } ================================================ FILE: server/embed/serve_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package embed import ( "fmt" "net/url" "os" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/server/v3/auth" ) // TestStartEtcdWrongToken ensures that StartEtcd with wrong configs returns with error. func TestStartEtcdWrongToken(t *testing.T) { tdir := t.TempDir() cfg := NewConfig() // Similar to function in integration/embed/embed_test.go for setting up Config. urls := newEmbedURLs(2) curls := []url.URL{urls[0]} purls := []url.URL{urls[1]} cfg.ListenClientUrls, cfg.AdvertiseClientUrls = curls, curls cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = purls, purls cfg.InitialCluster = "" for i := range purls { cfg.InitialCluster += ",default=" + purls[i].String() } cfg.InitialCluster = cfg.InitialCluster[1:] cfg.Dir = tdir cfg.AuthToken = "wrong-token" _, err := StartEtcd(cfg) require.ErrorIsf(t, err, auth.ErrInvalidAuthOpts, "expected %v, got %v", auth.ErrInvalidAuthOpts, err) } func newEmbedURLs(n int) (urls []url.URL) { scheme := "unix" for i := 0; i < n; i++ { u, _ := url.Parse(fmt.Sprintf("%s://localhost:%d%06d", scheme, os.Getpid(), i)) urls = append(urls, *u) } return urls } ================================================ FILE: server/embed/util.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package embed import ( "path/filepath" "go.etcd.io/etcd/server/v3/storage/wal" ) func isMemberInitialized(cfg *Config) bool { walDir := cfg.WalDir if walDir == "" { walDir = filepath.Join(cfg.Dir, "member", "wal") } return wal.Exist(walDir) } ================================================ FILE: server/etcdmain/config.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Every change should be reflected on help.go as well. package etcdmain import ( "errors" "flag" "fmt" "os" "runtime" "time" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/pkg/v3/flags" cconfig "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" ) var ( fallbackFlagExit = "exit" fallbackFlagProxy = "proxy" ignored = []string{ "cluster-active-size", "cluster-remove-delay", "cluster-sync-interval", "config", "force", "max-result-buffer", "max-retry-attempts", "peer-heartbeat-interval", "peer-election-timeout", "retry-interval", "snapshot", "v", "vv", // for coverage testing "test.coverprofile", "test.outputdir", } deprecatedFlags = map[string]string{ "max-snapshots": "--max-snapshots is deprecated in 3.6 and will be decommissioned in 3.8.", "v2-deprecation": "--v2-deprecation is deprecated and scheduled for removal in v3.8. The default value is enforced, ignoring user input.", } ) // config holds the config for a command line invocation of etcd type config struct { ec embed.Config cf configFlags configFile string printVersion bool ignored []string } // configFlags has the set of flags used for command line parsing a Config type configFlags struct { flagSet *flag.FlagSet clusterState *flags.SelectiveStringValue fallback *flags.SelectiveStringValue // Deprecated and scheduled for removal in v3.8. The default value is enforced, ignoring user input. // TODO: remove in v3.8. v2deprecation *flags.SelectiveStringsValue } func newConfig() *config { cfg := &config{ ec: *embed.NewConfig(), ignored: ignored, } cfg.cf = configFlags{ flagSet: flag.NewFlagSet("etcd", flag.ContinueOnError), clusterState: flags.NewSelectiveStringValue( embed.ClusterStateFlagNew, embed.ClusterStateFlagExisting, ), fallback: flags.NewSelectiveStringValue( fallbackFlagExit, fallbackFlagProxy, ), v2deprecation: flags.NewSelectiveStringsValue( string(cconfig.V2Depr1WriteOnly), string(cconfig.V2Depr1WriteOnlyDrop), string(cconfig.V2Depr2Gone)), } fs := cfg.cf.flagSet fs.Usage = func() { fmt.Fprintln(os.Stderr, usageline) } cfg.ec.AddFlags(fs) fs.StringVar(&cfg.configFile, "config-file", "", "Path to the server configuration file. Note that if a configuration file is provided, other command line flags and environment variables will be ignored.") fs.Var(cfg.cf.fallback, "discovery-fallback", fmt.Sprintf("Valid values include %q", cfg.cf.fallback.Valids())) fs.Var(cfg.cf.clusterState, "initial-cluster-state", "Initial cluster state ('new' when bootstrapping a new cluster or 'existing' when adding new members to an existing cluster). After successful initialization (bootstrapping or adding), flag is ignored on restarts.") fs.Var(cfg.cf.v2deprecation, "v2-deprecation", fmt.Sprintf("v2store deprecation stage: %q. Deprecated and scheduled for removal in v3.8. The default value is enforced, ignoring user input.", cfg.cf.v2deprecation.Valids())) fs.BoolVar(&cfg.printVersion, "version", false, "Print the version and exit.") // ignored for _, f := range cfg.ignored { fs.Var(&flags.IgnoredFlag{Name: f}, f, "") } return cfg } func (cfg *config) parse(arguments []string) error { perr := cfg.cf.flagSet.Parse(arguments) switch { case perr == nil: case errors.Is(perr, flag.ErrHelp): fmt.Println(flagsline) os.Exit(0) default: os.Exit(2) } if len(cfg.cf.flagSet.Args()) != 0 { return fmt.Errorf("%q is not a valid flag", cfg.cf.flagSet.Arg(0)) } if cfg.printVersion { fmt.Printf("etcd Version: %s\n", version.Version) fmt.Printf("Git SHA: %s\n", version.GitSHA) fmt.Printf("Go Version: %s\n", runtime.Version()) fmt.Printf("Go OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) os.Exit(0) } var err error // This env variable must be parsed separately // because we need to determine whether to use or // ignore the env variables based on if the config file is set. if cfg.configFile == "" { cfg.configFile = os.Getenv(flags.FlagToEnv("ETCD", "config-file")) } if cfg.configFile != "" { err = cfg.configFromFile(cfg.configFile) if lg := cfg.ec.GetLogger(); lg != nil { lg.Info( "loaded server configuration, other configuration command line flags and environment variables will be ignored if provided", zap.String("path", cfg.configFile), ) } } else { err = cfg.configFromCmdLine() } // `V2Deprecation` (--v2-deprecation) is deprecated and scheduled for removal in v3.8. The default value is enforced, ignoring user input. cfg.ec.V2Deprecation = cconfig.V2DeprDefault cfg.ec.WarningUnaryRequestDuration = cfg.parseWarningUnaryRequestDuration() // Check for deprecated options from both command line and config file var warningsForDeprecatedOpts []string for flagName := range cfg.ec.FlagsExplicitlySet { if msg, ok := deprecatedFlags[flagName]; ok { warningsForDeprecatedOpts = append(warningsForDeprecatedOpts, msg) } } // Log warnings if any deprecated options were found if len(warningsForDeprecatedOpts) > 0 { if lg := cfg.ec.GetLogger(); lg != nil { for _, msg := range warningsForDeprecatedOpts { lg.Warn(msg) } } } return err } func (cfg *config) configFromCmdLine() error { // user-specified logger is not setup yet, use this logger during flag parsing lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { return err } verKey := "ETCD_VERSION" if verVal := os.Getenv(verKey); verVal != "" { // unset to avoid any possible side-effect. os.Unsetenv(verKey) lg.Warn( "cannot set special environment variable", zap.String("key", verKey), zap.String("value", verVal), ) } err = flags.SetFlagsFromEnv(lg, "ETCD", cfg.cf.flagSet) if err != nil { return err } if rafthttp.ConnReadTimeout < rafthttp.DefaultConnReadTimeout { rafthttp.ConnReadTimeout = rafthttp.DefaultConnReadTimeout lg.Info(fmt.Sprintf("raft-read-timeout increased to minimum value: %v", rafthttp.DefaultConnReadTimeout)) } if rafthttp.ConnWriteTimeout < rafthttp.DefaultConnWriteTimeout { rafthttp.ConnWriteTimeout = rafthttp.DefaultConnWriteTimeout lg.Info(fmt.Sprintf("raft-write-timeout increased to minimum value: %v", rafthttp.DefaultConnWriteTimeout)) } cfg.ec.ListenPeerUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-peer-urls") cfg.ec.AdvertisePeerUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "initial-advertise-peer-urls") cfg.ec.ListenClientUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-client-urls") cfg.ec.ListenClientHttpUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-client-http-urls") cfg.ec.AdvertiseClientUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "advertise-client-urls") cfg.ec.ListenMetricsUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-metrics-urls") cfg.ec.DiscoveryCfg.Endpoints = flags.UniqueStringsFromFlag(cfg.cf.flagSet, "discovery-endpoints") cfg.ec.CORS = flags.UniqueURLsMapFromFlag(cfg.cf.flagSet, "cors") cfg.ec.HostWhitelist = flags.UniqueStringsMapFromFlag(cfg.cf.flagSet, "host-whitelist") cfg.ec.ClientTLSInfo.AllowedHostnames = flags.StringsFromFlag(cfg.cf.flagSet, "client-cert-allowed-hostname") cfg.ec.PeerTLSInfo.AllowedCNs = flags.StringsFromFlag(cfg.cf.flagSet, "peer-cert-allowed-cn") cfg.ec.PeerTLSInfo.AllowedHostnames = flags.StringsFromFlag(cfg.cf.flagSet, "peer-cert-allowed-hostname") cfg.ec.CipherSuites = flags.StringsFromFlag(cfg.cf.flagSet, "cipher-suites") cfg.ec.MaxConcurrentStreams = flags.Uint32FromFlag(cfg.cf.flagSet, "max-concurrent-streams") cfg.ec.LogOutputs = flags.UniqueStringsFromFlag(cfg.cf.flagSet, "log-outputs") cfg.ec.ClusterState = cfg.cf.clusterState.String() cfg.ec.V2Deprecation = cconfig.V2DeprecationEnum(cfg.cf.v2deprecation.String()) // disable default advertise-client-urls if lcurls is set missingAC := flags.IsSet(cfg.cf.flagSet, "listen-client-urls") && !flags.IsSet(cfg.cf.flagSet, "advertise-client-urls") if missingAC { cfg.ec.AdvertiseClientUrls = nil } // disable default initial-cluster if discovery is set if (cfg.ec.DNSCluster != "" || cfg.ec.DNSClusterServiceName != "" || len(cfg.ec.DiscoveryCfg.Endpoints) > 0) && !flags.IsSet(cfg.cf.flagSet, "initial-cluster") { cfg.ec.InitialCluster = "" } cfg.cf.flagSet.Visit(func(f *flag.Flag) { cfg.ec.FlagsExplicitlySet[f.Name] = true }) return cfg.validate() } func (cfg *config) configFromFile(path string) error { eCfg, err := embed.ConfigFromFile(path) if err != nil { return err } cfg.ec = *eCfg return nil } func (cfg *config) validate() error { if cfg.cf.fallback.String() == fallbackFlagProxy { return fmt.Errorf("v2 proxy is deprecated, and --discovery-fallback can't be configured as %q", fallbackFlagProxy) } return cfg.ec.Validate() } func (cfg *config) parseWarningUnaryRequestDuration() time.Duration { if cfg.ec.WarningUnaryRequestDuration != 0 { return cfg.ec.WarningUnaryRequestDuration } return embed.DefaultWarningUnaryRequestDuration } ================================================ FILE: server/etcdmain/config_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdmain import ( "errors" "flag" "fmt" "net/url" "os" "reflect" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sigs.k8s.io/yaml" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/featuregate" "go.etcd.io/etcd/pkg/v3/flags" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver/api/v3discovery" "go.etcd.io/etcd/server/v3/features" ) func TestConfigParsingMemberFlags(t *testing.T) { args := []string{ "-data-dir=testdir", "-name=testname", "-max-wals=10", "-max-snapshots=10", "-snapshot-count=10", "-snapshot-catchup-entries=1000", "-listen-peer-urls=http://localhost:8000,https://localhost:8001", "-listen-client-urls=http://localhost:7000,https://localhost:7001", "-listen-client-http-urls=http://localhost:7002,https://localhost:7003", // it should be set if -listen-client-urls is set "-advertise-client-urls=http://localhost:7000,https://localhost:7001", } cfg := newConfig() err := cfg.parse(args) if err != nil { t.Fatal(err) } validateMemberFlags(t, cfg) } func TestConfigFileMemberFields(t *testing.T) { yc := struct { Dir string `json:"data-dir"` MaxSnapFiles uint `json:"max-snapshots"` MaxWALFiles uint `json:"max-wals"` Name string `json:"name"` SnapshotCount uint64 `json:"snapshot-count"` SnapshotCatchUpEntries uint64 `json:"snapshot-catchup-entries"` ListenPeerURLs string `json:"listen-peer-urls"` ListenClientURLs string `json:"listen-client-urls"` ListenClientHTTPURLs string `json:"listen-client-http-urls"` AdvertiseClientURLs string `json:"advertise-client-urls"` }{ "testdir", 10, 10, "testname", 10, 1000, "http://localhost:8000,https://localhost:8001", "http://localhost:7000,https://localhost:7001", "http://localhost:7002,https://localhost:7003", "http://localhost:7000,https://localhost:7001", } b, err := yaml.Marshal(&yc) if err != nil { t.Fatal(err) } tmpfile := mustCreateCfgFile(t, b) defer os.Remove(tmpfile.Name()) args := []string{fmt.Sprintf("--config-file=%s", tmpfile.Name())} cfg := newConfig() if err = cfg.parse(args); err != nil { t.Fatal(err) } validateMemberFlags(t, cfg) } func TestConfigParsingClusteringFlags(t *testing.T) { args := []string{ "-initial-cluster=0=http://localhost:8000", "-initial-cluster-state=existing", "-initial-cluster-token=etcdtest", "-initial-advertise-peer-urls=http://localhost:8000,https://localhost:8001", "-advertise-client-urls=http://localhost:7000,https://localhost:7001", } cfg := newConfig() if err := cfg.parse(args); err != nil { t.Fatal(err) } validateClusteringFlags(t, cfg) } func TestConfigFileClusteringFields(t *testing.T) { yc := struct { InitialCluster string `json:"initial-cluster"` ClusterState string `json:"initial-cluster-state"` InitialClusterToken string `json:"initial-cluster-token"` AdvertisePeerUrls string `json:"initial-advertise-peer-urls"` AdvertiseClientUrls string `json:"advertise-client-urls"` }{ "0=http://localhost:8000", "existing", "etcdtest", "http://localhost:8000,https://localhost:8001", "http://localhost:7000,https://localhost:7001", } b, err := yaml.Marshal(&yc) if err != nil { t.Fatal(err) } tmpfile := mustCreateCfgFile(t, b) defer os.Remove(tmpfile.Name()) args := []string{fmt.Sprintf("--config-file=%s", tmpfile.Name())} cfg := newConfig() err = cfg.parse(args) if err != nil { t.Fatal(err) } validateClusteringFlags(t, cfg) } func TestConfigFileClusteringFlags(t *testing.T) { tests := []struct { Name string `json:"name"` InitialCluster string `json:"initial-cluster"` DNSCluster string `json:"discovery-srv"` Durl string `json:"discovery"` }{ // Use default name and generate a default initial-cluster {}, { Name: "non-default", }, { InitialCluster: "0=localhost:8000", }, { Name: "non-default", InitialCluster: "0=localhost:8000", }, { DNSCluster: "example.com", }, { Name: "non-default", DNSCluster: "example.com", }, { Durl: "http://example.com/abc", }, { Name: "non-default", Durl: "http://example.com/abc", }, } for i, tt := range tests { b, err := yaml.Marshal(&tt) if err != nil { t.Fatal(err) } tmpfile := mustCreateCfgFile(t, b) defer os.Remove(tmpfile.Name()) args := []string{fmt.Sprintf("--config-file=%s", tmpfile.Name())} cfg := newConfig() if err := cfg.parse(args); err != nil { t.Errorf("%d: err = %v", i, err) } } } func TestConfigParsingConflictClusteringFlags(t *testing.T) { conflictArgs := [][]string{ { "--initial-cluster=0=localhost:8000", "--discovery-endpoints=http://example.com/abc", }, { "--discovery-srv=example.com", "--discovery-endpoints=http://example.com/abc", }, { "--initial-cluster=0=localhost:8000", "--discovery-srv=example.com", }, { "--initial-cluster=0=localhost:8000", "--discovery-endpoints=http://example.com/abc", "--discovery-srv=example.com", }, } for i, tt := range conflictArgs { cfg := newConfig() if err := cfg.parse(tt); !errors.Is(err, embed.ErrConflictBootstrapFlags) { t.Errorf("%d: err = %v, want %v", i, err, embed.ErrConflictBootstrapFlags) } } } func TestConfigFileConflictClusteringFlags(t *testing.T) { tests := []struct { InitialCluster string `json:"initial-cluster"` DNSCluster string `json:"discovery-srv"` DiscoveryCfg v3discovery.DiscoveryConfig `json:"discovery-config"` }{ { InitialCluster: "0=localhost:8000", DiscoveryCfg: v3discovery.DiscoveryConfig{ ConfigSpec: clientv3.ConfigSpec{Endpoints: []string{"http://example.com/abc"}}, }, }, { DNSCluster: "example.com", DiscoveryCfg: v3discovery.DiscoveryConfig{ ConfigSpec: clientv3.ConfigSpec{Endpoints: []string{"http://example.com/abc"}}, }, }, { InitialCluster: "0=localhost:8000", DNSCluster: "example.com", }, { InitialCluster: "0=localhost:8000", DiscoveryCfg: v3discovery.DiscoveryConfig{ ConfigSpec: clientv3.ConfigSpec{Endpoints: []string{"http://example.com/abc"}}, }, DNSCluster: "example.com", }, } for i, tt := range tests { b, err := yaml.Marshal(&tt) if err != nil { t.Fatal(err) } tmpfile := mustCreateCfgFile(t, b) defer os.Remove(tmpfile.Name()) args := []string{fmt.Sprintf("--config-file=%s", tmpfile.Name())} cfg := newConfig() if err := cfg.parse(args); !errors.Is(err, embed.ErrConflictBootstrapFlags) { t.Errorf("%d: err = %v, want %v", i, err, embed.ErrConflictBootstrapFlags) } } } func TestConfigParsingMissedAdvertiseClientURLsFlag(t *testing.T) { tests := []struct { args []string werr error }{ { []string{ "--initial-cluster=infra1=http://127.0.0.1:2380", "--listen-client-urls=http://127.0.0.1:2379", }, embed.ErrUnsetAdvertiseClientURLsFlag, }, { []string{ "--discovery-srv=example.com", "--listen-client-urls=http://127.0.0.1:2379", }, embed.ErrUnsetAdvertiseClientURLsFlag, }, { []string{ "--discovery-fallback=exit", "--listen-client-urls=http://127.0.0.1:2379", }, embed.ErrUnsetAdvertiseClientURLsFlag, }, { []string{ "--listen-client-urls=http://127.0.0.1:2379", }, embed.ErrUnsetAdvertiseClientURLsFlag, }, } for i, tt := range tests { cfg := newConfig() if err := cfg.parse(tt.args); !errors.Is(err, tt.werr) { t.Errorf("%d: err = %v, want %v", i, err, tt.werr) } } } func TestConfigIsNewCluster(t *testing.T) { tests := []struct { state string wIsNew bool }{ {embed.ClusterStateFlagExisting, false}, {embed.ClusterStateFlagNew, true}, } for i, tt := range tests { cfg := newConfig() args := []string{"--initial-cluster-state", tests[i].state} err := cfg.parse(args) require.NoErrorf(t, err, "#%d: unexpected clusterState.Set error: %v", i, err) if g := cfg.ec.IsNewCluster(); g != tt.wIsNew { t.Errorf("#%d: isNewCluster = %v, want %v", i, g, tt.wIsNew) } } } func TestConfigFileElectionTimeout(t *testing.T) { tests := []struct { TickMs uint `json:"heartbeat-interval"` ElectionMs uint `json:"election-timeout"` errStr string }{ { ElectionMs: 1000, TickMs: 800, errStr: "should be at least as 5 times as", }, { ElectionMs: 60000, TickMs: 10000, errStr: "is too long, and should be set less than", }, { ElectionMs: 100, TickMs: 0, errStr: "--heartbeat-interval must be >0 (set to 0ms)", }, { ElectionMs: 0, TickMs: 100, errStr: "--election-timeout must be >0 (set to 0ms)", }, } for i, tt := range tests { b, err := yaml.Marshal(&tt) if err != nil { t.Fatal(err) } tmpfile := mustCreateCfgFile(t, b) defer os.Remove(tmpfile.Name()) args := []string{fmt.Sprintf("--config-file=%s", tmpfile.Name())} cfg := newConfig() if err := cfg.parse(args); err == nil || !strings.Contains(err.Error(), tt.errStr) { t.Errorf("%d: Wrong err = %v", i, err) } } } func TestFlagsPresentInHelp(t *testing.T) { cfg := newConfig() cfg.cf.flagSet.VisitAll(func(f *flag.Flag) { if _, ok := f.Value.(*flags.IgnoredFlag); ok { // Ignored flags do not need to be in the help return } flagText := fmt.Sprintf("--%s", f.Name) if !strings.Contains(flagsline, flagText) && !strings.Contains(usageline, flagText) { t.Errorf("Neither flagsline nor usageline in help.go contains flag named %s", flagText) } }) } func TestParseFeatureGateFlags(t *testing.T) { testCases := []struct { name string args []string expectErr bool expectedFeatures map[featuregate.Feature]bool }{ { name: "default", expectedFeatures: map[featuregate.Feature]bool{ features.StopGRPCServiceOnDefrag: false, }, }, { name: "can set feature gate from feature gate flag", args: []string{ "--feature-gates=StopGRPCServiceOnDefrag=true,InitialCorruptCheck=true", }, expectedFeatures: map[featuregate.Feature]bool{ features.StopGRPCServiceOnDefrag: true, features.InitialCorruptCheck: true, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { cfg := newConfig() err := cfg.parse(tc.args) if tc.expectErr { require.Errorf(t, err, "expect parse error") return } if err != nil { t.Fatal(err) } for k, v := range tc.expectedFeatures { if cfg.ec.ServerFeatureGate.Enabled(k) != v { t.Errorf("expected feature gate %s=%v, got %v", k, v, cfg.ec.ServerFeatureGate.Enabled(k)) } } }) } } func mustCreateCfgFile(t *testing.T, b []byte) *os.File { tmpfile, err := os.CreateTemp(t.TempDir(), "servercfg") if err != nil { t.Fatal(err) } _, err = tmpfile.Write(b) if err != nil { t.Fatal(err) } err = tmpfile.Close() if err != nil { t.Fatal(err) } return tmpfile } func validateMemberFlags(t *testing.T, cfg *config) { wcfg := &embed.Config{ Dir: "testdir", ListenPeerUrls: []url.URL{{Scheme: "http", Host: "localhost:8000"}, {Scheme: "https", Host: "localhost:8001"}}, ListenClientUrls: []url.URL{{Scheme: "http", Host: "localhost:7000"}, {Scheme: "https", Host: "localhost:7001"}}, ListenClientHttpUrls: []url.URL{{Scheme: "http", Host: "localhost:7002"}, {Scheme: "https", Host: "localhost:7003"}}, MaxSnapFiles: 10, MaxWalFiles: 10, Name: "testname", SnapshotCount: 10, SnapshotCatchUpEntries: 1000, } if cfg.ec.Dir != wcfg.Dir { t.Errorf("dir = %v, want %v", cfg.ec.Dir, wcfg.Dir) } if cfg.ec.MaxSnapFiles != wcfg.MaxSnapFiles { t.Errorf("maxsnap = %v, want %v", cfg.ec.MaxSnapFiles, wcfg.MaxSnapFiles) } if cfg.ec.MaxWalFiles != wcfg.MaxWalFiles { t.Errorf("maxwal = %v, want %v", cfg.ec.MaxWalFiles, wcfg.MaxWalFiles) } if cfg.ec.Name != wcfg.Name { t.Errorf("name = %v, want %v", cfg.ec.Name, wcfg.Name) } if cfg.ec.SnapshotCount != wcfg.SnapshotCount { t.Errorf("snapcount = %v, want %v", cfg.ec.SnapshotCount, wcfg.SnapshotCount) } if cfg.ec.SnapshotCatchUpEntries != wcfg.SnapshotCatchUpEntries { t.Errorf("snapshot catch up entries = %v, want %v", cfg.ec.SnapshotCatchUpEntries, wcfg.SnapshotCatchUpEntries) } if !reflect.DeepEqual(cfg.ec.ListenPeerUrls, wcfg.ListenPeerUrls) { t.Errorf("listen-peer-urls = %v, want %v", cfg.ec.ListenPeerUrls, wcfg.ListenPeerUrls) } if !reflect.DeepEqual(cfg.ec.ListenClientUrls, wcfg.ListenClientUrls) { t.Errorf("listen-client-urls = %v, want %v", cfg.ec.ListenClientUrls, wcfg.ListenClientUrls) } if !reflect.DeepEqual(cfg.ec.ListenClientHttpUrls, wcfg.ListenClientHttpUrls) { t.Errorf("listen-client-http-urls = %v, want %v", cfg.ec.ListenClientHttpUrls, wcfg.ListenClientHttpUrls) } } func validateClusteringFlags(t *testing.T, cfg *config) { wcfg := newConfig() wcfg.ec.AdvertisePeerUrls = []url.URL{{Scheme: "http", Host: "localhost:8000"}, {Scheme: "https", Host: "localhost:8001"}} wcfg.ec.AdvertiseClientUrls = []url.URL{{Scheme: "http", Host: "localhost:7000"}, {Scheme: "https", Host: "localhost:7001"}} wcfg.ec.ClusterState = embed.ClusterStateFlagExisting wcfg.ec.InitialCluster = "0=http://localhost:8000" wcfg.ec.InitialClusterToken = "etcdtest" if cfg.ec.ClusterState != wcfg.ec.ClusterState { t.Errorf("clusterState = %v, want %v", cfg.ec.ClusterState, wcfg.ec.ClusterState) } if cfg.ec.InitialCluster != wcfg.ec.InitialCluster { t.Errorf("initialCluster = %v, want %v", cfg.ec.InitialCluster, wcfg.ec.InitialCluster) } if cfg.ec.InitialClusterToken != wcfg.ec.InitialClusterToken { t.Errorf("initialClusterToken = %v, want %v", cfg.ec.InitialClusterToken, wcfg.ec.InitialClusterToken) } if !reflect.DeepEqual(cfg.ec.AdvertisePeerUrls, wcfg.ec.AdvertisePeerUrls) { t.Errorf("initial-advertise-peer-urls = %v, want %v", cfg.ec.AdvertisePeerUrls, wcfg.ec.AdvertisePeerUrls) } if !reflect.DeepEqual(cfg.ec.AdvertiseClientUrls, wcfg.ec.AdvertiseClientUrls) { t.Errorf("advertise-client-urls = %v, want %v", cfg.ec.AdvertiseClientUrls, wcfg.ec.AdvertiseClientUrls) } } func TestConfigFileDeprecatedOptions(t *testing.T) { // Define a minimal config struct with only the fields we need type configFileYAML struct { SnapshotCount uint64 `json:"snapshot-count,omitempty"` MaxSnapFiles uint `json:"max-snapshots,omitempty"` } testCases := []struct { name string configFileYAML configFileYAML expectedFlags map[string]struct{} }{ { name: "no deprecated options", configFileYAML: configFileYAML{}, expectedFlags: map[string]struct{}{}, }, { name: "deprecated snapshot options", configFileYAML: configFileYAML{ SnapshotCount: 10000, MaxSnapFiles: 5, }, expectedFlags: map[string]struct{}{ "max-snapshots": {}, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create config file b, err := yaml.Marshal(&tc.configFileYAML) if err != nil { t.Fatal(err) } tmpfile := mustCreateCfgFile(t, b) defer os.Remove(tmpfile.Name()) // Parse config cfg := newConfig() err = cfg.parse([]string{fmt.Sprintf("--config-file=%s", tmpfile.Name())}) if err != nil { t.Fatal(err) } // Check which flags were set and marked as deprecated foundFlags := make(map[string]struct{}) for flagName := range cfg.ec.FlagsExplicitlySet { if _, ok := deprecatedFlags[flagName]; ok { foundFlags[flagName] = struct{}{} } } // Compare sets of flags assert.Equalf(t, tc.expectedFlags, foundFlags, "deprecated flags mismatch - expected: %v, got: %v", tc.expectedFlags, foundFlags) }) } } ================================================ FILE: server/etcdmain/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package etcdmain contains the main entry point for the etcd binary. package etcdmain ================================================ FILE: server/etcdmain/etcd.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdmain import ( errorspkg "errors" "fmt" "os" "runtime" "strings" "go.uber.org/zap" "google.golang.org/grpc" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/osutil" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver/errors" ) type dirType string var ( dirMember = dirType("member") dirProxy = dirType("proxy") dirEmpty = dirType("empty") ) func startEtcdOrProxyV2(args []string) { grpc.EnableTracing = false cfg := newConfig() defaultInitialCluster := cfg.ec.InitialCluster err := cfg.parse(args[1:]) lg := cfg.ec.GetLogger() // If we failed to parse the whole configuration, print the error using // preferably the resolved logger from the config, // but if does not exists, create a new temporary logger. if lg == nil { var zapError error // use this logger lg, zapError = logutil.CreateDefaultZapLogger(zap.InfoLevel) if zapError != nil { fmt.Printf("error creating zap logger %v", zapError) os.Exit(1) } } lg.Info("Running: ", zap.Strings("args", args)) if err != nil { lg.Warn("failed to verify flags", zap.Error(err)) if errorspkg.Is(err, embed.ErrUnsetAdvertiseClientURLsFlag) { lg.Warn("advertise client URLs are not set", zap.Error(err)) } os.Exit(1) } cfg.ec.SetupGlobalLoggers() defer func() { logger := cfg.ec.GetLogger() if logger != nil { logger.Sync() } }() defaultHost, dhErr := (&cfg.ec).UpdateDefaultClusterFromName(defaultInitialCluster) if defaultHost != "" { lg.Info( "detected default host for advertise", zap.String("host", defaultHost), ) } if dhErr != nil { lg.Info("failed to detect default host", zap.Error(dhErr)) } if cfg.ec.Dir == "" { cfg.ec.Dir = fmt.Sprintf("%v.etcd", cfg.ec.Name) lg.Warn( "'data-dir' was empty; using default", zap.String("data-dir", cfg.ec.Dir), ) } var stopped <-chan struct{} var errc <-chan error which := identifyDataDirOrDie(cfg.ec.GetLogger(), cfg.ec.Dir) if which != dirEmpty { lg.Info( "server has already been initialized", zap.String("data-dir", cfg.ec.Dir), zap.String("dir-type", string(which)), ) switch which { case dirMember: stopped, errc, err = startEtcd(&cfg.ec) case dirProxy: lg.Panic("v2 http proxy has already been deprecated in 3.6", zap.String("dir-type", string(which))) default: lg.Panic( "unknown directory type", zap.String("dir-type", string(which)), ) } } else { lg.Info( "Initialize and start etcd server", zap.String("data-dir", cfg.ec.Dir), zap.String("dir-type", string(which)), ) stopped, errc, err = startEtcd(&cfg.ec) } if err != nil { var derr *errors.DiscoveryError if errorspkg.As(err, &derr) { lg.Warn( "failed to bootstrap; discovery token was already used", zap.String("discovery-token", cfg.ec.DiscoveryCfg.Token), zap.Strings("discovery-endpoints", cfg.ec.DiscoveryCfg.Endpoints), zap.Error(err), ) lg.Warn("do not reuse discovery token; generate a new one to bootstrap a cluster") os.Exit(1) } if strings.Contains(err.Error(), "include") && strings.Contains(err.Error(), "--initial-cluster") { lg.Warn("failed to start", zap.Error(err)) if cfg.ec.InitialCluster == cfg.ec.InitialClusterFromName(cfg.ec.Name) { lg.Warn("forgot to set --initial-cluster?") } if types.URLs(cfg.ec.AdvertisePeerUrls).String() == embed.DefaultInitialAdvertisePeerURLs { lg.Warn("forgot to set --initial-advertise-peer-urls?") } if cfg.ec.InitialCluster == cfg.ec.InitialClusterFromName(cfg.ec.Name) && len(cfg.ec.DiscoveryCfg.Endpoints) == 0 { lg.Warn("V3 discovery settings (i.e., --discovery-token, --discovery-endpoints) are not set") } os.Exit(1) } lg.Fatal("discovery failed", zap.Error(err)) } osutil.HandleInterrupts(lg) // At this point, the initialization of etcd is done. // The listeners are listening on the TCP ports and ready // for accepting connections. The etcd instance should be // joined with the cluster and ready to serve incoming // connections. notifySystemd(lg) select { case lerr := <-errc: // fatal out on listener errors lg.Fatal("listener failed", zap.Error(lerr)) case <-stopped: } osutil.Exit(0) } // startEtcd runs StartEtcd in addition to hooks needed for standalone etcd. func startEtcd(cfg *embed.Config) (<-chan struct{}, <-chan error, error) { e, err := embed.StartEtcd(cfg) if err != nil { return nil, nil, err } osutil.RegisterInterruptHandler(e.Close) select { case <-e.Server.ReadyNotify(): // wait for e.Server to join the cluster case <-e.Server.StopNotify(): // publish aborted from 'ErrStopped' } return e.Server.StopNotify(), e.Err(), nil } // identifyDataDirOrDie returns the type of the data dir. // Dies if the datadir is invalid. func identifyDataDirOrDie(lg *zap.Logger, dir string) dirType { names, err := fileutil.ReadDir(dir) if err != nil { if os.IsNotExist(err) { return dirEmpty } lg.Fatal("failed to list data directory", zap.String("dir", dir), zap.Error(err)) } var m, p bool for _, name := range names { switch dirType(name) { case dirMember: m = true case dirProxy: p = true default: lg.Warn( "found invalid file under data directory", zap.String("filename", name), zap.String("data-dir", dir), ) } } if m && p { lg.Fatal("invalid datadir; both member and proxy directories exist") } if m { return dirMember } if p { return dirProxy } return dirEmpty } func checkSupportArch() { lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { panic(err) } // To add a new platform, check https://github.com/etcd-io/website/blob/main/content/en/docs/${VERSION}/op-guide/supported-platform.md. // The ${VERSION} is the etcd version, e.g. v3.5, v3.6 etc. switch runtime.GOARCH { case "amd64", "arm64", "ppc64le", "s390x": return } // unsupported arch only configured via environment variable // so unset here to not parse through flag defer os.Unsetenv("ETCD_UNSUPPORTED_ARCH") if env, ok := os.LookupEnv("ETCD_UNSUPPORTED_ARCH"); ok && env == runtime.GOARCH { lg.Info("running etcd on unsupported architecture since ETCD_UNSUPPORTED_ARCH is set", zap.String("arch", env)) return } lg.Error("Refusing to run etcd on unsupported architecture since ETCD_UNSUPPORTED_ARCH is not set", zap.String("arch", runtime.GOARCH)) os.Exit(1) } ================================================ FILE: server/etcdmain/gateway.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdmain import ( "fmt" "net" "net/url" "os" "time" "github.com/spf13/cobra" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/server/v3/proxy/tcpproxy" ) var ( gatewayListenAddr string gatewayEndpoints []string gatewayDNSCluster string gatewayDNSClusterServiceName string gatewayInsecureDiscovery bool gatewayRetryDelay time.Duration gatewayCA string ) var rootCmd = &cobra.Command{ Use: "etcd", Short: "etcd server", SuggestFor: []string{"etcd"}, } func init() { rootCmd.AddCommand(newGatewayCommand()) } // newGatewayCommand returns the cobra command for "gateway". func newGatewayCommand() *cobra.Command { lpc := &cobra.Command{ Use: "gateway ", Short: "gateway related command", } lpc.AddCommand(newGatewayStartCommand()) return lpc } func newGatewayStartCommand() *cobra.Command { cmd := cobra.Command{ Use: "start", Short: "start the gateway", Run: startGateway, } cmd.Flags().StringVar(&gatewayListenAddr, "listen-addr", "127.0.0.1:23790", "listen address") cmd.Flags().StringVar(&gatewayDNSCluster, "discovery-srv", "", "DNS domain used to bootstrap initial cluster") cmd.Flags().StringVar(&gatewayDNSClusterServiceName, "discovery-srv-name", "", "service name to query when using DNS discovery") cmd.Flags().BoolVar(&gatewayInsecureDiscovery, "insecure-discovery", false, "accept insecure SRV records") cmd.Flags().StringVar(&gatewayCA, "trusted-ca-file", "", "path to the client server TLS CA file for verifying the discovered endpoints when discovery-srv is provided.") cmd.Flags().StringSliceVar(&gatewayEndpoints, "endpoints", []string{"127.0.0.1:2379"}, "comma separated etcd cluster endpoints") cmd.Flags().DurationVar(&gatewayRetryDelay, "retry-delay", time.Minute, "duration of delay before retrying failed endpoints") return &cmd } func stripSchema(eps []string) []string { var endpoints []string for _, ep := range eps { if u, err := url.Parse(ep); err == nil && u.Host != "" { ep = u.Host } endpoints = append(endpoints, ep) } return endpoints } func startGateway(cmd *cobra.Command, args []string) { lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } // We use os.Args to show all the arguments (not only passed-through Cobra). lg.Info("Running: ", zap.Strings("args", os.Args)) srvs := discoverEndpoints(lg, gatewayDNSCluster, gatewayCA, gatewayInsecureDiscovery, gatewayDNSClusterServiceName) if len(srvs.Endpoints) == 0 { // no endpoints discovered, fall back to provided endpoints srvs.Endpoints = gatewayEndpoints } // Strip the schema from the endpoints because we start just a TCP proxy srvs.Endpoints = stripSchema(srvs.Endpoints) if len(srvs.SRVs) == 0 { for _, ep := range srvs.Endpoints { h, p, serr := net.SplitHostPort(ep) if serr != nil { fmt.Printf("error parsing endpoint %q", ep) os.Exit(1) } var port uint16 fmt.Sscanf(p, "%d", &port) srvs.SRVs = append(srvs.SRVs, &net.SRV{Target: h, Port: port}) } } lhost, lport, err := net.SplitHostPort(gatewayListenAddr) if err != nil { fmt.Println("failed to validate listen address:", gatewayListenAddr) os.Exit(1) } laddrs, err := net.LookupHost(lhost) if err != nil { fmt.Println("failed to resolve listen host:", lhost) os.Exit(1) } laddrsMap := make(map[string]bool) for _, addr := range laddrs { laddrsMap[addr] = true } for _, srv := range srvs.SRVs { var eaddrs []string eaddrs, err = net.LookupHost(srv.Target) if err != nil { fmt.Println("failed to resolve endpoint host:", srv.Target) os.Exit(1) } if fmt.Sprintf("%d", srv.Port) != lport { continue } for _, ea := range eaddrs { if laddrsMap[ea] { fmt.Printf("SRV or endpoint (%s:%d->%s:%d) should not resolve to the gateway listen addr (%s)\n", srv.Target, srv.Port, ea, srv.Port, gatewayListenAddr) os.Exit(1) } } } if len(srvs.Endpoints) == 0 { fmt.Println("no endpoints found") os.Exit(1) } var l net.Listener l, err = net.Listen("tcp", gatewayListenAddr) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } tp := tcpproxy.TCPProxy{ Logger: lg, Listener: l, Endpoints: srvs.SRVs, MonitorInterval: gatewayRetryDelay, } // At this point, etcd gateway listener is initialized notifySystemd(lg) tp.Run() } ================================================ FILE: server/etcdmain/grpc_proxy.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdmain import ( "context" "crypto/tls" "crypto/x509" "fmt" "io" "log" "math" "net" "net/http" "net/url" "os" "path/filepath" "time" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors" "github.com/prometheus/client_golang/prometheus" "github.com/soheilhy/cmux" "github.com/spf13/cobra" "go.uber.org/zap" "go.uber.org/zap/zapgrpc" "golang.org/x/net/http2" "google.golang.org/grpc" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/keepalive" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/client/pkg/v3/tlsutil" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/leasing" "go.etcd.io/etcd/client/v3/namespace" "go.etcd.io/etcd/client/v3/ordering" "go.etcd.io/etcd/pkg/v3/debugutil" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb" "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb" "go.etcd.io/etcd/server/v3/proxy/grpcproxy" ) var ( grpcProxyListenAddr string grpcProxyMetricsListenAddr string grpcProxyEndpoints []string grpcProxyEndpointsAutoSyncInterval time.Duration grpcProxyDialKeepAliveTime time.Duration grpcProxyDialKeepAliveTimeout time.Duration grpcProxyPermitWithoutStream bool grpcProxyDNSCluster string grpcProxyDNSClusterServiceName string grpcProxyInsecureDiscovery bool grpcProxyDataDir string grpcMaxCallSendMsgSize int grpcMaxCallRecvMsgSize int // tls for connecting to etcd grpcProxyCA string grpcProxyCert string grpcProxyKey string grpcProxyInsecureSkipTLSVerify bool // tls for clients connecting to proxy grpcProxyListenCA string grpcProxyListenCert string grpcProxyListenKey string grpcProxyListenCipherSuites []string grpcProxyListenAutoTLS bool grpcProxyListenCRL string grpcProxyListenTLSMinVersion string grpcProxyListenTLSMaxVersion string selfSignedCertValidity uint grpcProxyAdvertiseClientURL string grpcProxyResolverPrefix string grpcProxyResolverTTL int grpcProxyNamespace string grpcProxyLeasing string grpcProxyEnablePprof bool grpcProxyEnableOrdering bool grpcProxyEnableLogging bool grpcProxyDebug bool // GRPC keep alive related options. grpcKeepAliveMinTime time.Duration grpcKeepAliveTimeout time.Duration grpcKeepAliveInterval time.Duration maxConcurrentStreams uint32 ) const defaultGRPCMaxCallSendMsgSize = 1.5 * 1024 * 1024 func init() { rootCmd.AddCommand(newGRPCProxyCommand()) } // newGRPCProxyCommand returns the cobra command for "grpc-proxy". func newGRPCProxyCommand() *cobra.Command { lpc := &cobra.Command{ Use: "grpc-proxy ", Short: "grpc-proxy related command", } lpc.AddCommand(newGRPCProxyStartCommand()) return lpc } func newGRPCProxyStartCommand() *cobra.Command { cmd := cobra.Command{ Use: "start", Short: "start the grpc proxy", Run: startGRPCProxy, } cmd.Flags().StringVar(&grpcProxyListenAddr, "listen-addr", "127.0.0.1:23790", "listen address") cmd.Flags().StringVar(&grpcProxyDNSCluster, "discovery-srv", "", "domain name to query for SRV records describing cluster endpoints") cmd.Flags().StringVar(&grpcProxyDNSClusterServiceName, "discovery-srv-name", "", "service name to query when using DNS discovery") cmd.Flags().StringVar(&grpcProxyMetricsListenAddr, "metrics-addr", "", "listen for endpoint /metrics requests on an additional interface") cmd.Flags().BoolVar(&grpcProxyInsecureDiscovery, "insecure-discovery", false, "accept insecure SRV records") cmd.Flags().StringSliceVar(&grpcProxyEndpoints, "endpoints", []string{"127.0.0.1:2379"}, "comma separated etcd cluster endpoints") cmd.Flags().DurationVar(&grpcProxyEndpointsAutoSyncInterval, "endpoints-auto-sync-interval", 0, "etcd endpoints auto sync interval (disabled by default)") cmd.Flags().DurationVar(&grpcProxyDialKeepAliveTime, "dial-keepalive-time", 0, "keepalive time for client(grpc-proxy) connections (default 0, disable).") cmd.Flags().DurationVar(&grpcProxyDialKeepAliveTimeout, "dial-keepalive-timeout", embed.DefaultGRPCKeepAliveTimeout, "keepalive timeout for client(grpc-proxy) connections (default 20s).") cmd.Flags().BoolVar(&grpcProxyPermitWithoutStream, "permit-without-stream", false, "Enable client(grpc-proxy) to send keepalive pings even with no active RPCs.") cmd.Flags().StringVar(&grpcProxyAdvertiseClientURL, "advertise-client-url", "127.0.0.1:23790", "advertise address to register (must be reachable by client)") cmd.Flags().StringVar(&grpcProxyResolverPrefix, "resolver-prefix", "", "prefix to use for registering proxy (must be shared with other grpc-proxy members)") cmd.Flags().IntVar(&grpcProxyResolverTTL, "resolver-ttl", 0, "specify TTL, in seconds, when registering proxy endpoints") cmd.Flags().StringVar(&grpcProxyNamespace, "namespace", "", "string to prefix to all keys for namespacing requests") cmd.Flags().BoolVar(&grpcProxyEnablePprof, "enable-pprof", false, `Enable runtime profiling data via HTTP server. Address is at client URL + "/debug/pprof/"`) cmd.Flags().StringVar(&grpcProxyDataDir, "data-dir", "default.proxy", "Data directory for persistent data") cmd.Flags().IntVar(&grpcMaxCallSendMsgSize, "max-send-bytes", defaultGRPCMaxCallSendMsgSize, "message send limits in bytes (default value is 1.5 MiB)") cmd.Flags().IntVar(&grpcMaxCallRecvMsgSize, "max-recv-bytes", math.MaxInt32, "message receive limits in bytes (default value is math.MaxInt32)") cmd.Flags().DurationVar(&grpcKeepAliveMinTime, "grpc-keepalive-min-time", embed.DefaultGRPCKeepAliveMinTime, "Minimum interval duration that a client should wait before pinging proxy.") cmd.Flags().DurationVar(&grpcKeepAliveInterval, "grpc-keepalive-interval", embed.DefaultGRPCKeepAliveInterval, "Frequency duration of server-to-client ping to check if a connection is alive (0 to disable).") cmd.Flags().DurationVar(&grpcKeepAliveTimeout, "grpc-keepalive-timeout", embed.DefaultGRPCKeepAliveTimeout, "Additional duration of wait before closing a non-responsive connection (0 to disable).") // client TLS for connecting to server cmd.Flags().StringVar(&grpcProxyCert, "cert", "", "identify secure connections with etcd servers using this TLS certificate file") cmd.Flags().StringVar(&grpcProxyKey, "key", "", "identify secure connections with etcd servers using this TLS key file") cmd.Flags().StringVar(&grpcProxyCA, "cacert", "", "verify certificates of TLS-enabled secure etcd servers using this CA bundle") cmd.Flags().BoolVar(&grpcProxyInsecureSkipTLSVerify, "insecure-skip-tls-verify", false, "skip authentication of etcd server TLS certificates (CAUTION: this option should be enabled only for testing purposes)") // client TLS for connecting to proxy cmd.Flags().StringVar(&grpcProxyListenCert, "cert-file", "", "identify secure connections to the proxy using this TLS certificate file") cmd.Flags().StringVar(&grpcProxyListenKey, "key-file", "", "identify secure connections to the proxy using this TLS key file") cmd.Flags().StringVar(&grpcProxyListenCA, "trusted-ca-file", "", "verify certificates of TLS-enabled secure proxy using this CA bundle") cmd.Flags().StringSliceVar(&grpcProxyListenCipherSuites, "listen-cipher-suites", grpcProxyListenCipherSuites, "Comma-separated list of supported TLS cipher suites between client/proxy (empty will be auto-populated by Go).") cmd.Flags().BoolVar(&grpcProxyListenAutoTLS, "auto-tls", false, "proxy TLS using generated certificates") cmd.Flags().StringVar(&grpcProxyListenCRL, "client-crl-file", "", "proxy client certificate revocation list file.") cmd.Flags().UintVar(&selfSignedCertValidity, "self-signed-cert-validity", 1, "The validity period of the proxy certificates, unit is year") cmd.Flags().StringVar(&grpcProxyListenTLSMinVersion, "tls-min-version", string(tlsutil.TLSVersion12), "Minimum TLS version supported by grpc proxy. Possible values: TLS1.2, TLS1.3.") cmd.Flags().StringVar(&grpcProxyListenTLSMaxVersion, "tls-max-version", string(tlsutil.TLSVersionDefault), "Maximum TLS version supported by grpc proxy. Possible values: TLS1.2, TLS1.3 (empty defers to Go).") // experimental flags cmd.Flags().BoolVar(&grpcProxyEnableOrdering, "experimental-serializable-ordering", false, "Ensure serializable reads have monotonically increasing store revisions across endpoints.") cmd.Flags().StringVar(&grpcProxyLeasing, "experimental-leasing-prefix", "", "leasing metadata prefix for disconnected linearized reads.") cmd.Flags().BoolVar(&grpcProxyEnableLogging, "experimental-enable-grpc-logging", false, "logging all grpc requests and responses") cmd.Flags().BoolVar(&grpcProxyDebug, "debug", false, "Enable debug-level logging for grpc-proxy.") cmd.Flags().Uint32Var(&maxConcurrentStreams, "max-concurrent-streams", math.MaxUint32, "Maximum concurrent streams that each client can open at a time.") return &cmd } func startGRPCProxy(cmd *cobra.Command, args []string) { checkArgs() lvl := zap.InfoLevel if grpcProxyDebug { lvl = zap.DebugLevel grpc.EnableTracing = true } lg, err := logutil.CreateDefaultZapLogger(lvl) if err != nil { panic(err) } defer lg.Sync() grpclog.SetLoggerV2(zapgrpc.NewLogger(lg)) // The proxy itself (ListenCert) can have not-empty CN. // The empty CN is required for grpcProxyCert. // Please see https://github.com/etcd-io/etcd/issues/11970#issuecomment-687875315 for more context. tlsInfo := newTLS(grpcProxyListenCA, grpcProxyListenCert, grpcProxyListenKey, false) if tlsInfo == nil && grpcProxyListenAutoTLS { host := []string{"https://" + grpcProxyListenAddr} dir := filepath.Join(grpcProxyDataDir, "fixtures", "proxy") autoTLS, err := transport.SelfCert(lg, dir, host, selfSignedCertValidity) if err != nil { log.Fatal(err) } tlsInfo = &autoTLS } if tlsInfo != nil { if len(grpcProxyListenCipherSuites) > 0 { cs, err := tlsutil.GetCipherSuites(grpcProxyListenCipherSuites) if err != nil { log.Fatal(err) } tlsInfo.CipherSuites = cs } if grpcProxyListenTLSMinVersion != "" { version, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMinVersion) if err != nil { log.Fatal(err) } tlsInfo.MinVersion = version } if grpcProxyListenTLSMaxVersion != "" { version, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMaxVersion) if err != nil { log.Fatal(err) } tlsInfo.MaxVersion = version } lg.Info("gRPC proxy server TLS", zap.String("tls-info", fmt.Sprintf("%+v", tlsInfo))) } m := mustListenCMux(lg, tlsInfo) grpcl := m.Match(cmux.HTTP2()) httpl := mustMatchHTTPListener(m, tlsInfo) defer func() { grpcl.Close() lg.Info("stop listening gRPC proxy client requests", zap.String("address", grpcProxyListenAddr)) }() client := mustNewClient(lg) grpcServer := newGRPCProxyServer(lg, client) errc := make(chan error, 3) // NOTE: // Start gRPC + cmux before creating proxyClient. // // proxyClient dials the proxy endpoint with a 5-second timeout. If cmux is not // serving yet, the self-dial can time out because the gRPC path is not being // accepted/dispatched. // // It is safe to start cmux before the HTTP server goroutine: HTTP has already // been matched/registered with cmux, so accepted HTTP connections are queued // and served once http.Serve starts. startServe(errc, func() error { return grpcServer.Serve(grpcl) }) startServe(errc, m.Serve) // The proxy client is used for self-healthchecking. // TODO: The mechanism should be refactored to use internal connection. // // Create it after gRPC/cmux serving goroutines have started proxyClient := newProxyHealthClient(lg, tlsInfo) httpClient := mustNewHTTPClient() srvhttp := mustHTTPServer(lg, tlsInfo, httpClient, client, proxyClient) startServe(errc, func() error { return srvhttp.Serve(httpl) }) maybeServeMetrics(lg, tlsInfo, httpClient, client, proxyClient) lg.Info("started gRPC proxy", zap.String("address", grpcProxyListenAddr)) // grpc-proxy is initialized, ready to serve notifySystemd(lg) fmt.Fprintln(os.Stderr, <-errc) os.Exit(1) } func checkArgs() { if grpcProxyResolverPrefix != "" && grpcProxyResolverTTL < 1 { fmt.Fprintln(os.Stderr, fmt.Errorf("invalid resolver-ttl %d", grpcProxyResolverTTL)) os.Exit(1) } if grpcProxyResolverPrefix == "" && grpcProxyResolverTTL > 0 { fmt.Fprintln(os.Stderr, fmt.Errorf("invalid resolver-prefix %q", grpcProxyResolverPrefix)) os.Exit(1) } if grpcProxyResolverPrefix != "" && grpcProxyResolverTTL > 0 && grpcProxyAdvertiseClientURL == "" { fmt.Fprintln(os.Stderr, fmt.Errorf("invalid advertise-client-url %q", grpcProxyAdvertiseClientURL)) os.Exit(1) } if grpcProxyListenAutoTLS && selfSignedCertValidity == 0 { fmt.Fprintln(os.Stderr, fmt.Errorf("selfSignedCertValidity is invalid,it should be greater than 0")) os.Exit(1) } minVersion, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMinVersion) if err != nil { fmt.Fprintln(os.Stderr, fmt.Errorf("tls-min-version is invalid: %w", err)) os.Exit(1) } maxVersion, err := tlsutil.GetTLSVersion(grpcProxyListenTLSMaxVersion) if err != nil { fmt.Fprintln(os.Stderr, fmt.Errorf("tls-max-version is invalid: %w", err)) os.Exit(1) } // maxVersion == 0 means that Go selects the highest available version. if maxVersion != 0 && minVersion > maxVersion { fmt.Fprintln(os.Stderr, fmt.Errorf("min version (%s) is greater than max version (%s)", grpcProxyListenTLSMinVersion, grpcProxyListenTLSMaxVersion)) os.Exit(1) } // Check if user attempted to configure ciphers for TLS1.3 only: Go does not support that currently. if minVersion == tls.VersionTLS13 && len(grpcProxyListenCipherSuites) > 0 { fmt.Fprintln(os.Stderr, fmt.Errorf("cipher suites cannot be configured when only TLS1.3 is enabled")) os.Exit(1) } } func mustNewClient(lg *zap.Logger) *clientv3.Client { srvs := discoverEndpoints(lg, grpcProxyDNSCluster, grpcProxyCA, grpcProxyInsecureDiscovery, grpcProxyDNSClusterServiceName) eps := srvs.Endpoints if len(eps) == 0 { eps = grpcProxyEndpoints } cfg, err := newClientCfg(lg, eps) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } cfg.DialOptions = append(cfg.DialOptions, grpc.WithUnaryInterceptor(grpcproxy.AuthUnaryClientInterceptor)) cfg.DialOptions = append(cfg.DialOptions, grpc.WithStreamInterceptor(grpcproxy.AuthStreamClientInterceptor)) cfg.Logger = lg.Named("client") client, err := clientv3.New(*cfg) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } return client } func mustNewProxyClient(lg *zap.Logger, tls *transport.TLSInfo) *clientv3.Client { eps := []string{grpcProxyAdvertiseClientURL} cfg, err := newProxyClientCfg(lg.Named("client"), eps, tls) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } client, err := clientv3.New(*cfg) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } lg.Info("create proxy client", zap.String("grpcProxyAdvertiseClientURL", grpcProxyAdvertiseClientURL)) return client } func newProxyHealthClient(lg *zap.Logger, tls *transport.TLSInfo) *clientv3.Client { if grpcProxyAdvertiseClientURL == "" { return nil } return mustNewProxyClient(lg, tls) } func newProxyClientCfg(lg *zap.Logger, eps []string, tls *transport.TLSInfo) (*clientv3.Config, error) { cfg := clientv3.Config{ Endpoints: eps, DialTimeout: 5 * time.Second, Logger: lg, } if tls != nil { clientTLS, err := tls.ClientConfig() if err != nil { return nil, err } cfg.TLS = clientTLS } return &cfg, nil } func newClientCfg(lg *zap.Logger, eps []string) (*clientv3.Config, error) { // set tls if any one tls option set cfg := clientv3.Config{ Endpoints: eps, AutoSyncInterval: grpcProxyEndpointsAutoSyncInterval, DialTimeout: 5 * time.Second, } if grpcMaxCallSendMsgSize > 0 { cfg.MaxCallSendMsgSize = grpcMaxCallSendMsgSize } if grpcMaxCallRecvMsgSize > 0 { cfg.MaxCallRecvMsgSize = grpcMaxCallRecvMsgSize } if grpcProxyDialKeepAliveTime > 0 { cfg.DialKeepAliveTime = grpcProxyDialKeepAliveTime } if grpcProxyDialKeepAliveTimeout > 0 { cfg.DialKeepAliveTimeout = grpcProxyDialKeepAliveTimeout } cfg.PermitWithoutStream = grpcProxyPermitWithoutStream tls := newTLS(grpcProxyCA, grpcProxyCert, grpcProxyKey, true) if tls == nil && grpcProxyInsecureSkipTLSVerify { tls = &transport.TLSInfo{} } if tls != nil { clientTLS, err := tls.ClientConfig() if err != nil { return nil, err } clientTLS.InsecureSkipVerify = grpcProxyInsecureSkipTLSVerify if clientTLS.InsecureSkipVerify { lg.Warn("--insecure-skip-tls-verify was given, this grpc proxy process skips authentication of etcd server TLS certificates. This option should be enabled only for testing purposes.") } cfg.TLS = clientTLS lg.Info("gRPC proxy client TLS", zap.String("tls-info", fmt.Sprintf("%+v", tls))) } return &cfg, nil } func newTLS(ca, cert, key string, requireEmptyCN bool) *transport.TLSInfo { if ca == "" && cert == "" && key == "" { return nil } return &transport.TLSInfo{TrustedCAFile: ca, CertFile: cert, KeyFile: key, EmptyCN: requireEmptyCN} } func mustListenCMux(lg *zap.Logger, tlsinfo *transport.TLSInfo) cmux.CMux { l, err := net.Listen("tcp", grpcProxyListenAddr) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } if l, err = transport.NewKeepAliveListener(l, "tcp", nil); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } if tlsinfo != nil { tlsinfo.CRLFile = grpcProxyListenCRL if l, err = transport.NewTLSListener(l, tlsinfo); err != nil { lg.Fatal("failed to create TLS listener", zap.Error(err)) } } lg.Info("listening for gRPC proxy client requests", zap.String("address", grpcProxyListenAddr)) return cmux.New(l) } func newGRPCProxyServer(lg *zap.Logger, client *clientv3.Client) *grpc.Server { if grpcProxyEnableOrdering { vf := ordering.NewOrderViolationSwitchEndpointClosure(client) client.KV = ordering.NewKV(client.KV, vf) lg.Info("waiting for linearized read from cluster to recover ordering") for { _, err := client.KV.Get(context.TODO(), "_", clientv3.WithKeysOnly()) if err == nil { break } lg.Warn("ordering recovery failed, retrying in 1s", zap.Error(err)) time.Sleep(time.Second) } } if len(grpcProxyNamespace) > 0 { client.KV = namespace.NewKV(client.KV, grpcProxyNamespace) client.Watcher = namespace.NewWatcher(client.Watcher, grpcProxyNamespace) client.Lease = namespace.NewLease(client.Lease, grpcProxyNamespace) } if len(grpcProxyLeasing) > 0 { client.KV, _, _ = leasing.NewKV(client, grpcProxyLeasing) } kvp, _ := grpcproxy.NewKvProxy(client) watchp, _ := grpcproxy.NewWatchProxy(client.Ctx(), lg, client) if grpcProxyResolverPrefix != "" { grpcproxy.Register(lg, client, grpcProxyResolverPrefix, grpcProxyAdvertiseClientURL, grpcProxyResolverTTL) } clusterp, _ := grpcproxy.NewClusterProxy(lg, client, grpcProxyAdvertiseClientURL, grpcProxyResolverPrefix) leasep, _ := grpcproxy.NewLeaseProxy(client.Ctx(), client) mainp := grpcproxy.NewMaintenanceProxy(client) authp := grpcproxy.NewAuthProxy(client) electionp := grpcproxy.NewElectionProxy(client) lockp := grpcproxy.NewLockProxy(client) serverMetrics := grpc_prometheus.NewServerMetrics() prometheus.MustRegister(serverMetrics) grpcChainStreamList := []grpc.StreamServerInterceptor{ serverMetrics.StreamServerInterceptor(), } grpcChainUnaryList := []grpc.UnaryServerInterceptor{ serverMetrics.UnaryServerInterceptor(), } if grpcProxyEnableLogging { grpcChainStreamList = append(grpcChainStreamList, interceptors.StreamServerInterceptor(reportable(lg)), ) grpcChainUnaryList = append(grpcChainUnaryList, interceptors.UnaryServerInterceptor(reportable(lg)), ) } gopts := []grpc.ServerOption{ grpc.ChainStreamInterceptor(grpcChainStreamList...), grpc.ChainUnaryInterceptor(grpcChainUnaryList...), grpc.MaxConcurrentStreams(math.MaxUint32), } if grpcKeepAliveMinTime > time.Duration(0) { gopts = append(gopts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ MinTime: grpcKeepAliveMinTime, PermitWithoutStream: false, })) } if grpcKeepAliveInterval > time.Duration(0) || grpcKeepAliveTimeout > time.Duration(0) { gopts = append(gopts, grpc.KeepaliveParams(keepalive.ServerParameters{ Time: grpcKeepAliveInterval, Timeout: grpcKeepAliveTimeout, })) } server := grpc.NewServer(gopts...) pb.RegisterKVServer(server, kvp) pb.RegisterWatchServer(server, watchp) pb.RegisterClusterServer(server, clusterp) pb.RegisterLeaseServer(server, leasep) pb.RegisterMaintenanceServer(server, mainp) pb.RegisterAuthServer(server, authp) v3electionpb.RegisterElectionServer(server, electionp) v3lockpb.RegisterLockServer(server, lockp) return server } func mustMatchHTTPListener(m cmux.CMux, tlsinfo *transport.TLSInfo) net.Listener { if tlsinfo == nil { return m.Match(cmux.HTTP1()) } return m.Match(cmux.Any()) } func mustHTTPServer(lg *zap.Logger, tlsinfo *transport.TLSInfo, httpClient *http.Client, c *clientv3.Client, proxyClient *clientv3.Client) *http.Server { httpmux := http.NewServeMux() httpmux.HandleFunc("/", http.NotFound) grpcproxy.HandleMetrics(httpmux, httpClient, c.Endpoints()) grpcproxy.HandleHealth(lg, httpmux, c) grpcproxy.HandleProxyMetrics(httpmux) grpcproxy.HandleProxyHealth(lg, httpmux, proxyClient) if grpcProxyEnablePprof { for p, h := range debugutil.PProfHandlers() { httpmux.Handle(p, h) } lg.Info("gRPC proxy enabled pprof", zap.String("path", debugutil.HTTPPrefixPProf)) } srvhttp := &http.Server{ Handler: httpmux, ErrorLog: log.New(io.Discard, "net/http", 0), } if err := http2.ConfigureServer(srvhttp, &http2.Server{ MaxConcurrentStreams: maxConcurrentStreams, }); err != nil { lg.Fatal("Failed to configure the http server", zap.Error(err)) } if tlsinfo == nil { return srvhttp } srvTLS, err := tlsinfo.ServerConfig() if err != nil { lg.Fatal("failed to set up TLS", zap.Error(err)) } srvhttp.TLSConfig = srvTLS return srvhttp } func maybeServeMetrics(lg *zap.Logger, tlsinfo *transport.TLSInfo, httpClient *http.Client, c *clientv3.Client, proxyClient *clientv3.Client) { if len(grpcProxyMetricsListenAddr) == 0 { return } mhttpl := mustMetricsListener(lg, tlsinfo) go func() { mux := http.NewServeMux() grpcproxy.HandleMetrics(mux, httpClient, c.Endpoints()) grpcproxy.HandleHealth(lg, mux, c) grpcproxy.HandleProxyMetrics(mux) grpcproxy.HandleProxyHealth(lg, mux, proxyClient) lg.Info("gRPC proxy server metrics URL serving") herr := http.Serve(mhttpl, mux) if herr != nil { lg.Fatal("gRPC proxy server metrics URL returned", zap.Error(herr)) } else { lg.Info("gRPC proxy server metrics URL returned") } }() } func startServe(errc chan<- error, serve func() error) { go func() { errc <- serve() }() } func mustNewHTTPClient() *http.Client { transport, err := newHTTPTransport(grpcProxyCA, grpcProxyCert, grpcProxyKey) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } return &http.Client{Transport: transport} } func newHTTPTransport(ca, cert, key string) (*http.Transport, error) { tr := &http.Transport{} if ca != "" && cert != "" && key != "" { caCert, err := os.ReadFile(ca) if err != nil { return nil, err } keyPair, err := tls.LoadX509KeyPair(cert, key) if err != nil { return nil, err } caPool := x509.NewCertPool() caPool.AppendCertsFromPEM(caCert) tlsConfig := &tls.Config{ Certificates: []tls.Certificate{keyPair}, RootCAs: caPool, } tr.TLSClientConfig = tlsConfig } else if grpcProxyInsecureSkipTLSVerify { tlsConfig := &tls.Config{InsecureSkipVerify: grpcProxyInsecureSkipTLSVerify} tr.TLSClientConfig = tlsConfig } return tr, nil } func mustMetricsListener(lg *zap.Logger, tlsinfo *transport.TLSInfo) net.Listener { murl, err := url.Parse(grpcProxyMetricsListenAddr) if err != nil { fmt.Fprintf(os.Stderr, "cannot parse %q", grpcProxyMetricsListenAddr) os.Exit(1) } ml, err := transport.NewListener(murl.Host, murl.Scheme, tlsinfo) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } lg.Info("gRPC proxy listening for metrics", zap.String("address", murl.String())) return ml } ================================================ FILE: server/etcdmain/grpc_proxy_logger.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdmain import ( "context" "fmt" "path" "reflect" "slices" "time" "github.com/golang/protobuf/jsonpb" //nolint:staticcheck // TODO: remove for a supported version "github.com/golang/protobuf/proto" //nolint:staticcheck // TODO: remove for a supported version "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors" "go.uber.org/zap" "google.golang.org/grpc/peer" ) // grpcProxyLogger implements the go-grpc-middleware v2 Reporter interface type grpcProxyLogger struct { logger *zap.Logger fields []zap.Field } var _ interceptors.Reporter = (*grpcProxyLogger)(nil) const ( responseCallType = "response" requestCallType = "request" ) func (r *grpcProxyLogger) PostCall(_ error, _ time.Duration) { // no op - no post-call payload logging } func (r *grpcProxyLogger) PostMsgReceive(payload any, err error, _ time.Duration) { callType := requestCallType f := logFieldsFromPayload(payload, err, callType) r.logger.Info(fmt.Sprintf("received payload logged as grpc.%s.content field", callType), slices.Concat(f, r.fields)...) } func (r *grpcProxyLogger) PostMsgSend(payload any, err error, _ time.Duration) { callType := responseCallType f := logFieldsFromPayload(payload, err, callType) r.logger.Info(fmt.Sprintf("returned response payload logged as grpc.%s.content field", callType), slices.Concat(f, r.fields)...) } func logFieldsFromPayload(payload any, err error, callType string) []zap.Field { fields := []zap.Field{} if err != nil { fields = append(fields, zap.NamedError(fmt.Sprintf("grpc.%s.error", callType), err)) } p, ok := payload.(proto.Message) if !ok { fields = append(fields, zap.NamedError("msg.type.error", fmt.Errorf("payload is not a github.com/golang/protobuf/proto message"))) } msg, pErr := protoToJSON(p) if pErr != nil { fields = append(fields, zap.NamedError("msg.proto.error", fmt.Errorf("error when converting payload to logging json: %w", pErr))) } fields = append(fields, zap.String(fmt.Sprintf("grpc.%s.content", callType), msg)) return fields } func protoToJSON(msg proto.Message) (string, error) { if reflect.ValueOf(msg).IsNil() { return "", nil } marshaler := jsonpb.Marshaler{} return marshaler.MarshalToString(msg) } func reportable(lg *zap.Logger) interceptors.CommonReportableFunc { return func(ctx context.Context, c interceptors.CallMeta) (interceptors.Reporter, context.Context) { fields := []zap.Field{ zap.String("grpc.service", path.Dir(c.FullMethod())[1:]), zap.String("grpc.method", path.Base(c.FullMethod())), } if peer, ok := peer.FromContext(ctx); ok { fields = append(fields, zap.String("peer.address", peer.Addr.String())) } return &grpcProxyLogger{ logger: lg, fields: fields, }, ctx } } ================================================ FILE: server/etcdmain/grpc_proxy_logger_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdmain import ( "context" "fmt" "io" "testing" "time" "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors" "github.com/grpc-ecosystem/go-grpc-middleware/v2/testing/testpb" "github.com/stretchr/testify/suite" "go.uber.org/zap" "go.uber.org/zap/zaptest/observer" "google.golang.org/grpc" ) type loggingPayloadSuite struct { *testpb.InterceptorTestSuite logger *zap.Logger logs *observer.ObservedLogs } func TestLoggingPayloadSuite(t *testing.T) { observer, logs := observer.New(zap.InfoLevel) logger := zap.New(observer) s := &loggingPayloadSuite{ InterceptorTestSuite: &testpb.InterceptorTestSuite{ TestService: &testpb.TestPingService{}, ServerOpts: []grpc.ServerOption{ grpc.UnaryInterceptor(interceptors.UnaryServerInterceptor(reportable(logger))), grpc.StreamInterceptor(interceptors.StreamServerInterceptor(reportable(logger))), }, }, logs: logs, logger: zap.New(observer), } suite.Run(t, s) } func (s *loggingPayloadSuite) SetupTest() { s.logs.TakeAll() // clear logs s.Require().Empty(s.logs.TakeAll()) } func (s *loggingPayloadSuite) TestPing_LogsBothRequestAndResponse() { _, err := s.Client.Ping(s.SimpleCtx(), testpb.GoodPing) s.Require().NoError(err) s.Require().Len(s.logs.All(), 2) // request and response s.assertField("grpc.request.content", `{"value":"something","sleepTimeMs":9999}`, 1) s.assertField("grpc.response.content", `{"value":"something"}`, 1) } func (s *loggingPayloadSuite) TestPingError_LogsError() { _, err := s.Client.PingError(s.SimpleCtx(), &testpb.PingErrorRequest{Value: "something", ErrorCodeReturned: uint32(4)}) s.Require().Error(err) s.Require().Len(s.logs.All(), 2) // request and response s.assertField("grpc.request.content", `{"value":"something","errorCodeReturned":4}`, 1) s.assertField("grpc.response.content", ``, 1) s.assertField("grpc.response.error", "rpc error: code = DeadlineExceeded desc = Userspace error", 1) } func (s *loggingPayloadSuite) TestPingStream_LogsAllRequestsAndResponses() { messagesExpected := 10 stream, err := s.Client.PingStream(s.SimpleCtx()) s.Require().NoError(err) for range messagesExpected { s.Require().NoError(stream.Send(testpb.GoodPingStream)) pong := &testpb.PingResponse{} err := stream.RecvMsg(pong) s.Require().NoError(err) } s.Require().NoError(stream.CloseSend()) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() s.Require().NoError(waitUntil(200*time.Millisecond, ctx.Done(), func() error { if len(s.logs.FilterFieldKey("grpc.request.error").All()) > 0 { return nil } return fmt.Errorf("no EOF log yet") })) eof := s.logs.FilterFieldKey("grpc.request.error").All() s.Len(eof, 1) s.Equal(io.EOF.Error(), eof[0].ContextMap()["grpc.request.error"]) s.assertField("grpc.request.content", `{"value":"something","sleepTimeMs":9999}`, messagesExpected+1) s.assertField("grpc.response.content", `{"value":"something"}`, messagesExpected) } func (s *loggingPayloadSuite) assertField(key, expectedValue string, expectedLineCount int) { s.T().Helper() filtered := s.logs.FilterFieldKey(key).All() s.Require().Len(filtered, expectedLineCount) actualValue, ok := filtered[0].ContextMap()[key].(string) s.Require().True(ok) s.Equal(expectedValue, actualValue) } // waitUntil executes f every interval seconds until timeout or no error is returned from f. func waitUntil(interval time.Duration, stopc <-chan struct{}, f func() error) error { tick := time.NewTicker(interval) defer tick.Stop() var err error for { if err = f(); err == nil { return nil } select { case <-stopc: return err case <-tick.C: } } } ================================================ FILE: server/etcdmain/help.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdmain import ( "fmt" "strconv" "strings" "golang.org/x/crypto/bcrypt" cconfig "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/features" ) var ( usageline = `Usage: etcd [flags] Start an etcd server. etcd --version Show the version of etcd. etcd -h | --help Show the help information about etcd. etcd --config-file Path to the server configuration file. Note that if a configuration file is provided, other command line flags and environment variables will be ignored. etcd gateway Run the stateless pass-through etcd TCP connection forwarding proxy. etcd grpc-proxy Run the stateless etcd v3 gRPC L7 reverse proxy. ` flagsline = ` Member: --name 'default' Human-readable name for this member. --data-dir '${name}.etcd' Path to the data directory. --wal-dir '' Path to the dedicated wal directory. --snapshot-count '10000' Number of committed transactions to trigger a snapshot. --heartbeat-interval '100' Time (in milliseconds) of a heartbeat interval. --election-timeout '1000' Time (in milliseconds) for an election to timeout. See tuning documentation for details. --initial-election-tick-advance 'true' Whether to fast-forward initial election ticks on boot for faster election. --listen-peer-urls 'http://localhost:2380' List of URLs to listen on for peer traffic. --listen-client-urls 'http://localhost:2379' List of URLs to listen on for client grpc traffic and http as long as --listen-client-http-urls is not specified. --listen-client-http-urls '' List of URLs to listen on for http only client traffic. Enabling this flag removes http services from --listen-client-urls. --max-snapshots '` + strconv.Itoa(embed.DefaultMaxSnapshots) + `' Maximum number of snapshot files to retain (0 is unlimited). Deprecated in v3.6 and will be decommissioned in v3.8. --max-wals '` + strconv.Itoa(embed.DefaultMaxWALs) + `' Maximum number of wal files to retain (0 is unlimited). --memory-mlock Enable to enforce etcd pages (in particular bbolt) to stay in RAM. --quota-backend-bytes '0' Sets the maximum size (in bytes) that the etcd backend database may consume. Exceeding this triggers an alarm and puts etcd in read-only mode. Set to 0 to use the default 2GiB limit. --backend-bbolt-freelist-type 'map' BackendFreelistType specifies the type of freelist that boltdb backend uses(array and map are supported types). --backend-batch-interval '' BackendBatchInterval is the maximum time before commit the backend transaction. --backend-batch-limit '0' BackendBatchLimit is the maximum operations before commit the backend transaction. --max-txn-ops '128' Maximum number of operations permitted in a transaction. --max-request-bytes '1572864' Maximum client request size in bytes the server will accept. --max-concurrent-streams 'math.MaxUint32' Maximum concurrent streams that each client can open at a time. --grpc-keepalive-min-time '5s' Minimum duration interval that a client should wait before pinging server. --grpc-keepalive-interval '2h' Frequency duration of server-to-client ping to check if a connection is alive (0 to disable). --grpc-keepalive-timeout '20s' Additional duration of wait before closing a non-responsive connection (0 to disable). --socket-reuse-port 'false' Enable to set socket option SO_REUSEPORT on listeners allowing rebinding of a port already in use. --socket-reuse-address 'false' Enable to set socket option SO_REUSEADDR on listeners allowing binding to an address in TIME_WAIT state. --enable-grpc-gateway Enable GRPC gateway. --raft-read-timeout '` + rafthttp.DefaultConnReadTimeout.String() + `' Read timeout set on each rafthttp connection --raft-write-timeout '` + rafthttp.DefaultConnWriteTimeout.String() + `' Write timeout set on each rafthttp connection --feature-gates '' A set of key=value pairs that describe server level feature gates for alpha/experimental features. Options are:` + "\n " + strings.Join(features.NewDefaultServerFeatureGate("", nil).KnownFeatures(), "\n ") + ` Clustering: --initial-advertise-peer-urls 'http://localhost:2380' List of this member's peer URLs to advertise to the rest of the cluster. --initial-cluster 'default=http://localhost:2380' Initial cluster configuration for bootstrapping. --initial-cluster-state 'new' Initial cluster state ('new' when bootstrapping a new cluster or 'existing' when adding new members to an existing cluster). After successful initialization (bootstrapping or adding), flag is ignored on restarts --initial-cluster-token 'etcd-cluster' Initial cluster token for the etcd cluster during bootstrap. Specifying this can protect you from unintended cross-cluster interaction when running multiple clusters. --advertise-client-urls 'http://localhost:2379' List of this member's client URLs to advertise to the public. The client URLs advertised should be accessible to machines that talk to etcd cluster. etcd client libraries parse these URLs to connect to the cluster. --discovery '' Discovery URL used to bootstrap the cluster for v2 discovery. Will be deprecated in v3.7, and be decommissioned in v3.8. --discovery-token '' V3 discovery: discovery token for the etcd cluster to be bootstrapped. --discovery-endpoints '' V3 discovery: List of gRPC endpoints of the discovery service. --discovery-dial-timeout '2s' V3 discovery: dial timeout for client connections. --discovery-request-timeout '5s' V3 discovery: timeout for discovery requests (excluding dial timeout). --discovery-keepalive-time '2s' V3 discovery: keepalive time for client connections. --discovery-keepalive-timeout '6s' V3 discovery: keepalive timeout for client connections. --discovery-insecure-transport 'true' V3 discovery: disable transport security for client connections. --discovery-insecure-skip-tls-verify 'false' V3 discovery: skip server certificate verification (CAUTION: this option should be enabled only for testing purposes). --discovery-cert '' V3 discovery: identify secure client using this TLS certificate file. --discovery-key '' V3 discovery: identify secure client using this TLS key file. --discovery-cacert '' V3 discovery: verify certificates of TLS-enabled secure servers using this CA bundle. --discovery-user '' V3 discovery: username[:password] for authentication (prompt if password is not supplied). --discovery-password '' V3 discovery: password for authentication (if this option is used, --user option shouldn't include password). --discovery-fallback 'exit' Expected behavior ('exit') when discovery services fails. Note that v2 proxy is removed. --discovery-proxy '' HTTP proxy to use for traffic to discovery service. Will be deprecated in v3.7, and be decommissioned in v3.8. --discovery-srv '' DNS srv domain used to bootstrap the cluster. --discovery-srv-name '' Suffix to the dns srv name queried when bootstrapping. --strict-reconfig-check '` + strconv.FormatBool(embed.DefaultStrictReconfigCheck) + `' Reject reconfiguration requests that would cause quorum loss. --pre-vote 'true' Enable the raft Pre-Vote algorithm to prevent disruption when a node that has been partitioned away rejoins the cluster. --auto-compaction-retention '0' Auto compaction retention length. 0 means disable auto compaction. --auto-compaction-mode 'periodic' Interpret 'auto-compaction-retention' one of: periodic|revision. 'periodic' for duration based retention, defaulting to hours if no time unit is provided (e.g. '5m'). 'revision' for revision number based retention. --v2-deprecation '` + string(cconfig.V2DeprDefault) + `' Phase of v2store deprecation. Deprecated and scheduled for removal in v3.8. The default value is enforced, ignoring user input. Supported values: 'not-yet' // Issues a warning if v2store have meaningful content (default in v3.5) 'write-only' // Custom v2 state is not allowed (default in v3.6) 'write-only-drop-data' // Custom v2 state will get DELETED ! (planned default in v3.7) 'gone' // v2store is not maintained any longer. (planned to cleanup anything related to v2store in v3.8) Security: --cert-file '' Path to the client server TLS cert file. --key-file '' Path to the client server TLS key file. --client-cert-auth 'false' Enable client cert authentication. --client-cert-file '' Path to an explicit peer client TLS cert file otherwise cert file will be used when client auth is required. --client-key-file '' Path to an explicit peer client TLS key file otherwise key file will be used when client auth is required. --client-crl-file '' Path to the client certificate revocation list file. --client-cert-allowed-hostname '' Comma-separated list of SAN hostnames for client cert authentication. --trusted-ca-file '' Path to the client server TLS trusted CA cert file. --auto-tls 'false' Client TLS using generated certificates. --peer-cert-file '' Path to the peer server TLS cert file. --peer-key-file '' Path to the peer server TLS key file. --peer-client-cert-auth 'false' Enable peer client cert authentication. --peer-client-cert-file '' Path to an explicit peer client TLS cert file otherwise peer cert file will be used when client auth is required. --peer-client-key-file '' Path to an explicit peer client TLS key file otherwise peer key file will be used when client auth is required. --peer-trusted-ca-file '' Path to the peer server TLS trusted CA file. --peer-cert-allowed-cn '' Comma-separated list of allowed CNs for inter-peer TLS authentication. --peer-cert-allowed-hostname '' Comma-separated list of allowed SAN hostnames for inter-peer TLS authentication. --peer-auto-tls 'false' Peer TLS using self-generated certificates if --peer-key-file and --peer-cert-file are not provided. --self-signed-cert-validity '1' The validity period of the client and peer certificates that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS, the unit is year, and the default is 1. --peer-crl-file '' Path to the peer certificate revocation list file. --cipher-suites '' Comma-separated list of supported TLS cipher suites between client/server and peers (empty will be auto-populated by Go). --cors '*' Comma-separated whitelist of origins for CORS, or cross-origin resource sharing, (empty or * means allow all). --host-whitelist '*' Acceptable hostnames from HTTP client requests, if server is not secure (empty or * means allow all). --tls-min-version 'TLS1.2' Minimum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3. --tls-max-version '' Maximum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3 (empty will be auto-populated by Go). Auth: --auth-token 'simple' Specify a v3 authentication token type and its options ('simple' or 'jwt'). --bcrypt-cost ` + fmt.Sprintf("%d", bcrypt.DefaultCost) + ` Specify the cost / strength of the bcrypt algorithm for hashing auth passwords. Valid values are between ` + fmt.Sprintf("%d", bcrypt.MinCost) + ` and ` + fmt.Sprintf("%d", bcrypt.MaxCost) + `. --auth-token-ttl 300 Time (in seconds) of the auth-token-ttl. Profiling and Monitoring: --enable-pprof 'false' Enable runtime profiling data via HTTP server. Address is at client URL + "/debug/pprof/" --metrics 'basic' Set level of detail for exported metrics, specify 'extensive' to include server side grpc histogram metrics. --listen-metrics-urls '' List of URLs to listen on for the /metrics and /health endpoints. For https, the client URL TLS info is used. Logging: --logger 'zap' Currently only supports 'zap' for structured logging. --log-outputs 'default' Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd, or list of comma separated output targets. --log-level 'info' Configures log level. Only supports debug, info, warn, error, panic, or fatal. --log-format 'json' Configures log format. Only supports json, console. --enable-log-rotation 'false' Enable log rotation of a single log-outputs file target. --log-rotation-config-json '{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}' Configures log rotation if enabled with a JSON logger config. MaxSize(MB), MaxAge(days,0=no limit), MaxBackups(0=no limit), LocalTime(use computers local time), Compress(gzip)". --warning-unary-request-duration '300ms' Set time duration after which a warning is logged if a unary request takes more than this duration. Distributed tracing: --enable-distributed-tracing 'false' Enable distributed tracing. --distributed-tracing-address 'localhost:4317' Distributed tracing collector address. --distributed-tracing-service-name 'etcd' Distributed tracing service name, must be same across all etcd instances. --distributed-tracing-instance-id '' Distributed tracing instance ID, must be unique per each etcd instance. --distributed-tracing-sampling-rate '0' Number of samples to collect per million spans for distributed tracing. Features: --corrupt-check-time '0s' Duration of time between cluster corruption check passes. --compact-hash-check-time '1m' Duration of time between leader checks followers compaction hashes. --compaction-batch-limit 1000 CompactionBatchLimit sets the maximum revisions deleted in each compaction batch. --peer-skip-client-san-verification 'false' Skip verification of SAN field in client certificate for peer connections. --watch-progress-notify-interval '10m' Duration of periodical watch progress notification. --warning-apply-duration '100ms' Warning is generated if requests take more than this duration. --bootstrap-defrag-threshold-megabytes Enable the defrag during etcd server bootstrap on condition that it will free at least the provided threshold of disk space. Needs to be set to non-zero value to take effect. --max-learners '1' Set the max number of learner members allowed in the cluster membership. --compaction-sleep-interval Sets the sleep interval between each compaction batch. --downgrade-check-time Duration of time between two downgrade status checks. --snapshot-catchup-entries Number of entries for a slow follower to catch up after compacting the raft storage entries. Unsafe feature: --force-new-cluster 'false' Force to create a new one-member cluster. --unsafe-no-fsync 'false' Disables fsync, unsafe, will cause data loss. CAUTIOUS with unsafe flag! It may break the guarantees given by the consensus protocol! ` ) // Add back "TO BE DEPRECATED" section if needed ================================================ FILE: server/etcdmain/main.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdmain import ( "fmt" "os" "github.com/coreos/go-systemd/v22/daemon" "go.uber.org/zap" ) func Main(args []string) { checkSupportArch() if len(args) > 1 { cmd := args[1] switch cmd { case "gateway", "grpc-proxy": if err := rootCmd.Execute(); err != nil { fmt.Fprint(os.Stderr, err) os.Exit(1) } return } } startEtcdOrProxyV2(args) } func notifySystemd(lg *zap.Logger) { lg.Info("notifying init daemon") _, err := daemon.SdNotify(false, daemon.SdNotifyReady) if err != nil { lg.Error("failed to notify systemd for readiness", zap.Error(err)) return } lg.Info("successfully notified init daemon") } ================================================ FILE: server/etcdmain/util.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdmain import ( "fmt" "os" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/srv" "go.etcd.io/etcd/client/pkg/v3/transport" ) func discoverEndpoints(lg *zap.Logger, dns string, ca string, insecure bool, serviceName string) (s srv.SRVClients) { if dns == "" { return s } srvs, err := srv.GetClient("etcd-client", dns, serviceName) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } endpoints := srvs.Endpoints lg.Info( "discovered cluster from SRV", zap.String("srv-server", dns), zap.Strings("endpoints", endpoints), ) if insecure { return *srvs } // confirm TLS connections are good tlsInfo := transport.TLSInfo{ TrustedCAFile: ca, ServerName: dns, } lg.Info( "validating discovered SRV endpoints", zap.String("srv-server", dns), zap.Strings("endpoints", endpoints), ) endpoints, err = transport.ValidateSecureEndpoints(tlsInfo, endpoints) if err != nil { lg.Warn( "failed to validate discovered endpoints", zap.String("srv-server", dns), zap.Strings("endpoints", endpoints), zap.Error(err), ) } else { lg.Info( "using validated discovered SRV endpoints", zap.String("srv-server", dns), zap.Strings("endpoints", endpoints), ) } // map endpoints back to SRVClients struct with SRV data eps := make(map[string]struct{}) for _, ep := range endpoints { eps[ep] = struct{}{} } for i := range srvs.Endpoints { if _, ok := eps[srvs.Endpoints[i]]; !ok { continue } s.Endpoints = append(s.Endpoints, srvs.Endpoints[i]) s.SRVs = append(s.SRVs, srvs.SRVs[i]) } return s } ================================================ FILE: server/etcdserver/adapters.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "context" "github.com/coreos/go-semver/semver" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/membershippb" "go.etcd.io/etcd/api/v3/version" serverversion "go.etcd.io/etcd/server/v3/etcdserver/version" "go.etcd.io/etcd/server/v3/storage/schema" ) // serverVersionAdapter implements the interface Server defined in package // go.etcd.io/etcd/server/v3/etcdserver/version, and it's needed by Monitor // in the same package. type serverVersionAdapter struct { *EtcdServer } func NewServerVersionAdapter(s *EtcdServer) serverversion.Server { return &serverVersionAdapter{ EtcdServer: s, } } var _ serverversion.Server = (*serverVersionAdapter)(nil) func (s *serverVersionAdapter) UpdateClusterVersion(version string) { s.GoAttach(func() { s.updateClusterVersionV3(version) }) } func (s *serverVersionAdapter) LinearizableReadNotify(ctx context.Context) error { return s.linearizableReadNotify(ctx) } func (s *serverVersionAdapter) DowngradeEnable(ctx context.Context, targetVersion *semver.Version) error { raftRequest := membershippb.DowngradeInfoSetRequest{Enabled: true, Ver: targetVersion.String()} _, err := s.raftRequest(ctx, pb.InternalRaftRequest{DowngradeInfoSet: &raftRequest}) return err } func (s *serverVersionAdapter) DowngradeCancel(ctx context.Context) error { raftRequest := membershippb.DowngradeInfoSetRequest{Enabled: false} _, err := s.raftRequest(ctx, pb.InternalRaftRequest{DowngradeInfoSet: &raftRequest}) return err } func (s *serverVersionAdapter) GetClusterVersion() *semver.Version { return s.cluster.Version() } func (s *serverVersionAdapter) GetDowngradeInfo() *serverversion.DowngradeInfo { return s.cluster.DowngradeInfo() } func (s *serverVersionAdapter) GetMembersVersions() map[string]*version.Versions { return getMembersVersions(s.lg, s.cluster, s.MemberID(), s.peerRt, s.Cfg.ReqTimeout()) } func (s *serverVersionAdapter) GetStorageVersion() *semver.Version { return s.StorageVersion() } func (s *serverVersionAdapter) UpdateStorageVersion(target semver.Version) error { // `applySnapshot` sets a new backend instance, so we need to acquire the bemu lock. s.bemu.RLock() defer s.bemu.RUnlock() tx := s.be.BatchTx() tx.LockOutsideApply() defer tx.Unlock() return schema.UnsafeMigrate(s.lg, tx, s.r.storage, target) } ================================================ FILE: server/etcdserver/api/capability.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package api import ( "sync" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/version" serverversion "go.etcd.io/etcd/server/v3/etcdserver/version" ) type Capability string const ( AuthCapability Capability = "auth" V3rpcCapability Capability = "v3rpc" ) var ( // capabilityMaps is a static map of version to capability map. capabilityMaps = map[string]map[Capability]bool{ "3.0.0": {AuthCapability: true, V3rpcCapability: true}, "3.1.0": {AuthCapability: true, V3rpcCapability: true}, "3.2.0": {AuthCapability: true, V3rpcCapability: true}, "3.3.0": {AuthCapability: true, V3rpcCapability: true}, "3.4.0": {AuthCapability: true, V3rpcCapability: true}, "3.5.0": {AuthCapability: true, V3rpcCapability: true}, "3.6.0": {AuthCapability: true, V3rpcCapability: true}, "3.7.0": {AuthCapability: true, V3rpcCapability: true}, } enableMapMu sync.RWMutex // enabledMap points to a map in capabilityMaps enabledMap map[Capability]bool curVersion *semver.Version ) func init() { enabledMap = map[Capability]bool{ AuthCapability: true, V3rpcCapability: true, } } // UpdateCapability updates the enabledMap when the cluster version increases. func UpdateCapability(lg *zap.Logger, v *semver.Version) { if v == nil { // if recovered but version was never set by cluster return } enableMapMu.Lock() if curVersion != nil && !serverversion.IsValidClusterVersionChange(curVersion, v) { enableMapMu.Unlock() return } curVersion = v enabledMap = capabilityMaps[curVersion.String()] enableMapMu.Unlock() if lg != nil { lg.Info( "enabled capabilities for version", zap.String("cluster-version", version.Cluster(v.String())), ) } } func IsCapabilityEnabled(c Capability) bool { enableMapMu.RLock() defer enableMapMu.RUnlock() if enabledMap == nil { return false } return enabledMap[c] } func EnableCapability(c Capability) { enableMapMu.Lock() defer enableMapMu.Unlock() enabledMap[c] = true } ================================================ FILE: server/etcdserver/api/cluster.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package api import ( "github.com/coreos/go-semver/semver" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" ) // Cluster is an interface representing a collection of members in one etcd cluster. type Cluster interface { // ID returns the cluster ID ID() types.ID // ClientURLs returns an aggregate set of all URLs on which this // cluster is listening for client requests ClientURLs() []string // Members returns a slice of members sorted by their ID Members() []*membership.Member // Member retrieves a particular member based on ID, or nil if the // member does not exist in the cluster Member(id types.ID) *membership.Member // Version is the cluster-wide minimum major.minor version. Version() *semver.Version } ================================================ FILE: server/etcdserver/api/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package api manages the capabilities and features that are exposed to clients by the etcd cluster. package api ================================================ FILE: server/etcdserver/api/etcdhttp/debug.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdhttp import ( "expvar" "fmt" "net/http" ) const ( varsPath = "/debug/vars" ) func HandleDebug(mux *http.ServeMux) { mux.HandleFunc(varsPath, serveVars) } func serveVars(w http.ResponseWriter, r *http.Request) { if !allowMethod(w, r, "GET") { return } w.Header().Set("Content-Type", "application/json; charset=utf-8") fmt.Fprint(w, "{\n") first := true expvar.Do(func(kv expvar.KeyValue) { if !first { fmt.Fprint(w, ",\n") } first = false fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value) }) fmt.Fprint(w, "\n}\n") } ================================================ FILE: server/etcdserver/api/etcdhttp/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package etcdhttp implements HTTP transportation layer for etcdserver. package etcdhttp ================================================ FILE: server/etcdserver/api/etcdhttp/health.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // This file defines the http endpoints for etcd health checks. // The endpoints include /livez, /readyz and /health. package etcdhttp import ( "bytes" "context" "encoding/json" "fmt" "net/http" "path" "strings" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/raft/v3" ) const ( PathHealth = "/health" PathProxyHealth = "/proxy/health" HealthStatusSuccess string = "success" HealthStatusError string = "error" checkTypeLivez = "livez" checkTypeReadyz = "readyz" checkTypeHealth = "health" ) type ServerHealth interface { Alarms() []*pb.AlarmMember Leader() types.ID Range(context.Context, *pb.RangeRequest) (*pb.RangeResponse, error) Config() config.ServerConfig AuthStore() auth.AuthStore IsLearner() bool } // HandleHealth registers metrics and health handlers. it checks health by using v3 range request // and its corresponding timeout. func HandleHealth(lg *zap.Logger, mux *http.ServeMux, srv ServerHealth) { mux.Handle(PathHealth, NewHealthHandler(lg, func(ctx context.Context, excludedAlarms StringSet, serializable bool) Health { if h := checkAlarms(lg, srv, excludedAlarms); h.Health != "true" { return h } if h := checkLeader(lg, srv, serializable); h.Health != "true" { return h } return checkAPI(ctx, lg, srv, serializable) })) installLivezEndpoints(lg, mux, srv) installReadyzEndpoints(lg, mux, srv) } // NewHealthHandler handles '/health' requests. func NewHealthHandler(lg *zap.Logger, hfunc func(ctx context.Context, excludedAlarms StringSet, Serializable bool) Health) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { w.Header().Set("Allow", http.MethodGet) http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) lg.Warn("/health error", zap.Int("status-code", http.StatusMethodNotAllowed)) return } excludedAlarms := getQuerySet(r, "exclude") // Passing the query parameter "serializable=true" ensures that the // health of the local etcd is checked vs the health of the cluster. // This is useful for probes attempting to validate the liveness of // the etcd process vs readiness of the cluster to serve requests. serializableFlag := getSerializableFlag(r) h := hfunc(r.Context(), excludedAlarms, serializableFlag) defer func() { if h.Health == "true" { healthSuccess.Inc() } else { healthFailed.Inc() } }() d, _ := json.Marshal(h) if h.Health != "true" { http.Error(w, string(d), http.StatusServiceUnavailable) lg.Warn("/health error", zap.String("output", string(d)), zap.Int("status-code", http.StatusServiceUnavailable)) return } w.WriteHeader(http.StatusOK) w.Write(d) lg.Debug("/health OK", zap.Int("status-code", http.StatusOK)) } } var ( healthSuccess = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "health_success", Help: "The total number of successful health checks", }) healthFailed = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "health_failures", Help: "The total number of failed health checks", }) healthCheckGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "healthcheck", Help: "The result of each kind of healthcheck.", }, []string{"type", "name"}, ) healthCheckCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "healthchecks_total", Help: "The total number of each kind of healthcheck.", }, []string{"type", "name", "status"}, ) ) func init() { prometheus.MustRegister(healthSuccess) prometheus.MustRegister(healthFailed) prometheus.MustRegister(healthCheckGauge) prometheus.MustRegister(healthCheckCounter) } // Health defines etcd server health status. // TODO: remove manual parsing in etcdctl cluster-health type Health struct { Health string `json:"health"` Reason string `json:"reason"` } // HealthStatus is used in new /readyz or /livez health checks instead of the Health struct. type HealthStatus struct { Reason string `json:"reason"` Status string `json:"status"` } func getQuerySet(r *http.Request, query string) StringSet { querySet := make(map[string]struct{}) qs, found := r.URL.Query()[query] if found { for _, q := range qs { if len(q) == 0 { continue } querySet[q] = struct{}{} } } return querySet } func getSerializableFlag(r *http.Request) bool { return r.URL.Query().Get("serializable") == "true" } // TODO: etcdserver.ErrNoLeader in health API func checkAlarms(lg *zap.Logger, srv ServerHealth, excludedAlarms StringSet) Health { h := Health{Health: "true"} for _, v := range srv.Alarms() { alarmName := v.Alarm.String() if _, found := excludedAlarms[alarmName]; found { lg.Debug("/health excluded alarm", zap.String("alarm", v.String())) continue } h.Health = "false" switch v.Alarm { case pb.AlarmType_NOSPACE: h.Reason = "ALARM NOSPACE" case pb.AlarmType_CORRUPT: h.Reason = "ALARM CORRUPT" default: h.Reason = "ALARM UNKNOWN" } lg.Warn("serving /health false due to an alarm", zap.String("alarm", v.String())) return h } return h } func checkLeader(lg *zap.Logger, srv ServerHealth, serializable bool) Health { h := Health{Health: "true"} if !serializable && (uint64(srv.Leader()) == raft.None) { h.Health = "false" h.Reason = "RAFT NO LEADER" lg.Warn("serving /health false; no leader") } return h } func checkAPI(ctx context.Context, lg *zap.Logger, srv ServerHealth, serializable bool) Health { h := Health{Health: "true"} cfg := srv.Config() ctx = srv.AuthStore().WithRoot(ctx) cctx, cancel := context.WithTimeout(ctx, cfg.ReqTimeout()) _, err := srv.Range(cctx, &pb.RangeRequest{KeysOnly: true, Limit: 1, Serializable: serializable}) cancel() if err != nil { h.Health = "false" h.Reason = fmt.Sprintf("RANGE ERROR:%s", err) lg.Warn("serving /health false; Range fails", zap.Error(err)) return h } lg.Debug("serving /health true") return h } type HealthCheck func(ctx context.Context) error type CheckRegistry struct { checkType string checks map[string]HealthCheck } func installLivezEndpoints(lg *zap.Logger, mux *http.ServeMux, server ServerHealth) { reg := CheckRegistry{checkType: checkTypeLivez, checks: make(map[string]HealthCheck)} reg.Register("serializable_read", readCheck(server, true /* serializable */)) reg.InstallHTTPEndpoints(lg, mux) } func installReadyzEndpoints(lg *zap.Logger, mux *http.ServeMux, server ServerHealth) { reg := CheckRegistry{checkType: checkTypeReadyz, checks: make(map[string]HealthCheck)} reg.Register("data_corruption", activeAlarmCheck(server, pb.AlarmType_CORRUPT)) // serializable_read checks if local read is ok. // linearizable_read checks if there is consensus in the cluster. // Having both serializable_read and linearizable_read helps isolate the cause of problems if there is a read failure. reg.Register("serializable_read", readCheck(server, true)) // linearizable_read check would be replaced by read_index check in 3.6 reg.Register("linearizable_read", readCheck(server, false)) // check if local is learner reg.Register("non_learner", learnerCheck(server)) reg.InstallHTTPEndpoints(lg, mux) } func (reg *CheckRegistry) Register(name string, check HealthCheck) { reg.checks[name] = check } func (reg *CheckRegistry) RootPath() string { return "/" + reg.checkType } // InstallHttpEndpoints installs the http handlers for the health checks. // // Deprecated: Please use (*CheckRegistry) InstallHTTPEndpoints instead. // //revive:disable-next-line:var-naming func (reg *CheckRegistry) InstallHttpEndpoints(lg *zap.Logger, mux *http.ServeMux) { reg.InstallHTTPEndpoints(lg, mux) } func (reg *CheckRegistry) InstallHTTPEndpoints(lg *zap.Logger, mux *http.ServeMux) { checkNames := make([]string, 0, len(reg.checks)) for k := range reg.checks { checkNames = append(checkNames, k) } // installs the http handler for the root path. reg.installRootHTTPEndpoint(lg, mux, checkNames...) for _, checkName := range checkNames { // installs the http handler for the individual check sub path. subpath := path.Join(reg.RootPath(), checkName) check := checkName mux.Handle(subpath, newHealthHandler(subpath, lg, func(r *http.Request) HealthStatus { return reg.runHealthChecks(r.Context(), check) })) } } func (reg *CheckRegistry) runHealthChecks(ctx context.Context, checkNames ...string) HealthStatus { h := HealthStatus{Status: HealthStatusSuccess} var individualCheckOutput bytes.Buffer for _, checkName := range checkNames { check, found := reg.checks[checkName] if !found { panic(fmt.Errorf("Health check: %s not registered", checkName)) } if err := check(ctx); err != nil { fmt.Fprintf(&individualCheckOutput, "[-]%s failed: %v\n", checkName, err) h.Status = HealthStatusError recordMetrics(reg.checkType, checkName, HealthStatusError) } else { fmt.Fprintf(&individualCheckOutput, "[+]%s ok\n", checkName) recordMetrics(reg.checkType, checkName, HealthStatusSuccess) } } h.Reason = individualCheckOutput.String() return h } // installRootHTTPEndpoint installs the http handler for the root path. func (reg *CheckRegistry) installRootHTTPEndpoint(lg *zap.Logger, mux *http.ServeMux, checks ...string) { hfunc := func(r *http.Request) HealthStatus { // extracts the health check names to be excludeList from the query param excluded := getQuerySet(r, "exclude") filteredCheckNames := filterCheckList(lg, listToStringSet(checks), excluded) h := reg.runHealthChecks(r.Context(), filteredCheckNames...) return h } mux.Handle(reg.RootPath(), newHealthHandler(reg.RootPath(), lg, hfunc)) } // newHealthHandler generates a http HandlerFunc for a health check function hfunc. func newHealthHandler(path string, lg *zap.Logger, hfunc func(*http.Request) HealthStatus) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { w.Header().Set("Allow", http.MethodGet) http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) lg.Warn("Health request error", zap.String("path", path), zap.Int("status-code", http.StatusMethodNotAllowed)) return } h := hfunc(r) // Always returns detailed reason for failed checks. if h.Status == HealthStatusError { http.Error(w, h.Reason, http.StatusServiceUnavailable) lg.Error("Health check error", zap.String("path", path), zap.String("reason", h.Reason), zap.Int("status-code", http.StatusServiceUnavailable)) return } w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("X-Content-Type-Options", "nosniff") // Only writes detailed reason for verbose requests. if _, found := r.URL.Query()["verbose"]; found { fmt.Fprint(w, h.Reason) } fmt.Fprint(w, "ok\n") lg.Debug("Health check OK", zap.String("path", path), zap.String("reason", h.Reason), zap.Int("status-code", http.StatusOK)) } } func filterCheckList(lg *zap.Logger, checks StringSet, excluded StringSet) []string { filteredList := []string{} for chk := range checks { if _, found := excluded[chk]; found { delete(excluded, chk) continue } filteredList = append(filteredList, chk) } if len(excluded) > 0 { // For version compatibility, excluding non-exist checks would not fail the request. lg.Warn("some health checks cannot be excluded", zap.String("missing-health-checks", formatQuoted(excluded.List()...))) } return filteredList } // formatQuoted returns a formatted string of the health check names, // preserving the order passed in. func formatQuoted(names ...string) string { quoted := make([]string, 0, len(names)) for _, name := range names { quoted = append(quoted, fmt.Sprintf("%q", name)) } return strings.Join(quoted, ",") } type StringSet map[string]struct{} func (s StringSet) List() []string { keys := make([]string, 0, len(s)) for k := range s { keys = append(keys, k) } return keys } func listToStringSet(list []string) StringSet { set := make(map[string]struct{}) for _, s := range list { set[s] = struct{}{} } return set } func recordMetrics(checkType, name string, status string) { val := 0.0 if status == HealthStatusSuccess { val = 1.0 } healthCheckGauge.With(prometheus.Labels{ "type": checkType, "name": name, }).Set(val) healthCheckCounter.With(prometheus.Labels{ "type": checkType, "name": name, "status": status, }).Inc() } // activeAlarmCheck checks if a specific alarm type is active in the server. func activeAlarmCheck(srv ServerHealth, at pb.AlarmType) func(context.Context) error { return func(ctx context.Context) error { as := srv.Alarms() for _, v := range as { if v.Alarm == at { return fmt.Errorf("alarm activated: %s", at.String()) } } return nil } } func readCheck(srv ServerHealth, serializable bool) func(ctx context.Context) error { return func(ctx context.Context) error { ctx = srv.AuthStore().WithRoot(ctx) _, err := srv.Range(ctx, &pb.RangeRequest{KeysOnly: true, Limit: 1, Serializable: serializable}) return err } } func learnerCheck(srv ServerHealth) func(ctx context.Context) error { return func(ctx context.Context) error { if srv.IsLearner() { return fmt.Errorf("not supported for learner") } return nil } } ================================================ FILE: server/etcdserver/api/etcdhttp/health_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdhttp import ( "context" "fmt" "io" "net/http" "net/http/httptest" "strings" "testing" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap/zaptest" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/config" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/raft/v3" ) type fakeHealthServer struct { fakeServer serializableReadError error linearizableReadError error missingLeader bool authStore auth.AuthStore isLearner bool } func (s *fakeHealthServer) Range(_ context.Context, req *pb.RangeRequest) (*pb.RangeResponse, error) { if req.Serializable { return nil, s.serializableReadError } return nil, s.linearizableReadError } func (s *fakeHealthServer) IsLearner() bool { return s.isLearner } func (s *fakeHealthServer) Config() config.ServerConfig { return config.ServerConfig{} } func (s *fakeHealthServer) Leader() types.ID { if !s.missingLeader { return 1 } return types.ID(raft.None) } func (s *fakeHealthServer) AuthStore() auth.AuthStore { return s.authStore } func (s *fakeHealthServer) ClientCertAuthEnabled() bool { return false } type healthTestCase struct { name string healthCheckURL string expectStatusCode int inResult []string notInResult []string alarms []*pb.AlarmMember apiError error missingLeader bool isLearner bool } func TestHealthHandler(t *testing.T) { // define the input and expected output // input: alarms, and healthCheckURL tests := []healthTestCase{ { name: "Healthy if no alarm", alarms: []*pb.AlarmMember{}, healthCheckURL: "/health", expectStatusCode: http.StatusOK, }, { name: "Unhealthy if NOSPACE alarm is on", alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}}, healthCheckURL: "/health", expectStatusCode: http.StatusServiceUnavailable, }, { name: "Healthy if NOSPACE alarm is on and excluded", alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}}, healthCheckURL: "/health?exclude=NOSPACE", expectStatusCode: http.StatusOK, }, { name: "Healthy if NOSPACE alarm is excluded", alarms: []*pb.AlarmMember{}, healthCheckURL: "/health?exclude=NOSPACE", expectStatusCode: http.StatusOK, }, { name: "Healthy if multiple NOSPACE alarms are on and excluded", alarms: []*pb.AlarmMember{{MemberID: uint64(1), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(2), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(3), Alarm: pb.AlarmType_NOSPACE}}, healthCheckURL: "/health?exclude=NOSPACE", expectStatusCode: http.StatusOK, }, { name: "Unhealthy if NOSPACE alarms is excluded and CORRUPT is on", alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}}, healthCheckURL: "/health?exclude=NOSPACE", expectStatusCode: http.StatusServiceUnavailable, }, { name: "Unhealthy if both NOSPACE and CORRUPT are on and excluded", alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}}, healthCheckURL: "/health?exclude=NOSPACE&exclude=CORRUPT", expectStatusCode: http.StatusOK, }, { name: "Unhealthy if api is not available", healthCheckURL: "/health", apiError: fmt.Errorf("Unexpected error"), expectStatusCode: http.StatusServiceUnavailable, }, { name: "Unhealthy if no leader", healthCheckURL: "/health", expectStatusCode: http.StatusServiceUnavailable, missingLeader: true, }, { name: "Healthy if no leader and serializable=true", healthCheckURL: "/health?serializable=true", expectStatusCode: http.StatusOK, missingLeader: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mux := http.NewServeMux() lg := zaptest.NewLogger(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) HandleHealth(zaptest.NewLogger(t), mux, &fakeHealthServer{ fakeServer: fakeServer{alarms: tt.alarms}, serializableReadError: tt.apiError, linearizableReadError: tt.apiError, missingLeader: tt.missingLeader, authStore: auth.NewAuthStore(lg, schema.NewAuthBackend(lg, be), nil, 0), }) ts := httptest.NewServer(mux) defer ts.Close() checkHTTPResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, nil, nil) }) } } func TestHTTPSubPath(t *testing.T) { be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) tests := []healthTestCase{ { name: "/readyz/data_corruption ok", healthCheckURL: "/readyz/data_corruption", expectStatusCode: http.StatusOK, }, { name: "/readyz/serializable_read not ok with error", apiError: fmt.Errorf("Unexpected error"), healthCheckURL: "/readyz/serializable_read", expectStatusCode: http.StatusServiceUnavailable, notInResult: []string{"data_corruption"}, }, { name: "/readyz/learner ok", healthCheckURL: "/readyz/non_learner", expectStatusCode: http.StatusOK, }, { name: "/readyz/non_exist 404", healthCheckURL: "/readyz/non_exist", expectStatusCode: http.StatusNotFound, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mux := http.NewServeMux() logger := zaptest.NewLogger(t) s := &fakeHealthServer{ serializableReadError: tt.apiError, authStore: auth.NewAuthStore(logger, schema.NewAuthBackend(logger, be), nil, 0), } HandleHealth(logger, mux, s) ts := httptest.NewServer(mux) defer ts.Close() checkHTTPResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, tt.inResult, tt.notInResult) checkMetrics(t, tt.healthCheckURL, "", tt.expectStatusCode) }) } } func TestDataCorruptionCheck(t *testing.T) { be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) tests := []healthTestCase{ { name: "Live if CORRUPT alarm is on", alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_CORRUPT}}, healthCheckURL: "/livez", expectStatusCode: http.StatusOK, notInResult: []string{"data_corruption"}, }, { name: "Not ready if CORRUPT alarm is on", alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_CORRUPT}}, healthCheckURL: "/readyz", expectStatusCode: http.StatusServiceUnavailable, inResult: []string{"[-]data_corruption failed: alarm activated: CORRUPT"}, }, { name: "ready if CORRUPT alarm is not on", alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}}, healthCheckURL: "/readyz", expectStatusCode: http.StatusOK, }, { name: "ready if CORRUPT alarm is excluded", alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_CORRUPT}, {MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}}, healthCheckURL: "/readyz?exclude=data_corruption", expectStatusCode: http.StatusOK, }, { name: "Not ready if CORRUPT alarm is on", alarms: []*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_CORRUPT}}, healthCheckURL: "/readyz?exclude=non_exist", expectStatusCode: http.StatusServiceUnavailable, inResult: []string{"[-]data_corruption failed: alarm activated: CORRUPT"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mux := http.NewServeMux() logger := zaptest.NewLogger(t) s := &fakeHealthServer{ authStore: auth.NewAuthStore(logger, schema.NewAuthBackend(logger, be), nil, 0), } HandleHealth(logger, mux, s) ts := httptest.NewServer(mux) defer ts.Close() // OK before alarms are activated. checkHTTPResponse(t, ts, tt.healthCheckURL, http.StatusOK, nil, nil) // Activate the alarms. s.alarms = tt.alarms checkHTTPResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, tt.inResult, tt.notInResult) }) } } func TestSerializableReadCheck(t *testing.T) { be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) tests := []healthTestCase{ { name: "Alive normal", healthCheckURL: "/livez?verbose", expectStatusCode: http.StatusOK, inResult: []string{"[+]serializable_read ok"}, }, { name: "Not alive if range api is not available", healthCheckURL: "/livez", apiError: fmt.Errorf("Unexpected error"), expectStatusCode: http.StatusServiceUnavailable, inResult: []string{"[-]serializable_read failed: Unexpected error"}, }, { name: "Not ready if range api is not available", healthCheckURL: "/readyz", apiError: fmt.Errorf("Unexpected error"), expectStatusCode: http.StatusServiceUnavailable, inResult: []string{"[-]serializable_read failed: Unexpected error"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mux := http.NewServeMux() logger := zaptest.NewLogger(t) s := &fakeHealthServer{ serializableReadError: tt.apiError, authStore: auth.NewAuthStore(logger, schema.NewAuthBackend(logger, be), nil, 0), } HandleHealth(logger, mux, s) ts := httptest.NewServer(mux) defer ts.Close() checkHTTPResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, tt.inResult, tt.notInResult) checkMetrics(t, tt.healthCheckURL, "serializable_read", tt.expectStatusCode) }) } } func TestLinearizableReadCheck(t *testing.T) { be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) tests := []healthTestCase{ { name: "Alive normal", healthCheckURL: "/livez?verbose", expectStatusCode: http.StatusOK, inResult: []string{"[+]serializable_read ok"}, }, { name: "Alive if lineariable range api is not available", healthCheckURL: "/livez", apiError: fmt.Errorf("Unexpected error"), expectStatusCode: http.StatusOK, }, { name: "Not ready if range api is not available", healthCheckURL: "/readyz", apiError: fmt.Errorf("Unexpected error"), expectStatusCode: http.StatusServiceUnavailable, inResult: []string{"[+]serializable_read ok", "[-]linearizable_read failed: Unexpected error"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mux := http.NewServeMux() logger := zaptest.NewLogger(t) s := &fakeHealthServer{ linearizableReadError: tt.apiError, authStore: auth.NewAuthStore(logger, schema.NewAuthBackend(logger, be), nil, 0), } HandleHealth(logger, mux, s) ts := httptest.NewServer(mux) defer ts.Close() checkHTTPResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, tt.inResult, tt.notInResult) checkMetrics(t, tt.healthCheckURL, "linearizable_read", tt.expectStatusCode) }) } } func TestLearnerReadyCheck(t *testing.T) { be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) tests := []healthTestCase{ { name: "readyz normal", healthCheckURL: "/readyz", expectStatusCode: http.StatusOK, isLearner: false, }, { name: "not ready because member is learner", healthCheckURL: "/readyz", expectStatusCode: http.StatusServiceUnavailable, isLearner: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mux := http.NewServeMux() logger := zaptest.NewLogger(t) s := &fakeHealthServer{ linearizableReadError: tt.apiError, authStore: auth.NewAuthStore(logger, schema.NewAuthBackend(logger, be), nil, 0), } s.isLearner = tt.isLearner HandleHealth(logger, mux, s) ts := httptest.NewServer(mux) defer ts.Close() checkHTTPResponse(t, ts, tt.healthCheckURL, tt.expectStatusCode, tt.inResult, tt.notInResult) checkMetrics(t, tt.healthCheckURL, "linearizable_read", tt.expectStatusCode) }) } } func checkHTTPResponse(t *testing.T, ts *httptest.Server, url string, expectStatusCode int, inResult []string, notInResult []string) { res, err := ts.Client().Do(&http.Request{Method: http.MethodGet, URL: testutil.MustNewURL(t, ts.URL+url)}) if err != nil { t.Fatalf("fail serve http request %s %v", url, err) } if res.StatusCode != expectStatusCode { t.Errorf("want statusCode %d but got %d", expectStatusCode, res.StatusCode) } defer res.Body.Close() b, err := io.ReadAll(res.Body) if err != nil { t.Fatalf("Failed to read response for %s", url) } result := string(b) for _, substr := range inResult { if !strings.Contains(result, substr) { t.Errorf("Could not find substring : %s, in response: %s", substr, result) return } } for _, substr := range notInResult { if strings.Contains(result, substr) { t.Errorf("Do not expect substring : %s, in response: %s", substr, result) return } } } func checkMetrics(t *testing.T, url, checkName string, expectStatusCode int) { defer healthCheckGauge.Reset() defer healthCheckCounter.Reset() typeName := strings.TrimPrefix(strings.Split(url, "?")[0], "/") if len(checkName) == 0 { checkName = strings.Split(typeName, "/")[1] typeName = strings.Split(typeName, "/")[0] } expectedSuccessCount := 1 expectedErrorCount := 0 if expectStatusCode != http.StatusOK { expectedSuccessCount = 0 expectedErrorCount = 1 } gather, _ := prometheus.DefaultGatherer.Gather() for _, mf := range gather { name := *mf.Name val := 0 switch name { case "etcd_server_healthcheck": val = int(mf.GetMetric()[0].GetGauge().GetValue()) case "etcd_server_healthcheck_total": val = int(mf.GetMetric()[0].GetCounter().GetValue()) default: continue } labelMap := make(map[string]string) for _, label := range mf.GetMetric()[0].Label { labelMap[label.GetName()] = label.GetValue() } if typeName != labelMap["type"] { continue } if labelMap["name"] != checkName { continue } if statusLabel, found := labelMap["status"]; found && statusLabel == HealthStatusError { if val != expectedErrorCount { t.Fatalf("%s got errorCount %d, wanted %d\n", name, val, expectedErrorCount) } } else { if val != expectedSuccessCount { t.Fatalf("%s got expectedSuccessCount %d, wanted %d\n", name, val, expectedSuccessCount) } } } } ================================================ FILE: server/etcdserver/api/etcdhttp/metrics.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdhttp import ( "net/http" "github.com/prometheus/client_golang/prometheus/promhttp" ) const ( PathMetrics = "/metrics" PathProxyMetrics = "/proxy/metrics" ) // HandleMetrics registers prometheus handler on '/metrics'. func HandleMetrics(mux *http.ServeMux) { mux.Handle(PathMetrics, promhttp.Handler()) } ================================================ FILE: server/etcdserver/api/etcdhttp/peer.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdhttp import ( "encoding/json" errorspkg "errors" "fmt" "net/http" "strconv" "strings" "go.uber.org/zap" "google.golang.org/grpc/metadata" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/etcdserver/errors" "go.etcd.io/etcd/server/v3/lease/leasehttp" ) const ( peerMembersPath = "/members" peerMemberPromotePrefix = "/members/promote/" ) // NewPeerHandler generates an http.Handler to handle etcd peer requests. func NewPeerHandler(lg *zap.Logger, s etcdserver.ServerPeerV2) http.Handler { return newPeerHandler(lg, s, s.RaftHandler(), s.LeaseHandler(), s.HashKVHandler(), s.DowngradeEnabledHandler()) } func newPeerHandler( lg *zap.Logger, s etcdserver.Server, raftHandler http.Handler, leaseHandler http.Handler, hashKVHandler http.Handler, downgradeEnabledHandler http.Handler, ) http.Handler { if lg == nil { lg = zap.NewNop() } peerMembersHandler := newPeerMembersHandler(lg, s.Cluster()) peerMemberPromoteHandler := newPeerMemberPromoteHandler(lg, s) mux := http.NewServeMux() mux.HandleFunc("/", http.NotFound) mux.Handle(rafthttp.RaftPrefix, raftHandler) mux.Handle(rafthttp.RaftPrefix+"/", raftHandler) mux.Handle(peerMembersPath, peerMembersHandler) mux.Handle(peerMemberPromotePrefix, peerMemberPromoteHandler) if leaseHandler != nil { mux.Handle(leasehttp.LeasePrefix, leaseHandler) mux.Handle(leasehttp.LeaseInternalPrefix, leaseHandler) } if downgradeEnabledHandler != nil { mux.Handle(etcdserver.DowngradeEnabledPath, downgradeEnabledHandler) } if hashKVHandler != nil { mux.Handle(etcdserver.PeerHashKVPath, hashKVHandler) } mux.HandleFunc(versionPath, versionHandler(s, serveVersion)) return mux } func newPeerMembersHandler(lg *zap.Logger, cluster api.Cluster) http.Handler { return &peerMembersHandler{ lg: lg, cluster: cluster, } } type peerMembersHandler struct { lg *zap.Logger cluster api.Cluster } func newPeerMemberPromoteHandler(lg *zap.Logger, s etcdserver.Server) http.Handler { return &peerMemberPromoteHandler{ lg: lg, cluster: s.Cluster(), server: s, } } type peerMemberPromoteHandler struct { lg *zap.Logger cluster api.Cluster server etcdserver.Server } func (h *peerMembersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !allowMethod(w, r, "GET") { return } w.Header().Set("X-Etcd-Cluster-ID", h.cluster.ID().String()) if r.URL.Path != peerMembersPath { http.Error(w, "bad path", http.StatusBadRequest) return } ms := h.cluster.Members() w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(ms); err != nil { h.lg.Warn("failed to encode membership members", zap.Error(err)) } } func (h *peerMemberPromoteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !allowMethod(w, r, "POST") { return } w.Header().Set("X-Etcd-Cluster-ID", h.cluster.ID().String()) if !strings.HasPrefix(r.URL.Path, peerMemberPromotePrefix) { http.Error(w, "bad path", http.StatusBadRequest) return } idStr := strings.TrimPrefix(r.URL.Path, peerMemberPromotePrefix) id, err := strconv.ParseUint(idStr, 10, 64) if err != nil { http.Error(w, fmt.Sprintf("member %s not found in cluster", idStr), http.StatusNotFound) return } // reconstruct gRPC metadata from HTTP header (if present) so admin check can pass ctx := r.Context() if tok := r.Header.Get("Authorization"); tok != "" { md := metadata.New(map[string]string{rpctypes.TokenFieldNameGRPC: tok}) ctx = metadata.NewIncomingContext(ctx, md) } resp, err := h.server.PromoteMember(ctx, id) if err != nil { switch { case errorspkg.Is(err, membership.ErrIDNotFound): http.Error(w, err.Error(), http.StatusNotFound) case errorspkg.Is(err, membership.ErrMemberNotLearner): http.Error(w, err.Error(), http.StatusPreconditionFailed) case errorspkg.Is(err, errors.ErrLearnerNotReady): http.Error(w, err.Error(), http.StatusPreconditionFailed) default: writeError(h.lg, w, r, err) } h.lg.Warn( "failed to promote a member", zap.String("member-id", types.ID(id).String()), zap.Error(err), ) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) if err := json.NewEncoder(w).Encode(resp); err != nil { h.lg.Warn("failed to encode members response", zap.Error(err)) } } ================================================ FILE: server/etcdserver/api/etcdhttp/peer_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdhttp import ( "context" "encoding/json" "fmt" "io" "net/http" "net/http/httptest" "path" "sort" "strings" "testing" "github.com/coreos/go-semver/semver" "go.uber.org/zap/zaptest" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" ) type fakeCluster struct { id uint64 clientURLs []string members map[uint64]*membership.Member } func (c *fakeCluster) ID() types.ID { return types.ID(c.id) } func (c *fakeCluster) ClientURLs() []string { return c.clientURLs } func (c *fakeCluster) Members() []*membership.Member { ms := make(membership.MembersByID, 0, len(c.members)) for _, m := range c.members { ms = append(ms, m) } sort.Sort(ms) return ms } func (c *fakeCluster) Member(id types.ID) *membership.Member { return c.members[uint64(id)] } func (c *fakeCluster) Version() *semver.Version { return nil } type fakeServer struct { cluster api.Cluster alarms []*pb.AlarmMember } func (s *fakeServer) AddMember(ctx context.Context, memb membership.Member) ([]*membership.Member, error) { return nil, fmt.Errorf("AddMember not implemented in fakeServer") } func (s *fakeServer) RemoveMember(ctx context.Context, id uint64) ([]*membership.Member, error) { return nil, fmt.Errorf("RemoveMember not implemented in fakeServer") } func (s *fakeServer) UpdateMember(ctx context.Context, updateMemb membership.Member) ([]*membership.Member, error) { return nil, fmt.Errorf("UpdateMember not implemented in fakeServer") } func (s *fakeServer) PromoteMember(ctx context.Context, id uint64) ([]*membership.Member, error) { return nil, fmt.Errorf("PromoteMember not implemented in fakeServer") } func (s *fakeServer) ClusterVersion() *semver.Version { return nil } func (s *fakeServer) StorageVersion() *semver.Version { return nil } func (s *fakeServer) Cluster() api.Cluster { return s.cluster } func (s *fakeServer) Alarms() []*pb.AlarmMember { return s.alarms } func (s *fakeServer) LeaderChangedNotify() <-chan struct{} { return nil } var fakeRaftHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("test data")) }) // TestNewPeerHandlerOnRaftPrefix tests that NewPeerHandler returns a handler that // handles raft-prefix requests well. func TestNewPeerHandlerOnRaftPrefix(t *testing.T) { ph := newPeerHandler(zaptest.NewLogger(t), &fakeServer{cluster: &fakeCluster{}}, fakeRaftHandler, nil, nil, nil) srv := httptest.NewServer(ph) defer srv.Close() tests := []string{ rafthttp.RaftPrefix, rafthttp.RaftPrefix + "/hello", } for i, tt := range tests { resp, err := http.Get(srv.URL + tt) if err != nil { t.Fatalf("unexpected http.Get error: %v", err) } body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("unexpected io.ReadAll error: %v", err) } resp.Body.Close() if w := "test data"; string(body) != w { t.Errorf("#%d: body = %s, want %s", i, body, w) } } } // TestServeMembersFails ensures peerMembersHandler only accepts GET request func TestServeMembersFails(t *testing.T) { tests := []struct { method string wcode int }{ { "POST", http.StatusMethodNotAllowed, }, { "PUT", http.StatusMethodNotAllowed, }, { "DELETE", http.StatusMethodNotAllowed, }, { "BAD", http.StatusMethodNotAllowed, }, } for i, tt := range tests { rw := httptest.NewRecorder() h := newPeerMembersHandler(nil, &fakeCluster{}) req, err := http.NewRequest(tt.method, "", nil) if err != nil { t.Fatalf("#%d: failed to create http request: %v", i, err) } h.ServeHTTP(rw, req) if rw.Code != tt.wcode { t.Errorf("#%d: code=%d, want %d", i, rw.Code, tt.wcode) } } } func TestServeMembersGet(t *testing.T) { memb1 := membership.Member{ID: 1, Attributes: membership.Attributes{ClientURLs: []string{"http://localhost:8080"}}} memb2 := membership.Member{ID: 2, Attributes: membership.Attributes{ClientURLs: []string{"http://localhost:8081"}}} cluster := &fakeCluster{ id: 1, members: map[uint64]*membership.Member{1: &memb1, 2: &memb2}, } h := newPeerMembersHandler(nil, cluster) msb, err := json.Marshal([]membership.Member{memb1, memb2}) if err != nil { t.Fatal(err) } wms := string(msb) + "\n" tests := []struct { path string wcode int wct string wbody string }{ {peerMembersPath, http.StatusOK, "application/json", wms}, {path.Join(peerMembersPath, "bad"), http.StatusBadRequest, "text/plain; charset=utf-8", "bad path\n"}, } for i, tt := range tests { req, err := http.NewRequest(http.MethodGet, testutil.MustNewURL(t, tt.path).String(), nil) if err != nil { t.Fatal(err) } rw := httptest.NewRecorder() h.ServeHTTP(rw, req) if rw.Code != tt.wcode { t.Errorf("#%d: code=%d, want %d", i, rw.Code, tt.wcode) } if gct := rw.Header().Get("Content-Type"); gct != tt.wct { t.Errorf("#%d: content-type = %s, want %s", i, gct, tt.wct) } if rw.Body.String() != tt.wbody { t.Errorf("#%d: body = %s, want %s", i, rw.Body.String(), tt.wbody) } gcid := rw.Header().Get("X-Etcd-Cluster-ID") wcid := cluster.ID().String() if gcid != wcid { t.Errorf("#%d: cid = %s, want %s", i, gcid, wcid) } } } // TestServeMemberPromoteFails ensures peerMemberPromoteHandler only accepts POST request func TestServeMemberPromoteFails(t *testing.T) { tests := []struct { method string wcode int }{ { "GET", http.StatusMethodNotAllowed, }, { "PUT", http.StatusMethodNotAllowed, }, { "DELETE", http.StatusMethodNotAllowed, }, { "BAD", http.StatusMethodNotAllowed, }, } for i, tt := range tests { rw := httptest.NewRecorder() h := newPeerMemberPromoteHandler(nil, &fakeServer{cluster: &fakeCluster{}}) req, err := http.NewRequest(tt.method, "", nil) if err != nil { t.Fatalf("#%d: failed to create http request: %v", i, err) } h.ServeHTTP(rw, req) if rw.Code != tt.wcode { t.Errorf("#%d: code=%d, want %d", i, rw.Code, tt.wcode) } } } // TestNewPeerHandlerOnMembersPromotePrefix verifies the request with members promote prefix is routed correctly func TestNewPeerHandlerOnMembersPromotePrefix(t *testing.T) { ph := newPeerHandler(zaptest.NewLogger(t), &fakeServer{cluster: &fakeCluster{}}, fakeRaftHandler, nil, nil, nil) srv := httptest.NewServer(ph) defer srv.Close() tests := []struct { path string wcode int checkBody bool wKeyWords string }{ { // does not contain member id in path peerMemberPromotePrefix, http.StatusNotFound, false, "", }, { // try to promote member id = 1 peerMemberPromotePrefix + "1", http.StatusInternalServerError, true, "PromoteMember not implemented in fakeServer", }, } for i, tt := range tests { req, err := http.NewRequest(http.MethodPost, srv.URL+tt.path, nil) if err != nil { t.Fatalf("failed to create request: %v", err) } resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("failed to get http response: %v", err) } body, err := io.ReadAll(resp.Body) resp.Body.Close() if err != nil { t.Fatalf("unexpected io.ReadAll error: %v", err) } if resp.StatusCode != tt.wcode { t.Fatalf("#%d: code = %d, want %d", i, resp.StatusCode, tt.wcode) } if tt.checkBody && strings.Contains(string(body), tt.wKeyWords) { t.Errorf("#%d: body: %s, want body to contain keywords: %s", i, body, tt.wKeyWords) } } } ================================================ FILE: server/etcdserver/api/etcdhttp/types/errors.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package httptypes import ( "encoding/json" "fmt" "net/http" ) type HTTPError struct { Message string `json:"message"` // Code is the HTTP status code Code int `json:"-"` } func (e HTTPError) Error() string { return e.Message } func (e HTTPError) WriteTo(w http.ResponseWriter) error { w.Header().Set("Content-Type", "application/json") w.WriteHeader(e.Code) b, err := json.Marshal(e) if err != nil { panic(fmt.Sprintf("failed to marshal HTTPError: %v", err)) } if _, err := w.Write(b); err != nil { return err } return nil } func NewHTTPError(code int, m string) *HTTPError { return &HTTPError{ Message: m, Code: code, } } ================================================ FILE: server/etcdserver/api/etcdhttp/types/errors_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package httptypes import ( "net/http" "net/http/httptest" "reflect" "testing" ) func TestHTTPErrorWriteTo(t *testing.T) { err := NewHTTPError(http.StatusBadRequest, "what a bad request you made!") rr := httptest.NewRecorder() if e := err.WriteTo(rr); e != nil { t.Fatalf("HTTPError.WriteTo error (%v)", e) } wcode := http.StatusBadRequest wheader := http.Header(map[string][]string{ "Content-Type": {"application/json"}, }) wbody := `{"message":"what a bad request you made!"}` if wcode != rr.Code { t.Errorf("HTTP status code %d, want %d", rr.Code, wcode) } if !reflect.DeepEqual(wheader, rr.HeaderMap) { //nolint:staticcheck // TODO: remove for a supported version t.Errorf("HTTP headers %v, want %v", rr.HeaderMap, wheader) //nolint:staticcheck // TODO: remove for a supported version } gbody := rr.Body.String() if wbody != gbody { t.Errorf("HTTP body %q, want %q", gbody, wbody) } } ================================================ FILE: server/etcdserver/api/etcdhttp/utils.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdhttp import ( errorspkg "errors" "net/http" "go.uber.org/zap" httptypes "go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/types" "go.etcd.io/etcd/server/v3/etcdserver/api/v2error" "go.etcd.io/etcd/server/v3/etcdserver/errors" ) func allowMethod(w http.ResponseWriter, r *http.Request, m string) bool { if m == r.Method { return true } w.Header().Set("Allow", m) http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) return false } // writeError logs and writes the given Error to the ResponseWriter // If Error is an etcdErr, it is rendered to the ResponseWriter // Otherwise, it is assumed to be a StatusInternalServerError func writeError(lg *zap.Logger, w http.ResponseWriter, r *http.Request, err error) { if err == nil { return } var v2Err *v2error.Error var httpErr *httptypes.HTTPError switch { case errorspkg.As(err, &v2Err): v2Err.WriteTo(w) case errorspkg.As(err, &httpErr): if et := httpErr.WriteTo(w); et != nil { if lg != nil { lg.Debug( "failed to write v2 HTTP error", zap.String("remote-addr", r.RemoteAddr), zap.String("internal-server-error", httpErr.Error()), zap.Error(et), ) } } default: switch { case errorspkg.Is(err, errors.ErrTimeoutDueToLeaderFail), errorspkg.Is(err, errors.ErrTimeoutDueToConnectionLost), errorspkg.Is(err, errors.ErrNotEnoughStartedMembers), errorspkg.Is(err, errors.ErrUnhealthy): if lg != nil { lg.Warn( "v2 response error", zap.String("remote-addr", r.RemoteAddr), zap.String("internal-server-error", err.Error()), ) } default: if lg != nil { lg.Warn( "unexpected v2 response error", zap.String("remote-addr", r.RemoteAddr), zap.String("internal-server-error", err.Error()), ) } } herr := httptypes.NewHTTPError(http.StatusInternalServerError, "Internal Server Error") if et := herr.WriteTo(w); et != nil { if lg != nil { lg.Debug( "failed to write v2 HTTP error", zap.String("remote-addr", r.RemoteAddr), zap.String("internal-server-error", err.Error()), zap.Error(et), ) } } } } ================================================ FILE: server/etcdserver/api/etcdhttp/version.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdhttp import ( "encoding/json" "fmt" "net/http" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/server/v3/etcdserver" ) const ( versionPath = "/version" ) func HandleVersion(mux *http.ServeMux, server etcdserver.Server) { mux.HandleFunc(versionPath, versionHandler(server, serveVersion)) } func versionHandler(server etcdserver.Server, fn func(http.ResponseWriter, *http.Request, string, string)) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { clusterVersion := server.ClusterVersion() storageVersion := server.StorageVersion() clusterVersionStr, storageVersionStr := "not_decided", "unknown" if clusterVersion != nil { clusterVersionStr = clusterVersion.String() } if storageVersion != nil { storageVersionStr = storageVersion.String() } fn(w, r, clusterVersionStr, storageVersionStr) } } func serveVersion(w http.ResponseWriter, r *http.Request, clusterV, storageV string) { if !allowMethod(w, r, "GET") { return } vs := version.Versions{ Server: version.Version, Cluster: clusterV, Storage: storageV, } w.Header().Set("Content-Type", "application/json") b, err := json.Marshal(&vs) if err != nil { panic(fmt.Sprintf("cannot marshal versions to json (%v)", err)) } w.Write(b) } ================================================ FILE: server/etcdserver/api/etcdhttp/version_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdhttp import ( "encoding/json" "net/http" "net/http/httptest" "testing" "go.etcd.io/etcd/api/v3/version" ) func TestServeVersion(t *testing.T) { req, err := http.NewRequest(http.MethodGet, "", nil) if err != nil { t.Fatalf("error creating request: %v", err) } rw := httptest.NewRecorder() serveVersion(rw, req, "3.6.0", "3.5.2") if rw.Code != http.StatusOK { t.Errorf("code=%d, want %d", rw.Code, http.StatusOK) } vs := version.Versions{ Server: version.Version, Cluster: "3.6.0", Storage: "3.5.2", } w, err := json.Marshal(&vs) if err != nil { t.Fatal(err) } if g := rw.Body.String(); g != string(w) { t.Fatalf("body = %q, want %q", g, string(w)) } if ct := rw.HeaderMap.Get("Content-Type"); ct != "application/json" { //nolint:staticcheck // TODO: remove for a supported version t.Errorf("contet-type header = %s, want %s", ct, "application/json") } } func TestServeVersionFails(t *testing.T) { for _, m := range []string{ "CONNECT", "TRACE", "PUT", "POST", "HEAD", } { t.Run(m, func(t *testing.T) { req, err := http.NewRequest(m, "", nil) if err != nil { t.Fatalf("error creating request: %v", err) } rw := httptest.NewRecorder() serveVersion(rw, req, "3.6.0", "3.5.2") if rw.Code != http.StatusMethodNotAllowed { t.Errorf("method %s: code=%d, want %d", m, rw.Code, http.StatusMethodNotAllowed) } }) } } ================================================ FILE: server/etcdserver/api/membership/cluster.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership import ( "context" "crypto/sha1" "encoding/binary" "encoding/json" "fmt" "sort" "strings" "sync" "time" "github.com/coreos/go-semver/semver" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/netutil" "go.etcd.io/etcd/pkg/v3/notify" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" serverversion "go.etcd.io/etcd/server/v3/etcdserver/version" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) // RaftCluster is a list of Members that belong to the same raft cluster type RaftCluster struct { lg *zap.Logger localID types.ID cid types.ID be MembershipBackend sync.Mutex // guards the fields below version *semver.Version members map[types.ID]*Member // removed contains the ids of removed members in the cluster. // removed id cannot be reused. removed map[types.ID]bool downgradeInfo *serverversion.DowngradeInfo maxLearners int versionChanged *notify.Notifier } // ConfigChangeContext represents a context for confChange. type ConfigChangeContext struct { Member // IsPromote indicates if the config change is for promoting a learner member. // This flag is needed because both adding a new member and promoting a learner member // uses the same config change type 'ConfChangeAddNode'. IsPromote bool `json:"isPromote"` } type ShouldApplyV3 bool const ( ApplyBoth = ShouldApplyV3(true) ApplyV2storeOnly = ShouldApplyV3(false) ) // NewClusterFromURLsMap creates a new raft cluster using provided urls map. Currently, it does not support creating // cluster with raft learner member. func NewClusterFromURLsMap(lg *zap.Logger, token string, urlsmap types.URLsMap, opts ...ClusterOption) (*RaftCluster, error) { c := NewCluster(lg, opts...) for name, urls := range urlsmap { m := NewMember(name, urls, token, nil) if _, ok := c.members[m.ID]; ok { return nil, fmt.Errorf("member exists with identical ID %v", m) } if uint64(m.ID) == raft.None { return nil, fmt.Errorf("cannot use %x as member id", raft.None) } c.members[m.ID] = m } c.genID() return c, nil } func NewClusterFromMembers(lg *zap.Logger, id types.ID, membs []*Member, opts ...ClusterOption) *RaftCluster { c := NewCluster(lg, opts...) c.cid = id for _, m := range membs { c.members[m.ID] = m } return c } func NewCluster(lg *zap.Logger, opts ...ClusterOption) *RaftCluster { if lg == nil { lg = zap.NewNop() } clOpts := newClusterOpts(opts...) return &RaftCluster{ lg: lg, members: make(map[types.ID]*Member), removed: make(map[types.ID]bool), downgradeInfo: &serverversion.DowngradeInfo{Enabled: false}, maxLearners: clOpts.maxLearners, } } func (c *RaftCluster) ID() types.ID { return c.cid } func (c *RaftCluster) Members() []*Member { c.Lock() defer c.Unlock() var ms MembersByID for _, m := range c.members { ms = append(ms, m.Clone()) } sort.Sort(ms) return ms } func (c *RaftCluster) Member(id types.ID) *Member { c.Lock() defer c.Unlock() return c.members[id].Clone() } func (c *RaftCluster) VotingMembers() []*Member { c.Lock() defer c.Unlock() var ms MembersByID for _, m := range c.members { if !m.IsLearner { ms = append(ms, m.Clone()) } } sort.Sort(ms) return ms } // MemberByName returns a Member with the given name if exists. // If more than one member has the given name, it will panic. func (c *RaftCluster) MemberByName(name string) *Member { c.Lock() defer c.Unlock() var memb *Member for _, m := range c.members { if m.Name == name { if memb != nil { c.lg.Panic("two member with same name found", zap.String("name", name)) } memb = m } } return memb.Clone() } func (c *RaftCluster) MemberIDs() []types.ID { c.Lock() defer c.Unlock() var ids []types.ID for _, m := range c.members { ids = append(ids, m.ID) } sort.Sort(types.IDSlice(ids)) return ids } func (c *RaftCluster) IsIDRemoved(id types.ID) bool { c.Lock() defer c.Unlock() return c.removed[id] } // PeerURLs returns a list of all peer addresses. // The returned list is sorted in ascending lexicographical order. func (c *RaftCluster) PeerURLs() []string { c.Lock() defer c.Unlock() urls := make([]string, 0) for _, p := range c.members { urls = append(urls, p.PeerURLs...) } sort.Strings(urls) return urls } // ClientURLs returns a list of all client addresses. // The returned list is sorted in ascending lexicographical order. func (c *RaftCluster) ClientURLs() []string { c.Lock() defer c.Unlock() urls := make([]string, 0) for _, p := range c.members { urls = append(urls, p.ClientURLs...) } sort.Strings(urls) return urls } func (c *RaftCluster) String() string { c.Lock() defer c.Unlock() b := &strings.Builder{} fmt.Fprintf(b, "{ClusterID:%s ", c.cid) var ms []string for _, m := range c.members { ms = append(ms, fmt.Sprintf("%+v", m)) } fmt.Fprintf(b, "Members:[%s] ", strings.Join(ms, " ")) var ids []string for id := range c.removed { ids = append(ids, id.String()) } fmt.Fprintf(b, "RemovedMemberIDs:[%s]}", strings.Join(ids, " ")) return b.String() } func (c *RaftCluster) genID() { mIDs := c.MemberIDs() b := make([]byte, 8*len(mIDs)) for i, id := range mIDs { binary.BigEndian.PutUint64(b[8*i:], uint64(id)) } hash := sha1.Sum(b) c.cid = types.ID(binary.BigEndian.Uint64(hash[:8])) } func (c *RaftCluster) SetID(localID, cid types.ID) { c.localID = localID c.cid = cid c.buildMembershipMetric() } func (c *RaftCluster) SetBackend(be MembershipBackend) { c.be = be c.be.MustCreateBackendBuckets() } func (c *RaftCluster) SetVersionChangedNotifier(n *notify.Notifier) { c.versionChanged = n } func (c *RaftCluster) UnsafeLoad() { c.version = c.be.ClusterVersionFromBackend() c.members, c.removed = c.be.MustReadMembersFromBackend() c.downgradeInfo = c.be.DowngradeInfoFromBackend() } func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) { c.Lock() defer c.Unlock() c.UnsafeLoad() c.buildMembershipMetric() sv := semver.Must(semver.NewVersion(version.Version)) if c.downgradeInfo != nil && c.downgradeInfo.Enabled { c.lg.Info( "cluster is downgrading to target version", zap.String("target-cluster-version", c.downgradeInfo.TargetVersion), zap.String("current-server-version", sv.String()), ) } serverversion.MustDetectDowngrade(c.lg, sv, c.version) onSet(c.lg, c.version) for _, m := range c.members { if c.localID == m.ID { setIsLearnerMetric(m) } c.lg.Info( "recovered/added member from store", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("recovered-remote-peer-id", m.ID.String()), zap.Strings("recovered-remote-peer-urls", m.PeerURLs), zap.Bool("recovered-remote-peer-is-learner", m.IsLearner), ) } if c.version != nil { c.lg.Info( "set cluster version from store", zap.String("cluster-version", version.Cluster(c.version.String())), ) } } // ValidateConfigurationChange takes a proposed ConfChange and // ensures that it is still valid. func (c *RaftCluster) ValidateConfigurationChange(cc raftpb.ConfChange, shouldApplyV3 ShouldApplyV3) error { if !shouldApplyV3 { return nil } membersMap, removedMap := c.be.MustReadMembersFromBackend() id := types.ID(cc.NodeID) if removedMap[id] { return ErrIDRemoved } switch cc.Type { case raftpb.ConfChangeAddNode, raftpb.ConfChangeAddLearnerNode: confChangeContext := new(ConfigChangeContext) if err := json.Unmarshal(cc.Context, confChangeContext); err != nil { c.lg.Panic("failed to unmarshal confChangeContext", zap.Error(err)) } if confChangeContext.IsPromote { // promoting a learner member to voting member if membersMap[id] == nil { return ErrIDNotFound } if !membersMap[id].IsLearner { return ErrMemberNotLearner } } else { // adding a new member if membersMap[id] != nil { return ErrIDExists } var members []*Member urls := make(map[string]bool) for _, m := range membersMap { members = append(members, m) for _, u := range m.PeerURLs { urls[u] = true } } for _, u := range confChangeContext.Member.PeerURLs { if urls[u] { return ErrPeerURLexists } } if confChangeContext.Member.RaftAttributes.IsLearner && cc.Type == raftpb.ConfChangeAddLearnerNode { // the new member is a learner scaleUpLearners := true if err := ValidateMaxLearnerConfig(c.maxLearners, members, scaleUpLearners); err != nil { return err } } } case raftpb.ConfChangeRemoveNode: if membersMap[id] == nil { return ErrIDNotFound } case raftpb.ConfChangeUpdateNode: if membersMap[id] == nil { return ErrIDNotFound } urls := make(map[string]bool) for _, m := range membersMap { if m.ID == id { continue } for _, u := range m.PeerURLs { urls[u] = true } } m := new(Member) if err := json.Unmarshal(cc.Context, m); err != nil { c.lg.Panic("failed to unmarshal member", zap.Error(err)) } for _, u := range m.PeerURLs { if urls[u] { return ErrPeerURLexists } } default: c.lg.Panic("unknown ConfChange type", zap.String("type", cc.Type.String())) } return nil } // AddMember adds a new Member into the cluster, and saves the given member's // raftAttributes into the store. The given member should have empty attributes. // A Member with a matching id must not exist. func (c *RaftCluster) AddMember(m *Member, shouldApplyV3 ShouldApplyV3) { c.Lock() defer c.Unlock() if m.ID == c.localID { setIsLearnerMetric(m) } if shouldApplyV3 { c.be.MustSaveMemberToBackend(m) c.members[m.ID] = m c.updateMembershipMetric(m.ID, true) c.lg.Info( "added member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("added-peer-id", m.ID.String()), zap.Strings("added-peer-peer-urls", m.PeerURLs), zap.Bool("added-peer-is-learner", m.IsLearner), ) } else { c.lg.Info( "ignore already added member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("added-peer-id", m.ID.String()), zap.Strings("added-peer-peer-urls", m.PeerURLs), zap.Bool("added-peer-is-learner", m.IsLearner)) } } // RemoveMember removes a member from the store. // The given id MUST exist, or the function panics. func (c *RaftCluster) RemoveMember(id types.ID, shouldApplyV3 ShouldApplyV3) { c.Lock() defer c.Unlock() if shouldApplyV3 { c.be.MustDeleteMemberFromBackend(id) m, ok := c.members[id] delete(c.members, id) c.removed[id] = true c.updateMembershipMetric(id, false) if ok { c.lg.Info( "removed member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("removed-remote-peer-id", id.String()), zap.Strings("removed-remote-peer-urls", m.PeerURLs), zap.Bool("removed-remote-peer-is-learner", m.IsLearner), ) } else { c.lg.Warn( "skipped removing already removed member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("removed-remote-peer-id", id.String()), ) } } else { c.lg.Info( "ignore already removed member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("removed-remote-peer-id", id.String()), ) } } func (c *RaftCluster) UpdateAttributes(id types.ID, attr Attributes, shouldApplyV3 ShouldApplyV3) { c.Lock() defer c.Unlock() if m, ok := c.members[id]; ok { m.Attributes = attr if shouldApplyV3 { c.be.MustSaveMemberToBackend(m) } return } _, ok := c.removed[id] if !ok { c.lg.Panic( "failed to update; member unknown", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("unknown-remote-peer-id", id.String()), ) } c.lg.Warn( "skipped attributes update of removed member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("updated-peer-id", id.String()), ) } // PromoteMember marks the member's IsLearner RaftAttributes to false. func (c *RaftCluster) PromoteMember(id types.ID, shouldApplyV3 ShouldApplyV3) { c.Lock() defer c.Unlock() if id == c.localID { isLearner.Set(0) } if shouldApplyV3 { if m, ok := c.members[id]; ok { m.RaftAttributes.IsLearner = false c.updateMembershipMetric(id, true) c.be.MustSaveMemberToBackend(m) c.lg.Info( "promote member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("promoted-member-id", id.String()), ) } else { c.lg.Info( "ignore promoting non-existent member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("promoted-member-id", id.String()), ) } } else { c.lg.Info( "ignore already promoted member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), ) } } func (c *RaftCluster) UpdateRaftAttributes(id types.ID, raftAttr RaftAttributes, shouldApplyV3 ShouldApplyV3) { c.Lock() defer c.Unlock() if _, ok := c.members[id]; ok { m := *(c.members[id]) m.RaftAttributes = raftAttr } else { c.lg.Info("Skipped updating non-existent member in v2store", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("updated-remote-peer-id", id.String()), zap.Strings("updated-remote-peer-urls", raftAttr.PeerURLs), zap.Bool("updated-remote-peer-is-learner", raftAttr.IsLearner), ) } if shouldApplyV3 { if m, ok := c.members[id]; ok { m.RaftAttributes = raftAttr c.be.MustSaveMemberToBackend(m) c.lg.Info( "updated member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("updated-remote-peer-id", id.String()), zap.Strings("updated-remote-peer-urls", raftAttr.PeerURLs), zap.Bool("updated-remote-peer-is-learner", raftAttr.IsLearner), ) } else { c.lg.Info( "ignore updating non-existent member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("updated-remote-peer-id", id.String()), zap.Strings("updated-remote-peer-urls", raftAttr.PeerURLs), zap.Bool("updated-remote-peer-is-learner", raftAttr.IsLearner), ) } } else { c.lg.Info( "ignored already updated member", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("updated-remote-peer-id", id.String()), zap.Strings("updated-remote-peer-urls", raftAttr.PeerURLs), zap.Bool("updated-remote-peer-is-learner", raftAttr.IsLearner), ) } } func (c *RaftCluster) Version() *semver.Version { c.Lock() defer c.Unlock() if c.version == nil { return nil } return semver.Must(semver.NewVersion(c.version.String())) } func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*zap.Logger, *semver.Version), shouldApplyV3 ShouldApplyV3) { c.Lock() defer c.Unlock() if c.version != nil { c.lg.Info( "updated cluster version", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("from", version.Cluster(c.version.String())), zap.String("to", version.Cluster(ver.String())), ) } else { c.lg.Info( "set initial cluster version", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), zap.String("cluster-version", version.Cluster(ver.String())), ) } oldVer := c.version c.version = ver sv := semver.Must(semver.NewVersion(version.Version)) serverversion.MustDetectDowngrade(c.lg, sv, c.version) if shouldApplyV3 { c.be.MustSaveClusterVersionToBackend(ver) } if oldVer != nil { ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(oldVer.String())}).Set(0) } ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(ver.String())}).Set(1) if c.versionChanged != nil { c.versionChanged.Notify() } onSet(c.lg, ver) } func (c *RaftCluster) IsReadyToAddVotingMember() bool { nmembers := 1 nstarted := 0 for _, member := range c.VotingMembers() { if member.IsStarted() { nstarted++ } nmembers++ } if nstarted == 1 && nmembers == 2 { // a case of adding a new node to 1-member cluster for restoring cluster data // https://github.com/etcd-io/website/blob/main/content/docs/v2/admin_guide.md#restoring-the-cluster c.lg.Debug("number of started member is 1; can accept add member request") return true } nquorum := nmembers/2 + 1 if nstarted < nquorum { c.lg.Warn( "rejecting member add; started member will be less than quorum", zap.Int("number-of-started-member", nstarted), zap.Int("quorum", nquorum), zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), ) return false } return true } func (c *RaftCluster) IsReadyToRemoveVotingMember(id uint64) bool { nmembers := 0 nstarted := 0 for _, member := range c.VotingMembers() { if uint64(member.ID) == id { continue } if member.IsStarted() { nstarted++ } nmembers++ } nquorum := nmembers/2 + 1 if nstarted < nquorum { c.lg.Warn( "rejecting member remove; started member will be less than quorum", zap.Int("number-of-started-member", nstarted), zap.Int("quorum", nquorum), zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), ) return false } return true } func (c *RaftCluster) IsReadyToPromoteMember(id uint64) bool { nmembers := 1 // We count the learner to be promoted for the future quorum nstarted := 1 // and we also count it as started. for _, member := range c.VotingMembers() { if member.IsStarted() { nstarted++ } nmembers++ } nquorum := nmembers/2 + 1 if nstarted < nquorum { c.lg.Warn( "rejecting member promote; started member will be less than quorum", zap.Int("number-of-started-member", nstarted), zap.Int("quorum", nquorum), zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), ) return false } return true } func MembersFromStore(lg *zap.Logger, st v2store.Store) (map[types.ID]*Member, map[types.ID]bool) { members := make(map[types.ID]*Member) removed := make(map[types.ID]bool) e, err := st.Get(StoreMembersPrefix, true, true) if err != nil { if isKeyNotFound(err) { return members, removed } lg.Panic("failed to get members from store", zap.String("path", StoreMembersPrefix), zap.Error(err)) } for _, n := range e.Node.Nodes { var m *Member m, err = nodeToMember(lg, n) if err != nil { lg.Panic("failed to nodeToMember", zap.Error(err)) } members[m.ID] = m } e, err = st.Get(storeRemovedMembersPrefix, true, true) if err != nil { if isKeyNotFound(err) { return members, removed } lg.Panic( "failed to get removed members from store", zap.String("path", storeRemovedMembersPrefix), zap.Error(err), ) } for _, n := range e.Node.Nodes { removed[MustParseMemberIDFromKey(lg, n.Key)] = true } return members, removed } // ValidateClusterAndAssignIDs validates the local cluster by matching the PeerURLs // with the existing cluster. If the validation succeeds, it assigns the IDs // from the existing cluster to the local cluster. // If the validation fails, an error will be returned. func ValidateClusterAndAssignIDs(lg *zap.Logger, local *RaftCluster, existing *RaftCluster) error { ems := existing.Members() lms := local.Members() if len(ems) != len(lms) { return fmt.Errorf("member count is unequal") } ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) defer cancel() for i := range ems { var err error ok := false for j := range lms { if ok, err = netutil.URLStringsEqual(ctx, lg, ems[i].PeerURLs, lms[j].PeerURLs); ok { lms[j].ID = ems[i].ID break } } if !ok { return fmt.Errorf("PeerURLs: no match found for existing member (%v, %v), last resolver error (%w)", ems[i].ID, ems[i].PeerURLs, err) } } local.members = make(map[types.ID]*Member) for _, m := range lms { local.members[m.ID] = m } local.buildMembershipMetric() return nil } // IsLocalMemberLearner returns if the local member is raft learner func (c *RaftCluster) IsLocalMemberLearner() bool { c.Lock() defer c.Unlock() localMember, ok := c.members[c.localID] if !ok { c.lg.Panic( "failed to find local ID in cluster members", zap.String("cluster-id", c.cid.String()), zap.String("local-member-id", c.localID.String()), ) } return localMember.IsLearner } // DowngradeInfo returns the downgrade status of the cluster func (c *RaftCluster) DowngradeInfo() *serverversion.DowngradeInfo { c.Lock() defer c.Unlock() if c.downgradeInfo == nil { return &serverversion.DowngradeInfo{Enabled: false} } d := &serverversion.DowngradeInfo{Enabled: c.downgradeInfo.Enabled, TargetVersion: c.downgradeInfo.TargetVersion} return d } func (c *RaftCluster) SetDowngradeInfo(d *serverversion.DowngradeInfo, shouldApplyV3 ShouldApplyV3) { c.Lock() defer c.Unlock() if shouldApplyV3 { c.be.MustSaveDowngradeToBackend(d) } c.downgradeInfo = d } // IsMemberExist returns if the member with the given id exists in cluster. func (c *RaftCluster) IsMemberExist(id types.ID) bool { c.Lock() _, ok := c.members[id] c.Unlock() // gofail: var afterIsMemberExist struct{} return ok } // VotingMemberIDs returns the ID of voting members in cluster. func (c *RaftCluster) VotingMemberIDs() []types.ID { c.Lock() defer c.Unlock() var ids []types.ID for _, m := range c.members { if !m.IsLearner { ids = append(ids, m.ID) } } sort.Sort(types.IDSlice(ids)) return ids } // buildMembershipMetric sets the knownPeers metric based on the current // members of the cluster. func (c *RaftCluster) buildMembershipMetric() { if c.localID == 0 { // We don't know our own id yet. return } for p := range c.members { knownPeers.WithLabelValues(c.localID.String(), p.String()).Set(1) } for p := range c.removed { knownPeers.WithLabelValues(c.localID.String(), p.String()).Set(0) } } // updateMembershipMetric updates the knownPeers metric to indicate that // the given peer is now (un)known. func (c *RaftCluster) updateMembershipMetric(peer types.ID, known bool) { if c.localID == 0 { // We don't know our own id yet. return } v := float64(0) if known { v = 1 } knownPeers.WithLabelValues(c.localID.String(), peer.String()).Set(v) } // ValidateMaxLearnerConfig verifies the existing learner members in the cluster membership and an optional N+1 learner // scale up are not more than maxLearners. func ValidateMaxLearnerConfig(maxLearners int, members []*Member, scaleUpLearners bool) error { numLearners := 0 for _, m := range members { if m.IsLearner { numLearners++ } } // Validate config can accommodate scale up. if scaleUpLearners { numLearners++ } if numLearners > maxLearners { return ErrTooManyLearners } return nil } func (c *RaftCluster) Store(store v2store.Store) { c.Lock() defer c.Unlock() verifyNoMembersInStore(c.lg, store) for _, m := range c.members { mustSaveMemberToStore(c.lg, store, m) if m.ClientURLs != nil { mustUpdateMemberAttrInStore(c.lg, store, m) } c.lg.Debug( "snapshot storing member", zap.String("id", m.ID.String()), zap.Strings("peer-urls", m.PeerURLs), zap.Bool("is-learner", m.IsLearner), ) } for id := range c.removed { // We do not need to delete the member since the store is empty. mustAddToRemovedMembersInStore(c.lg, store, id) } if c.version != nil { mustSaveClusterVersionToStore(c.lg, store, c.version) } } ================================================ FILE: server/etcdserver/api/membership/cluster_opts.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership const DefaultMaxLearners = 1 type ClusterOptions struct { maxLearners int } // ClusterOption are options which can be applied to the raft cluster. type ClusterOption func(*ClusterOptions) func newClusterOpts(opts ...ClusterOption) *ClusterOptions { clOpts := &ClusterOptions{} clOpts.applyOpts(opts) return clOpts } func (co *ClusterOptions) applyOpts(opts []ClusterOption) { for _, opt := range opts { opt(co) } } // WithMaxLearners sets the maximum number of learners that can exist in the cluster membership. func WithMaxLearners(max int) ClusterOption { return func(co *ClusterOptions) { co.maxLearners = max } } ================================================ FILE: server/etcdserver/api/membership/cluster_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership import ( "encoding/json" "errors" "fmt" "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/raft/v3/raftpb" ) func TestClusterMember(t *testing.T) { membs := []*Member{ newTestMember(1, nil, "node1", nil), newTestMember(2, nil, "node2", nil), } tests := []struct { id types.ID match bool }{ {1, true}, {2, true}, {3, false}, } for i, tt := range tests { c := newTestCluster(t, membs) m := c.Member(tt.id) if g := m != nil; g != tt.match { t.Errorf("#%d: find member = %v, want %v", i, g, tt.match) } if m != nil && m.ID != tt.id { t.Errorf("#%d: id = %x, want %x", i, m.ID, tt.id) } } } func TestClusterMemberByName(t *testing.T) { membs := []*Member{ newTestMember(1, nil, "node1", nil), newTestMember(2, nil, "node2", nil), } tests := []struct { name string match bool }{ {"node1", true}, {"node2", true}, {"node3", false}, } for i, tt := range tests { c := newTestCluster(t, membs) m := c.MemberByName(tt.name) if g := m != nil; g != tt.match { t.Errorf("#%d: find member = %v, want %v", i, g, tt.match) } if m != nil && m.Name != tt.name { t.Errorf("#%d: name = %v, want %v", i, m.Name, tt.name) } } } func TestClusterMemberIDs(t *testing.T) { c := newTestCluster(t, []*Member{ newTestMember(1, nil, "", nil), newTestMember(4, nil, "", nil), newTestMember(100, nil, "", nil), }) w := []types.ID{1, 4, 100} g := c.MemberIDs() if !reflect.DeepEqual(w, g) { t.Errorf("IDs = %+v, want %+v", g, w) } } func TestClusterPeerURLs(t *testing.T) { tests := []struct { mems []*Member wurls []string }{ // single peer with a single address { mems: []*Member{ newTestMember(1, []string{"http://192.0.2.1"}, "", nil), }, wurls: []string{"http://192.0.2.1"}, }, // single peer with a single address with a port { mems: []*Member{ newTestMember(1, []string{"http://192.0.2.1:8001"}, "", nil), }, wurls: []string{"http://192.0.2.1:8001"}, }, // several members explicitly unsorted { mems: []*Member{ newTestMember(2, []string{"http://192.0.2.3", "http://192.0.2.4"}, "", nil), newTestMember(3, []string{"http://192.0.2.5", "http://192.0.2.6"}, "", nil), newTestMember(1, []string{"http://192.0.2.1", "http://192.0.2.2"}, "", nil), }, wurls: []string{"http://192.0.2.1", "http://192.0.2.2", "http://192.0.2.3", "http://192.0.2.4", "http://192.0.2.5", "http://192.0.2.6"}, }, // no members { mems: []*Member{}, wurls: []string{}, }, // peer with no peer urls { mems: []*Member{ newTestMember(3, []string{}, "", nil), }, wurls: []string{}, }, } for i, tt := range tests { c := newTestCluster(t, tt.mems) urls := c.PeerURLs() if !reflect.DeepEqual(urls, tt.wurls) { t.Errorf("#%d: PeerURLs = %v, want %v", i, urls, tt.wurls) } } } func TestClusterClientURLs(t *testing.T) { tests := []struct { mems []*Member wurls []string }{ // single peer with a single address { mems: []*Member{ newTestMember(1, nil, "", []string{"http://192.0.2.1"}), }, wurls: []string{"http://192.0.2.1"}, }, // single peer with a single address with a port { mems: []*Member{ newTestMember(1, nil, "", []string{"http://192.0.2.1:8001"}), }, wurls: []string{"http://192.0.2.1:8001"}, }, // several members explicitly unsorted { mems: []*Member{ newTestMember(2, nil, "", []string{"http://192.0.2.3", "http://192.0.2.4"}), newTestMember(3, nil, "", []string{"http://192.0.2.5", "http://192.0.2.6"}), newTestMember(1, nil, "", []string{"http://192.0.2.1", "http://192.0.2.2"}), }, wurls: []string{"http://192.0.2.1", "http://192.0.2.2", "http://192.0.2.3", "http://192.0.2.4", "http://192.0.2.5", "http://192.0.2.6"}, }, // no members { mems: []*Member{}, wurls: []string{}, }, // peer with no client urls { mems: []*Member{ newTestMember(3, nil, "", []string{}), }, wurls: []string{}, }, } for i, tt := range tests { c := newTestCluster(t, tt.mems) urls := c.ClientURLs() if !reflect.DeepEqual(urls, tt.wurls) { t.Errorf("#%d: ClientURLs = %v, want %v", i, urls, tt.wurls) } } } func TestClusterValidateAndAssignIDsBad(t *testing.T) { tests := []struct { clmembs []*Member membs []*Member }{ { // unmatched length []*Member{ newTestMember(1, []string{"http://127.0.0.1:2379"}, "", nil), }, []*Member{}, }, { // unmatched peer urls []*Member{ newTestMember(1, []string{"http://127.0.0.1:2379"}, "", nil), }, []*Member{ newTestMember(1, []string{"http://127.0.0.1:4001"}, "", nil), }, }, { // unmatched peer urls []*Member{ newTestMember(1, []string{"http://127.0.0.1:2379"}, "", nil), newTestMember(2, []string{"http://127.0.0.2:2379"}, "", nil), }, []*Member{ newTestMember(1, []string{"http://127.0.0.1:2379"}, "", nil), newTestMember(2, []string{"http://127.0.0.2:4001"}, "", nil), }, }, } for i, tt := range tests { ecl := newTestCluster(t, tt.clmembs) lcl := newTestCluster(t, tt.membs) if err := ValidateClusterAndAssignIDs(zaptest.NewLogger(t), lcl, ecl); err == nil { t.Errorf("#%d: unexpected update success", i) } } } func TestClusterValidateAndAssignIDs(t *testing.T) { tests := []struct { clmembs []*Member membs []*Member wids []types.ID }{ { []*Member{ newTestMember(1, []string{"http://127.0.0.1:2379"}, "", nil), newTestMember(2, []string{"http://127.0.0.2:2379"}, "", nil), }, []*Member{ newTestMember(3, []string{"http://127.0.0.1:2379"}, "", nil), newTestMember(4, []string{"http://127.0.0.2:2379"}, "", nil), }, []types.ID{3, 4}, }, } for i, tt := range tests { lcl := newTestCluster(t, tt.clmembs) ecl := newTestCluster(t, tt.membs) if err := ValidateClusterAndAssignIDs(zaptest.NewLogger(t), lcl, ecl); err != nil { t.Errorf("#%d: unexpect update error: %v", i, err) } if !reflect.DeepEqual(lcl.MemberIDs(), tt.wids) { t.Errorf("#%d: ids = %v, want %v", i, lcl.MemberIDs(), tt.wids) } } } func TestClusterValidateConfigurationChangeV3(t *testing.T) { cl := NewCluster(zaptest.NewLogger(t), WithMaxLearners(1)) be := newMembershipBackend() cl.SetBackend(be) for i := 1; i <= 4; i++ { var isLearner bool if i == 1 { isLearner = true } attr := RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", i)}, IsLearner: isLearner} cl.AddMember(&Member{ID: types.ID(i), RaftAttributes: attr}, true) } cl.RemoveMember(4, true) attr := RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 1)}} ctx, err := json.Marshal(&Member{ID: types.ID(5), RaftAttributes: attr}) if err != nil { t.Fatal(err) } attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 1)}} ctx1, err := json.Marshal(&Member{ID: types.ID(1), RaftAttributes: attr}) if err != nil { t.Fatal(err) } attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 5)}} ctx5, err := json.Marshal(&Member{ID: types.ID(5), RaftAttributes: attr}) if err != nil { t.Fatal(err) } attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 3)}} ctx2to3, err := json.Marshal(&Member{ID: types.ID(2), RaftAttributes: attr}) if err != nil { t.Fatal(err) } attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 5)}} ctx2to5, err := json.Marshal(&Member{ID: types.ID(2), RaftAttributes: attr}) if err != nil { t.Fatal(err) } ctx3, err := json.Marshal(&ConfigChangeContext{Member: Member{ID: types.ID(3), RaftAttributes: attr}, IsPromote: true}) if err != nil { t.Fatal(err) } ctx6, err := json.Marshal(&ConfigChangeContext{Member: Member{ID: types.ID(6), RaftAttributes: attr}, IsPromote: true}) if err != nil { t.Fatal(err) } attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 7)}, IsLearner: true} ctx7, err := json.Marshal(&ConfigChangeContext{Member: Member{ID: types.ID(7), RaftAttributes: attr}}) if err != nil { t.Fatal(err) } attr = RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 1)}, IsLearner: true} ctx8, err := json.Marshal(&ConfigChangeContext{Member: Member{ID: types.ID(1), RaftAttributes: attr}, IsPromote: true}) if err != nil { t.Fatal(err) } tests := []struct { cc raftpb.ConfChange werr error }{ { raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: 3, }, nil, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: 4, }, ErrIDRemoved, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: 4, }, ErrIDRemoved, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: 1, Context: ctx1, }, ErrIDExists, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: 5, Context: ctx, }, ErrPeerURLexists, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: 5, }, ErrIDNotFound, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: 5, Context: ctx5, }, nil, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeUpdateNode, NodeID: 5, Context: ctx, }, ErrIDNotFound, }, // try to change the peer url of 2 to the peer url of 3 { raftpb.ConfChange{ Type: raftpb.ConfChangeUpdateNode, NodeID: 2, Context: ctx2to3, }, ErrPeerURLexists, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeUpdateNode, NodeID: 2, Context: ctx2to5, }, nil, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: 3, Context: ctx3, }, ErrMemberNotLearner, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: 6, Context: ctx6, }, ErrIDNotFound, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeAddLearnerNode, NodeID: 7, Context: ctx7, }, ErrTooManyLearners, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: 1, Context: ctx8, }, nil, }, } for i, tt := range tests { err := cl.ValidateConfigurationChange(tt.cc, true) if !errors.Is(err, tt.werr) { t.Errorf("#%d: validateConfigurationChange error = %v, want %v", i, err, tt.werr) } } } func TestClusterGenID(t *testing.T) { cs := newTestCluster(t, []*Member{ newTestMember(1, nil, "", nil), newTestMember(2, nil, "", nil), }) be := newMembershipBackend() cs.SetBackend(be) cs.genID() if cs.ID() == 0 { t.Fatalf("cluster.ID = %v, want not 0", cs.ID()) } previd := cs.ID() cs.AddMember(newTestMember(3, nil, "", nil), true) cs.genID() if cs.ID() == previd { t.Fatalf("cluster.ID = %v, want not %v", cs.ID(), previd) } } func TestNodeToMemberBad(t *testing.T) { tests := []*v2store.NodeExtern{ {Key: "/1234", Nodes: []*v2store.NodeExtern{ {Key: "/1234/strange"}, }}, {Key: "/1234", Nodes: []*v2store.NodeExtern{ {Key: "/1234/raftAttributes", Value: stringp("garbage")}, }}, {Key: "/1234", Nodes: []*v2store.NodeExtern{ {Key: "/1234/attributes", Value: stringp(`{"name":"node1","clientURLs":null}`)}, }}, {Key: "/1234", Nodes: []*v2store.NodeExtern{ {Key: "/1234/raftAttributes", Value: stringp(`{"peerURLs":null}`)}, {Key: "/1234/strange"}, }}, {Key: "/1234", Nodes: []*v2store.NodeExtern{ {Key: "/1234/raftAttributes", Value: stringp(`{"peerURLs":null}`)}, {Key: "/1234/attributes", Value: stringp("garbage")}, }}, {Key: "/1234", Nodes: []*v2store.NodeExtern{ {Key: "/1234/raftAttributes", Value: stringp(`{"peerURLs":null}`)}, {Key: "/1234/attributes", Value: stringp(`{"name":"node1","clientURLs":null}`)}, {Key: "/1234/strange"}, }}, } for i, tt := range tests { if _, err := nodeToMember(zaptest.NewLogger(t), tt); err == nil { t.Errorf("#%d: unexpected nil error", i) } } } func TestClusterAddMember(t *testing.T) { t.Run("V3", func(t *testing.T) { c := newTestCluster(t, nil) c.AddMember(newTestMember(1, nil, "node1", nil), true) members, _ := c.be.MustReadMembersFromBackend() if len(members) != 1 { t.Errorf("members = %v, want 1 member", members) } if _, ok := members[types.ID(1)]; !ok { t.Errorf("member 1 not found") } }) } func TestClusterAddMemberAsLearner(t *testing.T) { t.Run("V3", func(t *testing.T) { c := newTestCluster(t, nil) c.AddMember(newTestMemberAsLearner(1, []string{}, "node1", []string{"http://node1"}), true) members, _ := c.be.MustReadMembersFromBackend() if len(members) != 1 { t.Errorf("members = %v, want 1 member", members) } if m, ok := members[types.ID(1)]; !ok { t.Errorf("member 1 not found") } else if !m.IsLearner { t.Errorf("member 1 is not learner") } }) } func TestClusterMembers(t *testing.T) { cls := newTestCluster(t, []*Member{ {ID: 1}, {ID: 20}, {ID: 100}, {ID: 5}, {ID: 50}, }) w := []*Member{ {ID: 1}, {ID: 5}, {ID: 20}, {ID: 50}, {ID: 100}, } if g := cls.Members(); !reflect.DeepEqual(g, w) { t.Fatalf("Members()=%#v, want %#v", g, w) } } func TestClusterRemoveMember(t *testing.T) { t.Run("V3", func(t *testing.T) { c := newTestCluster(t, nil) c.AddMember(newTestMember(1, nil, "node1", nil), true) c.RemoveMember(1, true) members, removed := c.be.MustReadMembersFromBackend() if len(members) != 0 { t.Errorf("members = %v, want 0 member", members) } if !removed[types.ID(1)] { t.Errorf("member 1 not removed") } }) } func TestClusterUpdateAttributes(t *testing.T) { name := "etcd" clientURLs := []string{"http://127.0.0.1:4001"} tests := []struct { mems []*Member removed map[types.ID]bool wmems []*Member }{ // update attributes of existing member { []*Member{ newTestMember(1, nil, "", nil), }, nil, []*Member{ newTestMember(1, nil, name, clientURLs), }, }, // update attributes of removed member { nil, map[types.ID]bool{types.ID(1): true}, nil, }, } for i, tt := range tests { t.Run(fmt.Sprintf("V3-%d", i), func(t *testing.T) { c := newTestCluster(t, tt.mems) for id := range tt.removed { c.be.MustDeleteMemberFromBackend(id) } c.removed = tt.removed c.UpdateAttributes(types.ID(1), Attributes{Name: name, ClientURLs: clientURLs}, true) // Verify in-memory state if g := c.Members(); !reflect.DeepEqual(g, tt.wmems) { t.Errorf("#%d: members = %+v, want %+v", i, g, tt.wmems) } bmembers, _ := c.be.MustReadMembersFromBackend() if len(tt.wmems) > 0 { if m, ok := bmembers[types.ID(1)]; !ok { t.Errorf("member 1 not found in backend") } else { if m.Name != name { t.Errorf("member 1 name = %s, want %s", m.Name, name) } if !reflect.DeepEqual(m.ClientURLs, clientURLs) { t.Errorf("member 1 clientURLs = %v, want %v", m.ClientURLs, clientURLs) } } } }) } } func TestNodeToMember(t *testing.T) { n := &v2store.NodeExtern{Key: "/1234", Nodes: []*v2store.NodeExtern{ {Key: "/1234/attributes", Value: stringp(`{"name":"node1","clientURLs":null}`)}, {Key: "/1234/raftAttributes", Value: stringp(`{"peerURLs":null}`)}, }} wm := &Member{ID: 0x1234, RaftAttributes: RaftAttributes{}, Attributes: Attributes{Name: "node1"}} m, err := nodeToMember(zaptest.NewLogger(t), n) if err != nil { t.Fatalf("unexpected nodeToMember error: %v", err) } if !reflect.DeepEqual(m, wm) { t.Errorf("member = %+v, want %+v", m, wm) } } func newTestCluster(tb testing.TB, membs []*Member) *RaftCluster { c := &RaftCluster{ lg: zaptest.NewLogger(tb), members: make(map[types.ID]*Member), removed: make(map[types.ID]bool), be: newMembershipBackend(), } for _, m := range membs { c.AddMember(m, true) } return c } func stringp(s string) *string { return &s } func TestIsReadyToAddVotingMember(t *testing.T) { tests := []struct { members []*Member want bool }{ { // 0/3 members ready, should fail []*Member{ newTestMember(1, nil, "", nil), newTestMember(2, nil, "", nil), newTestMember(3, nil, "", nil), }, false, }, { // 1/2 members ready, should fail []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "", nil), }, false, }, { // 1/3 members ready, should fail []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "", nil), newTestMember(3, nil, "", nil), }, false, }, { // 1/1 members ready, should succeed (special case of 1-member cluster for recovery) []*Member{ newTestMember(1, nil, "1", nil), }, true, }, { // 2/3 members ready, should fail []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "2", nil), newTestMember(3, nil, "", nil), }, false, }, { // 3/3 members ready, should be fine to add one member and retain quorum []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "2", nil), newTestMember(3, nil, "3", nil), }, true, }, { // 3/4 members ready, should be fine to add one member and retain quorum []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "2", nil), newTestMember(3, nil, "3", nil), newTestMember(4, nil, "", nil), }, true, }, { // empty cluster, it is impossible but should fail []*Member{}, false, }, { // 2 voting members ready in cluster with 2 voting members and 2 unstarted learner member, should succeed // (the status of learner members does not affect the readiness of adding voting member) []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "2", nil), newTestMemberAsLearner(3, nil, "", nil), newTestMemberAsLearner(4, nil, "", nil), }, true, }, { // 1 voting member ready in cluster with 2 voting members and 2 ready learner member, should fail // (the status of learner members does not affect the readiness of adding voting member) []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "", nil), newTestMemberAsLearner(3, nil, "3", nil), newTestMemberAsLearner(4, nil, "4", nil), }, false, }, } for i, tt := range tests { c := newTestCluster(t, tt.members) if got := c.IsReadyToAddVotingMember(); got != tt.want { t.Errorf("%d: isReadyToAddNewMember returned %t, want %t", i, got, tt.want) } } } func TestIsReadyToRemoveVotingMember(t *testing.T) { tests := []struct { members []*Member removeID uint64 want bool }{ { // 1/1 members ready, should fail []*Member{ newTestMember(1, nil, "1", nil), }, 1, false, }, { // 0/3 members ready, should fail []*Member{ newTestMember(1, nil, "", nil), newTestMember(2, nil, "", nil), newTestMember(3, nil, "", nil), }, 1, false, }, { // 1/2 members ready, should be fine to remove unstarted member // (isReadyToRemoveMember() logic should return success, but operation itself would fail) []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "", nil), }, 2, true, }, { // 2/3 members ready, should fail []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "2", nil), newTestMember(3, nil, "", nil), }, 2, false, }, { // 3/3 members ready, should be fine to remove one member and retain quorum []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "2", nil), newTestMember(3, nil, "3", nil), }, 3, true, }, { // 3/4 members ready, should be fine to remove one member []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "2", nil), newTestMember(3, nil, "3", nil), newTestMember(4, nil, "", nil), }, 3, true, }, { // 3/4 members ready, should be fine to remove unstarted member []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "2", nil), newTestMember(3, nil, "3", nil), newTestMember(4, nil, "", nil), }, 4, true, }, { // 1 voting members ready in cluster with 1 voting member and 1 ready learner, // removing voting member should fail // (the status of learner members does not affect the readiness of removing voting member) []*Member{ newTestMember(1, nil, "1", nil), newTestMemberAsLearner(2, nil, "2", nil), }, 1, false, }, { // 1 voting members ready in cluster with 2 voting member and 1 ready learner, // removing ready voting member should fail // (the status of learner members does not affect the readiness of removing voting member) []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "", nil), newTestMemberAsLearner(3, nil, "3", nil), }, 1, false, }, { // 1 voting members ready in cluster with 2 voting member and 1 ready learner, // removing unstarted voting member should be fine. (Actual operation will fail) // (the status of learner members does not affect the readiness of removing voting member) []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "", nil), newTestMemberAsLearner(3, nil, "3", nil), }, 2, true, }, { // 1 voting members ready in cluster with 2 voting member and 1 unstarted learner, // removing not-ready voting member should be fine. (Actual operation will fail) // (the status of learner members does not affect the readiness of removing voting member) []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "", nil), newTestMemberAsLearner(3, nil, "", nil), }, 2, true, }, } for i, tt := range tests { c := newTestCluster(t, tt.members) if got := c.IsReadyToRemoveVotingMember(tt.removeID); got != tt.want { t.Errorf("%d: isReadyToAddNewMember returned %t, want %t", i, got, tt.want) } } } func TestIsReadyToPromoteMember(t *testing.T) { tests := []struct { members []*Member promoteID uint64 want bool }{ { // 1/1 members ready, should succeed (quorum = 1, new quorum = 2) []*Member{ newTestMember(1, nil, "1", nil), newTestMemberAsLearner(2, nil, "2", nil), }, 2, true, }, { // 0/1 members ready, should fail (quorum = 1) []*Member{ newTestMember(1, nil, "", nil), newTestMemberAsLearner(2, nil, "2", nil), }, 2, false, }, { // 2/2 members ready, should succeed (quorum = 2) []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "2", nil), newTestMemberAsLearner(3, nil, "3", nil), }, 3, true, }, { // 1/2 members ready, should succeed (quorum = 2) []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "", nil), newTestMemberAsLearner(3, nil, "3", nil), }, 3, true, }, { // 1/3 members ready, should fail (quorum = 2) []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "", nil), newTestMember(3, nil, "", nil), newTestMemberAsLearner(4, nil, "4", nil), }, 4, false, }, { // 2/3 members ready, should succeed (quorum = 2, new quorum = 3) []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "2", nil), newTestMember(3, nil, "", nil), newTestMemberAsLearner(4, nil, "4", nil), }, 4, true, }, { // 2/4 members ready, should succeed (quorum = 3) []*Member{ newTestMember(1, nil, "1", nil), newTestMember(2, nil, "2", nil), newTestMember(3, nil, "", nil), newTestMember(4, nil, "", nil), newTestMemberAsLearner(5, nil, "5", nil), }, 5, true, }, } for i, tt := range tests { c := newTestCluster(t, tt.members) if got := c.IsReadyToPromoteMember(tt.promoteID); got != tt.want { t.Errorf("%d: isReadyToPromoteMember returned %t, want %t", i, got, tt.want) } } } func TestPromoteMember(t *testing.T) { clientURLs := []string{"http://127.0.0.1:2379"} testCases := []struct { name string members []*Member promoteID types.ID wantMembers map[types.ID]*Member }{ { name: "promote a voting member", members: []*Member{ newTestMember(1, nil, "1", clientURLs), newTestMemberAsLearner(2, nil, "2", clientURLs), }, promoteID: 1, wantMembers: map[types.ID]*Member{ 1: newTestMember(1, nil, "1", clientURLs), 2: newTestMemberAsLearner(2, nil, "2", clientURLs), }, }, { name: "promote a learner", members: []*Member{ newTestMember(1, nil, "1", clientURLs), newTestMemberAsLearner(2, nil, "2", clientURLs), }, promoteID: 2, wantMembers: map[types.ID]*Member{ 1: newTestMember(1, nil, "1", clientURLs), 2: newTestMember(2, nil, "2", clientURLs), }, }, { name: "promote a non-exist member", members: []*Member{ newTestMember(1, nil, "1", clientURLs), newTestMemberAsLearner(2, nil, "2", clientURLs), }, promoteID: 3, wantMembers: map[types.ID]*Member{ 1: newTestMember(1, nil, "1", clientURLs), 2: newTestMemberAsLearner(2, nil, "2", clientURLs), }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { t.Run("V3", func(t *testing.T) { c := newTestCluster(t, tc.members) c.PromoteMember(tc.promoteID, true) mst, _ := c.be.MustReadMembersFromBackend() require.Equal(t, tc.wantMembers, mst) }) }) } } func TestUpdateRaftAttributes(t *testing.T) { clientURLs := []string{"http://127.0.0.1:2379"} oldPeerURLs := []string{"http://127.0.0.1:2380"} newPeerURLs := []string{"http://127.0.0.1:2382"} testCases := []struct { name string members []*Member updateMemberID types.ID wantMembers map[types.ID]*Member }{ { name: "update an existing member", members: []*Member{ newTestMember(1, oldPeerURLs, "1", clientURLs), newTestMember(2, oldPeerURLs, "2", clientURLs), }, updateMemberID: 2, wantMembers: map[types.ID]*Member{ 1: newTestMember(1, oldPeerURLs, "1", clientURLs), 2: newTestMember(2, newPeerURLs, "2", clientURLs), }, }, { name: "update a non-exist member", members: []*Member{ newTestMember(1, oldPeerURLs, "1", clientURLs), newTestMember(2, oldPeerURLs, "2", clientURLs), }, updateMemberID: 3, wantMembers: map[types.ID]*Member{ 1: newTestMember(1, oldPeerURLs, "1", clientURLs), 2: newTestMember(2, oldPeerURLs, "2", clientURLs), }, }, } for _, tc := range testCases { t.Run("V3", func(t *testing.T) { c := newTestCluster(t, tc.members) c.UpdateRaftAttributes(tc.updateMemberID, RaftAttributes{PeerURLs: newPeerURLs}, true) mst, _ := c.be.MustReadMembersFromBackend() require.Equal(t, tc.wantMembers, mst) }) } } func TestClusterStore(t *testing.T) { name := "etcd" clientURLs := []string{"http://127.0.0.1:4001"} tests := []struct { name string mems []*Member removed map[types.ID]bool }{ { name: "Single member, no removed members", mems: []*Member{ newTestMember(1, nil, name, clientURLs), }, removed: map[types.ID]bool{}, }, { name: "Multiple members, no removed members", mems: []*Member{ newTestMember(1, nil, name, clientURLs), newTestMember(2, nil, name, clientURLs), newTestMember(3, nil, name, clientURLs), }, removed: map[types.ID]bool{}, }, { name: "Single member, one removed member", mems: []*Member{ newTestMember(1, nil, name, clientURLs), }, removed: map[types.ID]bool{types.ID(2): true}, }, { name: "Multiple members, some removed members", mems: []*Member{ newTestMember(1, nil, name, clientURLs), newTestMember(2, nil, name, clientURLs), newTestMember(3, nil, name, clientURLs), }, removed: map[types.ID]bool{ types.ID(4): true, types.ID(5): true, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := newTestCluster(t, tt.mems) c.removed = tt.removed st := v2store.New("/0", "/1") c.Store(st) // Verify that the members are properly stored mst, rst := MembersFromStore(c.lg, st) for _, mem := range tt.mems { assert.Equal(t, mem, mst[mem.ID]) } // Verify that removed members are correctly stored assert.Equal(t, tt.removed, rst) }) } } ================================================ FILE: server/etcdserver/api/membership/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package membership describes individual etcd members and clusters of members. package membership ================================================ FILE: server/etcdserver/api/membership/errors.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership import ( "errors" "go.etcd.io/etcd/server/v3/etcdserver/api/v2error" ) var ( ErrIDRemoved = errors.New("membership: ID removed") ErrIDExists = errors.New("membership: ID exists") ErrIDNotFound = errors.New("membership: ID not found") ErrPeerURLexists = errors.New("membership: peerURL exists") ErrMemberNotLearner = errors.New("membership: can only promote a learner member") ErrTooManyLearners = errors.New("membership: too many learner members in cluster") ) func isKeyNotFound(err error) bool { var e *v2error.Error return errors.As(err, &e) && e.ErrorCode == v2error.EcodeKeyNotFound } ================================================ FILE: server/etcdserver/api/membership/member.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership import ( "crypto/sha1" "encoding/binary" "fmt" "sort" "strings" "time" "go.etcd.io/etcd/client/pkg/v3/types" ) // RaftAttributes represents the raft related attributes of an etcd member. type RaftAttributes struct { // PeerURLs is the list of peers in the raft cluster. // TODO(philips): ensure these are URLs PeerURLs []string `json:"peerURLs"` // IsLearner indicates if the member is raft learner. IsLearner bool `json:"isLearner,omitempty"` } // Attributes represents all the non-raft related attributes of an etcd member. type Attributes struct { Name string `json:"name,omitempty"` ClientURLs []string `json:"clientURLs,omitempty"` } type Member struct { ID types.ID `json:"id"` RaftAttributes Attributes } // NewMember creates a Member without an ID and generates one based on the // cluster name, peer URLs, and time. This is used for bootstrapping/adding new member. func NewMember(name string, peerURLs types.URLs, clusterName string, now *time.Time) *Member { memberID := computeMemberID(peerURLs, clusterName, now) return newMember(name, peerURLs, memberID, false) } // NewMemberAsLearner creates a learner Member without an ID and generates one based on the // cluster name, peer URLs, and time. This is used for adding new learner member. func NewMemberAsLearner(name string, peerURLs types.URLs, clusterName string, now *time.Time) *Member { memberID := computeMemberID(peerURLs, clusterName, now) return newMember(name, peerURLs, memberID, true) } func computeMemberID(peerURLs types.URLs, clusterName string, now *time.Time) types.ID { peerURLstrs := peerURLs.StringSlice() sort.Strings(peerURLstrs) joinedPeerUrls := strings.Join(peerURLstrs, "") b := []byte(joinedPeerUrls) b = append(b, []byte(clusterName)...) if now != nil { b = append(b, []byte(fmt.Sprintf("%d", now.Unix()))...) } hash := sha1.Sum(b) return types.ID(binary.BigEndian.Uint64(hash[:8])) } func newMember(name string, peerURLs types.URLs, memberID types.ID, isLearner bool) *Member { m := &Member{ RaftAttributes: RaftAttributes{ PeerURLs: peerURLs.StringSlice(), IsLearner: isLearner, }, Attributes: Attributes{Name: name}, ID: memberID, } return m } func (m *Member) Clone() *Member { if m == nil { return nil } mm := &Member{ ID: m.ID, RaftAttributes: RaftAttributes{ IsLearner: m.IsLearner, }, Attributes: Attributes{ Name: m.Name, }, } if m.PeerURLs != nil { mm.PeerURLs = make([]string, len(m.PeerURLs)) copy(mm.PeerURLs, m.PeerURLs) } if m.ClientURLs != nil { mm.ClientURLs = make([]string, len(m.ClientURLs)) copy(mm.ClientURLs, m.ClientURLs) } return mm } func (m *Member) IsStarted() bool { return len(m.Name) != 0 } // MembersByID implements sort by ID interface type MembersByID []*Member func (ms MembersByID) Len() int { return len(ms) } func (ms MembersByID) Less(i, j int) bool { return ms[i].ID < ms[j].ID } func (ms MembersByID) Swap(i, j int) { ms[i], ms[j] = ms[j], ms[i] } // MembersByPeerURLs implements sort by peer urls interface type MembersByPeerURLs []*Member func (ms MembersByPeerURLs) Len() int { return len(ms) } func (ms MembersByPeerURLs) Less(i, j int) bool { return ms[i].PeerURLs[0] < ms[j].PeerURLs[0] } func (ms MembersByPeerURLs) Swap(i, j int) { ms[i], ms[j] = ms[j], ms[i] } ================================================ FILE: server/etcdserver/api/membership/member_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership import ( "net/url" "reflect" "testing" "time" "go.etcd.io/etcd/client/pkg/v3/types" ) func timeParse(value string) *time.Time { t, err := time.Parse(time.RFC3339, value) if err != nil { panic(err) } return &t } func TestMemberTime(t *testing.T) { tests := []struct { mem *Member id types.ID }{ {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "", nil), 14544069596553697298}, // Same ID, different name (names shouldn't matter) {NewMember("memfoo", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "", nil), 14544069596553697298}, // Same ID, different Time {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "", timeParse("1984-12-23T15:04:05Z")), 2448790162483548276}, // Different cluster name {NewMember("mcm1", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "etcd", timeParse("1984-12-23T15:04:05Z")), 6973882743191604649}, {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}}, "", timeParse("1984-12-23T15:04:05Z")), 1466075294948436910}, // Order shouldn't matter {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "10.0.0.2:2379"}}, "", nil), 16552244735972308939}, {NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.2:2379"}, {Scheme: "http", Host: "10.0.0.1:2379"}}, "", nil), 16552244735972308939}, } for i, tt := range tests { if tt.mem.ID != tt.id { t.Errorf("#%d: mem.ID = %v, want %v", i, tt.mem.ID, tt.id) } } } func TestMemberClone(t *testing.T) { tests := []*Member{ newTestMember(1, nil, "abc", nil), newTestMember(1, []string{"http://a"}, "abc", nil), newTestMember(1, nil, "abc", []string{"http://b"}), newTestMember(1, []string{"http://a"}, "abc", []string{"http://b"}), } for i, tt := range tests { nm := tt.Clone() if nm == tt { t.Errorf("#%d: the pointers are the same, and clone doesn't happen", i) } if !reflect.DeepEqual(nm, tt) { t.Errorf("#%d: member = %+v, want %+v", i, nm, tt) } } } func newTestMember(id uint64, peerURLs []string, name string, clientURLs []string) *Member { return &Member{ ID: types.ID(id), RaftAttributes: RaftAttributes{PeerURLs: peerURLs}, Attributes: Attributes{Name: name, ClientURLs: clientURLs}, } } func newTestMemberAsLearner(id uint64, peerURLs []string, name string, clientURLs []string) *Member { return &Member{ ID: types.ID(id), RaftAttributes: RaftAttributes{PeerURLs: peerURLs, IsLearner: true}, Attributes: Attributes{Name: name, ClientURLs: clientURLs}, } } ================================================ FILE: server/etcdserver/api/membership/membership_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership import ( "testing" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/types" serverversion "go.etcd.io/etcd/server/v3/etcdserver/version" ) func TestAddRemoveMember(t *testing.T) { c := newTestCluster(t, nil) be := newMembershipBackend() c.SetBackend(be) c.AddMember(newTestMemberAsLearner(17, nil, "node17", nil), true) c.RemoveMember(17, true) c.AddMember(newTestMember(18, nil, "node18", nil), true) c.RemoveMember(18, true) c.AddMember(newTestMember(19, nil, "node19", nil), true) // Recover from backend c2 := newTestCluster(t, nil) c2.SetBackend(be) c2.Recover(func(*zap.Logger, *semver.Version) {}) assert.Equal(t, []*Member{{ ID: types.ID(19), Attributes: Attributes{Name: "node19"}, }}, c2.Members()) assert.True(t, c2.IsIDRemoved(17)) assert.True(t, c2.IsIDRemoved(18)) assert.False(t, c2.IsIDRemoved(19)) } type backendMock struct { members map[types.ID]*Member removed map[types.ID]bool version *semver.Version downgradeInfo *serverversion.DowngradeInfo } var _ MembershipBackend = (*backendMock)(nil) func newMembershipBackend() MembershipBackend { return &backendMock{ members: make(map[types.ID]*Member), removed: make(map[types.ID]bool), downgradeInfo: &serverversion.DowngradeInfo{Enabled: false}, } } func (b *backendMock) MustCreateBackendBuckets() {} func (b *backendMock) ClusterVersionFromBackend() *semver.Version { return b.version } func (b *backendMock) MustSaveClusterVersionToBackend(version *semver.Version) { b.version = version } func (b *backendMock) MustReadMembersFromBackend() (x map[types.ID]*Member, y map[types.ID]bool) { return b.members, b.removed } func (b *backendMock) MustSaveMemberToBackend(m *Member) { b.members[m.ID] = m } func (b *backendMock) TrimMembershipFromBackend() error { b.members = make(map[types.ID]*Member) b.removed = make(map[types.ID]bool) return nil } func (b *backendMock) MustDeleteMemberFromBackend(id types.ID) { delete(b.members, id) b.removed[id] = true } func (b *backendMock) MustSaveDowngradeToBackend(downgradeInfo *serverversion.DowngradeInfo) { b.downgradeInfo = downgradeInfo } func (b *backendMock) DowngradeInfoFromBackend() *serverversion.DowngradeInfo { return b.downgradeInfo } ================================================ FILE: server/etcdserver/api/membership/metrics.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership import "github.com/prometheus/client_golang/prometheus" var ( ClusterVersionMetrics = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "cluster", Name: "version", Help: "Which version is running. 1 for 'cluster_version' label with current cluster version", }, []string{"cluster_version"}, ) knownPeers = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "network", Name: "known_peers", Help: "The current number of known peers.", }, []string{"Local", "Remote"}, ) isLearner = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "is_learner", Help: "Whether or not this member is a learner. 1 if is, 0 otherwise.", }) ) func setIsLearnerMetric(m *Member) { if m.IsLearner { isLearner.Set(1) } else { isLearner.Set(0) } } func init() { prometheus.MustRegister(ClusterVersionMetrics) prometheus.MustRegister(knownPeers) prometheus.MustRegister(isLearner) } ================================================ FILE: server/etcdserver/api/membership/store.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership import ( "path" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/version" ) type MembershipBackend interface { ClusterVersionBackend MemberBackend DowngradeInfoBackend MustCreateBackendBuckets() } type ClusterVersionBackend interface { ClusterVersionFromBackend() *semver.Version MustSaveClusterVersionToBackend(version *semver.Version) } type MemberBackend interface { MustReadMembersFromBackend() (map[types.ID]*Member, map[types.ID]bool) MustSaveMemberToBackend(*Member) TrimMembershipFromBackend() error MustDeleteMemberFromBackend(types.ID) } type DowngradeInfoBackend interface { MustSaveDowngradeToBackend(*version.DowngradeInfo) DowngradeInfoFromBackend() *version.DowngradeInfo } func MustParseMemberIDFromKey(lg *zap.Logger, key string) types.ID { id, err := types.IDFromString(path.Base(key)) if err != nil { lg.Panic("failed to parse member id from key", zap.Error(err)) } return id } ================================================ FILE: server/etcdserver/api/membership/storev2.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership import ( "encoding/json" "fmt" "path" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" ) const ( // the prefix for storing membership related information in store provided by store pkg. storePrefix = "/0" attributesSuffix = "attributes" raftAttributesSuffix = "raftAttributes" ) var ( StoreMembersPrefix = path.Join(storePrefix, "members") storeRemovedMembersPrefix = path.Join(storePrefix, "removed_members") ) // IsMetaStoreOnly verifies if the given `store` contains only // a meta-information (members, version) that can be recovered from the // backend (storev3) as well as opposed to user-data. func IsMetaStoreOnly(store v2store.Store) (bool, error) { event, err := store.Get("/", true, false) if err != nil { return false, err } // storePermsPrefix is the internal prefix of the storage layer dedicated to storing user data. // refer to https://github.com/etcd-io/etcd/blob/v3.5.21/server/etcdserver/api/v2auth/auth.go#L40 storePermsPrefix := "/2" for _, n := range event.Node.Nodes { if n.Key == storePrefix { continue } // For auth data, even after we remove all users and roles, the node // "/2/roles" and "/2/users" are still present in the tree. We need // to exclude such case. See an example below, // Refer to https://github.com/etcd-io/etcd/discussions/20231#discussioncomment-13791940 /* "2": { "Path": "/2", "CreatedIndex": 204749, "ModifiedIndex": 204749, "ExpireTime": "0001-01-01T00:00:00Z", "Value": "", "Children": { "enabled": { "Path": "/2/enabled", "CreatedIndex": 204752, "ModifiedIndex": 16546016, "ExpireTime": "0001-01-01T00:00:00Z", "Value": "false", "Children": null }, "roles": { "Path": "/2/roles", "CreatedIndex": 204751, "ModifiedIndex": 204751, "ExpireTime": "0001-01-01T00:00:00Z", "Value": "", "Children": {} }, "users": { "Path": "/2/users", "CreatedIndex": 204750, "ModifiedIndex": 204750, "ExpireTime": "0001-01-01T00:00:00Z", "Value": "", "Children": {} } } } */ if n.Key == storePermsPrefix { if n.Nodes.Len() > 0 { for _, child := range n.Nodes { if child.Nodes.Len() > 0 { return false, nil } } } continue } if n.Nodes.Len() > 0 { return false, nil } } return true, nil } func verifyNoMembersInStore(lg *zap.Logger, s v2store.Store) { members, removed := MembersFromStore(lg, s) if len(members) != 0 || len(removed) != 0 { lg.Panic("store has membership info") } } func mustSaveMemberToStore(lg *zap.Logger, s v2store.Store, m *Member) { b, err := json.Marshal(m.RaftAttributes) if err != nil { lg.Panic("failed to marshal raftAttributes", zap.Error(err)) } p := path.Join(MemberStoreKey(m.ID), raftAttributesSuffix) if _, err := s.Create(p, false, string(b), false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}); err != nil { lg.Panic( "failed to save member to store", zap.String("path", p), zap.Error(err), ) } } func mustAddToRemovedMembersInStore(lg *zap.Logger, s v2store.Store, id types.ID) { if _, err := s.Create(RemovedMemberStoreKey(id), false, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}); err != nil { lg.Panic( "failed to create removedMember", zap.String("path", RemovedMemberStoreKey(id)), zap.Error(err), ) } } func mustUpdateMemberAttrInStore(lg *zap.Logger, s v2store.Store, m *Member) { b, err := json.Marshal(m.Attributes) if err != nil { lg.Panic("failed to marshal attributes", zap.Error(err)) } p := path.Join(MemberStoreKey(m.ID), attributesSuffix) if _, err := s.Set(p, false, string(b), v2store.TTLOptionSet{ExpireTime: v2store.Permanent}); err != nil { lg.Panic( "failed to update attributes", zap.String("path", p), zap.Error(err), ) } } func mustSaveClusterVersionToStore(lg *zap.Logger, s v2store.Store, ver *semver.Version) { if _, err := s.Set(StoreClusterVersionKey(), false, ver.String(), v2store.TTLOptionSet{ExpireTime: v2store.Permanent}); err != nil { lg.Panic( "failed to save cluster version to store", zap.String("path", StoreClusterVersionKey()), zap.Error(err), ) } } // nodeToMember builds member from a key value node. // the child nodes of the given node MUST be sorted by key. func nodeToMember(lg *zap.Logger, n *v2store.NodeExtern) (*Member, error) { m := &Member{ID: MustParseMemberIDFromKey(lg, n.Key)} attrs := make(map[string][]byte) raftAttrKey := path.Join(n.Key, raftAttributesSuffix) attrKey := path.Join(n.Key, attributesSuffix) for _, nn := range n.Nodes { if nn.Key != raftAttrKey && nn.Key != attrKey { return nil, fmt.Errorf("unknown key %q", nn.Key) } attrs[nn.Key] = []byte(*nn.Value) } if data := attrs[raftAttrKey]; data != nil { if err := json.Unmarshal(data, &m.RaftAttributes); err != nil { return nil, fmt.Errorf("unmarshal raftAttributes error: %w", err) } } else { return nil, fmt.Errorf("raftAttributes key doesn't exist") } if data := attrs[attrKey]; data != nil { if err := json.Unmarshal(data, &m.Attributes); err != nil { return m, fmt.Errorf("unmarshal attributes error: %w", err) } } return m, nil } func StoreClusterVersionKey() string { return path.Join(storePrefix, "version") } func RemovedMemberStoreKey(id types.ID) string { return path.Join(storeRemovedMembersPrefix, id.String()) } func MemberStoreKey(id types.ID) string { return path.Join(StoreMembersPrefix, id.String()) } func MemberAttributesStorePath(id types.ID) string { return path.Join(MemberStoreKey(id), attributesSuffix) } ================================================ FILE: server/etcdserver/api/membership/storev2_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package membership import ( "testing" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" ) func TestIsMetaStoreOnly(t *testing.T) { lg := zaptest.NewLogger(t) s := v2store.New("/0", "/1") metaOnly, err := IsMetaStoreOnly(s) require.NoError(t, err) assert.Truef(t, metaOnly, "Just created v2store should be meta-only") mustSaveClusterVersionToStore(lg, s, semver.New("3.5.17")) metaOnly, err = IsMetaStoreOnly(s) require.NoError(t, err) assert.Truef(t, metaOnly, "Just created v2store should be meta-only") mustSaveMemberToStore(lg, s, &Member{ID: 0x00abcd}) metaOnly, err = IsMetaStoreOnly(s) require.NoError(t, err) assert.Truef(t, metaOnly, "Just created v2store should be meta-only") _, err = s.Create("/1/foo", false, "v1", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) metaOnly, err = IsMetaStoreOnly(s) require.NoError(t, err) assert.Falsef(t, metaOnly, "Just created v2store should be meta-only") _, err = s.Delete("/1/foo", false, false) assert.NoError(t, err) assert.NoError(t, err) assert.Falsef(t, metaOnly, "Just created v2store should be meta-only") } func TestIsMetaStoreOnlyWithAuthData(t *testing.T) { s := v2store.New("/0", "/1") metaOnly, err := IsMetaStoreOnly(s) require.NoError(t, err) assert.Truef(t, metaOnly, "Just created v2store should be meta-only") _, err = s.Create("/2/roles", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) metaOnly, err = IsMetaStoreOnly(s) require.NoError(t, err) assert.Truef(t, metaOnly, "Just created empty roles directory should be meta-only") _, err = s.Create("/2/users", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) metaOnly, err = IsMetaStoreOnly(s) require.NoError(t, err) assert.Truef(t, metaOnly, "Just created empty users directory should be meta-only") } ================================================ FILE: server/etcdserver/api/rafthttp/coder.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import "go.etcd.io/raft/v3/raftpb" type encoder interface { // encode encodes the given message to an output stream. encode(m *raftpb.Message) error } type decoder interface { // decode decodes the message from an input stream. decode() (raftpb.Message, error) } ================================================ FILE: server/etcdserver/api/rafthttp/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package rafthttp implements HTTP transportation layer for raft pkg. package rafthttp ================================================ FILE: server/etcdserver/api/rafthttp/fake_roundtripper_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "errors" "net/http" ) func (t *roundTripperBlocker) RoundTrip(req *http.Request) (*http.Response, error) { c := make(chan struct{}, 1) t.mu.Lock() t.cancel[req] = c t.mu.Unlock() ctx := req.Context() select { case <-t.unblockc: return &http.Response{StatusCode: http.StatusNoContent, Body: &nopReadCloser{}}, nil case <-ctx.Done(): return nil, errors.New("request canceled") case <-c: return nil, errors.New("request canceled") } } ================================================ FILE: server/etcdserver/api/rafthttp/functional_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "context" "net/http/httptest" "reflect" "testing" "time" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/types" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) func TestSendMessage(t *testing.T) { // member 1 tr := &Transport{ ID: types.ID(1), ClusterID: types.ID(1), Raft: &fakeRaft{}, ServerStats: newServerStats(), LeaderStats: stats.NewLeaderStats(zaptest.NewLogger(t), "1"), } tr.Start() srv := httptest.NewServer(tr.Handler()) defer srv.Close() // member 2 recvc := make(chan raftpb.Message, 1) p := &fakeRaft{recvc: recvc} tr2 := &Transport{ ID: types.ID(2), ClusterID: types.ID(1), Raft: p, ServerStats: newServerStats(), LeaderStats: stats.NewLeaderStats(zaptest.NewLogger(t), "2"), } tr2.Start() srv2 := httptest.NewServer(tr2.Handler()) defer srv2.Close() tr.AddPeer(types.ID(2), []string{srv2.URL}) defer tr.Stop() tr2.AddPeer(types.ID(1), []string{srv.URL}) defer tr2.Stop() if !waitStreamWorking(tr.Get(types.ID(2)).(*peer)) { t.Fatalf("stream from 1 to 2 is not in work as expected") } data := []byte("some data") tests := []raftpb.Message{ // these messages are set to send to itself, which facilitates testing. {Type: raftpb.MsgProp, From: 1, To: 2, Entries: []raftpb.Entry{{Data: data}}}, {Type: raftpb.MsgApp, From: 1, To: 2, Term: 1, Index: 3, LogTerm: 0, Entries: []raftpb.Entry{{Index: 4, Term: 1, Data: data}}, Commit: 3}, {Type: raftpb.MsgAppResp, From: 1, To: 2, Term: 1, Index: 3}, {Type: raftpb.MsgVote, From: 1, To: 2, Term: 1, Index: 3, LogTerm: 0}, {Type: raftpb.MsgVoteResp, From: 1, To: 2, Term: 1}, {Type: raftpb.MsgSnap, From: 1, To: 2, Term: 1, Snapshot: &raftpb.Snapshot{Metadata: raftpb.SnapshotMetadata{Index: 1000, Term: 1}, Data: data}}, {Type: raftpb.MsgHeartbeat, From: 1, To: 2, Term: 1, Commit: 3}, {Type: raftpb.MsgHeartbeatResp, From: 1, To: 2, Term: 1}, } for i, tt := range tests { tr.Send([]raftpb.Message{tt}) msg := <-recvc if !reflect.DeepEqual(msg, tt) { t.Errorf("#%d: msg = %+v, want %+v", i, msg, tt) } } } // TestSendMessageWhenStreamIsBroken tests that message can be sent to the // remote in a limited time when all underlying connections are broken. func TestSendMessageWhenStreamIsBroken(t *testing.T) { // member 1 tr := &Transport{ ID: types.ID(1), ClusterID: types.ID(1), Raft: &fakeRaft{}, ServerStats: newServerStats(), LeaderStats: stats.NewLeaderStats(zaptest.NewLogger(t), "1"), } tr.Start() srv := httptest.NewServer(tr.Handler()) defer srv.Close() // member 2 recvc := make(chan raftpb.Message, 1) p := &fakeRaft{recvc: recvc} tr2 := &Transport{ ID: types.ID(2), ClusterID: types.ID(1), Raft: p, ServerStats: newServerStats(), LeaderStats: stats.NewLeaderStats(zaptest.NewLogger(t), "2"), } tr2.Start() srv2 := httptest.NewServer(tr2.Handler()) defer srv2.Close() tr.AddPeer(types.ID(2), []string{srv2.URL}) defer tr.Stop() tr2.AddPeer(types.ID(1), []string{srv.URL}) defer tr2.Stop() if !waitStreamWorking(tr.Get(types.ID(2)).(*peer)) { t.Fatalf("stream from 1 to 2 is not in work as expected") } // break the stream srv.CloseClientConnections() srv2.CloseClientConnections() var n int for { select { // TODO: remove this resend logic when we add retry logic into the code case <-time.After(time.Millisecond): n++ tr.Send([]raftpb.Message{{Type: raftpb.MsgHeartbeat, From: 1, To: 2, Term: 1, Commit: 3}}) case <-recvc: if n > 50 { t.Errorf("disconnection time = %dms, want < 50ms", n) } return } } } func newServerStats() *stats.ServerStats { return stats.NewServerStats("", "") } func waitStreamWorking(p *peer) bool { for i := 0; i < 1000; i++ { time.Sleep(time.Millisecond) if _, ok := p.msgAppV2Writer.writec(); !ok { continue } if _, ok := p.writer.writec(); !ok { continue } return true } return false } type fakeRaft struct { recvc chan<- raftpb.Message err error removedID uint64 } func (p *fakeRaft) Process(ctx context.Context, m raftpb.Message) error { select { case p.recvc <- m: default: } return p.err } func (p *fakeRaft) IsIDRemoved(id uint64) bool { return id == p.removedID } func (p *fakeRaft) ReportUnreachable(id uint64) {} func (p *fakeRaft) ReportSnapshot(id uint64, status raft.SnapshotStatus) {} ================================================ FILE: server/etcdserver/api/rafthttp/http.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "context" "errors" "fmt" "io" "net/http" "path" "strings" "time" humanize "github.com/dustin/go-humanize" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/types" pioutil "go.etcd.io/etcd/pkg/v3/ioutil" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/raft/v3/raftpb" ) const ( // connReadLimitByte limits the number of bytes // a single read can read out. // // 64KB should be large enough for not causing // throughput bottleneck as well as small enough // for not causing a read timeout. connReadLimitByte = 64 * 1024 // snapshotLimitByte limits the snapshot size to 1TB snapshotLimitByte = 1 * 1024 * 1024 * 1024 * 1024 ) var ( RaftPrefix = "/raft" ProbingPrefix = path.Join(RaftPrefix, "probing") RaftStreamPrefix = path.Join(RaftPrefix, "stream") RaftSnapshotPrefix = path.Join(RaftPrefix, "snapshot") errIncompatibleVersion = errors.New("incompatible version") ErrClusterIDMismatch = errors.New("cluster ID mismatch") ) type peerGetter interface { Get(id types.ID) Peer } type writerToResponse interface { WriteTo(w http.ResponseWriter) } type pipelineHandler struct { lg *zap.Logger localID types.ID tr Transporter r Raft cid types.ID } // newPipelineHandler returns a handler for handling raft messages // from pipeline for RaftPrefix. // // The handler reads out the raft message from request body, // and forwards it to the given raft state machine for processing. func newPipelineHandler(t *Transport, r Raft, cid types.ID) http.Handler { h := &pipelineHandler{ lg: t.Logger, localID: t.ID, tr: t, r: r, cid: cid, } if h.lg == nil { h.lg = zap.NewNop() } return h } func (h *pipelineHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { w.Header().Set("Allow", "POST") http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) return } w.Header().Set("X-Etcd-Cluster-ID", h.cid.String()) if err := checkClusterCompatibilityFromHeader(h.lg, h.localID, r.Header, h.cid); err != nil { http.Error(w, err.Error(), http.StatusPreconditionFailed) return } addRemoteFromRequest(h.tr, r) // Limit the data size that could be read from the request body, which ensures that read from // connection will not time out accidentally due to possible blocking in underlying implementation. limitedr := pioutil.NewLimitedBufferReader(r.Body, connReadLimitByte) b, err := io.ReadAll(limitedr) if err != nil { h.lg.Warn( "failed to read Raft message", zap.String("local-member-id", h.localID.String()), zap.Error(err), ) http.Error(w, "error reading raft message", http.StatusBadRequest) recvFailures.WithLabelValues(r.RemoteAddr).Inc() return } var m raftpb.Message if err := m.Unmarshal(b); err != nil { h.lg.Warn( "failed to unmarshal Raft message", zap.String("local-member-id", h.localID.String()), zap.Error(err), ) http.Error(w, "error unmarshalling raft message", http.StatusBadRequest) recvFailures.WithLabelValues(r.RemoteAddr).Inc() return } receivedBytes.WithLabelValues(types.ID(m.From).String()).Add(float64(len(b))) if err := h.r.Process(context.TODO(), m); err != nil { var writerErr writerToResponse switch { case errors.As(err, &writerErr): writerErr.WriteTo(w) default: h.lg.Warn( "failed to process Raft message", zap.String("local-member-id", h.localID.String()), zap.Error(err), ) http.Error(w, "error processing raft message", http.StatusInternalServerError) w.(http.Flusher).Flush() // disconnect the http stream panic(err) } return } // Write StatusNoContent header after the message has been processed by // raft, which facilitates the client to report MsgSnap status. w.WriteHeader(http.StatusNoContent) } type snapshotHandler struct { lg *zap.Logger tr Transporter r Raft snapshotter *snap.Snapshotter localID types.ID cid types.ID } func newSnapshotHandler(t *Transport, r Raft, snapshotter *snap.Snapshotter, cid types.ID) http.Handler { h := &snapshotHandler{ lg: t.Logger, tr: t, r: r, snapshotter: snapshotter, localID: t.ID, cid: cid, } if h.lg == nil { h.lg = zap.NewNop() } return h } const unknownSnapshotSender = "UNKNOWN_SNAPSHOT_SENDER" // ServeHTTP serves HTTP request to receive and process snapshot message. // // If request sender dies without closing underlying TCP connection, // the handler will keep waiting for the request body until TCP keepalive // finds out that the connection is broken after several minutes. // This is acceptable because // 1. snapshot messages sent through other TCP connections could still be // received and processed. // 2. this case should happen rarely, so no further optimization is done. func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { start := time.Now() if r.Method != http.MethodPost { w.Header().Set("Allow", "POST") http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) snapshotReceiveFailures.WithLabelValues(unknownSnapshotSender).Inc() return } w.Header().Set("X-Etcd-Cluster-ID", h.cid.String()) if err := checkClusterCompatibilityFromHeader(h.lg, h.localID, r.Header, h.cid); err != nil { http.Error(w, err.Error(), http.StatusPreconditionFailed) snapshotReceiveFailures.WithLabelValues(unknownSnapshotSender).Inc() return } addRemoteFromRequest(h.tr, r) dec := &messageDecoder{r: r.Body} // let snapshots be very large since they can exceed 512MB for large installations m, err := dec.decodeLimit(snapshotLimitByte) from := types.ID(m.From).String() if err != nil { msg := fmt.Sprintf("failed to decode raft message (%v)", err) h.lg.Warn( "failed to decode Raft message", zap.String("local-member-id", h.localID.String()), zap.String("remote-snapshot-sender-id", from), zap.Error(err), ) http.Error(w, msg, http.StatusBadRequest) recvFailures.WithLabelValues(r.RemoteAddr).Inc() snapshotReceiveFailures.WithLabelValues(from).Inc() return } msgSize := m.Size() receivedBytes.WithLabelValues(from).Add(float64(msgSize)) if m.Type != raftpb.MsgSnap { h.lg.Warn( "unexpected Raft message type", zap.String("local-member-id", h.localID.String()), zap.String("remote-snapshot-sender-id", from), zap.String("message-type", m.Type.String()), ) http.Error(w, "wrong raft message type", http.StatusBadRequest) snapshotReceiveFailures.WithLabelValues(from).Inc() return } snapshotReceiveInflights.WithLabelValues(from).Inc() defer func() { snapshotReceiveInflights.WithLabelValues(from).Dec() }() h.lg.Info( "receiving database snapshot", zap.String("local-member-id", h.localID.String()), zap.String("remote-snapshot-sender-id", from), zap.Uint64("incoming-snapshot-index", m.Snapshot.Metadata.Index), zap.Int("incoming-snapshot-message-size-bytes", msgSize), zap.String("incoming-snapshot-message-size", humanize.Bytes(uint64(msgSize))), ) // save incoming database snapshot. n, err := h.snapshotter.SaveDBFrom(r.Body, m.Snapshot.Metadata.Index) if err != nil { msg := fmt.Sprintf("failed to save KV snapshot (%v)", err) h.lg.Warn( "failed to save incoming database snapshot", zap.String("local-member-id", h.localID.String()), zap.String("remote-snapshot-sender-id", from), zap.Uint64("incoming-snapshot-index", m.Snapshot.Metadata.Index), zap.Error(err), ) http.Error(w, msg, http.StatusInternalServerError) snapshotReceiveFailures.WithLabelValues(from).Inc() return } receivedBytes.WithLabelValues(from).Add(float64(n)) downloadTook := time.Since(start) h.lg.Info( "received and saved database snapshot", zap.String("local-member-id", h.localID.String()), zap.String("remote-snapshot-sender-id", from), zap.Uint64("incoming-snapshot-index", m.Snapshot.Metadata.Index), zap.Int64("incoming-snapshot-size-bytes", n), zap.String("incoming-snapshot-size", humanize.Bytes(uint64(n))), zap.String("download-took", downloadTook.String()), ) if err := h.r.Process(context.TODO(), m); err != nil { var writerErr writerToResponse switch { // Process may return writerToResponse error when doing some // additional checks before calling raft.Node.Step. case errors.As(err, &writerErr): writerErr.WriteTo(w) default: msg := fmt.Sprintf("failed to process raft message (%v)", err) h.lg.Warn( "failed to process Raft message", zap.String("local-member-id", h.localID.String()), zap.String("remote-snapshot-sender-id", from), zap.Error(err), ) http.Error(w, msg, http.StatusInternalServerError) snapshotReceiveFailures.WithLabelValues(from).Inc() } return } // Write StatusNoContent header after the message has been processed by // raft, which facilitates the client to report MsgSnap status. w.WriteHeader(http.StatusNoContent) snapshotReceive.WithLabelValues(from).Inc() snapshotReceiveSeconds.WithLabelValues(from).Observe(time.Since(start).Seconds()) } type streamHandler struct { lg *zap.Logger tr *Transport peerGetter peerGetter r Raft id types.ID cid types.ID } func newStreamHandler(t *Transport, pg peerGetter, r Raft, id, cid types.ID) http.Handler { h := &streamHandler{ lg: t.Logger, tr: t, peerGetter: pg, r: r, id: id, cid: cid, } if h.lg == nil { h.lg = zap.NewNop() } return h } func (h *streamHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { w.Header().Set("Allow", "GET") http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) return } w.Header().Set("X-Server-Version", version.Version) w.Header().Set("X-Etcd-Cluster-ID", h.cid.String()) if err := checkClusterCompatibilityFromHeader(h.lg, h.tr.ID, r.Header, h.cid); err != nil { http.Error(w, err.Error(), http.StatusPreconditionFailed) return } var t streamType switch path.Dir(r.URL.Path) { case streamTypeMsgAppV2.endpoint(h.lg): t = streamTypeMsgAppV2 case streamTypeMessage.endpoint(h.lg): t = streamTypeMessage default: h.lg.Debug( "ignored unexpected streaming request path", zap.String("local-member-id", h.tr.ID.String()), zap.String("remote-peer-id-stream-handler", h.id.String()), zap.String("path", r.URL.Path), ) http.Error(w, "invalid path", http.StatusNotFound) return } fromStr := path.Base(r.URL.Path) from, err := types.IDFromString(fromStr) if err != nil { h.lg.Warn( "failed to parse path into ID", zap.String("local-member-id", h.tr.ID.String()), zap.String("remote-peer-id-stream-handler", h.id.String()), zap.String("path", fromStr), zap.Error(err), ) http.Error(w, "invalid from", http.StatusNotFound) return } if h.r.IsIDRemoved(uint64(from)) { h.lg.Warn( "rejected stream from remote peer because it was removed", zap.String("local-member-id", h.tr.ID.String()), zap.String("remote-peer-id-stream-handler", h.id.String()), zap.String("remote-peer-id-from", from.String()), ) http.Error(w, "removed member", http.StatusGone) return } p := h.peerGetter.Get(from) if p == nil { // This may happen in following cases: // 1. user starts a remote peer that belongs to a different cluster // with the same cluster ID. // 2. local etcd falls behind of the cluster, and cannot recognize // the members that joined after its current progress. if urls := r.Header.Get("X-PeerURLs"); urls != "" { h.tr.AddRemote(from, strings.Split(urls, ",")) } h.lg.Warn( "failed to find remote peer in cluster", zap.String("local-member-id", h.tr.ID.String()), zap.String("remote-peer-id-stream-handler", h.id.String()), zap.String("remote-peer-id-from", from.String()), zap.String("cluster-id", h.cid.String()), ) http.Error(w, "error sender not found", http.StatusNotFound) return } wto := h.id.String() if gto := r.Header.Get("X-Raft-To"); gto != wto { h.lg.Warn( "ignored streaming request; ID mismatch", zap.String("local-member-id", h.tr.ID.String()), zap.String("remote-peer-id-stream-handler", h.id.String()), zap.String("remote-peer-id-header", gto), zap.String("remote-peer-id-from", from.String()), zap.String("cluster-id", h.cid.String()), ) http.Error(w, "to field mismatch", http.StatusPreconditionFailed) return } w.WriteHeader(http.StatusOK) w.(http.Flusher).Flush() c := newCloseNotifier() conn := &outgoingConn{ t: t, Writer: w, Flusher: w.(http.Flusher), Closer: c, localID: h.tr.ID, peerID: from, } p.attachOutgoingConn(conn) <-c.closeNotify() } // checkClusterCompatibilityFromHeader checks the cluster compatibility of // the local member from the given header. // It checks whether the version of local member is compatible with // the versions in the header, and whether the cluster ID of local member // matches the one in the header. func checkClusterCompatibilityFromHeader(lg *zap.Logger, localID types.ID, header http.Header, cid types.ID) error { remoteName := header.Get("X-Server-From") remoteServer := serverVersion(header) remoteVs := "" if remoteServer != nil { remoteVs = remoteServer.String() } remoteMinClusterVer := minClusterVersion(header) remoteMinClusterVs := "" if remoteMinClusterVer != nil { remoteMinClusterVs = remoteMinClusterVer.String() } localServer, localMinCluster, err := checkVersionCompatibility(remoteName, remoteServer, remoteMinClusterVer) localVs := "" if localServer != nil { localVs = localServer.String() } localMinClusterVs := "" if localMinCluster != nil { localMinClusterVs = localMinCluster.String() } if err != nil { lg.Warn( "failed version compatibility check", zap.String("local-member-id", localID.String()), zap.String("local-member-cluster-id", cid.String()), zap.String("local-member-server-version", localVs), zap.String("local-member-server-minimum-cluster-version", localMinClusterVs), zap.String("remote-peer-server-name", remoteName), zap.String("remote-peer-server-version", remoteVs), zap.String("remote-peer-server-minimum-cluster-version", remoteMinClusterVs), zap.Error(err), ) return errIncompatibleVersion } if gcid := header.Get("X-Etcd-Cluster-ID"); gcid != cid.String() { lg.Warn( "request cluster ID mismatch", zap.String("local-member-id", localID.String()), zap.String("local-member-cluster-id", cid.String()), zap.String("local-member-server-version", localVs), zap.String("local-member-server-minimum-cluster-version", localMinClusterVs), zap.String("remote-peer-server-name", remoteName), zap.String("remote-peer-server-version", remoteVs), zap.String("remote-peer-server-minimum-cluster-version", remoteMinClusterVs), zap.String("remote-peer-cluster-id", gcid), ) return ErrClusterIDMismatch } return nil } type closeNotifier struct { done chan struct{} } func newCloseNotifier() *closeNotifier { return &closeNotifier{ done: make(chan struct{}), } } func (n *closeNotifier) Close() error { close(n.done) return nil } func (n *closeNotifier) closeNotify() <-chan struct{} { return n.done } ================================================ FILE: server/etcdserver/api/rafthttp/http_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "bytes" "errors" "fmt" "io" "net/http" "net/http/httptest" "net/url" "strings" "testing" "time" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/raft/v3/raftpb" ) func TestServeRaftPrefix(t *testing.T) { testCases := []struct { method string body io.Reader p Raft clusterID string wcode int }{ { // bad method "GET", bytes.NewReader( pbutil.MustMarshal(&raftpb.Message{}), ), &fakeRaft{}, "0", http.StatusMethodNotAllowed, }, { // bad method "PUT", bytes.NewReader( pbutil.MustMarshal(&raftpb.Message{}), ), &fakeRaft{}, "0", http.StatusMethodNotAllowed, }, { // bad method "DELETE", bytes.NewReader( pbutil.MustMarshal(&raftpb.Message{}), ), &fakeRaft{}, "0", http.StatusMethodNotAllowed, }, { // bad request body "POST", &errReader{}, &fakeRaft{}, "0", http.StatusBadRequest, }, { // bad request protobuf "POST", strings.NewReader("malformed garbage"), &fakeRaft{}, "0", http.StatusBadRequest, }, { // good request, wrong cluster ID "POST", bytes.NewReader( pbutil.MustMarshal(&raftpb.Message{}), ), &fakeRaft{}, "1", http.StatusPreconditionFailed, }, { // good request, Processor failure "POST", bytes.NewReader( pbutil.MustMarshal(&raftpb.Message{}), ), &fakeRaft{ err: &resWriterToError{code: http.StatusForbidden}, }, "0", http.StatusForbidden, }, { // good request, Processor failure "POST", bytes.NewReader( pbutil.MustMarshal(&raftpb.Message{}), ), &fakeRaft{ err: &resWriterToError{code: http.StatusInternalServerError}, }, "0", http.StatusInternalServerError, }, { // good request, Processor failure "POST", bytes.NewReader( pbutil.MustMarshal(&raftpb.Message{}), ), &fakeRaft{err: errors.New("blah")}, "0", http.StatusInternalServerError, }, { // good request "POST", bytes.NewReader( pbutil.MustMarshal(&raftpb.Message{}), ), &fakeRaft{}, "0", http.StatusNoContent, }, } for i, tt := range testCases { req, err := http.NewRequest(tt.method, "foo", tt.body) if err != nil { t.Fatalf("#%d: could not create request: %#v", i, err) } req.Header.Set("X-Etcd-Cluster-ID", tt.clusterID) req.Header.Set("X-Server-Version", version.Version) rw := httptest.NewRecorder() h := newPipelineHandler(&Transport{Logger: zaptest.NewLogger(t)}, tt.p, types.ID(0)) // goroutine because the handler panics to disconnect on raft error donec := make(chan struct{}) go func() { defer func() { recover() close(donec) }() h.ServeHTTP(rw, req) }() <-donec if rw.Code != tt.wcode { t.Errorf("#%d: got code=%d, want %d", i, rw.Code, tt.wcode) } } } func TestServeRaftStreamPrefix(t *testing.T) { tests := []struct { path string wtype streamType }{ { RaftStreamPrefix + "/message/1", streamTypeMessage, }, { RaftStreamPrefix + "/msgapp/1", streamTypeMsgAppV2, }, } for i, tt := range tests { req, err := http.NewRequest(http.MethodGet, "http://localhost:2380"+tt.path, nil) if err != nil { t.Fatalf("#%d: could not create request: %#v", i, err) } req.Header.Set("X-Etcd-Cluster-ID", "1") req.Header.Set("X-Server-Version", version.Version) req.Header.Set("X-Raft-To", "2") peer := newFakePeer() peerGetter := &fakePeerGetter{peers: map[types.ID]Peer{types.ID(1): peer}} tr := &Transport{} h := newStreamHandler(tr, peerGetter, &fakeRaft{}, types.ID(2), types.ID(1)) rw := httptest.NewRecorder() go h.ServeHTTP(rw, req) var conn *outgoingConn select { case conn = <-peer.connc: case <-time.After(time.Second): t.Fatalf("#%d: failed to attach outgoingConn", i) } if g := rw.Header().Get("X-Server-Version"); g != version.Version { t.Errorf("#%d: X-Server-Version = %s, want %s", i, g, version.Version) } if conn.t != tt.wtype { t.Errorf("#%d: type = %s, want %s", i, conn.t, tt.wtype) } conn.Close() } } func TestServeRaftStreamPrefixBad(t *testing.T) { removedID := uint64(5) tests := []struct { method string path string clusterID string remote string wcode int }{ // bad method { "PUT", RaftStreamPrefix + "/message/1", "1", "1", http.StatusMethodNotAllowed, }, // bad method { "POST", RaftStreamPrefix + "/message/1", "1", "1", http.StatusMethodNotAllowed, }, // bad method { "DELETE", RaftStreamPrefix + "/message/1", "1", "1", http.StatusMethodNotAllowed, }, // bad path { "GET", RaftStreamPrefix + "/strange/1", "1", "1", http.StatusNotFound, }, // bad path { "GET", RaftStreamPrefix + "/strange", "1", "1", http.StatusNotFound, }, // non-existent peer { "GET", RaftStreamPrefix + "/message/2", "1", "1", http.StatusNotFound, }, // removed peer { "GET", RaftStreamPrefix + "/message/" + fmt.Sprint(removedID), "1", "1", http.StatusGone, }, // wrong cluster ID { "GET", RaftStreamPrefix + "/message/1", "2", "1", http.StatusPreconditionFailed, }, // wrong remote id { "GET", RaftStreamPrefix + "/message/1", "1", "2", http.StatusPreconditionFailed, }, } for i, tt := range tests { req, err := http.NewRequest(tt.method, "http://localhost:2380"+tt.path, nil) if err != nil { t.Fatalf("#%d: could not create request: %#v", i, err) } req.Header.Set("X-Etcd-Cluster-ID", tt.clusterID) req.Header.Set("X-Server-Version", version.Version) req.Header.Set("X-Raft-To", tt.remote) rw := httptest.NewRecorder() tr := &Transport{} peerGetter := &fakePeerGetter{peers: map[types.ID]Peer{types.ID(1): newFakePeer()}} r := &fakeRaft{removedID: removedID} h := newStreamHandler(tr, peerGetter, r, types.ID(1), types.ID(1)) h.ServeHTTP(rw, req) if rw.Code != tt.wcode { t.Errorf("#%d: code = %d, want %d", i, rw.Code, tt.wcode) } } } func TestCloseNotifier(t *testing.T) { c := newCloseNotifier() select { case <-c.closeNotify(): t.Fatalf("received unexpected close notification") default: } c.Close() select { case <-c.closeNotify(): default: t.Fatalf("failed to get close notification") } } // errReader implements io.Reader to facilitate a broken request. type errReader struct{} func (er *errReader) Read(_ []byte) (int, error) { return 0, errors.New("some error") } type resWriterToError struct { code int } func (e *resWriterToError) Error() string { return "" } func (e *resWriterToError) WriteTo(w http.ResponseWriter) { w.WriteHeader(e.code) } type fakePeerGetter struct { peers map[types.ID]Peer } func (pg *fakePeerGetter) Get(id types.ID) Peer { return pg.peers[id] } type fakePeer struct { msgs []raftpb.Message snapMsgs []snap.Message peerURLs types.URLs connc chan *outgoingConn paused bool } func newFakePeer() *fakePeer { fakeURL, _ := url.Parse("http://localhost") return &fakePeer{ connc: make(chan *outgoingConn, 1), peerURLs: types.URLs{*fakeURL}, } } func (pr *fakePeer) send(m raftpb.Message) { if pr.paused { return } pr.msgs = append(pr.msgs, m) } func (pr *fakePeer) sendSnap(m snap.Message) { if pr.paused { return } pr.snapMsgs = append(pr.snapMsgs, m) } func (pr *fakePeer) update(urls types.URLs) { pr.peerURLs = urls } func (pr *fakePeer) attachOutgoingConn(conn *outgoingConn) { pr.connc <- conn } func (pr *fakePeer) activeSince() time.Time { return time.Time{} } func (pr *fakePeer) stop() {} func (pr *fakePeer) Pause() { pr.paused = true } func (pr *fakePeer) Resume() { pr.paused = false } ================================================ FILE: server/etcdserver/api/rafthttp/metrics.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import "github.com/prometheus/client_golang/prometheus" var ( activePeers = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "network", Name: "active_peers", Help: "The current number of active peer connections.", }, []string{"Local", "Remote"}, ) disconnectedPeers = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "disconnected_peers_total", Help: "The total number of disconnected peers.", }, []string{"Local", "Remote"}, ) sentBytes = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "peer_sent_bytes_total", Help: "The total number of bytes sent to peers.", }, []string{"To"}, ) receivedBytes = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "peer_received_bytes_total", Help: "The total number of bytes received from peers.", }, []string{"From"}, ) sentFailures = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "peer_sent_failures_total", Help: "The total number of send failures from peers.", }, []string{"To"}, ) recvFailures = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "peer_received_failures_total", Help: "The total number of receive failures from peers.", }, []string{"From"}, ) snapshotSend = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "snapshot_send_success", Help: "Total number of successful snapshot sends", }, []string{"To"}, ) snapshotSendInflights = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "network", Name: "snapshot_send_inflights_total", Help: "Total number of inflight snapshot sends", }, []string{"To"}, ) snapshotSendFailures = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "snapshot_send_failures", Help: "Total number of snapshot send failures", }, []string{"To"}, ) snapshotSendSeconds = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "network", Name: "snapshot_send_total_duration_seconds", Help: "Total latency distributions of v3 snapshot sends", // lowest bucket start of upper bound 0.1 sec (100 ms) with factor 2 // highest bucket start of 0.1 sec * 2^9 == 51.2 sec Buckets: prometheus.ExponentialBuckets(0.1, 2, 10), }, []string{"To"}, ) snapshotReceive = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "snapshot_receive_success", Help: "Total number of successful snapshot receives", }, []string{"From"}, ) snapshotReceiveInflights = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "network", Name: "snapshot_receive_inflights_total", Help: "Total number of inflight snapshot receives", }, []string{"From"}, ) snapshotReceiveFailures = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "snapshot_receive_failures", Help: "Total number of snapshot receive failures", }, []string{"From"}, ) snapshotReceiveSeconds = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "network", Name: "snapshot_receive_total_duration_seconds", Help: "Total latency distributions of v3 snapshot receives", // lowest bucket start of upper bound 0.1 sec (100 ms) with factor 2 // highest bucket start of 0.1 sec * 2^9 == 51.2 sec Buckets: prometheus.ExponentialBuckets(0.1, 2, 10), }, []string{"From"}, ) rttSec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "network", Name: "peer_round_trip_time_seconds", Help: "Round-Trip-Time histogram between peers", // lowest bucket start of upper bound 0.0001 sec (0.1 ms) with factor 2 // highest bucket start of 0.0001 sec * 2^15 == 3.2768 sec Buckets: prometheus.ExponentialBuckets(0.0001, 2, 16), }, []string{"To"}, ) ) func init() { prometheus.MustRegister(activePeers) prometheus.MustRegister(disconnectedPeers) prometheus.MustRegister(sentBytes) prometheus.MustRegister(receivedBytes) prometheus.MustRegister(sentFailures) prometheus.MustRegister(recvFailures) prometheus.MustRegister(snapshotSend) prometheus.MustRegister(snapshotSendInflights) prometheus.MustRegister(snapshotSendFailures) prometheus.MustRegister(snapshotSendSeconds) prometheus.MustRegister(snapshotReceive) prometheus.MustRegister(snapshotReceiveInflights) prometheus.MustRegister(snapshotReceiveFailures) prometheus.MustRegister(snapshotReceiveSeconds) prometheus.MustRegister(rttSec) } ================================================ FILE: server/etcdserver/api/rafthttp/msg_codec.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "encoding/binary" "errors" "io" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/raft/v3/raftpb" ) // messageEncoder is a encoder that can encode all kinds of messages. // It MUST be used with a paired messageDecoder. type messageEncoder struct { w io.Writer } func (enc *messageEncoder) encode(m *raftpb.Message) error { if err := binary.Write(enc.w, binary.BigEndian, uint64(m.Size())); err != nil { return err } _, err := enc.w.Write(pbutil.MustMarshal(m)) return err } // messageDecoder is a decoder that can decode all kinds of messages. type messageDecoder struct { r io.Reader } var ( readBytesLimit uint64 = 512 * 1024 * 1024 // 512 MB ErrExceedSizeLimit = errors.New("rafthttp: error limit exceeded") ) func (dec *messageDecoder) decode() (raftpb.Message, error) { return dec.decodeLimit(readBytesLimit) } func (dec *messageDecoder) decodeLimit(numBytes uint64) (raftpb.Message, error) { var m raftpb.Message var l uint64 if err := binary.Read(dec.r, binary.BigEndian, &l); err != nil { return m, err } if l > numBytes { return m, ErrExceedSizeLimit } buf := make([]byte, int(l)) if _, err := io.ReadFull(dec.r, buf); err != nil { return m, err } return m, m.Unmarshal(buf) } ================================================ FILE: server/etcdserver/api/rafthttp/msg_codec_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "bytes" "errors" "reflect" "testing" "go.etcd.io/raft/v3/raftpb" ) func TestMessage(t *testing.T) { // Lower readBytesLimit to make test pass in restricted resources environment originalLimit := readBytesLimit readBytesLimit = 1000 defer func() { readBytesLimit = originalLimit }() tests := []struct { msg raftpb.Message encodeErr error decodeErr error }{ { raftpb.Message{ Type: raftpb.MsgApp, From: 1, To: 2, Term: 1, LogTerm: 1, Index: 3, Entries: []raftpb.Entry{{Term: 1, Index: 4}}, }, nil, nil, }, { raftpb.Message{ Type: raftpb.MsgProp, From: 1, To: 2, Entries: []raftpb.Entry{ {Data: []byte("some data")}, {Data: []byte("some data")}, {Data: []byte("some data")}, }, }, nil, nil, }, { raftpb.Message{ Type: raftpb.MsgProp, From: 1, To: 2, Entries: []raftpb.Entry{ {Data: bytes.Repeat([]byte("a"), int(readBytesLimit+10))}, }, }, nil, ErrExceedSizeLimit, }, } for i, tt := range tests { b := &bytes.Buffer{} enc := &messageEncoder{w: b} if err := enc.encode(&tt.msg); !errors.Is(err, tt.encodeErr) { t.Errorf("#%d: encode message error expected %v, got %v", i, tt.encodeErr, err) continue } dec := &messageDecoder{r: b} m, err := dec.decode() if !errors.Is(err, tt.decodeErr) { t.Errorf("#%d: decode message error expected %v, got %v", i, tt.decodeErr, err) continue } if err == nil { if !reflect.DeepEqual(m, tt.msg) { t.Errorf("#%d: message = %+v, want %+v", i, m, tt.msg) } } } } ================================================ FILE: server/etcdserver/api/rafthttp/msgappv2_codec.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "encoding/binary" "fmt" "io" "time" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/pbutil" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/raft/v3/raftpb" ) const ( msgTypeLinkHeartbeat uint8 = 0 msgTypeAppEntries uint8 = 1 msgTypeApp uint8 = 2 msgAppV2BufSize = 1024 * 1024 ) // msgappv2 stream sends three types of message: linkHeartbeatMessage, // AppEntries and MsgApp. AppEntries is the MsgApp that is sent in // replicate state in raft, whose index and term are fully predictable. // // Data format of linkHeartbeatMessage: // | offset | bytes | description | // +--------+-------+-------------+ // | 0 | 1 | \x00 | // // Data format of AppEntries: // | offset | bytes | description | // +--------+-------+-------------+ // | 0 | 1 | \x01 | // | 1 | 8 | length of entries | // | 9 | 8 | length of first entry | // | 17 | n1 | first entry | // ... // | x | 8 | length of k-th entry data | // | x+8 | nk | k-th entry data | // | x+8+nk | 8 | commit index | // // Data format of MsgApp: // | offset | bytes | description | // +--------+-------+-------------+ // | 0 | 1 | \x02 | // | 1 | 8 | length of encoded message | // | 9 | n | encoded message | type msgAppV2Encoder struct { w io.Writer fs *stats.FollowerStats term uint64 index uint64 buf []byte uint64buf []byte uint8buf []byte } func newMsgAppV2Encoder(w io.Writer, fs *stats.FollowerStats) *msgAppV2Encoder { return &msgAppV2Encoder{ w: w, fs: fs, buf: make([]byte, msgAppV2BufSize), uint64buf: make([]byte, 8), uint8buf: make([]byte, 1), } } func (enc *msgAppV2Encoder) encode(m *raftpb.Message) error { start := time.Now() switch { case isLinkHeartbeatMessage(m): enc.uint8buf[0] = msgTypeLinkHeartbeat if _, err := enc.w.Write(enc.uint8buf); err != nil { return err } case enc.index == m.Index && enc.term == m.LogTerm && m.LogTerm == m.Term: enc.uint8buf[0] = msgTypeAppEntries if _, err := enc.w.Write(enc.uint8buf); err != nil { return err } // write length of entries binary.BigEndian.PutUint64(enc.uint64buf, uint64(len(m.Entries))) if _, err := enc.w.Write(enc.uint64buf); err != nil { return err } for i := 0; i < len(m.Entries); i++ { // write length of entry binary.BigEndian.PutUint64(enc.uint64buf, uint64(m.Entries[i].Size())) if _, err := enc.w.Write(enc.uint64buf); err != nil { return err } if n := m.Entries[i].Size(); n < msgAppV2BufSize { if _, err := m.Entries[i].MarshalTo(enc.buf); err != nil { return err } if _, err := enc.w.Write(enc.buf[:n]); err != nil { return err } } else { if _, err := enc.w.Write(pbutil.MustMarshal(&m.Entries[i])); err != nil { return err } } enc.index++ } // write commit index binary.BigEndian.PutUint64(enc.uint64buf, m.Commit) if _, err := enc.w.Write(enc.uint64buf); err != nil { return err } enc.fs.Succ(time.Since(start)) default: if err := binary.Write(enc.w, binary.BigEndian, msgTypeApp); err != nil { return err } // write size of message if err := binary.Write(enc.w, binary.BigEndian, uint64(m.Size())); err != nil { return err } // write message if _, err := enc.w.Write(pbutil.MustMarshal(m)); err != nil { return err } enc.term = m.Term enc.index = m.Index if l := len(m.Entries); l > 0 { enc.index = m.Entries[l-1].Index } enc.fs.Succ(time.Since(start)) } return nil } type msgAppV2Decoder struct { r io.Reader local, remote types.ID term uint64 index uint64 buf []byte uint64buf []byte uint8buf []byte } func newMsgAppV2Decoder(r io.Reader, local, remote types.ID) *msgAppV2Decoder { return &msgAppV2Decoder{ r: r, local: local, remote: remote, buf: make([]byte, msgAppV2BufSize), uint64buf: make([]byte, 8), uint8buf: make([]byte, 1), } } func (dec *msgAppV2Decoder) decode() (raftpb.Message, error) { var ( m raftpb.Message typ uint8 ) if _, err := io.ReadFull(dec.r, dec.uint8buf); err != nil { return m, err } typ = dec.uint8buf[0] switch typ { case msgTypeLinkHeartbeat: return linkHeartbeatMessage, nil case msgTypeAppEntries: m = raftpb.Message{ Type: raftpb.MsgApp, From: uint64(dec.remote), To: uint64(dec.local), Term: dec.term, LogTerm: dec.term, Index: dec.index, } // decode entries if _, err := io.ReadFull(dec.r, dec.uint64buf); err != nil { return m, err } l := binary.BigEndian.Uint64(dec.uint64buf) m.Entries = make([]raftpb.Entry, int(l)) for i := 0; i < int(l); i++ { if _, err := io.ReadFull(dec.r, dec.uint64buf); err != nil { return m, err } size := binary.BigEndian.Uint64(dec.uint64buf) var buf []byte if size < msgAppV2BufSize { buf = dec.buf[:size] if _, err := io.ReadFull(dec.r, buf); err != nil { return m, err } } else { buf = make([]byte, int(size)) if _, err := io.ReadFull(dec.r, buf); err != nil { return m, err } } dec.index++ // 1 alloc pbutil.MustUnmarshal(&m.Entries[i], buf) } // decode commit index if _, err := io.ReadFull(dec.r, dec.uint64buf); err != nil { return m, err } m.Commit = binary.BigEndian.Uint64(dec.uint64buf) case msgTypeApp: var size uint64 if err := binary.Read(dec.r, binary.BigEndian, &size); err != nil { return m, err } buf := make([]byte, int(size)) if _, err := io.ReadFull(dec.r, buf); err != nil { return m, err } pbutil.MustUnmarshal(&m, buf) dec.term = m.Term dec.index = m.Index if l := len(m.Entries); l > 0 { dec.index = m.Entries[l-1].Index } default: return m, fmt.Errorf("failed to parse type %d in msgappv2 stream", typ) } return m, nil } ================================================ FILE: server/etcdserver/api/rafthttp/msgappv2_codec_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "bytes" "reflect" "testing" "go.etcd.io/etcd/client/pkg/v3/types" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/raft/v3/raftpb" ) func TestMsgAppV2(t *testing.T) { tests := []raftpb.Message{ linkHeartbeatMessage, { Type: raftpb.MsgApp, From: 1, To: 2, Term: 1, LogTerm: 1, Index: 0, Entries: []raftpb.Entry{ {Term: 1, Index: 1, Data: []byte("some data")}, {Term: 1, Index: 2, Data: []byte("some data")}, {Term: 1, Index: 3, Data: []byte("some data")}, }, }, // consecutive MsgApp { Type: raftpb.MsgApp, From: 1, To: 2, Term: 1, LogTerm: 1, Index: 3, Entries: []raftpb.Entry{ {Term: 1, Index: 4, Data: []byte("some data")}, }, }, linkHeartbeatMessage, // consecutive MsgApp after linkHeartbeatMessage { Type: raftpb.MsgApp, From: 1, To: 2, Term: 1, LogTerm: 1, Index: 4, Entries: []raftpb.Entry{ {Term: 1, Index: 5, Data: []byte("some data")}, }, }, // MsgApp with higher term { Type: raftpb.MsgApp, From: 1, To: 2, Term: 3, LogTerm: 1, Index: 5, Entries: []raftpb.Entry{ {Term: 3, Index: 6, Data: []byte("some data")}, }, }, linkHeartbeatMessage, // consecutive MsgApp { Type: raftpb.MsgApp, From: 1, To: 2, Term: 3, LogTerm: 2, Index: 6, Entries: []raftpb.Entry{ {Term: 3, Index: 7, Data: []byte("some data")}, }, }, // consecutive empty MsgApp { Type: raftpb.MsgApp, From: 1, To: 2, Term: 3, LogTerm: 2, Index: 7, Entries: nil, }, linkHeartbeatMessage, } b := &bytes.Buffer{} enc := newMsgAppV2Encoder(b, &stats.FollowerStats{}) dec := newMsgAppV2Decoder(b, types.ID(2), types.ID(1)) for i, tt := range tests { if err := enc.encode(&tt); err != nil { t.Errorf("#%d: unexpected encode message error: %v", i, err) continue } m, err := dec.decode() if err != nil { t.Errorf("#%d: unexpected decode message error: %v", i, err) continue } if !reflect.DeepEqual(m, tt) { t.Errorf("#%d: message = %+v, want %+v", i, m, tt) } } } ================================================ FILE: server/etcdserver/api/rafthttp/peer.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "context" "sync" "time" "go.uber.org/zap" "golang.org/x/time/rate" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) const ( // ConnReadTimeout and ConnWriteTimeout are the i/o timeout set on each connection rafthttp pkg creates. // A 5 seconds timeout is good enough for recycling bad connections. Or we have to wait for // tcp keepalive failing to detect a bad connection, which is at minutes level. // For long term streaming connections, rafthttp pkg sends application level linkHeartbeatMessage // to keep the connection alive. // For short term pipeline connections, the connection MUST be killed to avoid it being // put back to http pkg connection pool. DefaultConnReadTimeout = 5 * time.Second DefaultConnWriteTimeout = 5 * time.Second recvBufSize = 4096 // maxPendingProposals holds the proposals during one leader election process. // Generally one leader election takes at most 1 sec. It should have // 0-2 election conflicts, and each one takes 0.5 sec. // We assume the number of concurrent proposers is smaller than 4096. // One client blocks on its proposal for at least 1 sec, so 4096 is enough // to hold all proposals. maxPendingProposals = 4096 streamAppV2 = "streamMsgAppV2" streamMsg = "streamMsg" pipelineMsg = "pipeline" sendSnap = "sendMsgSnap" ) var ( ConnReadTimeout = DefaultConnReadTimeout ConnWriteTimeout = DefaultConnWriteTimeout ) type Peer interface { // send sends the message to the remote peer. The function is non-blocking // and has no promise that the message will be received by the remote. // When it fails to send message out, it will report the status to underlying // raft. send(m raftpb.Message) // sendSnap sends the merged snapshot message to the remote peer. Its behavior // is similar to send. sendSnap(m snap.Message) // update updates the urls of remote peer. update(urls types.URLs) // attachOutgoingConn attaches the outgoing connection to the peer for // stream usage. After the call, the ownership of the outgoing // connection hands over to the peer. The peer will close the connection // when it is no longer used. attachOutgoingConn(conn *outgoingConn) // activeSince returns the time that the connection with the // peer becomes active. activeSince() time.Time // stop performs any necessary finalization and terminates the peer // elegantly. stop() } // peer is the representative of a remote raft node. Local raft node sends // messages to the remote through peer. // Each peer has two underlying mechanisms to send out a message: stream and // pipeline. // A stream is a receiver initialized long-polling connection, which // is always open to transfer messages. Besides general stream, peer also has // a optimized stream for sending msgApp since msgApp accounts for large part // of all messages. Only raft leader uses the optimized stream to send msgApp // to the remote follower node. // A pipeline is a series of http clients that send http requests to the remote. // It is only used when the stream has not been established. type peer struct { lg *zap.Logger localID types.ID // id of the remote raft peer node id types.ID r Raft status *peerStatus picker *urlPicker msgAppV2Writer *streamWriter writer *streamWriter pipeline *pipeline snapSender *snapshotSender // snapshot sender to send v3 snapshot messages msgAppV2Reader *streamReader msgAppReader *streamReader recvc chan raftpb.Message propc chan raftpb.Message mu sync.Mutex paused bool cancel context.CancelFunc // cancel pending works in go routine created by peer. stopc chan struct{} } func startPeer(t *Transport, urls types.URLs, peerID types.ID, fs *stats.FollowerStats) *peer { if t.Logger != nil { t.Logger.Info("starting remote peer", zap.String("remote-peer-id", peerID.String())) } defer func() { if t.Logger != nil { t.Logger.Info("started remote peer", zap.String("remote-peer-id", peerID.String())) } }() status := newPeerStatus(t.Logger, t.ID, peerID) picker := newURLPicker(urls) errorc := t.ErrorC r := t.Raft pipeline := &pipeline{ peerID: peerID, tr: t, picker: picker, status: status, followerStats: fs, raft: r, errorc: errorc, } pipeline.start() p := &peer{ lg: t.Logger, localID: t.ID, id: peerID, r: r, status: status, picker: picker, msgAppV2Writer: startStreamWriter(t.Logger, t.ID, peerID, status, fs, r), writer: startStreamWriter(t.Logger, t.ID, peerID, status, fs, r), pipeline: pipeline, snapSender: newSnapshotSender(t, picker, peerID, status), recvc: make(chan raftpb.Message, recvBufSize), propc: make(chan raftpb.Message, maxPendingProposals), stopc: make(chan struct{}), } ctx, cancel := context.WithCancel(context.Background()) p.cancel = cancel go func() { for { select { case mm := <-p.recvc: if err := r.Process(ctx, mm); err != nil { if t.Logger != nil { t.Logger.Warn("failed to process Raft message", zap.Error(err)) } } case <-p.stopc: return } } }() // r.Process might block for processing proposal when there is no leader. // Thus propc must be put into a separate routine with recvc to avoid blocking // processing other raft messages. go func() { for { select { case mm := <-p.propc: if err := r.Process(ctx, mm); err != nil { if t.Logger != nil { t.Logger.Warn("failed to process Raft message", zap.Error(err)) } } case <-p.stopc: return } } }() p.msgAppV2Reader = &streamReader{ lg: t.Logger, peerID: peerID, typ: streamTypeMsgAppV2, tr: t, picker: picker, status: status, recvc: p.recvc, propc: p.propc, rl: rate.NewLimiter(t.DialRetryFrequency, 1), } p.msgAppReader = &streamReader{ lg: t.Logger, peerID: peerID, typ: streamTypeMessage, tr: t, picker: picker, status: status, recvc: p.recvc, propc: p.propc, rl: rate.NewLimiter(t.DialRetryFrequency, 1), } p.msgAppV2Reader.start() p.msgAppReader.start() return p } func (p *peer) send(m raftpb.Message) { p.mu.Lock() paused := p.paused p.mu.Unlock() if paused { return } writec, name := p.pick(m) select { case writec <- m: default: p.r.ReportUnreachable(m.To) if isMsgSnap(m) { p.r.ReportSnapshot(m.To, raft.SnapshotFailure) } if p.lg != nil { p.lg.Warn( "dropped internal Raft message since sending buffer is full", zap.String("message-type", m.Type.String()), zap.String("local-member-id", p.localID.String()), zap.String("from", types.ID(m.From).String()), zap.String("remote-peer-id", p.id.String()), zap.String("remote-peer-name", name), zap.Bool("remote-peer-active", p.status.isActive()), ) } sentFailures.WithLabelValues(types.ID(m.To).String()).Inc() } } func (p *peer) sendSnap(m snap.Message) { go p.snapSender.send(m) } func (p *peer) update(urls types.URLs) { p.picker.update(urls) } func (p *peer) attachOutgoingConn(conn *outgoingConn) { var ok bool switch conn.t { case streamTypeMsgAppV2: ok = p.msgAppV2Writer.attach(conn) case streamTypeMessage: ok = p.writer.attach(conn) default: if p.lg != nil { p.lg.Panic("unknown stream type", zap.String("type", conn.t.String())) } } if !ok { conn.Close() } } func (p *peer) activeSince() time.Time { return p.status.activeSince() } // Pause pauses the peer. The peer will simply drops all incoming // messages without returning an error. func (p *peer) Pause() { p.mu.Lock() defer p.mu.Unlock() p.paused = true p.msgAppReader.pause() p.msgAppV2Reader.pause() } // Resume resumes a paused peer. func (p *peer) Resume() { p.mu.Lock() defer p.mu.Unlock() p.paused = false p.msgAppReader.resume() p.msgAppV2Reader.resume() } func (p *peer) stop() { if p.lg != nil { p.lg.Info("stopping remote peer", zap.String("remote-peer-id", p.id.String())) } defer func() { if p.lg != nil { p.lg.Info("stopped remote peer", zap.String("remote-peer-id", p.id.String())) } }() close(p.stopc) p.cancel() p.msgAppV2Writer.stop() p.writer.stop() p.pipeline.stop() p.snapSender.stop() p.msgAppV2Reader.stop() p.msgAppReader.stop() } // pick picks a chan for sending the given message. The picked chan and the picked chan // string name are returned. func (p *peer) pick(m raftpb.Message) (writec chan<- raftpb.Message, picked string) { var ok bool // Considering MsgSnap may have a big size, e.g., 1G, and will block // stream for a long time, only use one of the N pipelines to send MsgSnap. if isMsgSnap(m) { return p.pipeline.msgc, pipelineMsg } else if writec, ok = p.msgAppV2Writer.writec(); ok && isMsgApp(m) { return writec, streamAppV2 } else if writec, ok = p.writer.writec(); ok { return writec, streamMsg } return p.pipeline.msgc, pipelineMsg } func isMsgApp(m raftpb.Message) bool { return m.Type == raftpb.MsgApp } func isMsgSnap(m raftpb.Message) bool { return m.Type == raftpb.MsgSnap } ================================================ FILE: server/etcdserver/api/rafthttp/peer_status.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "errors" "fmt" "sync" "time" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/types" ) type failureType struct { source string action string } type peerStatus struct { lg *zap.Logger local types.ID id types.ID mu sync.Mutex // protect variables below active bool since time.Time } func newPeerStatus(lg *zap.Logger, local, id types.ID) *peerStatus { if lg == nil { lg = zap.NewNop() } return &peerStatus{lg: lg, local: local, id: id} } func (s *peerStatus) activate() { s.mu.Lock() defer s.mu.Unlock() if !s.active { s.lg.Info("peer became active", zap.String("peer-id", s.id.String())) s.active = true s.since = time.Now() activePeers.WithLabelValues(s.local.String(), s.id.String()).Inc() } } func (s *peerStatus) deactivate(failure failureType, reason string) { s.mu.Lock() defer s.mu.Unlock() msg := fmt.Sprintf("failed to %s %s on %s (%s)", failure.action, s.id, failure.source, reason) if s.active { s.lg.Warn("peer became inactive (message send to peer failed)", zap.String("peer-id", s.id.String()), zap.Error(errors.New(msg))) s.active = false s.since = time.Time{} activePeers.WithLabelValues(s.local.String(), s.id.String()).Dec() disconnectedPeers.WithLabelValues(s.local.String(), s.id.String()).Inc() return } if s.lg != nil { s.lg.Debug("peer deactivated again", zap.String("peer-id", s.id.String()), zap.Error(errors.New(msg))) } } func (s *peerStatus) isActive() bool { s.mu.Lock() defer s.mu.Unlock() return s.active } func (s *peerStatus) activeSince() time.Time { s.mu.Lock() defer s.mu.Unlock() return s.since } ================================================ FILE: server/etcdserver/api/rafthttp/peer_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "testing" "go.etcd.io/raft/v3/raftpb" ) func TestPeerPick(t *testing.T) { tests := []struct { msgappWorking bool messageWorking bool m raftpb.Message wpicked string }{ { true, true, raftpb.Message{Type: raftpb.MsgSnap}, pipelineMsg, }, { true, true, raftpb.Message{Type: raftpb.MsgApp, Term: 1, LogTerm: 1}, streamAppV2, }, { true, true, raftpb.Message{Type: raftpb.MsgProp}, streamMsg, }, { true, true, raftpb.Message{Type: raftpb.MsgHeartbeat}, streamMsg, }, { false, true, raftpb.Message{Type: raftpb.MsgApp, Term: 1, LogTerm: 1}, streamMsg, }, { false, false, raftpb.Message{Type: raftpb.MsgApp, Term: 1, LogTerm: 1}, pipelineMsg, }, { false, false, raftpb.Message{Type: raftpb.MsgProp}, pipelineMsg, }, { false, false, raftpb.Message{Type: raftpb.MsgSnap}, pipelineMsg, }, { false, false, raftpb.Message{Type: raftpb.MsgHeartbeat}, pipelineMsg, }, } for i, tt := range tests { peer := &peer{ msgAppV2Writer: &streamWriter{working: tt.msgappWorking}, writer: &streamWriter{working: tt.messageWorking}, pipeline: &pipeline{}, } _, picked := peer.pick(tt.m) if picked != tt.wpicked { t.Errorf("#%d: picked = %v, want %v", i, picked, tt.wpicked) } } } ================================================ FILE: server/etcdserver/api/rafthttp/pipeline.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "bytes" "context" "errors" "io" "runtime" "sync" "time" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/pbutil" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) const ( connPerPipeline = 4 // pipelineBufSize is the size of pipeline buffer, which helps hold the // temporary network latency. // The size ensures that pipeline does not drop messages when the network // is out of work for less than 1 second in good path. pipelineBufSize = 64 ) var errStopped = errors.New("stopped") type pipeline struct { peerID types.ID tr *Transport picker *urlPicker status *peerStatus raft Raft errorc chan error // deprecate when we depercate v2 API followerStats *stats.FollowerStats msgc chan raftpb.Message // wait for the handling routines wg sync.WaitGroup stopc chan struct{} } func (p *pipeline) start() { p.stopc = make(chan struct{}) p.msgc = make(chan raftpb.Message, pipelineBufSize) p.wg.Add(connPerPipeline) for i := 0; i < connPerPipeline; i++ { go p.handle() } if p.tr != nil && p.tr.Logger != nil { p.tr.Logger.Info( "started HTTP pipelining with remote peer", zap.String("local-member-id", p.tr.ID.String()), zap.String("remote-peer-id", p.peerID.String()), ) } } func (p *pipeline) stop() { close(p.stopc) p.wg.Wait() if p.tr != nil && p.tr.Logger != nil { p.tr.Logger.Info( "stopped HTTP pipelining with remote peer", zap.String("local-member-id", p.tr.ID.String()), zap.String("remote-peer-id", p.peerID.String()), ) } } func (p *pipeline) handle() { defer p.wg.Done() for { select { case m := <-p.msgc: start := time.Now() err := p.post(pbutil.MustMarshal(&m)) end := time.Now() if err != nil { p.status.deactivate(failureType{source: pipelineMsg, action: "write"}, err.Error()) if isMsgApp(m) && p.followerStats != nil { p.followerStats.Fail() } p.raft.ReportUnreachable(m.To) if isMsgSnap(m) { p.raft.ReportSnapshot(m.To, raft.SnapshotFailure) } sentFailures.WithLabelValues(types.ID(m.To).String()).Inc() continue } p.status.activate() if isMsgApp(m) && p.followerStats != nil { p.followerStats.Succ(end.Sub(start)) } if isMsgSnap(m) { p.raft.ReportSnapshot(m.To, raft.SnapshotFinish) } sentBytes.WithLabelValues(types.ID(m.To).String()).Add(float64(m.Size())) case <-p.stopc: return } } } // post POSTs a data payload to a url. Returns nil if the POST succeeds, // error on any failure. func (p *pipeline) post(data []byte) (err error) { u := p.picker.pick() req := createPostRequest(p.tr.Logger, u, RaftPrefix, bytes.NewBuffer(data), "application/protobuf", p.tr.URLs, p.tr.ID, p.tr.ClusterID) done := make(chan struct{}, 1) ctx, cancel := context.WithCancel(context.Background()) req = req.WithContext(ctx) go func() { select { case <-done: cancel() case <-p.stopc: waitSchedule() cancel() } }() resp, err := p.tr.pipelineRt.RoundTrip(req) done <- struct{}{} if err != nil { p.picker.unreachable(u) return err } defer resp.Body.Close() b, err := io.ReadAll(resp.Body) if err != nil { p.picker.unreachable(u) return err } err = checkPostResponse(p.tr.Logger, resp, b, req, p.peerID) if err != nil { p.picker.unreachable(u) // errMemberRemoved is a critical error since a removed member should // always be stopped. So we use reportCriticalError to report it to errorc. if errors.Is(err, errMemberRemoved) { reportCriticalError(err, p.errorc) } return err } return nil } // waitSchedule waits other goroutines to be scheduled for a while func waitSchedule() { runtime.Gosched() } ================================================ FILE: server/etcdserver/api/rafthttp/pipeline_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "errors" "fmt" "io" "net/http" "sync" "testing" "time" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/types" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/raft/v3/raftpb" ) // TestPipelineSend tests that pipeline could send data using roundtripper // and increase success count in stats. func TestPipelineSend(t *testing.T) { tr := &roundTripperRecorder{rec: testutil.NewRecorderStream()} picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) tp := &Transport{pipelineRt: tr} p := startTestPipeline(t, tp, picker) p.msgc <- raftpb.Message{Type: raftpb.MsgApp} tr.rec.Wait(1) p.stop() if p.followerStats.Counts.Success != 1 { t.Errorf("success = %d, want 1", p.followerStats.Counts.Success) } } // TestPipelineKeepSendingWhenPostError tests that pipeline can keep // sending messages if previous messages meet post error. func TestPipelineKeepSendingWhenPostError(t *testing.T) { tr := &respRoundTripper{rec: testutil.NewRecorderStream(), err: fmt.Errorf("roundtrip error")} picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) tp := &Transport{pipelineRt: tr} p := startTestPipeline(t, tp, picker) defer p.stop() for i := 0; i < 50; i++ { p.msgc <- raftpb.Message{Type: raftpb.MsgApp} } _, err := tr.rec.Wait(50) if err != nil { t.Errorf("unexpected wait error %v", err) } } func TestPipelineExceedMaximumServing(t *testing.T) { rt := newRoundTripperBlocker() picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) tp := &Transport{pipelineRt: rt} p := startTestPipeline(t, tp, picker) defer p.stop() // keep the sender busy and make the buffer full // nothing can go out as we block the sender for i := 0; i < connPerPipeline+pipelineBufSize; i++ { select { case p.msgc <- raftpb.Message{}: case <-time.After(time.Second): t.Errorf("failed to send out message") } } // try to send a data when we are sure the buffer is full select { case p.msgc <- raftpb.Message{}: t.Errorf("unexpected message sendout") default: } // unblock the senders and force them to send out the data rt.unblock() // It could send new data after previous ones succeed select { case p.msgc <- raftpb.Message{}: case <-time.After(time.Second): t.Errorf("failed to send out message") } } // TestPipelineSendFailed tests that when send func meets the post error, // it increases fail count in stats. func TestPipelineSendFailed(t *testing.T) { picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) rt := newRespRoundTripper(0, errors.New("blah")) rt.rec = testutil.NewRecorderStream() tp := &Transport{pipelineRt: rt} p := startTestPipeline(t, tp, picker) p.msgc <- raftpb.Message{Type: raftpb.MsgApp} if _, err := rt.rec.Wait(1); err != nil { t.Fatal(err) } p.stop() if p.followerStats.Counts.Fail != 1 { t.Errorf("fail = %d, want 1", p.followerStats.Counts.Fail) } } func TestPipelinePost(t *testing.T) { tr := &roundTripperRecorder{rec: &testutil.RecorderBuffered{}} picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) tp := &Transport{ClusterID: types.ID(1), pipelineRt: tr} p := startTestPipeline(t, tp, picker) if err := p.post([]byte("some data")); err != nil { t.Fatalf("unexpected post error: %v", err) } act, err := tr.rec.Wait(1) if err != nil { t.Fatal(err) } p.stop() req := act[0].Params[0].(*http.Request) if g := req.Method; g != "POST" { t.Errorf("method = %s, want %s", g, "POST") } if g := req.URL.String(); g != "http://localhost:2380/raft" { t.Errorf("url = %s, want %s", g, "http://localhost:2380/raft") } if g := req.Header.Get("Content-Type"); g != "application/protobuf" { t.Errorf("content type = %s, want %s", g, "application/protobuf") } if g := req.Header.Get("X-Server-Version"); g != version.Version { t.Errorf("version = %s, want %s", g, version.Version) } if g := req.Header.Get("X-Min-Cluster-Version"); g != version.MinClusterVersion { t.Errorf("min version = %s, want %s", g, version.MinClusterVersion) } if g := req.Header.Get("X-Etcd-Cluster-ID"); g != "1" { t.Errorf("cluster id = %s, want %s", g, "1") } b, err := io.ReadAll(req.Body) if err != nil { t.Fatalf("unexpected ReadAll error: %v", err) } if string(b) != "some data" { t.Errorf("body = %s, want %s", b, "some data") } } func TestPipelinePostBad(t *testing.T) { tests := []struct { u string code int err error }{ // RoundTrip returns error {"http://localhost:2380", 0, errors.New("blah")}, // unexpected response status code {"http://localhost:2380", http.StatusOK, nil}, {"http://localhost:2380", http.StatusCreated, nil}, } for i, tt := range tests { picker := mustNewURLPicker(t, []string{tt.u}) tp := &Transport{pipelineRt: newRespRoundTripper(tt.code, tt.err)} p := startTestPipeline(t, tp, picker) err := p.post([]byte("some data")) p.stop() if err == nil { t.Errorf("#%d: err = nil, want not nil", i) } } } func TestPipelinePostErrorc(t *testing.T) { tests := []struct { u string code int err error }{ {"http://localhost:2380", http.StatusForbidden, nil}, } for i, tt := range tests { picker := mustNewURLPicker(t, []string{tt.u}) tp := &Transport{pipelineRt: newRespRoundTripper(tt.code, tt.err)} p := startTestPipeline(t, tp, picker) p.post([]byte("some data")) p.stop() select { case <-p.errorc: default: t.Fatalf("#%d: cannot receive from errorc", i) } } } func TestStopBlockedPipeline(t *testing.T) { picker := mustNewURLPicker(t, []string{"http://localhost:2380"}) tp := &Transport{pipelineRt: newRoundTripperBlocker()} p := startTestPipeline(t, tp, picker) // send many messages that most of them will be blocked in buffer for i := 0; i < connPerPipeline*10; i++ { p.msgc <- raftpb.Message{} } done := make(chan struct{}) go func() { p.stop() done <- struct{}{} }() select { case <-done: case <-time.After(time.Second): t.Fatalf("failed to stop pipeline in 1s") } } type roundTripperBlocker struct { unblockc chan struct{} mu sync.Mutex cancel map[*http.Request]chan struct{} } func newRoundTripperBlocker() *roundTripperBlocker { return &roundTripperBlocker{ unblockc: make(chan struct{}), cancel: make(map[*http.Request]chan struct{}), } } func (t *roundTripperBlocker) unblock() { close(t.unblockc) } func (t *roundTripperBlocker) CancelRequest(req *http.Request) { t.mu.Lock() defer t.mu.Unlock() if c, ok := t.cancel[req]; ok { c <- struct{}{} delete(t.cancel, req) } } type respRoundTripper struct { mu sync.Mutex rec testutil.Recorder code int header http.Header err error } func newRespRoundTripper(code int, err error) *respRoundTripper { return &respRoundTripper{code: code, err: err} } func (t *respRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { t.mu.Lock() defer t.mu.Unlock() if t.rec != nil { t.rec.Record(testutil.Action{Name: "req", Params: []any{req}}) } return &http.Response{StatusCode: t.code, Header: t.header, Body: &nopReadCloser{}}, t.err } type roundTripperRecorder struct { rec testutil.Recorder } func (t *roundTripperRecorder) RoundTrip(req *http.Request) (*http.Response, error) { if t.rec != nil { t.rec.Record(testutil.Action{Name: "req", Params: []any{req}}) } return &http.Response{StatusCode: http.StatusNoContent, Body: &nopReadCloser{}}, nil } type nopReadCloser struct{} func (n *nopReadCloser) Read(p []byte) (int, error) { return 0, io.EOF } func (n *nopReadCloser) Close() error { return nil } func startTestPipeline(t *testing.T, tr *Transport, picker *urlPicker) *pipeline { p := &pipeline{ peerID: types.ID(1), tr: tr, picker: picker, status: newPeerStatus(zaptest.NewLogger(t), tr.ID, types.ID(1)), raft: &fakeRaft{}, followerStats: &stats.FollowerStats{}, errorc: make(chan error, 1), } p.start() return p } ================================================ FILE: server/etcdserver/api/rafthttp/probing_status.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "time" "github.com/prometheus/client_golang/prometheus" "github.com/xiang90/probing" "go.uber.org/zap" ) const ( // RoundTripperNameRaftMessage is the name of round-tripper that sends // all other Raft messages, other than "snap.Message". RoundTripperNameRaftMessage = "ROUND_TRIPPER_RAFT_MESSAGE" // RoundTripperNameSnapshot is the name of round-tripper that sends merged snapshot message. RoundTripperNameSnapshot = "ROUND_TRIPPER_SNAPSHOT" ) var ( // proberInterval must be shorter than read timeout. // Or the connection will time-out. proberInterval = ConnReadTimeout - time.Second statusMonitoringInterval = 30 * time.Second statusErrorInterval = 5 * time.Second ) func addPeerToProber(lg *zap.Logger, p probing.Prober, id string, us []string, roundTripperName string, rttSecProm *prometheus.HistogramVec) { hus := make([]string, len(us)) for i := range us { hus[i] = us[i] + ProbingPrefix } p.AddHTTP(id, proberInterval, hus) s, err := p.Status(id) if err != nil { if lg != nil { lg.Warn("failed to add peer into prober", zap.String("remote-peer-id", id), zap.Error(err)) } return } go monitorProbingStatus(lg, s, id, roundTripperName, rttSecProm) } func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string, roundTripperName string, rttSecProm *prometheus.HistogramVec) { // set the first interval short to log error early. interval := statusErrorInterval for { select { case <-time.After(interval): if !s.Health() { if lg != nil { lg.Warn( "prober detected unhealthy status", zap.String("round-tripper-name", roundTripperName), zap.String("remote-peer-id", id), zap.Duration("rtt", s.SRTT()), zap.Error(s.Err()), ) } interval = statusErrorInterval } else { interval = statusMonitoringInterval } if s.ClockDiff() > time.Second { if lg != nil { lg.Warn( "prober found high clock drift", zap.String("round-tripper-name", roundTripperName), zap.String("remote-peer-id", id), zap.Duration("clock-drift", s.ClockDiff()), zap.Duration("rtt", s.SRTT()), zap.Error(s.Err()), ) } } rttSecProm.WithLabelValues(id).Observe(s.SRTT().Seconds()) case <-s.StopNotify(): return } } } ================================================ FILE: server/etcdserver/api/rafthttp/remote.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/raft/v3/raftpb" ) type remote struct { lg *zap.Logger localID types.ID id types.ID status *peerStatus pipeline *pipeline } func startRemote(tr *Transport, urls types.URLs, id types.ID) *remote { picker := newURLPicker(urls) status := newPeerStatus(tr.Logger, tr.ID, id) pipeline := &pipeline{ peerID: id, tr: tr, picker: picker, status: status, raft: tr.Raft, errorc: tr.ErrorC, } pipeline.start() return &remote{ lg: tr.Logger, localID: tr.ID, id: id, status: status, pipeline: pipeline, } } func (g *remote) send(m raftpb.Message) { select { case g.pipeline.msgc <- m: default: if g.status.isActive() { if g.lg != nil { g.lg.Warn( "dropped internal Raft message since sending buffer is full (overloaded network)", zap.String("message-type", m.Type.String()), zap.String("local-member-id", g.localID.String()), zap.String("from", types.ID(m.From).String()), zap.String("remote-peer-id", g.id.String()), zap.Bool("remote-peer-active", g.status.isActive()), ) } } else { if g.lg != nil { g.lg.Warn( "dropped Raft message since sending buffer is full (overloaded network)", zap.String("message-type", m.Type.String()), zap.String("local-member-id", g.localID.String()), zap.String("from", types.ID(m.From).String()), zap.String("remote-peer-id", g.id.String()), zap.Bool("remote-peer-active", g.status.isActive()), ) } } sentFailures.WithLabelValues(types.ID(m.To).String()).Inc() } } func (g *remote) stop() { g.pipeline.stop() } func (g *remote) Pause() { g.stop() } func (g *remote) Resume() { g.pipeline.start() } ================================================ FILE: server/etcdserver/api/rafthttp/snapshot_sender.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "bytes" "context" "errors" "io" "net/http" "time" "github.com/dustin/go-humanize" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/httputil" pioutil "go.etcd.io/etcd/pkg/v3/ioutil" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/raft/v3" ) // timeout for reading snapshot response body var snapResponseReadTimeout = 5 * time.Second type snapshotSender struct { from, to types.ID cid types.ID tr *Transport picker *urlPicker status *peerStatus r Raft errorc chan error stopc chan struct{} } func newSnapshotSender(tr *Transport, picker *urlPicker, to types.ID, status *peerStatus) *snapshotSender { return &snapshotSender{ from: tr.ID, to: to, cid: tr.ClusterID, tr: tr, picker: picker, status: status, r: tr.Raft, errorc: tr.ErrorC, stopc: make(chan struct{}), } } func (s *snapshotSender) stop() { close(s.stopc) } func (s *snapshotSender) send(merged snap.Message) { start := time.Now() m := merged.Message to := types.ID(m.To).String() body := createSnapBody(s.tr.Logger, merged) defer body.Close() u := s.picker.pick() req := createPostRequest(s.tr.Logger, u, RaftSnapshotPrefix, body, "application/octet-stream", s.tr.URLs, s.from, s.cid) snapshotSizeVal := uint64(merged.TotalSize) snapshotSize := humanize.Bytes(snapshotSizeVal) if s.tr.Logger != nil { s.tr.Logger.Info( "sending database snapshot", zap.Uint64("snapshot-index", m.Snapshot.Metadata.Index), zap.String("remote-peer-id", to), zap.Uint64("bytes", snapshotSizeVal), zap.String("size", snapshotSize), ) } snapshotSendInflights.WithLabelValues(to).Inc() defer func() { snapshotSendInflights.WithLabelValues(to).Dec() }() err := s.post(req) defer merged.CloseWithError(err) if err != nil { if s.tr.Logger != nil { s.tr.Logger.Warn( "failed to send database snapshot", zap.Uint64("snapshot-index", m.Snapshot.Metadata.Index), zap.String("remote-peer-id", to), zap.Uint64("bytes", snapshotSizeVal), zap.String("size", snapshotSize), zap.Error(err), ) } // errMemberRemoved is a critical error since a removed member should // always be stopped. So we use reportCriticalError to report it to errorc. if errors.Is(err, errMemberRemoved) { reportCriticalError(err, s.errorc) } s.picker.unreachable(u) s.status.deactivate(failureType{source: sendSnap, action: "post"}, err.Error()) s.r.ReportUnreachable(m.To) // report SnapshotFailure to raft state machine. After raft state // machine knows about it, it would pause a while and retry sending // new snapshot message. s.r.ReportSnapshot(m.To, raft.SnapshotFailure) sentFailures.WithLabelValues(to).Inc() snapshotSendFailures.WithLabelValues(to).Inc() return } s.status.activate() s.r.ReportSnapshot(m.To, raft.SnapshotFinish) if s.tr.Logger != nil { s.tr.Logger.Info( "sent database snapshot", zap.Uint64("snapshot-index", m.Snapshot.Metadata.Index), zap.String("remote-peer-id", to), zap.Uint64("bytes", snapshotSizeVal), zap.String("size", snapshotSize), ) } sentBytes.WithLabelValues(to).Add(float64(merged.TotalSize)) snapshotSend.WithLabelValues(to).Inc() snapshotSendSeconds.WithLabelValues(to).Observe(time.Since(start).Seconds()) } // post posts the given request. // It returns nil when request is sent out and processed successfully. func (s *snapshotSender) post(req *http.Request) (err error) { ctx, cancel := context.WithCancel(context.Background()) req = req.WithContext(ctx) defer cancel() type responseAndError struct { resp *http.Response body []byte err error } result := make(chan responseAndError, 1) go func() { resp, err := s.tr.pipelineRt.RoundTrip(req) if err != nil { result <- responseAndError{resp, nil, err} return } // close the response body when timeouts. // prevents from reading the body forever when the other side dies right after // successfully receives the request body. time.AfterFunc(snapResponseReadTimeout, func() { httputil.GracefulClose(resp) }) body, err := io.ReadAll(resp.Body) result <- responseAndError{resp, body, err} }() select { case <-s.stopc: return errStopped case r := <-result: if r.err != nil { return r.err } return checkPostResponse(s.tr.Logger, r.resp, r.body, req, s.to) } } func createSnapBody(lg *zap.Logger, merged snap.Message) io.ReadCloser { buf := new(bytes.Buffer) enc := &messageEncoder{w: buf} // encode raft message if err := enc.encode(&merged.Message); err != nil { if lg != nil { lg.Panic("failed to encode message", zap.Error(err)) } } return &pioutil.ReaderAndCloser{ Reader: io.MultiReader(buf, merged.ReadCloser), Closer: merged.ReadCloser, } } ================================================ FILE: server/etcdserver/api/rafthttp/snapshot_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "fmt" "io" "net/http" "net/http/httptest" "os" "strings" "testing" "time" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/raft/v3/raftpb" ) type strReaderCloser struct{ *strings.Reader } func (s strReaderCloser) Close() error { return nil } func TestSnapshotSend(t *testing.T) { tests := []struct { m raftpb.Message rc io.ReadCloser size int64 wsent bool wfiles int }{ // sent and receive with no errors { m: raftpb.Message{Type: raftpb.MsgSnap, To: 1, Snapshot: &raftpb.Snapshot{}}, rc: strReaderCloser{strings.NewReader("hello")}, size: 5, wsent: true, wfiles: 1, }, // error when reading snapshot for send { m: raftpb.Message{Type: raftpb.MsgSnap, To: 1, Snapshot: &raftpb.Snapshot{}}, rc: &errReadCloser{fmt.Errorf("snapshot error")}, size: 1, wsent: false, wfiles: 0, }, // sends less than the given snapshot length { m: raftpb.Message{Type: raftpb.MsgSnap, To: 1, Snapshot: &raftpb.Snapshot{}}, rc: strReaderCloser{strings.NewReader("hello")}, size: 10000, wsent: false, wfiles: 0, }, // sends less than actual snapshot length { m: raftpb.Message{Type: raftpb.MsgSnap, To: 1, Snapshot: &raftpb.Snapshot{}}, rc: strReaderCloser{strings.NewReader("hello")}, size: 1, wsent: false, wfiles: 0, }, } for i, tt := range tests { sent, files := testSnapshotSend(t, snap.NewMessage(tt.m, tt.rc, tt.size)) if tt.wsent != sent { t.Errorf("#%d: snapshot expected %v, got %v", i, tt.wsent, sent) } if tt.wfiles != len(files) { t.Fatalf("#%d: expected %d files, got %d files", i, tt.wfiles, len(files)) } } } func testSnapshotSend(t *testing.T, sm *snap.Message) (bool, []os.DirEntry) { d := t.TempDir() r := &fakeRaft{} tr := &Transport{pipelineRt: &http.Transport{}, ClusterID: types.ID(1), Raft: r} ch := make(chan struct{}, 1) h := &syncHandler{newSnapshotHandler(tr, r, snap.New(zaptest.NewLogger(t), d), types.ID(1)), ch} srv := httptest.NewServer(h) defer srv.Close() picker := mustNewURLPicker(t, []string{srv.URL}) snapsend := newSnapshotSender(tr, picker, types.ID(1), newPeerStatus(zaptest.NewLogger(t), types.ID(0), types.ID(1))) defer snapsend.stop() snapsend.send(*sm) sent := false select { case <-time.After(time.Second): t.Fatalf("timed out sending snapshot") case sent = <-sm.CloseNotify(): } // wait for handler to finish accepting snapshot <-ch files, rerr := os.ReadDir(d) if rerr != nil { t.Fatal(rerr) } return sent, files } type errReadCloser struct{ err error } func (s *errReadCloser) Read(p []byte) (int, error) { return 0, s.err } func (s *errReadCloser) Close() error { return s.err } type syncHandler struct { h http.Handler ch chan<- struct{} } func (sh *syncHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { sh.h.ServeHTTP(w, r) sh.ch <- struct{}{} } ================================================ FILE: server/etcdserver/api/rafthttp/stream.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "context" "errors" "fmt" "io" "net/http" "path" "strings" "sync" "time" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "golang.org/x/time/rate" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/httputil" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/raft/v3/raftpb" ) const ( streamTypeMessage streamType = "message" streamTypeMsgAppV2 streamType = "msgappv2" streamBufSize = 4096 ) var ( errUnsupportedStreamType = fmt.Errorf("unsupported stream type") // the key is in string format "major.minor.patch" supportedStream = map[string][]streamType{ "2.0.0": {}, "2.1.0": {streamTypeMsgAppV2, streamTypeMessage}, "2.2.0": {streamTypeMsgAppV2, streamTypeMessage}, "2.3.0": {streamTypeMsgAppV2, streamTypeMessage}, "3.0.0": {streamTypeMsgAppV2, streamTypeMessage}, "3.1.0": {streamTypeMsgAppV2, streamTypeMessage}, "3.2.0": {streamTypeMsgAppV2, streamTypeMessage}, "3.3.0": {streamTypeMsgAppV2, streamTypeMessage}, "3.4.0": {streamTypeMsgAppV2, streamTypeMessage}, "3.5.0": {streamTypeMsgAppV2, streamTypeMessage}, "3.6.0": {streamTypeMsgAppV2, streamTypeMessage}, "3.7.0": {streamTypeMsgAppV2, streamTypeMessage}, } ) type streamType string func (t streamType) endpoint(lg *zap.Logger) string { switch t { case streamTypeMsgAppV2: return path.Join(RaftStreamPrefix, "msgapp") case streamTypeMessage: return path.Join(RaftStreamPrefix, "message") default: if lg != nil { lg.Panic("unhandled stream type", zap.String("stream-type", t.String())) } return "" } } func (t streamType) String() string { switch t { case streamTypeMsgAppV2: return "stream MsgApp v2" case streamTypeMessage: return "stream Message" default: return "unknown stream" } } // linkHeartbeatMessage is a special message used as heartbeat message in // link layer. It never conflicts with messages from raft because raft // doesn't send out messages without From and To fields. var linkHeartbeatMessage = raftpb.Message{Type: raftpb.MsgHeartbeat} func isLinkHeartbeatMessage(m *raftpb.Message) bool { return m.Type == raftpb.MsgHeartbeat && m.From == 0 && m.To == 0 } type outgoingConn struct { t streamType io.Writer http.Flusher io.Closer localID types.ID peerID types.ID } // streamWriter writes messages to the attached outgoingConn. type streamWriter struct { lg *zap.Logger localID types.ID peerID types.ID status *peerStatus fs *stats.FollowerStats r Raft mu sync.Mutex // guard field working and closer closer io.Closer working bool msgc chan raftpb.Message connc chan *outgoingConn stopc chan struct{} done chan struct{} } // startStreamWriter creates a streamWrite and starts a long running go-routine that accepts // messages and writes to the attached outgoing connection. func startStreamWriter(lg *zap.Logger, local, id types.ID, status *peerStatus, fs *stats.FollowerStats, r Raft) *streamWriter { w := &streamWriter{ lg: lg, localID: local, peerID: id, status: status, fs: fs, r: r, msgc: make(chan raftpb.Message, streamBufSize), connc: make(chan *outgoingConn), stopc: make(chan struct{}), done: make(chan struct{}), } go w.run() return w } func (cw *streamWriter) run() { var ( msgc chan raftpb.Message heartbeatc <-chan time.Time t streamType enc encoder flusher http.Flusher batched int ) tickc := time.NewTicker(ConnReadTimeout / 3) defer tickc.Stop() unflushed := 0 if cw.lg != nil { cw.lg.Info( "started stream writer with remote peer", zap.String("local-member-id", cw.localID.String()), zap.String("remote-peer-id", cw.peerID.String()), ) } for { select { case <-heartbeatc: err := enc.encode(&linkHeartbeatMessage) unflushed += linkHeartbeatMessage.Size() if err == nil { flusher.Flush() batched = 0 sentBytes.WithLabelValues(cw.peerID.String()).Add(float64(unflushed)) unflushed = 0 continue } cw.status.deactivate(failureType{source: t.String(), action: "heartbeat"}, err.Error()) sentFailures.WithLabelValues(cw.peerID.String()).Inc() cw.close() if cw.lg != nil { cw.lg.Warn( "lost TCP streaming connection with remote peer", zap.String("stream-writer-type", t.String()), zap.String("local-member-id", cw.localID.String()), zap.String("remote-peer-id", cw.peerID.String()), ) } heartbeatc, msgc = nil, nil case m := <-msgc: err := enc.encode(&m) if err == nil { unflushed += m.Size() if len(msgc) == 0 || batched > streamBufSize/2 { flusher.Flush() sentBytes.WithLabelValues(cw.peerID.String()).Add(float64(unflushed)) unflushed = 0 batched = 0 } else { batched++ } continue } cw.status.deactivate(failureType{source: t.String(), action: "write"}, err.Error()) cw.close() if cw.lg != nil { cw.lg.Warn( "lost TCP streaming connection with remote peer", zap.String("stream-writer-type", t.String()), zap.String("local-member-id", cw.localID.String()), zap.String("remote-peer-id", cw.peerID.String()), ) } heartbeatc, msgc = nil, nil cw.r.ReportUnreachable(m.To) sentFailures.WithLabelValues(cw.peerID.String()).Inc() case conn := <-cw.connc: cw.mu.Lock() closed := cw.closeUnlocked() t = conn.t switch conn.t { case streamTypeMsgAppV2: enc = newMsgAppV2Encoder(conn.Writer, cw.fs) case streamTypeMessage: enc = &messageEncoder{w: conn.Writer} default: if cw.lg != nil { cw.lg.Panic("unhandled stream type", zap.String("stream-type", t.String())) } } if cw.lg != nil { cw.lg.Info( "set message encoder", zap.String("from", conn.localID.String()), zap.String("to", conn.peerID.String()), zap.String("stream-type", t.String()), ) } flusher = conn.Flusher unflushed = 0 cw.status.activate() cw.closer = conn.Closer cw.working = true cw.mu.Unlock() if closed { if cw.lg != nil { cw.lg.Warn( "closed TCP streaming connection with remote peer", zap.String("stream-writer-type", t.String()), zap.String("local-member-id", cw.localID.String()), zap.String("remote-peer-id", cw.peerID.String()), ) } } if cw.lg != nil { cw.lg.Info( "established TCP streaming connection with remote peer", zap.String("stream-writer-type", t.String()), zap.String("local-member-id", cw.localID.String()), zap.String("remote-peer-id", cw.peerID.String()), ) } heartbeatc, msgc = tickc.C, cw.msgc case <-cw.stopc: if cw.close() { if cw.lg != nil { cw.lg.Warn( "closed TCP streaming connection with remote peer", zap.String("stream-writer-type", t.String()), zap.String("remote-peer-id", cw.peerID.String()), ) } } if cw.lg != nil { cw.lg.Info( "stopped TCP streaming connection with remote peer", zap.String("stream-writer-type", t.String()), zap.String("remote-peer-id", cw.peerID.String()), ) } close(cw.done) return } } } func (cw *streamWriter) writec() (chan<- raftpb.Message, bool) { cw.mu.Lock() defer cw.mu.Unlock() return cw.msgc, cw.working } func (cw *streamWriter) close() bool { cw.mu.Lock() defer cw.mu.Unlock() return cw.closeUnlocked() } func (cw *streamWriter) closeUnlocked() bool { if !cw.working { return false } if err := cw.closer.Close(); err != nil { if cw.lg != nil { cw.lg.Warn( "failed to close connection with remote peer", zap.String("remote-peer-id", cw.peerID.String()), zap.Error(err), ) } } if len(cw.msgc) > 0 { cw.r.ReportUnreachable(uint64(cw.peerID)) } cw.msgc = make(chan raftpb.Message, streamBufSize) cw.working = false return true } func (cw *streamWriter) attach(conn *outgoingConn) bool { select { case cw.connc <- conn: return true case <-cw.done: return false } } func (cw *streamWriter) stop() { close(cw.stopc) <-cw.done } // streamReader is a long-running go-routine that dials to the remote stream // endpoint and reads messages from the response body returned. type streamReader struct { lg *zap.Logger peerID types.ID typ streamType tr *Transport picker *urlPicker status *peerStatus recvc chan<- raftpb.Message propc chan<- raftpb.Message rl *rate.Limiter // alters the frequency of dial retrial attempts errorc chan<- error mu sync.Mutex paused bool closer io.Closer ctx context.Context cancel context.CancelFunc done chan struct{} } func (cr *streamReader) start() { cr.done = make(chan struct{}) if cr.errorc == nil { cr.errorc = cr.tr.ErrorC } if cr.ctx == nil { cr.ctx, cr.cancel = context.WithCancel(context.Background()) } go cr.run() } func (cr *streamReader) run() { t := cr.typ if cr.lg != nil { cr.lg.Info( "started stream reader with remote peer", zap.String("stream-reader-type", t.String()), zap.String("local-member-id", cr.tr.ID.String()), zap.String("remote-peer-id", cr.peerID.String()), ) } for { rc, err := cr.dial(t) if err != nil { if !errors.Is(err, errUnsupportedStreamType) { cr.status.deactivate(failureType{source: t.String(), action: "dial"}, err.Error()) } } else { cr.status.activate() if cr.lg != nil { cr.lg.Info( "established TCP streaming connection with remote peer", zap.String("stream-reader-type", cr.typ.String()), zap.String("local-member-id", cr.tr.ID.String()), zap.String("remote-peer-id", cr.peerID.String()), ) } err = cr.decodeLoop(rc, t) if cr.lg != nil { cr.lg.Warn( "lost TCP streaming connection with remote peer", zap.String("stream-reader-type", cr.typ.String()), zap.String("local-member-id", cr.tr.ID.String()), zap.String("remote-peer-id", cr.peerID.String()), zap.Error(err), ) } switch { // all data is read out case errors.Is(err, io.EOF): // connection is closed by the remote case transport.IsClosedConnError(err): default: cr.status.deactivate(failureType{source: t.String(), action: "read"}, err.Error()) } } // Wait for a while before new dial attempt err = cr.rl.Wait(cr.ctx) if cr.ctx.Err() != nil { if cr.lg != nil { cr.lg.Info( "stopped stream reader with remote peer", zap.String("stream-reader-type", t.String()), zap.String("local-member-id", cr.tr.ID.String()), zap.String("remote-peer-id", cr.peerID.String()), ) } close(cr.done) return } if err != nil { if cr.lg != nil { cr.lg.Warn( "rate limit on stream reader with remote peer", zap.String("stream-reader-type", t.String()), zap.String("local-member-id", cr.tr.ID.String()), zap.String("remote-peer-id", cr.peerID.String()), zap.Error(err), ) } } } } func (cr *streamReader) decodeLoop(rc io.ReadCloser, t streamType) error { var dec decoder cr.mu.Lock() switch t { case streamTypeMsgAppV2: dec = newMsgAppV2Decoder(rc, cr.tr.ID, cr.peerID) case streamTypeMessage: dec = &messageDecoder{r: rc} default: if cr.lg != nil { cr.lg.Panic("unknown stream type", zap.String("type", t.String())) } } select { case <-cr.ctx.Done(): cr.mu.Unlock() if err := rc.Close(); err != nil { return err } return io.EOF default: cr.closer = rc } cr.mu.Unlock() // gofail: labelRaftDropHeartbeat: for { m, err := dec.decode() if err != nil { cr.mu.Lock() cr.close() cr.mu.Unlock() return err } // gofail: var raftDropHeartbeat struct{} // continue labelRaftDropHeartbeat receivedBytes.WithLabelValues(types.ID(m.From).String()).Add(float64(m.Size())) cr.mu.Lock() paused := cr.paused cr.mu.Unlock() if paused { continue } if isLinkHeartbeatMessage(&m) { // raft is not interested in link layer // heartbeat message, so we should ignore // it. continue } recvc := cr.recvc if m.Type == raftpb.MsgProp { recvc = cr.propc } select { case recvc <- m: default: if cr.status.isActive() { if cr.lg != nil { cr.lg.Warn( "dropped internal Raft message since receiving buffer is full (overloaded network)", zap.String("message-type", m.Type.String()), zap.String("local-member-id", cr.tr.ID.String()), zap.String("from", types.ID(m.From).String()), zap.String("remote-peer-id", types.ID(m.To).String()), zap.Bool("remote-peer-active", cr.status.isActive()), ) } } else { if cr.lg != nil { cr.lg.Warn( "dropped Raft message since receiving buffer is full (overloaded network)", zap.String("message-type", m.Type.String()), zap.String("local-member-id", cr.tr.ID.String()), zap.String("from", types.ID(m.From).String()), zap.String("remote-peer-id", types.ID(m.To).String()), zap.Bool("remote-peer-active", cr.status.isActive()), ) } } recvFailures.WithLabelValues(types.ID(m.From).String()).Inc() } } } func (cr *streamReader) stop() { cr.mu.Lock() cr.cancel() cr.close() cr.mu.Unlock() <-cr.done } func (cr *streamReader) dial(t streamType) (io.ReadCloser, error) { u := cr.picker.pick() uu := u uu.Path = path.Join(t.endpoint(cr.lg), cr.tr.ID.String()) if cr.lg != nil { cr.lg.Debug( "dial stream reader", zap.String("from", cr.tr.ID.String()), zap.String("to", cr.peerID.String()), zap.String("address", uu.String()), ) } req, err := http.NewRequest(http.MethodGet, uu.String(), nil) if err != nil { cr.picker.unreachable(u) return nil, fmt.Errorf("failed to make http request to %v (%w)", u, err) } req.Header.Set("X-Server-From", cr.tr.ID.String()) req.Header.Set("X-Server-Version", version.Version) req.Header.Set("X-Min-Cluster-Version", version.MinClusterVersion) req.Header.Set("X-Etcd-Cluster-ID", cr.tr.ClusterID.String()) req.Header.Set("X-Raft-To", cr.peerID.String()) setPeerURLsHeader(req, cr.tr.URLs) req = req.WithContext(cr.ctx) cr.mu.Lock() select { case <-cr.ctx.Done(): cr.mu.Unlock() return nil, fmt.Errorf("stream reader is stopped") default: } cr.mu.Unlock() resp, err := cr.tr.streamRt.RoundTrip(req) if err != nil { cr.picker.unreachable(u) return nil, err } rv := serverVersion(resp.Header) lv := semver.Must(semver.NewVersion(version.Version)) if compareMajorMinorVersion(rv, lv) == -1 && !checkStreamSupport(rv, t) { httputil.GracefulClose(resp) cr.picker.unreachable(u) return nil, errUnsupportedStreamType } switch resp.StatusCode { case http.StatusGone: httputil.GracefulClose(resp) cr.picker.unreachable(u) reportCriticalError(errMemberRemoved, cr.errorc) return nil, errMemberRemoved case http.StatusOK: return resp.Body, nil case http.StatusNotFound: httputil.GracefulClose(resp) cr.picker.unreachable(u) return nil, fmt.Errorf("peer %s failed to find local node %s", cr.peerID, cr.tr.ID) case http.StatusPreconditionFailed: b, err := io.ReadAll(resp.Body) if err != nil { cr.picker.unreachable(u) return nil, err } httputil.GracefulClose(resp) cr.picker.unreachable(u) switch strings.TrimSuffix(string(b), "\n") { case errIncompatibleVersion.Error(): if cr.lg != nil { cr.lg.Warn( "request sent was ignored by remote peer due to server version incompatibility", zap.String("local-member-id", cr.tr.ID.String()), zap.String("remote-peer-id", cr.peerID.String()), zap.Error(errIncompatibleVersion), ) } return nil, errIncompatibleVersion case ErrClusterIDMismatch.Error(): if cr.lg != nil { cr.lg.Warn( "request sent was ignored by remote peer due to cluster ID mismatch", zap.String("remote-peer-id", cr.peerID.String()), zap.String("remote-peer-cluster-id", resp.Header.Get("X-Etcd-Cluster-ID")), zap.String("local-member-id", cr.tr.ID.String()), zap.String("local-member-cluster-id", cr.tr.ClusterID.String()), zap.Error(ErrClusterIDMismatch), ) } return nil, ErrClusterIDMismatch default: return nil, fmt.Errorf("unhandled error %q when precondition failed", string(b)) } default: httputil.GracefulClose(resp) cr.picker.unreachable(u) return nil, fmt.Errorf("unhandled http status %d", resp.StatusCode) } } func (cr *streamReader) close() { if cr.closer != nil { if err := cr.closer.Close(); err != nil { if cr.lg != nil { cr.lg.Warn( "failed to close remote peer connection", zap.String("local-member-id", cr.tr.ID.String()), zap.String("remote-peer-id", cr.peerID.String()), zap.Error(err), ) } } } cr.closer = nil } func (cr *streamReader) pause() { cr.mu.Lock() defer cr.mu.Unlock() cr.paused = true } func (cr *streamReader) resume() { cr.mu.Lock() defer cr.mu.Unlock() cr.paused = false } // checkStreamSupport checks whether the stream type is supported in the // given version. func checkStreamSupport(v *semver.Version, t streamType) bool { nv := &semver.Version{Major: v.Major, Minor: v.Minor} for _, s := range supportedStream[nv.String()] { if s == t { return true } } return false } ================================================ FILE: server/etcdserver/api/rafthttp/stream_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "errors" "io" "net/http" "net/http/httptest" "reflect" "sync" "testing" "time" "github.com/coreos/go-semver/semver" "go.uber.org/zap/zaptest" "golang.org/x/time/rate" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/types" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/raft/v3/raftpb" ) // TestStreamWriterAttachOutgoingConn tests that outgoingConn can be attached // to streamWriter. After that, streamWriter can use it to send messages // continuously, and closes it when stopped. func TestStreamWriterAttachOutgoingConn(t *testing.T) { sw := startStreamWriter(zaptest.NewLogger(t), types.ID(0), types.ID(1), newPeerStatus(zaptest.NewLogger(t), types.ID(0), types.ID(1)), &stats.FollowerStats{}, &fakeRaft{}) // the expected initial state of streamWriter is not working if _, ok := sw.writec(); ok { t.Errorf("initial working status = %v, want false", ok) } // repeat tests to ensure streamWriter can use last attached connection var wfc *fakeWriteFlushCloser for i := 0; i < 3; i++ { prevwfc := wfc wfc = newFakeWriteFlushCloser(nil) sw.attach(&outgoingConn{t: streamTypeMessage, Writer: wfc, Flusher: wfc, Closer: wfc}) // previous attached connection should be closed if prevwfc != nil { select { case <-prevwfc.closed: case <-time.After(time.Second): t.Errorf("#%d: close of previous connection timed out", i) } } // if prevwfc != nil, the new msgc is ready since prevwfc has closed // if prevwfc == nil, the first connection may be pending, but the first // msgc is already available since it's set on calling startStreamwriter msgc, _ := sw.writec() msgc <- raftpb.Message{} select { case <-wfc.writec: case <-time.After(time.Second): t.Errorf("#%d: failed to write to the underlying connection", i) } // write chan is still available if _, ok := sw.writec(); !ok { t.Errorf("#%d: working status = %v, want true", i, ok) } } sw.stop() // write chan is unavailable since the writer is stopped. if _, ok := sw.writec(); ok { t.Errorf("working status after stop = %v, want false", ok) } if !wfc.Closed() { t.Errorf("failed to close the underlying connection") } } // TestStreamWriterAttachBadOutgoingConn tests that streamWriter with bad // outgoingConn will close the outgoingConn and fall back to non-working status. func TestStreamWriterAttachBadOutgoingConn(t *testing.T) { sw := startStreamWriter(zaptest.NewLogger(t), types.ID(0), types.ID(1), newPeerStatus(zaptest.NewLogger(t), types.ID(0), types.ID(1)), &stats.FollowerStats{}, &fakeRaft{}) defer sw.stop() wfc := newFakeWriteFlushCloser(errors.New("blah")) sw.attach(&outgoingConn{t: streamTypeMessage, Writer: wfc, Flusher: wfc, Closer: wfc}) sw.msgc <- raftpb.Message{} select { case <-wfc.closed: case <-time.After(time.Second): t.Errorf("failed to close the underlying connection in time") } // no longer working if _, ok := sw.writec(); ok { t.Errorf("working = %v, want false", ok) } } func TestStreamReaderDialRequest(t *testing.T) { for i, tt := range []streamType{streamTypeMessage, streamTypeMsgAppV2} { tr := &roundTripperRecorder{rec: &testutil.RecorderBuffered{}} sr := &streamReader{ peerID: types.ID(2), tr: &Transport{streamRt: tr, ClusterID: types.ID(1), ID: types.ID(1)}, picker: mustNewURLPicker(t, []string{"http://localhost:2380"}), ctx: t.Context(), } sr.dial(tt) act, err := tr.rec.Wait(1) if err != nil { t.Fatal(err) } req := act[0].Params[0].(*http.Request) wurl := "http://localhost:2380" + tt.endpoint(zaptest.NewLogger(t)) + "/1" if req.URL.String() != wurl { t.Errorf("#%d: url = %s, want %s", i, req.URL.String(), wurl) } if w := "GET"; req.Method != w { t.Errorf("#%d: method = %s, want %s", i, req.Method, w) } if g := req.Header.Get("X-Etcd-Cluster-ID"); g != "1" { t.Errorf("#%d: header X-Etcd-Cluster-ID = %s, want 1", i, g) } if g := req.Header.Get("X-Raft-To"); g != "2" { t.Errorf("#%d: header X-Raft-To = %s, want 2", i, g) } } } // TestStreamReaderDialResult tests the result of the dial func call meets the // HTTP response received. func TestStreamReaderDialResult(t *testing.T) { tests := []struct { code int err error wok bool whalt bool }{ {0, errors.New("blah"), false, false}, {http.StatusOK, nil, true, false}, {http.StatusMethodNotAllowed, nil, false, false}, {http.StatusNotFound, nil, false, false}, {http.StatusPreconditionFailed, nil, false, false}, {http.StatusGone, nil, false, true}, } for i, tt := range tests { h := http.Header{} h.Add("X-Server-Version", version.Version) tr := &respRoundTripper{ code: tt.code, header: h, err: tt.err, } sr := &streamReader{ peerID: types.ID(2), tr: &Transport{streamRt: tr, ClusterID: types.ID(1)}, picker: mustNewURLPicker(t, []string{"http://localhost:2380"}), errorc: make(chan error, 1), ctx: t.Context(), } _, err := sr.dial(streamTypeMessage) if ok := err == nil; ok != tt.wok { t.Errorf("#%d: ok = %v, want %v", i, ok, tt.wok) } if halt := len(sr.errorc) > 0; halt != tt.whalt { t.Errorf("#%d: halt = %v, want %v", i, halt, tt.whalt) } } } // TestStreamReaderStopOnDial tests a stream reader closes the connection on stop. func TestStreamReaderStopOnDial(t *testing.T) { testutil.RegisterLeakDetection(t) h := http.Header{} h.Add("X-Server-Version", version.Version) tr := &respWaitRoundTripper{rrt: &respRoundTripper{code: http.StatusOK, header: h}} sr := &streamReader{ peerID: types.ID(2), tr: &Transport{streamRt: tr, ClusterID: types.ID(1)}, picker: mustNewURLPicker(t, []string{"http://localhost:2380"}), errorc: make(chan error, 1), typ: streamTypeMessage, status: newPeerStatus(zaptest.NewLogger(t), types.ID(1), types.ID(2)), rl: rate.NewLimiter(rate.Every(100*time.Millisecond), 1), } tr.onResp = func() { // stop() waits for the run() goroutine to exit, but that exit // needs a response from RoundTrip() first; use goroutine go sr.stop() // wait so that stop() is blocked on run() exiting time.Sleep(10 * time.Millisecond) // sr.run() completes dialing then begins decoding while stopped } sr.start() select { case <-sr.done: case <-time.After(time.Second): t.Fatal("streamReader did not stop in time") } } type respWaitRoundTripper struct { rrt *respRoundTripper onResp func() } func (t *respWaitRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { resp, err := t.rrt.RoundTrip(req) resp.Body = newWaitReadCloser() t.onResp() return resp, err } type waitReadCloser struct{ closec chan struct{} } func newWaitReadCloser() *waitReadCloser { return &waitReadCloser{make(chan struct{})} } func (wrc *waitReadCloser) Read(p []byte) (int, error) { <-wrc.closec return 0, io.EOF } func (wrc *waitReadCloser) Close() error { close(wrc.closec) return nil } // TestStreamReaderDialDetectUnsupport tests that dial func could find // out that the stream type is not supported by the remote. func TestStreamReaderDialDetectUnsupport(t *testing.T) { for i, typ := range []streamType{streamTypeMsgAppV2, streamTypeMessage} { // the response from etcd 2.0 tr := &respRoundTripper{ code: http.StatusNotFound, header: http.Header{}, } sr := &streamReader{ peerID: types.ID(2), tr: &Transport{streamRt: tr, ClusterID: types.ID(1)}, picker: mustNewURLPicker(t, []string{"http://localhost:2380"}), ctx: t.Context(), } _, err := sr.dial(typ) if !errors.Is(err, errUnsupportedStreamType) { t.Errorf("#%d: error = %v, want %v", i, err, errUnsupportedStreamType) } } } // TestStream tests that streamReader and streamWriter can build stream to // send messages between each other. func TestStream(t *testing.T) { recvc := make(chan raftpb.Message, streamBufSize) propc := make(chan raftpb.Message, streamBufSize) msgapp := raftpb.Message{ Type: raftpb.MsgApp, From: 2, To: 1, Term: 1, LogTerm: 1, Index: 3, Entries: []raftpb.Entry{{Term: 1, Index: 4}}, } tests := []struct { t streamType m raftpb.Message wc chan raftpb.Message }{ { streamTypeMessage, raftpb.Message{Type: raftpb.MsgProp, To: 2}, propc, }, { streamTypeMessage, msgapp, recvc, }, { streamTypeMsgAppV2, msgapp, recvc, }, } for i, tt := range tests { h := &fakeStreamHandler{t: tt.t} srv := httptest.NewServer(h) defer srv.Close() sw := startStreamWriter(zaptest.NewLogger(t), types.ID(0), types.ID(1), newPeerStatus(zaptest.NewLogger(t), types.ID(0), types.ID(1)), &stats.FollowerStats{}, &fakeRaft{}) defer sw.stop() h.sw = sw picker := mustNewURLPicker(t, []string{srv.URL}) tr := &Transport{streamRt: &http.Transport{}, ClusterID: types.ID(1)} sr := &streamReader{ peerID: types.ID(2), typ: tt.t, tr: tr, picker: picker, status: newPeerStatus(zaptest.NewLogger(t), types.ID(0), types.ID(2)), recvc: recvc, propc: propc, rl: rate.NewLimiter(rate.Every(100*time.Millisecond), 1), } sr.start() // wait for stream to work var writec chan<- raftpb.Message for { var ok bool if writec, ok = sw.writec(); ok { break } time.Sleep(time.Millisecond) } writec <- tt.m var m raftpb.Message select { case m = <-tt.wc: case <-time.After(time.Second): t.Fatalf("#%d: failed to receive message from the channel", i) } if !reflect.DeepEqual(m, tt.m) { t.Fatalf("#%d: message = %+v, want %+v", i, m, tt.m) } sr.stop() } } func TestCheckStreamSupport(t *testing.T) { tests := []struct { v *semver.Version t streamType w bool }{ // support { semver.Must(semver.NewVersion("2.1.0")), streamTypeMsgAppV2, true, }, // ignore patch { semver.Must(semver.NewVersion("2.1.9")), streamTypeMsgAppV2, true, }, // ignore prerelease { semver.Must(semver.NewVersion("2.1.0-alpha")), streamTypeMsgAppV2, true, }, } for i, tt := range tests { if g := checkStreamSupport(tt.v, tt.t); g != tt.w { t.Errorf("#%d: check = %v, want %v", i, g, tt.w) } } } func TestStreamSupportCurrentVersion(t *testing.T) { cv := version.Cluster(version.Version) cv = cv + ".0" if _, ok := supportedStream[cv]; !ok { t.Errorf("Current version does not have stream support.") } } type fakeWriteFlushCloser struct { mu sync.Mutex err error written int closed chan struct{} writec chan struct{} } func newFakeWriteFlushCloser(err error) *fakeWriteFlushCloser { return &fakeWriteFlushCloser{ err: err, closed: make(chan struct{}), writec: make(chan struct{}, 1), } } func (wfc *fakeWriteFlushCloser) Write(p []byte) (n int, err error) { wfc.mu.Lock() defer wfc.mu.Unlock() select { case wfc.writec <- struct{}{}: default: } wfc.written += len(p) return len(p), wfc.err } func (wfc *fakeWriteFlushCloser) Flush() {} func (wfc *fakeWriteFlushCloser) Close() error { close(wfc.closed) return wfc.err } func (wfc *fakeWriteFlushCloser) Written() int { wfc.mu.Lock() defer wfc.mu.Unlock() return wfc.written } func (wfc *fakeWriteFlushCloser) Closed() bool { select { case <-wfc.closed: return true default: return false } } type fakeStreamHandler struct { t streamType sw *streamWriter } func (h *fakeStreamHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Add("X-Server-Version", version.Version) w.(http.Flusher).Flush() c := newCloseNotifier() h.sw.attach(&outgoingConn{ t: h.t, Writer: w, Flusher: w.(http.Flusher), Closer: c, }) <-c.closeNotify() } ================================================ FILE: server/etcdserver/api/rafthttp/transport.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "context" "net/http" "sync" "time" "github.com/xiang90/probing" "go.uber.org/zap" "golang.org/x/time/rate" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) type Raft interface { Process(ctx context.Context, m raftpb.Message) error IsIDRemoved(id uint64) bool ReportUnreachable(id uint64) ReportSnapshot(id uint64, status raft.SnapshotStatus) } type Transporter interface { // Start starts the given Transporter. // Start MUST be called before calling other functions in the interface. Start() error // Handler returns the HTTP handler of the transporter. // A transporter HTTP handler handles the HTTP requests // from remote peers. // The handler MUST be used to handle RaftPrefix(/raft) // endpoint. Handler() http.Handler // Send sends out the given messages to the remote peers. // Each message has a To field, which is an id that maps // to an existing peer in the transport. // If the id cannot be found in the transport, the message // will be ignored. Send(m []raftpb.Message) // SendSnapshot sends out the given snapshot message to a remote peer. // The behavior of SendSnapshot is similar to Send. SendSnapshot(m snap.Message) // AddRemote adds a remote with given peer urls into the transport. // A remote helps newly joined member to catch up the progress of cluster, // and will not be used after that. // It is the caller's responsibility to ensure the urls are all valid, // or it panics. AddRemote(id types.ID, urls []string) // AddPeer adds a peer with given peer urls into the transport. // It is the caller's responsibility to ensure the urls are all valid, // or it panics. // Peer urls are used to connect to the remote peer. AddPeer(id types.ID, urls []string) // RemovePeer removes the peer with given id. RemovePeer(id types.ID) // RemoveAllPeers removes all the existing peers in the transport. RemoveAllPeers() // UpdatePeer updates the peer urls of the peer with the given id. // It is the caller's responsibility to ensure the urls are all valid, // or it panics. UpdatePeer(id types.ID, urls []string) // ActiveSince returns the time that the connection with the peer // of the given id becomes active. // If the connection is active since peer was added, it returns the adding time. // If the connection is currently inactive, it returns zero time. ActiveSince(id types.ID) time.Time // ActivePeers returns the number of active peers. ActivePeers() int // Stop closes the connections and stops the transporter. Stop() } // Transport implements Transporter interface. It provides the functionality // to send raft messages to peers, and receive raft messages from peers. // User should call Handler method to get a handler to serve requests // received from peerURLs. // User needs to call Start before calling other functions, and call // Stop when the Transport is no longer used. type Transport struct { Logger *zap.Logger DialTimeout time.Duration // maximum duration before timing out dial of the request // DialRetryFrequency defines the frequency of streamReader dial retrial attempts; // a distinct rate limiter is created per every peer (default value: 10 events/sec) DialRetryFrequency rate.Limit TLSInfo transport.TLSInfo // TLS information used when creating connection ID types.ID // local member ID URLs types.URLs // local peer URLs ClusterID types.ID // raft cluster ID for request validation Raft Raft // raft state machine, to which the Transport forwards received messages and reports status Snapshotter *snap.Snapshotter ServerStats *stats.ServerStats // used to record general transportation statistics // LeaderStats records transportation statistics with followers when // performing as leader in raft protocol LeaderStats *stats.LeaderStats // ErrorC is used to report detected critical errors, e.g., // the member has been permanently removed from the cluster // When an error is received from ErrorC, user should stop raft state // machine and thus stop the Transport. ErrorC chan error streamRt http.RoundTripper // roundTripper used by streams pipelineRt http.RoundTripper // roundTripper used by pipelines mu sync.RWMutex // protect the remote and peer map remotes map[types.ID]*remote // remotes map that helps newly joined member to catch up peers map[types.ID]Peer // peers map pipelineProber probing.Prober streamProber probing.Prober } func (t *Transport) Start() error { var err error t.streamRt, err = newStreamRoundTripper(t.TLSInfo, t.DialTimeout) if err != nil { return err } t.pipelineRt, err = NewRoundTripper(t.TLSInfo, t.DialTimeout) if err != nil { return err } t.remotes = make(map[types.ID]*remote) t.peers = make(map[types.ID]Peer) t.pipelineProber = probing.NewProber(t.pipelineRt) t.streamProber = probing.NewProber(t.streamRt) // If client didn't provide dial retry frequency, use the default // (100ms backoff between attempts to create a new stream), // so it doesn't bring too much overhead when retry. if t.DialRetryFrequency == 0 { t.DialRetryFrequency = rate.Every(100 * time.Millisecond) } return nil } func (t *Transport) Handler() http.Handler { pipelineHandler := newPipelineHandler(t, t.Raft, t.ClusterID) streamHandler := newStreamHandler(t, t, t.Raft, t.ID, t.ClusterID) snapHandler := newSnapshotHandler(t, t.Raft, t.Snapshotter, t.ClusterID) mux := http.NewServeMux() mux.Handle(RaftPrefix, pipelineHandler) mux.Handle(RaftStreamPrefix+"/", streamHandler) mux.Handle(RaftSnapshotPrefix, snapHandler) mux.Handle(ProbingPrefix, probing.NewHandler()) return mux } func (t *Transport) Get(id types.ID) Peer { t.mu.RLock() defer t.mu.RUnlock() return t.peers[id] } func (t *Transport) Send(msgs []raftpb.Message) { for _, m := range msgs { if m.To == 0 { // ignore intentionally dropped message continue } to := types.ID(m.To) t.mu.RLock() p, pok := t.peers[to] g, rok := t.remotes[to] t.mu.RUnlock() if pok { if isMsgApp(m) { t.ServerStats.SendAppendReq(m.Size()) } p.send(m) continue } if rok { g.send(m) continue } if t.Logger != nil { t.Logger.Debug( "ignored message send request; unknown remote peer target", zap.String("type", m.Type.String()), zap.String("unknown-target-peer-id", to.String()), ) } } } func (t *Transport) Stop() { t.mu.Lock() defer t.mu.Unlock() for _, r := range t.remotes { r.stop() } for _, p := range t.peers { p.stop() } t.pipelineProber.RemoveAll() t.streamProber.RemoveAll() if tr, ok := t.streamRt.(*http.Transport); ok { tr.CloseIdleConnections() } if tr, ok := t.pipelineRt.(*http.Transport); ok { tr.CloseIdleConnections() } t.peers = nil t.remotes = nil } // CutPeer drops messages to the specified peer. func (t *Transport) CutPeer(id types.ID) { t.mu.RLock() p, pok := t.peers[id] g, gok := t.remotes[id] t.mu.RUnlock() if pok { p.(Pausable).Pause() } if gok { g.Pause() } } // MendPeer recovers the message dropping behavior of the given peer. func (t *Transport) MendPeer(id types.ID) { t.mu.RLock() p, pok := t.peers[id] g, gok := t.remotes[id] t.mu.RUnlock() if pok { p.(Pausable).Resume() } if gok { g.Resume() } } func (t *Transport) AddRemote(id types.ID, us []string) { t.mu.Lock() defer t.mu.Unlock() if t.remotes == nil { // there's no clean way to shutdown the golang http server // (see: https://github.com/golang/go/issues/4674) before // stopping the transport; ignore any new connections. return } if _, ok := t.peers[id]; ok { return } if _, ok := t.remotes[id]; ok { return } urls, err := types.NewURLs(us) if err != nil { if t.Logger != nil { t.Logger.Panic("failed NewURLs", zap.Strings("urls", us), zap.Error(err)) } } t.remotes[id] = startRemote(t, urls, id) if t.Logger != nil { t.Logger.Info( "added new remote peer", zap.String("local-member-id", t.ID.String()), zap.String("remote-peer-id", id.String()), zap.Strings("remote-peer-urls", us), ) } } func (t *Transport) AddPeer(id types.ID, us []string) { t.mu.Lock() defer t.mu.Unlock() if t.peers == nil { panic("transport stopped") } if _, ok := t.peers[id]; ok { return } urls, err := types.NewURLs(us) if err != nil { if t.Logger != nil { t.Logger.Panic("failed NewURLs", zap.Strings("urls", us), zap.Error(err)) } } fs := t.LeaderStats.Follower(id.String()) t.peers[id] = startPeer(t, urls, id, fs) addPeerToProber(t.Logger, t.pipelineProber, id.String(), us, RoundTripperNameSnapshot, rttSec) addPeerToProber(t.Logger, t.streamProber, id.String(), us, RoundTripperNameRaftMessage, rttSec) if t.Logger != nil { t.Logger.Info( "added remote peer", zap.String("local-member-id", t.ID.String()), zap.String("remote-peer-id", id.String()), zap.Strings("remote-peer-urls", us), ) } } func (t *Transport) RemovePeer(id types.ID) { t.mu.Lock() defer t.mu.Unlock() t.removePeer(id) } func (t *Transport) RemoveAllPeers() { t.mu.Lock() defer t.mu.Unlock() for id := range t.peers { t.removePeer(id) } } // the caller of this function must have the peers mutex. func (t *Transport) removePeer(id types.ID) { // etcd may remove a member again on startup due to WAL files replaying. peer, ok := t.peers[id] if ok { peer.stop() delete(t.peers, id) delete(t.LeaderStats.Followers, id.String()) t.pipelineProber.Remove(id.String()) t.streamProber.Remove(id.String()) } if t.Logger != nil { if ok { t.Logger.Info( "removed remote peer", zap.String("local-member-id", t.ID.String()), zap.String("removed-remote-peer-id", id.String()), ) } else { t.Logger.Warn( "skipped removing already removed peer", zap.String("local-member-id", t.ID.String()), zap.String("removed-remote-peer-id", id.String()), ) } } } func (t *Transport) UpdatePeer(id types.ID, us []string) { t.mu.Lock() defer t.mu.Unlock() // TODO: return error or just panic? if _, ok := t.peers[id]; !ok { return } urls, err := types.NewURLs(us) if err != nil { if t.Logger != nil { t.Logger.Panic("failed NewURLs", zap.Strings("urls", us), zap.Error(err)) } } t.peers[id].update(urls) t.pipelineProber.Remove(id.String()) addPeerToProber(t.Logger, t.pipelineProber, id.String(), us, RoundTripperNameSnapshot, rttSec) t.streamProber.Remove(id.String()) addPeerToProber(t.Logger, t.streamProber, id.String(), us, RoundTripperNameRaftMessage, rttSec) if t.Logger != nil { t.Logger.Info( "updated remote peer", zap.String("local-member-id", t.ID.String()), zap.String("updated-remote-peer-id", id.String()), zap.Strings("updated-remote-peer-urls", us), ) } } func (t *Transport) ActiveSince(id types.ID) time.Time { t.mu.RLock() defer t.mu.RUnlock() if p, ok := t.peers[id]; ok { return p.activeSince() } return time.Time{} } func (t *Transport) SendSnapshot(m snap.Message) { t.mu.Lock() defer t.mu.Unlock() p := t.peers[types.ID(m.To)] if p == nil { m.CloseWithError(errMemberNotFound) return } p.sendSnap(m) } // Pausable is a testing interface for pausing transport traffic. type Pausable interface { Pause() Resume() } func (t *Transport) Pause() { t.mu.RLock() defer t.mu.RUnlock() for _, p := range t.peers { p.(Pausable).Pause() } } func (t *Transport) Resume() { t.mu.RLock() defer t.mu.RUnlock() for _, p := range t.peers { p.(Pausable).Resume() } } // ActivePeers returns a channel that closes when an initial // peer connection has been established. Use this to wait until the // first peer connection becomes active. func (t *Transport) ActivePeers() (cnt int) { t.mu.RLock() defer t.mu.RUnlock() for _, p := range t.peers { if !p.activeSince().IsZero() { cnt++ } } return cnt } ================================================ FILE: server/etcdserver/api/rafthttp/transport_bench_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "context" "net/http/httptest" "sync" "testing" "time" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/types" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) func BenchmarkSendingMsgApp(b *testing.B) { // member 1 tr := &Transport{ ID: types.ID(1), ClusterID: types.ID(1), Raft: &fakeRaft{}, ServerStats: newServerStats(), LeaderStats: stats.NewLeaderStats(zaptest.NewLogger(b), "1"), } tr.Start() srv := httptest.NewServer(tr.Handler()) defer srv.Close() // member 2 r := &countRaft{} tr2 := &Transport{ ID: types.ID(2), ClusterID: types.ID(1), Raft: r, ServerStats: newServerStats(), LeaderStats: stats.NewLeaderStats(zaptest.NewLogger(b), "2"), } tr2.Start() srv2 := httptest.NewServer(tr2.Handler()) defer srv2.Close() tr.AddPeer(types.ID(2), []string{srv2.URL}) defer tr.Stop() tr2.AddPeer(types.ID(1), []string{srv.URL}) defer tr2.Stop() if !waitStreamWorking(tr.Get(types.ID(2)).(*peer)) { b.Fatalf("stream from 1 to 2 is not in work as expected") } b.ReportAllocs() b.SetBytes(64) b.ResetTimer() data := make([]byte, 64) for i := 0; i < b.N; i++ { tr.Send([]raftpb.Message{ { Type: raftpb.MsgApp, From: 1, To: 2, Index: uint64(i), Entries: []raftpb.Entry{ { Index: uint64(i + 1), Data: data, }, }, }, }) } // wait until all messages are received by the target raft for r.count() != b.N { time.Sleep(time.Millisecond) } b.StopTimer() } type countRaft struct { mu sync.Mutex cnt int } func (r *countRaft) Process(ctx context.Context, m raftpb.Message) error { r.mu.Lock() defer r.mu.Unlock() r.cnt++ return nil } func (r *countRaft) IsIDRemoved(id uint64) bool { return false } func (r *countRaft) ReportUnreachable(id uint64) {} func (r *countRaft) ReportSnapshot(id uint64, status raft.SnapshotStatus) {} func (r *countRaft) count() int { r.mu.Lock() defer r.mu.Unlock() return r.cnt } ================================================ FILE: server/etcdserver/api/rafthttp/transport_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "net/http" "reflect" "testing" "time" "github.com/xiang90/probing" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/types" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/raft/v3/raftpb" ) // TestTransportSend tests that transport can send messages using correct // underlying peer, and drop local or unknown-target messages. func TestTransportSend(t *testing.T) { peer1 := newFakePeer() peer2 := newFakePeer() tr := &Transport{ ServerStats: stats.NewServerStats("", ""), peers: map[types.ID]Peer{types.ID(1): peer1, types.ID(2): peer2}, } wmsgsIgnored := []raftpb.Message{ // bad local message {Type: raftpb.MsgBeat}, // bad remote message {Type: raftpb.MsgProp, To: 3}, } wmsgsTo1 := []raftpb.Message{ // good message {Type: raftpb.MsgProp, To: 1}, {Type: raftpb.MsgApp, To: 1}, } wmsgsTo2 := []raftpb.Message{ // good message {Type: raftpb.MsgProp, To: 2}, {Type: raftpb.MsgApp, To: 2}, } tr.Send(wmsgsIgnored) tr.Send(wmsgsTo1) tr.Send(wmsgsTo2) if !reflect.DeepEqual(peer1.msgs, wmsgsTo1) { t.Errorf("msgs to peer 1 = %+v, want %+v", peer1.msgs, wmsgsTo1) } if !reflect.DeepEqual(peer2.msgs, wmsgsTo2) { t.Errorf("msgs to peer 2 = %+v, want %+v", peer2.msgs, wmsgsTo2) } } func TestTransportCutMend(t *testing.T) { peer1 := newFakePeer() peer2 := newFakePeer() tr := &Transport{ ServerStats: stats.NewServerStats("", ""), peers: map[types.ID]Peer{types.ID(1): peer1, types.ID(2): peer2}, } tr.CutPeer(types.ID(1)) wmsgsTo := []raftpb.Message{ // good message {Type: raftpb.MsgProp, To: 1}, {Type: raftpb.MsgApp, To: 1}, } tr.Send(wmsgsTo) if len(peer1.msgs) > 0 { t.Fatalf("msgs expected to be ignored, got %+v", peer1.msgs) } tr.MendPeer(types.ID(1)) tr.Send(wmsgsTo) if !reflect.DeepEqual(peer1.msgs, wmsgsTo) { t.Errorf("msgs to peer 1 = %+v, want %+v", peer1.msgs, wmsgsTo) } } func TestTransportAdd(t *testing.T) { ls := stats.NewLeaderStats(zaptest.NewLogger(t), "") tr := &Transport{ LeaderStats: ls, streamRt: &roundTripperRecorder{}, peers: make(map[types.ID]Peer), pipelineProber: probing.NewProber(nil), streamProber: probing.NewProber(nil), } tr.AddPeer(1, []string{"http://localhost:2380"}) if _, ok := ls.Followers["1"]; !ok { t.Errorf("FollowerStats[1] is nil, want exists") } s, ok := tr.peers[types.ID(1)] if !ok { tr.Stop() t.Fatalf("senders[1] is nil, want exists") } // duplicate AddPeer is ignored tr.AddPeer(1, []string{"http://localhost:2380"}) ns := tr.peers[types.ID(1)] if s != ns { t.Errorf("sender = %v, want %v", ns, s) } tr.Stop() } func TestTransportRemove(t *testing.T) { tr := &Transport{ LeaderStats: stats.NewLeaderStats(zaptest.NewLogger(t), ""), streamRt: &roundTripperRecorder{}, peers: make(map[types.ID]Peer), pipelineProber: probing.NewProber(nil), streamProber: probing.NewProber(nil), } tr.AddPeer(1, []string{"http://localhost:2380"}) tr.RemovePeer(types.ID(1)) defer tr.Stop() if _, ok := tr.peers[types.ID(1)]; ok { t.Fatalf("senders[1] exists, want removed") } } func TestTransportRemoveIsIdempotent(t *testing.T) { tr := &Transport{ LeaderStats: stats.NewLeaderStats(zaptest.NewLogger(t), ""), streamRt: &roundTripperRecorder{}, peers: make(map[types.ID]Peer), pipelineProber: probing.NewProber(nil), streamProber: probing.NewProber(nil), } tr.AddPeer(1, []string{"http://localhost:2380"}) tr.RemovePeer(types.ID(1)) tr.RemovePeer(types.ID(1)) defer tr.Stop() if _, ok := tr.peers[types.ID(1)]; ok { t.Fatalf("senders[1] exists, want removed") } } func TestTransportUpdate(t *testing.T) { peer := newFakePeer() tr := &Transport{ peers: map[types.ID]Peer{types.ID(1): peer}, pipelineProber: probing.NewProber(nil), streamProber: probing.NewProber(nil), } u := "http://localhost:2380" tr.UpdatePeer(types.ID(1), []string{u}) wurls := types.URLs(testutil.MustNewURLs(t, []string{"http://localhost:2380"})) if !reflect.DeepEqual(peer.peerURLs, wurls) { t.Errorf("urls = %+v, want %+v", peer.peerURLs, wurls) } } func TestTransportErrorc(t *testing.T) { errorc := make(chan error, 1) tr := &Transport{ Raft: &fakeRaft{}, LeaderStats: stats.NewLeaderStats(zaptest.NewLogger(t), ""), ErrorC: errorc, streamRt: newRespRoundTripper(http.StatusForbidden, nil), pipelineRt: newRespRoundTripper(http.StatusForbidden, nil), peers: make(map[types.ID]Peer), pipelineProber: probing.NewProber(nil), streamProber: probing.NewProber(nil), } tr.AddPeer(1, []string{"http://localhost:2380"}) defer tr.Stop() select { case <-errorc: t.Fatalf("received unexpected from errorc") case <-time.After(10 * time.Millisecond): } tr.peers[1].send(raftpb.Message{}) select { case <-errorc: case <-time.After(1 * time.Second): t.Fatalf("cannot receive error from errorc") } } ================================================ FILE: server/etcdserver/api/rafthttp/urlpick.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "net/url" "sync" "go.etcd.io/etcd/client/pkg/v3/types" ) type urlPicker struct { mu sync.Mutex // guards urls and picked urls types.URLs picked int } func newURLPicker(urls types.URLs) *urlPicker { return &urlPicker{ urls: urls, } } func (p *urlPicker) update(urls types.URLs) { p.mu.Lock() defer p.mu.Unlock() p.urls = urls p.picked = 0 } func (p *urlPicker) pick() url.URL { p.mu.Lock() defer p.mu.Unlock() return p.urls[p.picked] } // unreachable notices the picker that the given url is unreachable, // and it should use other possible urls. func (p *urlPicker) unreachable(u url.URL) { p.mu.Lock() defer p.mu.Unlock() if u == p.urls[p.picked] { p.picked = (p.picked + 1) % len(p.urls) } } ================================================ FILE: server/etcdserver/api/rafthttp/urlpick_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "net/url" "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" ) // TestURLPickerPickTwice tests that pick returns a possible url, // and always returns the same one. func TestURLPickerPickTwice(t *testing.T) { picker := mustNewURLPicker(t, []string{"http://127.0.0.1:2380", "http://127.0.0.1:7001"}) u := picker.pick() urlmap := map[url.URL]bool{ {Scheme: "http", Host: "127.0.0.1:2380"}: true, {Scheme: "http", Host: "127.0.0.1:7001"}: true, } if !urlmap[u] { t.Errorf("url picked = %+v, want a possible url in %+v", u, urlmap) } // pick out the same url when calling pick again uu := picker.pick() if u != uu { t.Errorf("url picked = %+v, want %+v", uu, u) } } func TestURLPickerUpdate(t *testing.T) { picker := mustNewURLPicker(t, []string{"http://127.0.0.1:2380", "http://127.0.0.1:7001"}) picker.update(testutil.MustNewURLs(t, []string{"http://localhost:2380", "http://localhost:7001"})) u := picker.pick() urlmap := map[url.URL]bool{ {Scheme: "http", Host: "localhost:2380"}: true, {Scheme: "http", Host: "localhost:7001"}: true, } if !urlmap[u] { t.Errorf("url picked = %+v, want a possible url in %+v", u, urlmap) } } func TestURLPickerUnreachable(t *testing.T) { picker := mustNewURLPicker(t, []string{"http://127.0.0.1:2380", "http://127.0.0.1:7001"}) u := picker.pick() picker.unreachable(u) uu := picker.pick() if u == uu { t.Errorf("url picked = %+v, want other possible urls", uu) } } func mustNewURLPicker(t *testing.T, us []string) *urlPicker { urls := testutil.MustNewURLs(t, us) return newURLPicker(urls) } ================================================ FILE: server/etcdserver/api/rafthttp/util.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "fmt" "io" "net" "net/http" "net/url" "strings" "time" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" ) var ( errMemberRemoved = fmt.Errorf("the member has been permanently removed from the cluster") errMemberNotFound = fmt.Errorf("member not found") ) // NewListener returns a listener for raft message transfer between peers. // It uses timeout listener to identify broken streams promptly. func NewListener(u url.URL, tlsinfo *transport.TLSInfo) (net.Listener, error) { return transport.NewListenerWithOpts(u.Host, u.Scheme, transport.WithTLSInfo(tlsinfo), transport.WithTimeout(ConnReadTimeout, ConnWriteTimeout)) } // NewRoundTripper returns a roundTripper used to send requests // to rafthttp listener of remote peers. func NewRoundTripper(tlsInfo transport.TLSInfo, dialTimeout time.Duration) (http.RoundTripper, error) { // It uses timeout transport to pair with remote timeout listeners. // It sets no read/write timeout, because message in requests may // take long time to write out before reading out the response. return transport.NewTimeoutTransport(tlsInfo, dialTimeout, 0, 0) } // newStreamRoundTripper returns a roundTripper used to send stream requests // to rafthttp listener of remote peers. // Read/write timeout is set for stream roundTripper to promptly // find out broken status, which minimizes the number of messages // sent on broken connection. func newStreamRoundTripper(tlsInfo transport.TLSInfo, dialTimeout time.Duration) (http.RoundTripper, error) { return transport.NewTimeoutTransport(tlsInfo, dialTimeout, ConnReadTimeout, ConnWriteTimeout) } // createPostRequest creates a HTTP POST request that sends raft message. func createPostRequest(lg *zap.Logger, u url.URL, path string, body io.Reader, ct string, urls types.URLs, from, cid types.ID) *http.Request { uu := u uu.Path = path req, err := http.NewRequest(http.MethodPost, uu.String(), body) if err != nil { if lg != nil { lg.Panic("unexpected new request error", zap.Error(err)) } } req.Header.Set("Content-Type", ct) req.Header.Set("X-Server-From", from.String()) req.Header.Set("X-Server-Version", version.Version) req.Header.Set("X-Min-Cluster-Version", version.MinClusterVersion) req.Header.Set("X-Etcd-Cluster-ID", cid.String()) setPeerURLsHeader(req, urls) return req } // checkPostResponse checks the response of the HTTP POST request that sends // raft message. func checkPostResponse(lg *zap.Logger, resp *http.Response, body []byte, req *http.Request, to types.ID) error { switch resp.StatusCode { case http.StatusPreconditionFailed: switch strings.TrimSuffix(string(body), "\n") { case errIncompatibleVersion.Error(): if lg != nil { lg.Error( "request sent was ignored by peer", zap.String("remote-peer-id", to.String()), ) } return errIncompatibleVersion case ErrClusterIDMismatch.Error(): if lg != nil { lg.Error( "request sent was ignored due to cluster ID mismatch", zap.String("remote-peer-id", to.String()), zap.String("remote-peer-cluster-id", resp.Header.Get("X-Etcd-Cluster-ID")), zap.String("local-member-cluster-id", req.Header.Get("X-Etcd-Cluster-ID")), ) } return ErrClusterIDMismatch default: return fmt.Errorf("unhandled error %q when precondition failed", string(body)) } case http.StatusForbidden: return errMemberRemoved case http.StatusNoContent: return nil default: return fmt.Errorf("unexpected http status %s while posting to %q", http.StatusText(resp.StatusCode), req.URL.String()) } } // reportCriticalError reports the given error through sending it into // the given error channel. // If the error channel is filled up when sending error, it drops the error // because the fact that error has happened is reported, which is // good enough. func reportCriticalError(err error, errc chan<- error) { select { case errc <- err: default: } } // compareMajorMinorVersion returns an integer comparing two versions based on // their major and minor version. The result will be 0 if a==b, -1 if a < b, // and 1 if a > b. func compareMajorMinorVersion(a, b *semver.Version) int { na := &semver.Version{Major: a.Major, Minor: a.Minor} nb := &semver.Version{Major: b.Major, Minor: b.Minor} switch { case na.LessThan(*nb): return -1 case nb.LessThan(*na): return 1 default: return 0 } } // serverVersion returns the server version from the given header. func serverVersion(h http.Header) *semver.Version { verStr := h.Get("X-Server-Version") // backward compatibility with etcd 2.0 if verStr == "" { verStr = "2.0.0" } return semver.Must(semver.NewVersion(verStr)) } // serverVersion returns the min cluster version from the given header. func minClusterVersion(h http.Header) *semver.Version { verStr := h.Get("X-Min-Cluster-Version") // backward compatibility with etcd 2.0 if verStr == "" { verStr = "2.0.0" } return semver.Must(semver.NewVersion(verStr)) } // checkVersionCompatibility checks whether the given version is compatible // with the local version. func checkVersionCompatibility(name string, server, minCluster *semver.Version) ( localServer *semver.Version, localMinCluster *semver.Version, err error, ) { localServer = semver.Must(semver.NewVersion(version.Version)) localMinCluster = semver.Must(semver.NewVersion(version.MinClusterVersion)) if compareMajorMinorVersion(server, localMinCluster) == -1 { return localServer, localMinCluster, fmt.Errorf("remote version is too low: remote[%s]=%s, local=%s", name, server, localServer) } if compareMajorMinorVersion(minCluster, localServer) == 1 { return localServer, localMinCluster, fmt.Errorf("local version is too low: remote[%s]=%s, local=%s", name, server, localServer) } return localServer, localMinCluster, nil } // setPeerURLsHeader reports local urls for peer discovery func setPeerURLsHeader(req *http.Request, urls types.URLs) { if urls == nil { // often not set in unit tests return } peerURLs := make([]string, urls.Len()) for i := range urls { peerURLs[i] = urls[i].String() } req.Header.Set("X-PeerURLs", strings.Join(peerURLs, ",")) } // addRemoteFromRequest adds a remote peer according to an http request header func addRemoteFromRequest(tr Transporter, r *http.Request) { if from, err := types.IDFromString(r.Header.Get("X-Server-From")); err == nil { if urls := r.Header.Get("X-PeerURLs"); urls != "" { tr.AddRemote(from, strings.Split(urls, ",")) } } } ================================================ FILE: server/etcdserver/api/rafthttp/util_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rafthttp import ( "bytes" "encoding/binary" "io" "net/http" "reflect" "testing" "github.com/coreos/go-semver/semver" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/raft/v3/raftpb" ) func TestEntry(t *testing.T) { tests := []raftpb.Entry{ {}, {Term: 1, Index: 1}, {Term: 1, Index: 1, Data: []byte("some data")}, } for i, tt := range tests { b := &bytes.Buffer{} if err := writeEntryTo(b, &tt); err != nil { t.Errorf("#%d: unexpected write ents error: %v", i, err) continue } var ent raftpb.Entry if err := readEntryFrom(b, &ent); err != nil { t.Errorf("#%d: unexpected read ents error: %v", i, err) continue } if !reflect.DeepEqual(ent, tt) { t.Errorf("#%d: ent = %+v, want %+v", i, ent, tt) } } } func TestCompareMajorMinorVersion(t *testing.T) { tests := []struct { va, vb *semver.Version w int }{ // equal to { semver.Must(semver.NewVersion("2.1.0")), semver.Must(semver.NewVersion("2.1.0")), 0, }, // smaller than { semver.Must(semver.NewVersion("2.0.0")), semver.Must(semver.NewVersion("2.1.0")), -1, }, // bigger than { semver.Must(semver.NewVersion("2.2.0")), semver.Must(semver.NewVersion("2.1.0")), 1, }, // ignore patch { semver.Must(semver.NewVersion("2.1.1")), semver.Must(semver.NewVersion("2.1.0")), 0, }, // ignore prerelease { semver.Must(semver.NewVersion("2.1.0-alpha.0")), semver.Must(semver.NewVersion("2.1.0")), 0, }, } for i, tt := range tests { if g := compareMajorMinorVersion(tt.va, tt.vb); g != tt.w { t.Errorf("#%d: compare = %d, want %d", i, g, tt.w) } } } func TestServerVersion(t *testing.T) { tests := []struct { h http.Header wv *semver.Version }{ // backward compatibility with etcd 2.0 { http.Header{}, semver.Must(semver.NewVersion("2.0.0")), }, { http.Header{"X-Server-Version": []string{"2.1.0"}}, semver.Must(semver.NewVersion("2.1.0")), }, { http.Header{"X-Server-Version": []string{"2.1.0-alpha.0+git"}}, semver.Must(semver.NewVersion("2.1.0-alpha.0+git")), }, } for i, tt := range tests { v := serverVersion(tt.h) if v.String() != tt.wv.String() { t.Errorf("#%d: version = %s, want %s", i, v, tt.wv) } } } func TestMinClusterVersion(t *testing.T) { tests := []struct { h http.Header wv *semver.Version }{ // backward compatibility with etcd 2.0 { http.Header{}, semver.Must(semver.NewVersion("2.0.0")), }, { http.Header{"X-Min-Cluster-Version": []string{"2.1.0"}}, semver.Must(semver.NewVersion("2.1.0")), }, { http.Header{"X-Min-Cluster-Version": []string{"2.1.0-alpha.0+git"}}, semver.Must(semver.NewVersion("2.1.0-alpha.0+git")), }, } for i, tt := range tests { v := minClusterVersion(tt.h) if v.String() != tt.wv.String() { t.Errorf("#%d: version = %s, want %s", i, v, tt.wv) } } } func TestCheckVersionCompatibility(t *testing.T) { ls := semver.Must(semver.NewVersion(version.Version)) lmc := semver.Must(semver.NewVersion(version.MinClusterVersion)) tests := []struct { server *semver.Version minCluster *semver.Version wok bool }{ // the same version as local { ls, lmc, true, }, // one version lower { lmc, &semver.Version{}, true, }, // one version higher { &semver.Version{Major: ls.Major + 1}, ls, true, }, // too low version { &semver.Version{Major: lmc.Major - 1}, &semver.Version{}, false, }, // too high version { &semver.Version{Major: ls.Major + 1, Minor: 1}, &semver.Version{Major: ls.Major + 1}, false, }, } for i, tt := range tests { _, _, err := checkVersionCompatibility("", tt.server, tt.minCluster) if ok := err == nil; ok != tt.wok { t.Errorf("#%d: ok = %v, want %v", i, ok, tt.wok) } } } func writeEntryTo(w io.Writer, ent *raftpb.Entry) error { size := ent.Size() if err := binary.Write(w, binary.BigEndian, uint64(size)); err != nil { return err } b, err := ent.Marshal() if err != nil { return err } _, err = w.Write(b) return err } func readEntryFrom(r io.Reader, ent *raftpb.Entry) error { var l uint64 if err := binary.Read(r, binary.BigEndian, &l); err != nil { return err } buf := make([]byte, int(l)) if _, err := io.ReadFull(r, buf); err != nil { return err } return ent.Unmarshal(buf) } ================================================ FILE: server/etcdserver/api/snap/db.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package snap import ( "errors" "fmt" "io" "os" "path/filepath" "time" humanize "github.com/dustin/go-humanize" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/fileutil" ) var ErrNoDBSnapshot = errors.New("snap: snapshot file doesn't exist") // SaveDBFrom saves snapshot of the database from the given reader. It // guarantees the save operation is atomic. func (s *Snapshotter) SaveDBFrom(r io.Reader, id uint64) (int64, error) { start := time.Now() f, err := os.CreateTemp(s.dir, "tmp") if err != nil { return 0, err } var n int64 n, err = io.Copy(f, r) if err == nil { fsyncStart := time.Now() err = fileutil.Fsync(f) snapDBFsyncSec.Observe(time.Since(fsyncStart).Seconds()) } f.Close() if err != nil { os.Remove(f.Name()) return n, err } fn := s.dbFilePath(id) if fileutil.Exist(fn) { os.Remove(f.Name()) return n, nil } err = os.Rename(f.Name(), fn) if err != nil { os.Remove(f.Name()) return n, err } s.lg.Info( "saved database snapshot to disk", zap.String("path", fn), zap.Int64("bytes", n), zap.String("size", humanize.Bytes(uint64(n))), ) snapDBSaveSec.Observe(time.Since(start).Seconds()) return n, nil } // DBFilePath returns the file path for the snapshot of the database with // given id. If the snapshot does not exist, it returns error. func (s *Snapshotter) DBFilePath(id uint64) (string, error) { if _, err := fileutil.ReadDir(s.dir); err != nil { return "", err } fn := s.dbFilePath(id) if fileutil.Exist(fn) { return fn, nil } if s.lg != nil { s.lg.Warn( "failed to find [SNAPSHOT-INDEX].snap.db", zap.Uint64("snapshot-index", id), zap.String("snapshot-file-path", fn), zap.Error(ErrNoDBSnapshot), ) } return "", ErrNoDBSnapshot } func (s *Snapshotter) dbFilePath(id uint64) string { return filepath.Join(s.dir, fmt.Sprintf("%016x.snap.db", id)) } ================================================ FILE: server/etcdserver/api/snap/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package snap handles Raft nodes' states with snapshots. // The snapshot logic is internal to etcd server and raft package. package snap ================================================ FILE: server/etcdserver/api/snap/message.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package snap import ( "io" "go.etcd.io/etcd/pkg/v3/ioutil" "go.etcd.io/raft/v3/raftpb" ) // Message is a struct that contains a raft Message and a ReadCloser. The type // of raft message MUST be MsgSnap, which contains the raft meta-data and an // additional data []byte field that contains the snapshot of the actual state // machine. // Message contains the ReadCloser field for handling large snapshot. This avoid // copying the entire snapshot into a byte array, which consumes a lot of memory. // // User of Message should close the Message after sending it. type Message struct { raftpb.Message ReadCloser io.ReadCloser TotalSize int64 closeC chan bool } func NewMessage(rs raftpb.Message, rc io.ReadCloser, rcSize int64) *Message { return &Message{ Message: rs, ReadCloser: ioutil.NewExactReadCloser(rc, rcSize), TotalSize: int64(rs.Size()) + rcSize, closeC: make(chan bool, 1), } } // CloseNotify returns a channel that receives a single value // when the message sent is finished. true indicates the sent // is successful. func (m Message) CloseNotify() <-chan bool { return m.closeC } func (m Message) CloseWithError(err error) { if cerr := m.ReadCloser.Close(); cerr != nil { err = cerr } if err == nil { m.closeC <- true } else { m.closeC <- false } } ================================================ FILE: server/etcdserver/api/snap/metrics.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package snap import "github.com/prometheus/client_golang/prometheus" var ( snapMarshallingSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "snap", Name: "save_marshalling_duration_seconds", Help: "The marshalling cost distributions of save called by snapshot.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) snapSaveSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "snap", Name: "save_total_duration_seconds", Help: "The total latency distributions of save called by snapshot.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) snapFsyncSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "snap", Name: "fsync_duration_seconds", Help: "The latency distributions of fsync called by snap.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) snapDBSaveSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "snap_db", Name: "save_total_duration_seconds", Help: "The total latency distributions of v3 snapshot save", // lowest bucket start of upper bound 0.1 sec (100 ms) with factor 2 // highest bucket start of 0.1 sec * 2^9 == 51.2 sec Buckets: prometheus.ExponentialBuckets(0.1, 2, 10), }) snapDBFsyncSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "snap_db", Name: "fsync_duration_seconds", Help: "The latency distributions of fsyncing .snap.db file", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) ) func init() { prometheus.MustRegister(snapMarshallingSec) prometheus.MustRegister(snapSaveSec) prometheus.MustRegister(snapFsyncSec) prometheus.MustRegister(snapDBSaveSec) prometheus.MustRegister(snapDBFsyncSec) } ================================================ FILE: server/etcdserver/api/snap/snappb/snap.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: snap.proto package snappb import ( fmt "fmt" io "io" math "math" math_bits "math/bits" proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Snapshot struct { Crc *uint32 `protobuf:"varint,1,opt,name=crc" json:"crc,omitempty"` Data []byte `protobuf:"bytes,2,opt,name=data" json:"data,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Snapshot) Reset() { *m = Snapshot{} } func (m *Snapshot) String() string { return proto.CompactTextString(m) } func (*Snapshot) ProtoMessage() {} func (*Snapshot) Descriptor() ([]byte, []int) { return fileDescriptor_f2e3c045ebf84d00, []int{0} } func (m *Snapshot) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Snapshot) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Snapshot.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Snapshot) XXX_Merge(src proto.Message) { xxx_messageInfo_Snapshot.Merge(m, src) } func (m *Snapshot) XXX_Size() int { return m.Size() } func (m *Snapshot) XXX_DiscardUnknown() { xxx_messageInfo_Snapshot.DiscardUnknown(m) } var xxx_messageInfo_Snapshot proto.InternalMessageInfo func (m *Snapshot) GetCrc() uint32 { if m != nil && m.Crc != nil { return *m.Crc } return 0 } func (m *Snapshot) GetData() []byte { if m != nil { return m.Data } return nil } func init() { proto.RegisterType((*Snapshot)(nil), "snappb.snapshot") } func init() { proto.RegisterFile("snap.proto", fileDescriptor_f2e3c045ebf84d00) } var fileDescriptor_f2e3c045ebf84d00 = []byte{ // 140 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2a, 0xce, 0x4b, 0x2c, 0xd0, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x03, 0xb1, 0x0b, 0x92, 0x94, 0x0c, 0xb8, 0x38, 0x40, 0xac, 0xe2, 0x8c, 0xfc, 0x12, 0x21, 0x01, 0x2e, 0xe6, 0xe4, 0xa2, 0x64, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xde, 0x20, 0x10, 0x53, 0x48, 0x88, 0x8b, 0x25, 0x25, 0xb1, 0x24, 0x51, 0x82, 0x49, 0x81, 0x51, 0x83, 0x27, 0x08, 0xcc, 0x76, 0x72, 0x3b, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x67, 0x3c, 0x96, 0x63, 0x88, 0x32, 0x49, 0xcf, 0xd7, 0x4b, 0x2d, 0x49, 0x4e, 0xd1, 0xcb, 0xcc, 0xd7, 0x07, 0xd1, 0xfa, 0xc5, 0xa9, 0x45, 0x65, 0xa9, 0x45, 0xfa, 0x65, 0xc6, 0x60, 0x2e, 0x94, 0x97, 0x58, 0x90, 0xa9, 0x0f, 0xb2, 0x4a, 0x1f, 0x62, 0x33, 0x20, 0x00, 0x00, 0xff, 0xff, 0x64, 0x15, 0x9e, 0x77, 0x8e, 0x00, 0x00, 0x00, } func (m *Snapshot) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Snapshot) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Snapshot) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Data != nil { i -= len(m.Data) copy(dAtA[i:], m.Data) i = encodeVarintSnap(dAtA, i, uint64(len(m.Data))) i-- dAtA[i] = 0x12 } if m.Crc != nil { i = encodeVarintSnap(dAtA, i, uint64(*m.Crc)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func encodeVarintSnap(dAtA []byte, offset int, v uint64) int { offset -= sovSnap(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) return base } func (m *Snapshot) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Crc != nil { n += 1 + sovSnap(uint64(*m.Crc)) } if m.Data != nil { l = len(m.Data) n += 1 + l + sovSnap(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func sovSnap(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } func sozSnap(x uint64) (n int) { return sovSnap(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (m *Snapshot) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowSnap } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: snapshot: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: snapshot: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Crc", wireType) } var v uint32 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowSnap } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= uint32(b&0x7F) << shift if b < 0x80 { break } } m.Crc = &v case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowSnap } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthSnap } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthSnap } if postIndex > l { return io.ErrUnexpectedEOF } m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) if m.Data == nil { m.Data = []byte{} } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSnap(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthSnap } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func skipSnap(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowSnap } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } wireType := int(wire & 0x7) switch wireType { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowSnap } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } iNdEx++ if dAtA[iNdEx-1] < 0x80 { break } } case 1: iNdEx += 8 case 2: var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowSnap } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { break } } if length < 0 { return 0, ErrInvalidLengthSnap } iNdEx += length case 3: depth++ case 4: if depth == 0 { return 0, ErrUnexpectedEndOfGroupSnap } depth-- case 5: iNdEx += 4 default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { return 0, ErrInvalidLengthSnap } if depth == 0 { return iNdEx, nil } } return 0, io.ErrUnexpectedEOF } var ( ErrInvalidLengthSnap = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowSnap = fmt.Errorf("proto: integer overflow") ErrUnexpectedEndOfGroupSnap = fmt.Errorf("proto: unexpected end of group") ) ================================================ FILE: server/etcdserver/api/snap/snappb/snap.proto ================================================ syntax = "proto2"; package snappb; option go_package = "go.etcd.io/etcd/server/v3/etcdserver/api/snap/snappb"; message snapshot { optional uint32 crc = 1; optional bytes data = 2; } ================================================ FILE: server/etcdserver/api/snap/snapshotter.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package snap import ( "errors" "fmt" "hash/crc32" "os" "path/filepath" "sort" "strconv" "strings" "time" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/verify" pioutil "go.etcd.io/etcd/pkg/v3/ioutil" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/etcdserver/api/snap/snappb" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) const snapSuffix = ".snap" var ( ErrNoSnapshot = errors.New("snap: no available snapshot") ErrEmptySnapshot = errors.New("snap: empty snapshot") ErrCRCMismatch = errors.New("snap: crc mismatch") crcTable = crc32.MakeTable(crc32.Castagnoli) // A map of valid files that can be present in the snap folder. validFiles = map[string]bool{ "db": true, } ) type Snapshotter struct { lg *zap.Logger dir string } func New(lg *zap.Logger, dir string) *Snapshotter { if lg == nil { lg = zap.NewNop() } return &Snapshotter{ lg: lg, dir: dir, } } func (s *Snapshotter) SaveSnap(snapshot raftpb.Snapshot) error { if raft.IsEmptySnap(snapshot) { return nil } return s.save(&snapshot) } func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error { start := time.Now() fname := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, snapSuffix) b := pbutil.MustMarshal(snapshot) crc := crc32.Update(0, crcTable, b) snap := snappb.Snapshot{Crc: &crc, Data: b} d, err := snap.Marshal() if err != nil { return err } snapMarshallingSec.Observe(time.Since(start).Seconds()) spath := filepath.Join(s.dir, fname) fsyncStart := time.Now() err = pioutil.WriteAndSyncFile(spath, d, 0o666) snapFsyncSec.Observe(time.Since(fsyncStart).Seconds()) if err != nil { s.lg.Warn("failed to write a snap file", zap.String("path", spath), zap.Error(err)) rerr := os.Remove(spath) if rerr != nil { s.lg.Warn("failed to remove a broken snap file", zap.String("path", spath), zap.Error(rerr)) } return err } snapSaveSec.Observe(time.Since(start).Seconds()) return nil } // Load returns the newest snapshot. func (s *Snapshotter) Load() (*raftpb.Snapshot, error) { return s.loadMatching(func(*raftpb.Snapshot) bool { return true }) } // LoadNewestAvailable loads the newest snapshot available that is in walSnaps. func (s *Snapshotter) LoadNewestAvailable(walSnaps []walpb.Snapshot) (*raftpb.Snapshot, error) { return s.loadMatching(func(snapshot *raftpb.Snapshot) bool { m := snapshot.Metadata for i := len(walSnaps) - 1; i >= 0; i-- { if m.Term == walSnaps[i].GetTerm() && m.Index == walSnaps[i].GetIndex() { return true } } return false }) } // loadMatching returns the newest snapshot where matchFn returns true. func (s *Snapshotter) loadMatching(matchFn func(*raftpb.Snapshot) bool) (*raftpb.Snapshot, error) { names, err := s.snapNames() if err != nil { return nil, err } var snap *raftpb.Snapshot for _, name := range names { if snap, err = s.loadSnap(name); err == nil && matchFn(snap) { return snap, nil } } return nil, ErrNoSnapshot } func (s *Snapshotter) loadSnap(name string) (*raftpb.Snapshot, error) { fpath := filepath.Join(s.dir, name) snap, err := Read(s.lg, fpath) if err != nil { brokenPath := fpath + ".broken" s.lg.Warn("failed to read a snap file", zap.String("path", fpath), zap.Error(err)) if rerr := os.Rename(fpath, brokenPath); rerr != nil { s.lg.Warn("failed to rename a broken snap file", zap.String("path", fpath), zap.String("broken-path", brokenPath), zap.Error(rerr)) } else { s.lg.Warn("renamed to a broken snap file", zap.String("path", fpath), zap.String("broken-path", brokenPath)) } } return snap, err } // Read reads the snapshot named by snapname and returns the snapshot. func Read(lg *zap.Logger, snapname string) (*raftpb.Snapshot, error) { verify.Assert(lg != nil, "the logger should not be nil") b, err := os.ReadFile(snapname) if err != nil { lg.Warn("failed to read a snap file", zap.String("path", snapname), zap.Error(err)) return nil, err } if len(b) == 0 { lg.Warn("failed to read empty snapshot file", zap.String("path", snapname)) return nil, ErrEmptySnapshot } var serializedSnap snappb.Snapshot if err = serializedSnap.Unmarshal(b); err != nil { lg.Warn("failed to unmarshal snappb.Snapshot", zap.String("path", snapname), zap.Error(err)) return nil, err } if len(serializedSnap.Data) == 0 || serializedSnap.GetCrc() == 0 { lg.Warn("failed to read empty snapshot data", zap.String("path", snapname)) return nil, ErrEmptySnapshot } crc := crc32.Update(0, crcTable, serializedSnap.Data) if crc != serializedSnap.GetCrc() { lg.Warn("snap file is corrupt", zap.String("path", snapname), zap.Uint32("prev-crc", serializedSnap.GetCrc()), zap.Uint32("new-crc", crc), ) return nil, ErrCRCMismatch } var snap raftpb.Snapshot if err = snap.Unmarshal(serializedSnap.Data); err != nil { lg.Warn("failed to unmarshal raftpb.Snapshot", zap.String("path", snapname), zap.Error(err)) return nil, err } return &snap, nil } // snapNames returns the filename of the snapshots in logical time order (from newest to oldest). // If there is no available snapshots, an ErrNoSnapshot will be returned. func (s *Snapshotter) snapNames() ([]string, error) { dir, err := os.Open(s.dir) if err != nil { return nil, err } defer dir.Close() names, err := dir.Readdirnames(-1) if err != nil { return nil, err } filenames, err := s.cleanupSnapdir(names) if err != nil { return nil, err } snaps := s.checkSuffix(filenames) if len(snaps) == 0 { return nil, ErrNoSnapshot } sort.Sort(sort.Reverse(sort.StringSlice(snaps))) return snaps, nil } func (s *Snapshotter) checkSuffix(names []string) []string { var snaps []string for i := range names { if strings.HasSuffix(names[i], snapSuffix) { snaps = append(snaps, names[i]) } else { // If we find a file which is not a snapshot then check if it's // a valid file. If not throw out a warning. if _, ok := validFiles[names[i]]; !ok { s.lg.Warn("found unexpected non-snap file; skipping", zap.String("path", names[i])) } } } return snaps } // cleanupSnapdir removes any files that should not be in the snapshot directory: // - db.tmp prefixed files that can be orphaned by defragmentation func (s *Snapshotter) cleanupSnapdir(filenames []string) (names []string, err error) { names = make([]string, 0, len(filenames)) for _, filename := range filenames { if strings.HasPrefix(filename, "db.tmp") { s.lg.Info("found orphaned defragmentation file; deleting", zap.String("path", filename)) if rmErr := os.Remove(filepath.Join(s.dir, filename)); rmErr != nil && !os.IsNotExist(rmErr) { return names, fmt.Errorf("failed to remove orphaned .snap.db file %s: %w", filename, rmErr) } } else { names = append(names, filename) } } return names, nil } func (s *Snapshotter) ReleaseSnapDBs(snap raftpb.Snapshot) error { dir, err := os.Open(s.dir) if err != nil { return err } defer dir.Close() filenames, err := dir.Readdirnames(-1) if err != nil { return err } for _, filename := range filenames { if strings.HasSuffix(filename, ".snap.db") { hexIndex := strings.TrimSuffix(filepath.Base(filename), ".snap.db") index, err := strconv.ParseUint(hexIndex, 16, 64) if err != nil { s.lg.Error("failed to parse index from filename", zap.String("path", filename), zap.String("error", err.Error())) continue } if index < snap.Metadata.Index { s.lg.Info("found orphaned .snap.db file; deleting", zap.String("path", filename)) if rmErr := os.Remove(filepath.Join(s.dir, filename)); rmErr != nil && !os.IsNotExist(rmErr) { s.lg.Error("failed to remove orphaned .snap.db file", zap.String("path", filename), zap.String("error", rmErr.Error())) } } } } return nil } ================================================ FILE: server/etcdserver/api/snap/snapshotter_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package snap import ( "errors" "fmt" "hash/crc32" "os" "path/filepath" "reflect" "testing" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) var testSnap = &raftpb.Snapshot{ Data: []byte("some snapshot"), Metadata: raftpb.SnapshotMetadata{ ConfState: raftpb.ConfState{ Voters: []uint64{1, 2, 3}, }, Index: 1, Term: 1, }, } func TestSaveAndLoad(t *testing.T) { dir := filepath.Join(os.TempDir(), "snapshot") err := os.Mkdir(dir, 0o700) if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) ss := New(zaptest.NewLogger(t), dir) err = ss.save(testSnap) if err != nil { t.Fatal(err) } g, err := ss.Load() if err != nil { t.Errorf("err = %v, want nil", err) } if !reflect.DeepEqual(g, testSnap) { t.Errorf("snap = %#v, want %#v", g, testSnap) } } func TestBadCRC(t *testing.T) { dir := filepath.Join(os.TempDir(), "snapshot") err := os.Mkdir(dir, 0o700) if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) ss := New(zaptest.NewLogger(t), dir) err = ss.save(testSnap) if err != nil { t.Fatal(err) } defer func() { crcTable = crc32.MakeTable(crc32.Castagnoli) }() // switch to use another crc table // fake a crc mismatch crcTable = crc32.MakeTable(crc32.Koopman) _, err = Read(zaptest.NewLogger(t), filepath.Join(dir, fmt.Sprintf("%016x-%016x.snap", 1, 1))) if err == nil || !errors.Is(err, ErrCRCMismatch) { t.Errorf("err = %v, want %v", err, ErrCRCMismatch) } } func TestFailback(t *testing.T) { dir := filepath.Join(os.TempDir(), "snapshot") err := os.Mkdir(dir, 0o700) if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) large := fmt.Sprintf("%016x-%016x-%016x.snap", 0xFFFF, 0xFFFF, 0xFFFF) err = os.WriteFile(filepath.Join(dir, large), []byte("bad data"), 0o666) if err != nil { t.Fatal(err) } ss := New(zaptest.NewLogger(t), dir) err = ss.save(testSnap) if err != nil { t.Fatal(err) } g, err := ss.Load() if err != nil { t.Errorf("err = %v, want nil", err) } if !reflect.DeepEqual(g, testSnap) { t.Errorf("snap = %#v, want %#v", g, testSnap) } if f, err := os.Open(filepath.Join(dir, large) + ".broken"); err != nil { t.Fatal("broken snapshot does not exist") } else { f.Close() } } func TestSnapNames(t *testing.T) { dir := filepath.Join(os.TempDir(), "snapshot") err := os.Mkdir(dir, 0o700) if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) for i := 1; i <= 5; i++ { var f *os.File if f, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.snap", i))); err != nil { t.Fatal(err) } else { f.Close() } } ss := New(zaptest.NewLogger(t), dir) names, err := ss.snapNames() if err != nil { t.Errorf("err = %v, want nil", err) } if len(names) != 5 { t.Errorf("len = %d, want 10", len(names)) } w := []string{"5.snap", "4.snap", "3.snap", "2.snap", "1.snap"} if !reflect.DeepEqual(names, w) { t.Errorf("names = %v, want %v", names, w) } } func TestLoadNewestSnap(t *testing.T) { dir := filepath.Join(os.TempDir(), "snapshot") err := os.Mkdir(dir, 0o700) if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) ss := New(zaptest.NewLogger(t), dir) err = ss.save(testSnap) if err != nil { t.Fatal(err) } newSnap := *testSnap newSnap.Metadata.Index = 5 err = ss.save(&newSnap) if err != nil { t.Fatal(err) } cases := []struct { name string availableWALSnaps []walpb.Snapshot expected *raftpb.Snapshot }{ { name: "load-newest", expected: &newSnap, }, { name: "loadnewestavailable-newest", availableWALSnaps: []walpb.Snapshot{{Index: new(uint64(0)), Term: new(uint64(0))}, {Index: new(uint64(1)), Term: new(uint64(1))}, {Index: new(uint64(5)), Term: new(uint64(1))}}, expected: &newSnap, }, { name: "loadnewestavailable-newest-unsorted", availableWALSnaps: []walpb.Snapshot{{Index: new(uint64(5)), Term: new(uint64(1))}, {Index: new(uint64(1)), Term: new(uint64(1))}, {Index: new(uint64(0)), Term: new(uint64(0))}}, expected: &newSnap, }, { name: "loadnewestavailable-previous", availableWALSnaps: []walpb.Snapshot{{Index: new(uint64(0)), Term: new(uint64(0))}, {Index: new(uint64(1)), Term: new(uint64(1))}}, expected: testSnap, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { var err error var g *raftpb.Snapshot if tc.availableWALSnaps != nil { g, err = ss.LoadNewestAvailable(tc.availableWALSnaps) } else { g, err = ss.Load() } if err != nil { t.Errorf("err = %v, want nil", err) } if !reflect.DeepEqual(g, tc.expected) { t.Errorf("snap = %#v, want %#v", g, tc.expected) } }) } } func TestNoSnapshot(t *testing.T) { dir := filepath.Join(os.TempDir(), "snapshot") err := os.Mkdir(dir, 0o700) if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) ss := New(zaptest.NewLogger(t), dir) _, err = ss.Load() if !errors.Is(err, ErrNoSnapshot) { t.Errorf("err = %v, want %v", err, ErrNoSnapshot) } } func TestEmptySnapshot(t *testing.T) { dir := filepath.Join(os.TempDir(), "snapshot") err := os.Mkdir(dir, 0o700) if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) err = os.WriteFile(filepath.Join(dir, "1.snap"), []byte(""), 0x700) if err != nil { t.Fatal(err) } _, err = Read(zaptest.NewLogger(t), filepath.Join(dir, "1.snap")) if !errors.Is(err, ErrEmptySnapshot) { t.Errorf("err = %v, want %v", err, ErrEmptySnapshot) } } // TestAllSnapshotBroken ensures snapshotter returns // ErrNoSnapshot if all the snapshots are broken. func TestAllSnapshotBroken(t *testing.T) { dir := filepath.Join(os.TempDir(), "snapshot") err := os.Mkdir(dir, 0o700) if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) err = os.WriteFile(filepath.Join(dir, "1.snap"), []byte("bad"), 0x700) if err != nil { t.Fatal(err) } ss := New(zaptest.NewLogger(t), dir) _, err = ss.Load() if !errors.Is(err, ErrNoSnapshot) { t.Errorf("err = %v, want %v", err, ErrNoSnapshot) } } func TestReleaseSnapDBs(t *testing.T) { dir := filepath.Join(os.TempDir(), "snapshot") err := os.Mkdir(dir, 0o700) if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) snapIndices := []uint64{100, 200, 300, 400} for _, index := range snapIndices { filename := filepath.Join(dir, fmt.Sprintf("%016x.snap.db", index)) if err := os.WriteFile(filename, []byte("snap file\n"), 0o644); err != nil { t.Fatal(err) } } ss := New(zaptest.NewLogger(t), dir) if err := ss.ReleaseSnapDBs(raftpb.Snapshot{Metadata: raftpb.SnapshotMetadata{Index: 300}}); err != nil { t.Fatal(err) } deleted := []uint64{100, 200} for _, index := range deleted { filename := filepath.Join(dir, fmt.Sprintf("%016x.snap.db", index)) if fileutil.Exist(filename) { t.Errorf("expected %s (index: %d) to be deleted, but it still exists", filename, index) } } retained := []uint64{300, 400} for _, index := range retained { filename := filepath.Join(dir, fmt.Sprintf("%016x.snap.db", index)) if !fileutil.Exist(filename) { t.Errorf("expected %s (index: %d) to be retained, but it no longer exists", filename, index) } } } ================================================ FILE: server/etcdserver/api/v2error/error.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package v2error describes errors in etcd project. When any change happens, // https://github.com/etcd-io/website/blob/main/content/docs/v2/errorcode.md // needs to be updated correspondingly. // To be deprecated in favor of v3 APIs. package v2error import ( "encoding/json" "fmt" "net/http" ) var errors = map[int]string{ // command related errors EcodeKeyNotFound: "Key not found", EcodeTestFailed: "Compare failed", // test and set EcodeNotFile: "Not a file", ecodeNoMorePeer: "Reached the max number of peers in the cluster", EcodeNotDir: "Not a directory", EcodeNodeExist: "Key already exists", // create ecodeKeyIsPreserved: "The prefix of given key is a keyword in etcd", EcodeRootROnly: "Root is read only", EcodeDirNotEmpty: "Directory not empty", ecodeExistingPeerAddr: "Peer address has existed", EcodeUnauthorized: "The request requires user authentication", // Post form related errors ecodeValueRequired: "Value is Required in POST form", EcodePrevValueRequired: "PrevValue is Required in POST form", EcodeTTLNaN: "The given TTL in POST form is not a number", EcodeIndexNaN: "The given index in POST form is not a number", ecodeValueOrTTLRequired: "Value or TTL is required in POST form", ecodeTimeoutNaN: "The given timeout in POST form is not a number", ecodeNameRequired: "Name is required in POST form", ecodeIndexOrValueRequired: "Index or value is required", ecodeIndexValueMutex: "Index and value cannot both be specified", EcodeInvalidField: "Invalid field", EcodeInvalidForm: "Invalid POST form", EcodeRefreshValue: "Value provided on refresh", EcodeRefreshTTLRequired: "A TTL must be provided on refresh", // raft related errors EcodeRaftInternal: "Raft Internal Error", EcodeLeaderElect: "During Leader Election", // etcd related errors EcodeWatcherCleared: "watcher is cleared due to etcd recovery", EcodeEventIndexCleared: "The event in requested index is outdated and cleared", ecodeStandbyInternal: "Standby Internal Error", ecodeInvalidActiveSize: "Invalid active size", ecodeInvalidRemoveDelay: "Standby remove delay", // client related errors ecodeClientInternal: "Client Internal Error", } var errorStatus = map[int]int{ EcodeKeyNotFound: http.StatusNotFound, EcodeNotFile: http.StatusForbidden, EcodeDirNotEmpty: http.StatusForbidden, EcodeUnauthorized: http.StatusUnauthorized, EcodeTestFailed: http.StatusPreconditionFailed, EcodeNodeExist: http.StatusPreconditionFailed, EcodeRaftInternal: http.StatusInternalServerError, EcodeLeaderElect: http.StatusInternalServerError, } const ( EcodeKeyNotFound = 100 EcodeTestFailed = 101 EcodeNotFile = 102 ecodeNoMorePeer = 103 EcodeNotDir = 104 EcodeNodeExist = 105 ecodeKeyIsPreserved = 106 EcodeRootROnly = 107 EcodeDirNotEmpty = 108 ecodeExistingPeerAddr = 109 EcodeUnauthorized = 110 ecodeValueRequired = 200 EcodePrevValueRequired = 201 EcodeTTLNaN = 202 EcodeIndexNaN = 203 ecodeValueOrTTLRequired = 204 ecodeTimeoutNaN = 205 ecodeNameRequired = 206 ecodeIndexOrValueRequired = 207 ecodeIndexValueMutex = 208 EcodeInvalidField = 209 EcodeInvalidForm = 210 EcodeRefreshValue = 211 EcodeRefreshTTLRequired = 212 EcodeRaftInternal = 300 EcodeLeaderElect = 301 EcodeWatcherCleared = 400 EcodeEventIndexCleared = 401 ecodeStandbyInternal = 402 ecodeInvalidActiveSize = 403 ecodeInvalidRemoveDelay = 404 ecodeClientInternal = 500 ) type Error struct { ErrorCode int `json:"errorCode"` Message string `json:"message"` Cause string `json:"cause,omitempty"` Index uint64 `json:"index"` } func NewError(errorCode int, cause string, index uint64) *Error { return &Error{ ErrorCode: errorCode, Message: errors[errorCode], Cause: cause, Index: index, } } // Error is for the error interface func (e Error) Error() string { return e.Message + " (" + e.Cause + ")" } func (e Error) toJSONString() string { b, _ := json.Marshal(e) return string(b) } func (e Error) StatusCode() int { status, ok := errorStatus[e.ErrorCode] if !ok { status = http.StatusBadRequest } return status } func (e Error) WriteTo(w http.ResponseWriter) error { w.Header().Add("X-Etcd-Index", fmt.Sprint(e.Index)) w.Header().Set("Content-Type", "application/json") w.WriteHeader(e.StatusCode()) _, err := w.Write([]byte(e.toJSONString() + "\n")) return err } ================================================ FILE: server/etcdserver/api/v2error/error_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2error import ( "net/http" "net/http/httptest" "reflect" "strings" "testing" ) func TestErrorWriteTo(t *testing.T) { for k := range errors { err := NewError(k, "", 1) rr := httptest.NewRecorder() err.WriteTo(rr) if err.StatusCode() != rr.Code { t.Errorf("HTTP status code %d, want %d", rr.Code, err.StatusCode()) } gbody := strings.TrimSuffix(rr.Body.String(), "\n") if err.toJSONString() != gbody { t.Errorf("HTTP body %q, want %q", gbody, err.toJSONString()) } wheader := http.Header(map[string][]string{ "Content-Type": {"application/json"}, "X-Etcd-Index": {"1"}, }) if !reflect.DeepEqual(wheader, rr.HeaderMap) { //nolint:staticcheck // TODO: remove for a supported version t.Errorf("HTTP headers %v, want %v", rr.HeaderMap, wheader) //nolint:staticcheck // TODO: remove for a supported version } } } ================================================ FILE: server/etcdserver/api/v2stats/leader.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2stats import ( "encoding/json" "math" "sync" "time" "go.uber.org/zap" ) // LeaderStats is used by the leader in an etcd cluster, and encapsulates // statistics about communication with its followers type LeaderStats struct { lg *zap.Logger leaderStats sync.Mutex } type leaderStats struct { // Leader is the ID of the leader in the etcd cluster. // TODO(jonboulle): clarify that these are IDs, not names Leader string `json:"leader"` Followers map[string]*FollowerStats `json:"followers"` } // NewLeaderStats generates a new LeaderStats with the given id as leader func NewLeaderStats(lg *zap.Logger, id string) *LeaderStats { if lg == nil { lg = zap.NewNop() } return &LeaderStats{ lg: lg, leaderStats: leaderStats{ Leader: id, Followers: make(map[string]*FollowerStats), }, } } func (ls *LeaderStats) JSON() []byte { ls.Lock() stats := ls.leaderStats ls.Unlock() b, err := json.Marshal(stats) // TODO(jonboulle): appropriate error handling? if err != nil { ls.lg.Error("failed to marshal leader stats", zap.Error(err)) } return b } func (ls *LeaderStats) Follower(name string) *FollowerStats { ls.Lock() defer ls.Unlock() fs, ok := ls.Followers[name] if !ok { fs = &FollowerStats{} fs.Latency.Minimum = 1 << 63 ls.Followers[name] = fs } return fs } // FollowerStats encapsulates various statistics about a follower in an etcd cluster type FollowerStats struct { Latency LatencyStats `json:"latency"` Counts CountsStats `json:"counts"` sync.Mutex } // LatencyStats encapsulates latency statistics. type LatencyStats struct { Current float64 `json:"current"` Average float64 `json:"average"` averageSquare float64 StandardDeviation float64 `json:"standardDeviation"` Minimum float64 `json:"minimum"` Maximum float64 `json:"maximum"` } // CountsStats encapsulates raft statistics. type CountsStats struct { Fail uint64 `json:"fail"` Success uint64 `json:"success"` } // Succ updates the FollowerStats with a successful send func (fs *FollowerStats) Succ(d time.Duration) { fs.Lock() defer fs.Unlock() total := float64(fs.Counts.Success) * fs.Latency.Average totalSquare := float64(fs.Counts.Success) * fs.Latency.averageSquare fs.Counts.Success++ fs.Latency.Current = float64(d) / (1000000.0) if fs.Latency.Current > fs.Latency.Maximum { fs.Latency.Maximum = fs.Latency.Current } if fs.Latency.Current < fs.Latency.Minimum { fs.Latency.Minimum = fs.Latency.Current } fs.Latency.Average = (total + fs.Latency.Current) / float64(fs.Counts.Success) fs.Latency.averageSquare = (totalSquare + fs.Latency.Current*fs.Latency.Current) / float64(fs.Counts.Success) // sdv = sqrt(avg(x^2) - avg(x)^2) fs.Latency.StandardDeviation = math.Sqrt(fs.Latency.averageSquare - fs.Latency.Average*fs.Latency.Average) } // Fail updates the FollowerStats with an unsuccessful send func (fs *FollowerStats) Fail() { fs.Lock() defer fs.Unlock() fs.Counts.Fail++ } ================================================ FILE: server/etcdserver/api/v2stats/queue.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2stats import ( "sync" "time" ) const ( queueCapacity = 200 ) // RequestStats represent the stats for a request. // It encapsulates the sending time and the size of the request. type RequestStats struct { SendingTime time.Time Size int } type statsQueue struct { items [queueCapacity]*RequestStats size int front int back int totalReqSize int rwl sync.RWMutex } func (q *statsQueue) Len() int { return q.size } func (q *statsQueue) ReqSize() int { return q.totalReqSize } // FrontAndBack gets the front and back elements in the queue // We must grab front and back together with the protection of the lock func (q *statsQueue) frontAndBack() (*RequestStats, *RequestStats) { q.rwl.RLock() defer q.rwl.RUnlock() if q.size != 0 { return q.items[q.front], q.items[q.back] } return nil, nil } // Insert function insert a RequestStats into the queue and update the records func (q *statsQueue) Insert(p *RequestStats) { q.rwl.Lock() defer q.rwl.Unlock() q.back = (q.back + 1) % queueCapacity if q.size == queueCapacity { // dequeue q.totalReqSize -= q.items[q.front].Size q.front = (q.back + 1) % queueCapacity } else { q.size++ } q.items[q.back] = p q.totalReqSize += q.items[q.back].Size } // Rate function returns the package rate and byte rate func (q *statsQueue) Rate() (float64, float64) { front, back := q.frontAndBack() if front == nil || back == nil { return 0, 0 } if time.Since(back.SendingTime) > time.Second { q.Clear() return 0, 0 } sampleDuration := back.SendingTime.Sub(front.SendingTime) pr := float64(q.Len()) / float64(sampleDuration) * float64(time.Second) br := float64(q.ReqSize()) / float64(sampleDuration) * float64(time.Second) return pr, br } // Clear function clear up the statsQueue func (q *statsQueue) Clear() { q.rwl.Lock() defer q.rwl.Unlock() q.back = -1 q.front = 0 q.size = 0 q.totalReqSize = 0 } ================================================ FILE: server/etcdserver/api/v2stats/server.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2stats import ( "encoding/json" "log" "sync" "time" "go.etcd.io/raft/v3" ) // ServerStats encapsulates various statistics about an EtcdServer and its // communication with other members of the cluster type ServerStats struct { serverStats sync.Mutex } func NewServerStats(name, id string) *ServerStats { ss := &ServerStats{ serverStats: serverStats{ Name: name, ID: id, }, } now := time.Now() ss.StartTime = now ss.LeaderInfo.StartTime = now ss.sendRateQueue = &statsQueue{back: -1} ss.recvRateQueue = &statsQueue{back: -1} return ss } type serverStats struct { Name string `json:"name"` // ID is the raft ID of the node. // TODO(jonboulle): use ID instead of name? ID string `json:"id"` State raft.StateType `json:"state"` StartTime time.Time `json:"startTime"` LeaderInfo struct { Name string `json:"leader"` Uptime string `json:"uptime"` StartTime time.Time `json:"startTime"` } `json:"leaderInfo"` RecvAppendRequestCnt uint64 `json:"recvAppendRequestCnt"` RecvingPkgRate float64 `json:"recvPkgRate,omitempty"` RecvingBandwidthRate float64 `json:"recvBandwidthRate,omitempty"` SendAppendRequestCnt uint64 `json:"sendAppendRequestCnt"` SendingPkgRate float64 `json:"sendPkgRate,omitempty"` SendingBandwidthRate float64 `json:"sendBandwidthRate,omitempty"` sendRateQueue *statsQueue recvRateQueue *statsQueue } func (ss *ServerStats) JSON() []byte { ss.Lock() stats := ss.serverStats stats.SendingPkgRate, stats.SendingBandwidthRate = stats.sendRateQueue.Rate() stats.RecvingPkgRate, stats.RecvingBandwidthRate = stats.recvRateQueue.Rate() stats.LeaderInfo.Uptime = time.Since(stats.LeaderInfo.StartTime).String() ss.Unlock() b, err := json.Marshal(stats) // TODO(jonboulle): appropriate error handling? if err != nil { log.Printf("stats: error marshalling server stats: %v", err) } return b } // RecvAppendReq updates the ServerStats in response to an AppendRequest // from the given leader being received func (ss *ServerStats) RecvAppendReq(leader string, reqSize int) { ss.Lock() defer ss.Unlock() now := time.Now() ss.State = raft.StateFollower if leader != ss.LeaderInfo.Name { ss.LeaderInfo.Name = leader ss.LeaderInfo.StartTime = now } ss.recvRateQueue.Insert( &RequestStats{ SendingTime: now, Size: reqSize, }, ) ss.RecvAppendRequestCnt++ } // SendAppendReq updates the ServerStats in response to an AppendRequest // being sent by this server func (ss *ServerStats) SendAppendReq(reqSize int) { ss.Lock() defer ss.Unlock() ss.becomeLeader() ss.sendRateQueue.Insert( &RequestStats{ SendingTime: time.Now(), Size: reqSize, }, ) ss.SendAppendRequestCnt++ } func (ss *ServerStats) BecomeLeader() { ss.Lock() defer ss.Unlock() ss.becomeLeader() } func (ss *ServerStats) becomeLeader() { if ss.State != raft.StateLeader { ss.State = raft.StateLeader ss.LeaderInfo.Name = ss.ID ss.LeaderInfo.StartTime = time.Now() } } ================================================ FILE: server/etcdserver/api/v2store/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package v2store defines etcd's in-memory key/value store in v2 API. // To be deprecated in favor of v3 storage. package v2store ================================================ FILE: server/etcdserver/api/v2store/event.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store const ( Get = "get" Create = "create" Set = "set" Update = "update" Delete = "delete" CompareAndSwap = "compareAndSwap" CompareAndDelete = "compareAndDelete" Expire = "expire" ) type Event struct { Action string `json:"action"` Node *NodeExtern `json:"node,omitempty"` PrevNode *NodeExtern `json:"prevNode,omitempty"` EtcdIndex uint64 `json:"-"` Refresh bool `json:"refresh,omitempty"` } func newEvent(action string, key string, modifiedIndex, createdIndex uint64) *Event { n := &NodeExtern{ Key: key, ModifiedIndex: modifiedIndex, CreatedIndex: createdIndex, } return &Event{ Action: action, Node: n, } } func (e *Event) IsCreated() bool { if e.Action == Create { return true } return e.Action == Set && e.PrevNode == nil } func (e *Event) Index() uint64 { return e.Node.ModifiedIndex } func (e *Event) Clone() *Event { return &Event{ Action: e.Action, EtcdIndex: e.EtcdIndex, Node: e.Node.Clone(), PrevNode: e.PrevNode.Clone(), } } func (e *Event) SetRefresh() { e.Refresh = true } ================================================ FILE: server/etcdserver/api/v2store/event_history.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "fmt" "path" "strings" "sync" "go.etcd.io/etcd/server/v3/etcdserver/api/v2error" ) type EventHistory struct { Queue eventQueue StartIndex uint64 LastIndex uint64 rwl sync.RWMutex } func newEventHistory(capacity int) *EventHistory { return &EventHistory{ Queue: eventQueue{ Capacity: capacity, Events: make([]*Event, capacity), }, } } // addEvent function adds event into the eventHistory func (eh *EventHistory) addEvent(e *Event) *Event { eh.rwl.Lock() defer eh.rwl.Unlock() eh.Queue.insert(e) eh.LastIndex = e.Index() eh.StartIndex = eh.Queue.Events[eh.Queue.Front].Index() return e } // scan enumerates events from the index history and stops at the first point // where the key matches. func (eh *EventHistory) scan(key string, recursive bool, index uint64) (*Event, *v2error.Error) { eh.rwl.RLock() defer eh.rwl.RUnlock() // index should be after the event history's StartIndex if index < eh.StartIndex { return nil, v2error.NewError(v2error.EcodeEventIndexCleared, fmt.Sprintf("the requested history has been cleared [%v/%v]", eh.StartIndex, index), 0) } // the index should come before the size of the queue minus the duplicate count if index > eh.LastIndex { // future index return nil, nil } offset := index - eh.StartIndex i := (eh.Queue.Front + int(offset)) % eh.Queue.Capacity for { e := eh.Queue.Events[i] if !e.Refresh { ok := e.Node.Key == key if recursive { // add tailing slash nkey := path.Clean(key) if nkey[len(nkey)-1] != '/' { nkey = nkey + "/" } ok = ok || strings.HasPrefix(e.Node.Key, nkey) } if (e.Action == Delete || e.Action == Expire) && e.PrevNode != nil && e.PrevNode.Dir { ok = ok || strings.HasPrefix(key, e.PrevNode.Key) } if ok { return e, nil } } i = (i + 1) % eh.Queue.Capacity if i == eh.Queue.Back { return nil, nil } } } // clone will be protected by a stop-world lock // do not need to obtain internal lock func (eh *EventHistory) clone() *EventHistory { clonedQueue := eventQueue{ Capacity: eh.Queue.Capacity, Events: make([]*Event, eh.Queue.Capacity), Size: eh.Queue.Size, Front: eh.Queue.Front, Back: eh.Queue.Back, } copy(clonedQueue.Events, eh.Queue.Events) return &EventHistory{ StartIndex: eh.StartIndex, Queue: clonedQueue, LastIndex: eh.LastIndex, } } ================================================ FILE: server/etcdserver/api/v2store/event_queue.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store type eventQueue struct { Events []*Event Size int Front int Back int Capacity int } func (eq *eventQueue) insert(e *Event) { eq.Events[eq.Back] = e eq.Back = (eq.Back + 1) % eq.Capacity if eq.Size == eq.Capacity { // dequeue eq.Front = (eq.Front + 1) % eq.Capacity } else { eq.Size++ } } ================================================ FILE: server/etcdserver/api/v2store/event_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "testing" "go.etcd.io/etcd/server/v3/etcdserver/api/v2error" ) // TestEventQueue tests a queue with capacity = 100 // Add 200 events into that queue, and test if the // previous 100 events have been swapped out. func TestEventQueue(t *testing.T) { eh := newEventHistory(100) // Add for i := 0; i < 200; i++ { e := newEvent(Create, "/foo", uint64(i), uint64(i)) eh.addEvent(e) } // Test j := 100 i := eh.Queue.Front n := eh.Queue.Size for ; n > 0; n-- { e := eh.Queue.Events[i] if e.Index() != uint64(j) { t.Fatalf("queue error!") } j++ i = (i + 1) % eh.Queue.Capacity } } func TestScanHistory(t *testing.T) { eh := newEventHistory(100) // Add eh.addEvent(newEvent(Create, "/foo", 1, 1)) eh.addEvent(newEvent(Create, "/foo/bar", 2, 2)) eh.addEvent(newEvent(Create, "/foo/foo", 3, 3)) eh.addEvent(newEvent(Create, "/foo/bar/bar", 4, 4)) eh.addEvent(newEvent(Create, "/foo/foo/foo", 5, 5)) // Delete a dir de := newEvent(Delete, "/foo", 6, 6) de.PrevNode = newDir(nil, "/foo", 1, nil, Permanent).Repr(false, false, nil) eh.addEvent(de) e, err := eh.scan("/foo", false, 1) if err != nil || e.Index() != 1 { t.Fatalf("scan error [/foo] [1] %d (%v)", e.Index(), err) } e, err = eh.scan("/foo/bar", false, 1) if err != nil || e.Index() != 2 { t.Fatalf("scan error [/foo/bar] [2] %d (%v)", e.Index(), err) } e, err = eh.scan("/foo/bar", true, 3) if err != nil || e.Index() != 4 { t.Fatalf("scan error [/foo/bar/bar] [4] %d (%v)", e.Index(), err) } e, err = eh.scan("/foo/foo/foo", false, 6) if err != nil || e.Index() != 6 { t.Fatalf("scan error [/foo/foo/foo] [6] %d (%v)", e.Index(), err) } e, _ = eh.scan("/foo/bar", true, 7) if e != nil { t.Fatalf("bad index shoud reuturn nil") } } func TestEventIndexHistoryCleared(t *testing.T) { eh := newEventHistory(5) // Add eh.addEvent(newEvent(Create, "/foo", 1, 1)) eh.addEvent(newEvent(Create, "/foo/bar", 2, 2)) eh.addEvent(newEvent(Create, "/foo/foo", 3, 3)) eh.addEvent(newEvent(Create, "/foo/bar/bar", 4, 4)) eh.addEvent(newEvent(Create, "/foo/foo/foo", 5, 5)) // Add a new event which will replace/de-queue the first entry eh.addEvent(newEvent(Create, "/foo/bar/bar/bar", 6, 6)) // test for the event which has been replaced. _, err := eh.scan("/foo", false, 1) if err == nil || err.ErrorCode != v2error.EcodeEventIndexCleared { t.Fatalf("scan error cleared index should return err with %d got (%v)", v2error.EcodeEventIndexCleared, err) } } // TestFullEventQueue tests a queue with capacity = 10 // Add 1000 events into that queue, and test if scanning // works still for previous events. func TestFullEventQueue(t *testing.T) { eh := newEventHistory(10) // Add for i := 0; i < 1000; i++ { ce := newEvent(Create, "/foo", uint64(i), uint64(i)) eh.addEvent(ce) e, err := eh.scan("/foo", true, uint64(i-1)) if i > 0 { if e == nil || err != nil { t.Fatalf("scan error [/foo] [%v] %v", i-1, i) } } } } func TestCloneEvent(t *testing.T) { e1 := &Event{ Action: Create, EtcdIndex: 1, Node: nil, PrevNode: nil, } e2 := e1.Clone() if e2.Action != Create { t.Fatalf("Action=%q, want %q", e2.Action, Create) } if e2.EtcdIndex != e1.EtcdIndex { t.Fatalf("EtcdIndex=%d, want %d", e2.EtcdIndex, e1.EtcdIndex) } // Changing the cloned node should not affect the original e2.Action = Delete e2.EtcdIndex = uint64(5) if e1.Action != Create { t.Fatalf("Action=%q, want %q", e1.Action, Create) } if e1.EtcdIndex != uint64(1) { t.Fatalf("EtcdIndex=%d, want %d", e1.EtcdIndex, uint64(1)) } if e2.Action != Delete { t.Fatalf("Action=%q, want %q", e2.Action, Delete) } if e2.EtcdIndex != uint64(5) { t.Fatalf("EtcdIndex=%d, want %d", e2.EtcdIndex, uint64(5)) } } ================================================ FILE: server/etcdserver/api/v2store/heap_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "fmt" "testing" "time" ) func TestHeapPushPop(t *testing.T) { h := newTTLKeyHeap() // add from older expire time to earlier expire time // the path is equal to ttl from now for i := 0; i < 10; i++ { path := fmt.Sprintf("%v", 10-i) m := time.Duration(10 - i) n := newKV(nil, path, path, 0, nil, time.Now().Add(time.Second*m)) h.push(n) } min := time.Now() for i := 0; i < 10; i++ { node := h.pop() if node.ExpireTime.Before(min) { t.Fatal("heap sort wrong!") } min = node.ExpireTime } } func TestHeapUpdate(t *testing.T) { h := newTTLKeyHeap() kvs := make([]*node, 10) // add from older expire time to earlier expire time // the path is equal to ttl from now for i := range kvs { path := fmt.Sprintf("%v", 10-i) m := time.Duration(10 - i) n := newKV(nil, path, path, 0, nil, time.Now().Add(time.Second*m)) kvs[i] = n h.push(n) } // Path 7 kvs[3].ExpireTime = time.Now().Add(time.Second * 11) // Path 5 kvs[5].ExpireTime = time.Now().Add(time.Second * 12) h.update(kvs[3]) h.update(kvs[5]) min := time.Now() for i := 0; i < 10; i++ { node := h.pop() if node.ExpireTime.Before(min) { t.Fatal("heap sort wrong!") } min = node.ExpireTime if i == 8 { if node.Path != "7" { t.Fatal("heap sort wrong!", node.Path) } } if i == 9 { if node.Path != "5" { t.Fatal("heap sort wrong!") } } } } ================================================ FILE: server/etcdserver/api/v2store/metrics.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import "github.com/prometheus/client_golang/prometheus" // Set of raw Prometheus metrics. // Labels // * action = declared in event.go // * outcome = Outcome // Do not increment directly, use Report* methods. var ( readCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "store", Name: "reads_total", Help: "Total number of reads action by (get/getRecursive), local to this member.", }, []string{"action"}, ) writeCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "store", Name: "writes_total", Help: "Total number of writes (e.g. set/compareAndDelete) seen by this member.", }, []string{"action"}, ) readFailedCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "store", Name: "reads_failed_total", Help: "Failed read actions by (get/getRecursive), local to this member.", }, []string{"action"}, ) writeFailedCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "store", Name: "writes_failed_total", Help: "Failed write actions (e.g. set/compareAndDelete), seen by this member.", }, []string{"action"}, ) expireCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "store", Name: "expires_total", Help: "Total number of expired keys.", }, ) watchRequests = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "store", Name: "watch_requests_total", Help: "Total number of incoming watch requests (new or reestablished).", }, ) watcherCount = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "store", Name: "watchers", Help: "Count of currently active watchers.", }, ) ) const ( GetRecursive = "getRecursive" ) func init() { if prometheus.Register(readCounter) != nil { // Tests will try to double register since the tests use both // store and store_test packages; ignore second attempts. return } prometheus.MustRegister(writeCounter) prometheus.MustRegister(expireCounter) prometheus.MustRegister(watchRequests) prometheus.MustRegister(watcherCount) } func reportReadSuccess(readAction string) { readCounter.WithLabelValues(readAction).Inc() } func reportReadFailure(readAction string) { readCounter.WithLabelValues(readAction).Inc() readFailedCounter.WithLabelValues(readAction).Inc() } func reportWriteSuccess(writeAction string) { writeCounter.WithLabelValues(writeAction).Inc() } func reportWriteFailure(writeAction string) { writeCounter.WithLabelValues(writeAction).Inc() writeFailedCounter.WithLabelValues(writeAction).Inc() } func reportExpiredKey() { expireCounter.Inc() } func reportWatchRequest() { watchRequests.Inc() } func reportWatcherAdded() { watcherCount.Inc() } func reportWatcherRemoved() { watcherCount.Dec() } ================================================ FILE: server/etcdserver/api/v2store/node.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "path" "sort" "time" "github.com/jonboulle/clockwork" "go.etcd.io/etcd/server/v3/etcdserver/api/v2error" ) // explanations of Compare function result const ( CompareMatch = iota CompareIndexNotMatch CompareValueNotMatch CompareNotMatch ) var Permanent time.Time // node is the basic element in the store system. // A key-value pair will have a string value // A directory will have a children map type node struct { Path string CreatedIndex uint64 ModifiedIndex uint64 Parent *node `json:"-"` // should not encode this field! avoid circular dependency. ExpireTime time.Time Value string // for key-value pair Children map[string]*node // for directory // A reference to the store this node is attached to. store *store } // newKV creates a Key-Value pair func newKV(store *store, nodePath string, value string, createdIndex uint64, parent *node, expireTime time.Time) *node { return &node{ Path: nodePath, CreatedIndex: createdIndex, ModifiedIndex: createdIndex, Parent: parent, store: store, ExpireTime: expireTime, Value: value, } } // newDir creates a directory func newDir(store *store, nodePath string, createdIndex uint64, parent *node, expireTime time.Time) *node { return &node{ Path: nodePath, CreatedIndex: createdIndex, ModifiedIndex: createdIndex, Parent: parent, ExpireTime: expireTime, Children: make(map[string]*node), store: store, } } // IsHidden function checks if the node is a hidden node. A hidden node // will begin with '_' // A hidden node will not be shown via get command under a directory // For example if we have /foo/_hidden and /foo/notHidden, get "/foo" // will only return /foo/notHidden func (n *node) IsHidden() bool { _, name := path.Split(n.Path) return name[0] == '_' } // IsPermanent function checks if the node is a permanent one. func (n *node) IsPermanent() bool { // we use a uninitialized time.Time to indicate the node is a // permanent one. // the uninitialized time.Time should equal zero. return n.ExpireTime.IsZero() } // IsDir function checks whether the node is a directory. // If the node is a directory, the function will return true. // Otherwise the function will return false. func (n *node) IsDir() bool { return n.Children != nil } // Read function gets the value of the node. // If the receiver node is not a key-value pair, a "Not A File" error will be returned. func (n *node) Read() (string, *v2error.Error) { if n.IsDir() { return "", v2error.NewError(v2error.EcodeNotFile, "", n.store.CurrentIndex) } return n.Value, nil } // Write function set the value of the node to the given value. // If the receiver node is a directory, a "Not A File" error will be returned. func (n *node) Write(value string, index uint64) *v2error.Error { if n.IsDir() { return v2error.NewError(v2error.EcodeNotFile, "", n.store.CurrentIndex) } n.Value = value n.ModifiedIndex = index return nil } func (n *node) expirationAndTTL(clock clockwork.Clock) (*time.Time, int64) { if !n.IsPermanent() { /* compute ttl as: ceiling( (expireTime - timeNow) / nanosecondsPerSecond ) which ranges from 1..n rather than as: ( (expireTime - timeNow) / nanosecondsPerSecond ) + 1 which ranges 1..n+1 */ ttlN := n.ExpireTime.Sub(clock.Now()) ttl := ttlN / time.Second if (ttlN % time.Second) > 0 { ttl++ } t := n.ExpireTime.UTC() return &t, int64(ttl) } return nil, 0 } // List function return a slice of nodes under the receiver node. // If the receiver node is not a directory, a "Not A Directory" error will be returned. func (n *node) List() ([]*node, *v2error.Error) { if !n.IsDir() { return nil, v2error.NewError(v2error.EcodeNotDir, "", n.store.CurrentIndex) } nodes := make([]*node, len(n.Children)) i := 0 for _, node := range n.Children { nodes[i] = node i++ } return nodes, nil } // GetChild function returns the child node under the directory node. // On success, it returns the file node func (n *node) GetChild(name string) (*node, *v2error.Error) { if !n.IsDir() { return nil, v2error.NewError(v2error.EcodeNotDir, n.Path, n.store.CurrentIndex) } child, ok := n.Children[name] if ok { return child, nil } return nil, nil } // Add function adds a node to the receiver node. // If the receiver is not a directory, a "Not A Directory" error will be returned. // If there is an existing node with the same name under the directory, a "Already Exist" // error will be returned func (n *node) Add(child *node) *v2error.Error { if !n.IsDir() { return v2error.NewError(v2error.EcodeNotDir, "", n.store.CurrentIndex) } _, name := path.Split(child.Path) if _, ok := n.Children[name]; ok { return v2error.NewError(v2error.EcodeNodeExist, "", n.store.CurrentIndex) } n.Children[name] = child return nil } // Remove function remove the node. func (n *node) Remove(dir, recursive bool, callback func(path string)) *v2error.Error { if !n.IsDir() { // key-value pair _, name := path.Split(n.Path) // find its parent and remove the node from the map if n.Parent != nil && n.Parent.Children[name] == n { delete(n.Parent.Children, name) } if callback != nil { callback(n.Path) } if !n.IsPermanent() { n.store.ttlKeyHeap.remove(n) } return nil } if !dir { // cannot delete a directory without dir set to true return v2error.NewError(v2error.EcodeNotFile, n.Path, n.store.CurrentIndex) } if len(n.Children) != 0 && !recursive { // cannot delete a directory if it is not empty and the operation // is not recursive return v2error.NewError(v2error.EcodeDirNotEmpty, n.Path, n.store.CurrentIndex) } for _, child := range n.Children { // delete all children child.Remove(true, true, callback) } // delete self _, name := path.Split(n.Path) if n.Parent != nil && n.Parent.Children[name] == n { delete(n.Parent.Children, name) if callback != nil { callback(n.Path) } if !n.IsPermanent() { n.store.ttlKeyHeap.remove(n) } } return nil } func (n *node) Repr(recursive, sorted bool, clock clockwork.Clock) *NodeExtern { if n.IsDir() { node := &NodeExtern{ Key: n.Path, Dir: true, ModifiedIndex: n.ModifiedIndex, CreatedIndex: n.CreatedIndex, } node.Expiration, node.TTL = n.expirationAndTTL(clock) if !recursive { return node } children, _ := n.List() node.Nodes = make(NodeExterns, len(children)) // we do not use the index in the children slice directly // we need to skip the hidden one i := 0 for _, child := range children { if child.IsHidden() { // get will not list hidden node continue } node.Nodes[i] = child.Repr(recursive, sorted, clock) i++ } // eliminate hidden nodes node.Nodes = node.Nodes[:i] if sorted { sort.Sort(node.Nodes) } return node } // since n.Value could be changed later, so we need to copy the value out value := n.Value node := &NodeExtern{ Key: n.Path, Value: &value, ModifiedIndex: n.ModifiedIndex, CreatedIndex: n.CreatedIndex, } node.Expiration, node.TTL = n.expirationAndTTL(clock) return node } func (n *node) UpdateTTL(expireTime time.Time) { if !n.IsPermanent() { if expireTime.IsZero() { // from ttl to permanent n.ExpireTime = expireTime // remove from ttl heap n.store.ttlKeyHeap.remove(n) return } // update ttl n.ExpireTime = expireTime // update ttl heap n.store.ttlKeyHeap.update(n) return } if expireTime.IsZero() { return } // from permanent to ttl n.ExpireTime = expireTime // push into ttl heap n.store.ttlKeyHeap.push(n) } // Compare function compares node index and value with provided ones. // second result value explains result and equals to one of Compare.. constants func (n *node) Compare(prevValue string, prevIndex uint64) (ok bool, which int) { indexMatch := prevIndex == 0 || n.ModifiedIndex == prevIndex valueMatch := prevValue == "" || n.Value == prevValue ok = valueMatch && indexMatch switch { case valueMatch && indexMatch: which = CompareMatch case indexMatch && !valueMatch: which = CompareValueNotMatch case valueMatch && !indexMatch: which = CompareIndexNotMatch default: which = CompareNotMatch } return ok, which } // Clone function clone the node recursively and return the new node. // If the node is a directory, it will clone all the content under this directory. // If the node is a key-value pair, it will clone the pair. func (n *node) Clone() *node { if !n.IsDir() { newkv := newKV(n.store, n.Path, n.Value, n.CreatedIndex, n.Parent, n.ExpireTime) newkv.ModifiedIndex = n.ModifiedIndex return newkv } clone := newDir(n.store, n.Path, n.CreatedIndex, n.Parent, n.ExpireTime) clone.ModifiedIndex = n.ModifiedIndex for key, child := range n.Children { clone.Children[key] = child.Clone() } return clone } // recoverAndclean function help to do recovery. // Two things need to be done: 1. recovery structure; 2. delete expired nodes // // If the node is a directory, it will help recover children's parent pointer and recursively // call this function on its children. // We check the expire last since we need to recover the whole structure first and add all the // notifications into the event history. func (n *node) recoverAndclean() { if n.IsDir() { for _, child := range n.Children { child.Parent = n child.store = n.store child.recoverAndclean() } } if !n.ExpireTime.IsZero() { n.store.ttlKeyHeap.push(n) } } ================================================ FILE: server/etcdserver/api/v2store/node_extern.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "sort" "time" "github.com/jonboulle/clockwork" ) // NodeExtern is the external representation of the // internal node with additional fields // PrevValue is the previous value of the node // TTL is time to live in second type NodeExtern struct { Key string `json:"key,omitempty"` Value *string `json:"value,omitempty"` Dir bool `json:"dir,omitempty"` Expiration *time.Time `json:"expiration,omitempty"` TTL int64 `json:"ttl,omitempty"` Nodes NodeExterns `json:"nodes,omitempty"` ModifiedIndex uint64 `json:"modifiedIndex,omitempty"` CreatedIndex uint64 `json:"createdIndex,omitempty"` } func (eNode *NodeExtern) loadInternalNode(n *node, recursive, sorted bool, clock clockwork.Clock) { if n.IsDir() { // node is a directory eNode.Dir = true children, _ := n.List() eNode.Nodes = make(NodeExterns, len(children)) // we do not use the index in the children slice directly // we need to skip the hidden one i := 0 for _, child := range children { if child.IsHidden() { // get will not return hidden nodes continue } eNode.Nodes[i] = child.Repr(recursive, sorted, clock) i++ } // eliminate hidden nodes eNode.Nodes = eNode.Nodes[:i] if sorted { sort.Sort(eNode.Nodes) } } else { // node is a file value, _ := n.Read() eNode.Value = &value } eNode.Expiration, eNode.TTL = n.expirationAndTTL(clock) } func (eNode *NodeExtern) Clone() *NodeExtern { if eNode == nil { return nil } nn := &NodeExtern{ Key: eNode.Key, Dir: eNode.Dir, TTL: eNode.TTL, ModifiedIndex: eNode.ModifiedIndex, CreatedIndex: eNode.CreatedIndex, } if eNode.Value != nil { s := *eNode.Value nn.Value = &s } if eNode.Expiration != nil { t := *eNode.Expiration nn.Expiration = &t } if eNode.Nodes != nil { nn.Nodes = make(NodeExterns, len(eNode.Nodes)) for i, n := range eNode.Nodes { nn.Nodes[i] = n.Clone() } } return nn } type NodeExterns []*NodeExtern // interfaces for sorting func (ns NodeExterns) Len() int { return len(ns) } func (ns NodeExterns) Less(i, j int) bool { return ns[i].Key < ns[j].Key } func (ns NodeExterns) Swap(i, j int) { ns[i], ns[j] = ns[j], ns[i] } ================================================ FILE: server/etcdserver/api/v2store/node_extern_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "reflect" "testing" "time" "github.com/stretchr/testify/assert" ) func TestNodeExternClone(t *testing.T) { var eNode *NodeExtern if g := eNode.Clone(); g != nil { t.Fatalf("nil.Clone=%v, want nil", g) } const ( key string = "/foo/bar" ttl int64 = 123456789 ci uint64 = 123 mi uint64 = 321 ) var ( val = "some_data" valp = &val exp = time.Unix(12345, 67890) expp = &exp child = NodeExtern{} childp = &child childs = []*NodeExtern{childp} ) eNode = &NodeExtern{ Key: key, TTL: ttl, CreatedIndex: ci, ModifiedIndex: mi, Value: valp, Expiration: expp, Nodes: childs, } gNode := eNode.Clone() // Check the clone is as expected assert.Equal(t, key, gNode.Key) assert.Equal(t, ttl, gNode.TTL) assert.Equal(t, ci, gNode.CreatedIndex) assert.Equal(t, mi, gNode.ModifiedIndex) // values should be the same assert.Equal(t, val, *gNode.Value) assert.Equal(t, exp, *gNode.Expiration) assert.Len(t, gNode.Nodes, len(childs)) assert.Equal(t, child, *gNode.Nodes[0]) // but pointers should differ if gNode.Value == eNode.Value { t.Fatalf("expected value pointers to differ, but got same!") } if gNode.Expiration == eNode.Expiration { t.Fatalf("expected expiration pointers to differ, but got same!") } if sameSlice(gNode.Nodes, eNode.Nodes) { t.Fatalf("expected nodes pointers to differ, but got same!") } // Original should be the same assert.Equal(t, key, eNode.Key) assert.Equal(t, ttl, eNode.TTL) assert.Equal(t, ci, eNode.CreatedIndex) assert.Equal(t, mi, eNode.ModifiedIndex) assert.Equal(t, valp, eNode.Value) assert.Equal(t, expp, eNode.Expiration) if !sameSlice(eNode.Nodes, childs) { t.Fatalf("expected nodes pointer to same, but got different!") } // Change the clone and ensure the original is not affected gNode.Key = "/baz" gNode.TTL = 0 gNode.Nodes[0].Key = "uno" assert.Equal(t, key, eNode.Key) assert.Equal(t, ttl, eNode.TTL) assert.Equal(t, ci, eNode.CreatedIndex) assert.Equal(t, mi, eNode.ModifiedIndex) assert.Equal(t, child, *eNode.Nodes[0]) // Change the original and ensure the clone is not affected eNode.Key = "/wuf" assert.Equal(t, "/wuf", eNode.Key) assert.Equal(t, "/baz", gNode.Key) } func sameSlice(a, b []*NodeExtern) bool { va := reflect.ValueOf(a) vb := reflect.ValueOf(b) return va.Len() == vb.Len() && va.Pointer() == vb.Pointer() } ================================================ FILE: server/etcdserver/api/v2store/node_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "testing" "time" "github.com/jonboulle/clockwork" ) var ( key, val = "foo", "bar" val1, val2 = "bar1", "bar2" expiration = time.Minute ) func TestNewKVIs(t *testing.T) { nd := newTestNode() if nd.IsHidden() { t.Errorf("nd.Hidden() = %v, want = false", nd.IsHidden()) } if nd.IsPermanent() { t.Errorf("nd.IsPermanent() = %v, want = false", nd.IsPermanent()) } if nd.IsDir() { t.Errorf("nd.IsDir() = %v, want = false", nd.IsDir()) } } func TestNewKVReadWriteCompare(t *testing.T) { nd := newTestNode() if v, err := nd.Read(); v != val || err != nil { t.Errorf("value = %s and err = %v, want value = %s and err = nil", v, err, val) } if err := nd.Write(val1, nd.CreatedIndex+1); err != nil { t.Errorf("nd.Write error = %v, want = nil", err) } else { if v, err := nd.Read(); v != val1 || err != nil { t.Errorf("value = %s and err = %v, want value = %s and err = nil", v, err, val1) } } if err := nd.Write(val2, nd.CreatedIndex+2); err != nil { t.Errorf("nd.Write error = %v, want = nil", err) } else { if v, err := nd.Read(); v != val2 || err != nil { t.Errorf("value = %s and err = %v, want value = %s and err = nil", v, err, val2) } } if ok, which := nd.Compare(val2, 2); !ok || which != 0 { t.Errorf("ok = %v and which = %d, want ok = true and which = 0", ok, which) } } func TestNewKVExpiration(t *testing.T) { nd := newTestNode() if _, ttl := nd.expirationAndTTL(clockwork.NewFakeClock()); ttl > expiration.Nanoseconds() { t.Errorf("ttl = %d, want %d < %d", ttl, ttl, expiration.Nanoseconds()) } newExpiration := time.Hour nd.UpdateTTL(time.Now().Add(newExpiration)) if _, ttl := nd.expirationAndTTL(clockwork.NewFakeClock()); ttl > newExpiration.Nanoseconds() { t.Errorf("ttl = %d, want %d < %d", ttl, ttl, newExpiration.Nanoseconds()) } if ns, err := nd.List(); ns != nil || err == nil { t.Errorf("nodes = %v and err = %v, want nodes = nil and err != nil", ns, err) } en := nd.Repr(false, false, clockwork.NewFakeClock()) if en.Key != nd.Path { t.Errorf("en.Key = %s, want = %s", en.Key, nd.Path) } if *(en.Value) != nd.Value { t.Errorf("*(en.Key) = %s, want = %s", *(en.Value), nd.Value) } } func TestNewKVListReprCompareClone(t *testing.T) { nd := newTestNode() if ns, err := nd.List(); ns != nil || err == nil { t.Errorf("nodes = %v and err = %v, want nodes = nil and err != nil", ns, err) } en := nd.Repr(false, false, clockwork.NewFakeClock()) if en.Key != nd.Path { t.Errorf("en.Key = %s, want = %s", en.Key, nd.Path) } if *(en.Value) != nd.Value { t.Errorf("*(en.Key) = %s, want = %s", *(en.Value), nd.Value) } cn := nd.Clone() if cn.Path != nd.Path { t.Errorf("cn.Path = %s, want = %s", cn.Path, nd.Path) } if cn.Value != nd.Value { t.Errorf("cn.Value = %s, want = %s", cn.Value, nd.Value) } } func TestNewKVRemove(t *testing.T) { nd := newTestNode() if v, err := nd.Read(); v != val || err != nil { t.Errorf("value = %s and err = %v, want value = %s and err = nil", v, err, val) } if err := nd.Write(val1, nd.CreatedIndex+1); err != nil { t.Errorf("nd.Write error = %v, want = nil", err) } else { if v, err := nd.Read(); v != val1 || err != nil { t.Errorf("value = %s and err = %v, want value = %s and err = nil", v, err, val1) } } if err := nd.Write(val2, nd.CreatedIndex+2); err != nil { t.Errorf("nd.Write error = %v, want = nil", err) } else { if v, err := nd.Read(); v != val2 || err != nil { t.Errorf("value = %s and err = %v, want value = %s and err = nil", v, err, val2) } } if err := nd.Remove(false, false, nil); err != nil { t.Errorf("nd.Remove err = %v, want = nil", err) } else { // still readable if v, err := nd.Read(); v != val2 || err != nil { t.Errorf("value = %s and err = %v, want value = %s and err = nil", v, err, val2) } if len(nd.store.ttlKeyHeap.array) != 0 { t.Errorf("len(nd.store.ttlKeyHeap.array) = %d, want = 0", len(nd.store.ttlKeyHeap.array)) } if len(nd.store.ttlKeyHeap.keyMap) != 0 { t.Errorf("len(nd.store.ttlKeyHeap.keyMap) = %d, want = 0", len(nd.store.ttlKeyHeap.keyMap)) } } } func TestNewDirIs(t *testing.T) { nd, _ := newTestNodeDir() if nd.IsHidden() { t.Errorf("nd.Hidden() = %v, want = false", nd.IsHidden()) } if nd.IsPermanent() { t.Errorf("nd.IsPermanent() = %v, want = false", nd.IsPermanent()) } if !nd.IsDir() { t.Errorf("nd.IsDir() = %v, want = true", nd.IsDir()) } } func TestNewDirReadWriteListReprClone(t *testing.T) { nd, _ := newTestNodeDir() if _, err := nd.Read(); err == nil { t.Errorf("err = %v, want err != nil", err) } if err := nd.Write(val, nd.CreatedIndex+1); err == nil { t.Errorf("err = %v, want err != nil", err) } if ns, err := nd.List(); ns == nil && err != nil { t.Errorf("nodes = %v and err = %v, want nodes = nil and err == nil", ns, err) } en := nd.Repr(false, false, clockwork.NewFakeClock()) if en.Key != nd.Path { t.Errorf("en.Key = %s, want = %s", en.Key, nd.Path) } cn := nd.Clone() if cn.Path != nd.Path { t.Errorf("cn.Path = %s, want = %s", cn.Path, nd.Path) } } func TestNewDirExpirationTTL(t *testing.T) { nd, _ := newTestNodeDir() if _, ttl := nd.expirationAndTTL(clockwork.NewFakeClock()); ttl > expiration.Nanoseconds() { t.Errorf("ttl = %d, want %d < %d", ttl, ttl, expiration.Nanoseconds()) } newExpiration := time.Hour nd.UpdateTTL(time.Now().Add(newExpiration)) if _, ttl := nd.expirationAndTTL(clockwork.NewFakeClock()); ttl > newExpiration.Nanoseconds() { t.Errorf("ttl = %d, want %d < %d", ttl, ttl, newExpiration.Nanoseconds()) } } func TestNewDirChild(t *testing.T) { nd, child := newTestNodeDir() if err := nd.Add(child); err != nil { t.Errorf("nd.Add(child) err = %v, want = nil", err) } else { if len(nd.Children) == 0 { t.Errorf("len(nd.Children) = %d, want = 1", len(nd.Children)) } } if err := child.Remove(true, true, nil); err != nil { t.Errorf("child.Remove err = %v, want = nil", err) } else { if len(nd.Children) != 0 { t.Errorf("len(nd.Children) = %d, want = 0", len(nd.Children)) } } } func newTestNode() *node { nd := newKV(newStore(), key, val, 0, nil, time.Now().Add(expiration)) return nd } func newTestNodeDir() (*node, *node) { s := newStore() nd := newDir(s, key, 0, nil, time.Now().Add(expiration)) cKey, cVal := "hello", "world" child := newKV(s, cKey, cVal, 0, nd, time.Now().Add(expiration)) return nd, child } ================================================ FILE: server/etcdserver/api/v2store/stats.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "encoding/json" "sync/atomic" ) const ( SetSuccess = iota SetFail DeleteSuccess DeleteFail CreateSuccess CreateFail UpdateSuccess UpdateFail CompareAndSwapSuccess CompareAndSwapFail GetSuccess GetFail ExpireCount CompareAndDeleteSuccess CompareAndDeleteFail ) type Stats struct { // Number of get requests GetSuccess uint64 `json:"getsSuccess"` GetFail uint64 `json:"getsFail"` // Number of sets requests SetSuccess uint64 `json:"setsSuccess"` SetFail uint64 `json:"setsFail"` // Number of delete requests DeleteSuccess uint64 `json:"deleteSuccess"` DeleteFail uint64 `json:"deleteFail"` // Number of update requests UpdateSuccess uint64 `json:"updateSuccess"` UpdateFail uint64 `json:"updateFail"` // Number of create requests CreateSuccess uint64 `json:"createSuccess"` CreateFail uint64 `json:"createFail"` // Number of testAndSet requests CompareAndSwapSuccess uint64 `json:"compareAndSwapSuccess"` CompareAndSwapFail uint64 `json:"compareAndSwapFail"` // Number of compareAndDelete requests CompareAndDeleteSuccess uint64 `json:"compareAndDeleteSuccess"` CompareAndDeleteFail uint64 `json:"compareAndDeleteFail"` ExpireCount uint64 `json:"expireCount"` Watchers uint64 `json:"watchers"` } func newStats() *Stats { s := new(Stats) return s } func (s *Stats) clone() *Stats { return &Stats{ GetSuccess: atomic.LoadUint64(&s.GetSuccess), GetFail: atomic.LoadUint64(&s.GetFail), SetSuccess: atomic.LoadUint64(&s.SetSuccess), SetFail: atomic.LoadUint64(&s.SetFail), DeleteSuccess: atomic.LoadUint64(&s.DeleteSuccess), DeleteFail: atomic.LoadUint64(&s.DeleteFail), UpdateSuccess: atomic.LoadUint64(&s.UpdateSuccess), UpdateFail: atomic.LoadUint64(&s.UpdateFail), CreateSuccess: atomic.LoadUint64(&s.CreateSuccess), CreateFail: atomic.LoadUint64(&s.CreateFail), CompareAndSwapSuccess: atomic.LoadUint64(&s.CompareAndSwapSuccess), CompareAndSwapFail: atomic.LoadUint64(&s.CompareAndSwapFail), CompareAndDeleteSuccess: atomic.LoadUint64(&s.CompareAndDeleteSuccess), CompareAndDeleteFail: atomic.LoadUint64(&s.CompareAndDeleteFail), ExpireCount: atomic.LoadUint64(&s.ExpireCount), Watchers: atomic.LoadUint64(&s.Watchers), } } func (s *Stats) toJSON() []byte { b, _ := json.Marshal(s) return b } func (s *Stats) Inc(field int) { switch field { case SetSuccess: atomic.AddUint64(&s.SetSuccess, 1) case SetFail: atomic.AddUint64(&s.SetFail, 1) case CreateSuccess: atomic.AddUint64(&s.CreateSuccess, 1) case CreateFail: atomic.AddUint64(&s.CreateFail, 1) case DeleteSuccess: atomic.AddUint64(&s.DeleteSuccess, 1) case DeleteFail: atomic.AddUint64(&s.DeleteFail, 1) case GetSuccess: atomic.AddUint64(&s.GetSuccess, 1) case GetFail: atomic.AddUint64(&s.GetFail, 1) case UpdateSuccess: atomic.AddUint64(&s.UpdateSuccess, 1) case UpdateFail: atomic.AddUint64(&s.UpdateFail, 1) case CompareAndSwapSuccess: atomic.AddUint64(&s.CompareAndSwapSuccess, 1) case CompareAndSwapFail: atomic.AddUint64(&s.CompareAndSwapFail, 1) case CompareAndDeleteSuccess: atomic.AddUint64(&s.CompareAndDeleteSuccess, 1) case CompareAndDeleteFail: atomic.AddUint64(&s.CompareAndDeleteFail, 1) case ExpireCount: atomic.AddUint64(&s.ExpireCount, 1) } } ================================================ FILE: server/etcdserver/api/v2store/stats_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "testing" "time" "github.com/stretchr/testify/assert" ) // TestStoreStatsGetSuccess ensures that a successful Get is recorded in the stats. func TestStoreStatsGetSuccess(t *testing.T) { s := newStore() s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: Permanent}) s.Get("/foo", false, false) assert.Equal(t, uint64(1), s.Stats.GetSuccess) } // TestStoreStatsGetFail ensures that a failed Get is recorded in the stats. func TestStoreStatsGetFail(t *testing.T) { s := newStore() s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: Permanent}) s.Get("/no_such_key", false, false) assert.Equal(t, uint64(1), s.Stats.GetFail) } // TestStoreStatsCreateSuccess ensures that a successful Create is recorded in the stats. func TestStoreStatsCreateSuccess(t *testing.T) { s := newStore() s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: Permanent}) assert.Equal(t, uint64(1), s.Stats.CreateSuccess) } // TestStoreStatsCreateFail ensures that a failed Create is recorded in the stats. func TestStoreStatsCreateFail(t *testing.T) { s := newStore() s.Create("/foo", true, "", false, TTLOptionSet{ExpireTime: Permanent}) s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: Permanent}) assert.Equal(t, uint64(1), s.Stats.CreateFail) } // TestStoreStatsUpdateSuccess ensures that a successful Update is recorded in the stats. func TestStoreStatsUpdateSuccess(t *testing.T) { s := newStore() s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: Permanent}) s.Update("/foo", "baz", TTLOptionSet{ExpireTime: Permanent}) assert.Equal(t, uint64(1), s.Stats.UpdateSuccess) } // TestStoreStatsUpdateFail ensures that a failed Update is recorded in the stats. func TestStoreStatsUpdateFail(t *testing.T) { s := newStore() s.Update("/foo", "bar", TTLOptionSet{ExpireTime: Permanent}) assert.Equal(t, uint64(1), s.Stats.UpdateFail) } // TestStoreStatsCompareAndSwapSuccess ensures that a successful CAS is recorded in the stats. func TestStoreStatsCompareAndSwapSuccess(t *testing.T) { s := newStore() s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: Permanent}) s.CompareAndSwap("/foo", "bar", 0, "baz", TTLOptionSet{ExpireTime: Permanent}) assert.Equal(t, uint64(1), s.Stats.CompareAndSwapSuccess) } // TestStoreStatsCompareAndSwapFail ensures that a failed CAS is recorded in the stats. func TestStoreStatsCompareAndSwapFail(t *testing.T) { s := newStore() s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: Permanent}) s.CompareAndSwap("/foo", "wrong_value", 0, "baz", TTLOptionSet{ExpireTime: Permanent}) assert.Equal(t, uint64(1), s.Stats.CompareAndSwapFail) } // TestStoreStatsDeleteSuccess ensures that a successful Delete is recorded in the stats. func TestStoreStatsDeleteSuccess(t *testing.T) { s := newStore() s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: Permanent}) s.Delete("/foo", false, false) assert.Equal(t, uint64(1), s.Stats.DeleteSuccess) } // TestStoreStatsDeleteFail ensures that a failed Delete is recorded in the stats. func TestStoreStatsDeleteFail(t *testing.T) { s := newStore() s.Delete("/foo", false, false) assert.Equal(t, uint64(1), s.Stats.DeleteFail) } // TestStoreStatsExpireCount ensures that the number of expirations is recorded in the stats. func TestStoreStatsExpireCount(t *testing.T) { s := newStore() fc := newFakeClock() s.clock = fc s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond)}) assert.Equal(t, uint64(0), s.Stats.ExpireCount) fc.Advance(600 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) assert.Equal(t, uint64(1), s.Stats.ExpireCount) } ================================================ FILE: server/etcdserver/api/v2store/store.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "encoding/json" "fmt" "path" "strconv" "strings" "sync" "time" "github.com/jonboulle/clockwork" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/v2error" ) // The default version to set when the store is first initialized. const defaultVersion = 2 var minExpireTime time.Time func init() { minExpireTime, _ = time.Parse(time.RFC3339, "2000-01-01T00:00:00Z") } type Store interface { Version() int Index() uint64 Get(nodePath string, recursive, sorted bool) (*Event, error) Set(nodePath string, dir bool, value string, expireOpts TTLOptionSet) (*Event, error) Update(nodePath string, newValue string, expireOpts TTLOptionSet) (*Event, error) Create(nodePath string, dir bool, value string, unique bool, expireOpts TTLOptionSet) (*Event, error) CompareAndSwap(nodePath string, prevValue string, prevIndex uint64, value string, expireOpts TTLOptionSet) (*Event, error) Delete(nodePath string, dir, recursive bool) (*Event, error) CompareAndDelete(nodePath string, prevValue string, prevIndex uint64) (*Event, error) Watch(prefix string, recursive, stream bool, sinceIndex uint64) (Watcher, error) Save() ([]byte, error) Recovery(state []byte) error Clone() Store SaveNoCopy() ([]byte, error) JsonStats() []byte DeleteExpiredKeys(cutoff time.Time) HasTTLKeys() bool } type TTLOptionSet struct { ExpireTime time.Time Refresh bool } type store struct { Root *node WatcherHub *watcherHub CurrentIndex uint64 Stats *Stats CurrentVersion int ttlKeyHeap *ttlKeyHeap // need to recovery manually worldLock sync.RWMutex // stop the world lock clock clockwork.Clock readonlySet types.Set } // New creates a store where the given namespaces will be created as initial directories. func New(namespaces ...string) Store { s := newStore(namespaces...) s.clock = clockwork.NewRealClock() return s } func newStore(namespaces ...string) *store { s := new(store) s.CurrentVersion = defaultVersion s.Root = newDir(s, "/", s.CurrentIndex, nil, Permanent) for _, namespace := range namespaces { s.Root.Add(newDir(s, namespace, s.CurrentIndex, s.Root, Permanent)) } s.Stats = newStats() s.WatcherHub = newWatchHub(1000) s.ttlKeyHeap = newTTLKeyHeap() s.readonlySet = types.NewUnsafeSet(append(namespaces, "/")...) return s } // Version retrieves current version of the store. func (s *store) Version() int { return s.CurrentVersion } // Index retrieves the current index of the store. func (s *store) Index() uint64 { s.worldLock.RLock() defer s.worldLock.RUnlock() return s.CurrentIndex } // Get returns a get event. // If recursive is true, it will return all the content under the node path. // If sorted is true, it will sort the content by keys. func (s *store) Get(nodePath string, recursive, sorted bool) (*Event, error) { var err *v2error.Error s.worldLock.RLock() defer s.worldLock.RUnlock() defer func() { if err == nil { s.Stats.Inc(GetSuccess) if recursive { reportReadSuccess(GetRecursive) } else { reportReadSuccess(Get) } return } s.Stats.Inc(GetFail) if recursive { reportReadFailure(GetRecursive) } else { reportReadFailure(Get) } }() n, err := s.internalGet(nodePath) if err != nil { return nil, err } e := newEvent(Get, nodePath, n.ModifiedIndex, n.CreatedIndex) e.EtcdIndex = s.CurrentIndex e.Node.loadInternalNode(n, recursive, sorted, s.clock) return e, nil } // Create creates the node at nodePath. Create will help to create intermediate directories with no ttl. // If the node has already existed, create will fail. // If any node on the path is a file, create will fail. func (s *store) Create(nodePath string, dir bool, value string, unique bool, expireOpts TTLOptionSet) (*Event, error) { var err *v2error.Error s.worldLock.Lock() defer s.worldLock.Unlock() defer func() { if err == nil { s.Stats.Inc(CreateSuccess) reportWriteSuccess(Create) return } s.Stats.Inc(CreateFail) reportWriteFailure(Create) }() e, err := s.internalCreate(nodePath, dir, value, unique, false, expireOpts.ExpireTime, Create) if err != nil { return nil, err } e.EtcdIndex = s.CurrentIndex s.WatcherHub.notify(e) return e, nil } // Set creates or replace the node at nodePath. func (s *store) Set(nodePath string, dir bool, value string, expireOpts TTLOptionSet) (*Event, error) { var err *v2error.Error s.worldLock.Lock() defer s.worldLock.Unlock() defer func() { if err == nil { s.Stats.Inc(SetSuccess) reportWriteSuccess(Set) return } s.Stats.Inc(SetFail) reportWriteFailure(Set) }() // Get prevNode value n, getErr := s.internalGet(nodePath) if getErr != nil && getErr.ErrorCode != v2error.EcodeKeyNotFound { err = getErr return nil, err } if expireOpts.Refresh { if getErr != nil { err = getErr return nil, err } value = n.Value } // Set new value e, err := s.internalCreate(nodePath, dir, value, false, true, expireOpts.ExpireTime, Set) if err != nil { return nil, err } e.EtcdIndex = s.CurrentIndex // Put prevNode into event if getErr == nil { prev := newEvent(Get, nodePath, n.ModifiedIndex, n.CreatedIndex) prev.Node.loadInternalNode(n, false, false, s.clock) e.PrevNode = prev.Node } if !expireOpts.Refresh { s.WatcherHub.notify(e) } else { e.SetRefresh() s.WatcherHub.add(e) } return e, nil } // returns user-readable cause of failed comparison func getCompareFailCause(n *node, which int, prevValue string, prevIndex uint64) string { switch which { case CompareIndexNotMatch: return fmt.Sprintf("[%v != %v]", prevIndex, n.ModifiedIndex) case CompareValueNotMatch: return fmt.Sprintf("[%v != %v]", prevValue, n.Value) default: return fmt.Sprintf("[%v != %v] [%v != %v]", prevValue, n.Value, prevIndex, n.ModifiedIndex) } } func (s *store) CompareAndSwap(nodePath string, prevValue string, prevIndex uint64, value string, expireOpts TTLOptionSet, ) (*Event, error) { var err *v2error.Error s.worldLock.Lock() defer s.worldLock.Unlock() defer func() { if err == nil { s.Stats.Inc(CompareAndSwapSuccess) reportWriteSuccess(CompareAndSwap) return } s.Stats.Inc(CompareAndSwapFail) reportWriteFailure(CompareAndSwap) }() nodePath = path.Clean(path.Join("/", nodePath)) // we do not allow the user to change "/" if s.readonlySet.Contains(nodePath) { return nil, v2error.NewError(v2error.EcodeRootROnly, "/", s.CurrentIndex) } n, err := s.internalGet(nodePath) if err != nil { return nil, err } if n.IsDir() { // can only compare and swap file err = v2error.NewError(v2error.EcodeNotFile, nodePath, s.CurrentIndex) return nil, err } // If both of the prevValue and prevIndex are given, we will test both of them. // Command will be executed, only if both of the tests are successful. if ok, which := n.Compare(prevValue, prevIndex); !ok { cause := getCompareFailCause(n, which, prevValue, prevIndex) err = v2error.NewError(v2error.EcodeTestFailed, cause, s.CurrentIndex) return nil, err } if expireOpts.Refresh { value = n.Value } // update etcd index s.CurrentIndex++ e := newEvent(CompareAndSwap, nodePath, s.CurrentIndex, n.CreatedIndex) e.EtcdIndex = s.CurrentIndex e.PrevNode = n.Repr(false, false, s.clock) eNode := e.Node // if test succeed, write the value if err := n.Write(value, s.CurrentIndex); err != nil { return nil, err } n.UpdateTTL(expireOpts.ExpireTime) // copy the value for safety valueCopy := value eNode.Value = &valueCopy eNode.Expiration, eNode.TTL = n.expirationAndTTL(s.clock) if !expireOpts.Refresh { s.WatcherHub.notify(e) } else { e.SetRefresh() s.WatcherHub.add(e) } return e, nil } // Delete deletes the node at the given path. // If the node is a directory, recursive must be true to delete it. func (s *store) Delete(nodePath string, dir, recursive bool) (*Event, error) { var err *v2error.Error s.worldLock.Lock() defer s.worldLock.Unlock() defer func() { if err == nil { s.Stats.Inc(DeleteSuccess) reportWriteSuccess(Delete) return } s.Stats.Inc(DeleteFail) reportWriteFailure(Delete) }() nodePath = path.Clean(path.Join("/", nodePath)) // we do not allow the user to change "/" if s.readonlySet.Contains(nodePath) { return nil, v2error.NewError(v2error.EcodeRootROnly, "/", s.CurrentIndex) } // recursive implies dir if recursive { dir = true } n, err := s.internalGet(nodePath) if err != nil { // if the node does not exist, return error return nil, err } nextIndex := s.CurrentIndex + 1 e := newEvent(Delete, nodePath, nextIndex, n.CreatedIndex) e.EtcdIndex = nextIndex e.PrevNode = n.Repr(false, false, s.clock) eNode := e.Node if n.IsDir() { eNode.Dir = true } callback := func(path string) { // notify function // notify the watchers with deleted set true s.WatcherHub.notifyWatchers(e, path, true) } err = n.Remove(dir, recursive, callback) if err != nil { return nil, err } // update etcd index s.CurrentIndex++ s.WatcherHub.notify(e) return e, nil } func (s *store) CompareAndDelete(nodePath string, prevValue string, prevIndex uint64) (*Event, error) { var err *v2error.Error s.worldLock.Lock() defer s.worldLock.Unlock() defer func() { if err == nil { s.Stats.Inc(CompareAndDeleteSuccess) reportWriteSuccess(CompareAndDelete) return } s.Stats.Inc(CompareAndDeleteFail) reportWriteFailure(CompareAndDelete) }() nodePath = path.Clean(path.Join("/", nodePath)) n, err := s.internalGet(nodePath) if err != nil { // if the node does not exist, return error return nil, err } if n.IsDir() { // can only compare and delete file return nil, v2error.NewError(v2error.EcodeNotFile, nodePath, s.CurrentIndex) } // If both of the prevValue and prevIndex are given, we will test both of them. // Command will be executed, only if both of the tests are successful. if ok, which := n.Compare(prevValue, prevIndex); !ok { cause := getCompareFailCause(n, which, prevValue, prevIndex) return nil, v2error.NewError(v2error.EcodeTestFailed, cause, s.CurrentIndex) } // update etcd index s.CurrentIndex++ e := newEvent(CompareAndDelete, nodePath, s.CurrentIndex, n.CreatedIndex) e.EtcdIndex = s.CurrentIndex e.PrevNode = n.Repr(false, false, s.clock) callback := func(path string) { // notify function // notify the watchers with deleted set true s.WatcherHub.notifyWatchers(e, path, true) } err = n.Remove(false, false, callback) if err != nil { return nil, err } s.WatcherHub.notify(e) return e, nil } func (s *store) Watch(key string, recursive, stream bool, sinceIndex uint64) (Watcher, error) { s.worldLock.RLock() defer s.worldLock.RUnlock() key = path.Clean(path.Join("/", key)) if sinceIndex == 0 { sinceIndex = s.CurrentIndex + 1 } // WatcherHub does not know about the current index, so we need to pass it in w, err := s.WatcherHub.watch(key, recursive, stream, sinceIndex, s.CurrentIndex) if err != nil { return nil, err } return w, nil } // walk walks all the nodePath and apply the walkFunc on each directory func (s *store) walk(nodePath string, walkFunc func(prev *node, component string) (*node, *v2error.Error)) (*node, *v2error.Error) { components := strings.Split(nodePath, "/") curr := s.Root var err *v2error.Error for i := 1; i < len(components); i++ { if len(components[i]) == 0 { // ignore empty string return curr, nil } curr, err = walkFunc(curr, components[i]) if err != nil { return nil, err } } return curr, nil } // Update updates the value/ttl of the node. // If the node is a file, the value and the ttl can be updated. // If the node is a directory, only the ttl can be updated. func (s *store) Update(nodePath string, newValue string, expireOpts TTLOptionSet) (*Event, error) { var err *v2error.Error s.worldLock.Lock() defer s.worldLock.Unlock() defer func() { if err == nil { s.Stats.Inc(UpdateSuccess) reportWriteSuccess(Update) return } s.Stats.Inc(UpdateFail) reportWriteFailure(Update) }() nodePath = path.Clean(path.Join("/", nodePath)) // we do not allow the user to change "/" if s.readonlySet.Contains(nodePath) { return nil, v2error.NewError(v2error.EcodeRootROnly, "/", s.CurrentIndex) } currIndex, nextIndex := s.CurrentIndex, s.CurrentIndex+1 n, err := s.internalGet(nodePath) if err != nil { // if the node does not exist, return error return nil, err } if n.IsDir() && len(newValue) != 0 { // if the node is a directory, we cannot update value to non-empty return nil, v2error.NewError(v2error.EcodeNotFile, nodePath, currIndex) } if expireOpts.Refresh { newValue = n.Value } e := newEvent(Update, nodePath, nextIndex, n.CreatedIndex) e.EtcdIndex = nextIndex e.PrevNode = n.Repr(false, false, s.clock) eNode := e.Node if err := n.Write(newValue, nextIndex); err != nil { return nil, fmt.Errorf("nodePath %v : %w", nodePath, err) } if n.IsDir() { eNode.Dir = true } else { // copy the value for safety newValueCopy := newValue eNode.Value = &newValueCopy } // update ttl n.UpdateTTL(expireOpts.ExpireTime) eNode.Expiration, eNode.TTL = n.expirationAndTTL(s.clock) if !expireOpts.Refresh { s.WatcherHub.notify(e) } else { e.SetRefresh() s.WatcherHub.add(e) } s.CurrentIndex = nextIndex return e, nil } func (s *store) internalCreate(nodePath string, dir bool, value string, unique, replace bool, expireTime time.Time, action string, ) (*Event, *v2error.Error) { currIndex, nextIndex := s.CurrentIndex, s.CurrentIndex+1 if unique { // append unique item under the node path nodePath += "/" + fmt.Sprintf("%020s", strconv.FormatUint(nextIndex, 10)) } nodePath = path.Clean(path.Join("/", nodePath)) // we do not allow the user to change "/" if s.readonlySet.Contains(nodePath) { return nil, v2error.NewError(v2error.EcodeRootROnly, "/", currIndex) } // Assume expire times that are way in the past are // This can occur when the time is serialized to JS if expireTime.Before(minExpireTime) { expireTime = Permanent } dirName, nodeName := path.Split(nodePath) // walk through the nodePath, create dirs and get the last directory node d, err := s.walk(dirName, s.checkDir) if err != nil { s.Stats.Inc(SetFail) reportWriteFailure(action) err.Index = currIndex return nil, err } e := newEvent(action, nodePath, nextIndex, nextIndex) eNode := e.Node n, _ := d.GetChild(nodeName) // force will try to replace an existing file if n != nil { if !replace { return nil, v2error.NewError(v2error.EcodeNodeExist, nodePath, currIndex) } if n.IsDir() { return nil, v2error.NewError(v2error.EcodeNotFile, nodePath, currIndex) } e.PrevNode = n.Repr(false, false, s.clock) if err := n.Remove(false, false, nil); err != nil { return nil, err } } if !dir { // create file // copy the value for safety valueCopy := value eNode.Value = &valueCopy n = newKV(s, nodePath, value, nextIndex, d, expireTime) } else { // create directory eNode.Dir = true n = newDir(s, nodePath, nextIndex, d, expireTime) } // we are sure d is a directory and does not have the children with name n.Name if err := d.Add(n); err != nil { return nil, err } // node with TTL if !n.IsPermanent() { s.ttlKeyHeap.push(n) eNode.Expiration, eNode.TTL = n.expirationAndTTL(s.clock) } s.CurrentIndex = nextIndex return e, nil } // InternalGet gets the node of the given nodePath. func (s *store) internalGet(nodePath string) (*node, *v2error.Error) { nodePath = path.Clean(path.Join("/", nodePath)) walkFunc := func(parent *node, name string) (*node, *v2error.Error) { if !parent.IsDir() { err := v2error.NewError(v2error.EcodeNotDir, parent.Path, s.CurrentIndex) return nil, err } child, ok := parent.Children[name] if ok { return child, nil } return nil, v2error.NewError(v2error.EcodeKeyNotFound, path.Join(parent.Path, name), s.CurrentIndex) } f, err := s.walk(nodePath, walkFunc) if err != nil { return nil, err } return f, nil } // DeleteExpiredKeys will delete all expired keys func (s *store) DeleteExpiredKeys(cutoff time.Time) { s.worldLock.Lock() defer s.worldLock.Unlock() for { node := s.ttlKeyHeap.top() if node == nil || node.ExpireTime.After(cutoff) { break } s.CurrentIndex++ e := newEvent(Expire, node.Path, s.CurrentIndex, node.CreatedIndex) e.EtcdIndex = s.CurrentIndex e.PrevNode = node.Repr(false, false, s.clock) if node.IsDir() { e.Node.Dir = true } callback := func(path string) { // notify function // notify the watchers with deleted set true s.WatcherHub.notifyWatchers(e, path, true) } s.ttlKeyHeap.pop() node.Remove(true, true, callback) reportExpiredKey() s.Stats.Inc(ExpireCount) s.WatcherHub.notify(e) } } // checkDir will check whether the component is a directory under parent node. // If it is a directory, this function will return the pointer to that node. // If it does not exist, this function will create a new directory and return the pointer to that node. // If it is a file, this function will return error. func (s *store) checkDir(parent *node, dirName string) (*node, *v2error.Error) { node, ok := parent.Children[dirName] if ok { if node.IsDir() { return node, nil } return nil, v2error.NewError(v2error.EcodeNotDir, node.Path, s.CurrentIndex) } n := newDir(s, path.Join(parent.Path, dirName), s.CurrentIndex+1, parent, Permanent) parent.Children[dirName] = n return n, nil } // Save saves the static state of the store system. // It will not be able to save the state of watchers. // It will not save the parent field of the node. Or there will // be cyclic dependencies issue for the json package. func (s *store) Save() ([]byte, error) { b, err := json.Marshal(s.Clone()) if err != nil { return nil, err } return b, nil } func (s *store) SaveNoCopy() ([]byte, error) { b, err := json.Marshal(s) if err != nil { return nil, err } return b, nil } func (s *store) Clone() Store { s.worldLock.RLock() clonedStore := newStore() clonedStore.CurrentIndex = s.CurrentIndex clonedStore.Root = s.Root.Clone() clonedStore.WatcherHub = s.WatcherHub.clone() clonedStore.Stats = s.Stats.clone() clonedStore.CurrentVersion = s.CurrentVersion s.worldLock.RUnlock() return clonedStore } // Recovery recovers the store system from a static state // It needs to recover the parent field of the nodes. // It needs to delete the expired nodes since the saved time and also // needs to create monitoring goroutines. func (s *store) Recovery(state []byte) error { s.worldLock.Lock() defer s.worldLock.Unlock() err := json.Unmarshal(state, s) if err != nil { return err } s.ttlKeyHeap = newTTLKeyHeap() s.Root.recoverAndclean() return nil } //revive:disable:var-naming func (s *store) JsonStats() []byte { //revive:enable:var-naming s.Stats.Watchers = uint64(s.WatcherHub.count) return s.Stats.toJSON() } func (s *store) HasTTLKeys() bool { s.worldLock.RLock() defer s.worldLock.RUnlock() return s.ttlKeyHeap.Len() != 0 } ================================================ FILE: server/etcdserver/api/v2store/store_bench_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "encoding/json" "fmt" "runtime" "testing" ) func BenchmarkStoreSet128Bytes(b *testing.B) { benchStoreSet(b, 128, nil) } func BenchmarkStoreSet1024Bytes(b *testing.B) { benchStoreSet(b, 1024, nil) } func BenchmarkStoreSet4096Bytes(b *testing.B) { benchStoreSet(b, 4096, nil) } func BenchmarkStoreSetWithJson128Bytes(b *testing.B) { benchStoreSet(b, 128, json.Marshal) } func BenchmarkStoreSetWithJson1024Bytes(b *testing.B) { benchStoreSet(b, 1024, json.Marshal) } func BenchmarkStoreSetWithJson4096Bytes(b *testing.B) { benchStoreSet(b, 4096, json.Marshal) } func BenchmarkStoreDelete(b *testing.B) { b.StopTimer() s := newStore() kvs, _ := generateNRandomKV(b.N, 128) memStats := new(runtime.MemStats) runtime.GC() runtime.ReadMemStats(memStats) for i := 0; i < b.N; i++ { _, err := s.Set(kvs[i][0], false, kvs[i][1], TTLOptionSet{ExpireTime: Permanent}) if err != nil { panic(err) } } setMemStats := new(runtime.MemStats) runtime.GC() runtime.ReadMemStats(setMemStats) b.StartTimer() for i := range kvs { s.Delete(kvs[i][0], false, false) } b.StopTimer() // clean up e, err := s.Get("/", false, false) if err != nil { panic(err) } for _, n := range e.Node.Nodes { _, err := s.Delete(n.Key, true, true) if err != nil { panic(err) } } s.WatcherHub.EventHistory = nil deleteMemStats := new(runtime.MemStats) runtime.GC() runtime.ReadMemStats(deleteMemStats) fmt.Printf("\nBefore set Alloc: %v; After set Alloc: %v, After delete Alloc: %v\n", memStats.Alloc/1000, setMemStats.Alloc/1000, deleteMemStats.Alloc/1000) } func BenchmarkWatch(b *testing.B) { b.StopTimer() s := newStore() kvs, _ := generateNRandomKV(b.N, 128) b.StartTimer() memStats := new(runtime.MemStats) runtime.GC() runtime.ReadMemStats(memStats) for i := 0; i < b.N; i++ { w, _ := s.Watch(kvs[i][0], false, false, 0) e := newEvent("set", kvs[i][0], uint64(i+1), uint64(i+1)) s.WatcherHub.notify(e) <-w.EventChan() s.CurrentIndex++ } s.WatcherHub.EventHistory = nil afterMemStats := new(runtime.MemStats) runtime.GC() runtime.ReadMemStats(afterMemStats) fmt.Printf("\nBefore Alloc: %v; After Alloc: %v\n", memStats.Alloc/1000, afterMemStats.Alloc/1000) } func BenchmarkWatchWithSet(b *testing.B) { b.StopTimer() s := newStore() kvs, _ := generateNRandomKV(b.N, 128) b.StartTimer() for i := 0; i < b.N; i++ { w, _ := s.Watch(kvs[i][0], false, false, 0) s.Set(kvs[i][0], false, "test", TTLOptionSet{ExpireTime: Permanent}) <-w.EventChan() } } func BenchmarkWatchWithSetBatch(b *testing.B) { b.StopTimer() s := newStore() kvs, _ := generateNRandomKV(b.N, 128) b.StartTimer() watchers := make([]Watcher, b.N) for i := 0; i < b.N; i++ { watchers[i], _ = s.Watch(kvs[i][0], false, false, 0) } for i := 0; i < b.N; i++ { s.Set(kvs[i][0], false, "test", TTLOptionSet{ExpireTime: Permanent}) } for i := 0; i < b.N; i++ { <-watchers[i].EventChan() } } func BenchmarkWatchOneKey(b *testing.B) { s := newStore() watchers := make([]Watcher, b.N) for i := 0; i < b.N; i++ { watchers[i], _ = s.Watch("/foo", false, false, 0) } s.Set("/foo", false, "", TTLOptionSet{ExpireTime: Permanent}) for i := 0; i < b.N; i++ { <-watchers[i].EventChan() } } func benchStoreSet(b *testing.B, valueSize int, process func(any) ([]byte, error)) { s := newStore() b.StopTimer() kvs, size := generateNRandomKV(b.N, valueSize) b.StartTimer() for i := 0; i < b.N; i++ { resp, err := s.Set(kvs[i][0], false, kvs[i][1], TTLOptionSet{ExpireTime: Permanent}) if err != nil { panic(err) } if process != nil { _, err = process(resp) if err != nil { panic(err) } } } b.StopTimer() memStats := new(runtime.MemStats) runtime.GC() runtime.ReadMemStats(memStats) fmt.Printf("\nAlloc: %vKB; Data: %vKB; Kvs: %v; Alloc/Data:%v\n", memStats.Alloc/1000, size/1000, b.N, memStats.Alloc/size) } func generateNRandomKV(n int, valueSize int) ([][]string, uint64) { var size uint64 kvs := make([][]string, n) bytes := make([]byte, valueSize) for i := 0; i < n; i++ { kvs[i] = make([]string, 2) kvs[i][0] = fmt.Sprintf("/%010d/%010d/%010d", n, n, n) kvs[i][1] = string(bytes) size = size + uint64(len(kvs[i][0])) + uint64(len(kvs[i][1])) } return kvs, size } ================================================ FILE: server/etcdserver/api/v2store/store_ttl_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "testing" "time" "github.com/jonboulle/clockwork" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/server/v3/etcdserver/api/v2error" ) // TestMinExpireTime ensures that any TTL <= minExpireTime becomes Permanent func TestMinExpireTime(t *testing.T) { s := newStore() fc := clockwork.NewFakeClockAt(time.Date(1984, time.April, 4, 0, 0, 0, 0, time.UTC)) s.clock = fc // FakeClock starts at 0, so minExpireTime should be far in the future.. but just in case assert.Truef(t, minExpireTime.After(fc.Now()), "minExpireTime should be ahead of FakeClock!") s.Create("/foo", false, "Y", false, TTLOptionSet{ExpireTime: fc.Now().Add(3 * time.Second)}) fc.Advance(5 * time.Second) // Ensure it hasn't expired s.DeleteExpiredKeys(fc.Now()) var eidx uint64 = 1 e, err := s.Get("/foo", true, false) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "get", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.Equal(t, int64(0), e.Node.TTL) } // TestStoreGetDirectory ensures that the store can recursively retrieve a directory listing. // Note that hidden files should not be returned. func TestStoreGetDirectory(t *testing.T) { s := newStore() fc := newFakeClock() s.clock = fc s.Create("/foo", true, "", false, TTLOptionSet{ExpireTime: Permanent}) s.Create("/foo/bar", false, "X", false, TTLOptionSet{ExpireTime: Permanent}) s.Create("/foo/_hidden", false, "*", false, TTLOptionSet{ExpireTime: Permanent}) s.Create("/foo/baz", true, "", false, TTLOptionSet{ExpireTime: Permanent}) s.Create("/foo/baz/bat", false, "Y", false, TTLOptionSet{ExpireTime: Permanent}) s.Create("/foo/baz/_hidden", false, "*", false, TTLOptionSet{ExpireTime: Permanent}) s.Create("/foo/baz/ttl", false, "Y", false, TTLOptionSet{ExpireTime: fc.Now().Add(time.Second * 3)}) var eidx uint64 = 7 e, err := s.Get("/foo", true, false) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "get", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.Len(t, e.Node.Nodes, 2) var bazNodes NodeExterns for _, node := range e.Node.Nodes { switch node.Key { case "/foo/bar": assert.Equal(t, "X", *node.Value) assert.False(t, node.Dir) case "/foo/baz": assert.True(t, node.Dir) assert.Len(t, node.Nodes, 2) bazNodes = node.Nodes default: t.Errorf("key = %s, not matched", node.Key) } } for _, node := range bazNodes { switch node.Key { case "/foo/baz/bat": assert.Equal(t, "Y", *node.Value) assert.False(t, node.Dir) case "/foo/baz/ttl": assert.Equal(t, "Y", *node.Value) assert.False(t, node.Dir) assert.Equal(t, int64(3), node.TTL) default: t.Errorf("key = %s, not matched", node.Key) } } } // TestStoreUpdateValueTTL ensures that the store can update the TTL on a value. func TestStoreUpdateValueTTL(t *testing.T) { s := newStore() fc := newFakeClock() s.clock = fc var eidx uint64 = 2 s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: Permanent}) _, err := s.Update("/foo", "baz", TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond)}) require.NoError(t, err) e, _ := s.Get("/foo", false, false) assert.Equal(t, "baz", *e.Node.Value) assert.Equal(t, eidx, e.EtcdIndex) fc.Advance(600 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) e, err = s.Get("/foo", false, false) assert.Nil(t, e) var v2Err *v2error.Error require.ErrorAs(t, err, &v2Err) assert.Equal(t, v2error.EcodeKeyNotFound, v2Err.ErrorCode) } // TestStoreUpdateDirTTL ensures that the store can update the TTL on a directory. func TestStoreUpdateDirTTL(t *testing.T) { s := newStore() fc := newFakeClock() s.clock = fc var eidx uint64 = 3 _, err := s.Create("/foo", true, "", false, TTLOptionSet{ExpireTime: Permanent}) require.NoError(t, err) _, err = s.Create("/foo/bar", false, "baz", false, TTLOptionSet{ExpireTime: Permanent}) require.NoError(t, err) e, err := s.Update("/foo/bar", "", TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond)}) require.NoError(t, err) assert.False(t, e.Node.Dir) assert.Equal(t, eidx, e.EtcdIndex) e, _ = s.Get("/foo/bar", false, false) assert.Empty(t, *e.Node.Value) assert.Equal(t, eidx, e.EtcdIndex) fc.Advance(600 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) e, err = s.Get("/foo/bar", false, false) assert.Nil(t, e) var v2Err *v2error.Error require.ErrorAs(t, err, &v2Err) assert.Equal(t, v2error.EcodeKeyNotFound, v2Err.ErrorCode) } // TestStoreWatchExpire ensures that the store can watch for key expiration. func TestStoreWatchExpire(t *testing.T) { s := newStore() fc := newFakeClock() s.clock = fc var eidx uint64 = 3 s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: fc.Now().Add(400 * time.Millisecond)}) s.Create("/foofoo", false, "barbarbar", false, TTLOptionSet{ExpireTime: fc.Now().Add(450 * time.Millisecond)}) s.Create("/foodir", true, "", false, TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond)}) w, _ := s.Watch("/", true, false, 0) assert.Equal(t, eidx, w.StartIndex()) c := w.EventChan() e := nbselect(c) assert.Nil(t, e) fc.Advance(600 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) eidx = 4 e = nbselect(c) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "expire", e.Action) assert.Equal(t, "/foo", e.Node.Key) w, _ = s.Watch("/", true, false, 5) eidx = 6 assert.Equal(t, eidx, w.StartIndex()) e = nbselect(w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "expire", e.Action) assert.Equal(t, "/foofoo", e.Node.Key) w, _ = s.Watch("/", true, false, 6) e = nbselect(w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "expire", e.Action) assert.Equal(t, "/foodir", e.Node.Key) assert.True(t, e.Node.Dir) } // TestStoreWatchExpireRefresh ensures that the store can watch for key expiration when refreshing. func TestStoreWatchExpireRefresh(t *testing.T) { s := newStore() fc := newFakeClock() s.clock = fc var eidx uint64 = 2 s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond), Refresh: true}) s.Create("/foofoo", false, "barbarbar", false, TTLOptionSet{ExpireTime: fc.Now().Add(1200 * time.Millisecond), Refresh: true}) // Make sure we set watch updates when Refresh is true for newly created keys w, _ := s.Watch("/", true, false, 0) assert.Equal(t, eidx, w.StartIndex()) c := w.EventChan() e := nbselect(c) assert.Nil(t, e) fc.Advance(600 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) eidx = 3 e = nbselect(c) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "expire", e.Action) assert.Equal(t, "/foo", e.Node.Key) s.Update("/foofoo", "", TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond), Refresh: true}) w, _ = s.Watch("/", true, false, 4) fc.Advance(700 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) eidx = 5 // We should skip 4 because a TTL update should occur with no watch notification if set `TTLOptionSet.Refresh` to true assert.Equal(t, eidx-1, w.StartIndex()) e = nbselect(w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "expire", e.Action) assert.Equal(t, "/foofoo", e.Node.Key) } // TestStoreWatchExpireEmptyRefresh ensures that the store can watch for key expiration when refreshing with an empty value. func TestStoreWatchExpireEmptyRefresh(t *testing.T) { s := newStore() fc := newFakeClock() s.clock = fc var eidx uint64 s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond), Refresh: true}) // Should be no-op fc.Advance(200 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) s.Update("/foo", "", TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond), Refresh: true}) w, _ := s.Watch("/", true, false, 2) fc.Advance(700 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) eidx = 3 // We should skip 2 because a TTL update should occur with no watch notification if set `TTLOptionSet.Refresh` to true assert.Equal(t, eidx-1, w.StartIndex()) e := nbselect(w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "expire", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.Equal(t, "bar", *e.PrevNode.Value) } // TestStoreWatchNoRefresh updates TTL of a key (set TTLOptionSet.Refresh to false) and send notification func TestStoreWatchNoRefresh(t *testing.T) { s := newStore() fc := newFakeClock() s.clock = fc var eidx uint64 s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond), Refresh: true}) // Should be no-op fc.Advance(200 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) // Update key's TTL with setting `TTLOptionSet.Refresh` to false will cause an update event s.Update("/foo", "", TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond), Refresh: false}) w, _ := s.Watch("/", true, false, 2) fc.Advance(700 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) eidx = 2 assert.Equal(t, eidx, w.StartIndex()) e := nbselect(w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "update", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.Equal(t, "bar", *e.PrevNode.Value) } // TestStoreRefresh ensures that the store can update the TTL on a value with refresh. func TestStoreRefresh(t *testing.T) { s := newStore() fc := newFakeClock() s.clock = fc s.Create("/foo", false, "bar", false, TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond)}) s.Create("/bar", true, "bar", false, TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond)}) s.Create("/bar/z", false, "bar", false, TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond)}) _, err := s.Update("/foo", "", TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond), Refresh: true}) require.NoError(t, err) _, err = s.Set("/foo", false, "", TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond), Refresh: true}) require.NoError(t, err) _, err = s.Update("/bar/z", "", TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond), Refresh: true}) require.NoError(t, err) _, err = s.CompareAndSwap("/foo", "bar", 0, "", TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond), Refresh: true}) assert.NoError(t, err) } // TestStoreRecoverWithExpiration ensures that the store can recover from a previously saved state that includes an expiring key. func TestStoreRecoverWithExpiration(t *testing.T) { s := newStore() s.clock = newFakeClock() fc := newFakeClock() var eidx uint64 = 4 s.Create("/foo", true, "", false, TTLOptionSet{ExpireTime: Permanent}) s.Create("/foo/x", false, "bar", false, TTLOptionSet{ExpireTime: Permanent}) s.Create("/foo/y", false, "baz", false, TTLOptionSet{ExpireTime: fc.Now().Add(5 * time.Millisecond)}) b, err := s.Save() require.NoError(t, err) time.Sleep(10 * time.Millisecond) s2 := newStore() s2.clock = fc s2.Recovery(b) fc.Advance(600 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) e, err := s.Get("/foo/x", false, false) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "bar", *e.Node.Value) e, err = s.Get("/foo/y", false, false) require.Error(t, err) assert.Nil(t, e) } // TestStoreWatchExpireWithHiddenKey ensures that the store doesn't see expirations of hidden keys. func TestStoreWatchExpireWithHiddenKey(t *testing.T) { s := newStore() fc := newFakeClock() s.clock = fc s.Create("/_foo", false, "bar", false, TTLOptionSet{ExpireTime: fc.Now().Add(500 * time.Millisecond)}) s.Create("/foofoo", false, "barbarbar", false, TTLOptionSet{ExpireTime: fc.Now().Add(time.Second)}) w, _ := s.Watch("/", true, false, 0) c := w.EventChan() e := nbselect(c) assert.Nil(t, e) fc.Advance(600 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) e = nbselect(c) assert.Nil(t, e) fc.Advance(600 * time.Millisecond) s.DeleteExpiredKeys(fc.Now()) e = nbselect(c) assert.Equal(t, "expire", e.Action) assert.Equal(t, "/foofoo", e.Node.Key) } // newFakeClock creates a new FakeClock that has been advanced to at least minExpireTime func newFakeClock() *clockwork.FakeClock { fc := clockwork.NewFakeClock() for minExpireTime.After(fc.Now()) { fc.Advance((0x1 << 62) * time.Nanosecond) } return fc } // Performs a non-blocking select on an event channel. func nbselect(c <-chan *Event) *Event { select { case e := <-c: return e default: return nil } } ================================================ FILE: server/etcdserver/api/v2store/ttl_key_heap.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import "container/heap" // An TTLKeyHeap is a min-heap of TTLKeys order by expiration time type ttlKeyHeap struct { array []*node keyMap map[*node]int } func newTTLKeyHeap() *ttlKeyHeap { h := &ttlKeyHeap{keyMap: make(map[*node]int)} heap.Init(h) return h } func (h ttlKeyHeap) Len() int { return len(h.array) } func (h ttlKeyHeap) Less(i, j int) bool { return h.array[i].ExpireTime.Before(h.array[j].ExpireTime) } func (h ttlKeyHeap) Swap(i, j int) { // swap node h.array[i], h.array[j] = h.array[j], h.array[i] // update map h.keyMap[h.array[i]] = i h.keyMap[h.array[j]] = j } func (h *ttlKeyHeap) Push(x any) { n, _ := x.(*node) h.keyMap[n] = len(h.array) h.array = append(h.array, n) } func (h *ttlKeyHeap) Pop() any { old := h.array n := len(old) x := old[n-1] // Set slice element to nil, so GC can recycle the node. // This is due to golang GC doesn't support partial recycling: // https://github.com/golang/go/issues/9618 old[n-1] = nil h.array = old[0 : n-1] delete(h.keyMap, x) return x } func (h *ttlKeyHeap) top() *node { if h.Len() != 0 { return h.array[0] } return nil } func (h *ttlKeyHeap) pop() *node { x := heap.Pop(h) n, _ := x.(*node) return n } func (h *ttlKeyHeap) push(x any) { heap.Push(h, x) } func (h *ttlKeyHeap) update(n *node) { index, ok := h.keyMap[n] if ok { heap.Remove(h, index) heap.Push(h, n) } } func (h *ttlKeyHeap) remove(n *node) { index, ok := h.keyMap[n] if ok { heap.Remove(h, index) } } ================================================ FILE: server/etcdserver/api/v2store/watcher.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store type Watcher interface { EventChan() chan *Event StartIndex() uint64 // The EtcdIndex at which the Watcher was created Remove() } type watcher struct { eventChan chan *Event stream bool recursive bool sinceIndex uint64 startIndex uint64 hub *watcherHub removed bool remove func() } func (w *watcher) EventChan() chan *Event { return w.eventChan } func (w *watcher) StartIndex() uint64 { return w.startIndex } // notify function notifies the watcher. If the watcher interests in the given path, // the function will return true. func (w *watcher) notify(e *Event, originalPath bool, deleted bool) bool { // watcher is interested the path in three cases and under one condition // the condition is that the event happens after the watcher's sinceIndex // 1. the path at which the event happens is the path the watcher is watching at. // For example if the watcher is watching at "/foo" and the event happens at "/foo", // the watcher must be interested in that event. // 2. the watcher is a recursive watcher, it interests in the event happens after // its watching path. For example if watcher A watches at "/foo" and it is a recursive // one, it will interest in the event happens at "/foo/bar". // 3. when we delete a directory, we need to force notify all the watchers who watches // at the file we need to delete. // For example a watcher is watching at "/foo/bar". And we deletes "/foo". The watcher // should get notified even if "/foo" is not the path it is watching. if (w.recursive || originalPath || deleted) && e.Index() >= w.sinceIndex { // We cannot block here if the eventChan capacity is full, otherwise // etcd will hang. eventChan capacity is full when the rate of // notifications are higher than our send rate. // If this happens, we close the channel. select { case w.eventChan <- e: default: // We have missed a notification. Remove the watcher. // Removing the watcher also closes the eventChan. w.remove() } return true } return false } // Remove removes the watcher from watcherHub // The actual remove function is guaranteed to only be executed once func (w *watcher) Remove() { w.hub.mutex.Lock() defer w.hub.mutex.Unlock() close(w.eventChan) if w.remove != nil { w.remove() } } // nopWatcher is a watcher that receives nothing, always blocking. type nopWatcher struct{} func NewNopWatcher() Watcher { return &nopWatcher{} } func (w *nopWatcher) EventChan() chan *Event { return nil } func (w *nopWatcher) StartIndex() uint64 { return 0 } func (w *nopWatcher) Remove() {} ================================================ FILE: server/etcdserver/api/v2store/watcher_hub.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import ( "container/list" "path" "strings" "sync" "sync/atomic" "go.etcd.io/etcd/server/v3/etcdserver/api/v2error" ) // A watcherHub contains all subscribed watchers // watchers is a map with watched path as key and watcher as value // EventHistory keeps the old events for watcherHub. It is used to help // watcher to get a continuous event history. Or a watcher might miss the // event happens between the end of the first watch command and the start // of the second command. type watcherHub struct { // count must be the first element to keep 64-bit alignment for atomic // access count int64 // current number of watchers. mutex sync.Mutex watchers map[string]*list.List EventHistory *EventHistory } // newWatchHub creates a watcherHub. The capacity determines how many events we will // keep in the eventHistory. // Typically, we only need to keep a small size of history[smaller than 20K]. // Ideally, it should smaller than 20K/s[max throughput] * 2 * 50ms[RTT] = 2000 func newWatchHub(capacity int) *watcherHub { return &watcherHub{ watchers: make(map[string]*list.List), EventHistory: newEventHistory(capacity), } } // Watch function returns a Watcher. // If recursive is true, the first change after index under key will be sent to the event channel of the watcher. // If recursive is false, the first change after index at key will be sent to the event channel of the watcher. // If index is zero, watch will start from the current index + 1. func (wh *watcherHub) watch(key string, recursive, stream bool, index, storeIndex uint64) (Watcher, *v2error.Error) { reportWatchRequest() event, err := wh.EventHistory.scan(key, recursive, index) if err != nil { err.Index = storeIndex return nil, err } w := &watcher{ eventChan: make(chan *Event, 100), // use a buffered channel recursive: recursive, stream: stream, sinceIndex: index, startIndex: storeIndex, hub: wh, } wh.mutex.Lock() defer wh.mutex.Unlock() // If the event exists in the known history, append the EtcdIndex and return immediately if event != nil { ne := event.Clone() ne.EtcdIndex = storeIndex w.eventChan <- ne return w, nil } l, ok := wh.watchers[key] var elem *list.Element if ok { // add the new watcher to the back of the list elem = l.PushBack(w) } else { // create a new list and add the new watcher l = list.New() elem = l.PushBack(w) wh.watchers[key] = l } w.remove = func() { if w.removed { // avoid removing it twice return } w.removed = true l.Remove(elem) atomic.AddInt64(&wh.count, -1) reportWatcherRemoved() if l.Len() == 0 { delete(wh.watchers, key) } } atomic.AddInt64(&wh.count, 1) reportWatcherAdded() return w, nil } func (wh *watcherHub) add(e *Event) { wh.EventHistory.addEvent(e) } // notify function accepts an event and notify to the watchers. func (wh *watcherHub) notify(e *Event) { e = wh.EventHistory.addEvent(e) // add event into the eventHistory segments := strings.Split(e.Node.Key, "/") currPath := "/" // walk through all the segments of the path and notify the watchers // if the path is "/foo/bar", it will notify watchers with path "/", // "/foo" and "/foo/bar" for _, segment := range segments { currPath = path.Join(currPath, segment) // notify the watchers who interests in the changes of current path wh.notifyWatchers(e, currPath, false) } } func (wh *watcherHub) notifyWatchers(e *Event, nodePath string, deleted bool) { wh.mutex.Lock() defer wh.mutex.Unlock() l, ok := wh.watchers[nodePath] if ok { curr := l.Front() for curr != nil { next := curr.Next() // save reference to the next one in the list w, _ := curr.Value.(*watcher) originalPath := e.Node.Key == nodePath if (originalPath || !isHidden(nodePath, e.Node.Key)) && w.notify(e, originalPath, deleted) { if !w.stream { // do not remove the stream watcher // if we successfully notify a watcher // we need to remove the watcher from the list // and decrease the counter w.removed = true l.Remove(curr) atomic.AddInt64(&wh.count, -1) reportWatcherRemoved() } } curr = next // update current to the next element in the list } if l.Len() == 0 { // if we have notified all watcher in the list // we can delete the list delete(wh.watchers, nodePath) } } } // clone function clones the watcherHub and return the cloned one. // only clone the static content. do not clone the current watchers. func (wh *watcherHub) clone() *watcherHub { clonedHistory := wh.EventHistory.clone() return &watcherHub{ EventHistory: clonedHistory, } } // isHidden checks to see if key path is considered hidden to watch path i.e. the // last element is hidden or it's within a hidden directory func isHidden(watchPath, keyPath string) bool { // When deleting a directory, watchPath might be deeper than the actual keyPath // For example, when deleting /foo we also need to notify watchers on /foo/bar. if len(watchPath) > len(keyPath) { return false } // if watch path is just a "/", after path will start without "/" // add a "/" to deal with the special case when watchPath is "/" afterPath := path.Clean("/" + keyPath[len(watchPath):]) return strings.Contains(afterPath, "/_") } ================================================ FILE: server/etcdserver/api/v2store/watcher_hub_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import "testing" // TestIsHidden tests isHidden functions. func TestIsHidden(t *testing.T) { // watch at "/" // key is "/_foo", hidden to "/" // expected: hidden = true watch := "/" key := "/_foo" hidden := isHidden(watch, key) if !hidden { t.Fatalf("%v should be hidden to %v\n", key, watch) } // watch at "/_foo" // key is "/_foo", not hidden to "/_foo" // expected: hidden = false watch = "/_foo" hidden = isHidden(watch, key) if hidden { t.Fatalf("%v should not be hidden to %v\n", key, watch) } // watch at "/_foo/" // key is "/_foo/foo", not hidden to "/_foo" key = "/_foo/foo" hidden = isHidden(watch, key) if hidden { t.Fatalf("%v should not be hidden to %v\n", key, watch) } // watch at "/_foo/" // key is "/_foo/_foo", hidden to "/_foo" key = "/_foo/_foo" hidden = isHidden(watch, key) if !hidden { t.Fatalf("%v should be hidden to %v\n", key, watch) } // watch at "/_foo/foo" // key is "/_foo" watch = "_foo/foo" key = "/_foo/" hidden = isHidden(watch, key) if hidden { t.Fatalf("%v should not be hidden to %v\n", key, watch) } } ================================================ FILE: server/etcdserver/api/v2store/watcher_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store import "testing" func TestWatcher(t *testing.T) { s := newStore() wh := s.WatcherHub w, err := wh.watch("/foo", true, false, 1, 1) if err != nil { t.Fatalf("%v", err) } c := w.EventChan() select { case <-c: t.Fatal("should not receive from channel before send the event") default: // do nothing } e := newEvent(Create, "/foo/bar", 1, 1) wh.notify(e) re := <-c if e != re { t.Fatal("recv != send") } w, _ = wh.watch("/foo", false, false, 2, 1) c = w.EventChan() e = newEvent(Create, "/foo/bar", 2, 2) wh.notify(e) select { case re = <-c: t.Fatal("should not receive from channel if not recursive ", re) default: // do nothing } e = newEvent(Create, "/foo", 3, 3) wh.notify(e) re = <-c if e != re { t.Fatal("recv != send") } // ensure we are doing exact matching rather than prefix matching w, _ = wh.watch("/fo", true, false, 1, 1) c = w.EventChan() select { case re = <-c: t.Fatal("should not receive from channel:", re) default: // do nothing } e = newEvent(Create, "/fo/bar", 3, 3) wh.notify(e) re = <-c if e != re { t.Fatal("recv != send") } } ================================================ FILE: server/etcdserver/api/v3alarm/alarms.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package v3alarm manages health status alarms in etcd. package v3alarm import ( "sync" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" ) type BackendGetter interface { Backend() backend.Backend } type alarmSet map[types.ID]*pb.AlarmMember // AlarmStore persists alarms to the backend. type AlarmStore struct { lg *zap.Logger mu sync.Mutex types map[pb.AlarmType]alarmSet be schema.AlarmBackend } func NewAlarmStore(lg *zap.Logger, be schema.AlarmBackend) (*AlarmStore, error) { if lg == nil { lg = zap.NewNop() } ret := &AlarmStore{lg: lg, types: make(map[pb.AlarmType]alarmSet), be: be} err := ret.restore() return ret, err } func (a *AlarmStore) Activate(id types.ID, at pb.AlarmType) *pb.AlarmMember { a.mu.Lock() defer a.mu.Unlock() newAlarm := &pb.AlarmMember{MemberID: uint64(id), Alarm: at} if m := a.addToMap(newAlarm); m != newAlarm { return m } a.be.MustPutAlarm(newAlarm) return newAlarm } func (a *AlarmStore) Deactivate(id types.ID, at pb.AlarmType) *pb.AlarmMember { a.mu.Lock() defer a.mu.Unlock() t := a.types[at] if t == nil { t = make(alarmSet) a.types[at] = t } m := t[id] if m == nil { return nil } delete(t, id) a.be.MustDeleteAlarm(m) return m } func (a *AlarmStore) Get(at pb.AlarmType) (ret []*pb.AlarmMember) { a.mu.Lock() defer a.mu.Unlock() if at == pb.AlarmType_NONE { for _, t := range a.types { for _, m := range t { ret = append(ret, m) } } return ret } for _, m := range a.types[at] { ret = append(ret, m) } return ret } func (a *AlarmStore) restore() error { a.be.CreateAlarmBucket() ms, err := a.be.GetAllAlarms() if err != nil { return err } for _, m := range ms { a.addToMap(m) } a.be.ForceCommit() return err } func (a *AlarmStore) addToMap(newAlarm *pb.AlarmMember) *pb.AlarmMember { t := a.types[newAlarm.Alarm] if t == nil { t = make(alarmSet) a.types[newAlarm.Alarm] = t } m := t[types.ID(newAlarm.MemberID)] if m != nil { return m } t[types.ID(newAlarm.MemberID)] = newAlarm return newAlarm } ================================================ FILE: server/etcdserver/api/v3client/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package v3client provides clientv3 interfaces from an etcdserver. // // Use v3client by creating an EtcdServer instance, then wrapping it with v3client.New: // // import ( // "context" // // "go.etcd.io/etcd/server/v3/embed" // "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" // ) // // ... // // // create an embedded EtcdServer from the default configuration // cfg := embed.NewConfig() // cfg.Dir = "default.etcd" // e, err := embed.StartEtcd(cfg) // if err != nil { // // handle error! // } // // // wrap the EtcdServer with v3client // cli := v3client.New(e.Server) // // // use like an ordinary clientv3 // resp, err := cli.Put(context.TODO(), "some-key", "it works!") // if err != nil { // // handle error! // } package v3client ================================================ FILE: server/etcdserver/api/v3client/v3client.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3client import ( "context" "time" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc" "go.etcd.io/etcd/server/v3/proxy/grpcproxy/adapter" ) // New creates a clientv3 client that wraps an in-process EtcdServer. Instead // of making gRPC calls through sockets, the client makes direct function calls // to the etcd server through its api/v3rpc function interfaces. func New(s *etcdserver.EtcdServer) *clientv3.Client { c := clientv3.NewCtxClient(context.Background(), clientv3.WithZapLogger(s.Logger())) kvc := adapter.KvServerToKvClient(v3rpc.NewQuotaKVServer(s)) c.KV = clientv3.NewKVFromKVClient(kvc, c) lc := adapter.LeaseServerToLeaseClient(v3rpc.NewQuotaLeaseServer(s)) c.Lease = clientv3.NewLeaseFromLeaseClient(lc, c, time.Second) wc := adapter.WatchServerToWatchClient(v3rpc.NewWatchServer(s)) c.Watcher = &watchWrapper{clientv3.NewWatchFromWatchClient(wc, c)} mc := adapter.MaintenanceServerToMaintenanceClient(v3rpc.NewMaintenanceServer(s, nil)) c.Maintenance = clientv3.NewMaintenanceFromMaintenanceClient(mc, c) clc := adapter.ClusterServerToClusterClient(v3rpc.NewClusterServer(s)) c.Cluster = clientv3.NewClusterFromClusterClient(clc, c) a := adapter.AuthServerToAuthClient(v3rpc.NewAuthServer(s)) c.Auth = clientv3.NewAuthFromAuthClient(a, c) return c } // BlankContext implements Stringer on a context so the ctx string doesn't // depend on the context's WithValue data, which tends to be unsynchronized // (e.g., x/net/trace), causing ctx.String() to throw data races. type blankContext struct{ context.Context } func (*blankContext) String() string { return "(blankCtx)" } // watchWrapper wraps clientv3 watch calls to blank out the context // to avoid races on trace data. type watchWrapper struct{ clientv3.Watcher } func (ww *watchWrapper) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan { return ww.Watcher.Watch(&blankContext{ctx}, key, opts...) } ================================================ FILE: server/etcdserver/api/v3compactor/compactor.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3compactor import ( "context" "fmt" "time" "github.com/jonboulle/clockwork" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) const ( ModePeriodic = "periodic" ModeRevision = "revision" ) // Compactor purges old log from the storage periodically. type Compactor interface { // Run starts the main loop of the compactor in background. // Use Stop() to halt the loop and release the resource. Run() // Stop halts the main loop of the compactor. Stop() // Pause temporally suspend the compactor not to run compaction. Resume() to unpose. Pause() // Resume restarts the compactor suspended by Pause(). Resume() } type Compactable interface { Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) } type RevGetter interface { Rev() int64 } // New returns a new Compactor based on given "mode". func New( lg *zap.Logger, mode string, retention time.Duration, rg RevGetter, c Compactable, ) (Compactor, error) { if lg == nil { lg = zap.NewNop() } switch mode { case ModePeriodic: return newPeriodic(lg, clockwork.NewRealClock(), retention, rg, c), nil case ModeRevision: return newRevision(lg, clockwork.NewRealClock(), int64(retention), rg, c), nil default: return nil, fmt.Errorf("unsupported compaction mode %s", mode) } } ================================================ FILE: server/etcdserver/api/v3compactor/compactor_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3compactor import ( "context" "sync/atomic" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/testutil" ) type fakeCompactable struct { testutil.Recorder } func (fc *fakeCompactable) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) { fc.Record(testutil.Action{Name: "c", Params: []any{r}}) return &pb.CompactionResponse{}, nil } type fakeRevGetter struct { testutil.Recorder rev int64 } func (fr *fakeRevGetter) Rev() int64 { fr.Record(testutil.Action{Name: "g"}) rev := atomic.AddInt64(&fr.rev, 1) return rev } func (fr *fakeRevGetter) SetRev(rev int64) { atomic.StoreInt64(&fr.rev, rev) } ================================================ FILE: server/etcdserver/api/v3compactor/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package v3compactor implements automated policies for compacting etcd's mvcc storage. package v3compactor ================================================ FILE: server/etcdserver/api/v3compactor/periodic.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3compactor import ( "context" "errors" "sync" "time" "github.com/jonboulle/clockwork" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/storage/mvcc" ) // Periodic compacts the log by purging revisions older than // the configured retention time. type Periodic struct { lg *zap.Logger clock clockwork.Clock period time.Duration rg RevGetter c Compactable revs []int64 ctx context.Context cancel context.CancelFunc // mu protects paused mu sync.RWMutex paused bool } // newPeriodic creates a new instance of Periodic compactor that purges // the log older than h Duration. func newPeriodic(lg *zap.Logger, clock clockwork.Clock, h time.Duration, rg RevGetter, c Compactable) *Periodic { pc := &Periodic{ lg: lg, clock: clock, period: h, rg: rg, c: c, } // revs won't be longer than the retentions. pc.revs = make([]int64, 0, pc.getRetentions()) pc.ctx, pc.cancel = context.WithCancel(context.Background()) return pc } /* Compaction period 1-hour: 1. compute compaction period, which is 1-hour 2. record revisions for every 1/10 of 1-hour (6-minute) 3. keep recording revisions with no compaction for first 1-hour 4. do compact with revs[0] - success? continue on for-loop and move sliding window; revs = revs[1:] - failure? update revs, and retry after 1/10 of 1-hour (6-minute) Compaction period 24-hour: 1. compute compaction period, which is 24-hour 2. record revisions for every 1/10 of 24-hour (144-minute) 3. keep recording revisions with no compaction for first 24-hour 4. do compact with revs[0] - success? continue on for-loop and move sliding window; revs = revs[1:] - failure? update revs, and retry after 1/10 of 24-hour (144-minute) Compaction period 59-min: 1. compute compaction period, which is 59-min 2. record revisions for every 1/10 of 59-min (5.9-min) 3. keep recording revisions with no compaction for first 59-min 4. do compact with revs[0] - success? continue on for-loop and move sliding window; revs = revs[1:] - failure? update revs, and retry after 1/10 of 59-min (5.9-min) Compaction period 5-sec: 1. compute compaction period, which is 5-sec 2. record revisions for every 1/10 of 5-sec (0.5-sec) 3. keep recording revisions with no compaction for first 5-sec 4. do compact with revs[0] - success? continue on for-loop and move sliding window; revs = revs[1:] - failure? update revs, and retry after 1/10 of 5-sec (0.5-sec) */ // Run runs periodic compactor. func (pc *Periodic) Run() { compactInterval := pc.getCompactInterval() retryInterval := pc.getRetryInterval() retentions := pc.getRetentions() go func() { lastRevision := int64(0) lastSuccess := pc.clock.Now() baseInterval := pc.period for { pc.revs = append(pc.revs, pc.rg.Rev()) if len(pc.revs) > retentions { pc.revs = pc.revs[1:] // pc.revs[0] is always the rev at pc.period ago } select { case <-pc.ctx.Done(): return case <-pc.clock.After(retryInterval): pc.mu.RLock() p := pc.paused pc.mu.RUnlock() if p { continue } } rev := pc.revs[0] if pc.clock.Now().Sub(lastSuccess) < baseInterval || rev == lastRevision { continue } // wait up to initial given period if baseInterval == pc.period { baseInterval = compactInterval } pc.lg.Info( "starting auto periodic compaction", zap.Int64("revision", rev), zap.Duration("compact-period", pc.period), ) startTime := pc.clock.Now() _, err := pc.c.Compact(pc.ctx, &pb.CompactionRequest{Revision: rev}) if err == nil || errors.Is(err, mvcc.ErrCompacted) { pc.lg.Info( "completed auto periodic compaction", zap.Int64("revision", rev), zap.Duration("compact-period", pc.period), zap.Duration("took", pc.clock.Now().Sub(startTime)), ) lastRevision = rev lastSuccess = pc.clock.Now() } else { pc.lg.Warn( "failed auto periodic compaction", zap.Int64("revision", rev), zap.Duration("compact-period", pc.period), zap.Duration("retry-interval", retryInterval), zap.Error(err), ) } } }() } // if given compaction period x is <1-hour, compact every x duration. // (e.g. --auto-compaction-mode 'periodic' --auto-compaction-retention='10m', then compact every 10-minute) // if given compaction period x is >1-hour, compact every hour. // (e.g. --auto-compaction-mode 'periodic' --auto-compaction-retention='2h', then compact every 1-hour) func (pc *Periodic) getCompactInterval() time.Duration { itv := pc.period if itv > time.Hour { itv = time.Hour } return itv } func (pc *Periodic) getRetentions() int { return int(pc.period/pc.getRetryInterval()) + 1 } const retryDivisor = 10 func (pc *Periodic) getRetryInterval() time.Duration { itv := pc.period if itv > time.Hour { itv = time.Hour } return itv / retryDivisor } // Stop stops periodic compactor. func (pc *Periodic) Stop() { pc.cancel() } // Pause pauses periodic compactor. func (pc *Periodic) Pause() { pc.mu.Lock() pc.paused = true pc.mu.Unlock() } // Resume resumes periodic compactor. func (pc *Periodic) Resume() { pc.mu.Lock() pc.paused = false pc.mu.Unlock() } ================================================ FILE: server/etcdserver/api/v3compactor/periodic_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3compactor import ( "errors" "fmt" "reflect" "testing" "time" "github.com/jonboulle/clockwork" "go.uber.org/zap/zaptest" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func TestPeriodicHourly(t *testing.T) { retentionHours := 2 retentionDuration := time.Duration(retentionHours) * time.Hour fc := clockwork.NewFakeClock() // TODO: Do not depand or real time (Recorder.Wait) in unit tests. rg := &fakeRevGetter{testutil.NewRecorderStreamWithWaitTimout(0), 0} compactable := &fakeCompactable{testutil.NewRecorderStreamWithWaitTimout(10 * time.Millisecond)} tb := newPeriodic(zaptest.NewLogger(t), fc, retentionDuration, rg, compactable) tb.Run() defer tb.Stop() initialIntervals, intervalsPerPeriod := tb.getRetentions(), 10 // compaction doesn't happen til 2 hours elapse for i := 0; i < initialIntervals-1; i++ { waitOneAction(t, rg) fc.Advance(tb.getRetryInterval()) } // very first compaction a, err := waitWithRetry(t, compactable) if err != nil { t.Fatal(err) } expectedRevision := int64(1) if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) { t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) } // simulate 3 hours // now compactor kicks in, every hour for i := 0; i < 3; i++ { // advance one hour, one revision for each interval for j := 0; j < intervalsPerPeriod; j++ { waitOneAction(t, rg) fc.Advance(tb.getRetryInterval()) } a, err = waitWithRetry(t, compactable) if err != nil { t.Fatal(err) } expectedRevision = int64((i + 1) * 10) if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) { t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) } } } func TestPeriodicMinutes(t *testing.T) { retentionMinutes := 5 retentionDuration := time.Duration(retentionMinutes) * time.Minute fc := clockwork.NewFakeClock() rg := &fakeRevGetter{testutil.NewRecorderStreamWithWaitTimout(0), 0} compactable := &fakeCompactable{testutil.NewRecorderStreamWithWaitTimout(10 * time.Millisecond)} tb := newPeriodic(zaptest.NewLogger(t), fc, retentionDuration, rg, compactable) tb.Run() defer tb.Stop() initialIntervals, intervalsPerPeriod := tb.getRetentions(), 10 // compaction doesn't happen til 5 minutes elapse for i := 0; i < initialIntervals-1; i++ { waitOneAction(t, rg) fc.Advance(tb.getRetryInterval()) } // very first compaction a, err := waitWithRetry(t, compactable) if err != nil { t.Fatal(err) } expectedRevision := int64(1) if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) { t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) } // compaction happens at every interval for i := 0; i < 5; i++ { // advance 5-minute, one revision for each interval for j := 0; j < intervalsPerPeriod; j++ { waitOneAction(t, rg) fc.Advance(tb.getRetryInterval()) } a, err := waitWithRetry(t, compactable) if err != nil { t.Fatal(err) } expectedRevision = int64((i + 1) * 10) if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) { t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) } } } func TestPeriodicPause(t *testing.T) { fc := clockwork.NewFakeClock() retentionDuration := time.Hour rg := &fakeRevGetter{testutil.NewRecorderStreamWithWaitTimout(0), 0} compactable := &fakeCompactable{testutil.NewRecorderStreamWithWaitTimout(10 * time.Millisecond)} tb := newPeriodic(zaptest.NewLogger(t), fc, retentionDuration, rg, compactable) tb.Run() tb.Pause() n := tb.getRetentions() // tb will collect 3 hours of revisions but not compact since paused for i := 0; i < n*3; i++ { waitOneAction(t, rg) fc.Advance(tb.getRetryInterval()) } // t.revs = [21 22 23 24 25 26 27 28 29 30] select { case a := <-compactable.Chan(): t.Fatalf("unexpected action %v", a) case <-time.After(10 * time.Millisecond): } // tb resumes to being blocked on the clock tb.Resume() waitOneAction(t, rg) // unblock clock, will kick off a compaction at T=3h6m by retry fc.Advance(tb.getRetryInterval()) // T=3h6m a, err := waitWithRetry(t, compactable) if err != nil { t.Fatal(err) } // compact the revision from hour 2:06 wreq := &pb.CompactionRequest{Revision: int64(1 + 2*n + 1)} if !reflect.DeepEqual(a[0].Params[0], wreq) { t.Errorf("compact request = %v, want %v", a[0].Params[0], wreq.Revision) } } func TestPeriodicSkipRevNotChange(t *testing.T) { retentionMinutes := 5 retentionDuration := time.Duration(retentionMinutes) * time.Minute fc := clockwork.NewFakeClock() rg := &fakeRevGetter{testutil.NewRecorderStreamWithWaitTimout(0), 0} compactable := &fakeCompactable{testutil.NewRecorderStreamWithWaitTimout(20 * time.Millisecond)} tb := newPeriodic(zaptest.NewLogger(t), fc, retentionDuration, rg, compactable) tb.Run() defer tb.Stop() initialIntervals, intervalsPerPeriod := tb.getRetentions(), 10 // first compaction happens til 5 minutes elapsed for i := 0; i < initialIntervals-1; i++ { // every time set the same revision with 100 rg.SetRev(int64(100)) waitOneAction(t, rg) fc.Advance(tb.getRetryInterval()) } // very first compaction a, err := waitWithRetry(t, compactable) if err != nil { t.Fatal(err) } // first compaction the compact revision will be 100+1 expectedRevision := int64(100 + 1) if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) { t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) } // compaction doesn't happens at every interval since revision not change for i := 0; i < 5; i++ { for j := 0; j < intervalsPerPeriod; j++ { rg.SetRev(int64(100)) waitOneAction(t, rg) fc.Advance(tb.getRetryInterval()) } _, err = compactable.Wait(1) if err == nil { t.Fatal(errors.New("should not compact since the revision not change")) } } // when revision changed, compaction is normally for i := 0; i < initialIntervals; i++ { waitOneAction(t, rg) fc.Advance(tb.getRetryInterval()) } a, err = waitWithRetry(t, compactable) if err != nil { t.Fatal(err) } expectedRevision = int64(100 + 2) if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) { t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) } } func waitOneAction(t *testing.T, r testutil.Recorder) { if actions, _ := r.Wait(1); len(actions) != 1 { t.Errorf("expect 1 action, got %v instead", len(actions)) } } func waitWithRetry(t *testing.T, compactable *fakeCompactable) ([]testutil.Action, error) { t.Helper() var lastErr error var actions []testutil.Action expectedActions, maxRetries := 1, 5 for retry := 0; retry < maxRetries; retry++ { actions, lastErr = compactable.Wait(expectedActions) if lastErr == nil || len(actions) >= expectedActions { return actions, nil } // Exponential backoff backoffTime := time.Duration(10*(1</members/". peerRegKey string // peerURLsMap format: "peerName=peerURLs", i.e., "member1=http://127.0.0.1:2380". peerURLsMap string // createRev is the member's CreateRevision in the etcd cluster backing // the discovery service. createRev int64 } type clusterInfo struct { clusterToken string members []memberInfo } // key prefix for each cluster: "/_etcd/registry/". func getClusterKeyPrefix(cluster string) string { return path.Join(discoveryPrefix, cluster) } // key format for cluster size: "/_etcd/registry//_config/size". func getClusterSizeKey(cluster string) string { return path.Join(getClusterKeyPrefix(cluster), "_config/size") } // key prefix for each member: "/_etcd/registry//members". func getMemberKeyPrefix(clusterToken string) string { return path.Join(getClusterKeyPrefix(clusterToken), "members") } // key format for each member: "/_etcd/registry//members/". func getMemberKey(cluster, memberID string) string { return path.Join(getMemberKeyPrefix(cluster), memberID) } // GetCluster will connect to the discovery service at the given endpoints and // retrieve a string describing the cluster func GetCluster(lg *zap.Logger, cfg *DiscoveryConfig) (cs string, rerr error) { d, err := newDiscovery(lg, cfg, 0) if err != nil { return "", err } defer d.close() defer func() { if rerr != nil { d.lg.Error( "discovery failed to get cluster", zap.String("cluster", cs), zap.Error(rerr), ) } else { d.lg.Info( "discovery got cluster successfully", zap.String("cluster", cs), ) } }() return d.getCluster() } // JoinCluster will connect to the discovery service at the endpoints, and // register the server represented by the given id and config to the cluster. // The parameter `config` is supposed to be in the format "memberName=peerURLs", // such as "member1=http://127.0.0.1:2380". // // The final returned string has the same format as "--initial-cluster", such as // "infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380". func JoinCluster(lg *zap.Logger, cfg *DiscoveryConfig, id types.ID, config string) (cs string, rerr error) { d, err := newDiscovery(lg, cfg, id) if err != nil { return "", err } defer d.close() defer func() { if rerr != nil { d.lg.Error( "discovery failed to join cluster", zap.String("cluster", cs), zap.Error(rerr), ) } else { d.lg.Info( "discovery joined cluster successfully", zap.String("cluster", cs), ) } }() return d.joinCluster(config) } type discovery struct { lg *zap.Logger clusterToken string memberID types.ID c *clientv3.Client retries uint cfg *DiscoveryConfig clock clockwork.Clock } func newDiscovery(lg *zap.Logger, dcfg *DiscoveryConfig, id types.ID) (*discovery, error) { if lg == nil { lg = zap.NewNop() } lg = lg.With(zap.String("discovery-token", dcfg.Token), zap.String("discovery-endpoints", strings.Join(dcfg.Endpoints, ","))) cfg, err := clientv3.NewClientConfig(&dcfg.ConfigSpec, lg) if err != nil { return nil, err } c, err := clientv3.New(*cfg) if err != nil { return nil, err } return &discovery{ lg: lg, clusterToken: dcfg.Token, memberID: id, c: c, cfg: dcfg, clock: clockwork.NewRealClock(), }, nil } func (d *discovery) getCluster() (string, error) { cls, clusterSize, rev, err := d.checkCluster() if err != nil { if errors.Is(err, ErrFullCluster) { return cls.getInitClusterStr(clusterSize) } return "", err } for cls.Len() < clusterSize { d.waitPeers(cls, clusterSize, rev) } return cls.getInitClusterStr(clusterSize) } func (d *discovery) joinCluster(config string) (string, error) { _, _, _, err := d.checkCluster() if err != nil { return "", err } if err = d.registerSelf(config); err != nil { return "", err } cls, clusterSize, rev, err := d.checkCluster() if err != nil { return "", err } for cls.Len() < clusterSize { d.waitPeers(cls, clusterSize, rev) } return cls.getInitClusterStr(clusterSize) } func (d *discovery) getClusterSize() (int, error) { configKey := getClusterSizeKey(d.clusterToken) ctx, cancel := context.WithTimeout(context.Background(), d.cfg.RequestTimeout) defer cancel() resp, err := d.c.Get(ctx, configKey) if err != nil { d.lg.Warn( "failed to get cluster size from discovery service", zap.String("clusterSizeKey", configKey), zap.Error(err), ) return 0, err } if len(resp.Kvs) == 0 { return 0, ErrSizeNotFound } clusterSize, err := strconv.ParseInt(string(resp.Kvs[0].Value), 10, 0) if err != nil || clusterSize <= 0 { return 0, ErrBadSizeKey } return int(clusterSize), nil } func (d *discovery) getClusterMembers() (*clusterInfo, int64, error) { membersKeyPrefix := getMemberKeyPrefix(d.clusterToken) ctx, cancel := context.WithTimeout(context.Background(), d.cfg.RequestTimeout) defer cancel() resp, err := d.c.Get(ctx, membersKeyPrefix, clientv3.WithPrefix()) if err != nil { d.lg.Warn( "failed to get cluster members from discovery service", zap.String("membersKeyPrefix", membersKeyPrefix), zap.Error(err), ) return nil, 0, err } cls := &clusterInfo{clusterToken: d.clusterToken} for _, kv := range resp.Kvs { mKey := strings.TrimSpace(string(kv.Key)) mValue := strings.TrimSpace(string(kv.Value)) if err := cls.add(mKey, mValue, kv.CreateRevision); err != nil { d.lg.Warn( err.Error(), zap.String("memberKey", mKey), zap.String("memberInfo", mValue), ) } else { d.lg.Info( "found peer from discovery service", zap.String("memberKey", mKey), zap.String("memberInfo", mValue), ) } } return cls, resp.Header.Revision, nil } func (d *discovery) checkClusterRetry() (*clusterInfo, int, int64, error) { if d.retries < nRetries { d.logAndBackoffForRetry("cluster status check") return d.checkCluster() } return nil, 0, 0, ErrTooManyRetries } func (d *discovery) checkCluster() (*clusterInfo, int, int64, error) { clusterSize, err := d.getClusterSize() if err != nil { if errors.Is(err, ErrSizeNotFound) || errors.Is(err, ErrBadSizeKey) { return nil, 0, 0, err } return d.checkClusterRetry() } cls, rev, err := d.getClusterMembers() if err != nil { return d.checkClusterRetry() } d.retries = 0 // find self position memberSelfID := getMemberKey(d.clusterToken, d.memberID.String()) idx := 0 for _, m := range cls.members { if m.peerRegKey == memberSelfID { break } if idx >= clusterSize-1 { return cls, clusterSize, rev, ErrFullCluster } idx++ } return cls, clusterSize, rev, nil } func (d *discovery) registerSelfRetry(contents string) error { if d.retries < nRetries { d.logAndBackoffForRetry("register member itself") return d.registerSelf(contents) } return ErrTooManyRetries } func (d *discovery) registerSelf(contents string) error { ctx, cancel := context.WithTimeout(context.Background(), d.cfg.RequestTimeout) memberKey := getMemberKey(d.clusterToken, d.memberID.String()) _, err := d.c.Put(ctx, memberKey, contents) cancel() if err != nil { d.lg.Warn( "failed to register members itself to the discovery service", zap.String("memberKey", memberKey), zap.Error(err), ) return d.registerSelfRetry(contents) } d.retries = 0 d.lg.Info( "register member itself successfully", zap.String("memberKey", memberKey), zap.String("memberInfo", contents), ) return nil } func (d *discovery) waitPeers(cls *clusterInfo, clusterSize int, rev int64) { // watch from the next revision membersKeyPrefix := getMemberKeyPrefix(d.clusterToken) w := d.c.Watch(context.Background(), membersKeyPrefix, clientv3.WithPrefix(), clientv3.WithRev(rev+1)) d.lg.Info( "waiting for peers from discovery service", zap.Int("clusterSize", clusterSize), zap.Int("found-peers", cls.Len()), ) // waiting for peers until all needed peers are returned for wresp := range w { for _, ev := range wresp.Events { mKey := strings.TrimSpace(string(ev.Kv.Key)) mValue := strings.TrimSpace(string(ev.Kv.Value)) if err := cls.add(mKey, mValue, ev.Kv.CreateRevision); err != nil { d.lg.Warn( err.Error(), zap.String("memberKey", mKey), zap.String("memberInfo", mValue), ) } else { d.lg.Info( "found peer from discovery service", zap.String("memberKey", mKey), zap.String("memberInfo", mValue), ) } } if cls.Len() >= clusterSize { break } } d.lg.Info( "found all needed peers from discovery service", zap.Int("clusterSize", clusterSize), zap.Int("found-peers", cls.Len()), ) } func (d *discovery) logAndBackoffForRetry(step string) { d.retries++ // logAndBackoffForRetry stops exponential backoff when the retries are // more than maxExpoentialRetries and is set to a constant backoff afterward. retries := d.retries if retries > maxExponentialRetries { retries = maxExponentialRetries } retryTimeInSecond := time.Duration(0x1< clusterSize { peerURLs = peerURLs[:clusterSize] } us := strings.Join(peerURLs, ",") _, err := types.NewURLsMap(us) if err != nil { return us, ErrInvalidURL } return us, nil } func (cls *clusterInfo) getPeerURLs() []string { var peerURLs []string for _, peer := range cls.members { peerURLs = append(peerURLs, peer.peerURLsMap) } return peerURLs } ================================================ FILE: server/etcdserver/api/v3discovery/discovery_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3discovery import ( "context" "errors" "fmt" "testing" "github.com/jonboulle/clockwork" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/client/pkg/v3/types" clientv3 "go.etcd.io/etcd/client/v3" ) // fakeKVForClusterSize is used to test getClusterSize. type fakeKVForClusterSize struct { *fakeBaseKV clusterSizeStr string } // Get when we only need to overwrite the method `Get`. func (fkv *fakeKVForClusterSize) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { if fkv.clusterSizeStr == "" { // cluster size isn't configured in this case. return &clientv3.GetResponse{}, nil } return &clientv3.GetResponse{ Kvs: []*mvccpb.KeyValue{ { Value: []byte(fkv.clusterSizeStr), }, }, }, nil } func TestGetClusterSize(t *testing.T) { cases := []struct { name string clusterSizeStr string expectedErr error expectedSize int }{ { name: "cluster size not defined", clusterSizeStr: "", expectedErr: ErrSizeNotFound, }, { name: "invalid cluster size", clusterSizeStr: "invalidSize", expectedErr: ErrBadSizeKey, }, { name: "valid cluster size", clusterSizeStr: "3", expectedErr: nil, expectedSize: 3, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) d := &discovery{ lg: lg, c: &clientv3.Client{ KV: &fakeKVForClusterSize{ fakeBaseKV: &fakeBaseKV{}, clusterSizeStr: tc.clusterSizeStr, }, }, cfg: &DiscoveryConfig{}, clusterToken: "fakeToken", } if cs, err := d.getClusterSize(); !errors.Is(err, tc.expectedErr) { t.Errorf("Unexpected error, expected: %v got: %v", tc.expectedErr, err) } else { if err == nil && cs != tc.expectedSize { t.Errorf("Unexpected cluster size, expected: %d got: %d", tc.expectedSize, cs) } } }) } } // fakeKVForClusterMembers is used to test getClusterMembers. type fakeKVForClusterMembers struct { *fakeBaseKV members []memberInfo } // Get when we only need to overwrite method `Get`. func (fkv *fakeKVForClusterMembers) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { kvs := memberInfoToKeyValues(fkv.members) return &clientv3.GetResponse{ Header: &etcdserverpb.ResponseHeader{ Revision: 10, }, Kvs: kvs, }, nil } func memberInfoToKeyValues(members []memberInfo) []*mvccpb.KeyValue { kvs := make([]*mvccpb.KeyValue, 0) for _, mi := range members { kvs = append(kvs, &mvccpb.KeyValue{ Key: []byte(mi.peerRegKey), Value: []byte(mi.peerURLsMap), CreateRevision: mi.createRev, }) } return kvs } func TestGetClusterMembers(t *testing.T) { actualMemberInfo := []memberInfo{ { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(101).String(), peerURLsMap: "infra1=http://192.168.0.100:2380", createRev: 8, }, { // invalid peer registry key peerRegKey: "/invalidPrefix/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "infra2=http://192.168.0.102:2380", createRev: 6, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "infra2=http://192.168.0.102:2380", createRev: 6, }, { // invalid peer info format peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "http://192.168.0.102:2380", createRev: 6, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(103).String(), peerURLsMap: "infra3=http://192.168.0.103:2380", createRev: 7, }, { // duplicate peer peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(101).String(), peerURLsMap: "infra1=http://192.168.0.100:2380", createRev: 2, }, } // sort by CreateRevision expectedMemberInfo := []memberInfo{ { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "infra2=http://192.168.0.102:2380", createRev: 6, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(103).String(), peerURLsMap: "infra3=http://192.168.0.103:2380", createRev: 7, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(101).String(), peerURLsMap: "infra1=http://192.168.0.100:2380", createRev: 8, }, } lg := zaptest.NewLogger(t) d := &discovery{ lg: lg, c: &clientv3.Client{ KV: &fakeKVForClusterMembers{ fakeBaseKV: &fakeBaseKV{}, members: actualMemberInfo, }, }, cfg: &DiscoveryConfig{}, clusterToken: "fakeToken", } clsInfo, _, err := d.getClusterMembers() if err != nil { t.Errorf("Failed to get cluster members, error: %v", err) } if clsInfo.Len() != len(expectedMemberInfo) { t.Errorf("unexpected member count, expected: %d, got: %d", len(expectedMemberInfo), clsInfo.Len()) } for i, m := range clsInfo.members { if m != expectedMemberInfo[i] { t.Errorf("unexpected member[%d], expected: %v, got: %v", i, expectedMemberInfo[i], m) } } } // fakeKVForCheckCluster is used to test checkCluster. type fakeKVForCheckCluster struct { *fakeBaseKV t *testing.T token string clusterSizeStr string members []memberInfo getSizeRetries int getMembersRetries int } // Get when we only need to overwrite method `Get`. func (fkv *fakeKVForCheckCluster) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { clusterSizeKey := fmt.Sprintf("/_etcd/registry/%s/_config/size", fkv.token) clusterMembersKey := fmt.Sprintf("/_etcd/registry/%s/members", fkv.token) if key == clusterSizeKey { if fkv.getSizeRetries > 0 { fkv.getSizeRetries-- // discovery client should retry on error. return nil, errors.New("get cluster size failed") } return &clientv3.GetResponse{ Kvs: []*mvccpb.KeyValue{ { Value: []byte(fkv.clusterSizeStr), }, }, }, nil } if key == clusterMembersKey { if fkv.getMembersRetries > 0 { fkv.getMembersRetries-- // discovery client should retry on error. return nil, errors.New("get cluster members failed") } kvs := memberInfoToKeyValues(fkv.members) return &clientv3.GetResponse{ Header: &etcdserverpb.ResponseHeader{ Revision: 10, }, Kvs: kvs, }, nil } fkv.t.Errorf("unexpected key: %s", key) return nil, fmt.Errorf("unexpected key: %s", key) } func TestCheckCluster(t *testing.T) { actualMemberInfo := []memberInfo{ { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(101).String(), peerURLsMap: "infra1=http://192.168.0.100:2380", createRev: 8, }, { // invalid peer registry key peerRegKey: "/invalidPrefix/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "infra2=http://192.168.0.102:2380", createRev: 6, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "infra2=http://192.168.0.102:2380", createRev: 6, }, { // invalid peer info format peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "http://192.168.0.102:2380", createRev: 6, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(103).String(), peerURLsMap: "infra3=http://192.168.0.103:2380", createRev: 7, }, { // duplicate peer peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(101).String(), peerURLsMap: "infra1=http://192.168.0.100:2380", createRev: 2, }, } // sort by CreateRevision expectedMemberInfo := []memberInfo{ { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "infra2=http://192.168.0.102:2380", createRev: 6, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(103).String(), peerURLsMap: "infra3=http://192.168.0.103:2380", createRev: 7, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(101).String(), peerURLsMap: "infra1=http://192.168.0.100:2380", createRev: 8, }, } cases := []struct { name string memberID types.ID getSizeRetries int getMembersRetries int expectedError error }{ { name: "no retries", memberID: 101, getSizeRetries: 0, getMembersRetries: 0, expectedError: nil, }, { name: "2 retries for getClusterSize", memberID: 102, getSizeRetries: 2, getMembersRetries: 0, expectedError: nil, }, { name: "2 retries for getClusterMembers", memberID: 103, getSizeRetries: 0, getMembersRetries: 2, expectedError: nil, }, { name: "error due to cluster full", memberID: 104, getSizeRetries: 0, getMembersRetries: 0, expectedError: ErrFullCluster, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) fkv := &fakeKVForCheckCluster{ fakeBaseKV: &fakeBaseKV{}, t: t, token: "fakeToken", clusterSizeStr: "3", members: actualMemberInfo, getSizeRetries: tc.getSizeRetries, getMembersRetries: tc.getMembersRetries, } d := &discovery{ lg: lg, c: &clientv3.Client{ KV: fkv, }, cfg: &DiscoveryConfig{}, clusterToken: "fakeToken", memberID: tc.memberID, clock: clockwork.NewRealClock(), } clsInfo, _, _, err := d.checkCluster() if !errors.Is(err, tc.expectedError) { t.Errorf("Unexpected error, expected: %v, got: %v", tc.expectedError, err) } if err == nil { if fkv.getSizeRetries != 0 || fkv.getMembersRetries != 0 { t.Errorf("Discovery client did not retry checking cluster on error, remaining etries: (%d, %d)", fkv.getSizeRetries, fkv.getMembersRetries) } if clsInfo.Len() != len(expectedMemberInfo) { t.Errorf("Unexpected member count, expected: %d, got: %d", len(expectedMemberInfo), clsInfo.Len()) } for mIdx, m := range clsInfo.members { if m != expectedMemberInfo[mIdx] { t.Errorf("Unexpected member[%d], expected: %v, got: %v", mIdx, expectedMemberInfo[mIdx], m) } } } }) } } // fakeKVForRegisterSelf is used to test registerSelf. type fakeKVForRegisterSelf struct { *fakeBaseKV t *testing.T expectedRegKey string expectedRegValue string retries int } // Put when we only need to overwrite method `Put`. func (fkv *fakeKVForRegisterSelf) Put(ctx context.Context, key string, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { if key != fkv.expectedRegKey { fkv.t.Errorf("unexpected register key, expected: %s, got: %s", fkv.expectedRegKey, key) } if val != fkv.expectedRegValue { fkv.t.Errorf("unexpected register value, expected: %s, got: %s", fkv.expectedRegValue, val) } if fkv.retries > 0 { fkv.retries-- // discovery client should retry on error. return nil, errors.New("register self failed") } return nil, nil } func TestRegisterSelf(t *testing.T) { cases := []struct { name string token string memberID types.ID expectedRegKey string expectedRegValue string retries int // when retries > 0, then return an error on Put request. }{ { name: "no retry with token1", token: "token1", memberID: 101, expectedRegKey: "/_etcd/registry/token1/members/" + types.ID(101).String(), expectedRegValue: "infra=http://127.0.0.1:2380", retries: 0, }, { name: "no retry with token2", token: "token2", memberID: 102, expectedRegKey: "/_etcd/registry/token2/members/" + types.ID(102).String(), expectedRegValue: "infra=http://127.0.0.1:2380", retries: 0, }, { name: "2 retries", token: "token3", memberID: 103, expectedRegKey: "/_etcd/registry/token3/members/" + types.ID(103).String(), expectedRegValue: "infra=http://127.0.0.1:2380", retries: 2, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) fkv := &fakeKVForRegisterSelf{ fakeBaseKV: &fakeBaseKV{}, t: t, expectedRegKey: tc.expectedRegKey, expectedRegValue: tc.expectedRegValue, retries: tc.retries, } d := &discovery{ lg: lg, clusterToken: tc.token, memberID: tc.memberID, cfg: &DiscoveryConfig{}, c: &clientv3.Client{ KV: fkv, }, clock: clockwork.NewRealClock(), } if err := d.registerSelf(tc.expectedRegValue); err != nil { t.Errorf("Error occurring on register member self: %v", err) } if fkv.retries != 0 { t.Errorf("Discovery client did not retry registering itself on error, remaining retries: %d", fkv.retries) } }) } } // fakeWatcherForWaitPeers is used to test waitPeers. type fakeWatcherForWaitPeers struct { *fakeBaseWatcher t *testing.T token string members []memberInfo } // Watch we only need to overwrite method `Watch`. func (fw *fakeWatcherForWaitPeers) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan { expectedWatchKey := fmt.Sprintf("/_etcd/registry/%s/members", fw.token) if key != expectedWatchKey { fw.t.Errorf("unexpected watch key, expected: %s, got: %s", expectedWatchKey, key) } ch := make(chan clientv3.WatchResponse, 1) go func() { for _, mi := range fw.members { ch <- clientv3.WatchResponse{ Events: []*clientv3.Event{ { Kv: &mvccpb.KeyValue{ Key: []byte(mi.peerRegKey), Value: []byte(mi.peerURLsMap), CreateRevision: mi.createRev, }, }, }, } } close(ch) }() return ch } func TestWaitPeers(t *testing.T) { actualMemberInfo := []memberInfo{ { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(101).String(), peerURLsMap: "infra1=http://192.168.0.100:2380", createRev: 8, }, { // invalid peer registry key peerRegKey: "/invalidPrefix/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "infra2=http://192.168.0.102:2380", createRev: 6, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "infra2=http://192.168.0.102:2380", createRev: 6, }, { // invalid peer info format peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "http://192.168.0.102:2380", createRev: 6, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(103).String(), peerURLsMap: "infra3=http://192.168.0.103:2380", createRev: 7, }, { // duplicate peer peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(101).String(), peerURLsMap: "infra1=http://192.168.0.100:2380", createRev: 2, }, } // sort by CreateRevision expectedMemberInfo := []memberInfo{ { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(102).String(), peerURLsMap: "infra2=http://192.168.0.102:2380", createRev: 6, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(103).String(), peerURLsMap: "infra3=http://192.168.0.103:2380", createRev: 7, }, { peerRegKey: "/_etcd/registry/fakeToken/members/" + types.ID(101).String(), peerURLsMap: "infra1=http://192.168.0.100:2380", createRev: 8, }, } lg := zaptest.NewLogger(t) d := &discovery{ lg: lg, c: &clientv3.Client{ KV: &fakeBaseKV{}, Watcher: &fakeWatcherForWaitPeers{ fakeBaseWatcher: &fakeBaseWatcher{}, t: t, token: "fakeToken", members: actualMemberInfo, }, }, cfg: &DiscoveryConfig{}, clusterToken: "fakeToken", } cls := clusterInfo{ clusterToken: "fakeToken", } d.waitPeers(&cls, 3, 0) if cls.Len() != len(expectedMemberInfo) { t.Errorf("unexpected member number returned by watch, expected: %d, got: %d", len(expectedMemberInfo), cls.Len()) } for i, m := range cls.members { if m != expectedMemberInfo[i] { t.Errorf("unexpected member[%d] returned by watch, expected: %v, got: %v", i, expectedMemberInfo[i], m) } } } func TestGetInitClusterStr(t *testing.T) { cases := []struct { name string members []memberInfo clusterSize int expectedResult string expectedError error }{ { name: "1 member", members: []memberInfo{ { peerURLsMap: "infra2=http://192.168.0.102:2380", }, }, clusterSize: 1, expectedResult: "infra2=http://192.168.0.102:2380", expectedError: nil, }, { name: "2 members", members: []memberInfo{ { peerURLsMap: "infra2=http://192.168.0.102:2380", }, { peerURLsMap: "infra3=http://192.168.0.103:2380", }, }, clusterSize: 2, expectedResult: "infra2=http://192.168.0.102:2380,infra3=http://192.168.0.103:2380", expectedError: nil, }, { name: "3 members", members: []memberInfo{ { peerURLsMap: "infra2=http://192.168.0.102:2380", }, { peerURLsMap: "infra3=http://192.168.0.103:2380", }, { peerURLsMap: "infra1=http://192.168.0.100:2380", }, }, clusterSize: 3, expectedResult: "infra2=http://192.168.0.102:2380,infra3=http://192.168.0.103:2380,infra1=http://192.168.0.100:2380", expectedError: nil, }, { name: "should ignore redundant member", members: []memberInfo{ { peerURLsMap: "infra2=http://192.168.0.102:2380", }, { peerURLsMap: "infra3=http://192.168.0.103:2380", }, { peerURLsMap: "infra1=http://192.168.0.100:2380", }, { peerURLsMap: "infra4=http://192.168.0.104:2380", }, }, clusterSize: 3, expectedResult: "infra2=http://192.168.0.102:2380,infra3=http://192.168.0.103:2380,infra1=http://192.168.0.100:2380", expectedError: nil, }, { name: "invalid_peer_url", members: []memberInfo{ { peerURLsMap: "infra2=http://192.168.0.102:2380", }, { peerURLsMap: "infra3=http://192.168.0.103", // not host:port }, }, clusterSize: 2, expectedResult: "infra2=http://192.168.0.102:2380,infra3=http://192.168.0.103:2380", expectedError: ErrInvalidURL, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { clsInfo := &clusterInfo{ members: tc.members, } retStr, err := clsInfo.getInitClusterStr(tc.clusterSize) if !errors.Is(err, tc.expectedError) { t.Errorf("Unexpected error, expected: %v, got: %v", tc.expectedError, err) } if err == nil { if retStr != tc.expectedResult { t.Errorf("Unexpected result, expected: %s, got: %s", tc.expectedResult, retStr) } } }) } } // fakeBaseKV is the base struct implementing the interface `clientv3.KV`. type fakeBaseKV struct{} func (fkv *fakeBaseKV) Put(ctx context.Context, key string, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) { return nil, nil } func (fkv *fakeBaseKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { return nil, nil } func (fkv *fakeBaseKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { return nil, nil } func (fkv *fakeBaseKV) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) { return nil, nil } func (fkv *fakeBaseKV) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) { return clientv3.OpResponse{}, nil } func (fkv *fakeBaseKV) Txn(ctx context.Context) clientv3.Txn { return nil } // fakeBaseWatcher is the base struct implementing the interface `clientv3.Watcher`. type fakeBaseWatcher struct{} func (fw *fakeBaseWatcher) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan { return nil } func (fw *fakeBaseWatcher) RequestProgress(ctx context.Context) error { return nil } func (fw *fakeBaseWatcher) Close() error { return nil } ================================================ FILE: server/etcdserver/api/v3election/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package v3election provides a v3 election service from an etcdserver. package v3election ================================================ FILE: server/etcdserver/api/v3election/election.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3election import ( "context" "errors" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" epb "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb" ) // ErrMissingLeaderKey is returned when election API request // is missing the "leader" field. var ErrMissingLeaderKey = errors.New(`"leader" field must be provided`) type electionServer struct { c *clientv3.Client // we want compile errors if new methods are added epb.UnsafeElectionServer } func NewElectionServer(c *clientv3.Client) epb.ElectionServer { return &electionServer{c: c} } func (es *electionServer) Campaign(ctx context.Context, req *epb.CampaignRequest) (*epb.CampaignResponse, error) { s, err := es.session(ctx, req.Lease) if err != nil { return nil, err } e := concurrency.NewElection(s, string(req.Name)) if err = e.Campaign(ctx, string(req.Value)); err != nil { return nil, err } return &epb.CampaignResponse{ Header: e.Header(), Leader: &epb.LeaderKey{ Name: req.Name, Key: []byte(e.Key()), Rev: e.Rev(), Lease: int64(s.Lease()), }, }, nil } func (es *electionServer) Proclaim(ctx context.Context, req *epb.ProclaimRequest) (*epb.ProclaimResponse, error) { if req.Leader == nil { return nil, ErrMissingLeaderKey } s, err := es.session(ctx, req.Leader.Lease) if err != nil { return nil, err } e := concurrency.ResumeElection(s, string(req.Leader.Name), string(req.Leader.Key), req.Leader.Rev) if err := e.Proclaim(ctx, string(req.Value)); err != nil { return nil, err } return &epb.ProclaimResponse{Header: e.Header()}, nil } func (es *electionServer) Observe(req *epb.LeaderRequest, stream epb.Election_ObserveServer) error { s, err := es.session(stream.Context(), -1) if err != nil { return err } e := concurrency.NewElection(s, string(req.Name)) ch := e.Observe(stream.Context()) for stream.Context().Err() == nil { select { case <-stream.Context().Done(): case resp, ok := <-ch: if !ok { return nil } lresp := &epb.LeaderResponse{Header: resp.Header, Kv: resp.Kvs[0]} if err := stream.Send(lresp); err != nil { return err } } } return stream.Context().Err() } func (es *electionServer) Leader(ctx context.Context, req *epb.LeaderRequest) (*epb.LeaderResponse, error) { s, err := es.session(ctx, -1) if err != nil { return nil, err } l, lerr := concurrency.NewElection(s, string(req.Name)).Leader(ctx) if lerr != nil { return nil, lerr } return &epb.LeaderResponse{Header: l.Header, Kv: l.Kvs[0]}, nil } func (es *electionServer) Resign(ctx context.Context, req *epb.ResignRequest) (*epb.ResignResponse, error) { if req.Leader == nil { return nil, ErrMissingLeaderKey } s, err := es.session(ctx, req.Leader.Lease) if err != nil { return nil, err } e := concurrency.ResumeElection(s, string(req.Leader.Name), string(req.Leader.Key), req.Leader.Rev) if err := e.Resign(ctx); err != nil { return nil, err } return &epb.ResignResponse{Header: e.Header()}, nil } func (es *electionServer) session(ctx context.Context, lease int64) (*concurrency.Session, error) { s, err := concurrency.NewSession( es.c, concurrency.WithLease(clientv3.LeaseID(lease)), concurrency.WithContext(ctx), ) if err != nil { return nil, err } s.Orphan() return s, nil } ================================================ FILE: server/etcdserver/api/v3election/v3electionpb/gw/v3election.pb.gw.go ================================================ // Code generated by protoc-gen-grpc-gateway. DO NOT EDIT. // source: server/etcdserver/api/v3election/v3electionpb/v3election.proto /* Package v3electionpb is a reverse proxy. It translates gRPC into RESTful JSON APIs. */ package gw import ( protov1 "github.com/golang/protobuf/proto" "context" "errors" "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb" "io" "net/http" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/grpc-ecosystem/grpc-gateway/v2/utilities" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" ) // Suppress "imported and not used" errors var ( _ codes.Code _ io.Reader _ status.Status _ = errors.New _ = runtime.String _ = utilities.NewDoubleArray _ = metadata.Join ) func request_Election_Campaign_0(ctx context.Context, marshaler runtime.Marshaler, client v3electionpb.ElectionClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3electionpb.CampaignRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Campaign(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Election_Campaign_0(ctx context.Context, marshaler runtime.Marshaler, server v3electionpb.ElectionServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3electionpb.CampaignRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Campaign(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Election_Proclaim_0(ctx context.Context, marshaler runtime.Marshaler, client v3electionpb.ElectionClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3electionpb.ProclaimRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Proclaim(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Election_Proclaim_0(ctx context.Context, marshaler runtime.Marshaler, server v3electionpb.ElectionServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3electionpb.ProclaimRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Proclaim(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Election_Leader_0(ctx context.Context, marshaler runtime.Marshaler, client v3electionpb.ElectionClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3electionpb.LeaderRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Leader(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Election_Leader_0(ctx context.Context, marshaler runtime.Marshaler, server v3electionpb.ElectionServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3electionpb.LeaderRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Leader(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Election_Observe_0(ctx context.Context, marshaler runtime.Marshaler, client v3electionpb.ElectionClient, req *http.Request, pathParams map[string]string) (v3electionpb.Election_ObserveClient, runtime.ServerMetadata, error) { var ( protoReq v3electionpb.LeaderRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } stream, err := client.Observe(ctx, &protoReq) if err != nil { return nil, metadata, err } header, err := stream.Header() if err != nil { return nil, metadata, err } metadata.HeaderMD = header return stream, metadata, nil } func request_Election_Resign_0(ctx context.Context, marshaler runtime.Marshaler, client v3electionpb.ElectionClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3electionpb.ResignRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Resign(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Election_Resign_0(ctx context.Context, marshaler runtime.Marshaler, server v3electionpb.ElectionServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3electionpb.ResignRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Resign(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } // v3electionpb.RegisterElectionHandlerServer registers the http handlers for service Election to "mux". // UnaryRPC :call v3electionpb.ElectionServer directly. // StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906. // Note that using this registration option will cause many gRPC library features to stop working. Consider using RegisterElectionHandlerFromEndpoint instead. // GRPC interceptors will not work for this type of registration. To use interceptors, you must use the "runtime.WithMiddlewares" option in the "runtime.NewServeMux" call. func RegisterElectionHandlerServer(ctx context.Context, mux *runtime.ServeMux, server v3electionpb.ElectionServer) error { mux.Handle(http.MethodPost, pattern_Election_Campaign_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/v3electionpb.Election/Campaign", runtime.WithHTTPPathPattern("/v3/election/campaign")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Election_Campaign_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Election_Campaign_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Election_Proclaim_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/v3electionpb.Election/Proclaim", runtime.WithHTTPPathPattern("/v3/election/proclaim")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Election_Proclaim_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Election_Proclaim_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Election_Leader_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/v3electionpb.Election/Leader", runtime.WithHTTPPathPattern("/v3/election/leader")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Election_Leader_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Election_Leader_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Election_Observe_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { err := status.Error(codes.Unimplemented, "streaming calls are not yet supported in the in-process transport") _, outboundMarshaler := runtime.MarshalerForRequest(mux, req) runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return }) mux.Handle(http.MethodPost, pattern_Election_Resign_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/v3electionpb.Election/Resign", runtime.WithHTTPPathPattern("/v3/election/resign")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Election_Resign_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Election_Resign_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } // RegisterElectionHandlerFromEndpoint is same as RegisterElectionHandler but // automatically dials to "endpoint" and closes the connection when "ctx" gets done. func RegisterElectionHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { conn, err := grpc.NewClient(endpoint, opts...) if err != nil { return err } defer func() { if err != nil { if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } return } go func() { <-ctx.Done() if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } }() }() return RegisterElectionHandler(ctx, mux, conn) } // RegisterElectionHandler registers the http handlers for service Election to "mux". // The handlers forward requests to the grpc endpoint over "conn". func RegisterElectionHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { return RegisterElectionHandlerClient(ctx, mux, v3electionpb.NewElectionClient(conn)) } // v3electionpb.RegisterElectionHandlerClient registers the http handlers for service Election // to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "ElectionClient". // Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "ElectionClient" // doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in // "ElectionClient" to call the correct interceptors. This client ignores the HTTP middlewares. func RegisterElectionHandlerClient(ctx context.Context, mux *runtime.ServeMux, client v3electionpb.ElectionClient) error { mux.Handle(http.MethodPost, pattern_Election_Campaign_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/v3electionpb.Election/Campaign", runtime.WithHTTPPathPattern("/v3/election/campaign")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Election_Campaign_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Election_Campaign_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Election_Proclaim_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/v3electionpb.Election/Proclaim", runtime.WithHTTPPathPattern("/v3/election/proclaim")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Election_Proclaim_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Election_Proclaim_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Election_Leader_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/v3electionpb.Election/Leader", runtime.WithHTTPPathPattern("/v3/election/leader")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Election_Leader_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Election_Leader_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Election_Observe_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/v3electionpb.Election/Observe", runtime.WithHTTPPathPattern("/v3/election/observe")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Election_Observe_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Election_Observe_0(annotatedContext, mux, outboundMarshaler, w, req, func() (proto.Message, error) { m1, err := resp.Recv() return protov1.MessageV2(m1), err }, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Election_Resign_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/v3electionpb.Election/Resign", runtime.WithHTTPPathPattern("/v3/election/resign")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Election_Resign_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Election_Resign_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } var ( pattern_Election_Campaign_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "election", "campaign"}, "")) pattern_Election_Proclaim_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "election", "proclaim"}, "")) pattern_Election_Leader_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "election", "leader"}, "")) pattern_Election_Observe_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "election", "observe"}, "")) pattern_Election_Resign_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "election", "resign"}, "")) ) var ( forward_Election_Campaign_0 = runtime.ForwardResponseMessage forward_Election_Proclaim_0 = runtime.ForwardResponseMessage forward_Election_Leader_0 = runtime.ForwardResponseMessage forward_Election_Observe_0 = runtime.ForwardResponseStream forward_Election_Resign_0 = runtime.ForwardResponseMessage ) ================================================ FILE: server/etcdserver/api/v3election/v3electionpb/v3election.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: v3election.proto package v3electionpb import ( fmt "fmt" io "io" math "math" math_bits "math/bits" proto "github.com/golang/protobuf/proto" etcdserverpb "go.etcd.io/etcd/api/v3/etcdserverpb" mvccpb "go.etcd.io/etcd/api/v3/mvccpb" _ "google.golang.org/genproto/googleapis/api/annotations" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type CampaignRequest struct { // name is the election's identifier for the campaign. Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // lease is the ID of the lease attached to leadership of the election. If the // lease expires or is revoked before resigning leadership, then the // leadership is transferred to the next campaigner, if any. Lease int64 `protobuf:"varint,2,opt,name=lease,proto3" json:"lease,omitempty"` // value is the initial proclaimed value set when the campaigner wins the // election. Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *CampaignRequest) Reset() { *m = CampaignRequest{} } func (m *CampaignRequest) String() string { return proto.CompactTextString(m) } func (*CampaignRequest) ProtoMessage() {} func (*CampaignRequest) Descriptor() ([]byte, []int) { return fileDescriptor_c9b1f26cc432a035, []int{0} } func (m *CampaignRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *CampaignRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_CampaignRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *CampaignRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_CampaignRequest.Merge(m, src) } func (m *CampaignRequest) XXX_Size() int { return m.Size() } func (m *CampaignRequest) XXX_DiscardUnknown() { xxx_messageInfo_CampaignRequest.DiscardUnknown(m) } var xxx_messageInfo_CampaignRequest proto.InternalMessageInfo func (m *CampaignRequest) GetName() []byte { if m != nil { return m.Name } return nil } func (m *CampaignRequest) GetLease() int64 { if m != nil { return m.Lease } return 0 } func (m *CampaignRequest) GetValue() []byte { if m != nil { return m.Value } return nil } type CampaignResponse struct { Header *etcdserverpb.ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // leader describes the resources used for holding leadereship of the election. Leader *LeaderKey `protobuf:"bytes,2,opt,name=leader,proto3" json:"leader,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *CampaignResponse) Reset() { *m = CampaignResponse{} } func (m *CampaignResponse) String() string { return proto.CompactTextString(m) } func (*CampaignResponse) ProtoMessage() {} func (*CampaignResponse) Descriptor() ([]byte, []int) { return fileDescriptor_c9b1f26cc432a035, []int{1} } func (m *CampaignResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *CampaignResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_CampaignResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *CampaignResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_CampaignResponse.Merge(m, src) } func (m *CampaignResponse) XXX_Size() int { return m.Size() } func (m *CampaignResponse) XXX_DiscardUnknown() { xxx_messageInfo_CampaignResponse.DiscardUnknown(m) } var xxx_messageInfo_CampaignResponse proto.InternalMessageInfo func (m *CampaignResponse) GetHeader() *etcdserverpb.ResponseHeader { if m != nil { return m.Header } return nil } func (m *CampaignResponse) GetLeader() *LeaderKey { if m != nil { return m.Leader } return nil } type LeaderKey struct { // name is the election identifier that corresponds to the leadership key. Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // key is an opaque key representing the ownership of the election. If the key // is deleted, then leadership is lost. Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` // rev is the creation revision of the key. It can be used to test for ownership // of an election during transactions by testing the key's creation revision // matches rev. Rev int64 `protobuf:"varint,3,opt,name=rev,proto3" json:"rev,omitempty"` // lease is the lease ID of the election leader. Lease int64 `protobuf:"varint,4,opt,name=lease,proto3" json:"lease,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaderKey) Reset() { *m = LeaderKey{} } func (m *LeaderKey) String() string { return proto.CompactTextString(m) } func (*LeaderKey) ProtoMessage() {} func (*LeaderKey) Descriptor() ([]byte, []int) { return fileDescriptor_c9b1f26cc432a035, []int{2} } func (m *LeaderKey) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaderKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaderKey.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaderKey) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaderKey.Merge(m, src) } func (m *LeaderKey) XXX_Size() int { return m.Size() } func (m *LeaderKey) XXX_DiscardUnknown() { xxx_messageInfo_LeaderKey.DiscardUnknown(m) } var xxx_messageInfo_LeaderKey proto.InternalMessageInfo func (m *LeaderKey) GetName() []byte { if m != nil { return m.Name } return nil } func (m *LeaderKey) GetKey() []byte { if m != nil { return m.Key } return nil } func (m *LeaderKey) GetRev() int64 { if m != nil { return m.Rev } return 0 } func (m *LeaderKey) GetLease() int64 { if m != nil { return m.Lease } return 0 } type LeaderRequest struct { // name is the election identifier for the leadership information. Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaderRequest) Reset() { *m = LeaderRequest{} } func (m *LeaderRequest) String() string { return proto.CompactTextString(m) } func (*LeaderRequest) ProtoMessage() {} func (*LeaderRequest) Descriptor() ([]byte, []int) { return fileDescriptor_c9b1f26cc432a035, []int{3} } func (m *LeaderRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaderRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaderRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaderRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaderRequest.Merge(m, src) } func (m *LeaderRequest) XXX_Size() int { return m.Size() } func (m *LeaderRequest) XXX_DiscardUnknown() { xxx_messageInfo_LeaderRequest.DiscardUnknown(m) } var xxx_messageInfo_LeaderRequest proto.InternalMessageInfo func (m *LeaderRequest) GetName() []byte { if m != nil { return m.Name } return nil } type LeaderResponse struct { Header *etcdserverpb.ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // kv is the key-value pair representing the latest leader update. Kv *mvccpb.KeyValue `protobuf:"bytes,2,opt,name=kv,proto3" json:"kv,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaderResponse) Reset() { *m = LeaderResponse{} } func (m *LeaderResponse) String() string { return proto.CompactTextString(m) } func (*LeaderResponse) ProtoMessage() {} func (*LeaderResponse) Descriptor() ([]byte, []int) { return fileDescriptor_c9b1f26cc432a035, []int{4} } func (m *LeaderResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaderResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaderResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaderResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaderResponse.Merge(m, src) } func (m *LeaderResponse) XXX_Size() int { return m.Size() } func (m *LeaderResponse) XXX_DiscardUnknown() { xxx_messageInfo_LeaderResponse.DiscardUnknown(m) } var xxx_messageInfo_LeaderResponse proto.InternalMessageInfo func (m *LeaderResponse) GetHeader() *etcdserverpb.ResponseHeader { if m != nil { return m.Header } return nil } func (m *LeaderResponse) GetKv() *mvccpb.KeyValue { if m != nil { return m.Kv } return nil } type ResignRequest struct { // leader is the leadership to relinquish by resignation. Leader *LeaderKey `protobuf:"bytes,1,opt,name=leader,proto3" json:"leader,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *ResignRequest) Reset() { *m = ResignRequest{} } func (m *ResignRequest) String() string { return proto.CompactTextString(m) } func (*ResignRequest) ProtoMessage() {} func (*ResignRequest) Descriptor() ([]byte, []int) { return fileDescriptor_c9b1f26cc432a035, []int{5} } func (m *ResignRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *ResignRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_ResignRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *ResignRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_ResignRequest.Merge(m, src) } func (m *ResignRequest) XXX_Size() int { return m.Size() } func (m *ResignRequest) XXX_DiscardUnknown() { xxx_messageInfo_ResignRequest.DiscardUnknown(m) } var xxx_messageInfo_ResignRequest proto.InternalMessageInfo func (m *ResignRequest) GetLeader() *LeaderKey { if m != nil { return m.Leader } return nil } type ResignResponse struct { Header *etcdserverpb.ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *ResignResponse) Reset() { *m = ResignResponse{} } func (m *ResignResponse) String() string { return proto.CompactTextString(m) } func (*ResignResponse) ProtoMessage() {} func (*ResignResponse) Descriptor() ([]byte, []int) { return fileDescriptor_c9b1f26cc432a035, []int{6} } func (m *ResignResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *ResignResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_ResignResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *ResignResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_ResignResponse.Merge(m, src) } func (m *ResignResponse) XXX_Size() int { return m.Size() } func (m *ResignResponse) XXX_DiscardUnknown() { xxx_messageInfo_ResignResponse.DiscardUnknown(m) } var xxx_messageInfo_ResignResponse proto.InternalMessageInfo func (m *ResignResponse) GetHeader() *etcdserverpb.ResponseHeader { if m != nil { return m.Header } return nil } type ProclaimRequest struct { // leader is the leadership hold on the election. Leader *LeaderKey `protobuf:"bytes,1,opt,name=leader,proto3" json:"leader,omitempty"` // value is an update meant to overwrite the leader's current value. Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *ProclaimRequest) Reset() { *m = ProclaimRequest{} } func (m *ProclaimRequest) String() string { return proto.CompactTextString(m) } func (*ProclaimRequest) ProtoMessage() {} func (*ProclaimRequest) Descriptor() ([]byte, []int) { return fileDescriptor_c9b1f26cc432a035, []int{7} } func (m *ProclaimRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *ProclaimRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_ProclaimRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *ProclaimRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_ProclaimRequest.Merge(m, src) } func (m *ProclaimRequest) XXX_Size() int { return m.Size() } func (m *ProclaimRequest) XXX_DiscardUnknown() { xxx_messageInfo_ProclaimRequest.DiscardUnknown(m) } var xxx_messageInfo_ProclaimRequest proto.InternalMessageInfo func (m *ProclaimRequest) GetLeader() *LeaderKey { if m != nil { return m.Leader } return nil } func (m *ProclaimRequest) GetValue() []byte { if m != nil { return m.Value } return nil } type ProclaimResponse struct { Header *etcdserverpb.ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *ProclaimResponse) Reset() { *m = ProclaimResponse{} } func (m *ProclaimResponse) String() string { return proto.CompactTextString(m) } func (*ProclaimResponse) ProtoMessage() {} func (*ProclaimResponse) Descriptor() ([]byte, []int) { return fileDescriptor_c9b1f26cc432a035, []int{8} } func (m *ProclaimResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *ProclaimResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_ProclaimResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *ProclaimResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_ProclaimResponse.Merge(m, src) } func (m *ProclaimResponse) XXX_Size() int { return m.Size() } func (m *ProclaimResponse) XXX_DiscardUnknown() { xxx_messageInfo_ProclaimResponse.DiscardUnknown(m) } var xxx_messageInfo_ProclaimResponse proto.InternalMessageInfo func (m *ProclaimResponse) GetHeader() *etcdserverpb.ResponseHeader { if m != nil { return m.Header } return nil } func init() { proto.RegisterType((*CampaignRequest)(nil), "v3electionpb.CampaignRequest") proto.RegisterType((*CampaignResponse)(nil), "v3electionpb.CampaignResponse") proto.RegisterType((*LeaderKey)(nil), "v3electionpb.LeaderKey") proto.RegisterType((*LeaderRequest)(nil), "v3electionpb.LeaderRequest") proto.RegisterType((*LeaderResponse)(nil), "v3electionpb.LeaderResponse") proto.RegisterType((*ResignRequest)(nil), "v3electionpb.ResignRequest") proto.RegisterType((*ResignResponse)(nil), "v3electionpb.ResignResponse") proto.RegisterType((*ProclaimRequest)(nil), "v3electionpb.ProclaimRequest") proto.RegisterType((*ProclaimResponse)(nil), "v3electionpb.ProclaimResponse") } func init() { proto.RegisterFile("v3election.proto", fileDescriptor_c9b1f26cc432a035) } var fileDescriptor_c9b1f26cc432a035 = []byte{ // 548 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0xcd, 0x6e, 0xd3, 0x4c, 0x14, 0xfd, 0xc6, 0xc9, 0x17, 0xca, 0x25, 0x6d, 0x23, 0x53, 0x44, 0x08, 0xc1, 0x8d, 0x86, 0x4d, 0x95, 0x85, 0x07, 0x35, 0xac, 0xb2, 0xaa, 0x40, 0xa0, 0x4a, 0x45, 0x02, 0x66, 0x81, 0x80, 0xdd, 0xc4, 0xbd, 0x4a, 0xa3, 0x38, 0x1e, 0x63, 0xbb, 0x96, 0xb2, 0xe5, 0x15, 0x58, 0xc0, 0x23, 0xb1, 0x44, 0xe2, 0x05, 0x50, 0xe0, 0x41, 0xd0, 0xcc, 0xd8, 0xf1, 0x8f, 0x12, 0x84, 0xc8, 0x6e, 0x3c, 0xf7, 0xcc, 0x3d, 0xf7, 0x9c, 0x39, 0x1e, 0xe8, 0xa4, 0x23, 0xf4, 0xd1, 0x4b, 0x66, 0x32, 0x70, 0xc3, 0x48, 0x26, 0xd2, 0x6e, 0x17, 0x3b, 0xe1, 0xa4, 0x77, 0x8c, 0x89, 0x77, 0xc9, 0x44, 0x38, 0x63, 0x6a, 0x11, 0x63, 0x94, 0x62, 0x14, 0x4e, 0x58, 0x14, 0x7a, 0x06, 0xde, 0xeb, 0xae, 0x01, 0x8b, 0xd4, 0xf3, 0xc2, 0x09, 0x9b, 0xa7, 0x59, 0xa5, 0x3f, 0x95, 0x72, 0xea, 0xa3, 0xae, 0x89, 0x20, 0x90, 0x89, 0x50, 0x3d, 0x63, 0x53, 0xa5, 0xaf, 0xe1, 0xf0, 0xa9, 0x58, 0x84, 0x62, 0x36, 0x0d, 0x38, 0x7e, 0xb8, 0xc6, 0x38, 0xb1, 0x6d, 0x68, 0x06, 0x62, 0x81, 0x5d, 0x32, 0x20, 0x27, 0x6d, 0xae, 0xd7, 0xf6, 0x11, 0xfc, 0xef, 0xa3, 0x88, 0xb1, 0x6b, 0x0d, 0xc8, 0x49, 0x83, 0x9b, 0x0f, 0xb5, 0x9b, 0x0a, 0xff, 0x1a, 0xbb, 0x0d, 0x0d, 0x35, 0x1f, 0x74, 0x09, 0x9d, 0xa2, 0x65, 0x1c, 0xca, 0x20, 0x46, 0xfb, 0x31, 0xb4, 0xae, 0x50, 0x5c, 0x62, 0xa4, 0xbb, 0xde, 0x3a, 0xed, 0xbb, 0x65, 0x1d, 0x6e, 0x8e, 0x3b, 0xd7, 0x18, 0x9e, 0x61, 0x6d, 0x06, 0x2d, 0xdf, 0x9c, 0xb2, 0xf4, 0xa9, 0xbb, 0x6e, 0xd9, 0x14, 0xf7, 0x85, 0xae, 0x5d, 0xe0, 0x92, 0x67, 0x30, 0xfa, 0x0e, 0x6e, 0xae, 0x37, 0x37, 0xea, 0xe8, 0x40, 0x63, 0x8e, 0x4b, 0xdd, 0xae, 0xcd, 0xd5, 0x52, 0xed, 0x44, 0x98, 0x6a, 0x05, 0x0d, 0xae, 0x96, 0x85, 0xd6, 0x66, 0x49, 0x2b, 0x7d, 0x08, 0xfb, 0xa6, 0xf5, 0x1f, 0x6c, 0xa2, 0x57, 0x70, 0x90, 0x83, 0x76, 0x12, 0x3e, 0x00, 0x6b, 0x9e, 0x66, 0xa2, 0x3b, 0xae, 0xb9, 0x51, 0xf7, 0x02, 0x97, 0x6f, 0x94, 0xc1, 0xdc, 0x9a, 0xa7, 0xf4, 0x0c, 0xf6, 0x39, 0xc6, 0xa5, 0x5b, 0x2b, 0xbc, 0x22, 0x7f, 0xe7, 0xd5, 0x73, 0x38, 0xc8, 0x3b, 0xec, 0x32, 0x2b, 0x7d, 0x0b, 0x87, 0xaf, 0x22, 0xe9, 0xf9, 0x62, 0xb6, 0xf8, 0xd7, 0x59, 0x8a, 0x20, 0x59, 0xe5, 0x20, 0x9d, 0x43, 0xa7, 0xe8, 0xbc, 0xcb, 0x8c, 0xa7, 0x9f, 0x9b, 0xb0, 0xf7, 0x2c, 0x1b, 0xc0, 0x9e, 0xc3, 0x5e, 0x9e, 0x4f, 0xfb, 0x41, 0x75, 0xb2, 0xda, 0xaf, 0xd0, 0x73, 0xb6, 0x95, 0x0d, 0x0b, 0x1d, 0x7c, 0xfc, 0xfe, 0xeb, 0x93, 0xd5, 0xa3, 0x77, 0x58, 0x3a, 0x62, 0x39, 0x90, 0x79, 0x19, 0x6c, 0x4c, 0x86, 0x8a, 0x2c, 0xd7, 0x50, 0x27, 0xab, 0xb9, 0x56, 0x27, 0xab, 0x4b, 0xdf, 0x42, 0x16, 0x66, 0x30, 0x45, 0xe6, 0x41, 0xcb, 0x78, 0x6b, 0xdf, 0xdf, 0xe4, 0x78, 0x4e, 0xd4, 0xdf, 0x5c, 0xcc, 0x68, 0x1c, 0x4d, 0xd3, 0xa5, 0xb7, 0x2b, 0x34, 0xe6, 0xa2, 0x14, 0xc9, 0x14, 0x6e, 0xbc, 0x9c, 0x68, 0xc3, 0x77, 0x61, 0x39, 0xd6, 0x2c, 0xf7, 0xe8, 0x51, 0x85, 0x45, 0x9a, 0xc6, 0x63, 0x32, 0x7c, 0x44, 0x94, 0x1a, 0x13, 0xd0, 0x3a, 0x4f, 0x25, 0xf8, 0x75, 0x9e, 0x6a, 0xa6, 0xb7, 0xa8, 0x89, 0x34, 0x68, 0x4c, 0x86, 0x4f, 0xf8, 0xd7, 0x95, 0x43, 0xbe, 0xad, 0x1c, 0xf2, 0x63, 0xe5, 0x90, 0x2f, 0x3f, 0x9d, 0xff, 0xde, 0x9f, 0x4d, 0xa5, 0xce, 0x94, 0x3b, 0x93, 0xfa, 0xb1, 0x65, 0x26, 0x5c, 0xfa, 0xfc, 0x3a, 0x6a, 0xfa, 0x35, 0x2d, 0x78, 0x59, 0x79, 0x84, 0x49, 0x4b, 0x3f, 0xad, 0xa3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x8d, 0x13, 0xc0, 0xca, 0xd5, 0x05, 0x00, 0x00, } func (m *CampaignRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *CampaignRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *CampaignRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Value) > 0 { i -= len(m.Value) copy(dAtA[i:], m.Value) i = encodeVarintV3Election(dAtA, i, uint64(len(m.Value))) i-- dAtA[i] = 0x1a } if m.Lease != 0 { i = encodeVarintV3Election(dAtA, i, uint64(m.Lease)) i-- dAtA[i] = 0x10 } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintV3Election(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *CampaignResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *CampaignResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *CampaignResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Leader != nil { { size, err := m.Leader.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintV3Election(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintV3Election(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *LeaderKey) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaderKey) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaderKey) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Lease != 0 { i = encodeVarintV3Election(dAtA, i, uint64(m.Lease)) i-- dAtA[i] = 0x20 } if m.Rev != 0 { i = encodeVarintV3Election(dAtA, i, uint64(m.Rev)) i-- dAtA[i] = 0x18 } if len(m.Key) > 0 { i -= len(m.Key) copy(dAtA[i:], m.Key) i = encodeVarintV3Election(dAtA, i, uint64(len(m.Key))) i-- dAtA[i] = 0x12 } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintV3Election(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *LeaderRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaderRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaderRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintV3Election(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *LeaderResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaderResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaderResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Kv != nil { { size, err := m.Kv.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintV3Election(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x12 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintV3Election(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *ResignRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *ResignRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ResignRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Leader != nil { { size, err := m.Leader.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintV3Election(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *ResignResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *ResignResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ResignResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintV3Election(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *ProclaimRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *ProclaimRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ProclaimRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Value) > 0 { i -= len(m.Value) copy(dAtA[i:], m.Value) i = encodeVarintV3Election(dAtA, i, uint64(len(m.Value))) i-- dAtA[i] = 0x12 } if m.Leader != nil { { size, err := m.Leader.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintV3Election(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *ProclaimResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *ProclaimResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *ProclaimResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintV3Election(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func encodeVarintV3Election(dAtA []byte, offset int, v uint64) int { offset -= sovV3Election(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) return base } func (m *CampaignRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovV3Election(uint64(l)) } if m.Lease != 0 { n += 1 + sovV3Election(uint64(m.Lease)) } l = len(m.Value) if l > 0 { n += 1 + l + sovV3Election(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *CampaignResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovV3Election(uint64(l)) } if m.Leader != nil { l = m.Leader.Size() n += 1 + l + sovV3Election(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaderKey) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovV3Election(uint64(l)) } l = len(m.Key) if l > 0 { n += 1 + l + sovV3Election(uint64(l)) } if m.Rev != 0 { n += 1 + sovV3Election(uint64(m.Rev)) } if m.Lease != 0 { n += 1 + sovV3Election(uint64(m.Lease)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaderRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovV3Election(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaderResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovV3Election(uint64(l)) } if m.Kv != nil { l = m.Kv.Size() n += 1 + l + sovV3Election(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *ResignRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Leader != nil { l = m.Leader.Size() n += 1 + l + sovV3Election(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *ResignResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovV3Election(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *ProclaimRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Leader != nil { l = m.Leader.Size() n += 1 + l + sovV3Election(uint64(l)) } l = len(m.Value) if l > 0 { n += 1 + l + sovV3Election(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *ProclaimResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovV3Election(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func sovV3Election(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } func sozV3Election(x uint64) (n int) { return sovV3Election(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (m *CampaignRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: CampaignRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: CampaignRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) if m.Name == nil { m.Name = []byte{} } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Lease", wireType) } m.Lease = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Lease |= int64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } m.Value = append(m.Value[:0], dAtA[iNdEx:postIndex]...) if m.Value == nil { m.Value = []byte{} } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipV3Election(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Election } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *CampaignResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: CampaignResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: CampaignResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &etcdserverpb.ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Leader", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } if m.Leader == nil { m.Leader = &LeaderKey{} } if err := m.Leader.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipV3Election(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Election } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaderKey) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaderKey: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaderKey: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) if m.Name == nil { m.Name = []byte{} } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) if m.Key == nil { m.Key = []byte{} } iNdEx = postIndex case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Rev", wireType) } m.Rev = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Rev |= int64(b&0x7F) << shift if b < 0x80 { break } } case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Lease", wireType) } m.Lease = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Lease |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipV3Election(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Election } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaderRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaderRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaderRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) if m.Name == nil { m.Name = []byte{} } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipV3Election(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Election } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaderResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaderResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaderResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &etcdserverpb.ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Kv", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } if m.Kv == nil { m.Kv = &mvccpb.KeyValue{} } if err := m.Kv.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipV3Election(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Election } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *ResignRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: ResignRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: ResignRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Leader", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } if m.Leader == nil { m.Leader = &LeaderKey{} } if err := m.Leader.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipV3Election(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Election } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *ResignResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: ResignResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: ResignResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &etcdserverpb.ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipV3Election(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Election } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *ProclaimRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: ProclaimRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: ProclaimRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Leader", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } if m.Leader == nil { m.Leader = &LeaderKey{} } if err := m.Leader.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } m.Value = append(m.Value[:0], dAtA[iNdEx:postIndex]...) if m.Value == nil { m.Value = []byte{} } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipV3Election(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Election } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *ProclaimResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: ProclaimResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: ProclaimResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Election } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthV3Election } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthV3Election } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &etcdserverpb.ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipV3Election(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Election } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func skipV3Election(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowV3Election } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } wireType := int(wire & 0x7) switch wireType { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowV3Election } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } iNdEx++ if dAtA[iNdEx-1] < 0x80 { break } } case 1: iNdEx += 8 case 2: var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowV3Election } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { break } } if length < 0 { return 0, ErrInvalidLengthV3Election } iNdEx += length case 3: depth++ case 4: if depth == 0 { return 0, ErrUnexpectedEndOfGroupV3Election } depth-- case 5: iNdEx += 4 default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { return 0, ErrInvalidLengthV3Election } if depth == 0 { return iNdEx, nil } } return 0, io.ErrUnexpectedEOF } var ( ErrInvalidLengthV3Election = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowV3Election = fmt.Errorf("proto: integer overflow") ErrUnexpectedEndOfGroupV3Election = fmt.Errorf("proto: unexpected end of group") ) ================================================ FILE: server/etcdserver/api/v3election/v3electionpb/v3election.proto ================================================ syntax = "proto3"; package v3electionpb; import "etcd/api/etcdserverpb/rpc.proto"; import "etcd/api/mvccpb/kv.proto"; // for grpc-gateway import "google/api/annotations.proto"; option go_package = "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb"; // The election service exposes client-side election facilities as a gRPC interface. service Election { // Campaign waits to acquire leadership in an election, returning a LeaderKey // representing the leadership if successful. The LeaderKey can then be used // to issue new values on the election, transactionally guard API requests on // leadership still being held, and resign from the election. rpc Campaign(CampaignRequest) returns (CampaignResponse) { option (google.api.http) = { post: "/v3/election/campaign" body: "*" }; } // Proclaim updates the leader's posted value with a new value. rpc Proclaim(ProclaimRequest) returns (ProclaimResponse) { option (google.api.http) = { post: "/v3/election/proclaim" body: "*" }; } // Leader returns the current election proclamation, if any. rpc Leader(LeaderRequest) returns (LeaderResponse) { option (google.api.http) = { post: "/v3/election/leader" body: "*" }; } // Observe streams election proclamations in-order as made by the election's // elected leaders. rpc Observe(LeaderRequest) returns (stream LeaderResponse) { option (google.api.http) = { post: "/v3/election/observe" body: "*" }; } // Resign releases election leadership so other campaigners may acquire // leadership on the election. rpc Resign(ResignRequest) returns (ResignResponse) { option (google.api.http) = { post: "/v3/election/resign" body: "*" }; } } message CampaignRequest { // name is the election's identifier for the campaign. bytes name = 1; // lease is the ID of the lease attached to leadership of the election. If the // lease expires or is revoked before resigning leadership, then the // leadership is transferred to the next campaigner, if any. int64 lease = 2; // value is the initial proclaimed value set when the campaigner wins the // election. bytes value = 3; } message CampaignResponse { etcdserverpb.ResponseHeader header = 1; // leader describes the resources used for holding leadereship of the election. LeaderKey leader = 2; } message LeaderKey { // name is the election identifier that corresponds to the leadership key. bytes name = 1; // key is an opaque key representing the ownership of the election. If the key // is deleted, then leadership is lost. bytes key = 2; // rev is the creation revision of the key. It can be used to test for ownership // of an election during transactions by testing the key's creation revision // matches rev. int64 rev = 3; // lease is the lease ID of the election leader. int64 lease = 4; } message LeaderRequest { // name is the election identifier for the leadership information. bytes name = 1; } message LeaderResponse { etcdserverpb.ResponseHeader header = 1; // kv is the key-value pair representing the latest leader update. mvccpb.KeyValue kv = 2; } message ResignRequest { // leader is the leadership to relinquish by resignation. LeaderKey leader = 1; } message ResignResponse { etcdserverpb.ResponseHeader header = 1; } message ProclaimRequest { // leader is the leadership hold on the election. LeaderKey leader = 1; // value is an update meant to overwrite the leader's current value. bytes value = 2; } message ProclaimResponse { etcdserverpb.ResponseHeader header = 1; } ================================================ FILE: server/etcdserver/api/v3election/v3electionpb/v3election_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.1 // - protoc v3.20.3 // source: v3election.proto package v3electionpb import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.64.0 or later. const _ = grpc.SupportPackageIsVersion9 const ( Election_Campaign_FullMethodName = "/v3electionpb.Election/Campaign" Election_Proclaim_FullMethodName = "/v3electionpb.Election/Proclaim" Election_Leader_FullMethodName = "/v3electionpb.Election/Leader" Election_Observe_FullMethodName = "/v3electionpb.Election/Observe" Election_Resign_FullMethodName = "/v3electionpb.Election/Resign" ) // ElectionClient is the client API for Election service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // // The election service exposes client-side election facilities as a gRPC interface. type ElectionClient interface { // Campaign waits to acquire leadership in an election, returning a LeaderKey // representing the leadership if successful. The LeaderKey can then be used // to issue new values on the election, transactionally guard API requests on // leadership still being held, and resign from the election. Campaign(ctx context.Context, in *CampaignRequest, opts ...grpc.CallOption) (*CampaignResponse, error) // Proclaim updates the leader's posted value with a new value. Proclaim(ctx context.Context, in *ProclaimRequest, opts ...grpc.CallOption) (*ProclaimResponse, error) // Leader returns the current election proclamation, if any. Leader(ctx context.Context, in *LeaderRequest, opts ...grpc.CallOption) (*LeaderResponse, error) // Observe streams election proclamations in-order as made by the election's // elected leaders. Observe(ctx context.Context, in *LeaderRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[LeaderResponse], error) // Resign releases election leadership so other campaigners may acquire // leadership on the election. Resign(ctx context.Context, in *ResignRequest, opts ...grpc.CallOption) (*ResignResponse, error) } type electionClient struct { cc grpc.ClientConnInterface } func NewElectionClient(cc grpc.ClientConnInterface) ElectionClient { return &electionClient{cc} } func (c *electionClient) Campaign(ctx context.Context, in *CampaignRequest, opts ...grpc.CallOption) (*CampaignResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CampaignResponse) err := c.cc.Invoke(ctx, Election_Campaign_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *electionClient) Proclaim(ctx context.Context, in *ProclaimRequest, opts ...grpc.CallOption) (*ProclaimResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ProclaimResponse) err := c.cc.Invoke(ctx, Election_Proclaim_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *electionClient) Leader(ctx context.Context, in *LeaderRequest, opts ...grpc.CallOption) (*LeaderResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(LeaderResponse) err := c.cc.Invoke(ctx, Election_Leader_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *electionClient) Observe(ctx context.Context, in *LeaderRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[LeaderResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Election_ServiceDesc.Streams[0], Election_Observe_FullMethodName, cOpts...) if err != nil { return nil, err } x := &grpc.GenericClientStream[LeaderRequest, LeaderResponse]{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } if err := x.ClientStream.CloseSend(); err != nil { return nil, err } return x, nil } // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type Election_ObserveClient = grpc.ServerStreamingClient[LeaderResponse] func (c *electionClient) Resign(ctx context.Context, in *ResignRequest, opts ...grpc.CallOption) (*ResignResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ResignResponse) err := c.cc.Invoke(ctx, Election_Resign_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } // ElectionServer is the server API for Election service. // All implementations must embed UnimplementedElectionServer // for forward compatibility. // // The election service exposes client-side election facilities as a gRPC interface. type ElectionServer interface { // Campaign waits to acquire leadership in an election, returning a LeaderKey // representing the leadership if successful. The LeaderKey can then be used // to issue new values on the election, transactionally guard API requests on // leadership still being held, and resign from the election. Campaign(context.Context, *CampaignRequest) (*CampaignResponse, error) // Proclaim updates the leader's posted value with a new value. Proclaim(context.Context, *ProclaimRequest) (*ProclaimResponse, error) // Leader returns the current election proclamation, if any. Leader(context.Context, *LeaderRequest) (*LeaderResponse, error) // Observe streams election proclamations in-order as made by the election's // elected leaders. Observe(*LeaderRequest, grpc.ServerStreamingServer[LeaderResponse]) error // Resign releases election leadership so other campaigners may acquire // leadership on the election. Resign(context.Context, *ResignRequest) (*ResignResponse, error) mustEmbedUnimplementedElectionServer() } // UnimplementedElectionServer must be embedded to have // forward compatible implementations. // // NOTE: this should be embedded by value instead of pointer to avoid a nil // pointer dereference when methods are called. type UnimplementedElectionServer struct{} func (UnimplementedElectionServer) Campaign(context.Context, *CampaignRequest) (*CampaignResponse, error) { return nil, status.Error(codes.Unimplemented, "method Campaign not implemented") } func (UnimplementedElectionServer) Proclaim(context.Context, *ProclaimRequest) (*ProclaimResponse, error) { return nil, status.Error(codes.Unimplemented, "method Proclaim not implemented") } func (UnimplementedElectionServer) Leader(context.Context, *LeaderRequest) (*LeaderResponse, error) { return nil, status.Error(codes.Unimplemented, "method Leader not implemented") } func (UnimplementedElectionServer) Observe(*LeaderRequest, grpc.ServerStreamingServer[LeaderResponse]) error { return status.Error(codes.Unimplemented, "method Observe not implemented") } func (UnimplementedElectionServer) Resign(context.Context, *ResignRequest) (*ResignResponse, error) { return nil, status.Error(codes.Unimplemented, "method Resign not implemented") } func (UnimplementedElectionServer) mustEmbedUnimplementedElectionServer() {} func (UnimplementedElectionServer) testEmbeddedByValue() {} // UnsafeElectionServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ElectionServer will // result in compilation errors. type UnsafeElectionServer interface { mustEmbedUnimplementedElectionServer() } func RegisterElectionServer(s grpc.ServiceRegistrar, srv ElectionServer) { // If the following call panics, it indicates UnimplementedElectionServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { t.testEmbeddedByValue() } s.RegisterService(&Election_ServiceDesc, srv) } func _Election_Campaign_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CampaignRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ElectionServer).Campaign(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Election_Campaign_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ElectionServer).Campaign(ctx, req.(*CampaignRequest)) } return interceptor(ctx, in, info, handler) } func _Election_Proclaim_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ProclaimRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ElectionServer).Proclaim(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Election_Proclaim_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ElectionServer).Proclaim(ctx, req.(*ProclaimRequest)) } return interceptor(ctx, in, info, handler) } func _Election_Leader_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(LeaderRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ElectionServer).Leader(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Election_Leader_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ElectionServer).Leader(ctx, req.(*LeaderRequest)) } return interceptor(ctx, in, info, handler) } func _Election_Observe_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(LeaderRequest) if err := stream.RecvMsg(m); err != nil { return err } return srv.(ElectionServer).Observe(m, &grpc.GenericServerStream[LeaderRequest, LeaderResponse]{ServerStream: stream}) } // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type Election_ObserveServer = grpc.ServerStreamingServer[LeaderResponse] func _Election_Resign_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ResignRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(ElectionServer).Resign(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Election_Resign_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ElectionServer).Resign(ctx, req.(*ResignRequest)) } return interceptor(ctx, in, info, handler) } // Election_ServiceDesc is the grpc.ServiceDesc for Election service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var Election_ServiceDesc = grpc.ServiceDesc{ ServiceName: "v3electionpb.Election", HandlerType: (*ElectionServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "Campaign", Handler: _Election_Campaign_Handler, }, { MethodName: "Proclaim", Handler: _Election_Proclaim_Handler, }, { MethodName: "Leader", Handler: _Election_Leader_Handler, }, { MethodName: "Resign", Handler: _Election_Resign_Handler, }, }, Streams: []grpc.StreamDesc{ { StreamName: "Observe", Handler: _Election_Observe_Handler, ServerStreams: true, }, }, Metadata: "v3election.proto", } ================================================ FILE: server/etcdserver/api/v3lock/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package v3lock provides a v3 locking service from an etcdserver. package v3lock ================================================ FILE: server/etcdserver/api/v3lock/lock.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3lock import ( "context" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb" ) type lockServer struct { c *clientv3.Client // we want compile errors if new methods are added v3lockpb.UnsafeLockServer } func NewLockServer(c *clientv3.Client) v3lockpb.LockServer { return &lockServer{c: c} } func (ls *lockServer) Lock(ctx context.Context, req *v3lockpb.LockRequest) (*v3lockpb.LockResponse, error) { s, err := concurrency.NewSession( ls.c, concurrency.WithLease(clientv3.LeaseID(req.Lease)), concurrency.WithContext(ctx), ) if err != nil { return nil, err } s.Orphan() m := concurrency.NewMutex(s, string(req.Name)) if err = m.Lock(ctx); err != nil { return nil, err } return &v3lockpb.LockResponse{Header: m.Header(), Key: []byte(m.Key())}, nil } func (ls *lockServer) Unlock(ctx context.Context, req *v3lockpb.UnlockRequest) (*v3lockpb.UnlockResponse, error) { resp, err := ls.c.Delete(ctx, string(req.Key)) if err != nil { return nil, err } return &v3lockpb.UnlockResponse{Header: resp.Header}, nil } ================================================ FILE: server/etcdserver/api/v3lock/v3lockpb/gw/v3lock.pb.gw.go ================================================ // Code generated by protoc-gen-grpc-gateway. DO NOT EDIT. // source: server/etcdserver/api/v3lock/v3lockpb/v3lock.proto /* Package v3lockpb is a reverse proxy. It translates gRPC into RESTful JSON APIs. */ package gw import ( protov1 "github.com/golang/protobuf/proto" "context" "errors" "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb" "io" "net/http" "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/grpc-ecosystem/grpc-gateway/v2/utilities" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" ) // Suppress "imported and not used" errors var ( _ codes.Code _ io.Reader _ status.Status _ = errors.New _ = runtime.String _ = utilities.NewDoubleArray _ = metadata.Join ) func request_Lock_Lock_0(ctx context.Context, marshaler runtime.Marshaler, client v3lockpb.LockClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3lockpb.LockRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Lock(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Lock_Lock_0(ctx context.Context, marshaler runtime.Marshaler, server v3lockpb.LockServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3lockpb.LockRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Lock(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } func request_Lock_Unlock_0(ctx context.Context, marshaler runtime.Marshaler, client v3lockpb.LockClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3lockpb.UnlockRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } if req.Body != nil { _, _ = io.Copy(io.Discard, req.Body) } msg, err := client.Unlock(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return protov1.MessageV2(msg), metadata, err } func local_request_Lock_Unlock_0(ctx context.Context, marshaler runtime.Marshaler, server v3lockpb.LockServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq v3lockpb.UnlockRequest metadata runtime.ServerMetadata ) if err := marshaler.NewDecoder(req.Body).Decode(protov1.MessageV2(&protoReq)); err != nil && !errors.Is(err, io.EOF) { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := server.Unlock(ctx, &protoReq) return protov1.MessageV2(msg), metadata, err } // v3lockpb.RegisterLockHandlerServer registers the http handlers for service Lock to "mux". // UnaryRPC :call v3lockpb.LockServer directly. // StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906. // Note that using this registration option will cause many gRPC library features to stop working. Consider using RegisterLockHandlerFromEndpoint instead. // GRPC interceptors will not work for this type of registration. To use interceptors, you must use the "runtime.WithMiddlewares" option in the "runtime.NewServeMux" call. func RegisterLockHandlerServer(ctx context.Context, mux *runtime.ServeMux, server v3lockpb.LockServer) error { mux.Handle(http.MethodPost, pattern_Lock_Lock_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/v3lockpb.Lock/Lock", runtime.WithHTTPPathPattern("/v3/lock/lock")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Lock_Lock_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lock_Lock_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lock_Unlock_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() var stream runtime.ServerTransportStream ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/v3lockpb.Lock/Unlock", runtime.WithHTTPPathPattern("/v3/lock/unlock")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := local_request_Lock_Unlock_0(annotatedContext, inboundMarshaler, server, req, pathParams) md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lock_Unlock_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } // RegisterLockHandlerFromEndpoint is same as RegisterLockHandler but // automatically dials to "endpoint" and closes the connection when "ctx" gets done. func RegisterLockHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { conn, err := grpc.NewClient(endpoint, opts...) if err != nil { return err } defer func() { if err != nil { if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } return } go func() { <-ctx.Done() if cerr := conn.Close(); cerr != nil { grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) } }() }() return RegisterLockHandler(ctx, mux, conn) } // RegisterLockHandler registers the http handlers for service Lock to "mux". // The handlers forward requests to the grpc endpoint over "conn". func RegisterLockHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { return RegisterLockHandlerClient(ctx, mux, v3lockpb.NewLockClient(conn)) } // v3lockpb.RegisterLockHandlerClient registers the http handlers for service Lock // to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "LockClient". // Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "LockClient" // doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in // "LockClient" to call the correct interceptors. This client ignores the HTTP middlewares. func RegisterLockHandlerClient(ctx context.Context, mux *runtime.ServeMux, client v3lockpb.LockClient) error { mux.Handle(http.MethodPost, pattern_Lock_Lock_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/v3lockpb.Lock/Lock", runtime.WithHTTPPathPattern("/v3/lock/lock")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Lock_Lock_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lock_Lock_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) mux.Handle(http.MethodPost, pattern_Lock_Unlock_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/v3lockpb.Lock/Unlock", runtime.WithHTTPPathPattern("/v3/lock/unlock")) if err != nil { runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } resp, md, err := request_Lock_Unlock_0(annotatedContext, inboundMarshaler, client, req, pathParams) annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) if err != nil { runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) return } forward_Lock_Unlock_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) return nil } var ( pattern_Lock_Lock_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 1}, []string{"v3", "lock"}, "")) pattern_Lock_Unlock_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"v3", "lock", "unlock"}, "")) ) var ( forward_Lock_Lock_0 = runtime.ForwardResponseMessage forward_Lock_Unlock_0 = runtime.ForwardResponseMessage ) ================================================ FILE: server/etcdserver/api/v3lock/v3lockpb/v3lock.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: v3lock.proto package v3lockpb import ( fmt "fmt" io "io" math "math" math_bits "math/bits" proto "github.com/golang/protobuf/proto" etcdserverpb "go.etcd.io/etcd/api/v3/etcdserverpb" _ "google.golang.org/genproto/googleapis/api/annotations" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type LockRequest struct { // name is the identifier for the distributed shared lock to be acquired. Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // lease is the ID of the lease that will be attached to ownership of the // lock. If the lease expires or is revoked and currently holds the lock, // the lock is automatically released. Calls to Lock with the same lease will // be treated as a single acquisition; locking twice with the same lease is a // no-op. Lease int64 `protobuf:"varint,2,opt,name=lease,proto3" json:"lease,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LockRequest) Reset() { *m = LockRequest{} } func (m *LockRequest) String() string { return proto.CompactTextString(m) } func (*LockRequest) ProtoMessage() {} func (*LockRequest) Descriptor() ([]byte, []int) { return fileDescriptor_52389b3e2f253201, []int{0} } func (m *LockRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LockRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LockRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LockRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_LockRequest.Merge(m, src) } func (m *LockRequest) XXX_Size() int { return m.Size() } func (m *LockRequest) XXX_DiscardUnknown() { xxx_messageInfo_LockRequest.DiscardUnknown(m) } var xxx_messageInfo_LockRequest proto.InternalMessageInfo func (m *LockRequest) GetName() []byte { if m != nil { return m.Name } return nil } func (m *LockRequest) GetLease() int64 { if m != nil { return m.Lease } return 0 } type LockResponse struct { Header *etcdserverpb.ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` // key is a key that will exist on etcd for the duration that the Lock caller // owns the lock. Users should not modify this key or the lock may exhibit // undefined behavior. Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LockResponse) Reset() { *m = LockResponse{} } func (m *LockResponse) String() string { return proto.CompactTextString(m) } func (*LockResponse) ProtoMessage() {} func (*LockResponse) Descriptor() ([]byte, []int) { return fileDescriptor_52389b3e2f253201, []int{1} } func (m *LockResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LockResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LockResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LockResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_LockResponse.Merge(m, src) } func (m *LockResponse) XXX_Size() int { return m.Size() } func (m *LockResponse) XXX_DiscardUnknown() { xxx_messageInfo_LockResponse.DiscardUnknown(m) } var xxx_messageInfo_LockResponse proto.InternalMessageInfo func (m *LockResponse) GetHeader() *etcdserverpb.ResponseHeader { if m != nil { return m.Header } return nil } func (m *LockResponse) GetKey() []byte { if m != nil { return m.Key } return nil } type UnlockRequest struct { // key is the lock ownership key granted by Lock. Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *UnlockRequest) Reset() { *m = UnlockRequest{} } func (m *UnlockRequest) String() string { return proto.CompactTextString(m) } func (*UnlockRequest) ProtoMessage() {} func (*UnlockRequest) Descriptor() ([]byte, []int) { return fileDescriptor_52389b3e2f253201, []int{2} } func (m *UnlockRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *UnlockRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_UnlockRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *UnlockRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_UnlockRequest.Merge(m, src) } func (m *UnlockRequest) XXX_Size() int { return m.Size() } func (m *UnlockRequest) XXX_DiscardUnknown() { xxx_messageInfo_UnlockRequest.DiscardUnknown(m) } var xxx_messageInfo_UnlockRequest proto.InternalMessageInfo func (m *UnlockRequest) GetKey() []byte { if m != nil { return m.Key } return nil } type UnlockResponse struct { Header *etcdserverpb.ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *UnlockResponse) Reset() { *m = UnlockResponse{} } func (m *UnlockResponse) String() string { return proto.CompactTextString(m) } func (*UnlockResponse) ProtoMessage() {} func (*UnlockResponse) Descriptor() ([]byte, []int) { return fileDescriptor_52389b3e2f253201, []int{3} } func (m *UnlockResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *UnlockResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_UnlockResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *UnlockResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_UnlockResponse.Merge(m, src) } func (m *UnlockResponse) XXX_Size() int { return m.Size() } func (m *UnlockResponse) XXX_DiscardUnknown() { xxx_messageInfo_UnlockResponse.DiscardUnknown(m) } var xxx_messageInfo_UnlockResponse proto.InternalMessageInfo func (m *UnlockResponse) GetHeader() *etcdserverpb.ResponseHeader { if m != nil { return m.Header } return nil } func init() { proto.RegisterType((*LockRequest)(nil), "v3lockpb.LockRequest") proto.RegisterType((*LockResponse)(nil), "v3lockpb.LockResponse") proto.RegisterType((*UnlockRequest)(nil), "v3lockpb.UnlockRequest") proto.RegisterType((*UnlockResponse)(nil), "v3lockpb.UnlockResponse") } func init() { proto.RegisterFile("v3lock.proto", fileDescriptor_52389b3e2f253201) } var fileDescriptor_52389b3e2f253201 = []byte{ // 346 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x33, 0xce, 0xc9, 0x4f, 0xce, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x80, 0xf0, 0x0a, 0x92, 0xa4, 0xe4, 0x53, 0x4b, 0x92, 0x53, 0xf4, 0x13, 0x0b, 0x32, 0xf5, 0x41, 0x8c, 0xe2, 0xd4, 0xa2, 0xb2, 0xd4, 0xa2, 0x82, 0x24, 0xfd, 0xa2, 0x82, 0x64, 0x88, 0x52, 0x29, 0x99, 0xf4, 0xfc, 0xfc, 0xf4, 0x9c, 0x54, 0xb0, 0x92, 0xc4, 0xbc, 0xbc, 0xfc, 0x92, 0xc4, 0x92, 0xcc, 0xfc, 0xbc, 0x62, 0x88, 0xac, 0x92, 0x39, 0x17, 0xb7, 0x4f, 0x7e, 0x72, 0x76, 0x50, 0x6a, 0x61, 0x69, 0x6a, 0x71, 0x89, 0x90, 0x10, 0x17, 0x4b, 0x5e, 0x62, 0x6e, 0xaa, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0x4f, 0x10, 0x98, 0x2d, 0x24, 0xc2, 0xc5, 0x9a, 0x93, 0x9a, 0x58, 0x9c, 0x2a, 0xc1, 0xa4, 0xc0, 0xa8, 0xc1, 0x1c, 0x04, 0xe1, 0x28, 0x85, 0x71, 0xf1, 0x40, 0x34, 0x16, 0x17, 0xe4, 0xe7, 0x15, 0xa7, 0x0a, 0x99, 0x70, 0xb1, 0x65, 0xa4, 0x26, 0xa6, 0xa4, 0x16, 0x81, 0xf5, 0x72, 0x1b, 0xc9, 0xe8, 0x21, 0xbb, 0x47, 0x0f, 0xa6, 0xce, 0x03, 0xac, 0x26, 0x08, 0xaa, 0x56, 0x48, 0x80, 0x8b, 0x39, 0x3b, 0xb5, 0x12, 0x6c, 0x32, 0x4f, 0x10, 0x88, 0xa9, 0xa4, 0xc8, 0xc5, 0x1b, 0x9a, 0x97, 0x83, 0xe4, 0x24, 0xa8, 0x12, 0x46, 0x84, 0x12, 0x37, 0x2e, 0x3e, 0x98, 0x12, 0x4a, 0x2c, 0x37, 0xda, 0xc0, 0xc8, 0xc5, 0x02, 0xf2, 0x83, 0x90, 0x3f, 0x94, 0x16, 0xd5, 0x83, 0x05, 0xab, 0x1e, 0x52, 0xa0, 0x48, 0x89, 0xa1, 0x0b, 0x43, 0x4c, 0x53, 0x92, 0x68, 0xba, 0xfc, 0x64, 0x32, 0x93, 0x90, 0x12, 0xaf, 0x7e, 0x99, 0xb1, 0x3e, 0x48, 0x01, 0x98, 0xb0, 0x62, 0xd4, 0x12, 0x0a, 0xe7, 0x62, 0x83, 0xb8, 0x50, 0x48, 0x1c, 0xa1, 0x17, 0xc5, 0x5b, 0x52, 0x12, 0x98, 0x12, 0x50, 0x63, 0xa5, 0xc0, 0xc6, 0x8a, 0x28, 0xf1, 0xc3, 0x8d, 0x2d, 0xcd, 0x83, 0x1a, 0xec, 0xe4, 0x75, 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, 0x1e, 0xc9, 0x31, 0xce, 0x78, 0x2c, 0xc7, 0x10, 0x65, 0x91, 0x9e, 0x0f, 0xf6, 0xac, 0x5e, 0x66, 0x3e, 0x38, 0x05, 0xe8, 0x43, 0x7c, 0x0d, 0xd2, 0x8b, 0x08, 0x03, 0x70, 0xe4, 0x43, 0xec, 0xd3, 0x87, 0x59, 0x9b, 0xc4, 0x06, 0x4e, 0x01, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2a, 0x20, 0x0d, 0x43, 0x5a, 0x02, 0x00, 0x00, } func (m *LockRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LockRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LockRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Lease != 0 { i = encodeVarintV3Lock(dAtA, i, uint64(m.Lease)) i-- dAtA[i] = 0x10 } if len(m.Name) > 0 { i -= len(m.Name) copy(dAtA[i:], m.Name) i = encodeVarintV3Lock(dAtA, i, uint64(len(m.Name))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *LockResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LockResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LockResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Key) > 0 { i -= len(m.Key) copy(dAtA[i:], m.Key) i = encodeVarintV3Lock(dAtA, i, uint64(len(m.Key))) i-- dAtA[i] = 0x12 } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintV3Lock(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *UnlockRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *UnlockRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *UnlockRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Key) > 0 { i -= len(m.Key) copy(dAtA[i:], m.Key) i = encodeVarintV3Lock(dAtA, i, uint64(len(m.Key))) i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *UnlockResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *UnlockResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *UnlockResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Header != nil { { size, err := m.Header.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintV3Lock(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func encodeVarintV3Lock(dAtA []byte, offset int, v uint64) int { offset -= sovV3Lock(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) return base } func (m *LockRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Name) if l > 0 { n += 1 + l + sovV3Lock(uint64(l)) } if m.Lease != 0 { n += 1 + sovV3Lock(uint64(m.Lease)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LockResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovV3Lock(uint64(l)) } l = len(m.Key) if l > 0 { n += 1 + l + sovV3Lock(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *UnlockRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l l = len(m.Key) if l > 0 { n += 1 + l + sovV3Lock(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *UnlockResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Header != nil { l = m.Header.Size() n += 1 + l + sovV3Lock(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func sovV3Lock(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } func sozV3Lock(x uint64) (n int) { return sovV3Lock(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (m *LockRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Lock } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LockRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LockRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Lock } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthV3Lock } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthV3Lock } if postIndex > l { return io.ErrUnexpectedEOF } m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...) if m.Name == nil { m.Name = []byte{} } iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Lease", wireType) } m.Lease = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Lock } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.Lease |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipV3Lock(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Lock } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LockResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Lock } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LockResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LockResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Lock } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthV3Lock } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthV3Lock } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &etcdserverpb.ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Lock } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthV3Lock } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthV3Lock } if postIndex > l { return io.ErrUnexpectedEOF } m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) if m.Key == nil { m.Key = []byte{} } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipV3Lock(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Lock } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *UnlockRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Lock } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: UnlockRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: UnlockRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Lock } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthV3Lock } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthV3Lock } if postIndex > l { return io.ErrUnexpectedEOF } m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...) if m.Key == nil { m.Key = []byte{} } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipV3Lock(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Lock } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *UnlockResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Lock } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: UnlockResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: UnlockResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Header", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowV3Lock } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthV3Lock } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthV3Lock } if postIndex > l { return io.ErrUnexpectedEOF } if m.Header == nil { m.Header = &etcdserverpb.ResponseHeader{} } if err := m.Header.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipV3Lock(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthV3Lock } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func skipV3Lock(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowV3Lock } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } wireType := int(wire & 0x7) switch wireType { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowV3Lock } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } iNdEx++ if dAtA[iNdEx-1] < 0x80 { break } } case 1: iNdEx += 8 case 2: var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowV3Lock } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { break } } if length < 0 { return 0, ErrInvalidLengthV3Lock } iNdEx += length case 3: depth++ case 4: if depth == 0 { return 0, ErrUnexpectedEndOfGroupV3Lock } depth-- case 5: iNdEx += 4 default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { return 0, ErrInvalidLengthV3Lock } if depth == 0 { return iNdEx, nil } } return 0, io.ErrUnexpectedEOF } var ( ErrInvalidLengthV3Lock = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowV3Lock = fmt.Errorf("proto: integer overflow") ErrUnexpectedEndOfGroupV3Lock = fmt.Errorf("proto: unexpected end of group") ) ================================================ FILE: server/etcdserver/api/v3lock/v3lockpb/v3lock.proto ================================================ syntax = "proto3"; package v3lockpb; import "etcd/api/etcdserverpb/rpc.proto"; // for grpc-gateway import "google/api/annotations.proto"; option go_package = "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb"; // The lock service exposes client-side locking facilities as a gRPC interface. service Lock { // Lock acquires a distributed shared lock on a given named lock. // On success, it will return a unique key that exists so long as the // lock is held by the caller. This key can be used in conjunction with // transactions to safely ensure updates to etcd only occur while holding // lock ownership. The lock is held until Unlock is called on the key or the // lease associate with the owner expires. rpc Lock(LockRequest) returns (LockResponse) { option (google.api.http) = { post: "/v3/lock/lock" body: "*" }; } // Unlock takes a key returned by Lock and releases the hold on lock. The // next Lock caller waiting for the lock will then be woken up and given // ownership of the lock. rpc Unlock(UnlockRequest) returns (UnlockResponse) { option (google.api.http) = { post: "/v3/lock/unlock" body: "*" }; } } message LockRequest { // name is the identifier for the distributed shared lock to be acquired. bytes name = 1; // lease is the ID of the lease that will be attached to ownership of the // lock. If the lease expires or is revoked and currently holds the lock, // the lock is automatically released. Calls to Lock with the same lease will // be treated as a single acquisition; locking twice with the same lease is a // no-op. int64 lease = 2; } message LockResponse { etcdserverpb.ResponseHeader header = 1; // key is a key that will exist on etcd for the duration that the Lock caller // owns the lock. Users should not modify this key or the lock may exhibit // undefined behavior. bytes key = 2; } message UnlockRequest { // key is the lock ownership key granted by Lock. bytes key = 1; } message UnlockResponse { etcdserverpb.ResponseHeader header = 1; } ================================================ FILE: server/etcdserver/api/v3lock/v3lockpb/v3lock_grpc.pb.go ================================================ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.1 // - protoc v3.20.3 // source: v3lock.proto package v3lockpb import ( context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. // Requires gRPC-Go v1.64.0 or later. const _ = grpc.SupportPackageIsVersion9 const ( Lock_Lock_FullMethodName = "/v3lockpb.Lock/Lock" Lock_Unlock_FullMethodName = "/v3lockpb.Lock/Unlock" ) // LockClient is the client API for Lock service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // // The lock service exposes client-side locking facilities as a gRPC interface. type LockClient interface { // Lock acquires a distributed shared lock on a given named lock. // On success, it will return a unique key that exists so long as the // lock is held by the caller. This key can be used in conjunction with // transactions to safely ensure updates to etcd only occur while holding // lock ownership. The lock is held until Unlock is called on the key or the // lease associate with the owner expires. Lock(ctx context.Context, in *LockRequest, opts ...grpc.CallOption) (*LockResponse, error) // Unlock takes a key returned by Lock and releases the hold on lock. The // next Lock caller waiting for the lock will then be woken up and given // ownership of the lock. Unlock(ctx context.Context, in *UnlockRequest, opts ...grpc.CallOption) (*UnlockResponse, error) } type lockClient struct { cc grpc.ClientConnInterface } func NewLockClient(cc grpc.ClientConnInterface) LockClient { return &lockClient{cc} } func (c *lockClient) Lock(ctx context.Context, in *LockRequest, opts ...grpc.CallOption) (*LockResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(LockResponse) err := c.cc.Invoke(ctx, Lock_Lock_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } func (c *lockClient) Unlock(ctx context.Context, in *UnlockRequest, opts ...grpc.CallOption) (*UnlockResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(UnlockResponse) err := c.cc.Invoke(ctx, Lock_Unlock_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } // LockServer is the server API for Lock service. // All implementations must embed UnimplementedLockServer // for forward compatibility. // // The lock service exposes client-side locking facilities as a gRPC interface. type LockServer interface { // Lock acquires a distributed shared lock on a given named lock. // On success, it will return a unique key that exists so long as the // lock is held by the caller. This key can be used in conjunction with // transactions to safely ensure updates to etcd only occur while holding // lock ownership. The lock is held until Unlock is called on the key or the // lease associate with the owner expires. Lock(context.Context, *LockRequest) (*LockResponse, error) // Unlock takes a key returned by Lock and releases the hold on lock. The // next Lock caller waiting for the lock will then be woken up and given // ownership of the lock. Unlock(context.Context, *UnlockRequest) (*UnlockResponse, error) mustEmbedUnimplementedLockServer() } // UnimplementedLockServer must be embedded to have // forward compatible implementations. // // NOTE: this should be embedded by value instead of pointer to avoid a nil // pointer dereference when methods are called. type UnimplementedLockServer struct{} func (UnimplementedLockServer) Lock(context.Context, *LockRequest) (*LockResponse, error) { return nil, status.Error(codes.Unimplemented, "method Lock not implemented") } func (UnimplementedLockServer) Unlock(context.Context, *UnlockRequest) (*UnlockResponse, error) { return nil, status.Error(codes.Unimplemented, "method Unlock not implemented") } func (UnimplementedLockServer) mustEmbedUnimplementedLockServer() {} func (UnimplementedLockServer) testEmbeddedByValue() {} // UnsafeLockServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to LockServer will // result in compilation errors. type UnsafeLockServer interface { mustEmbedUnimplementedLockServer() } func RegisterLockServer(s grpc.ServiceRegistrar, srv LockServer) { // If the following call panics, it indicates UnimplementedLockServer was // embedded by pointer and is nil. This will cause panics if an // unimplemented method is ever invoked, so we test this at initialization // time to prevent it from happening at runtime later due to I/O. if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { t.testEmbeddedByValue() } s.RegisterService(&Lock_ServiceDesc, srv) } func _Lock_Lock_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(LockRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(LockServer).Lock(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Lock_Lock_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(LockServer).Lock(ctx, req.(*LockRequest)) } return interceptor(ctx, in, info, handler) } func _Lock_Unlock_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(UnlockRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { return srv.(LockServer).Unlock(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, FullMethod: Lock_Unlock_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(LockServer).Unlock(ctx, req.(*UnlockRequest)) } return interceptor(ctx, in, info, handler) } // Lock_ServiceDesc is the grpc.ServiceDesc for Lock service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var Lock_ServiceDesc = grpc.ServiceDesc{ ServiceName: "v3lockpb.Lock", HandlerType: (*LockServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "Lock", Handler: _Lock_Lock_Handler, }, { MethodName: "Unlock", Handler: _Lock_Unlock_Handler, }, }, Streams: []grpc.StreamDesc{}, Metadata: "v3lock.proto", } ================================================ FILE: server/etcdserver/api/v3rpc/auth.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "context" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/etcdserver" ) type AuthServer struct { authenticator etcdserver.Authenticator // we want compile errors if new methods are added pb.UnsafeAuthServer } func NewAuthServer(s *etcdserver.EtcdServer) *AuthServer { return &AuthServer{authenticator: s} } func (as *AuthServer) AuthEnable(ctx context.Context, r *pb.AuthEnableRequest) (*pb.AuthEnableResponse, error) { resp, err := as.authenticator.AuthEnable(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) AuthDisable(ctx context.Context, r *pb.AuthDisableRequest) (*pb.AuthDisableResponse, error) { resp, err := as.authenticator.AuthDisable(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) AuthStatus(ctx context.Context, r *pb.AuthStatusRequest) (*pb.AuthStatusResponse, error) { resp, err := as.authenticator.AuthStatus(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) Authenticate(ctx context.Context, r *pb.AuthenticateRequest) (*pb.AuthenticateResponse, error) { resp, err := as.authenticator.Authenticate(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) RoleAdd(ctx context.Context, r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, error) { resp, err := as.authenticator.RoleAdd(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) RoleDelete(ctx context.Context, r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDeleteResponse, error) { resp, err := as.authenticator.RoleDelete(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) RoleGet(ctx context.Context, r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) { resp, err := as.authenticator.RoleGet(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) RoleList(ctx context.Context, r *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error) { resp, err := as.authenticator.RoleList(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) RoleRevokePermission(ctx context.Context, r *pb.AuthRoleRevokePermissionRequest) (*pb.AuthRoleRevokePermissionResponse, error) { resp, err := as.authenticator.RoleRevokePermission(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) RoleGrantPermission(ctx context.Context, r *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) { resp, err := as.authenticator.RoleGrantPermission(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) UserAdd(ctx context.Context, r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, error) { resp, err := as.authenticator.UserAdd(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) UserDelete(ctx context.Context, r *pb.AuthUserDeleteRequest) (*pb.AuthUserDeleteResponse, error) { resp, err := as.authenticator.UserDelete(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) UserGet(ctx context.Context, r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) { resp, err := as.authenticator.UserGet(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) UserList(ctx context.Context, r *pb.AuthUserListRequest) (*pb.AuthUserListResponse, error) { resp, err := as.authenticator.UserList(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) UserGrantRole(ctx context.Context, r *pb.AuthUserGrantRoleRequest) (*pb.AuthUserGrantRoleResponse, error) { resp, err := as.authenticator.UserGrantRole(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) UserRevokeRole(ctx context.Context, r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUserRevokeRoleResponse, error) { resp, err := as.authenticator.UserRevokeRole(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } func (as *AuthServer) UserChangePassword(ctx context.Context, r *pb.AuthUserChangePasswordRequest) (*pb.AuthUserChangePasswordResponse, error) { resp, err := as.authenticator.UserChangePassword(ctx, r) if err != nil { return nil, togRPCError(err) } return resp, nil } type AuthGetter interface { AuthInfoFromCtx(ctx context.Context) (*auth.AuthInfo, error) AuthStore() auth.AuthStore } type AuthAdmin struct { ag AuthGetter } // isPermitted verifies the user has admin privilege. // Only users with "root" role are permitted. func (aa *AuthAdmin) isPermitted(ctx context.Context) error { authInfo, err := aa.ag.AuthInfoFromCtx(ctx) if err != nil { return err } return aa.ag.AuthStore().IsAdminPermitted(authInfo) } ================================================ FILE: server/etcdserver/api/v3rpc/codec.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import "github.com/golang/protobuf/proto" //nolint:staticcheck // TODO: remove for a supported version type codec struct{} func (c *codec) Marshal(v any) ([]byte, error) { b, err := proto.Marshal(v.(proto.Message)) sentBytes.Add(float64(len(b))) return b, err } func (c *codec) Unmarshal(data []byte, v any) error { receivedBytes.Add(float64(len(data))) return proto.Unmarshal(data, v.(proto.Message)) } func (c *codec) String() string { return "proto" } ================================================ FILE: server/etcdserver/api/v3rpc/grpc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "crypto/tls" "math" "sync" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/v3/credentials" "go.etcd.io/etcd/server/v3/etcdserver" ) const ( maxSendBytes = math.MaxInt32 ) var ( metricsServerLock sync.Mutex metricsServerCached *grpc_prometheus.ServerMetrics ) func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnaryServerInterceptor, gopts ...grpc.ServerOption) *grpc.Server { var opts []grpc.ServerOption opts = append(opts, grpc.CustomCodec(&codec{})) //nolint:staticcheck // TODO: remove for a supported version if tls != nil { opts = append(opts, grpc.Creds(credentials.NewTransportCredential(tls))) } serverMetrics := getServerMetrics(s.Cfg.Metrics, s.Cfg.Logger) chainUnaryInterceptors := []grpc.UnaryServerInterceptor{ newLogUnaryInterceptor(s), serverMetrics.UnaryServerInterceptor(), newUnaryInterceptor(s), } if interceptor != nil { chainUnaryInterceptors = append(chainUnaryInterceptors, interceptor) } chainStreamInterceptors := []grpc.StreamServerInterceptor{ serverMetrics.StreamServerInterceptor(), newStreamInterceptor(s), } if s.Cfg.EnableDistributedTracing { opts = append(opts, grpc.StatsHandler(otelgrpc.NewServerHandler(s.Cfg.TracerOptions...))) } opts = append(opts, grpc.ChainUnaryInterceptor(chainUnaryInterceptors...)) opts = append(opts, grpc.ChainStreamInterceptor(chainStreamInterceptors...)) opts = append(opts, grpc.MaxRecvMsgSize(int(s.Cfg.MaxRequestBytesWithOverhead()))) opts = append(opts, grpc.MaxSendMsgSize(maxSendBytes)) opts = append(opts, grpc.MaxConcurrentStreams(s.Cfg.MaxConcurrentStreams)) grpcServer := grpc.NewServer(append(opts, gopts...)...) pb.RegisterKVServer(grpcServer, NewQuotaKVServer(s)) pb.RegisterWatchServer(grpcServer, NewWatchServer(s)) pb.RegisterLeaseServer(grpcServer, NewQuotaLeaseServer(s)) pb.RegisterClusterServer(grpcServer, NewClusterServer(s)) pb.RegisterAuthServer(grpcServer, NewAuthServer(s)) hsrv := health.NewServer() healthNotifier := newHealthNotifier(hsrv, s) healthpb.RegisterHealthServer(grpcServer, hsrv) pb.RegisterMaintenanceServer(grpcServer, NewMaintenanceServer(s, healthNotifier)) // set zero values for metrics registered for this grpc server serverMetrics.InitializeMetrics(grpcServer) return grpcServer } func getServerMetrics(metricType string, lg *zap.Logger) *grpc_prometheus.ServerMetrics { metricsServerLock.Lock() defer metricsServerLock.Unlock() if metricsServerCached == nil { var mopts []grpc_prometheus.ServerMetricsOption if metricType == "extensive" { mopts = append(mopts, grpc_prometheus.WithServerHandlingTimeHistogram()) } metricsServerCached = grpc_prometheus.NewServerMetrics(mopts...) err := prometheus.Register(metricsServerCached) if err != nil { lg.Warn("etcdserver: failed to register grpc metrics", zap.Error(err)) } } return metricsServerCached } ================================================ FILE: server/etcdserver/api/v3rpc/header.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/apply" ) type header struct { clusterID int64 memberID int64 sg apply.RaftStatusGetter rev func() int64 } func newHeader(s *etcdserver.EtcdServer) header { return header{ clusterID: int64(s.Cluster().ID()), memberID: int64(s.MemberID()), sg: s, rev: func() int64 { return s.KV().Rev() }, } } // fill populates pb.ResponseHeader using etcdserver information func (h *header) fill(rh *pb.ResponseHeader) { if rh == nil { panic("unexpected nil resp.Header") } rh.ClusterId = uint64(h.clusterID) rh.MemberId = uint64(h.memberID) rh.RaftTerm = h.sg.Term() if rh.Revision == 0 { rh.Revision = h.rev() } } ================================================ FILE: server/etcdserver/api/v3rpc/health.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "go.uber.org/zap" "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/features" ) const ( allGRPCServices = "" ) type notifier interface { defragStarted() defragFinished() } func newHealthNotifier(hs *health.Server, s *etcdserver.EtcdServer) notifier { if hs == nil { panic("unexpected nil gRPC health server") } hc := &healthNotifier{hs: hs, lg: s.Logger(), stopGRPCServiceOnDefrag: s.FeatureEnabled(features.StopGRPCServiceOnDefrag)} // set grpc health server as serving status blindly since // the grpc server will serve iff s.ReadyNotify() is closed. hc.startServe() return hc } type healthNotifier struct { hs *health.Server lg *zap.Logger stopGRPCServiceOnDefrag bool } func (hc *healthNotifier) defragStarted() { if !hc.stopGRPCServiceOnDefrag { return } hc.stopServe("defrag is active") } func (hc *healthNotifier) defragFinished() { hc.startServe() } func (hc *healthNotifier) startServe() { hc.lg.Info( "grpc service status changed", zap.String("service", allGRPCServices), zap.String("status", healthpb.HealthCheckResponse_SERVING.String()), ) hc.hs.SetServingStatus(allGRPCServices, healthpb.HealthCheckResponse_SERVING) } func (hc *healthNotifier) stopServe(reason string) { hc.lg.Warn( "grpc service status changed", zap.String("service", allGRPCServices), zap.String("status", healthpb.HealthCheckResponse_NOT_SERVING.String()), zap.String("reason", reason), ) hc.hs.SetServingStatus(allGRPCServices, healthpb.HealthCheckResponse_NOT_SERVING) } ================================================ FILE: server/etcdserver/api/v3rpc/interceptor.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "context" "sync" "time" "unicode/utf8" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api" "go.etcd.io/raft/v3" ) const ( maxNoLeaderCnt = 3 snapshotMethod = "/etcdserverpb.Maintenance/Snapshot" ) type streamsMap struct { mu sync.Mutex streams map[grpc.ServerStream]struct{} } func newUnaryInterceptor(s *etcdserver.EtcdServer) grpc.UnaryServerInterceptor { return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { if !api.IsCapabilityEnabled(api.V3rpcCapability) { return nil, rpctypes.ErrGRPCNotCapable } if s.IsMemberExist(s.MemberID()) && s.IsLearner() && !isRPCSupportedForLearner(req) { return nil, rpctypes.ErrGRPCNotSupportedForLearner } md, ok := metadata.FromIncomingContext(ctx) if ok { ver, vs := "unknown", md.Get(rpctypes.MetadataClientAPIVersionKey) if len(vs) > 0 { ver = vs[0] } if !utf8.ValidString(ver) { return nil, rpctypes.ErrGRPCInvalidClientAPIVersion } clientRequests.WithLabelValues("unary", ver).Inc() if ks := md[rpctypes.MetadataRequireLeaderKey]; len(ks) > 0 && ks[0] == rpctypes.MetadataHasLeader { if s.Leader() == types.ID(raft.None) { return nil, rpctypes.ErrGRPCNoLeader } } } return handler(ctx, req) } } func newLogUnaryInterceptor(s *etcdserver.EtcdServer) grpc.UnaryServerInterceptor { return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { startTime := time.Now() resp, err := handler(ctx, req) lg := s.Logger() if lg != nil { // acquire stats if debug level is enabled or RequestInfo is expensive defer logUnaryRequestStats(ctx, lg, s.Cfg.WarningUnaryRequestDuration, info, startTime, req, resp) } return resp, err } } func logUnaryRequestStats(ctx context.Context, lg *zap.Logger, warnLatency time.Duration, info *grpc.UnaryServerInfo, startTime time.Time, req any, resp any) { duration := time.Since(startTime) var enabledDebugLevel, expensiveRequest bool if lg.Core().Enabled(zap.DebugLevel) { enabledDebugLevel = true } if duration > warnLatency { expensiveRequest = true } if !enabledDebugLevel && !expensiveRequest { return } remote := "No remote client info." peerInfo, ok := peer.FromContext(ctx) if ok { remote = peerInfo.Addr.String() } responseType := info.FullMethod var reqCount, respCount int64 var reqSize, respSize int var reqContent string switch _resp := resp.(type) { case *pb.RangeResponse: _req, ok := req.(*pb.RangeRequest) if ok { reqCount = 0 reqSize = _req.Size() reqContent = _req.String() } if _resp != nil { respCount = _resp.GetCount() respSize = _resp.Size() } case *pb.PutResponse: _req, ok := req.(*pb.PutRequest) if ok { reqCount = 1 reqSize = _req.Size() reqContent = pb.NewLoggablePutRequest(_req).String() // redact value field from request content, see PR #9821 } if _resp != nil { respCount = 0 respSize = _resp.Size() } case *pb.DeleteRangeResponse: _req, ok := req.(*pb.DeleteRangeRequest) if ok { reqCount = 0 reqSize = _req.Size() reqContent = _req.String() } if _resp != nil { respCount = _resp.GetDeleted() respSize = _resp.Size() } case *pb.TxnResponse: _req, ok := req.(*pb.TxnRequest) if ok && _resp != nil { if _resp.GetSucceeded() { // determine the 'actual' count and size of request based on success or failure reqCount = int64(len(_req.GetSuccess())) reqSize = 0 for _, r := range _req.GetSuccess() { reqSize += r.Size() } } else { reqCount = int64(len(_req.GetFailure())) reqSize = 0 for _, r := range _req.GetFailure() { reqSize += r.Size() } } reqContent = pb.NewLoggableTxnRequest(_req).String() // redact value field from request content, see PR #9821 } if _resp != nil { respCount = 0 respSize = _resp.Size() } default: reqCount = -1 reqSize = -1 respCount = -1 respSize = -1 } if enabledDebugLevel { logGenericRequestStats(lg, startTime, duration, remote, responseType, reqCount, reqSize, respCount, respSize, reqContent) } else if expensiveRequest { logExpensiveRequestStats(lg, startTime, duration, remote, responseType, reqCount, reqSize, respCount, respSize, reqContent) } } func logGenericRequestStats(lg *zap.Logger, startTime time.Time, duration time.Duration, remote string, responseType string, reqCount int64, reqSize int, respCount int64, respSize int, reqContent string, ) { lg.Debug("request stats", zap.Time("start time", startTime), zap.Duration("time spent", duration), zap.String("remote", remote), zap.String("response type", responseType), zap.Int64("request count", reqCount), zap.Int("request size", reqSize), zap.Int64("response count", respCount), zap.Int("response size", respSize), zap.String("request content", reqContent), ) } func logExpensiveRequestStats(lg *zap.Logger, startTime time.Time, duration time.Duration, remote string, responseType string, reqCount int64, reqSize int, respCount int64, respSize int, reqContent string, ) { lg.Warn("request stats", zap.Time("start time", startTime), zap.Duration("time spent", duration), zap.String("remote", remote), zap.String("response type", responseType), zap.Int64("request count", reqCount), zap.Int("request size", reqSize), zap.Int64("response count", respCount), zap.Int("response size", respSize), zap.String("request content", reqContent), ) } func newStreamInterceptor(s *etcdserver.EtcdServer) grpc.StreamServerInterceptor { smap := monitorLeader(s) return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { if !api.IsCapabilityEnabled(api.V3rpcCapability) { return rpctypes.ErrGRPCNotCapable } if s.IsMemberExist(s.MemberID()) && s.IsLearner() && info.FullMethod != snapshotMethod { // learner does not support stream RPC except Snapshot return rpctypes.ErrGRPCNotSupportedForLearner } md, ok := metadata.FromIncomingContext(ss.Context()) if ok { ver, vs := "unknown", md.Get(rpctypes.MetadataClientAPIVersionKey) if len(vs) > 0 { ver = vs[0] } if !utf8.ValidString(ver) { return rpctypes.ErrGRPCInvalidClientAPIVersion } clientRequests.WithLabelValues("stream", ver).Inc() if ks := md[rpctypes.MetadataRequireLeaderKey]; len(ks) > 0 && ks[0] == rpctypes.MetadataHasLeader { if s.Leader() == types.ID(raft.None) { return rpctypes.ErrGRPCNoLeader } ctx := newCancellableContext(ss.Context()) ss = serverStreamWithCtx{ctx: ctx, ServerStream: ss} smap.mu.Lock() smap.streams[ss] = struct{}{} smap.mu.Unlock() defer func() { smap.mu.Lock() delete(smap.streams, ss) smap.mu.Unlock() // TODO: investigate whether the reason for cancellation here is useful to know ctx.Cancel(nil) }() } } return handler(srv, ss) } } // cancellableContext wraps a context with new cancellable context that allows a // specific cancellation error to be preserved and later retrieved using the // Context.Err() function. This is so downstream context users can disambiguate // the reason for the cancellation which could be from the client (for example) // or from this interceptor code. type cancellableContext struct { context.Context lock sync.RWMutex cancel context.CancelFunc cancelReason error } func newCancellableContext(parent context.Context) *cancellableContext { ctx, cancel := context.WithCancel(parent) return &cancellableContext{ Context: ctx, cancel: cancel, } } // Cancel stores the cancellation reason and then delegates to context.WithCancel // against the parent context. func (c *cancellableContext) Cancel(reason error) { c.lock.Lock() c.cancelReason = reason c.lock.Unlock() c.cancel() } // Err will return the preserved cancel reason error if present, and will // otherwise return the underlying error from the parent context. func (c *cancellableContext) Err() error { c.lock.RLock() defer c.lock.RUnlock() if c.cancelReason != nil { return c.cancelReason } return c.Context.Err() } type serverStreamWithCtx struct { grpc.ServerStream // ctx is used so that we can preserve a reason for cancellation. ctx *cancellableContext } func (ssc serverStreamWithCtx) Context() context.Context { return ssc.ctx } func monitorLeader(s *etcdserver.EtcdServer) *streamsMap { smap := &streamsMap{ streams: make(map[grpc.ServerStream]struct{}), } s.GoAttach(func() { election := time.Duration(s.Cfg.TickMs) * time.Duration(s.Cfg.ElectionTicks) * time.Millisecond noLeaderCnt := 0 for { select { case <-s.StoppingNotify(): return case <-time.After(election): if s.Leader() == types.ID(raft.None) { noLeaderCnt++ } else { noLeaderCnt = 0 } // We are more conservative on canceling existing streams. Reconnecting streams // cost much more than just rejecting new requests. So we wait until the member // cannot find a leader for maxNoLeaderCnt election timeouts to cancel existing streams. if noLeaderCnt >= maxNoLeaderCnt { smap.mu.Lock() for ss := range smap.streams { if ssWithCtx, ok := ss.(serverStreamWithCtx); ok { ssWithCtx.ctx.Cancel(rpctypes.ErrGRPCNoLeader) <-ss.Context().Done() } } smap.streams = make(map[grpc.ServerStream]struct{}) smap.mu.Unlock() } } } }) return smap } ================================================ FILE: server/etcdserver/api/v3rpc/key.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package v3rpc implements etcd v3 RPC system based on gRPC. package v3rpc import ( "context" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/pkg/v3/adt" "go.etcd.io/etcd/server/v3/etcdserver" ) type kvServer struct { hdr header kv etcdserver.RaftKV aa *AuthAdmin // maxTxnOps is the max operations per txn. // e.g suppose maxTxnOps = 128. // Txn.Success can have at most 128 operations, // and Txn.Failure can have at most 128 operations. maxTxnOps uint // we want compile errors if new methods are added pb.UnsafeKVServer } func NewKVServer(s *etcdserver.EtcdServer) pb.KVServer { return &kvServer{hdr: newHeader(s), kv: s, aa: &AuthAdmin{s}, maxTxnOps: s.Cfg.MaxTxnOps} } func (s *kvServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) { if err := checkRangeRequest(r); err != nil { return nil, err } resp, err := s.kv.Range(ctx, r) if err != nil { return nil, togRPCError(err) } s.hdr.fill(resp.Header) return resp, nil } func (s *kvServer) Put(ctx context.Context, r *pb.PutRequest) (*pb.PutResponse, error) { if err := checkPutRequest(r); err != nil { return nil, err } resp, err := s.kv.Put(ctx, r) if err != nil { return nil, togRPCError(err) } s.hdr.fill(resp.Header) return resp, nil } func (s *kvServer) DeleteRange(ctx context.Context, r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) { if err := checkDeleteRequest(r); err != nil { return nil, err } resp, err := s.kv.DeleteRange(ctx, r) if err != nil { return nil, togRPCError(err) } s.hdr.fill(resp.Header) return resp, nil } func (s *kvServer) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse, error) { if err := checkTxnRequest(r, int(s.maxTxnOps)); err != nil { return nil, err } // check for forbidden put/del overlaps after checking request to avoid quadratic blowup if _, _, err := checkIntervals(r.Success); err != nil { return nil, err } if _, _, err := checkIntervals(r.Failure); err != nil { return nil, err } resp, err := s.kv.Txn(ctx, r) if err != nil { return nil, togRPCError(err) } s.hdr.fill(resp.Header) return resp, nil } func (s *kvServer) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) { if err := s.aa.isPermitted(ctx); err != nil { return nil, togRPCError(err) } resp, err := s.kv.Compact(ctx, r) if err != nil { return nil, togRPCError(err) } s.hdr.fill(resp.Header) return resp, nil } func checkRangeRequest(r *pb.RangeRequest) error { if len(r.Key) == 0 { return rpctypes.ErrGRPCEmptyKey } if _, ok := pb.RangeRequest_SortOrder_name[int32(r.SortOrder)]; !ok { return rpctypes.ErrGRPCInvalidSortOption } if _, ok := pb.RangeRequest_SortTarget_name[int32(r.SortTarget)]; !ok { return rpctypes.ErrGRPCInvalidSortOption } return nil } func checkPutRequest(r *pb.PutRequest) error { if len(r.Key) == 0 { return rpctypes.ErrGRPCEmptyKey } if r.IgnoreValue && len(r.Value) != 0 { return rpctypes.ErrGRPCValueProvided } if r.IgnoreLease && r.Lease != 0 { return rpctypes.ErrGRPCLeaseProvided } return nil } func checkDeleteRequest(r *pb.DeleteRangeRequest) error { if len(r.Key) == 0 { return rpctypes.ErrGRPCEmptyKey } return nil } func checkTxnRequest(r *pb.TxnRequest, maxTxnOps int) error { opc := len(r.Compare) if opc < len(r.Success) { opc = len(r.Success) } if opc < len(r.Failure) { opc = len(r.Failure) } if opc > maxTxnOps { return rpctypes.ErrGRPCTooManyOps } for _, c := range r.Compare { if len(c.Key) == 0 { return rpctypes.ErrGRPCEmptyKey } } for _, u := range r.Success { if err := checkRequestOp(u, maxTxnOps-opc); err != nil { return err } } for _, u := range r.Failure { if err := checkRequestOp(u, maxTxnOps-opc); err != nil { return err } } return nil } // checkIntervals tests whether puts and deletes overlap for a list of ops. If // there is an overlap, returns an error. If no overlap, return put and delete // sets for recursive evaluation. func checkIntervals(reqs []*pb.RequestOp) (map[string]struct{}, adt.IntervalTree, error) { dels := adt.NewIntervalTree() // collect deletes from this level; build first to check lower level overlapped puts for _, req := range reqs { tv, ok := req.Request.(*pb.RequestOp_RequestDeleteRange) if !ok { continue } dreq := tv.RequestDeleteRange if dreq == nil { continue } var iv adt.Interval if len(dreq.RangeEnd) != 0 { iv = adt.NewStringAffineInterval(string(dreq.Key), string(dreq.RangeEnd)) } else { iv = adt.NewStringAffinePoint(string(dreq.Key)) } dels.Insert(iv, struct{}{}) } // collect children puts/deletes puts := make(map[string]struct{}) for _, req := range reqs { tv, ok := req.Request.(*pb.RequestOp_RequestTxn) if !ok { continue } putsThen, delsThen, err := checkIntervals(tv.RequestTxn.Success) if err != nil { return nil, dels, err } putsElse, delsElse, err := checkIntervals(tv.RequestTxn.Failure) if err != nil { return nil, dels, err } for k := range putsThen { if _, ok := puts[k]; ok { return nil, dels, rpctypes.ErrGRPCDuplicateKey } if dels.Intersects(adt.NewStringAffinePoint(k)) { return nil, dels, rpctypes.ErrGRPCDuplicateKey } puts[k] = struct{}{} } for k := range putsElse { if _, ok := puts[k]; ok { // if key is from putsThen, overlap is OK since // either then/else are mutually exclusive if _, isSafe := putsThen[k]; !isSafe { return nil, dels, rpctypes.ErrGRPCDuplicateKey } } if dels.Intersects(adt.NewStringAffinePoint(k)) { return nil, dels, rpctypes.ErrGRPCDuplicateKey } puts[k] = struct{}{} } dels.Union(delsThen, adt.NewStringAffineInterval("\x00", "")) dels.Union(delsElse, adt.NewStringAffineInterval("\x00", "")) } // collect and check this level's puts for _, req := range reqs { tv, ok := req.Request.(*pb.RequestOp_RequestPut) if !ok || tv.RequestPut == nil { continue } k := string(tv.RequestPut.Key) if _, ok := puts[k]; ok { return nil, dels, rpctypes.ErrGRPCDuplicateKey } if dels.Intersects(adt.NewStringAffinePoint(k)) { return nil, dels, rpctypes.ErrGRPCDuplicateKey } puts[k] = struct{}{} } return puts, dels, nil } func checkRequestOp(u *pb.RequestOp, maxTxnOps int) error { // TODO: ensure only one of the field is set. switch uv := u.Request.(type) { case *pb.RequestOp_RequestRange: return checkRangeRequest(uv.RequestRange) case *pb.RequestOp_RequestPut: return checkPutRequest(uv.RequestPut) case *pb.RequestOp_RequestDeleteRange: return checkDeleteRequest(uv.RequestDeleteRange) case *pb.RequestOp_RequestTxn: return checkTxnRequest(uv.RequestTxn, maxTxnOps) default: // empty op / nil entry return rpctypes.ErrGRPCKeyNotFound } } ================================================ FILE: server/etcdserver/api/v3rpc/key_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "testing" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" ) func TestCheckRangeRequest(t *testing.T) { rangeReqs := []struct { sortOrder pb.RangeRequest_SortOrder sortTarget pb.RangeRequest_SortTarget expectedError error }{ { sortOrder: pb.RangeRequest_ASCEND, sortTarget: pb.RangeRequest_CREATE, expectedError: nil, }, { sortOrder: pb.RangeRequest_ASCEND, sortTarget: 100, expectedError: rpctypes.ErrGRPCInvalidSortOption, }, { sortOrder: 200, sortTarget: pb.RangeRequest_MOD, expectedError: rpctypes.ErrGRPCInvalidSortOption, }, } for _, req := range rangeReqs { rangeReq := pb.RangeRequest{ Key: []byte{1, 2, 3}, SortOrder: req.sortOrder, SortTarget: req.sortTarget, } actualRet := checkRangeRequest(&rangeReq) if getError(actualRet) != getError(req.expectedError) { t.Errorf("expected sortOrder (%d) and sortTarget (%d) to be %q, but got %q", req.sortOrder, req.sortTarget, getError(req.expectedError), getError(actualRet)) } } } func getError(err error) string { if err == nil { return "" } return err.Error() } ================================================ FILE: server/etcdserver/api/v3rpc/lease.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "context" "errors" "io" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/lease" ) type LeaseServer struct { lg *zap.Logger hdr header le etcdserver.Lessor pb.UnsafeLeaseServer } func NewLeaseServer(s *etcdserver.EtcdServer) pb.LeaseServer { srv := &LeaseServer{lg: s.Cfg.Logger, le: s, hdr: newHeader(s)} if srv.lg == nil { srv.lg = zap.NewNop() } return srv } func (ls *LeaseServer) LeaseGrant(ctx context.Context, cr *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { resp, err := ls.le.LeaseGrant(ctx, cr) if err != nil { return nil, togRPCError(err) } ls.hdr.fill(resp.Header) return resp, nil } func (ls *LeaseServer) LeaseRevoke(ctx context.Context, rr *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) { resp, err := ls.le.LeaseRevoke(ctx, rr) if err != nil { return nil, togRPCError(err) } ls.hdr.fill(resp.Header) return resp, nil } func (ls *LeaseServer) LeaseTimeToLive(ctx context.Context, rr *pb.LeaseTimeToLiveRequest) (*pb.LeaseTimeToLiveResponse, error) { resp, err := ls.le.LeaseTimeToLive(ctx, rr) if err != nil && !errors.Is(err, lease.ErrLeaseNotFound) { return nil, togRPCError(err) } if errors.Is(err, lease.ErrLeaseNotFound) { resp = &pb.LeaseTimeToLiveResponse{ Header: &pb.ResponseHeader{}, ID: rr.ID, TTL: -1, } } ls.hdr.fill(resp.Header) return resp, nil } func (ls *LeaseServer) LeaseLeases(ctx context.Context, rr *pb.LeaseLeasesRequest) (*pb.LeaseLeasesResponse, error) { resp, err := ls.le.LeaseLeases(ctx, rr) if err != nil && !errors.Is(err, lease.ErrLeaseNotFound) { return nil, togRPCError(err) } if errors.Is(err, lease.ErrLeaseNotFound) { resp = &pb.LeaseLeasesResponse{ Header: &pb.ResponseHeader{}, Leases: []*pb.LeaseStatus{}, } } ls.hdr.fill(resp.Header) return resp, nil } func (ls *LeaseServer) LeaseKeepAlive(stream pb.Lease_LeaseKeepAliveServer) (err error) { errc := make(chan error, 1) go func() { errc <- ls.leaseKeepAlive(stream) }() select { case err = <-errc: case <-stream.Context().Done(): // We end up here due to: // 1. Client cancellation // 2. Server cancellation: the client ctx is wrapped with WithRequireLeader, // monitorLeader() detects no leader and thus cancels this stream with ErrGRPCNoLeader. // 3. Server cancellation: the server is shutting down. err = stream.Context().Err() } return err } func (ls *LeaseServer) leaseKeepAlive(stream pb.Lease_LeaseKeepAliveServer) error { for { req, err := stream.Recv() if errors.Is(err, io.EOF) { return nil } if err != nil { if isClientCtxErr(stream.Context().Err(), err) { ls.lg.Debug("failed to receive lease keepalive request from gRPC stream", zap.Error(err)) } else { ls.lg.Warn("failed to receive lease keepalive request from gRPC stream", zap.Error(err)) streamFailures.WithLabelValues("receive", "lease-keepalive").Inc() } return err } // Create header before we sent out the renew request. // This can make sure that the revision is strictly smaller or equal to // when the keepalive happened at the local server (when the local server is the leader) // or remote leader. // Without this, a lease might be revoked at rev 3 but client can see the keepalive succeeded // at rev 4. resp := &pb.LeaseKeepAliveResponse{ID: req.ID, Header: &pb.ResponseHeader{}} ls.hdr.fill(resp.Header) ttl, err := ls.le.LeaseRenew(stream.Context(), lease.LeaseID(req.ID)) if errors.Is(err, lease.ErrLeaseNotFound) { err = nil ttl = 0 } if err != nil { return togRPCError(err) } resp.TTL = ttl err = stream.Send(resp) if err != nil { if isClientCtxErr(stream.Context().Err(), err) { ls.lg.Debug("failed to send lease keepalive response to gRPC stream", zap.Error(err)) } else { ls.lg.Warn("failed to send lease keepalive response to gRPC stream", zap.Error(err)) streamFailures.WithLabelValues("send", "lease-keepalive").Inc() } return err } } } ================================================ FILE: server/etcdserver/api/v3rpc/maintenance.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "context" "crypto/sha256" errorspkg "errors" "io" "time" "github.com/dustin/go-humanize" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/apply" "go.etcd.io/etcd/server/v3/etcdserver/errors" serverversion "go.etcd.io/etcd/server/v3/etcdserver/version" "go.etcd.io/etcd/server/v3/storage" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/raft/v3" ) type KVGetter interface { KV() mvcc.WatchableKV } type BackendGetter interface { Backend() backend.Backend } type Defrager interface { Defragment() error } type Alarmer interface { // Alarms is implemented in Server interface located in etcdserver/server.go // It returns a list of alarms present in the AlarmStore Alarms() []*pb.AlarmMember Alarm(ctx context.Context, ar *pb.AlarmRequest) (*pb.AlarmResponse, error) } type Downgrader interface { Downgrade(ctx context.Context, dr *pb.DowngradeRequest) (*pb.DowngradeResponse, error) } type LeaderTransferrer interface { MoveLeader(ctx context.Context, lead, target uint64) error } type ClusterStatusGetter interface { IsLearner() bool } type ConfigGetter interface { Config() config.ServerConfig } type maintenanceServer struct { lg *zap.Logger rg apply.RaftStatusGetter hasher mvcc.HashStorage bg BackendGetter defrag Defrager a Alarmer lt LeaderTransferrer hdr header cs ClusterStatusGetter d Downgrader vs serverversion.Server cg ConfigGetter healthNotifier notifier // we want compile errors if new methods are added pb.UnsafeMaintenanceServer } func NewMaintenanceServer(s *etcdserver.EtcdServer, healthNotifier notifier) pb.MaintenanceServer { srv := &maintenanceServer{ lg: s.Cfg.Logger, rg: s, hasher: s.KV().HashStorage(), bg: s, defrag: s, a: s, lt: s, hdr: newHeader(s), cs: s, d: s, vs: etcdserver.NewServerVersionAdapter(s), healthNotifier: healthNotifier, cg: s, } if srv.lg == nil { srv.lg = zap.NewNop() } return &authMaintenanceServer{srv, &AuthAdmin{s}} } func (ms *maintenanceServer) Defragment(ctx context.Context, sr *pb.DefragmentRequest) (*pb.DefragmentResponse, error) { ms.lg.Info("starting defragment") ms.healthNotifier.defragStarted() defer ms.healthNotifier.defragFinished() err := ms.defrag.Defragment() if err != nil { ms.lg.Warn("failed to defragment", zap.Error(err)) return nil, togRPCError(err) } ms.lg.Info("finished defragment") return &pb.DefragmentResponse{}, nil } // big enough size to hold >1 OS pages in the buffer const snapshotSendBufferSize = 32 * 1024 func (ms *maintenanceServer) Snapshot(sr *pb.SnapshotRequest, srv pb.Maintenance_SnapshotServer) error { ver := schema.ReadStorageVersion(ms.bg.Backend().ReadTx()) storageVersion := "" if ver != nil { storageVersion = ver.String() } snap := ms.bg.Backend().Snapshot() pr, pw := io.Pipe() defer pr.Close() go func() { snap.WriteTo(pw) if err := snap.Close(); err != nil { ms.lg.Warn("failed to close snapshot", zap.Error(err)) } pw.Close() }() // record SHA digest of snapshot data // used for integrity checks during snapshot restore operation h := sha256.New() sent := int64(0) total := snap.Size() size := humanize.Bytes(uint64(total)) start := time.Now() ms.lg.Info("sending database snapshot to client", zap.Int64("total-bytes", total), zap.String("size", size), zap.String("storage-version", storageVersion), ) for total-sent > 0 { // buffer just holds read bytes from stream // response size is multiple of OS page size, fetched in boltdb // e.g. 4*1024 // NOTE: srv.Send does not wait until the message is received by the client. // Therefore the buffer can not be safely reused between Send operations buf := make([]byte, snapshotSendBufferSize) n, err := io.ReadFull(pr, buf) if err != nil && !errorspkg.Is(err, io.EOF) && !errorspkg.Is(err, io.ErrUnexpectedEOF) { return togRPCError(err) } sent += int64(n) // if total is x * snapshotSendBufferSize. it is possible that // resp.RemainingBytes == 0 // resp.Blob == zero byte but not nil // does this make server response sent to client nil in proto // and client stops receiving from snapshot stream before // server sends snapshot SHA? // No, the client will still receive non-nil response // until server closes the stream with EOF resp := &pb.SnapshotResponse{ RemainingBytes: uint64(total - sent), Blob: buf[:n], Version: storageVersion, } if err = srv.Send(resp); err != nil { return togRPCError(err) } h.Write(buf[:n]) } // send SHA digest for integrity checks // during snapshot restore operation sha := h.Sum(nil) ms.lg.Info("sending database sha256 checksum to client", zap.Int64("total-bytes", total), zap.Int("checksum-size", len(sha)), ) hresp := &pb.SnapshotResponse{RemainingBytes: 0, Blob: sha, Version: storageVersion} if err := srv.Send(hresp); err != nil { return togRPCError(err) } ms.lg.Info("successfully sent database snapshot to client", zap.Int64("total-bytes", total), zap.String("size", size), zap.Duration("took", time.Since(start)), zap.String("storage-version", storageVersion), ) return nil } func (ms *maintenanceServer) Hash(ctx context.Context, r *pb.HashRequest) (*pb.HashResponse, error) { h, rev, err := ms.hasher.Hash() if err != nil { return nil, togRPCError(err) } resp := &pb.HashResponse{Header: &pb.ResponseHeader{Revision: rev}, Hash: h} ms.hdr.fill(resp.Header) return resp, nil } func (ms *maintenanceServer) HashKV(ctx context.Context, r *pb.HashKVRequest) (*pb.HashKVResponse, error) { h, rev, err := ms.hasher.HashByRev(r.Revision) if err != nil { return nil, togRPCError(err) } resp := &pb.HashKVResponse{ Header: &pb.ResponseHeader{Revision: rev}, Hash: h.Hash, CompactRevision: h.CompactRevision, HashRevision: h.Revision, } ms.hdr.fill(resp.Header) return resp, nil } func (ms *maintenanceServer) Alarm(ctx context.Context, ar *pb.AlarmRequest) (*pb.AlarmResponse, error) { resp, err := ms.a.Alarm(ctx, ar) if err != nil { return nil, togRPCError(err) } if resp.Header == nil { resp.Header = &pb.ResponseHeader{} } ms.hdr.fill(resp.Header) return resp, nil } func (ms *maintenanceServer) Status(ctx context.Context, ar *pb.StatusRequest) (*pb.StatusResponse, error) { hdr := &pb.ResponseHeader{} ms.hdr.fill(hdr) resp := &pb.StatusResponse{ Header: hdr, Version: version.Version, Leader: uint64(ms.rg.Leader()), RaftIndex: ms.rg.CommittedIndex(), RaftAppliedIndex: ms.rg.AppliedIndex(), RaftTerm: ms.rg.Term(), DbSize: ms.bg.Backend().Size(), DbSizeInUse: ms.bg.Backend().SizeInUse(), IsLearner: ms.cs.IsLearner(), DbSizeQuota: ms.cg.Config().QuotaBackendBytes, DowngradeInfo: &pb.DowngradeInfo{Enabled: false}, } if resp.DbSizeQuota == 0 { resp.DbSizeQuota = storage.DefaultQuotaBytes } if storageVersion := ms.vs.GetStorageVersion(); storageVersion != nil { resp.StorageVersion = storageVersion.String() } if downgradeInfo := ms.vs.GetDowngradeInfo(); downgradeInfo != nil { resp.DowngradeInfo = &pb.DowngradeInfo{ Enabled: downgradeInfo.Enabled, TargetVersion: downgradeInfo.TargetVersion, } } if resp.Leader == raft.None { resp.Errors = append(resp.Errors, errors.ErrNoLeader.Error()) } for _, a := range ms.a.Alarms() { resp.Errors = append(resp.Errors, a.String()) } return resp, nil } func (ms *maintenanceServer) MoveLeader(ctx context.Context, tr *pb.MoveLeaderRequest) (*pb.MoveLeaderResponse, error) { if ms.rg.MemberID() != ms.rg.Leader() { return nil, rpctypes.ErrGRPCNotLeader } if err := ms.lt.MoveLeader(ctx, uint64(ms.rg.Leader()), tr.TargetID); err != nil { return nil, togRPCError(err) } return &pb.MoveLeaderResponse{}, nil } func (ms *maintenanceServer) Downgrade(ctx context.Context, r *pb.DowngradeRequest) (*pb.DowngradeResponse, error) { resp, err := ms.d.Downgrade(ctx, r) if err != nil { return nil, togRPCError(err) } resp.Header = &pb.ResponseHeader{} ms.hdr.fill(resp.Header) return resp, nil } type authMaintenanceServer struct { *maintenanceServer *AuthAdmin } func (ams *authMaintenanceServer) Defragment(ctx context.Context, sr *pb.DefragmentRequest) (*pb.DefragmentResponse, error) { if err := ams.isPermitted(ctx); err != nil { return nil, togRPCError(err) } return ams.maintenanceServer.Defragment(ctx, sr) } func (ams *authMaintenanceServer) Snapshot(sr *pb.SnapshotRequest, srv pb.Maintenance_SnapshotServer) error { if err := ams.isPermitted(srv.Context()); err != nil { return togRPCError(err) } return ams.maintenanceServer.Snapshot(sr, srv) } func (ams *authMaintenanceServer) Hash(ctx context.Context, r *pb.HashRequest) (*pb.HashResponse, error) { if err := ams.isPermitted(ctx); err != nil { return nil, togRPCError(err) } return ams.maintenanceServer.Hash(ctx, r) } func (ams *authMaintenanceServer) HashKV(ctx context.Context, r *pb.HashKVRequest) (*pb.HashKVResponse, error) { if err := ams.isPermitted(ctx); err != nil { return nil, togRPCError(err) } return ams.maintenanceServer.HashKV(ctx, r) } func (ams *authMaintenanceServer) Alarm(ctx context.Context, ar *pb.AlarmRequest) (*pb.AlarmResponse, error) { if err := ams.isPermitted(ctx); err != nil { return nil, togRPCError(err) } return ams.maintenanceServer.Alarm(ctx, ar) } func (ams *authMaintenanceServer) Status(ctx context.Context, ar *pb.StatusRequest) (*pb.StatusResponse, error) { if err := ams.isPermitted(ctx); err != nil { return nil, togRPCError(err) } return ams.maintenanceServer.Status(ctx, ar) } func (ams *authMaintenanceServer) MoveLeader(ctx context.Context, tr *pb.MoveLeaderRequest) (*pb.MoveLeaderResponse, error) { if err := ams.isPermitted(ctx); err != nil { return nil, togRPCError(err) } return ams.maintenanceServer.MoveLeader(ctx, tr) } func (ams *authMaintenanceServer) Downgrade(ctx context.Context, r *pb.DowngradeRequest) (*pb.DowngradeResponse, error) { if err := ams.isPermitted(ctx); err != nil { return nil, togRPCError(err) } return ams.maintenanceServer.Downgrade(ctx, r) } ================================================ FILE: server/etcdserver/api/v3rpc/member.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "context" "time" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" ) type ClusterServer struct { cluster api.Cluster server *etcdserver.EtcdServer // we want compile errors if new methods are added pb.UnsafeClusterServer } func NewClusterServer(s *etcdserver.EtcdServer) *ClusterServer { return &ClusterServer{ cluster: s.Cluster(), server: s, } } func (cs *ClusterServer) MemberAdd(ctx context.Context, r *pb.MemberAddRequest) (*pb.MemberAddResponse, error) { urls, err := types.NewURLs(r.PeerURLs) if err != nil { return nil, rpctypes.ErrGRPCMemberBadURLs } now := time.Now() var m *membership.Member if r.IsLearner { m = membership.NewMemberAsLearner("", urls, "", &now) } else { m = membership.NewMember("", urls, "", &now) } membs, merr := cs.server.AddMember(ctx, *m) if merr != nil { return nil, togRPCError(merr) } return &pb.MemberAddResponse{ Header: cs.header(), Member: &pb.Member{ ID: uint64(m.ID), PeerURLs: m.PeerURLs, IsLearner: m.IsLearner, }, Members: membersToProtoMembers(membs), }, nil } func (cs *ClusterServer) MemberRemove(ctx context.Context, r *pb.MemberRemoveRequest) (*pb.MemberRemoveResponse, error) { membs, err := cs.server.RemoveMember(ctx, r.ID) if err != nil { return nil, togRPCError(err) } return &pb.MemberRemoveResponse{Header: cs.header(), Members: membersToProtoMembers(membs)}, nil } func (cs *ClusterServer) MemberUpdate(ctx context.Context, r *pb.MemberUpdateRequest) (*pb.MemberUpdateResponse, error) { m := membership.Member{ ID: types.ID(r.ID), RaftAttributes: membership.RaftAttributes{PeerURLs: r.PeerURLs}, } membs, err := cs.server.UpdateMember(ctx, m) if err != nil { return nil, togRPCError(err) } return &pb.MemberUpdateResponse{Header: cs.header(), Members: membersToProtoMembers(membs)}, nil } func (cs *ClusterServer) MemberList(ctx context.Context, r *pb.MemberListRequest) (*pb.MemberListResponse, error) { members, err := cs.server.MemberList(ctx, r) if err != nil { return nil, togRPCError(err) } membs := membersToProtoMembers(members) return &pb.MemberListResponse{Header: cs.header(), Members: membs}, nil } func (cs *ClusterServer) MemberPromote(ctx context.Context, r *pb.MemberPromoteRequest) (*pb.MemberPromoteResponse, error) { membs, err := cs.server.PromoteMember(ctx, r.ID) if err != nil { return nil, togRPCError(err) } return &pb.MemberPromoteResponse{Header: cs.header(), Members: membersToProtoMembers(membs)}, nil } func (cs *ClusterServer) header() *pb.ResponseHeader { return &pb.ResponseHeader{ClusterId: uint64(cs.cluster.ID()), MemberId: uint64(cs.server.MemberID()), RaftTerm: cs.server.Term()} } func membersToProtoMembers(membs []*membership.Member) []*pb.Member { protoMembs := make([]*pb.Member, len(membs)) for i := range membs { protoMembs[i] = &pb.Member{ Name: membs[i].Name, ID: uint64(membs[i].ID), PeerURLs: membs[i].PeerURLs, ClientURLs: membs[i].ClientURLs, IsLearner: membs[i].IsLearner, } } return protoMembs } ================================================ FILE: server/etcdserver/api/v3rpc/metrics.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "github.com/prometheus/client_golang/prometheus" ) var ( sentBytes = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "client_grpc_sent_bytes_total", Help: "The total number of bytes sent to grpc clients.", }) receivedBytes = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "client_grpc_received_bytes_total", Help: "The total number of bytes received from grpc clients.", }) streamFailures = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", Name: "server_stream_failures_total", Help: "The total number of stream failures from the local server.", }, []string{"Type", "API"}, ) clientRequests = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "client_requests_total", Help: "The total number of client requests per client version.", }, []string{"type", "client_api_version"}, ) watchSendLoopWatchStreamDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "server", Name: "watch_send_loop_watch_stream_duration_seconds", Help: "The total duration in seconds of running through the send loop watch stream response all events.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }, ) watchSendLoopWatchStreamDurationPerEvent = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "server", Name: "watch_send_loop_watch_stream_duration_per_event_seconds", Help: "The average duration in seconds of running through the send loop watch stream response, per event.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }, ) watchSendLoopControlStreamDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "server", Name: "watch_send_loop_control_stream_duration_seconds", Help: "The total duration in seconds of running through the send loop control stream response.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }, ) watchSendLoopProgressDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "server", Name: "watch_send_loop_progress_duration_seconds", Help: "The total duration in seconds of running through the progress loop control stream response.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }, ) ) func init() { prometheus.MustRegister(sentBytes) prometheus.MustRegister(receivedBytes) prometheus.MustRegister(streamFailures) prometheus.MustRegister(clientRequests) prometheus.MustRegister(watchSendLoopWatchStreamDuration) prometheus.MustRegister(watchSendLoopWatchStreamDurationPerEvent) prometheus.MustRegister(watchSendLoopControlStreamDuration) prometheus.MustRegister(watchSendLoopProgressDuration) } ================================================ FILE: server/etcdserver/api/v3rpc/quota.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "context" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/storage" ) type quotaKVServer struct { pb.KVServer qa quotaAlarmer } type quotaAlarmer struct { q storage.Quota a Alarmer id types.ID } // check whether request satisfies the quota. If there is not enough space, // ignore request and raise the free space alarm. func (qa *quotaAlarmer) check(ctx context.Context, r any) error { if qa.q.Available(r) { return nil } req := &pb.AlarmRequest{ MemberID: uint64(qa.id), Action: pb.AlarmRequest_ACTIVATE, Alarm: pb.AlarmType_NOSPACE, } qa.a.Alarm(ctx, req) return rpctypes.ErrGRPCNoSpace } func NewQuotaKVServer(s *etcdserver.EtcdServer) pb.KVServer { return "aKVServer{ NewKVServer(s), quotaAlarmer{newBackendQuota(s, "kv"), s, s.MemberID()}, } } func (s *quotaKVServer) Put(ctx context.Context, r *pb.PutRequest) (*pb.PutResponse, error) { if err := s.qa.check(ctx, r); err != nil { return nil, err } return s.KVServer.Put(ctx, r) } func (s *quotaKVServer) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse, error) { if err := s.qa.check(ctx, r); err != nil { return nil, err } return s.KVServer.Txn(ctx, r) } type quotaLeaseServer struct { pb.LeaseServer qa quotaAlarmer } func (s *quotaLeaseServer) LeaseGrant(ctx context.Context, cr *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { if err := s.qa.check(ctx, cr); err != nil { return nil, err } return s.LeaseServer.LeaseGrant(ctx, cr) } func NewQuotaLeaseServer(s *etcdserver.EtcdServer) pb.LeaseServer { return "aLeaseServer{ NewLeaseServer(s), quotaAlarmer{newBackendQuota(s, "lease"), s, s.MemberID()}, } } func newBackendQuota(s *etcdserver.EtcdServer, name string) storage.Quota { return storage.NewBackendQuota(s.Logger(), s.Cfg.QuotaBackendBytes, s.Backend(), name) } ================================================ FILE: server/etcdserver/api/v3rpc/util.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "context" errorspkg "errors" "strings" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/errors" "go.etcd.io/etcd/server/v3/etcdserver/version" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/mvcc" ) var toGRPCErrorMap = map[error]error{ membership.ErrIDRemoved: rpctypes.ErrGRPCMemberNotFound, membership.ErrIDNotFound: rpctypes.ErrGRPCMemberNotFound, membership.ErrIDExists: rpctypes.ErrGRPCMemberExist, membership.ErrPeerURLexists: rpctypes.ErrGRPCPeerURLExist, membership.ErrMemberNotLearner: rpctypes.ErrGRPCMemberNotLearner, membership.ErrTooManyLearners: rpctypes.ErrGRPCTooManyLearners, errors.ErrNotEnoughStartedMembers: rpctypes.ErrMemberNotEnoughStarted, errors.ErrLearnerNotReady: rpctypes.ErrGRPCLearnerNotReady, mvcc.ErrCompacted: rpctypes.ErrGRPCCompacted, mvcc.ErrFutureRev: rpctypes.ErrGRPCFutureRev, errors.ErrRequestTooLarge: rpctypes.ErrGRPCRequestTooLarge, errors.ErrNoSpace: rpctypes.ErrGRPCNoSpace, errors.ErrTooManyRequests: rpctypes.ErrTooManyRequests, errors.ErrNoLeader: rpctypes.ErrGRPCNoLeader, errors.ErrNotLeader: rpctypes.ErrGRPCNotLeader, errors.ErrLeaderChanged: rpctypes.ErrGRPCLeaderChanged, errors.ErrCanceled: rpctypes.ErrGRPCCanceled, errors.ErrStopped: rpctypes.ErrGRPCStopped, errors.ErrTimeout: rpctypes.ErrGRPCTimeout, errors.ErrTimeoutDueToLeaderFail: rpctypes.ErrGRPCTimeoutDueToLeaderFail, errors.ErrTimeoutDueToConnectionLost: rpctypes.ErrGRPCTimeoutDueToConnectionLost, errors.ErrTimeoutWaitAppliedIndex: rpctypes.ErrGRPCTimeoutWaitAppliedIndex, errors.ErrUnhealthy: rpctypes.ErrGRPCUnhealthy, errors.ErrKeyNotFound: rpctypes.ErrGRPCKeyNotFound, errors.ErrCorrupt: rpctypes.ErrGRPCCorrupt, errors.ErrBadLeaderTransferee: rpctypes.ErrGRPCBadLeaderTransferee, errors.ErrClusterVersionUnavailable: rpctypes.ErrGRPCClusterVersionUnavailable, errors.ErrWrongDowngradeVersionFormat: rpctypes.ErrGRPCWrongDowngradeVersionFormat, version.ErrInvalidDowngradeTargetVersion: rpctypes.ErrGRPCInvalidDowngradeTargetVersion, version.ErrDowngradeInProcess: rpctypes.ErrGRPCDowngradeInProcess, version.ErrNoInflightDowngrade: rpctypes.ErrGRPCNoInflightDowngrade, lease.ErrLeaseNotFound: rpctypes.ErrGRPCLeaseNotFound, lease.ErrLeaseExists: rpctypes.ErrGRPCLeaseExist, lease.ErrLeaseTTLTooLarge: rpctypes.ErrGRPCLeaseTTLTooLarge, auth.ErrRootUserNotExist: rpctypes.ErrGRPCRootUserNotExist, auth.ErrRootRoleNotExist: rpctypes.ErrGRPCRootRoleNotExist, auth.ErrUserAlreadyExist: rpctypes.ErrGRPCUserAlreadyExist, auth.ErrUserEmpty: rpctypes.ErrGRPCUserEmpty, auth.ErrUserNotFound: rpctypes.ErrGRPCUserNotFound, auth.ErrRoleAlreadyExist: rpctypes.ErrGRPCRoleAlreadyExist, auth.ErrRoleNotFound: rpctypes.ErrGRPCRoleNotFound, auth.ErrRoleEmpty: rpctypes.ErrGRPCRoleEmpty, auth.ErrAuthFailed: rpctypes.ErrGRPCAuthFailed, auth.ErrPermissionNotGiven: rpctypes.ErrGRPCPermissionNotGiven, auth.ErrPermissionDenied: rpctypes.ErrGRPCPermissionDenied, auth.ErrRoleNotGranted: rpctypes.ErrGRPCRoleNotGranted, auth.ErrPermissionNotGranted: rpctypes.ErrGRPCPermissionNotGranted, auth.ErrAuthNotEnabled: rpctypes.ErrGRPCAuthNotEnabled, auth.ErrInvalidAuthToken: rpctypes.ErrGRPCInvalidAuthToken, auth.ErrInvalidAuthMgmt: rpctypes.ErrGRPCInvalidAuthMgmt, auth.ErrAuthOldRevision: rpctypes.ErrGRPCAuthOldRevision, // In sync with status.FromContextError context.Canceled: rpctypes.ErrGRPCCanceled, context.DeadlineExceeded: rpctypes.ErrGRPCDeadlineExceeded, } func togRPCError(err error) error { // let gRPC server convert to codes.Canceled, codes.DeadlineExceeded if errorspkg.Is(err, context.Canceled) || errorspkg.Is(err, context.DeadlineExceeded) { return err } grpcErr, ok := toGRPCErrorMap[err] if !ok { return status.Error(codes.Unknown, err.Error()) } return grpcErr } func isClientCtxErr(ctxErr error, err error) bool { if ctxErr != nil { return true } ev, ok := status.FromError(err) if !ok { return false } switch ev.Code() { case codes.Canceled, codes.DeadlineExceeded: // client-side context cancel or deadline exceeded // "rpc error: code = Canceled desc = context canceled" // "rpc error: code = DeadlineExceeded desc = context deadline exceeded" return true case codes.Unavailable: msg := ev.Message() // client-side context cancel or deadline exceeded with TLS ("http2.errClientDisconnected") // "rpc error: code = Unavailable desc = client disconnected" if msg == "client disconnected" { return true } // "grpc/transport.ClientTransport.CloseStream" on canceled streams // "rpc error: code = Unavailable desc = stream error: stream ID 21; CANCEL") if strings.HasPrefix(msg, "stream error: ") && strings.HasSuffix(msg, "; CANCEL") { return true } } return false } // in v3.4, learner is allowed to serve serializable read and endpoint status func isRPCSupportedForLearner(req any) bool { switch r := req.(type) { case *pb.StatusRequest: return true case *pb.RangeRequest: return r.Serializable default: return false } } ================================================ FILE: server/etcdserver/api/v3rpc/util_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "context" "errors" "testing" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/server/v3/storage/mvcc" ) func TestGRPCError(t *testing.T) { tt := []struct { err error exp error }{ {err: mvcc.ErrCompacted, exp: rpctypes.ErrGRPCCompacted}, {err: mvcc.ErrFutureRev, exp: rpctypes.ErrGRPCFutureRev}, {err: context.Canceled, exp: context.Canceled}, {err: context.DeadlineExceeded, exp: context.DeadlineExceeded}, {err: errors.New("foo"), exp: status.Error(codes.Unknown, "foo")}, } for i := range tt { if err := togRPCError(tt[i].err); !errors.Is(err, tt[i].exp) { if _, ok := status.FromError(err); ok { if err.Error() == tt[i].exp.Error() { continue } } t.Errorf("#%d: got %v, expected %v", i, err, tt[i].exp) } } } ================================================ FILE: server/etcdserver/api/v3rpc/validationfuzz_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "context" "testing" "go.uber.org/zap/zaptest" pb "go.etcd.io/etcd/api/v3/etcdserverpb" txn "go.etcd.io/etcd/server/v3/etcdserver/txn" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/mvcc" ) func FuzzTxnRangeRequest(f *testing.F) { testcases := []pb.RangeRequest{ { Key: []byte{2}, RangeEnd: []byte{2}, Limit: 3, Revision: 3, SortOrder: 2, SortTarget: 2, }, } for _, tc := range testcases { soValue := pb.RangeRequest_SortOrder_value[tc.SortOrder.String()] soTarget := pb.RangeRequest_SortTarget_value[tc.SortTarget.String()] f.Add(tc.Key, tc.RangeEnd, tc.Limit, tc.Revision, soValue, soTarget) } f.Fuzz(func(t *testing.T, key []byte, rangeEnd []byte, limit int64, revision int64, sortOrder int32, sortTarget int32, ) { fuzzRequest := &pb.RangeRequest{ Key: key, RangeEnd: rangeEnd, Limit: limit, SortOrder: pb.RangeRequest_SortOrder(sortOrder), SortTarget: pb.RangeRequest_SortTarget(sortTarget), } verifyCheck(t, func() error { return checkRangeRequest(fuzzRequest) }) execTransaction(t, &pb.RequestOp{ Request: &pb.RequestOp_RequestRange{ RequestRange: fuzzRequest, }, }) }) } func FuzzTxnPutRequest(f *testing.F) { testcases := []pb.PutRequest{ { Key: []byte{2}, Value: []byte{2}, Lease: 2, PrevKv: false, IgnoreValue: false, IgnoreLease: false, }, } for _, tc := range testcases { f.Add(tc.Key, tc.Value, tc.Lease, tc.PrevKv, tc.IgnoreValue, tc.IgnoreLease) } f.Fuzz(func(t *testing.T, key []byte, value []byte, leaseValue int64, prevKv bool, ignoreValue bool, IgnoreLease bool, ) { fuzzRequest := &pb.PutRequest{ Key: key, Value: value, Lease: leaseValue, PrevKv: prevKv, IgnoreValue: ignoreValue, IgnoreLease: IgnoreLease, } verifyCheck(t, func() error { return checkPutRequest(fuzzRequest) }) execTransaction(t, &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: fuzzRequest, }, }) }) } func FuzzTxnDeleteRangeRequest(f *testing.F) { testcases := []pb.DeleteRangeRequest{ { Key: []byte{2}, RangeEnd: []byte{2}, PrevKv: false, }, } for _, tc := range testcases { f.Add(tc.Key, tc.RangeEnd, tc.PrevKv) } f.Fuzz(func(t *testing.T, key []byte, rangeEnd []byte, prevKv bool, ) { fuzzRequest := &pb.DeleteRangeRequest{ Key: key, RangeEnd: rangeEnd, PrevKv: prevKv, } verifyCheck(t, func() error { return checkDeleteRequest(fuzzRequest) }) execTransaction(t, &pb.RequestOp{ Request: &pb.RequestOp_RequestDeleteRange{ RequestDeleteRange: fuzzRequest, }, }) }) } func verifyCheck(t *testing.T, check func() error) { errCheck := check() if errCheck != nil { t.Skip("Validation not passing. Skipping the apply.") } } func execTransaction(t *testing.T, req *pb.RequestOp) { b, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, b) s := mvcc.NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, mvcc.StoreConfig{}) defer s.Close() // setup cancelled context ctx, cancel := context.WithCancel(t.Context()) cancel() request := &pb.TxnRequest{ Success: []*pb.RequestOp{req}, } _, _, err := txn.Txn(ctx, zaptest.NewLogger(t), request, false, s, &lease.FakeLessor{}) if err != nil { t.Skipf("Application erroring. %s", err.Error()) } } ================================================ FILE: server/etcdserver/api/v3rpc/watch.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "context" "errors" "io" "math/rand" "sync" "time" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/verify" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/apply" "go.etcd.io/etcd/server/v3/storage/mvcc" ) const minWatchProgressInterval = 100 * time.Millisecond type watchServer struct { lg *zap.Logger clusterID int64 memberID int64 maxRequestBytes uint sg apply.RaftStatusGetter watchable mvcc.WatchableKV ag AuthGetter // we want compile errors if new methods are added pb.UnsafeWatchServer } // NewWatchServer returns a new watch server. func NewWatchServer(s *etcdserver.EtcdServer) pb.WatchServer { srv := &watchServer{ lg: s.Cfg.Logger, clusterID: int64(s.Cluster().ID()), memberID: int64(s.MemberID()), maxRequestBytes: s.Cfg.MaxRequestBytesWithOverhead(), sg: s, watchable: s.Watchable(), ag: s, } if srv.lg == nil { srv.lg = zap.NewNop() } if s.Cfg.WatchProgressNotifyInterval > 0 { if s.Cfg.WatchProgressNotifyInterval < minWatchProgressInterval { srv.lg.Warn( "adjusting watch progress notify interval to minimum period", zap.Duration("min-watch-progress-notify-interval", minWatchProgressInterval), ) s.Cfg.WatchProgressNotifyInterval = minWatchProgressInterval } SetProgressReportInterval(s.Cfg.WatchProgressNotifyInterval) } return srv } var ( // External test can read this with GetProgressReportInterval() // and change this to a small value to finish fast with // SetProgressReportInterval(). progressReportInterval = 10 * time.Minute progressReportIntervalMu sync.RWMutex ) // GetProgressReportInterval returns the current progress report interval (for testing). func GetProgressReportInterval() time.Duration { progressReportIntervalMu.RLock() interval := progressReportInterval progressReportIntervalMu.RUnlock() // add rand(1/10*progressReportInterval) as jitter so that etcdserver will not // send progress notifications to watchers around the same time even when watchers // are created around the same time (which is common when a client restarts itself). jitter := time.Duration(rand.Int63n(int64(interval) / 10)) return interval + jitter } // SetProgressReportInterval updates the current progress report interval (for testing). func SetProgressReportInterval(newTimeout time.Duration) { progressReportIntervalMu.Lock() progressReportInterval = newTimeout progressReportIntervalMu.Unlock() } // We send ctrl response inside the read loop. We do not want // send to block read, but we still want ctrl response we sent to // be serialized. Thus we use a buffered chan to solve the problem. // A small buffer should be OK for most cases, since we expect the // ctrl requests are infrequent. const ctrlStreamBufLen = 16 // serverWatchStream is an etcd server side stream. It receives requests // from client side gRPC stream. It receives watch events from mvcc.WatchStream, // and creates responses that forwarded to gRPC stream. // It also forwards control message like watch created and canceled. type serverWatchStream struct { lg *zap.Logger clusterID int64 memberID int64 maxRequestBytes uint sg apply.RaftStatusGetter watchable mvcc.WatchableKV ag AuthGetter gRPCStream pb.Watch_WatchServer watchStream mvcc.WatchStream ctrlStream chan *pb.WatchResponse // mu protects progress, prevKV, fragment mu sync.RWMutex // tracks the watchID that stream might need to send progress to // TODO: combine progress and prevKV into a single struct? progress map[mvcc.WatchID]bool // record watch IDs that need return previous key-value pair prevKV map[mvcc.WatchID]bool // records fragmented watch IDs fragment map[mvcc.WatchID]bool // closec indicates the stream is closed. closec chan struct{} // wg waits for the send loop to complete wg sync.WaitGroup } func (ws *watchServer) Watch(stream pb.Watch_WatchServer) (err error) { sws := serverWatchStream{ lg: ws.lg, clusterID: ws.clusterID, memberID: ws.memberID, maxRequestBytes: ws.maxRequestBytes, sg: ws.sg, watchable: ws.watchable, ag: ws.ag, gRPCStream: stream, watchStream: ws.watchable.NewWatchStream(), // chan for sending control response like watcher created and canceled. ctrlStream: make(chan *pb.WatchResponse, ctrlStreamBufLen), progress: make(map[mvcc.WatchID]bool), prevKV: make(map[mvcc.WatchID]bool), fragment: make(map[mvcc.WatchID]bool), closec: make(chan struct{}), } sws.wg.Add(1) go func() { sws.sendLoop() sws.wg.Done() }() errc := make(chan error, 1) // Ideally recvLoop would also use sws.wg to signal its completion // but when stream.Context().Done() is closed, the stream's recv // may continue to block since it uses a different context, leading to // deadlock when calling sws.close(). go func() { if rerr := sws.recvLoop(); rerr != nil { if isClientCtxErr(stream.Context().Err(), rerr) { sws.lg.Debug("failed to receive watch request from gRPC stream", zap.Error(rerr)) } else { sws.lg.Warn("failed to receive watch request from gRPC stream", zap.Error(rerr)) streamFailures.WithLabelValues("receive", "watch").Inc() } errc <- rerr } }() // TODO: There's a race here. When a stream is closed (e.g. due to a cancellation), // the underlying error (e.g. a gRPC stream error) may be returned and handled // through errc if the recv goroutine finishes before the send goroutine. // When the recv goroutine wins, the stream error is retained. When recv loses // the race, the underlying error is lost (unless the root error is propagated // through Context.Err() which is not always the case (as callers have to decide // to implement a custom context to do so). The stdlib context package builtins // may be insufficient to carry semantically useful errors around and should be // revisited. select { case err = <-errc: if errors.Is(err, context.Canceled) { err = rpctypes.ErrGRPCWatchCanceled } close(sws.ctrlStream) case <-stream.Context().Done(): err = stream.Context().Err() if errors.Is(err, context.Canceled) { err = rpctypes.ErrGRPCWatchCanceled } } sws.close() return err } func (sws *serverWatchStream) isWatchPermitted(wcr *pb.WatchCreateRequest) error { authInfo, err := sws.ag.AuthInfoFromCtx(sws.gRPCStream.Context()) if err != nil { return err } if authInfo == nil { // if auth is enabled, IsRangePermitted() can cause an error authInfo = &auth.AuthInfo{} } return sws.ag.AuthStore().IsRangePermitted(authInfo, wcr.Key, wcr.RangeEnd) } func (sws *serverWatchStream) recvLoop() error { for { req, err := sws.gRPCStream.Recv() if errors.Is(err, io.EOF) { return nil } if err != nil { return err } switch uv := req.RequestUnion.(type) { case *pb.WatchRequest_CreateRequest: if uv.CreateRequest == nil { break } creq := uv.CreateRequest if len(creq.Key) == 0 { // \x00 is the smallest key creq.Key = []byte{0} } if len(creq.RangeEnd) == 0 { // force nil since watchstream.Watch distinguishes // between nil and []byte{} for single key / >= creq.RangeEnd = nil } if len(creq.RangeEnd) == 1 && creq.RangeEnd[0] == 0 { // support >= key queries creq.RangeEnd = []byte{} } if creq.StartRevision < 0 { wr := &pb.WatchResponse{ Header: sws.newResponseHeader(sws.watchStream.Rev()), WatchId: clientv3.InvalidWatchID, Canceled: true, Created: true, CancelReason: rpctypes.ErrCompacted.Error(), } select { case sws.ctrlStream <- wr: continue case <-sws.closec: return nil } } err := sws.isWatchPermitted(creq) if err != nil { var cancelReason string switch { case errors.Is(err, auth.ErrInvalidAuthToken): cancelReason = rpctypes.ErrGRPCInvalidAuthToken.Error() case errors.Is(err, auth.ErrAuthOldRevision): cancelReason = rpctypes.ErrGRPCAuthOldRevision.Error() case errors.Is(err, auth.ErrUserEmpty): cancelReason = rpctypes.ErrGRPCUserEmpty.Error() default: if !errors.Is(err, auth.ErrPermissionDenied) { sws.lg.Error("unexpected error code", zap.Error(err)) } cancelReason = rpctypes.ErrGRPCPermissionDenied.Error() } wr := &pb.WatchResponse{ Header: sws.newResponseHeader(sws.watchStream.Rev()), WatchId: clientv3.InvalidWatchID, Canceled: true, Created: true, CancelReason: cancelReason, } select { case sws.ctrlStream <- wr: continue case <-sws.closec: return nil } } filters := FiltersFromRequest(creq) ctx, _ := traceutil.Tracer.Start(sws.gRPCStream.Context(), "watch", trace.WithAttributes( attribute.String("key", string(creq.Key)), attribute.String("range_end", string(creq.RangeEnd)), attribute.Int64("start_rev", creq.StartRevision), attribute.Bool("progress_notify", creq.ProgressNotify), attribute.Bool("prev_kv", creq.PrevKv), attribute.Bool("fragment", creq.Fragment), )) id, err := sws.watchStream.Watch(ctx, mvcc.WatchID(creq.WatchId), creq.Key, creq.RangeEnd, creq.StartRevision, filters...) if err == nil { sws.mu.Lock() if creq.ProgressNotify { sws.progress[id] = true } if creq.PrevKv { sws.prevKV[id] = true } if creq.Fragment { sws.fragment[id] = true } sws.mu.Unlock() } else { id = clientv3.InvalidWatchID } wr := &pb.WatchResponse{ Header: sws.newResponseHeader(sws.watchStream.Rev()), WatchId: int64(id), Created: true, Canceled: err != nil, } if err != nil { wr.CancelReason = err.Error() } select { case sws.ctrlStream <- wr: case <-sws.closec: return nil } case *pb.WatchRequest_CancelRequest: if uv.CancelRequest != nil { id := uv.CancelRequest.WatchId err := sws.watchStream.Cancel(mvcc.WatchID(id)) if err == nil { wr := &pb.WatchResponse{ Header: sws.newResponseHeader(sws.watchStream.Rev()), WatchId: id, Canceled: true, } select { case sws.ctrlStream <- wr: case <-sws.closec: return nil } sws.mu.Lock() delete(sws.progress, mvcc.WatchID(id)) delete(sws.prevKV, mvcc.WatchID(id)) delete(sws.fragment, mvcc.WatchID(id)) sws.mu.Unlock() } } case *pb.WatchRequest_ProgressRequest: if uv.ProgressRequest != nil { sws.mu.Lock() sws.watchStream.RequestProgressAll() sws.mu.Unlock() } default: // we probably should not shutdown the entire stream when // receive an invalid command. // so just do nothing instead. sws.lg.Sugar().Infof("invalid watch request type %T received in gRPC stream", uv) continue } } } func (sws *serverWatchStream) sendLoop() { // watch ids that are currently active ids := make(map[mvcc.WatchID]struct{}) // watch responses pending on a watch id creation message pending := make(map[mvcc.WatchID][]*pb.WatchResponse) interval := GetProgressReportInterval() progressTicker := time.NewTicker(interval) defer func() { progressTicker.Stop() // drain the chan to clean up pending events for ws := range sws.watchStream.Chan() { mvcc.ReportEventReceived(len(ws.Events)) } for _, wrs := range pending { for _, ws := range wrs { mvcc.ReportEventReceived(len(ws.Events)) } } }() for { select { case wresp, ok := <-sws.watchStream.Chan(): if !ok { return } start := time.Now() // TODO: evs is []mvccpb.Event type // either return []*mvccpb.Event from the mvcc package // or define protocol buffer with []mvccpb.Event. evs := wresp.Events events := make([]*mvccpb.Event, len(evs)) sws.mu.RLock() needPrevKV := sws.prevKV[wresp.WatchID] sws.mu.RUnlock() for i := range evs { events[i] = &evs[i] if needPrevKV && !IsCreateEvent(evs[i]) { opt := mvcc.RangeOptions{Rev: evs[i].Kv.ModRevision - 1} r, err := sws.watchable.Range(context.TODO(), evs[i].Kv.Key, nil, opt) if err == nil && len(r.KVs) != 0 { events[i].PrevKv = &(r.KVs[0]) } } } canceled := wresp.CompactRevision != 0 wr := &pb.WatchResponse{ Header: sws.newResponseHeader(wresp.Revision), WatchId: int64(wresp.WatchID), Events: events, CompactRevision: wresp.CompactRevision, Canceled: canceled, } // Progress notifications can have WatchID -1 // if they announce on behalf of multiple watchers if wresp.WatchID != clientv3.InvalidWatchID { if _, okID := ids[wresp.WatchID]; !okID { // buffer if id not yet announced wrs := append(pending[wresp.WatchID], wr) pending[wresp.WatchID] = wrs continue } } mvcc.ReportEventReceived(len(evs)) sws.mu.RLock() fragmented, ok := sws.fragment[wresp.WatchID] sws.mu.RUnlock() var serr error // gofail: var beforeSendWatchResponse struct{} if !fragmented && !ok { serr = sws.gRPCStream.Send(wr) } else { serr = sendFragments(wr, sws.maxRequestBytes, sws.gRPCStream.Send) } if serr != nil { if isClientCtxErr(sws.gRPCStream.Context().Err(), serr) { sws.lg.Debug("failed to send watch response to gRPC stream", zap.Error(serr)) } else { sws.lg.Warn("failed to send watch response to gRPC stream", zap.Error(serr)) streamFailures.WithLabelValues("send", "watch").Inc() } return } sws.mu.Lock() if len(evs) > 0 && sws.progress[wresp.WatchID] { // elide next progress update if sent a key update sws.progress[wresp.WatchID] = false } sws.mu.Unlock() totalDur := time.Since(start) watchSendLoopWatchStreamDuration.Observe(totalDur.Seconds()) watchSendLoopWatchStreamDurationPerEvent.Observe(totalDur.Seconds() / float64(len(evs))) case c, ok := <-sws.ctrlStream: if !ok { return } start := time.Now() if err := sws.gRPCStream.Send(c); err != nil { if isClientCtxErr(sws.gRPCStream.Context().Err(), err) { sws.lg.Debug("failed to send watch control response to gRPC stream", zap.Error(err)) } else { sws.lg.Warn("failed to send watch control response to gRPC stream", zap.Error(err)) streamFailures.WithLabelValues("send", "watch").Inc() } return } // track id creation wid := mvcc.WatchID(c.WatchId) verify.Assert(!(c.Canceled && c.Created) || wid == clientv3.InvalidWatchID, "unexpected watchId: %d, wanted: %d, since both 'Canceled' and 'Created' are true", wid, clientv3.InvalidWatchID) if c.Canceled && wid != clientv3.InvalidWatchID { delete(ids, wid) continue } if c.Created { // flush buffered events ids[wid] = struct{}{} for _, v := range pending[wid] { mvcc.ReportEventReceived(len(v.Events)) if err := sws.gRPCStream.Send(v); err != nil { if isClientCtxErr(sws.gRPCStream.Context().Err(), err) { sws.lg.Debug("failed to send pending watch response to gRPC stream", zap.Error(err)) } else { sws.lg.Warn("failed to send pending watch response to gRPC stream", zap.Error(err)) streamFailures.WithLabelValues("send", "watch").Inc() } return } } delete(pending, wid) } watchSendLoopControlStreamDuration.Observe(time.Since(start).Seconds()) case <-progressTicker.C: start := time.Now() sws.mu.Lock() for id, ok := range sws.progress { if ok { sws.watchStream.RequestProgress(id) } sws.progress[id] = true } sws.mu.Unlock() watchSendLoopProgressDuration.Observe(time.Since(start).Seconds()) case <-sws.closec: return } } } func IsCreateEvent(e mvccpb.Event) bool { return e.Type == mvccpb.Event_PUT && e.Kv.CreateRevision == e.Kv.ModRevision } func sendFragments( wr *pb.WatchResponse, maxRequestBytes uint, sendFunc func(*pb.WatchResponse) error, ) error { // no need to fragment if total request size is smaller // than max request limit or response contains only one event if uint(wr.Size()) < maxRequestBytes || len(wr.Events) < 2 { return sendFunc(wr) } ow := *wr ow.Events = make([]*mvccpb.Event, 0) ow.Fragment = true var idx int for { cur := ow for _, ev := range wr.Events[idx:] { cur.Events = append(cur.Events, ev) if len(cur.Events) > 1 && uint(cur.Size()) >= maxRequestBytes { cur.Events = cur.Events[:len(cur.Events)-1] break } idx++ } if idx == len(wr.Events) { // last response has no more fragment cur.Fragment = false } if err := sendFunc(&cur); err != nil { return err } if !cur.Fragment { break } } return nil } func (sws *serverWatchStream) close() { sws.watchStream.Close() close(sws.closec) sws.wg.Wait() } func (sws *serverWatchStream) newResponseHeader(rev int64) *pb.ResponseHeader { return &pb.ResponseHeader{ ClusterId: uint64(sws.clusterID), MemberId: uint64(sws.memberID), Revision: rev, RaftTerm: sws.sg.Term(), } } func filterNoDelete(e mvccpb.Event) bool { return e.Type == mvccpb.Event_DELETE } func filterNoPut(e mvccpb.Event) bool { return e.Type == mvccpb.Event_PUT } // FiltersFromRequest returns "mvcc.FilterFunc" from a given watch create request. func FiltersFromRequest(creq *pb.WatchCreateRequest) []mvcc.FilterFunc { filters := make([]mvcc.FilterFunc, 0, len(creq.Filters)) for _, ft := range creq.Filters { switch ft { case pb.WatchCreateRequest_NOPUT: filters = append(filters, filterNoPut) case pb.WatchCreateRequest_NODELETE: filters = append(filters, filterNoDelete) default: } } return filters } ================================================ FILE: server/etcdserver/api/v3rpc/watch_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v3rpc import ( "bytes" "errors" "math" "testing" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" ) func TestSendFragment(t *testing.T) { tt := []struct { wr *pb.WatchResponse maxRequestBytes uint fragments int werr error }{ { // large limit should not fragment wr: createResponse(100, 1), maxRequestBytes: math.MaxInt32, fragments: 1, }, { // large limit for two messages, expect no fragment wr: createResponse(10, 2), maxRequestBytes: 50, fragments: 1, }, { // limit is small but only one message, expect no fragment wr: createResponse(1024, 1), maxRequestBytes: 1, fragments: 1, }, { // exceed limit only when combined, expect fragments wr: createResponse(11, 5), maxRequestBytes: 20, fragments: 5, }, { // 5 events with each event exceeding limits, expect fragments wr: createResponse(15, 5), maxRequestBytes: 10, fragments: 5, }, { // 4 events with some combined events exceeding limits wr: createResponse(10, 4), maxRequestBytes: 35, fragments: 2, }, } for i := range tt { fragmentedResp := make([]*pb.WatchResponse, 0) testSend := func(wr *pb.WatchResponse) error { fragmentedResp = append(fragmentedResp, wr) return nil } err := sendFragments(tt[i].wr, tt[i].maxRequestBytes, testSend) if !errors.Is(err, tt[i].werr) { t.Errorf("#%d: expected error %v, got %v", i, tt[i].werr, err) } got := len(fragmentedResp) if got != tt[i].fragments { t.Errorf("#%d: expected response number %d, got %d", i, tt[i].fragments, got) } if got > 0 && fragmentedResp[got-1].Fragment { t.Errorf("#%d: expected fragment=false in last response, got %+v", i, fragmentedResp[got-1]) } } } func createResponse(dataSize, events int) (resp *pb.WatchResponse) { resp = &pb.WatchResponse{Events: make([]*mvccpb.Event, events)} for i := range resp.Events { resp.Events[i] = &mvccpb.Event{ Kv: &mvccpb.KeyValue{ Key: bytes.Repeat([]byte("a"), dataSize), }, } } return resp } ================================================ FILE: server/etcdserver/apply/apply.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package apply import ( "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/pkg/v3/wait" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/raft/v3/raftpb" ) func Apply(lg *zap.Logger, e *raftpb.Entry, uberApply UberApplier, w wait.Wait, shouldApplyV3 membership.ShouldApplyV3) (ar *Result, id uint64) { var raftReq pb.InternalRaftRequest pbutil.MustUnmarshal(&raftReq, e.Data) lg.Debug("Apply", zap.Stringer("raftReq", &raftReq)) id = raftReq.ID if id == 0 { if raftReq.Header == nil { lg.Panic("Apply, could not find a header") } id = raftReq.Header.ID } needResult := w.IsRegistered(id) if needResult || !noSideEffect(&raftReq) { if !needResult && raftReq.Txn != nil { removeNeedlessRangeReqs(raftReq.Txn) } return uberApply.Apply(&raftReq, shouldApplyV3), id } return nil, id } func noSideEffect(r *pb.InternalRaftRequest) bool { return r.Range != nil || r.AuthUserGet != nil || r.AuthRoleGet != nil || r.AuthStatus != nil } func removeNeedlessRangeReqs(txn *pb.TxnRequest) { f := func(ops []*pb.RequestOp) []*pb.RequestOp { j := 0 for i := 0; i < len(ops); i++ { if _, ok := ops[i].Request.(*pb.RequestOp_RequestRange); ok { continue } ops[j] = ops[i] j++ } return ops[:j] } txn.Success = f(txn.Success) txn.Failure = f(txn.Failure) } ================================================ FILE: server/etcdserver/apply/auth.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package apply import ( "sync" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/txn" "go.etcd.io/etcd/server/v3/lease" ) type authApplierV3 struct { applierV3 as auth.AuthStore lessor lease.Lessor // mu serializes Apply so that user isn't corrupted and so that // serialized requests don't leak data from TOCTOU errors mu sync.Mutex authInfo auth.AuthInfo } func newAuthApplierV3(as auth.AuthStore, base applierV3, lessor lease.Lessor) *authApplierV3 { return &authApplierV3{applierV3: base, as: as, lessor: lessor} } func (aa *authApplierV3) Apply(r *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3, applyFunc applyFunc) *Result { aa.mu.Lock() defer aa.mu.Unlock() if r.Header != nil { // backward-compatible with pre-3.0 releases when internalRaftRequest // does not have header field aa.authInfo.Username = r.Header.Username aa.authInfo.Revision = r.Header.AuthRevision } if needAdminPermission(r) { if err := aa.as.IsAdminPermitted(&aa.authInfo); err != nil { aa.authInfo.Username = "" aa.authInfo.Revision = 0 return &Result{Err: err} } } ret := aa.applierV3.Apply(r, shouldApplyV3, applyFunc) aa.authInfo.Username = "" aa.authInfo.Revision = 0 return ret } func (aa *authApplierV3) Put(r *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) { if err := aa.as.IsPutPermitted(&aa.authInfo, r.Key); err != nil { return nil, nil, err } if err := aa.checkLeasePuts(lease.LeaseID(r.Lease)); err != nil { // The specified lease is already attached with a key that cannot // be written by this user. It means the user cannot revoke the // lease so attaching the lease to the newly written key should // be forbidden. return nil, nil, err } if r.PrevKv { err := aa.as.IsRangePermitted(&aa.authInfo, r.Key, nil) if err != nil { return nil, nil, err } } return aa.applierV3.Put(r) } func (aa *authApplierV3) Range(r *pb.RangeRequest) (*pb.RangeResponse, *traceutil.Trace, error) { if err := aa.as.IsRangePermitted(&aa.authInfo, r.Key, r.RangeEnd); err != nil { return nil, nil, err } return aa.applierV3.Range(r) } func (aa *authApplierV3) DeleteRange(r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, *traceutil.Trace, error) { if err := aa.as.IsDeleteRangePermitted(&aa.authInfo, r.Key, r.RangeEnd); err != nil { return nil, nil, err } if r.PrevKv { err := aa.as.IsRangePermitted(&aa.authInfo, r.Key, r.RangeEnd) if err != nil { return nil, nil, err } } return aa.applierV3.DeleteRange(r) } func (aa *authApplierV3) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, *traceutil.Trace, error) { if err := txn.CheckTxnAuth(aa.as, &aa.authInfo, rt); err != nil { return nil, nil, err } return aa.applierV3.Txn(rt) } func (aa *authApplierV3) LeaseRevoke(lc *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) { if err := aa.checkLeasePuts(lease.LeaseID(lc.ID)); err != nil { return nil, err } return aa.applierV3.LeaseRevoke(lc) } func (aa *authApplierV3) checkLeasePuts(leaseID lease.LeaseID) error { l := aa.lessor.Lookup(leaseID) if l != nil { return aa.checkLeasePutsKeys(l) } return nil } func (aa *authApplierV3) checkLeasePutsKeys(l *lease.Lease) error { // early return for most-common scenario of either disabled auth or admin user. // IsAdminPermitted also checks whether auth is enabled if err := aa.as.IsAdminPermitted(&aa.authInfo); err == nil { return nil } for _, key := range l.Keys() { if err := aa.as.IsPutPermitted(&aa.authInfo, []byte(key)); err != nil { return err } } return nil } func (aa *authApplierV3) UserGet(r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) { err := aa.as.IsAdminPermitted(&aa.authInfo) if err != nil && r.Name != aa.authInfo.Username { aa.authInfo.Username = "" aa.authInfo.Revision = 0 return &pb.AuthUserGetResponse{}, err } return aa.applierV3.UserGet(r) } func (aa *authApplierV3) RoleGet(r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) { err := aa.as.IsAdminPermitted(&aa.authInfo) if err != nil && !aa.as.HasRole(aa.authInfo.Username, r.Role) { aa.authInfo.Username = "" aa.authInfo.Revision = 0 return &pb.AuthRoleGetResponse{}, err } return aa.applierV3.RoleGet(r) } func needAdminPermission(r *pb.InternalRaftRequest) bool { switch { case r.AuthEnable != nil: return true case r.AuthDisable != nil: return true case r.AuthUserAdd != nil: return true case r.AuthUserDelete != nil: return true case r.AuthUserChangePassword != nil: return true case r.AuthUserGrantRole != nil: return true case r.AuthUserRevokeRole != nil: return true case r.AuthRoleAdd != nil: return true case r.AuthRoleGrantPermission != nil: return true case r.AuthRoleRevokePermission != nil: return true case r.AuthRoleDelete != nil: return true case r.AuthUserList != nil: return true case r.AuthRoleList != nil: return true default: return false } } ================================================ FILE: server/etcdserver/apply/auth_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package apply import ( "errors" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "golang.org/x/crypto/bcrypt" "go.etcd.io/etcd/api/v3/authpb" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/membershippb" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/v3alarm" "go.etcd.io/etcd/server/v3/etcdserver/cindex" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" ) func dummyIndexWaiter(_ uint64) <-chan struct{} { ch := make(chan struct{}, 1) ch <- struct{}{} return ch } func dummyApplyFunc(_ *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3) *Result { return &Result{} } type fakeRaftStatusGetter struct{} func (*fakeRaftStatusGetter) MemberID() types.ID { return 0 } func (*fakeRaftStatusGetter) Leader() types.ID { return 0 } func (*fakeRaftStatusGetter) CommittedIndex() uint64 { return 0 } func (*fakeRaftStatusGetter) AppliedIndex() uint64 { return 0 } func (*fakeRaftStatusGetter) Term() uint64 { return 0 } type fakeSnapshotServer struct{} func (*fakeSnapshotServer) ForceSnapshot() {} func defaultAuthApplierV3(t *testing.T) *authApplierV3 { lg := zaptest.NewLogger(t) be, _ := betesting.NewDefaultTmpBackend(t) t.Cleanup(func() { betesting.Close(t, be) }) cluster := membership.NewCluster(lg) lessor := lease.NewLessor(lg, be, cluster, lease.LessorConfig{}) kv := mvcc.NewStore(lg, be, lessor, mvcc.StoreConfig{}) alarmStore, err := v3alarm.NewAlarmStore(lg, schema.NewAlarmBackend(lg, be)) require.NoError(t, err) tp, err := auth.NewTokenProvider(lg, "simple", dummyIndexWaiter, 300*time.Second) require.NoError(t, err) authStore := auth.NewAuthStore( lg, schema.NewAuthBackend(lg, be), tp, bcrypt.DefaultCost, ) consistentIndex := cindex.NewConsistentIndex(be) return newAuthApplierV3( authStore, newApplierV3Backend(ApplierOptions{ Logger: lg, KV: kv, AlarmStore: alarmStore, ConsistentIndex: consistentIndex, AuthStore: authStore, Lessor: lessor, Cluster: cluster, RaftStatus: &fakeRaftStatusGetter{}, SnapshotServer: &fakeSnapshotServer{}, TxnModeWriteWithSharedBuffer: false, }), lessor) } const ( userRoot = "root" roleRoot = "root" userReadOnly = "user_read_only" roleReadOnly = "role_read_only" userWriteOnly = "user_write_only" roleWriteOnly = "role_write_only" key = "key" rangeEnd = "rangeEnd" keyOutsideRange = "rangeEnd_outside" leaseID = 1 ) func mustCreateRolesAndEnableAuth(t *testing.T, authApplier *authApplierV3) { _, err := authApplier.UserAdd(&pb.AuthUserAddRequest{Name: userRoot, Options: &authpb.UserAddOptions{NoPassword: true}}) require.NoError(t, err) _, err = authApplier.RoleAdd(&pb.AuthRoleAddRequest{Name: roleRoot}) require.NoError(t, err) _, err = authApplier.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: userRoot, Role: roleRoot}) require.NoError(t, err) _, err = authApplier.UserAdd(&pb.AuthUserAddRequest{Name: userReadOnly, Options: &authpb.UserAddOptions{NoPassword: true}}) require.NoError(t, err) _, err = authApplier.RoleAdd(&pb.AuthRoleAddRequest{Name: roleReadOnly}) require.NoError(t, err) _, err = authApplier.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: userReadOnly, Role: roleReadOnly}) require.NoError(t, err) _, err = authApplier.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{Name: roleReadOnly, Perm: &authpb.Permission{ PermType: authpb.Permission_READ, Key: []byte(key), RangeEnd: []byte(rangeEnd), }}) require.NoError(t, err) _, err = authApplier.UserAdd(&pb.AuthUserAddRequest{Name: userWriteOnly, Options: &authpb.UserAddOptions{NoPassword: true}}) require.NoError(t, err) _, err = authApplier.RoleAdd(&pb.AuthRoleAddRequest{Name: roleWriteOnly}) require.NoError(t, err) _, err = authApplier.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: userWriteOnly, Role: roleWriteOnly}) require.NoError(t, err) _, err = authApplier.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{Name: roleWriteOnly, Perm: &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte(key), RangeEnd: []byte(rangeEnd), }}) require.NoError(t, err) _, err = authApplier.AuthEnable() require.NoError(t, err) } // setAuthInfo manually sets the authInfo of the applier. In reality, authInfo is filled before Apply() func setAuthInfo(authApplier *authApplierV3, userName string) { authApplier.authInfo = auth.AuthInfo{ Username: userName, Revision: authApplier.as.Revision(), } } // TestAuthApplierV3_Apply ensures Apply() calls applyFunc() when permission is granted // and returns an error when permission is denied func TestAuthApplierV3_Apply(t *testing.T) { tcs := []struct { name string request *pb.InternalRaftRequest expectResult *Result }{ { name: "request does not need admin permission", request: &pb.InternalRaftRequest{ Header: &pb.RequestHeader{}, }, expectResult: &Result{}, }, { name: "request needs admin permission but permission denied", request: &pb.InternalRaftRequest{ Header: &pb.RequestHeader{ Username: userReadOnly, }, AuthEnable: &pb.AuthEnableRequest{}, }, expectResult: &Result{ Err: auth.ErrPermissionDenied, }, }, { name: "request needs admin permission and permitted", request: &pb.InternalRaftRequest{ Header: &pb.RequestHeader{ Username: userRoot, }, AuthEnable: &pb.AuthEnableRequest{}, }, expectResult: &Result{}, }, } authApplier := defaultAuthApplierV3(t) mustCreateRolesAndEnableAuth(t, authApplier) for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { result := authApplier.Apply(tc.request, membership.ApplyBoth, dummyApplyFunc) require.Equalf(t, result, tc.expectResult, "Apply: got %v, expect: %v", result, tc.expectResult) }) } } // TestAuthApplierV3_AdminPermission ensures the admin permission is checked for certain // operations func TestAuthApplierV3_AdminPermission(t *testing.T) { tcs := []struct { name string request *pb.InternalRaftRequest adminPermissionNeeded bool }{ { name: "Range does not need admin permission", request: &pb.InternalRaftRequest{Range: &pb.RangeRequest{}}, adminPermissionNeeded: false, }, { name: "Put does not need admin permission", request: &pb.InternalRaftRequest{Put: &pb.PutRequest{}}, adminPermissionNeeded: false, }, { name: "DeleteRange does not need admin permission", request: &pb.InternalRaftRequest{DeleteRange: &pb.DeleteRangeRequest{}}, adminPermissionNeeded: false, }, { name: "Txn does not need admin permission", request: &pb.InternalRaftRequest{Txn: &pb.TxnRequest{}}, adminPermissionNeeded: false, }, { name: "Compaction does not need admin permission", request: &pb.InternalRaftRequest{Compaction: &pb.CompactionRequest{}}, adminPermissionNeeded: false, }, { name: "LeaseGrant does not need admin permission", request: &pb.InternalRaftRequest{LeaseGrant: &pb.LeaseGrantRequest{}}, adminPermissionNeeded: false, }, { name: "LeaseRevoke does not need admin permission", request: &pb.InternalRaftRequest{LeaseRevoke: &pb.LeaseRevokeRequest{}}, adminPermissionNeeded: false, }, { name: "Alarm does not need admin permission", request: &pb.InternalRaftRequest{Alarm: &pb.AlarmRequest{}}, adminPermissionNeeded: false, }, { name: "LeaseCheckpoint does not need admin permission", request: &pb.InternalRaftRequest{LeaseCheckpoint: &pb.LeaseCheckpointRequest{}}, adminPermissionNeeded: false, }, { name: "Authenticate does not need admin permission", request: &pb.InternalRaftRequest{Authenticate: &pb.InternalAuthenticateRequest{}}, adminPermissionNeeded: false, }, { name: "ClusterVersionSet does not need admin permission", request: &pb.InternalRaftRequest{ClusterVersionSet: &membershippb.ClusterVersionSetRequest{}}, adminPermissionNeeded: false, }, { name: "ClusterMemberAttrSet does not need admin permission", request: &pb.InternalRaftRequest{ClusterMemberAttrSet: &membershippb.ClusterMemberAttrSetRequest{}}, adminPermissionNeeded: false, }, { name: "DowngradeInfoSet does not need admin permission", request: &pb.InternalRaftRequest{DowngradeInfoSet: &membershippb.DowngradeInfoSetRequest{}}, adminPermissionNeeded: false, }, { name: "AuthUserGet does not need admin permission", request: &pb.InternalRaftRequest{AuthUserGet: &pb.AuthUserGetRequest{}}, adminPermissionNeeded: false, }, { name: "AuthRoleGet does not need admin permission", request: &pb.InternalRaftRequest{AuthRoleGet: &pb.AuthRoleGetRequest{}}, adminPermissionNeeded: false, }, { name: "AuthEnable needs admin permission", request: &pb.InternalRaftRequest{AuthEnable: &pb.AuthEnableRequest{}}, adminPermissionNeeded: true, }, { name: "AuthDisable needs admin permission", request: &pb.InternalRaftRequest{AuthDisable: &pb.AuthDisableRequest{}}, adminPermissionNeeded: true, }, { name: "AuthUserAdd needs admin permission", request: &pb.InternalRaftRequest{AuthUserAdd: &pb.AuthUserAddRequest{}}, adminPermissionNeeded: true, }, { name: "AuthUserDelete needs admin permission", request: &pb.InternalRaftRequest{AuthUserDelete: &pb.AuthUserDeleteRequest{}}, adminPermissionNeeded: true, }, { name: "AuthUserChangePassword needs admin permission", request: &pb.InternalRaftRequest{AuthUserChangePassword: &pb.AuthUserChangePasswordRequest{}}, adminPermissionNeeded: true, }, { name: "AuthUserGrantRole needs admin permission", request: &pb.InternalRaftRequest{AuthUserGrantRole: &pb.AuthUserGrantRoleRequest{}}, adminPermissionNeeded: true, }, { name: "AuthUserRevokeRole needs admin permission", request: &pb.InternalRaftRequest{AuthUserRevokeRole: &pb.AuthUserRevokeRoleRequest{}}, adminPermissionNeeded: true, }, { name: "AuthUserList needs admin permission", request: &pb.InternalRaftRequest{AuthUserList: &pb.AuthUserListRequest{}}, adminPermissionNeeded: true, }, { name: "AuthRoleList needs admin permission", request: &pb.InternalRaftRequest{AuthRoleList: &pb.AuthRoleListRequest{}}, adminPermissionNeeded: true, }, { name: "AuthRoleAdd needs admin permission", request: &pb.InternalRaftRequest{AuthRoleAdd: &pb.AuthRoleAddRequest{}}, adminPermissionNeeded: true, }, { name: "AuthRoleDelete needs admin permission", request: &pb.InternalRaftRequest{AuthRoleDelete: &pb.AuthRoleDeleteRequest{}}, adminPermissionNeeded: true, }, { name: "AuthRoleGrantPermission needs admin permission", request: &pb.InternalRaftRequest{AuthRoleGrantPermission: &pb.AuthRoleGrantPermissionRequest{}}, adminPermissionNeeded: true, }, { name: "AuthRoleRevokePermission needs admin permission", request: &pb.InternalRaftRequest{AuthRoleRevokePermission: &pb.AuthRoleRevokePermissionRequest{}}, adminPermissionNeeded: true, }, } authApplier := defaultAuthApplierV3(t) mustCreateRolesAndEnableAuth(t, authApplier) for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { if tc.adminPermissionNeeded { tc.request.Header = &pb.RequestHeader{Username: userReadOnly} } result := authApplier.Apply(tc.request, membership.ApplyBoth, dummyApplyFunc) require.Equalf(t, errors.Is(result.Err, auth.ErrPermissionDenied), tc.adminPermissionNeeded, "Admin permission needed") }) } } // TestAuthApplierV3_Put verifies only users with write permissions in the key range can put func TestAuthApplierV3_Put(t *testing.T) { tcs := []struct { name string userName string request *pb.PutRequest expectError error }{ { name: "put permission denied", userName: userReadOnly, request: &pb.PutRequest{}, expectError: auth.ErrPermissionDenied, }, { name: "prevKv is set, but user does not have read permission", userName: userWriteOnly, request: &pb.PutRequest{ Key: []byte(key), Value: []byte("1"), PrevKv: true, }, expectError: auth.ErrPermissionDenied, }, { name: "put success", userName: userWriteOnly, request: &pb.PutRequest{ Key: []byte(key), Value: []byte("1"), }, expectError: nil, }, { name: "put success with PrevKv set", userName: userRoot, request: &pb.PutRequest{ Key: []byte(key), Value: []byte("1"), PrevKv: true, }, expectError: nil, }, } authApplier := defaultAuthApplierV3(t) mustCreateRolesAndEnableAuth(t, authApplier) for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { setAuthInfo(authApplier, tc.userName) _, _, err := authApplier.Put(tc.request) require.Equalf(t, tc.expectError, err, "Put returned unexpected error (or lack thereof), expected: %v, got: %v", tc.expectError, err) }) } } // TestAuthApplierV3_LeasePut verifies users cannot put with lease if the lease is attached with a key out of range func TestAuthApplierV3_LeasePut(t *testing.T) { authApplier := defaultAuthApplierV3(t) mustCreateRolesAndEnableAuth(t, authApplier) _, err := authApplier.LeaseGrant(&pb.LeaseGrantRequest{ TTL: lease.MaxLeaseTTL, ID: leaseID, }) require.NoError(t, err) // The user should be able to put the key setAuthInfo(authApplier, userWriteOnly) _, _, err = authApplier.Put(&pb.PutRequest{ Key: []byte(key), Value: []byte("1"), Lease: leaseID, }) require.NoError(t, err) // Put a key under the lease outside user's key range setAuthInfo(authApplier, userRoot) _, _, err = authApplier.Put(&pb.PutRequest{ Key: []byte(keyOutsideRange), Value: []byte("1"), Lease: leaseID, }) require.NoError(t, err) // The user should not be able to put the key anymore setAuthInfo(authApplier, userWriteOnly) _, _, err = authApplier.Put(&pb.PutRequest{ Key: []byte(key), Value: []byte("1"), Lease: leaseID, }) require.Equal(t, err, auth.ErrPermissionDenied) } // TestAuthApplierV3_Range verifies only users with read permissions can do range in the key range func TestAuthApplierV3_Range(t *testing.T) { tcs := []struct { name string userName string request *pb.RangeRequest expectError error }{ { name: "range permission denied", userName: userWriteOnly, request: &pb.RangeRequest{}, expectError: auth.ErrPermissionDenied, }, { name: "range key out of range", userName: userReadOnly, request: &pb.RangeRequest{ Key: []byte(keyOutsideRange), }, expectError: auth.ErrPermissionDenied, }, { name: "range success", userName: userReadOnly, request: &pb.RangeRequest{ Key: []byte(key), RangeEnd: []byte(rangeEnd), }, expectError: nil, }, } authApplier := defaultAuthApplierV3(t) mustCreateRolesAndEnableAuth(t, authApplier) for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { setAuthInfo(authApplier, tc.userName) _, _, err := authApplier.Range(tc.request) require.Equalf(t, tc.expectError, err, "Range returned unexpected error (or lack thereof), expected: %v, got: %v", tc.expectError, err) }) } } // TestAuthApplierV3_DeleteRange verifies only users with write permissions can do delete range in the key range func TestAuthApplierV3_DeleteRange(t *testing.T) { tcs := []struct { name string userName string request *pb.DeleteRangeRequest expectError error }{ { name: "delete range permission denied", userName: userReadOnly, request: &pb.DeleteRangeRequest{}, expectError: auth.ErrPermissionDenied, }, { name: "delete range key out of range", userName: userWriteOnly, request: &pb.DeleteRangeRequest{ Key: []byte(keyOutsideRange), }, expectError: auth.ErrPermissionDenied, }, { name: "prevKv is set, but user does not have read permission", userName: userWriteOnly, request: &pb.DeleteRangeRequest{ Key: []byte(key), RangeEnd: []byte(rangeEnd), PrevKv: true, }, expectError: auth.ErrPermissionDenied, }, { name: "delete range success", userName: userWriteOnly, request: &pb.DeleteRangeRequest{ Key: []byte(key), RangeEnd: []byte(rangeEnd), }, expectError: nil, }, { name: "delete range success with PrevKv", userName: userRoot, request: &pb.DeleteRangeRequest{ Key: []byte(key), RangeEnd: []byte(rangeEnd), PrevKv: true, }, expectError: nil, }, } authApplier := defaultAuthApplierV3(t) mustCreateRolesAndEnableAuth(t, authApplier) for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { setAuthInfo(authApplier, tc.userName) _, _, err := authApplier.DeleteRange(tc.request) require.Equalf(t, tc.expectError, err, "Range returned unexpected error (or lack thereof), expected: %v, got: %v", tc.expectError, err) }) } } // TestAuthApplierV3_Txn verifies txns can only be applied with proper permissions func TestAuthApplierV3_Txn(t *testing.T) { tcs := []struct { name string userName string request *pb.TxnRequest expectError error }{ { name: "txn range permission denied", userName: userWriteOnly, request: &pb.TxnRequest{ Compare: []*pb.Compare{ { Key: []byte(key), }, }, }, expectError: auth.ErrPermissionDenied, }, { name: "txn put permission denied", userName: userReadOnly, request: &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Key: []byte(key), }, }, }, }, }, expectError: auth.ErrPermissionDenied, }, { name: "txn success", userName: userRoot, request: &pb.TxnRequest{ Compare: []*pb.Compare{ { Key: []byte(key), }, }, Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Key: []byte(key), }, }, }, }, }, expectError: nil, }, } authApplier := defaultAuthApplierV3(t) mustCreateRolesAndEnableAuth(t, authApplier) for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { setAuthInfo(authApplier, tc.userName) _, _, err := authApplier.Txn(tc.request) require.Equalf(t, tc.expectError, err, "Range returned unexpected error (or lack thereof), expected: %v, got: %v", tc.expectError, err) }) } } // TestAuthApplierV3_LeaseRevoke verifies user cannot revoke a lease if the lease is attached with // a key out of range by someone else func TestAuthApplierV3_LeaseRevoke(t *testing.T) { authApplier := defaultAuthApplierV3(t) mustCreateRolesAndEnableAuth(t, authApplier) _, err := authApplier.LeaseGrant(&pb.LeaseGrantRequest{ TTL: lease.MaxLeaseTTL, ID: leaseID, }) require.NoError(t, err) // The user should be able to revoke the lease setAuthInfo(authApplier, userWriteOnly) _, err = authApplier.LeaseRevoke(&pb.LeaseRevokeRequest{ ID: leaseID, }) require.NoError(t, err) _, err = authApplier.LeaseGrant(&pb.LeaseGrantRequest{ TTL: lease.MaxLeaseTTL, ID: leaseID, }) require.NoError(t, err) // Put a key under the lease outside user's key range setAuthInfo(authApplier, userRoot) _, _, err = authApplier.Put(&pb.PutRequest{ Key: []byte(keyOutsideRange), Value: []byte("1"), Lease: leaseID, }) require.NoError(t, err) // The user should not be able to revoke the lease anymore setAuthInfo(authApplier, userWriteOnly) _, err = authApplier.LeaseRevoke(&pb.LeaseRevokeRequest{ ID: leaseID, }) require.Equal(t, err, auth.ErrPermissionDenied) } // TestAuthApplierV3_UserGet verifies UserGet can only be performed by the user itself or the root func TestAuthApplierV3_UserGet(t *testing.T) { tcs := []struct { name string userName string request *pb.AuthUserGetRequest expectError error }{ { name: "UserGet permission denied with non-root role and requests other user", userName: userWriteOnly, request: &pb.AuthUserGetRequest{Name: userReadOnly}, expectError: auth.ErrPermissionDenied, }, { name: "UserGet success with non-root role but requests itself", userName: userWriteOnly, request: &pb.AuthUserGetRequest{Name: userWriteOnly}, expectError: nil, }, { name: "UserGet success with root role", userName: userRoot, request: &pb.AuthUserGetRequest{Name: userWriteOnly}, expectError: nil, }, } authApplier := defaultAuthApplierV3(t) mustCreateRolesAndEnableAuth(t, authApplier) for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { setAuthInfo(authApplier, tc.userName) _, err := authApplier.UserGet(tc.request) require.Equalf(t, tc.expectError, err, "Range returned unexpected error (or lack thereof), expected: %v, got: %v", tc.expectError, err) }) } } // TestAuthApplierV3_RoleGet verifies RoleGet can only be performed by the user in the role itself or the root func TestAuthApplierV3_RoleGet(t *testing.T) { tcs := []struct { name string userName string request *pb.AuthRoleGetRequest expectError error }{ { name: "RoleGet permission denied with non-root role and requests other role", userName: userWriteOnly, request: &pb.AuthRoleGetRequest{Role: roleReadOnly}, expectError: auth.ErrPermissionDenied, }, { name: "RoleGet success with non-root role but requests itself", userName: userWriteOnly, request: &pb.AuthRoleGetRequest{Role: roleWriteOnly}, expectError: nil, }, { name: "RoleGet success with root role", userName: userRoot, request: &pb.AuthRoleGetRequest{Role: roleWriteOnly}, expectError: nil, }, } authApplier := defaultAuthApplierV3(t) mustCreateRolesAndEnableAuth(t, authApplier) for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { setAuthInfo(authApplier, tc.userName) _, err := authApplier.RoleGet(tc.request) require.Equalf(t, tc.expectError, err, "Range returned unexpected error (or lack thereof), expected: %v, got: %v", tc.expectError, err) }) } } func TestCheckLeasePutsKeys(t *testing.T) { aa := defaultAuthApplierV3(t) require.NoErrorf(t, aa.checkLeasePutsKeys(lease.NewLease(lease.LeaseID(1), 3600)), "auth is disabled, should allow puts") mustCreateRolesAndEnableAuth(t, aa) aa.authInfo = auth.AuthInfo{Username: "root"} require.NoErrorf(t, aa.checkLeasePutsKeys(lease.NewLease(lease.LeaseID(1), 3600)), "auth is enabled, should allow puts for root") l := lease.NewLease(lease.LeaseID(1), 3600) l.SetLeaseItem(lease.LeaseItem{Key: "a"}) aa.authInfo = auth.AuthInfo{Username: "bob", Revision: 0} require.ErrorIsf(t, aa.checkLeasePutsKeys(l), auth.ErrUserEmpty, "auth is enabled, should not allow bob, non existing at rev 0") aa.authInfo = auth.AuthInfo{Username: "bob", Revision: 1} require.ErrorIsf(t, aa.checkLeasePutsKeys(l), auth.ErrAuthOldRevision, "auth is enabled, old revision") aa.authInfo = auth.AuthInfo{Username: "bob", Revision: aa.as.Revision()} require.ErrorIsf(t, aa.checkLeasePutsKeys(l), auth.ErrPermissionDenied, "auth is enabled, bob does not have permissions, bob does not exist") _, err := aa.as.UserAdd(&pb.AuthUserAddRequest{Name: "bob", Options: &authpb.UserAddOptions{NoPassword: true}}) require.NoErrorf(t, err, "bob should be added without error") aa.authInfo = auth.AuthInfo{Username: "bob", Revision: aa.as.Revision()} require.ErrorIsf(t, aa.checkLeasePutsKeys(l), auth.ErrPermissionDenied, "auth is enabled, bob exists yet does not have permissions") // allow bob to access "a" _, err = aa.as.RoleAdd(&pb.AuthRoleAddRequest{Name: "bobsrole"}) require.NoErrorf(t, err, "bobsrole should be added without error") _, err = aa.as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ Name: "bobsrole", Perm: &authpb.Permission{ PermType: authpb.Permission_READWRITE, Key: []byte("a"), RangeEnd: nil, }, }) require.NoErrorf(t, err, "bobsrole should be granted permissions without error") _, err = aa.as.UserGrantRole(&pb.AuthUserGrantRoleRequest{ User: "bob", Role: "bobsrole", }) require.NoErrorf(t, err, "bob should be granted bobsrole without error") aa.authInfo = auth.AuthInfo{Username: "bob", Revision: aa.as.Revision()} assert.NoErrorf(t, aa.checkLeasePutsKeys(l), "bob should be able to access key 'a'") } ================================================ FILE: server/etcdserver/apply/backend.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package apply import ( "context" "github.com/coreos/go-semver/semver" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/membershippb" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/etcdserver/api" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" mvcctxn "go.etcd.io/etcd/server/v3/etcdserver/txn" "go.etcd.io/etcd/server/v3/etcdserver/version" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/mvcc" ) type applierV3backend struct { options ApplierOptions } func newApplierV3Backend(opts ApplierOptions) applierV3 { return &applierV3backend{ options: opts, } } func (a *applierV3backend) Apply(r *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3, applyFunc applyFunc) *Result { return applyFunc(r, shouldApplyV3) } func (a *applierV3backend) Put(p *pb.PutRequest) (resp *pb.PutResponse, trace *traceutil.Trace, err error) { return mvcctxn.Put(context.TODO(), a.options.Logger, a.options.Lessor, a.options.KV, p) } func (a *applierV3backend) DeleteRange(dr *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, *traceutil.Trace, error) { return mvcctxn.DeleteRange(context.TODO(), a.options.Logger, a.options.KV, dr) } func (a *applierV3backend) Range(r *pb.RangeRequest) (*pb.RangeResponse, *traceutil.Trace, error) { return mvcctxn.Range(context.TODO(), a.options.Logger, a.options.KV, r) } func (a *applierV3backend) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, *traceutil.Trace, error) { return mvcctxn.Txn(context.TODO(), a.options.Logger, rt, a.options.TxnModeWriteWithSharedBuffer, a.options.KV, a.options.Lessor) } func (a *applierV3backend) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, *traceutil.Trace, error) { resp := &pb.CompactionResponse{} resp.Header = &pb.ResponseHeader{} ctx, trace := traceutil.EnsureTrace(context.TODO(), a.options.Logger, "compact", traceutil.Field{Key: "revision", Value: compaction.Revision}, ) ch, err := a.options.KV.Compact(trace, compaction.Revision) if err != nil { return nil, ch, nil, err } // get the current revision. which key to get is not important. rr, _ := a.options.KV.Range(ctx, []byte("compaction"), nil, mvcc.RangeOptions{}) resp.Header.Revision = rr.Rev return resp, ch, trace, err } func (a *applierV3backend) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { l, err := a.options.Lessor.Grant(lease.LeaseID(lc.ID), lc.TTL) resp := &pb.LeaseGrantResponse{} if err == nil { resp.ID = int64(l.ID) resp.TTL = l.TTL() resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) LeaseRevoke(lc *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) { err := a.options.Lessor.Revoke(lease.LeaseID(lc.ID)) return &pb.LeaseRevokeResponse{Header: a.newHeader()}, err } func (a *applierV3backend) LeaseCheckpoint(lc *pb.LeaseCheckpointRequest) (*pb.LeaseCheckpointResponse, error) { for _, c := range lc.Checkpoints { err := a.options.Lessor.Checkpoint(lease.LeaseID(c.ID), c.Remaining_TTL) if err != nil { return &pb.LeaseCheckpointResponse{Header: a.newHeader()}, err } } return &pb.LeaseCheckpointResponse{Header: a.newHeader()}, nil } func (a *applierV3backend) Alarm(ar *pb.AlarmRequest) (*pb.AlarmResponse, error) { resp := &pb.AlarmResponse{} switch ar.Action { case pb.AlarmRequest_GET: resp.Alarms = a.options.AlarmStore.Get(ar.Alarm) case pb.AlarmRequest_ACTIVATE: if ar.Alarm == pb.AlarmType_NONE { break } m := a.options.AlarmStore.Activate(types.ID(ar.MemberID), ar.Alarm) if m == nil { break } resp.Alarms = append(resp.Alarms, m) alarms.WithLabelValues(types.ID(ar.MemberID).String(), m.Alarm.String()).Inc() case pb.AlarmRequest_DEACTIVATE: m := a.options.AlarmStore.Deactivate(types.ID(ar.MemberID), ar.Alarm) if m == nil { break } resp.Alarms = append(resp.Alarms, m) alarms.WithLabelValues(types.ID(ar.MemberID).String(), m.Alarm.String()).Dec() default: return nil, nil } return resp, nil } func (a *applierV3backend) AuthEnable() (*pb.AuthEnableResponse, error) { err := a.options.AuthStore.AuthEnable() if err != nil { return nil, err } return &pb.AuthEnableResponse{Header: a.newHeader()}, nil } func (a *applierV3backend) AuthDisable() (*pb.AuthDisableResponse, error) { a.options.AuthStore.AuthDisable() return &pb.AuthDisableResponse{Header: a.newHeader()}, nil } func (a *applierV3backend) AuthStatus() (*pb.AuthStatusResponse, error) { enabled := a.options.AuthStore.IsAuthEnabled() authRevision := a.options.AuthStore.Revision() return &pb.AuthStatusResponse{Header: a.newHeader(), Enabled: enabled, AuthRevision: authRevision}, nil } func (a *applierV3backend) Authenticate(r *pb.InternalAuthenticateRequest) (*pb.AuthenticateResponse, error) { ctx := context.WithValue(context.WithValue(context.Background(), auth.AuthenticateParamIndex{}, a.options.ConsistentIndex.ConsistentIndex()), auth.AuthenticateParamSimpleTokenPrefix{}, r.SimpleToken) resp, err := a.options.AuthStore.Authenticate(ctx, r.Name, r.Password) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, error) { resp, err := a.options.AuthStore.UserAdd(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) UserDelete(r *pb.AuthUserDeleteRequest) (*pb.AuthUserDeleteResponse, error) { resp, err := a.options.AuthStore.UserDelete(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*pb.AuthUserChangePasswordResponse, error) { resp, err := a.options.AuthStore.UserChangePassword(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) UserGrantRole(r *pb.AuthUserGrantRoleRequest) (*pb.AuthUserGrantRoleResponse, error) { resp, err := a.options.AuthStore.UserGrantRole(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) UserGet(r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) { resp, err := a.options.AuthStore.UserGet(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) UserRevokeRole(r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUserRevokeRoleResponse, error) { resp, err := a.options.AuthStore.UserRevokeRole(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) RoleAdd(r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, error) { resp, err := a.options.AuthStore.RoleAdd(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) RoleGrantPermission(r *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) { resp, err := a.options.AuthStore.RoleGrantPermission(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) RoleGet(r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) { resp, err := a.options.AuthStore.RoleGet(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) RoleRevokePermission(r *pb.AuthRoleRevokePermissionRequest) (*pb.AuthRoleRevokePermissionResponse, error) { resp, err := a.options.AuthStore.RoleRevokePermission(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) RoleDelete(r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDeleteResponse, error) { resp, err := a.options.AuthStore.RoleDelete(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) UserList(r *pb.AuthUserListRequest) (*pb.AuthUserListResponse, error) { resp, err := a.options.AuthStore.UserList(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) RoleList(r *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error) { resp, err := a.options.AuthStore.RoleList(r) if resp != nil { resp.Header = a.newHeader() } return resp, err } func (a *applierV3backend) ClusterVersionSet(r *membershippb.ClusterVersionSetRequest, shouldApplyV3 membership.ShouldApplyV3) { prevVersion := a.options.Cluster.Version() newVersion := semver.Must(semver.NewVersion(r.Ver)) a.options.Cluster.SetVersion(newVersion, api.UpdateCapability, shouldApplyV3) // Force snapshot after cluster version downgrade. if prevVersion != nil && newVersion.LessThan(*prevVersion) { lg := a.options.Logger if lg != nil { lg.Info("Cluster version downgrade detected, forcing snapshot", zap.String("prev-cluster-version", prevVersion.String()), zap.String("new-cluster-version", newVersion.String()), ) } a.options.SnapshotServer.ForceSnapshot() } } func (a *applierV3backend) ClusterMemberAttrSet(r *membershippb.ClusterMemberAttrSetRequest, shouldApplyV3 membership.ShouldApplyV3) { a.options.Cluster.UpdateAttributes( types.ID(r.Member_ID), membership.Attributes{ Name: r.MemberAttributes.Name, ClientURLs: r.MemberAttributes.ClientUrls, }, shouldApplyV3, ) } func (a *applierV3backend) DowngradeInfoSet(r *membershippb.DowngradeInfoSetRequest, shouldApplyV3 membership.ShouldApplyV3) { d := version.DowngradeInfo{Enabled: false} if r.Enabled { d = version.DowngradeInfo{Enabled: true, TargetVersion: r.Ver} } a.options.Cluster.SetDowngradeInfo(&d, shouldApplyV3) } func (a *applierV3backend) newHeader() *pb.ResponseHeader { return &pb.ResponseHeader{ ClusterId: uint64(a.options.Cluster.ID()), MemberId: uint64(a.options.RaftStatus.MemberID()), Revision: a.options.KV.Rev(), RaftTerm: a.options.RaftStatus.Term(), } } ================================================ FILE: server/etcdserver/apply/capped.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package apply import ( pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/etcdserver/errors" serverstorage "go.etcd.io/etcd/server/v3/storage" ) type applierV3Capped struct { applierV3 q serverstorage.BackendQuota } // newApplierV3Capped creates an applyV3 that will reject Puts and transactions // with Puts so that the number of keys in the store is capped. func newApplierV3Capped(base applierV3) applierV3 { return &applierV3Capped{applierV3: base} } func (a *applierV3Capped) Put(_ *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) { return nil, nil, errors.ErrNoSpace } func (a *applierV3Capped) Txn(r *pb.TxnRequest) (*pb.TxnResponse, *traceutil.Trace, error) { if a.q.Cost(r) > 0 { return nil, nil, errors.ErrNoSpace } return a.applierV3.Txn(r) } func (a *applierV3Capped) LeaseGrant(_ *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { return nil, errors.ErrNoSpace } ================================================ FILE: server/etcdserver/apply/corrupt.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package apply import ( pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/etcdserver/errors" ) type applierV3Corrupt struct { applierV3 } func newApplierV3Corrupt(a applierV3) *applierV3Corrupt { return &applierV3Corrupt{a} } func (a *applierV3Corrupt) Put(_ *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) { return nil, nil, errors.ErrCorrupt } func (a *applierV3Corrupt) Range(_ *pb.RangeRequest) (*pb.RangeResponse, *traceutil.Trace, error) { return nil, nil, errors.ErrCorrupt } func (a *applierV3Corrupt) DeleteRange(_ *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, *traceutil.Trace, error) { return nil, nil, errors.ErrCorrupt } func (a *applierV3Corrupt) Txn(_ *pb.TxnRequest) (*pb.TxnResponse, *traceutil.Trace, error) { return nil, nil, errors.ErrCorrupt } func (a *applierV3Corrupt) Compaction(_ *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, *traceutil.Trace, error) { return nil, nil, nil, errors.ErrCorrupt } func (a *applierV3Corrupt) LeaseGrant(_ *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { return nil, errors.ErrCorrupt } func (a *applierV3Corrupt) LeaseRevoke(_ *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) { return nil, errors.ErrCorrupt } ================================================ FILE: server/etcdserver/apply/interface.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package apply import ( "time" "github.com/gogo/protobuf/proto" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/membershippb" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/v3alarm" "go.etcd.io/etcd/server/v3/etcdserver/cindex" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" ) // applierV3 is the interface for processing V3 raft messages type applierV3 interface { // Apply executes the generic portion of application logic for the current applier, but // delegates the actual execution to the applyFunc method. Apply(r *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3, applyFunc applyFunc) *Result Put(p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) Range(r *pb.RangeRequest) (*pb.RangeResponse, *traceutil.Trace, error) DeleteRange(dr *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, *traceutil.Trace, error) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, *traceutil.Trace, error) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, *traceutil.Trace, error) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) LeaseRevoke(lc *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) LeaseCheckpoint(lc *pb.LeaseCheckpointRequest) (*pb.LeaseCheckpointResponse, error) Alarm(*pb.AlarmRequest) (*pb.AlarmResponse, error) Authenticate(r *pb.InternalAuthenticateRequest) (*pb.AuthenticateResponse, error) AuthEnable() (*pb.AuthEnableResponse, error) AuthDisable() (*pb.AuthDisableResponse, error) AuthStatus() (*pb.AuthStatusResponse, error) UserAdd(ua *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, error) UserDelete(ua *pb.AuthUserDeleteRequest) (*pb.AuthUserDeleteResponse, error) UserChangePassword(ua *pb.AuthUserChangePasswordRequest) (*pb.AuthUserChangePasswordResponse, error) UserGrantRole(ua *pb.AuthUserGrantRoleRequest) (*pb.AuthUserGrantRoleResponse, error) UserGet(ua *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) UserRevokeRole(ua *pb.AuthUserRevokeRoleRequest) (*pb.AuthUserRevokeRoleResponse, error) RoleAdd(ua *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, error) RoleGrantPermission(ua *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) RoleGet(ua *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) RoleRevokePermission(ua *pb.AuthRoleRevokePermissionRequest) (*pb.AuthRoleRevokePermissionResponse, error) RoleDelete(ua *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDeleteResponse, error) UserList(ua *pb.AuthUserListRequest) (*pb.AuthUserListResponse, error) RoleList(ua *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error) ClusterVersionSet(r *membershippb.ClusterVersionSetRequest, shouldApplyV3 membership.ShouldApplyV3) ClusterMemberAttrSet(r *membershippb.ClusterMemberAttrSetRequest, shouldApplyV3 membership.ShouldApplyV3) DowngradeInfoSet(r *membershippb.DowngradeInfoSetRequest, shouldApplyV3 membership.ShouldApplyV3) } type ApplierOptions struct { Logger *zap.Logger KV mvcc.KV AlarmStore *v3alarm.AlarmStore AuthStore auth.AuthStore Lessor lease.Lessor Cluster *membership.RaftCluster RaftStatus RaftStatusGetter SnapshotServer SnapshotServer ConsistentIndex cindex.ConsistentIndexer TxnModeWriteWithSharedBuffer bool Backend backend.Backend QuotaBackendBytesCfg int64 WarningApplyDuration time.Duration } type SnapshotServer interface { ForceSnapshot() } // RaftStatusGetter represents etcd server and Raft progress. type RaftStatusGetter interface { MemberID() types.ID Leader() types.ID CommittedIndex() uint64 AppliedIndex() uint64 Term() uint64 } type Result struct { Resp proto.Message Err error // Physc signals the physical effect of the request has completed in addition // to being logically reflected by the node. Currently, only used for // Compaction requests. Physc <-chan struct{} Trace *traceutil.Trace } type applyFunc func(*pb.InternalRaftRequest, membership.ShouldApplyV3) *Result ================================================ FILE: server/etcdserver/apply/metrics.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package apply import "github.com/prometheus/client_golang/prometheus" var alarms = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "server", Name: "alarms", Help: "Alarms for every member in cluster. 1 for 'server_id' label with current ID. 2 for 'alarm_type' label with type of this alarm", }, []string{"server_id", "alarm_type"}, ) func init() { prometheus.MustRegister(alarms) } ================================================ FILE: server/etcdserver/apply/quota.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package apply import ( "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/etcdserver/errors" serverstorage "go.etcd.io/etcd/server/v3/storage" "go.etcd.io/etcd/server/v3/storage/backend" ) type quotaApplierV3 struct { applierV3 q serverstorage.Quota } func newQuotaApplierV3(lg *zap.Logger, quotaBackendBytesCfg int64, be backend.Backend, app applierV3) applierV3 { return "aApplierV3{app, serverstorage.NewBackendQuota(lg, quotaBackendBytesCfg, be, "v3-applier")} } func (a *quotaApplierV3) Put(p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) { ok := a.q.Available(p) resp, trace, err := a.applierV3.Put(p) if err == nil && !ok { err = errors.ErrNoSpace } return resp, trace, err } func (a *quotaApplierV3) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, *traceutil.Trace, error) { ok := a.q.Available(rt) resp, trace, err := a.applierV3.Txn(rt) if err == nil && !ok { err = errors.ErrNoSpace } return resp, trace, err } func (a *quotaApplierV3) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { ok := a.q.Available(lc) resp, err := a.applierV3.LeaseGrant(lc) if err == nil && !ok { err = errors.ErrNoSpace } return resp, err } ================================================ FILE: server/etcdserver/apply/uber_applier.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package apply import ( "errors" "time" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/v3alarm" "go.etcd.io/etcd/server/v3/etcdserver/txn" "go.etcd.io/etcd/server/v3/storage/mvcc" ) type UberApplier interface { Apply(r *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3) *Result } type uberApplier struct { lg *zap.Logger alarmStore *v3alarm.AlarmStore warningApplyDuration time.Duration // This is the applier that is taking in consideration current alarms applyV3 applierV3 // This is the applier used for wrapping when alarms change applyV3base applierV3 } func NewUberApplier(opts ApplierOptions) UberApplier { applyV3base := newApplierV3(opts) ua := &uberApplier{ lg: opts.Logger, alarmStore: opts.AlarmStore, warningApplyDuration: opts.WarningApplyDuration, applyV3: applyV3base, applyV3base: applyV3base, } ua.restoreAlarms() return ua } func newApplierV3(opts ApplierOptions) applierV3 { applierBackend := newApplierV3Backend(opts) return newAuthApplierV3( opts.AuthStore, newQuotaApplierV3(opts.Logger, opts.QuotaBackendBytesCfg, opts.Backend, applierBackend), opts.Lessor, ) } func (a *uberApplier) restoreAlarms() { noSpaceAlarms := len(a.alarmStore.Get(pb.AlarmType_NOSPACE)) > 0 corruptAlarms := len(a.alarmStore.Get(pb.AlarmType_CORRUPT)) > 0 a.applyV3 = a.applyV3base if noSpaceAlarms { a.applyV3 = newApplierV3Capped(a.applyV3) } if corruptAlarms { a.applyV3 = newApplierV3Corrupt(a.applyV3) } } func (a *uberApplier) Apply(r *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3) *Result { // We first execute chain of Apply() calls down the hierarchy: // (i.e. CorruptApplier -> CappedApplier -> Auth -> Quota -> Backend), // then dispatch() unpacks the request to a specific method (like Put), // that gets executed down the hierarchy again: // i.e. CorruptApplier.Put(CappedApplier.Put(...(BackendApplier.Put(...)))). return a.applyV3.Apply(r, shouldApplyV3, a.dispatch) } // dispatch translates the request (r) into appropriate call (like Put) on // the underlying applyV3 object. func (a *uberApplier) dispatch(r *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3) *Result { op := "unknown" ar := &Result{} defer func(start time.Time) { success := ar.Err == nil || errors.Is(ar.Err, mvcc.ErrCompacted) txn.ApplySecObserve("v3", op, success, time.Since(start)) txn.WarnOfExpensiveRequest(a.lg, a.warningApplyDuration, start, &pb.InternalRaftStringer{Request: r}, ar.Resp, ar.Err) if !success { txn.WarnOfFailedRequest(a.lg, start, &pb.InternalRaftStringer{Request: r}, ar.Resp, ar.Err) } }(time.Now()) switch { case r.ClusterVersionSet != nil: op = "ClusterVersionSet" // Implemented in 3.5.x a.applyV3.ClusterVersionSet(r.ClusterVersionSet, shouldApplyV3) return ar case r.ClusterMemberAttrSet != nil: op = "ClusterMemberAttrSet" // Implemented in 3.5.x a.applyV3.ClusterMemberAttrSet(r.ClusterMemberAttrSet, shouldApplyV3) return ar case r.DowngradeInfoSet != nil: op = "DowngradeInfoSet" // Implemented in 3.5.x a.applyV3.DowngradeInfoSet(r.DowngradeInfoSet, shouldApplyV3) return ar case r.DowngradeVersionTest != nil: op = "DowngradeVersionTest" // Implemented in 3.6 for test only // do nothing, we are just to ensure etcdserver don't panic in case // users(test cases) intentionally inject DowngradeVersionTestRequest // into the WAL files. return ar default: } if !shouldApplyV3 { return nil } switch { case r.Range != nil: op = "Range" ar.Resp, ar.Trace, ar.Err = a.applyV3.Range(r.Range) case r.Put != nil: op = "Put" ar.Resp, ar.Trace, ar.Err = a.applyV3.Put(r.Put) case r.DeleteRange != nil: op = "DeleteRange" ar.Resp, ar.Trace, ar.Err = a.applyV3.DeleteRange(r.DeleteRange) case r.Txn != nil: op = "Txn" ar.Resp, ar.Trace, ar.Err = a.applyV3.Txn(r.Txn) case r.Compaction != nil: op = "Compaction" ar.Resp, ar.Physc, ar.Trace, ar.Err = a.applyV3.Compaction(r.Compaction) case r.LeaseGrant != nil: op = "LeaseGrant" ar.Resp, ar.Err = a.applyV3.LeaseGrant(r.LeaseGrant) case r.LeaseRevoke != nil: op = "LeaseRevoke" ar.Resp, ar.Err = a.applyV3.LeaseRevoke(r.LeaseRevoke) case r.LeaseCheckpoint != nil: op = "LeaseCheckpoint" ar.Resp, ar.Err = a.applyV3.LeaseCheckpoint(r.LeaseCheckpoint) case r.Alarm != nil: op = "Alarm" ar.Resp, ar.Err = a.Alarm(r.Alarm) case r.Authenticate != nil: op = "Authenticate" ar.Resp, ar.Err = a.applyV3.Authenticate(r.Authenticate) case r.AuthEnable != nil: op = "AuthEnable" ar.Resp, ar.Err = a.applyV3.AuthEnable() case r.AuthDisable != nil: op = "AuthDisable" ar.Resp, ar.Err = a.applyV3.AuthDisable() case r.AuthStatus != nil: ar.Resp, ar.Err = a.applyV3.AuthStatus() case r.AuthUserAdd != nil: op = "AuthUserAdd" ar.Resp, ar.Err = a.applyV3.UserAdd(r.AuthUserAdd) case r.AuthUserDelete != nil: op = "AuthUserDelete" ar.Resp, ar.Err = a.applyV3.UserDelete(r.AuthUserDelete) case r.AuthUserChangePassword != nil: op = "AuthUserChangePassword" ar.Resp, ar.Err = a.applyV3.UserChangePassword(r.AuthUserChangePassword) case r.AuthUserGrantRole != nil: op = "AuthUserGrantRole" ar.Resp, ar.Err = a.applyV3.UserGrantRole(r.AuthUserGrantRole) case r.AuthUserGet != nil: op = "AuthUserGet" ar.Resp, ar.Err = a.applyV3.UserGet(r.AuthUserGet) case r.AuthUserRevokeRole != nil: op = "AuthUserRevokeRole" ar.Resp, ar.Err = a.applyV3.UserRevokeRole(r.AuthUserRevokeRole) case r.AuthRoleAdd != nil: op = "AuthRoleAdd" ar.Resp, ar.Err = a.applyV3.RoleAdd(r.AuthRoleAdd) case r.AuthRoleGrantPermission != nil: op = "AuthRoleGrantPermission" ar.Resp, ar.Err = a.applyV3.RoleGrantPermission(r.AuthRoleGrantPermission) case r.AuthRoleGet != nil: op = "AuthRoleGet" ar.Resp, ar.Err = a.applyV3.RoleGet(r.AuthRoleGet) case r.AuthRoleRevokePermission != nil: op = "AuthRoleRevokePermission" ar.Resp, ar.Err = a.applyV3.RoleRevokePermission(r.AuthRoleRevokePermission) case r.AuthRoleDelete != nil: op = "AuthRoleDelete" ar.Resp, ar.Err = a.applyV3.RoleDelete(r.AuthRoleDelete) case r.AuthUserList != nil: op = "AuthUserList" ar.Resp, ar.Err = a.applyV3.UserList(r.AuthUserList) case r.AuthRoleList != nil: op = "AuthRoleList" ar.Resp, ar.Err = a.applyV3.RoleList(r.AuthRoleList) default: a.lg.Panic("not implemented apply", zap.Stringer("raft-request", r)) } return ar } func (a *uberApplier) Alarm(ar *pb.AlarmRequest) (*pb.AlarmResponse, error) { resp, err := a.applyV3.Alarm(ar) if ar.Action == pb.AlarmRequest_ACTIVATE || ar.Action == pb.AlarmRequest_DEACTIVATE { a.restoreAlarms() } return resp, err } ================================================ FILE: server/etcdserver/apply/uber_applier_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package apply import ( "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "golang.org/x/crypto/bcrypt" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/v3alarm" "go.etcd.io/etcd/server/v3/etcdserver/cindex" "go.etcd.io/etcd/server/v3/etcdserver/errors" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" ) const memberID = 111195 func defaultUberApplier(t *testing.T) UberApplier { lg := zaptest.NewLogger(t) be, _ := betesting.NewDefaultTmpBackend(t) t.Cleanup(func() { betesting.Close(t, be) }) cluster := membership.NewCluster(lg) cluster.SetBackend(schema.NewMembershipBackend(lg, be)) cluster.AddMember(&membership.Member{ID: memberID}, true) lessor := lease.NewLessor(lg, be, cluster, lease.LessorConfig{}) kv := mvcc.NewStore(lg, be, lessor, mvcc.StoreConfig{}) alarmStore, err := v3alarm.NewAlarmStore(lg, schema.NewAlarmBackend(lg, be)) require.NoError(t, err) tp, err := auth.NewTokenProvider(lg, "simple", dummyIndexWaiter, 300*time.Second) require.NoError(t, err) authStore := auth.NewAuthStore( lg, schema.NewAuthBackend(lg, be), tp, bcrypt.DefaultCost, ) consistentIndex := cindex.NewConsistentIndex(be) opts := ApplierOptions{ Logger: lg, KV: kv, AlarmStore: alarmStore, AuthStore: authStore, Lessor: lessor, Cluster: cluster, RaftStatus: &fakeRaftStatusGetter{}, SnapshotServer: &fakeSnapshotServer{}, ConsistentIndex: consistentIndex, TxnModeWriteWithSharedBuffer: false, Backend: be, QuotaBackendBytesCfg: 16 * 1024 * 1024, // 16MB WarningApplyDuration: time.Hour, } return NewUberApplier(opts) } // TestUberApplier_Alarm_Corrupt tests the applier returns ErrCorrupt after alarm CORRUPT is activated func TestUberApplier_Alarm_Corrupt(t *testing.T) { tcs := []struct { name string request *pb.InternalRaftRequest expectError error }{ { name: "Put request returns ErrCorrupt after alarm CORRUPT is activated", request: &pb.InternalRaftRequest{Put: &pb.PutRequest{}}, expectError: errors.ErrCorrupt, }, { name: "Range request returns ErrCorrupt after alarm CORRUPT is activated", request: &pb.InternalRaftRequest{Range: &pb.RangeRequest{}}, expectError: errors.ErrCorrupt, }, { name: "DeleteRange request returns ErrCorrupt after alarm CORRUPT is activated", request: &pb.InternalRaftRequest{DeleteRange: &pb.DeleteRangeRequest{}}, expectError: errors.ErrCorrupt, }, { name: "Txn request returns ErrCorrupt after alarm CORRUPT is activated", request: &pb.InternalRaftRequest{Txn: &pb.TxnRequest{}}, expectError: errors.ErrCorrupt, }, { name: "Compaction request returns ErrCorrupt after alarm CORRUPT is activated", request: &pb.InternalRaftRequest{Compaction: &pb.CompactionRequest{}}, expectError: errors.ErrCorrupt, }, { name: "LeaseGrant request returns ErrCorrupt after alarm CORRUPT is activated", request: &pb.InternalRaftRequest{LeaseGrant: &pb.LeaseGrantRequest{}}, expectError: errors.ErrCorrupt, }, { name: "LeaseRevoke request returns ErrCorrupt after alarm CORRUPT is activated", request: &pb.InternalRaftRequest{LeaseRevoke: &pb.LeaseRevokeRequest{}}, expectError: errors.ErrCorrupt, }, } ua := defaultUberApplier(t) result := ua.Apply(&pb.InternalRaftRequest{ Header: &pb.RequestHeader{}, Alarm: &pb.AlarmRequest{ Action: pb.AlarmRequest_ACTIVATE, MemberID: memberID, Alarm: pb.AlarmType_CORRUPT, }, }, membership.ApplyBoth) require.NotNil(t, result) require.NoError(t, result.Err) for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { result = ua.Apply(tc.request, membership.ApplyBoth) require.NotNil(t, result) require.Equalf(t, tc.expectError, result.Err, "Apply: got %v, expect: %v", result.Err, tc.expectError) }) } } // TestUberApplier_Alarm_Quota tests the applier returns ErrNoSpace after alarm NOSPACE is activated func TestUberApplier_Alarm_Quota(t *testing.T) { tcs := []struct { name string request *pb.InternalRaftRequest expectError error }{ { name: "Put request returns ErrCorrupt after alarm NOSPACE is activated", request: &pb.InternalRaftRequest{Put: &pb.PutRequest{Key: []byte(key)}}, expectError: errors.ErrNoSpace, }, { name: "Txn request cost > 0 returns ErrCorrupt after alarm NOSPACE is activated", request: &pb.InternalRaftRequest{Txn: &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Key: []byte(key), }, }, }, }, }}, expectError: errors.ErrNoSpace, }, { name: "Txn request cost = 0 is still allowed after alarm NOSPACE is activated", request: &pb.InternalRaftRequest{Txn: &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestRange{ RequestRange: &pb.RangeRequest{ Key: []byte(key), }, }, }, }, }}, expectError: nil, }, { name: "Txn request cost = 0 in both branches is still allowed after alarm NOSPACE is activated", request: &pb.InternalRaftRequest{Txn: &pb.TxnRequest{ Compare: []*pb.Compare{ { Key: []byte(key), Result: pb.Compare_EQUAL, Target: pb.Compare_CREATE, TargetUnion: &pb.Compare_CreateRevision{CreateRevision: 0}, }, }, Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestRange{ RequestRange: &pb.RangeRequest{ Key: []byte(key), }, }, }, }, Failure: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestDeleteRange{ RequestDeleteRange: &pb.DeleteRangeRequest{ Key: []byte(key), }, }, }, }, }}, expectError: nil, }, { name: "LeaseGrant request returns ErrCorrupt after alarm NOSPACE is activated", request: &pb.InternalRaftRequest{LeaseGrant: &pb.LeaseGrantRequest{}}, expectError: errors.ErrNoSpace, }, } ua := defaultUberApplier(t) result := ua.Apply(&pb.InternalRaftRequest{ Header: &pb.RequestHeader{}, Alarm: &pb.AlarmRequest{ Action: pb.AlarmRequest_ACTIVATE, MemberID: memberID, Alarm: pb.AlarmType_NOSPACE, }, }, membership.ApplyBoth) require.NotNil(t, result) require.NoError(t, result.Err) for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { result = ua.Apply(tc.request, membership.ApplyBoth) require.NotNil(t, result) require.Equalf(t, tc.expectError, result.Err, "Apply: got %v, expect: %v", result.Err, tc.expectError) }) } } // TestUberApplier_Alarm_Deactivate tests the applier should be able to apply after alarm is deactivated func TestUberApplier_Alarm_Deactivate(t *testing.T) { ua := defaultUberApplier(t) result := ua.Apply(&pb.InternalRaftRequest{ Header: &pb.RequestHeader{}, Alarm: &pb.AlarmRequest{ Action: pb.AlarmRequest_ACTIVATE, MemberID: memberID, Alarm: pb.AlarmType_NOSPACE, }, }, membership.ApplyBoth) require.NotNil(t, result) require.NoError(t, result.Err) result = ua.Apply(&pb.InternalRaftRequest{Put: &pb.PutRequest{Key: []byte(key)}}, membership.ApplyBoth) require.NotNil(t, result) require.Equalf(t, errors.ErrNoSpace, result.Err, "Apply: got %v, expect: %v", result.Err, errors.ErrNoSpace) result = ua.Apply(&pb.InternalRaftRequest{ Header: &pb.RequestHeader{}, Alarm: &pb.AlarmRequest{ Action: pb.AlarmRequest_DEACTIVATE, MemberID: memberID, Alarm: pb.AlarmType_NOSPACE, }, }, membership.ApplyBoth) require.NotNil(t, result) require.NoError(t, result.Err) result = ua.Apply(&pb.InternalRaftRequest{Put: &pb.PutRequest{Key: []byte(key)}}, membership.ApplyBoth) require.NotNil(t, result) assert.NoError(t, result.Err) } ================================================ FILE: server/etcdserver/bootstrap.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "encoding/json" "errors" "fmt" "io" "net/http" "os" "strings" "time" "github.com/coreos/go-semver/semver" "github.com/dustin/go-humanize" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver/api" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/etcd/server/v3/etcdserver/api/v3discovery" "go.etcd.io/etcd/server/v3/etcdserver/cindex" servererrors "go.etcd.io/etcd/server/v3/etcdserver/errors" serverstorage "go.etcd.io/etcd/server/v3/storage" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) func bootstrap(cfg config.ServerConfig) (b *bootstrappedServer, err error) { if cfg.MaxRequestBytes > recommendedMaxRequestBytes { cfg.Logger.Warn( "exceeded recommended request limit", zap.Uint("max-request-bytes", cfg.MaxRequestBytes), zap.String("max-request-size", humanize.Bytes(uint64(cfg.MaxRequestBytes))), zap.Int("recommended-request-bytes", recommendedMaxRequestBytes), zap.String("recommended-request-size", recommendedMaxRequestBytesString), ) } if terr := fileutil.TouchDirAll(cfg.Logger, cfg.DataDir); terr != nil { return nil, fmt.Errorf("cannot access data directory: %w", terr) } if terr := fileutil.TouchDirAll(cfg.Logger, cfg.MemberDir()); terr != nil { return nil, fmt.Errorf("cannot access member directory: %w", terr) } ss := bootstrapSnapshot(cfg) prt, err := rafthttp.NewRoundTripper(cfg.PeerTLSInfo, cfg.PeerDialTimeout()) if err != nil { return nil, err } haveWAL := wal.Exist(cfg.WALDir()) backend, err := bootstrapBackend(cfg, haveWAL) if err != nil { return nil, err } var bwal *bootstrappedWAL if haveWAL { if err = fileutil.IsDirWriteable(cfg.WALDir()); err != nil { return nil, fmt.Errorf("cannot write to WAL directory: %w", err) } cfg.Logger.Info("Bootstrapping WAL from snapshot") bwal = bootstrapWALFromSnapshot(cfg, backend.snapshot, backend.ci) } cfg.Logger.Info("bootstrapping cluster") cluster, err := bootstrapCluster(cfg, bwal, prt) if err != nil { backend.Close() return nil, err } cfg.Logger.Info("bootstrapping storage") s := bootstrapStorage(cfg, backend, bwal, cluster) if err = cluster.Finalize(cfg, s); err != nil { backend.Close() return nil, err } if haveWAL { sn := s.wal.snapshot if sn == nil { sn = &raftpb.Snapshot{} } cs := buildConfStateFromV3store(cfg.Logger, backend.be) sn.Metadata.ConfState = cs s.wal.snapshot = sn cfg.Logger.Info("Constructed a new raft snapshot from v3 state", zap.Uint64("index", sn.Metadata.Index), zap.Uint64("term", sn.Metadata.Term), zap.String("confState", sn.Metadata.ConfState.String())) } cfg.Logger.Info("bootstrapping raft") raft := bootstrapRaft(cfg, cluster, s.wal) return &bootstrappedServer{ prt: prt, ss: ss, storage: s, cluster: cluster, raft: raft, }, nil } func buildConfStateFromV3store(lg *zap.Logger, be backend.Backend) raftpb.ConfState { members, _ := schema.NewMembershipBackend(lg, be).MustReadMembersFromBackend() var ( voters []uint64 learners []uint64 ) for _, m := range members { if m.IsLearner { learners = append(learners, uint64(m.ID)) } else { voters = append(voters, uint64(m.ID)) } } return raftpb.ConfState{ Voters: voters, Learners: learners, } } type bootstrappedServer struct { storage *bootstrappedStorage cluster *bootstrappedCluster raft *bootstrappedRaft prt http.RoundTripper ss *snap.Snapshotter } func (s *bootstrappedServer) Close() { s.storage.Close() } type bootstrappedStorage struct { backend *bootstrappedBackend wal *bootstrappedWAL } func (s *bootstrappedStorage) Close() { s.backend.Close() } type bootstrappedBackend struct { beHooks *serverstorage.BackendHooks be backend.Backend ci cindex.ConsistentIndexer beExist bool snapshot *raftpb.Snapshot } func (s *bootstrappedBackend) Close() { s.be.Close() } type bootstrappedCluster struct { remotes []*membership.Member cl *membership.RaftCluster nodeID types.ID } type bootstrappedRaft struct { lg *zap.Logger heartbeat time.Duration peers []raft.Peer config *raft.Config storage *raft.MemoryStorage } func bootstrapStorage(cfg config.ServerConfig, be *bootstrappedBackend, wal *bootstrappedWAL, cl *bootstrappedCluster) *bootstrappedStorage { if wal == nil { wal = bootstrapNewWAL(cfg, cl) } return &bootstrappedStorage{ backend: be, wal: wal, } } func bootstrapSnapshot(cfg config.ServerConfig) *snap.Snapshotter { if err := fileutil.TouchDirAll(cfg.Logger, cfg.SnapDir()); err != nil { cfg.Logger.Fatal( "failed to create snapshot directory", zap.String("path", cfg.SnapDir()), zap.Error(err), ) } if err := fileutil.RemoveMatchFile(cfg.Logger, cfg.SnapDir(), func(fileName string) bool { return strings.HasPrefix(fileName, "tmp") }); err != nil { cfg.Logger.Error( "failed to remove temp file(s) in snapshot directory", zap.String("path", cfg.SnapDir()), zap.Error(err), ) } return snap.New(cfg.Logger, cfg.SnapDir()) } func bootstrapBackend(cfg config.ServerConfig, haveWAL bool) (backend *bootstrappedBackend, err error) { beExist := fileutil.Exist(cfg.BackendPath()) ci := cindex.NewConsistentIndex(nil) beHooks := serverstorage.NewBackendHooks(cfg.Logger, ci) be := serverstorage.OpenBackend(cfg, beHooks) defer func() { if err != nil && be != nil { be.Close() } }() ci.SetBackend(be) schema.CreateMetaBucket(be.BatchTx()) if cfg.BootstrapDefragThresholdMegabytes != 0 { err = maybeDefragBackend(cfg, be) if err != nil { return nil, err } } cfg.Logger.Info("restore consistentIndex", zap.Uint64("index", ci.ConsistentIndex())) // TODO(serathius): Implement schema setup in fresh storage var snapshot *raftpb.Snapshot if haveWAL { snapshot, be, err = recoverSnapshot(cfg, be, beExist, beHooks, ci) if err != nil { return nil, err } } if beExist { s1, s2 := be.Size(), be.SizeInUse() cfg.Logger.Info( "recovered v3 backend", zap.Int64("backend-size-bytes", s1), zap.String("backend-size", humanize.Bytes(uint64(s1))), zap.Int64("backend-size-in-use-bytes", s2), zap.String("backend-size-in-use", humanize.Bytes(uint64(s2))), ) if err = schema.Validate(cfg.Logger, be.ReadTx()); err != nil { cfg.Logger.Error("Failed to validate schema", zap.Error(err)) return nil, err } } return &bootstrappedBackend{ beHooks: beHooks, be: be, ci: ci, beExist: beExist, snapshot: snapshot, }, nil } func maybeDefragBackend(cfg config.ServerConfig, be backend.Backend) error { size := be.Size() sizeInUse := be.SizeInUse() freeableMemory := uint(size - sizeInUse) thresholdBytes := cfg.BootstrapDefragThresholdMegabytes * 1024 * 1024 if freeableMemory < thresholdBytes { cfg.Logger.Info("Skipping defragmentation", zap.Int64("current-db-size-bytes", size), zap.String("current-db-size", humanize.Bytes(uint64(size))), zap.Int64("current-db-size-in-use-bytes", sizeInUse), zap.String("current-db-size-in-use", humanize.Bytes(uint64(sizeInUse))), zap.Uint("bootstrap-defrag-threshold-bytes", thresholdBytes), zap.String("bootstrap-defrag-threshold", humanize.Bytes(uint64(thresholdBytes))), ) return nil } return be.Defrag() } func bootstrapCluster(cfg config.ServerConfig, bwal *bootstrappedWAL, prt http.RoundTripper) (c *bootstrappedCluster, err error) { switch { case bwal == nil && !cfg.NewCluster: c, err = bootstrapExistingClusterNoWAL(cfg, prt) case bwal == nil && cfg.NewCluster: c, err = bootstrapNewClusterNoWAL(cfg, prt) case bwal != nil && bwal.haveWAL: c, err = bootstrapClusterWithWAL(cfg, bwal.meta) default: return nil, fmt.Errorf("unsupported bootstrap config") } if err != nil { return nil, err } return c, nil } func bootstrapExistingClusterNoWAL(cfg config.ServerConfig, prt http.RoundTripper) (*bootstrappedCluster, error) { if err := cfg.VerifyJoinExisting(); err != nil { return nil, err } cl, err := membership.NewClusterFromURLsMap(cfg.Logger, cfg.InitialClusterToken, cfg.InitialPeerURLsMap, membership.WithMaxLearners(cfg.MaxLearners)) if err != nil { return nil, err } existingCluster, gerr := GetClusterFromRemotePeers(cfg.Logger, getRemotePeerURLs(cl, cfg.Name), prt) if gerr != nil { return nil, fmt.Errorf("cannot fetch cluster info from peer urls: %w", gerr) } if err := membership.ValidateClusterAndAssignIDs(cfg.Logger, cl, existingCluster); err != nil { return nil, fmt.Errorf("error validating peerURLs %s: %w", existingCluster, err) } if !isCompatibleWithCluster(cfg.Logger, cl, cl.MemberByName(cfg.Name).ID, prt, cfg.ReqTimeout()) { return nil, fmt.Errorf("incompatible with current running cluster") } scaleUpLearners := false if err := membership.ValidateMaxLearnerConfig(cfg.MaxLearners, existingCluster.Members(), scaleUpLearners); err != nil { return nil, err } remotes := existingCluster.Members() cl.SetID(types.ID(0), existingCluster.ID()) member := cl.MemberByName(cfg.Name) return &bootstrappedCluster{ remotes: remotes, cl: cl, nodeID: member.ID, }, nil } func bootstrapNewClusterNoWAL(cfg config.ServerConfig, prt http.RoundTripper) (*bootstrappedCluster, error) { if err := cfg.VerifyBootstrap(); err != nil { return nil, err } cl, err := membership.NewClusterFromURLsMap(cfg.Logger, cfg.InitialClusterToken, cfg.InitialPeerURLsMap, membership.WithMaxLearners(cfg.MaxLearners)) if err != nil { return nil, err } m := cl.MemberByName(cfg.Name) if isMemberBootstrapped(cfg.Logger, cl, cfg.Name, prt, cfg.BootstrapTimeoutEffective()) { return nil, fmt.Errorf("member %s has already been bootstrapped", m.ID) } if cfg.ShouldDiscover() { cfg.Logger.Info("Bootstrapping cluster using v3 discovery.") str, err := v3discovery.JoinCluster(cfg.Logger, &cfg.DiscoveryCfg, m.ID, cfg.InitialPeerURLsMap.String()) if err != nil { return nil, &servererrors.DiscoveryError{Op: "join", Err: err} } var urlsmap types.URLsMap urlsmap, err = types.NewURLsMap(str) if err != nil { return nil, err } if config.CheckDuplicateURL(urlsmap) { return nil, fmt.Errorf("discovery cluster %s has duplicate url", urlsmap) } if cl, err = membership.NewClusterFromURLsMap(cfg.Logger, cfg.InitialClusterToken, urlsmap, membership.WithMaxLearners(cfg.MaxLearners)); err != nil { return nil, err } } return &bootstrappedCluster{ remotes: nil, cl: cl, nodeID: m.ID, }, nil } func bootstrapClusterWithWAL(cfg config.ServerConfig, meta *snapshotMetadata) (*bootstrappedCluster, error) { if err := fileutil.IsDirWriteable(cfg.MemberDir()); err != nil { return nil, fmt.Errorf("cannot write to member directory: %w", err) } if cfg.ShouldDiscover() { cfg.Logger.Warn( "discovery token is ignored since cluster already initialized; valid logs are found", zap.String("wal-dir", cfg.WALDir()), ) } cl := membership.NewCluster(cfg.Logger, membership.WithMaxLearners(cfg.MaxLearners)) scaleUpLearners := false if err := membership.ValidateMaxLearnerConfig(cfg.MaxLearners, cl.Members(), scaleUpLearners); err != nil { return nil, err } cl.SetID(meta.nodeID, meta.clusterID) return &bootstrappedCluster{ cl: cl, nodeID: meta.nodeID, }, nil } func recoverSnapshot(cfg config.ServerConfig, be backend.Backend, beExist bool, beHooks *serverstorage.BackendHooks, ci cindex.ConsistentIndexer) (*raftpb.Snapshot, backend.Backend, error) { // Find a snapshot to start/restart a raft node walSnaps, err := wal.ValidSnapshotEntries(cfg.Logger, cfg.WALDir()) if err != nil { return nil, be, err } var snapshot *raftpb.Snapshot if len(walSnaps) > 0 { idx := len(walSnaps) - 1 snapshot = &raftpb.Snapshot{ Metadata: raftpb.SnapshotMetadata{ Term: walSnaps[idx].GetTerm(), Index: walSnaps[idx].GetIndex(), }, } if walSnaps[idx].ConfState != nil { snapshot.Metadata.ConfState = *walSnaps[idx].ConfState } cfg.Logger.Info("constructed a snapshot from WAL record", zap.Uint64("snapshot-index", snapshot.Metadata.Index), zap.String("snapshot-size", humanize.Bytes(uint64(snapshot.Size()))), zap.String("confState", snapshot.Metadata.ConfState.String()), zap.Int("walSnaps-count", len(walSnaps)), ) } if snapshot != nil { if be, err = serverstorage.RecoverSnapshotBackend(cfg, be, *snapshot, beExist, beHooks); err != nil { cfg.Logger.Panic("failed to recover v3 backend from snapshot", zap.Error(err)) } // A snapshot db may have already been recovered, and the old db should have // already been closed in this case, so we should set the backend again. ci.SetBackend(be) if beExist { // TODO: remove kvindex != 0 checking when we do not expect users to upgrade // etcd from pre-3.0 release. kvindex := ci.ConsistentIndex() if kvindex < snapshot.Metadata.Index { if kvindex != 0 { return nil, be, fmt.Errorf("database file (%v index %d) does not match with snapshot (index %d)", cfg.BackendPath(), kvindex, snapshot.Metadata.Index) } cfg.Logger.Warn( "consistent index was never saved", zap.Uint64("snapshot-index", snapshot.Metadata.Index), ) } } } else { cfg.Logger.Info("No snapshot found. Recovering WAL from scratch!") } return snapshot, be, nil } func (c *bootstrappedCluster) Finalize(cfg config.ServerConfig, s *bootstrappedStorage) error { if !s.wal.haveWAL { c.cl.SetID(c.nodeID, c.cl.ID()) } c.cl.SetBackend(schema.NewMembershipBackend(cfg.Logger, s.backend.be)) if s.wal.haveWAL { c.cl.Recover(api.UpdateCapability) if c.databaseFileMissing(s) { bepath := cfg.BackendPath() os.RemoveAll(bepath) return fmt.Errorf("database file (%v) of the backend is missing", bepath) } } scaleUpLearners := false return membership.ValidateMaxLearnerConfig(cfg.MaxLearners, c.cl.Members(), scaleUpLearners) } func (c *bootstrappedCluster) databaseFileMissing(s *bootstrappedStorage) bool { v3Cluster := c.cl.Version() != nil && !c.cl.Version().LessThan(semver.Version{Major: 3}) return v3Cluster && !s.backend.beExist } func bootstrapRaft(cfg config.ServerConfig, cluster *bootstrappedCluster, bwal *bootstrappedWAL) *bootstrappedRaft { switch { case !bwal.haveWAL && !cfg.NewCluster: return bootstrapRaftFromCluster(cfg, cluster.cl, nil, bwal) case !bwal.haveWAL && cfg.NewCluster: return bootstrapRaftFromCluster(cfg, cluster.cl, cluster.cl.MemberIDs(), bwal) case bwal.haveWAL: return bootstrapRaftFromWAL(cfg, bwal) default: cfg.Logger.Panic("unsupported bootstrap config") return nil } } func bootstrapRaftFromCluster(cfg config.ServerConfig, cl *membership.RaftCluster, ids []types.ID, bwal *bootstrappedWAL) *bootstrappedRaft { member := cl.MemberByName(cfg.Name) peers := make([]raft.Peer, len(ids)) for i, id := range ids { var ctx []byte ctx, err := json.Marshal((*cl).Member(id)) if err != nil { cfg.Logger.Panic("failed to marshal member", zap.Error(err)) } peers[i] = raft.Peer{ID: uint64(id), Context: ctx} } cfg.Logger.Info( "starting local member", zap.String("local-member-id", member.ID.String()), zap.String("cluster-id", cl.ID().String()), ) s := bwal.MemoryStorage() return &bootstrappedRaft{ lg: cfg.Logger, heartbeat: time.Duration(cfg.TickMs) * time.Millisecond, config: raftConfig(cfg, uint64(member.ID), s), peers: peers, storage: s, } } func bootstrapRaftFromWAL(cfg config.ServerConfig, bwal *bootstrappedWAL) *bootstrappedRaft { s := bwal.MemoryStorage() return &bootstrappedRaft{ lg: cfg.Logger, heartbeat: time.Duration(cfg.TickMs) * time.Millisecond, config: raftConfig(cfg, uint64(bwal.meta.nodeID), s), storage: s, } } func raftConfig(cfg config.ServerConfig, id uint64, s *raft.MemoryStorage) *raft.Config { return &raft.Config{ ID: id, ElectionTick: cfg.ElectionTicks, HeartbeatTick: 1, Storage: s, MaxSizePerMsg: maxSizePerMsg, MaxInflightMsgs: maxInflightMsgs, CheckQuorum: true, PreVote: cfg.PreVote, Logger: NewRaftLoggerZap(cfg.Logger.Named("raft")), } } func (b *bootstrappedRaft) newRaftNode(ss *snap.Snapshotter, wal *wal.WAL, cl *membership.RaftCluster) *raftNode { var n raft.Node if len(b.peers) == 0 { n = raft.RestartNode(b.config) } else { n = raft.StartNode(b.config, b.peers) } raftStatusMu.Lock() raftStatus = n.Status raftStatusMu.Unlock() return newRaftNode( raftNodeConfig{ lg: b.lg, isIDRemoved: func(id uint64) bool { return cl.IsIDRemoved(types.ID(id)) }, Node: n, heartbeat: b.heartbeat, raftStorage: b.storage, storage: serverstorage.NewStorage(b.lg, wal, ss), }, ) } func bootstrapWALFromSnapshot(cfg config.ServerConfig, snapshot *raftpb.Snapshot, ci cindex.ConsistentIndexer) *bootstrappedWAL { wal, st, ents, snap, meta := openWALFromSnapshot(cfg, snapshot) bwal := &bootstrappedWAL{ lg: cfg.Logger, w: wal, st: st, ents: ents, snapshot: snap, meta: meta, haveWAL: true, } if cfg.ForceNewCluster { consistentIndex := ci.ConsistentIndex() oldCommitIndex := bwal.st.Commit // If only `HardState.Commit` increases, HardState won't be persisted // to disk, even though the committed entries might have already been // applied. This can result in consistent_index > CommitIndex. // // When restarting etcd with `--force-new-cluster`, all uncommitted // entries are dropped. To avoid losing entries that were actually // committed, we reset Commit to max(HardState.Commit, consistent_index). // // See: https://github.com/etcd-io/raft/pull/300 for more details. bwal.st.Commit = max(oldCommitIndex, consistentIndex) // discard the previously uncommitted entries bwal.ents = bwal.CommitedEntries() entries := bwal.NewConfigChangeEntries() // force commit config change entries bwal.AppendAndCommitEntries(entries) cfg.Logger.Info( "forcing restart member", zap.String("cluster-id", meta.clusterID.String()), zap.String("local-member-id", meta.nodeID.String()), zap.Uint64("wal-commit-index", oldCommitIndex), zap.Uint64("commit-index", bwal.st.Commit), ) } else { cfg.Logger.Info( "restarting local member", zap.String("cluster-id", meta.clusterID.String()), zap.String("local-member-id", meta.nodeID.String()), zap.Uint64("commit-index", bwal.st.Commit), ) } return bwal } // openWALFromSnapshot reads the WAL at the given snap and returns the wal, its latest HardState and cluster ID, and all entries that appear // after the position of the given snap in the WAL. // The snap must have been previously saved to the WAL, or this call will panic. func openWALFromSnapshot(cfg config.ServerConfig, snapshot *raftpb.Snapshot) (*wal.WAL, *raftpb.HardState, []raftpb.Entry, *raftpb.Snapshot, *snapshotMetadata) { var walsnap walpb.Snapshot if snapshot != nil { walsnap.Index, walsnap.Term = new(snapshot.Metadata.Index), new(snapshot.Metadata.Term) } repaired := false for { w, err := wal.Open(cfg.Logger, cfg.WALDir(), walsnap) if err != nil { cfg.Logger.Fatal("failed to open WAL", zap.Error(err)) } if cfg.UnsafeNoFsync { w.SetUnsafeNoFsync() } wmetadata, st, ents, err := w.ReadAll() if err != nil { w.Close() // we can only repair ErrUnexpectedEOF and we never repair twice. if repaired || !errors.Is(err, io.ErrUnexpectedEOF) { cfg.Logger.Fatal("failed to read WAL, cannot be repaired", zap.Error(err)) } if !wal.Repair(cfg.Logger, cfg.WALDir()) { cfg.Logger.Fatal("failed to repair WAL", zap.Error(err)) } else { cfg.Logger.Info("repaired WAL", zap.Error(err)) repaired = true } continue } var metadata etcdserverpb.Metadata pbutil.MustUnmarshal(&metadata, wmetadata) id := types.ID(metadata.GetNodeID()) cid := types.ID(metadata.GetClusterID()) meta := &snapshotMetadata{clusterID: cid, nodeID: id} return w, &st, ents, snapshot, meta } } type snapshotMetadata struct { nodeID, clusterID types.ID } func bootstrapNewWAL(cfg config.ServerConfig, cl *bootstrappedCluster) *bootstrappedWAL { metadata := pbutil.MustMarshal( &etcdserverpb.Metadata{ NodeID: new(uint64(cl.nodeID)), ClusterID: new(uint64(cl.cl.ID())), }, ) w, err := wal.Create(cfg.Logger, cfg.WALDir(), metadata) if err != nil { cfg.Logger.Panic("failed to create WAL", zap.Error(err)) } if cfg.UnsafeNoFsync { w.SetUnsafeNoFsync() } return &bootstrappedWAL{ lg: cfg.Logger, w: w, } } type bootstrappedWAL struct { lg *zap.Logger haveWAL bool w *wal.WAL st *raftpb.HardState ents []raftpb.Entry snapshot *raftpb.Snapshot meta *snapshotMetadata } func (wal *bootstrappedWAL) MemoryStorage() *raft.MemoryStorage { s := raft.NewMemoryStorage() if wal.snapshot != nil { s.ApplySnapshot(*wal.snapshot) } if wal.st != nil { s.SetHardState(*wal.st) } if len(wal.ents) != 0 { s.Append(wal.ents) } return s } func (wal *bootstrappedWAL) CommitedEntries() []raftpb.Entry { for i, ent := range wal.ents { if ent.Index > wal.st.Commit { wal.lg.Info( "discarding uncommitted WAL entries", zap.Uint64("entry-index", ent.Index), zap.Uint64("commit-index-from-wal", wal.st.Commit), zap.Int("number-of-discarded-entries", len(wal.ents)-i), ) return wal.ents[:i] } } return wal.ents } func (wal *bootstrappedWAL) NewConfigChangeEntries() []raftpb.Entry { return serverstorage.CreateConfigChangeEnts( wal.lg, serverstorage.GetEffectiveNodeIDsFromWALEntries(wal.lg, wal.snapshot, wal.ents), uint64(wal.meta.nodeID), wal.st.Term, wal.st.Commit, ) } func (wal *bootstrappedWAL) AppendAndCommitEntries(ents []raftpb.Entry) { wal.ents = append(wal.ents, ents...) err := wal.w.Save(raftpb.HardState{}, ents) if err != nil { wal.lg.Fatal("failed to save hard state and entries", zap.Error(err)) } if len(wal.ents) != 0 { wal.st.Commit = wal.ents[len(wal.ents)-1].Index } } ================================================ FILE: server/etcdserver/bootstrap_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package version implements etcd version parsing and contains latest version // information. package etcdserver import ( "encoding/json" "fmt" "io" "net/http" "os" "path/filepath" "strings" "testing" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" bolt "go.etcd.io/bbolt" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" serverstorage "go.etcd.io/etcd/server/v3/storage" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) func TestBootstrapExistingClusterNoWALMaxLearner(t *testing.T) { tests := []struct { name string members []etcdserverpb.Member maxLearner int hasError bool expectedError error }{ { name: "bootstrap success: maxLearner gt learner count", members: []etcdserverpb.Member{ {ID: 4512484362714696085, PeerURLs: []string{"http://localhost:2380"}}, {ID: 5321713336100798248, PeerURLs: []string{"http://localhost:2381"}}, {ID: 5670219998796287055, PeerURLs: []string{"http://localhost:2382"}}, }, maxLearner: 1, hasError: false, expectedError: nil, }, { name: "bootstrap success: maxLearner eq learner count", members: []etcdserverpb.Member{ {ID: 4512484362714696085, PeerURLs: []string{"http://localhost:2380"}, IsLearner: true}, {ID: 5321713336100798248, PeerURLs: []string{"http://localhost:2381"}}, {ID: 5670219998796287055, PeerURLs: []string{"http://localhost:2382"}, IsLearner: true}, }, maxLearner: 2, hasError: false, expectedError: nil, }, { name: "bootstrap fail: maxLearner lt learner count", members: []etcdserverpb.Member{ {ID: 4512484362714696085, PeerURLs: []string{"http://localhost:2380"}}, {ID: 5321713336100798248, PeerURLs: []string{"http://localhost:2381"}, IsLearner: true}, {ID: 5670219998796287055, PeerURLs: []string{"http://localhost:2382"}, IsLearner: true}, }, maxLearner: 1, hasError: true, expectedError: membership.ErrTooManyLearners, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cluster, err := types.NewURLsMap("node0=http://localhost:2380,node1=http://localhost:2381,node2=http://localhost:2382") require.NoErrorf(t, err, "unexpected error: %v", err) cfg := config.ServerConfig{ Name: "node0", InitialPeerURLsMap: cluster, Logger: zaptest.NewLogger(t), MaxLearners: tt.maxLearner, } _, err = bootstrapExistingClusterNoWAL(cfg, mockBootstrapRoundTrip(tt.members)) hasError := err != nil if hasError != tt.hasError { t.Errorf("expected error: %v got: %v", tt.hasError, err) } if hasError { require.Containsf(t, err.Error(), tt.expectedError.Error(), "expected error to contain: %q, got: %q", tt.expectedError.Error(), err.Error()) } }) } } type roundTripFunc func(r *http.Request) (*http.Response, error) func (s roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return s(r) } func mockBootstrapRoundTrip(members []etcdserverpb.Member) roundTripFunc { return func(r *http.Request) (*http.Response, error) { switch { case strings.Contains(r.URL.String(), "/members"): return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(mockMembersJSON(members))), Header: http.Header{"X-Etcd-Cluster-Id": []string{"f4588138892a16b0"}}, }, nil case strings.Contains(r.URL.String(), "/version"): return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(mockVersionJSON())), }, nil case strings.Contains(r.URL.String(), DowngradeEnabledPath): return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`false`)), }, nil } return nil, nil } } func mockVersionJSON() string { v := version.Versions{Server: "3.7.0", Cluster: "3.7.0"} version, _ := json.Marshal(v) return string(version) } func mockMembersJSON(m []etcdserverpb.Member) string { members, _ := json.Marshal(m) return string(members) } func TestBootstrapBackend(t *testing.T) { tests := []struct { name string prepareData func(config.ServerConfig) error expectedConsistentIdx uint64 expectedError error }{ { name: "bootstrap backend success: no data files", prepareData: nil, expectedConsistentIdx: 0, expectedError: nil, }, { name: "bootstrap backend success: have data files and snapshot db file", prepareData: prepareData, expectedConsistentIdx: 5, expectedError: nil, }, // TODO(ahrtr): add more test cases // https://github.com/etcd-io/etcd/issues/13507 } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { dataDir, err := createDataDir(t) require.NoErrorf(t, err, "Failed to create the data dir, unexpected error: %v", err) cfg := config.ServerConfig{ Name: "demoNode", DataDir: dataDir, BackendFreelistType: bolt.FreelistArrayType, Logger: zaptest.NewLogger(t), } if tt.prepareData != nil { err = tt.prepareData(cfg) require.NoErrorf(t, err, "failed to prepare data, unexpected error: %v", err) } haveWAL := wal.Exist(cfg.WALDir()) backend, err := bootstrapBackend(cfg, haveWAL) defer t.Cleanup(func() { backend.Close() }) hasError := err != nil expectedHasError := tt.expectedError != nil if hasError != expectedHasError { t.Errorf("expected error: %v got: %v", expectedHasError, err) } if hasError { require.Containsf(t, err.Error(), tt.expectedError.Error(), "expected error to contain: %q, got: %q", tt.expectedError.Error(), err.Error()) } if backend.ci.ConsistentIndex() != tt.expectedConsistentIdx { t.Errorf("expected consistent index: %d, got: %d", tt.expectedConsistentIdx, backend.ci.ConsistentIndex()) } }) } } func createDataDir(t *testing.T) (string, error) { var err error // create the temporary data dir dataDir := t.TempDir() // create ${dataDir}/member/snap if err = os.MkdirAll(datadir.ToSnapDir(dataDir), 0o700); err != nil { return "", err } // create ${dataDir}/member/wal err = os.MkdirAll(datadir.ToWALDir(dataDir), 0o700) if err != nil { return "", err } return dataDir, nil } // prepare data for the test case func prepareData(cfg config.ServerConfig) error { var snapshotTerm, snapshotIndex uint64 = 2, 5 if err := createWALFileWithSnapshotRecord(cfg, snapshotTerm, snapshotIndex); err != nil { return err } return createSnapshotAndBackendDB(cfg, snapshotTerm, snapshotIndex) } func createWALFileWithSnapshotRecord(cfg config.ServerConfig, snapshotTerm, snapshotIndex uint64) (err error) { var w *wal.WAL if w, err = wal.Create(cfg.Logger, cfg.WALDir(), []byte("somedata")); err != nil { return err } defer func() { err = w.Close() }() walSnap := walpb.Snapshot{ Index: &snapshotIndex, Term: &snapshotTerm, ConfState: &raftpb.ConfState{ Voters: []uint64{0x00ffca74}, AutoLeave: false, }, } if err = w.SaveSnapshot(walSnap); err != nil { return err } return w.Save(raftpb.HardState{Term: snapshotTerm, Vote: 3, Commit: snapshotIndex}, nil) } func createSnapshotAndBackendDB(cfg config.ServerConfig, snapshotTerm, snapshotIndex uint64) error { var err error confState := raftpb.ConfState{ Voters: []uint64{1, 2, 3}, } // create snapshot file ss := snap.New(cfg.Logger, cfg.SnapDir()) if err = ss.SaveSnap(raftpb.Snapshot{ Data: []byte("{}"), Metadata: raftpb.SnapshotMetadata{ ConfState: confState, Index: snapshotIndex, Term: snapshotTerm, }, }); err != nil { return err } // create snapshot db file: "%016x.snap.db" be := serverstorage.OpenBackend(cfg, nil) schema.CreateMetaBucket(be.BatchTx()) schema.UnsafeUpdateConsistentIndex(be.BatchTx(), snapshotIndex, snapshotTerm) schema.MustUnsafeSaveConfStateToBackend(cfg.Logger, be.BatchTx(), &confState) if err = be.Close(); err != nil { return err } sdb := filepath.Join(cfg.SnapDir(), fmt.Sprintf("%016x.snap.db", snapshotIndex)) if err = os.Rename(cfg.BackendPath(), sdb); err != nil { return err } // create backend db file be = serverstorage.OpenBackend(cfg, nil) schema.CreateMetaBucket(be.BatchTx()) schema.UnsafeUpdateConsistentIndex(be.BatchTx(), 1, 1) return be.Close() } ================================================ FILE: server/etcdserver/cindex/cindex.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cindex import ( "sync" "sync/atomic" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" ) type Backend interface { ReadTx() backend.ReadTx } // ConsistentIndexer is an interface that wraps the Get/Set/Save method for consistentIndex. type ConsistentIndexer interface { // ConsistentIndex returns the consistent index of current executing entry. ConsistentIndex() uint64 // ConsistentApplyingIndex returns the consistent applying index of current executing entry. ConsistentApplyingIndex() (uint64, uint64) // UnsafeConsistentIndex is similar to ConsistentIndex, but it doesn't lock the transaction. UnsafeConsistentIndex() uint64 // SetConsistentIndex set the consistent index of current executing entry. SetConsistentIndex(v uint64, term uint64) // SetConsistentApplyingIndex set the consistent applying index of current executing entry. SetConsistentApplyingIndex(v uint64, term uint64) // UnsafeSave must be called holding the lock on the tx. // It saves consistentIndex to the underlying stable storage. UnsafeSave(tx backend.UnsafeReadWriter) // SetBackend set the available backend.BatchTx for ConsistentIndexer. SetBackend(be Backend) } // consistentIndex implements the ConsistentIndexer interface. type consistentIndex struct { // consistentIndex represents the offset of an entry in a consistent replica log. // It caches the "consistent_index" key's value. // Accessed through atomics so must be 64-bit aligned. consistentIndex uint64 // term represents the RAFT term of committed entry in a consistent replica log. // Accessed through atomics so must be 64-bit aligned. // The value is being persisted in the backend since v3.5. term uint64 // applyingIndex and applyingTerm are just temporary cache of the raftpb.Entry.Index // and raftpb.Entry.Term, and they are not ready to be persisted yet. They will be // saved to consistentIndex and term above in the txPostLockInsideApplyHook. // // TODO(ahrtr): try to remove the OnPreCommitUnsafe, and compare the // performance difference. Afterwards we can make a decision on whether // or not we should remove OnPreCommitUnsafe. If it is true, then we // can remove applyingIndex and applyingTerm, and save the e.Index and // e.Term to consistentIndex and term directly in applyEntries, and // persist them into db in the txPostLockInsideApplyHook. applyingIndex uint64 applyingTerm uint64 // be is used for initial read consistentIndex be Backend // mutex is protecting be. mutex sync.Mutex } // NewConsistentIndex creates a new consistent index. // If `be` is nil, it must be set (SetBackend) before first access using `ConsistentIndex()`. func NewConsistentIndex(be Backend) ConsistentIndexer { return &consistentIndex{be: be} } func (ci *consistentIndex) ConsistentIndex() uint64 { if index := atomic.LoadUint64(&ci.consistentIndex); index > 0 { return index } ci.mutex.Lock() defer ci.mutex.Unlock() v, term := schema.ReadConsistentIndex(ci.be.ReadTx()) ci.SetConsistentIndex(v, term) return v } func (ci *consistentIndex) UnsafeConsistentIndex() uint64 { if index := atomic.LoadUint64(&ci.consistentIndex); index > 0 { return index } v, term := schema.UnsafeReadConsistentIndex(ci.be.ReadTx()) ci.SetConsistentIndex(v, term) return v } func (ci *consistentIndex) SetConsistentIndex(v uint64, term uint64) { atomic.StoreUint64(&ci.consistentIndex, v) atomic.StoreUint64(&ci.term, term) } func (ci *consistentIndex) UnsafeSave(tx backend.UnsafeReadWriter) { index := atomic.LoadUint64(&ci.consistentIndex) term := atomic.LoadUint64(&ci.term) schema.UnsafeUpdateConsistentIndex(tx, index, term) } func (ci *consistentIndex) SetBackend(be Backend) { ci.mutex.Lock() defer ci.mutex.Unlock() ci.be = be // After the backend is changed, the first access should re-read it. ci.SetConsistentIndex(0, 0) } func (ci *consistentIndex) ConsistentApplyingIndex() (uint64, uint64) { return atomic.LoadUint64(&ci.applyingIndex), atomic.LoadUint64(&ci.applyingTerm) } func (ci *consistentIndex) SetConsistentApplyingIndex(v uint64, term uint64) { atomic.StoreUint64(&ci.applyingIndex, v) atomic.StoreUint64(&ci.applyingTerm, term) } func NewFakeConsistentIndex(index uint64) ConsistentIndexer { return &fakeConsistentIndex{index: index} } type fakeConsistentIndex struct { index uint64 term uint64 } func (f *fakeConsistentIndex) ConsistentIndex() uint64 { return atomic.LoadUint64(&f.index) } func (f *fakeConsistentIndex) ConsistentApplyingIndex() (uint64, uint64) { return atomic.LoadUint64(&f.index), atomic.LoadUint64(&f.term) } func (f *fakeConsistentIndex) UnsafeConsistentIndex() uint64 { return atomic.LoadUint64(&f.index) } func (f *fakeConsistentIndex) SetConsistentIndex(index uint64, term uint64) { atomic.StoreUint64(&f.index, index) atomic.StoreUint64(&f.term, term) } func (f *fakeConsistentIndex) SetConsistentApplyingIndex(index uint64, term uint64) { atomic.StoreUint64(&f.index, index) atomic.StoreUint64(&f.term, term) } func (f *fakeConsistentIndex) UnsafeSave(_ backend.UnsafeReadWriter) {} func (f *fakeConsistentIndex) SetBackend(_ Backend) {} func UpdateConsistentIndexForce(tx backend.BatchTx, index uint64, term uint64) { tx.LockOutsideApply() defer tx.Unlock() schema.UnsafeUpdateConsistentIndexForce(tx, index, term) } ================================================ FILE: server/etcdserver/cindex/cindex_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cindex import ( "math/rand" "testing" "time" "github.com/stretchr/testify/assert" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/schema" ) // TestConsistentIndex ensures that LoadConsistentIndex/Save/ConsistentIndex and backend.BatchTx can work well together. func TestConsistentIndex(t *testing.T) { be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) ci := NewConsistentIndex(be) tx := be.BatchTx() if tx == nil { t.Fatal("batch tx is nil") } tx.Lock() schema.UnsafeCreateMetaBucket(tx) tx.Unlock() be.ForceCommit() r := uint64(7890123) term := uint64(234) ci.SetConsistentIndex(r, term) index := ci.ConsistentIndex() if index != r { t.Errorf("expected %d,got %d", r, index) } tx.Lock() ci.UnsafeSave(tx) tx.Unlock() be.ForceCommit() be.Close() b := backend.NewDefaultBackend(zaptest.NewLogger(t), tmpPath) defer b.Close() ci.SetBackend(b) index = ci.ConsistentIndex() assert.Equal(t, r, index) ci = NewConsistentIndex(b) index = ci.ConsistentIndex() assert.Equal(t, r, index) } func TestConsistentIndexDecrease(t *testing.T) { testutil.BeforeTest(t) initIndex := uint64(100) initTerm := uint64(10) tcs := []struct { name string index uint64 term uint64 panicExpected bool }{ { name: "Decrease term", index: initIndex + 1, term: initTerm - 1, panicExpected: false, // TODO: Change in v3.7 }, { name: "Decrease CI", index: initIndex - 1, term: initTerm + 1, panicExpected: true, }, { name: "Decrease CI and term", index: initIndex - 1, term: initTerm - 1, panicExpected: true, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) tx := be.BatchTx() tx.Lock() schema.UnsafeCreateMetaBucket(tx) schema.UnsafeUpdateConsistentIndex(tx, initIndex, initTerm) tx.Unlock() be.ForceCommit() be.Close() be = backend.NewDefaultBackend(zaptest.NewLogger(t), tmpPath) defer be.Close() ci := NewConsistentIndex(be) ci.SetConsistentIndex(tc.index, tc.term) tx = be.BatchTx() func() { tx.Lock() defer tx.Unlock() if tc.panicExpected { assert.Panicsf(t, func() { ci.UnsafeSave(tx) }, "Should refuse to decrease cindex") return } ci.UnsafeSave(tx) }() if !tc.panicExpected { assert.Equal(t, tc.index, ci.ConsistentIndex()) ci = NewConsistentIndex(be) assert.Equal(t, tc.index, ci.ConsistentIndex()) } }) } } func TestFakeConsistentIndex(t *testing.T) { r := rand.Uint64() ci := NewFakeConsistentIndex(r) index := ci.ConsistentIndex() if index != r { t.Errorf("expected %d,got %d", r, index) } r = rand.Uint64() ci.SetConsistentIndex(r, 5) index = ci.ConsistentIndex() if index != r { t.Errorf("expected %d,got %d", r, index) } } ================================================ FILE: server/etcdserver/cindex/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package cindex provides an interface and implementation for getting/saving consistentIndex. package cindex ================================================ FILE: server/etcdserver/cluster_util.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "context" "encoding/json" "fmt" "io" "net/http" "sort" "strconv" "strings" "time" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "google.golang.org/grpc/metadata" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/etcd/server/v3/etcdserver/errors" ) // isMemberBootstrapped tries to check if the given member has been bootstrapped // in the given cluster. func isMemberBootstrapped(lg *zap.Logger, cl *membership.RaftCluster, member string, rt http.RoundTripper, timeout time.Duration) bool { rcl, err := getClusterFromRemotePeers(lg, getRemotePeerURLs(cl, member), timeout, false, rt) if err != nil { return false } id := cl.MemberByName(member).ID m := rcl.Member(id) if m == nil { return false } if len(m.ClientURLs) > 0 { return true } return false } // GetClusterFromRemotePeers takes a set of URLs representing etcd peers, and // attempts to construct a Cluster by accessing the members endpoint on one of // these URLs. The first URL to provide a response is used. If no URLs provide // a response, or a Cluster cannot be successfully created from a received // response, an error is returned. // Each request has a 10-second timeout. Because the upper limit of TTL is 5s, // 10 second is enough for building connection and finishing request. func GetClusterFromRemotePeers(lg *zap.Logger, urls []string, rt http.RoundTripper) (*membership.RaftCluster, error) { return getClusterFromRemotePeers(lg, urls, 10*time.Second, true, rt) } // If logerr is true, it prints out more error messages. func getClusterFromRemotePeers(lg *zap.Logger, urls []string, timeout time.Duration, logerr bool, rt http.RoundTripper) (*membership.RaftCluster, error) { if lg == nil { lg = zap.NewNop() } cc := &http.Client{ Transport: rt, Timeout: timeout, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } for _, u := range urls { addr := u + "/members" resp, err := cc.Get(addr) if err != nil { if logerr { lg.Warn("failed to get cluster response", zap.String("address", addr), zap.Error(err)) } continue } b, err := io.ReadAll(resp.Body) resp.Body.Close() if err != nil { if logerr { lg.Warn("failed to read body of cluster response", zap.String("address", addr), zap.Error(err)) } continue } var membs []*membership.Member if err = json.Unmarshal(b, &membs); err != nil { if logerr { lg.Warn("failed to unmarshal cluster response", zap.String("address", addr), zap.Error(err)) } continue } id, err := types.IDFromString(resp.Header.Get("X-Etcd-Cluster-ID")) if err != nil { if logerr { lg.Warn( "failed to parse cluster ID", zap.String("address", addr), zap.String("header", resp.Header.Get("X-Etcd-Cluster-ID")), zap.Error(err), ) } continue } // check the length of membership members // if the membership members are present then prepare and return raft cluster // if membership members are not present then the raft cluster formed will be // an invalid empty cluster hence return failed to get raft cluster member(s) from the given urls error if len(membs) > 0 { return membership.NewClusterFromMembers(lg, id, membs), nil } return nil, fmt.Errorf("failed to get raft cluster member(s) from the given URLs") } return nil, fmt.Errorf("could not retrieve cluster information from the given URLs") } // getRemotePeerURLs returns peer urls of remote members in the cluster. The // returned list is sorted in ascending lexicographical order. func getRemotePeerURLs(cl *membership.RaftCluster, local string) []string { us := make([]string, 0) for _, m := range cl.Members() { if m.Name == local { continue } us = append(us, m.PeerURLs...) } sort.Strings(us) return us } // getMembersVersions returns the versions of the members in the given cluster. // The key of the returned map is the member's ID. The value of the returned map // is the semver versions string, including server and cluster. // If it fails to get the version of a member, the key will be nil. func getMembersVersions(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper, timeout time.Duration) map[string]*version.Versions { members := cl.Members() vers := make(map[string]*version.Versions) for _, m := range members { if m.ID == local { cv := "not_decided" if cl.Version() != nil { cv = cl.Version().String() } vers[m.ID.String()] = &version.Versions{Server: version.Version, Cluster: cv} continue } ver, err := getVersion(lg, m, rt, timeout) if err != nil { lg.Warn("failed to get version", zap.String("remote-member-id", m.ID.String()), zap.Error(err)) vers[m.ID.String()] = nil } else { vers[m.ID.String()] = ver } } return vers } // allowedVersionRange decides the available version range of the cluster that local server can join in; // if the downgrade enabled status is true, the version window is [oneMinorHigher, oneMinorHigher] // if the downgrade is not enabled, the version window is [MinClusterVersion, localVersion] func allowedVersionRange(downgradeEnabled bool) (minV *semver.Version, maxV *semver.Version) { minV = semver.Must(semver.NewVersion(version.MinClusterVersion)) maxV = semver.Must(semver.NewVersion(version.Version)) maxV = &semver.Version{Major: maxV.Major, Minor: maxV.Minor} if downgradeEnabled { // Todo: handle the case that downgrading from higher major version(e.g. downgrade from v4.0 to v3.x) maxV.Minor = maxV.Minor + 1 minV = &semver.Version{Major: maxV.Major, Minor: maxV.Minor} } return minV, maxV } // isCompatibleWithCluster return true if the local member has a compatible version with // the current running cluster. // The version is considered as compatible when at least one of the other members in the cluster has a // cluster version in the range of [MinV, MaxV] and no known members has a cluster version // out of the range. // We set this rule since when the local member joins, another member might be offline. func isCompatibleWithCluster(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper, timeout time.Duration) bool { vers := getMembersVersions(lg, cl, local, rt, timeout) minV, maxV := allowedVersionRange(getDowngradeEnabledFromRemotePeers(lg, cl, local, rt, timeout)) return isCompatibleWithVers(lg, vers, local, minV, maxV) } func isCompatibleWithVers(lg *zap.Logger, vers map[string]*version.Versions, local types.ID, minV, maxV *semver.Version) bool { var ok bool for id, v := range vers { // ignore comparison with local version if id == local.String() { continue } if v == nil { continue } clusterv, err := semver.NewVersion(v.Cluster) if err != nil { lg.Warn( "failed to parse cluster version of remote member", zap.String("remote-member-id", id), zap.String("remote-member-cluster-version", v.Cluster), zap.Error(err), ) continue } if clusterv.LessThan(*minV) { lg.Warn( "cluster version of remote member is not compatible; too low", zap.String("remote-member-id", id), zap.String("remote-member-cluster-version", clusterv.String()), zap.String("minimum-cluster-version-supported", minV.String()), ) return false } if maxV.LessThan(*clusterv) { lg.Warn( "cluster version of remote member is not compatible; too high", zap.String("remote-member-id", id), zap.String("remote-member-cluster-version", clusterv.String()), zap.String("maximum-cluster-version-supported", maxV.String()), ) return false } ok = true } return ok } // getVersion returns the Versions of the given member via its // peerURLs. Returns the last error if it fails to get the version. func getVersion(lg *zap.Logger, m *membership.Member, rt http.RoundTripper, timeout time.Duration) (*version.Versions, error) { cc := &http.Client{ Transport: rt, Timeout: timeout, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } var ( err error resp *http.Response ) for _, u := range m.PeerURLs { addr := u + "/version" resp, err = cc.Get(addr) if err != nil { lg.Warn( "failed to reach the peer URL", zap.String("address", addr), zap.String("remote-member-id", m.ID.String()), zap.Error(err), ) continue } var b []byte b, err = io.ReadAll(resp.Body) resp.Body.Close() if err != nil { lg.Warn( "failed to read body of response", zap.String("address", addr), zap.String("remote-member-id", m.ID.String()), zap.Error(err), ) continue } var vers version.Versions if err = json.Unmarshal(b, &vers); err != nil { lg.Warn( "failed to unmarshal response", zap.String("address", addr), zap.String("remote-member-id", m.ID.String()), zap.Error(err), ) continue } return &vers, nil } return nil, err } func promoteMemberHTTP(ctx context.Context, url string, id uint64, peerRt http.RoundTripper) ([]*membership.Member, error) { cc := &http.Client{ Transport: peerRt, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } // TODO: refactor member http handler code // cannot import etcdhttp, so manually construct url requestURL := url + "/members/promote/" + fmt.Sprintf("%d", id) req, err := http.NewRequest(http.MethodPost, requestURL, nil) if err != nil { return nil, err } // add the auth token via HTTP header if present in gRPC metadata if md, ok := metadata.FromIncomingContext(ctx); ok { ts, ok := md[rpctypes.TokenFieldNameGRPC] if !ok { ts, ok = md[rpctypes.TokenFieldNameSwagger] } if ok && len(ts) > 0 { token := ts[0] req.Header.Set("Authorization", token) } } req = req.WithContext(ctx) resp, err := cc.Do(req) if err != nil { return nil, err } defer resp.Body.Close() b, err := io.ReadAll(resp.Body) if err != nil { return nil, err } if resp.StatusCode == http.StatusRequestTimeout { return nil, errors.ErrTimeout } if resp.StatusCode == http.StatusPreconditionFailed { // both ErrMemberNotLearner and ErrLearnerNotReady have same http status code if strings.Contains(string(b), errors.ErrLearnerNotReady.Error()) { return nil, errors.ErrLearnerNotReady } if strings.Contains(string(b), membership.ErrMemberNotLearner.Error()) { return nil, membership.ErrMemberNotLearner } return nil, fmt.Errorf("member promote: unknown error(%s)", b) } if resp.StatusCode == http.StatusNotFound { return nil, membership.ErrIDNotFound } if resp.StatusCode != http.StatusOK { // all other types of errors return nil, fmt.Errorf("member promote: unknown error(%s)", b) } var membs []*membership.Member if err := json.Unmarshal(b, &membs); err != nil { return nil, err } return membs, nil } // getDowngradeEnabledFromRemotePeers will get the downgrade enabled status of the cluster. func getDowngradeEnabledFromRemotePeers(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper, timeout time.Duration) bool { members := cl.Members() for _, m := range members { if m.ID == local { continue } enable, err := getDowngradeEnabled(lg, m, rt, timeout) if err == nil { // Since the "/downgrade/enabled" serves linearized data, // this function can return once it gets a non-error response from the endpoint. return enable } lg.Warn("failed to get downgrade enabled status", zap.String("remote-member-id", m.ID.String()), zap.Error(err)) } return false } // getDowngradeEnabled returns the downgrade enabled status of the given member // via its peerURLs. Returns the last error if it fails to get it. func getDowngradeEnabled(lg *zap.Logger, m *membership.Member, rt http.RoundTripper, timeout time.Duration) (bool, error) { cc := &http.Client{ Transport: rt, Timeout: timeout, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } var ( err error resp *http.Response ) for _, u := range m.PeerURLs { addr := u + DowngradeEnabledPath resp, err = cc.Get(addr) if err != nil { lg.Warn( "failed to reach the peer URL", zap.String("address", addr), zap.String("remote-member-id", m.ID.String()), zap.Error(err), ) continue } var b []byte b, err = io.ReadAll(resp.Body) resp.Body.Close() if err != nil { lg.Warn( "failed to read body of response", zap.String("address", addr), zap.String("remote-member-id", m.ID.String()), zap.Error(err), ) continue } var enable bool if enable, err = strconv.ParseBool(string(b)); err != nil { lg.Warn( "failed to convert response", zap.String("address", addr), zap.String("remote-member-id", m.ID.String()), zap.Error(err), ) continue } return enable, nil } return false, err } func convertToClusterVersion(v string) (*semver.Version, error) { ver, err := semver.NewVersion(v) if err != nil { // allow input version format Major.Minor ver, err = semver.NewVersion(v + ".0") if err != nil { return nil, errors.ErrWrongDowngradeVersionFormat } } // cluster version only keeps major.minor, remove patch version ver = &semver.Version{Major: ver.Major, Minor: ver.Minor} return ver, nil } func GetMembershipInfoInV2Format(lg *zap.Logger, cl *membership.RaftCluster) []byte { st := v2store.New(StoreClusterPrefix, StoreKeysPrefix) cl.Store(st) d, err := st.SaveNoCopy() if err != nil { lg.Panic("failed to save v2 store", zap.Error(err)) } return d } ================================================ FILE: server/etcdserver/cluster_util_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "testing" "github.com/coreos/go-semver/semver" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/types" ) func TestIsCompatibleWithVers(t *testing.T) { tests := []struct { vers map[string]*version.Versions local types.ID minV, maxV *semver.Version wok bool }{ // too low { map[string]*version.Versions{ "a": {Server: "2.0.0", Cluster: "not_decided"}, "b": {Server: "2.1.0", Cluster: "2.1.0"}, "c": {Server: "2.1.0", Cluster: "2.1.0"}, }, 0xa, semver.Must(semver.NewVersion("2.0.0")), semver.Must(semver.NewVersion("2.0.0")), false, }, { map[string]*version.Versions{ "a": {Server: "2.1.0", Cluster: "not_decided"}, "b": {Server: "2.1.0", Cluster: "2.1.0"}, "c": {Server: "2.1.0", Cluster: "2.1.0"}, }, 0xa, semver.Must(semver.NewVersion("2.0.0")), semver.Must(semver.NewVersion("2.1.0")), true, }, // too high { map[string]*version.Versions{ "a": {Server: "2.2.0", Cluster: "not_decided"}, "b": {Server: "2.0.0", Cluster: "2.0.0"}, "c": {Server: "2.0.0", Cluster: "2.0.0"}, }, 0xa, semver.Must(semver.NewVersion("2.1.0")), semver.Must(semver.NewVersion("2.2.0")), false, }, // cannot get b's version, expect ok { map[string]*version.Versions{ "a": {Server: "2.1.0", Cluster: "not_decided"}, "b": nil, "c": {Server: "2.1.0", Cluster: "2.1.0"}, }, 0xa, semver.Must(semver.NewVersion("2.0.0")), semver.Must(semver.NewVersion("2.1.0")), true, }, // cannot get b and c's version, expect not ok { map[string]*version.Versions{ "a": {Server: "2.1.0", Cluster: "not_decided"}, "b": nil, "c": nil, }, 0xa, semver.Must(semver.NewVersion("2.0.0")), semver.Must(semver.NewVersion("2.1.0")), false, }, } for i, tt := range tests { ok := isCompatibleWithVers(zaptest.NewLogger(t), tt.vers, tt.local, tt.minV, tt.maxV) if ok != tt.wok { t.Errorf("#%d: ok = %+v, want %+v", i, ok, tt.wok) } } } func TestConvertToClusterVersion(t *testing.T) { tests := []struct { name string inputVerStr string expectedVer string hasError bool }{ { "Succeeded: Major.Minor.Patch", "3.4.2", "3.4.0", false, }, { "Succeeded: Major.Minor", "3.4", "3.4.0", false, }, { "Failed: wrong version format", "3*.9", "", true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ver, err := convertToClusterVersion(tt.inputVerStr) hasError := err != nil if hasError != tt.hasError { t.Errorf("Expected error status is %v; Got %v", tt.hasError, err) } if tt.hasError { return } if ver == nil || tt.expectedVer != ver.String() { t.Errorf("Expected output cluster version is %v; Got %v", tt.expectedVer, ver) } }) } } func TestDecideAllowedVersionRange(t *testing.T) { minClusterV := semver.Must(semver.NewVersion(version.MinClusterVersion)) localV := semver.Must(semver.NewVersion(version.Version)) localV = &semver.Version{Major: localV.Major, Minor: localV.Minor} tests := []struct { name string downgradeEnabled bool expectedMinV *semver.Version expectedMaxV *semver.Version }{ { "When cluster enables downgrade", true, &semver.Version{Major: localV.Major, Minor: localV.Minor + 1}, &semver.Version{Major: localV.Major, Minor: localV.Minor + 1}, }, { "When cluster disables downgrade", false, minClusterV, localV, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { minV, maxV := allowedVersionRange(tt.downgradeEnabled) if !minV.Equal(*tt.expectedMinV) { t.Errorf("Expected minV is %v; Got %v", tt.expectedMinV.String(), minV.String()) } if !maxV.Equal(*tt.expectedMaxV) { t.Errorf("Expected maxV is %v; Got %v", tt.expectedMaxV.String(), maxV.String()) } }) } } ================================================ FILE: server/etcdserver/corrupt.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "sort" "strings" "sync" "time" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/storage/mvcc" ) type CorruptionChecker interface { InitialCheck() error PeriodicCheck() error CompactHashCheck() } type corruptionChecker struct { lg *zap.Logger hasher Hasher mux sync.RWMutex latestRevisionChecked int64 } type Hasher interface { mvcc.HashStorage ReqTimeout() time.Duration MemberID() types.ID PeerHashByRev(int64) []*peerHashKVResp LinearizableReadNotify(context.Context) error TriggerCorruptAlarm(types.ID) } func newCorruptionChecker(lg *zap.Logger, s *EtcdServer, storage mvcc.HashStorage) *corruptionChecker { return &corruptionChecker{ lg: lg, hasher: hasherAdapter{s, storage}, } } type hasherAdapter struct { *EtcdServer mvcc.HashStorage } func (h hasherAdapter) ReqTimeout() time.Duration { return h.EtcdServer.Cfg.ReqTimeout() } func (h hasherAdapter) PeerHashByRev(rev int64) []*peerHashKVResp { return h.EtcdServer.getPeerHashKVs(rev) } func (h hasherAdapter) TriggerCorruptAlarm(memberID types.ID) { h.EtcdServer.triggerCorruptAlarm(memberID) } // InitialCheck compares initial hash values with its peers // before serving any peer/client traffic. Only mismatch when hashes // are different at requested revision, with same compact revision. func (cm *corruptionChecker) InitialCheck() error { cm.lg.Info( "starting initial corruption check", zap.String("local-member-id", cm.hasher.MemberID().String()), zap.Duration("timeout", cm.hasher.ReqTimeout()), ) h, _, err := cm.hasher.HashByRev(0) if err != nil { return fmt.Errorf("%s failed to fetch hash (%w)", cm.hasher.MemberID(), err) } peers := cm.hasher.PeerHashByRev(h.Revision) mismatch := 0 for _, p := range peers { if p.resp != nil { peerID := types.ID(p.resp.Header.MemberId) fields := []zap.Field{ zap.String("local-member-id", cm.hasher.MemberID().String()), zap.Int64("local-member-revision", h.Revision), zap.Int64("local-member-compact-revision", h.CompactRevision), zap.Uint32("local-member-hash", h.Hash), zap.String("remote-peer-id", peerID.String()), zap.Strings("remote-peer-endpoints", p.eps), zap.Int64("remote-peer-revision", p.resp.Header.Revision), zap.Int64("remote-peer-compact-revision", p.resp.CompactRevision), zap.Uint32("remote-peer-hash", p.resp.Hash), } if h.Hash != p.resp.Hash { if h.CompactRevision == p.resp.CompactRevision { cm.lg.Warn("found different hash values from remote peer", fields...) mismatch++ } else { cm.lg.Warn("found different compact revision values from remote peer", fields...) } } continue } if p.err != nil { switch { case errors.Is(p.err, rpctypes.ErrFutureRev): cm.lg.Warn( "cannot fetch hash from slow remote peer", zap.String("local-member-id", cm.hasher.MemberID().String()), zap.Int64("local-member-revision", h.Revision), zap.Int64("local-member-compact-revision", h.CompactRevision), zap.Uint32("local-member-hash", h.Hash), zap.String("remote-peer-id", p.id.String()), zap.Strings("remote-peer-endpoints", p.eps), zap.Error(err), ) case errors.Is(p.err, rpctypes.ErrCompacted): cm.lg.Warn( "cannot fetch hash from remote peer; local member is behind", zap.String("local-member-id", cm.hasher.MemberID().String()), zap.Int64("local-member-revision", h.Revision), zap.Int64("local-member-compact-revision", h.CompactRevision), zap.Uint32("local-member-hash", h.Hash), zap.String("remote-peer-id", p.id.String()), zap.Strings("remote-peer-endpoints", p.eps), zap.Error(err), ) case errors.Is(p.err, rpctypes.ErrClusterIDMismatch): cm.lg.Warn( "cluster ID mismatch", zap.String("local-member-id", cm.hasher.MemberID().String()), zap.Int64("local-member-revision", h.Revision), zap.Int64("local-member-compact-revision", h.CompactRevision), zap.Uint32("local-member-hash", h.Hash), zap.String("remote-peer-id", p.id.String()), zap.Strings("remote-peer-endpoints", p.eps), zap.Error(err), ) } } } if mismatch > 0 { return fmt.Errorf("%s found data inconsistency with peers", cm.hasher.MemberID()) } cm.lg.Info( "initial corruption checking passed; no corruption", zap.String("local-member-id", cm.hasher.MemberID().String()), ) return nil } func (cm *corruptionChecker) PeriodicCheck() error { h, _, err := cm.hasher.HashByRev(0) if err != nil { return err } peers := cm.hasher.PeerHashByRev(h.Revision) ctx, cancel := context.WithTimeout(context.Background(), cm.hasher.ReqTimeout()) err = cm.hasher.LinearizableReadNotify(ctx) cancel() if err != nil { return err } h2, rev2, err := cm.hasher.HashByRev(0) if err != nil { return err } alarmed := false mismatch := func(id types.ID) { if alarmed { return } alarmed = true cm.hasher.TriggerCorruptAlarm(id) } if h2.Hash != h.Hash && h2.Revision == h.Revision && h.CompactRevision == h2.CompactRevision { cm.lg.Warn( "found hash mismatch", zap.Int64("revision-1", h.Revision), zap.Int64("compact-revision-1", h.CompactRevision), zap.Uint32("hash-1", h.Hash), zap.Int64("revision-2", h2.Revision), zap.Int64("compact-revision-2", h2.CompactRevision), zap.Uint32("hash-2", h2.Hash), ) mismatch(cm.hasher.MemberID()) } checkedCount := 0 for _, p := range peers { if p.resp == nil { continue } checkedCount++ // leader expects follower's latest revision less than or equal to leader's if p.resp.Header.Revision > rev2 { cm.lg.Warn( "revision from follower must be less than or equal to leader's", zap.Int64("leader-revision", rev2), zap.Int64("follower-revision", p.resp.Header.Revision), zap.String("follower-peer-id", p.id.String()), ) mismatch(p.id) } // leader expects follower's latest compact revision less than or equal to leader's if p.resp.CompactRevision > h2.CompactRevision { cm.lg.Warn( "compact revision from follower must be less than or equal to leader's", zap.Int64("leader-compact-revision", h2.CompactRevision), zap.Int64("follower-compact-revision", p.resp.CompactRevision), zap.String("follower-peer-id", p.id.String()), ) mismatch(p.id) } // follower's compact revision is leader's old one, then hashes must match if p.resp.CompactRevision == h.CompactRevision && p.resp.Hash != h.Hash { cm.lg.Warn( "same compact revision then hashes must match", zap.Int64("leader-compact-revision", h2.CompactRevision), zap.Uint32("leader-hash", h.Hash), zap.Int64("follower-compact-revision", p.resp.CompactRevision), zap.Uint32("follower-hash", p.resp.Hash), zap.String("follower-peer-id", p.id.String()), ) mismatch(p.id) } } cm.lg.Info("finished peer corruption check", zap.Int("number-of-peers-checked", checkedCount)) return nil } // CompactHashCheck is based on the fact that 'compactions' are coordinated // between raft members and performed at the same revision. For each compacted // revision there is KV store hash computed and saved for some time. // // This method communicates with peers to find a recent common revision across // members, and raises alarm if 2 or more members at the same compact revision // have different hashes. // // We might miss opportunity to perform the check if the compaction is still // ongoing on one of the members, or it was unresponsive. In such situation the // method still passes without raising alarm. func (cm *corruptionChecker) CompactHashCheck() { cm.lg.Info("starting compact hash check", zap.String("local-member-id", cm.hasher.MemberID().String()), zap.Duration("timeout", cm.hasher.ReqTimeout()), ) hashes := cm.uncheckedRevisions() // Assume that revisions are ordered from largest to smallest for i, hash := range hashes { peers := cm.hasher.PeerHashByRev(hash.Revision) if len(peers) == 0 { continue } if cm.checkPeerHashes(hash, peers) { cm.lg.Info("finished compaction hash check", zap.Int("number-of-hashes-checked", i+1)) return } } cm.lg.Info("finished compaction hash check", zap.Int("number-of-hashes-checked", len(hashes))) } // check peers hash and raise alarms if detected corruption. // return a bool indicate whether to check next hash. // // true: successfully checked hash on whole cluster or raised alarms, so no need to check next hash // false: skipped some members, so need to check next hash func (cm *corruptionChecker) checkPeerHashes(leaderHash mvcc.KeyValueHash, peers []*peerHashKVResp) bool { leaderID := cm.hasher.MemberID() hash2members := map[uint32]types.IDSlice{leaderHash.Hash: {leaderID}} peersChecked := 0 // group all peers by hash for _, peer := range peers { skipped := false reason := "" if peer.resp == nil { skipped = true reason = "no response" } else if peer.resp.CompactRevision != leaderHash.CompactRevision { skipped = true reason = fmt.Sprintf("the peer's CompactRevision %d doesn't match leader's CompactRevision %d", peer.resp.CompactRevision, leaderHash.CompactRevision) } if skipped { cm.lg.Warn("Skipped peer's hash", zap.Int("number-of-peers", len(peers)), zap.String("leader-id", leaderID.String()), zap.String("peer-id", peer.id.String()), zap.String("reason", reason)) continue } peersChecked++ if ids, ok := hash2members[peer.resp.Hash]; !ok { hash2members[peer.resp.Hash] = []types.ID{peer.id} } else { ids = append(ids, peer.id) hash2members[peer.resp.Hash] = ids } } // All members have the same CompactRevision and Hash. if len(hash2members) == 1 { return cm.handleConsistentHash(leaderHash, peersChecked, len(peers)) } // Detected hashes mismatch // The first step is to figure out the majority with the same hash. memberCnt := len(peers) + 1 quorum := memberCnt/2 + 1 quorumExist := false for k, v := range hash2members { if len(v) >= quorum { quorumExist = true // remove the majority, and we might raise alarms for the left members. delete(hash2members, k) break } } if !quorumExist { // If quorum doesn't exist, we don't know which members data are // corrupted. In such situation, we intentionally set the memberID // as 0, it means it affects the whole cluster. cm.lg.Error("Detected compaction hash mismatch but cannot identify the corrupted members, so intentionally set the memberID as 0", zap.String("leader-id", leaderID.String()), zap.Int64("leader-revision", leaderHash.Revision), zap.Int64("leader-compact-revision", leaderHash.CompactRevision), zap.Uint32("leader-hash", leaderHash.Hash), ) cm.hasher.TriggerCorruptAlarm(0) } // Raise alarm for the left members if the quorum is present. // But we should always generate error log for debugging. for k, v := range hash2members { if quorumExist { for _, pid := range v { cm.hasher.TriggerCorruptAlarm(pid) } } cm.lg.Error("Detected compaction hash mismatch", zap.String("leader-id", leaderID.String()), zap.Int64("leader-revision", leaderHash.Revision), zap.Int64("leader-compact-revision", leaderHash.CompactRevision), zap.Uint32("leader-hash", leaderHash.Hash), zap.Uint32("peer-hash", k), zap.String("peer-ids", v.String()), zap.Bool("quorum-exist", quorumExist), ) } return true } func (cm *corruptionChecker) handleConsistentHash(hash mvcc.KeyValueHash, peersChecked, peerCnt int) bool { if peersChecked == peerCnt { cm.lg.Info("successfully checked hash on whole cluster", zap.Int("number-of-peers-checked", peersChecked), zap.Int64("revision", hash.Revision), zap.Int64("compactRevision", hash.CompactRevision), ) cm.mux.Lock() if hash.Revision > cm.latestRevisionChecked { cm.latestRevisionChecked = hash.Revision } cm.mux.Unlock() return true } cm.lg.Warn("skipped revision in compaction hash check; was not able to check all peers", zap.Int("number-of-peers-checked", peersChecked), zap.Int("number-of-peers", peerCnt), zap.Int64("revision", hash.Revision), zap.Int64("compactRevision", hash.CompactRevision), ) // The only case which needs to check next hash return false } func (cm *corruptionChecker) uncheckedRevisions() []mvcc.KeyValueHash { cm.mux.RLock() lastRevisionChecked := cm.latestRevisionChecked cm.mux.RUnlock() hashes := cm.hasher.Hashes() // Sort in descending order sort.Slice(hashes, func(i, j int) bool { return hashes[i].Revision > hashes[j].Revision }) for i, hash := range hashes { if hash.Revision <= lastRevisionChecked { return hashes[:i] } } return hashes } func (s *EtcdServer) triggerCorruptAlarm(id types.ID) { a := &pb.AlarmRequest{ MemberID: uint64(id), Action: pb.AlarmRequest_ACTIVATE, Alarm: pb.AlarmType_CORRUPT, } s.GoAttach(func() { s.raftRequest(s.ctx, pb.InternalRaftRequest{Alarm: a}) }) } type peerInfo struct { id types.ID eps []string } type peerHashKVResp struct { peerInfo resp *pb.HashKVResponse err error } func (s *EtcdServer) getPeerHashKVs(rev int64) []*peerHashKVResp { // TODO: handle the case when "s.cluster.Members" have not // been populated (e.g. no snapshot to load from disk) members := s.cluster.Members() peers := make([]peerInfo, 0, len(members)) for _, m := range members { if m.ID == s.MemberID() { continue } peers = append(peers, peerInfo{id: m.ID, eps: m.PeerURLs}) } lg := s.Logger() cc := &http.Client{ Transport: s.peerRt, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } var resps []*peerHashKVResp for _, p := range peers { if len(p.eps) == 0 { continue } respsLen := len(resps) var lastErr error for _, ep := range p.eps { var resp *pb.HashKVResponse ctx, cancel := context.WithTimeout(context.Background(), s.Cfg.ReqTimeout()) resp, lastErr = HashByRev(ctx, s.cluster.ID(), cc, ep, rev) cancel() if lastErr == nil { resps = append(resps, &peerHashKVResp{peerInfo: p, resp: resp, err: nil}) break } lg.Warn( "failed hash kv request", zap.String("local-member-id", s.MemberID().String()), zap.Int64("requested-revision", rev), zap.String("remote-peer-endpoint", ep), zap.Error(lastErr), ) } // failed to get hashKV from all endpoints of this peer if respsLen == len(resps) { resps = append(resps, &peerHashKVResp{peerInfo: p, resp: nil, err: lastErr}) } } return resps } const PeerHashKVPath = "/members/hashkv" type hashKVHandler struct { lg *zap.Logger server *EtcdServer } func (s *EtcdServer) HashKVHandler() http.Handler { return &hashKVHandler{lg: s.Logger(), server: s} } func (h *hashKVHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { w.Header().Set("Allow", http.MethodGet) http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) return } if r.URL.Path != PeerHashKVPath { http.Error(w, "bad path", http.StatusBadRequest) return } if gcid := r.Header.Get("X-Etcd-Cluster-ID"); gcid != "" && gcid != h.server.cluster.ID().String() { http.Error(w, rafthttp.ErrClusterIDMismatch.Error(), http.StatusPreconditionFailed) return } defer r.Body.Close() b, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "error reading body", http.StatusBadRequest) return } req := &pb.HashKVRequest{} if err = json.Unmarshal(b, req); err != nil { h.lg.Warn("failed to unmarshal request", zap.Error(err)) http.Error(w, "error unmarshalling request", http.StatusBadRequest) return } hash, rev, err := h.server.KV().HashStorage().HashByRev(req.Revision) if err != nil { h.lg.Warn( "failed to get hashKV", zap.Int64("requested-revision", req.Revision), zap.Error(err), ) http.Error(w, err.Error(), http.StatusBadRequest) return } resp := &pb.HashKVResponse{ Header: &pb.ResponseHeader{Revision: rev}, Hash: hash.Hash, CompactRevision: hash.CompactRevision, HashRevision: hash.Revision, } respBytes, err := json.Marshal(resp) if err != nil { h.lg.Warn("failed to marshal hashKV response", zap.Error(err)) http.Error(w, err.Error(), http.StatusInternalServerError) return } w.Header().Set("X-Etcd-Cluster-ID", h.server.Cluster().ID().String()) w.Header().Set("Content-Type", "application/json") w.Write(respBytes) } // HashByRev fetch hash of kv store at the given rev via http call to the given url func HashByRev(ctx context.Context, cid types.ID, cc *http.Client, url string, rev int64) (*pb.HashKVResponse, error) { hashReq := &pb.HashKVRequest{Revision: rev} hashReqBytes, err := json.Marshal(hashReq) if err != nil { return nil, err } requestURL := url + PeerHashKVPath req, err := http.NewRequest(http.MethodGet, requestURL, bytes.NewReader(hashReqBytes)) if err != nil { return nil, err } req = req.WithContext(ctx) req.Header.Set("Content-Type", "application/json") req.Header.Set("X-Etcd-Cluster-ID", cid.String()) req.Cancel = ctx.Done() //nolint:staticcheck // TODO: remove for a supported version resp, err := cc.Do(req) if err != nil { return nil, err } defer resp.Body.Close() b, err := io.ReadAll(resp.Body) if err != nil { return nil, err } if resp.StatusCode == http.StatusBadRequest { if strings.Contains(string(b), mvcc.ErrCompacted.Error()) { return nil, rpctypes.ErrCompacted } if strings.Contains(string(b), mvcc.ErrFutureRev.Error()) { return nil, rpctypes.ErrFutureRev } } else if resp.StatusCode == http.StatusPreconditionFailed { if strings.Contains(string(b), rafthttp.ErrClusterIDMismatch.Error()) { return nil, rpctypes.ErrClusterIDMismatch } } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("unknown error: %s", b) } hashResp := &pb.HashKVResponse{} if err := json.Unmarshal(b, hashResp); err != nil { return nil, err } return hashResp, nil } ================================================ FILE: server/etcdserver/corrupt_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "net/http/httptest" "strconv" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/mvcc" ) func TestInitialCheck(t *testing.T) { tcs := []struct { name string hasher fakeHasher expectError bool expectCorrupt bool expectActions []string }{ { name: "No peers", hasher: fakeHasher{ hashByRevResponses: []hashByRev{{hash: mvcc.KeyValueHash{Revision: 10}}}, }, expectActions: []string{"MemberID()", "ReqTimeout()", "HashByRev(0)", "PeerHashByRev(10)", "MemberID()"}, }, { name: "Error getting hash", hasher: fakeHasher{hashByRevResponses: []hashByRev{{err: fmt.Errorf("error getting hash")}}}, expectActions: []string{"MemberID()", "ReqTimeout()", "HashByRev(0)", "MemberID()"}, expectError: true, }, { name: "Peer with empty response", hasher: fakeHasher{peerHashes: []*peerHashKVResp{{}}}, expectActions: []string{"MemberID()", "ReqTimeout()", "HashByRev(0)", "PeerHashByRev(0)", "MemberID()"}, }, { name: "Peer returned ErrFutureRev", hasher: fakeHasher{peerHashes: []*peerHashKVResp{{err: rpctypes.ErrFutureRev}}}, expectActions: []string{"MemberID()", "ReqTimeout()", "HashByRev(0)", "PeerHashByRev(0)", "MemberID()", "MemberID()"}, }, { name: "Peer returned ErrCompacted", hasher: fakeHasher{peerHashes: []*peerHashKVResp{{err: rpctypes.ErrCompacted}}}, expectActions: []string{"MemberID()", "ReqTimeout()", "HashByRev(0)", "PeerHashByRev(0)", "MemberID()", "MemberID()"}, }, { name: "Peer returned other error", hasher: fakeHasher{peerHashes: []*peerHashKVResp{{err: rpctypes.ErrCorrupt}}}, expectActions: []string{"MemberID()", "ReqTimeout()", "HashByRev(0)", "PeerHashByRev(0)", "MemberID()"}, }, { name: "Peer returned same hash", hasher: fakeHasher{hashByRevResponses: []hashByRev{{hash: mvcc.KeyValueHash{Hash: 1}}}, peerHashes: []*peerHashKVResp{{resp: &pb.HashKVResponse{Header: &pb.ResponseHeader{}, Hash: 1}}}}, expectActions: []string{"MemberID()", "ReqTimeout()", "HashByRev(0)", "PeerHashByRev(0)", "MemberID()", "MemberID()"}, }, { name: "Peer returned different hash with same compaction rev", hasher: fakeHasher{hashByRevResponses: []hashByRev{{hash: mvcc.KeyValueHash{Hash: 1, CompactRevision: 1}}}, peerHashes: []*peerHashKVResp{{resp: &pb.HashKVResponse{Header: &pb.ResponseHeader{}, Hash: 2, CompactRevision: 1}}}}, expectActions: []string{"MemberID()", "ReqTimeout()", "HashByRev(0)", "PeerHashByRev(0)", "MemberID()", "MemberID()"}, expectError: true, }, { name: "Peer returned different hash and compaction rev", hasher: fakeHasher{hashByRevResponses: []hashByRev{{hash: mvcc.KeyValueHash{Hash: 1, CompactRevision: 1}}}, peerHashes: []*peerHashKVResp{{resp: &pb.HashKVResponse{Header: &pb.ResponseHeader{}, Hash: 2, CompactRevision: 2}}}}, expectActions: []string{"MemberID()", "ReqTimeout()", "HashByRev(0)", "PeerHashByRev(0)", "MemberID()", "MemberID()"}, }, { name: "Cluster ID Mismatch does not fail CorruptionChecker.InitialCheck()", hasher: fakeHasher{ peerHashes: []*peerHashKVResp{{err: rpctypes.ErrClusterIDMismatch}}, }, expectActions: []string{"MemberID()", "ReqTimeout()", "HashByRev(0)", "PeerHashByRev(0)", "MemberID()", "MemberID()"}, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { monitor := corruptionChecker{ lg: zaptest.NewLogger(t), hasher: &tc.hasher, } err := monitor.InitialCheck() if gotError := err != nil; gotError != tc.expectError { t.Errorf("Unexpected error, got: %v, expected?: %v", err, tc.expectError) } if tc.hasher.alarmTriggered != tc.expectCorrupt { t.Errorf("Unexpected corrupt triggered, got: %v, expected?: %v", tc.hasher.alarmTriggered, tc.expectCorrupt) } assert.Equal(t, tc.expectActions, tc.hasher.actions) }) } } func TestPeriodicCheck(t *testing.T) { tcs := []struct { name string hasher fakeHasher expectError bool expectCorrupt bool expectActions []string }{ { name: "Same local hash and no peers", hasher: fakeHasher{hashByRevResponses: []hashByRev{{hash: mvcc.KeyValueHash{Revision: 10}}, {hash: mvcc.KeyValueHash{Revision: 10}}}}, expectActions: []string{"HashByRev(0)", "PeerHashByRev(10)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)"}, }, { name: "Error getting hash first time", hasher: fakeHasher{hashByRevResponses: []hashByRev{{err: fmt.Errorf("error getting hash")}}}, expectActions: []string{"HashByRev(0)"}, expectError: true, }, { name: "Error getting hash second time", hasher: fakeHasher{hashByRevResponses: []hashByRev{{hash: mvcc.KeyValueHash{Revision: 11}}, {err: fmt.Errorf("error getting hash")}}}, expectActions: []string{"HashByRev(0)", "PeerHashByRev(11)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)"}, expectError: true, }, { name: "Error linearizableReadNotify", hasher: fakeHasher{linearizableReadNotify: fmt.Errorf("error getting linearizableReadNotify")}, expectActions: []string{"HashByRev(0)", "PeerHashByRev(0)", "ReqTimeout()", "LinearizableReadNotify()"}, expectError: true, }, { name: "Different local hash and revision", hasher: fakeHasher{hashByRevResponses: []hashByRev{{hash: mvcc.KeyValueHash{Hash: 1, Revision: 1}, revision: 1}, {hash: mvcc.KeyValueHash{Hash: 2}, revision: 2}}}, expectActions: []string{"HashByRev(0)", "PeerHashByRev(1)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)"}, }, { name: "Different local hash and compaction revision", hasher: fakeHasher{hashByRevResponses: []hashByRev{{hash: mvcc.KeyValueHash{Hash: 1, CompactRevision: 1}}, {hash: mvcc.KeyValueHash{Hash: 2, CompactRevision: 2}}}}, expectActions: []string{"HashByRev(0)", "PeerHashByRev(0)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)"}, }, { name: "Different local hash and same revisions", hasher: fakeHasher{hashByRevResponses: []hashByRev{{hash: mvcc.KeyValueHash{Hash: 1, CompactRevision: 1, Revision: 1}, revision: 1}, {hash: mvcc.KeyValueHash{Hash: 2, CompactRevision: 1, Revision: 1}, revision: 1}}}, expectActions: []string{"HashByRev(0)", "PeerHashByRev(1)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)", "MemberID()", "TriggerCorruptAlarm(1)"}, expectCorrupt: true, }, { name: "Peer with nil response", hasher: fakeHasher{ peerHashes: []*peerHashKVResp{{}}, }, expectActions: []string{"HashByRev(0)", "PeerHashByRev(0)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)"}, }, { name: "Peer with newer revision", hasher: fakeHasher{ peerHashes: []*peerHashKVResp{{peerInfo: peerInfo{id: 42}, resp: &pb.HashKVResponse{Header: &pb.ResponseHeader{Revision: 1}}}}, }, expectActions: []string{"HashByRev(0)", "PeerHashByRev(0)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)", "TriggerCorruptAlarm(42)"}, expectCorrupt: true, }, { name: "Peer with newer compact revision", hasher: fakeHasher{ peerHashes: []*peerHashKVResp{{peerInfo: peerInfo{id: 88}, resp: &pb.HashKVResponse{Header: &pb.ResponseHeader{Revision: 10}, CompactRevision: 2}}}, }, expectActions: []string{"HashByRev(0)", "PeerHashByRev(0)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)", "TriggerCorruptAlarm(88)"}, expectCorrupt: true, }, { name: "Peer with same hash and compact revision", hasher: fakeHasher{ hashByRevResponses: []hashByRev{{hash: mvcc.KeyValueHash{Hash: 1, CompactRevision: 1, Revision: 1}, revision: 1}, {hash: mvcc.KeyValueHash{Hash: 2, CompactRevision: 2, Revision: 2}, revision: 2}}, peerHashes: []*peerHashKVResp{{resp: &pb.HashKVResponse{Header: &pb.ResponseHeader{Revision: 1}, CompactRevision: 1, Hash: 1}}}, }, expectActions: []string{"HashByRev(0)", "PeerHashByRev(1)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)"}, }, { name: "Peer with different hash and same compact revision as first local", hasher: fakeHasher{ hashByRevResponses: []hashByRev{{hash: mvcc.KeyValueHash{Hash: 1, CompactRevision: 1, Revision: 1}, revision: 1}, {hash: mvcc.KeyValueHash{Hash: 2, CompactRevision: 2}, revision: 2}}, peerHashes: []*peerHashKVResp{{peerInfo: peerInfo{id: 666}, resp: &pb.HashKVResponse{Header: &pb.ResponseHeader{Revision: 1}, CompactRevision: 1, Hash: 2}}}, }, expectActions: []string{"HashByRev(0)", "PeerHashByRev(1)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)", "TriggerCorruptAlarm(666)"}, expectCorrupt: true, }, { name: "Multiple corrupted peers trigger one alarm", hasher: fakeHasher{ peerHashes: []*peerHashKVResp{ {peerInfo: peerInfo{id: 88}, resp: &pb.HashKVResponse{Header: &pb.ResponseHeader{Revision: 10}, CompactRevision: 2}}, {peerInfo: peerInfo{id: 89}, resp: &pb.HashKVResponse{Header: &pb.ResponseHeader{Revision: 10}, CompactRevision: 2}}, }, }, expectActions: []string{"HashByRev(0)", "PeerHashByRev(0)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)", "TriggerCorruptAlarm(88)"}, expectCorrupt: true, }, { name: "Cluster ID Mismatch does not fail CorruptionChecker.PeriodicCheck()", hasher: fakeHasher{ peerHashes: []*peerHashKVResp{{err: rpctypes.ErrClusterIDMismatch}}, }, expectActions: []string{"HashByRev(0)", "PeerHashByRev(0)", "ReqTimeout()", "LinearizableReadNotify()", "HashByRev(0)"}, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { monitor := corruptionChecker{ lg: zaptest.NewLogger(t), hasher: &tc.hasher, } err := monitor.PeriodicCheck() if gotError := err != nil; gotError != tc.expectError { t.Errorf("Unexpected error, got: %v, expected?: %v", err, tc.expectError) } if tc.hasher.alarmTriggered != tc.expectCorrupt { t.Errorf("Unexpected corrupt triggered, got: %v, expected?: %v", tc.hasher.alarmTriggered, tc.expectCorrupt) } assert.Equal(t, tc.expectActions, tc.hasher.actions) }) } } func TestCompactHashCheck(t *testing.T) { tcs := []struct { name string hasher fakeHasher lastRevisionChecked int64 expectError bool expectCorrupt bool expectActions []string expectLastRevisionChecked int64 }{ { name: "No hashes", expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()"}, }, { name: "No peers, check new checked from largest to smallest", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1}, {Revision: 2}, {Revision: 3}, {Revision: 4}}, }, lastRevisionChecked: 2, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(4)", "PeerHashByRev(3)"}, expectLastRevisionChecked: 2, }, { name: "Peer error", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1}, {Revision: 2}}, peerHashes: []*peerHashKVResp{{err: fmt.Errorf("failed getting hash")}}, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(2)", "MemberID()", "PeerHashByRev(1)", "MemberID()"}, }, { name: "Peer returned different compaction revision is skipped", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1}, {Revision: 2, CompactRevision: 2}}, peerHashes: []*peerHashKVResp{{resp: &pb.HashKVResponse{CompactRevision: 3}}}, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(2)", "MemberID()", "PeerHashByRev(1)", "MemberID()"}, }, { name: "Etcd can identify two corrupted members in 5 member cluster", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1, Hash: 1}, {Revision: 2, CompactRevision: 1, Hash: 2}}, peerHashes: []*peerHashKVResp{ {peerInfo: peerInfo{id: 42}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 2}}, {peerInfo: peerInfo{id: 43}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 2}}, {peerInfo: peerInfo{id: 44}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 7}}, {peerInfo: peerInfo{id: 45}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 7}}, }, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(2)", "MemberID()", "TriggerCorruptAlarm(44)", "TriggerCorruptAlarm(45)"}, expectCorrupt: true, }, { name: "Etcd checks next hash when one member is unresponsive in 3 member cluster", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1, Hash: 2}, {Revision: 2, CompactRevision: 1, Hash: 2}}, peerHashes: []*peerHashKVResp{ {err: fmt.Errorf("failed getting hash")}, {peerInfo: peerInfo{id: 43}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 2}}, }, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(2)", "MemberID()", "PeerHashByRev(1)", "MemberID()"}, expectCorrupt: false, }, { name: "Etcd can identify single corrupted member in 3 member cluster", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1, Hash: 2}, {Revision: 2, CompactRevision: 1, Hash: 2}}, peerHashes: []*peerHashKVResp{ {peerInfo: peerInfo{id: 42}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 2}}, {peerInfo: peerInfo{id: 43}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 3}}, }, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(2)", "MemberID()", "TriggerCorruptAlarm(43)"}, expectCorrupt: true, }, { name: "Etcd can identify single corrupted member in 5 member cluster", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1, Hash: 2}, {Revision: 2, CompactRevision: 1, Hash: 2}}, peerHashes: []*peerHashKVResp{ {peerInfo: peerInfo{id: 42}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 2}}, {peerInfo: peerInfo{id: 43}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 2}}, {peerInfo: peerInfo{id: 44}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 3}}, {peerInfo: peerInfo{id: 45}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 2}}, }, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(2)", "MemberID()", "TriggerCorruptAlarm(44)"}, expectCorrupt: true, }, { name: "Etcd triggers corrupted alarm on whole cluster if in 3 member cluster one member is down and one member corrupted", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1, Hash: 2}, {Revision: 2, CompactRevision: 1, Hash: 2}}, peerHashes: []*peerHashKVResp{ {err: fmt.Errorf("failed getting hash")}, {peerInfo: peerInfo{id: 43}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 3}}, }, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(2)", "MemberID()", "TriggerCorruptAlarm(0)"}, expectCorrupt: true, }, { name: "Etcd triggers corrupted alarm on whole cluster if no quorum in 5 member cluster", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1, Hash: 1}, {Revision: 2, CompactRevision: 1, Hash: 2}}, peerHashes: []*peerHashKVResp{ {peerInfo: peerInfo{id: 42}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 2}}, {peerInfo: peerInfo{id: 43}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 3}}, {peerInfo: peerInfo{id: 44}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 3}}, {peerInfo: peerInfo{id: 45}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 3}}, {peerInfo: peerInfo{id: 46}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 4}}, {peerInfo: peerInfo{id: 47}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 2}}, }, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(2)", "MemberID()", "TriggerCorruptAlarm(0)"}, expectCorrupt: true, }, { name: "Etcd can identify corrupted member in 5 member cluster even if one member is down", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1, Hash: 2}, {Revision: 2, CompactRevision: 1, Hash: 2}}, peerHashes: []*peerHashKVResp{ {peerInfo: peerInfo{id: 42}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 2}}, {err: fmt.Errorf("failed getting hash")}, {peerInfo: peerInfo{id: 44}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 3}}, {peerInfo: peerInfo{id: 45}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 2}}, }, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(2)", "MemberID()", "TriggerCorruptAlarm(44)"}, expectCorrupt: true, }, { name: "Etcd can identify that leader is corrupted", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1, Hash: 2}, {Revision: 2, CompactRevision: 1, Hash: 2}}, peerHashes: []*peerHashKVResp{ {peerInfo: peerInfo{id: 42}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 3}}, {peerInfo: peerInfo{id: 43}, resp: &pb.HashKVResponse{CompactRevision: 1, Hash: 3}}, }, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(2)", "MemberID()", "TriggerCorruptAlarm(1)"}, expectCorrupt: true, }, { name: "Peer returned same hash bumps last revision checked", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1, Hash: 1}, {Revision: 2, CompactRevision: 1, Hash: 1}}, peerHashes: []*peerHashKVResp{{resp: &pb.HashKVResponse{Header: &pb.ResponseHeader{MemberId: 42}, CompactRevision: 1, Hash: 1}}}, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(2)", "MemberID()"}, expectLastRevisionChecked: 2, }, { name: "Only one peer succeeded check", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1, Hash: 1}}, peerHashes: []*peerHashKVResp{ {resp: &pb.HashKVResponse{Header: &pb.ResponseHeader{MemberId: 42}, CompactRevision: 1, Hash: 1}}, {err: fmt.Errorf("failed getting hash")}, }, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(1)", "MemberID()"}, }, { name: "Cluster ID Mismatch does not fail CorruptionChecker.CompactHashCheck()", hasher: fakeHasher{ hashes: []mvcc.KeyValueHash{{Revision: 1, CompactRevision: 1, Hash: 1}}, peerHashes: []*peerHashKVResp{{err: rpctypes.ErrClusterIDMismatch}}, }, expectActions: []string{"MemberID()", "ReqTimeout()", "Hashes()", "PeerHashByRev(1)", "MemberID()"}, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { monitor := corruptionChecker{ latestRevisionChecked: tc.lastRevisionChecked, lg: zaptest.NewLogger(t), hasher: &tc.hasher, } monitor.CompactHashCheck() if tc.hasher.alarmTriggered != tc.expectCorrupt { t.Errorf("Unexpected corrupt triggered, got: %v, expected?: %v", tc.hasher.alarmTriggered, tc.expectCorrupt) } if tc.expectLastRevisionChecked != monitor.latestRevisionChecked { t.Errorf("Unexpected last revision checked, got: %v, expected?: %v", monitor.latestRevisionChecked, tc.expectLastRevisionChecked) } assert.Equal(t, tc.expectActions, tc.hasher.actions) }) } } type fakeHasher struct { peerHashes []*peerHashKVResp hashByRevIndex int hashByRevResponses []hashByRev linearizableReadNotify error hashes []mvcc.KeyValueHash alarmTriggered bool actions []string } type hashByRev struct { hash mvcc.KeyValueHash revision int64 err error } func (f *fakeHasher) Hash() (hash uint32, revision int64, err error) { panic("not implemented") } func (f *fakeHasher) HashByRev(rev int64) (hash mvcc.KeyValueHash, revision int64, err error) { f.actions = append(f.actions, fmt.Sprintf("HashByRev(%d)", rev)) if len(f.hashByRevResponses) == 0 { return mvcc.KeyValueHash{}, 0, nil } hashByRev := f.hashByRevResponses[f.hashByRevIndex] f.hashByRevIndex++ return hashByRev.hash, hashByRev.revision, hashByRev.err } func (f *fakeHasher) Store(hash mvcc.KeyValueHash) { f.actions = append(f.actions, fmt.Sprintf("Store(%v)", hash)) f.hashes = append(f.hashes, hash) } func (f *fakeHasher) Hashes() []mvcc.KeyValueHash { f.actions = append(f.actions, "Hashes()") return f.hashes } func (f *fakeHasher) ReqTimeout() time.Duration { f.actions = append(f.actions, "ReqTimeout()") return time.Second } func (f *fakeHasher) MemberID() types.ID { f.actions = append(f.actions, "MemberID()") return 1 } func (f *fakeHasher) PeerHashByRev(rev int64) []*peerHashKVResp { f.actions = append(f.actions, fmt.Sprintf("PeerHashByRev(%d)", rev)) return f.peerHashes } func (f *fakeHasher) LinearizableReadNotify(ctx context.Context) error { f.actions = append(f.actions, "LinearizableReadNotify()") return f.linearizableReadNotify } func (f *fakeHasher) TriggerCorruptAlarm(memberID types.ID) { f.actions = append(f.actions, fmt.Sprintf("TriggerCorruptAlarm(%d)", memberID)) f.alarmTriggered = true } func TestHashKVHandler(t *testing.T) { remoteClusterID := 111195 localClusterID := 111196 revision := 1 etcdSrv := &EtcdServer{} etcdSrv.cluster = newTestCluster(t) etcdSrv.cluster.SetID(types.ID(localClusterID), types.ID(localClusterID)) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) etcdSrv.kv = mvcc.New(zap.NewNop(), be, &lease.FakeLessor{}, mvcc.StoreConfig{}) defer func() { assert.NoError(t, etcdSrv.kv.Close()) }() ph := &hashKVHandler{ lg: zap.NewNop(), server: etcdSrv, } srv := httptest.NewServer(ph) defer srv.Close() tests := []struct { name string remoteClusterID int wcode int wKeyWords string }{ { name: "HashKV returns 200 if cluster hash matches", remoteClusterID: localClusterID, wcode: http.StatusOK, wKeyWords: "", }, { name: "HashKV returns 400 if cluster hash doesn't matche", remoteClusterID: remoteClusterID, wcode: http.StatusPreconditionFailed, wKeyWords: "cluster ID mismatch", }, } for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { hashReq := &pb.HashKVRequest{Revision: int64(revision)} hashReqBytes, err := json.Marshal(hashReq) require.NoErrorf(t, err, "failed to marshal request: %v", err) req, err := http.NewRequest(http.MethodGet, srv.URL+PeerHashKVPath, bytes.NewReader(hashReqBytes)) require.NoErrorf(t, err, "failed to create request: %v", err) req.Header.Set("X-Etcd-Cluster-ID", strconv.FormatUint(uint64(tt.remoteClusterID), 16)) resp, err := http.DefaultClient.Do(req) require.NoErrorf(t, err, "failed to get http response: %v", err) body, err := io.ReadAll(resp.Body) resp.Body.Close() require.NoErrorf(t, err, "unexpected io.ReadAll error: %v", err) require.Equalf(t, resp.StatusCode, tt.wcode, "#%d: code = %d, want %d", i, resp.StatusCode, tt.wcode) if resp.StatusCode != http.StatusOK { if !strings.Contains(string(body), tt.wKeyWords) { t.Errorf("#%d: body: %s, want body to contain keywords: %s", i, body, tt.wKeyWords) } return } hashKVResponse := pb.HashKVResponse{} err = json.Unmarshal(body, &hashKVResponse) require.NoErrorf(t, err, "unmarshal response error: %v", err) hashValue, _, err := etcdSrv.KV().HashStorage().HashByRev(int64(revision)) require.NoErrorf(t, err, "etcd server hash failed: %v", err) require.Equalf(t, hashKVResponse.Hash, hashValue.Hash, "hash value inconsistent: %d != %d", hashKVResponse.Hash, hashValue) }) } } ================================================ FILE: server/etcdserver/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package etcdserver defines how etcd servers interact and store their states. package etcdserver ================================================ FILE: server/etcdserver/errors/errors.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //revive:disable-next-line:var-naming package errors import ( "errors" "fmt" ) var ( ErrUnknownMethod = errors.New("etcdserver: unknown method") ErrStopped = errors.New("etcdserver: server stopped") ErrCanceled = errors.New("etcdserver: request cancelled") ErrTimeout = errors.New("etcdserver: request timed out") ErrTimeoutDueToLeaderFail = errors.New("etcdserver: request timed out, possibly due to previous leader failure") ErrTimeoutDueToConnectionLost = errors.New("etcdserver: request timed out, possibly due to connection lost") ErrTimeoutLeaderTransfer = errors.New("etcdserver: request timed out, leader transfer took too long") ErrTimeoutWaitAppliedIndex = errors.New("etcdserver: request timed out, waiting for the applied index took too long") ErrLeaderChanged = errors.New("etcdserver: leader changed") ErrNotEnoughStartedMembers = errors.New("etcdserver: re-configuration failed due to not enough started members") ErrLearnerNotReady = errors.New("etcdserver: can only promote a learner member which is in sync with leader") ErrNoLeader = errors.New("etcdserver: no leader") ErrNotLeader = errors.New("etcdserver: not leader") ErrRequestTooLarge = errors.New("etcdserver: request is too large") ErrNoSpace = errors.New("etcdserver: no space") ErrTooManyRequests = errors.New("etcdserver: too many requests") ErrUnhealthy = errors.New("etcdserver: unhealthy cluster") ErrCorrupt = errors.New("etcdserver: corrupt cluster") ErrBadLeaderTransferee = errors.New("etcdserver: bad leader transferee") ErrClusterVersionUnavailable = errors.New("etcdserver: cluster version not found during downgrade") ErrWrongDowngradeVersionFormat = errors.New("etcdserver: wrong downgrade target version format") ErrKeyNotFound = errors.New("etcdserver: key not found") ) type DiscoveryError struct { Op string Err error } func (e DiscoveryError) Error() string { return fmt.Sprintf("failed to %s discovery cluster (%v)", e.Op, e.Err) } ================================================ FILE: server/etcdserver/metrics.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( goruntime "runtime" "time" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/pkg/v3/runtime" ) var ( hasLeader = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "has_leader", Help: "Whether or not a leader exists. 1 is existence, 0 is not.", }) isLeader = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "is_leader", Help: "Whether or not this member is a leader. 1 if is, 0 otherwise.", }) leaderChanges = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "leader_changes_seen_total", Help: "The number of leader changes seen.", }) learnerPromoteFailed = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "learner_promote_failures", Help: "The total number of failed learner promotions (likely learner not ready) while this member is leader.", }, []string{"Reason"}, ) learnerPromoteSucceed = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "learner_promote_successes", Help: "The total number of successful learner promotions while this member is leader.", }) heartbeatSendFailures = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "heartbeat_send_failures_total", Help: "The total number of leader heartbeat send failures (likely overloaded from slow disk).", }) applySnapshotInProgress = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "snapshot_apply_in_progress_total", Help: "1 if the server is applying the incoming snapshot. 0 if none.", }) proposalsCommitted = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "proposals_committed_total", Help: "The total number of consensus proposals committed.", }) proposalsApplied = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "proposals_applied_total", Help: "The total number of consensus proposals applied.", }) proposalsPending = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "proposals_pending", Help: "The current number of pending proposals to commit.", }) proposalsFailed = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "proposals_failed_total", Help: "The total number of failed proposals seen.", }) slowReadIndex = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "slow_read_indexes_total", Help: "The total number of pending read indexes not in sync with leader's or timed out read index requests.", }) readIndexFailed = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "read_indexes_failed_total", Help: "The total number of failed read indexes seen.", }) requestDurationSec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "server", Name: "request_duration_seconds", Help: "Response latency distribution in seconds for each type.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }, []string{"type", "success"}, ) leaseExpired = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "server", Name: "lease_expired_total", Help: "The total number of expired leases.", }) currentVersion = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "version", Help: "Which version is running. 1 for 'server_version' label with current version.", }, []string{"server_version"}, ) currentGoVersion = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "go_version", Help: "Which Go version server is running with. 1 for 'server_go_version' label with current version.", }, []string{"server_go_version"}, ) serverID = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "id", Help: "Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID.", }, []string{"server_id"}, ) serverFeatureEnabled = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "etcd_server_feature_enabled", Help: "Whether or not a feature is enabled. 1 is enabled, 0 is not.", }, []string{"name", "stage"}, ) fdUsed = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "os", Subsystem: "fd", Name: "used", Help: "The number of used file descriptors.", }) fdLimit = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "os", Subsystem: "fd", Name: "limit", Help: "The file descriptor limit.", }) ) func init() { prometheus.MustRegister(hasLeader) prometheus.MustRegister(isLeader) prometheus.MustRegister(leaderChanges) prometheus.MustRegister(heartbeatSendFailures) prometheus.MustRegister(applySnapshotInProgress) prometheus.MustRegister(proposalsCommitted) prometheus.MustRegister(proposalsApplied) prometheus.MustRegister(proposalsPending) prometheus.MustRegister(proposalsFailed) prometheus.MustRegister(slowReadIndex) prometheus.MustRegister(readIndexFailed) prometheus.MustRegister(requestDurationSec) prometheus.MustRegister(leaseExpired) prometheus.MustRegister(currentVersion) prometheus.MustRegister(currentGoVersion) prometheus.MustRegister(serverID) prometheus.MustRegister(serverFeatureEnabled) prometheus.MustRegister(learnerPromoteSucceed) prometheus.MustRegister(learnerPromoteFailed) prometheus.MustRegister(fdUsed) prometheus.MustRegister(fdLimit) currentVersion.With(prometheus.Labels{ "server_version": version.Version, }).Set(1) currentGoVersion.With(prometheus.Labels{ "server_go_version": goruntime.Version(), }).Set(1) } func monitorFileDescriptor(lg *zap.Logger, done <-chan struct{}) { // This ticker will check File Descriptor Requirements ,and count all fds in used. // And recorded some logs when in used >= limit/5*4. Just recorded message. // If fds was more than 10K,It's low performance due to FDUsage() works. // So need to increase it. // See https://github.com/etcd-io/etcd/issues/11969 for more detail. ticker := time.NewTicker(10 * time.Minute) defer ticker.Stop() for { used, err := runtime.FDUsage() if err != nil { lg.Warn("failed to get file descriptor usage", zap.Error(err)) return } fdUsed.Set(float64(used)) limit, err := runtime.FDLimit() if err != nil { lg.Warn("failed to get file descriptor limit", zap.Error(err)) return } fdLimit.Set(float64(limit)) if used >= limit/5*4 { lg.Warn("80% of file descriptors are used", zap.Uint64("used", used), zap.Uint64("limit", limit)) } select { case <-ticker.C: case <-done: return } } } ================================================ FILE: server/etcdserver/raft.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "expvar" "fmt" "log" "sync" "time" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/pkg/v3/contention" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" serverstorage "go.etcd.io/etcd/server/v3/storage" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) const ( // The max throughput of etcd will not exceed 100MB/s (100K * 1KB value). // Assuming the RTT is around 10ms, 1MB max size is large enough. maxSizePerMsg = 1 * 1024 * 1024 // Never overflow the rafthttp buffer, which is 4096. // TODO: a better const? maxInflightMsgs = 4096 / 8 ) var ( // protects raftStatus raftStatusMu sync.Mutex // indirection for expvar func interface // expvar panics when publishing duplicate name // expvar does not support remove a registered name // so only register a func that calls raftStatus // and change raftStatus as we need. raftStatus func() raft.Status ) func init() { expvar.Publish("raft.status", expvar.Func(func() any { raftStatusMu.Lock() defer raftStatusMu.Unlock() if raftStatus == nil { return nil } return raftStatus() })) } // toApply contains entries, snapshot to be applied. Once // an toApply is consumed, the entries will be persisted to // raft storage concurrently; the application must read // notifyc before assuming the raft messages are stable. type toApply struct { entries []raftpb.Entry snapshot raftpb.Snapshot // notifyc synchronizes etcd server applies with the raft node notifyc chan struct{} // raftAdvancedC notifies EtcdServer.apply that // 'raftLog.applied' has advanced by r.Advance // it should be used only when entries contain raftpb.EntryConfChange raftAdvancedC <-chan struct{} } type raftNode struct { lg *zap.Logger tickMu *sync.RWMutex // timestamp of the latest tick latestTickTs time.Time raftNodeConfig // a chan to send/receive snapshot msgSnapC chan raftpb.Message // a chan to send out apply applyc chan toApply // a chan to send out readState readStateC chan raft.ReadState // utility ticker *time.Ticker // contention detectors for raft heartbeat message td *contention.TimeoutDetector stopped chan struct{} done chan struct{} } type raftNodeConfig struct { lg *zap.Logger // to check if msg receiver is removed from cluster isIDRemoved func(id uint64) bool raft.Node raftStorage *raft.MemoryStorage storage serverstorage.Storage heartbeat time.Duration // for logging // transport specifies the transport to send and receive msgs to members. // Sending messages MUST NOT block. It is okay to drop messages, since // clients should timeout and reissue their messages. // If transport is nil, server will panic. transport rafthttp.Transporter } func newRaftNode(cfg raftNodeConfig) *raftNode { var lg raft.Logger if cfg.lg != nil { lg = NewRaftLoggerZap(cfg.lg) } else { lcfg := logutil.DefaultZapLoggerConfig var err error lg, err = NewRaftLogger(&lcfg) if err != nil { log.Fatalf("cannot create raft logger %v", err) } } raft.SetLogger(lg) r := &raftNode{ lg: cfg.lg, tickMu: new(sync.RWMutex), raftNodeConfig: cfg, latestTickTs: time.Now(), // set up contention detectors for raft heartbeat message. // expect to send a heartbeat within 2 heartbeat intervals. td: contention.NewTimeoutDetector(2 * cfg.heartbeat), readStateC: make(chan raft.ReadState, 1), msgSnapC: make(chan raftpb.Message, maxInFlightMsgSnap), applyc: make(chan toApply), stopped: make(chan struct{}), done: make(chan struct{}), } if r.heartbeat == 0 { r.ticker = &time.Ticker{} } else { r.ticker = time.NewTicker(r.heartbeat) } return r } // raft.Node does not have locks in Raft package func (r *raftNode) tick() { r.tickMu.Lock() r.Tick() r.latestTickTs = time.Now() r.tickMu.Unlock() } func (r *raftNode) getLatestTickTs() time.Time { r.tickMu.RLock() defer r.tickMu.RUnlock() return r.latestTickTs } // start prepares and starts raftNode in a new goroutine. It is no longer safe // to modify the fields after it has been started. func (r *raftNode) start(rh *raftReadyHandler) { internalTimeout := time.Second go func() { defer r.onStop() islead := false for { select { case <-r.ticker.C: r.tick() case rd := <-r.Ready(): if rd.SoftState != nil { newLeader := rd.SoftState.Lead != raft.None && rh.getLead() != rd.SoftState.Lead if newLeader { leaderChanges.Inc() } if rd.SoftState.Lead == raft.None { hasLeader.Set(0) } else { hasLeader.Set(1) } rh.updateLead(rd.SoftState.Lead) islead = rd.RaftState == raft.StateLeader if islead { isLeader.Set(1) } else { isLeader.Set(0) } rh.updateLeadership(newLeader) r.td.Reset() } if len(rd.ReadStates) != 0 { select { case r.readStateC <- rd.ReadStates[len(rd.ReadStates)-1]: case <-time.After(internalTimeout): r.lg.Warn("timed out sending read state", zap.Duration("timeout", internalTimeout)) case <-r.stopped: return } } notifyc := make(chan struct{}, 1) raftAdvancedC := make(chan struct{}, 1) ap := toApply{ entries: rd.CommittedEntries, snapshot: rd.Snapshot, notifyc: notifyc, raftAdvancedC: raftAdvancedC, } updateCommittedIndex(&ap, rh) select { case r.applyc <- ap: case <-r.stopped: return } // the leader can write to its disk in parallel with replicating to the followers and then // writing to their disks. // For more details, check raft thesis 10.2.1 if islead { // gofail: var raftBeforeLeaderSend struct{} r.transport.Send(r.processMessages(rd.Messages)) } // Must save the snapshot file and WAL snapshot entry before saving any other entries or hardstate to // ensure that recovery after a snapshot restore is possible. if !raft.IsEmptySnap(rd.Snapshot) { // gofail: var raftBeforeSaveSnap struct{} if err := r.storage.SaveSnap(rd.Snapshot); err != nil { r.lg.Fatal("failed to save Raft snapshot", zap.Error(err)) } // gofail: var raftAfterSaveSnap struct{} } // gofail: var raftBeforeSave struct{} if err := r.storage.Save(rd.HardState, rd.Entries); err != nil { r.lg.Fatal("failed to save Raft hard state and entries", zap.Error(err)) } if !raft.IsEmptyHardState(rd.HardState) { proposalsCommitted.Set(float64(rd.HardState.Commit)) } // gofail: var raftAfterSave struct{} if !raft.IsEmptySnap(rd.Snapshot) { // Force WAL to fsync its hard state before Release() releases // old data from the WAL. Otherwise could get an error like: // panic: tocommit(107) is out of range [lastIndex(84)]. Was the raft log corrupted, truncated, or lost? // See https://github.com/etcd-io/etcd/issues/10219 for more details. if err := r.storage.Sync(); err != nil { r.lg.Fatal("failed to sync Raft snapshot", zap.Error(err)) } // etcdserver now claim the snapshot has been persisted onto the disk notifyc <- struct{}{} // gofail: var raftBeforeApplySnap struct{} r.raftStorage.ApplySnapshot(rd.Snapshot) r.lg.Info("applied incoming Raft snapshot", zap.Uint64("snapshot-index", rd.Snapshot.Metadata.Index)) // gofail: var raftAfterApplySnap struct{} if err := r.storage.Release(rd.Snapshot); err != nil { r.lg.Fatal("failed to release Raft wal", zap.Error(err)) } // gofail: var raftAfterWALRelease struct{} } r.raftStorage.Append(rd.Entries) confChanged := false for _, ent := range rd.CommittedEntries { if ent.Type == raftpb.EntryConfChange { confChanged = true break } } if !islead { // finish processing incoming messages before we signal notifyc chan msgs := r.processMessages(rd.Messages) // now unblocks 'applyAll' that waits on Raft log disk writes before triggering snapshots notifyc <- struct{}{} // Candidate or follower needs to wait for all pending configuration // changes to be applied before sending messages. // Otherwise we might incorrectly count votes (e.g. votes from removed members). // Also slow machine's follower raft-layer could proceed to become the leader // on its own single-node cluster, before toApply-layer applies the config change. // We simply wait for ALL pending entries to be applied for now. // We might improve this later on if it causes unnecessary long blocking issues. if confChanged { // blocks until 'applyAll' calls 'applyWait.Trigger' // to be in sync with scheduled config-change job // (assume notifyc has cap of 1) select { case notifyc <- struct{}{}: case <-r.stopped: return } } // gofail: var raftBeforeFollowerSend struct{} r.transport.Send(msgs) } else { // leader already processed 'MsgSnap' and signaled notifyc <- struct{}{} } // gofail: var raftBeforeAdvance struct{} r.Advance() if confChanged { // notify etcdserver that raft has already been notified or advanced. raftAdvancedC <- struct{}{} } case <-r.stopped: return } } }() } func updateCommittedIndex(ap *toApply, rh *raftReadyHandler) { var ci uint64 if len(ap.entries) != 0 { ci = ap.entries[len(ap.entries)-1].Index } if ap.snapshot.Metadata.Index > ci { ci = ap.snapshot.Metadata.Index } if ci != 0 { rh.updateCommittedIndex(ci) } } func (r *raftNode) processMessages(ms []raftpb.Message) []raftpb.Message { sentAppResp := false for i := len(ms) - 1; i >= 0; i-- { if r.isIDRemoved(ms[i].To) { ms[i].To = 0 continue } if ms[i].Type == raftpb.MsgAppResp { if sentAppResp { ms[i].To = 0 } else { sentAppResp = true } } if ms[i].Type == raftpb.MsgSnap { // There are two separate data store: the store for v2, and the KV for v3. // The msgSnap only contains the most recent snapshot of store without KV. // So we need to redirect the msgSnap to etcd server main loop for merging in the // current store snapshot and KV snapshot. select { case r.msgSnapC <- ms[i]: default: // drop msgSnap if the inflight chan if full. } ms[i].To = 0 } if ms[i].Type == raftpb.MsgHeartbeat { ok, exceed := r.td.Observe(ms[i].To) if !ok { // TODO: limit request rate. r.lg.Warn( "leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk", zap.String("to", fmt.Sprintf("%x", ms[i].To)), zap.Duration("heartbeat-interval", r.heartbeat), zap.Duration("expected-duration", 2*r.heartbeat), zap.Duration("exceeded-duration", exceed), ) heartbeatSendFailures.Inc() } } } return ms } func (r *raftNode) apply() chan toApply { return r.applyc } func (r *raftNode) stop() { select { case r.stopped <- struct{}{}: // Not already stopped, so trigger it case <-r.done: // Has already been stopped - no need to do anything return } // Block until the stop has been acknowledged by start() <-r.done } func (r *raftNode) onStop() { r.Stop() r.ticker.Stop() r.transport.Stop() if err := r.storage.Close(); err != nil { r.lg.Panic("failed to close Raft storage", zap.Error(err)) } close(r.done) } // for testing func (r *raftNode) pauseSending() { p := r.transport.(rafthttp.Pausable) p.Pause() } func (r *raftNode) resumeSending() { p := r.transport.(rafthttp.Pausable) p.Resume() } // advanceTicks advances ticks of Raft node. // This can be used for fast-forwarding election // ticks in multi data-center deployments, thus // speeding up election process. func (r *raftNode) advanceTicks(ticks int) { for i := 0; i < ticks; i++ { r.tick() } } ================================================ FILE: server/etcdserver/raft_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "encoding/json" "expvar" "reflect" "sync" "testing" "time" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/mock/mockstorage" serverstorage "go.etcd.io/etcd/server/v3/storage" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) func TestGetIDs(t *testing.T) { lg := zaptest.NewLogger(t) addcc := &raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 2} addEntry := raftpb.Entry{Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(addcc)} removecc := &raftpb.ConfChange{Type: raftpb.ConfChangeRemoveNode, NodeID: 2} removeEntry := raftpb.Entry{Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc)} normalEntry := raftpb.Entry{Type: raftpb.EntryNormal} updatecc := &raftpb.ConfChange{Type: raftpb.ConfChangeUpdateNode, NodeID: 2} updateEntry := raftpb.Entry{Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(updatecc)} tests := []struct { confState *raftpb.ConfState ents []raftpb.Entry widSet []uint64 }{ {nil, []raftpb.Entry{}, []uint64{}}, { &raftpb.ConfState{Voters: []uint64{1}}, []raftpb.Entry{}, []uint64{1}, }, { &raftpb.ConfState{Voters: []uint64{1}}, []raftpb.Entry{addEntry}, []uint64{1, 2}, }, { &raftpb.ConfState{Voters: []uint64{1}}, []raftpb.Entry{addEntry, removeEntry}, []uint64{1}, }, { &raftpb.ConfState{Voters: []uint64{1}}, []raftpb.Entry{addEntry, normalEntry}, []uint64{1, 2}, }, { &raftpb.ConfState{Voters: []uint64{1}}, []raftpb.Entry{addEntry, normalEntry, updateEntry}, []uint64{1, 2}, }, { &raftpb.ConfState{Voters: []uint64{1}}, []raftpb.Entry{addEntry, removeEntry, normalEntry}, []uint64{1}, }, } for i, tt := range tests { var snap raftpb.Snapshot if tt.confState != nil { snap.Metadata.ConfState = *tt.confState } idSet := serverstorage.GetEffectiveNodeIDsFromWALEntries(lg, &snap, tt.ents) if !reflect.DeepEqual(idSet, tt.widSet) { t.Errorf("#%d: idset = %#v, want %#v", i, idSet, tt.widSet) } } } func TestCreateConfigChangeEnts(t *testing.T) { lg := zaptest.NewLogger(t) m := membership.Member{ ID: types.ID(1), RaftAttributes: membership.RaftAttributes{PeerURLs: []string{"http://localhost:2380"}}, } ctx, err := json.Marshal(m) if err != nil { t.Fatal(err) } addcc1 := &raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1, Context: ctx} removecc2 := &raftpb.ConfChange{Type: raftpb.ConfChangeRemoveNode, NodeID: 2} removecc3 := &raftpb.ConfChange{Type: raftpb.ConfChangeRemoveNode, NodeID: 3} tests := []struct { ids []uint64 self uint64 term, index uint64 wents []raftpb.Entry }{ { []uint64{1}, 1, 1, 1, nil, }, { []uint64{1, 2}, 1, 1, 1, []raftpb.Entry{{Term: 1, Index: 2, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc2)}}, }, { []uint64{1, 2}, 1, 2, 2, []raftpb.Entry{{Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc2)}}, }, { []uint64{1, 2, 3}, 1, 2, 2, []raftpb.Entry{ {Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc2)}, {Term: 2, Index: 4, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc3)}, }, }, { []uint64{2, 3}, 2, 2, 2, []raftpb.Entry{ {Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc3)}, }, }, { []uint64{2, 3}, 1, 2, 2, []raftpb.Entry{ {Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(addcc1)}, {Term: 2, Index: 4, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc2)}, {Term: 2, Index: 5, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc3)}, }, }, } for i, tt := range tests { gents := serverstorage.CreateConfigChangeEnts(lg, tt.ids, tt.self, tt.term, tt.index) if !reflect.DeepEqual(gents, tt.wents) { t.Errorf("#%d: ents = %v, want %v", i, gents, tt.wents) } } } func TestStopRaftWhenWaitingForApplyDone(t *testing.T) { n := newNopReadyNode() r := newRaftNode(raftNodeConfig{ lg: zaptest.NewLogger(t), Node: n, storage: mockstorage.NewStorageRecorder(""), raftStorage: raft.NewMemoryStorage(), transport: newNopTransporter(), }) srv := &EtcdServer{lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), r: *r} srv.r.start(nil) n.readyc <- raft.Ready{} stop := func() { srv.r.stopped <- struct{}{} select { case <-srv.r.done: case <-time.After(time.Second): t.Fatalf("failed to stop raft loop") } } select { case <-srv.r.applyc: case <-time.After(time.Second): stop() t.Fatalf("failed to receive toApply struct") } stop() } // TestConfigChangeBlocksApply ensures toApply blocks if committed entries contain config-change. func TestConfigChangeBlocksApply(t *testing.T) { n := newNopReadyNode() r := newRaftNode(raftNodeConfig{ lg: zaptest.NewLogger(t), Node: n, storage: mockstorage.NewStorageRecorder(""), raftStorage: raft.NewMemoryStorage(), transport: newNopTransporter(), }) srv := &EtcdServer{lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), r: *r} srv.r.start(&raftReadyHandler{ getLead: func() uint64 { return 0 }, updateLead: func(uint64) {}, updateLeadership: func(bool) {}, }) defer srv.r.stop() n.readyc <- raft.Ready{ SoftState: &raft.SoftState{RaftState: raft.StateFollower}, CommittedEntries: []raftpb.Entry{{Type: raftpb.EntryConfChange}}, } ap := <-srv.r.applyc continueC := make(chan struct{}) go func() { n.readyc <- raft.Ready{} <-srv.r.applyc close(continueC) }() select { case <-continueC: t.Fatalf("unexpected execution: raft routine should block waiting for toApply") case <-time.After(time.Second): } // finish toApply, unblock raft routine <-ap.notifyc select { case <-ap.raftAdvancedC: t.Log("recevied raft advance notification") } select { case <-continueC: case <-time.After(time.Second): t.Fatalf("unexpected blocking on execution") } } func TestProcessDuplicatedAppRespMessage(t *testing.T) { n := newNopReadyNode() cl := membership.NewCluster(zaptest.NewLogger(t)) rs := raft.NewMemoryStorage() p := mockstorage.NewStorageRecorder("") tr, sendc := newSendMsgAppRespTransporter() r := newRaftNode(raftNodeConfig{ lg: zaptest.NewLogger(t), isIDRemoved: func(id uint64) bool { return cl.IsIDRemoved(types.ID(id)) }, Node: n, transport: tr, storage: p, raftStorage: rs, }) s := &EtcdServer{ lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), r: *r, cluster: cl, } s.start() defer s.Stop() lead := uint64(1) n.readyc <- raft.Ready{Messages: []raftpb.Message{ {Type: raftpb.MsgAppResp, From: 2, To: lead, Term: 1, Index: 1}, {Type: raftpb.MsgAppResp, From: 2, To: lead, Term: 1, Index: 2}, {Type: raftpb.MsgAppResp, From: 2, To: lead, Term: 1, Index: 3}, }} got, want := <-sendc, 1 if got != want { t.Errorf("count = %d, want %d", got, want) } } // TestExpvarWithNoRaftStatus to test that none of the expvars that get added during init panic. // This matters if another package imports etcdserver, doesn't use it, but does use expvars. func TestExpvarWithNoRaftStatus(t *testing.T) { defer func() { if err := recover(); err != nil { t.Fatal(err) } }() expvar.Do(func(kv expvar.KeyValue) { _ = kv.Value.String() }) } func TestStopRaftNodeMoreThanOnce(t *testing.T) { n := newNopReadyNode() r := newRaftNode(raftNodeConfig{ lg: zaptest.NewLogger(t), Node: n, storage: mockstorage.NewStorageRecorder(""), raftStorage: raft.NewMemoryStorage(), transport: newNopTransporter(), }) r.start(&raftReadyHandler{}) for i := 0; i < 2; i++ { stopped := make(chan struct{}) go func() { r.stop() close(stopped) }() select { case <-stopped: case <-time.After(time.Second): t.Errorf("*raftNode.stop() is blocked !") } } } ================================================ FILE: server/etcdserver/server.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "context" "encoding/json" errorspkg "errors" "expvar" "fmt" "math" "net/http" "strconv" "sync" "sync/atomic" "time" "github.com/coreos/go-semver/semver" humanize "github.com/dustin/go-humanize" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/membershippb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/client/pkg/v3/verify" "go.etcd.io/etcd/pkg/v3/featuregate" "go.etcd.io/etcd/pkg/v3/idutil" "go.etcd.io/etcd/pkg/v3/notify" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/pkg/v3/runtime" "go.etcd.io/etcd/pkg/v3/schedule" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/pkg/v3/wait" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver/api" httptypes "go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/types" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" stats "go.etcd.io/etcd/server/v3/etcdserver/api/v2stats" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/etcd/server/v3/etcdserver/api/v3alarm" "go.etcd.io/etcd/server/v3/etcdserver/api/v3compactor" "go.etcd.io/etcd/server/v3/etcdserver/apply" "go.etcd.io/etcd/server/v3/etcdserver/cindex" "go.etcd.io/etcd/server/v3/etcdserver/errors" serverversion "go.etcd.io/etcd/server/v3/etcdserver/version" "go.etcd.io/etcd/server/v3/features" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/lease/leasehttp" serverstorage "go.etcd.io/etcd/server/v3/storage" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) const ( DefaultSnapshotCount = 10000 // DefaultSnapshotCatchUpEntries is the number of entries for a slow follower // to catch-up after compacting the raft storage entries. // We expect the follower has a millisecond level latency with the leader. // The max throughput is around 10K. Keep a 5K entries is enough for helping // follower to catch up. DefaultSnapshotCatchUpEntries uint64 = 5000 StoreClusterPrefix = "/0" StoreKeysPrefix = "/1" // HealthInterval is the minimum time the cluster should be healthy // before accepting add and delete member requests. HealthInterval = 5 * time.Second purgeFileInterval = 30 * time.Second // max number of in-flight snapshot messages etcdserver allows to have // This number is more than enough for most clusters with 5 machines. maxInFlightMsgSnap = 16 releaseDelayAfterSnapshot = 30 * time.Second // maxPendingRevokes is the maximum number of outstanding expired lease revocations. maxPendingRevokes = 16 recommendedMaxRequestBytes = 10 * 1024 * 1024 // readyPercentThreshold is a threshold used to determine // whether a learner is ready for a transition into a full voting member or not. readyPercentThreshold = 0.9 DowngradeEnabledPath = "/downgrade/enabled" memorySnapshotCount = 100 ) var ( // monitorVersionInterval should be smaller than the timeout // on the connection. Or we will not be able to reuse the connection // (since it will timeout). monitorVersionInterval = rafthttp.ConnWriteTimeout - time.Second recommendedMaxRequestBytesString = humanize.Bytes(uint64(recommendedMaxRequestBytes)) ) func init() { expvar.Publish( "file_descriptor_limit", expvar.Func( func() any { n, _ := runtime.FDLimit() return n }, ), ) } type Response struct { Term uint64 Index uint64 Event *v2store.Event Watcher v2store.Watcher Err error } type ServerV2 interface { Server Leader() types.ID ClientCertAuthEnabled() bool } type ServerV3 interface { Server apply.RaftStatusGetter } func (s *EtcdServer) ClientCertAuthEnabled() bool { return s.Cfg.ClientCertAuthEnabled } type Server interface { // AddMember attempts to add a member into the cluster. It will return // ErrIDRemoved if member ID is removed from the cluster, or return // ErrIDExists if member ID exists in the cluster. AddMember(ctx context.Context, memb membership.Member) ([]*membership.Member, error) // RemoveMember attempts to remove a member from the cluster. It will // return ErrIDRemoved if member ID is removed from the cluster, or return // ErrIDNotFound if member ID is not in the cluster. RemoveMember(ctx context.Context, id uint64) ([]*membership.Member, error) // UpdateMember attempts to update an existing member in the cluster. It will // return ErrIDNotFound if the member ID does not exist. UpdateMember(ctx context.Context, updateMemb membership.Member) ([]*membership.Member, error) // PromoteMember attempts to promote a non-voting node to a voting node. It will // return ErrIDNotFound if the member ID does not exist. // return ErrLearnerNotReady if the member are not ready. // return ErrMemberNotLearner if the member is not a learner. PromoteMember(ctx context.Context, id uint64) ([]*membership.Member, error) // ClusterVersion is the cluster-wide minimum major.minor version. // Cluster version is set to the min version that an etcd member is // compatible with when first bootstrap. // // ClusterVersion is nil until the cluster is bootstrapped (has a quorum). // // During a rolling upgrades, the ClusterVersion will be updated // automatically after a sync. (5 second by default) // // The API/raft component can utilize ClusterVersion to determine if // it can accept a client request or a raft RPC. // NOTE: ClusterVersion might be nil when etcd 2.1 works with etcd 2.0 and // the leader is etcd 2.0. etcd 2.0 leader will not update clusterVersion since // this feature is introduced post 2.0. ClusterVersion() *semver.Version // StorageVersion is the storage schema version. It's supported starting // from 3.6. StorageVersion() *semver.Version Cluster() api.Cluster Alarms() []*pb.AlarmMember // LeaderChangedNotify returns a channel for application level code to be notified // when etcd leader changes, this function is intend to be used only in application // which embed etcd. // Caution: // 1. the returned channel is being closed when the leadership changes. // 2. so the new channel needs to be obtained for each raft term. // 3. user can loose some consecutive channel changes using this API. LeaderChangedNotify() <-chan struct{} } // EtcdServer is the production implementation of the Server interface type EtcdServer struct { // inflightSnapshots holds count the number of snapshots currently inflight. inflightSnapshots atomic.Int64 appliedIndex atomic.Uint64 committedIndex atomic.Uint64 term atomic.Uint64 lead atomic.Uint64 consistIndex cindex.ConsistentIndexer // consistIndex is used to get/set/save consistentIndex r raftNode // uses 64-bit atomics; keep 64-bit aligned. readych chan struct{} Cfg config.ServerConfig lgMu *sync.RWMutex lg *zap.Logger w wait.Wait readMu sync.RWMutex // read routine notifies etcd server that it waits for reading by sending an empty struct to // readwaitC readwaitc chan struct{} // readNotifier is used to notify the read routine that it can process the request // when there is no error readNotifier *notifier // stop signals the run goroutine should shutdown. stop chan struct{} // stopping is closed by run goroutine on shutdown. stopping chan struct{} // done is closed when all goroutines from start() complete. done chan struct{} // leaderChanged is used to notify the linearizable read loop to drop the old read requests. leaderChanged *notify.Notifier errorc chan error memberID types.ID attributes membership.Attributes cluster *membership.RaftCluster snapshotter *snap.Snapshotter uberApply apply.UberApplier applyWait wait.WaitTime kv mvcc.WatchableKV lessor lease.Lessor bemu sync.RWMutex be backend.Backend beHooks *serverstorage.BackendHooks authStore auth.AuthStore alarmStore *v3alarm.AlarmStore stats *stats.ServerStats lstats *stats.LeaderStats // compactor is used to auto-compact the KV. compactor v3compactor.Compactor // peerRt used to send requests (version, lease) to peers. peerRt http.RoundTripper reqIDGen *idutil.Generator // wgMu blocks concurrent waitgroup mutation while server stopping wgMu sync.RWMutex // wg is used to wait for the goroutines that depends on the server state // to exit when stopping the server. wg sync.WaitGroup // ctx is used for etcd-initiated requests that may need to be canceled // on etcd server shutdown. ctx context.Context cancel context.CancelFunc leadTimeMu sync.RWMutex leadElectedTime time.Time firstCommitInTerm *notify.Notifier clusterVersionChanged *notify.Notifier *AccessController // forceDiskSnapshot can force snapshot be triggered after apply, independent of the snapshotCount. // Should only be set within apply code path. Used to force snapshot after cluster version downgrade. // TODO: Replace with flush db in v3.7 assuming v3.6 bootstraps from db file. forceDiskSnapshot bool corruptionChecker CorruptionChecker } // NewServer creates a new EtcdServer from the supplied configuration. The // configuration is considered static for the lifetime of the EtcdServer. func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { b, err := bootstrap(cfg) if err != nil { cfg.Logger.Error("bootstrap failed", zap.Error(err)) return nil, err } cfg.Logger.Info("bootstrap successfully") defer func() { if err != nil { b.Close() } }() sstats := stats.NewServerStats(cfg.Name, b.cluster.cl.String()) lstats := stats.NewLeaderStats(cfg.Logger, b.cluster.nodeID.String()) heartbeat := time.Duration(cfg.TickMs) * time.Millisecond srv = &EtcdServer{ readych: make(chan struct{}), Cfg: cfg, lgMu: new(sync.RWMutex), lg: cfg.Logger, errorc: make(chan error, 1), snapshotter: b.ss, r: *b.raft.newRaftNode(b.ss, b.storage.wal.w, b.cluster.cl), memberID: b.cluster.nodeID, attributes: membership.Attributes{Name: cfg.Name, ClientURLs: cfg.ClientURLs.StringSlice()}, cluster: b.cluster.cl, stats: sstats, lstats: lstats, peerRt: b.prt, reqIDGen: idutil.NewGenerator(uint16(b.cluster.nodeID), time.Now()), AccessController: &AccessController{CORS: cfg.CORS, HostWhitelist: cfg.HostWhitelist}, consistIndex: b.storage.backend.ci, firstCommitInTerm: notify.NewNotifier(), clusterVersionChanged: notify.NewNotifier(), } addFeatureGateMetrics(cfg.ServerFeatureGate, serverFeatureEnabled) serverID.With(prometheus.Labels{"server_id": b.cluster.nodeID.String()}).Set(1) srv.cluster.SetVersionChangedNotifier(srv.clusterVersionChanged) srv.be = b.storage.backend.be srv.beHooks = b.storage.backend.beHooks minTTL := time.Duration((3*cfg.ElectionTicks)/2) * heartbeat // always recover lessor before kv. When we recover the mvcc.KV it will reattach keys to its leases. // If we recover mvcc.KV first, it will attach the keys to the wrong lessor before it recovers. srv.lessor = lease.NewLessor(srv.Logger(), srv.be, srv.cluster, lease.LessorConfig{ MinLeaseTTL: int64(math.Ceil(minTTL.Seconds())), CheckpointInterval: cfg.LeaseCheckpointInterval, CheckpointPersist: cfg.ServerFeatureGate.Enabled(features.LeaseCheckpointPersist), ExpiredLeasesRetryInterval: srv.Cfg.ReqTimeout(), }) tp, err := auth.NewTokenProvider(cfg.Logger, cfg.AuthToken, func(index uint64) <-chan struct{} { return srv.applyWait.Wait(index) }, time.Duration(cfg.TokenTTL)*time.Second, ) if err != nil { cfg.Logger.Warn("failed to create token provider", zap.Error(err)) return nil, err } mvccStoreConfig := mvcc.StoreConfig{ CompactionBatchLimit: cfg.CompactionBatchLimit, CompactionSleepInterval: cfg.CompactionSleepInterval, } srv.kv = mvcc.New(srv.Logger(), srv.be, srv.lessor, mvccStoreConfig) srv.corruptionChecker = newCorruptionChecker(cfg.Logger, srv, srv.kv.HashStorage()) srv.authStore = auth.NewAuthStore(srv.Logger(), schema.NewAuthBackend(srv.Logger(), srv.be), tp, int(cfg.BcryptCost)) newSrv := srv // since srv == nil in defer if srv is returned as nil defer func() { // closing backend without first closing kv can cause // resumed compactions to fail with closed tx errors if err != nil { newSrv.kv.Close() } }() if num := cfg.AutoCompactionRetention; num != 0 { srv.compactor, err = v3compactor.New(cfg.Logger, cfg.AutoCompactionMode, num, srv.kv, srv) if err != nil { return nil, err } srv.compactor.Run() } if err = srv.restoreAlarms(); err != nil { return nil, err } srv.uberApply = srv.NewUberApplier() if srv.FeatureEnabled(features.LeaseCheckpoint) { // setting checkpointer enables lease checkpoint feature. srv.lessor.SetCheckpointer(func(ctx context.Context, cp *pb.LeaseCheckpointRequest) error { if !srv.ensureLeadership() { srv.lg.Warn("Ignore the checkpoint request because current member isn't a leader", zap.Uint64("local-member-id", uint64(srv.MemberID()))) return lease.ErrNotPrimary } srv.raftRequestOnce(ctx, pb.InternalRaftRequest{LeaseCheckpoint: cp}) return nil }) } // Set the hook after EtcdServer finishes the initialization to avoid // the hook being called during the initialization process. srv.be.SetTxPostLockInsideApplyHook(srv.getTxPostLockInsideApplyHook()) // TODO: move transport initialization near the definition of remote tr := &rafthttp.Transport{ Logger: cfg.Logger, TLSInfo: cfg.PeerTLSInfo, DialTimeout: cfg.PeerDialTimeout(), ID: b.cluster.nodeID, URLs: cfg.PeerURLs, ClusterID: b.cluster.cl.ID(), Raft: srv, Snapshotter: b.ss, ServerStats: sstats, LeaderStats: lstats, ErrorC: srv.errorc, } if err = tr.Start(); err != nil { return nil, err } // add all remotes into transport for _, m := range b.cluster.remotes { if m.ID != b.cluster.nodeID { tr.AddRemote(m.ID, m.PeerURLs) } } for _, m := range b.cluster.cl.Members() { if m.ID != b.cluster.nodeID { tr.AddPeer(m.ID, m.PeerURLs) } } srv.r.transport = tr return srv, nil } func (s *EtcdServer) Logger() *zap.Logger { s.lgMu.RLock() l := s.lg s.lgMu.RUnlock() return l } func (s *EtcdServer) Config() config.ServerConfig { return s.Cfg } // FeatureEnabled returns true if the feature is enabled by the etcd server, false otherwise. func (s *EtcdServer) FeatureEnabled(f featuregate.Feature) bool { return s.Cfg.ServerFeatureGate.Enabled(f) } func tickToDur(ticks int, tickMs uint) string { return fmt.Sprintf("%v", time.Duration(ticks)*time.Duration(tickMs)*time.Millisecond) } func (s *EtcdServer) adjustTicks() { lg := s.Logger() clusterN := len(s.cluster.Members()) // single-node fresh start, or single-node recovers from snapshot if clusterN == 1 { ticks := s.Cfg.ElectionTicks - 1 lg.Info( "started as single-node; fast-forwarding election ticks", zap.String("local-member-id", s.MemberID().String()), zap.Int("forward-ticks", ticks), zap.String("forward-duration", tickToDur(ticks, s.Cfg.TickMs)), zap.Int("election-ticks", s.Cfg.ElectionTicks), zap.String("election-timeout", tickToDur(s.Cfg.ElectionTicks, s.Cfg.TickMs)), ) s.r.advanceTicks(ticks) return } if !s.Cfg.InitialElectionTickAdvance { lg.Info("skipping initial election tick advance", zap.Int("election-ticks", s.Cfg.ElectionTicks)) return } lg.Info("starting initial election tick advance", zap.Int("election-ticks", s.Cfg.ElectionTicks)) // retry up to "rafthttp.ConnReadTimeout", which is 5-sec // until peer connection reports; otherwise: // 1. all connections failed, or // 2. no active peers, or // 3. restarted single-node with no snapshot // then, do nothing, because advancing ticks would have no effect waitTime := rafthttp.ConnReadTimeout itv := 50 * time.Millisecond for i := int64(0); i < int64(waitTime/itv); i++ { select { case <-time.After(itv): case <-s.stopping: return } peerN := s.r.transport.ActivePeers() if peerN > 1 { // multi-node received peer connection reports // adjust ticks, in case slow leader message receive ticks := s.Cfg.ElectionTicks - 2 lg.Info( "initialized peer connections; fast-forwarding election ticks", zap.String("local-member-id", s.MemberID().String()), zap.Int("forward-ticks", ticks), zap.String("forward-duration", tickToDur(ticks, s.Cfg.TickMs)), zap.Int("election-ticks", s.Cfg.ElectionTicks), zap.String("election-timeout", tickToDur(s.Cfg.ElectionTicks, s.Cfg.TickMs)), zap.Int("active-remote-members", peerN), ) s.r.advanceTicks(ticks) return } } } // Start performs any initialization of the Server necessary for it to // begin serving requests. It must be called before Do or Process. // Start must be non-blocking; any long-running server functionality // should be implemented in goroutines. func (s *EtcdServer) Start() { s.start() s.GoAttach(func() { s.adjustTicks() }) s.GoAttach(func() { s.publishV3(s.Cfg.ReqTimeout()) }) s.GoAttach(s.purgeFile) s.GoAttach(func() { monitorFileDescriptor(s.Logger(), s.stopping) }) s.GoAttach(s.monitorClusterVersions) s.GoAttach(s.monitorStorageVersion) s.GoAttach(s.linearizableReadLoop) s.GoAttach(s.monitorKVHash) s.GoAttach(s.monitorCompactHash) s.GoAttach(s.monitorDowngrade) } // start prepares and starts server in a new goroutine. It is no longer safe to // modify a server's fields after it has been sent to Start. // This function is just used for testing. func (s *EtcdServer) start() { lg := s.Logger() if s.Cfg.SnapshotCount == 0 { lg.Info( "updating snapshot-count to default", zap.Uint64("given-snapshot-count", s.Cfg.SnapshotCount), zap.Uint64("updated-snapshot-count", DefaultSnapshotCount), ) s.Cfg.SnapshotCount = DefaultSnapshotCount } if s.Cfg.SnapshotCatchUpEntries == 0 { lg.Info( "updating snapshot catch-up entries to default", zap.Uint64("given-snapshot-catchup-entries", s.Cfg.SnapshotCatchUpEntries), zap.Uint64("updated-snapshot-catchup-entries", DefaultSnapshotCatchUpEntries), ) s.Cfg.SnapshotCatchUpEntries = DefaultSnapshotCatchUpEntries } s.w = wait.New() s.applyWait = wait.NewTimeList() s.done = make(chan struct{}) s.stop = make(chan struct{}) s.stopping = make(chan struct{}, 1) s.ctx, s.cancel = context.WithCancel(context.Background()) s.readwaitc = make(chan struct{}, 1) s.readNotifier = newNotifier() s.leaderChanged = notify.NewNotifier() if s.ClusterVersion() != nil { lg.Info( "starting etcd server", zap.String("local-member-id", s.MemberID().String()), zap.String("local-server-version", version.Version), zap.String("cluster-id", s.Cluster().ID().String()), zap.String("cluster-version", version.Cluster(s.ClusterVersion().String())), ) membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(s.ClusterVersion().String())}).Set(1) } else { lg.Info( "starting etcd server", zap.String("local-member-id", s.MemberID().String()), zap.String("local-server-version", version.Version), zap.String("cluster-version", "to_be_decided"), ) } // TODO: if this is an empty log, writes all peer infos // into the first entry go s.run() } func (s *EtcdServer) purgeFile() { lg := s.Logger() var dberrc, serrc, werrc <-chan error var dbdonec, sdonec, wdonec <-chan struct{} if s.Cfg.MaxSnapFiles > 0 { dbdonec, dberrc = fileutil.PurgeFileWithoutFlock(lg, s.Cfg.SnapDir(), "snap.db", s.Cfg.MaxSnapFiles, purgeFileInterval, s.stopping) sdonec, serrc = fileutil.PurgeFileWithoutFlock(lg, s.Cfg.SnapDir(), "snap", s.Cfg.MaxSnapFiles, purgeFileInterval, s.stopping) } if s.Cfg.MaxWALFiles > 0 { wdonec, werrc = fileutil.PurgeFileWithDoneNotify(lg, s.Cfg.WALDir(), "wal", s.Cfg.MaxWALFiles, purgeFileInterval, s.stopping) } select { case e := <-dberrc: lg.Fatal("failed to purge snap db file", zap.Error(e)) case e := <-serrc: lg.Fatal("failed to purge snap file", zap.Error(e)) case e := <-werrc: lg.Fatal("failed to purge wal file", zap.Error(e)) case <-s.stopping: if dbdonec != nil { <-dbdonec } if sdonec != nil { <-sdonec } if wdonec != nil { <-wdonec } return } } func (s *EtcdServer) Cluster() api.Cluster { return s.cluster } func (s *EtcdServer) ApplyWait() <-chan struct{} { return s.applyWait.Wait(s.getCommittedIndex()) } type ServerPeer interface { ServerV2 RaftHandler() http.Handler LeaseHandler() http.Handler } func (s *EtcdServer) LeaseHandler() http.Handler { if s.lessor == nil { return nil } return leasehttp.NewHandler(s.lessor, s.ApplyWait) } func (s *EtcdServer) RaftHandler() http.Handler { return s.r.transport.Handler() } type ServerPeerV2 interface { ServerPeer HashKVHandler() http.Handler DowngradeEnabledHandler() http.Handler } func (s *EtcdServer) DowngradeInfo() *serverversion.DowngradeInfo { return s.cluster.DowngradeInfo() } type downgradeEnabledHandler struct { lg *zap.Logger cluster api.Cluster server *EtcdServer } func (s *EtcdServer) DowngradeEnabledHandler() http.Handler { return &downgradeEnabledHandler{ lg: s.Logger(), cluster: s.cluster, server: s, } } func (h *downgradeEnabledHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { w.Header().Set("Allow", http.MethodGet) http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) return } w.Header().Set("X-Etcd-Cluster-ID", h.cluster.ID().String()) if r.URL.Path != DowngradeEnabledPath { http.Error(w, "bad path", http.StatusBadRequest) return } ctx, cancel := context.WithTimeout(context.Background(), h.server.Cfg.ReqTimeout()) defer cancel() // serve with linearized downgrade info if err := h.server.linearizableReadNotify(ctx); err != nil { http.Error(w, fmt.Sprintf("failed linearized read: %v", err), http.StatusInternalServerError) return } enabled := h.server.DowngradeInfo().Enabled w.Header().Set("Content-Type", "text/plain") w.Write([]byte(strconv.FormatBool(enabled))) } // Process takes a raft message and applies it to the server's raft state // machine, respecting any timeout of the given context. func (s *EtcdServer) Process(ctx context.Context, m raftpb.Message) error { lg := s.Logger() if s.cluster.IsIDRemoved(types.ID(m.From)) { lg.Warn( "rejected Raft message from removed member", zap.String("local-member-id", s.MemberID().String()), zap.String("removed-member-id", types.ID(m.From).String()), ) return httptypes.NewHTTPError(http.StatusForbidden, "cannot process message from removed member") } if s.MemberID() != types.ID(m.To) { lg.Warn( "rejected Raft message to mismatch member", zap.String("local-member-id", s.MemberID().String()), zap.String("mismatch-member-id", types.ID(m.To).String()), ) return httptypes.NewHTTPError(http.StatusForbidden, "cannot process message to mismatch member") } if m.Type == raftpb.MsgApp { s.stats.RecvAppendReq(types.ID(m.From).String(), m.Size()) } return s.r.Step(ctx, m) } func (s *EtcdServer) IsIDRemoved(id uint64) bool { return s.cluster.IsIDRemoved(types.ID(id)) } func (s *EtcdServer) ReportUnreachable(id uint64) { s.r.ReportUnreachable(id) } // ReportSnapshot reports snapshot sent status to the raft state machine, // and clears the used snapshot from the snapshot store. func (s *EtcdServer) ReportSnapshot(id uint64, status raft.SnapshotStatus) { s.r.ReportSnapshot(id, status) } type etcdProgress struct { confState raftpb.ConfState diskSnapshotIndex uint64 memorySnapshotIndex uint64 appliedt uint64 appliedi uint64 } // raftReadyHandler contains a set of EtcdServer operations to be called by raftNode, // and helps decouple state machine logic from Raft algorithms. // TODO: add a state machine interface to toApply the commit entries and do snapshot/recover type raftReadyHandler struct { getLead func() (lead uint64) updateLead func(lead uint64) updateLeadership func(newLeader bool) updateCommittedIndex func(uint64) } func (s *EtcdServer) run() { lg := s.Logger() sn, err := s.r.raftStorage.Snapshot() if err != nil { lg.Panic("failed to get snapshot from Raft storage", zap.Error(err)) } // asynchronously accept toApply packets, dispatch progress in-order sched := schedule.NewFIFOScheduler(lg) rh := &raftReadyHandler{ getLead: func() (lead uint64) { return s.getLead() }, updateLead: func(lead uint64) { s.setLead(lead) }, updateLeadership: func(newLeader bool) { if !s.isLeader() { if s.lessor != nil { s.lessor.Demote() } if s.compactor != nil { s.compactor.Pause() } } else { if newLeader { t := time.Now() s.leadTimeMu.Lock() s.leadElectedTime = t s.leadTimeMu.Unlock() } if s.compactor != nil { s.compactor.Resume() } } if newLeader { s.leaderChanged.Notify() } // TODO: remove the nil checking // current test utility does not provide the stats if s.stats != nil { s.stats.BecomeLeader() } }, updateCommittedIndex: func(ci uint64) { cci := s.getCommittedIndex() if ci > cci { s.setCommittedIndex(ci) } }, } s.r.start(rh) ep := etcdProgress{ confState: sn.Metadata.ConfState, diskSnapshotIndex: sn.Metadata.Index, memorySnapshotIndex: sn.Metadata.Index, appliedt: sn.Metadata.Term, appliedi: sn.Metadata.Index, } defer func() { s.wgMu.Lock() // block concurrent waitgroup adds in GoAttach while stopping close(s.stopping) s.wgMu.Unlock() s.cancel() sched.Stop() // wait for goroutines before closing raft so wal stays open s.wg.Wait() // must stop raft after scheduler-- etcdserver can leak rafthttp pipelines // by adding a peer after raft stops the transport s.r.stop() s.Cleanup() close(s.done) }() var expiredLeaseC <-chan []*lease.Lease if s.lessor != nil { expiredLeaseC = s.lessor.ExpiredLeasesC() } for { select { case ap := <-s.r.apply(): f := schedule.NewJob("server_applyAll", func(context.Context) { s.applyAll(&ep, &ap) }) sched.Schedule(f) case leases := <-expiredLeaseC: s.revokeExpiredLeases(leases) case err := <-s.errorc: lg.Warn("server error", zap.Error(err)) lg.Warn("data-dir used by this member must be removed") return case <-s.stop: return } } } func (s *EtcdServer) revokeExpiredLeases(leases []*lease.Lease) { s.GoAttach(func() { // We shouldn't revoke any leases if current member isn't a leader, // because the operation should only be performed by the leader. When // the leader gets blocked on the raft loop, such as writing WAL entries, // it can't process any events or messages from raft. It may think it // is still the leader even the leader has already changed. // Refer to https://github.com/etcd-io/etcd/issues/15247 lg := s.Logger() if !s.ensureLeadership() { lg.Warn("Ignore the lease revoking request because current member isn't a leader", zap.Uint64("local-member-id", uint64(s.MemberID()))) return } // Increases throughput of expired leases deletion process through parallelization c := make(chan struct{}, maxPendingRevokes) for _, curLease := range leases { select { case c <- struct{}{}: case <-s.stopping: return } f := func(lid int64) { s.GoAttach(func() { ctx := s.authStore.WithRoot(s.ctx) _, lerr := s.LeaseRevoke(ctx, &pb.LeaseRevokeRequest{ID: lid}) if lerr == nil { leaseExpired.Inc() } else { lg.Warn( "failed to revoke lease", zap.String("lease-id", fmt.Sprintf("%016x", lid)), zap.Error(lerr), ) } <-c }) } f(int64(curLease.ID)) } }) } // isActive checks if the etcd instance is still actively processing the // heartbeat message (ticks). It returns false if no heartbeat has been // received within 3 * tickMs. func (s *EtcdServer) isActive() bool { latestTickTs := s.r.getLatestTickTs() threshold := 3 * time.Duration(s.Cfg.TickMs) * time.Millisecond return latestTickTs.Add(threshold).After(time.Now()) } // ensureLeadership checks whether current member is still the leader. func (s *EtcdServer) ensureLeadership() bool { lg := s.Logger() if s.isActive() { lg.Debug("The member is active, skip checking leadership", zap.Time("latestTickTs", s.r.getLatestTickTs()), zap.Time("now", time.Now())) return true } ctx, cancel := context.WithTimeout(s.ctx, s.Cfg.ReqTimeout()) defer cancel() if err := s.linearizableReadNotify(ctx); err != nil { lg.Warn("Failed to check current member's leadership", zap.Error(err)) return false } newLeaderID := s.raftStatus().Lead if newLeaderID != uint64(s.MemberID()) { lg.Warn("Current member isn't a leader", zap.Uint64("local-member-id", uint64(s.MemberID())), zap.Uint64("new-lead", newLeaderID)) return false } return true } // Cleanup removes allocated objects by EtcdServer.NewServer in // situation that EtcdServer::Start was not called (that takes care of cleanup). func (s *EtcdServer) Cleanup() { // kv, lessor and backend can be nil if running without v3 enabled // or running unit tests. if s.lessor != nil { s.lessor.Stop() } if s.kv != nil { s.kv.Close() } if s.authStore != nil { s.authStore.Close() } if s.be != nil { s.be.Close() } if s.compactor != nil { s.compactor.Stop() } } func (s *EtcdServer) Defragment() error { s.bemu.Lock() defer s.bemu.Unlock() return s.be.Defrag() } func (s *EtcdServer) applyAll(ep *etcdProgress, apply *toApply) { s.applySnapshot(ep, apply) s.applyEntries(ep, apply) backend.VerifyBackendConsistency(s.Backend(), s.Logger(), true, schema.AllBuckets...) proposalsApplied.Set(float64(ep.appliedi)) s.applyWait.Trigger(ep.appliedi) // wait for the raft routine to finish the disk writes before triggering a // snapshot. or applied index might be greater than the last index in raft // storage, since the raft routine might be slower than toApply routine. <-apply.notifyc s.snapshotIfNeededAndCompactRaftLog(ep) select { // snapshot requested via send() case m := <-s.r.msgSnapC: merged := s.createMergedSnapshotMessage(m, ep.appliedt, ep.appliedi, ep.confState) s.sendMergedSnap(merged) default: } } func (s *EtcdServer) applySnapshot(ep *etcdProgress, toApply *toApply) { if raft.IsEmptySnap(toApply.snapshot) { return } applySnapshotInProgress.Inc() lg := s.Logger() lg.Info( "applying snapshot", zap.Uint64("current-snapshot-index", ep.diskSnapshotIndex), zap.Uint64("current-applied-index", ep.appliedi), zap.Uint64("incoming-leader-snapshot-index", toApply.snapshot.Metadata.Index), zap.Uint64("incoming-leader-snapshot-term", toApply.snapshot.Metadata.Term), ) defer func() { lg.Info( "applied snapshot", zap.Uint64("current-snapshot-index", ep.diskSnapshotIndex), zap.Uint64("current-applied-index", ep.appliedi), zap.Uint64("incoming-leader-snapshot-index", toApply.snapshot.Metadata.Index), zap.Uint64("incoming-leader-snapshot-term", toApply.snapshot.Metadata.Term), ) applySnapshotInProgress.Dec() }() if toApply.snapshot.Metadata.Index <= ep.appliedi { lg.Panic( "unexpected leader snapshot from outdated index", zap.Uint64("current-snapshot-index", ep.diskSnapshotIndex), zap.Uint64("current-applied-index", ep.appliedi), zap.Uint64("incoming-leader-snapshot-index", toApply.snapshot.Metadata.Index), zap.Uint64("incoming-leader-snapshot-term", toApply.snapshot.Metadata.Term), ) } // wait for raftNode to persist snapshot onto the disk <-toApply.notifyc bemuUnlocked := false s.bemu.Lock() defer func() { if !bemuUnlocked { s.bemu.Unlock() } }() // gofail: var applyBeforeOpenSnapshot struct{} newbe, err := serverstorage.OpenSnapshotBackend(s.Cfg, s.snapshotter, toApply.snapshot, s.beHooks) if err != nil { lg.Panic("failed to open snapshot backend", zap.Error(err)) } lg.Info("applySnapshot: opened snapshot backend") // gofail: var applyAfterOpenSnapshot struct{} // We need to set the backend to consistIndex before recovering the lessor, // because lessor.Recover will commit the boltDB transaction, accordingly it // will get the old consistent_index persisted into the db in OnPreCommitUnsafe. // Eventually the new consistent_index value coming from snapshot is overwritten // by the old value. s.consistIndex.SetBackend(newbe) verifySnapshotIndex(toApply.snapshot, s.consistIndex.ConsistentIndex()) // always recover lessor before kv. When we recover the mvcc.KV it will reattach keys to its leases. // If we recover mvcc.KV first, it will attach the keys to the wrong lessor before it recovers. if s.lessor != nil { lg.Info("restoring lease store") s.lessor.Recover(newbe, func() lease.TxnDelete { return s.kv.Write(traceutil.TODO()) }) lg.Info("restored lease store") } lg.Info("restoring mvcc store") if err := s.kv.Restore(newbe); err != nil { lg.Panic("failed to restore mvcc store", zap.Error(err)) } newbe.SetTxPostLockInsideApplyHook(s.getTxPostLockInsideApplyHook()) lg.Info("restored mvcc store", zap.Uint64("consistent-index", s.consistIndex.ConsistentIndex())) oldbe := s.be s.be = newbe s.bemu.Unlock() bemuUnlocked = true // Closing old backend might block until all the txns // on the backend are finished. // We do not want to wait on closing the old backend. go func() { lg.Info("closing old backend file") defer func() { lg.Info("closed old backend file") }() if err := oldbe.Close(); err != nil { lg.Panic("failed to close old backend", zap.Error(err)) } }() lg.Info("restoring alarm store") if err := s.restoreAlarms(); err != nil { lg.Panic("failed to restore alarm store", zap.Error(err)) } lg.Info("restored alarm store") if s.authStore != nil { lg.Info("restoring auth store") s.authStore.Recover(schema.NewAuthBackend(lg, newbe)) lg.Info("restored auth store") } s.cluster.SetBackend(schema.NewMembershipBackend(lg, newbe)) lg.Info("restoring cluster configuration") s.cluster.Recover(api.UpdateCapability) lg.Info("restored cluster configuration") lg.Info("removing old peers from network") // recover raft transport s.r.transport.RemoveAllPeers() lg.Info("removed old peers from network") lg.Info("adding peers from new cluster configuration") for _, m := range s.cluster.Members() { if m.ID == s.MemberID() { continue } s.r.transport.AddPeer(m.ID, m.PeerURLs) } lg.Info("added peers from new cluster configuration") ep.appliedt = toApply.snapshot.Metadata.Term ep.appliedi = toApply.snapshot.Metadata.Index ep.diskSnapshotIndex = ep.appliedi ep.memorySnapshotIndex = ep.appliedi ep.confState = toApply.snapshot.Metadata.ConfState // As backends and implementations like alarmsStore changed, we need // to re-bootstrap Appliers. s.uberApply = s.NewUberApplier() } func (s *EtcdServer) NewUberApplier() apply.UberApplier { opts := apply.ApplierOptions{ Logger: s.lg, KV: s.KV(), AlarmStore: s.alarmStore, AuthStore: s.authStore, Lessor: s.lessor, Cluster: s.cluster, RaftStatus: s, SnapshotServer: s, ConsistentIndex: s.consistIndex, TxnModeWriteWithSharedBuffer: s.Cfg.ServerFeatureGate.Enabled(features.TxnModeWriteWithSharedBuffer), Backend: s.be, QuotaBackendBytesCfg: s.Cfg.QuotaBackendBytes, WarningApplyDuration: s.Cfg.WarningApplyDuration, } return apply.NewUberApplier(opts) } func verifySnapshotIndex(snapshot raftpb.Snapshot, cindex uint64) { verify.Verify("consistent_index isn't equal to snapshot index", func() (bool, map[string]any) { return cindex == snapshot.Metadata.Index, map[string]any{ "consistent_index": cindex, "snapshot_index": snapshot.Metadata.Index, } }) } func verifyConsistentIndexIsLatest(snapshot raftpb.Snapshot, cindex uint64) { verify.Verify("consistent_index is older than snapshot_index", func() (bool, map[string]any) { return cindex >= snapshot.Metadata.Index, map[string]any{ "consistent_index": cindex, "snapshot_index": snapshot.Metadata.Index, } }) } func (s *EtcdServer) applyEntries(ep *etcdProgress, apply *toApply) { if len(apply.entries) == 0 { return } firsti := apply.entries[0].Index if firsti > ep.appliedi+1 { lg := s.Logger() lg.Panic( "unexpected committed entry index", zap.Uint64("current-applied-index", ep.appliedi), zap.Uint64("first-committed-entry-index", firsti), ) } var ents []raftpb.Entry if ep.appliedi+1-firsti < uint64(len(apply.entries)) { ents = apply.entries[ep.appliedi+1-firsti:] } if len(ents) == 0 { return } var shouldstop bool if ep.appliedt, ep.appliedi, shouldstop = s.apply(ents, &ep.confState, apply.raftAdvancedC); shouldstop { go s.stopWithDelay(10*100*time.Millisecond, fmt.Errorf("the member has been permanently removed from the cluster")) } } func (s *EtcdServer) ForceSnapshot() { s.forceDiskSnapshot = true } func (s *EtcdServer) snapshotIfNeededAndCompactRaftLog(ep *etcdProgress) { // TODO: Remove disk snapshot in v3.7 shouldSnapshotToDisk := s.shouldSnapshotToDisk(ep) shouldSnapshotToMemory := s.shouldSnapshotToMemory(ep) if !shouldSnapshotToDisk && !shouldSnapshotToMemory { return } s.snapshot(ep, shouldSnapshotToDisk) s.compactRaftLog(ep.appliedi) } func (s *EtcdServer) shouldSnapshotToDisk(ep *etcdProgress) bool { return (s.forceDiskSnapshot && ep.appliedi != ep.diskSnapshotIndex) || (ep.appliedi-ep.diskSnapshotIndex > s.Cfg.SnapshotCount) } func (s *EtcdServer) shouldSnapshotToMemory(ep *etcdProgress) bool { return ep.appliedi > ep.memorySnapshotIndex+memorySnapshotCount } func (s *EtcdServer) hasMultipleVotingMembers() bool { return s.cluster != nil && len(s.cluster.VotingMemberIDs()) > 1 } func (s *EtcdServer) isLeader() bool { return uint64(s.MemberID()) == s.Lead() } // MoveLeader transfers the leader to the given transferee. func (s *EtcdServer) MoveLeader(ctx context.Context, lead, transferee uint64) error { member := s.cluster.Member(types.ID(transferee)) if member == nil || member.IsLearner { return errors.ErrBadLeaderTransferee } now := time.Now() interval := time.Duration(s.Cfg.TickMs) * time.Millisecond lg := s.Logger() lg.Info( "leadership transfer starting", zap.String("local-member-id", s.MemberID().String()), zap.String("current-leader-member-id", types.ID(lead).String()), zap.String("transferee-member-id", types.ID(transferee).String()), ) s.r.TransferLeadership(ctx, lead, transferee) for s.Lead() != transferee { select { case <-ctx.Done(): // time out return errors.ErrTimeoutLeaderTransfer case <-time.After(interval): } } // TODO: drain all requests, or drop all messages to the old leader lg.Info( "leadership transfer finished", zap.String("local-member-id", s.MemberID().String()), zap.String("old-leader-member-id", types.ID(lead).String()), zap.String("new-leader-member-id", types.ID(transferee).String()), zap.Duration("took", time.Since(now)), ) return nil } // TryTransferLeadershipOnShutdown transfers the leader to the chosen transferee. It is only used in server graceful shutdown. func (s *EtcdServer) TryTransferLeadershipOnShutdown() error { lg := s.Logger() if !s.isLeader() { lg.Info( "skipped leadership transfer; local server is not leader", zap.String("local-member-id", s.MemberID().String()), zap.String("current-leader-member-id", types.ID(s.Lead()).String()), ) return nil } if !s.hasMultipleVotingMembers() { lg.Info( "skipped leadership transfer for single voting member cluster", zap.String("local-member-id", s.MemberID().String()), zap.String("current-leader-member-id", types.ID(s.Lead()).String()), ) return nil } transferee, ok := longestConnected(s.r.transport, s.cluster.VotingMemberIDs()) if !ok { return errors.ErrUnhealthy } tm := s.Cfg.ReqTimeout() ctx, cancel := context.WithTimeout(s.ctx, tm) err := s.MoveLeader(ctx, s.Lead(), uint64(transferee)) cancel() return err } // HardStop stops the server without coordination with other members in the cluster. func (s *EtcdServer) HardStop() { select { case s.stop <- struct{}{}: case <-s.done: return } <-s.done } // Stop stops the server gracefully, and shuts down the running goroutine. // Stop should be called after a Start(s), otherwise it will block forever. // When stopping leader, Stop transfers its leadership to one of its peers // before stopping the server. // Stop terminates the Server and performs any necessary finalization. // Do and Process cannot be called after Stop has been invoked. func (s *EtcdServer) Stop() { lg := s.Logger() if err := s.TryTransferLeadershipOnShutdown(); err != nil { lg.Warn("leadership transfer failed", zap.String("local-member-id", s.MemberID().String()), zap.Error(err)) } s.HardStop() } // ReadyNotify returns a channel that will be closed when the server // is ready to serve client requests func (s *EtcdServer) ReadyNotify() <-chan struct{} { return s.readych } func (s *EtcdServer) stopWithDelay(d time.Duration, err error) { select { case <-time.After(d): case <-s.done: } select { case s.errorc <- err: default: } } // StopNotify returns a channel that receives an empty struct // when the server is stopped. func (s *EtcdServer) StopNotify() <-chan struct{} { return s.done } // StoppingNotify returns a channel that receives an empty struct // when the server is being stopped. func (s *EtcdServer) StoppingNotify() <-chan struct{} { return s.stopping } func (s *EtcdServer) checkMembershipOperationPermission(ctx context.Context) error { if s.authStore == nil { // In the context of ordinary etcd process, s.authStore will never be nil. // This branch is for handling cases in server_test.go return nil } // Note that this permission check is done in the API layer, // so TOCTOU problem can be caused potentially in a schedule like this: // update membership with user A -> revoke root role of A -> toApply membership change // in the state machine layer // However, both of membership change and role management requires the root privilege. // So careful operation by admins can prevent the problem. authInfo, err := s.AuthInfoFromCtx(ctx) if err != nil { return err } return s.AuthStore().IsAdminPermitted(authInfo) } func (s *EtcdServer) AddMember(ctx context.Context, memb membership.Member) ([]*membership.Member, error) { if err := s.checkMembershipOperationPermission(ctx); err != nil { return nil, err } // TODO: move Member to protobuf type b, err := json.Marshal(memb) if err != nil { return nil, err } // by default StrictReconfigCheck is enabled; reject new members if unhealthy. if err := s.mayAddMember(memb); err != nil { return nil, err } cc := raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: uint64(memb.ID), Context: b, } if memb.IsLearner { cc.Type = raftpb.ConfChangeAddLearnerNode } return s.configure(ctx, cc) } func (s *EtcdServer) mayAddMember(memb membership.Member) error { lg := s.Logger() if !s.Cfg.StrictReconfigCheck { return nil } // protect quorum when adding voting member if !memb.IsLearner && !s.cluster.IsReadyToAddVotingMember() { lg.Warn( "rejecting member add request; not enough healthy members", zap.String("local-member-id", s.MemberID().String()), zap.String("requested-member-add", fmt.Sprintf("%+v", memb)), zap.Error(errors.ErrNotEnoughStartedMembers), ) return errors.ErrNotEnoughStartedMembers } if !isConnectedFullySince(s.r.transport, time.Now().Add(-HealthInterval), s.MemberID(), s.cluster.VotingMembers()) { lg.Warn( "rejecting member add request; local member has not been connected to all peers, reconfigure breaks active quorum", zap.String("local-member-id", s.MemberID().String()), zap.String("requested-member-add", fmt.Sprintf("%+v", memb)), zap.Error(errors.ErrUnhealthy), ) return errors.ErrUnhealthy } return nil } func (s *EtcdServer) RemoveMember(ctx context.Context, id uint64) ([]*membership.Member, error) { if err := s.checkMembershipOperationPermission(ctx); err != nil { return nil, err } // by default StrictReconfigCheck is enabled; reject removal if leads to quorum loss if err := s.mayRemoveMember(types.ID(id)); err != nil { return nil, err } cc := raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: id, } return s.configure(ctx, cc) } // PromoteMember promotes a learner node to a voting node. func (s *EtcdServer) PromoteMember(ctx context.Context, id uint64) ([]*membership.Member, error) { // only raft leader has information on whether the to-be-promoted learner node is ready. If promoteMember call // fails with ErrNotLeader, forward the request to leader node via HTTP. If promoteMember call fails with error // other than ErrNotLeader, return the error. resp, err := s.promoteMember(ctx, id) if err == nil { learnerPromoteSucceed.Inc() return resp, nil } if !errorspkg.Is(err, errors.ErrNotLeader) { learnerPromoteFailed.WithLabelValues(err.Error()).Inc() return resp, err } cctx, cancel := context.WithTimeout(ctx, s.Cfg.ReqTimeout()) defer cancel() // forward to leader for cctx.Err() == nil { leader, err := s.waitLeader(cctx) if err != nil { return nil, err } for _, url := range leader.PeerURLs { resp, err := promoteMemberHTTP(cctx, url, id, s.peerRt) if err == nil { return resp, nil } // If member promotion failed, return early. Otherwise keep retry. if errorspkg.Is(err, errors.ErrLearnerNotReady) || errorspkg.Is(err, membership.ErrIDNotFound) || errorspkg.Is(err, membership.ErrMemberNotLearner) { return nil, err } } } if errorspkg.Is(cctx.Err(), context.DeadlineExceeded) { return nil, errors.ErrTimeout } return nil, errors.ErrCanceled } // promoteMember checks whether the to-be-promoted learner node is ready before sending the promote // request to raft. // The function returns ErrNotLeader if the local node is not raft leader (therefore does not have // enough information to determine if the learner node is ready), returns ErrLearnerNotReady if the // local node is leader (therefore has enough information) but decided the learner node is not ready // to be promoted. func (s *EtcdServer) promoteMember(ctx context.Context, id uint64) ([]*membership.Member, error) { if err := s.checkMembershipOperationPermission(ctx); err != nil { return nil, err } // check if we can promote this learner. if err := s.mayPromoteMember(types.ID(id)); err != nil { return nil, err } // build the context for the promote confChange. mark IsLearner to false and IsPromote to true. promoteChangeContext := membership.ConfigChangeContext{ Member: membership.Member{ ID: types.ID(id), }, IsPromote: true, } b, err := json.Marshal(promoteChangeContext) if err != nil { return nil, err } cc := raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: id, Context: b, } return s.configure(ctx, cc) } func (s *EtcdServer) mayPromoteMember(id types.ID) error { lg := s.Logger() if err := s.isLearnerReady(lg, uint64(id)); err != nil { return err } if !s.Cfg.StrictReconfigCheck { return nil } if !s.cluster.IsReadyToPromoteMember(uint64(id)) { lg.Warn( "rejecting member promote request; not enough healthy members", zap.String("local-member-id", s.MemberID().String()), zap.String("requested-member-remove-id", id.String()), zap.Error(errors.ErrNotEnoughStartedMembers), ) return errors.ErrNotEnoughStartedMembers } return nil } // check whether the learner catches up with leader or not. // Note: it will return nil if member is not found in cluster or if member is not learner. // These two conditions will be checked before toApply phase later. func (s *EtcdServer) isLearnerReady(lg *zap.Logger, id uint64) error { if err := s.waitAppliedIndex(); err != nil { return err } rs := s.raftStatus() // leader's raftStatus.Progress is not nil if rs.Progress == nil { return errors.ErrNotLeader } var learnerMatch uint64 isFound := false leaderID := rs.ID for memberID, progress := range rs.Progress { if id == memberID { // check its status learnerMatch = progress.Match isFound = true break } } // We should return an error in API directly, to avoid the request // being unnecessarily delivered to raft. if !isFound { return membership.ErrIDNotFound } leaderMatch := rs.Progress[leaderID].Match learnerReadyPercent := float64(learnerMatch) / float64(leaderMatch) // the learner's Match not caught up with leader yet if learnerReadyPercent < readyPercentThreshold { lg.Error( "rejecting promote learner: learner is not ready", zap.Float64("learner-ready-percent", learnerReadyPercent), zap.Float64("ready-percent-threshold", readyPercentThreshold), ) return errors.ErrLearnerNotReady } return nil } func (s *EtcdServer) mayRemoveMember(id types.ID) error { if !s.Cfg.StrictReconfigCheck { return nil } lg := s.Logger() member := s.cluster.Member(id) // no need to check quorum when removing non-voting member if member != nil && member.IsLearner { return nil } if !s.cluster.IsReadyToRemoveVotingMember(uint64(id)) { lg.Warn( "rejecting member remove request; not enough healthy members", zap.String("local-member-id", s.MemberID().String()), zap.String("requested-member-remove-id", id.String()), zap.Error(errors.ErrNotEnoughStartedMembers), ) return errors.ErrNotEnoughStartedMembers } // downed member is safe to remove since it's not part of the active quorum if t := s.r.transport.ActiveSince(id); id != s.MemberID() && t.IsZero() { return nil } // protect quorum if some members are down m := s.cluster.VotingMembers() active := numConnectedSince(s.r.transport, time.Now().Add(-HealthInterval), s.MemberID(), m) if (active - 1) < 1+((len(m)-1)/2) { lg.Warn( "rejecting member remove request; local member has not been connected to all peers, reconfigure breaks active quorum", zap.String("local-member-id", s.MemberID().String()), zap.String("requested-member-remove", id.String()), zap.Int("active-peers", active), zap.Error(errors.ErrUnhealthy), ) return errors.ErrUnhealthy } return nil } func (s *EtcdServer) UpdateMember(ctx context.Context, memb membership.Member) ([]*membership.Member, error) { b, merr := json.Marshal(memb) if merr != nil { return nil, merr } if err := s.checkMembershipOperationPermission(ctx); err != nil { return nil, err } cc := raftpb.ConfChange{ Type: raftpb.ConfChangeUpdateNode, NodeID: uint64(memb.ID), Context: b, } return s.configure(ctx, cc) } func (s *EtcdServer) MemberList(ctx context.Context, r *pb.MemberListRequest) ([]*membership.Member, error) { if r.Linearizable { if err := s.LinearizableReadNotify(ctx); err != nil { return nil, err } } if err := s.checkMembershipOperationPermission(ctx); err != nil { return nil, err } return s.cluster.Members(), nil } func (s *EtcdServer) setCommittedIndex(v uint64) { s.committedIndex.Store(v) } func (s *EtcdServer) getCommittedIndex() uint64 { return s.committedIndex.Load() } func (s *EtcdServer) setAppliedIndex(v uint64) { s.appliedIndex.Store(v) } func (s *EtcdServer) getAppliedIndex() uint64 { return s.appliedIndex.Load() } func (s *EtcdServer) setTerm(v uint64) { s.term.Store(v) } func (s *EtcdServer) getTerm() uint64 { return s.term.Load() } func (s *EtcdServer) setLead(v uint64) { s.lead.Store(v) } func (s *EtcdServer) getLead() uint64 { return s.lead.Load() } func (s *EtcdServer) LeaderChangedNotify() <-chan struct{} { return s.leaderChanged.Receive() } // FirstCommitInTermNotify returns channel that will be unlocked on first // entry committed in new term, which is necessary for new leader to answer // read-only requests (leader is not able to respond any read-only requests // as long as linearizable semantic is required) func (s *EtcdServer) FirstCommitInTermNotify() <-chan struct{} { return s.firstCommitInTerm.Receive() } // MemberId returns the ID of the local member. // Deprecated: Please use (*EtcdServer) MemberID instead. // //revive:disable:var-naming func (s *EtcdServer) MemberId() types.ID { return s.MemberID() } //revive:enable:var-naming func (s *EtcdServer) MemberID() types.ID { return s.memberID } func (s *EtcdServer) Leader() types.ID { return types.ID(s.getLead()) } func (s *EtcdServer) Lead() uint64 { return s.getLead() } func (s *EtcdServer) CommittedIndex() uint64 { return s.getCommittedIndex() } func (s *EtcdServer) AppliedIndex() uint64 { return s.getAppliedIndex() } func (s *EtcdServer) Term() uint64 { return s.getTerm() } type confChangeResponse struct { membs []*membership.Member raftAdvanceC <-chan struct{} err error } // configure sends a configuration change through consensus and // then waits for it to be applied to the server. It // will block until the change is performed or there is an error. func (s *EtcdServer) configure(ctx context.Context, cc raftpb.ConfChange) ([]*membership.Member, error) { lg := s.Logger() cc.ID = s.reqIDGen.Next() ch := s.w.Register(cc.ID) start := time.Now() if err := s.r.ProposeConfChange(ctx, cc); err != nil { s.w.Trigger(cc.ID, nil) return nil, err } select { case x := <-ch: if x == nil { lg.Panic("failed to configure") } resp := x.(*confChangeResponse) // etcdserver need to ensure the raft has already been notified // or advanced before it responds to the client. Otherwise, the // following config change request may be rejected. // See https://github.com/etcd-io/etcd/issues/15528. <-resp.raftAdvanceC lg.Info( "applied a configuration change through raft", zap.String("local-member-id", s.MemberID().String()), zap.String("raft-conf-change", cc.Type.String()), zap.String("raft-conf-change-node-id", types.ID(cc.NodeID).String()), ) return resp.membs, resp.err case <-ctx.Done(): s.w.Trigger(cc.ID, nil) // GC wait return nil, s.parseProposeCtxErr(ctx.Err(), start) case <-s.stopping: return nil, errors.ErrStopped } } // publishV3 registers server information into the cluster using v3 request. The // information is the JSON representation of this server's member struct, updated // with the static clientURLs of the server. // The function keeps attempting to register until it succeeds, // or its server is stopped. func (s *EtcdServer) publishV3(timeout time.Duration) { req := &membershippb.ClusterMemberAttrSetRequest{ Member_ID: uint64(s.MemberID()), MemberAttributes: &membershippb.Attributes{ Name: s.attributes.Name, ClientUrls: s.attributes.ClientURLs, }, } // gofail: var beforePublishing struct{} lg := s.Logger() for { select { case <-s.stopping: lg.Warn( "stopped publish because server is stopping", zap.String("local-member-id", s.MemberID().String()), zap.String("local-member-attributes", fmt.Sprintf("%+v", s.attributes)), zap.Duration("publish-timeout", timeout), ) return default: } ctx, cancel := context.WithTimeout(s.ctx, timeout) _, err := s.raftRequest(ctx, pb.InternalRaftRequest{ClusterMemberAttrSet: req}) cancel() switch err { case nil: close(s.readych) lg.Info( "published local member to cluster through raft", zap.String("local-member-id", s.MemberID().String()), zap.String("local-member-attributes", fmt.Sprintf("%+v", s.attributes)), zap.String("cluster-id", s.cluster.ID().String()), zap.Duration("publish-timeout", timeout), ) return default: lg.Warn( "failed to publish local member to cluster through raft", zap.String("local-member-id", s.MemberID().String()), zap.String("local-member-attributes", fmt.Sprintf("%+v", s.attributes)), zap.Duration("publish-timeout", timeout), zap.Error(err), ) } } } func (s *EtcdServer) sendMergedSnap(merged snap.Message) { s.inflightSnapshots.Add(1) lg := s.Logger() fields := []zap.Field{ zap.String("from", s.MemberID().String()), zap.String("to", types.ID(merged.To).String()), zap.Int64("bytes", merged.TotalSize), zap.String("size", humanize.Bytes(uint64(merged.TotalSize))), } now := time.Now() s.r.transport.SendSnapshot(merged) lg.Info("sending merged snapshot", fields...) s.GoAttach(func() { select { case ok := <-merged.CloseNotify(): // delay releasing inflight snapshot for another 30 seconds to // block log compaction. // If the follower still fails to catch up, it is probably just too slow // to catch up. We cannot avoid the snapshot cycle anyway. if ok { select { case <-time.After(releaseDelayAfterSnapshot): case <-s.stopping: } } s.inflightSnapshots.Add(-1) lg.Info("sent merged snapshot", append(fields, zap.Duration("took", time.Since(now)))...) case <-s.stopping: lg.Warn("canceled sending merged snapshot; server stopping", fields...) return } }) } // toApply takes entries received from Raft (after it has been committed) and // applies them to the current state of the EtcdServer. // The given entries should not be empty. func (s *EtcdServer) apply( es []raftpb.Entry, confState *raftpb.ConfState, raftAdvancedC <-chan struct{}, ) (appliedt uint64, appliedi uint64, shouldStop bool) { s.lg.Debug("Applying entries", zap.Int("num-entries", len(es))) for i := range es { e := es[i] index := s.consistIndex.ConsistentIndex() s.lg.Debug("Applying entry", zap.Uint64("consistent-index", index), zap.Uint64("entry-index", e.Index), zap.Uint64("entry-term", e.Term), zap.Stringer("entry-type", e.Type)) // We need to toApply all WAL entries on top of v2store // and only 'unapplied' (e.Index>backend.ConsistentIndex) on the backend. shouldApplyV3 := membership.ApplyV2storeOnly if e.Index > index { shouldApplyV3 = membership.ApplyBoth // set the consistent index of current executing entry s.consistIndex.SetConsistentApplyingIndex(e.Index, e.Term) } switch e.Type { case raftpb.EntryNormal: // gofail: var beforeApplyOneEntryNormal struct{} s.applyEntryNormal(&e, shouldApplyV3) s.setAppliedIndex(e.Index) s.setTerm(e.Term) case raftpb.EntryConfChange: // gofail: var beforeApplyOneConfChange struct{} var cc raftpb.ConfChange pbutil.MustUnmarshal(&cc, e.Data) removedSelf, err := s.applyConfChange(cc, confState, shouldApplyV3) s.setAppliedIndex(e.Index) s.setTerm(e.Term) shouldStop = shouldStop || removedSelf s.w.Trigger(cc.ID, &confChangeResponse{s.cluster.Members(), raftAdvancedC, err}) default: lg := s.Logger() lg.Panic( "unknown entry type; must be either EntryNormal or EntryConfChange", zap.String("type", e.Type.String()), ) } appliedi, appliedt = e.Index, e.Term } return appliedt, appliedi, shouldStop } // applyEntryNormal applies an EntryNormal type raftpb request to the EtcdServer func (s *EtcdServer) applyEntryNormal(e *raftpb.Entry, shouldApplyV3 membership.ShouldApplyV3) { if shouldApplyV3 { defer func() { // The txPostLockInsideApplyHook will not get called in some cases, // in which we should move the consistent index forward directly. newIndex := s.consistIndex.ConsistentIndex() if newIndex < e.Index { s.consistIndex.SetConsistentIndex(e.Index, e.Term) } }() } // raft state machine may generate noop entry when leader confirmation. // skip it in advance to avoid some potential bug in the future if len(e.Data) == 0 { s.firstCommitInTerm.Notify() // promote lessor when the local member is leader and finished // applying all entries from the last term. if s.isLeader() { s.lessor.Promote(s.Cfg.ElectionTimeout()) } return } ar, id := apply.Apply(s.lg, e, s.uberApply, s.w, shouldApplyV3) // do not re-toApply applied entries. if !shouldApplyV3 { return } if ar == nil { return } if !errorspkg.Is(ar.Err, errors.ErrNoSpace) || len(s.alarmStore.Get(pb.AlarmType_NOSPACE)) > 0 { s.w.Trigger(id, ar) return } lg := s.Logger() lg.Warn( "message exceeded backend quota; raising alarm", zap.Int64("quota-size-bytes", s.Cfg.QuotaBackendBytes), zap.String("quota-size", humanize.Bytes(uint64(s.Cfg.QuotaBackendBytes))), zap.Error(ar.Err), ) s.GoAttach(func() { a := &pb.AlarmRequest{ MemberID: uint64(s.MemberID()), Action: pb.AlarmRequest_ACTIVATE, Alarm: pb.AlarmType_NOSPACE, } s.raftRequest(s.ctx, pb.InternalRaftRequest{Alarm: a}) s.w.Trigger(id, ar) }) } // applyConfChange applies a ConfChange to the server. It is only // invoked with a ConfChange that has already passed through Raft func (s *EtcdServer) applyConfChange(cc raftpb.ConfChange, confState *raftpb.ConfState, shouldApplyV3 membership.ShouldApplyV3) (bool, error) { lg := s.Logger() if err := s.cluster.ValidateConfigurationChange(cc, shouldApplyV3); err != nil { lg.Error("Validation on configuration change failed", zap.Bool("shouldApplyV3", bool(shouldApplyV3)), zap.Error(err)) cc.NodeID = raft.None s.r.ApplyConfChange(cc) // The txPostLock callback will not get called in this case, // so we should set the consistent index directly. if s.consistIndex != nil && membership.ApplyBoth == shouldApplyV3 { applyingIndex, applyingTerm := s.consistIndex.ConsistentApplyingIndex() s.consistIndex.SetConsistentIndex(applyingIndex, applyingTerm) } return false, err } // We don't validate the configuration change when `shouldApplyV3` // is false, so we shouldn't apply it to raft either in this case. // Otherwise, we might apply an invalid confChange (which failed // the validation previously) to raft on bootstrap. if shouldApplyV3 { *confState = *s.r.ApplyConfChange(cc) } s.beHooks.SetConfState(confState) switch cc.Type { case raftpb.ConfChangeAddNode, raftpb.ConfChangeAddLearnerNode: confChangeContext := new(membership.ConfigChangeContext) if err := json.Unmarshal(cc.Context, confChangeContext); err != nil { lg.Panic("failed to unmarshal member", zap.Error(err)) } if cc.NodeID != uint64(confChangeContext.Member.ID) { lg.Panic( "got different member ID", zap.String("member-id-from-config-change-entry", types.ID(cc.NodeID).String()), zap.String("member-id-from-message", confChangeContext.Member.ID.String()), ) } if confChangeContext.IsPromote { s.cluster.PromoteMember(confChangeContext.Member.ID, shouldApplyV3) } else { s.cluster.AddMember(&confChangeContext.Member, shouldApplyV3) if confChangeContext.Member.ID != s.MemberID() { s.r.transport.AddPeer(confChangeContext.Member.ID, confChangeContext.PeerURLs) } } case raftpb.ConfChangeRemoveNode: id := types.ID(cc.NodeID) s.cluster.RemoveMember(id, shouldApplyV3) if id == s.MemberID() { return true, nil } s.r.transport.RemovePeer(id) case raftpb.ConfChangeUpdateNode: m := new(membership.Member) if err := json.Unmarshal(cc.Context, m); err != nil { lg.Panic("failed to unmarshal member", zap.Error(err)) } if cc.NodeID != uint64(m.ID) { lg.Panic( "got different member ID", zap.String("member-id-from-config-change-entry", types.ID(cc.NodeID).String()), zap.String("member-id-from-message", m.ID.String()), ) } s.cluster.UpdateRaftAttributes(m.ID, m.RaftAttributes, shouldApplyV3) if m.ID != s.MemberID() { s.r.transport.UpdatePeer(m.ID, m.PeerURLs) } } return false, nil } // TODO: non-blocking snapshot func (s *EtcdServer) snapshot(ep *etcdProgress, toDisk bool) { lg := s.Logger() d := GetMembershipInfoInV2Format(lg, s.cluster) if toDisk { s.Logger().Info( "triggering snapshot", zap.String("local-member-id", s.MemberID().String()), zap.Uint64("local-member-applied-index", ep.appliedi), zap.Uint64("local-member-snapshot-index", ep.diskSnapshotIndex), zap.Uint64("local-member-snapshot-count", s.Cfg.SnapshotCount), zap.Bool("snapshot-forced", s.forceDiskSnapshot), ) s.forceDiskSnapshot = false // commit kv to write metadata (for example: consistent index) to disk. // // This guarantees that Backend's consistent_index is >= index of last snapshot. // // KV().commit() updates the consistent index in backend. // All operations that update consistent index must be called sequentially // from applyAll function. // So KV().Commit() cannot run in parallel with toApply. It has to be called outside // the go routine created below. s.KV().Commit() } // For backward compatibility, generate v2 snapshot from v3 state. snap, err := s.r.raftStorage.CreateSnapshot(ep.appliedi, &ep.confState, d) if err != nil { // the snapshot was done asynchronously with the progress of raft. // raft might have already got a newer snapshot. if errorspkg.Is(err, raft.ErrSnapOutOfDate) { return } lg.Panic("failed to create snapshot", zap.Error(err)) } ep.memorySnapshotIndex = ep.appliedi verifyConsistentIndexIsLatest(snap, s.consistIndex.ConsistentIndex()) if toDisk { // SaveSnap saves the snapshot to file and appends the corresponding WAL entry. if err = s.r.storage.SaveSnap(snap); err != nil { lg.Panic("failed to save snapshot", zap.Error(err)) } ep.diskSnapshotIndex = ep.appliedi if err = s.r.storage.Release(snap); err != nil { lg.Panic("failed to release wal", zap.Error(err)) } lg.Info( "saved snapshot to disk", zap.Uint64("snapshot-index", snap.Metadata.Index), ) } } func (s *EtcdServer) compactRaftLog(snapi uint64) { lg := s.Logger() // When sending a snapshot, etcd will pause compaction. // After receives a snapshot, the slow follower needs to get all the entries right after // the snapshot sent to catch up. If we do not pause compaction, the log entries right after // the snapshot sent might already be compacted. It happens when the snapshot takes long time // to send and save. Pausing compaction avoids triggering a snapshot sending cycle. if s.inflightSnapshots.Load() != 0 { lg.Info("skip compaction since there is an inflight snapshot") return } // keep some in memory log entries for slow followers. compacti := uint64(1) if snapi > s.Cfg.SnapshotCatchUpEntries { compacti = snapi - s.Cfg.SnapshotCatchUpEntries } err := s.r.raftStorage.Compact(compacti) if err != nil { // the compaction was done asynchronously with the progress of raft. // raft log might already been compact. if errorspkg.Is(err, raft.ErrCompacted) { return } lg.Panic("failed to compact", zap.Error(err)) } lg.Debug( "compacted Raft logs", zap.Uint64("compact-index", compacti), ) } // CutPeer drops messages to the specified peer. func (s *EtcdServer) CutPeer(id types.ID) { tr, ok := s.r.transport.(*rafthttp.Transport) if ok { tr.CutPeer(id) } } // MendPeer recovers the message dropping behavior of the given peer. func (s *EtcdServer) MendPeer(id types.ID) { tr, ok := s.r.transport.(*rafthttp.Transport) if ok { tr.MendPeer(id) } } func (s *EtcdServer) PauseSending() { s.r.pauseSending() } func (s *EtcdServer) ResumeSending() { s.r.resumeSending() } func (s *EtcdServer) ClusterVersion() *semver.Version { if s.cluster == nil { return nil } return s.cluster.Version() } func (s *EtcdServer) StorageVersion() *semver.Version { // `applySnapshot` sets a new backend instance, so we need to acquire the bemu lock. s.bemu.RLock() defer s.bemu.RUnlock() v, err := schema.DetectSchemaVersion(s.lg, s.be.ReadTx()) if err != nil { s.lg.Warn("Failed to detect schema version", zap.Error(err)) return nil } return &v } // monitorClusterVersions every monitorVersionInterval checks if it's the leader and updates cluster version if needed. func (s *EtcdServer) monitorClusterVersions() { lg := s.Logger() monitor := serverversion.NewMonitor(lg, NewServerVersionAdapter(s)) for { select { case <-s.firstCommitInTerm.Receive(): case <-time.After(monitorVersionInterval): case <-s.stopping: lg.Info("server has stopped; stopping cluster version's monitor") return } if s.Leader() != s.MemberID() { continue } err := monitor.UpdateClusterVersionIfNeeded() if err != nil { s.lg.Error("Failed to monitor cluster version", zap.Error(err)) } } } // monitorStorageVersion every monitorVersionInterval updates storage version if needed. func (s *EtcdServer) monitorStorageVersion() { lg := s.Logger() monitor := serverversion.NewMonitor(lg, NewServerVersionAdapter(s)) for { select { case <-time.After(monitorVersionInterval): case <-s.clusterVersionChanged.Receive(): case <-s.stopping: lg.Info("server has stopped; stopping storage version's monitor") return } monitor.UpdateStorageVersionIfNeeded() } } func (s *EtcdServer) monitorKVHash() { t := s.Cfg.CorruptCheckTime if t == 0 { return } checkTicker := time.NewTicker(t) defer checkTicker.Stop() lg := s.Logger() lg.Info( "enabled corruption checking", zap.String("local-member-id", s.MemberID().String()), zap.Duration("interval", t), ) for { select { case <-s.stopping: lg.Info("server has stopped; stopping kv hash's monitor") return case <-checkTicker.C: } backend.VerifyBackendConsistency(s.be, lg, false, schema.AllBuckets...) if !s.isLeader() { continue } if err := s.corruptionChecker.PeriodicCheck(); err != nil { lg.Warn("failed to check hash KV", zap.Error(err)) } } } func (s *EtcdServer) monitorCompactHash() { if !s.FeatureEnabled(features.CompactHashCheck) { return } t := s.Cfg.CompactHashCheckTime for { select { case <-time.After(t): case <-s.stopping: lg := s.Logger() lg.Info("server has stopped; stopping compact hash's monitor") return } if !s.isLeader() { continue } s.corruptionChecker.CompactHashCheck() } } func (s *EtcdServer) updateClusterVersionV3(ver string) { lg := s.Logger() if s.cluster.Version() == nil { lg.Info( "setting up initial cluster version using v3 API", zap.String("cluster-version", version.Cluster(ver)), ) } else { lg.Info( "updating cluster version using v3 API", zap.String("from", version.Cluster(s.cluster.Version().String())), zap.String("to", version.Cluster(ver)), ) } req := membershippb.ClusterVersionSetRequest{Ver: ver} ctx, cancel := context.WithTimeout(s.ctx, s.Cfg.ReqTimeout()) _, err := s.raftRequest(ctx, pb.InternalRaftRequest{ClusterVersionSet: &req}) cancel() switch { case errorspkg.Is(err, nil): lg.Info("cluster version is updated", zap.String("cluster-version", version.Cluster(ver))) return case errorspkg.Is(err, errors.ErrStopped): lg.Warn("aborting cluster version update; server is stopped", zap.Error(err)) return default: lg.Warn("failed to update cluster version", zap.Error(err)) } } // monitorDowngrade every DowngradeCheckTime checks if it's the leader and cancels downgrade if needed. func (s *EtcdServer) monitorDowngrade() { monitor := serverversion.NewMonitor(s.Logger(), NewServerVersionAdapter(s)) t := s.Cfg.DowngradeCheckTime if t == 0 { return } for { select { case <-time.After(t): case <-s.stopping: return } if !s.isLeader() { continue } monitor.CancelDowngradeIfNeeded() } } func (s *EtcdServer) parseProposeCtxErr(err error, start time.Time) error { switch { case errorspkg.Is(err, context.Canceled): return errors.ErrCanceled case errorspkg.Is(err, context.DeadlineExceeded): s.leadTimeMu.RLock() curLeadElected := s.leadElectedTime s.leadTimeMu.RUnlock() prevLeadLost := curLeadElected.Add(-2 * time.Duration(s.Cfg.ElectionTicks) * time.Duration(s.Cfg.TickMs) * time.Millisecond) if start.After(prevLeadLost) && start.Before(curLeadElected) { return errors.ErrTimeoutDueToLeaderFail } lead := types.ID(s.getLead()) switch lead { case types.ID(raft.None): // TODO: return error to specify it happens because the cluster does not have leader now case s.MemberID(): if !isConnectedToQuorumSince(s.r.transport, start, s.MemberID(), s.cluster.Members()) { return errors.ErrTimeoutDueToConnectionLost } default: if !isConnectedSince(s.r.transport, start, lead) { return errors.ErrTimeoutDueToConnectionLost } } return errors.ErrTimeout default: return err } } func (s *EtcdServer) KV() mvcc.WatchableKV { return s.kv } func (s *EtcdServer) Backend() backend.Backend { s.bemu.RLock() defer s.bemu.RUnlock() return s.be } func (s *EtcdServer) AuthStore() auth.AuthStore { return s.authStore } func (s *EtcdServer) restoreAlarms() error { as, err := v3alarm.NewAlarmStore(s.lg, schema.NewAlarmBackend(s.lg, s.be)) if err != nil { return err } s.alarmStore = as return nil } // GoAttach creates a goroutine on a given function and tracks it using // the etcdserver waitgroup. // The passed function should interrupt on s.StoppingNotify(). func (s *EtcdServer) GoAttach(f func()) { s.wgMu.RLock() // this blocks with ongoing close(s.stopping) defer s.wgMu.RUnlock() select { case <-s.stopping: lg := s.Logger() lg.Warn("server has stopped; skipping GoAttach") return default: } // now safe to add since waitgroup wait has not started yet s.wg.Add(1) go func() { defer s.wg.Done() f() }() } func (s *EtcdServer) Alarms() []*pb.AlarmMember { return s.alarmStore.Get(pb.AlarmType_NONE) } // IsLearner returns if the local member is raft learner func (s *EtcdServer) IsLearner() bool { return s.cluster.IsLocalMemberLearner() } // IsMemberExist returns if the member with the given id exists in cluster. func (s *EtcdServer) IsMemberExist(id types.ID) bool { return s.cluster.IsMemberExist(id) } // raftStatus returns the raft status of this etcd node. func (s *EtcdServer) raftStatus() raft.Status { return s.r.Node.Status() } func (s *EtcdServer) Version() *serverversion.Manager { return serverversion.NewManager(s.Logger(), NewServerVersionAdapter(s)) } func (s *EtcdServer) getTxPostLockInsideApplyHook() func() { return func() { applyingIdx, applyingTerm := s.consistIndex.ConsistentApplyingIndex() if applyingIdx > s.consistIndex.UnsafeConsistentIndex() { s.consistIndex.SetConsistentIndex(applyingIdx, applyingTerm) } } } func (s *EtcdServer) CorruptionChecker() CorruptionChecker { return s.corruptionChecker } func addFeatureGateMetrics(fg featuregate.FeatureGate, guageVec *prometheus.GaugeVec) { for feature, featureSpec := range fg.(featuregate.MutableFeatureGate).GetAll() { var metricVal float64 if fg.Enabled(feature) { metricVal = 1 } else { metricVal = 0 } guageVec.With(prometheus.Labels{"name": string(feature), "stage": string(featureSpec.PreRelease)}).Set(metricVal) } } ================================================ FILE: server/etcdserver/server_access_control.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import "sync" // AccessController controls etcd server HTTP request access. type AccessController struct { corsMu sync.RWMutex CORS map[string]struct{} hostWhitelistMu sync.RWMutex HostWhitelist map[string]struct{} } // NewAccessController returns a new "AccessController" with default "*" values. func NewAccessController() *AccessController { return &AccessController{ CORS: map[string]struct{}{"*": {}}, HostWhitelist: map[string]struct{}{"*": {}}, } } // OriginAllowed determines whether the server will allow a given CORS origin. // If CORS is empty, allow all. func (ac *AccessController) OriginAllowed(origin string) bool { ac.corsMu.RLock() defer ac.corsMu.RUnlock() if len(ac.CORS) == 0 { // allow all return true } _, ok := ac.CORS["*"] if ok { return true } _, ok = ac.CORS[origin] return ok } // IsHostWhitelisted returns true if the host is whitelisted. // If whitelist is empty, allow all. func (ac *AccessController) IsHostWhitelisted(host string) bool { ac.hostWhitelistMu.RLock() defer ac.hostWhitelistMu.RUnlock() if len(ac.HostWhitelist) == 0 { // allow all return true } _, ok := ac.HostWhitelist["*"] if ok { return true } _, ok = ac.HostWhitelist[host] return ok } ================================================ FILE: server/etcdserver/server_access_control_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "testing" "github.com/stretchr/testify/assert" ) func TestOriginAllowed(t *testing.T) { tests := []struct { accessController *AccessController origin string allowed bool }{ { &AccessController{ CORS: map[string]struct{}{}, }, "https://example.com", true, }, { &AccessController{ CORS: map[string]struct{}{"*": {}}, }, "https://example.com", true, }, { &AccessController{ CORS: map[string]struct{}{"https://example.com": {}, "http://example.org": {}}, }, "https://example.com", true, }, { &AccessController{ CORS: map[string]struct{}{"http://example.org": {}}, }, "https://example.com", false, }, { &AccessController{ CORS: map[string]struct{}{"*": {}, "http://example.org/": {}}, }, "https://example.com", true, }, } for _, tt := range tests { allowed := tt.accessController.OriginAllowed(tt.origin) assert.Equal(t, allowed, tt.allowed) } } func TestIsHostWhitelisted(t *testing.T) { tests := []struct { accessController *AccessController host string whitelisted bool }{ { &AccessController{ HostWhitelist: map[string]struct{}{}, }, "example.com", true, }, { &AccessController{ HostWhitelist: map[string]struct{}{"*": {}}, }, "example.com", true, }, { &AccessController{ HostWhitelist: map[string]struct{}{"example.com": {}, "example.org": {}}, }, "example.com", true, }, { &AccessController{ HostWhitelist: map[string]struct{}{"example.org": {}}, }, "example.com", false, }, { &AccessController{ HostWhitelist: map[string]struct{}{"*": {}, "example.org/": {}}, }, "example.com", true, }, } for _, tt := range tests { whitelisted := tt.accessController.IsHostWhitelisted(tt.host) assert.Equal(t, whitelisted, tt.whitelisted) } } ================================================ FILE: server/etcdserver/server_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "context" "encoding/binary" "encoding/json" errorspkg "errors" "fmt" "math" "net/http" "os" "path/filepath" "reflect" "strings" "sync" "testing" "time" "github.com/prometheus/client_golang/prometheus" ptestutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/membershippb" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/client/pkg/v3/verify" "go.etcd.io/etcd/pkg/v3/featuregate" "go.etcd.io/etcd/pkg/v3/idutil" "go.etcd.io/etcd/pkg/v3/notify" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/pkg/v3/wait" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" apply2 "go.etcd.io/etcd/server/v3/etcdserver/apply" "go.etcd.io/etcd/server/v3/etcdserver/cindex" "go.etcd.io/etcd/server/v3/etcdserver/errors" "go.etcd.io/etcd/server/v3/features" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/mock/mockstorage" "go.etcd.io/etcd/server/v3/mock/mockstore" "go.etcd.io/etcd/server/v3/mock/mockwait" serverstorage "go.etcd.io/etcd/server/v3/storage" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) // TestApplyRepeat tests that server handles repeat raft messages gracefully func TestApplyRepeat(t *testing.T) { lg := zaptest.NewLogger(t) n := newNodeConfChangeCommitterStream() n.readyc <- raft.Ready{ SoftState: &raft.SoftState{RaftState: raft.StateLeader}, } cl := newTestCluster(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) cl.SetBackend(schema.NewMembershipBackend(lg, be)) cl.AddMember(&membership.Member{ID: 1234}, true) r := newRaftNode(raftNodeConfig{ lg: zaptest.NewLogger(t), Node: n, raftStorage: raft.NewMemoryStorage(), storage: mockstorage.NewStorageRecorder(""), transport: newNopTransporter(), }) s := &EtcdServer{ lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), r: *r, cluster: cl, reqIDGen: idutil.NewGenerator(0, time.Time{}), consistIndex: cindex.NewFakeConsistentIndex(0), uberApply: uberApplierMock{}, } s.start() req := &pb.InternalRaftRequest{ Header: &pb.RequestHeader{ID: 1}, Put: &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}, } ents := []raftpb.Entry{{Index: 1, Data: pbutil.MustMarshal(req)}} n.readyc <- raft.Ready{CommittedEntries: ents} // dup msg n.readyc <- raft.Ready{CommittedEntries: ents} // use a conf change to block until dup msgs are all processed cc := &raftpb.ConfChange{Type: raftpb.ConfChangeRemoveNode, NodeID: 2} ents = []raftpb.Entry{{ Index: 2, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(cc), }} n.readyc <- raft.Ready{CommittedEntries: ents} // wait for conf change message act, err := n.Wait(1) // wait for stop message (async to avoid deadlock) stopc := make(chan error, 1) go func() { _, werr := n.Wait(1) stopc <- werr }() s.Stop() // only want to confirm etcdserver won't panic; no data to check if err != nil { t.Fatal(err) } require.NotEmptyf(t, act, "expected len(act)=0, got %d", len(act)) err = <-stopc require.NoErrorf(t, err, "error on stop (%v)", err) } type uberApplierMock struct{} func (uberApplierMock) Apply(r *pb.InternalRaftRequest, shouldApplyV3 membership.ShouldApplyV3) *apply2.Result { return &apply2.Result{} } func TestApplyConfStateWithRestart(t *testing.T) { n := newNodeRecorder() srv := newServer(t, n) defer srv.Cleanup() assert.Equal(t, uint64(0), srv.consistIndex.ConsistentIndex()) var nodeID uint64 = 1 memberData, err := json.Marshal(&membership.Member{ID: types.ID(nodeID), RaftAttributes: membership.RaftAttributes{PeerURLs: []string{""}}}) if err != nil { t.Fatal(err) } entries := []raftpb.Entry{ { Term: 1, Index: 1, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: nodeID, Context: memberData, }), }, { Term: 1, Index: 2, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: nodeID, }), }, { Term: 1, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&raftpb.ConfChange{ Type: raftpb.ConfChangeUpdateNode, NodeID: nodeID, Context: memberData, }), }, } want := []testutil.Action{ { Name: "ApplyConfChange", Params: []any{raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: nodeID, Context: memberData, }}, }, { Name: "ApplyConfChange", Params: []any{raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: nodeID, }}, }, // This action is expected to fail validation, thus NodeID is set to 0 { Name: "ApplyConfChange", Params: []any{raftpb.ConfChange{ Type: raftpb.ConfChangeUpdateNode, Context: memberData, NodeID: 0, }}, }, } confState := raftpb.ConfState{} t.Log("Applying entries for the first time") srv.apply(entries, &confState, nil) if got, _ := n.Wait(len(want)); !reflect.DeepEqual(got, want) { t.Errorf("actions don't match\n got %+v\n want %+v", got, want) } t.Log("Simulating etcd restart by clearing v3 store") be, _ := betesting.NewDefaultTmpBackend(t) t.Cleanup(func() { betesting.Close(t, be) }) lg := zaptest.NewLogger(t) srv.cluster.SetBackend(schema.NewMembershipBackend(lg, be)) srv.beHooks = serverstorage.NewBackendHooks(lg, srv.consistIndex) srv.consistIndex.SetBackend(be) t.Log("Reapplying same entries after restart") srv.apply(entries, &confState, nil) if got, _ := n.Wait(2 * len(want)); !reflect.DeepEqual(got[len(want):], want) { t.Errorf("actions don't match\n got %+v\n want %+v", got, want) } } func newServer(t *testing.T, recorder *nodeRecorder) *EtcdServer { lg := zaptest.NewLogger(t) be, _ := betesting.NewDefaultTmpBackend(t) t.Cleanup(func() { betesting.Close(t, be) }) srv := &EtcdServer{ lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), r: *newRaftNode(raftNodeConfig{lg: lg, Node: recorder}), cluster: membership.NewCluster(lg), consistIndex: cindex.NewConsistentIndex(be), } srv.cluster.SetBackend(schema.NewMembershipBackend(lg, be)) srv.beHooks = serverstorage.NewBackendHooks(lg, srv.consistIndex) srv.r.transport = newNopTransporter() srv.w = mockwait.NewNop() return srv } func TestApplyConfChangeError(t *testing.T) { lg := zaptest.NewLogger(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) cl := membership.NewCluster(lg) cl.SetBackend(schema.NewMembershipBackend(lg, be)) for i := 1; i <= 4; i++ { cl.AddMember(&membership.Member{ID: types.ID(i)}, true) } cl.RemoveMember(4, true) attr := membership.RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 1)}} ctx, err := json.Marshal(&membership.Member{ID: types.ID(1), RaftAttributes: attr}) if err != nil { t.Fatal(err) } attr = membership.RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 4)}} ctx4, err := json.Marshal(&membership.Member{ID: types.ID(1), RaftAttributes: attr}) if err != nil { t.Fatal(err) } attr = membership.RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 5)}} ctx5, err := json.Marshal(&membership.Member{ID: types.ID(1), RaftAttributes: attr}) if err != nil { t.Fatal(err) } tests := []struct { cc raftpb.ConfChange werr error }{ { raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: 4, Context: ctx4, }, membership.ErrIDRemoved, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeUpdateNode, NodeID: 4, Context: ctx4, }, membership.ErrIDRemoved, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: 1, Context: ctx, }, membership.ErrIDExists, }, { raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: 5, Context: ctx5, }, membership.ErrIDNotFound, }, } for i, tt := range tests { n := newNodeRecorder() srv := &EtcdServer{ lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), r: *newRaftNode(raftNodeConfig{lg: zaptest.NewLogger(t), Node: n}), cluster: cl, } _, err := srv.applyConfChange(tt.cc, nil, true) if !errorspkg.Is(err, tt.werr) { t.Errorf("#%d: applyConfChange error = %v, want %v", i, err, tt.werr) } cc := raftpb.ConfChange{Type: tt.cc.Type, NodeID: raft.None, Context: tt.cc.Context} w := []testutil.Action{ { Name: "ApplyConfChange", Params: []any{cc}, }, } if g, _ := n.Wait(1); !reflect.DeepEqual(g, w) { t.Errorf("#%d: action = %+v, want %+v", i, g, w) } } } func TestApplyConfChangeShouldStop(t *testing.T) { lg := zaptest.NewLogger(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) cl := membership.NewCluster(lg) cl.SetBackend(schema.NewMembershipBackend(lg, be)) for i := 1; i <= 3; i++ { cl.AddMember(&membership.Member{ID: types.ID(i)}, true) } r := newRaftNode(raftNodeConfig{ lg: zaptest.NewLogger(t), Node: newNodeNop(), transport: newNopTransporter(), }) srv := &EtcdServer{ lgMu: new(sync.RWMutex), lg: lg, memberID: 1, r: *r, cluster: cl, beHooks: serverstorage.NewBackendHooks(lg, nil), } cc := raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: 2, } // remove non-local member shouldStop, err := srv.applyConfChange(cc, &raftpb.ConfState{}, true) if err != nil { t.Fatalf("unexpected error %v", err) } if shouldStop { t.Errorf("shouldStop = %t, want %t", shouldStop, false) } // remove local member cc.NodeID = 1 shouldStop, err = srv.applyConfChange(cc, &raftpb.ConfState{}, true) if err != nil { t.Fatalf("unexpected error %v", err) } if !shouldStop { t.Errorf("shouldStop = %t, want %t", shouldStop, true) } } // TestApplyConfigChangeUpdatesConsistIndex ensures a config change also updates the consistIndex // where consistIndex equals to applied index. func TestApplyConfigChangeUpdatesConsistIndex(t *testing.T) { lg := zaptest.NewLogger(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) cl := membership.NewCluster(zaptest.NewLogger(t)) cl.SetBackend(schema.NewMembershipBackend(lg, be)) cl.AddMember(&membership.Member{ID: types.ID(1)}, true) schema.CreateMetaBucket(be.BatchTx()) ci := cindex.NewConsistentIndex(be) srv := &EtcdServer{ lgMu: new(sync.RWMutex), lg: lg, memberID: 1, r: *realisticRaftNode(lg, 1, nil), cluster: cl, w: wait.New(), consistIndex: ci, beHooks: serverstorage.NewBackendHooks(lg, ci), } defer srv.r.raftNodeConfig.Stop() // create EntryConfChange entry now := time.Now() urls, err := types.NewURLs([]string{"http://whatever:123"}) if err != nil { t.Fatal(err) } m := membership.NewMember("", urls, "", &now) m.ID = types.ID(2) b, err := json.Marshal(m) if err != nil { t.Fatal(err) } cc := &raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 2, Context: b} ents := []raftpb.Entry{{ Index: 2, Term: 4, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(cc), }} raftAdvancedC := make(chan struct{}, 1) raftAdvancedC <- struct{}{} _, appliedi, _ := srv.apply(ents, &raftpb.ConfState{}, raftAdvancedC) consistIndex := srv.consistIndex.ConsistentIndex() assert.Equal(t, uint64(2), appliedi) t.Run("verify-backend", func(t *testing.T) { tx := be.BatchTx() tx.Lock() defer tx.Unlock() srv.beHooks.OnPreCommitUnsafe(tx) assert.Equal(t, raftpb.ConfState{Voters: []uint64{2}}, *schema.UnsafeConfStateFromBackend(lg, tx)) }) rindex, _ := schema.ReadConsistentIndex(be.ReadTx()) assert.Equal(t, consistIndex, rindex) } func realisticRaftNode(lg *zap.Logger, id uint64, snap *raftpb.Snapshot) *raftNode { storage := raft.NewMemoryStorage() storage.SetHardState(raftpb.HardState{Commit: 0, Term: 0}) if snap != nil { err := storage.ApplySnapshot(*snap) if err != nil { panic(err) } } c := &raft.Config{ ID: id, ElectionTick: 10, HeartbeatTick: 1, Storage: storage, MaxSizePerMsg: math.MaxUint64, MaxInflightMsgs: 256, } n := raft.RestartNode(c) r := newRaftNode(raftNodeConfig{ lg: lg, Node: n, transport: newNopTransporter(), }) return r } // TestApplyMultiConfChangeShouldStop ensures that toApply will return shouldStop // if the local member is removed along with other conf updates. func TestApplyMultiConfChangeShouldStop(t *testing.T) { lg := zaptest.NewLogger(t) cl := membership.NewCluster(lg) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) cl.SetBackend(schema.NewMembershipBackend(lg, be)) for i := 1; i <= 5; i++ { cl.AddMember(&membership.Member{ID: types.ID(i)}, true) } r := newRaftNode(raftNodeConfig{ lg: lg, Node: newNodeNop(), transport: newNopTransporter(), }) ci := cindex.NewFakeConsistentIndex(0) srv := &EtcdServer{ lgMu: new(sync.RWMutex), lg: lg, memberID: 2, r: *r, cluster: cl, w: wait.New(), consistIndex: ci, beHooks: serverstorage.NewBackendHooks(lg, ci), } var ents []raftpb.Entry for i := 1; i <= 4; i++ { ent := raftpb.Entry{ Term: 1, Index: uint64(i), Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal( &raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: uint64(i), }), } ents = append(ents, ent) } raftAdvancedC := make(chan struct{}, 1) raftAdvancedC <- struct{}{} _, _, shouldStop := srv.apply(ents, &raftpb.ConfState{}, raftAdvancedC) if !shouldStop { t.Errorf("shouldStop = %t, want %t", shouldStop, true) } } // TestSnapshotDisk should save the snapshot to disk and release old snapshots func TestSnapshotDisk(t *testing.T) { revertFunc := verify.DisableVerifications() defer revertFunc() be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) s := raft.NewMemoryStorage() s.Append([]raftpb.Entry{{Index: 1}}) st := mockstore.NewRecorderStream() p := mockstorage.NewStorageRecorderStream("") r := newRaftNode(raftNodeConfig{ lg: zaptest.NewLogger(t), Node: newNodeNop(), raftStorage: s, storage: p, }) srv := &EtcdServer{ lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), r: *r, consistIndex: cindex.NewConsistentIndex(be), } srv.kv = mvcc.New(zaptest.NewLogger(t), be, &lease.FakeLessor{}, mvcc.StoreConfig{}) defer func() { assert.NoError(t, srv.kv.Close()) }() srv.be = be cl := membership.NewCluster(zaptest.NewLogger(t)) srv.cluster = cl ch := make(chan struct{}, 1) go func() { gaction, _ := p.Wait(2) defer func() { ch <- struct{}{} }() assert.Len(t, gaction, 2) assert.Equal(t, testutil.Action{Name: "SaveSnap"}, gaction[0]) assert.Equal(t, testutil.Action{Name: "Release"}, gaction[1]) }() ep := etcdProgress{appliedi: 1, confState: raftpb.ConfState{Voters: []uint64{1}}} srv.snapshot(&ep, true) <-ch assert.Empty(t, st.Action()) assert.Equal(t, uint64(1), ep.diskSnapshotIndex) assert.Equal(t, uint64(1), ep.memorySnapshotIndex) } func TestSnapshotMemory(t *testing.T) { revertFunc := verify.DisableVerifications() defer revertFunc() be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) s := raft.NewMemoryStorage() s.Append([]raftpb.Entry{{Index: 1}}) st := mockstore.NewRecorderStream() p := mockstorage.NewStorageRecorderStream("") r := newRaftNode(raftNodeConfig{ lg: zaptest.NewLogger(t), Node: newNodeNop(), raftStorage: s, storage: p, }) srv := &EtcdServer{ lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), r: *r, consistIndex: cindex.NewConsistentIndex(be), } srv.kv = mvcc.New(zaptest.NewLogger(t), be, &lease.FakeLessor{}, mvcc.StoreConfig{}) defer func() { assert.NoError(t, srv.kv.Close()) }() srv.be = be cl := membership.NewCluster(zaptest.NewLogger(t)) srv.cluster = cl ch := make(chan struct{}, 1) go func() { gaction, _ := p.Wait(1) defer func() { ch <- struct{}{} }() assert.Empty(t, gaction) }() ep := etcdProgress{appliedi: 1, confState: raftpb.ConfState{Voters: []uint64{1}}} srv.snapshot(&ep, false) <-ch assert.Empty(t, st.Action()) assert.Equal(t, uint64(0), ep.diskSnapshotIndex) assert.Equal(t, uint64(1), ep.memorySnapshotIndex) } // TestSnapshotOrdering ensures raft persists snapshot onto disk before // snapshot db is applied. func TestSnapshotOrdering(t *testing.T) { // Ignore the snapshot index verification in unit test, because // it doesn't follow the e2e applying logic. revertFunc := verify.DisableVerifications() defer revertFunc() lg := zaptest.NewLogger(t) n := newNopReadyNode() cl := membership.NewCluster(lg) be, _ := betesting.NewDefaultTmpBackend(t) cl.SetBackend(schema.NewMembershipBackend(lg, be)) testdir := t.TempDir() snapdir := filepath.Join(testdir, "member", "snap") if err := os.MkdirAll(snapdir, 0o755); err != nil { t.Fatalf("couldn't make snap dir (%v)", err) } rs := raft.NewMemoryStorage() p := mockstorage.NewStorageRecorderStream(testdir) tr, snapDoneC := newSnapTransporter(lg, snapdir) r := newRaftNode(raftNodeConfig{ lg: lg, isIDRemoved: func(id uint64) bool { return cl.IsIDRemoved(types.ID(id)) }, Node: n, transport: tr, storage: p, raftStorage: rs, }) ci := cindex.NewConsistentIndex(be) cfg := config.ServerConfig{ Logger: lg, DataDir: testdir, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries, ServerFeatureGate: features.NewDefaultServerFeatureGate("test", lg), } s := &EtcdServer{ lgMu: new(sync.RWMutex), lg: lg, Cfg: cfg, r: *r, snapshotter: snap.New(lg, snapdir), cluster: cl, consistIndex: ci, beHooks: serverstorage.NewBackendHooks(lg, ci), } s.kv = mvcc.New(lg, be, &lease.FakeLessor{}, mvcc.StoreConfig{}) s.be = be s.start() defer s.Stop() n.readyc <- raft.Ready{Messages: []raftpb.Message{{Type: raftpb.MsgSnap}}} go func() { // get the snapshot sent by the transport snapMsg := <-snapDoneC // Snapshot first triggers raftnode to persists the snapshot onto disk // before renaming db snapshot file to db snapMsg.Snapshot.Metadata.Index = 1 n.readyc <- raft.Ready{Snapshot: *snapMsg.Snapshot} }() ac := <-p.Chan() if ac.Name != "Save" { t.Fatalf("expected Save, got %+v", ac) } if ac := <-p.Chan(); ac.Name != "SaveSnap" { t.Fatalf("expected SaveSnap, got %+v", ac) } if ac := <-p.Chan(); ac.Name != "Save" { t.Fatalf("expected Save, got %+v", ac) } // confirm snapshot file still present before calling SaveSnap snapPath := filepath.Join(snapdir, fmt.Sprintf("%016x.snap.db", 1)) if !fileutil.Exist(snapPath) { t.Fatalf("expected file %q, got missing", snapPath) } // unblock SaveSnapshot, etcdserver now permitted to move snapshot file if ac := <-p.Chan(); ac.Name != "Sync" { t.Fatalf("expected Sync, got %+v", ac) } if ac := <-p.Chan(); ac.Name != "Release" { t.Fatalf("expected Release, got %+v", ac) } } // TestConcurrentApplyAndSnapshotV3 will send out snapshots concurrently with // proposals. func TestConcurrentApplyAndSnapshotV3(t *testing.T) { // Ignore the snapshot index verification in unit test, because // it doesn't follow the e2e applying logic. revertFunc := verify.DisableVerifications() defer revertFunc() lg := zaptest.NewLogger(t) n := newNopReadyNode() cl := membership.NewCluster(lg) be, _ := betesting.NewDefaultTmpBackend(t) cl.SetBackend(schema.NewMembershipBackend(lg, be)) testdir := t.TempDir() if err := os.MkdirAll(testdir+"/member/snap", 0o755); err != nil { t.Fatalf("Couldn't make snap dir (%v)", err) } rs := raft.NewMemoryStorage() tr, snapDoneC := newSnapTransporter(lg, testdir) r := newRaftNode(raftNodeConfig{ lg: lg, isIDRemoved: func(id uint64) bool { return cl.IsIDRemoved(types.ID(id)) }, Node: n, transport: tr, storage: mockstorage.NewStorageRecorder(testdir), raftStorage: rs, }) ci := cindex.NewConsistentIndex(be) s := &EtcdServer{ lgMu: new(sync.RWMutex), lg: lg, Cfg: config.ServerConfig{ Logger: lg, DataDir: testdir, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries, ServerFeatureGate: features.NewDefaultServerFeatureGate("test", lg), }, r: *r, snapshotter: snap.New(lg, testdir), cluster: cl, consistIndex: ci, beHooks: serverstorage.NewBackendHooks(lg, ci), firstCommitInTerm: notify.NewNotifier(), lessor: &lease.FakeLessor{}, uberApply: uberApplierMock{}, authStore: auth.NewAuthStore(lg, schema.NewAuthBackend(lg, be), nil, 1), } s.kv = mvcc.New(lg, be, &lease.FakeLessor{}, mvcc.StoreConfig{}) s.be = be s.start() defer s.Stop() // submit applied entries and snap entries idx := uint64(0) outdated := 0 accepted := 0 for k := 1; k <= 101; k++ { idx++ ch := s.w.Register(idx) req := &pb.InternalRaftRequest{ Header: &pb.RequestHeader{ID: idx}, Put: &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}, } ent := raftpb.Entry{Index: idx, Data: pbutil.MustMarshal(req)} ready := raft.Ready{Entries: []raftpb.Entry{ent}} n.readyc <- ready ready = raft.Ready{CommittedEntries: []raftpb.Entry{ent}} n.readyc <- ready // "idx" applied <-ch // one snapshot for every two messages if k%2 != 0 { continue } n.readyc <- raft.Ready{Messages: []raftpb.Message{{Type: raftpb.MsgSnap}}} // get the snapshot sent by the transport snapMsg := <-snapDoneC // If the snapshot trails applied records, recovery will panic // since there's no allocated snapshot at the place of the // snapshot record. This only happens when the applier and the // snapshot sender get out of sync. if snapMsg.Snapshot.Metadata.Index == idx { idx++ snapMsg.Snapshot.Metadata.Index = idx ready = raft.Ready{Snapshot: *snapMsg.Snapshot} n.readyc <- ready accepted++ } else { outdated++ } // don't wait for the snapshot to complete, move to next message } if accepted != 50 { t.Errorf("accepted=%v, want 50", accepted) } if outdated != 0 { t.Errorf("outdated=%v, want 0", outdated) } } // TestAddMember tests AddMember can propose and perform node addition. func TestAddMember(t *testing.T) { lg := zaptest.NewLogger(t) n := newNodeConfChangeCommitterRecorder() n.readyc <- raft.Ready{ SoftState: &raft.SoftState{RaftState: raft.StateLeader}, } cl := newTestCluster(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) cl.SetBackend(schema.NewMembershipBackend(lg, be)) r := newRaftNode(raftNodeConfig{ lg: lg, Node: n, raftStorage: raft.NewMemoryStorage(), storage: mockstorage.NewStorageRecorder(""), transport: newNopTransporter(), }) s := &EtcdServer{ lgMu: new(sync.RWMutex), lg: lg, r: *r, cluster: cl, reqIDGen: idutil.NewGenerator(0, time.Time{}), consistIndex: cindex.NewFakeConsistentIndex(0), beHooks: serverstorage.NewBackendHooks(lg, nil), } s.start() m := membership.Member{ID: 1234, RaftAttributes: membership.RaftAttributes{PeerURLs: []string{"foo"}}} _, err := s.AddMember(t.Context(), m) gaction := n.Action() s.Stop() if err != nil { t.Fatalf("AddMember error: %v", err) } wactions := []testutil.Action{{Name: "ProposeConfChange:ConfChangeAddNode"}, {Name: "ApplyConfChange:ConfChangeAddNode"}} if !reflect.DeepEqual(gaction, wactions) { t.Errorf("action = %v, want %v", gaction, wactions) } if cl.Member(1234) == nil { t.Errorf("member with id 1234 is not added") } } // TestProcessIgnoreMismatchMessage tests Process must ignore messages to // mismatch member. func TestProcessIgnoreMismatchMessage(t *testing.T) { lg := zaptest.NewLogger(t) cl := newTestCluster(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) cl.SetBackend(schema.NewMembershipBackend(lg, be)) // Bootstrap a 3-node cluster, member IDs: 1 2 3. cl.AddMember(&membership.Member{ID: types.ID(1)}, true) cl.AddMember(&membership.Member{ID: types.ID(2)}, true) cl.AddMember(&membership.Member{ID: types.ID(3)}, true) // r is initialized with ID 1. r := realisticRaftNode(lg, 1, &raftpb.Snapshot{ Metadata: raftpb.SnapshotMetadata{ Index: 11, // Magic number. Term: 11, // Magic number. ConfState: raftpb.ConfState{ // Member ID list. Voters: []uint64{1, 2, 3}, }, }, }) defer r.raftNodeConfig.Stop() s := &EtcdServer{ lgMu: new(sync.RWMutex), lg: lg, memberID: 1, r: *r, cluster: cl, reqIDGen: idutil.NewGenerator(0, time.Time{}), consistIndex: cindex.NewFakeConsistentIndex(0), beHooks: serverstorage.NewBackendHooks(lg, nil), } // Mock a mad switch dispatching messages to wrong node. m := raftpb.Message{ Type: raftpb.MsgHeartbeat, To: 2, // Wrong ID, s.MemberID() is 1. From: 3, Term: 11, Commit: 42, // Commit is larger than the last index 11. } if types.ID(m.To) == s.MemberID() { t.Fatalf("m.To (%d) is expected to mismatch s.MemberID (%d)", m.To, s.MemberID()) } err := s.Process(t.Context(), m) if err == nil { t.Fatalf("Must ignore the message and return an error") } } // TestRemoveMember tests RemoveMember can propose and perform node removal. func TestRemoveMember(t *testing.T) { lg := zaptest.NewLogger(t) n := newNodeConfChangeCommitterRecorder() n.readyc <- raft.Ready{ SoftState: &raft.SoftState{RaftState: raft.StateLeader}, } cl := newTestCluster(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) cl.SetBackend(schema.NewMembershipBackend(lg, be)) cl.AddMember(&membership.Member{ID: 1234}, true) r := newRaftNode(raftNodeConfig{ lg: lg, Node: n, raftStorage: raft.NewMemoryStorage(), storage: mockstorage.NewStorageRecorder(""), transport: newNopTransporter(), }) s := &EtcdServer{ lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), r: *r, cluster: cl, reqIDGen: idutil.NewGenerator(0, time.Time{}), consistIndex: cindex.NewFakeConsistentIndex(0), beHooks: serverstorage.NewBackendHooks(lg, nil), } s.start() _, err := s.RemoveMember(t.Context(), 1234) gaction := n.Action() s.Stop() if err != nil { t.Fatalf("RemoveMember error: %v", err) } wactions := []testutil.Action{{Name: "ProposeConfChange:ConfChangeRemoveNode"}, {Name: "ApplyConfChange:ConfChangeRemoveNode"}} if !reflect.DeepEqual(gaction, wactions) { t.Errorf("action = %v, want %v", gaction, wactions) } if cl.Member(1234) != nil { t.Errorf("member with id 1234 is not removed") } } // TestUpdateMember tests RemoveMember can propose and perform node update. func TestUpdateMember(t *testing.T) { lg := zaptest.NewLogger(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) n := newNodeConfChangeCommitterRecorder() n.readyc <- raft.Ready{ SoftState: &raft.SoftState{RaftState: raft.StateLeader}, } cl := newTestCluster(t) cl.SetBackend(schema.NewMembershipBackend(lg, be)) cl.AddMember(&membership.Member{ID: 1234}, true) r := newRaftNode(raftNodeConfig{ lg: lg, Node: n, raftStorage: raft.NewMemoryStorage(), storage: mockstorage.NewStorageRecorder(""), transport: newNopTransporter(), }) s := &EtcdServer{ lgMu: new(sync.RWMutex), lg: lg, r: *r, cluster: cl, reqIDGen: idutil.NewGenerator(0, time.Time{}), consistIndex: cindex.NewFakeConsistentIndex(0), beHooks: serverstorage.NewBackendHooks(lg, nil), } s.start() wm := membership.Member{ID: 1234, RaftAttributes: membership.RaftAttributes{PeerURLs: []string{"http://127.0.0.1:1"}}} _, err := s.UpdateMember(t.Context(), wm) gaction := n.Action() s.Stop() if err != nil { t.Fatalf("UpdateMember error: %v", err) } wactions := []testutil.Action{{Name: "ProposeConfChange:ConfChangeUpdateNode"}, {Name: "ApplyConfChange:ConfChangeUpdateNode"}} if !reflect.DeepEqual(gaction, wactions) { t.Errorf("action = %v, want %v", gaction, wactions) } if !reflect.DeepEqual(cl.Member(1234), &wm) { t.Errorf("member = %v, want %v", cl.Member(1234), &wm) } } // TODO: test server could stop itself when being removed func TestPublishV3(t *testing.T) { n := newNodeRecorder() ch := make(chan any, 1) // simulate that request has gone through consensus ch <- &apply2.Result{} w := wait.NewWithResponse(ch) ctx, cancel := context.WithCancel(t.Context()) lg := zaptest.NewLogger(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) srv := &EtcdServer{ lgMu: new(sync.RWMutex), lg: lg, readych: make(chan struct{}), Cfg: config.ServerConfig{Logger: lg, TickMs: 1, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries, MaxRequestBytes: 1000}, memberID: 1, r: *newRaftNode(raftNodeConfig{lg: lg, Node: n}), attributes: membership.Attributes{Name: "node1", ClientURLs: []string{"http://a", "http://b"}}, cluster: &membership.RaftCluster{}, w: w, reqIDGen: idutil.NewGenerator(0, time.Time{}), authStore: auth.NewAuthStore(lg, schema.NewAuthBackend(lg, be), nil, 0), be: be, ctx: ctx, cancel: cancel, } srv.publishV3(time.Hour) action := n.Action() if len(action) != 1 { t.Fatalf("len(action) = %d, want 1", len(action)) } if action[0].Name != "Propose" { t.Fatalf("action = %s, want Propose", action[0].Name) } data := action[0].Params[0].([]byte) var r pb.InternalRaftRequest if err := r.Unmarshal(data); err != nil { t.Fatalf("unmarshal request error: %v", err) } assert.Equal(t, &membershippb.ClusterMemberAttrSetRequest{Member_ID: 0x1, MemberAttributes: &membershippb.Attributes{ Name: "node1", ClientUrls: []string{"http://a", "http://b"}, }}, r.ClusterMemberAttrSet) } // TestPublishV3Stopped tests that publish will be stopped if server is stopped. func TestPublishV3Stopped(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) r := newRaftNode(raftNodeConfig{ lg: zaptest.NewLogger(t), Node: newNodeNop(), transport: newNopTransporter(), }) srv := &EtcdServer{ lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), Cfg: config.ServerConfig{Logger: zaptest.NewLogger(t), TickMs: 1, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries}, r: *r, cluster: &membership.RaftCluster{}, w: mockwait.NewNop(), done: make(chan struct{}), stopping: make(chan struct{}), stop: make(chan struct{}), reqIDGen: idutil.NewGenerator(0, time.Time{}), ctx: ctx, cancel: cancel, } close(srv.stopping) srv.publishV3(time.Hour) } // TestPublishV3Retry tests that publish will keep retry until success. func TestPublishV3Retry(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) n := newNodeRecorderStream() lg := zaptest.NewLogger(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) srv := &EtcdServer{ lgMu: new(sync.RWMutex), lg: lg, readych: make(chan struct{}), Cfg: config.ServerConfig{Logger: lg, TickMs: 1, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries, MaxRequestBytes: 1000}, memberID: 1, r: *newRaftNode(raftNodeConfig{lg: lg, Node: n}), w: mockwait.NewNop(), stopping: make(chan struct{}), attributes: membership.Attributes{Name: "node1", ClientURLs: []string{"http://a", "http://b"}}, cluster: &membership.RaftCluster{}, reqIDGen: idutil.NewGenerator(0, time.Time{}), authStore: auth.NewAuthStore(lg, schema.NewAuthBackend(lg, be), nil, 0), be: be, ctx: ctx, cancel: cancel, } // expect multiple proposals from retrying ch := make(chan struct{}) go func() { defer close(ch) if action, err := n.Wait(2); err != nil { t.Errorf("len(action) = %d, want >= 2 (%v)", len(action), err) } close(srv.stopping) // drain remaining actions, if any, so publish can terminate for { select { case <-ch: return default: n.Action() } } }() srv.publishV3(10 * time.Nanosecond) ch <- struct{}{} <-ch } func TestUpdateVersionV3(t *testing.T) { n := newNodeRecorder() ch := make(chan any, 1) // simulate that request has gone through consensus ch <- &apply2.Result{} w := wait.NewWithResponse(ch) ctx, cancel := context.WithCancel(t.Context()) lg := zaptest.NewLogger(t) be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) srv := &EtcdServer{ lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), memberID: 1, Cfg: config.ServerConfig{Logger: lg, TickMs: 1, SnapshotCatchUpEntries: DefaultSnapshotCatchUpEntries, MaxRequestBytes: 1000}, r: *newRaftNode(raftNodeConfig{lg: zaptest.NewLogger(t), Node: n}), attributes: membership.Attributes{Name: "node1", ClientURLs: []string{"http://node1.com"}}, cluster: &membership.RaftCluster{}, w: w, reqIDGen: idutil.NewGenerator(0, time.Time{}), authStore: auth.NewAuthStore(lg, schema.NewAuthBackend(lg, be), nil, 0), be: be, ctx: ctx, cancel: cancel, } ver := "2.0.0" srv.updateClusterVersionV3(ver) action := n.Action() if len(action) != 1 { t.Fatalf("len(action) = %d, want 1", len(action)) } if action[0].Name != "Propose" { t.Fatalf("action = %s, want Propose", action[0].Name) } data := action[0].Params[0].([]byte) var r pb.InternalRaftRequest if err := r.Unmarshal(data); err != nil { t.Fatalf("unmarshal request error: %v", err) } assert.Equal(t, &membershippb.ClusterVersionSetRequest{Ver: ver}, r.ClusterVersionSet) } func TestStopNotify(t *testing.T) { s := &EtcdServer{ lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), stop: make(chan struct{}), done: make(chan struct{}), } go func() { <-s.stop close(s.done) }() notifier := s.StopNotify() select { case <-notifier: t.Fatalf("received unexpected stop notification") default: } s.Stop() select { case <-notifier: default: t.Fatalf("cannot receive stop notification") } } func TestGetOtherPeerURLs(t *testing.T) { lg := zaptest.NewLogger(t) tests := []struct { membs []*membership.Member wurls []string }{ { []*membership.Member{ membership.NewMember("1", types.MustNewURLs([]string{"http://10.0.0.1:1"}), "a", nil), }, []string{}, }, { []*membership.Member{ membership.NewMember("1", types.MustNewURLs([]string{"http://10.0.0.1:1"}), "a", nil), membership.NewMember("2", types.MustNewURLs([]string{"http://10.0.0.2:2"}), "a", nil), membership.NewMember("3", types.MustNewURLs([]string{"http://10.0.0.3:3"}), "a", nil), }, []string{"http://10.0.0.2:2", "http://10.0.0.3:3"}, }, { []*membership.Member{ membership.NewMember("1", types.MustNewURLs([]string{"http://10.0.0.1:1"}), "a", nil), membership.NewMember("3", types.MustNewURLs([]string{"http://10.0.0.3:3"}), "a", nil), membership.NewMember("2", types.MustNewURLs([]string{"http://10.0.0.2:2"}), "a", nil), }, []string{"http://10.0.0.2:2", "http://10.0.0.3:3"}, }, } for i, tt := range tests { cl := membership.NewClusterFromMembers(lg, types.ID(0), tt.membs) self := "1" urls := getRemotePeerURLs(cl, self) if !reflect.DeepEqual(urls, tt.wurls) { t.Errorf("#%d: urls = %+v, want %+v", i, urls, tt.wurls) } } } type nodeRecorder struct{ testutil.Recorder } func newNodeRecorder() *nodeRecorder { return &nodeRecorder{&testutil.RecorderBuffered{}} } func newNodeRecorderStream() *nodeRecorder { return &nodeRecorder{testutil.NewRecorderStream()} } func newNodeNop() raft.Node { return newNodeRecorder() } func (n *nodeRecorder) Tick() { n.Record(testutil.Action{Name: "Tick"}) } func (n *nodeRecorder) Campaign(ctx context.Context) error { n.Record(testutil.Action{Name: "Campaign"}) return nil } func (n *nodeRecorder) Propose(ctx context.Context, data []byte) error { n.Record(testutil.Action{Name: "Propose", Params: []any{data}}) return nil } func (n *nodeRecorder) ProposeConfChange(ctx context.Context, conf raftpb.ConfChangeI) error { n.Record(testutil.Action{Name: "ProposeConfChange"}) return nil } func (n *nodeRecorder) Step(ctx context.Context, msg raftpb.Message) error { n.Record(testutil.Action{Name: "Step"}) return nil } func (n *nodeRecorder) Status() raft.Status { return raft.Status{} } func (n *nodeRecorder) Ready() <-chan raft.Ready { return nil } func (n *nodeRecorder) TransferLeadership(ctx context.Context, lead, transferee uint64) {} func (n *nodeRecorder) ReadIndex(ctx context.Context, rctx []byte) error { return nil } func (n *nodeRecorder) Advance() {} func (n *nodeRecorder) ApplyConfChange(conf raftpb.ConfChangeI) *raftpb.ConfState { n.Record(testutil.Action{Name: "ApplyConfChange", Params: []any{conf}}) return &raftpb.ConfState{} } func (n *nodeRecorder) Stop() { n.Record(testutil.Action{Name: "Stop"}) } func (n *nodeRecorder) ReportUnreachable(id uint64) {} func (n *nodeRecorder) ReportSnapshot(id uint64, status raft.SnapshotStatus) {} func (n *nodeRecorder) Compact(index uint64, nodes []uint64, d []byte) { n.Record(testutil.Action{Name: "Compact"}) } func (n *nodeRecorder) ForgetLeader(ctx context.Context) error { return nil } // readyNode is a nodeRecorder with a user-writeable ready channel type readyNode struct { nodeRecorder readyc chan raft.Ready } func newReadyNode() *readyNode { return &readyNode{ nodeRecorder{testutil.NewRecorderStream()}, make(chan raft.Ready, 1), } } func newNopReadyNode() *readyNode { return &readyNode{*newNodeRecorder(), make(chan raft.Ready, 1)} } func (n *readyNode) Ready() <-chan raft.Ready { return n.readyc } type nodeConfChangeCommitterRecorder struct { readyNode index uint64 } func newNodeConfChangeCommitterRecorder() *nodeConfChangeCommitterRecorder { return &nodeConfChangeCommitterRecorder{*newNopReadyNode(), 0} } func newNodeConfChangeCommitterStream() *nodeConfChangeCommitterRecorder { return &nodeConfChangeCommitterRecorder{*newReadyNode(), 0} } func confChangeActionName(conf raftpb.ConfChangeI) string { var s string if confV1, ok := conf.AsV1(); ok { s = confV1.Type.String() } else { for i, chg := range conf.AsV2().Changes { if i > 0 { s += "/" } s += chg.Type.String() } } return s } func (n *nodeConfChangeCommitterRecorder) ProposeConfChange(ctx context.Context, conf raftpb.ConfChangeI) error { typ, data, err := raftpb.MarshalConfChange(conf) if err != nil { return err } n.index++ n.Record(testutil.Action{Name: "ProposeConfChange:" + confChangeActionName(conf)}) n.readyc <- raft.Ready{CommittedEntries: []raftpb.Entry{{Index: n.index, Type: typ, Data: data}}} return nil } func (n *nodeConfChangeCommitterRecorder) Ready() <-chan raft.Ready { return n.readyc } func (n *nodeConfChangeCommitterRecorder) ApplyConfChange(conf raftpb.ConfChangeI) *raftpb.ConfState { n.Record(testutil.Action{Name: "ApplyConfChange:" + confChangeActionName(conf)}) return &raftpb.ConfState{} } func newTestCluster(tb testing.TB) *membership.RaftCluster { return membership.NewCluster(zaptest.NewLogger(tb)) } type nopTransporter struct{} func newNopTransporter() rafthttp.Transporter { return &nopTransporter{} } func (s *nopTransporter) Start() error { return nil } func (s *nopTransporter) Handler() http.Handler { return nil } func (s *nopTransporter) Send(m []raftpb.Message) {} func (s *nopTransporter) SendSnapshot(m snap.Message) {} func (s *nopTransporter) AddRemote(id types.ID, us []string) {} func (s *nopTransporter) AddPeer(id types.ID, us []string) {} func (s *nopTransporter) RemovePeer(id types.ID) {} func (s *nopTransporter) RemoveAllPeers() {} func (s *nopTransporter) UpdatePeer(id types.ID, us []string) {} func (s *nopTransporter) ActiveSince(id types.ID) time.Time { return time.Time{} } func (s *nopTransporter) ActivePeers() int { return 0 } func (s *nopTransporter) Stop() {} func (s *nopTransporter) Pause() {} func (s *nopTransporter) Resume() {} type snapTransporter struct { nopTransporter snapDoneC chan snap.Message snapDir string lg *zap.Logger } func newSnapTransporter(lg *zap.Logger, snapDir string) (rafthttp.Transporter, <-chan snap.Message) { ch := make(chan snap.Message, 1) tr := &snapTransporter{snapDoneC: ch, snapDir: snapDir, lg: lg} return tr, ch } func (s *snapTransporter) SendSnapshot(m snap.Message) { ss := snap.New(s.lg, s.snapDir) ss.SaveDBFrom(m.ReadCloser, m.Snapshot.Metadata.Index+1) m.CloseWithError(nil) s.snapDoneC <- m } type sendMsgAppRespTransporter struct { nopTransporter sendC chan int } func newSendMsgAppRespTransporter() (rafthttp.Transporter, <-chan int) { ch := make(chan int, 1) tr := &sendMsgAppRespTransporter{sendC: ch} return tr, ch } func (s *sendMsgAppRespTransporter) Send(m []raftpb.Message) { var send int for _, msg := range m { if msg.To != 0 { send++ } } s.sendC <- send } func TestWaitAppliedIndex(t *testing.T) { cases := []struct { name string appliedIndex uint64 committedIndex uint64 action func(s *EtcdServer) ExpectedError error }{ { name: "The applied Id is already equal to the commitId", appliedIndex: 10, committedIndex: 10, action: func(s *EtcdServer) { s.applyWait.Trigger(10) }, ExpectedError: nil, }, { name: "The etcd server has already stopped", appliedIndex: 10, committedIndex: 12, action: func(s *EtcdServer) { s.stopping <- struct{}{} }, ExpectedError: errors.ErrStopped, }, { name: "Timed out waiting for the applied index", appliedIndex: 10, committedIndex: 12, action: nil, ExpectedError: errors.ErrTimeoutWaitAppliedIndex, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s := &EtcdServer{ stopping: make(chan struct{}, 1), applyWait: wait.NewTimeList(), } s.appliedIndex.Store(tc.appliedIndex) s.committedIndex.Store(tc.committedIndex) if tc.action != nil { go tc.action(s) } err := s.waitAppliedIndex() if !errorspkg.Is(err, tc.ExpectedError) { t.Errorf("Unexpected error, want (%v), got (%v)", tc.ExpectedError, err) } }) } } func TestIsActive(t *testing.T) { cases := []struct { name string tickMs uint durationSinceLastTick time.Duration expectActive bool }{ { name: "1.5*tickMs,active", tickMs: 100, durationSinceLastTick: 150 * time.Millisecond, expectActive: true, }, { name: "2*tickMs,active", tickMs: 200, durationSinceLastTick: 400 * time.Millisecond, expectActive: true, }, { name: "4*tickMs,not active", tickMs: 150, durationSinceLastTick: 600 * time.Millisecond, expectActive: false, }, } for _, tc := range cases { s := EtcdServer{ Cfg: config.ServerConfig{ TickMs: tc.tickMs, }, r: raftNode{ tickMu: new(sync.RWMutex), latestTickTs: time.Now().Add(-tc.durationSinceLastTick), }, } require.Equal(t, tc.expectActive, s.isActive()) } } func TestAddFeatureGateMetrics(t *testing.T) { const testAlphaGate featuregate.Feature = "TestAlpha" const testBetaGate featuregate.Feature = "TestBeta" const testGAGate featuregate.Feature = "TestGA" featuremap := map[featuregate.Feature]featuregate.FeatureSpec{ testGAGate: {Default: true, PreRelease: featuregate.GA}, testAlphaGate: {Default: true, PreRelease: featuregate.Alpha}, testBetaGate: {Default: false, PreRelease: featuregate.Beta}, } fg := featuregate.New("test", zaptest.NewLogger(t)) fg.Add(featuremap) addFeatureGateMetrics(fg, serverFeatureEnabled) expected := `# HELP etcd_server_feature_enabled Whether or not a feature is enabled. 1 is enabled, 0 is not. # TYPE etcd_server_feature_enabled gauge etcd_server_feature_enabled{name="AllAlpha",stage="ALPHA"} 0 etcd_server_feature_enabled{name="AllBeta",stage="BETA"} 0 etcd_server_feature_enabled{name="TestAlpha",stage="ALPHA"} 1 etcd_server_feature_enabled{name="TestBeta",stage="BETA"} 0 etcd_server_feature_enabled{name="TestGA",stage=""} 1 ` err := ptestutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected), "etcd_server_feature_enabled") require.NoErrorf(t, err, "unexpected metric collection result: \n%s", err) } func TestRequestCurrentIndex_LeaderChangedRace(t *testing.T) { s, _ := setupTestRequestCurrentIndex(t) for i := 0; i < 100; i++ { s.r.readStateC <- raft.ReadState{Index: 100} leaderChangedNotifier := s.leaderChanged.Receive() s.leaderChanged.Notify() index, err := s.requestCurrentIndex(leaderChangedNotifier) require.ErrorIs(t, err, errors.ErrLeaderChanged) require.Equal(t, uint64(0), index) // Clear the readStateC channel for the next iteration, select { case <-s.r.readStateC: default: } } } func TestRequestCurrentIndex_UniqueRequestID(t *testing.T) { s, mockRaft := setupTestRequestCurrentIndex(t) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() s.requestCurrentIndex(s.leaderChanged.Receive()) }() require.Eventually(t, func() bool { return len(mockRaft.getRequests()) >= 2 }, time.Second, 100*time.Millisecond) s.leaderChanged.Notify() wg.Wait() seen := make(map[uint64]bool) for _, id := range mockRaft.getRequests() { require.Falsef(t, seen[id], "Found duplicate request ID: %d", id) seen[id] = true } } func TestRequestCurrentIndex_Success(t *testing.T) { s, mockRaft := setupTestRequestCurrentIndex(t) wg := sync.WaitGroup{} wg.Add(1) var index uint64 var err error go func() { defer wg.Done() index, err = s.requestCurrentIndex(s.leaderChanged.Receive()) }() require.Eventually(t, func() bool { return len(mockRaft.getRequests()) == 1 }, time.Second, 100*time.Millisecond) reqID := mockRaft.getRequests()[0] reqIDBytes := make([]byte, 8) binary.BigEndian.PutUint64(reqIDBytes, reqID) s.r.readStateC <- raft.ReadState{ Index: 100, RequestCtx: reqIDBytes, } wg.Wait() require.NoError(t, err) require.Equal(t, uint64(100), index) require.Lenf(t, mockRaft.getRequests(), 1, "Expected exactly 1 ReadIndex request") } func TestRequestCurrentIndex_WrongRequestID(t *testing.T) { s, mockRaft := setupTestRequestCurrentIndex(t) wg := sync.WaitGroup{} wg.Add(1) var index uint64 var err error go func() { defer wg.Done() index, err = s.requestCurrentIndex(s.leaderChanged.Receive()) }() require.Eventually(t, func() bool { return len(mockRaft.getRequests()) == 1 }, time.Second, 10*time.Millisecond) wrongReqIDBytes := make([]byte, 8) binary.BigEndian.PutUint64(wrongReqIDBytes, 99999) s.r.readStateC <- raft.ReadState{ Index: 100, RequestCtx: wrongReqIDBytes, } time.Sleep(100 * time.Millisecond) requests := mockRaft.getRequests() require.Lenf(t, requests, 1, "Expected exactly 1 ReadIndex request") reqID := requests[0] reqIDBytes := make([]byte, 8) binary.BigEndian.PutUint64(reqIDBytes, reqID) s.r.readStateC <- raft.ReadState{ Index: 99, RequestCtx: reqIDBytes, } wg.Wait() require.NoError(t, err) require.Equal(t, uint64(99), index) require.Lenf(t, mockRaft.getRequests(), 1, "Expected exactly 1 ReadIndex request") } func TestRequestCurrentIndex_DelayedResponse(t *testing.T) { s, mockRaft := setupTestRequestCurrentIndex(t) wg := sync.WaitGroup{} wg.Add(1) var index uint64 var err error go func() { defer wg.Done() index, err = s.requestCurrentIndex(s.leaderChanged.Receive()) }() require.Eventually(t, func() bool { return len(mockRaft.getRequests()) >= 3 }, 2*time.Second, 100*time.Millisecond) requests := mockRaft.getRequests() reqID := requests[1] reqIDBytes := make([]byte, 8) binary.BigEndian.PutUint64(reqIDBytes, reqID) select { case s.r.readStateC <- raft.ReadState{ Index: 100, RequestCtx: reqIDBytes, }: case <-time.After(time.Second): t.Fatal("timed out sending read state") } wg.Wait() require.NoError(t, err) require.Equal(t, uint64(100), index) } func setupTestRequestCurrentIndex(t *testing.T) (*EtcdServer, *testRaftNode) { mockRaft := &testRaftNode{} s := &EtcdServer{ lgMu: new(sync.RWMutex), lg: zaptest.NewLogger(t), reqIDGen: idutil.NewGenerator(0, time.Time{}), firstCommitInTerm: notify.NewNotifier(), leaderChanged: notify.NewNotifier(), r: raftNode{ raftNodeConfig: raftNodeConfig{ Node: mockRaft, }, readStateC: make(chan raft.ReadState, 1), }, } return s, mockRaft } type testRaftNode struct { raft.Node mu sync.Mutex readIndexRequests []uint64 } func (m *testRaftNode) ReadIndex(ctx context.Context, rctx []byte) error { m.mu.Lock() defer m.mu.Unlock() if len(rctx) == 8 { m.readIndexRequests = append(m.readIndexRequests, binary.BigEndian.Uint64(rctx)) } return nil } func (m *testRaftNode) getRequests() []uint64 { m.mu.Lock() defer m.mu.Unlock() res := make([]uint64, len(m.readIndexRequests)) copy(res, m.readIndexRequests) return res } ================================================ FILE: server/etcdserver/snapshot_merge.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "io" humanize "github.com/dustin/go-humanize" "go.uber.org/zap" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/raft/v3/raftpb" ) // createMergedSnapshotMessage creates a snapshot message that contains: raft status (term, conf), // a snapshot of v2 store inside raft.Snapshot as []byte, a snapshot of v3 KV in the top level message // as ReadCloser. func (s *EtcdServer) createMergedSnapshotMessage(m raftpb.Message, snapt, snapi uint64, confState raftpb.ConfState) snap.Message { lg := s.Logger() // get a snapshot of v2 store as []byte d := GetMembershipInfoInV2Format(lg, s.cluster) // commit kv to write metadata(for example: consistent index). s.KV().Commit() dbsnap := s.be.Snapshot() // get a snapshot of v3 KV as readCloser rc := newSnapshotReaderCloser(lg, dbsnap) // put the []byte snapshot of store into raft snapshot and return the merged snapshot with // KV readCloser snapshot. snapshot := raftpb.Snapshot{ Metadata: raftpb.SnapshotMetadata{ Index: snapi, Term: snapt, ConfState: confState, }, Data: d, } m.Snapshot = &snapshot verifySnapshotIndex(snapshot, s.consistIndex.ConsistentIndex()) return *snap.NewMessage(m, rc, dbsnap.Size()) } func newSnapshotReaderCloser(lg *zap.Logger, snapshot backend.Snapshot) io.ReadCloser { pr, pw := io.Pipe() go func() { n, err := snapshot.WriteTo(pw) if err == nil { lg.Info( "sent database snapshot to writer", zap.Int64("bytes", n), zap.String("size", humanize.Bytes(uint64(n))), ) } else { lg.Warn( "failed to send database snapshot to writer", zap.String("size", humanize.Bytes(uint64(n))), zap.Error(err), ) } pw.CloseWithError(err) err = snapshot.Close() if err != nil { lg.Panic("failed to close database snapshot", zap.Error(err)) } }() return pr } ================================================ FILE: server/etcdserver/tracing.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import pb "go.etcd.io/etcd/api/v3/etcdserverpb" // firstCompareKey returns first non-empty key in the list of comparison operations. func firstCompareKey(c []*pb.Compare) string { for _, op := range c { key := string(op.GetKey()) if key != "" { return key } } return "" } // firstOpKey returns first non-empty key in the list of request operations. func firstOpKey(ops []*pb.RequestOp) string { for _, operation := range ops { var key string switch op := operation.GetRequest().(type) { case *pb.RequestOp_RequestPut: key = string(op.RequestPut.GetKey()) case *pb.RequestOp_RequestRange: key = string(op.RequestRange.GetKey()) case *pb.RequestOp_RequestDeleteRange: key = string(op.RequestDeleteRange.GetKey()) } if key != "" { return key } } return "" } // firstOpType returns type of the first operation in the list. func firstOpType(ops []*pb.RequestOp) string { for _, operation := range ops { switch operation.GetRequest().(type) { case *pb.RequestOp_RequestPut: return "put" case *pb.RequestOp_RequestRange: return "range" case *pb.RequestOp_RequestDeleteRange: return "delete_range" case *pb.RequestOp_RequestTxn: return "txn" } } return "" } // firstOpLease returns lease ID of the first PUT operation in the list. func firstOpLease(ops []*pb.RequestOp) int64 { for _, operation := range ops { if op, ok := operation.GetRequest().(*pb.RequestOp_RequestPut); ok { return op.RequestPut.GetLease() } } return -1 } ================================================ FILE: server/etcdserver/txn/delete.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package txn import ( "context" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/storage/mvcc" ) func DeleteRange(ctx context.Context, lg *zap.Logger, kv mvcc.KV, dr *pb.DeleteRangeRequest) (resp *pb.DeleteRangeResponse, trace *traceutil.Trace, err error) { ctx, trace = traceutil.EnsureTrace(ctx, lg, "delete_range", traceutil.Field{Key: "key", Value: string(dr.Key)}, traceutil.Field{Key: "range_end", Value: string(dr.RangeEnd)}, ) txnWrite := kv.Write(trace) defer txnWrite.End() resp, err = deleteRange(ctx, txnWrite, dr) return resp, trace, err } func deleteRange(ctx context.Context, txnWrite mvcc.TxnWrite, dr *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) { resp := &pb.DeleteRangeResponse{} resp.Header = &pb.ResponseHeader{} end := mkGteRange(dr.RangeEnd) if dr.PrevKv { rr, err := txnWrite.Range(ctx, dr.Key, end, mvcc.RangeOptions{}) if err != nil { return nil, err } if rr != nil { resp.PrevKvs = make([]*mvccpb.KeyValue, len(rr.KVs)) for i := range rr.KVs { resp.PrevKvs[i] = &rr.KVs[i] } } } resp.Deleted, resp.Header.Revision = txnWrite.DeleteRange(dr.Key, end) return resp, nil } // mkGteRange determines if the range end is a >= range. This works around grpc // sending empty byte strings as nil; >= is encoded in the range end as '\0'. // If it is a GTE range, then []byte{} is returned to indicate the empty byte // string (vs nil being no byte string). func mkGteRange(rangeEnd []byte) []byte { if len(rangeEnd) == 1 && rangeEnd[0] == 0 { return []byte{} } return rangeEnd } ================================================ FILE: server/etcdserver/txn/metrics.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package txn import ( "strconv" "time" "github.com/prometheus/client_golang/prometheus" ) var ( slowApplies = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "server", Name: "slow_apply_total", Help: "The total number of slow apply requests (likely overloaded from slow disk).", }) applySec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "server", Name: "apply_duration_seconds", Help: "The latency distributions of v2 apply called by backend.", // lowest bucket start of upper bound 0.0001 sec (0.1 ms) with factor 2 // highest bucket start of 0.0001 sec * 2^19 == 52.4288 sec Buckets: prometheus.ExponentialBuckets(0.0001, 2, 20), }, []string{"version", "op", "success"}, ) rangeSec = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "server", Name: "range_duration_seconds", Help: "The latency distributions of txn.Range", // lowest bucket start of upper bound 0.0001 sec (0.1 ms) with factor 2 // highest bucket start of 0.0001 sec * 2^19 == 52.4288 sec Buckets: prometheus.ExponentialBuckets(0.0001, 2, 20), }, []string{"success"}, ) ) func ApplySecObserve(version, op string, success bool, latency time.Duration) { applySec.WithLabelValues(version, op, strconv.FormatBool(success)).Observe(float64(latency.Microseconds()) / 1000000.0) } func RangeSecObserve(success bool, latency time.Duration) { rangeSec.WithLabelValues(strconv.FormatBool(success)).Observe(float64(latency.Microseconds()) / 1000000.0) } func init() { prometheus.MustRegister(applySec) prometheus.MustRegister(rangeSec) prometheus.MustRegister(slowApplies) } ================================================ FILE: server/etcdserver/txn/metrics_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package txn import ( "strings" "testing" "time" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" ) func TestRangeSecObserve(t *testing.T) { // Simulate a range operation taking 500 milliseconds. latency := 500 * time.Millisecond RangeSecObserve(true, latency) // Use testutil to collect the results and check against expected value expected := ` # HELP etcd_server_range_duration_seconds The latency distributions of txn.Range # TYPE etcd_server_range_duration_seconds histogram etcd_server_range_duration_seconds_bucket{success="true",le="0.0001"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.0002"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.0004"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.0008"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.0016"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.0032"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.0064"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.0128"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.0256"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.0512"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.1024"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.2048"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.4096"} 0 etcd_server_range_duration_seconds_bucket{success="true",le="0.8192"} 1 etcd_server_range_duration_seconds_bucket{success="true",le="1.6384"} 1 etcd_server_range_duration_seconds_bucket{success="true",le="3.2768"} 1 etcd_server_range_duration_seconds_bucket{success="true",le="6.5536"} 1 etcd_server_range_duration_seconds_bucket{success="true",le="13.1072"} 1 etcd_server_range_duration_seconds_bucket{success="true",le="26.2144"} 1 etcd_server_range_duration_seconds_bucket{success="true",le="52.4288"} 1 etcd_server_range_duration_seconds_bucket{success="true",le="+Inf"} 1 etcd_server_range_duration_seconds_sum{success="true"} 0.5 etcd_server_range_duration_seconds_count{success="true"} 1 ` err := testutil.CollectAndCompare(rangeSec, strings.NewReader(expected)) require.NoErrorf(t, err, "Collected metrics did not match expected metrics") } ================================================ FILE: server/etcdserver/txn/put.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package txn import ( "context" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/etcdserver/errors" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/mvcc" ) func Put(ctx context.Context, lg *zap.Logger, lessor lease.Lessor, kv mvcc.KV, p *pb.PutRequest) (resp *pb.PutResponse, trace *traceutil.Trace, err error) { ctx, trace = traceutil.EnsureTrace(ctx, lg, "put", traceutil.Field{Key: "key", Value: string(p.Key)}, traceutil.Field{Key: "req_size", Value: p.Size()}, ) err = checkLease(lessor, p) if err != nil { return nil, trace, err } txnWrite := kv.Write(trace) defer txnWrite.End() prevKV, err := checkAndGetPrevKV(trace, txnWrite, p) if err != nil { return nil, trace, err } return put(ctx, txnWrite, p, prevKV), trace, nil } func put(ctx context.Context, txnWrite mvcc.TxnWrite, p *pb.PutRequest, prevKV *mvcc.RangeResult) *pb.PutResponse { trace := traceutil.Get(ctx) resp := &pb.PutResponse{} resp.Header = &pb.ResponseHeader{} val, leaseID := p.Value, lease.LeaseID(p.Lease) if p.IgnoreValue { val = prevKV.KVs[0].Value } if p.IgnoreLease { leaseID = lease.LeaseID(prevKV.KVs[0].Lease) } if p.PrevKv { if prevKV != nil && len(prevKV.KVs) != 0 { resp.PrevKv = &prevKV.KVs[0] } } resp.Header.Revision = txnWrite.Put(p.Key, val, leaseID) trace.AddField(traceutil.Field{Key: "response_revision", Value: resp.Header.Revision}) return resp } func checkPut(trace *traceutil.Trace, txnWrite mvcc.ReadView, lessor lease.Lessor, p *pb.PutRequest) error { err := checkLease(lessor, p) if err != nil { return err } _, err = checkAndGetPrevKV(trace, txnWrite, p) return err } func checkLease(lessor lease.Lessor, p *pb.PutRequest) error { leaseID := lease.LeaseID(p.Lease) if leaseID != lease.NoLease { if l := lessor.Lookup(leaseID); l == nil { return lease.ErrLeaseNotFound } } return nil } func checkAndGetPrevKV(trace *traceutil.Trace, txnWrite mvcc.ReadView, p *pb.PutRequest) (prevKV *mvcc.RangeResult, err error) { prevKV, err = getPrevKV(trace, txnWrite, p) if err != nil { return nil, err } if p.IgnoreValue || p.IgnoreLease { if prevKV == nil || len(prevKV.KVs) == 0 { // ignore_{lease,value} flag expects previous key-value pair return nil, errors.ErrKeyNotFound } } return prevKV, nil } func getPrevKV(trace *traceutil.Trace, txnWrite mvcc.ReadView, p *pb.PutRequest) (prevKV *mvcc.RangeResult, err error) { if p.IgnoreValue || p.IgnoreLease || p.PrevKv { trace.StepWithFunction(func() { prevKV, err = txnWrite.Range(context.TODO(), p.Key, nil, mvcc.RangeOptions{}) }, "get previous kv pair") if err != nil { return nil, err } } return prevKV, nil } ================================================ FILE: server/etcdserver/txn/range.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package txn import ( "bytes" "context" "sort" "time" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/storage/mvcc" ) func Range(ctx context.Context, lg *zap.Logger, kv mvcc.KV, r *pb.RangeRequest) (resp *pb.RangeResponse, trace *traceutil.Trace, err error) { ctx, trace = traceutil.EnsureTrace(ctx, lg, "range") defer func(start time.Time) { success := err == nil RangeSecObserve(success, time.Since(start)) }(time.Now()) txnRead := kv.Read(mvcc.ConcurrentReadTxMode, trace) defer txnRead.End() resp, err = executeRange(ctx, lg, txnRead, r) return resp, trace, err } func executeRange(ctx context.Context, lg *zap.Logger, txnRead mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) { trace := traceutil.Get(ctx) limit := rangeLimit(r) ro := mvcc.RangeOptions{ Limit: limit, Rev: r.Revision, Count: r.CountOnly, } rr, err := txnRead.Range(ctx, r.Key, mkGteRange(r.RangeEnd), ro) if err != nil { return nil, err } filterRangeResults(rr, r) sortRangeResults(rr, r, lg) trace.Step("filter and sort the key-value pairs") resp := asembleRangeResponse(rr, r) trace.Step("assemble the response") return resp, nil } func rangeLimit(r *pb.RangeRequest) int64 { limit := r.Limit if r.SortOrder != pb.RangeRequest_NONE || r.MinModRevision != 0 || r.MaxModRevision != 0 || r.MinCreateRevision != 0 || r.MaxCreateRevision != 0 { // fetch everything; sort and truncate afterwards limit = 0 } if limit > 0 { // fetch one extra for 'more' flag limit = limit + 1 } return limit } func filterRangeResults(rr *mvcc.RangeResult, r *pb.RangeRequest) { if r.MaxModRevision != 0 { f := func(kv *mvccpb.KeyValue) bool { return kv.ModRevision > r.MaxModRevision } pruneKVs(rr, f) } if r.MinModRevision != 0 { f := func(kv *mvccpb.KeyValue) bool { return kv.ModRevision < r.MinModRevision } pruneKVs(rr, f) } if r.MaxCreateRevision != 0 { f := func(kv *mvccpb.KeyValue) bool { return kv.CreateRevision > r.MaxCreateRevision } pruneKVs(rr, f) } if r.MinCreateRevision != 0 { f := func(kv *mvccpb.KeyValue) bool { return kv.CreateRevision < r.MinCreateRevision } pruneKVs(rr, f) } } func sortRangeResults(rr *mvcc.RangeResult, r *pb.RangeRequest, lg *zap.Logger) { sortOrder := r.SortOrder if r.SortTarget != pb.RangeRequest_KEY && sortOrder == pb.RangeRequest_NONE { // Since current mvcc.Range implementation returns results // sorted by keys in lexiographically ascending order, // sort ASCEND by default only when target is not 'KEY' sortOrder = pb.RangeRequest_ASCEND } else if r.SortTarget == pb.RangeRequest_KEY && sortOrder == pb.RangeRequest_ASCEND { // Since current mvcc.Range implementation returns results // sorted by keys in lexiographically ascending order, // don't re-sort when target is 'KEY' and order is ASCEND sortOrder = pb.RangeRequest_NONE } if sortOrder != pb.RangeRequest_NONE { var sorter sort.Interface switch { case r.SortTarget == pb.RangeRequest_KEY: sorter = &kvSortByKey{&kvSort{rr.KVs}} case r.SortTarget == pb.RangeRequest_VERSION: sorter = &kvSortByVersion{&kvSort{rr.KVs}} case r.SortTarget == pb.RangeRequest_CREATE: sorter = &kvSortByCreate{&kvSort{rr.KVs}} case r.SortTarget == pb.RangeRequest_MOD: sorter = &kvSortByMod{&kvSort{rr.KVs}} case r.SortTarget == pb.RangeRequest_VALUE: sorter = &kvSortByValue{&kvSort{rr.KVs}} default: lg.Panic("unexpected sort target", zap.Int32("sort-target", int32(r.SortTarget))) } switch { case sortOrder == pb.RangeRequest_ASCEND: sort.Sort(sorter) case sortOrder == pb.RangeRequest_DESCEND: sort.Sort(sort.Reverse(sorter)) } } } func asembleRangeResponse(rr *mvcc.RangeResult, r *pb.RangeRequest) *pb.RangeResponse { resp := &pb.RangeResponse{Header: &pb.ResponseHeader{}} if r.Limit > 0 && len(rr.KVs) > int(r.Limit) { rr.KVs = rr.KVs[:r.Limit] resp.More = true } resp.Header.Revision = rr.Rev resp.Count = int64(rr.Count) resp.Kvs = make([]*mvccpb.KeyValue, len(rr.KVs)) for i := range rr.KVs { if r.KeysOnly { rr.KVs[i].Value = nil } resp.Kvs[i] = &rr.KVs[i] } return resp } func checkRange(rv mvcc.ReadView, req *pb.RangeRequest) error { switch { case req.Revision == 0: return nil case req.Revision > rv.Rev(): return mvcc.ErrFutureRev case req.Revision < rv.FirstRev(): return mvcc.ErrCompacted } return nil } func pruneKVs(rr *mvcc.RangeResult, isPrunable func(*mvccpb.KeyValue) bool) { j := 0 for i := range rr.KVs { rr.KVs[j] = rr.KVs[i] if !isPrunable(&rr.KVs[i]) { j++ } } rr.KVs = rr.KVs[:j] } type kvSort struct{ kvs []mvccpb.KeyValue } func (s *kvSort) Swap(i, j int) { t := s.kvs[i] s.kvs[i] = s.kvs[j] s.kvs[j] = t } func (s *kvSort) Len() int { return len(s.kvs) } type kvSortByKey struct{ *kvSort } func (s *kvSortByKey) Less(i, j int) bool { return bytes.Compare(s.kvs[i].Key, s.kvs[j].Key) < 0 } type kvSortByVersion struct{ *kvSort } func (s *kvSortByVersion) Less(i, j int) bool { return (s.kvs[i].Version - s.kvs[j].Version) < 0 } type kvSortByCreate struct{ *kvSort } func (s *kvSortByCreate) Less(i, j int) bool { return (s.kvs[i].CreateRevision - s.kvs[j].CreateRevision) < 0 } type kvSortByMod struct{ *kvSort } func (s *kvSortByMod) Less(i, j int) bool { return (s.kvs[i].ModRevision - s.kvs[j].ModRevision) < 0 } type kvSortByValue struct{ *kvSort } func (s *kvSortByValue) Less(i, j int) bool { return bytes.Compare(s.kvs[i].Value, s.kvs[j].Value) < 0 } ================================================ FILE: server/etcdserver/txn/txn.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package txn import ( "bytes" "context" "fmt" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/mvcc" ) func Txn(ctx context.Context, lg *zap.Logger, rt *pb.TxnRequest, txnModeWriteWithSharedBuffer bool, kv mvcc.KV, lessor lease.Lessor) (txnResp *pb.TxnResponse, trace *traceutil.Trace, err error) { ctx, trace = traceutil.EnsureTrace(ctx, lg, "transaction") isWrite := !IsTxnReadonly(rt) // When the transaction contains write operations, we use ReadTx instead of // ConcurrentReadTx to avoid extra overhead of copying buffer. var mode mvcc.ReadTxMode if isWrite && txnModeWriteWithSharedBuffer /*a.s.Cfg.ServerFeatureGate.Enabled(features.TxnModeWriteWithSharedBuffer)*/ { mode = mvcc.SharedBufReadTxMode } else { mode = mvcc.ConcurrentReadTxMode } txnRead := kv.Read(mode, trace) var txnPath []bool trace.StepWithFunction( func() { txnPath = compareToPath(txnRead, rt) }, "compare", ) if isWrite { trace.AddField(traceutil.Field{Key: "read_only", Value: false}) } _, err = checkTxn(trace, txnRead, rt, lessor, txnPath) if err != nil { txnRead.End() return nil, nil, err } trace.Step("check requests") // When executing mutable txnWrite ops, etcd must hold the txnWrite lock so // readers do not see any intermediate results. Since writes are // serialized on the raft loop, the revision in the read view will // be the revision of the write txnWrite. var txnWrite mvcc.TxnWrite if isWrite { txnRead.End() txnWrite = kv.Write(trace) } else { txnWrite = mvcc.NewReadOnlyTxnWrite(txnRead) } txnResp, err = txn(ctx, lg, txnWrite, rt, isWrite, txnPath) txnWrite.End() trace.AddField( traceutil.Field{Key: "number_of_response", Value: len(txnResp.Responses)}, traceutil.Field{Key: "response_revision", Value: txnResp.Header.Revision}, ) return txnResp, trace, err } func txn(ctx context.Context, lg *zap.Logger, txnWrite mvcc.TxnWrite, rt *pb.TxnRequest, isWrite bool, txnPath []bool) (*pb.TxnResponse, error) { txnResp, _ := newTxnResp(rt, txnPath) _, err := executeTxn(ctx, lg, txnWrite, rt, txnPath, txnResp) if err != nil { if isWrite { // CAUTION: When a txn performing write operations starts, we always expect it to be successful. // If a write failure is seen we SHOULD NOT try to recover the server, but crash with a panic to make the failure explicit. // Trying to silently recover (e.g by ignoring the failed txn or calling txn.End() early) poses serious risks: // - violation of transaction atomicity if some write operations have been partially executed // - data inconsistency across different etcd members if they applied the txn asymmetrically lg.Panic("unexpected error during txn with writes", zap.Error(err)) } else { lg.Error("unexpected error during readonly txn", zap.Error(err)) } } rev := txnWrite.Rev() if len(txnWrite.Changes()) != 0 { rev++ } txnResp.Header.Revision = rev return txnResp, err } // newTxnResp allocates a txn response for a txn request given a path. func newTxnResp(rt *pb.TxnRequest, txnPath []bool) (txnResp *pb.TxnResponse, txnCount int) { reqs := rt.Success if !txnPath[0] { reqs = rt.Failure } resps := make([]*pb.ResponseOp, len(reqs)) txnResp = &pb.TxnResponse{ Responses: resps, Succeeded: txnPath[0], Header: &pb.ResponseHeader{}, } for i, req := range reqs { switch tv := req.Request.(type) { case *pb.RequestOp_RequestRange: resps[i] = &pb.ResponseOp{Response: &pb.ResponseOp_ResponseRange{}} case *pb.RequestOp_RequestPut: resps[i] = &pb.ResponseOp{Response: &pb.ResponseOp_ResponsePut{}} case *pb.RequestOp_RequestDeleteRange: resps[i] = &pb.ResponseOp{Response: &pb.ResponseOp_ResponseDeleteRange{}} case *pb.RequestOp_RequestTxn: resp, txns := newTxnResp(tv.RequestTxn, txnPath[1:]) resps[i] = &pb.ResponseOp{Response: &pb.ResponseOp_ResponseTxn{ResponseTxn: resp}} txnPath = txnPath[1+txns:] txnCount += txns + 1 default: } } return txnResp, txnCount } func executeTxn(ctx context.Context, lg *zap.Logger, txnWrite mvcc.TxnWrite, rt *pb.TxnRequest, txnPath []bool, tresp *pb.TxnResponse) (txns int, err error) { trace := traceutil.Get(ctx) reqs := rt.Success if !txnPath[0] { reqs = rt.Failure } for i, req := range reqs { respi := tresp.Responses[i].Response switch tv := req.Request.(type) { case *pb.RequestOp_RequestRange: trace.StartSubTrace( traceutil.Field{Key: "req_type", Value: "range"}, traceutil.Field{Key: "range_begin", Value: string(tv.RequestRange.Key)}, traceutil.Field{Key: "range_end", Value: string(tv.RequestRange.RangeEnd)}) resp, err := executeRange(ctx, lg, txnWrite, tv.RequestRange) if err != nil { return 0, fmt.Errorf("applyTxn: failed Range: %w", err) } respi.(*pb.ResponseOp_ResponseRange).ResponseRange = resp trace.StopSubTrace() case *pb.RequestOp_RequestPut: trace.StartSubTrace( traceutil.Field{Key: "req_type", Value: "put"}, traceutil.Field{Key: "key", Value: string(tv.RequestPut.Key)}, traceutil.Field{Key: "req_size", Value: tv.RequestPut.Size()}) prevKV, err := getPrevKV(trace, txnWrite, tv.RequestPut) if err != nil { return 0, fmt.Errorf("applyTxn: failed to get prevKV on put: %w", err) } resp := put(ctx, txnWrite, tv.RequestPut, prevKV) respi.(*pb.ResponseOp_ResponsePut).ResponsePut = resp trace.StopSubTrace() case *pb.RequestOp_RequestDeleteRange: resp, err := deleteRange(ctx, txnWrite, tv.RequestDeleteRange) if err != nil { return 0, fmt.Errorf("applyTxn: failed DeleteRange: %w", err) } respi.(*pb.ResponseOp_ResponseDeleteRange).ResponseDeleteRange = resp case *pb.RequestOp_RequestTxn: resp := respi.(*pb.ResponseOp_ResponseTxn).ResponseTxn applyTxns, err := executeTxn(ctx, lg, txnWrite, tv.RequestTxn, txnPath[1:], resp) if err != nil { // don't wrap the error. It's a recursive call and err should be already wrapped return 0, err } txns += applyTxns + 1 txnPath = txnPath[applyTxns+1:] default: // empty union } } return txns, nil } func checkTxn(trace *traceutil.Trace, rv mvcc.ReadView, rt *pb.TxnRequest, lessor lease.Lessor, txnPath []bool) (int, error) { txnCount := 0 reqs := rt.Success if !txnPath[0] { reqs = rt.Failure } for _, req := range reqs { var err error var txns int switch tv := req.Request.(type) { case *pb.RequestOp_RequestRange: err = checkRange(rv, tv.RequestRange) case *pb.RequestOp_RequestPut: err = checkPut(trace, rv, lessor, tv.RequestPut) case *pb.RequestOp_RequestDeleteRange: case *pb.RequestOp_RequestTxn: txns, err = checkTxn(trace, rv, tv.RequestTxn, lessor, txnPath[1:]) txnCount += txns + 1 txnPath = txnPath[txns+1:] default: // empty union } if err != nil { return 0, err } } return txnCount, nil } func compareInt64(a, b int64) int { switch { case a < b: return -1 case a > b: return 1 default: return 0 } } func compareToPath(rv mvcc.ReadView, rt *pb.TxnRequest) []bool { txnPath := make([]bool, 1) ops := rt.Success if txnPath[0] = applyCompares(rv, rt.Compare); !txnPath[0] { ops = rt.Failure } for _, op := range ops { tv, ok := op.Request.(*pb.RequestOp_RequestTxn) if !ok || tv.RequestTxn == nil { continue } txnPath = append(txnPath, compareToPath(rv, tv.RequestTxn)...) } return txnPath } func applyCompares(rv mvcc.ReadView, cmps []*pb.Compare) bool { for _, c := range cmps { if !applyCompare(rv, c) { return false } } return true } // applyCompare applies the compare request. // If the comparison succeeds, it returns true. Otherwise, returns false. func applyCompare(rv mvcc.ReadView, c *pb.Compare) bool { // TODO: possible optimizations // * chunk reads for large ranges to conserve memory // * rewrite rules for common patterns: // ex. "[a, b) createrev > 0" => "limit 1 /\ kvs > 0" // * caching rr, err := rv.Range(context.TODO(), c.Key, mkGteRange(c.RangeEnd), mvcc.RangeOptions{}) if err != nil { return false } if len(rr.KVs) == 0 { if c.Target == pb.Compare_VALUE { // Always fail if comparing a value on a key/keys that doesn't exist; // nil == empty string in grpc; no way to represent missing value return false } return compareKV(c, mvccpb.KeyValue{}) } for _, kv := range rr.KVs { if !compareKV(c, kv) { return false } } return true } func compareKV(c *pb.Compare, ckv mvccpb.KeyValue) bool { var result int rev := int64(0) switch c.Target { case pb.Compare_VALUE: var v []byte if tv, _ := c.TargetUnion.(*pb.Compare_Value); tv != nil { v = tv.Value } result = bytes.Compare(ckv.Value, v) case pb.Compare_CREATE: if tv, _ := c.TargetUnion.(*pb.Compare_CreateRevision); tv != nil { rev = tv.CreateRevision } result = compareInt64(ckv.CreateRevision, rev) case pb.Compare_MOD: if tv, _ := c.TargetUnion.(*pb.Compare_ModRevision); tv != nil { rev = tv.ModRevision } result = compareInt64(ckv.ModRevision, rev) case pb.Compare_VERSION: if tv, _ := c.TargetUnion.(*pb.Compare_Version); tv != nil { rev = tv.Version } result = compareInt64(ckv.Version, rev) case pb.Compare_LEASE: if tv, _ := c.TargetUnion.(*pb.Compare_Lease); tv != nil { rev = tv.Lease } result = compareInt64(ckv.Lease, rev) } switch c.Result { case pb.Compare_EQUAL: return result == 0 case pb.Compare_NOT_EQUAL: return result != 0 case pb.Compare_GREATER: return result > 0 case pb.Compare_LESS: return result < 0 } return true } func IsTxnSerializable(r *pb.TxnRequest) bool { for _, u := range r.Success { if r := u.GetRequestRange(); r == nil || !r.Serializable { return false } } for _, u := range r.Failure { if r := u.GetRequestRange(); r == nil || !r.Serializable { return false } } return true } func IsTxnReadonly(r *pb.TxnRequest) bool { for _, u := range r.Success { if r := u.GetRequestRange(); r == nil { return false } } for _, u := range r.Failure { if r := u.GetRequestRange(); r == nil { return false } } return true } func CheckTxnAuth(as auth.AuthStore, ai *auth.AuthInfo, rt *pb.TxnRequest) error { return checkTxnPermission(as, ai, rt) } func checkTxnPermission(as auth.AuthStore, ai *auth.AuthInfo, rt *pb.TxnRequest) error { for _, c := range rt.Compare { if err := as.IsRangePermitted(ai, c.Key, c.RangeEnd); err != nil { return err } } if err := checkTxnReqsPermission(as, ai, rt.Success); err != nil { return err } return checkTxnReqsPermission(as, ai, rt.Failure) } func checkTxnReqsPermission(as auth.AuthStore, ai *auth.AuthInfo, reqs []*pb.RequestOp) error { for _, requ := range reqs { switch tv := requ.Request.(type) { case *pb.RequestOp_RequestRange: if tv.RequestRange == nil { continue } if err := as.IsRangePermitted(ai, tv.RequestRange.Key, tv.RequestRange.RangeEnd); err != nil { return err } case *pb.RequestOp_RequestPut: if tv.RequestPut == nil { continue } if err := as.IsPutPermitted(ai, tv.RequestPut.Key); err != nil { return err } case *pb.RequestOp_RequestDeleteRange: if tv.RequestDeleteRange == nil { continue } if tv.RequestDeleteRange.PrevKv { err := as.IsRangePermitted(ai, tv.RequestDeleteRange.Key, tv.RequestDeleteRange.RangeEnd) if err != nil { return err } } err := as.IsDeleteRangePermitted(ai, tv.RequestDeleteRange.Key, tv.RequestDeleteRange.RangeEnd) if err != nil { return err } case *pb.RequestOp_RequestTxn: if tv.RequestTxn == nil { continue } err := checkTxnPermission(as, ai, tv.RequestTxn) if err != nil { return err } } } return nil } ================================================ FILE: server/etcdserver/txn/txn_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package txn import ( "context" "crypto/sha256" "io" "os" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/authpb" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" ) type testCase struct { name string setup testSetup op *pb.RequestOp expectError string } type testSetup struct { compactRevision int64 lease int64 key []byte } var futureRev int64 = 1000 var rangeTestCases = []testCase{ { name: "Range with revision 0 should succeed", op: &pb.RequestOp{ Request: &pb.RequestOp_RequestRange{ RequestRange: &pb.RangeRequest{ Revision: 0, }, }, }, }, { name: "Range on future rev should fail", op: &pb.RequestOp{ Request: &pb.RequestOp_RequestRange{ RequestRange: &pb.RangeRequest{ Revision: futureRev, }, }, }, expectError: "mvcc: required revision is a future revision", }, { name: "Range on compacted rev should fail", setup: testSetup{compactRevision: 10}, op: &pb.RequestOp{ Request: &pb.RequestOp_RequestRange{ RequestRange: &pb.RangeRequest{ Revision: 9, }, }, }, expectError: "mvcc: required revision has been compacted", }, } var putTestCases = []testCase{ { name: "Put without lease should succeed", op: &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{}, }, }, }, { name: "Put with non-existing lease should fail", op: &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Lease: 123, }, }, }, expectError: "lease not found", }, { name: "Put with existing lease should succeed", setup: testSetup{lease: 123}, op: &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Lease: 123, }, }, }, }, { name: "Put with ignore value without previous key should fail", op: &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ IgnoreValue: true, }, }, }, expectError: "etcdserver: key not found", }, { name: "Put with ignore lease without previous key should fail", op: &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ IgnoreLease: true, }, }, }, expectError: "etcdserver: key not found", }, { name: "Put with ignore value with previous key should succeeded", setup: testSetup{key: []byte("ignore-value")}, op: &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ IgnoreValue: true, Key: []byte("ignore-value"), }, }, }, }, { name: "Put with ignore lease with previous key should succeed ", setup: testSetup{key: []byte("ignore-lease")}, op: &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ IgnoreLease: true, Key: []byte("ignore-lease"), }, }, }, }, } func TestCheckTxn(t *testing.T) { type txnTestCase struct { name string setup testSetup txn *pb.TxnRequest expectError string } testCases := []txnTestCase{} for _, tc := range append(rangeTestCases, putTestCases...) { testCases = append(testCases, txnTestCase{ name: tc.name, setup: tc.setup, txn: &pb.TxnRequest{ Success: []*pb.RequestOp{ tc.op, }, }, expectError: tc.expectError, }) } invalidOperation := &pb.RequestOp{ Request: &pb.RequestOp_RequestRange{ RequestRange: &pb.RangeRequest{ Revision: futureRev, }, }, } testCases = append(testCases, txnTestCase{ name: "Invalid operation on failed path should succeed", txn: &pb.TxnRequest{ Failure: []*pb.RequestOp{ invalidOperation, }, }, }) testCases = append(testCases, txnTestCase{ name: "Invalid operation on subtransaction should fail", txn: &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{ Success: []*pb.RequestOp{ invalidOperation, }, }, }, }, }, }, expectError: "mvcc: required revision is a future revision", }) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { s, lessor := setup(t, tc.setup) ctx, cancel := context.WithCancel(t.Context()) defer cancel() _, _, err := Txn(ctx, zaptest.NewLogger(t), tc.txn, false, s, lessor) gotErr := "" if err != nil { gotErr = err.Error() } if gotErr != tc.expectError { t.Errorf("Error not matching, got %q, expected %q", gotErr, tc.expectError) } }) } } func TestCheckPut(t *testing.T) { for _, tc := range putTestCases { t.Run(tc.name, func(t *testing.T) { s, lessor := setup(t, tc.setup) ctx, cancel := context.WithCancel(t.Context()) defer cancel() _, _, err := Put(ctx, zaptest.NewLogger(t), lessor, s, tc.op.GetRequestPut()) gotErr := "" if err != nil { gotErr = err.Error() } if gotErr != tc.expectError { t.Errorf("Error not matching, got %q, expected %q", gotErr, tc.expectError) } }) } } func TestCheckRange(t *testing.T) { for _, tc := range rangeTestCases { t.Run(tc.name, func(t *testing.T) { s, _ := setup(t, tc.setup) ctx, cancel := context.WithCancel(t.Context()) defer cancel() _, _, err := Range(ctx, zaptest.NewLogger(t), s, tc.op.GetRequestRange()) gotErr := "" if err != nil { gotErr = err.Error() } if gotErr != tc.expectError { t.Errorf("Error not matching, got %q, expected %q", gotErr, tc.expectError) } }) } } func setup(t *testing.T, setup testSetup) (mvcc.KV, lease.Lessor) { b, _ := betesting.NewDefaultTmpBackend(t) t.Cleanup(func() { betesting.Close(t, b) }) lessor := &lease.FakeLessor{LeaseSet: map[lease.LeaseID]struct{}{}} s := mvcc.NewStore(zaptest.NewLogger(t), b, lessor, mvcc.StoreConfig{}) t.Cleanup(func() { s.Close() }) if setup.compactRevision != 0 { for i := 0; int64(i) < setup.compactRevision; i++ { s.Put([]byte("a"), []byte("b"), 0) } s.Compact(traceutil.TODO(), setup.compactRevision) } if setup.lease != 0 { lessor.Grant(lease.LeaseID(setup.lease), 0) } if len(setup.key) != 0 { s.Put(setup.key, []byte("b"), 0) } return s, lessor } func TestReadonlyTxnError(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, b) s := mvcc.NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, mvcc.StoreConfig{}) defer s.Close() // setup cancelled context ctx, cancel := context.WithCancel(t.Context()) cancel() // put some data to prevent early termination in rangeKeys // we are expecting failure on cancelled context check s.Put([]byte("foo"), []byte("bar"), lease.NoLease) txn := &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestRange{ RequestRange: &pb.RangeRequest{ Key: []byte("foo"), }, }, }, }, } _, _, err := Txn(ctx, zaptest.NewLogger(t), txn, false, s, &lease.FakeLessor{}) if err == nil || !strings.Contains(err.Error(), "applyTxn: failed Range: rangeKeys: context cancelled: context canceled") { t.Fatalf("Expected context canceled error, got %v", err) } } func TestWriteTxnPanicWithoutApply(t *testing.T) { b, bePath := betesting.NewDefaultTmpBackend(t) s := mvcc.NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, mvcc.StoreConfig{}) defer s.Close() // setup cancelled context ctx, cancel := context.WithCancel(t.Context()) cancel() // write txn that puts some data and then fails in range due to cancelled context txn := &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Key: []byte("foo"), Value: []byte("bar"), }, }, }, { Request: &pb.RequestOp_RequestRange{ RequestRange: &pb.RangeRequest{ Key: []byte("foo"), }, }, }, }, } // compute DB file hash before applying the txn dbHashBefore, err := computeFileHash(bePath) require.NoErrorf(t, err, "failed to compute DB file hash before txn") // we verify the following properties below: // 1. server panics after a write txn aply fails (invariant: server should never try to move on from a failed write) // 2. no writes from the txn are applied to the backend (invariant: failed write should have no side-effect on DB state besides panic) assert.Panicsf(t, func() { Txn(ctx, zaptest.NewLogger(t), txn, false, s, &lease.FakeLessor{}) }, "Expected panic in Txn with writes") dbHashAfter, err := computeFileHash(bePath) require.NoErrorf(t, err, "failed to compute DB file hash after txn") require.Equalf(t, dbHashBefore, dbHashAfter, "mismatch in DB hash before and after failed write txn") } func TestCheckTxnAuth(t *testing.T) { be, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, be) as := setupAuth(t, be) tests := []struct { name string txnRequest *pb.TxnRequest err error }{ { name: "Out of range compare is unauthorized", txnRequest: &pb.TxnRequest{ Compare: []*pb.Compare{outOfRangeCompare}, }, err: auth.ErrPermissionDenied, }, { name: "In range compare is authorized", txnRequest: &pb.TxnRequest{ Compare: []*pb.Compare{inRangeCompare}, }, err: nil, }, { name: "Nil request range is always authorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{nilRequestRange}, }, err: nil, }, { name: "Range request in range is authorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{inRangeRequestRange}, Failure: []*pb.RequestOp{inRangeRequestRange}, }, err: nil, }, { name: "Range request out of range success case is unauthorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{outOfRangeRequestRange}, Failure: []*pb.RequestOp{inRangeRequestRange}, }, err: auth.ErrPermissionDenied, }, { name: "Range request out of range failure case is unauthorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{inRangeRequestRange}, Failure: []*pb.RequestOp{outOfRangeRequestRange}, }, err: auth.ErrPermissionDenied, }, { name: "Nil Put request is always authorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{nilRequestPut}, }, err: nil, }, { name: "Put request in range in authorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{inRangeRequestPut}, Failure: []*pb.RequestOp{inRangeRequestPut}, }, err: nil, }, { name: "Put request out of range success case is unauthorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{outOfRangeRequestPut}, Failure: []*pb.RequestOp{inRangeRequestPut}, }, err: auth.ErrPermissionDenied, }, { name: "Put request out of range failure case is unauthorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{inRangeRequestPut}, Failure: []*pb.RequestOp{outOfRangeRequestPut}, }, err: auth.ErrPermissionDenied, }, { name: "Nil delete request is authorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{nilRequestDeleteRange}, }, err: nil, }, { name: "Delete range request in range is authorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{inRangeRequestDeleteRange}, Failure: []*pb.RequestOp{inRangeRequestDeleteRange}, }, err: nil, }, { name: "Delete range request out of range success case is unauthorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{outOfRangeRequestDeleteRange}, Failure: []*pb.RequestOp{inRangeRequestDeleteRange}, }, err: auth.ErrPermissionDenied, }, { name: "Delete range request out of range failure case is unauthorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{inRangeRequestDeleteRange}, Failure: []*pb.RequestOp{outOfRangeRequestDeleteRange}, }, err: auth.ErrPermissionDenied, }, { name: "Delete range request out of range and PrevKv false success case is unauthorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{outOfRangeRequestDeleteRangeKvFalse}, Failure: []*pb.RequestOp{inRangeRequestDeleteRange}, }, err: auth.ErrPermissionDenied, }, { name: "Delete range request out of range and PrevKv false failure case is unauthorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{inRangeRequestDeleteRange}, Failure: []*pb.RequestOp{outOfRangeRequestDeleteRangeKvFalse}, }, err: auth.ErrPermissionDenied, }, { name: "Nested txn request in range is authorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{ Success: []*pb.RequestOp{inRangeRequestRange, inRangeRequestPut}, Failure: []*pb.RequestOp{inRangeRequestDeleteRange}, }, }, }, }, }, err: nil, }, { name: "Nested txn request out of range success case is unauthorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{ Success: []*pb.RequestOp{outOfRangeRequestRange}, }, }, }, }, }, err: auth.ErrPermissionDenied, }, { name: "Nested txn request out of range failure case is unauthorized", txnRequest: &pb.TxnRequest{ Failure: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{ Failure: []*pb.RequestOp{outOfRangeRequestPut}, }, }, }, }, }, err: auth.ErrPermissionDenied, }, { name: "Nested txn request out of range delete is unauthorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{ Success: []*pb.RequestOp{outOfRangeRequestDeleteRange}, }, }, }, }, }, err: auth.ErrPermissionDenied, }, { name: "Two level nested txn request out of range delete is unauthorized", txnRequest: &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{ Failure: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{ Success: []*pb.RequestOp{outOfRangeRequestDeleteRange}, }, }, }, }, }, }, }, }, }, err: auth.ErrPermissionDenied, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { err := CheckTxnAuth(as, &auth.AuthInfo{Username: "foo", Revision: 8}, tt.txnRequest) assert.Equal(t, tt.err, err) }) } } // CheckTxnAuth test setup. func setupAuth(t *testing.T, be backend.Backend) auth.AuthStore { lg := zaptest.NewLogger(t) simpleTokenTTLDefault := 300 * time.Second tokenTypeSimple := "simple" dummyIndexWaiter := func(index uint64) <-chan struct{} { ch := make(chan struct{}, 1) go func() { ch <- struct{}{} }() return ch } tp, _ := auth.NewTokenProvider(zaptest.NewLogger(t), tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault) as := auth.NewAuthStore(lg, schema.NewAuthBackend(lg, be), tp, 4) // create "root" user and "foo" user with limited range _, err := as.RoleAdd(&pb.AuthRoleAddRequest{Name: "root"}) require.NoError(t, err) _, err = as.RoleAdd(&pb.AuthRoleAddRequest{Name: "rw"}) require.NoError(t, err) _, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{ Name: "rw", Perm: &authpb.Permission{ PermType: authpb.Permission_READWRITE, Key: []byte("foo"), RangeEnd: []byte("zoo"), }, }) require.NoError(t, err) _, err = as.UserAdd(&pb.AuthUserAddRequest{Name: "root", Password: "foo"}) require.NoError(t, err) _, err = as.UserAdd(&pb.AuthUserAddRequest{Name: "foo", Password: "foo"}) require.NoError(t, err) _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "root", Role: "root"}) require.NoError(t, err) _, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "foo", Role: "rw"}) require.NoError(t, err) err = as.AuthEnable() require.NoError(t, err) return as } func computeFileHash(filePath string) (string, error) { file, err := os.Open(filePath) if err != nil { return "", err } defer file.Close() h := sha256.New() if _, err := io.Copy(h, file); err != nil { return "", err } return string(h.Sum(nil)), nil } // CheckTxnAuth variables setup. var ( inRangeCompare = &pb.Compare{ Key: []byte("foo"), RangeEnd: []byte("zoo"), } outOfRangeCompare = &pb.Compare{ Key: []byte("boo"), RangeEnd: []byte("zoo"), } nilRequestPut = &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: nil, }, } inRangeRequestPut = &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Key: []byte("foo"), }, }, } outOfRangeRequestPut = &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Key: []byte("boo"), }, }, } nilRequestRange = &pb.RequestOp{ Request: &pb.RequestOp_RequestRange{ RequestRange: nil, }, } inRangeRequestRange = &pb.RequestOp{ Request: &pb.RequestOp_RequestRange{ RequestRange: &pb.RangeRequest{ Key: []byte("foo"), RangeEnd: []byte("zoo"), }, }, } outOfRangeRequestRange = &pb.RequestOp{ Request: &pb.RequestOp_RequestRange{ RequestRange: &pb.RangeRequest{ Key: []byte("boo"), RangeEnd: []byte("zoo"), }, }, } nilRequestDeleteRange = &pb.RequestOp{ Request: &pb.RequestOp_RequestDeleteRange{ RequestDeleteRange: nil, }, } inRangeRequestDeleteRange = &pb.RequestOp{ Request: &pb.RequestOp_RequestDeleteRange{ RequestDeleteRange: &pb.DeleteRangeRequest{ Key: []byte("foo"), RangeEnd: []byte("zoo"), PrevKv: true, }, }, } outOfRangeRequestDeleteRange = &pb.RequestOp{ Request: &pb.RequestOp_RequestDeleteRange{ RequestDeleteRange: &pb.DeleteRangeRequest{ Key: []byte("boo"), RangeEnd: []byte("zoo"), PrevKv: true, }, }, } outOfRangeRequestDeleteRangeKvFalse = &pb.RequestOp{ Request: &pb.RequestOp_RequestDeleteRange{ RequestDeleteRange: &pb.DeleteRangeRequest{ Key: []byte("boo"), RangeEnd: []byte("zoo"), PrevKv: false, }, }, } ) ================================================ FILE: server/etcdserver/txn/util.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package txn import ( "fmt" "reflect" "strings" "time" "github.com/golang/protobuf/proto" //nolint:staticcheck // TODO: remove for a supported version "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) func WarnOfExpensiveRequest(lg *zap.Logger, warningApplyDuration time.Duration, now time.Time, reqStringer fmt.Stringer, respMsg proto.Message, err error) { if time.Since(now) <= warningApplyDuration { return } var resp string if !isNil(respMsg) { resp = fmt.Sprintf("size:%d", proto.Size(respMsg)) } warnOfExpensiveGenericRequest(lg, warningApplyDuration, now, reqStringer, "", resp, err) } func WarnOfFailedRequest(lg *zap.Logger, now time.Time, reqStringer fmt.Stringer, respMsg proto.Message, err error) { var resp string if !isNil(respMsg) { resp = fmt.Sprintf("size:%d", proto.Size(respMsg)) } d := time.Since(now) lg.Warn( "failed to apply request", zap.Duration("took", d), zap.String("request", reqStringer.String()), zap.String("response", resp), zap.Error(err), ) } func WarnOfExpensiveReadOnlyTxnRequest(lg *zap.Logger, warningApplyDuration time.Duration, now time.Time, r *pb.TxnRequest, txnResponse *pb.TxnResponse, err error) { if time.Since(now) <= warningApplyDuration { return } reqStringer := pb.NewLoggableTxnRequest(r) var resp string if !isNil(txnResponse) { var resps []string for _, r := range txnResponse.Responses { switch r.Response.(type) { case *pb.ResponseOp_ResponseRange: if op := r.GetResponseRange(); op != nil { resps = append(resps, fmt.Sprintf("range_response_count:%d", len(op.GetKvs()))) } else { resps = append(resps, "range_response:nil") } default: // only range responses should be in a read only txn request } } resp = fmt.Sprintf("responses:<%s> size:%d", strings.Join(resps, " "), txnResponse.Size()) } warnOfExpensiveGenericRequest(lg, warningApplyDuration, now, reqStringer, "read-only txn ", resp, err) } func WarnOfExpensiveReadOnlyRangeRequest(lg *zap.Logger, warningApplyDuration time.Duration, now time.Time, reqStringer fmt.Stringer, rangeResponse *pb.RangeResponse, err error) { if time.Since(now) <= warningApplyDuration { return } var resp string if !isNil(rangeResponse) { resp = fmt.Sprintf("range_response_count:%d size:%d", len(rangeResponse.Kvs), rangeResponse.Size()) } warnOfExpensiveGenericRequest(lg, warningApplyDuration, now, reqStringer, "read-only range ", resp, err) } // callers need make sure time has passed warningApplyDuration func warnOfExpensiveGenericRequest(lg *zap.Logger, warningApplyDuration time.Duration, now time.Time, reqStringer fmt.Stringer, prefix string, resp string, err error) { lg.Warn( "apply request took too long", zap.Duration("took", time.Since(now)), zap.Duration("expected-duration", warningApplyDuration), zap.String("prefix", prefix), zap.String("request", reqStringer.String()), zap.String("response", resp), zap.Error(err), ) slowApplies.Inc() } func isNil(msg proto.Message) bool { return msg == nil || reflect.ValueOf(msg).IsNil() } ================================================ FILE: server/etcdserver/txn/util_bench_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package txn import ( "errors" "testing" "time" "go.uber.org/zap/zaptest" "go.etcd.io/raft/v3/raftpb" ) func BenchmarkWarnOfExpensiveRequestNoLog(b *testing.B) { m := &raftpb.Message{ Type: 0, To: 0, From: 1, Term: 2, LogTerm: 3, Index: 0, Entries: []raftpb.Entry{ { Term: 0, Index: 0, Type: 0, Data: make([]byte, 1024), }, }, Commit: 0, Snapshot: nil, Reject: false, RejectHint: 0, Context: nil, } err := errors.New("benchmarking warn of expensive request") lg := zaptest.NewLogger(b) for n := 0; n < b.N; n++ { WarnOfExpensiveRequest(lg, time.Second, time.Now(), nil, m, err) } } ================================================ FILE: server/etcdserver/txn/util_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package txn import ( "testing" "time" "go.uber.org/zap/zaptest" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" ) // TestWarnOfExpensiveReadOnlyTxnRequest verifies WarnOfExpensiveReadOnlyTxnRequest // never panic no matter what data the txnResponse contains. func TestWarnOfExpensiveReadOnlyTxnRequest(t *testing.T) { kvs := []*mvccpb.KeyValue{ {Key: []byte("k1"), Value: []byte("v1")}, {Key: []byte("k2"), Value: []byte("v2")}, } testCases := []struct { name string txnResp *pb.TxnResponse }{ { name: "all readonly responses", txnResp: &pb.TxnResponse{ Responses: []*pb.ResponseOp{ { Response: &pb.ResponseOp_ResponseRange{ ResponseRange: &pb.RangeResponse{ Kvs: kvs, }, }, }, { Response: &pb.ResponseOp_ResponseRange{ ResponseRange: &pb.RangeResponse{}, }, }, }, }, }, { name: "all readonly responses with partial nil responses", txnResp: &pb.TxnResponse{ Responses: []*pb.ResponseOp{ { Response: &pb.ResponseOp_ResponseRange{ ResponseRange: &pb.RangeResponse{}, }, }, { Response: &pb.ResponseOp_ResponseRange{ ResponseRange: nil, }, }, { Response: &pb.ResponseOp_ResponseRange{ ResponseRange: &pb.RangeResponse{ Kvs: kvs, }, }, }, }, }, }, { name: "all readonly responses with all nil responses", txnResp: &pb.TxnResponse{ Responses: []*pb.ResponseOp{ { Response: &pb.ResponseOp_ResponseRange{ ResponseRange: nil, }, }, { Response: &pb.ResponseOp_ResponseRange{ ResponseRange: nil, }, }, }, }, }, { name: "partial non readonly responses", txnResp: &pb.TxnResponse{ Responses: []*pb.ResponseOp{ { Response: &pb.ResponseOp_ResponseRange{ ResponseRange: nil, }, }, { Response: &pb.ResponseOp_ResponsePut{}, }, { Response: &pb.ResponseOp_ResponseDeleteRange{}, }, }, }, }, { name: "all non readonly responses", txnResp: &pb.TxnResponse{ Responses: []*pb.ResponseOp{ { Response: &pb.ResponseOp_ResponsePut{}, }, { Response: &pb.ResponseOp_ResponseDeleteRange{}, }, }, }, }, } for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) start := time.Now().Add(-1 * time.Second) // WarnOfExpensiveReadOnlyTxnRequest shouldn't panic. WarnOfExpensiveReadOnlyTxnRequest(lg, 0, start, &pb.TxnRequest{}, tc.txnResp, nil) }) } } ================================================ FILE: server/etcdserver/util.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "time" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" ) // isConnectedToQuorumSince checks whether the local member is connected to the // quorum of the cluster since the given time. func isConnectedToQuorumSince(transport rafthttp.Transporter, since time.Time, self types.ID, members []*membership.Member) bool { return numConnectedSince(transport, since, self, members) >= (len(members)/2)+1 } // isConnectedSince checks whether the local member is connected to the // remote member since the given time. func isConnectedSince(transport rafthttp.Transporter, since time.Time, remote types.ID) bool { t := transport.ActiveSince(remote) return !t.IsZero() && t.Before(since) } // isConnectedFullySince checks whether the local member is connected to all // members in the cluster since the given time. func isConnectedFullySince(transport rafthttp.Transporter, since time.Time, self types.ID, members []*membership.Member) bool { return numConnectedSince(transport, since, self, members) == len(members) } // numConnectedSince counts how many members are connected to the local member // since the given time. func numConnectedSince(transport rafthttp.Transporter, since time.Time, self types.ID, members []*membership.Member) int { connectedNum := 0 for _, m := range members { if m.ID == self || isConnectedSince(transport, since, m.ID) { connectedNum++ } } return connectedNum } // longestConnected chooses the member with longest active-since-time. // It returns false, if nothing is active. func longestConnected(tp rafthttp.Transporter, membs []types.ID) (types.ID, bool) { var longest types.ID var oldest time.Time for _, id := range membs { tm := tp.ActiveSince(id) if tm.IsZero() { // inactive continue } if oldest.IsZero() { // first longest candidate oldest = tm longest = id } if tm.Before(oldest) { oldest = tm longest = id } } if uint64(longest) == 0 { return longest, false } return longest, true } type notifier struct { c chan struct{} err error } func newNotifier() *notifier { return ¬ifier{ c: make(chan struct{}), } } func (nc *notifier) notify(err error) { nc.err = err close(nc.c) } ================================================ FILE: server/etcdserver/util_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "net/http" "testing" "time" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/raft/v3/raftpb" ) func TestLongestConnected(t *testing.T) { umap, err := types.NewURLsMap("mem1=http://10.1:2379,mem2=http://10.2:2379,mem3=http://10.3:2379") if err != nil { t.Fatal(err) } clus, err := membership.NewClusterFromURLsMap(zaptest.NewLogger(t), "test", umap) if err != nil { t.Fatal(err) } memberIDs := clus.MemberIDs() tr := newNopTransporterWithActiveTime(memberIDs) transferee, ok := longestConnected(tr, memberIDs) if !ok { t.Fatalf("unexpected ok %v", ok) } if memberIDs[0] != transferee { t.Fatalf("expected first member %s to be transferee, got %s", memberIDs[0], transferee) } // make all members non-active amap := make(map[types.ID]time.Time) for _, id := range memberIDs { amap[id] = time.Time{} } tr.(*nopTransporterWithActiveTime).reset(amap) _, ok2 := longestConnected(tr, memberIDs) if ok2 { t.Fatalf("unexpected ok %v", ok) } } type nopTransporterWithActiveTime struct { activeMap map[types.ID]time.Time } // newNopTransporterWithActiveTime creates nopTransporterWithActiveTime with the first member // being the most stable (longest active-since time). func newNopTransporterWithActiveTime(memberIDs []types.ID) rafthttp.Transporter { am := make(map[types.ID]time.Time) for i, id := range memberIDs { am[id] = time.Now().Add(time.Duration(i) * time.Second) } return &nopTransporterWithActiveTime{activeMap: am} } func (s *nopTransporterWithActiveTime) Start() error { return nil } func (s *nopTransporterWithActiveTime) Handler() http.Handler { return nil } func (s *nopTransporterWithActiveTime) Send(m []raftpb.Message) {} func (s *nopTransporterWithActiveTime) SendSnapshot(m snap.Message) {} func (s *nopTransporterWithActiveTime) AddRemote(id types.ID, us []string) {} func (s *nopTransporterWithActiveTime) AddPeer(id types.ID, us []string) {} func (s *nopTransporterWithActiveTime) RemovePeer(id types.ID) {} func (s *nopTransporterWithActiveTime) RemoveAllPeers() {} func (s *nopTransporterWithActiveTime) UpdatePeer(id types.ID, us []string) {} func (s *nopTransporterWithActiveTime) ActiveSince(id types.ID) time.Time { return s.activeMap[id] } func (s *nopTransporterWithActiveTime) ActivePeers() int { return 0 } func (s *nopTransporterWithActiveTime) Stop() {} func (s *nopTransporterWithActiveTime) Pause() {} func (s *nopTransporterWithActiveTime) Resume() {} func (s *nopTransporterWithActiveTime) reset(am map[types.ID]time.Time) { s.activeMap = am } ================================================ FILE: server/etcdserver/v3_server.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "context" "encoding/base64" "encoding/binary" errorspkg "errors" "strconv" "time" "github.com/gogo/protobuf/proto" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" "golang.org/x/crypto/bcrypt" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" apply2 "go.etcd.io/etcd/server/v3/etcdserver/apply" "go.etcd.io/etcd/server/v3/etcdserver/errors" "go.etcd.io/etcd/server/v3/etcdserver/txn" "go.etcd.io/etcd/server/v3/features" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/lease/leasehttp" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/raft/v3" ) const ( // In the health case, there might be a small gap (10s of entries) between // the applied index and committed index. // However, if the committed entries are very heavy to toApply, the gap might grow. // We should stop accepting new proposals if the gap growing to a certain point. maxGapBetweenApplyAndCommitIndex = 5000 traceThreshold = 100 * time.Millisecond readIndexRetryTime = 500 * time.Millisecond // The timeout for the node to catch up its applied index, and is used in // lease related operations, such as LeaseRenew and LeaseTimeToLive. applyTimeout = time.Second ) type RaftKV interface { Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) Put(ctx context.Context, r *pb.PutRequest) (*pb.PutResponse, error) DeleteRange(ctx context.Context, r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse, error) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) } type Lessor interface { // LeaseGrant sends LeaseGrant request to raft and toApply it after committed. LeaseGrant(ctx context.Context, r *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) // LeaseRevoke sends LeaseRevoke request to raft and toApply it after committed. LeaseRevoke(ctx context.Context, r *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) // LeaseRenew renews the lease with given ID. The renewed TTL is returned. Or an error // is returned. LeaseRenew(ctx context.Context, id lease.LeaseID) (int64, error) // LeaseTimeToLive retrieves lease information. LeaseTimeToLive(ctx context.Context, r *pb.LeaseTimeToLiveRequest) (*pb.LeaseTimeToLiveResponse, error) // LeaseLeases lists all leases. LeaseLeases(ctx context.Context, r *pb.LeaseLeasesRequest) (*pb.LeaseLeasesResponse, error) } type Authenticator interface { AuthEnable(ctx context.Context, r *pb.AuthEnableRequest) (*pb.AuthEnableResponse, error) AuthDisable(ctx context.Context, r *pb.AuthDisableRequest) (*pb.AuthDisableResponse, error) AuthStatus(ctx context.Context, r *pb.AuthStatusRequest) (*pb.AuthStatusResponse, error) Authenticate(ctx context.Context, r *pb.AuthenticateRequest) (*pb.AuthenticateResponse, error) UserAdd(ctx context.Context, r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, error) UserDelete(ctx context.Context, r *pb.AuthUserDeleteRequest) (*pb.AuthUserDeleteResponse, error) UserChangePassword(ctx context.Context, r *pb.AuthUserChangePasswordRequest) (*pb.AuthUserChangePasswordResponse, error) UserGrantRole(ctx context.Context, r *pb.AuthUserGrantRoleRequest) (*pb.AuthUserGrantRoleResponse, error) UserGet(ctx context.Context, r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) UserRevokeRole(ctx context.Context, r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUserRevokeRoleResponse, error) RoleAdd(ctx context.Context, r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, error) RoleGrantPermission(ctx context.Context, r *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) RoleGet(ctx context.Context, r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) RoleRevokePermission(ctx context.Context, r *pb.AuthRoleRevokePermissionRequest) (*pb.AuthRoleRevokePermissionResponse, error) RoleDelete(ctx context.Context, r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDeleteResponse, error) UserList(ctx context.Context, r *pb.AuthUserListRequest) (*pb.AuthUserListResponse, error) RoleList(ctx context.Context, r *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error) } func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) { var span trace.Span ctx, span = traceutil.Tracer.Start(ctx, "range", trace.WithAttributes( attribute.String("range_begin", string(r.GetKey())), attribute.String("range_end", string(r.GetRangeEnd())), attribute.Int64("rev", r.GetRevision()), attribute.Int64("limit", r.GetLimit()), attribute.Bool("count_only", r.GetCountOnly()), attribute.Bool("keys_only", r.GetKeysOnly()), )) defer span.End() ctx, trace := traceutil.EnsureTrace(ctx, s.Logger(), "range", traceutil.Field{Key: "range_begin", Value: string(r.Key)}, traceutil.Field{Key: "range_end", Value: string(r.RangeEnd)}, ) var resp *pb.RangeResponse var err error defer func(start time.Time) { txn.WarnOfExpensiveReadOnlyRangeRequest(s.Logger(), s.Cfg.WarningApplyDuration, start, r, resp, err) if resp != nil { trace.AddField( traceutil.Field{Key: "response_count", Value: len(resp.Kvs)}, traceutil.Field{Key: "response_revision", Value: resp.Header.Revision}, ) } trace.LogIfLong(traceThreshold) success := err == nil requestDurationSec.WithLabelValues("Range", strconv.FormatBool(success)).Observe(time.Since(start).Seconds()) }(time.Now()) if !r.Serializable { err = s.linearizableReadNotify(ctx) trace.Step("agreement among raft nodes before linearized reading") if err != nil { return nil, err } } chk := func(ai *auth.AuthInfo) error { return s.authStore.IsRangePermitted(ai, r.Key, r.RangeEnd) } get := func() { resp, _, err = txn.Range(ctx, s.Logger(), s.KV(), r) } if serr := s.doSerialize(ctx, chk, get); serr != nil { err = serr return nil, err } return resp, err } func (s *EtcdServer) Put(ctx context.Context, r *pb.PutRequest) (*pb.PutResponse, error) { var span trace.Span ctx, span = traceutil.Tracer.Start(ctx, "put", trace.WithAttributes( attribute.String("key", string(r.GetKey())), )) defer span.End() ctx = context.WithValue(ctx, traceutil.StartTimeKey{}, time.Now()) resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{Put: r}) if err != nil { return nil, err } return resp.(*pb.PutResponse), nil } func (s *EtcdServer) DeleteRange(ctx context.Context, r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) { var span trace.Span ctx, span = traceutil.Tracer.Start(ctx, "delete_range", trace.WithAttributes( attribute.String("range_begin", string(r.GetKey())), attribute.String("range_end", string(r.GetRangeEnd())), )) defer span.End() resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{DeleteRange: r}) if err != nil { return nil, err } return resp.(*pb.DeleteRangeResponse), nil } func (s *EtcdServer) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse, error) { readOnly := txn.IsTxnReadonly(r) var span trace.Span ctx, span = traceutil.Tracer.Start(ctx, "txn", trace.WithAttributes( attribute.String("compare_first_key", firstCompareKey(r.GetCompare())), attribute.String("success_first_key", firstOpKey(r.GetSuccess())), attribute.String("success_first_type", firstOpType(r.GetSuccess())), attribute.Int64("success_first_lease", firstOpLease(r.GetSuccess())), attribute.Int("compare_len", len(r.GetCompare())), attribute.Int("success_len", len(r.GetSuccess())), attribute.Int("failure_len", len(r.GetFailure())), attribute.Bool("read_only", readOnly), )) defer span.End() ctx, trace := traceutil.EnsureTrace(ctx, s.Logger(), "transaction", traceutil.Field{Key: "read_only", Value: readOnly}, ) if readOnly { if !txn.IsTxnSerializable(r) { err := s.linearizableReadNotify(ctx) trace.Step("agreement among raft nodes before linearized reading") if err != nil { return nil, err } } var resp *pb.TxnResponse var err error chk := func(ai *auth.AuthInfo) error { return txn.CheckTxnAuth(s.authStore, ai, r) } defer func(start time.Time) { txn.WarnOfExpensiveReadOnlyTxnRequest(s.Logger(), s.Cfg.WarningApplyDuration, start, r, resp, err) trace.LogIfLong(traceThreshold) success := err == nil requestDurationSec.WithLabelValues("ReadonlyTxn", strconv.FormatBool(success)).Observe(time.Since(start).Seconds()) }(time.Now()) get := func() { resp, _, err = txn.Txn(ctx, s.Logger(), r, s.Cfg.ServerFeatureGate.Enabled(features.TxnModeWriteWithSharedBuffer), s.KV(), s.lessor) } if serr := s.doSerialize(ctx, chk, get); serr != nil { return nil, serr } return resp, err } ctx = context.WithValue(ctx, traceutil.StartTimeKey{}, time.Now()) resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{Txn: r}) if err != nil { return nil, err } return resp.(*pb.TxnResponse), nil } func (s *EtcdServer) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) { var span trace.Span ctx, span = traceutil.Tracer.Start(ctx, "compact", trace.WithAttributes( attribute.Bool("is_physical", r.GetPhysical()), attribute.Int64("rev", r.GetRevision()), )) defer span.End() startTime := time.Now() ctx, trace := traceutil.EnsureTrace(ctx, s.Logger(), "compact") result, err := s.processInternalRaftRequestOnce(ctx, pb.InternalRaftRequest{Compaction: r}) if result != nil && result.Trace != nil { trace = result.Trace defer func() { trace.LogIfLong(traceThreshold) }() applyStart := result.Trace.GetStartTime() result.Trace.SetStartTime(startTime) trace.InsertStep(0, applyStart, "process raft request") } if r.Physical && result != nil && result.Physc != nil { <-result.Physc // The compaction is done deleting keys; the hash is now settled // but the data is not necessarily committed. If there's a crash, // the hash may revert to a hash prior to compaction completing // if the compaction resumes. Force the finished compaction to // commit so it won't resume following a crash. // // `applySnapshot` sets a new backend instance, so we need to acquire the bemu lock. s.bemu.RLock() s.be.ForceCommit() s.bemu.RUnlock() trace.Step("physically toApply compaction") } if err != nil { return nil, err } if result.Err != nil { return nil, result.Err } resp := result.Resp.(*pb.CompactionResponse) if resp == nil { resp = &pb.CompactionResponse{} } if resp.Header == nil { resp.Header = &pb.ResponseHeader{} } resp.Header.Revision = s.kv.Rev() trace.AddField(traceutil.Field{Key: "response_revision", Value: resp.Header.Revision}) return resp, nil } func (s *EtcdServer) LeaseGrant(ctx context.Context, r *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { // no id given? choose one for r.ID == int64(lease.NoLease) { // only use positive int64 id's r.ID = int64(s.reqIDGen.Next() & ((1 << 63) - 1)) } var span trace.Span ctx, span = traceutil.Tracer.Start(ctx, "lease_grant", trace.WithAttributes( attribute.Int64("id", r.ID), attribute.Int64("ttl", r.GetTTL()), )) defer span.End() if err := s.requireAuthInfo(ctx); err != nil { return nil, err } resp, err := s.raftRequestOnce(ctx, pb.InternalRaftRequest{LeaseGrant: r}) if err != nil { return nil, err } return resp.(*pb.LeaseGrantResponse), nil } func (s *EtcdServer) waitAppliedIndex() error { select { case <-s.ApplyWait(): case <-s.stopping: return errors.ErrStopped case <-time.After(applyTimeout): return errors.ErrTimeoutWaitAppliedIndex } return nil } func (s *EtcdServer) LeaseRevoke(ctx context.Context, r *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) { var span trace.Span ctx, span = traceutil.Tracer.Start(ctx, "lease_revoke", trace.WithAttributes( attribute.Int64("id", r.GetID()), )) defer span.End() if err := s.requireAuthInfo(ctx); err != nil { return nil, err } resp, err := s.raftRequestOnce(ctx, pb.InternalRaftRequest{LeaseRevoke: r}) if err != nil { return nil, err } return resp.(*pb.LeaseRevokeResponse), nil } func (s *EtcdServer) LeaseRenew(ctx context.Context, id lease.LeaseID) (int64, error) { var span trace.Span ctx, span = traceutil.Tracer.Start(ctx, "lease_renew", trace.WithAttributes( attribute.Int64("id", int64(id)), )) defer span.End() if s.isLeader() { // If s.isLeader() returns true, but we fail to ensure the current // member's leadership, there are a couple of possibilities: // 1. current member gets stuck on writing WAL entries; // 2. current member is in network isolation status; // 3. current member isn't a leader anymore (possibly due to #1 above). // In such case, we just return error to client, so that the client can // switch to another member to continue the lease keep-alive operation. if !s.ensureLeadership() { return -1, lease.ErrNotPrimary } // This change aims to make lease renewal faster under high server load // while preserving correctness. If a lease is not found, it might still be in // the process of being created. We must wait for the applied index to advance // to verify whether the lease truly does not exist. if s.FeatureEnabled(features.FastLeaseKeepAlive) { le := s.lessor.Lookup(id) if le == nil { if err := s.waitAppliedIndex(); err != nil { return 0, err } } } else { if err := s.waitAppliedIndex(); err != nil { return 0, err } } if err := s.checkLeaseRenew(ctx, id); err != nil { return 0, err } ttl, err := s.lessor.Renew(id) if err == nil { // already requested to primary lessor(leader) return ttl, nil } if !errorspkg.Is(err, lease.ErrNotPrimary) { return -1, err } } cctx, cancel := context.WithTimeout(ctx, s.Cfg.ReqTimeout()) defer cancel() // renewals don't go through raft; forward to leader manually for cctx.Err() == nil { leader, lerr := s.waitLeader(cctx) if lerr != nil { return -1, lerr } if err := s.checkLeaseRenew(ctx, id); err != nil { return 0, err } for _, url := range leader.PeerURLs { lurl := url + leasehttp.LeasePrefix ttl, err := leasehttp.RenewHTTP(cctx, id, lurl, s.peerRt) if err == nil || errorspkg.Is(err, lease.ErrLeaseNotFound) { return ttl, err } } // Throttle in case of e.g. connection problems. time.Sleep(50 * time.Millisecond) } err := cctx.Err() switch { case errorspkg.Is(err, context.DeadlineExceeded): return -1, errors.ErrTimeout case errorspkg.Is(err, context.Canceled): return -1, errors.ErrCanceled default: s.Logger().Warn("Unexpected lease renew context error", zap.Error(err)) return -1, errors.ErrCanceled } } func (s *EtcdServer) checkLeaseRenew(ctx context.Context, leaseID lease.LeaseID) error { rev := s.AuthStore().Revision() if !s.AuthStore().IsAuthEnabled() { return nil } authInfo, err := s.AuthInfoFromCtx(ctx) if err != nil { return err } if authInfo == nil { return auth.ErrUserEmpty } if s.AuthStore().IsAdminPermitted(authInfo) == nil { return nil } l := s.lessor.Lookup(leaseID) if l != nil { for _, key := range l.Keys() { if err := s.AuthStore().IsPutPermitted(authInfo, []byte(key)); err != nil { return err } } } if rev != s.AuthStore().Revision() { return auth.ErrAuthOldRevision } return nil } func (s *EtcdServer) checkLeaseTimeToLive(ctx context.Context, leaseID lease.LeaseID) (uint64, error) { rev := s.AuthStore().Revision() if !s.AuthStore().IsAuthEnabled() { return rev, nil } authInfo, err := s.AuthInfoFromCtx(ctx) if err != nil { return rev, err } if authInfo == nil { return rev, auth.ErrUserEmpty } if s.AuthStore().IsAdminPermitted(authInfo) == nil { return rev, nil } l := s.lessor.Lookup(leaseID) if l != nil { for _, key := range l.Keys() { if err := s.AuthStore().IsRangePermitted(authInfo, []byte(key), []byte{}); err != nil { return 0, err } } } return rev, nil } func (s *EtcdServer) leaseTimeToLive(ctx context.Context, r *pb.LeaseTimeToLiveRequest) (*pb.LeaseTimeToLiveResponse, error) { if s.isLeader() { if err := s.waitAppliedIndex(); err != nil { return nil, err } // gofail: var beforeLookupWhenLeaseTimeToLive struct{} // primary; timetolive directly from leader le := s.lessor.Lookup(lease.LeaseID(r.ID)) if le == nil { return nil, lease.ErrLeaseNotFound } // TODO: fill out ResponseHeader resp := &pb.LeaseTimeToLiveResponse{Header: &pb.ResponseHeader{}, ID: r.ID, TTL: int64(le.Remaining().Seconds()), GrantedTTL: le.TTL()} if r.Keys { ks := le.Keys() kbs := make([][]byte, len(ks)) for i := range ks { kbs[i] = []byte(ks[i]) } resp.Keys = kbs } // The leasor could be demoted if leader changed during lookup. // We should return error to force retry instead of returning // incorrect remaining TTL. if le.Demoted() { // NOTE: lease.ErrNotPrimary is not retryable error for // client. Instead, uses ErrLeaderChanged. return nil, errors.ErrLeaderChanged } return resp, nil } cctx, cancel := context.WithTimeout(ctx, s.Cfg.ReqTimeout()) defer cancel() // forward to leader for cctx.Err() == nil { leader, err := s.waitLeader(cctx) if err != nil { return nil, err } for _, url := range leader.PeerURLs { lurl := url + leasehttp.LeaseInternalPrefix resp, err := leasehttp.TimeToLiveHTTP(cctx, lease.LeaseID(r.ID), r.Keys, lurl, s.peerRt) if err == nil { return resp.LeaseTimeToLiveResponse, nil } if errorspkg.Is(err, lease.ErrLeaseNotFound) { return nil, err } } } if errorspkg.Is(cctx.Err(), context.DeadlineExceeded) { return nil, errors.ErrTimeout } return nil, errors.ErrCanceled } func (s *EtcdServer) LeaseTimeToLive(ctx context.Context, r *pb.LeaseTimeToLiveRequest) (*pb.LeaseTimeToLiveResponse, error) { if err := s.requireAuthInfo(ctx); err != nil { return nil, err } var rev uint64 var err error if r.Keys { // check RBAC permission only if Keys is true rev, err = s.checkLeaseTimeToLive(ctx, lease.LeaseID(r.ID)) if err != nil { return nil, err } } resp, err := s.leaseTimeToLive(ctx, r) if err != nil { return nil, err } if r.Keys { if s.AuthStore().IsAuthEnabled() && rev != s.AuthStore().Revision() { return nil, auth.ErrAuthOldRevision } } return resp, nil } func (s *EtcdServer) newHeader() *pb.ResponseHeader { return &pb.ResponseHeader{ ClusterId: uint64(s.cluster.ID()), MemberId: uint64(s.MemberID()), Revision: s.KV().Rev(), RaftTerm: s.Term(), } } // LeaseLeases is really ListLeases !??? func (s *EtcdServer) LeaseLeases(ctx context.Context, _ *pb.LeaseLeasesRequest) (*pb.LeaseLeasesResponse, error) { ls := s.lessor.Leases() if err := s.checkLeaseLeases(ctx, ls); err != nil { return nil, err } lss := make([]*pb.LeaseStatus, len(ls)) for i := range ls { lss[i] = &pb.LeaseStatus{ID: int64(ls[i].ID)} } return &pb.LeaseLeasesResponse{Header: s.newHeader(), Leases: lss}, nil } func (s *EtcdServer) checkLeaseLeases(ctx context.Context, leases []*lease.Lease) error { rev := s.AuthStore().Revision() if !s.AuthStore().IsAuthEnabled() { return nil } authInfo, err := s.AuthInfoFromCtx(ctx) if err != nil { return err } if authInfo == nil { return auth.ErrUserEmpty } if err := s.AuthStore().IsAdminPermitted(authInfo); err == nil { return nil } for _, l := range leases { for _, key := range l.Keys() { if err := s.AuthStore().IsRangePermitted(authInfo, []byte(key), []byte{}); err != nil { return err } } } if rev != s.AuthStore().Revision() { return auth.ErrAuthOldRevision } return nil } func (s *EtcdServer) waitLeader(ctx context.Context) (*membership.Member, error) { leader := s.cluster.Member(s.Leader()) for leader == nil { // wait an election dur := time.Duration(s.Cfg.ElectionTicks) * time.Duration(s.Cfg.TickMs) * time.Millisecond select { case <-time.After(dur): leader = s.cluster.Member(s.Leader()) case <-s.stopping: return nil, errors.ErrStopped case <-ctx.Done(): return nil, errors.ErrNoLeader } } if len(leader.PeerURLs) == 0 { return nil, errors.ErrNoLeader } return leader, nil } func (s *EtcdServer) Alarm(ctx context.Context, r *pb.AlarmRequest) (*pb.AlarmResponse, error) { resp, err := s.raftRequestOnce(ctx, pb.InternalRaftRequest{Alarm: r}) if err != nil { return nil, err } return resp.(*pb.AlarmResponse), nil } func (s *EtcdServer) AuthEnable(ctx context.Context, r *pb.AuthEnableRequest) (*pb.AuthEnableResponse, error) { resp, err := s.raftRequestOnce(ctx, pb.InternalRaftRequest{AuthEnable: r}) if err != nil { return nil, err } return resp.(*pb.AuthEnableResponse), nil } func (s *EtcdServer) AuthDisable(ctx context.Context, r *pb.AuthDisableRequest) (*pb.AuthDisableResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthDisable: r}) if err != nil { return nil, err } return resp.(*pb.AuthDisableResponse), nil } func (s *EtcdServer) AuthStatus(ctx context.Context, r *pb.AuthStatusRequest) (*pb.AuthStatusResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthStatus: r}) if err != nil { return nil, err } return resp.(*pb.AuthStatusResponse), nil } func (s *EtcdServer) Authenticate(ctx context.Context, r *pb.AuthenticateRequest) (*pb.AuthenticateResponse, error) { if err := s.linearizableReadNotify(ctx); err != nil { return nil, err } lg := s.Logger() // fix https://nvd.nist.gov/vuln/detail/CVE-2021-28235 defer func() { if r != nil { r.Password = "" } }() var resp proto.Message for { checkedRevision, err := s.AuthStore().CheckPassword(r.Name, r.Password) if err != nil { if !errorspkg.Is(err, auth.ErrAuthNotEnabled) { lg.Warn( "invalid authentication was requested", zap.String("user", r.Name), zap.Error(err), ) } return nil, err } st, err := s.AuthStore().GenTokenPrefix() if err != nil { return nil, err } // internalReq doesn't need to have Password because the above s.AuthStore().CheckPassword() already did it. // In addition, it will let a WAL entry not record password as a plain text. internalReq := &pb.InternalAuthenticateRequest{ Name: r.Name, SimpleToken: st, } resp, err = s.raftRequestOnce(ctx, pb.InternalRaftRequest{Authenticate: internalReq}) if err != nil { return nil, err } if checkedRevision == s.AuthStore().Revision() { break } lg.Info("revision when password checked became stale; retrying") } return resp.(*pb.AuthenticateResponse), nil } func (s *EtcdServer) UserAdd(ctx context.Context, r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, error) { if r.Options == nil || !r.Options.NoPassword { hashedPassword, err := bcrypt.GenerateFromPassword([]byte(r.Password), s.authStore.BcryptCost()) if err != nil { return nil, err } r.HashedPassword = base64.StdEncoding.EncodeToString(hashedPassword) r.Password = "" } resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthUserAdd: r}) if err != nil { return nil, err } return resp.(*pb.AuthUserAddResponse), nil } func (s *EtcdServer) UserDelete(ctx context.Context, r *pb.AuthUserDeleteRequest) (*pb.AuthUserDeleteResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthUserDelete: r}) if err != nil { return nil, err } return resp.(*pb.AuthUserDeleteResponse), nil } func (s *EtcdServer) UserChangePassword(ctx context.Context, r *pb.AuthUserChangePasswordRequest) (*pb.AuthUserChangePasswordResponse, error) { if r.Password != "" { hashedPassword, err := bcrypt.GenerateFromPassword([]byte(r.Password), s.authStore.BcryptCost()) if err != nil { return nil, err } r.HashedPassword = base64.StdEncoding.EncodeToString(hashedPassword) r.Password = "" } resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthUserChangePassword: r}) if err != nil { return nil, err } return resp.(*pb.AuthUserChangePasswordResponse), nil } func (s *EtcdServer) UserGrantRole(ctx context.Context, r *pb.AuthUserGrantRoleRequest) (*pb.AuthUserGrantRoleResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthUserGrantRole: r}) if err != nil { return nil, err } return resp.(*pb.AuthUserGrantRoleResponse), nil } func (s *EtcdServer) UserGet(ctx context.Context, r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthUserGet: r}) if err != nil { return nil, err } return resp.(*pb.AuthUserGetResponse), nil } func (s *EtcdServer) UserList(ctx context.Context, r *pb.AuthUserListRequest) (*pb.AuthUserListResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthUserList: r}) if err != nil { return nil, err } return resp.(*pb.AuthUserListResponse), nil } func (s *EtcdServer) UserRevokeRole(ctx context.Context, r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUserRevokeRoleResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthUserRevokeRole: r}) if err != nil { return nil, err } return resp.(*pb.AuthUserRevokeRoleResponse), nil } func (s *EtcdServer) RoleAdd(ctx context.Context, r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthRoleAdd: r}) if err != nil { return nil, err } return resp.(*pb.AuthRoleAddResponse), nil } func (s *EtcdServer) RoleGrantPermission(ctx context.Context, r *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthRoleGrantPermission: r}) if err != nil { return nil, err } return resp.(*pb.AuthRoleGrantPermissionResponse), nil } func (s *EtcdServer) RoleGet(ctx context.Context, r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthRoleGet: r}) if err != nil { return nil, err } return resp.(*pb.AuthRoleGetResponse), nil } func (s *EtcdServer) RoleList(ctx context.Context, r *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthRoleList: r}) if err != nil { return nil, err } return resp.(*pb.AuthRoleListResponse), nil } func (s *EtcdServer) RoleRevokePermission(ctx context.Context, r *pb.AuthRoleRevokePermissionRequest) (*pb.AuthRoleRevokePermissionResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthRoleRevokePermission: r}) if err != nil { return nil, err } return resp.(*pb.AuthRoleRevokePermissionResponse), nil } func (s *EtcdServer) RoleDelete(ctx context.Context, r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDeleteResponse, error) { resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{AuthRoleDelete: r}) if err != nil { return nil, err } return resp.(*pb.AuthRoleDeleteResponse), nil } func (s *EtcdServer) raftRequestOnce(ctx context.Context, r pb.InternalRaftRequest) (proto.Message, error) { result, err := s.processInternalRaftRequestOnce(ctx, r) if err != nil { trace.SpanFromContext(ctx).RecordError(err) return nil, err } if result.Err != nil { return nil, result.Err } if startTime, ok := ctx.Value(traceutil.StartTimeKey{}).(time.Time); ok && result.Trace != nil { applyStart := result.Trace.GetStartTime() // The trace object is created in toApply. Here reset the start time to trace // the raft request time by the difference between the request start time // and toApply start time result.Trace.SetStartTime(startTime) result.Trace.InsertStep(0, applyStart, "process raft request") result.Trace.LogIfLong(traceThreshold) } return result.Resp, nil } func (s *EtcdServer) raftRequest(ctx context.Context, r pb.InternalRaftRequest) (proto.Message, error) { return s.raftRequestOnce(ctx, r) } // doSerialize handles the auth logic, with permissions checked by "chk", for a serialized request "get". Returns a non-nil error on authentication failure. func (s *EtcdServer) doSerialize(ctx context.Context, chk func(*auth.AuthInfo) error, get func()) error { trace := traceutil.Get(ctx) ai, err := s.AuthInfoFromCtx(ctx) if err != nil { return err } if ai == nil { // chk expects non-nil AuthInfo; use empty credentials ai = &auth.AuthInfo{} } if err = chk(ai); err != nil { return err } trace.Step("get authentication metadata") // fetch response for serialized request get() // check for stale token revision in case the auth store was updated while // the request has been handled. if ai.Revision != 0 && ai.Revision != s.authStore.Revision() { return auth.ErrAuthOldRevision } return nil } func (s *EtcdServer) processInternalRaftRequestOnce(ctx context.Context, r pb.InternalRaftRequest) (*apply2.Result, error) { ai := s.getAppliedIndex() ci := s.getCommittedIndex() if ci > ai+maxGapBetweenApplyAndCommitIndex { return nil, errors.ErrTooManyRequests } r.Header = &pb.RequestHeader{ ID: s.reqIDGen.Next(), } // check authinfo if it is not InternalAuthenticateRequest if r.Authenticate == nil { authInfo, err := s.AuthInfoFromCtx(ctx) if err != nil { return nil, err } if authInfo != nil { r.Header.Username = authInfo.Username r.Header.AuthRevision = authInfo.Revision } } var ( data []byte err error start = time.Now() reqType = getRequestType(&r) ) defer func() { success := err == nil requestDurationSec.WithLabelValues(reqType, strconv.FormatBool(success)).Observe(time.Since(start).Seconds()) }() data, err = r.Marshal() if err != nil { return nil, err } if len(data) > int(s.Cfg.MaxRequestBytes) { return nil, errors.ErrRequestTooLarge } id := r.ID if id == 0 { id = r.Header.ID } ch := s.w.Register(id) cctx, cancel := context.WithTimeout(ctx, s.Cfg.ReqTimeout()) defer cancel() span := trace.SpanFromContext(ctx) span.AddEvent("Send raft proposal") err = s.r.Propose(cctx, data) if err != nil { proposalsFailed.Inc() s.w.Trigger(id, nil) // GC wait return nil, err } proposalsPending.Inc() defer proposalsPending.Dec() select { case x := <-ch: span.AddEvent("Receive raft result") return x.(*apply2.Result), nil case <-cctx.Done(): proposalsFailed.Inc() s.w.Trigger(id, nil) // GC wait return nil, s.parseProposeCtxErr(cctx.Err(), start) case <-s.done: return nil, errors.ErrStopped } } func getRequestType(r *pb.InternalRaftRequest) string { switch { case r.Range != nil: return "Range" case r.Put != nil: return "Put" case r.DeleteRange != nil: return "DeleteRange" case r.Txn != nil: return "Txn" case r.Compaction != nil: return "Compaction" case r.LeaseGrant != nil: return "LeaseGrant" case r.LeaseRevoke != nil: return "LeaseRevoke" case r.LeaseCheckpoint != nil: return "LeaseCheckpoint" case r.Alarm != nil: return "Alarm" case r.Authenticate != nil: return "Authenticate" case r.AuthEnable != nil: return "AuthEnable" case r.AuthDisable != nil: return "AuthDisable" case r.AuthStatus != nil: return "AuthStatus" case r.AuthUserAdd != nil: return "AuthUserAdd" case r.AuthUserDelete != nil: return "AuthUserDelete" case r.AuthUserChangePassword != nil: return "AuthUserChangePassword" case r.AuthUserGrantRole != nil: return "AuthUserGrantRole" case r.AuthUserGet != nil: return "AuthUserGet" case r.AuthUserRevokeRole != nil: return "AuthUserRevokeRole" case r.AuthRoleAdd != nil: return "AuthRoleAdd" case r.AuthRoleGrantPermission != nil: return "AuthRoleGrantPermission" case r.AuthRoleGet != nil: return "AuthRoleGet" case r.AuthRoleRevokePermission != nil: return "AuthRoleRevokePermission" case r.AuthRoleDelete != nil: return "AuthRoleDelete" case r.AuthUserList != nil: return "AuthUserList" case r.AuthRoleList != nil: return "AuthRoleList" case r.ClusterVersionSet != nil: return "ClusterVersionSet" case r.ClusterMemberAttrSet != nil: return "ClusterMemberAttrSet" case r.DowngradeInfoSet != nil: return "DowngradeInfoSet" case r.DowngradeVersionTest != nil: return "DowngradeVersionTest" default: return "Unknown" } } // Watchable returns a watchable interface attached to the etcdserver. func (s *EtcdServer) Watchable() mvcc.WatchableKV { return s.KV() } func (s *EtcdServer) linearizableReadLoop() { for { leaderChangedNotifier := s.leaderChanged.Receive() select { case <-leaderChangedNotifier: continue case <-s.readwaitc: case <-s.stopping: return } // as a single loop is can unlock multiple reads, it is not very useful // to propagate the trace from Txn or Range. _, trace := traceutil.EnsureTrace(context.Background(), s.Logger(), "linearizableReadLoop") nextnr := newNotifier() s.readMu.Lock() nr := s.readNotifier s.readNotifier = nextnr s.readMu.Unlock() confirmedIndex, err := s.requestCurrentIndex(leaderChangedNotifier) if isStopped(err) { return } if err != nil { nr.notify(err) continue } trace.Step("read index received") trace.AddField(traceutil.Field{Key: "readStateIndex", Value: confirmedIndex}) appliedIndex := s.getAppliedIndex() trace.AddField(traceutil.Field{Key: "appliedIndex", Value: strconv.FormatUint(appliedIndex, 10)}) if appliedIndex < confirmedIndex { select { case <-s.applyWait.Wait(confirmedIndex): case <-s.stopping: return } } // unblock all l-reads requested at indices before confirmedIndex nr.notify(nil) trace.Step("applied index is now lower than readState.Index") trace.LogAllStepsIfLong(traceThreshold) } } func isStopped(err error) bool { return errorspkg.Is(err, raft.ErrStopped) || errorspkg.Is(err, errors.ErrStopped) } func (s *EtcdServer) requestCurrentIndex(leaderChangedNotifier <-chan struct{}) (uint64, error) { requestIDs := map[uint64]struct{}{} requestID := s.reqIDGen.Next() requestIDs[requestID] = struct{}{} err := s.sendReadIndex(requestID) if err != nil { return 0, err } lg := s.Logger() errorTimer := time.NewTimer(s.Cfg.ReqTimeout()) defer errorTimer.Stop() retryTimer := time.NewTimer(readIndexRetryTime) defer retryTimer.Stop() firstCommitInTermNotifier := s.firstCommitInTerm.Receive() for { select { case rs := <-s.r.readStateC: // Check again if leader changed as when multiple channels are ready, select picks randomly. select { case <-leaderChangedNotifier: readIndexFailed.Inc() return 0, errors.ErrLeaderChanged default: } responseID := uint64(0) if len(rs.RequestCtx) == 8 { responseID = binary.BigEndian.Uint64(rs.RequestCtx) } if _, ok := requestIDs[responseID]; !ok { // a previous request might time out. now we should ignore the response of it and // continue waiting for the response of the current requests. lg.Warn( "ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader", zap.Uint64("received-request-id", responseID), ) slowReadIndex.Inc() continue } return rs.Index, nil case <-leaderChangedNotifier: readIndexFailed.Inc() // return a retryable error. return 0, errors.ErrLeaderChanged case <-firstCommitInTermNotifier: firstCommitInTermNotifier = s.firstCommitInTerm.Receive() lg.Info("first commit in current term: resending ReadIndex request") requestID = s.reqIDGen.Next() requestIDs[requestID] = struct{}{} err := s.sendReadIndex(requestID) if err != nil { return 0, err } retryTimer.Reset(readIndexRetryTime) continue case <-retryTimer.C: lg.Warn( "waiting for ReadIndex response took too long, retrying", zap.Uint64("sent-request-id", requestID), zap.Duration("retry-timeout", readIndexRetryTime), ) requestID = s.reqIDGen.Next() requestIDs[requestID] = struct{}{} err := s.sendReadIndex(requestID) if err != nil { return 0, err } retryTimer.Reset(readIndexRetryTime) continue case <-errorTimer.C: lg.Warn( "timed out waiting for read index response (local node might have slow network)", zap.Duration("timeout", s.Cfg.ReqTimeout()), ) slowReadIndex.Inc() return 0, errors.ErrTimeout case <-s.stopping: return 0, errors.ErrStopped } } } func uint64ToBigEndianBytes(number uint64) []byte { byteResult := make([]byte, 8) binary.BigEndian.PutUint64(byteResult, number) return byteResult } func (s *EtcdServer) sendReadIndex(requestIndex uint64) error { ctxToSend := uint64ToBigEndianBytes(requestIndex) cctx, cancel := context.WithTimeout(context.Background(), s.Cfg.ReqTimeout()) err := s.r.ReadIndex(cctx, ctxToSend) cancel() if errorspkg.Is(err, raft.ErrStopped) { return err } if err != nil { lg := s.Logger() lg.Warn("failed to get read index from Raft", zap.Error(err)) readIndexFailed.Inc() return err } return nil } func (s *EtcdServer) LinearizableReadNotify(ctx context.Context) error { return s.linearizableReadNotify(ctx) } func (s *EtcdServer) linearizableReadNotify(ctx context.Context) error { s.readMu.RLock() nc := s.readNotifier s.readMu.RUnlock() // signal linearizable loop for current notify if it hasn't been already select { case s.readwaitc <- struct{}{}: default: } // wait for read state notification select { case <-nc.c: return nc.err case <-ctx.Done(): return ctx.Err() case <-s.done: return errors.ErrStopped } } func (s *EtcdServer) AuthInfoFromCtx(ctx context.Context) (*auth.AuthInfo, error) { authInfo, err := s.AuthStore().AuthInfoFromCtx(ctx) if authInfo != nil || err != nil { return authInfo, err } if !s.Cfg.ClientCertAuthEnabled { return nil, nil } authInfo = s.AuthStore().AuthInfoFromTLS(ctx) return authInfo, nil } func (s *EtcdServer) Downgrade(ctx context.Context, r *pb.DowngradeRequest) (*pb.DowngradeResponse, error) { switch r.Action { case pb.DowngradeRequest_VALIDATE: return s.downgradeValidate(ctx, r.Version) case pb.DowngradeRequest_ENABLE: return s.downgradeEnable(ctx, r) case pb.DowngradeRequest_CANCEL: return s.downgradeCancel(ctx) default: return nil, errors.ErrUnknownMethod } } func (s *EtcdServer) downgradeValidate(ctx context.Context, v string) (*pb.DowngradeResponse, error) { resp := &pb.DowngradeResponse{} targetVersion, err := convertToClusterVersion(v) if err != nil { return nil, err } cv := s.ClusterVersion() if cv == nil { return nil, errors.ErrClusterVersionUnavailable } resp.Version = version.Cluster(cv.String()) err = s.Version().DowngradeValidate(ctx, targetVersion) if err != nil { return nil, err } return resp, nil } func (s *EtcdServer) downgradeEnable(ctx context.Context, r *pb.DowngradeRequest) (*pb.DowngradeResponse, error) { lg := s.Logger() targetVersion, err := convertToClusterVersion(r.Version) if err != nil { lg.Warn("reject downgrade request", zap.Error(err)) return nil, err } err = s.Version().DowngradeEnable(ctx, targetVersion) if err != nil { lg.Warn("reject downgrade request", zap.Error(err)) return nil, err } resp := pb.DowngradeResponse{Version: version.Cluster(s.ClusterVersion().String())} return &resp, nil } func (s *EtcdServer) downgradeCancel(ctx context.Context) (*pb.DowngradeResponse, error) { err := s.Version().DowngradeCancel(ctx) if err != nil { s.lg.Warn("failed to cancel downgrade", zap.Error(err)) } resp := pb.DowngradeResponse{Version: version.Cluster(s.ClusterVersion().String())} return &resp, nil } func (s *EtcdServer) requireAuthInfo(ctx context.Context) error { if !s.authStore.IsAuthEnabled() { return nil } authInfo, err := s.AuthInfoFromCtx(ctx) if err != nil { return err } if authInfo == nil { return auth.ErrUserEmpty } return nil } ================================================ FILE: server/etcdserver/version/doc.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package version provides functions for getting/saving storage version. package version ================================================ FILE: server/etcdserver/version/downgrade.go ================================================ // Copyright 2020 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package version import ( "github.com/coreos/go-semver/semver" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/version" ) type DowngradeInfo struct { // TargetVersion is the target downgrade version, if the cluster is not under downgrading, // the targetVersion will be an empty string TargetVersion string `json:"target-version"` // Enabled indicates whether the cluster is enabled to downgrade Enabled bool `json:"enabled"` } func (d *DowngradeInfo) GetTargetVersion() *semver.Version { return semver.Must(semver.NewVersion(d.TargetVersion)) } // isValidDowngrade verifies whether the cluster can be downgraded from verFrom to verTo func isValidDowngrade(verFrom *semver.Version, verTo *semver.Version) bool { return verTo.Equal(*allowedDowngradeVersion(verFrom)) } // MustDetectDowngrade will detect local server joining cluster that doesn't support it's version. func MustDetectDowngrade(lg *zap.Logger, sv, cv *semver.Version) { // only keep major.minor version for comparison against cluster version sv = &semver.Version{Major: sv.Major, Minor: sv.Minor} // if the cluster disables downgrade, check local version against determined cluster version. // the validation passes when local version is not less than cluster version if cv != nil && sv.LessThan(*cv) { lg.Panic( "invalid downgrade; server version is lower than determined cluster version", zap.String("current-server-version", sv.String()), zap.String("determined-cluster-version", version.Cluster(cv.String())), ) } } func allowedDowngradeVersion(ver *semver.Version) *semver.Version { // Todo: handle the case that downgrading from higher major version(e.g. downgrade from v4.0 to v3.x) return &semver.Version{Major: ver.Major, Minor: ver.Minor - 1} } // IsValidClusterVersionChange checks the two scenario when version is valid to change: // 1. Downgrade: cluster version is 1 minor version higher than local version, // cluster version should change. // 2. Cluster start: when not all members version are available, cluster version // is set to MinVersion(3.0), when all members are at higher version, cluster version // is lower than minimal server version, cluster version should change func IsValidClusterVersionChange(verFrom *semver.Version, verTo *semver.Version) bool { verFrom = &semver.Version{Major: verFrom.Major, Minor: verFrom.Minor} verTo = &semver.Version{Major: verTo.Major, Minor: verTo.Minor} if isValidDowngrade(verFrom, verTo) || (verFrom.Major == verTo.Major && verFrom.LessThan(*verTo)) { return true } return false } ================================================ FILE: server/etcdserver/version/downgrade_test.go ================================================ // Copyright 2020 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package version import ( "fmt" "testing" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" "go.uber.org/zap" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/version" ) func TestMustDetectDowngrade(t *testing.T) { lv := semver.Must(semver.NewVersion(version.Version)) lv = &semver.Version{Major: lv.Major, Minor: lv.Minor} oneMinorHigher := &semver.Version{Major: lv.Major, Minor: lv.Minor + 1} oneMinorLower := &semver.Version{Major: lv.Major, Minor: lv.Minor - 1} tests := []struct { name string clusterVersion *semver.Version success bool message string }{ { "Succeeded when cluster version is nil", nil, true, "", }, { "Succeeded when cluster version is one minor lower", oneMinorLower, true, "", }, { "Succeeded when cluster version is server version", lv, true, "", }, { "Failed when server version is lower than determined cluster version ", oneMinorHigher, false, "invalid downgrade; server version is lower than determined cluster version", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { lg := zaptest.NewLogger(t) sv := semver.Must(semver.NewVersion(version.Version)) err := tryMustDetectDowngrade(lg, sv, tt.clusterVersion) if tt.success != (err == nil) { t.Errorf("Unexpected success, got: %v, wanted: %v", err == nil, tt.success) // TODO test err } if err != nil && tt.message != fmt.Sprintf("%s", err) { t.Errorf("Unexpected message, got %q, wanted: %v", err, tt.message) } }) } } func tryMustDetectDowngrade(lg *zap.Logger, sv, cv *semver.Version) (err any) { defer func() { err = recover() }() MustDetectDowngrade(lg, sv, cv) return err } func TestIsValidDowngrade(t *testing.T) { tests := []struct { name string verFrom string verTo string result bool }{ { "Valid downgrade", "3.5.0", "3.4.0", true, }, { "Invalid downgrade", "3.5.2", "3.3.0", false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { res := isValidDowngrade( semver.Must(semver.NewVersion(tt.verFrom)), semver.Must(semver.NewVersion(tt.verTo))) if res != tt.result { t.Errorf("Expected downgrade valid is %v; Got %v", tt.result, res) } }) } } func TestIsVersionChangable(t *testing.T) { tests := []struct { name string verFrom string verTo string expectedResult bool }{ { name: "When local version is one minor lower than cluster version", verFrom: "3.5.0", verTo: "3.4.0", expectedResult: true, }, { name: "When local version is one minor and one patch lower than cluster version", verFrom: "3.5.1", verTo: "3.4.0", expectedResult: true, }, { name: "When local version is one minor higher than cluster version", verFrom: "3.4.0", verTo: "3.5.0", expectedResult: true, }, { name: "When local version is two minor higher than cluster version", verFrom: "3.4.0", verTo: "3.6.0", expectedResult: true, }, { name: "When local version is one major higher than cluster version", verFrom: "2.4.0", verTo: "3.4.0", expectedResult: false, }, { name: "When local version is equal to cluster version", verFrom: "3.4.0", verTo: "3.4.0", expectedResult: false, }, { name: "When local version is one patch higher than cluster version", verFrom: "3.5.0", verTo: "3.5.1", expectedResult: false, }, { name: "When local version is two minor lower than cluster version", verFrom: "3.6.0", verTo: "3.4.0", expectedResult: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { verFrom := semver.Must(semver.NewVersion(tt.verFrom)) verTo := semver.Must(semver.NewVersion(tt.verTo)) ret := IsValidClusterVersionChange(verFrom, verTo) assert.Equal(t, tt.expectedResult, ret) }) } } ================================================ FILE: server/etcdserver/version/errors.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package version import "errors" var ( ErrInvalidDowngradeTargetVersion = errors.New("etcdserver: invalid downgrade target version") ErrDowngradeInProcess = errors.New("etcdserver: cluster has a downgrade job in progress") ErrNoInflightDowngrade = errors.New("etcdserver: no inflight downgrade job") ) ================================================ FILE: server/etcdserver/version/monitor.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package version import ( "context" "errors" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/version" ) // Monitor contains logic used by cluster leader to monitor version changes and decide on cluster version or downgrade progress. type Monitor struct { lg *zap.Logger s Server } // Server lists EtcdServer methods needed by Monitor type Server interface { GetClusterVersion() *semver.Version GetDowngradeInfo() *DowngradeInfo GetMembersVersions() map[string]*version.Versions UpdateClusterVersion(string) LinearizableReadNotify(ctx context.Context) error DowngradeEnable(ctx context.Context, targetVersion *semver.Version) error DowngradeCancel(ctx context.Context) error GetStorageVersion() *semver.Version UpdateStorageVersion(semver.Version) error } func NewMonitor(lg *zap.Logger, storage Server) *Monitor { return &Monitor{ lg: lg, s: storage, } } // UpdateClusterVersionIfNeeded updates the cluster version. func (m *Monitor) UpdateClusterVersionIfNeeded() error { newClusterVersion, err := m.decideClusterVersion() if newClusterVersion != nil { newClusterVersion = &semver.Version{Major: newClusterVersion.Major, Minor: newClusterVersion.Minor} m.s.UpdateClusterVersion(newClusterVersion.String()) } return err } // decideClusterVersion decides whether to change cluster version and its next value. // New cluster version is based on the members versions server and whether cluster is downgrading. // Returns nil if cluster version should be left unchanged. func (m *Monitor) decideClusterVersion() (*semver.Version, error) { clusterVersion := m.s.GetClusterVersion() minimalServerVersion := m.membersMinimalServerVersion() if clusterVersion == nil { if minimalServerVersion != nil { return minimalServerVersion, nil } return semver.New(version.MinClusterVersion), nil } if minimalServerVersion == nil { return nil, nil } downgrade := m.s.GetDowngradeInfo() if downgrade != nil && downgrade.Enabled { if downgrade.GetTargetVersion().Equal(*clusterVersion) { return nil, nil } if !isValidDowngrade(clusterVersion, downgrade.GetTargetVersion()) { m.lg.Error("Cannot downgrade from cluster-version to downgrade-target", zap.String("downgrade-target", downgrade.TargetVersion), zap.String("cluster-version", clusterVersion.String()), ) return nil, errors.New("invalid downgrade target") } if !isValidDowngrade(minimalServerVersion, downgrade.GetTargetVersion()) { m.lg.Error("Cannot downgrade from minimal-server-version to downgrade-target", zap.String("downgrade-target", downgrade.TargetVersion), zap.String("minimal-server-version", minimalServerVersion.String()), ) return nil, errors.New("invalid downgrade target") } return downgrade.GetTargetVersion(), nil } if clusterVersion.LessThan(*minimalServerVersion) && IsValidClusterVersionChange(clusterVersion, minimalServerVersion) { return minimalServerVersion, nil } return nil, nil } // UpdateStorageVersionIfNeeded updates the storage version if it differs from cluster version. func (m *Monitor) UpdateStorageVersionIfNeeded() { cv := m.s.GetClusterVersion() if cv == nil || cv.String() == version.MinClusterVersion { return } sv := m.s.GetStorageVersion() if sv == nil || sv.Major != cv.Major || sv.Minor != cv.Minor { if sv != nil { m.lg.Info("cluster version differs from storage version.", zap.String("cluster-version", cv.String()), zap.String("storage-version", sv.String())) } err := m.s.UpdateStorageVersion(semver.Version{Major: cv.Major, Minor: cv.Minor}) if err != nil { m.lg.Error("failed to update storage version", zap.String("cluster-version", cv.String()), zap.Error(err)) return } d := m.s.GetDowngradeInfo() if d != nil && d.Enabled { m.lg.Info( "The server is ready to downgrade", zap.String("target-version", d.TargetVersion), zap.String("server-version", version.Version), ) } } } func (m *Monitor) CancelDowngradeIfNeeded() { d := m.s.GetDowngradeInfo() if d == nil || !d.Enabled { return } targetVersion := d.TargetVersion v := semver.Must(semver.NewVersion(targetVersion)) if m.versionsMatchTarget(v) { m.lg.Info("the cluster has been downgraded", zap.String("cluster-version", targetVersion)) err := m.s.DowngradeCancel(context.Background()) if err != nil { m.lg.Warn("failed to cancel downgrade", zap.Error(err)) } } } // membersMinimalServerVersion returns the min server version in the map, or nil if the min // version in unknown. // It prints out log if there is a member with a higher version than the // local version. func (m *Monitor) membersMinimalServerVersion() *semver.Version { vers := m.s.GetMembersVersions() var minV *semver.Version lv := semver.Must(semver.NewVersion(version.Version)) for mid, ver := range vers { if ver == nil { return nil } v, err := semver.NewVersion(ver.Server) if err != nil { m.lg.Warn( "failed to parse server version of remote member", zap.String("remote-member-id", mid), zap.String("remote-member-version", ver.Server), zap.Error(err), ) return nil } if lv.LessThan(*v) { m.lg.Warn( "leader found higher-versioned member", zap.String("local-member-version", lv.String()), zap.String("remote-member-id", mid), zap.String("remote-member-version", ver.Server), ) } if minV == nil { minV = v } else if v.LessThan(*minV) { minV = v } } return minV } // versionsMatchTarget returns true if all server versions are equal to target version, otherwise return false. // It can be used to decide the whether the cluster finishes downgrading to target version. func (m *Monitor) versionsMatchTarget(targetVersion *semver.Version) bool { vers := m.s.GetMembersVersions() targetVersion = &semver.Version{Major: targetVersion.Major, Minor: targetVersion.Minor} for mid, ver := range vers { if ver == nil { return false } v, err := semver.NewVersion(ver.Server) if err != nil { m.lg.Warn( "failed to parse server version of remote member", zap.String("remote-member-id", mid), zap.String("remote-member-version", ver.Server), zap.Error(err), ) return false } v = &semver.Version{Major: v.Major, Minor: v.Minor} if !targetVersion.Equal(*v) { m.lg.Warn("remotes server has mismatching etcd version", zap.String("remote-member-id", mid), zap.String("current-server-version", v.String()), zap.String("target-version", targetVersion.String()), ) return false } } return true } ================================================ FILE: server/etcdserver/version/monitor_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package version import ( "context" "fmt" "reflect" "testing" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/version" ) func TestMemberMinimalVersion(t *testing.T) { tests := []struct { memberVersions map[string]*version.Versions wantVersion *semver.Version }{ { map[string]*version.Versions{"a": {Server: "2.0.0"}}, semver.Must(semver.NewVersion("2.0.0")), }, // unknown { map[string]*version.Versions{"a": nil}, nil, }, { map[string]*version.Versions{"a": {Server: "2.0.0"}, "b": {Server: "2.1.0"}, "c": {Server: "2.1.0"}}, semver.Must(semver.NewVersion("2.0.0")), }, { map[string]*version.Versions{"a": {Server: "2.1.0"}, "b": {Server: "2.1.0"}, "c": {Server: "2.1.0"}}, semver.Must(semver.NewVersion("2.1.0")), }, { map[string]*version.Versions{"a": nil, "b": {Server: "2.1.0"}, "c": {Server: "2.1.0"}}, nil, }, } for i, tt := range tests { monitor := NewMonitor(zaptest.NewLogger(t), &storageMock{ memberVersions: tt.memberVersions, }) minV := monitor.membersMinimalServerVersion() if !reflect.DeepEqual(minV, tt.wantVersion) { t.Errorf("#%d: ver = %+v, want %+v", i, minV, tt.wantVersion) } } } func TestDecideStorageVersion(t *testing.T) { tests := []struct { name string clusterVersion *semver.Version storageVersion *semver.Version expectStorageVersion *semver.Version }{ { name: "No action if cluster version is nil", }, { name: "Should set storage version if cluster version is set", clusterVersion: &version.V3_5, expectStorageVersion: &version.V3_5, }, { name: "No action if storage version was already set", storageVersion: &version.V3_5, expectStorageVersion: &version.V3_5, }, { name: "No action if storage version equals cluster version", clusterVersion: &version.V3_5, storageVersion: &version.V3_5, expectStorageVersion: &version.V3_5, }, { name: "Should set storage version to cluster version", clusterVersion: &version.V3_6, storageVersion: &version.V3_5, expectStorageVersion: &version.V3_6, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &storageMock{ clusterVersion: tt.clusterVersion, storageVersion: tt.storageVersion, } monitor := NewMonitor(zaptest.NewLogger(t), s) monitor.UpdateStorageVersionIfNeeded() if !reflect.DeepEqual(s.storageVersion, tt.expectStorageVersion) { t.Errorf("Unexpected storage version value, got = %+v, want %+v", s.storageVersion, tt.expectStorageVersion) } }) } } func TestVersionMatchTarget(t *testing.T) { tests := []struct { name string targetVersion *semver.Version versionMap map[string]*version.Versions expectedFinished bool }{ { "When downgrade finished", &semver.Version{Major: 3, Minor: 4}, map[string]*version.Versions{ "mem1": {Server: "3.4.1", Cluster: "3.4.0"}, "mem2": {Server: "3.4.2-pre", Cluster: "3.4.0"}, "mem3": {Server: "3.4.2", Cluster: "3.4.0"}, }, true, }, { "When cannot parse peer version", &semver.Version{Major: 3, Minor: 4}, map[string]*version.Versions{ "mem1": {Server: "3.4", Cluster: "3.4.0"}, "mem2": {Server: "3.4.2-pre", Cluster: "3.4.0"}, "mem3": {Server: "3.4.2", Cluster: "3.4.0"}, }, false, }, { "When downgrade not finished", &semver.Version{Major: 3, Minor: 4}, map[string]*version.Versions{ "mem1": {Server: "3.4.1", Cluster: "3.4.0"}, "mem2": {Server: "3.4.2-pre", Cluster: "3.4.0"}, "mem3": {Server: "3.5.2", Cluster: "3.5.0"}, }, false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { monitor := NewMonitor(zaptest.NewLogger(t), &storageMock{ memberVersions: tt.versionMap, }) actual := monitor.versionsMatchTarget(tt.targetVersion) if actual != tt.expectedFinished { t.Errorf("expected downgrade finished is %v; got %v", tt.expectedFinished, actual) } }) } } func TestUpdateClusterVersionIfNeeded(t *testing.T) { tests := []struct { name string clusterVersion *semver.Version memberVersions map[string]*version.Versions downgrade *DowngradeInfo expectClusterVersion *semver.Version expectError error }{ { name: "Default to 3.0 if there are no members", expectClusterVersion: &version.V3_0, }, { name: "Should pick lowest server version from members", memberVersions: map[string]*version.Versions{ "a": {Server: "3.6.0"}, "b": {Server: "3.5.0"}, }, expectClusterVersion: &version.V3_5, }, { name: "Should support not full releases", memberVersions: map[string]*version.Versions{ "b": {Server: "3.5.0-alpha.0"}, }, expectClusterVersion: &version.V3_5, }, { name: "Sets minimal version when member has broken version", memberVersions: map[string]*version.Versions{ "a": {Server: "3.6.0"}, "b": {Server: "yyyy"}, }, expectClusterVersion: &version.V3_0, }, { name: "Should not downgrade cluster version without explicit downgrade request", memberVersions: map[string]*version.Versions{ "a": {Server: "3.5.0"}, "b": {Server: "3.6.0"}, }, clusterVersion: &version.V3_6, expectClusterVersion: &version.V3_6, }, { name: "Should not upgrade cluster version if there is still member old member", memberVersions: map[string]*version.Versions{ "a": {Server: "3.5.0"}, "b": {Server: "3.6.0"}, }, clusterVersion: &version.V3_5, expectClusterVersion: &version.V3_5, }, { name: "Should upgrade cluster version if all members have upgraded (have higher server version)", memberVersions: map[string]*version.Versions{ "a": {Server: "3.6.0"}, "b": {Server: "3.6.0"}, }, clusterVersion: &version.V3_5, expectClusterVersion: &version.V3_6, }, { name: "Should downgrade cluster version if downgrade is set to allow older members to join", memberVersions: map[string]*version.Versions{ "a": {Server: "3.6.0"}, "b": {Server: "3.6.0"}, }, clusterVersion: &version.V3_6, downgrade: &DowngradeInfo{TargetVersion: "3.5.0", Enabled: true}, expectClusterVersion: &version.V3_5, }, { name: "Don't downgrade below supported range", memberVersions: map[string]*version.Versions{ "a": {Server: "3.6.0"}, "b": {Server: "3.6.0"}, }, clusterVersion: &version.V3_5, downgrade: &DowngradeInfo{TargetVersion: "3.4.0", Enabled: true}, expectClusterVersion: &version.V3_5, expectError: fmt.Errorf("invalid downgrade target"), }, { name: "Don't downgrade above cluster version", memberVersions: map[string]*version.Versions{ "a": {Server: "3.5.0"}, "b": {Server: "3.5.0"}, }, clusterVersion: &version.V3_5, downgrade: &DowngradeInfo{TargetVersion: "3.6.0", Enabled: true}, expectClusterVersion: &version.V3_5, expectError: fmt.Errorf("invalid downgrade target"), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &storageMock{ clusterVersion: tt.clusterVersion, memberVersions: tt.memberVersions, downgradeInfo: tt.downgrade, } monitor := NewMonitor(zaptest.NewLogger(t), s) err := monitor.UpdateClusterVersionIfNeeded() assert.Equal(t, tt.expectClusterVersion, s.clusterVersion) assert.Equal(t, tt.expectError, err) // Ensure results are stable newVersion, err := monitor.decideClusterVersion() assert.Nil(t, newVersion) assert.Equal(t, tt.expectError, err) }) } } func TestCancelDowngradeIfNeeded(t *testing.T) { tests := []struct { name string memberVersions map[string]*version.Versions downgrade *DowngradeInfo expectDowngrade *DowngradeInfo }{ { name: "No action if there no downgrade in progress", }, { name: "Cancel downgrade if there are no members", downgrade: &DowngradeInfo{TargetVersion: "3.5.0", Enabled: true}, expectDowngrade: nil, }, // Next entries go through all states that should happen during downgrade { name: "No action if downgrade was not started", memberVersions: map[string]*version.Versions{ "a": {Cluster: "3.6.0", Server: "3.6.1"}, "b": {Cluster: "3.6.0", Server: "3.6.2"}, }, }, { name: "Continue downgrade if just started", memberVersions: map[string]*version.Versions{ "a": {Cluster: "3.5.0", Server: "3.6.1"}, "b": {Cluster: "3.5.0", Server: "3.6.2"}, }, downgrade: &DowngradeInfo{TargetVersion: "3.5.0", Enabled: true}, expectDowngrade: &DowngradeInfo{TargetVersion: "3.5.0", Enabled: true}, }, { name: "Continue downgrade if there is at least one member with not matching", memberVersions: map[string]*version.Versions{ "a": {Cluster: "3.5.0", Server: "3.5.1"}, "b": {Cluster: "3.5.0", Server: "3.6.2"}, }, downgrade: &DowngradeInfo{TargetVersion: "3.5.0", Enabled: true}, expectDowngrade: &DowngradeInfo{TargetVersion: "3.5.0", Enabled: true}, }, { name: "Cancel downgrade if all members have downgraded", memberVersions: map[string]*version.Versions{ "a": {Cluster: "3.5.0", Server: "3.5.1"}, "b": {Cluster: "3.5.0", Server: "3.5.2"}, }, downgrade: &DowngradeInfo{TargetVersion: "3.5.0", Enabled: true}, expectDowngrade: nil, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &storageMock{ memberVersions: tt.memberVersions, downgradeInfo: tt.downgrade, } monitor := NewMonitor(zaptest.NewLogger(t), s) // Run multiple times to ensure that results are stable for i := 0; i < 3; i++ { monitor.CancelDowngradeIfNeeded() assert.Equal(t, tt.expectDowngrade, s.downgradeInfo) } }) } } func TestUpdateStorageVersionIfNeeded(t *testing.T) { tests := []struct { name string clusterVersion *semver.Version storageVersion *semver.Version expectStorageVersion *semver.Version }{ { name: "No action if cluster version is nil", }, { name: "Should set storage version if cluster version is set", clusterVersion: &version.V3_5, expectStorageVersion: &version.V3_5, }, { name: "No action if storage version was already set", storageVersion: &version.V3_5, expectStorageVersion: &version.V3_5, }, { name: "No action if storage version equals cluster version", clusterVersion: &version.V3_5, storageVersion: &version.V3_5, expectStorageVersion: &version.V3_5, }, { name: "Should set storage version to cluster version", clusterVersion: &version.V3_6, storageVersion: &version.V3_5, expectStorageVersion: &version.V3_6, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &storageMock{ clusterVersion: tt.clusterVersion, storageVersion: tt.storageVersion, } monitor := NewMonitor(zaptest.NewLogger(t), s) // Run multiple times to ensure that results are stable for i := 0; i < 3; i++ { monitor.UpdateStorageVersionIfNeeded() assert.Equal(t, tt.expectStorageVersion, s.storageVersion) } }) } } type storageMock struct { memberVersions map[string]*version.Versions clusterVersion *semver.Version storageVersion *semver.Version downgradeInfo *DowngradeInfo } var _ Server = (*storageMock)(nil) func (s *storageMock) UpdateClusterVersion(version string) { s.clusterVersion = semver.New(version) } func (s *storageMock) LinearizableReadNotify(ctx context.Context) error { return nil } func (s *storageMock) DowngradeEnable(ctx context.Context, targetVersion *semver.Version) error { return nil } func (s *storageMock) DowngradeCancel(ctx context.Context) error { s.downgradeInfo = nil return nil } func (s *storageMock) GetClusterVersion() *semver.Version { return s.clusterVersion } func (s *storageMock) GetDowngradeInfo() *DowngradeInfo { return s.downgradeInfo } func (s *storageMock) GetMembersVersions() map[string]*version.Versions { return s.memberVersions } func (s *storageMock) GetStorageVersion() *semver.Version { return s.storageVersion } func (s *storageMock) UpdateStorageVersion(v semver.Version) error { s.storageVersion = &v return nil } ================================================ FILE: server/etcdserver/version/version.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package version import ( "context" "github.com/coreos/go-semver/semver" "go.uber.org/zap" ) // Manager contains logic to manage etcd cluster version downgrade process. type Manager struct { lg *zap.Logger s Server } // NewManager returns a new manager instance func NewManager(lg *zap.Logger, s Server) *Manager { return &Manager{ lg: lg, s: s, } } // DowngradeValidate validates if cluster is downloadable to provided target version and returns error if not. func (m *Manager) DowngradeValidate(ctx context.Context, targetVersion *semver.Version) error { // gets leaders commit index and wait for local store to finish applying that index // to avoid using stale downgrade information err := m.s.LinearizableReadNotify(ctx) if err != nil { return err } cv := m.s.GetClusterVersion() allowedTargetVersion := allowedDowngradeVersion(cv) if !targetVersion.Equal(*allowedTargetVersion) { return ErrInvalidDowngradeTargetVersion } downgradeInfo := m.s.GetDowngradeInfo() if downgradeInfo != nil && downgradeInfo.Enabled { // Todo: return the downgrade status along with the error msg return ErrDowngradeInProcess } return nil } // DowngradeEnable initiates etcd cluster version downgrade process. func (m *Manager) DowngradeEnable(ctx context.Context, targetVersion *semver.Version) error { // validate downgrade capability before starting downgrade err := m.DowngradeValidate(ctx, targetVersion) if err != nil { return err } return m.s.DowngradeEnable(ctx, targetVersion) } // DowngradeCancel cancels ongoing downgrade process. func (m *Manager) DowngradeCancel(ctx context.Context) error { err := m.s.LinearizableReadNotify(ctx) if err != nil { return err } downgradeInfo := m.s.GetDowngradeInfo() if !downgradeInfo.Enabled { return ErrNoInflightDowngrade } return m.s.DowngradeCancel(ctx) } ================================================ FILE: server/etcdserver/version/version_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package version import ( "context" "fmt" "math/rand" "testing" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/version" ) func TestUpgradeSingleNode(t *testing.T) { lg := zaptest.NewLogger(t) c := newCluster(lg, 1, version.V3_6) c.StepMonitors() assert.Equal(t, newCluster(lg, 1, version.V3_6), c) c.ReplaceMemberBinary(0, version.V3_7) c.StepMonitors() c.StepMonitors() assert.Equal(t, newCluster(lg, 1, version.V3_7), c) } func TestUpgradeThreeNodes(t *testing.T) { lg := zaptest.NewLogger(t) c := newCluster(lg, 3, version.V3_6) c.StepMonitors() assert.Equal(t, newCluster(lg, 3, version.V3_6), c) c.ReplaceMemberBinary(0, version.V3_7) c.StepMonitors() c.ReplaceMemberBinary(1, version.V3_7) c.StepMonitors() c.ReplaceMemberBinary(2, version.V3_7) c.StepMonitors() c.StepMonitors() assert.Equal(t, newCluster(lg, 3, version.V3_7), c) } func TestDowngradeSingleNode(t *testing.T) { lg := zaptest.NewLogger(t) c := newCluster(lg, 1, version.V3_6) c.StepMonitors() assert.Equal(t, newCluster(lg, 1, version.V3_6), c) require.NoError(t, c.Version().DowngradeEnable(t.Context(), &version.V3_5)) c.StepMonitors() assert.Equal(t, version.V3_5, c.clusterVersion) c.ReplaceMemberBinary(0, version.V3_5) c.StepMonitors() assert.Equal(t, newCluster(lg, 1, version.V3_5), c) } func TestDowngradeThreeNode(t *testing.T) { lg := zaptest.NewLogger(t) c := newCluster(lg, 3, version.V3_6) c.StepMonitors() assert.Equal(t, newCluster(lg, 3, version.V3_6), c) require.NoError(t, c.Version().DowngradeEnable(t.Context(), &version.V3_5)) c.StepMonitors() assert.Equal(t, version.V3_5, c.clusterVersion) c.ReplaceMemberBinary(0, version.V3_5) c.StepMonitors() c.ReplaceMemberBinary(1, version.V3_5) c.StepMonitors() c.ReplaceMemberBinary(2, version.V3_5) c.StepMonitors() assert.Equal(t, newCluster(lg, 3, version.V3_5), c) } func TestNewerMemberCanReconnectDuringDowngrade(t *testing.T) { lg := zaptest.NewLogger(t) c := newCluster(lg, 3, version.V3_6) c.StepMonitors() assert.Equal(t, newCluster(lg, 3, version.V3_6), c) require.NoError(t, c.Version().DowngradeEnable(t.Context(), &version.V3_5)) c.StepMonitors() assert.Equal(t, version.V3_5, c.clusterVersion) c.ReplaceMemberBinary(0, version.V3_5) c.StepMonitors() c.MemberCrashes(2) c.StepMonitors() c.MemberReconnects(2) c.StepMonitors() c.ReplaceMemberBinary(1, version.V3_5) c.StepMonitors() c.ReplaceMemberBinary(2, version.V3_5) c.StepMonitors() assert.Equal(t, newCluster(lg, 3, version.V3_5), c) } func newCluster(lg *zap.Logger, memberCount int, ver semver.Version) *clusterMock { cluster := &clusterMock{ lg: lg, clusterVersion: ver, members: make([]*memberMock, 0, memberCount), } majorMinVer := semver.Version{Major: ver.Major, Minor: ver.Minor} for i := 0; i < memberCount; i++ { m := &memberMock{ isRunning: true, cluster: cluster, serverVersion: ver, storageVersion: majorMinVer, } m.monitor = NewMonitor(lg.Named(fmt.Sprintf("m%d", i)), m) cluster.members = append(cluster.members, m) } cluster.members[0].isLeader = true return cluster } func (c *clusterMock) StepMonitors() { // Execute monitor functions in random order as it is not guaranteed var fs []func() for _, m := range c.members { fs = append(fs, m.monitor.UpdateStorageVersionIfNeeded) if m.isLeader { fs = append(fs, m.monitor.CancelDowngradeIfNeeded, func() { m.monitor.UpdateClusterVersionIfNeeded() }) } } rand.Shuffle(len(fs), func(i, j int) { fs[i], fs[j] = fs[j], fs[i] }) for _, f := range fs { f() } } type clusterMock struct { lg *zap.Logger clusterVersion semver.Version downgradeInfo *DowngradeInfo members []*memberMock } func (c *clusterMock) Version() *Manager { return NewManager(c.lg, c.members[0]) } func (c *clusterMock) MembersVersions() map[string]*version.Versions { result := map[string]*version.Versions{} for i, m := range c.members { if m.isRunning { result[fmt.Sprintf("%d", i)] = &version.Versions{ Server: m.serverVersion.String(), Cluster: c.clusterVersion.String(), } } } return result } func (c *clusterMock) ReplaceMemberBinary(mid int, newServerVersion semver.Version) { MustDetectDowngrade(c.lg, &c.members[mid].serverVersion, &c.clusterVersion) c.members[mid].serverVersion = newServerVersion } func (c *clusterMock) MemberCrashes(mid int) { c.members[mid].isRunning = false } func (c *clusterMock) MemberReconnects(mid int) { MustDetectDowngrade(c.lg, &c.members[mid].serverVersion, &c.clusterVersion) c.members[mid].isRunning = true } type memberMock struct { cluster *clusterMock isRunning bool isLeader bool serverVersion semver.Version storageVersion semver.Version monitor *Monitor } var _ Server = (*memberMock)(nil) func (m *memberMock) UpdateClusterVersion(version string) { m.cluster.clusterVersion = *semver.New(version) } func (m *memberMock) LinearizableReadNotify(ctx context.Context) error { return nil } func (m *memberMock) DowngradeEnable(ctx context.Context, targetVersion *semver.Version) error { m.cluster.downgradeInfo = &DowngradeInfo{ TargetVersion: targetVersion.String(), Enabled: true, } return nil } func (m *memberMock) DowngradeCancel(context.Context) error { m.cluster.downgradeInfo = nil return nil } func (m *memberMock) GetClusterVersion() *semver.Version { return &m.cluster.clusterVersion } func (m *memberMock) GetDowngradeInfo() *DowngradeInfo { return m.cluster.downgradeInfo } func (m *memberMock) GetMembersVersions() map[string]*version.Versions { return m.cluster.MembersVersions() } func (m *memberMock) GetStorageVersion() *semver.Version { return &m.storageVersion } func (m *memberMock) UpdateStorageVersion(v semver.Version) error { m.storageVersion = v return nil } func (m *memberMock) TriggerSnapshot() { } ================================================ FILE: server/etcdserver/zap_raft.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "errors" "go.uber.org/zap" "go.uber.org/zap/zapcore" "go.etcd.io/raft/v3" ) // NewRaftLogger builds "raft.Logger" from "*zap.Config". func NewRaftLogger(lcfg *zap.Config) (raft.Logger, error) { if lcfg == nil { return nil, errors.New("nil zap.Config") } lg, err := lcfg.Build(zap.AddCallerSkip(1)) // to annotate caller outside of "logutil" if err != nil { return nil, err } return &zapRaftLogger{lg: lg, sugar: lg.Sugar()}, nil } // NewRaftLoggerZap converts "*zap.Logger" to "raft.Logger". func NewRaftLoggerZap(lg *zap.Logger) raft.Logger { skipCallerLg := lg.WithOptions(zap.AddCallerSkip(1)) return &zapRaftLogger{lg: skipCallerLg, sugar: skipCallerLg.Sugar()} } // NewRaftLoggerFromZapCore creates "raft.Logger" from "zap.Core" // and "zapcore.WriteSyncer". func NewRaftLoggerFromZapCore(cr zapcore.Core, syncer zapcore.WriteSyncer) raft.Logger { // "AddCallerSkip" to annotate caller outside of "logutil" lg := zap.New(cr, zap.AddCaller(), zap.AddCallerSkip(1), zap.ErrorOutput(syncer)) return &zapRaftLogger{lg: lg, sugar: lg.Sugar()} } type zapRaftLogger struct { lg *zap.Logger sugar *zap.SugaredLogger } func (zl *zapRaftLogger) Debug(args ...any) { zl.sugar.Debug(args...) } func (zl *zapRaftLogger) Debugf(format string, args ...any) { zl.sugar.Debugf(format, args...) } func (zl *zapRaftLogger) Error(args ...any) { zl.sugar.Error(args...) } func (zl *zapRaftLogger) Errorf(format string, args ...any) { zl.sugar.Errorf(format, args...) } func (zl *zapRaftLogger) Info(args ...any) { zl.sugar.Info(args...) } func (zl *zapRaftLogger) Infof(format string, args ...any) { zl.sugar.Infof(format, args...) } func (zl *zapRaftLogger) Warning(args ...any) { zl.sugar.Warn(args...) } func (zl *zapRaftLogger) Warningf(format string, args ...any) { zl.sugar.Warnf(format, args...) } func (zl *zapRaftLogger) Fatal(args ...any) { zl.sugar.Fatal(args...) } func (zl *zapRaftLogger) Fatalf(format string, args ...any) { zl.sugar.Fatalf(format, args...) } func (zl *zapRaftLogger) Panic(args ...any) { zl.sugar.Panic(args...) } func (zl *zapRaftLogger) Panicf(format string, args ...any) { zl.sugar.Panicf(format, args...) } ================================================ FILE: server/etcdserver/zap_raft_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package etcdserver import ( "bytes" "fmt" "os" "path/filepath" "strings" "testing" "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" "go.etcd.io/etcd/client/pkg/v3/logutil" ) func TestNewRaftLogger(t *testing.T) { logPath := filepath.Join(os.TempDir(), fmt.Sprintf("test-log-%d", time.Now().UnixNano())) defer os.RemoveAll(logPath) lcfg := &zap.Config{ Level: zap.NewAtomicLevelAt(zap.DebugLevel), Development: false, Sampling: &zap.SamplingConfig{ Initial: 100, Thereafter: 100, }, Encoding: "json", EncoderConfig: logutil.DefaultZapLoggerConfig.EncoderConfig, OutputPaths: []string{logPath}, ErrorOutputPaths: []string{logPath}, } gl, err := NewRaftLogger(lcfg) if err != nil { t.Fatal(err) } gl.Info("etcd-logutil-1") data, err := os.ReadFile(logPath) if err != nil { t.Fatal(err) } if !bytes.Contains(data, []byte("etcd-logutil-1")) { t.Fatalf("can't find data in log %q", string(data)) } gl.Warning("etcd-logutil-2") data, err = os.ReadFile(logPath) if err != nil { t.Fatal(err) } if !bytes.Contains(data, []byte("etcd-logutil-2")) { t.Fatalf("can't find data in log %q", string(data)) } if !bytes.Contains(data, []byte("zap_raft_test.go:")) { t.Fatalf("unexpected caller; %q", string(data)) } } func TestNewRaftLoggerFromZapCore(t *testing.T) { buf := bytes.NewBuffer(nil) syncer := zapcore.AddSync(buf) cr := zapcore.NewCore( zapcore.NewJSONEncoder(logutil.DefaultZapLoggerConfig.EncoderConfig), syncer, zap.NewAtomicLevelAt(zap.InfoLevel), ) lg := NewRaftLoggerFromZapCore(cr, syncer) lg.Info("TestNewRaftLoggerFromZapCore") txt := buf.String() if !strings.Contains(txt, "TestNewRaftLoggerFromZapCore") { t.Fatalf("unexpected log %q", txt) } } ================================================ FILE: server/features/etcd_features.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package features import ( "fmt" "go.uber.org/zap" "go.etcd.io/etcd/pkg/v3/featuregate" ) const ( // Every feature gate should add method here following this template: // // // owner: @username // // kep: https://kep.k8s.io/NNN (or issue: https://github.com/etcd-io/etcd/issues/NNN, or main PR: https://github.com/etcd-io/etcd/pull/NNN) // // alpha: v3.X // MyFeature featuregate.Feature = "MyFeature" // // Feature gates should be listed in alphabetical, case-sensitive // (upper before any lower case character) order. This reduces the risk // of code conflicts because changes are more likely to be scattered // across the file. // StopGRPCServiceOnDefrag enables etcd gRPC service to stop serving client requests on defragmentation. // owner: @chaochn47 // alpha: v3.6 // main PR: https://github.com/etcd-io/etcd/pull/18279 StopGRPCServiceOnDefrag featuregate.Feature = "StopGRPCServiceOnDefrag" // TxnModeWriteWithSharedBuffer enables the write transaction to use a shared buffer in its readonly check operations. // owner: @wilsonwang371 // beta: v3.5 // main PR: https://github.com/etcd-io/etcd/pull/12896 TxnModeWriteWithSharedBuffer featuregate.Feature = "TxnModeWriteWithSharedBuffer" // InitialCorruptCheck enable to check data corruption before serving any client/peer traffic. // owner: @serathius // alpha: v3.6 // main PR: https://github.com/etcd-io/etcd/pull/10524 InitialCorruptCheck featuregate.Feature = "InitialCorruptCheck" // CompactHashCheck enables leader to periodically check followers compaction hashes. // owner: @serathius // alpha: v3.6 // main PR: https://github.com/etcd-io/etcd/pull/14120 CompactHashCheck featuregate.Feature = "CompactHashCheck" // LeaseCheckpoint enables leader to send regular checkpoints to other members to prevent reset of remaining TTL on leader change. // owner: @serathius // alpha: v3.6 // main PR: https://github.com/etcd-io/etcd/pull/13508 LeaseCheckpoint featuregate.Feature = "LeaseCheckpoint" // LeaseCheckpointPersist enables persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. // Requires EnableLeaseCheckpoint featuragate to be enabled. // TODO: Delete in v3.7 // owner: @serathius // alpha: v3.6 // main PR: https://github.com/etcd-io/etcd/pull/13508 // Deprecated: Enabled by default in v3.6, to be removed in v3.7. LeaseCheckpointPersist featuregate.Feature = "LeaseCheckpointPersist" // SetMemberLocalAddr enables using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer. // Requires SetMemberLocalAddr featuragate to be enabled. // owner: @flawedmatrix // alpha: v3.6 // main PR: https://github.com/etcd-io/etcd/pull/17661 SetMemberLocalAddr featuregate.Feature = "SetMemberLocalAddr" // FastLeaseKeepAlive enables lease renewal to skip waiting for the applied index. // owner: @aaronjzhang // beta: v3.7 // main PR: https://github.com/etcd-io/etcd/pull/20589 FastLeaseKeepAlive featuregate.Feature = "FastLeaseKeepAlive" ) var DefaultEtcdServerFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ StopGRPCServiceOnDefrag: {Default: false, PreRelease: featuregate.Alpha}, InitialCorruptCheck: {Default: false, PreRelease: featuregate.Alpha}, CompactHashCheck: {Default: false, PreRelease: featuregate.Alpha}, TxnModeWriteWithSharedBuffer: {Default: true, PreRelease: featuregate.Beta}, LeaseCheckpoint: {Default: false, PreRelease: featuregate.Alpha}, LeaseCheckpointPersist: {Default: false, PreRelease: featuregate.Alpha}, SetMemberLocalAddr: {Default: false, PreRelease: featuregate.Alpha}, FastLeaseKeepAlive: {Default: true, PreRelease: featuregate.Beta}, } func NewDefaultServerFeatureGate(name string, lg *zap.Logger) featuregate.FeatureGate { fg := featuregate.New(fmt.Sprintf("%sServerFeatureGate", name), lg) if err := fg.Add(DefaultEtcdServerFeatureGates); err != nil { panic(err) } return fg } ================================================ FILE: server/go.mod ================================================ module go.etcd.io/etcd/server/v3 go 1.26 toolchain go1.26.1 require ( github.com/coreos/go-semver v0.3.1 github.com/coreos/go-systemd/v22 v22.7.0 github.com/dustin/go-humanize v1.0.1 github.com/gogo/protobuf v1.3.2 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/golang/protobuf v1.5.4 github.com/google/go-cmp v0.7.0 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 github.com/jonboulle/clockwork v0.5.0 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 github.com/soheilhy/cmux v0.1.5 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 go.etcd.io/bbolt v1.4.3 go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/v3 v3.6.0-alpha.0 go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0 go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 go.opentelemetry.io/otel v1.42.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 go.opentelemetry.io/otel/sdk v1.42.0 go.opentelemetry.io/otel/trace v1.42.0 go.uber.org/zap v1.27.1 golang.org/x/crypto v0.48.0 golang.org/x/net v0.51.0 golang.org/x/time v0.14.0 google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 google.golang.org/grpc v1.79.2 google.golang.org/protobuf v1.36.11 gopkg.in/natefinch/lumberjack.v2 v2.2.1 k8s.io/utils v0.0.0-20260108192941-914a6e750570 sigs.k8s.io/yaml v1.6.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/pflag v1.0.10 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace ( go.etcd.io/etcd/api/v3 => ../api go.etcd.io/etcd/client/pkg/v3 => ../client/pkg go.etcd.io/etcd/client/v3 => ../client/v3 go.etcd.io/etcd/pkg/v3 => ../pkg ) ================================================ FILE: server/go.sum ================================================ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee h1:9s5V0M58uCy51LgP6SUjROx7Ofqf8lGmeD/cCLaoagI= go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee/go.mod h1:VteWcRz3UV3TOpfex1x8jgPKAyjRXLKw3j8RdK3UAps= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 h1:XmiuHzgJt067+a6kwyAzkhXooYVv3/TOw9cM2VfJgUM= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/utils v0.0.0-20260108192941-914a6e750570 h1:JT4W8lsdrGENg9W+YwwdLJxklIuKWdRm+BC+xt33FOY= k8s.io/utils v0.0.0-20260108192941-914a6e750570/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= ================================================ FILE: server/lease/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package lease provides an interface and implementation for time-limited leases over arbitrary resources. package lease ================================================ FILE: server/lease/lease.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lease import ( "math" "sync" "time" "go.etcd.io/etcd/server/v3/lease/leasepb" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" ) type Lease struct { ID LeaseID ttl int64 // time to live of the lease in seconds remainingTTL int64 // remaining time to live in seconds, if zero valued it is considered unset and the full ttl should be used // expiryMu protects concurrent accesses to expiry expiryMu sync.RWMutex // expiry is time when lease should expire. no expiration when expiry.IsZero() is true expiry time.Time // mu protects concurrent accesses to itemSet mu sync.RWMutex itemSet map[LeaseItem]struct{} revokec chan struct{} } func NewLease(id LeaseID, ttl int64) *Lease { return &Lease{ ID: id, ttl: ttl, itemSet: make(map[LeaseItem]struct{}), revokec: make(chan struct{}), } } func (l *Lease) expired() bool { return l.Remaining() <= 0 } func (l *Lease) persistTo(b backend.Backend) { lpb := leasepb.Lease{ID: int64(l.ID), TTL: l.ttl, RemainingTTL: l.remainingTTL} tx := b.BatchTx() tx.LockInsideApply() defer tx.Unlock() schema.MustUnsafePutLease(tx, &lpb) } // TTL returns the TTL of the Lease. func (l *Lease) TTL() int64 { return l.ttl } // SetLeaseItem sets the given lease item, this func is thread-safe func (l *Lease) SetLeaseItem(item LeaseItem) { l.mu.Lock() defer l.mu.Unlock() l.itemSet[item] = struct{}{} } // getRemainingTTL returns the last checkpointed remaining TTL of the lease. func (l *Lease) getRemainingTTL() int64 { if l.remainingTTL > 0 { return l.remainingTTL } return l.ttl } // refresh refreshes the expiry of the lease. func (l *Lease) refresh(extend time.Duration) { newExpiry := time.Now().Add(extend + time.Duration(l.getRemainingTTL())*time.Second) l.expiryMu.Lock() defer l.expiryMu.Unlock() l.expiry = newExpiry } // forever sets the expiry of lease to be forever. func (l *Lease) forever() { l.expiryMu.Lock() defer l.expiryMu.Unlock() l.expiry = forever } // Demoted returns true if the lease's expiry has been reset to forever. func (l *Lease) Demoted() bool { l.expiryMu.RLock() defer l.expiryMu.RUnlock() return l.expiry == forever } // Keys returns all the keys attached to the lease. func (l *Lease) Keys() []string { l.mu.RLock() keys := make([]string, 0, len(l.itemSet)) for k := range l.itemSet { keys = append(keys, k.Key) } l.mu.RUnlock() return keys } // Remaining returns the remaining time of the lease. func (l *Lease) Remaining() time.Duration { l.expiryMu.RLock() defer l.expiryMu.RUnlock() if l.expiry.IsZero() { return time.Duration(math.MaxInt64) } return time.Until(l.expiry) } type LeaseItem struct { Key string } // leasesByExpiry implements the sort.Interface. type leasesByExpiry []*Lease func (le leasesByExpiry) Len() int { return len(le) } func (le leasesByExpiry) Less(i, j int) bool { return le[i].Remaining() < le[j].Remaining() } func (le leasesByExpiry) Swap(i, j int) { le[i], le[j] = le[j], le[i] } ================================================ FILE: server/lease/lease_queue.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lease import ( "container/heap" "time" ) // LeaseWithTime contains lease object with a time. // For the lessor's lease heap, time identifies the lease expiration time. // For the lessor's lease checkpoint heap, the time identifies the next lease checkpoint time. type LeaseWithTime struct { id LeaseID time time.Time index int } type LeaseQueue []*LeaseWithTime func (pq LeaseQueue) Len() int { return len(pq) } func (pq LeaseQueue) Less(i, j int) bool { return pq[i].time.Before(pq[j].time) } func (pq LeaseQueue) Swap(i, j int) { pq[i], pq[j] = pq[j], pq[i] pq[i].index = i pq[j].index = j } func (pq *LeaseQueue) Push(x any) { n := len(*pq) item := x.(*LeaseWithTime) item.index = n *pq = append(*pq, item) } func (pq *LeaseQueue) Pop() any { old := *pq n := len(old) item := old[n-1] item.index = -1 // for safety *pq = old[0 : n-1] return item } // LeaseExpiredNotifier is a queue used to notify lessor to revoke expired lease. // Only save one item for a lease, `Register` will update time of the corresponding lease. type LeaseExpiredNotifier struct { m map[LeaseID]*LeaseWithTime queue LeaseQueue } func newLeaseExpiredNotifier() *LeaseExpiredNotifier { return &LeaseExpiredNotifier{ m: make(map[LeaseID]*LeaseWithTime), queue: make(LeaseQueue, 0), } } func (mq *LeaseExpiredNotifier) Init() { heap.Init(&mq.queue) mq.m = make(map[LeaseID]*LeaseWithTime) for _, item := range mq.queue { mq.m[item.id] = item } } func (mq *LeaseExpiredNotifier) RegisterOrUpdate(item *LeaseWithTime) { if old, ok := mq.m[item.id]; ok { old.time = item.time heap.Fix(&mq.queue, old.index) } else { heap.Push(&mq.queue, item) mq.m[item.id] = item } } func (mq *LeaseExpiredNotifier) Unregister() *LeaseWithTime { item := heap.Pop(&mq.queue).(*LeaseWithTime) delete(mq.m, item.id) return item } func (mq *LeaseExpiredNotifier) Peek() *LeaseWithTime { if mq.Len() == 0 { return nil } return mq.queue[0] } func (mq *LeaseExpiredNotifier) Len() int { return len(mq.m) } ================================================ FILE: server/lease/lease_queue_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lease import ( "testing" "time" ) func TestLeaseQueue(t *testing.T) { expiredRetryInterval := 100 * time.Millisecond le := &lessor{ leaseExpiredNotifier: newLeaseExpiredNotifier(), leaseMap: make(map[LeaseID]*Lease), expiredLeaseRetryInterval: expiredRetryInterval, } le.leaseExpiredNotifier.Init() // insert in reverse order of expiration time for i := 50; i >= 1; i-- { now := time.Now() exp := now.Add(time.Hour) if i == 1 { exp = now } le.leaseMap[LeaseID(i)] = &Lease{ID: LeaseID(i)} le.leaseExpiredNotifier.RegisterOrUpdate(&LeaseWithTime{id: LeaseID(i), time: exp}) } // first element is expired. if le.leaseExpiredNotifier.Peek().id != LeaseID(1) { t.Fatalf("first item expected lease ID %d, got %d", LeaseID(1), le.leaseExpiredNotifier.Peek().id) } existExpiredEvent := func() { l, more := le.expireExists() if l == nil { t.Fatalf("expect expiry lease exists") } if l.ID != 1 { t.Fatalf("first item expected lease ID %d, got %d", 1, l.ID) } if more { t.Fatal("expect no more expiry lease") } if le.leaseExpiredNotifier.Len() != 50 { t.Fatalf("expected the expired lease to be pushed back to the heap, heap size got %d", le.leaseExpiredNotifier.Len()) } if le.leaseExpiredNotifier.Peek().id != LeaseID(1) { t.Fatalf("first item expected lease ID %d, got %d", LeaseID(1), le.leaseExpiredNotifier.Peek().id) } } noExpiredEvent := func() { // re-acquire the expired item, nothing exists l, more := le.expireExists() if l != nil { t.Fatal("expect no expiry lease exists") } if more { t.Fatal("expect no more expiry lease") } } existExpiredEvent() // first acquire noExpiredEvent() // second acquire time.Sleep(expiredRetryInterval) existExpiredEvent() // acquire after retry interval } ================================================ FILE: server/lease/leasehttp/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package leasehttp serves lease renewals made through HTTP requests. package leasehttp ================================================ FILE: server/lease/leasehttp/http.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package leasehttp import ( "bytes" "context" "errors" "fmt" "io" "net/http" "time" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/httputil" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/lease/leasepb" ) var ( LeasePrefix = "/leases" LeaseInternalPrefix = "/leases/internal" applyTimeout = time.Second ErrLeaseHTTPTimeout = errors.New("waiting for node to catch up its applied index has timed out") ) // NewHandler returns an http Handler for lease renewals func NewHandler(l lease.Lessor, waitch func() <-chan struct{}) http.Handler { return &leaseHandler{l, waitch} } type leaseHandler struct { l lease.Lessor waitch func() <-chan struct{} } func (h *leaseHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) return } defer r.Body.Close() b, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "error reading body", http.StatusBadRequest) return } var v []byte switch r.URL.Path { case LeasePrefix: lreq := pb.LeaseKeepAliveRequest{} if uerr := lreq.Unmarshal(b); uerr != nil { http.Error(w, "error unmarshalling request", http.StatusBadRequest) return } select { case <-h.waitch(): case <-time.After(applyTimeout): http.Error(w, ErrLeaseHTTPTimeout.Error(), http.StatusRequestTimeout) return } // gofail: var beforeServeHTTPLeaseRenew struct{} ttl, rerr := h.l.Renew(lease.LeaseID(lreq.ID)) if rerr != nil { if errors.Is(rerr, lease.ErrLeaseNotFound) { http.Error(w, rerr.Error(), http.StatusNotFound) return } http.Error(w, rerr.Error(), http.StatusBadRequest) return } // TODO: fill out ResponseHeader resp := &pb.LeaseKeepAliveResponse{ID: lreq.ID, TTL: ttl} v, err = resp.Marshal() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } case LeaseInternalPrefix: lreq := leasepb.LeaseInternalRequest{} if lerr := lreq.Unmarshal(b); lerr != nil { http.Error(w, "error unmarshalling request", http.StatusBadRequest) return } select { case <-h.waitch(): case <-time.After(applyTimeout): http.Error(w, ErrLeaseHTTPTimeout.Error(), http.StatusRequestTimeout) return } // gofail: var beforeLookupWhenForwardLeaseTimeToLive struct{} l := h.l.Lookup(lease.LeaseID(lreq.LeaseTimeToLiveRequest.ID)) if l == nil { http.Error(w, lease.ErrLeaseNotFound.Error(), http.StatusNotFound) return } // TODO: fill out ResponseHeader resp := &leasepb.LeaseInternalResponse{ LeaseTimeToLiveResponse: &pb.LeaseTimeToLiveResponse{ Header: &pb.ResponseHeader{}, ID: lreq.LeaseTimeToLiveRequest.ID, TTL: int64(l.Remaining().Seconds()), GrantedTTL: l.TTL(), }, } if lreq.LeaseTimeToLiveRequest.Keys { ks := l.Keys() kbs := make([][]byte, len(ks)) for i := range ks { kbs[i] = []byte(ks[i]) } resp.LeaseTimeToLiveResponse.Keys = kbs } // The leasor could be demoted if leader changed during lookup. // We should return error to force retry instead of returning // incorrect remaining TTL. if l.Demoted() { http.Error(w, lease.ErrNotPrimary.Error(), http.StatusInternalServerError) return } v, err = resp.Marshal() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } default: http.Error(w, fmt.Sprintf("unknown request path %q", r.URL.Path), http.StatusBadRequest) return } w.Header().Set("Content-Type", "application/protobuf") w.Write(v) } // RenewHTTP renews a lease at a given primary server. // TODO: Batch request in future? func RenewHTTP(ctx context.Context, id lease.LeaseID, url string, rt http.RoundTripper) (int64, error) { // will post lreq protobuf to leader lreq, err := (&pb.LeaseKeepAliveRequest{ID: int64(id)}).Marshal() if err != nil { return -1, err } cc := &http.Client{ Transport: rt, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(lreq)) if err != nil { return -1, err } req = req.WithContext(ctx) req.Header.Set("Content-Type", "application/protobuf") resp, err := cc.Do(req) if err != nil { return -1, err } b, err := readResponse(resp) if err != nil { return -1, err } if resp.StatusCode == http.StatusRequestTimeout { return -1, ErrLeaseHTTPTimeout } if resp.StatusCode == http.StatusNotFound { return -1, lease.ErrLeaseNotFound } if resp.StatusCode != http.StatusOK { return -1, fmt.Errorf("lease: unknown error(%s)", b) } lresp := &pb.LeaseKeepAliveResponse{} if err := lresp.Unmarshal(b); err != nil { return -1, fmt.Errorf(`lease: %w. data = "%s"`, err, b) } if lresp.ID != int64(id) { return -1, fmt.Errorf("lease: renew id mismatch") } return lresp.TTL, nil } // TimeToLiveHTTP retrieves lease information of the given lease ID. func TimeToLiveHTTP(ctx context.Context, id lease.LeaseID, keys bool, url string, rt http.RoundTripper) (*leasepb.LeaseInternalResponse, error) { // will post lreq protobuf to leader lreq, err := (&leasepb.LeaseInternalRequest{ LeaseTimeToLiveRequest: &pb.LeaseTimeToLiveRequest{ ID: int64(id), Keys: keys, }, }).Marshal() if err != nil { return nil, err } req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(lreq)) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/protobuf") req = req.WithContext(ctx) cc := &http.Client{ Transport: rt, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } var b []byte // buffer errc channel so that errc don't block inside the go routinue resp, err := cc.Do(req) if err != nil { return nil, err } b, err = readResponse(resp) if err != nil { return nil, err } if resp.StatusCode == http.StatusRequestTimeout { return nil, ErrLeaseHTTPTimeout } if resp.StatusCode == http.StatusNotFound { return nil, lease.ErrLeaseNotFound } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("lease: unknown error(%s)", string(b)) } lresp := &leasepb.LeaseInternalResponse{} if err := lresp.Unmarshal(b); err != nil { return nil, fmt.Errorf(`lease: %w. data = "%s"`, err, string(b)) } if lresp.LeaseTimeToLiveResponse.ID != int64(id) { return nil, fmt.Errorf("lease: TTL id mismatch") } return lresp, nil } func readResponse(resp *http.Response) (b []byte, err error) { b, err = io.ReadAll(resp.Body) httputil.GracefulClose(resp) return } ================================================ FILE: server/lease/leasehttp/http_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package leasehttp import ( "net/http" "net/http/httptest" "testing" "time" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) func TestRenewHTTP(t *testing.T) { lg := zaptest.NewLogger(t) be, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, be) le := lease.NewLessor(lg, be, nil, lease.LessorConfig{MinLeaseTTL: int64(5)}) le.Promote(time.Second) l, err := le.Grant(1, int64(5)) if err != nil { t.Fatalf("failed to create lease: %v", err) } ts := httptest.NewServer(NewHandler(le, waitReady)) defer ts.Close() ttl, err := RenewHTTP(t.Context(), l.ID, ts.URL+LeasePrefix, http.DefaultTransport) if err != nil { t.Fatal(err) } if ttl != 5 { t.Fatalf("ttl expected 5, got %d", ttl) } } func TestTimeToLiveHTTP(t *testing.T) { lg := zaptest.NewLogger(t) be, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, be) le := lease.NewLessor(lg, be, nil, lease.LessorConfig{MinLeaseTTL: int64(5)}) le.Promote(time.Second) l, err := le.Grant(1, int64(5)) if err != nil { t.Fatalf("failed to create lease: %v", err) } ts := httptest.NewServer(NewHandler(le, waitReady)) defer ts.Close() resp, err := TimeToLiveHTTP(t.Context(), l.ID, true, ts.URL+LeaseInternalPrefix, http.DefaultTransport) if err != nil { t.Fatal(err) } if resp.LeaseTimeToLiveResponse.ID != 1 { t.Fatalf("lease id expected 1, got %d", resp.LeaseTimeToLiveResponse.ID) } if resp.LeaseTimeToLiveResponse.GrantedTTL != 5 { t.Fatalf("granted TTL expected 5, got %d", resp.LeaseTimeToLiveResponse.GrantedTTL) } } func TestRenewHTTPTimeout(t *testing.T) { testApplyTimeout(t, func(l *lease.Lease, serverURL string) error { _, err := RenewHTTP(t.Context(), l.ID, serverURL+LeasePrefix, http.DefaultTransport) return err }) } func TestTimeToLiveHTTPTimeout(t *testing.T) { testApplyTimeout(t, func(l *lease.Lease, serverURL string) error { _, err := TimeToLiveHTTP(t.Context(), l.ID, true, serverURL+LeaseInternalPrefix, http.DefaultTransport) return err }) } func testApplyTimeout(t *testing.T, f func(*lease.Lease, string) error) { lg := zaptest.NewLogger(t) be, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, be) le := lease.NewLessor(lg, be, nil, lease.LessorConfig{MinLeaseTTL: int64(5)}) le.Promote(time.Second) l, err := le.Grant(1, int64(5)) if err != nil { t.Fatalf("failed to create lease: %v", err) } ts := httptest.NewServer(NewHandler(le, waitNotReady)) defer ts.Close() err = f(l, ts.URL) if err == nil { t.Fatalf("expected timeout error, got nil") } if err.Error() != ErrLeaseHTTPTimeout.Error() { t.Fatalf("expected (%v), got (%v)", ErrLeaseHTTPTimeout.Error(), err.Error()) } } func waitReady() <-chan struct{} { ch := make(chan struct{}) close(ch) return ch } func waitNotReady() <-chan struct{} { return nil } ================================================ FILE: server/lease/leasepb/lease.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: lease.proto package leasepb import ( fmt "fmt" io "io" math "math" math_bits "math/bits" proto "github.com/golang/protobuf/proto" etcdserverpb "go.etcd.io/etcd/api/v3/etcdserverpb" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Lease struct { ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` TTL int64 `protobuf:"varint,2,opt,name=TTL,proto3" json:"TTL,omitempty"` RemainingTTL int64 `protobuf:"varint,3,opt,name=RemainingTTL,proto3" json:"RemainingTTL,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Lease) Reset() { *m = Lease{} } func (m *Lease) String() string { return proto.CompactTextString(m) } func (*Lease) ProtoMessage() {} func (*Lease) Descriptor() ([]byte, []int) { return fileDescriptor_3dd57e402472b33a, []int{0} } func (m *Lease) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Lease) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Lease.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Lease) XXX_Merge(src proto.Message) { xxx_messageInfo_Lease.Merge(m, src) } func (m *Lease) XXX_Size() int { return m.Size() } func (m *Lease) XXX_DiscardUnknown() { xxx_messageInfo_Lease.DiscardUnknown(m) } var xxx_messageInfo_Lease proto.InternalMessageInfo func (m *Lease) GetID() int64 { if m != nil { return m.ID } return 0 } func (m *Lease) GetTTL() int64 { if m != nil { return m.TTL } return 0 } func (m *Lease) GetRemainingTTL() int64 { if m != nil { return m.RemainingTTL } return 0 } type LeaseInternalRequest struct { LeaseTimeToLiveRequest *etcdserverpb.LeaseTimeToLiveRequest `protobuf:"bytes,1,opt,name=LeaseTimeToLiveRequest,proto3" json:"LeaseTimeToLiveRequest,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseInternalRequest) Reset() { *m = LeaseInternalRequest{} } func (m *LeaseInternalRequest) String() string { return proto.CompactTextString(m) } func (*LeaseInternalRequest) ProtoMessage() {} func (*LeaseInternalRequest) Descriptor() ([]byte, []int) { return fileDescriptor_3dd57e402472b33a, []int{1} } func (m *LeaseInternalRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseInternalRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseInternalRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseInternalRequest) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseInternalRequest.Merge(m, src) } func (m *LeaseInternalRequest) XXX_Size() int { return m.Size() } func (m *LeaseInternalRequest) XXX_DiscardUnknown() { xxx_messageInfo_LeaseInternalRequest.DiscardUnknown(m) } var xxx_messageInfo_LeaseInternalRequest proto.InternalMessageInfo func (m *LeaseInternalRequest) GetLeaseTimeToLiveRequest() *etcdserverpb.LeaseTimeToLiveRequest { if m != nil { return m.LeaseTimeToLiveRequest } return nil } type LeaseInternalResponse struct { LeaseTimeToLiveResponse *etcdserverpb.LeaseTimeToLiveResponse `protobuf:"bytes,1,opt,name=LeaseTimeToLiveResponse,proto3" json:"LeaseTimeToLiveResponse,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *LeaseInternalResponse) Reset() { *m = LeaseInternalResponse{} } func (m *LeaseInternalResponse) String() string { return proto.CompactTextString(m) } func (*LeaseInternalResponse) ProtoMessage() {} func (*LeaseInternalResponse) Descriptor() ([]byte, []int) { return fileDescriptor_3dd57e402472b33a, []int{2} } func (m *LeaseInternalResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *LeaseInternalResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_LeaseInternalResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *LeaseInternalResponse) XXX_Merge(src proto.Message) { xxx_messageInfo_LeaseInternalResponse.Merge(m, src) } func (m *LeaseInternalResponse) XXX_Size() int { return m.Size() } func (m *LeaseInternalResponse) XXX_DiscardUnknown() { xxx_messageInfo_LeaseInternalResponse.DiscardUnknown(m) } var xxx_messageInfo_LeaseInternalResponse proto.InternalMessageInfo func (m *LeaseInternalResponse) GetLeaseTimeToLiveResponse() *etcdserverpb.LeaseTimeToLiveResponse { if m != nil { return m.LeaseTimeToLiveResponse } return nil } func init() { proto.RegisterType((*Lease)(nil), "leasepb.Lease") proto.RegisterType((*LeaseInternalRequest)(nil), "leasepb.LeaseInternalRequest") proto.RegisterType((*LeaseInternalResponse)(nil), "leasepb.LeaseInternalResponse") } func init() { proto.RegisterFile("lease.proto", fileDescriptor_3dd57e402472b33a) } var fileDescriptor_3dd57e402472b33a = []byte{ // 261 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xce, 0x49, 0x4d, 0x2c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x07, 0x73, 0x0a, 0x92, 0xa4, 0xe4, 0x53, 0x4b, 0x92, 0x53, 0xf4, 0x13, 0x0b, 0x32, 0xf5, 0x41, 0x8c, 0xe2, 0xd4, 0xa2, 0xb2, 0xd4, 0xa2, 0x82, 0x24, 0xfd, 0xa2, 0x82, 0x64, 0x88, 0x4a, 0x25, 0x5f, 0x2e, 0x56, 0x1f, 0x90, 0x5a, 0x21, 0x3e, 0x2e, 0x26, 0x4f, 0x17, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xe6, 0x20, 0x26, 0x4f, 0x17, 0x21, 0x01, 0x2e, 0xe6, 0x90, 0x10, 0x1f, 0x09, 0x26, 0xb0, 0x00, 0x88, 0x29, 0xa4, 0xc4, 0xc5, 0x13, 0x94, 0x9a, 0x9b, 0x98, 0x99, 0x97, 0x99, 0x97, 0x0e, 0x92, 0x62, 0x06, 0x4b, 0xa1, 0x88, 0x29, 0x95, 0x70, 0x89, 0x80, 0x8d, 0xf3, 0xcc, 0x2b, 0x49, 0x2d, 0xca, 0x4b, 0xcc, 0x09, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x8a, 0xe1, 0x12, 0x03, 0x8b, 0x87, 0x64, 0xe6, 0xa6, 0x86, 0xe4, 0xfb, 0x64, 0x96, 0xa5, 0x42, 0x65, 0xc0, 0x36, 0x72, 0x1b, 0xa9, 0xe8, 0x21, 0xbb, 0x4f, 0x0f, 0xbb, 0xda, 0x20, 0x1c, 0x66, 0x28, 0x55, 0x70, 0x89, 0xa2, 0xd9, 0x5a, 0x5c, 0x90, 0x9f, 0x57, 0x9c, 0x2a, 0x14, 0xcf, 0x25, 0x8e, 0xa1, 0x05, 0x22, 0x05, 0xb5, 0x57, 0x95, 0x80, 0xbd, 0x10, 0xc5, 0x41, 0xb8, 0x4c, 0x71, 0x72, 0x3c, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x67, 0x3c, 0x96, 0x63, 0x88, 0xd2, 0x4f, 0xcf, 0x07, 0x9b, 0xa9, 0x97, 0x99, 0x0f, 0x0e, 0x73, 0x7d, 0x88, 0xe1, 0xfa, 0x65, 0xc6, 0xfa, 0xe0, 0x48, 0xd1, 0x87, 0x46, 0x8d, 0x35, 0x94, 0x4e, 0x62, 0x03, 0x47, 0x84, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x2c, 0x7a, 0xc1, 0x88, 0xc1, 0x01, 0x00, 0x00, } func (m *Lease) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Lease) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Lease) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.RemainingTTL != 0 { i = encodeVarintLease(dAtA, i, uint64(m.RemainingTTL)) i-- dAtA[i] = 0x18 } if m.TTL != 0 { i = encodeVarintLease(dAtA, i, uint64(m.TTL)) i-- dAtA[i] = 0x10 } if m.ID != 0 { i = encodeVarintLease(dAtA, i, uint64(m.ID)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *LeaseInternalRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseInternalRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseInternalRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.LeaseTimeToLiveRequest != nil { { size, err := m.LeaseTimeToLiveRequest.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintLease(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func (m *LeaseInternalResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *LeaseInternalResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *LeaseInternalResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.LeaseTimeToLiveResponse != nil { { size, err := m.LeaseTimeToLiveResponse.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintLease(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } func encodeVarintLease(dAtA []byte, offset int, v uint64) int { offset -= sovLease(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) return base } func (m *Lease) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.ID != 0 { n += 1 + sovLease(uint64(m.ID)) } if m.TTL != 0 { n += 1 + sovLease(uint64(m.TTL)) } if m.RemainingTTL != 0 { n += 1 + sovLease(uint64(m.RemainingTTL)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseInternalRequest) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.LeaseTimeToLiveRequest != nil { l = m.LeaseTimeToLiveRequest.Size() n += 1 + l + sovLease(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *LeaseInternalResponse) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.LeaseTimeToLiveResponse != nil { l = m.LeaseTimeToLiveResponse.Size() n += 1 + l + sovLease(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func sovLease(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } func sozLease(x uint64) (n int) { return sovLease(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (m *Lease) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowLease } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: Lease: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: Lease: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) } m.ID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowLease } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.ID |= int64(b&0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field TTL", wireType) } m.TTL = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowLease } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.TTL |= int64(b&0x7F) << shift if b < 0x80 { break } } case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field RemainingTTL", wireType) } m.RemainingTTL = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowLease } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ m.RemainingTTL |= int64(b&0x7F) << shift if b < 0x80 { break } } default: iNdEx = preIndex skippy, err := skipLease(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthLease } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseInternalRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowLease } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseInternalRequest: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseInternalRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field LeaseTimeToLiveRequest", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowLease } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthLease } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthLease } if postIndex > l { return io.ErrUnexpectedEOF } if m.LeaseTimeToLiveRequest == nil { m.LeaseTimeToLiveRequest = &etcdserverpb.LeaseTimeToLiveRequest{} } if err := m.LeaseTimeToLiveRequest.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLease(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthLease } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *LeaseInternalResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowLease } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: LeaseInternalResponse: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: LeaseInternalResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field LeaseTimeToLiveResponse", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowLease } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthLease } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthLease } if postIndex > l { return io.ErrUnexpectedEOF } if m.LeaseTimeToLiveResponse == nil { m.LeaseTimeToLiveResponse = &etcdserverpb.LeaseTimeToLiveResponse{} } if err := m.LeaseTimeToLiveResponse.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLease(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthLease } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func skipLease(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowLease } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } wireType := int(wire & 0x7) switch wireType { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowLease } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } iNdEx++ if dAtA[iNdEx-1] < 0x80 { break } } case 1: iNdEx += 8 case 2: var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowLease } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { break } } if length < 0 { return 0, ErrInvalidLengthLease } iNdEx += length case 3: depth++ case 4: if depth == 0 { return 0, ErrUnexpectedEndOfGroupLease } depth-- case 5: iNdEx += 4 default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { return 0, ErrInvalidLengthLease } if depth == 0 { return iNdEx, nil } } return 0, io.ErrUnexpectedEOF } var ( ErrInvalidLengthLease = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowLease = fmt.Errorf("proto: integer overflow") ErrUnexpectedEndOfGroupLease = fmt.Errorf("proto: unexpected end of group") ) ================================================ FILE: server/lease/leasepb/lease.proto ================================================ syntax = "proto3"; package leasepb; import "etcd/api/etcdserverpb/rpc.proto"; option go_package = "go.etcd.io/etcd/server/v3/lease/leasepb;leasepb"; message Lease { int64 ID = 1; int64 TTL = 2; int64 RemainingTTL = 3; } message LeaseInternalRequest { etcdserverpb.LeaseTimeToLiveRequest LeaseTimeToLiveRequest = 1; } message LeaseInternalResponse { etcdserverpb.LeaseTimeToLiveResponse LeaseTimeToLiveResponse = 1; } ================================================ FILE: server/lease/lessor.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lease import ( "container/heap" "context" "errors" "math" "sort" "sync" "time" "github.com/coreos/go-semver/semver" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/server/v3/lease/leasepb" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" ) // NoLease is a special LeaseID representing the absence of a lease. const NoLease = LeaseID(0) // MaxLeaseTTL is the maximum lease TTL value const MaxLeaseTTL = 9000000000 var ( forever = time.Time{} // default number of leases to revoke per second; configurable for tests defaultLeaseRevokeRate = 1000 // maximum number of lease checkpoints recorded to the consensus log per second; configurable for tests leaseCheckpointRate = 1000 // the default interval of lease checkpoint defaultLeaseCheckpointInterval = 5 * time.Minute // maximum number of lease checkpoints to batch into a single consensus log entry maxLeaseCheckpointBatchSize = 1000 // the default interval to check if the expired lease is revoked defaultExpiredleaseRetryInterval = 3 * time.Second ErrNotPrimary = errors.New("not a primary lessor") ErrLeaseNotFound = errors.New("lease not found") ErrLeaseExists = errors.New("lease already exists") ErrLeaseTTLTooLarge = errors.New("too large lease TTL") ) // TxnDelete is a TxnWrite that only permits deletes. Defined here // to avoid circular dependency with mvcc. type TxnDelete interface { DeleteRange(key, end []byte) (n, rev int64) End() } // RangeDeleter is a TxnDelete constructor. type RangeDeleter func() TxnDelete // Checkpointer permits checkpointing of lease remaining TTLs to the consensus log. Defined here to // avoid circular dependency with mvcc. type Checkpointer func(ctx context.Context, lc *pb.LeaseCheckpointRequest) error type LeaseID int64 // Lessor owns leases. It can grant, revoke, renew and modify leases for lessee. type Lessor interface { // SetRangeDeleter lets the lessor create TxnDeletes to the store. // Lessor deletes the items in the revoked or expired lease by creating // new TxnDeletes. SetRangeDeleter(rd RangeDeleter) SetCheckpointer(cp Checkpointer) // Grant grants a lease that expires at least after TTL seconds. Grant(id LeaseID, ttl int64) (*Lease, error) // Revoke revokes a lease with given ID. The item attached to the // given lease will be removed. If the ID does not exist, an error // will be returned. Revoke(id LeaseID) error // Checkpoint applies the remainingTTL of a lease. The remainingTTL is used in Promote to set // the expiry of leases to less than the full TTL when possible. Checkpoint(id LeaseID, remainingTTL int64) error // Attach attaches given leaseItem to the lease with given LeaseID. // If the lease does not exist, an error will be returned. Attach(id LeaseID, items []LeaseItem) error // GetLease returns LeaseID for given item. // If no lease found, NoLease value will be returned. GetLease(item LeaseItem) LeaseID // Detach detaches given leaseItem from the lease with given LeaseID. // If the lease does not exist, an error will be returned. Detach(id LeaseID, items []LeaseItem) error // Promote promotes the lessor to be the primary lessor. Primary lessor manages // the expiration and renew of leases. // Newly promoted lessor renew the TTL of all lease to extend + previous TTL. Promote(extend time.Duration) // Demote demotes the lessor from being the primary lessor. Demote() // Renew renews a lease with given ID. It returns the renewed TTL. If the ID does not exist, // an error will be returned. Renew(id LeaseID) (int64, error) // Lookup gives the lease at a given lease id, if any Lookup(id LeaseID) *Lease // Leases lists all leases. Leases() []*Lease // ExpiredLeasesC returns a chan that is used to receive expired leases. ExpiredLeasesC() <-chan []*Lease // Recover recovers the lessor state from the given backend and RangeDeleter. Recover(b backend.Backend, rd RangeDeleter) // Stop stops the lessor for managing leases. The behavior of calling Stop multiple // times is undefined. Stop() } // lessor implements Lessor interface. // TODO: use clockwork for testability. type lessor struct { mu sync.RWMutex // demotec is set when the lessor is the primary. // demotec will be closed if the lessor is demoted. demotec chan struct{} leaseMap map[LeaseID]*Lease leaseExpiredNotifier *LeaseExpiredNotifier leaseCheckpointHeap LeaseQueue itemMap map[LeaseItem]LeaseID // When a lease expires, the lessor will delete the // leased range (or key) by the RangeDeleter. rd RangeDeleter // When a lease's deadline should be persisted to preserve the remaining TTL across leader // elections and restarts, the lessor will checkpoint the lease by the Checkpointer. cp Checkpointer // backend to persist leases. We only persist lease ID and expiry for now. // The leased items can be recovered by iterating all the keys in kv. b backend.Backend // minLeaseTTL is the minimum lease TTL that can be granted for a lease. Any // requests for shorter TTLs are extended to the minimum TTL. minLeaseTTL int64 // maximum number of leases to revoke per second leaseRevokeRate int expiredC chan []*Lease // stopC is a channel whose closure indicates that the lessor should be stopped. stopC chan struct{} // doneC is a channel whose closure indicates that the lessor is stopped. doneC chan struct{} lg *zap.Logger // Wait duration between lease checkpoints. checkpointInterval time.Duration // the interval to check if the expired lease is revoked expiredLeaseRetryInterval time.Duration // whether lessor should always persist remaining TTL (always enabled in v3.6). checkpointPersist bool // cluster is used to adapt lessor logic based on cluster version cluster cluster } type cluster interface { // Version is the cluster-wide minimum major.minor version. Version() *semver.Version } type LessorConfig struct { MinLeaseTTL int64 CheckpointInterval time.Duration ExpiredLeasesRetryInterval time.Duration CheckpointPersist bool leaseRevokeRate int } func NewLessor(lg *zap.Logger, b backend.Backend, cluster cluster, cfg LessorConfig) Lessor { return newLessor(lg, b, cluster, cfg) } func newLessor(lg *zap.Logger, b backend.Backend, cluster cluster, cfg LessorConfig) *lessor { checkpointInterval := cfg.CheckpointInterval expiredLeaseRetryInterval := cfg.ExpiredLeasesRetryInterval leaseRevokeRate := cfg.leaseRevokeRate if checkpointInterval == 0 { checkpointInterval = defaultLeaseCheckpointInterval } if expiredLeaseRetryInterval == 0 { expiredLeaseRetryInterval = defaultExpiredleaseRetryInterval } if leaseRevokeRate == 0 { leaseRevokeRate = defaultLeaseRevokeRate } l := &lessor{ leaseMap: make(map[LeaseID]*Lease), itemMap: make(map[LeaseItem]LeaseID), leaseExpiredNotifier: newLeaseExpiredNotifier(), leaseCheckpointHeap: make(LeaseQueue, 0), b: b, minLeaseTTL: cfg.MinLeaseTTL, leaseRevokeRate: leaseRevokeRate, checkpointInterval: checkpointInterval, expiredLeaseRetryInterval: expiredLeaseRetryInterval, checkpointPersist: cfg.CheckpointPersist, // expiredC is a small buffered chan to avoid unnecessary blocking. expiredC: make(chan []*Lease, 16), stopC: make(chan struct{}), doneC: make(chan struct{}), lg: lg, cluster: cluster, } l.initAndRecover() go l.runLoop() return l } // isPrimary indicates if this lessor is the primary lessor. The primary // lessor manages lease expiration and renew. // // in etcd, raft leader is the primary. Thus there might be two primary // leaders at the same time (raft allows concurrent leader but with different term) // for at most a leader election timeout. // The old primary leader cannot affect the correctness since its proposal has a // smaller term and will not be committed. // // TODO: raft follower do not forward lease management proposals. There might be a // very small window (within second normally which depends on go scheduling) that // a raft follow is the primary between the raft leader demotion and lessor demotion. // Usually this should not be a problem. Lease should not be that sensitive to timing. func (le *lessor) isPrimary() bool { return le.demotec != nil } func (le *lessor) SetRangeDeleter(rd RangeDeleter) { le.mu.Lock() defer le.mu.Unlock() le.rd = rd } func (le *lessor) SetCheckpointer(cp Checkpointer) { le.mu.Lock() defer le.mu.Unlock() le.cp = cp } func (le *lessor) Grant(id LeaseID, ttl int64) (*Lease, error) { if id == NoLease { return nil, ErrLeaseNotFound } if ttl > MaxLeaseTTL { return nil, ErrLeaseTTLTooLarge } // TODO: when lessor is under high load, it should give out lease // with longer TTL to reduce renew load. l := NewLease(id, ttl) le.mu.Lock() defer le.mu.Unlock() if _, ok := le.leaseMap[id]; ok { return nil, ErrLeaseExists } if l.ttl < le.minLeaseTTL { l.ttl = le.minLeaseTTL } if le.isPrimary() { l.refresh(0) } else { l.forever() } le.leaseMap[id] = l l.persistTo(le.b) leaseTotalTTLs.Observe(float64(l.ttl)) leaseGranted.Inc() if le.isPrimary() { item := &LeaseWithTime{id: l.ID, time: l.expiry} le.leaseExpiredNotifier.RegisterOrUpdate(item) le.scheduleCheckpointIfNeeded(l) } return l, nil } func (le *lessor) Revoke(id LeaseID) error { le.mu.Lock() l := le.leaseMap[id] if l == nil { le.mu.Unlock() return ErrLeaseNotFound } defer close(l.revokec) // unlock before doing external work le.mu.Unlock() if le.rd == nil { return nil } txn := le.rd() // sort keys so deletes are in same order among all members, // otherwise the backend hashes will be different keys := l.Keys() sort.StringSlice(keys).Sort() for _, key := range keys { txn.DeleteRange([]byte(key), nil) } le.mu.Lock() defer le.mu.Unlock() delete(le.leaseMap, l.ID) // lease deletion needs to be in the same backend transaction with the // kv deletion. Or we might end up with not executing the revoke or not // deleting the keys if etcdserver fails in between. schema.UnsafeDeleteLease(le.b.BatchTx(), &leasepb.Lease{ID: int64(l.ID)}) txn.End() leaseRevoked.Inc() return nil } func (le *lessor) Checkpoint(id LeaseID, remainingTTL int64) error { le.mu.Lock() defer le.mu.Unlock() if l, ok := le.leaseMap[id]; ok { // when checkpointing, we only update the remainingTTL, Promote is responsible for applying this to lease expiry l.remainingTTL = remainingTTL if le.shouldPersistCheckpoints() { l.persistTo(le.b) } if le.isPrimary() { // schedule the next checkpoint as needed le.scheduleCheckpointIfNeeded(l) } } return nil } func (le *lessor) shouldPersistCheckpoints() bool { cv := le.cluster.Version() return le.checkpointPersist || (cv != nil && greaterOrEqual(*cv, version.V3_6)) } func greaterOrEqual(first, second semver.Version) bool { return !version.LessThan(first, second) } // Renew renews an existing lease. If the given lease does not exist or // has expired, an error will be returned. func (le *lessor) Renew(id LeaseID) (int64, error) { le.mu.RLock() if !le.isPrimary() { // forward renew request to primary instead of returning error. le.mu.RUnlock() return -1, ErrNotPrimary } demotec := le.demotec l := le.leaseMap[id] if l == nil { le.mu.RUnlock() return -1, ErrLeaseNotFound } // Clear remaining TTL when we renew if it is set clearRemainingTTL := le.cp != nil && l.remainingTTL > 0 le.mu.RUnlock() if l.expired() { select { // A expired lease might be pending for revoking or going through // quorum to be revoked. To be accurate, renew request must wait for the // deletion to complete. case <-l.revokec: return -1, ErrLeaseNotFound // The expired lease might fail to be revoked if the primary changes. // The caller will retry on ErrNotPrimary. case <-demotec: return -1, ErrNotPrimary case <-le.stopC: return -1, ErrNotPrimary } } // gofail: var beforeCheckpointInLeaseRenew struct{} // Clear remaining TTL when we renew if it is set // By applying a RAFT entry only when the remainingTTL is already set, we limit the number // of RAFT entries written per lease to a max of 2 per checkpoint interval. if clearRemainingTTL { if err := le.cp(context.Background(), &pb.LeaseCheckpointRequest{Checkpoints: []*pb.LeaseCheckpoint{{ID: int64(l.ID), Remaining_TTL: 0}}}); err != nil { return -1, err } } le.mu.Lock() // Re-check in case the lease was revoked immediately after the previous check l = le.leaseMap[id] if l == nil { le.mu.Unlock() return -1, ErrLeaseNotFound } l.refresh(0) item := &LeaseWithTime{id: l.ID, time: l.expiry} le.leaseExpiredNotifier.RegisterOrUpdate(item) le.mu.Unlock() leaseRenewed.Inc() return l.ttl, nil } func (le *lessor) Lookup(id LeaseID) *Lease { le.mu.RLock() defer le.mu.RUnlock() return le.leaseMap[id] } func (le *lessor) unsafeLeases() []*Lease { leases := make([]*Lease, 0, len(le.leaseMap)) for _, l := range le.leaseMap { leases = append(leases, l) } return leases } func (le *lessor) Leases() []*Lease { le.mu.RLock() ls := le.unsafeLeases() le.mu.RUnlock() sort.Sort(leasesByExpiry(ls)) return ls } func (le *lessor) Promote(extend time.Duration) { le.mu.Lock() defer le.mu.Unlock() le.demotec = make(chan struct{}) // refresh the expiries of all leases. for _, l := range le.leaseMap { l.refresh(extend) item := &LeaseWithTime{id: l.ID, time: l.expiry} le.leaseExpiredNotifier.RegisterOrUpdate(item) le.scheduleCheckpointIfNeeded(l) } if len(le.leaseMap) < le.leaseRevokeRate { // no possibility of lease pile-up return } // adjust expiries in case of overlap leases := le.unsafeLeases() sort.Sort(leasesByExpiry(leases)) baseWindow := leases[0].Remaining() nextWindow := baseWindow + time.Second expires := 0 // have fewer expires than the total revoke rate so piled up leases // don't consume the entire revoke limit targetExpiresPerSecond := (3 * le.leaseRevokeRate) / 4 for _, l := range leases { remaining := l.Remaining() if remaining > nextWindow { baseWindow = remaining nextWindow = baseWindow + time.Second expires = 1 continue } expires++ if expires <= targetExpiresPerSecond { continue } rateDelay := float64(time.Second) * (float64(expires) / float64(targetExpiresPerSecond)) // If leases are extended by n seconds, leases n seconds ahead of the // base window should be extended by only one second. rateDelay -= float64(remaining - baseWindow) delay := time.Duration(rateDelay) nextWindow = baseWindow + delay l.refresh(delay + extend) item := &LeaseWithTime{id: l.ID, time: l.expiry} le.leaseExpiredNotifier.RegisterOrUpdate(item) le.scheduleCheckpointIfNeeded(l) } } func (le *lessor) Demote() { le.mu.Lock() defer le.mu.Unlock() // set the expiries of all leases to forever for _, l := range le.leaseMap { l.forever() } le.clearScheduledLeasesCheckpoints() le.clearLeaseExpiredNotifier() if le.demotec != nil { close(le.demotec) le.demotec = nil } } // Attach attaches items to the lease with given ID. When the lease // expires, the attached items will be automatically removed. // If the given lease does not exist, an error will be returned. func (le *lessor) Attach(id LeaseID, items []LeaseItem) error { le.mu.Lock() defer le.mu.Unlock() l := le.leaseMap[id] if l == nil { return ErrLeaseNotFound } l.mu.Lock() for _, it := range items { l.itemSet[it] = struct{}{} le.itemMap[it] = id } l.mu.Unlock() return nil } func (le *lessor) GetLease(item LeaseItem) LeaseID { le.mu.RLock() id := le.itemMap[item] le.mu.RUnlock() return id } // Detach detaches items from the lease with given ID. // If the given lease does not exist, an error will be returned. func (le *lessor) Detach(id LeaseID, items []LeaseItem) error { le.mu.Lock() defer le.mu.Unlock() l := le.leaseMap[id] if l == nil { return ErrLeaseNotFound } l.mu.Lock() for _, it := range items { delete(l.itemSet, it) delete(le.itemMap, it) } l.mu.Unlock() return nil } func (le *lessor) Recover(b backend.Backend, rd RangeDeleter) { le.mu.Lock() defer le.mu.Unlock() le.b = b le.rd = rd le.leaseMap = make(map[LeaseID]*Lease) le.itemMap = make(map[LeaseItem]LeaseID) le.initAndRecover() } func (le *lessor) ExpiredLeasesC() <-chan []*Lease { return le.expiredC } func (le *lessor) Stop() { close(le.stopC) <-le.doneC } func (le *lessor) runLoop() { defer close(le.doneC) delayTicker := time.NewTicker(500 * time.Millisecond) defer delayTicker.Stop() for { le.revokeExpiredLeases() le.checkpointScheduledLeases() select { case <-delayTicker.C: case <-le.stopC: return } } } // revokeExpiredLeases finds all leases past their expiry and sends them to expired channel for // to be revoked. func (le *lessor) revokeExpiredLeases() { var ls []*Lease // rate limit revokeLimit := le.leaseRevokeRate / 2 le.mu.RLock() if le.isPrimary() { ls = le.findExpiredLeases(revokeLimit) } le.mu.RUnlock() if len(ls) != 0 { select { case <-le.stopC: return case le.expiredC <- ls: default: // the receiver of expiredC is probably busy handling // other stuff // let's try this next time after 500ms } } } // checkpointScheduledLeases finds all scheduled lease checkpoints that are due and // submits them to the checkpointer to persist them to the consensus log. func (le *lessor) checkpointScheduledLeases() { // rate limit for i := 0; i < leaseCheckpointRate/2; i++ { var cps []*pb.LeaseCheckpoint le.mu.Lock() if le.isPrimary() { cps = le.findDueScheduledCheckpoints(maxLeaseCheckpointBatchSize) } le.mu.Unlock() if len(cps) != 0 { if err := le.cp(context.Background(), &pb.LeaseCheckpointRequest{Checkpoints: cps}); err != nil { return } } if len(cps) < maxLeaseCheckpointBatchSize { return } } } func (le *lessor) clearScheduledLeasesCheckpoints() { le.leaseCheckpointHeap = make(LeaseQueue, 0) } func (le *lessor) clearLeaseExpiredNotifier() { le.leaseExpiredNotifier = newLeaseExpiredNotifier() } // expireExists returns "l" which is not nil if expiry items exist. // It pops only when expiry item exists. // "next" is true, to indicate that it may exist in next attempt. func (le *lessor) expireExists() (l *Lease, next bool) { if le.leaseExpiredNotifier.Len() == 0 { return nil, false } item := le.leaseExpiredNotifier.Peek() l = le.leaseMap[item.id] if l == nil { // lease has expired or been revoked // no need to revoke (nothing is expiry) le.leaseExpiredNotifier.Unregister() // O(log N) return nil, true } now := time.Now() if now.Before(item.time) /* item.time: expiration time */ { // Candidate expirations are caught up, reinsert this item // and no need to revoke (nothing is expiry) return nil, false } // recheck if revoke is complete after retry interval item.time = now.Add(le.expiredLeaseRetryInterval) le.leaseExpiredNotifier.RegisterOrUpdate(item) return l, false } // findExpiredLeases loops leases in the leaseMap until reaching expired limit // and returns the expired leases that needed to be revoked. func (le *lessor) findExpiredLeases(limit int) []*Lease { leases := make([]*Lease, 0, 16) for { l, next := le.expireExists() if l == nil && !next { break } if next { continue } if l.expired() { leases = append(leases, l) // reach expired limit if len(leases) == limit { break } } } return leases } func (le *lessor) scheduleCheckpointIfNeeded(lease *Lease) { if le.cp == nil { return } if lease.getRemainingTTL() > int64(le.checkpointInterval.Seconds()) { if le.lg != nil { le.lg.Debug("Scheduling lease checkpoint", zap.Int64("leaseID", int64(lease.ID)), zap.Duration("intervalSeconds", le.checkpointInterval), ) } heap.Push(&le.leaseCheckpointHeap, &LeaseWithTime{ id: lease.ID, time: time.Now().Add(le.checkpointInterval), }) } } func (le *lessor) findDueScheduledCheckpoints(checkpointLimit int) []*pb.LeaseCheckpoint { if le.cp == nil { return nil } now := time.Now() var cps []*pb.LeaseCheckpoint for le.leaseCheckpointHeap.Len() > 0 && len(cps) < checkpointLimit { lt := le.leaseCheckpointHeap[0] if lt.time.After(now) /* lt.time: next checkpoint time */ { return cps } heap.Pop(&le.leaseCheckpointHeap) var l *Lease var ok bool if l, ok = le.leaseMap[lt.id]; !ok { continue } if !now.Before(l.expiry) { continue } remainingTTL := int64(math.Ceil(l.expiry.Sub(now).Seconds())) if remainingTTL >= l.ttl { continue } if le.lg != nil { le.lg.Debug("Checkpointing lease", zap.Int64("leaseID", int64(lt.id)), zap.Int64("remainingTTL", remainingTTL), ) } cps = append(cps, &pb.LeaseCheckpoint{ID: int64(lt.id), Remaining_TTL: remainingTTL}) } return cps } func (le *lessor) initAndRecover() { tx := le.b.BatchTx() tx.LockOutsideApply() schema.UnsafeCreateLeaseBucket(tx) lpbs := schema.MustUnsafeGetAllLeases(tx) tx.Unlock() for _, lpb := range lpbs { ID := LeaseID(lpb.ID) if lpb.TTL < le.minLeaseTTL { lpb.TTL = le.minLeaseTTL } le.leaseMap[ID] = &Lease{ ID: ID, ttl: lpb.TTL, // itemSet will be filled in when recover key-value pairs // set expiry to forever, refresh when promoted itemSet: make(map[LeaseItem]struct{}), expiry: forever, revokec: make(chan struct{}), remainingTTL: lpb.RemainingTTL, } } le.leaseExpiredNotifier.Init() heap.Init(&le.leaseCheckpointHeap) le.b.ForceCommit() } // FakeLessor is a fake implementation of Lessor interface. // Used for testing only. type FakeLessor struct { LeaseSet map[LeaseID]struct{} } func (fl *FakeLessor) SetRangeDeleter(dr RangeDeleter) {} func (fl *FakeLessor) SetCheckpointer(cp Checkpointer) {} func (fl *FakeLessor) Grant(id LeaseID, ttl int64) (*Lease, error) { fl.LeaseSet[id] = struct{}{} return nil, nil } func (fl *FakeLessor) Revoke(id LeaseID) error { return nil } func (fl *FakeLessor) Checkpoint(id LeaseID, remainingTTL int64) error { return nil } func (fl *FakeLessor) Attach(id LeaseID, items []LeaseItem) error { return nil } func (fl *FakeLessor) GetLease(item LeaseItem) LeaseID { return 0 } func (fl *FakeLessor) Detach(id LeaseID, items []LeaseItem) error { return nil } func (fl *FakeLessor) Promote(extend time.Duration) {} func (fl *FakeLessor) Demote() {} func (fl *FakeLessor) Renew(id LeaseID) (int64, error) { return 10, nil } func (fl *FakeLessor) Lookup(id LeaseID) *Lease { if _, ok := fl.LeaseSet[id]; ok { return &Lease{ID: id} } return nil } func (fl *FakeLessor) Leases() []*Lease { return nil } func (fl *FakeLessor) ExpiredLeasesC() <-chan []*Lease { return nil } func (fl *FakeLessor) Recover(b backend.Backend, rd RangeDeleter) {} func (fl *FakeLessor) Stop() {} type FakeTxnDelete struct { backend.BatchTx } func (ftd *FakeTxnDelete) DeleteRange(key, end []byte) (n, rev int64) { return 0, 0 } func (ftd *FakeTxnDelete) End() { ftd.Unlock() } ================================================ FILE: server/lease/lessor_bench_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lease import ( "math/rand" "testing" "time" "go.uber.org/zap" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) func BenchmarkLessorGrant1000(b *testing.B) { benchmarkLessorGrant(1000, b) } func BenchmarkLessorGrant100000(b *testing.B) { benchmarkLessorGrant(100000, b) } func BenchmarkLessorRevoke1000(b *testing.B) { benchmarkLessorRevoke(1000, b) } func BenchmarkLessorRevoke100000(b *testing.B) { benchmarkLessorRevoke(100000, b) } func BenchmarkLessorRenew1000(b *testing.B) { benchmarkLessorRenew(1000, b) } func BenchmarkLessorRenew100000(b *testing.B) { benchmarkLessorRenew(100000, b) } // BenchmarkLessorFindExpired10000 uses findExpired10000 replace findExpired1000, which takes too long. func BenchmarkLessorFindExpired10000(b *testing.B) { benchmarkLessorFindExpired(10000, b) } func BenchmarkLessorFindExpired100000(b *testing.B) { benchmarkLessorFindExpired(100000, b) } const ( // minTTL keep lease will not auto expire in benchmark minTTL = 1000 // maxTTL control repeat probability of ttls maxTTL = 2000 ) func randomTTL(n int, min, max int64) (out []int64) { for i := 0; i < n; i++ { out = append(out, rand.Int63n(max-min)+min) } return out } // demote lessor from being the primary, but don't change any lease's expiry func demote(le *lessor) { le.mu.Lock() defer le.mu.Unlock() close(le.demotec) le.demotec = nil } // return new lessor and tearDown to release resource func setUp(tb testing.TB) (le *lessor, tearDown func()) { lg := zap.NewNop() be, _ := betesting.NewDefaultTmpBackend(tb) // MinLeaseTTL is negative, so we can grant expired lease in benchmark. // ExpiredLeasesRetryInterval should small, so benchmark of findExpired will recheck expired lease. le = newLessor(lg, be, nil, LessorConfig{MinLeaseTTL: -1000, ExpiredLeasesRetryInterval: 10 * time.Microsecond}) le.SetRangeDeleter(func() TxnDelete { ftd := &FakeTxnDelete{be.BatchTx()} ftd.Lock() return ftd }) le.Promote(0) return le, func() { le.Stop() be.Close() } } func benchmarkLessorGrant(benchSize int, b *testing.B) { ttls := randomTTL(benchSize, minTTL, maxTTL) var le *lessor var tearDown func() b.ResetTimer() for i := 0; i < b.N; { b.StopTimer() if tearDown != nil { tearDown() } le, tearDown = setUp(b) b.StartTimer() for j := 1; j <= benchSize; j++ { le.Grant(LeaseID(j), ttls[j-1]) } i += benchSize } b.StopTimer() if tearDown != nil { tearDown() } } func benchmarkLessorRevoke(benchSize int, b *testing.B) { ttls := randomTTL(benchSize, minTTL, maxTTL) var le *lessor var tearDown func() b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() if tearDown != nil { tearDown() } le, tearDown = setUp(b) for j := 1; j <= benchSize; j++ { le.Grant(LeaseID(j), ttls[j-1]) } b.StartTimer() for j := 1; j <= benchSize; j++ { le.Revoke(LeaseID(j)) } i += benchSize } b.StopTimer() if tearDown != nil { tearDown() } } func benchmarkLessorRenew(benchSize int, b *testing.B) { ttls := randomTTL(benchSize, minTTL, maxTTL) var le *lessor var tearDown func() b.ResetTimer() for i := 0; i < b.N; { b.StopTimer() if tearDown != nil { tearDown() } le, tearDown = setUp(b) for j := 1; j <= benchSize; j++ { le.Grant(LeaseID(j), ttls[j-1]) } b.StartTimer() for j := 1; j <= benchSize; j++ { le.Renew(LeaseID(j)) } i += benchSize } b.StopTimer() if tearDown != nil { tearDown() } } func benchmarkLessorFindExpired(benchSize int, b *testing.B) { // 50% lease are expired. ttls := randomTTL(benchSize, -500, 500) findExpiredLimit := 50 var le *lessor var tearDown func() b.ResetTimer() for i := 0; i < b.N; { b.StopTimer() if tearDown != nil { tearDown() } le, tearDown = setUp(b) for j := 1; j <= benchSize; j++ { le.Grant(LeaseID(j), ttls[j-1]) } // lessor's runLoop should not call findExpired demote(le) b.StartTimer() // refresh fixture after pop all expired lease for ; ; i++ { le.mu.Lock() ls := le.findExpiredLeases(findExpiredLimit) if len(ls) == 0 { le.mu.Unlock() break } le.mu.Unlock() // simulation: revoke lease after expired b.StopTimer() for _, lease := range ls { le.Revoke(lease.ID) } b.StartTimer() } } b.StopTimer() if tearDown != nil { tearDown() } } ================================================ FILE: server/lease/lessor_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lease import ( "context" "errors" "fmt" "os" "path/filepath" "reflect" "sort" "sync" "testing" "time" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "go.uber.org/zap/zaptest" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" ) const ( minLeaseTTL = int64(5) minLeaseTTLDuration = time.Duration(minLeaseTTL) * time.Second ) // TestLessorGrant ensures Lessor can grant wanted lease. // The granted lease should have a unique ID with a term // that is greater than minLeaseTTL. func TestLessorGrant(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) defer be.Close() le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL}) defer le.Stop() le.Promote(0) l, err := le.Grant(1, 1) if err != nil { t.Fatalf("could not grant lease 1 (%v)", err) } if l.ttl != minLeaseTTL { t.Fatalf("ttl = %v, expect minLeaseTTL %v", l.ttl, minLeaseTTL) } gl := le.Lookup(l.ID) if !reflect.DeepEqual(gl, l) { t.Errorf("lease = %v, want %v", gl, l) } if l.Remaining() < minLeaseTTLDuration-time.Second { t.Errorf("term = %v, want at least %v", l.Remaining(), minLeaseTTLDuration-time.Second) } _, err = le.Grant(1, 1) if err == nil { t.Errorf("allocated the same lease") } var nl *Lease nl, err = le.Grant(2, 1) if err != nil { t.Errorf("could not grant lease 2 (%v)", err) } if nl.ID == l.ID { t.Errorf("new lease.id = %x, want != %x", nl.ID, l.ID) } lss := []*Lease{gl, nl} leases := le.Leases() for i := range lss { if lss[i].ID != leases[i].ID { t.Fatalf("lease ID expected %d, got %d", lss[i].ID, leases[i].ID) } if lss[i].ttl != leases[i].ttl { t.Fatalf("ttl expected %d, got %d", lss[i].ttl, leases[i].ttl) } } tx := be.BatchTx() tx.Lock() defer tx.Unlock() lpb := schema.MustUnsafeGetLease(tx, int64(l.ID)) if lpb == nil { t.Errorf("lpb = %d, want not nil", lpb) } } // TestLeaseConcurrentKeys ensures Lease.Keys method calls are guarded // from concurrent map writes on 'itemSet'. func TestLeaseConcurrentKeys(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) defer be.Close() le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL}) defer le.Stop() le.SetRangeDeleter(func() TxnDelete { return newFakeDeleter(be) }) // grant a lease with long term (100 seconds) to // avoid early termination during the test. l, err := le.Grant(1, 100) if err != nil { t.Fatalf("could not grant lease for 100s ttl (%v)", err) } itemn := 10 items := make([]LeaseItem, itemn) for i := 0; i < itemn; i++ { items[i] = LeaseItem{Key: fmt.Sprintf("foo%d", i)} } if err = le.Attach(l.ID, items); err != nil { t.Fatalf("failed to attach items to the lease: %v", err) } donec := make(chan struct{}) go func() { le.Detach(l.ID, items) close(donec) }() var wg sync.WaitGroup wg.Add(itemn) for i := 0; i < itemn; i++ { go func() { defer wg.Done() l.Keys() }() } <-donec wg.Wait() } // TestLessorRevoke ensures Lessor can revoke a lease. // The items in the revoked lease should be removed from // the backend. // The revoked lease cannot be got from Lessor again. func TestLessorRevoke(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) defer be.Close() le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL}) defer le.Stop() var fd *fakeDeleter le.SetRangeDeleter(func() TxnDelete { fd = newFakeDeleter(be) return fd }) // grant a lease with long term (100 seconds) to // avoid early termination during the test. l, err := le.Grant(1, 100) if err != nil { t.Fatalf("could not grant lease for 100s ttl (%v)", err) } items := []LeaseItem{ {"foo"}, {"bar"}, } if err = le.Attach(l.ID, items); err != nil { t.Fatalf("failed to attach items to the lease: %v", err) } if err = le.Revoke(l.ID); err != nil { t.Fatal("failed to revoke lease:", err) } if le.Lookup(l.ID) != nil { t.Errorf("got revoked lease %x", l.ID) } wdeleted := []string{"bar_", "foo_"} sort.Strings(fd.deleted) if !reflect.DeepEqual(fd.deleted, wdeleted) { t.Errorf("deleted= %v, want %v", fd.deleted, wdeleted) } tx := be.BatchTx() tx.Lock() defer tx.Unlock() lpb := schema.MustUnsafeGetLease(tx, int64(l.ID)) if lpb != nil { t.Errorf("lpb = %d, want nil", lpb) } } func renew(t *testing.T, le *lessor, id LeaseID) int64 { ch := make(chan int64, 1) errch := make(chan error, 1) go func() { ttl, err := le.Renew(id) if err != nil { errch <- err } else { ch <- ttl } }() select { case ttl := <-ch: return ttl case err := <-errch: t.Fatalf("failed to renew lease (%v)", err) case <-time.After(10 * time.Second): t.Fatal("timed out while renewing lease") } panic("unreachable") } // TestLessorRenew ensures Lessor can renew an existing lease. func TestLessorRenew(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer be.Close() defer os.RemoveAll(dir) le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL}) defer le.Stop() le.Promote(0) l, err := le.Grant(1, minLeaseTTL) if err != nil { t.Fatalf("failed to grant lease (%v)", err) } // manually change the ttl field le.mu.Lock() l.ttl = 10 le.mu.Unlock() ttl := renew(t, le, l.ID) if ttl != l.ttl { t.Errorf("ttl = %d, want %d", ttl, l.ttl) } l = le.Lookup(l.ID) if l.Remaining() < 9*time.Second { t.Errorf("failed to renew the lease") } } func TestLessorRenewWithCheckpointer(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer be.Close() defer os.RemoveAll(dir) le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL}) fakerCheckerpointer := func(ctx context.Context, cp *pb.LeaseCheckpointRequest) error { for _, cp := range cp.GetCheckpoints() { le.Checkpoint(LeaseID(cp.GetID()), cp.GetRemaining_TTL()) } return nil } defer le.Stop() // Set checkpointer le.SetCheckpointer(fakerCheckerpointer) le.Promote(0) l, err := le.Grant(1, minLeaseTTL) if err != nil { t.Fatalf("failed to grant lease (%v)", err) } // manually change the ttl field le.mu.Lock() l.ttl = 10 l.remainingTTL = 10 le.mu.Unlock() ttl := renew(t, le, l.ID) if ttl != l.ttl { t.Errorf("ttl = %d, want %d", ttl, l.ttl) } if l.remainingTTL != 0 { t.Fatalf("remianingTTL = %d, want %d", l.remainingTTL, 0) } l = le.Lookup(l.ID) if l.Remaining() < 9*time.Second { t.Errorf("failed to renew the lease") } } // TestLessorRenewExtendPileup ensures Lessor extends leases on promotion if too many // expire at the same time. func TestLessorRenewExtendPileup(t *testing.T) { leaseRevokeRate := 10 lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL, leaseRevokeRate: leaseRevokeRate}) ttl := int64(10) for i := 1; i <= le.leaseRevokeRate*10; i++ { if _, err := le.Grant(LeaseID(2*i), ttl); err != nil { t.Fatal(err) } // ttls that overlap spillover for ttl=10 if _, err := le.Grant(LeaseID(2*i+1), ttl+1); err != nil { t.Fatal(err) } } // simulate stop and recovery le.Stop() be.Close() bcfg := backend.DefaultBackendConfig(lg) bcfg.Path = filepath.Join(dir, "be") be = backend.New(bcfg) defer be.Close() le = newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL, leaseRevokeRate: leaseRevokeRate}) defer le.Stop() // extend after recovery should extend expiration on lease pile-up le.Promote(0) windowCounts := make(map[int64]int) for _, l := range le.leaseMap { // round up slightly for baseline ttl s := int64(l.Remaining().Seconds() + 0.1) windowCounts[s]++ } for i := ttl; i < ttl+20; i++ { c := windowCounts[i] if c > le.leaseRevokeRate { t.Errorf("expected at most %d expiring at %ds, got %d", le.leaseRevokeRate, i, c) } if c < le.leaseRevokeRate/2 { t.Errorf("expected at least %d expiring at %ds, got %d", le.leaseRevokeRate/2, i, c) } } } func TestLessorDetach(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) defer be.Close() le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL}) defer le.Stop() le.SetRangeDeleter(func() TxnDelete { return newFakeDeleter(be) }) // grant a lease with long term (100 seconds) to // avoid early termination during the test. l, err := le.Grant(1, 100) if err != nil { t.Fatalf("could not grant lease for 100s ttl (%v)", err) } items := []LeaseItem{ {"foo"}, {"bar"}, } if err := le.Attach(l.ID, items); err != nil { t.Fatalf("failed to attach items to the lease: %v", err) } if err := le.Detach(l.ID, items[0:1]); err != nil { t.Fatalf("failed to de-attach items to the lease: %v", err) } l = le.Lookup(l.ID) if len(l.itemSet) != 1 { t.Fatalf("len(l.itemSet) = %d, failed to de-attach items", len(l.itemSet)) } if _, ok := l.itemSet[LeaseItem{"bar"}]; !ok { t.Fatalf("de-attached wrong item, want %q exists", "bar") } } // TestLessorRecover ensures Lessor recovers leases from // persist backend. func TestLessorRecover(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) defer be.Close() le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL}) defer le.Stop() l1, err1 := le.Grant(1, 10) l2, err2 := le.Grant(2, 20) if err1 != nil || err2 != nil { t.Fatalf("could not grant initial leases (%v, %v)", err1, err2) } // Create a new lessor with the same backend nle := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL}) defer nle.Stop() nl1 := nle.Lookup(l1.ID) if nl1 == nil || nl1.ttl != l1.ttl { t.Errorf("nl1 = %v, want nl1.ttl= %d", nl1.ttl, l1.ttl) } nl2 := nle.Lookup(l2.ID) if nl2 == nil || nl2.ttl != l2.ttl { t.Errorf("nl2 = %v, want nl2.ttl= %d", nl2.ttl, l2.ttl) } } func TestLessorExpire(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) defer be.Close() testMinTTL := int64(1) le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: testMinTTL}) defer le.Stop() le.Promote(1 * time.Second) l, err := le.Grant(1, testMinTTL) if err != nil { t.Fatalf("failed to create lease: %v", err) } select { case el := <-le.ExpiredLeasesC(): if el[0].ID != l.ID { t.Fatalf("expired id = %x, want %x", el[0].ID, l.ID) } case <-time.After(10 * time.Second): t.Fatalf("failed to receive expired lease") } donec := make(chan struct{}, 1) go func() { // expired lease cannot be renewed if _, err := le.Renew(l.ID); !errors.Is(err, ErrLeaseNotFound) { t.Errorf("unexpected renew") } donec <- struct{}{} }() select { case <-donec: t.Fatalf("renew finished before lease revocation") case <-time.After(50 * time.Millisecond): } // expired lease can be revoked if err := le.Revoke(l.ID); err != nil { t.Fatalf("failed to revoke expired lease: %v", err) } select { case <-donec: case <-time.After(10 * time.Second): t.Fatalf("renew has not returned after lease revocation") } } func TestLessorExpireAndDemote(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) defer be.Close() testMinTTL := int64(1) le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: testMinTTL}) defer le.Stop() le.Promote(1 * time.Second) l, err := le.Grant(1, testMinTTL) if err != nil { t.Fatalf("failed to create lease: %v", err) } select { case el := <-le.ExpiredLeasesC(): if el[0].ID != l.ID { t.Fatalf("expired id = %x, want %x", el[0].ID, l.ID) } case <-time.After(10 * time.Second): t.Fatalf("failed to receive expired lease") } donec := make(chan struct{}, 1) go func() { // expired lease cannot be renewed if _, err := le.Renew(l.ID); !errors.Is(err, ErrNotPrimary) { t.Errorf("unexpected renew: %v", err) } donec <- struct{}{} }() select { case <-donec: t.Fatalf("renew finished before demotion") case <-time.After(50 * time.Millisecond): } // demote will cause the renew request to fail with ErrNotPrimary le.Demote() select { case <-donec: case <-time.After(10 * time.Second): t.Fatalf("renew has not returned after lessor demotion") } } func TestLessorMaxTTL(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) defer be.Close() le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL}) defer le.Stop() _, err := le.Grant(1, MaxLeaseTTL+1) if !errors.Is(err, ErrLeaseTTLTooLarge) { t.Fatalf("grant unexpectedly succeeded") } } func TestLessorCheckpointScheduling(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) defer be.Close() le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL, CheckpointInterval: 1 * time.Second}) defer le.Stop() le.minLeaseTTL = 1 checkpointedC := make(chan struct{}) le.SetCheckpointer(func(ctx context.Context, lc *pb.LeaseCheckpointRequest) error { close(checkpointedC) if len(lc.Checkpoints) != 1 { t.Errorf("expected 1 checkpoint but got %d", len(lc.Checkpoints)) } c := lc.Checkpoints[0] if c.Remaining_TTL != 1 { t.Errorf("expected checkpoint to be called with Remaining_TTL=%d but got %d", 1, c.Remaining_TTL) } return nil }) _, err := le.Grant(1, 2) if err != nil { t.Fatal(err) } le.Promote(0) // TODO: Is there any way to avoid doing this wait? Lease TTL granularity is in seconds. select { case <-checkpointedC: case <-time.After(2 * time.Second): t.Fatal("expected checkpointer to be called, but it was not") } } func TestLessorCheckpointsRestoredOnPromote(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) defer be.Close() le := newLessor(lg, be, clusterLatest(), LessorConfig{MinLeaseTTL: minLeaseTTL}) defer le.Stop() l, err := le.Grant(1, 10) if err != nil { t.Fatal(err) } le.Checkpoint(l.ID, 5) le.Promote(0) remaining := l.Remaining().Seconds() if !(remaining > 4 && remaining < 5) { t.Fatalf("expected expiry to be less than 1s in the future, but got %f seconds", remaining) } } func TestLessorCheckpointPersistenceAfterRestart(t *testing.T) { const ttl int64 = 10 const checkpointTTL int64 = 5 tcs := []struct { name string cluster cluster checkpointPersist bool expectRemainingTTL int64 }{ { name: "Etcd v3.6 and newer persist remainingTTL on checkpoint", cluster: clusterLatest(), expectRemainingTTL: checkpointTTL, }, { name: "Etcd v3.5 and older persist remainingTTL if CheckpointPersist is set", cluster: clusterV3_5(), checkpointPersist: true, expectRemainingTTL: checkpointTTL, }, { name: "Etcd with version unknown persists remainingTTL if CheckpointPersist is set", cluster: clusterNil(), checkpointPersist: true, expectRemainingTTL: checkpointTTL, }, { name: "Etcd v3.5 and older reset remainingTTL on checkpoint", cluster: clusterV3_5(), expectRemainingTTL: ttl, }, { name: "Etcd with version unknown fallbacks to v3.5 behavior", cluster: clusterNil(), expectRemainingTTL: ttl, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { lg := zap.NewNop() dir, be := NewTestBackend(t) defer os.RemoveAll(dir) defer be.Close() cfg := LessorConfig{MinLeaseTTL: minLeaseTTL} cfg.CheckpointPersist = tc.checkpointPersist le := newLessor(lg, be, tc.cluster, cfg) l, err := le.Grant(2, ttl) if err != nil { t.Fatal(err) } if l.getRemainingTTL() != ttl { t.Errorf("getRemainingTTL() = %d, expected: %d", l.getRemainingTTL(), ttl) } le.Checkpoint(2, checkpointTTL) if l.getRemainingTTL() != checkpointTTL { t.Errorf("getRemainingTTL() = %d, expected: %d", l.getRemainingTTL(), checkpointTTL) } le.Stop() le2 := newLessor(lg, be, clusterLatest(), cfg) l = le2.Lookup(2) if l.getRemainingTTL() != tc.expectRemainingTTL { t.Errorf("getRemainingTTL() = %d, expected: %d", l.getRemainingTTL(), tc.expectRemainingTTL) } }) } } type fakeDeleter struct { deleted []string tx backend.BatchTx } func newFakeDeleter(be backend.Backend) *fakeDeleter { fd := &fakeDeleter{nil, be.BatchTx()} fd.tx.Lock() return fd } func (fd *fakeDeleter) End() { fd.tx.Unlock() } func (fd *fakeDeleter) DeleteRange(key, end []byte) (int64, int64) { fd.deleted = append(fd.deleted, string(key)+"_"+string(end)) return 0, 0 } func NewTestBackend(t *testing.T) (string, backend.Backend) { lg := zaptest.NewLogger(t) tmpPath := t.TempDir() bcfg := backend.DefaultBackendConfig(lg) bcfg.Path = filepath.Join(tmpPath, "be") return tmpPath, backend.New(bcfg) } func clusterLatest() cluster { return fakeCluster{semver.New(version.Cluster(version.Version) + ".0")} } func clusterV3_5() cluster { return fakeCluster{semver.New("3.5.0")} } func clusterNil() cluster { return fakeCluster{} } type fakeCluster struct { version *semver.Version } func (c fakeCluster) Version() *semver.Version { return c.version } ================================================ FILE: server/lease/metrics.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lease import ( "github.com/prometheus/client_golang/prometheus" ) var ( leaseGranted = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "lease", Name: "granted_total", Help: "The total number of granted leases.", }) leaseRevoked = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "lease", Name: "revoked_total", Help: "The total number of revoked leases.", }) leaseRenewed = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "lease", Name: "renewed_total", Help: "The number of renewed leases seen by the leader.", }) leaseTotalTTLs = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "lease", Name: "ttl_total", Help: "Bucketed histogram of lease TTLs.", // 1 second -> 3 months Buckets: prometheus.ExponentialBuckets(1, 2, 24), }, ) ) func init() { prometheus.MustRegister(leaseGranted) prometheus.MustRegister(leaseRevoked) prometheus.MustRegister(leaseRenewed) prometheus.MustRegister(leaseTotalTTLs) } ================================================ FILE: server/main.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package main is a simple wrapper of the real etcd entrypoint package // (located at go.etcd.io/etcd/etcdmain) to ensure that etcd is still // "go getable"; e.g. `go get go.etcd.io/etcd` works as expected and // builds a binary in $GOBIN/etcd // // This package should NOT be extended or modified in any way; to modify the // etcd binary, work in the `go.etcd.io/etcd/etcdmain` package. package main import ( "os" "go.etcd.io/etcd/server/v3/etcdmain" ) func main() { etcdmain.Main(os.Args) } ================================================ FILE: server/mock/mockstorage/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package mockstorage provides mock implementations for etcdserver's storage interface. package mockstorage ================================================ FILE: server/mock/mockstorage/storage_recorder.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mockstorage import ( "github.com/coreos/go-semver/semver" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) type StorageRecorder struct { testutil.Recorder dbPath string // must have '/' suffix if set } func NewStorageRecorder(db string) *StorageRecorder { return &StorageRecorder{&testutil.RecorderBuffered{}, db} } func NewStorageRecorderStream(db string) *StorageRecorder { return &StorageRecorder{testutil.NewRecorderStream(), db} } func (p *StorageRecorder) Save(st raftpb.HardState, ents []raftpb.Entry) error { p.Record(testutil.Action{Name: "Save"}) return nil } func (p *StorageRecorder) SaveSnap(st raftpb.Snapshot) error { if !raft.IsEmptySnap(st) { p.Record(testutil.Action{Name: "SaveSnap"}) } return nil } func (p *StorageRecorder) Release(st raftpb.Snapshot) error { if !raft.IsEmptySnap(st) { p.Record(testutil.Action{Name: "Release"}) } return nil } func (p *StorageRecorder) Sync() error { p.Record(testutil.Action{Name: "Sync"}) return nil } func (p *StorageRecorder) Close() error { return nil } func (p *StorageRecorder) MinimalEtcdVersion() *semver.Version { return nil } ================================================ FILE: server/mock/mockstore/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package mockstore provides mock structures for the etcd store package. package mockstore ================================================ FILE: server/mock/mockstore/store_recorder.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mockstore import ( "time" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" ) // StoreRecorder provides a Store interface with a testutil.Recorder type StoreRecorder struct { v2store.Store testutil.Recorder } // storeRecorder records all the methods it receives. // storeRecorder DOES NOT work as a actual v2store. // It always returns invalid empty response and no error. type storeRecorder struct { v2store.Store testutil.Recorder } func NewNop() v2store.Store { return &storeRecorder{Recorder: &testutil.RecorderBuffered{}} } func NewRecorder() *StoreRecorder { sr := &storeRecorder{Recorder: &testutil.RecorderBuffered{}} return &StoreRecorder{Store: sr, Recorder: sr.Recorder} } func NewRecorderStream() *StoreRecorder { sr := &storeRecorder{Recorder: testutil.NewRecorderStream()} return &StoreRecorder{Store: sr, Recorder: sr.Recorder} } func (s *storeRecorder) Version() int { return 0 } func (s *storeRecorder) Index() uint64 { return 0 } func (s *storeRecorder) Get(path string, recursive, sorted bool) (*v2store.Event, error) { s.Record(testutil.Action{ Name: "Get", Params: []any{path, recursive, sorted}, }) return &v2store.Event{}, nil } func (s *storeRecorder) Set(path string, dir bool, val string, expireOpts v2store.TTLOptionSet) (*v2store.Event, error) { s.Record(testutil.Action{ Name: "Set", Params: []any{path, dir, val, expireOpts}, }) return &v2store.Event{}, nil } func (s *storeRecorder) Update(path, val string, expireOpts v2store.TTLOptionSet) (*v2store.Event, error) { s.Record(testutil.Action{ Name: "Update", Params: []any{path, val, expireOpts}, }) return &v2store.Event{}, nil } func (s *storeRecorder) Create(path string, dir bool, val string, uniq bool, expireOpts v2store.TTLOptionSet) (*v2store.Event, error) { s.Record(testutil.Action{ Name: "Create", Params: []any{path, dir, val, uniq, expireOpts}, }) return &v2store.Event{}, nil } func (s *storeRecorder) CompareAndSwap(path, prevVal string, prevIdx uint64, val string, expireOpts v2store.TTLOptionSet) (*v2store.Event, error) { s.Record(testutil.Action{ Name: "CompareAndSwap", Params: []any{path, prevVal, prevIdx, val, expireOpts}, }) return &v2store.Event{}, nil } func (s *storeRecorder) Delete(path string, dir, recursive bool) (*v2store.Event, error) { s.Record(testutil.Action{ Name: "Delete", Params: []any{path, dir, recursive}, }) return &v2store.Event{}, nil } func (s *storeRecorder) CompareAndDelete(path, prevVal string, prevIdx uint64) (*v2store.Event, error) { s.Record(testutil.Action{ Name: "CompareAndDelete", Params: []any{path, prevVal, prevIdx}, }) return &v2store.Event{}, nil } func (s *storeRecorder) Watch(_ string, _, _ bool, _ uint64) (v2store.Watcher, error) { s.Record(testutil.Action{Name: "Watch"}) return v2store.NewNopWatcher(), nil } func (s *storeRecorder) Save() ([]byte, error) { s.Record(testutil.Action{Name: "Save"}) return nil, nil } func (s *storeRecorder) Recovery(b []byte) error { s.Record(testutil.Action{Name: "Recovery"}) return nil } func (s *storeRecorder) SaveNoCopy() ([]byte, error) { s.Record(testutil.Action{Name: "SaveNoCopy"}) return nil, nil } func (s *storeRecorder) Clone() v2store.Store { s.Record(testutil.Action{Name: "Clone"}) return s } //revive:disable:var-naming func (s *storeRecorder) JsonStats() []byte { return nil } //revive:enable:var-naming func (s *storeRecorder) DeleteExpiredKeys(cutoff time.Time) { s.Record(testutil.Action{ Name: "DeleteExpiredKeys", Params: []any{cutoff}, }) } func (s *storeRecorder) HasTTLKeys() bool { s.Record(testutil.Action{ Name: "HasTTLKeys", }) return true } // errStoreRecorder is a storeRecorder, but returns the given error on // Get, Watch methods. type errStoreRecorder struct { storeRecorder err error } func NewErrRecorder(err error) *StoreRecorder { sr := &errStoreRecorder{err: err} sr.Recorder = &testutil.RecorderBuffered{} return &StoreRecorder{Store: sr, Recorder: sr.Recorder} } func (s *errStoreRecorder) Get(path string, recursive, sorted bool) (*v2store.Event, error) { s.storeRecorder.Get(path, recursive, sorted) return nil, s.err } func (s *errStoreRecorder) Watch(path string, recursive, sorted bool, index uint64) (v2store.Watcher, error) { s.storeRecorder.Watch(path, recursive, sorted, index) return nil, s.err } ================================================ FILE: server/mock/mockwait/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package mockwait provides mock implementations for pkg/wait. package mockwait ================================================ FILE: server/mock/mockwait/wait_recorder.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mockwait import ( "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/pkg/v3/wait" ) type WaitRecorder struct { wait.Wait testutil.Recorder } type waitRecorder struct { testutil.RecorderBuffered } func NewRecorder() *WaitRecorder { wr := &waitRecorder{} return &WaitRecorder{Wait: wr, Recorder: wr} } func NewNop() wait.Wait { return NewRecorder() } func (w *waitRecorder) Register(id uint64) <-chan any { w.Record(testutil.Action{Name: "Register"}) return nil } func (w *waitRecorder) Trigger(id uint64, x any) { w.Record(testutil.Action{Name: "Trigger"}) } func (w *waitRecorder) IsRegistered(id uint64) bool { panic("waitRecorder.IsRegistered() shouldn't be called") } ================================================ FILE: server/proxy/grpcproxy/adapter/auth_client_adapter.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adapter import ( "context" grpc "google.golang.org/grpc" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) type as2ac struct{ as pb.AuthServer } func AuthServerToAuthClient(as pb.AuthServer) pb.AuthClient { return &as2ac{as} } func (s *as2ac) AuthEnable(ctx context.Context, in *pb.AuthEnableRequest, opts ...grpc.CallOption) (*pb.AuthEnableResponse, error) { return s.as.AuthEnable(ctx, in) } func (s *as2ac) AuthDisable(ctx context.Context, in *pb.AuthDisableRequest, opts ...grpc.CallOption) (*pb.AuthDisableResponse, error) { return s.as.AuthDisable(ctx, in) } func (s *as2ac) AuthStatus(ctx context.Context, in *pb.AuthStatusRequest, opts ...grpc.CallOption) (*pb.AuthStatusResponse, error) { return s.as.AuthStatus(ctx, in) } func (s *as2ac) Authenticate(ctx context.Context, in *pb.AuthenticateRequest, opts ...grpc.CallOption) (*pb.AuthenticateResponse, error) { return s.as.Authenticate(ctx, in) } func (s *as2ac) RoleAdd(ctx context.Context, in *pb.AuthRoleAddRequest, opts ...grpc.CallOption) (*pb.AuthRoleAddResponse, error) { return s.as.RoleAdd(ctx, in) } func (s *as2ac) RoleDelete(ctx context.Context, in *pb.AuthRoleDeleteRequest, opts ...grpc.CallOption) (*pb.AuthRoleDeleteResponse, error) { return s.as.RoleDelete(ctx, in) } func (s *as2ac) RoleGet(ctx context.Context, in *pb.AuthRoleGetRequest, opts ...grpc.CallOption) (*pb.AuthRoleGetResponse, error) { return s.as.RoleGet(ctx, in) } func (s *as2ac) RoleList(ctx context.Context, in *pb.AuthRoleListRequest, opts ...grpc.CallOption) (*pb.AuthRoleListResponse, error) { return s.as.RoleList(ctx, in) } func (s *as2ac) RoleRevokePermission(ctx context.Context, in *pb.AuthRoleRevokePermissionRequest, opts ...grpc.CallOption) (*pb.AuthRoleRevokePermissionResponse, error) { return s.as.RoleRevokePermission(ctx, in) } func (s *as2ac) RoleGrantPermission(ctx context.Context, in *pb.AuthRoleGrantPermissionRequest, opts ...grpc.CallOption) (*pb.AuthRoleGrantPermissionResponse, error) { return s.as.RoleGrantPermission(ctx, in) } func (s *as2ac) UserDelete(ctx context.Context, in *pb.AuthUserDeleteRequest, opts ...grpc.CallOption) (*pb.AuthUserDeleteResponse, error) { return s.as.UserDelete(ctx, in) } func (s *as2ac) UserAdd(ctx context.Context, in *pb.AuthUserAddRequest, opts ...grpc.CallOption) (*pb.AuthUserAddResponse, error) { return s.as.UserAdd(ctx, in) } func (s *as2ac) UserGet(ctx context.Context, in *pb.AuthUserGetRequest, opts ...grpc.CallOption) (*pb.AuthUserGetResponse, error) { return s.as.UserGet(ctx, in) } func (s *as2ac) UserList(ctx context.Context, in *pb.AuthUserListRequest, opts ...grpc.CallOption) (*pb.AuthUserListResponse, error) { return s.as.UserList(ctx, in) } func (s *as2ac) UserGrantRole(ctx context.Context, in *pb.AuthUserGrantRoleRequest, opts ...grpc.CallOption) (*pb.AuthUserGrantRoleResponse, error) { return s.as.UserGrantRole(ctx, in) } func (s *as2ac) UserRevokeRole(ctx context.Context, in *pb.AuthUserRevokeRoleRequest, opts ...grpc.CallOption) (*pb.AuthUserRevokeRoleResponse, error) { return s.as.UserRevokeRole(ctx, in) } func (s *as2ac) UserChangePassword(ctx context.Context, in *pb.AuthUserChangePasswordRequest, opts ...grpc.CallOption) (*pb.AuthUserChangePasswordResponse, error) { return s.as.UserChangePassword(ctx, in) } ================================================ FILE: server/proxy/grpcproxy/adapter/chan_stream.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adapter import ( "context" "maps" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" ) // chanServerStream implements grpc.ServerStream with a chanStream type chanServerStream struct { headerc chan<- metadata.MD trailerc chan<- metadata.MD grpc.Stream headers []metadata.MD } func (ss *chanServerStream) SendHeader(md metadata.MD) error { if ss.headerc == nil { return errAlreadySentHeader } outmd := make(map[string][]string) for _, h := range append(ss.headers, md) { maps.Copy(outmd, h) } select { case ss.headerc <- outmd: ss.headerc = nil ss.headers = nil return nil case <-ss.Context().Done(): //nolint:staticcheck // TODO: remove for a supported version } return ss.Context().Err() //nolint:staticcheck // TODO: remove for a supported version } func (ss *chanServerStream) SetHeader(md metadata.MD) error { if ss.headerc == nil { return errAlreadySentHeader } ss.headers = append(ss.headers, md) return nil } func (ss *chanServerStream) SetTrailer(md metadata.MD) { ss.trailerc <- md } // chanClientStream implements grpc.ClientStream with a chanStream type chanClientStream struct { headerc <-chan metadata.MD trailerc <-chan metadata.MD *chanStream } func (cs *chanClientStream) Header() (metadata.MD, error) { select { case md := <-cs.headerc: return md, nil case <-cs.Context().Done(): } return nil, cs.Context().Err() } func (cs *chanClientStream) Trailer() metadata.MD { select { case md := <-cs.trailerc: return md case <-cs.Context().Done(): return nil } } func (cs *chanClientStream) CloseSend() error { close(cs.chanStream.sendc) return nil } // chanStream implements grpc.Stream using channels type chanStream struct { recvc <-chan any sendc chan<- any ctx context.Context cancel context.CancelFunc } func (s *chanStream) Context() context.Context { return s.ctx } func (s *chanStream) SendMsg(m any) error { select { case s.sendc <- m: if err, ok := m.(error); ok { return err } return nil case <-s.ctx.Done(): } return s.ctx.Err() } func (s *chanStream) RecvMsg(m any) error { v := m.(*any) for { select { case msg, ok := <-s.recvc: if !ok { return status.Error(codes.Canceled, "the client connection is closing") } if err, ok := msg.(error); ok { return err } *v = msg return nil case <-s.ctx.Done(): } if len(s.recvc) == 0 { // prioritize any pending recv messages over canceled context break } } return s.ctx.Err() } func newPipeStream(ctx context.Context, ssHandler func(chanServerStream) error) chanClientStream { // ch1 is buffered so server can send error on close ch1, ch2 := make(chan any, 1), make(chan any) headerc, trailerc := make(chan metadata.MD, 1), make(chan metadata.MD, 1) cctx, ccancel := context.WithCancel(ctx) cli := &chanStream{recvc: ch1, sendc: ch2, ctx: cctx, cancel: ccancel} cs := chanClientStream{headerc, trailerc, cli} sctx, scancel := context.WithCancel(ctx) srv := &chanStream{recvc: ch2, sendc: ch1, ctx: sctx, cancel: scancel} ss := chanServerStream{headerc, trailerc, srv, nil} go func() { if err := ssHandler(ss); err != nil { select { case srv.sendc <- err: case <-sctx.Done(): case <-cctx.Done(): } } scancel() ccancel() }() return cs } ================================================ FILE: server/proxy/grpcproxy/adapter/cluster_client_adapter.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adapter import ( "context" "google.golang.org/grpc" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) type cls2clc struct{ cls pb.ClusterServer } func ClusterServerToClusterClient(cls pb.ClusterServer) pb.ClusterClient { return &cls2clc{cls} } func (s *cls2clc) MemberList(ctx context.Context, r *pb.MemberListRequest, opts ...grpc.CallOption) (*pb.MemberListResponse, error) { return s.cls.MemberList(ctx, r) } func (s *cls2clc) MemberAdd(ctx context.Context, r *pb.MemberAddRequest, opts ...grpc.CallOption) (*pb.MemberAddResponse, error) { return s.cls.MemberAdd(ctx, r) } func (s *cls2clc) MemberUpdate(ctx context.Context, r *pb.MemberUpdateRequest, opts ...grpc.CallOption) (*pb.MemberUpdateResponse, error) { return s.cls.MemberUpdate(ctx, r) } func (s *cls2clc) MemberRemove(ctx context.Context, r *pb.MemberRemoveRequest, opts ...grpc.CallOption) (*pb.MemberRemoveResponse, error) { return s.cls.MemberRemove(ctx, r) } func (s *cls2clc) MemberPromote(ctx context.Context, r *pb.MemberPromoteRequest, opts ...grpc.CallOption) (*pb.MemberPromoteResponse, error) { return s.cls.MemberPromote(ctx, r) } ================================================ FILE: server/proxy/grpcproxy/adapter/doc.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package adapter provides gRPC adapters between client and server // gRPC interfaces without needing to go through a gRPC connection. package adapter ================================================ FILE: server/proxy/grpcproxy/adapter/election_client_adapter.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adapter import ( "context" "google.golang.org/grpc" "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb" ) type es2ec struct{ es v3electionpb.ElectionServer } func ElectionServerToElectionClient(es v3electionpb.ElectionServer) v3electionpb.ElectionClient { return &es2ec{es} } func (s *es2ec) Campaign(ctx context.Context, r *v3electionpb.CampaignRequest, opts ...grpc.CallOption) (*v3electionpb.CampaignResponse, error) { return s.es.Campaign(ctx, r) } func (s *es2ec) Proclaim(ctx context.Context, r *v3electionpb.ProclaimRequest, opts ...grpc.CallOption) (*v3electionpb.ProclaimResponse, error) { return s.es.Proclaim(ctx, r) } func (s *es2ec) Leader(ctx context.Context, r *v3electionpb.LeaderRequest, opts ...grpc.CallOption) (*v3electionpb.LeaderResponse, error) { return s.es.Leader(ctx, r) } func (s *es2ec) Resign(ctx context.Context, r *v3electionpb.ResignRequest, opts ...grpc.CallOption) (*v3electionpb.ResignResponse, error) { return s.es.Resign(ctx, r) } func (s *es2ec) Observe(ctx context.Context, in *v3electionpb.LeaderRequest, opts ...grpc.CallOption) (v3electionpb.Election_ObserveClient, error) { cs := newPipeStream(ctx, func(ss chanServerStream) error { return s.es.Observe(in, &es2ecServerStream{ss}) }) return &es2ecClientStream{cs}, nil } // es2ecClientStream implements Election_ObserveClient type es2ecClientStream struct{ chanClientStream } // es2ecServerStream implements Election_ObserveServer type es2ecServerStream struct{ chanServerStream } func (s *es2ecClientStream) Send(rr *v3electionpb.LeaderRequest) error { return s.SendMsg(rr) //nolint:staticcheck // TODO: remove for a supported version } func (s *es2ecClientStream) Recv() (*v3electionpb.LeaderResponse, error) { var v any if err := s.RecvMsg(&v); err != nil { //nolint:staticcheck // TODO: remove for a supported version return nil, err } return v.(*v3electionpb.LeaderResponse), nil } func (s *es2ecServerStream) Send(rr *v3electionpb.LeaderResponse) error { return s.SendMsg(rr) //nolint:staticcheck // TODO: remove for a supported version } func (s *es2ecServerStream) Recv() (*v3electionpb.LeaderRequest, error) { var v any if err := s.RecvMsg(&v); err != nil { //nolint:staticcheck // TODO: remove for a supported version return nil, err } return v.(*v3electionpb.LeaderRequest), nil } ================================================ FILE: server/proxy/grpcproxy/adapter/kv_client_adapter.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adapter import ( "context" grpc "google.golang.org/grpc" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) type kvs2kvc struct{ kvs pb.KVServer } func KvServerToKvClient(kvs pb.KVServer) pb.KVClient { return &kvs2kvc{kvs} } func (s *kvs2kvc) Range(ctx context.Context, in *pb.RangeRequest, opts ...grpc.CallOption) (*pb.RangeResponse, error) { return s.kvs.Range(ctx, in) } func (s *kvs2kvc) Put(ctx context.Context, in *pb.PutRequest, opts ...grpc.CallOption) (*pb.PutResponse, error) { return s.kvs.Put(ctx, in) } func (s *kvs2kvc) DeleteRange(ctx context.Context, in *pb.DeleteRangeRequest, opts ...grpc.CallOption) (*pb.DeleteRangeResponse, error) { return s.kvs.DeleteRange(ctx, in) } func (s *kvs2kvc) Txn(ctx context.Context, in *pb.TxnRequest, opts ...grpc.CallOption) (*pb.TxnResponse, error) { return s.kvs.Txn(ctx, in) } func (s *kvs2kvc) Compact(ctx context.Context, in *pb.CompactionRequest, opts ...grpc.CallOption) (*pb.CompactionResponse, error) { return s.kvs.Compact(ctx, in) } ================================================ FILE: server/proxy/grpcproxy/adapter/lease_client_adapter.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adapter import ( "context" "google.golang.org/grpc" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) type ls2lc struct { leaseServer pb.LeaseServer } func LeaseServerToLeaseClient(ls pb.LeaseServer) pb.LeaseClient { return &ls2lc{ls} } func (c *ls2lc) LeaseGrant(ctx context.Context, in *pb.LeaseGrantRequest, opts ...grpc.CallOption) (*pb.LeaseGrantResponse, error) { return c.leaseServer.LeaseGrant(ctx, in) } func (c *ls2lc) LeaseRevoke(ctx context.Context, in *pb.LeaseRevokeRequest, opts ...grpc.CallOption) (*pb.LeaseRevokeResponse, error) { return c.leaseServer.LeaseRevoke(ctx, in) } func (c *ls2lc) LeaseKeepAlive(ctx context.Context, opts ...grpc.CallOption) (pb.Lease_LeaseKeepAliveClient, error) { cs := newPipeStream(ctx, func(ss chanServerStream) error { return c.leaseServer.LeaseKeepAlive(&ls2lcServerStream{ss}) }) return &ls2lcClientStream{cs}, nil } func (c *ls2lc) LeaseTimeToLive(ctx context.Context, in *pb.LeaseTimeToLiveRequest, opts ...grpc.CallOption) (*pb.LeaseTimeToLiveResponse, error) { return c.leaseServer.LeaseTimeToLive(ctx, in) } func (c *ls2lc) LeaseLeases(ctx context.Context, in *pb.LeaseLeasesRequest, opts ...grpc.CallOption) (*pb.LeaseLeasesResponse, error) { return c.leaseServer.LeaseLeases(ctx, in) } // ls2lcClientStream implements Lease_LeaseKeepAliveClient type ls2lcClientStream struct{ chanClientStream } // ls2lcServerStream implements Lease_LeaseKeepAliveServer type ls2lcServerStream struct{ chanServerStream } func (s *ls2lcClientStream) Send(rr *pb.LeaseKeepAliveRequest) error { return s.SendMsg(rr) //nolint:staticcheck // TODO: remove for a supported version } func (s *ls2lcClientStream) Recv() (*pb.LeaseKeepAliveResponse, error) { var v any if err := s.RecvMsg(&v); err != nil { //nolint:staticcheck // TODO: remove for a supported version return nil, err } return v.(*pb.LeaseKeepAliveResponse), nil } func (s *ls2lcServerStream) Send(rr *pb.LeaseKeepAliveResponse) error { return s.SendMsg(rr) //nolint:staticcheck // TODO: remove for a supported version } func (s *ls2lcServerStream) Recv() (*pb.LeaseKeepAliveRequest, error) { var v any if err := s.RecvMsg(&v); err != nil { //nolint:staticcheck // TODO: remove for a supported version return nil, err } return v.(*pb.LeaseKeepAliveRequest), nil } ================================================ FILE: server/proxy/grpcproxy/adapter/lock_client_adapter.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adapter import ( "context" "google.golang.org/grpc" "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb" ) type ls2lsc struct{ ls v3lockpb.LockServer } func LockServerToLockClient(ls v3lockpb.LockServer) v3lockpb.LockClient { return &ls2lsc{ls} } func (s *ls2lsc) Lock(ctx context.Context, r *v3lockpb.LockRequest, opts ...grpc.CallOption) (*v3lockpb.LockResponse, error) { return s.ls.Lock(ctx, r) } func (s *ls2lsc) Unlock(ctx context.Context, r *v3lockpb.UnlockRequest, opts ...grpc.CallOption) (*v3lockpb.UnlockResponse, error) { return s.ls.Unlock(ctx, r) } ================================================ FILE: server/proxy/grpcproxy/adapter/maintenance_client_adapter.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adapter import ( "context" "google.golang.org/grpc" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) type mts2mtc struct{ mts pb.MaintenanceServer } func MaintenanceServerToMaintenanceClient(mts pb.MaintenanceServer) pb.MaintenanceClient { return &mts2mtc{mts} } func (s *mts2mtc) Alarm(ctx context.Context, r *pb.AlarmRequest, opts ...grpc.CallOption) (*pb.AlarmResponse, error) { return s.mts.Alarm(ctx, r) } func (s *mts2mtc) Status(ctx context.Context, r *pb.StatusRequest, opts ...grpc.CallOption) (*pb.StatusResponse, error) { return s.mts.Status(ctx, r) } func (s *mts2mtc) Defragment(ctx context.Context, dr *pb.DefragmentRequest, opts ...grpc.CallOption) (*pb.DefragmentResponse, error) { return s.mts.Defragment(ctx, dr) } func (s *mts2mtc) Hash(ctx context.Context, r *pb.HashRequest, opts ...grpc.CallOption) (*pb.HashResponse, error) { return s.mts.Hash(ctx, r) } func (s *mts2mtc) HashKV(ctx context.Context, r *pb.HashKVRequest, opts ...grpc.CallOption) (*pb.HashKVResponse, error) { return s.mts.HashKV(ctx, r) } func (s *mts2mtc) MoveLeader(ctx context.Context, r *pb.MoveLeaderRequest, opts ...grpc.CallOption) (*pb.MoveLeaderResponse, error) { return s.mts.MoveLeader(ctx, r) } func (s *mts2mtc) Downgrade(ctx context.Context, r *pb.DowngradeRequest, opts ...grpc.CallOption) (*pb.DowngradeResponse, error) { return s.mts.Downgrade(ctx, r) } func (s *mts2mtc) Snapshot(ctx context.Context, in *pb.SnapshotRequest, opts ...grpc.CallOption) (pb.Maintenance_SnapshotClient, error) { cs := newPipeStream(ctx, func(ss chanServerStream) error { return s.mts.Snapshot(in, &ss2scServerStream{ss}) }) return &ss2scClientStream{cs}, nil } // ss2scClientStream implements Maintenance_SnapshotClient type ss2scClientStream struct{ chanClientStream } // ss2scServerStream implements Maintenance_SnapshotServer type ss2scServerStream struct{ chanServerStream } func (s *ss2scClientStream) Send(rr *pb.SnapshotRequest) error { return s.SendMsg(rr) //nolint:staticcheck // TODO: remove for a supported version } func (s *ss2scClientStream) Recv() (*pb.SnapshotResponse, error) { var v any if err := s.RecvMsg(&v); err != nil { //nolint:staticcheck // TODO: remove for a supported version return nil, err } return v.(*pb.SnapshotResponse), nil } func (s *ss2scServerStream) Send(rr *pb.SnapshotResponse) error { return s.SendMsg(rr) //nolint:staticcheck // TODO: remove for a supported version } func (s *ss2scServerStream) Recv() (*pb.SnapshotRequest, error) { var v any if err := s.RecvMsg(&v); err != nil { //nolint:staticcheck // TODO: remove for a supported version return nil, err } return v.(*pb.SnapshotRequest), nil } ================================================ FILE: server/proxy/grpcproxy/adapter/watch_client_adapter.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package adapter import ( "context" "errors" "google.golang.org/grpc" pb "go.etcd.io/etcd/api/v3/etcdserverpb" ) var errAlreadySentHeader = errors.New("adapter: already sent header") type ws2wc struct{ wserv pb.WatchServer } func WatchServerToWatchClient(wserv pb.WatchServer) pb.WatchClient { return &ws2wc{wserv} } func (s *ws2wc) Watch(ctx context.Context, opts ...grpc.CallOption) (pb.Watch_WatchClient, error) { cs := newPipeStream(ctx, func(ss chanServerStream) error { return s.wserv.Watch(&ws2wcServerStream{ss}) }) return &ws2wcClientStream{cs}, nil } // ws2wcClientStream implements Watch_WatchClient type ws2wcClientStream struct{ chanClientStream } // ws2wcServerStream implements Watch_WatchServer type ws2wcServerStream struct{ chanServerStream } func (s *ws2wcClientStream) Send(wr *pb.WatchRequest) error { return s.SendMsg(wr) //nolint:staticcheck // TODO: remove for a supported version } func (s *ws2wcClientStream) Recv() (*pb.WatchResponse, error) { var v any if err := s.RecvMsg(&v); err != nil { //nolint:staticcheck // TODO: remove for a supported version return nil, err } return v.(*pb.WatchResponse), nil } func (s *ws2wcServerStream) Send(wr *pb.WatchResponse) error { return s.SendMsg(wr) //nolint:staticcheck // TODO: remove for a supported version } func (s *ws2wcServerStream) Recv() (*pb.WatchRequest, error) { var v any if err := s.RecvMsg(&v); err != nil { //nolint:staticcheck // TODO: remove for a supported version return nil, err } return v.(*pb.WatchRequest), nil } ================================================ FILE: server/proxy/grpcproxy/auth.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" ) type AuthProxy struct { authClient pb.AuthClient // we want compile errors if new methods are added pb.UnsafeAuthServer } func NewAuthProxy(c *clientv3.Client) pb.AuthServer { return &AuthProxy{authClient: pb.NewAuthClient(c.ActiveConnection())} } func (ap *AuthProxy) AuthEnable(ctx context.Context, r *pb.AuthEnableRequest) (*pb.AuthEnableResponse, error) { return ap.authClient.AuthEnable(ctx, r) } func (ap *AuthProxy) AuthDisable(ctx context.Context, r *pb.AuthDisableRequest) (*pb.AuthDisableResponse, error) { return ap.authClient.AuthDisable(ctx, r) } func (ap *AuthProxy) AuthStatus(ctx context.Context, r *pb.AuthStatusRequest) (*pb.AuthStatusResponse, error) { return ap.authClient.AuthStatus(ctx, r) } func (ap *AuthProxy) Authenticate(ctx context.Context, r *pb.AuthenticateRequest) (*pb.AuthenticateResponse, error) { return ap.authClient.Authenticate(ctx, r) } func (ap *AuthProxy) RoleAdd(ctx context.Context, r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, error) { return ap.authClient.RoleAdd(ctx, r) } func (ap *AuthProxy) RoleDelete(ctx context.Context, r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDeleteResponse, error) { return ap.authClient.RoleDelete(ctx, r) } func (ap *AuthProxy) RoleGet(ctx context.Context, r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse, error) { return ap.authClient.RoleGet(ctx, r) } func (ap *AuthProxy) RoleList(ctx context.Context, r *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error) { return ap.authClient.RoleList(ctx, r) } func (ap *AuthProxy) RoleRevokePermission(ctx context.Context, r *pb.AuthRoleRevokePermissionRequest) (*pb.AuthRoleRevokePermissionResponse, error) { return ap.authClient.RoleRevokePermission(ctx, r) } func (ap *AuthProxy) RoleGrantPermission(ctx context.Context, r *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) { return ap.authClient.RoleGrantPermission(ctx, r) } func (ap *AuthProxy) UserAdd(ctx context.Context, r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, error) { return ap.authClient.UserAdd(ctx, r) } func (ap *AuthProxy) UserDelete(ctx context.Context, r *pb.AuthUserDeleteRequest) (*pb.AuthUserDeleteResponse, error) { return ap.authClient.UserDelete(ctx, r) } func (ap *AuthProxy) UserGet(ctx context.Context, r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) { return ap.authClient.UserGet(ctx, r) } func (ap *AuthProxy) UserList(ctx context.Context, r *pb.AuthUserListRequest) (*pb.AuthUserListResponse, error) { return ap.authClient.UserList(ctx, r) } func (ap *AuthProxy) UserGrantRole(ctx context.Context, r *pb.AuthUserGrantRoleRequest) (*pb.AuthUserGrantRoleResponse, error) { return ap.authClient.UserGrantRole(ctx, r) } func (ap *AuthProxy) UserRevokeRole(ctx context.Context, r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUserRevokeRoleResponse, error) { return ap.authClient.UserRevokeRole(ctx, r) } func (ap *AuthProxy) UserChangePassword(ctx context.Context, r *pb.AuthUserChangePasswordRequest) (*pb.AuthUserChangePasswordResponse, error) { return ap.authClient.UserChangePassword(ctx, r) } ================================================ FILE: server/proxy/grpcproxy/cache/store.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package cache exports functionality for efficiently caching and mapping // `RangeRequest`s to corresponding `RangeResponse`s. package cache import ( "errors" "sync" "k8s.io/utils/lru" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/pkg/v3/adt" ) var ( DefaultMaxEntries = 2048 ErrCompacted = rpctypes.ErrGRPCCompacted ) type Cache interface { Add(req *pb.RangeRequest, resp *pb.RangeResponse) Get(req *pb.RangeRequest) (*pb.RangeResponse, error) Compact(revision int64) Invalidate(key []byte, endkey []byte) Size() int Close() } // keyFunc returns the key of a request, which is used to look up its caching response in the cache. func keyFunc(req *pb.RangeRequest) string { // TODO: use marshalTo to reduce allocation b, err := req.Marshal() if err != nil { panic(err) } return string(b) } func NewCache(maxCacheEntries int) Cache { return &cache{ lru: lru.New(maxCacheEntries), cachedRanges: adt.NewIntervalTree(), compactedRev: -1, } } func (c *cache) Close() {} // cache implements Cache type cache struct { mu sync.RWMutex lru *lru.Cache // a reverse index for cache invalidation cachedRanges adt.IntervalTree compactedRev int64 } // Add adds the response of a request to the cache if its revision is larger than the compacted revision of the cache. func (c *cache) Add(req *pb.RangeRequest, resp *pb.RangeResponse) { key := keyFunc(req) c.mu.Lock() defer c.mu.Unlock() if req.Revision > c.compactedRev { c.lru.Add(key, resp) } // we do not need to invalidate a request with a revision specified. // so we do not need to add it into the reverse index. if req.Revision != 0 { return } var ( iv *adt.IntervalValue ivl adt.Interval ) if len(req.RangeEnd) != 0 { ivl = adt.NewStringAffineInterval(string(req.Key), string(req.RangeEnd)) } else { ivl = adt.NewStringAffinePoint(string(req.Key)) } iv = c.cachedRanges.Find(ivl) if iv == nil { val := map[string]struct{}{key: {}} c.cachedRanges.Insert(ivl, val) } else { val := iv.Val.(map[string]struct{}) val[key] = struct{}{} iv.Val = val } } // Get looks up the caching response for a given request. // Get is also responsible for lazy eviction when accessing compacted entries. func (c *cache) Get(req *pb.RangeRequest) (*pb.RangeResponse, error) { key := keyFunc(req) c.mu.Lock() defer c.mu.Unlock() if req.Revision > 0 && req.Revision < c.compactedRev { c.lru.Remove(key) return nil, ErrCompacted } if resp, ok := c.lru.Get(key); ok { return resp.(*pb.RangeResponse), nil } return nil, errors.New("not exist") } // Invalidate invalidates the cache entries that intersecting with the given range from key to endkey. func (c *cache) Invalidate(key, endkey []byte) { c.mu.Lock() defer c.mu.Unlock() var ( ivs []*adt.IntervalValue ivl adt.Interval ) if len(endkey) == 0 { ivl = adt.NewStringAffinePoint(string(key)) } else { ivl = adt.NewStringAffineInterval(string(key), string(endkey)) } ivs = c.cachedRanges.Stab(ivl) for _, iv := range ivs { keys := iv.Val.(map[string]struct{}) for key := range keys { c.lru.Remove(key) } } // delete after removing all keys since it is destructive to 'ivs' c.cachedRanges.Delete(ivl) } // Compact invalidate all caching response before the given rev. // Replace with the invalidation is lazy. The actual removal happens when the entries is accessed. func (c *cache) Compact(revision int64) { c.mu.Lock() defer c.mu.Unlock() if revision > c.compactedRev { c.compactedRev = revision } } func (c *cache) Size() int { c.mu.RLock() defer c.mu.RUnlock() return c.lru.Len() } ================================================ FILE: server/proxy/grpcproxy/cluster.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" "errors" "fmt" "os" "sync" "go.uber.org/zap" "golang.org/x/time/rate" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/naming/endpoints" ) // allow maximum 1 retry per second const resolveRetryRate = 1 type clusterProxy struct { lg *zap.Logger clus pb.ClusterClient ctx context.Context // advertise client URL advaddr string prefix string em endpoints.Manager umu sync.RWMutex umap map[string]endpoints.Endpoint // we want compile errors if new methods are added pb.UnsafeClusterServer } // NewClusterProxy takes optional prefix to fetch grpc-proxy member endpoints. // The returned channel is closed when there is grpc-proxy endpoint registered // and the client's context is canceled so the 'register' loop returns. // TODO: Expand the API to report creation errors func NewClusterProxy(lg *zap.Logger, c *clientv3.Client, advaddr string, prefix string) (pb.ClusterServer, <-chan struct{}) { if lg == nil { lg = zap.NewNop() } var em endpoints.Manager if advaddr != "" && prefix != "" { var err error if em, err = endpoints.NewManager(c, prefix); err != nil { lg.Error("failed to provision endpointsManager", zap.String("prefix", prefix), zap.Error(err)) return nil, nil } } cp := &clusterProxy{ lg: lg, clus: pb.NewClusterClient(c.ActiveConnection()), ctx: c.Ctx(), advaddr: advaddr, prefix: prefix, umap: make(map[string]endpoints.Endpoint), em: em, } donec := make(chan struct{}) if em != nil { go func() { defer close(donec) cp.establishEndpointWatch(prefix) }() return cp, donec } close(donec) return cp, donec } func (cp *clusterProxy) establishEndpointWatch(prefix string) { rm := rate.NewLimiter(rate.Limit(resolveRetryRate), resolveRetryRate) for rm.Wait(cp.ctx) == nil { wc, err := cp.em.NewWatchChannel(cp.ctx) if err != nil { cp.lg.Warn("failed to establish endpoint watch", zap.String("prefix", prefix), zap.Error(err)) continue } cp.monitor(wc) } } func (cp *clusterProxy) monitor(wa endpoints.WatchChannel) { for { select { case <-cp.ctx.Done(): cp.lg.Info("watching endpoints interrupted", zap.Error(cp.ctx.Err())) return case updates, ok := <-wa: if !ok { cp.lg.Info("endpoints watch channel closed") return } cp.umu.Lock() for _, up := range updates { switch up.Op { case endpoints.Add: cp.umap[up.Key] = up.Endpoint case endpoints.Delete: delete(cp.umap, up.Key) } } cp.umu.Unlock() } } } func (cp *clusterProxy) MemberAdd(ctx context.Context, r *pb.MemberAddRequest) (*pb.MemberAddResponse, error) { return cp.clus.MemberAdd(ctx, r) } func (cp *clusterProxy) MemberRemove(ctx context.Context, r *pb.MemberRemoveRequest) (*pb.MemberRemoveResponse, error) { return cp.clus.MemberRemove(ctx, r) } func (cp *clusterProxy) MemberUpdate(ctx context.Context, r *pb.MemberUpdateRequest) (*pb.MemberUpdateResponse, error) { return cp.clus.MemberUpdate(ctx, r) } func (cp *clusterProxy) membersFromUpdates() ([]*pb.Member, error) { cp.umu.RLock() defer cp.umu.RUnlock() mbs := make([]*pb.Member, 0, len(cp.umap)) for _, upt := range cp.umap { m, err := decodeMeta(fmt.Sprint(upt.Metadata)) //nolint:staticcheck // TODO: remove for a supported version if err != nil { return nil, err } mbs = append(mbs, &pb.Member{Name: m.Name, ClientURLs: []string{upt.Addr}}) } return mbs, nil } // MemberList wraps member list API with following rules: // - If 'advaddr' is not empty and 'prefix' is not empty, return registered member lists via resolver // - If 'advaddr' is not empty and 'prefix' is not empty and registered grpc-proxy members haven't been fetched, return the 'advaddr' // - If 'advaddr' is not empty and 'prefix' is empty, return 'advaddr' without forcing it to 'register' // - If 'advaddr' is empty, forward to member list API func (cp *clusterProxy) MemberList(ctx context.Context, r *pb.MemberListRequest) (*pb.MemberListResponse, error) { if cp.advaddr != "" { if cp.prefix != "" { mbs, err := cp.membersFromUpdates() if err != nil { return nil, err } if len(mbs) > 0 { return &pb.MemberListResponse{Members: mbs}, nil } } // prefix is empty or no grpc-proxy members haven't been registered hostname, _ := os.Hostname() return &pb.MemberListResponse{Members: []*pb.Member{{Name: hostname, ClientURLs: []string{cp.advaddr}}}}, nil } return cp.clus.MemberList(ctx, r) } func (cp *clusterProxy) MemberPromote(ctx context.Context, r *pb.MemberPromoteRequest) (*pb.MemberPromoteResponse, error) { // TODO: implement return nil, errors.New("not implemented") } ================================================ FILE: server/proxy/grpcproxy/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package grpcproxy is an OSI level 7 proxy for etcd v3 API requests. package grpcproxy ================================================ FILE: server/proxy/grpcproxy/election.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb" ) type electionProxy struct { electionClient v3electionpb.ElectionClient // we want compile errors if new methods are added v3electionpb.UnsafeElectionServer } func NewElectionProxy(client *clientv3.Client) v3electionpb.ElectionServer { return &electionProxy{electionClient: v3electionpb.NewElectionClient(client.ActiveConnection())} } func (ep *electionProxy) Campaign(ctx context.Context, req *v3electionpb.CampaignRequest) (*v3electionpb.CampaignResponse, error) { return ep.electionClient.Campaign(ctx, req) } func (ep *electionProxy) Proclaim(ctx context.Context, req *v3electionpb.ProclaimRequest) (*v3electionpb.ProclaimResponse, error) { return ep.electionClient.Proclaim(ctx, req) } func (ep *electionProxy) Leader(ctx context.Context, req *v3electionpb.LeaderRequest) (*v3electionpb.LeaderResponse, error) { return ep.electionClient.Leader(ctx, req) } func (ep *electionProxy) Observe(req *v3electionpb.LeaderRequest, s v3electionpb.Election_ObserveServer) error { ctx, cancel := context.WithCancel(s.Context()) defer cancel() sc, err := ep.electionClient.Observe(ctx, req) if err != nil { return err } for { rr, err := sc.Recv() if err != nil { return err } if err = s.Send(rr); err != nil { return err } } } func (ep *electionProxy) Resign(ctx context.Context, req *v3electionpb.ResignRequest) (*v3electionpb.ResignResponse, error) { return ep.electionClient.Resign(ctx, req) } ================================================ FILE: server/proxy/grpcproxy/health.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" "errors" "fmt" "net/http" "time" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp" ) // HandleHealth registers health handler on '/health'. func HandleHealth(lg *zap.Logger, mux *http.ServeMux, c *clientv3.Client) { if lg == nil { lg = zap.NewNop() } mux.Handle(etcdhttp.PathHealth, etcdhttp.NewHealthHandler(lg, func(ctx context.Context, excludedAlarms etcdhttp.StringSet, serializable bool) etcdhttp.Health { return checkHealth(c) })) } // HandleProxyHealth registers health handler on '/proxy/health'. func HandleProxyHealth(lg *zap.Logger, mux *http.ServeMux, c *clientv3.Client) { if lg == nil { lg = zap.NewNop() } mux.Handle(etcdhttp.PathProxyHealth, etcdhttp.NewHealthHandler(lg, func(ctx context.Context, excludedAlarms etcdhttp.StringSet, serializable bool) etcdhttp.Health { return checkProxyHealth(c) })) } func checkHealth(c *clientv3.Client) etcdhttp.Health { h := etcdhttp.Health{Health: "false"} ctx, cancel := context.WithTimeout(c.Ctx(), time.Second) _, err := c.Get(ctx, "a") cancel() if err == nil || errors.Is(err, rpctypes.ErrPermissionDenied) { h.Health = "true" } else { h.Reason = fmt.Sprintf("GET ERROR:%s", err) } return h } func checkProxyHealth(c *clientv3.Client) etcdhttp.Health { if c == nil { return etcdhttp.Health{Health: "false", Reason: "no connection to proxy"} } h := checkHealth(c) if h.Health != "true" { return h } ctx, cancel := context.WithTimeout(c.Ctx(), time.Second*3) ch := c.Watch(ctx, "a", clientv3.WithCreatedNotify()) select { case <-ch: case <-ctx.Done(): h.Health = "false" h.Reason = "WATCH TIMEOUT" } cancel() return h } ================================================ FILE: server/proxy/grpcproxy/kv.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" "errors" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/proxy/grpcproxy/cache" ) type kvProxy struct { kv clientv3.KV cache cache.Cache // we want compile errors if new methods are added pb.UnsafeKVServer } func NewKvProxy(c *clientv3.Client) (pb.KVServer, <-chan struct{}) { kv := &kvProxy{ kv: c.KV, cache: cache.NewCache(cache.DefaultMaxEntries), } donec := make(chan struct{}) close(donec) return kv, donec } func (p *kvProxy) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) { if r.Serializable { resp, err := p.cache.Get(r) switch { case err == nil: cacheHits.Inc() return resp, nil case errors.Is(err, cache.ErrCompacted): cacheHits.Inc() return nil, err } cachedMisses.Inc() } resp, err := p.kv.Do(ctx, RangeRequestToOp(r)) if err != nil { return nil, err } // cache linearizable as serializable req := *r req.Serializable = true gresp := (*pb.RangeResponse)(resp.Get()) p.cache.Add(&req, gresp) cacheKeys.Set(float64(p.cache.Size())) return gresp, nil } func (p *kvProxy) Put(ctx context.Context, r *pb.PutRequest) (*pb.PutResponse, error) { p.cache.Invalidate(r.Key, nil) cacheKeys.Set(float64(p.cache.Size())) resp, err := p.kv.Do(ctx, PutRequestToOp(r)) return (*pb.PutResponse)(resp.Put()), err } func (p *kvProxy) DeleteRange(ctx context.Context, r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) { p.cache.Invalidate(r.Key, r.RangeEnd) cacheKeys.Set(float64(p.cache.Size())) resp, err := p.kv.Do(ctx, DelRequestToOp(r)) return (*pb.DeleteRangeResponse)(resp.Del()), err } func (p *kvProxy) txnToCache(reqs []*pb.RequestOp, resps []*pb.ResponseOp) { for i := range resps { switch tv := resps[i].Response.(type) { case *pb.ResponseOp_ResponsePut: p.cache.Invalidate(reqs[i].GetRequestPut().Key, nil) case *pb.ResponseOp_ResponseDeleteRange: rdr := reqs[i].GetRequestDeleteRange() p.cache.Invalidate(rdr.Key, rdr.RangeEnd) case *pb.ResponseOp_ResponseRange: req := *(reqs[i].GetRequestRange()) req.Serializable = true p.cache.Add(&req, tv.ResponseRange) } } } func (p *kvProxy) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse, error) { op := TxnRequestToOp(r) opResp, err := p.kv.Do(ctx, op) if err != nil { return nil, err } resp := opResp.Txn() // txn may claim an outdated key is updated; be safe and invalidate for _, cmp := range r.Compare { p.cache.Invalidate(cmp.Key, cmp.RangeEnd) } // update any fetched keys if resp.Succeeded { p.txnToCache(r.Success, resp.Responses) } else { p.txnToCache(r.Failure, resp.Responses) } cacheKeys.Set(float64(p.cache.Size())) return (*pb.TxnResponse)(resp), nil } func (p *kvProxy) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) { var opts []clientv3.CompactOption if r.Physical { opts = append(opts, clientv3.WithCompactPhysical()) } resp, err := p.kv.Compact(ctx, r.Revision, opts...) if err == nil { p.cache.Compact(r.Revision) } cacheKeys.Set(float64(p.cache.Size())) return (*pb.CompactionResponse)(resp), err } func requestOpToOp(union *pb.RequestOp) clientv3.Op { switch tv := union.Request.(type) { case *pb.RequestOp_RequestRange: if tv.RequestRange != nil { return RangeRequestToOp(tv.RequestRange) } case *pb.RequestOp_RequestPut: if tv.RequestPut != nil { return PutRequestToOp(tv.RequestPut) } case *pb.RequestOp_RequestDeleteRange: if tv.RequestDeleteRange != nil { return DelRequestToOp(tv.RequestDeleteRange) } case *pb.RequestOp_RequestTxn: if tv.RequestTxn != nil { return TxnRequestToOp(tv.RequestTxn) } } panic("unknown request") } func RangeRequestToOp(r *pb.RangeRequest) clientv3.Op { var opts []clientv3.OpOption if len(r.RangeEnd) != 0 { opts = append(opts, clientv3.WithRange(string(r.RangeEnd))) } opts = append(opts, clientv3.WithRev(r.Revision)) opts = append(opts, clientv3.WithLimit(r.Limit)) opts = append(opts, clientv3.WithSort( clientv3.SortTarget(r.SortTarget), clientv3.SortOrder(r.SortOrder)), ) opts = append(opts, clientv3.WithMaxCreateRev(r.MaxCreateRevision)) opts = append(opts, clientv3.WithMinCreateRev(r.MinCreateRevision)) opts = append(opts, clientv3.WithMaxModRev(r.MaxModRevision)) opts = append(opts, clientv3.WithMinModRev(r.MinModRevision)) if r.CountOnly { opts = append(opts, clientv3.WithCountOnly()) } if r.KeysOnly { opts = append(opts, clientv3.WithKeysOnly()) } if r.Serializable { opts = append(opts, clientv3.WithSerializable()) } return clientv3.OpGet(string(r.Key), opts...) } func PutRequestToOp(r *pb.PutRequest) clientv3.Op { var opts []clientv3.OpOption opts = append(opts, clientv3.WithLease(clientv3.LeaseID(r.Lease))) if r.IgnoreValue { opts = append(opts, clientv3.WithIgnoreValue()) } if r.IgnoreLease { opts = append(opts, clientv3.WithIgnoreLease()) } if r.PrevKv { opts = append(opts, clientv3.WithPrevKV()) } return clientv3.OpPut(string(r.Key), string(r.Value), opts...) } func DelRequestToOp(r *pb.DeleteRangeRequest) clientv3.Op { var opts []clientv3.OpOption if len(r.RangeEnd) != 0 { opts = append(opts, clientv3.WithRange(string(r.RangeEnd))) } if r.PrevKv { opts = append(opts, clientv3.WithPrevKV()) } return clientv3.OpDelete(string(r.Key), opts...) } func TxnRequestToOp(r *pb.TxnRequest) clientv3.Op { cmps := make([]clientv3.Cmp, len(r.Compare)) thenops := make([]clientv3.Op, len(r.Success)) elseops := make([]clientv3.Op, len(r.Failure)) for i := range r.Compare { cmps[i] = (clientv3.Cmp)(*r.Compare[i]) } for i := range r.Success { thenops[i] = requestOpToOp(r.Success[i]) } for i := range r.Failure { elseops[i] = requestOpToOp(r.Failure[i]) } return clientv3.OpTxn(cmps, thenops, elseops) } ================================================ FILE: server/proxy/grpcproxy/leader.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" "math" "sync" "golang.org/x/time/rate" clientv3 "go.etcd.io/etcd/client/v3" ) const ( lostLeaderKey = "__lostleader" // watched to detect leader loss retryPerSecond = 10 ) type leader struct { ctx context.Context w clientv3.Watcher // mu protects leaderc updates. mu sync.RWMutex leaderc chan struct{} disconnc chan struct{} donec chan struct{} } func newLeader(ctx context.Context, w clientv3.Watcher) *leader { l := &leader{ ctx: clientv3.WithRequireLeader(ctx), w: w, leaderc: make(chan struct{}), disconnc: make(chan struct{}), donec: make(chan struct{}), } // begin assuming leader is lost close(l.leaderc) go l.recvLoop() return l } func (l *leader) recvLoop() { defer close(l.donec) limiter := rate.NewLimiter(rate.Limit(retryPerSecond), retryPerSecond) rev := int64(math.MaxInt64 - 2) for limiter.Wait(l.ctx) == nil { wch := l.w.Watch(l.ctx, lostLeaderKey, clientv3.WithRev(rev), clientv3.WithCreatedNotify()) cresp, ok := <-wch if !ok { l.loseLeader() continue } if cresp.Err() != nil { l.loseLeader() if clientv3.IsConnCanceled(cresp.Err()) { close(l.disconnc) return } continue } l.gotLeader() <-wch l.loseLeader() } } func (l *leader) loseLeader() { l.mu.RLock() defer l.mu.RUnlock() select { case <-l.leaderc: default: close(l.leaderc) } } // gotLeader will force update the leadership status to having a leader. func (l *leader) gotLeader() { l.mu.Lock() defer l.mu.Unlock() select { case <-l.leaderc: l.leaderc = make(chan struct{}) default: } } func (l *leader) disconnectNotify() <-chan struct{} { return l.disconnc } func (l *leader) stopNotify() <-chan struct{} { return l.donec } // lostNotify returns a channel that is closed if there has been // a leader loss not yet followed by a leader reacquire. func (l *leader) lostNotify() <-chan struct{} { l.mu.RLock() defer l.mu.RUnlock() return l.leaderc } ================================================ FILE: server/proxy/grpcproxy/lease.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" "errors" "io" "sync" "sync/atomic" "time" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" ) type leaseProxy struct { // leaseClient handles req from LeaseGrant() that requires a lease ID. leaseClient pb.LeaseClient lessor clientv3.Lease ctx context.Context leader *leader // mu protects adding outstanding leaseProxyStream through wg. mu sync.RWMutex // wg waits until all outstanding leaseProxyStream quit. wg sync.WaitGroup // we want compile errors if new methods are added pb.LeaseServer } func NewLeaseProxy(ctx context.Context, c *clientv3.Client) (pb.LeaseServer, <-chan struct{}) { cctx, cancel := context.WithCancel(ctx) lp := &leaseProxy{ leaseClient: pb.NewLeaseClient(c.ActiveConnection()), lessor: c.Lease, ctx: cctx, leader: newLeader(cctx, c.Watcher), } ch := make(chan struct{}) go func() { defer close(ch) <-lp.leader.stopNotify() lp.mu.Lock() select { case <-lp.ctx.Done(): case <-lp.leader.disconnectNotify(): cancel() } <-lp.ctx.Done() lp.mu.Unlock() lp.wg.Wait() }() return lp, ch } func (lp *leaseProxy) LeaseGrant(ctx context.Context, cr *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { rp, err := lp.leaseClient.LeaseGrant(ctx, cr, grpc.WaitForReady(true)) if err != nil { return nil, err } lp.leader.gotLeader() return rp, nil } func (lp *leaseProxy) LeaseRevoke(ctx context.Context, rr *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) { r, err := lp.lessor.Revoke(ctx, clientv3.LeaseID(rr.ID)) if err != nil { return nil, err } lp.leader.gotLeader() return (*pb.LeaseRevokeResponse)(r), nil } func (lp *leaseProxy) LeaseTimeToLive(ctx context.Context, rr *pb.LeaseTimeToLiveRequest) (*pb.LeaseTimeToLiveResponse, error) { var ( r *clientv3.LeaseTimeToLiveResponse err error ) if rr.Keys { r, err = lp.lessor.TimeToLive(ctx, clientv3.LeaseID(rr.ID), clientv3.WithAttachedKeys()) } else { r, err = lp.lessor.TimeToLive(ctx, clientv3.LeaseID(rr.ID)) } if err != nil { return nil, err } rp := &pb.LeaseTimeToLiveResponse{ Header: r.ResponseHeader, ID: int64(r.ID), TTL: r.TTL, GrantedTTL: r.GrantedTTL, Keys: r.Keys, } return rp, err } func (lp *leaseProxy) LeaseLeases(ctx context.Context, rr *pb.LeaseLeasesRequest) (*pb.LeaseLeasesResponse, error) { r, err := lp.lessor.Leases(ctx) if err != nil { return nil, err } leases := make([]*pb.LeaseStatus, len(r.Leases)) for i := range r.Leases { leases[i] = &pb.LeaseStatus{ID: int64(r.Leases[i].ID)} } rp := &pb.LeaseLeasesResponse{ Header: r.ResponseHeader, Leases: leases, } return rp, err } func (lp *leaseProxy) LeaseKeepAlive(stream pb.Lease_LeaseKeepAliveServer) error { lp.mu.Lock() select { case <-lp.ctx.Done(): lp.mu.Unlock() return lp.ctx.Err() default: lp.wg.Add(1) } lp.mu.Unlock() ctx, cancel := context.WithCancel(stream.Context()) lps := leaseProxyStream{ stream: stream, lessor: lp.lessor, keepAliveLeases: make(map[int64]*atomicCounter), respc: make(chan *pb.LeaseKeepAliveResponse), ctx: ctx, cancel: cancel, } errc := make(chan error, 2) var lostLeaderC <-chan struct{} if md, ok := metadata.FromOutgoingContext(stream.Context()); ok { v := md[rpctypes.MetadataRequireLeaderKey] if len(v) > 0 && v[0] == rpctypes.MetadataHasLeader { lostLeaderC = lp.leader.lostNotify() // if leader is known to be lost at creation time, avoid // letting events through at all select { case <-lostLeaderC: lp.wg.Done() return rpctypes.ErrNoLeader default: } } } stopc := make(chan struct{}, 3) go func() { defer func() { stopc <- struct{}{} }() if err := lps.recvLoop(); err != nil { errc <- err } }() go func() { defer func() { stopc <- struct{}{} }() if err := lps.sendLoop(); err != nil { errc <- err } }() // tears down LeaseKeepAlive stream if leader goes down or entire leaseProxy is terminated. go func() { defer func() { stopc <- struct{}{} }() select { case <-lostLeaderC: case <-ctx.Done(): case <-lp.ctx.Done(): } }() var err error select { case <-stopc: stopc <- struct{}{} case err = <-errc: } cancel() // recv/send may only shutdown after function exits; // this goroutine notifies lease proxy that the stream is through go func() { <-stopc <-stopc <-stopc lps.close() close(errc) lp.wg.Done() }() select { case <-lostLeaderC: return rpctypes.ErrNoLeader case <-lp.leader.disconnectNotify(): return status.Error(codes.Canceled, "the client connection is closing") default: if err != nil { return err } return ctx.Err() } } type leaseProxyStream struct { stream pb.Lease_LeaseKeepAliveServer lessor clientv3.Lease // wg tracks keepAliveLoop goroutines wg sync.WaitGroup // mu protects keepAliveLeases mu sync.RWMutex // keepAliveLeases tracks how many outstanding keepalive requests which need responses are on a lease. keepAliveLeases map[int64]*atomicCounter // respc receives lease keepalive responses from etcd backend respc chan *pb.LeaseKeepAliveResponse ctx context.Context cancel context.CancelFunc } func (lps *leaseProxyStream) recvLoop() error { for { rr, err := lps.stream.Recv() if errors.Is(err, io.EOF) { return nil } if err != nil { return err } lps.mu.Lock() neededResps, ok := lps.keepAliveLeases[rr.ID] if !ok { neededResps = &atomicCounter{} lps.keepAliveLeases[rr.ID] = neededResps lps.wg.Add(1) go func() { defer lps.wg.Done() if err := lps.keepAliveLoop(rr.ID, neededResps); err != nil { lps.cancel() } }() } neededResps.add(1) lps.mu.Unlock() } } func (lps *leaseProxyStream) keepAliveLoop(leaseID int64, neededResps *atomicCounter) error { cctx, ccancel := context.WithCancel(lps.ctx) defer ccancel() respc, err := lps.lessor.KeepAlive(cctx, clientv3.LeaseID(leaseID)) if err != nil { return err } // ticker expires when loop hasn't received keepalive within TTL var ticker <-chan time.Time for { select { case <-ticker: lps.mu.Lock() // if there are outstanding keepAlive reqs at the moment of ticker firing, // don't close keepAliveLoop(), let it continuing to process the KeepAlive reqs. if neededResps.get() > 0 { lps.mu.Unlock() ticker = nil continue } delete(lps.keepAliveLeases, leaseID) lps.mu.Unlock() return nil case rp, ok := <-respc: if !ok { lps.mu.Lock() delete(lps.keepAliveLeases, leaseID) lps.mu.Unlock() if neededResps.get() == 0 { return nil } ttlResp, err := lps.lessor.TimeToLive(cctx, clientv3.LeaseID(leaseID)) if err != nil { return err } r := &pb.LeaseKeepAliveResponse{ Header: ttlResp.ResponseHeader, ID: int64(ttlResp.ID), TTL: ttlResp.TTL, } for neededResps.get() > 0 { select { case lps.respc <- r: neededResps.add(-1) case <-lps.ctx.Done(): return nil } } return nil } if neededResps.get() == 0 { continue } ticker = time.After(time.Duration(rp.TTL) * time.Second) r := &pb.LeaseKeepAliveResponse{ Header: rp.ResponseHeader, ID: int64(rp.ID), TTL: rp.TTL, } lps.replyToClient(r, neededResps) } } } func (lps *leaseProxyStream) replyToClient(r *pb.LeaseKeepAliveResponse, neededResps *atomicCounter) { timer := time.After(500 * time.Millisecond) for neededResps.get() > 0 { select { case lps.respc <- r: neededResps.add(-1) case <-timer: return case <-lps.ctx.Done(): return } } } func (lps *leaseProxyStream) sendLoop() error { for { select { case lrp, ok := <-lps.respc: if !ok { return nil } if err := lps.stream.Send(lrp); err != nil { return err } case <-lps.ctx.Done(): return lps.ctx.Err() } } } func (lps *leaseProxyStream) close() { lps.cancel() lps.wg.Wait() // only close respc channel if all the keepAliveLoop() goroutines have finished // this ensures those goroutines don't send resp to a closed resp channel close(lps.respc) } type atomicCounter struct { counter int64 } func (ac *atomicCounter) add(delta int64) { atomic.AddInt64(&ac.counter, delta) } func (ac *atomicCounter) get() int64 { return atomic.LoadInt64(&ac.counter) } ================================================ FILE: server/proxy/grpcproxy/lock.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb" ) type lockProxy struct { lockClient v3lockpb.LockClient // we want compile errors if new methods are added v3lockpb.UnsafeLockServer } func NewLockProxy(client *clientv3.Client) v3lockpb.LockServer { return &lockProxy{lockClient: v3lockpb.NewLockClient(client.ActiveConnection())} } func (lp *lockProxy) Lock(ctx context.Context, req *v3lockpb.LockRequest) (*v3lockpb.LockResponse, error) { return lp.lockClient.Lock(ctx, req) } func (lp *lockProxy) Unlock(ctx context.Context, req *v3lockpb.UnlockRequest) (*v3lockpb.UnlockResponse, error) { return lp.lockClient.Unlock(ctx, req) } ================================================ FILE: server/proxy/grpcproxy/maintenance.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" "errors" "io" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" ) type maintenanceProxy struct { maintenanceClient pb.MaintenanceClient // we want compile errors if new methods are added pb.UnsafeMaintenanceServer } func NewMaintenanceProxy(c *clientv3.Client) pb.MaintenanceServer { return &maintenanceProxy{ maintenanceClient: pb.NewMaintenanceClient(c.ActiveConnection()), } } func (mp *maintenanceProxy) Defragment(ctx context.Context, dr *pb.DefragmentRequest) (*pb.DefragmentResponse, error) { return mp.maintenanceClient.Defragment(ctx, dr) } func (mp *maintenanceProxy) Snapshot(sr *pb.SnapshotRequest, stream pb.Maintenance_SnapshotServer) error { ctx, cancel := context.WithCancel(stream.Context()) defer cancel() ctx = withClientAuthToken(ctx, stream.Context()) sc, err := mp.maintenanceClient.Snapshot(ctx, sr) if err != nil { return err } for { rr, err := sc.Recv() if err != nil { if errors.Is(err, io.EOF) { return nil } return err } err = stream.Send(rr) if err != nil { return err } } } func (mp *maintenanceProxy) Hash(ctx context.Context, r *pb.HashRequest) (*pb.HashResponse, error) { return mp.maintenanceClient.Hash(ctx, r) } func (mp *maintenanceProxy) HashKV(ctx context.Context, r *pb.HashKVRequest) (*pb.HashKVResponse, error) { return mp.maintenanceClient.HashKV(ctx, r) } func (mp *maintenanceProxy) Alarm(ctx context.Context, r *pb.AlarmRequest) (*pb.AlarmResponse, error) { return mp.maintenanceClient.Alarm(ctx, r) } func (mp *maintenanceProxy) Status(ctx context.Context, r *pb.StatusRequest) (*pb.StatusResponse, error) { return mp.maintenanceClient.Status(ctx, r) } func (mp *maintenanceProxy) MoveLeader(ctx context.Context, r *pb.MoveLeaderRequest) (*pb.MoveLeaderResponse, error) { return mp.maintenanceClient.MoveLeader(ctx, r) } func (mp *maintenanceProxy) Downgrade(ctx context.Context, r *pb.DowngradeRequest) (*pb.DowngradeResponse, error) { return mp.maintenanceClient.Downgrade(ctx, r) } ================================================ FILE: server/proxy/grpcproxy/metrics.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "fmt" "io" "math/rand" "net/http" "strings" "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp" ) var ( watchersCoalescing = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "grpc_proxy", Name: "watchers_coalescing_total", Help: "Total number of current watchers coalescing", }) eventsCoalescing = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "grpc_proxy", Name: "events_coalescing_total", Help: "Total number of events coalescing", }) cacheKeys = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "grpc_proxy", Name: "cache_keys_total", Help: "Total number of keys/ranges cached", }) cacheHits = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "grpc_proxy", Name: "cache_hits_total", Help: "Total number of cache hits", }) cachedMisses = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "grpc_proxy", Name: "cache_misses_total", Help: "Total number of cache misses", }) ) func init() { prometheus.MustRegister(watchersCoalescing) prometheus.MustRegister(eventsCoalescing) prometheus.MustRegister(cacheKeys) prometheus.MustRegister(cacheHits) prometheus.MustRegister(cachedMisses) } // HandleMetrics performs a GET request against etcd endpoint and returns '/metrics'. func HandleMetrics(mux *http.ServeMux, c *http.Client, eps []string) { // random shuffle endpoints r := rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) if len(eps) > 1 { eps = shuffleEndpoints(r, eps) } pathMetrics := etcdhttp.PathMetrics mux.HandleFunc(pathMetrics, func(w http.ResponseWriter, r *http.Request) { target := fmt.Sprintf("%s%s", eps[0], pathMetrics) if !strings.HasPrefix(target, "http") { scheme := "http" if r.TLS != nil { scheme = "https" } target = fmt.Sprintf("%s://%s", scheme, target) } resp, err := c.Get(target) if err != nil { http.Error(w, "Internal server error", http.StatusInternalServerError) return } defer resp.Body.Close() w.Header().Set("Content-Type", "text/plain; version=0.0.4") body, _ := io.ReadAll(resp.Body) fmt.Fprintf(w, "%s", body) }) } // HandleProxyMetrics registers metrics handler on '/proxy/metrics'. func HandleProxyMetrics(mux *http.ServeMux) { mux.Handle(etcdhttp.PathProxyMetrics, promhttp.Handler()) } func shuffleEndpoints(r *rand.Rand, eps []string) []string { // copied from Go 1.9<= rand.Rand.Perm n := len(eps) p := make([]int, n) for i := 0; i < n; i++ { j := r.Intn(i + 1) p[i] = p[j] p[j] = i } neps := make([]string, n) for i, k := range p { neps[i] = eps[k] } return neps } ================================================ FILE: server/proxy/grpcproxy/register.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "encoding/json" "os" "go.uber.org/zap" "golang.org/x/time/rate" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/client/v3/naming/endpoints" ) // allow maximum 1 retry per second const registerRetryRate = 1 // Register registers itself as a grpc-proxy server by writing prefixed-key // with session of specified TTL (in seconds). The returned channel is closed // when the client's context is canceled. func Register(lg *zap.Logger, c *clientv3.Client, prefix string, addr string, ttl int) <-chan struct{} { rm := rate.NewLimiter(rate.Limit(registerRetryRate), registerRetryRate) donec := make(chan struct{}) go func() { defer close(donec) for rm.Wait(c.Ctx()) == nil { ss, err := registerSession(lg, c, prefix, addr, ttl) if err != nil { lg.Warn("failed to create a session", zap.Error(err)) continue } select { case <-c.Ctx().Done(): ss.Close() return case <-ss.Done(): lg.Warn("session expired; possible network partition or server restart") lg.Warn("creating a new session to rejoin") continue } } }() return donec } func registerSession(lg *zap.Logger, c *clientv3.Client, prefix string, addr string, ttl int) (*concurrency.Session, error) { ss, err := concurrency.NewSession(c, concurrency.WithTTL(ttl)) if err != nil { return nil, err } em, err := endpoints.NewManager(c, prefix) if err != nil { ss.Close() return nil, err } endpoint := endpoints.Endpoint{Addr: addr, Metadata: getMeta()} if err = em.AddEndpoint(c.Ctx(), prefix+"/"+addr, endpoint, clientv3.WithLease(ss.Lease())); err != nil { ss.Close() return nil, err } lg.Info( "registered session with lease", zap.String("addr", addr), zap.Int("lease-ttl", ttl), ) return ss, nil } // meta represents metadata of proxy register. type meta struct { Name string `json:"name"` } func getMeta() string { hostname, _ := os.Hostname() bts, _ := json.Marshal(meta{Name: hostname}) return string(bts) } func decodeMeta(s string) (meta, error) { m := meta{} err := json.Unmarshal([]byte(s), &m) return m, err } ================================================ FILE: server/proxy/grpcproxy/util.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" "google.golang.org/grpc" "google.golang.org/grpc/metadata" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" ) func getAuthTokenFromClient(ctx context.Context) string { md, ok := metadata.FromIncomingContext(ctx) if ok { ts, ok := md[rpctypes.TokenFieldNameGRPC] if ok { return ts[0] } } return "" } func withClientAuthToken(ctx, ctxWithToken context.Context) context.Context { token := getAuthTokenFromClient(ctxWithToken) if token != "" { ctx = context.WithValue(ctx, rpctypes.TokenFieldNameGRPCKey{}, token) } return ctx } type proxyTokenCredential struct { token string } func (cred *proxyTokenCredential) RequireTransportSecurity() bool { return false } func (cred *proxyTokenCredential) GetRequestMetadata(ctx context.Context, s ...string) (map[string]string, error) { return map[string]string{ rpctypes.TokenFieldNameGRPC: cred.token, }, nil } func AuthUnaryClientInterceptor(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { token := getAuthTokenFromClient(ctx) if token != "" { tokenCred := &proxyTokenCredential{token} opts = append(opts, grpc.PerRPCCredentials(tokenCred)) } return invoker(ctx, method, req, reply, cc, opts...) } func AuthStreamClientInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { tokenif := ctx.Value(rpctypes.TokenFieldNameGRPCKey{}) if tokenif != nil { tokenCred := &proxyTokenCredential{tokenif.(string)} opts = append(opts, grpc.PerRPCCredentials(tokenCred)) } return streamer(ctx, desc, cc, method, opts...) } ================================================ FILE: server/proxy/grpcproxy/watch.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" "sync" "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc" ) type watchProxy struct { cw clientv3.Watcher ctx context.Context leader *leader ranges *watchRanges // mu protects adding outstanding watch servers through wg. mu sync.Mutex // wg waits until all outstanding watch servers quit. wg sync.WaitGroup // kv is used for permission checking kv clientv3.KV lg *zap.Logger // we want compile errors if new methods are added pb.UnsafeWatchServer } func NewWatchProxy(ctx context.Context, lg *zap.Logger, c *clientv3.Client) (pb.WatchServer, <-chan struct{}) { cctx, cancel := context.WithCancel(ctx) wp := &watchProxy{ cw: c.Watcher, ctx: cctx, leader: newLeader(cctx, c.Watcher), kv: c.KV, // for permission checking lg: lg, } wp.ranges = newWatchRanges(wp) ch := make(chan struct{}) go func() { defer close(ch) <-wp.leader.stopNotify() wp.mu.Lock() select { case <-wp.ctx.Done(): case <-wp.leader.disconnectNotify(): cancel() } <-wp.ctx.Done() wp.mu.Unlock() wp.wg.Wait() wp.ranges.stop() }() return wp, ch } func (wp *watchProxy) Watch(stream pb.Watch_WatchServer) (err error) { wp.mu.Lock() select { case <-wp.ctx.Done(): wp.mu.Unlock() select { case <-wp.leader.disconnectNotify(): return status.Error(codes.Canceled, "the client connection is closing") default: return wp.ctx.Err() } default: wp.wg.Add(1) } wp.mu.Unlock() ctx, cancel := context.WithCancel(stream.Context()) wps := &watchProxyStream{ ranges: wp.ranges, watchers: make(map[int64]*watcher), stream: stream, watchCh: make(chan *pb.WatchResponse, 1024), ctx: ctx, cancel: cancel, kv: wp.kv, lg: wp.lg, } var lostLeaderC <-chan struct{} if md, ok := metadata.FromOutgoingContext(stream.Context()); ok { v := md[rpctypes.MetadataRequireLeaderKey] if len(v) > 0 && v[0] == rpctypes.MetadataHasLeader { lostLeaderC = wp.leader.lostNotify() // if leader is known to be lost at creation time, avoid // letting events through at all select { case <-lostLeaderC: wp.wg.Done() return rpctypes.ErrNoLeader default: } } } // post to stopc => terminate server stream; can't use a waitgroup // since all goroutines will only terminate after Watch() exits. stopc := make(chan struct{}, 3) go func() { defer func() { stopc <- struct{}{} }() wps.recvLoop() }() go func() { defer func() { stopc <- struct{}{} }() wps.sendLoop() }() // tear down watch if leader goes down or entire watch proxy is terminated go func() { defer func() { stopc <- struct{}{} }() select { case <-lostLeaderC: case <-ctx.Done(): case <-wp.ctx.Done(): } }() <-stopc cancel() // recv/send may only shutdown after function exits; // goroutine notifies proxy that stream is through go func() { <-stopc <-stopc wps.close() wp.wg.Done() }() select { case <-lostLeaderC: return rpctypes.ErrNoLeader case <-wp.leader.disconnectNotify(): return status.Error(codes.Canceled, "the client connection is closing") default: return wps.ctx.Err() } } // watchProxyStream forwards etcd watch events to a proxied client stream. type watchProxyStream struct { ranges *watchRanges // mu protects watchers and nextWatcherID mu sync.Mutex // watchers receive events from watch broadcast. watchers map[int64]*watcher // nextWatcherID is the id to assign the next watcher on this stream. nextWatcherID int64 stream pb.Watch_WatchServer // watchCh receives watch responses from the watchers. watchCh chan *pb.WatchResponse ctx context.Context cancel context.CancelFunc // kv is used for permission checking kv clientv3.KV lg *zap.Logger } func (wps *watchProxyStream) close() { var wg sync.WaitGroup wps.cancel() wps.mu.Lock() wg.Add(len(wps.watchers)) for _, wpsw := range wps.watchers { go func(w *watcher) { wps.ranges.delete(w) wg.Done() }(wpsw) } wps.watchers = nil wps.mu.Unlock() wg.Wait() close(wps.watchCh) } func (wps *watchProxyStream) checkPermissionForWatch(key, rangeEnd []byte) error { if len(key) == 0 { // If the length of the key is 0, we need to obtain full range. // look at clientv3.WithPrefix() key = []byte{0} rangeEnd = []byte{0} } req := &pb.RangeRequest{ Serializable: true, Key: key, RangeEnd: rangeEnd, CountOnly: true, Limit: 1, } _, err := wps.kv.Do(wps.ctx, RangeRequestToOp(req)) return err } func (wps *watchProxyStream) recvLoop() error { for { req, err := wps.stream.Recv() if err != nil { return err } switch uv := req.RequestUnion.(type) { case *pb.WatchRequest_CreateRequest: cr := uv.CreateRequest if cr.StartRevision < 0 { wps.watchCh <- &pb.WatchResponse{ Header: &pb.ResponseHeader{}, WatchId: clientv3.InvalidWatchID, Created: true, Canceled: true, CancelReason: rpctypes.ErrCompacted.Error(), } continue } if err := wps.checkPermissionForWatch(cr.Key, cr.RangeEnd); err != nil { wps.watchCh <- &pb.WatchResponse{ Header: &pb.ResponseHeader{}, WatchId: clientv3.InvalidWatchID, Created: true, Canceled: true, CancelReason: err.Error(), } continue } wps.mu.Lock() w := &watcher{ wr: watchRange{string(cr.Key), string(cr.RangeEnd)}, id: wps.nextWatcherID, wps: wps, nextrev: cr.StartRevision, progress: cr.ProgressNotify, prevKV: cr.PrevKv, filters: v3rpc.FiltersFromRequest(cr), } if !w.wr.valid() { w.post(&pb.WatchResponse{WatchId: clientv3.InvalidWatchID, Created: true, Canceled: true}) wps.mu.Unlock() continue } wps.nextWatcherID++ w.nextrev = cr.StartRevision wps.watchers[w.id] = w wps.ranges.add(w) wps.mu.Unlock() wps.lg.Debug("create watcher", zap.String("key", w.wr.key), zap.String("end", w.wr.end), zap.Int64("watcherId", wps.nextWatcherID)) case *pb.WatchRequest_CancelRequest: wps.delete(uv.CancelRequest.WatchId) wps.lg.Debug("cancel watcher", zap.Int64("watcherId", uv.CancelRequest.WatchId)) default: // Panic or Fatalf would allow to network clients to crash the serve remotely. wps.lg.Error("not supported request type by gRPC proxy", zap.Stringer("request", req)) } } } func (wps *watchProxyStream) sendLoop() { for { select { case wresp, ok := <-wps.watchCh: if !ok { return } if err := wps.stream.Send(wresp); err != nil { return } case <-wps.ctx.Done(): return } } } func (wps *watchProxyStream) delete(id int64) { wps.mu.Lock() defer wps.mu.Unlock() w, ok := wps.watchers[id] if !ok { return } wps.ranges.delete(w) delete(wps.watchers, id) resp := &pb.WatchResponse{ Header: &w.lastHeader, WatchId: id, Canceled: true, } wps.watchCh <- resp } ================================================ FILE: server/proxy/grpcproxy/watch_broadcast.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "context" "sync" "time" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" ) // watchBroadcast broadcasts a server watcher to many client watchers. type watchBroadcast struct { // cancel stops the underlying etcd server watcher and closes ch. cancel context.CancelFunc donec chan struct{} // mu protects rev and receivers. mu sync.RWMutex // nextrev is the minimum expected next revision of the watcher on ch. nextrev int64 // receivers contains all the client-side watchers to serve. receivers map[*watcher]struct{} // responses counts the number of responses responses int lg *zap.Logger } func newWatchBroadcast(lg *zap.Logger, wp *watchProxy, w *watcher, update func(*watchBroadcast)) *watchBroadcast { cctx, cancel := context.WithCancel(wp.ctx) wb := &watchBroadcast{ cancel: cancel, nextrev: w.nextrev, receivers: make(map[*watcher]struct{}), donec: make(chan struct{}), lg: lg, } wb.add(w) go func() { defer close(wb.donec) opts := []clientv3.OpOption{ clientv3.WithRange(w.wr.end), clientv3.WithProgressNotify(), clientv3.WithRev(wb.nextrev), clientv3.WithPrevKV(), clientv3.WithCreatedNotify(), } cctx = withClientAuthToken(cctx, w.wps.stream.Context()) wch := wp.cw.Watch(cctx, w.wr.key, opts...) wp.lg.Debug("watch", zap.String("key", w.wr.key)) for wr := range wch { wb.bcast(wr) update(wb) } }() return wb } func (wb *watchBroadcast) bcast(wr clientv3.WatchResponse) { wb.mu.Lock() defer wb.mu.Unlock() // watchers start on the given revision, if any; ignore header rev on create if wb.responses > 0 || wb.nextrev == 0 { wb.nextrev = wr.Header.Revision + 1 } wb.responses++ for r := range wb.receivers { r.send(wr) } if len(wb.receivers) > 0 { eventsCoalescing.Add(float64(len(wb.receivers) - 1)) } } // add puts a watcher into receiving a broadcast if its revision at least // meets the broadcast revision. Returns true if added. func (wb *watchBroadcast) add(w *watcher) bool { wb.mu.Lock() defer wb.mu.Unlock() if wb.nextrev > w.nextrev || (wb.nextrev == 0 && w.nextrev != 0) { // wb is too far ahead, w will miss events // or wb is being established with a current watcher return false } if wb.responses == 0 { // Newly created; create event will be sent by etcd. wb.receivers[w] = struct{}{} return true } // already sent by etcd; emulate create event ok := w.post(&pb.WatchResponse{ Header: &pb.ResponseHeader{ // todo: fill in ClusterId // todo: fill in MemberId: Revision: w.nextrev, // todo: fill in RaftTerm: }, WatchId: w.id, Created: true, }) if !ok { return false } wb.receivers[w] = struct{}{} watchersCoalescing.Inc() return true } func (wb *watchBroadcast) delete(w *watcher) { wb.mu.Lock() defer wb.mu.Unlock() if _, ok := wb.receivers[w]; !ok { panic("deleting missing watcher from broadcast") } delete(wb.receivers, w) if len(wb.receivers) > 0 { // do not dec the only left watcher for coalescing. watchersCoalescing.Dec() } } func (wb *watchBroadcast) size() int { wb.mu.RLock() defer wb.mu.RUnlock() return len(wb.receivers) } func (wb *watchBroadcast) empty() bool { return wb.size() == 0 } func (wb *watchBroadcast) stop() { if !wb.empty() { // do not dec the only left watcher for coalescing. watchersCoalescing.Sub(float64(wb.size() - 1)) } wb.cancel() select { case <-wb.donec: // watchProxyStream will hold watchRanges global mutex lock all the time if client failed to cancel etcd watchers. // and it will cause the watch proxy to not work. // please see pr https://github.com/etcd-io/etcd/pull/12030 to get more detail info. case <-time.After(time.Second): wb.lg.Error("failed to cancel etcd watcher") } } ================================================ FILE: server/proxy/grpcproxy/watch_broadcasts.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "sync" ) type watchBroadcasts struct { wp *watchProxy // mu protects bcasts and watchers from the coalesce loop. mu sync.Mutex bcasts map[*watchBroadcast]struct{} watchers map[*watcher]*watchBroadcast updatec chan *watchBroadcast donec chan struct{} } // maxCoalesceRecievers prevents a popular watchBroadcast from being coalesced. const maxCoalesceReceivers = 5 func newWatchBroadcasts(wp *watchProxy) *watchBroadcasts { wbs := &watchBroadcasts{ wp: wp, bcasts: make(map[*watchBroadcast]struct{}), watchers: make(map[*watcher]*watchBroadcast), updatec: make(chan *watchBroadcast, 1), donec: make(chan struct{}), } go func() { defer close(wbs.donec) for wb := range wbs.updatec { wbs.coalesce(wb) } }() return wbs } func (wbs *watchBroadcasts) coalesce(wb *watchBroadcast) { if wb.size() >= maxCoalesceReceivers { return } wbs.mu.Lock() for wbswb := range wbs.bcasts { if wbswb == wb { continue } wb.mu.Lock() wbswb.mu.Lock() // 1. check if wbswb is behind wb so it won't skip any events in wb // 2. ensure wbswb started; nextrev == 0 may mean wbswb is waiting // for a current watcher and expects a create event from the server. if wb.nextrev >= wbswb.nextrev && wbswb.responses > 0 { for w := range wb.receivers { wbswb.receivers[w] = struct{}{} wbs.watchers[w] = wbswb } wb.receivers = nil } wbswb.mu.Unlock() wb.mu.Unlock() if wb.empty() { delete(wbs.bcasts, wb) wb.stop() break } } wbs.mu.Unlock() } func (wbs *watchBroadcasts) add(w *watcher) { wbs.mu.Lock() defer wbs.mu.Unlock() // find fitting bcast for wb := range wbs.bcasts { if wb.add(w) { wbs.watchers[w] = wb return } } // no fit; create a bcast wb := newWatchBroadcast(wbs.wp.lg, wbs.wp, w, wbs.update) wbs.watchers[w] = wb wbs.bcasts[wb] = struct{}{} } // delete removes a watcher and returns the number of remaining watchers. func (wbs *watchBroadcasts) delete(w *watcher) int { wbs.mu.Lock() defer wbs.mu.Unlock() wb, ok := wbs.watchers[w] if !ok { panic("deleting missing watcher from broadcasts") } delete(wbs.watchers, w) wb.delete(w) if wb.empty() { delete(wbs.bcasts, wb) wb.stop() } return len(wbs.bcasts) } func (wbs *watchBroadcasts) stop() { wbs.mu.Lock() for wb := range wbs.bcasts { wb.stop() } wbs.bcasts = nil close(wbs.updatec) wbs.mu.Unlock() <-wbs.donec } func (wbs *watchBroadcasts) update(wb *watchBroadcast) { select { case wbs.updatec <- wb: default: } } ================================================ FILE: server/proxy/grpcproxy/watch_ranges.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "sync" ) // watchRanges tracks all open watches for the proxy. type watchRanges struct { wp *watchProxy mu sync.Mutex bcasts map[watchRange]*watchBroadcasts } func newWatchRanges(wp *watchProxy) *watchRanges { return &watchRanges{ wp: wp, bcasts: make(map[watchRange]*watchBroadcasts), } } func (wrs *watchRanges) add(w *watcher) { wrs.mu.Lock() defer wrs.mu.Unlock() if wbs := wrs.bcasts[w.wr]; wbs != nil { wbs.add(w) return } wbs := newWatchBroadcasts(wrs.wp) wrs.bcasts[w.wr] = wbs wbs.add(w) } func (wrs *watchRanges) delete(w *watcher) { wrs.mu.Lock() defer wrs.mu.Unlock() wbs, ok := wrs.bcasts[w.wr] if !ok { panic("deleting missing range") } if wbs.delete(w) == 0 { wbs.stop() delete(wrs.bcasts, w.wr) } } func (wrs *watchRanges) stop() { wrs.mu.Lock() defer wrs.mu.Unlock() for _, wb := range wrs.bcasts { wb.stop() } wrs.bcasts = nil } ================================================ FILE: server/proxy/grpcproxy/watcher.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "time" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/storage/mvcc" ) type watchRange struct { key, end string } func (wr *watchRange) valid() bool { return len(wr.end) == 0 || wr.end > wr.key || (wr.end[0] == 0 && len(wr.end) == 1) } type watcher struct { // user configuration wr watchRange filters []mvcc.FilterFunc progress bool prevKV bool // id is the id returned to the client on its watch stream. id int64 // nextrev is the minimum expected next event revision. nextrev int64 // lastHeader has the last header sent over the stream. lastHeader pb.ResponseHeader // wps is the parent. wps *watchProxyStream } // send filters out repeated events by discarding revisions older // than the last one sent over the watch channel. func (w *watcher) send(wr clientv3.WatchResponse) { if wr.IsProgressNotify() && !w.progress { return } if w.nextrev > wr.Header.Revision && len(wr.Events) > 0 { return } if w.nextrev == 0 { // current watch; expect updates following this revision w.nextrev = wr.Header.Revision + 1 } events := make([]*mvccpb.Event, 0, len(wr.Events)) var lastRev int64 for i := range wr.Events { ev := (*mvccpb.Event)(wr.Events[i]) if ev.Kv.ModRevision < w.nextrev { continue } // We cannot update w.rev here. // txn can have multiple events with the same rev. // If w.nextrev updates here, it would skip events in the same txn. lastRev = ev.Kv.ModRevision filtered := false for _, filter := range w.filters { if filter(*ev) { filtered = true break } } if filtered { continue } if !w.prevKV { evCopy := *ev evCopy.PrevKv = nil ev = &evCopy } events = append(events, ev) } if lastRev >= w.nextrev { w.nextrev = lastRev + 1 } // all events are filtered out? if !wr.IsProgressNotify() && !wr.Created && len(events) == 0 && wr.CompactRevision == 0 { return } w.lastHeader = wr.Header w.post(&pb.WatchResponse{ Header: &wr.Header, Created: wr.Created, CompactRevision: wr.CompactRevision, Canceled: wr.Canceled, WatchId: w.id, Events: events, }) } // post puts a watch response on the watcher's proxy stream channel func (w *watcher) post(wr *pb.WatchResponse) bool { select { case w.wps.watchCh <- wr: case <-time.After(50 * time.Millisecond): w.wps.cancel() w.wps.lg.Error("failed to put a watch response on the watcher's proxy stream channel,err is timeout") return false } return true } ================================================ FILE: server/proxy/tcpproxy/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package tcpproxy is an OSI level 4 proxy for routing etcd clients to etcd servers. package tcpproxy ================================================ FILE: server/proxy/tcpproxy/userspace.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package tcpproxy import ( "fmt" "io" "math/rand" "net" "sync" "time" "go.uber.org/zap" ) type remote struct { mu sync.Mutex srv *net.SRV addr string inactive bool } func (r *remote) inactivate() { r.mu.Lock() defer r.mu.Unlock() r.inactive = true } func (r *remote) tryReactivate() error { conn, err := net.Dial("tcp", r.addr) if err != nil { return err } conn.Close() r.mu.Lock() defer r.mu.Unlock() r.inactive = false return nil } func (r *remote) isActive() bool { r.mu.Lock() defer r.mu.Unlock() return !r.inactive } type TCPProxy struct { Logger *zap.Logger Listener net.Listener Endpoints []*net.SRV MonitorInterval time.Duration donec chan struct{} mu sync.Mutex // guards the following fields remotes []*remote pickCount int // for round robin } func (tp *TCPProxy) Run() error { tp.donec = make(chan struct{}) if tp.MonitorInterval == 0 { tp.MonitorInterval = 5 * time.Minute } var eps []string // for logging for _, srv := range tp.Endpoints { addr := net.JoinHostPort(srv.Target, fmt.Sprintf("%d", srv.Port)) tp.remotes = append(tp.remotes, &remote{srv: srv, addr: addr}) eps = append(eps, addr) } if tp.Logger != nil { tp.Logger.Info("ready to proxy client requests", zap.Strings("endpoints", eps)) } go tp.runMonitor() for { in, err := tp.Listener.Accept() if err != nil { return err } go tp.serve(in) } } func (tp *TCPProxy) pick() *remote { var weighted []*remote var unweighted []*remote bestPr := uint16(65535) w := 0 // find best priority class for _, r := range tp.remotes { switch { case !r.isActive(): case r.srv.Priority < bestPr: bestPr = r.srv.Priority w = 0 weighted = nil unweighted = nil fallthrough case r.srv.Priority == bestPr: if r.srv.Weight > 0 { weighted = append(weighted, r) w += int(r.srv.Weight) } else { unweighted = append(unweighted, r) } } } if weighted != nil { if len(unweighted) > 0 && rand.Intn(100) == 1 { // In the presence of records containing weights greater // than 0, records with weight 0 should have a very small // chance of being selected. r := unweighted[tp.pickCount%len(unweighted)] tp.pickCount++ return r } // choose a uniform random number between 0 and the sum computed // (inclusive), and select the RR whose running sum value is the // first in the selected order choose := rand.Intn(w) for i := 0; i < len(weighted); i++ { choose -= int(weighted[i].srv.Weight) if choose <= 0 { return weighted[i] } } } if unweighted != nil { for i := 0; i < len(tp.remotes); i++ { picked := tp.remotes[tp.pickCount%len(tp.remotes)] tp.pickCount++ if picked.isActive() { return picked } } } return nil } func (tp *TCPProxy) serve(in net.Conn) { var ( err error out net.Conn ) for { tp.mu.Lock() remote := tp.pick() tp.mu.Unlock() if remote == nil { break } // TODO: add timeout out, err = net.Dial("tcp", remote.addr) if err == nil { break } remote.inactivate() if tp.Logger != nil { tp.Logger.Warn("deactivated endpoint", zap.String("address", remote.addr), zap.Duration("interval", tp.MonitorInterval), zap.Error(err)) } } if out == nil { in.Close() return } go func() { io.Copy(in, out) in.Close() out.Close() }() io.Copy(out, in) out.Close() in.Close() } func (tp *TCPProxy) runMonitor() { for { select { case <-time.After(tp.MonitorInterval): tp.mu.Lock() for _, rem := range tp.remotes { if rem.isActive() { continue } go func(r *remote) { if err := r.tryReactivate(); err != nil { if tp.Logger != nil { tp.Logger.Warn("failed to activate endpoint (stay inactive for another interval)", zap.String("address", r.addr), zap.Duration("interval", tp.MonitorInterval), zap.Error(err)) } } else { if tp.Logger != nil { tp.Logger.Info("activated", zap.String("address", r.addr)) } } }(rem) } tp.mu.Unlock() case <-tp.donec: return } } } func (tp *TCPProxy) Stop() { // graceful shutdown? // shutdown current connections? tp.Listener.Close() close(tp.donec) } ================================================ FILE: server/proxy/tcpproxy/userspace_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package tcpproxy import ( "fmt" "io" "net" "net/http" "net/http/httptest" "net/url" "testing" ) func TestUserspaceProxy(t *testing.T) { l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer l.Close() want := "hello proxy" ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, want) })) defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatal(err) } var port uint16 fmt.Sscanf(u.Port(), "%d", &port) p := TCPProxy{ Listener: l, Endpoints: []*net.SRV{{Target: u.Hostname(), Port: port}}, } go p.Run() defer p.Stop() u.Host = l.Addr().String() res, err := http.Get(u.String()) if err != nil { t.Fatal(err) } got, gerr := io.ReadAll(res.Body) res.Body.Close() if gerr != nil { t.Fatal(gerr) } if string(got) != want { t.Errorf("got = %s, want %s", got, want) } } func TestUserspaceProxyPriority(t *testing.T) { l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } defer l.Close() backends := []struct { Payload string Priority uint16 }{ {"hello proxy 1", 1}, {"hello proxy 2", 2}, {"hello proxy 3", 3}, } var eps []*net.SRV var front *url.URL for _, b := range backends { backend := b ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, backend.Payload) })) defer ts.Close() front, err = url.Parse(ts.URL) if err != nil { t.Fatal(err) } var port uint16 fmt.Sscanf(front.Port(), "%d", &port) ep := &net.SRV{Target: front.Hostname(), Port: port, Priority: backend.Priority} eps = append(eps, ep) } p := TCPProxy{ Listener: l, Endpoints: eps, } go p.Run() defer p.Stop() front.Host = l.Addr().String() res, err := http.Get(front.String()) if err != nil { t.Fatal(err) } got, gerr := io.ReadAll(res.Body) res.Body.Close() if gerr != nil { t.Fatal(gerr) } want := "hello proxy 1" if string(got) != want { t.Errorf("got = %s, want %s", got, want) } } ================================================ FILE: server/storage/backend/backend.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend import ( "fmt" "hash/crc32" "io" "os" "path/filepath" "sync" "sync/atomic" "time" humanize "github.com/dustin/go-humanize" "go.uber.org/zap" bolt "go.etcd.io/bbolt" "go.etcd.io/etcd/client/pkg/v3/verify" ) var ( defaultBatchLimit = 10000 defaultBatchInterval = 100 * time.Millisecond defragLimit = 10000 // InitialMmapSize is the initial size of the mmapped region. Setting this larger than // the potential max db size can prevent writer from blocking reader. // This only works for linux. InitialMmapSize = uint64(10 * 1024 * 1024 * 1024) // minSnapshotWarningTimeout is the minimum threshold to trigger a long running snapshot warning. minSnapshotWarningTimeout = 30 * time.Second ) type Backend interface { // ReadTx returns a read transaction. It is replaced by ConcurrentReadTx in the main data path, see #10523. ReadTx() ReadTx BatchTx() BatchTx // ConcurrentReadTx returns a non-blocking read transaction. ConcurrentReadTx() ReadTx Snapshot() Snapshot Hash(ignores func(bucketName, keyName []byte) bool) (uint32, error) // Size returns the current size of the backend physically allocated. // The backend can hold DB space that is not utilized at the moment, // since it can conduct pre-allocation or spare unused space for recycling. // Use SizeInUse() instead for the actual DB size. Size() int64 // SizeInUse returns the current size of the backend logically in use. // Since the backend can manage free space in a non-byte unit such as // number of pages, the returned value can be not exactly accurate in bytes. SizeInUse() int64 // OpenReadTxN returns the number of currently open read transactions in the backend. OpenReadTxN() int64 Defrag() error ForceCommit() Close() error // SetTxPostLockInsideApplyHook sets a txPostLockInsideApplyHook. SetTxPostLockInsideApplyHook(func()) } type Snapshot interface { // Size gets the size of the snapshot. Size() int64 // WriteTo writes the snapshot into the given writer. WriteTo(w io.Writer) (n int64, err error) // Close closes the snapshot. Close() error } type txReadBufferCache struct { mu sync.Mutex buf *txReadBuffer bufVersion uint64 } type backend struct { // size and commits are used with atomic operations so they must be // 64-bit aligned, otherwise 32-bit tests will crash // size is the number of bytes allocated in the backend size int64 // sizeInUse is the number of bytes actually used in the backend sizeInUse int64 // commits counts number of commits since start commits int64 // openReadTxN is the number of currently open read transactions in the backend openReadTxN int64 // mlock prevents backend database file to be swapped mlock bool mu sync.RWMutex bopts *bolt.Options db *bolt.DB batchInterval time.Duration batchLimit int batchTx *batchTxBuffered readTx *readTx // txReadBufferCache mirrors "txReadBuffer" within "readTx" -- readTx.baseReadTx.buf. // When creating "concurrentReadTx": // - if the cache is up-to-date, "readTx.baseReadTx.buf" copy can be skipped // - if the cache is empty or outdated, "readTx.baseReadTx.buf" copy is required txReadBufferCache txReadBufferCache stopc chan struct{} donec chan struct{} hooks Hooks // txPostLockInsideApplyHook is called each time right after locking the tx. txPostLockInsideApplyHook func() lg *zap.Logger } type BackendConfig struct { // Path is the file path to the backend file. Path string // BatchInterval is the maximum time before flushing the BatchTx. BatchInterval time.Duration // BatchLimit is the maximum puts before flushing the BatchTx. BatchLimit int // BackendFreelistType is the backend boltdb's freelist type. BackendFreelistType bolt.FreelistType // MmapSize is the number of bytes to mmap for the backend. MmapSize uint64 // Logger logs backend-side operations. Logger *zap.Logger // UnsafeNoFsync disables all uses of fsync. UnsafeNoFsync bool `json:"unsafe-no-fsync"` // Mlock prevents backend database file to be swapped Mlock bool // Timeout is the amount of time to wait to obtain a file lock. // When set to zero it will wait indefinitely. Timeout time.Duration // Hooks are getting executed during lifecycle of Backend's transactions. Hooks Hooks } type BackendConfigOption func(*BackendConfig) func DefaultBackendConfig(lg *zap.Logger) BackendConfig { return BackendConfig{ BatchInterval: defaultBatchInterval, BatchLimit: defaultBatchLimit, MmapSize: InitialMmapSize, Logger: lg, } } func New(bcfg BackendConfig) Backend { return newBackend(bcfg) } func WithMmapSize(size uint64) BackendConfigOption { return func(bcfg *BackendConfig) { bcfg.MmapSize = size } } func WithTimeout(timeout time.Duration) BackendConfigOption { return func(bcfg *BackendConfig) { bcfg.Timeout = timeout } } func NewDefaultBackend(lg *zap.Logger, path string, opts ...BackendConfigOption) Backend { bcfg := DefaultBackendConfig(lg) bcfg.Path = path for _, opt := range opts { opt(&bcfg) } return newBackend(bcfg) } func newBackend(bcfg BackendConfig) *backend { bopts := &bolt.Options{} if boltOpenOptions != nil { *bopts = *boltOpenOptions } if bcfg.Logger == nil { bcfg.Logger = zap.NewNop() } bopts.InitialMmapSize = bcfg.mmapSize() bopts.FreelistType = bcfg.BackendFreelistType bopts.NoSync = bcfg.UnsafeNoFsync bopts.NoGrowSync = bcfg.UnsafeNoFsync bopts.Mlock = bcfg.Mlock bopts.Logger = newBoltLoggerZap(bcfg) bopts.Timeout = bcfg.Timeout db, err := bolt.Open(bcfg.Path, 0o600, bopts) if err != nil { bcfg.Logger.Panic("failed to open database", zap.String("path", bcfg.Path), zap.Error(err)) } // In future, may want to make buffering optional for low-concurrency systems // or dynamically swap between buffered/non-buffered depending on workload. b := &backend{ bopts: bopts, db: db, batchInterval: bcfg.BatchInterval, batchLimit: bcfg.BatchLimit, mlock: bcfg.Mlock, readTx: &readTx{ baseReadTx: baseReadTx{ buf: txReadBuffer{ txBuffer: txBuffer{make(map[BucketID]*bucketBuffer)}, bufVersion: 0, }, buckets: make(map[BucketID]*bolt.Bucket), txWg: new(sync.WaitGroup), txMu: new(sync.RWMutex), }, }, txReadBufferCache: txReadBufferCache{ mu: sync.Mutex{}, bufVersion: 0, buf: nil, }, stopc: make(chan struct{}), donec: make(chan struct{}), lg: bcfg.Logger, } b.batchTx = newBatchTxBuffered(b) // We set it after newBatchTxBuffered to skip the 'empty' commit. b.hooks = bcfg.Hooks go b.run() return b } // BatchTx returns the current batch tx in coalescer. The tx can be used for read and // write operations. The write result can be retrieved within the same tx immediately. // The write result is isolated with other txs until the current one get committed. func (b *backend) BatchTx() BatchTx { return b.batchTx } func (b *backend) SetTxPostLockInsideApplyHook(hook func()) { // It needs to lock the batchTx, because the periodic commit // may be accessing the txPostLockInsideApplyHook at the moment. b.batchTx.lock() defer b.batchTx.Unlock() b.txPostLockInsideApplyHook = hook } func (b *backend) ReadTx() ReadTx { return b.readTx } // ConcurrentReadTx creates and returns a new ReadTx, which: // A) creates and keeps a copy of backend.readTx.txReadBuffer, // B) references the boltdb read Tx (and its bucket cache) of current batch interval. func (b *backend) ConcurrentReadTx() ReadTx { b.readTx.RLock() defer b.readTx.RUnlock() // prevent boltdb read Tx from been rolled back until store read Tx is done. Needs to be called when holding readTx.RLock(). b.readTx.txWg.Add(1) // TODO: might want to copy the read buffer lazily - create copy when A) end of a write transaction B) end of a batch interval. // inspect/update cache recency iff there's no ongoing update to the cache // this falls through if there's no cache update // by this line, "ConcurrentReadTx" code path is already protected against concurrent "writeback" operations // which requires write lock to update "readTx.baseReadTx.buf". // Which means setting "buf *txReadBuffer" with "readTx.buf.unsafeCopy()" is guaranteed to be up-to-date, // whereas "txReadBufferCache.buf" may be stale from concurrent "writeback" operations. // We only update "txReadBufferCache.buf" if we know "buf *txReadBuffer" is up-to-date. // The update to "txReadBufferCache.buf" will benefit the following "ConcurrentReadTx" creation // by avoiding copying "readTx.baseReadTx.buf". b.txReadBufferCache.mu.Lock() curCache := b.txReadBufferCache.buf curCacheVer := b.txReadBufferCache.bufVersion curBufVer := b.readTx.buf.bufVersion isEmptyCache := curCache == nil isStaleCache := curCacheVer != curBufVer var buf *txReadBuffer switch { case isEmptyCache: // perform safe copy of buffer while holding "b.txReadBufferCache.mu.Lock" // this is only supposed to run once so there won't be much overhead curBuf := b.readTx.buf.unsafeCopy() buf = &curBuf case isStaleCache: // to maximize the concurrency, try unsafe copy of buffer // release the lock while copying buffer -- cache may become stale again and // get overwritten by someone else. // therefore, we need to check the readTx buffer version again b.txReadBufferCache.mu.Unlock() curBuf := b.readTx.buf.unsafeCopy() b.txReadBufferCache.mu.Lock() buf = &curBuf default: // neither empty nor stale cache, just use the current buffer buf = curCache } // txReadBufferCache.bufVersion can be modified when we doing an unsafeCopy() // as a result, curCacheVer could be no longer the same as // txReadBufferCache.bufVersion // if !isEmptyCache && curCacheVer != b.txReadBufferCache.bufVersion // then the cache became stale while copying "readTx.baseReadTx.buf". // It is safe to not update "txReadBufferCache.buf", because the next following // "ConcurrentReadTx" creation will trigger a new "readTx.baseReadTx.buf" copy // and "buf" is still used for the current "concurrentReadTx.baseReadTx.buf". if isEmptyCache || curCacheVer == b.txReadBufferCache.bufVersion { // continue if the cache is never set or no one has modified the cache b.txReadBufferCache.buf = buf b.txReadBufferCache.bufVersion = curBufVer } b.txReadBufferCache.mu.Unlock() // concurrentReadTx is not supposed to write to its txReadBuffer return &concurrentReadTx{ baseReadTx: baseReadTx{ buf: *buf, txMu: b.readTx.txMu, tx: b.readTx.tx, buckets: b.readTx.buckets, txWg: b.readTx.txWg, }, } } // ForceCommit forces the current batching tx to commit. func (b *backend) ForceCommit() { b.batchTx.Commit() } func (b *backend) Snapshot() Snapshot { b.batchTx.Commit() b.mu.RLock() defer b.mu.RUnlock() tx, err := b.db.Begin(false) if err != nil { b.lg.Fatal("failed to begin tx", zap.Error(err)) } stopc, donec := make(chan struct{}), make(chan struct{}) dbBytes := tx.Size() go func() { defer close(donec) // sendRateBytes is based on transferring snapshot data over a 1 gigabit/s connection // assuming a min tcp throughput of 100MB/s. var sendRateBytes int64 = 100 * 1024 * 1024 warningTimeout := time.Duration(int64((float64(dbBytes) / float64(sendRateBytes)) * float64(time.Second))) if warningTimeout < minSnapshotWarningTimeout { warningTimeout = minSnapshotWarningTimeout } start := time.Now() ticker := time.NewTicker(warningTimeout) defer ticker.Stop() for { select { case <-ticker.C: b.lg.Warn( "snapshotting taking too long to transfer", zap.Duration("taking", time.Since(start)), zap.Int64("bytes", dbBytes), zap.String("size", humanize.Bytes(uint64(dbBytes))), ) case <-stopc: snapshotTransferSec.Observe(time.Since(start).Seconds()) return } } }() return &snapshot{tx, stopc, donec} } func (b *backend) Hash(ignores func(bucketName, keyName []byte) bool) (uint32, error) { h := crc32.New(crc32.MakeTable(crc32.Castagnoli)) b.mu.RLock() defer b.mu.RUnlock() err := b.db.View(func(tx *bolt.Tx) error { c := tx.Cursor() for next, _ := c.First(); next != nil; next, _ = c.Next() { b := tx.Bucket(next) if b == nil { return fmt.Errorf("cannot get hash of bucket %s", next) } h.Write(next) b.ForEach(func(k, v []byte) error { if ignores != nil && !ignores(next, k) { h.Write(k) h.Write(v) } return nil }) } return nil }) if err != nil { return 0, err } return h.Sum32(), nil } func (b *backend) Size() int64 { return atomic.LoadInt64(&b.size) } func (b *backend) SizeInUse() int64 { return atomic.LoadInt64(&b.sizeInUse) } func (b *backend) run() { defer close(b.donec) t := time.NewTimer(b.batchInterval) defer t.Stop() for { select { case <-t.C: case <-b.stopc: b.batchTx.CommitAndStop() return } if b.batchTx.safePending() != 0 { b.batchTx.Commit() } t.Reset(b.batchInterval) } } func (b *backend) Close() error { close(b.stopc) <-b.donec b.mu.Lock() defer b.mu.Unlock() return b.db.Close() } // Commits returns total number of commits since start func (b *backend) Commits() int64 { return atomic.LoadInt64(&b.commits) } func (b *backend) Defrag() error { return b.defrag() } func (b *backend) defrag() error { verify.Assert(b.lg != nil, "the logger should not be nil") now := time.Now() isDefragActive.Set(1) defer isDefragActive.Set(0) // TODO: make this non-blocking? // lock batchTx to ensure nobody is using previous tx, and then // close previous ongoing tx. b.batchTx.LockOutsideApply() defer b.batchTx.Unlock() // lock database after lock tx to avoid deadlock. b.mu.Lock() defer b.mu.Unlock() // block concurrent read requests while resetting tx b.readTx.Lock() defer b.readTx.Unlock() // Create a temporary file to ensure we start with a clean slate. // Snapshotter.cleanupSnapdir cleans up any of these that are found during startup. dir := filepath.Dir(b.db.Path()) temp, err := os.CreateTemp(dir, "db.tmp.*") if err != nil { return err } options := bolt.Options{} if boltOpenOptions != nil { options = *boltOpenOptions } options.OpenFile = func(_ string, _ int, _ os.FileMode) (file *os.File, err error) { // gofail: var defragOpenFileError string // return nil, fmt.Errorf(defragOpenFileError) return temp, nil } // Don't load tmp db into memory regardless of opening options options.Mlock = false tdbp := temp.Name() tmpdb, err := bolt.Open(tdbp, 0o600, &options) if err != nil { temp.Close() if rmErr := os.Remove(temp.Name()); rmErr != nil { b.lg.Error( "failed to remove temporary file", zap.String("path", temp.Name()), zap.Error(rmErr), ) } return err } dbp := b.db.Path() size1, sizeInUse1 := b.Size(), b.SizeInUse() b.lg.Info( "defragmenting", zap.String("path", dbp), zap.Int64("current-db-size-bytes", size1), zap.String("current-db-size", humanize.Bytes(uint64(size1))), zap.Int64("current-db-size-in-use-bytes", sizeInUse1), zap.String("current-db-size-in-use", humanize.Bytes(uint64(sizeInUse1))), ) defer func() { // NOTE: We should exit as soon as possible because that tx // might be closed. The inflight request might use invalid // tx and then panic as well. The real panic reason might be // shadowed by new panic. So, we should fatal here with lock. if rerr := recover(); rerr != nil { b.lg.Fatal("unexpected panic during defrag", zap.Any("panic", rerr)) } }() // Commit/stop and then reset current transactions (including the readTx) b.batchTx.unsafeCommit(true) b.batchTx.tx = nil // gofail: var defragBeforeCopy struct{} err = defragdb(b.db, tmpdb, defragLimit) if err != nil { tmpdb.Close() if rmErr := os.RemoveAll(tmpdb.Path()); rmErr != nil { b.lg.Error("failed to remove db.tmp after defragmentation completed", zap.Error(rmErr)) } // restore the bbolt transactions if defragmentation fails b.batchTx.tx = b.unsafeBegin(true) b.readTx.tx = b.unsafeBegin(false) return err } err = b.db.Close() if err != nil { b.lg.Fatal("failed to close database", zap.Error(err)) } err = tmpdb.Close() if err != nil { b.lg.Fatal("failed to close tmp database", zap.Error(err)) } // gofail: var defragBeforeRename struct{} err = os.Rename(tdbp, dbp) if err != nil { b.lg.Fatal("failed to rename tmp database", zap.Error(err)) } b.db, err = bolt.Open(dbp, 0o600, b.bopts) if err != nil { b.lg.Fatal("failed to open database", zap.String("path", dbp), zap.Error(err)) } b.batchTx.tx = b.unsafeBegin(true) b.readTx.reset() b.readTx.tx = b.unsafeBegin(false) size := b.readTx.tx.Size() db := b.readTx.tx.DB() atomic.StoreInt64(&b.size, size) atomic.StoreInt64(&b.sizeInUse, size-(int64(db.Stats().FreePageN)*int64(db.Info().PageSize))) took := time.Since(now) defragSec.Observe(took.Seconds()) size2, sizeInUse2 := b.Size(), b.SizeInUse() b.lg.Info( "finished defragmenting directory", zap.String("path", dbp), zap.Int64("current-db-size-bytes-diff", size2-size1), zap.Int64("current-db-size-bytes", size2), zap.String("current-db-size", humanize.Bytes(uint64(size2))), zap.Int64("current-db-size-in-use-bytes-diff", sizeInUse2-sizeInUse1), zap.Int64("current-db-size-in-use-bytes", sizeInUse2), zap.String("current-db-size-in-use", humanize.Bytes(uint64(sizeInUse2))), zap.Duration("took", took), ) return nil } func defragdb(odb, tmpdb *bolt.DB, limit int) error { // gofail: var defragdbFail string // return fmt.Errorf(defragdbFail) // open a tx on tmpdb for writes tmptx, err := tmpdb.Begin(true) if err != nil { return err } defer func() { if err != nil { tmptx.Rollback() } }() // open a tx on old db for read tx, err := odb.Begin(false) if err != nil { return err } defer tx.Rollback() c := tx.Cursor() count := 0 for next, _ := c.First(); next != nil; next, _ = c.Next() { b := tx.Bucket(next) if b == nil { return fmt.Errorf("backend: cannot defrag bucket %s", next) } tmpb, berr := tmptx.CreateBucketIfNotExists(next) if berr != nil { return berr } tmpb.FillPercent = 0.9 // for bucket2seq write in for each if err = b.ForEach(func(k, v []byte) error { count++ if count > limit { err = tmptx.Commit() if err != nil { return err } tmptx, err = tmpdb.Begin(true) if err != nil { return err } tmpb = tmptx.Bucket(next) tmpb.FillPercent = 0.9 // for bucket2seq write in for each count = 0 } return tmpb.Put(k, v) }); err != nil { return err } } return tmptx.Commit() } func (b *backend) begin(write bool) *bolt.Tx { b.mu.RLock() tx := b.unsafeBegin(write) b.mu.RUnlock() size := tx.Size() db := tx.DB() stats := db.Stats() atomic.StoreInt64(&b.size, size) atomic.StoreInt64(&b.sizeInUse, size-(int64(stats.FreePageN)*int64(db.Info().PageSize))) atomic.StoreInt64(&b.openReadTxN, int64(stats.OpenTxN)) return tx } func (b *backend) unsafeBegin(write bool) *bolt.Tx { // gofail: var beforeStartDBTxn struct{} tx, err := b.db.Begin(write) // gofail: var afterStartDBTxn struct{} if err != nil { b.lg.Fatal("failed to begin tx", zap.Error(err)) } return tx } func (b *backend) OpenReadTxN() int64 { return atomic.LoadInt64(&b.openReadTxN) } type snapshot struct { *bolt.Tx stopc chan struct{} donec chan struct{} } func (s *snapshot) Close() error { close(s.stopc) <-s.donec return s.Tx.Rollback() } func newBoltLoggerZap(bcfg BackendConfig) bolt.Logger { lg := bcfg.Logger.Named("bbolt") return &zapBoltLogger{lg.WithOptions(zap.AddCallerSkip(1)).Sugar()} } type zapBoltLogger struct { *zap.SugaredLogger } func (zl *zapBoltLogger) Warning(args ...any) { zl.SugaredLogger.Warn(args...) } func (zl *zapBoltLogger) Warningf(format string, args ...any) { zl.SugaredLogger.Warnf(format, args...) } ================================================ FILE: server/storage/backend/backend_bench_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend_test import ( "crypto/rand" "testing" "time" "github.com/stretchr/testify/require" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/schema" ) func BenchmarkBackendPut(b *testing.B) { backend, _ := betesting.NewTmpBackend(b, 100*time.Millisecond, 10000) defer betesting.Close(b, backend) // prepare keys keys := make([][]byte, b.N) for i := 0; i < b.N; i++ { keys[i] = make([]byte, 64) _, err := rand.Read(keys[i]) require.NoError(b, err) } value := make([]byte, 128) _, err := rand.Read(value) require.NoError(b, err) batchTx := backend.BatchTx() batchTx.Lock() batchTx.UnsafeCreateBucket(schema.Test) batchTx.Unlock() b.ResetTimer() for i := 0; i < b.N; i++ { batchTx.Lock() batchTx.UnsafePut(schema.Test, keys[i], value) batchTx.Unlock() } } ================================================ FILE: server/storage/backend/backend_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend_test import ( "fmt" "os" "reflect" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" bolt "go.etcd.io/bbolt" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/schema" ) func TestBackendClose(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) // check close could work done := make(chan struct{}, 1) go func() { err := b.Close() if err != nil { t.Errorf("close error = %v, want nil", err) } done <- struct{}{} }() select { case <-done: case <-time.After(10 * time.Second): t.Errorf("failed to close database in 10s") } } func TestBackendSnapshot(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar")) tx.Unlock() b.ForceCommit() // write snapshot to a new file f, err := os.CreateTemp(t.TempDir(), "etcd_backend_test") if err != nil { t.Fatal(err) } snap := b.Snapshot() defer func() { assert.NoError(t, snap.Close()) }() if _, err := snap.WriteTo(f); err != nil { t.Fatal(err) } require.NoError(t, f.Close()) // bootstrap new backend from the snapshot bcfg := backend.DefaultBackendConfig(zaptest.NewLogger(t)) bcfg.Path, bcfg.BatchInterval, bcfg.BatchLimit = f.Name(), time.Hour, 10000 nb := backend.New(bcfg) defer betesting.Close(t, nb) newTx := nb.BatchTx() newTx.Lock() ks, _ := newTx.UnsafeRange(schema.Test, []byte("foo"), []byte("goo"), 0) if len(ks) != 1 { t.Errorf("len(kvs) = %d, want 1", len(ks)) } newTx.Unlock() } func TestBackendBatchIntervalCommit(t *testing.T) { // start backend with super short batch interval so // we do not need to wait long before commit to happen. b, _ := betesting.NewTmpBackend(t, time.Nanosecond, 10000) defer betesting.Close(t, b) pc := backend.CommitsForTest(b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar")) tx.Unlock() for i := 0; i < 10; i++ { if backend.CommitsForTest(b) >= pc+1 { break } time.Sleep(time.Duration(i*100) * time.Millisecond) } // check whether put happens via db view assert.NoError(t, backend.DbFromBackendForTest(b).View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte("test")) if bucket == nil { t.Errorf("bucket test does not exit") return nil } v := bucket.Get([]byte("foo")) if v == nil { t.Errorf("foo key failed to written in backend") } return nil })) } func TestBackendDefrag(t *testing.T) { bcfg := backend.DefaultBackendConfig(zaptest.NewLogger(t)) // Make sure we change BackendFreelistType // The goal is to verify that we restore config option after defrag. if bcfg.BackendFreelistType == bolt.FreelistMapType { bcfg.BackendFreelistType = bolt.FreelistArrayType } else { bcfg.BackendFreelistType = bolt.FreelistMapType } b, _ := betesting.NewTmpBackendFromCfg(t, bcfg) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) for i := 0; i < backend.DefragLimitForTest()+100; i++ { tx.UnsafePut(schema.Test, []byte(fmt.Sprintf("foo_%d", i)), []byte("bar")) } tx.Unlock() b.ForceCommit() // remove some keys to ensure the disk space will be reclaimed after defrag tx = b.BatchTx() tx.Lock() for i := 0; i < 50; i++ { tx.UnsafeDelete(schema.Test, []byte(fmt.Sprintf("foo_%d", i))) } tx.Unlock() b.ForceCommit() size := b.Size() // shrink and check hash oh, err := b.Hash(nil) if err != nil { t.Fatal(err) } err = b.Defrag() if err != nil { t.Fatal(err) } nh, err := b.Hash(nil) if err != nil { t.Fatal(err) } if oh != nh { t.Errorf("hash = %v, want %v", nh, oh) } nsize := b.Size() if nsize >= size { t.Errorf("new size = %v, want < %d", nsize, size) } db := backend.DbFromBackendForTest(b) if db.FreelistType != bcfg.BackendFreelistType { t.Errorf("db FreelistType = [%v], want [%v]", db.FreelistType, bcfg.BackendFreelistType) } // try put more keys after shrink. tx = b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafePut(schema.Test, []byte("more"), []byte("bar")) tx.Unlock() b.ForceCommit() } // TestBackendWriteback ensures writes are stored to the read txn on write txn unlock. func TestBackendWriteback(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Key) tx.UnsafePut(schema.Key, []byte("abc"), []byte("bar")) tx.UnsafePut(schema.Key, []byte("def"), []byte("baz")) tx.UnsafePut(schema.Key, []byte("overwrite"), []byte("1")) tx.Unlock() // overwrites should be propagated too tx.Lock() tx.UnsafePut(schema.Key, []byte("overwrite"), []byte("2")) tx.Unlock() keys := []struct { key []byte end []byte limit int64 wkey [][]byte wval [][]byte }{ { key: []byte("abc"), end: nil, wkey: [][]byte{[]byte("abc")}, wval: [][]byte{[]byte("bar")}, }, { key: []byte("abc"), end: []byte("def"), wkey: [][]byte{[]byte("abc")}, wval: [][]byte{[]byte("bar")}, }, { key: []byte("abc"), end: []byte("deg"), wkey: [][]byte{[]byte("abc"), []byte("def")}, wval: [][]byte{[]byte("bar"), []byte("baz")}, }, { key: []byte("abc"), end: []byte("\xff"), limit: 1, wkey: [][]byte{[]byte("abc")}, wval: [][]byte{[]byte("bar")}, }, { key: []byte("abc"), end: []byte("\xff"), wkey: [][]byte{[]byte("abc"), []byte("def"), []byte("overwrite")}, wval: [][]byte{[]byte("bar"), []byte("baz"), []byte("2")}, }, } rtx := b.ReadTx() for i, tt := range keys { func() { rtx.RLock() defer rtx.RUnlock() k, v := rtx.UnsafeRange(schema.Key, tt.key, tt.end, tt.limit) if !reflect.DeepEqual(tt.wkey, k) || !reflect.DeepEqual(tt.wval, v) { t.Errorf("#%d: want k=%+v, v=%+v; got k=%+v, v=%+v", i, tt.wkey, tt.wval, k, v) } }() } } // TestConcurrentReadTx ensures that current read transaction can see all prior writes stored in read buffer func TestConcurrentReadTx(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) wtx1 := b.BatchTx() wtx1.Lock() wtx1.UnsafeCreateBucket(schema.Key) wtx1.UnsafePut(schema.Key, []byte("abc"), []byte("ABC")) wtx1.UnsafePut(schema.Key, []byte("overwrite"), []byte("1")) wtx1.Unlock() wtx2 := b.BatchTx() wtx2.Lock() wtx2.UnsafePut(schema.Key, []byte("def"), []byte("DEF")) wtx2.UnsafePut(schema.Key, []byte("overwrite"), []byte("2")) wtx2.Unlock() rtx := b.ConcurrentReadTx() rtx.RLock() // no-op k, v := rtx.UnsafeRange(schema.Key, []byte("abc"), []byte("\xff"), 0) rtx.RUnlock() wKey := [][]byte{[]byte("abc"), []byte("def"), []byte("overwrite")} wVal := [][]byte{[]byte("ABC"), []byte("DEF"), []byte("2")} if !reflect.DeepEqual(wKey, k) || !reflect.DeepEqual(wVal, v) { t.Errorf("want k=%+v, v=%+v; got k=%+v, v=%+v", wKey, wVal, k, v) } } // TestBackendWritebackForEach checks that partially written / buffered // data is visited in the same order as fully committed data. func TestBackendWritebackForEach(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Key) for i := 0; i < 5; i++ { k := []byte(fmt.Sprintf("%04d", i)) tx.UnsafePut(schema.Key, k, []byte("bar")) } tx.Unlock() // writeback b.ForceCommit() tx.Lock() tx.UnsafeCreateBucket(schema.Key) for i := 5; i < 20; i++ { k := []byte(fmt.Sprintf("%04d", i)) tx.UnsafePut(schema.Key, k, []byte("bar")) } tx.Unlock() seq := "" getSeq := func(k, v []byte) error { seq += string(k) return nil } rtx := b.ReadTx() rtx.RLock() require.NoError(t, rtx.UnsafeForEach(schema.Key, getSeq)) rtx.RUnlock() partialSeq := seq seq = "" b.ForceCommit() tx.Lock() require.NoError(t, tx.UnsafeForEach(schema.Key, getSeq)) tx.Unlock() if seq != partialSeq { t.Fatalf("expected %q, got %q", seq, partialSeq) } } ================================================ FILE: server/storage/backend/batch_tx.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend import ( "bytes" "errors" "math" "sync" "sync/atomic" "time" "go.uber.org/zap" bolt "go.etcd.io/bbolt" bolterrors "go.etcd.io/bbolt/errors" ) type BucketID int type Bucket interface { // ID returns a unique identifier of a bucket. // The id must NOT be persisted and can be used as lightweight identificator // in the in-memory maps. ID() BucketID Name() []byte // String implements Stringer (human readable name). String() string // IsSafeRangeBucket is a hack to avoid inadvertently reading duplicate keys; // overwrites on a bucket should only fetch with limit=1, but safeRangeBucket // is known to never overwrite any key so range is safe. IsSafeRangeBucket() bool } type BatchTx interface { Lock() Unlock() // Commit commits a previous tx and begins a new writable one. Commit() // CommitAndStop commits the previous tx and does not create a new one. CommitAndStop() LockInsideApply() LockOutsideApply() UnsafeReadWriter } type UnsafeReadWriter interface { UnsafeReader UnsafeWriter } type UnsafeWriter interface { UnsafeCreateBucket(bucket Bucket) UnsafeDeleteBucket(bucket Bucket) UnsafePut(bucket Bucket, key []byte, value []byte) UnsafeSeqPut(bucket Bucket, key []byte, value []byte) UnsafeDelete(bucket Bucket, key []byte) } type batchTx struct { sync.Mutex tx *bolt.Tx backend *backend pending int } // Lock is supposed to be called only by the unit test. func (t *batchTx) Lock() { ValidateCalledInsideUnittest(t.backend.lg) t.lock() } func (t *batchTx) lock() { t.Mutex.Lock() } func (t *batchTx) LockInsideApply() { t.lock() if t.backend.txPostLockInsideApplyHook != nil { // The callers of some methods (i.e., (*RaftCluster).AddMember) // can be coming from both InsideApply and OutsideApply, but the // callers from OutsideApply will have a nil txPostLockInsideApplyHook. // So we should check the txPostLockInsideApplyHook before validating // the callstack. ValidateCalledInsideApply(t.backend.lg) t.backend.txPostLockInsideApplyHook() } } func (t *batchTx) LockOutsideApply() { ValidateCalledOutSideApply(t.backend.lg) t.lock() } func (t *batchTx) Unlock() { if t.pending >= t.backend.batchLimit { t.commit(false) } t.Mutex.Unlock() } func (t *batchTx) UnsafeCreateBucket(bucket Bucket) { if _, err := t.tx.CreateBucketIfNotExists(bucket.Name()); err != nil { t.backend.lg.Fatal( "failed to create a bucket", zap.Stringer("bucket-name", bucket), zap.Error(err), ) } t.pending++ } func (t *batchTx) UnsafeDeleteBucket(bucket Bucket) { err := t.tx.DeleteBucket(bucket.Name()) if err != nil && !errors.Is(err, bolterrors.ErrBucketNotFound) { t.backend.lg.Fatal( "failed to delete a bucket", zap.Stringer("bucket-name", bucket), zap.Error(err), ) } t.pending++ } // UnsafePut must be called holding the lock on the tx. func (t *batchTx) UnsafePut(bucket Bucket, key []byte, value []byte) { t.unsafePut(bucket, key, value, false) } // UnsafeSeqPut must be called holding the lock on the tx. func (t *batchTx) UnsafeSeqPut(bucket Bucket, key []byte, value []byte) { t.unsafePut(bucket, key, value, true) } func (t *batchTx) unsafePut(bucketType Bucket, key []byte, value []byte, seq bool) { bucket := t.tx.Bucket(bucketType.Name()) if bucket == nil { t.backend.lg.Fatal( "failed to find a bucket", zap.Stringer("bucket-name", bucketType), zap.Stack("stack"), ) } if seq { // it is useful to increase fill percent when the workloads are mostly append-only. // this can delay the page split and reduce space usage. bucket.FillPercent = 0.9 } if err := bucket.Put(key, value); err != nil { t.backend.lg.Fatal( "failed to write to a bucket", zap.Stringer("bucket-name", bucketType), zap.Error(err), ) } t.pending++ } // UnsafeRange must be called holding the lock on the tx. func (t *batchTx) UnsafeRange(bucketType Bucket, key, endKey []byte, limit int64) ([][]byte, [][]byte) { bucket := t.tx.Bucket(bucketType.Name()) if bucket == nil { t.backend.lg.Fatal( "failed to find a bucket", zap.Stringer("bucket-name", bucketType), zap.Stack("stack"), ) } return unsafeRange(bucket.Cursor(), key, endKey, limit) } func unsafeRange(c *bolt.Cursor, key, endKey []byte, limit int64) (keys [][]byte, vs [][]byte) { if limit <= 0 { limit = math.MaxInt64 } var isMatch func(b []byte) bool if len(endKey) > 0 { isMatch = func(b []byte) bool { return bytes.Compare(b, endKey) < 0 } } else { isMatch = func(b []byte) bool { return bytes.Equal(b, key) } limit = 1 } for ck, cv := c.Seek(key); ck != nil && isMatch(ck); ck, cv = c.Next() { vs = append(vs, cv) keys = append(keys, ck) if limit == int64(len(keys)) { break } } return keys, vs } // UnsafeDelete must be called holding the lock on the tx. func (t *batchTx) UnsafeDelete(bucketType Bucket, key []byte) { bucket := t.tx.Bucket(bucketType.Name()) if bucket == nil { t.backend.lg.Fatal( "failed to find a bucket", zap.Stringer("bucket-name", bucketType), zap.Stack("stack"), ) } err := bucket.Delete(key) if err != nil { t.backend.lg.Fatal( "failed to delete a key", zap.Stringer("bucket-name", bucketType), zap.Error(err), ) } t.pending++ } // UnsafeForEach must be called holding the lock on the tx. func (t *batchTx) UnsafeForEach(bucket Bucket, visitor func(k, v []byte) error) error { return unsafeForEach(t.tx, bucket, visitor) } func unsafeForEach(tx *bolt.Tx, bucket Bucket, visitor func(k, v []byte) error) error { if b := tx.Bucket(bucket.Name()); b != nil { return b.ForEach(visitor) } return nil } // Commit commits a previous tx and begins a new writable one. func (t *batchTx) Commit() { t.lock() t.commit(false) t.Unlock() } // CommitAndStop commits the previous tx and does not create a new one. func (t *batchTx) CommitAndStop() { t.lock() t.commit(true) t.Unlock() } func (t *batchTx) safePending() int { t.Mutex.Lock() defer t.Mutex.Unlock() return t.pending } func (t *batchTx) commit(stop bool) { // commit the last tx if t.tx != nil { if t.pending == 0 && !stop { return } start := time.Now() // gofail: var beforeCommit struct{} err := t.tx.Commit() // gofail: var afterCommit struct{} rebalanceSec.Observe(t.tx.Stats().RebalanceTime.Seconds()) spillSec.Observe(t.tx.Stats().SpillTime.Seconds()) writeSec.Observe(t.tx.Stats().WriteTime.Seconds()) commitSec.Observe(time.Since(start).Seconds()) atomic.AddInt64(&t.backend.commits, 1) t.pending = 0 if err != nil { t.backend.lg.Fatal("failed to commit tx", zap.Error(err)) } } if !stop { t.tx = t.backend.begin(true) } } type batchTxBuffered struct { batchTx buf txWriteBuffer pendingDeleteOperations int } func newBatchTxBuffered(backend *backend) *batchTxBuffered { tx := &batchTxBuffered{ batchTx: batchTx{backend: backend}, buf: txWriteBuffer{ txBuffer: txBuffer{make(map[BucketID]*bucketBuffer)}, bucket2seq: make(map[BucketID]bool), }, } tx.Commit() return tx } func (t *batchTxBuffered) Unlock() { if t.pending != 0 { t.backend.readTx.Lock() // blocks txReadBuffer for writing. // gofail: var beforeWritebackBuf struct{} t.buf.writeback(&t.backend.readTx.buf) // gofail: var afterWritebackBuf struct{} t.backend.readTx.Unlock() // We commit the transaction when the number of pending operations // reaches the configured limit(batchLimit) to prevent it from // becoming excessively large. // // But we also need to commit the transaction immediately if there // is any pending deleting operation, otherwise etcd might run into // a situation that it haven't finished committing the data into backend // storage (note: etcd periodically commits the bbolt transactions // instead of on each request) when it applies next request. Accordingly, // etcd may still read the stale data from bbolt when processing next // request. So it breaks the linearizability. // // Note we don't need to commit the transaction for put requests if // it doesn't exceed the batch limit, because there is a buffer on top // of the bbolt. Each time when etcd reads data from backend storage, // it will read data from both bbolt and the buffer. But there is no // such a buffer for delete requests. // // Please also refer to // https://github.com/etcd-io/etcd/pull/17119#issuecomment-1857547158 if t.pending >= t.backend.batchLimit || t.pendingDeleteOperations > 0 { t.commit(false) } } t.batchTx.Unlock() } func (t *batchTxBuffered) Commit() { t.lock() t.commit(false) t.Unlock() } func (t *batchTxBuffered) CommitAndStop() { t.lock() t.commit(true) t.Unlock() } func (t *batchTxBuffered) commit(stop bool) { // all read txs must be closed to acquire boltdb commit rwlock t.backend.readTx.Lock() t.unsafeCommit(stop) t.backend.readTx.Unlock() } func (t *batchTxBuffered) unsafeCommit(stop bool) { if t.backend.hooks != nil { // gofail: var commitBeforePreCommitHook struct{} t.backend.hooks.OnPreCommitUnsafe(t) // gofail: var commitAfterPreCommitHook struct{} } if t.backend.readTx.tx != nil { // wait all store read transactions using the current boltdb tx to finish, // then close the boltdb tx go func(tx *bolt.Tx, wg *sync.WaitGroup) { wg.Wait() if err := tx.Rollback(); err != nil { t.backend.lg.Fatal("failed to rollback tx", zap.Error(err)) } }(t.backend.readTx.tx, t.backend.readTx.txWg) t.backend.readTx.reset() } t.batchTx.commit(stop) t.pendingDeleteOperations = 0 if !stop { t.backend.readTx.tx = t.backend.begin(false) } } func (t *batchTxBuffered) UnsafePut(bucket Bucket, key []byte, value []byte) { t.batchTx.UnsafePut(bucket, key, value) t.buf.put(bucket, key, value) } func (t *batchTxBuffered) UnsafeSeqPut(bucket Bucket, key []byte, value []byte) { t.batchTx.UnsafeSeqPut(bucket, key, value) t.buf.putSeq(bucket, key, value) } func (t *batchTxBuffered) UnsafeDelete(bucketType Bucket, key []byte) { t.batchTx.UnsafeDelete(bucketType, key) t.pendingDeleteOperations++ } func (t *batchTxBuffered) UnsafeDeleteBucket(bucket Bucket) { t.batchTx.UnsafeDeleteBucket(bucket) t.pendingDeleteOperations++ } ================================================ FILE: server/storage/backend/batch_tx_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend_test import ( "fmt" "math/rand" "reflect" "testing" "time" "github.com/google/go-cmp/cmp" bolt "go.etcd.io/bbolt" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/schema" ) func TestBatchTxPut(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() // create bucket tx.UnsafeCreateBucket(schema.Test) // put v := []byte("bar") tx.UnsafePut(schema.Test, []byte("foo"), v) tx.Unlock() // check put result before and after tx is committed for k := 0; k < 2; k++ { tx.Lock() _, gv := tx.UnsafeRange(schema.Test, []byte("foo"), nil, 0) tx.Unlock() if !reflect.DeepEqual(gv[0], v) { t.Errorf("v = %s, want %s", gv[0], v) } tx.Commit() } } func TestBatchTxRange(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() defer tx.Unlock() tx.UnsafeCreateBucket(schema.Test) // put keys allKeys := [][]byte{[]byte("foo"), []byte("foo1"), []byte("foo2")} allVals := [][]byte{[]byte("bar"), []byte("bar1"), []byte("bar2")} for i := range allKeys { tx.UnsafePut(schema.Test, allKeys[i], allVals[i]) } tests := []struct { key []byte endKey []byte limit int64 wkeys [][]byte wvals [][]byte }{ // single key { []byte("foo"), nil, 0, allKeys[:1], allVals[:1], }, // single key, bad { []byte("doo"), nil, 0, nil, nil, }, // key range { []byte("foo"), []byte("foo1"), 0, allKeys[:1], allVals[:1], }, // key range, get all keys { []byte("foo"), []byte("foo3"), 0, allKeys, allVals, }, // key range, bad { []byte("goo"), []byte("goo3"), 0, nil, nil, }, // key range with effective limit { []byte("foo"), []byte("foo3"), 1, allKeys[:1], allVals[:1], }, // key range with limit { []byte("foo"), []byte("foo3"), 4, allKeys, allVals, }, } for i, tt := range tests { keys, vals := tx.UnsafeRange(schema.Test, tt.key, tt.endKey, tt.limit) if !reflect.DeepEqual(keys, tt.wkeys) { t.Errorf("#%d: keys = %+v, want %+v", i, keys, tt.wkeys) } if !reflect.DeepEqual(vals, tt.wvals) { t.Errorf("#%d: vals = %+v, want %+v", i, vals, tt.wvals) } } } func TestBatchTxDelete(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar")) tx.UnsafeDelete(schema.Test, []byte("foo")) tx.Unlock() // check put result before and after tx is committed for k := 0; k < 2; k++ { tx.Lock() ks, _ := tx.UnsafeRange(schema.Test, []byte("foo"), nil, 0) tx.Unlock() if len(ks) != 0 { t.Errorf("keys on foo = %v, want nil", ks) } tx.Commit() } } func TestBatchTxCommit(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar")) tx.Unlock() tx.Commit() // check whether put happens via db view backend.DbFromBackendForTest(b).View(func(tx *bolt.Tx) error { bucket := tx.Bucket(schema.Test.Name()) if bucket == nil { t.Errorf("bucket test does not exit") return nil } v := bucket.Get([]byte("foo")) if v == nil { t.Errorf("foo key failed to written in backend") } return nil }) } func TestBatchTxBatchLimitCommit(t *testing.T) { // start backend with batch limit 1 so one write can // trigger a commit b, _ := betesting.NewTmpBackend(t, time.Hour, 1) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar")) tx.Unlock() // batch limit commit should have been triggered // check whether put happens via db view backend.DbFromBackendForTest(b).View(func(tx *bolt.Tx) error { bucket := tx.Bucket(schema.Test.Name()) if bucket == nil { t.Errorf("bucket test does not exit") return nil } v := bucket.Get([]byte("foo")) if v == nil { t.Errorf("foo key failed to written in backend") } return nil }) } func TestRangeAfterDeleteBucketMatch(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar")) tx.Unlock() tx.Commit() checkForEach(t, b.BatchTx(), b.ReadTx(), [][]byte{[]byte("foo")}, [][]byte{[]byte("bar")}) tx.Lock() tx.UnsafeDeleteBucket(schema.Test) tx.Unlock() checkForEach(t, b.BatchTx(), b.ReadTx(), nil, nil) } func TestRangeAfterDeleteMatch(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar")) tx.Unlock() tx.Commit() checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), schema.Test, []byte("foo"), nil, 0) checkForEach(t, b.BatchTx(), b.ReadTx(), [][]byte{[]byte("foo")}, [][]byte{[]byte("bar")}) tx.Lock() tx.UnsafeDelete(schema.Test, []byte("foo")) tx.Unlock() checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), schema.Test, []byte("foo"), nil, 0) checkForEach(t, b.BatchTx(), b.ReadTx(), nil, nil) } func TestRangeAfterUnorderedKeyWriteMatch(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafePut(schema.Test, []byte("foo5"), []byte("bar5")) tx.UnsafePut(schema.Test, []byte("foo2"), []byte("bar2")) tx.UnsafePut(schema.Test, []byte("foo1"), []byte("bar1")) tx.UnsafePut(schema.Test, []byte("foo3"), []byte("bar3")) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar")) tx.UnsafePut(schema.Test, []byte("foo4"), []byte("bar4")) tx.Unlock() checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), schema.Test, []byte("foo"), nil, 1) } func TestRangeAfterAlternatingBucketWriteMatch(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Key) tx.UnsafeCreateBucket(schema.Test) tx.UnsafeSeqPut(schema.Key, []byte("key1"), []byte("val1")) tx.Unlock() tx.Lock() tx.UnsafeSeqPut(schema.Key, []byte("key2"), []byte("val2")) tx.Unlock() tx.Commit() // only in the 2nd commit the schema.Key key is removed from the readBuffer.buckets. // This makes sure to test the case when an empty writeBuffer.bucket // is used to replace the read buffer bucket. tx.Commit() tx.Lock() tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar")) tx.Unlock() checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), schema.Key, []byte("key"), []byte("key5"), 100) checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), schema.Test, []byte("foo"), []byte("foo3"), 1) } func TestRangeAfterOverwriteMatch(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar2")) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar0")) tx.UnsafePut(schema.Test, []byte("foo1"), []byte("bar10")) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar1")) tx.UnsafePut(schema.Test, []byte("foo1"), []byte("bar11")) tx.Unlock() checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), schema.Test, []byte("foo"), []byte("foo3"), 1) checkForEach(t, b.BatchTx(), b.ReadTx(), [][]byte{[]byte("foo"), []byte("foo1")}, [][]byte{[]byte("bar1"), []byte("bar11")}) } func TestRangeAfterOverwriteAndDeleteMatch(t *testing.T) { b, _ := betesting.NewTmpBackend(t, time.Hour, 10000) defer betesting.Close(t, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar2")) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar0")) tx.UnsafePut(schema.Test, []byte("foo1"), []byte("bar10")) tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar1")) tx.UnsafePut(schema.Test, []byte("foo1"), []byte("bar11")) tx.Unlock() checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), schema.Test, []byte("foo"), nil, 0) checkForEach(t, b.BatchTx(), b.ReadTx(), [][]byte{[]byte("foo"), []byte("foo1")}, [][]byte{[]byte("bar1"), []byte("bar11")}) tx.Lock() tx.UnsafePut(schema.Test, []byte("foo"), []byte("bar3")) tx.UnsafeDelete(schema.Test, []byte("foo1")) tx.Unlock() checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), schema.Test, []byte("foo"), nil, 0) checkRangeResponseMatch(t, b.BatchTx(), b.ReadTx(), schema.Test, []byte("foo1"), nil, 0) checkForEach(t, b.BatchTx(), b.ReadTx(), [][]byte{[]byte("foo")}, [][]byte{[]byte("bar3")}) } func checkRangeResponseMatch(t *testing.T, tx backend.BatchTx, rtx backend.ReadTx, bucket backend.Bucket, key, endKey []byte, limit int64) { tx.Lock() ks1, vs1 := tx.UnsafeRange(bucket, key, endKey, limit) tx.Unlock() rtx.RLock() ks2, vs2 := rtx.UnsafeRange(bucket, key, endKey, limit) rtx.RUnlock() if diff := cmp.Diff(ks1, ks2); diff != "" { t.Errorf("keys on read and batch transaction doesn't match, diff: %s", diff) } if diff := cmp.Diff(vs1, vs2); diff != "" { t.Errorf("values on read and batch transaction doesn't match, diff: %s", diff) } } func checkForEach(t *testing.T, tx backend.BatchTx, rtx backend.ReadTx, expectedKeys, expectedValues [][]byte) { tx.Lock() checkUnsafeForEach(t, tx, expectedKeys, expectedValues) tx.Unlock() rtx.RLock() checkUnsafeForEach(t, rtx, expectedKeys, expectedValues) rtx.RUnlock() } func checkUnsafeForEach(t *testing.T, tx backend.UnsafeReader, expectedKeys, expectedValues [][]byte) { var ks, vs [][]byte tx.UnsafeForEach(schema.Test, func(k, v []byte) error { ks = append(ks, k) vs = append(vs, v) return nil }) if diff := cmp.Diff(ks, expectedKeys); diff != "" { t.Errorf("keys on transaction doesn't match expected, diff: %s", diff) } if diff := cmp.Diff(vs, expectedValues); diff != "" { t.Errorf("values on transaction doesn't match expected, diff: %s", diff) } } // runWriteback is used test the txWriteBuffer.writeback function, which is called inside tx.Unlock(). // The parameters are chosen based on defaultBatchLimit = 10000 func runWriteback(tb testing.TB, kss, vss [][]string, isSeq bool) { b, _ := betesting.NewTmpBackend(tb, time.Hour, 10000) defer betesting.Close(tb, b) tx := b.BatchTx() tx.Lock() tx.UnsafeCreateBucket(schema.Test) tx.UnsafeCreateBucket(schema.Key) tx.Unlock() for i, ks := range kss { vs := vss[i] tx.Lock() for j := 0; j < len(ks); j++ { if isSeq { tx.UnsafeSeqPut(schema.Key, []byte(ks[j]), []byte(vs[j])) } else { tx.UnsafePut(schema.Test, []byte(ks[j]), []byte(vs[j])) } } tx.Unlock() } } func BenchmarkWritebackSeqBatches1BatchSize10000(b *testing.B) { benchmarkWriteback(b, 1, 10000, true) } func BenchmarkWritebackSeqBatches10BatchSize1000(b *testing.B) { benchmarkWriteback(b, 10, 1000, true) } func BenchmarkWritebackSeqBatches100BatchSize100(b *testing.B) { benchmarkWriteback(b, 100, 100, true) } func BenchmarkWritebackSeqBatches1000BatchSize10(b *testing.B) { benchmarkWriteback(b, 1000, 10, true) } func BenchmarkWritebackNonSeqBatches1000BatchSize1(b *testing.B) { // for non sequential writes, the batch size is usually small, 1 or the order of cluster size. benchmarkWriteback(b, 1000, 1, false) } func BenchmarkWritebackNonSeqBatches10000BatchSize1(b *testing.B) { benchmarkWriteback(b, 10000, 1, false) } func BenchmarkWritebackNonSeqBatches100BatchSize10(b *testing.B) { benchmarkWriteback(b, 100, 10, false) } func BenchmarkWritebackNonSeqBatches1000BatchSize10(b *testing.B) { benchmarkWriteback(b, 1000, 10, false) } func benchmarkWriteback(b *testing.B, batches, batchSize int, isSeq bool) { // kss and vss are key and value arrays to write with size batches*batchSize var kss, vss [][]string for i := 0; i < batches; i++ { var ks, vs []string for j := i * batchSize; j < (i+1)*batchSize; j++ { k := fmt.Sprintf("key%d", j) v := fmt.Sprintf("val%d", j) ks = append(ks, k) vs = append(vs, v) } if !isSeq { // make sure each batch is shuffled differently but the same for different test runs. shuffleList(ks, i*batchSize) } kss = append(kss, ks) vss = append(vss, vs) } b.ResetTimer() for n := 1; n < b.N; n++ { runWriteback(b, kss, vss, isSeq) } } func shuffleList(l []string, seed int) { r := rand.New(rand.NewSource(int64(seed))) for i := 0; i < len(l); i++ { j := r.Intn(i + 1) l[i], l[j] = l[j], l[i] } } ================================================ FILE: server/storage/backend/config_default.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !linux && !windows package backend import bolt "go.etcd.io/bbolt" var boltOpenOptions *bolt.Options func (bcfg *BackendConfig) mmapSize() int { return int(bcfg.MmapSize) } ================================================ FILE: server/storage/backend/config_linux.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend import ( "syscall" bolt "go.etcd.io/bbolt" ) // syscall.MAP_POPULATE on linux 2.6.23+ does sequential read-ahead // which can speed up entire-database read with boltdb. We want to // enable MAP_POPULATE for faster key-value store recovery in storage // package. If your kernel version is lower than 2.6.23 // (https://github.com/torvalds/linux/releases/tag/v2.6.23), mmap might // silently ignore this flag. Please update your kernel to prevent this. var boltOpenOptions = &bolt.Options{ MmapFlags: syscall.MAP_POPULATE, NoFreelistSync: true, } func (bcfg *BackendConfig) mmapSize() int { return int(bcfg.MmapSize) } ================================================ FILE: server/storage/backend/config_windows.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows package backend import bolt "go.etcd.io/bbolt" var boltOpenOptions *bolt.Options = nil // setting mmap size != 0 on windows will allocate the entire // mmap size for the file, instead of growing it. So, force 0. func (bcfg *BackendConfig) mmapSize() int { return 0 } ================================================ FILE: server/storage/backend/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package backend defines a standard interface for etcd's backend MVCC storage. package backend ================================================ FILE: server/storage/backend/export_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend import bolt "go.etcd.io/bbolt" func DbFromBackendForTest(b Backend) *bolt.DB { return b.(*backend).db } func DefragLimitForTest() int { return defragLimit } func CommitsForTest(b Backend) int64 { return b.(*backend).Commits() } ================================================ FILE: server/storage/backend/hooks.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend type HookFunc func(tx UnsafeReadWriter) // Hooks allow to add additional logic executed during transaction lifetime. type Hooks interface { // OnPreCommitUnsafe is executed before Commit of transactions. // The given transaction is already locked. OnPreCommitUnsafe(tx UnsafeReadWriter) } type hooks struct { onPreCommitUnsafe HookFunc } func (h hooks) OnPreCommitUnsafe(tx UnsafeReadWriter) { h.onPreCommitUnsafe(tx) } func NewHooks(onPreCommitUnsafe HookFunc) Hooks { return hooks{onPreCommitUnsafe: onPreCommitUnsafe} } ================================================ FILE: server/storage/backend/hooks_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend_test import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/schema" ) var ( bucket = schema.Test key = []byte("key") ) func TestBackendPreCommitHook(t *testing.T) { be := newTestHooksBackend(t, backend.DefaultBackendConfig(zaptest.NewLogger(t))) tx := be.BatchTx() prepareBuckenAndKey(tx) tx.Commit() // Empty commit. tx.Commit() assert.Equalf(t, ">cc", getCommitsKey(t, be), "expected 2 explicit commits") tx.Commit() assert.Equalf(t, ">ccc", getCommitsKey(t, be), "expected 3 explicit commits") } func TestBackendAutoCommitLimitHook(t *testing.T) { cfg := backend.DefaultBackendConfig(zaptest.NewLogger(t)) cfg.BatchLimit = 3 be := newTestHooksBackend(t, cfg) tx := be.BatchTx() prepareBuckenAndKey(tx) // writes 2 entries. for i := 3; i <= 9; i++ { write(tx, []byte("i"), []byte{byte(i)}) } assert.Equal(t, ">ccc", getCommitsKey(t, be)) } func write(tx backend.BatchTx, k, v []byte) { tx.Lock() defer tx.Unlock() tx.UnsafePut(bucket, k, v) } func TestBackendAutoCommitBatchIntervalHook(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), time.Minute) defer cancel() cfg := backend.DefaultBackendConfig(zaptest.NewLogger(t)) cfg.BatchInterval = 10 * time.Millisecond be := newTestHooksBackend(t, cfg) tx := be.BatchTx() prepareBuckenAndKey(tx) // Edits trigger an auto-commit waitUntil(ctx, t, func() bool { return getCommitsKey(t, be) == ">c" }) time.Sleep(time.Second) // No additional auto-commits, as there were no more edits assert.Equal(t, ">c", getCommitsKey(t, be)) write(tx, []byte("foo"), []byte("bar1")) waitUntil(ctx, t, func() bool { return getCommitsKey(t, be) == ">cc" }) write(tx, []byte("foo"), []byte("bar1")) waitUntil(ctx, t, func() bool { return getCommitsKey(t, be) == ">ccc" }) } func waitUntil(ctx context.Context, tb testing.TB, f func() bool) { for !f() { select { case <-ctx.Done(): tb.Fatalf("Context cancelled/timedout without condition met: %v", ctx.Err()) default: } time.Sleep(10 * time.Millisecond) } } func prepareBuckenAndKey(tx backend.BatchTx) { tx.Lock() defer tx.Unlock() tx.UnsafeCreateBucket(bucket) tx.UnsafePut(bucket, key, []byte(">")) } func newTestHooksBackend(tb testing.TB, baseConfig backend.BackendConfig) backend.Backend { cfg := baseConfig cfg.Hooks = backend.NewHooks(func(tx backend.UnsafeReadWriter) { k, v := tx.UnsafeRange(bucket, key, nil, 1) tb.Logf("OnPreCommit executed: %v %v", string(k[0]), string(v[0])) assert.Len(tb, k, 1) assert.Len(tb, v, 1) tx.UnsafePut(bucket, key, append(v[0], byte('c'))) }) be, _ := betesting.NewTmpBackendFromCfg(tb, cfg) tb.Cleanup(func() { betesting.Close(tb, be) }) return be } func getCommitsKey(tb testing.TB, be backend.Backend) string { rtx := be.BatchTx() rtx.Lock() defer rtx.Unlock() _, v := rtx.UnsafeRange(bucket, key, nil, 1) assert.Len(tb, v, 1) return string(v[0]) } ================================================ FILE: server/storage/backend/metrics.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend import "github.com/prometheus/client_golang/prometheus" var ( commitSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "disk", Name: "backend_commit_duration_seconds", Help: "The latency distributions of commit called by backend.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) rebalanceSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "disk", Name: "backend_commit_rebalance_duration_seconds", Help: "The latency distributions of commit.rebalance called by bboltdb backend.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) spillSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "disk", Name: "backend_commit_spill_duration_seconds", Help: "The latency distributions of commit.spill called by bboltdb backend.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) writeSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "disk", Name: "backend_commit_write_duration_seconds", Help: "The latency distributions of commit.write called by bboltdb backend.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) defragSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "disk", Name: "backend_defrag_duration_seconds", Help: "The latency distribution of backend defragmentation.", // 100 MB usually takes 1 sec, so start with 10 MB of 100 ms // lowest bucket start of upper bound 0.1 sec (100 ms) with factor 2 // highest bucket start of 0.1 sec * 2^12 == 409.6 sec Buckets: prometheus.ExponentialBuckets(.1, 2, 13), }) snapshotTransferSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "disk", Name: "backend_snapshot_duration_seconds", Help: "The latency distribution of backend snapshots.", // lowest bucket start of upper bound 0.01 sec (10 ms) with factor 2 // highest bucket start of 0.01 sec * 2^16 == 655.36 sec Buckets: prometheus.ExponentialBuckets(.01, 2, 17), }) isDefragActive = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "disk", Name: "defrag_inflight", Help: "Whether or not defrag is active on the member. 1 means active, 0 means not.", }) ) func init() { prometheus.MustRegister(commitSec) prometheus.MustRegister(rebalanceSec) prometheus.MustRegister(spillSec) prometheus.MustRegister(writeSec) prometheus.MustRegister(defragSec) prometheus.MustRegister(snapshotTransferSec) prometheus.MustRegister(isDefragActive) } ================================================ FILE: server/storage/backend/read_tx.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend import ( "math" "sync" bolt "go.etcd.io/bbolt" ) // IsSafeRangeBucket is a hack to avoid inadvertently reading duplicate keys; // overwrites on a bucket should only fetch with limit=1, but IsSafeRangeBucket // is known to never overwrite any key so range is safe. type ReadTx interface { RLock() RUnlock() UnsafeReader } type UnsafeReader interface { UnsafeRange(bucket Bucket, key, endKey []byte, limit int64) (keys [][]byte, vals [][]byte) UnsafeForEach(bucket Bucket, visitor func(k, v []byte) error) error } // Base type for readTx and concurrentReadTx to eliminate duplicate functions between these type baseReadTx struct { // mu protects accesses to the txReadBuffer mu sync.RWMutex buf txReadBuffer // TODO: group and encapsulate {txMu, tx, buckets, txWg}, as they share the same lifecycle. // txMu protects accesses to buckets and tx on Range requests. txMu *sync.RWMutex tx *bolt.Tx buckets map[BucketID]*bolt.Bucket // txWg protects tx from being rolled back at the end of a batch interval until all reads using this tx are done. txWg *sync.WaitGroup } func (baseReadTx *baseReadTx) UnsafeForEach(bucket Bucket, visitor func(k, v []byte) error) error { dups := make(map[string]struct{}) getDups := func(k, v []byte) error { dups[string(k)] = struct{}{} return nil } visitNoDup := func(k, v []byte) error { if _, ok := dups[string(k)]; ok { return nil } return visitor(k, v) } if err := baseReadTx.buf.ForEach(bucket, getDups); err != nil { return err } baseReadTx.txMu.Lock() err := unsafeForEach(baseReadTx.tx, bucket, visitNoDup) baseReadTx.txMu.Unlock() if err != nil { return err } return baseReadTx.buf.ForEach(bucket, visitor) } func (baseReadTx *baseReadTx) UnsafeRange(bucketType Bucket, key, endKey []byte, limit int64) ([][]byte, [][]byte) { if endKey == nil { // forbid duplicates for single keys limit = 1 } if limit <= 0 { limit = math.MaxInt64 } if limit > 1 && !bucketType.IsSafeRangeBucket() { panic("do not use unsafeRange on non-keys bucket") } keys, vals := baseReadTx.buf.Range(bucketType, key, endKey, limit) if int64(len(keys)) == limit { return keys, vals } // find/cache bucket bn := bucketType.ID() baseReadTx.txMu.RLock() bucket, ok := baseReadTx.buckets[bn] baseReadTx.txMu.RUnlock() lockHeld := false if !ok { baseReadTx.txMu.Lock() lockHeld = true bucket = baseReadTx.tx.Bucket(bucketType.Name()) baseReadTx.buckets[bn] = bucket } // ignore missing bucket since may have been created in this batch if bucket == nil { if lockHeld { baseReadTx.txMu.Unlock() } return keys, vals } if !lockHeld { baseReadTx.txMu.Lock() } c := bucket.Cursor() baseReadTx.txMu.Unlock() k2, v2 := unsafeRange(c, key, endKey, limit-int64(len(keys))) return append(k2, keys...), append(v2, vals...) } type readTx struct { baseReadTx } func (rt *readTx) Lock() { rt.mu.Lock() } func (rt *readTx) Unlock() { rt.mu.Unlock() } func (rt *readTx) RLock() { rt.mu.RLock() } func (rt *readTx) RUnlock() { rt.mu.RUnlock() } func (rt *readTx) reset() { rt.buf.reset() rt.buckets = make(map[BucketID]*bolt.Bucket) rt.tx = nil rt.txWg = new(sync.WaitGroup) } type concurrentReadTx struct { baseReadTx } func (rt *concurrentReadTx) Lock() {} func (rt *concurrentReadTx) Unlock() {} // RLock is no-op. concurrentReadTx does not need to be locked after it is created. func (rt *concurrentReadTx) RLock() {} // RUnlock signals the end of concurrentReadTx. func (rt *concurrentReadTx) RUnlock() { rt.txWg.Done() } ================================================ FILE: server/storage/backend/testing/betesting.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package betesting import ( "os" "path/filepath" "testing" "time" "github.com/stretchr/testify/assert" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/server/v3/storage/backend" ) func NewTmpBackendFromCfg(tb testing.TB, bcfg backend.BackendConfig) (backend.Backend, string) { dir, err := os.MkdirTemp(tb.TempDir(), "etcd_backend_test") if err != nil { panic(err) } tmpPath := filepath.Join(dir, "database") bcfg.Path = tmpPath bcfg.Logger = zaptest.NewLogger(tb) return backend.New(bcfg), tmpPath } // NewTmpBackend creates a backend implementation for testing. func NewTmpBackend(tb testing.TB, batchInterval time.Duration, batchLimit int) (backend.Backend, string) { bcfg := backend.DefaultBackendConfig(zaptest.NewLogger(tb)) bcfg.BatchInterval, bcfg.BatchLimit = batchInterval, batchLimit return NewTmpBackendFromCfg(tb, bcfg) } func NewDefaultTmpBackend(tb testing.TB) (backend.Backend, string) { return NewTmpBackendFromCfg(tb, backend.DefaultBackendConfig(zaptest.NewLogger(tb))) } func Close(tb testing.TB, b backend.Backend) { assert.NoError(tb, b.Close()) } ================================================ FILE: server/storage/backend/tx_buffer.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend import ( "bytes" "encoding/hex" "sort" "go.etcd.io/etcd/client/pkg/v3/verify" ) const bucketBufferInitialSize = 512 // txBuffer handles functionality shared between txWriteBuffer and txReadBuffer. type txBuffer struct { buckets map[BucketID]*bucketBuffer } func (txb *txBuffer) reset() { for k, v := range txb.buckets { if v.used == 0 { // demote delete(txb.buckets, k) } v.used = 0 } } // txWriteBuffer buffers writes of pending updates that have not yet committed. type txWriteBuffer struct { txBuffer // Map from bucket ID into information whether this bucket is edited // sequentially (i.e. keys are growing monotonically). bucket2seq map[BucketID]bool } func (txw *txWriteBuffer) put(bucket Bucket, k, v []byte) { txw.bucket2seq[bucket.ID()] = false txw.putInternal(bucket, k, v) } func (txw *txWriteBuffer) putSeq(bucket Bucket, k, v []byte) { // putSeq is only be called for the data in the Key bucket. The keys // in the Key bucket should be monotonically increasing revisions. verify.Verify("Broke the rule of monotonically increasing", func() (bool, map[string]any) { b, ok := txw.buckets[bucket.ID()] if !ok || b.used == 0 { return true, nil } existingMaxKey := b.buf[b.used-1].key if bytes.Compare(k, existingMaxKey) <= 0 { return false, map[string]any{ "existingMaxKey": hex.EncodeToString(existingMaxKey), "currentKey": hex.EncodeToString(k), } } return true, nil }) txw.putInternal(bucket, k, v) } func (txw *txWriteBuffer) putInternal(bucket Bucket, k, v []byte) { b, ok := txw.buckets[bucket.ID()] if !ok { b = newBucketBuffer() txw.buckets[bucket.ID()] = b } b.add(k, v) } func (txw *txWriteBuffer) reset() { txw.txBuffer.reset() for k := range txw.bucket2seq { v, ok := txw.buckets[k] if !ok { delete(txw.bucket2seq, k) } else if v.used == 0 { txw.bucket2seq[k] = true } } } func (txw *txWriteBuffer) writeback(txr *txReadBuffer) { for k, wb := range txw.buckets { rb, ok := txr.buckets[k] if !ok { delete(txw.buckets, k) if seq, ok := txw.bucket2seq[k]; ok && !seq { wb.dedupe() } txr.buckets[k] = wb continue } if seq, ok := txw.bucket2seq[k]; ok && !seq && wb.used > 1 { // assume no duplicate keys sort.Sort(wb) } rb.merge(wb) } txw.reset() // increase the buffer version txr.bufVersion++ } // txReadBuffer accesses buffered updates. type txReadBuffer struct { txBuffer // bufVersion is used to check if the buffer is modified recently bufVersion uint64 } func (txr *txReadBuffer) Range(bucket Bucket, key, endKey []byte, limit int64) ([][]byte, [][]byte) { if b := txr.buckets[bucket.ID()]; b != nil { return b.Range(key, endKey, limit) } return nil, nil } func (txr *txReadBuffer) ForEach(bucket Bucket, visitor func(k, v []byte) error) error { if b := txr.buckets[bucket.ID()]; b != nil { return b.ForEach(visitor) } return nil } // unsafeCopy returns a copy of txReadBuffer, caller should acquire backend.readTx.RLock() func (txr *txReadBuffer) unsafeCopy() txReadBuffer { txrCopy := txReadBuffer{ txBuffer: txBuffer{ buckets: make(map[BucketID]*bucketBuffer, len(txr.txBuffer.buckets)), }, bufVersion: 0, } for bucketName, bucket := range txr.txBuffer.buckets { txrCopy.txBuffer.buckets[bucketName] = bucket.CopyUsed() } return txrCopy } type kv struct { key []byte val []byte } // bucketBuffer buffers key-value pairs that are pending commit. type bucketBuffer struct { buf []kv // used tracks number of elements in use so buf can be reused without reallocation. used int } func newBucketBuffer() *bucketBuffer { return &bucketBuffer{buf: make([]kv, bucketBufferInitialSize), used: 0} } func (bb *bucketBuffer) Range(key, endKey []byte, limit int64) (keys [][]byte, vals [][]byte) { f := func(i int) bool { return bytes.Compare(bb.buf[i].key, key) >= 0 } idx := sort.Search(bb.used, f) if idx < 0 || idx >= bb.used { return nil, nil } if len(endKey) == 0 { if bytes.Equal(key, bb.buf[idx].key) { keys = append(keys, bb.buf[idx].key) vals = append(vals, bb.buf[idx].val) } return keys, vals } if bytes.Compare(endKey, bb.buf[idx].key) <= 0 { return nil, nil } for i := idx; i < bb.used && int64(len(keys)) < limit; i++ { if bytes.Compare(endKey, bb.buf[i].key) <= 0 { break } keys = append(keys, bb.buf[i].key) vals = append(vals, bb.buf[i].val) } return keys, vals } func (bb *bucketBuffer) ForEach(visitor func(k, v []byte) error) error { for i := 0; i < bb.used; i++ { if err := visitor(bb.buf[i].key, bb.buf[i].val); err != nil { return err } } return nil } func (bb *bucketBuffer) add(k, v []byte) { bb.buf[bb.used].key, bb.buf[bb.used].val = k, v bb.used++ if bb.used == len(bb.buf) { buf := make([]kv, (3*len(bb.buf))/2) copy(buf, bb.buf) bb.buf = buf } } // merge merges data from bbsrc into bb. func (bb *bucketBuffer) merge(bbsrc *bucketBuffer) { for i := 0; i < bbsrc.used; i++ { bb.add(bbsrc.buf[i].key, bbsrc.buf[i].val) } if bb.used == bbsrc.used { return } if bytes.Compare(bb.buf[(bb.used-bbsrc.used)-1].key, bbsrc.buf[0].key) < 0 { return } bb.dedupe() } // dedupe removes duplicates, using only newest update func (bb *bucketBuffer) dedupe() { if bb.used <= 1 { return } sort.Stable(bb) widx := 0 for ridx := 1; ridx < bb.used; ridx++ { if !bytes.Equal(bb.buf[ridx].key, bb.buf[widx].key) { widx++ } bb.buf[widx] = bb.buf[ridx] } bb.used = widx + 1 } func (bb *bucketBuffer) Len() int { return bb.used } func (bb *bucketBuffer) Less(i, j int) bool { return bytes.Compare(bb.buf[i].key, bb.buf[j].key) < 0 } func (bb *bucketBuffer) Swap(i, j int) { bb.buf[i], bb.buf[j] = bb.buf[j], bb.buf[i] } func (bb *bucketBuffer) CopyUsed() *bucketBuffer { verify.Assert(bb.used <= len(bb.buf), "used (%d) should never be bigger than the length of buf (%d)", bb.used, len(bb.buf)) bbCopy := bucketBuffer{ buf: make([]kv, bb.used), used: bb.used, } copy(bbCopy.buf, bb.buf[:bb.used]) return &bbCopy } ================================================ FILE: server/storage/backend/tx_buffer_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend import ( "fmt" "testing" "github.com/stretchr/testify/assert" ) func Test_bucketBuffer_CopyUsed_After_Add(t *testing.T) { bb := &bucketBuffer{buf: make([]kv, 10), used: 0} for i := 0; i < 20; i++ { k := fmt.Sprintf("key%d", i) v := fmt.Sprintf("val%d", i) bb.add([]byte(k), []byte(v)) bbCopy := bb.CopyUsed() assert.Equal(t, bb.used, bbCopy.used) assert.Len(t, bbCopy.buf, bbCopy.used) assert.GreaterOrEqual(t, len(bb.buf), len(bbCopy.buf)) } } func Test_bucketBuffer_CopyUsed(t *testing.T) { tests := []struct { name string bufLen int used int wantPanic bool wantUsed int wantBufLen int }{ { name: "used is 0", bufLen: 10, used: 0, wantPanic: false, wantUsed: 0, wantBufLen: 0, }, { name: "used is greater than 0 and less than len(buf)", bufLen: 10, used: 5, wantPanic: false, wantUsed: 5, wantBufLen: 5, }, { name: "used is equal to len(buf)", bufLen: 10, used: 10, wantPanic: false, wantUsed: 10, wantBufLen: 10, }, { name: "used is greater than len(buf)", bufLen: 10, used: 11, wantPanic: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { bb := &bucketBuffer{buf: make([]kv, tt.bufLen), used: tt.used} if tt.wantPanic { assert.Panicsf(t, func() { bb.CopyUsed() }, "expected panic when used (%d) and the length of buf (%d)", tt.used, tt.bufLen) } else { bbCopy := bb.CopyUsed() assert.Equal(t, tt.wantUsed, bbCopy.used) assert.Len(t, bbCopy.buf, tt.wantBufLen) } }) } } func TestDedupe(t *testing.T) { tests := []struct { name string keys, vals, expectedKeys, expectedVals []string }{ { name: "empty", keys: []string{}, vals: []string{}, expectedKeys: []string{}, expectedVals: []string{}, }, { name: "single kv", keys: []string{"key1"}, vals: []string{"val1"}, expectedKeys: []string{"key1"}, expectedVals: []string{"val1"}, }, { name: "duplicate key", keys: []string{"key1", "key1"}, vals: []string{"val1", "val2"}, expectedKeys: []string{"key1"}, expectedVals: []string{"val2"}, }, { name: "unordered keys", keys: []string{"key3", "key1", "key4", "key2", "key1", "key4"}, vals: []string{"val1", "val5", "val3", "val4", "val2", "val6"}, expectedKeys: []string{"key1", "key2", "key3", "key4"}, expectedVals: []string{"val2", "val4", "val1", "val6"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { bb := &bucketBuffer{buf: make([]kv, 10), used: 0} for i := 0; i < len(tt.keys); i++ { bb.add([]byte(tt.keys[i]), []byte(tt.vals[i])) } bb.dedupe() assert.Len(t, tt.expectedKeys, bb.used) for i := 0; i < bb.used; i++ { assert.Equal(t, bb.buf[i].key, []byte(tt.expectedKeys[i])) assert.Equal(t, bb.buf[i].val, []byte(tt.expectedVals[i])) } }) } } ================================================ FILE: server/storage/backend/verify.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend import ( "runtime/debug" "strings" "github.com/google/go-cmp/cmp" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/verify" ) const ( EnvVerifyValueLock verify.VerificationType = "lock" ) func ValidateCalledInsideApply(lg *zap.Logger) { if !verifyLockEnabled() { return } if !insideApply() { lg.Panic("Called outside of APPLY!", zap.Stack("stacktrace")) } } func ValidateCalledOutSideApply(lg *zap.Logger) { if !verifyLockEnabled() { return } if insideApply() { lg.Panic("Called inside of APPLY!", zap.Stack("stacktrace")) } } func ValidateCalledInsideUnittest(lg *zap.Logger) { if !verifyLockEnabled() { return } if !insideUnittest() { lg.Fatal("Lock called outside of unit test!", zap.Stack("stacktrace")) } } func verifyLockEnabled() bool { return verify.IsVerificationEnabled(EnvVerifyValueLock) } func insideApply() bool { stackTraceStr := string(debug.Stack()) return strings.Contains(stackTraceStr, ".applyEntries") } func insideUnittest() bool { stackTraceStr := string(debug.Stack()) return strings.Contains(stackTraceStr, "_test.go") && !strings.Contains(stackTraceStr, "tests/") } // VerifyBackendConsistency verifies data in ReadTx and BatchTx are consistent. func VerifyBackendConsistency(b Backend, lg *zap.Logger, skipSafeRangeBucket bool, bucket ...Bucket) { verify.Verify("bucket data mismatch", func() (bool, map[string]any) { if b == nil { return true, nil } if lg != nil { lg.Debug("verifyBackendConsistency", zap.Bool("skipSafeRangeBucket", skipSafeRangeBucket)) } b.BatchTx().LockOutsideApply() defer b.BatchTx().Unlock() b.ReadTx().RLock() defer b.ReadTx().RUnlock() for _, bkt := range bucket { if skipSafeRangeBucket && bkt.IsSafeRangeBucket() { continue } if ok, details := unsafeVerifyTxConsistency(b, bkt); !ok { return false, details } } return true, nil }) } func unsafeVerifyTxConsistency(b Backend, bucket Bucket) (bool, map[string]any) { dataFromWriteTxn := map[string]string{} b.BatchTx().UnsafeForEach(bucket, func(k, v []byte) error { dataFromWriteTxn[string(k)] = string(v) return nil }) dataFromReadTxn := map[string]string{} b.ReadTx().UnsafeForEach(bucket, func(k, v []byte) error { dataFromReadTxn[string(k)] = string(v) return nil }) if diff := cmp.Diff(dataFromWriteTxn, dataFromReadTxn); diff != "" { return false, map[string]any{ "bucket": bucket.String(), "write TXN": dataFromWriteTxn, "read TXN": dataFromReadTxn, "diff": diff, } } return true, nil } ================================================ FILE: server/storage/backend/verify_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package backend_test import ( "testing" "time" "go.etcd.io/etcd/client/pkg/v3/verify" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) func TestLockVerify(t *testing.T) { tcs := []struct { name string insideApply bool lock func(tx backend.BatchTx) txPostLockInsideApplyHook func() expectPanic bool }{ { name: "call lockInsideApply from inside apply", insideApply: true, lock: lockInsideApply, expectPanic: false, }, { name: "call lockInsideApply from outside apply (without txPostLockInsideApplyHook)", insideApply: false, lock: lockInsideApply, expectPanic: false, }, { name: "call lockInsideApply from outside apply (with txPostLockInsideApplyHook)", insideApply: false, lock: lockInsideApply, txPostLockInsideApplyHook: func() {}, expectPanic: true, }, { name: "call lockOutsideApply from outside apply", insideApply: false, lock: lockOutsideApply, expectPanic: false, }, { name: "call lockOutsideApply from inside apply", insideApply: true, lock: lockOutsideApply, expectPanic: true, }, { name: "call Lock from unit test", insideApply: false, lock: lockFromUT, expectPanic: false, }, } revertVerifyFunc := verify.EnableVerifications(backend.EnvVerifyValueLock) defer revertVerifyFunc() for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { be, _ := betesting.NewTmpBackend(t, time.Hour, 10000) be.SetTxPostLockInsideApplyHook(tc.txPostLockInsideApplyHook) hasPaniced := handlePanic(func() { if tc.insideApply { applyEntries(be, tc.lock) } else { tc.lock(be.BatchTx()) } }) != nil if hasPaniced != tc.expectPanic { t.Errorf("%v != %v", hasPaniced, tc.expectPanic) } }) } } func handlePanic(f func()) (result any) { defer func() { result = recover() }() f() return result } func applyEntries(be backend.Backend, f func(tx backend.BatchTx)) { f(be.BatchTx()) } func lockInsideApply(tx backend.BatchTx) { tx.LockInsideApply() } func lockOutsideApply(tx backend.BatchTx) { tx.LockOutsideApply() } func lockFromUT(tx backend.BatchTx) { tx.Lock() } ================================================ FILE: server/storage/backend.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage import ( "fmt" "os" "time" "go.uber.org/zap" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/raft/v3/raftpb" ) func newBackend(cfg config.ServerConfig, hooks backend.Hooks) backend.Backend { bcfg := backend.DefaultBackendConfig(cfg.Logger) bcfg.Path = cfg.BackendPath() bcfg.UnsafeNoFsync = cfg.UnsafeNoFsync if cfg.BackendBatchLimit != 0 { bcfg.BatchLimit = cfg.BackendBatchLimit if cfg.Logger != nil { cfg.Logger.Info("setting backend batch limit", zap.Int("batch limit", cfg.BackendBatchLimit)) } } if cfg.BackendBatchInterval != 0 { bcfg.BatchInterval = cfg.BackendBatchInterval if cfg.Logger != nil { cfg.Logger.Info("setting backend batch interval", zap.Duration("batch interval", cfg.BackendBatchInterval)) } } bcfg.BackendFreelistType = cfg.BackendFreelistType bcfg.Logger = cfg.Logger if cfg.QuotaBackendBytes > 0 && cfg.QuotaBackendBytes != DefaultQuotaBytes { // permit 10% excess over quota for disarm bcfg.MmapSize = uint64(cfg.QuotaBackendBytes + cfg.QuotaBackendBytes/10) } bcfg.Mlock = cfg.MemoryMlock bcfg.Hooks = hooks return backend.New(bcfg) } // OpenSnapshotBackend renames a snapshot db to the current etcd db and opens it. func OpenSnapshotBackend(cfg config.ServerConfig, ss *snap.Snapshotter, snapshot raftpb.Snapshot, hooks *BackendHooks) (backend.Backend, error) { snapPath, err := ss.DBFilePath(snapshot.Metadata.Index) if err != nil { return nil, fmt.Errorf("failed to find database snapshot file (%w)", err) } if err := os.Rename(snapPath, cfg.BackendPath()); err != nil { return nil, fmt.Errorf("failed to rename database snapshot file (%w)", err) } return OpenBackend(cfg, hooks), nil } // OpenBackend returns a backend using the current etcd db. func OpenBackend(cfg config.ServerConfig, hooks backend.Hooks) backend.Backend { fn := cfg.BackendPath() now, beOpened := time.Now(), make(chan backend.Backend) go func() { beOpened <- newBackend(cfg, hooks) }() defer func() { cfg.Logger.Info("opened backend db", zap.String("path", fn), zap.Duration("took", time.Since(now))) }() select { case be := <-beOpened: return be case <-time.After(10 * time.Second): cfg.Logger.Info( "db file is flocked by another process, or taking too long", zap.String("path", fn), zap.Duration("took", time.Since(now)), ) } return <-beOpened } // RecoverSnapshotBackend recovers the DB from a snapshot in case etcd crashes // before updating the backend db after persisting raft snapshot to disk, // violating the invariant snapshot.Metadata.Index < db.consistentIndex. In this // case, replace the db with the snapshot db sent by the leader. func RecoverSnapshotBackend(cfg config.ServerConfig, oldbe backend.Backend, snapshot raftpb.Snapshot, beExist bool, hooks *BackendHooks) (backend.Backend, error) { consistentIndex := uint64(0) if beExist { consistentIndex, _ = schema.ReadConsistentIndex(oldbe.ReadTx()) } if snapshot.Metadata.Index <= consistentIndex { cfg.Logger.Info("Skipping snapshot backend", zap.Uint64("consistent-index", consistentIndex), zap.Uint64("snapshot-index", snapshot.Metadata.Index)) return oldbe, nil } cfg.Logger.Info("Recovering from snapshot backend", zap.Uint64("consistent-index", consistentIndex), zap.Uint64("snapshot-index", snapshot.Metadata.Index)) oldbe.Close() return OpenSnapshotBackend(cfg, snap.New(cfg.Logger, cfg.SnapDir()), snapshot, hooks) } ================================================ FILE: server/storage/datadir/datadir.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package datadir import "path/filepath" const ( memberDirSegment = "member" snapDirSegment = "snap" walDirSegment = "wal" backendFileSegment = "db" ) func ToBackendFileName(dataDir string) string { return filepath.Join(ToSnapDir(dataDir), backendFileSegment) } func ToSnapDir(dataDir string) string { return filepath.Join(ToMemberDir(dataDir), snapDirSegment) } // ToWalDir returns the directory path for the member's WAL. // // Deprecated: use ToWALDir instead. // //revive:disable-next-line:var-naming func ToWalDir(dataDir string) string { return ToWALDir(dataDir) } func ToWALDir(dataDir string) string { return filepath.Join(ToMemberDir(dataDir), walDirSegment) } func ToMemberDir(dataDir string) string { return filepath.Join(dataDir, memberDirSegment) } ================================================ FILE: server/storage/datadir/datadir_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package datadir_test import ( "testing" "github.com/stretchr/testify/assert" "go.etcd.io/etcd/server/v3/storage/datadir" ) func TestToBackendFileName(t *testing.T) { result := datadir.ToBackendFileName("/dir/data-dir") assert.Equal(t, "/dir/data-dir/member/snap/db", result) } func TestToMemberDir(t *testing.T) { result := datadir.ToMemberDir("/dir/data-dir") assert.Equal(t, "/dir/data-dir/member", result) } func TestToSnapDir(t *testing.T) { result := datadir.ToSnapDir("/dir/data-dir") assert.Equal(t, "/dir/data-dir/member/snap", result) } func TestToWALDir(t *testing.T) { result := datadir.ToWALDir("/dir/data-dir") assert.Equal(t, "/dir/data-dir/member/wal", result) } func TestToWALDirSlash(t *testing.T) { result := datadir.ToWALDir("/dir/data-dir/") assert.Equal(t, "/dir/data-dir/member/wal", result) } ================================================ FILE: server/storage/datadir/doc.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package datadir // datadir contains functions to navigate file-layout of etcd data-directory. ================================================ FILE: server/storage/hooks.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage import ( "sync" "go.uber.org/zap" "go.etcd.io/etcd/server/v3/etcdserver/cindex" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/raft/v3/raftpb" ) type BackendHooks struct { indexer cindex.ConsistentIndexer lg *zap.Logger // confState to Be written in the next submitted Backend transaction (if dirty) confState raftpb.ConfState // first write changes it to 'dirty'. false by default, so // not initialized `confState` is meaningless. confStateDirty bool confStateLock sync.Mutex } func NewBackendHooks(lg *zap.Logger, indexer cindex.ConsistentIndexer) *BackendHooks { return &BackendHooks{lg: lg, indexer: indexer} } func (bh *BackendHooks) OnPreCommitUnsafe(tx backend.UnsafeReadWriter) { bh.indexer.UnsafeSave(tx) bh.confStateLock.Lock() defer bh.confStateLock.Unlock() if bh.confStateDirty { schema.MustUnsafeSaveConfStateToBackend(bh.lg, tx, &bh.confState) // save bh.confState bh.confStateDirty = false } } func (bh *BackendHooks) SetConfState(confState *raftpb.ConfState) { bh.confStateLock.Lock() defer bh.confStateLock.Unlock() bh.confState = *confState bh.confStateDirty = true } ================================================ FILE: server/storage/metrics.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage import ( "github.com/prometheus/client_golang/prometheus" ) var quotaBackendBytes = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", Name: "quota_backend_bytes", Help: "Current backend storage quota size in bytes.", }) func init() { prometheus.MustRegister(quotaBackendBytes) } ================================================ FILE: server/storage/mvcc/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package mvcc defines etcd's stable MVCC storage. package mvcc ================================================ FILE: server/storage/mvcc/hash.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "hash" "hash/crc32" "sort" "sync" "go.uber.org/zap" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" ) const ( hashStorageMaxSize = 10 ) func unsafeHashByRev(tx backend.UnsafeReader, compactRevision, revision int64, keep map[Revision]struct{}) (KeyValueHash, error) { h := newKVHasher(compactRevision, revision, keep) err := tx.UnsafeForEach(schema.Key, func(k, v []byte) error { h.WriteKeyValue(k, v) return nil }) return h.Hash(), err } type kvHasher struct { hash hash.Hash32 compactRevision int64 revision int64 keep map[Revision]struct{} } func newKVHasher(compactRev, rev int64, keep map[Revision]struct{}) kvHasher { h := crc32.New(crc32.MakeTable(crc32.Castagnoli)) h.Write(schema.Key.Name()) return kvHasher{ hash: h, compactRevision: compactRev, revision: rev, keep: keep, } } func (h *kvHasher) WriteKeyValue(k, v []byte) { kr := BytesToRev(k) upper := Revision{Main: h.revision + 1} if !upper.GreaterThan(kr) { return } isTombstone := BytesToBucketKey(k).tombstone lower := Revision{Main: h.compactRevision + 1} // skip revisions that are scheduled for deletion // due to compacting; don't skip if there isn't one. if lower.GreaterThan(kr) && len(h.keep) > 0 { if _, ok := h.keep[kr]; !ok { return } } // When performing compaction, if the compacted revision is a // tombstone, older versions (<= 3.5.15 or <= 3.4.33) will delete // the tombstone. But newer versions (> 3.5.15 or > 3.4.33) won't // delete it. So we should skip the tombstone in such cases when // computing the hash to ensure that both older and newer versions // can always generate the same hash values. if kr.Main == h.compactRevision && isTombstone { return } h.hash.Write(k) h.hash.Write(v) } func (h *kvHasher) Hash() KeyValueHash { return KeyValueHash{Hash: h.hash.Sum32(), CompactRevision: h.compactRevision, Revision: h.revision} } type KeyValueHash struct { Hash uint32 CompactRevision int64 Revision int64 } type HashStorage interface { // Hash computes the hash of the whole backend keyspace, // including key, lease, and other buckets in storage. // This is designed for testing ONLY! // Do not rely on this in production with ongoing transactions, // since Hash operation does not hold MVCC locks. // Use "HashByRev" method instead for "key" bucket consistency checks. Hash() (hash uint32, revision int64, err error) // HashByRev computes the hash of all MVCC revisions up to a given revision. HashByRev(rev int64) (hash KeyValueHash, currentRev int64, err error) // Store adds hash value in local cache, allowing it to be returned by HashByRev. Store(valueHash KeyValueHash) // Hashes returns list of up to `hashStorageMaxSize` newest previously stored hashes. Hashes() []KeyValueHash } type hashStorage struct { store *store hashMu sync.RWMutex hashes []KeyValueHash lg *zap.Logger } func NewHashStorage(lg *zap.Logger, s *store) HashStorage { return &hashStorage{ store: s, lg: lg, } } func (s *hashStorage) Hash() (hash uint32, revision int64, err error) { return s.store.hash() } func (s *hashStorage) HashByRev(rev int64) (KeyValueHash, int64, error) { s.hashMu.RLock() for _, h := range s.hashes { if rev == h.Revision { s.hashMu.RUnlock() s.store.revMu.RLock() currentRev := s.store.currentRev s.store.revMu.RUnlock() return h, currentRev, nil } } s.hashMu.RUnlock() return s.store.hashByRev(rev) } func (s *hashStorage) Store(hash KeyValueHash) { s.lg.Info("storing new hash", zap.Uint32("hash", hash.Hash), zap.Int64("revision", hash.Revision), zap.Int64("compact-revision", hash.CompactRevision), ) s.hashMu.Lock() defer s.hashMu.Unlock() s.hashes = append(s.hashes, hash) sort.Slice(s.hashes, func(i, j int) bool { return s.hashes[i].Revision < s.hashes[j].Revision }) if len(s.hashes) > hashStorageMaxSize { s.hashes = s.hashes[len(s.hashes)-hashStorageMaxSize:] } } func (s *hashStorage) Hashes() []KeyValueHash { s.hashMu.RLock() // Copy out hashes under lock just to be safe hashes := make([]KeyValueHash, 0, len(s.hashes)) hashes = append(hashes, s.hashes...) s.hashMu.RUnlock() return hashes } ================================================ FILE: server/storage/mvcc/hash_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "context" "fmt" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/mvcc/testutil" ) // TestHashByRevValue test HashByRevValue values to ensure we don't change the // output which would have catastrophic consequences. Expected output is just // hardcoded, so please regenerate it every time you change input parameters. func TestHashByRevValue(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) var totalRevisions int64 = 1210 assert.Less(t, int64(s.cfg.CompactionBatchLimit), totalRevisions) assert.Less(t, int64(testutil.CompactionCycle*10), totalRevisions) var rev int64 var got []KeyValueHash for ; rev < totalRevisions; rev += testutil.CompactionCycle { putKVs(s, rev, testutil.CompactionCycle) hash := testHashByRev(t, s, rev+testutil.CompactionCycle/2) got = append(got, hash) } putKVs(s, rev, totalRevisions) hash := testHashByRev(t, s, rev+totalRevisions/2) got = append(got, hash) assert.Equal(t, []KeyValueHash{ {4082599214, -1, 35}, {2279933401, 35, 106}, {3284231217, 106, 177}, {126286495, 177, 248}, {900108730, 248, 319}, {2475485232, 319, 390}, {1226296507, 390, 461}, {2503661030, 461, 532}, {4155130747, 532, 603}, {106915399, 603, 674}, {406914006, 674, 745}, {1882211381, 745, 816}, {806177088, 816, 887}, {664311366, 887, 958}, {1496914449, 958, 1029}, {2434525091, 1029, 1100}, {3988652253, 1100, 1171}, {1122462288, 1171, 1242}, {724436716, 1242, 1883}, }, got) } func TestHashByRevValueLastRevision(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) var totalRevisions int64 = 1210 assert.Less(t, int64(s.cfg.CompactionBatchLimit), totalRevisions) assert.Less(t, int64(testutil.CompactionCycle*10), totalRevisions) var rev int64 var got []KeyValueHash for ; rev < totalRevisions; rev += testutil.CompactionCycle { putKVs(s, rev, testutil.CompactionCycle) hash := testHashByRev(t, s, 0) got = append(got, hash) } putKVs(s, rev, totalRevisions) hash := testHashByRev(t, s, 0) got = append(got, hash) assert.Equal(t, []KeyValueHash{ {1913897190, -1, 73}, {224860069, 73, 145}, {1565167519, 145, 217}, {1566261620, 217, 289}, {2037173024, 289, 361}, {691659396, 361, 433}, {2713730748, 433, 505}, {3919322507, 505, 577}, {769967540, 577, 649}, {2909194793, 649, 721}, {1576921157, 721, 793}, {4067701532, 793, 865}, {2226384237, 865, 937}, {2923408134, 937, 1009}, {2680329256, 1009, 1081}, {1546717673, 1081, 1153}, {2713657846, 1153, 1225}, {1046575299, 1225, 1297}, {2017735779, 1297, 2508}, }, got) } func putKVs(s *store, rev, count int64) { for i := rev; i <= rev+count; i++ { s.Put([]byte(testutil.PickKey(i)), []byte(fmt.Sprint(i)), 0) } } func testHashByRev(t *testing.T, s *store, rev int64) KeyValueHash { if rev == 0 { rev = s.Rev() } hash, _, err := s.hashByRev(rev) require.NoErrorf(t, err, "error on rev %v", rev) _, err = s.Compact(traceutil.TODO(), rev) assert.NoErrorf(t, err, "error on compact %v", rev) return hash } // TestCompactionHash tests compaction hash // TODO: Change this to fuzz test func TestCompactionHash(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) testutil.TestCompactionHash(t.Context(), t, hashTestCase{s}, s.cfg.CompactionBatchLimit) } type hashTestCase struct { *store } func (tc hashTestCase) Put(ctx context.Context, key, value string) error { tc.store.Put([]byte(key), []byte(value), 0) return nil } func (tc hashTestCase) Delete(ctx context.Context, key string) error { tc.store.DeleteRange([]byte(key), nil) return nil } func (tc hashTestCase) HashByRev(ctx context.Context, rev int64) (testutil.KeyValueHash, error) { hash, _, err := tc.store.HashStorage().HashByRev(rev) return testutil.KeyValueHash{Hash: hash.Hash, CompactRevision: hash.CompactRevision, Revision: hash.Revision}, err } func (tc hashTestCase) Defrag(ctx context.Context) error { return tc.store.b.Defrag() } func (tc hashTestCase) Compact(ctx context.Context, rev int64) error { done, err := tc.store.Compact(traceutil.TODO(), rev) if err != nil { return err } select { case <-done: case <-ctx.Done(): return ctx.Err() } return nil } func TestHasherStore(t *testing.T) { lg := zaptest.NewLogger(t) store := newFakeStore(lg) s := NewHashStorage(lg, store) defer store.Close() var hashes []KeyValueHash for i := 0; i < hashStorageMaxSize; i++ { hash := KeyValueHash{Hash: uint32(i), Revision: int64(i) + 10, CompactRevision: int64(i) + 100} hashes = append(hashes, hash) s.Store(hash) } for _, want := range hashes { got, _, err := s.HashByRev(want.Revision) if err != nil { t.Fatal(err) } if want.Hash != got.Hash { t.Errorf("Expected stored hash to match, got: %d, expected: %d", want.Hash, got.Hash) } if want.Revision != got.Revision { t.Errorf("Expected stored revision to match, got: %d, expected: %d", want.Revision, got.Revision) } if want.CompactRevision != got.CompactRevision { t.Errorf("Expected stored compact revision to match, got: %d, expected: %d", want.CompactRevision, got.CompactRevision) } } } func TestHasherStoreFull(t *testing.T) { lg := zaptest.NewLogger(t) store := newFakeStore(lg) s := NewHashStorage(lg, store) defer store.Close() var minRevision int64 = 100 maxRevision := minRevision + hashStorageMaxSize for i := 0; i < hashStorageMaxSize; i++ { s.Store(KeyValueHash{Revision: int64(i) + minRevision}) } // Hash for old revision should be discarded as storage is already full s.Store(KeyValueHash{Revision: minRevision - 1}) hash, _, err := s.HashByRev(minRevision - 1) if err == nil { t.Errorf("Expected an error as old revision should be discarded, got: %v", hash) } // Hash for new revision should be stored even when storage is full s.Store(KeyValueHash{Revision: maxRevision + 1}) _, _, err = s.HashByRev(maxRevision + 1) if err != nil { t.Errorf("Didn't expect error for new revision, err: %v", err) } } ================================================ FILE: server/storage/mvcc/index.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "sync" "go.uber.org/zap" "k8s.io/utils/third_party/forked/golang/btree" ) type index interface { Get(key []byte, atRev int64) (rev, created Revision, ver int64, err error) Range(key, end []byte, atRev int64) ([][]byte, []Revision) Revisions(key, end []byte, atRev int64, limit int) ([]Revision, int) CountRevisions(key, end []byte, atRev int64) int Put(key []byte, rev Revision) Tombstone(key []byte, rev Revision) error Compact(rev int64) map[Revision]struct{} Keep(rev int64) map[Revision]struct{} Equal(b index) bool Insert(ki *keyIndex) KeyIndex(ki *keyIndex) *keyIndex } type treeIndex struct { sync.RWMutex tree *btree.BTree[*keyIndex] lg *zap.Logger } func newTreeIndex(lg *zap.Logger) index { return &treeIndex{ tree: btree.New(32, func(aki *keyIndex, bki *keyIndex) bool { return aki.Less(bki) }), lg: lg, } } func (ti *treeIndex) Put(key []byte, rev Revision) { keyi := &keyIndex{key: key} ti.Lock() defer ti.Unlock() okeyi, ok := ti.tree.Get(keyi) if !ok { keyi.put(ti.lg, rev.Main, rev.Sub) ti.tree.ReplaceOrInsert(keyi) return } okeyi.put(ti.lg, rev.Main, rev.Sub) } func (ti *treeIndex) Get(key []byte, atRev int64) (modified, created Revision, ver int64, err error) { ti.RLock() defer ti.RUnlock() return ti.unsafeGet(key, atRev) } func (ti *treeIndex) unsafeGet(key []byte, atRev int64) (modified, created Revision, ver int64, err error) { keyi := &keyIndex{key: key} if keyi = ti.keyIndex(keyi); keyi == nil { return Revision{}, Revision{}, 0, ErrRevisionNotFound } return keyi.get(ti.lg, atRev) } func (ti *treeIndex) KeyIndex(keyi *keyIndex) *keyIndex { ti.RLock() defer ti.RUnlock() return ti.keyIndex(keyi) } func (ti *treeIndex) keyIndex(keyi *keyIndex) *keyIndex { if ki, ok := ti.tree.Get(keyi); ok { return ki } return nil } func (ti *treeIndex) unsafeVisit(key, end []byte, f func(ki *keyIndex) bool) { keyi, endi := &keyIndex{key: key}, &keyIndex{key: end} ti.tree.AscendGreaterOrEqual(keyi, func(item *keyIndex) bool { if len(endi.key) > 0 && !item.Less(endi) { return false } if !f(item) { return false } return true }) } // Revisions returns limited number of revisions from key(included) to end(excluded) // at the given rev. The returned slice is sorted in the order of key. There is no limit if limit <= 0. // The second return parameter isn't capped by the limit and reflects the total number of revisions. func (ti *treeIndex) Revisions(key, end []byte, atRev int64, limit int) (revs []Revision, total int) { ti.RLock() defer ti.RUnlock() if end == nil { rev, _, _, err := ti.unsafeGet(key, atRev) if err != nil { return nil, 0 } return []Revision{rev}, 1 } ti.unsafeVisit(key, end, func(ki *keyIndex) bool { if rev, _, _, err := ki.get(ti.lg, atRev); err == nil { if limit <= 0 || len(revs) < limit { revs = append(revs, rev) } total++ } return true }) return revs, total } // CountRevisions returns the number of revisions // from key(included) to end(excluded) at the given rev. func (ti *treeIndex) CountRevisions(key, end []byte, atRev int64) int { ti.RLock() defer ti.RUnlock() if end == nil { _, _, _, err := ti.unsafeGet(key, atRev) if err != nil { return 0 } return 1 } total := 0 ti.unsafeVisit(key, end, func(ki *keyIndex) bool { if _, _, _, err := ki.get(ti.lg, atRev); err == nil { total++ } return true }) return total } func (ti *treeIndex) Range(key, end []byte, atRev int64) (keys [][]byte, revs []Revision) { ti.RLock() defer ti.RUnlock() if end == nil { rev, _, _, err := ti.unsafeGet(key, atRev) if err != nil { return nil, nil } return [][]byte{key}, []Revision{rev} } ti.unsafeVisit(key, end, func(ki *keyIndex) bool { if rev, _, _, err := ki.get(ti.lg, atRev); err == nil { revs = append(revs, rev) keys = append(keys, ki.key) } return true }) return keys, revs } func (ti *treeIndex) Tombstone(key []byte, rev Revision) error { keyi := &keyIndex{key: key} ti.Lock() defer ti.Unlock() ki, ok := ti.tree.Get(keyi) if !ok { return ErrRevisionNotFound } return ki.tombstone(ti.lg, rev.Main, rev.Sub) } func (ti *treeIndex) Compact(rev int64) map[Revision]struct{} { available := make(map[Revision]struct{}) ti.lg.Info("compact tree index", zap.Int64("revision", rev)) ti.Lock() clone := ti.tree.Clone() ti.Unlock() clone.Ascend(func(keyi *keyIndex) bool { // Lock is needed here to prevent modification to the keyIndex while // compaction is going on or revision added to empty before deletion ti.Lock() keyi.compact(ti.lg, rev, available) if keyi.isEmpty() { _, ok := ti.tree.Delete(keyi) if !ok { ti.lg.Panic("failed to delete during compaction") } } ti.Unlock() return true }) return available } // Keep finds all revisions to be kept for a Compaction at the given rev. func (ti *treeIndex) Keep(rev int64) map[Revision]struct{} { available := make(map[Revision]struct{}) ti.RLock() defer ti.RUnlock() ti.tree.Ascend(func(keyi *keyIndex) bool { keyi.keep(rev, available) return true }) return available } func (ti *treeIndex) Equal(bi index) bool { b := bi.(*treeIndex) if ti.tree.Len() != b.tree.Len() { return false } equal := true ti.tree.Ascend(func(aki *keyIndex) bool { bki, _ := b.tree.Get(aki) if !aki.equal(bki) { equal = false return false } return true }) return equal } func (ti *treeIndex) Insert(ki *keyIndex) { ti.Lock() defer ti.Unlock() ti.tree.ReplaceOrInsert(ki) } ================================================ FILE: server/storage/mvcc/index_bench_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "testing" "go.uber.org/zap" ) func BenchmarkIndexCompact1(b *testing.B) { benchmarkIndexCompact(b, 1) } func BenchmarkIndexCompact100(b *testing.B) { benchmarkIndexCompact(b, 100) } func BenchmarkIndexCompact10000(b *testing.B) { benchmarkIndexCompact(b, 10000) } func BenchmarkIndexCompact100000(b *testing.B) { benchmarkIndexCompact(b, 100000) } func BenchmarkIndexCompact1000000(b *testing.B) { benchmarkIndexCompact(b, 1000000) } func benchmarkIndexCompact(b *testing.B, size int) { log := zap.NewNop() kvindex := newTreeIndex(log) bytesN := 64 keys := createBytesSlice(bytesN, size) for i := 1; i < size; i++ { kvindex.Put(keys[i], Revision{Main: int64(i), Sub: int64(i)}) } b.ResetTimer() for i := 1; i < b.N; i++ { kvindex.Compact(int64(i)) } } func BenchmarkIndexPut(b *testing.B) { log := zap.NewNop() kvindex := newTreeIndex(log) bytesN := 64 keys := createBytesSlice(bytesN, b.N) b.ResetTimer() for i := 1; i < b.N; i++ { kvindex.Put(keys[i], Revision{Main: int64(i), Sub: int64(i)}) } } func BenchmarkIndexGet(b *testing.B) { log := zap.NewNop() kvindex := newTreeIndex(log) bytesN := 64 keys := createBytesSlice(bytesN, b.N) for i := 1; i < b.N; i++ { kvindex.Put(keys[i], Revision{Main: int64(i), Sub: int64(i)}) } b.ResetTimer() for i := 1; i < b.N; i++ { kvindex.Get(keys[i], int64(i)) } } ================================================ FILE: server/storage/mvcc/index_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "errors" "reflect" "testing" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" ) func TestIndexGet(t *testing.T) { ti := newTreeIndex(zaptest.NewLogger(t)) ti.Put([]byte("foo"), Revision{Main: 2}) ti.Put([]byte("foo"), Revision{Main: 4}) ti.Tombstone([]byte("foo"), Revision{Main: 6}) tests := []struct { rev int64 wrev Revision wcreated Revision wver int64 werr error }{ {0, Revision{}, Revision{}, 0, ErrRevisionNotFound}, {1, Revision{}, Revision{}, 0, ErrRevisionNotFound}, {2, Revision{Main: 2}, Revision{Main: 2}, 1, nil}, {3, Revision{Main: 2}, Revision{Main: 2}, 1, nil}, {4, Revision{Main: 4}, Revision{Main: 2}, 2, nil}, {5, Revision{Main: 4}, Revision{Main: 2}, 2, nil}, {6, Revision{}, Revision{}, 0, ErrRevisionNotFound}, } for i, tt := range tests { rev, created, ver, err := ti.Get([]byte("foo"), tt.rev) if !errors.Is(err, tt.werr) { t.Errorf("#%d: err = %v, want %v", i, err, tt.werr) } if rev != tt.wrev { t.Errorf("#%d: rev = %+v, want %+v", i, rev, tt.wrev) } if created != tt.wcreated { t.Errorf("#%d: created = %+v, want %+v", i, created, tt.wcreated) } if ver != tt.wver { t.Errorf("#%d: ver = %d, want %d", i, ver, tt.wver) } } } func TestIndexRange(t *testing.T) { allKeys := [][]byte{[]byte("foo"), []byte("foo1"), []byte("foo2")} allRevs := []Revision{{Main: 1}, {Main: 2}, {Main: 3}} ti := newTreeIndex(zaptest.NewLogger(t)) for i := range allKeys { ti.Put(allKeys[i], allRevs[i]) } atRev := int64(3) tests := []struct { key, end []byte wkeys [][]byte wrevs []Revision }{ // single key that not found { []byte("bar"), nil, nil, nil, }, // single key that found { []byte("foo"), nil, allKeys[:1], allRevs[:1], }, // range keys, return first member { []byte("foo"), []byte("foo1"), allKeys[:1], allRevs[:1], }, // range keys, return first two members { []byte("foo"), []byte("foo2"), allKeys[:2], allRevs[:2], }, // range keys, return all members { []byte("foo"), []byte("fop"), allKeys, allRevs, }, // range keys, return last two members { []byte("foo1"), []byte("fop"), allKeys[1:], allRevs[1:], }, // range keys, return last member { []byte("foo2"), []byte("fop"), allKeys[2:], allRevs[2:], }, // range keys, return nothing { []byte("foo3"), []byte("fop"), nil, nil, }, } for i, tt := range tests { keys, revs := ti.Range(tt.key, tt.end, atRev) if !reflect.DeepEqual(keys, tt.wkeys) { t.Errorf("#%d: keys = %+v, want %+v", i, keys, tt.wkeys) } if !reflect.DeepEqual(revs, tt.wrevs) { t.Errorf("#%d: revs = %+v, want %+v", i, revs, tt.wrevs) } } } func TestIndexTombstone(t *testing.T) { ti := newTreeIndex(zaptest.NewLogger(t)) ti.Put([]byte("foo"), Revision{Main: 1}) err := ti.Tombstone([]byte("foo"), Revision{Main: 2}) if err != nil { t.Errorf("tombstone error = %v, want nil", err) } _, _, _, err = ti.Get([]byte("foo"), 2) if !errors.Is(err, ErrRevisionNotFound) { t.Errorf("get error = %v, want ErrRevisionNotFound", err) } err = ti.Tombstone([]byte("foo"), Revision{Main: 3}) if !errors.Is(err, ErrRevisionNotFound) { t.Errorf("tombstone error = %v, want %v", err, ErrRevisionNotFound) } } func TestIndexRevision(t *testing.T) { allKeys := [][]byte{[]byte("foo"), []byte("foo1"), []byte("foo2"), []byte("foo2"), []byte("foo1"), []byte("foo")} allRevs := []Revision{{Main: 1}, {Main: 2}, {Main: 3}, {Main: 4}, {Main: 5}, {Main: 6}} ti := newTreeIndex(zaptest.NewLogger(t)) for i := range allKeys { ti.Put(allKeys[i], allRevs[i]) } tests := []struct { key, end []byte atRev int64 limit int wrevs []Revision wcounts int }{ // single key that not found { []byte("bar"), nil, 6, 0, nil, 0, }, // single key that found { []byte("foo"), nil, 6, 0, []Revision{{Main: 6}}, 1, }, // various range keys, fixed atRev, unlimited { []byte("foo"), []byte("foo1"), 6, 0, []Revision{{Main: 6}}, 1, }, { []byte("foo"), []byte("foo2"), 6, 0, []Revision{{Main: 6}, {Main: 5}}, 2, }, { []byte("foo"), []byte("fop"), 6, 0, []Revision{{Main: 6}, {Main: 5}, {Main: 4}}, 3, }, { []byte("foo1"), []byte("fop"), 6, 0, []Revision{{Main: 5}, {Main: 4}}, 2, }, { []byte("foo2"), []byte("fop"), 6, 0, []Revision{{Main: 4}}, 1, }, { []byte("foo3"), []byte("fop"), 6, 0, nil, 0, }, // fixed range keys, various atRev, unlimited { []byte("foo1"), []byte("fop"), 1, 0, nil, 0, }, { []byte("foo1"), []byte("fop"), 2, 0, []Revision{{Main: 2}}, 1, }, { []byte("foo1"), []byte("fop"), 3, 0, []Revision{{Main: 2}, {Main: 3}}, 2, }, { []byte("foo1"), []byte("fop"), 4, 0, []Revision{{Main: 2}, {Main: 4}}, 2, }, { []byte("foo1"), []byte("fop"), 5, 0, []Revision{{Main: 5}, {Main: 4}}, 2, }, { []byte("foo1"), []byte("fop"), 6, 0, []Revision{{Main: 5}, {Main: 4}}, 2, }, // fixed range keys, fixed atRev, various limit { []byte("foo"), []byte("fop"), 6, 1, []Revision{{Main: 6}}, 3, }, { []byte("foo"), []byte("fop"), 6, 2, []Revision{{Main: 6}, {Main: 5}}, 3, }, { []byte("foo"), []byte("fop"), 6, 3, []Revision{{Main: 6}, {Main: 5}, {Main: 4}}, 3, }, { []byte("foo"), []byte("fop"), 3, 1, []Revision{{Main: 1}}, 3, }, { []byte("foo"), []byte("fop"), 3, 2, []Revision{{Main: 1}, {Main: 2}}, 3, }, { []byte("foo"), []byte("fop"), 3, 3, []Revision{{Main: 1}, {Main: 2}, {Main: 3}}, 3, }, } for i, tt := range tests { revs, _ := ti.Revisions(tt.key, tt.end, tt.atRev, tt.limit) if !reflect.DeepEqual(revs, tt.wrevs) { t.Errorf("#%d limit %d: revs = %+v, want %+v", i, tt.limit, revs, tt.wrevs) } count := ti.CountRevisions(tt.key, tt.end, tt.atRev) if count != tt.wcounts { t.Errorf("#%d: count = %d, want %v", i, count, tt.wcounts) } } } func TestIndexCompactAndKeep(t *testing.T) { maxRev := int64(20) // key: "foo" // modified: 10 // generations: // {{10, 0}} // {{1, 0}, {5, 0}, {9, 0}(t)} // // key: "foo1" // modified: 10, 1 // generations: // {{10, 1}} // {{2, 0}, {6, 0}, {7, 0}(t)} // // key: "foo2" // modified: 8 // generations: // {empty} // {{3, 0}, {4, 0}, {8, 0}(t)} // buildTreeIndex := func() index { ti := newTreeIndex(zaptest.NewLogger(t)) ti.Put([]byte("foo"), Revision{Main: 1}) ti.Put([]byte("foo1"), Revision{Main: 2}) ti.Put([]byte("foo2"), Revision{Main: 3}) ti.Put([]byte("foo2"), Revision{Main: 4}) ti.Put([]byte("foo"), Revision{Main: 5}) ti.Put([]byte("foo1"), Revision{Main: 6}) require.NoError(t, ti.Tombstone([]byte("foo1"), Revision{Main: 7})) require.NoError(t, ti.Tombstone([]byte("foo2"), Revision{Main: 8})) require.NoError(t, ti.Tombstone([]byte("foo"), Revision{Main: 9})) ti.Put([]byte("foo"), Revision{Main: 10}) ti.Put([]byte("foo1"), Revision{Main: 10, Sub: 1}) return ti } afterCompacts := []struct { atRev int keyIndexes []keyIndex keep map[Revision]struct{} compacted map[Revision]struct{} }{ { atRev: 1, keyIndexes: []keyIndex{ { key: []byte("foo"), modified: Revision{Main: 10}, generations: []generation{ {ver: 3, created: Revision{Main: 1}, revs: []Revision{{Main: 1}, {Main: 5}, {Main: 9}}}, {ver: 1, created: Revision{Main: 10}, revs: []Revision{{Main: 10}}}, }, }, { key: []byte("foo1"), modified: Revision{Main: 10, Sub: 1}, generations: []generation{ {ver: 3, created: Revision{Main: 2}, revs: []Revision{{Main: 2}, {Main: 6}, {Main: 7}}}, {ver: 1, created: Revision{Main: 10, Sub: 1}, revs: []Revision{{Main: 10, Sub: 1}}}, }, }, { key: []byte("foo2"), modified: Revision{Main: 8}, generations: []generation{ {ver: 3, created: Revision{Main: 3}, revs: []Revision{{Main: 3}, {Main: 4}, {Main: 8}}}, {}, }, }, }, keep: map[Revision]struct{}{ {Main: 1}: {}, }, compacted: map[Revision]struct{}{ {Main: 1}: {}, }, }, { atRev: 2, keyIndexes: []keyIndex{ { key: []byte("foo"), modified: Revision{Main: 10}, generations: []generation{ {ver: 3, created: Revision{Main: 1}, revs: []Revision{{Main: 1}, {Main: 5}, {Main: 9}}}, {ver: 1, created: Revision{Main: 10}, revs: []Revision{{Main: 10}}}, }, }, { key: []byte("foo1"), modified: Revision{Main: 10, Sub: 1}, generations: []generation{ {ver: 3, created: Revision{Main: 2}, revs: []Revision{{Main: 2}, {Main: 6}, {Main: 7}}}, {ver: 1, created: Revision{Main: 10, Sub: 1}, revs: []Revision{{Main: 10, Sub: 1}}}, }, }, { key: []byte("foo2"), modified: Revision{Main: 8}, generations: []generation{ {ver: 3, created: Revision{Main: 3}, revs: []Revision{{Main: 3}, {Main: 4}, {Main: 8}}}, {}, }, }, }, keep: map[Revision]struct{}{ {Main: 1}: {}, {Main: 2}: {}, }, compacted: map[Revision]struct{}{ {Main: 1}: {}, {Main: 2}: {}, }, }, { atRev: 3, keyIndexes: []keyIndex{ { key: []byte("foo"), modified: Revision{Main: 10}, generations: []generation{ {ver: 3, created: Revision{Main: 1}, revs: []Revision{{Main: 1}, {Main: 5}, {Main: 9}}}, {ver: 1, created: Revision{Main: 10}, revs: []Revision{{Main: 10}}}, }, }, { key: []byte("foo1"), modified: Revision{Main: 10, Sub: 1}, generations: []generation{ {ver: 3, created: Revision{Main: 2}, revs: []Revision{{Main: 2}, {Main: 6}, {Main: 7}}}, {ver: 1, created: Revision{Main: 10, Sub: 1}, revs: []Revision{{Main: 10, Sub: 1}}}, }, }, { key: []byte("foo2"), modified: Revision{Main: 8}, generations: []generation{ {ver: 3, created: Revision{Main: 3}, revs: []Revision{{Main: 3}, {Main: 4}, {Main: 8}}}, {}, }, }, }, keep: map[Revision]struct{}{ {Main: 1}: {}, {Main: 2}: {}, {Main: 3}: {}, }, compacted: map[Revision]struct{}{ {Main: 1}: {}, {Main: 2}: {}, {Main: 3}: {}, }, }, { atRev: 4, keyIndexes: []keyIndex{ { key: []byte("foo"), modified: Revision{Main: 10}, generations: []generation{ {ver: 3, created: Revision{Main: 1}, revs: []Revision{{Main: 1}, {Main: 5}, {Main: 9}}}, {ver: 1, created: Revision{Main: 10}, revs: []Revision{{Main: 10}}}, }, }, { key: []byte("foo1"), modified: Revision{Main: 10, Sub: 1}, generations: []generation{ {ver: 3, created: Revision{Main: 2}, revs: []Revision{{Main: 2}, {Main: 6}, {Main: 7}}}, {ver: 1, created: Revision{Main: 10, Sub: 1}, revs: []Revision{{Main: 10, Sub: 1}}}, }, }, { key: []byte("foo2"), modified: Revision{Main: 8}, generations: []generation{ {ver: 3, created: Revision{Main: 3}, revs: []Revision{{Main: 4}, {Main: 8}}}, {}, }, }, }, keep: map[Revision]struct{}{ {Main: 1}: {}, {Main: 2}: {}, {Main: 4}: {}, }, compacted: map[Revision]struct{}{ {Main: 1}: {}, {Main: 2}: {}, {Main: 4}: {}, }, }, { atRev: 5, keyIndexes: []keyIndex{ { key: []byte("foo"), modified: Revision{Main: 10}, generations: []generation{ {ver: 3, created: Revision{Main: 1}, revs: []Revision{{Main: 5}, {Main: 9}}}, {ver: 1, created: Revision{Main: 10}, revs: []Revision{{Main: 10}}}, }, }, { key: []byte("foo1"), modified: Revision{Main: 10, Sub: 1}, generations: []generation{ {ver: 3, created: Revision{Main: 2}, revs: []Revision{{Main: 2}, {Main: 6}, {Main: 7}}}, {ver: 1, created: Revision{Main: 10, Sub: 1}, revs: []Revision{{Main: 10, Sub: 1}}}, }, }, { key: []byte("foo2"), modified: Revision{Main: 8}, generations: []generation{ {ver: 3, created: Revision{Main: 3}, revs: []Revision{{Main: 4}, {Main: 8}}}, {}, }, }, }, keep: map[Revision]struct{}{ {Main: 2}: {}, {Main: 4}: {}, {Main: 5}: {}, }, compacted: map[Revision]struct{}{ {Main: 2}: {}, {Main: 4}: {}, {Main: 5}: {}, }, }, { atRev: 6, keyIndexes: []keyIndex{ { key: []byte("foo"), modified: Revision{Main: 10}, generations: []generation{ {ver: 3, created: Revision{Main: 1}, revs: []Revision{{Main: 5}, {Main: 9}}}, {ver: 1, created: Revision{Main: 10}, revs: []Revision{{Main: 10}}}, }, }, { key: []byte("foo1"), modified: Revision{Main: 10, Sub: 1}, generations: []generation{ {ver: 3, created: Revision{Main: 2}, revs: []Revision{{Main: 6}, {Main: 7}}}, {ver: 1, created: Revision{Main: 10, Sub: 1}, revs: []Revision{{Main: 10, Sub: 1}}}, }, }, { key: []byte("foo2"), modified: Revision{Main: 8}, generations: []generation{ {ver: 3, created: Revision{Main: 3}, revs: []Revision{{Main: 4}, {Main: 8}}}, {}, }, }, }, keep: map[Revision]struct{}{ {Main: 6}: {}, {Main: 4}: {}, {Main: 5}: {}, }, compacted: map[Revision]struct{}{ {Main: 6}: {}, {Main: 4}: {}, {Main: 5}: {}, }, }, { atRev: 7, keyIndexes: []keyIndex{ { key: []byte("foo"), modified: Revision{Main: 10}, generations: []generation{ {ver: 3, created: Revision{Main: 1}, revs: []Revision{{Main: 5}, {Main: 9}}}, {ver: 1, created: Revision{Main: 10}, revs: []Revision{{Main: 10}}}, }, }, { key: []byte("foo1"), modified: Revision{Main: 10, Sub: 1}, generations: []generation{ {ver: 3, created: Revision{Main: 2}, revs: []Revision{{Main: 7}}}, {ver: 1, created: Revision{Main: 10, Sub: 1}, revs: []Revision{{Main: 10, Sub: 1}}}, }, }, { key: []byte("foo2"), modified: Revision{Main: 8}, generations: []generation{ {ver: 3, created: Revision{Main: 3}, revs: []Revision{{Main: 4}, {Main: 8}}}, {}, }, }, }, keep: map[Revision]struct{}{ {Main: 4}: {}, {Main: 5}: {}, }, compacted: map[Revision]struct{}{ {Main: 7}: {}, {Main: 4}: {}, {Main: 5}: {}, }, }, { atRev: 8, keyIndexes: []keyIndex{ { key: []byte("foo"), modified: Revision{Main: 10}, generations: []generation{ {ver: 3, created: Revision{Main: 1}, revs: []Revision{{Main: 5}, {Main: 9}}}, {ver: 1, created: Revision{Main: 10}, revs: []Revision{{Main: 10}}}, }, }, { key: []byte("foo1"), modified: Revision{Main: 10, Sub: 1}, generations: []generation{ {ver: 1, created: Revision{Main: 10, Sub: 1}, revs: []Revision{{Main: 10, Sub: 1}}}, }, }, { key: []byte("foo2"), modified: Revision{Main: 8}, generations: []generation{ {ver: 3, created: Revision{Main: 3}, revs: []Revision{{Main: 8}}}, {}, }, }, }, keep: map[Revision]struct{}{ {Main: 5}: {}, }, compacted: map[Revision]struct{}{ {Main: 8}: {}, {Main: 5}: {}, }, }, { atRev: 9, keyIndexes: []keyIndex{ { key: []byte("foo"), modified: Revision{Main: 10}, generations: []generation{ {ver: 3, created: Revision{Main: 1}, revs: []Revision{{Main: 9}}}, {ver: 1, created: Revision{Main: 10}, revs: []Revision{{Main: 10}}}, }, }, { key: []byte("foo1"), modified: Revision{Main: 10, Sub: 1}, generations: []generation{ {ver: 1, created: Revision{Main: 10, Sub: 1}, revs: []Revision{{Main: 10, Sub: 1}}}, }, }, }, keep: map[Revision]struct{}{}, compacted: map[Revision]struct{}{ {Main: 9}: {}, }, }, { atRev: 10, keyIndexes: []keyIndex{ { key: []byte("foo"), modified: Revision{Main: 10}, generations: []generation{ {ver: 1, created: Revision{Main: 10}, revs: []Revision{{Main: 10}}}, }, }, { key: []byte("foo1"), modified: Revision{Main: 10, Sub: 1}, generations: []generation{ {ver: 1, created: Revision{Main: 10, Sub: 1}, revs: []Revision{{Main: 10, Sub: 1}}}, }, }, }, keep: map[Revision]struct{}{ {Main: 10}: {}, {Main: 10, Sub: 1}: {}, }, compacted: map[Revision]struct{}{ {Main: 10}: {}, {Main: 10, Sub: 1}: {}, }, }, } ti := buildTreeIndex() // Continuous Compact and Keep for i := int64(1); i < maxRev; i++ { j := i - 1 if i >= int64(len(afterCompacts)) { j = int64(len(afterCompacts)) - 1 } am := ti.Compact(i) require.Equalf(t, afterCompacts[j].compacted, am, "#%d: compact(%d) != expected", i, i) keep := ti.Keep(i) require.Equalf(t, afterCompacts[j].keep, keep, "#%d: keep(%d) != expected", i, i) nti := newTreeIndex(zaptest.NewLogger(t)).(*treeIndex) for k := range afterCompacts[j].keyIndexes { ki := afterCompacts[j].keyIndexes[k] nti.tree.ReplaceOrInsert(&ki) } require.Truef(t, ti.Equal(nti), "#%d: not equal ti", i) } // Once Compact and Keep for i := int64(1); i < maxRev; i++ { ti := buildTreeIndex() j := i - 1 if i >= int64(len(afterCompacts)) { j = int64(len(afterCompacts)) - 1 } am := ti.Compact(i) require.Equalf(t, afterCompacts[j].compacted, am, "#%d: compact(%d) != expected", i, i) keep := ti.Keep(i) require.Equalf(t, afterCompacts[j].keep, keep, "#%d: keep(%d) != expected", i, i) nti := newTreeIndex(zaptest.NewLogger(t)).(*treeIndex) for k := range afterCompacts[j].keyIndexes { ki := afterCompacts[j].keyIndexes[k] nti.tree.ReplaceOrInsert(&ki) } require.Truef(t, ti.Equal(nti), "#%d: not equal ti", i) } } ================================================ FILE: server/storage/mvcc/key_index.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "bytes" "errors" "fmt" "go.uber.org/zap" ) var ErrRevisionNotFound = errors.New("mvcc: revision not found") // keyIndex stores the revisions of a key in the backend. // Each keyIndex has at least one key generation. // Each generation might have several key versions. // Tombstone on a key appends a tombstone version at the end // of the current generation and creates a new empty generation. // Each version of a key has an index pointing to the backend. // // For example: put(1.0);put(2.0);tombstone(3.0);put(4.0);tombstone(5.0) on key "foo" // generate a keyIndex: // key: "foo" // modified: 5 // generations: // // {empty} // {4.0, 5.0(t)} // {1.0, 2.0, 3.0(t)} // // Compact a keyIndex removes the versions with smaller or equal to // rev except the largest one. If the generation becomes empty // during compaction, it will be removed. if all the generations get // removed, the keyIndex should be removed. // // For example: // compact(2) on the previous example // generations: // // {empty} // {4.0, 5.0(t)} // {2.0, 3.0(t)} // // compact(4) // generations: // // {empty} // {4.0, 5.0(t)} // // compact(5): // generations: // // {empty} // {5.0(t)} // // compact(6): // generations: // // {empty} -> key SHOULD be removed. type keyIndex struct { key []byte modified Revision // the main rev of the last modification generations []generation } // put puts a revision to the keyIndex. func (ki *keyIndex) put(lg *zap.Logger, main int64, sub int64) { rev := Revision{Main: main, Sub: sub} if !rev.GreaterThan(ki.modified) { lg.Panic( "'put' with an unexpected smaller revision", zap.Int64("given-revision-main", rev.Main), zap.Int64("given-revision-sub", rev.Sub), zap.Int64("modified-revision-main", ki.modified.Main), zap.Int64("modified-revision-sub", ki.modified.Sub), ) } if len(ki.generations) == 0 { ki.generations = append(ki.generations, generation{}) } g := &ki.generations[len(ki.generations)-1] if len(g.revs) == 0 { // create a new key keysGauge.Inc() g.created = rev } g.revs = append(g.revs, rev) g.ver++ ki.modified = rev } func (ki *keyIndex) restore(lg *zap.Logger, created, modified Revision, ver int64) { if len(ki.generations) != 0 { lg.Panic( "'restore' got an unexpected non-empty generations", zap.Int("generations-size", len(ki.generations)), ) } ki.modified = modified g := generation{created: created, ver: ver, revs: []Revision{modified}} ki.generations = append(ki.generations, g) keysGauge.Inc() } // restoreTombstone is used to restore a tombstone revision, which is the only // revision so far for a key. We don't know the creating revision (i.e. already // compacted) of the key, so set it empty. func (ki *keyIndex) restoreTombstone(lg *zap.Logger, main, sub int64) { ki.restore(lg, Revision{}, Revision{main, sub}, 1) ki.generations = append(ki.generations, generation{}) keysGauge.Dec() } // tombstone puts a revision, pointing to a tombstone, to the keyIndex. // It also creates a new empty generation in the keyIndex. // It returns ErrRevisionNotFound when tombstone on an empty generation. func (ki *keyIndex) tombstone(lg *zap.Logger, main int64, sub int64) error { if ki.isEmpty() { lg.Panic( "'tombstone' got an unexpected empty keyIndex", zap.String("key", string(ki.key)), ) } if ki.generations[len(ki.generations)-1].isEmpty() { return ErrRevisionNotFound } ki.put(lg, main, sub) ki.generations = append(ki.generations, generation{}) keysGauge.Dec() return nil } // get gets the modified, created revision and version of the key that satisfies the given atRev. // Rev must be smaller than or equal to the given atRev. func (ki *keyIndex) get(lg *zap.Logger, atRev int64) (modified, created Revision, ver int64, err error) { if ki.isEmpty() { lg.Panic( "'get' got an unexpected empty keyIndex", zap.String("key", string(ki.key)), ) } g := ki.findGeneration(atRev) if g.isEmpty() { return Revision{}, Revision{}, 0, ErrRevisionNotFound } n := g.walk(func(rev Revision) bool { return rev.Main > atRev }) if n != -1 { return g.revs[n], g.created, g.ver - int64(len(g.revs)-n-1), nil } return Revision{}, Revision{}, 0, ErrRevisionNotFound } // since returns revisions since the given rev. Only the revision with the // largest sub revision will be returned if multiple revisions have the same // main revision. func (ki *keyIndex) since(lg *zap.Logger, rev int64) []Revision { if ki.isEmpty() { lg.Panic( "'since' got an unexpected empty keyIndex", zap.String("key", string(ki.key)), ) } since := Revision{Main: rev} var gi int // find the generations to start checking for gi = len(ki.generations) - 1; gi > 0; gi-- { g := ki.generations[gi] if g.isEmpty() { continue } if since.GreaterThan(g.created) { break } } var revs []Revision var last int64 for ; gi < len(ki.generations); gi++ { for _, r := range ki.generations[gi].revs { if since.GreaterThan(r) { continue } if r.Main == last { // replace the revision with a new one that has higher sub value, // because the original one should not be seen by external revs[len(revs)-1] = r continue } revs = append(revs, r) last = r.Main } } return revs } // compact compacts a keyIndex by removing the versions with smaller or equal // revision than the given atRev except the largest one. // If a generation becomes empty during compaction, it will be removed. func (ki *keyIndex) compact(lg *zap.Logger, atRev int64, available map[Revision]struct{}) { if ki.isEmpty() { lg.Panic( "'compact' got an unexpected empty keyIndex", zap.String("key", string(ki.key)), ) } genIdx, revIndex := ki.doCompact(atRev, available) g := &ki.generations[genIdx] if !g.isEmpty() { // remove the previous contents. if revIndex != -1 { g.revs = g.revs[revIndex:] } } // remove the previous generations. ki.generations = ki.generations[genIdx:] } // keep finds the revision to be kept if compact is called at given atRev. func (ki *keyIndex) keep(atRev int64, available map[Revision]struct{}) { if ki.isEmpty() { return } genIdx, revIndex := ki.doCompact(atRev, available) g := &ki.generations[genIdx] if !g.isEmpty() { // If the given `atRev` is a tombstone, we need to skip it. // // Note that this s different from the `compact` function which // keeps tombstone in such case. We need to stay consistent with // existing versions, ensuring they always generate the same hash // values. if revIndex == len(g.revs)-1 && genIdx != len(ki.generations)-1 { delete(available, g.revs[revIndex]) } } } func (ki *keyIndex) doCompact(atRev int64, available map[Revision]struct{}) (genIdx int, revIndex int) { // walk until reaching the first revision smaller or equal to "atRev", // and add the revision to the available map f := func(rev Revision) bool { if rev.Main <= atRev { available[rev] = struct{}{} return false } return true } genIdx, g := 0, &ki.generations[0] // find first generation includes atRev or created after atRev for genIdx < len(ki.generations)-1 { if tomb := g.revs[len(g.revs)-1].Main; tomb >= atRev { break } genIdx++ g = &ki.generations[genIdx] } revIndex = g.walk(f) return genIdx, revIndex } func (ki *keyIndex) isEmpty() bool { return len(ki.generations) == 1 && ki.generations[0].isEmpty() } // findGeneration finds out the generation of the keyIndex that the // given rev belongs to. If the given rev is at the gap of two generations, // which means that the key does not exist at the given rev, it returns nil. func (ki *keyIndex) findGeneration(rev int64) *generation { lastg := len(ki.generations) - 1 cg := lastg for cg >= 0 { if len(ki.generations[cg].revs) == 0 { cg-- continue } g := ki.generations[cg] if cg != lastg { if tomb := g.revs[len(g.revs)-1].Main; tomb <= rev { return nil } } if g.revs[0].Main <= rev { return &ki.generations[cg] } cg-- } return nil } func (ki *keyIndex) Less(bki *keyIndex) bool { return bytes.Compare(ki.key, bki.key) == -1 } func (ki *keyIndex) equal(b *keyIndex) bool { if !bytes.Equal(ki.key, b.key) { return false } if ki.modified != b.modified { return false } if len(ki.generations) != len(b.generations) { return false } for i := range ki.generations { ag, bg := ki.generations[i], b.generations[i] if !ag.equal(bg) { return false } } return true } func (ki *keyIndex) String() string { var s string for _, g := range ki.generations { s += g.String() } return s } // generation contains multiple revisions of a key. type generation struct { ver int64 created Revision // when the generation is created (put in first revision). revs []Revision } func (g *generation) isEmpty() bool { return g == nil || len(g.revs) == 0 } // walk walks through the revisions in the generation in descending order. // It passes the revision to the given function. // walk returns until: 1. it finishes walking all pairs 2. the function returns false. // walk returns the position at where it stopped. If it stopped after // finishing walking, -1 will be returned. func (g *generation) walk(f func(rev Revision) bool) int { l := len(g.revs) for i := range g.revs { ok := f(g.revs[l-i-1]) if !ok { return l - i - 1 } } return -1 } func (g *generation) String() string { return fmt.Sprintf("g: created[%d] ver[%d], revs %#v\n", g.created, g.ver, g.revs) } func (g generation) equal(b generation) bool { if g.ver != b.ver { return false } if len(g.revs) != len(b.revs) { return false } for i := range g.revs { ar, br := g.revs[i], b.revs[i] if ar != br { return false } } return true } ================================================ FILE: server/storage/mvcc/key_index_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "errors" "reflect" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" ) func TestRestoreTombstone(t *testing.T) { lg := zaptest.NewLogger(t) // restore from tombstone // // key: "foo" // modified: 16 // "created": 16 // generations: // {empty} // {{16, 0}(t)[0]} // ki := &keyIndex{key: []byte("foo")} ki.restoreTombstone(lg, 16, 0) // get should return not found for retAt := 16; retAt <= 20; retAt++ { _, _, _, err := ki.get(lg, int64(retAt)) require.ErrorIs(t, err, ErrRevisionNotFound) } // doCompact should keep that tombstone availables := map[Revision]struct{}{} ki.doCompact(16, availables) require.Len(t, availables, 1) _, ok := availables[Revision{Main: 16}] require.True(t, ok) // should be able to put new revisions ki.put(lg, 17, 0) ki.put(lg, 18, 0) revs := ki.since(lg, 16) require.Equal(t, []Revision{{16, 0}, {17, 0}, {18, 0}}, revs) // compaction should remove restored tombstone ki.compact(lg, 17, map[Revision]struct{}{}) require.Len(t, ki.generations, 1) require.Equal(t, []Revision{{17, 0}, {18, 0}}, ki.generations[0].revs) } func TestKeyIndexGet(t *testing.T) { // key: "foo" // modified: 16 // generations: // {empty} // {{14, 0}[1], {15, 1}[2], {16, 0}(t)[3]} // {{8, 0}[1], {10, 0}[2], {12, 0}(t)[3]} // {{2, 0}[1], {4, 0}[2], {6, 0}(t)[3]} ki := newTestKeyIndex(zaptest.NewLogger(t)) ki.compact(zaptest.NewLogger(t), 4, make(map[Revision]struct{})) tests := []struct { rev int64 wmod Revision wcreat Revision wver int64 werr error }{ {17, Revision{}, Revision{}, 0, ErrRevisionNotFound}, {16, Revision{}, Revision{}, 0, ErrRevisionNotFound}, // get on generation 3 {15, Revision{Main: 15, Sub: 1}, Revision{Main: 14}, 2, nil}, {14, Revision{Main: 14}, Revision{Main: 14}, 1, nil}, {13, Revision{}, Revision{}, 0, ErrRevisionNotFound}, {12, Revision{}, Revision{}, 0, ErrRevisionNotFound}, // get on generation 2 {11, Revision{Main: 10}, Revision{Main: 8}, 2, nil}, {10, Revision{Main: 10}, Revision{Main: 8}, 2, nil}, {9, Revision{Main: 8}, Revision{Main: 8}, 1, nil}, {8, Revision{Main: 8}, Revision{Main: 8}, 1, nil}, {7, Revision{}, Revision{}, 0, ErrRevisionNotFound}, {6, Revision{}, Revision{}, 0, ErrRevisionNotFound}, // get on generation 1 {5, Revision{Main: 4}, Revision{Main: 2}, 2, nil}, {4, Revision{Main: 4}, Revision{Main: 2}, 2, nil}, {3, Revision{}, Revision{}, 0, ErrRevisionNotFound}, {2, Revision{}, Revision{}, 0, ErrRevisionNotFound}, {1, Revision{}, Revision{}, 0, ErrRevisionNotFound}, {0, Revision{}, Revision{}, 0, ErrRevisionNotFound}, } for i, tt := range tests { mod, creat, ver, err := ki.get(zaptest.NewLogger(t), tt.rev) if !errors.Is(err, tt.werr) { t.Errorf("#%d: err = %v, want %v", i, err, tt.werr) } if mod != tt.wmod { t.Errorf("#%d: modified = %+v, want %+v", i, mod, tt.wmod) } if creat != tt.wcreat { t.Errorf("#%d: created = %+v, want %+v", i, creat, tt.wcreat) } if ver != tt.wver { t.Errorf("#%d: version = %d, want %d", i, ver, tt.wver) } } } func TestKeyIndexSince(t *testing.T) { ki := newTestKeyIndex(zaptest.NewLogger(t)) ki.compact(zaptest.NewLogger(t), 4, make(map[Revision]struct{})) allRevs := []Revision{ {Main: 4}, {Main: 6}, {Main: 8}, {Main: 10}, {Main: 12}, {Main: 14}, {Main: 15, Sub: 1}, {Main: 16}, } tests := []struct { rev int64 wrevs []Revision }{ {17, nil}, {16, allRevs[7:]}, {15, allRevs[6:]}, {14, allRevs[5:]}, {13, allRevs[5:]}, {12, allRevs[4:]}, {11, allRevs[4:]}, {10, allRevs[3:]}, {9, allRevs[3:]}, {8, allRevs[2:]}, {7, allRevs[2:]}, {6, allRevs[1:]}, {5, allRevs[1:]}, {4, allRevs}, {3, allRevs}, {2, allRevs}, {1, allRevs}, {0, allRevs}, } for i, tt := range tests { revs := ki.since(zaptest.NewLogger(t), tt.rev) if !reflect.DeepEqual(revs, tt.wrevs) { t.Errorf("#%d: revs = %+v, want %+v", i, revs, tt.wrevs) } } } func TestKeyIndexPut(t *testing.T) { ki := &keyIndex{key: []byte("foo")} ki.put(zaptest.NewLogger(t), 5, 0) wki := &keyIndex{ key: []byte("foo"), modified: Revision{Main: 5}, generations: []generation{{created: Revision{Main: 5}, ver: 1, revs: []Revision{{Main: 5}}}}, } if !reflect.DeepEqual(ki, wki) { t.Errorf("ki = %+v, want %+v", ki, wki) } ki.put(zaptest.NewLogger(t), 7, 0) wki = &keyIndex{ key: []byte("foo"), modified: Revision{Main: 7}, generations: []generation{{created: Revision{Main: 5}, ver: 2, revs: []Revision{{Main: 5}, {Main: 7}}}}, } if !reflect.DeepEqual(ki, wki) { t.Errorf("ki = %+v, want %+v", ki, wki) } } func TestKeyIndexRestore(t *testing.T) { ki := &keyIndex{key: []byte("foo")} ki.restore(zaptest.NewLogger(t), Revision{Main: 5}, Revision{Main: 7}, 2) wki := &keyIndex{ key: []byte("foo"), modified: Revision{Main: 7}, generations: []generation{{created: Revision{Main: 5}, ver: 2, revs: []Revision{{Main: 7}}}}, } if !reflect.DeepEqual(ki, wki) { t.Errorf("ki = %+v, want %+v", ki, wki) } } func TestKeyIndexTombstone(t *testing.T) { ki := &keyIndex{key: []byte("foo")} ki.put(zaptest.NewLogger(t), 5, 0) err := ki.tombstone(zaptest.NewLogger(t), 7, 0) if err != nil { t.Errorf("unexpected tombstone error: %v", err) } wki := &keyIndex{ key: []byte("foo"), modified: Revision{Main: 7}, generations: []generation{{created: Revision{Main: 5}, ver: 2, revs: []Revision{{Main: 5}, {Main: 7}}}, {}}, } if !reflect.DeepEqual(ki, wki) { t.Errorf("ki = %+v, want %+v", ki, wki) } ki.put(zaptest.NewLogger(t), 8, 0) ki.put(zaptest.NewLogger(t), 9, 0) err = ki.tombstone(zaptest.NewLogger(t), 15, 0) if err != nil { t.Errorf("unexpected tombstone error: %v", err) } wki = &keyIndex{ key: []byte("foo"), modified: Revision{Main: 15}, generations: []generation{ {created: Revision{Main: 5}, ver: 2, revs: []Revision{{Main: 5}, {Main: 7}}}, {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 8}, {Main: 9}, {Main: 15}}}, {}, }, } if !reflect.DeepEqual(ki, wki) { t.Errorf("ki = %+v, want %+v", ki, wki) } err = ki.tombstone(zaptest.NewLogger(t), 16, 0) if !errors.Is(err, ErrRevisionNotFound) { t.Errorf("tombstone error = %v, want %v", err, ErrRevisionNotFound) } } func TestKeyIndexCompactAndKeep(t *testing.T) { tests := []struct { compact int64 wki *keyIndex wam map[Revision]struct{} }{ { 1, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 2}, ver: 3, revs: []Revision{{Main: 2}, {Main: 4}, {Main: 6}}}, {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 8}, {Main: 10}, {Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{}, }, { 2, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 2}, ver: 3, revs: []Revision{{Main: 2}, {Main: 4}, {Main: 6}}}, {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 8}, {Main: 10}, {Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 2}: {}, }, }, { 3, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 2}, ver: 3, revs: []Revision{{Main: 2}, {Main: 4}, {Main: 6}}}, {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 8}, {Main: 10}, {Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 2}: {}, }, }, { 4, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 2}, ver: 3, revs: []Revision{{Main: 4}, {Main: 6}}}, {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 8}, {Main: 10}, {Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 4}: {}, }, }, { 5, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 2}, ver: 3, revs: []Revision{{Main: 4}, {Main: 6}}}, {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 8}, {Main: 10}, {Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 4}: {}, }, }, { 6, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 2}, ver: 3, revs: []Revision{{Main: 6}}}, {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 8}, {Main: 10}, {Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 6}: {}, }, }, { 7, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 8}, {Main: 10}, {Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{}, }, { 8, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 8}, {Main: 10}, {Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 8}: {}, }, }, { 9, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 8}, {Main: 10}, {Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 8}: {}, }, }, { 10, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 10}, {Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 10}: {}, }, }, { 11, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 10}, {Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 10}: {}, }, }, { 12, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 8}, ver: 3, revs: []Revision{{Main: 12}}}, {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 12}: {}, }, }, { 13, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{}, }, { 14, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 14}, {Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 14}: {}, }, }, { 15, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 15, Sub: 1}, {Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 15, Sub: 1}: {}, }, }, { 16, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {created: Revision{Main: 14}, ver: 3, revs: []Revision{{Main: 16}}}, {}, }, }, map[Revision]struct{}{ {Main: 16}: {}, }, }, { 17, &keyIndex{ key: []byte("foo"), modified: Revision{Main: 16}, generations: []generation{ {}, }, }, map[Revision]struct{}{}, }, } isTombstoneRevFn := func(ki *keyIndex, rev int64) bool { for i := 0; i < len(ki.generations)-1; i++ { g := ki.generations[i] if l := len(g.revs); l > 0 && g.revs[l-1].Main == rev { return true } } return false } // Continuous Compaction and finding Keep ki := newTestKeyIndex(zaptest.NewLogger(t)) for i, tt := range tests { isTombstone := isTombstoneRevFn(ki, tt.compact) am := make(map[Revision]struct{}) kiclone := cloneKeyIndex(ki) ki.keep(tt.compact, am) if !reflect.DeepEqual(ki, kiclone) { t.Errorf("#%d: ki = %+v, want %+v", i, ki, kiclone) } if isTombstone { assert.Emptyf(t, am, "#%d: ki = %d, keep result wants empty because tombstone", i, ki) } else { assert.Equalf(t, tt.wam, am, "#%d: ki = %d, compact keep should be equal to keep keep if it's not tombstone", i, ki) } am = make(map[Revision]struct{}) ki.compact(zaptest.NewLogger(t), tt.compact, am) if !reflect.DeepEqual(ki, tt.wki) { t.Errorf("#%d: ki = %+v, want %+v", i, ki, tt.wki) } if !reflect.DeepEqual(am, tt.wam) { t.Errorf("#%d: am = %+v, want %+v", i, am, tt.wam) } } // Jump Compaction and finding Keep ki = newTestKeyIndex(zaptest.NewLogger(t)) for i, tt := range tests { if !isTombstoneRevFn(ki, tt.compact) { am := make(map[Revision]struct{}) kiclone := cloneKeyIndex(ki) ki.keep(tt.compact, am) if !reflect.DeepEqual(ki, kiclone) { t.Errorf("#%d: ki = %+v, want %+v", i, ki, kiclone) } if !reflect.DeepEqual(am, tt.wam) { t.Errorf("#%d: am = %+v, want %+v", i, am, tt.wam) } am = make(map[Revision]struct{}) ki.compact(zaptest.NewLogger(t), tt.compact, am) if !reflect.DeepEqual(ki, tt.wki) { t.Errorf("#%d: ki = %+v, want %+v", i, ki, tt.wki) } if !reflect.DeepEqual(am, tt.wam) { t.Errorf("#%d: am = %+v, want %+v", i, am, tt.wam) } } } kiClone := newTestKeyIndex(zaptest.NewLogger(t)) // Once Compaction and finding Keep for i, tt := range tests { ki := newTestKeyIndex(zaptest.NewLogger(t)) am := make(map[Revision]struct{}) ki.keep(tt.compact, am) if !reflect.DeepEqual(ki, kiClone) { t.Errorf("#%d: ki = %+v, want %+v", i, ki, kiClone) } if isTombstoneRevFn(ki, tt.compact) { assert.Emptyf(t, am, "#%d: ki = %d, keep result wants empty because tombstone", i, ki) } else { assert.Equalf(t, tt.wam, am, "#%d: ki = %d, compact keep should be equal to keep keep if it's not tombstone", i, ki) } am = make(map[Revision]struct{}) ki.compact(zaptest.NewLogger(t), tt.compact, am) if !reflect.DeepEqual(ki, tt.wki) { t.Errorf("#%d: ki = %+v, want %+v", i, ki, tt.wki) } if !reflect.DeepEqual(am, tt.wam) { t.Errorf("#%d: am = %+v, want %+v", i, am, tt.wam) } } } func cloneKeyIndex(ki *keyIndex) *keyIndex { generations := make([]generation, len(ki.generations)) for i, gen := range ki.generations { generations[i] = *cloneGeneration(&gen) } return &keyIndex{ki.key, ki.modified, generations} } func cloneGeneration(g *generation) *generation { if g.revs == nil { return &generation{g.ver, g.created, nil} } tmp := make([]Revision, len(g.revs)) copy(tmp, g.revs) return &generation{g.ver, g.created, tmp} } // TestKeyIndexCompactOnFurtherRev tests that compact on version that // higher than last modified version works well func TestKeyIndexCompactOnFurtherRev(t *testing.T) { ki := &keyIndex{key: []byte("foo")} ki.put(zaptest.NewLogger(t), 1, 0) ki.put(zaptest.NewLogger(t), 2, 0) am := make(map[Revision]struct{}) ki.compact(zaptest.NewLogger(t), 3, am) wki := &keyIndex{ key: []byte("foo"), modified: Revision{Main: 2}, generations: []generation{ {created: Revision{Main: 1}, ver: 2, revs: []Revision{{Main: 2}}}, }, } wam := map[Revision]struct{}{ {Main: 2}: {}, } if !reflect.DeepEqual(ki, wki) { t.Errorf("ki = %+v, want %+v", ki, wki) } if !reflect.DeepEqual(am, wam) { t.Errorf("am = %+v, want %+v", am, wam) } } func TestKeyIndexIsEmpty(t *testing.T) { tests := []struct { ki *keyIndex w bool }{ { &keyIndex{ key: []byte("foo"), generations: []generation{{}}, }, true, }, { &keyIndex{ key: []byte("foo"), modified: Revision{Main: 2}, generations: []generation{ {created: Revision{Main: 1}, ver: 2, revs: []Revision{{Main: 2}}}, }, }, false, }, } for i, tt := range tests { g := tt.ki.isEmpty() if g != tt.w { t.Errorf("#%d: isEmpty = %v, want %v", i, g, tt.w) } } } func TestKeyIndexFindGeneration(t *testing.T) { ki := newTestKeyIndex(zaptest.NewLogger(t)) tests := []struct { rev int64 wg *generation }{ {0, nil}, {1, nil}, {2, &ki.generations[0]}, {3, &ki.generations[0]}, {4, &ki.generations[0]}, {5, &ki.generations[0]}, {6, nil}, {7, nil}, {8, &ki.generations[1]}, {9, &ki.generations[1]}, {10, &ki.generations[1]}, {11, &ki.generations[1]}, {12, nil}, {13, nil}, } for i, tt := range tests { g := ki.findGeneration(tt.rev) if g != tt.wg { t.Errorf("#%d: generation = %+v, want %+v", i, g, tt.wg) } } } func TestKeyIndexLess(t *testing.T) { ki := &keyIndex{key: []byte("foo")} tests := []struct { ki *keyIndex w bool }{ {&keyIndex{key: []byte("doo")}, false}, {&keyIndex{key: []byte("foo")}, false}, {&keyIndex{key: []byte("goo")}, true}, } for i, tt := range tests { g := ki.Less(tt.ki) if g != tt.w { t.Errorf("#%d: Less = %v, want %v", i, g, tt.w) } } } func TestGenerationIsEmpty(t *testing.T) { tests := []struct { g *generation w bool }{ {nil, true}, {&generation{}, true}, {&generation{revs: []Revision{{Main: 1}}}, false}, } for i, tt := range tests { g := tt.g.isEmpty() if g != tt.w { t.Errorf("#%d: isEmpty = %v, want %v", i, g, tt.w) } } } func TestGenerationWalk(t *testing.T) { g := &generation{ ver: 3, created: Revision{Main: 2}, revs: []Revision{{Main: 2}, {Main: 4}, {Main: 6}}, } tests := []struct { f func(rev Revision) bool wi int }{ {func(rev Revision) bool { return rev.Main >= 7 }, 2}, {func(rev Revision) bool { return rev.Main >= 6 }, 1}, {func(rev Revision) bool { return rev.Main >= 5 }, 1}, {func(rev Revision) bool { return rev.Main >= 4 }, 0}, {func(rev Revision) bool { return rev.Main >= 3 }, 0}, {func(rev Revision) bool { return rev.Main >= 2 }, -1}, } for i, tt := range tests { idx := g.walk(tt.f) if idx != tt.wi { t.Errorf("#%d: index = %d, want %d", i, idx, tt.wi) } } } func newTestKeyIndex(lg *zap.Logger) *keyIndex { // key: "foo" // modified: 16 // generations: // {empty} // {{14, 0}[1], {15, 1}[2], {16, 0}(t)[3]} // {{8, 0}[1], {10, 0}[2], {12, 0}(t)[3]} // {{2, 0}[1], {4, 0}[2], {6, 0}(t)[3]} ki := &keyIndex{key: []byte("foo")} ki.put(lg, 2, 0) ki.put(lg, 4, 0) ki.tombstone(lg, 6, 0) ki.put(lg, 8, 0) ki.put(lg, 10, 0) ki.tombstone(lg, 12, 0) ki.put(lg, 14, 0) ki.put(lg, 15, 1) ki.tombstone(lg, 16, 0) return ki } ================================================ FILE: server/storage/mvcc/kv.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "context" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" ) type RangeOptions struct { Limit int64 Rev int64 Count bool } type RangeResult struct { KVs []mvccpb.KeyValue Rev int64 Count int } type ReadView interface { // FirstRev returns the first KV revision at the time of opening the txn. // After a compaction, the first revision increases to the compaction // revision. FirstRev() int64 // Rev returns the revision of the KV at the time of opening the txn. Rev() int64 // Range gets the keys in the range at rangeRev. // The returned rev is the current revision of the KV when the operation is executed. // If rangeRev <=0, range gets the keys at currentRev. // If `end` is nil, the request returns the key. // If `end` is not nil and not empty, it gets the keys in range [key, range_end). // If `end` is not nil and empty, it gets the keys greater than or equal to key. // Limit limits the number of keys returned. // If the required rev is compacted, ErrCompacted will be returned. Range(ctx context.Context, key, end []byte, ro RangeOptions) (r *RangeResult, err error) } // TxnRead represents a read-only transaction with operations that will not // block other read transactions. type TxnRead interface { ReadView // End marks the transaction is complete and ready to commit. End() } type WriteView interface { // DeleteRange deletes the given range from the store. // A deleteRange increases the rev of the store if any key in the range exists. // The number of key deleted will be returned. // The returned rev is the current revision of the KV when the operation is executed. // It also generates one event for each key delete in the event history. // if the `end` is nil, deleteRange deletes the key. // if the `end` is not nil, deleteRange deletes the keys in range [key, range_end). DeleteRange(key, end []byte) (n, rev int64) // Put puts the given key, value into the store. Put also takes additional argument lease to // attach a lease to a key-value pair as meta-data. KV implementation does not validate the lease // id. // A put also increases the rev of the store, and generates one event in the event history. // The returned rev is the current revision of the KV when the operation is executed. Put(key, value []byte, lease lease.LeaseID) (rev int64) } // TxnWrite represents a transaction that can modify the store. type TxnWrite interface { TxnRead WriteView // Changes gets the changes made since opening the write txn. Changes() []mvccpb.KeyValue } // txnReadWrite coerces a read txn to a write, panicking on any write operation. type txnReadWrite struct{ TxnRead } func (trw *txnReadWrite) DeleteRange(key, end []byte) (n, rev int64) { panic("unexpected DeleteRange") } func (trw *txnReadWrite) Put(key, value []byte, lease lease.LeaseID) (rev int64) { panic("unexpected Put") } func (trw *txnReadWrite) Changes() []mvccpb.KeyValue { return nil } func NewReadOnlyTxnWrite(txn TxnRead) TxnWrite { return &txnReadWrite{txn} } type ReadTxMode uint32 const ( // Use ConcurrentReadTx and the txReadBuffer is copied ConcurrentReadTxMode = ReadTxMode(1) // Use backend ReadTx and txReadBuffer is not copied SharedBufReadTxMode = ReadTxMode(2) ) type KV interface { ReadView WriteView // Read creates a read transaction. Read(mode ReadTxMode, trace *traceutil.Trace) TxnRead // Write creates a write transaction. Write(trace *traceutil.Trace) TxnWrite // HashStorage returns HashStorage interface for KV storage. HashStorage() HashStorage // Compact frees all superseded keys with revisions less than rev. Compact(trace *traceutil.Trace, rev int64) (<-chan struct{}, error) // Commit commits outstanding txns into the underlying backend. Commit() // Restore restores the KV store from a backend. Restore(b backend.Backend) error Close() error } // WatchableKV is a KV that can be watched. type WatchableKV interface { KV Watchable } // Watchable is the interface that wraps the NewWatchStream function. type Watchable interface { // NewWatchStream returns a WatchStream that can be used to // watch events happened or happening on the KV. NewWatchStream() WatchStream } ================================================ FILE: server/storage/mvcc/kv_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "context" "errors" "fmt" "os" "reflect" "testing" "time" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) // Functional tests for features implemented in v3 store. It treats v3 store // as a black box, and tests it by feeding the input and validating the output. // TODO: add similar tests on operations in one txn/rev type ( rangeFunc func(kv KV, key, end []byte, ro RangeOptions) (*RangeResult, error) putFunc func(kv KV, key, value []byte, lease lease.LeaseID) int64 deleteRangeFunc func(kv KV, key, end []byte) (n, rev int64) ) var ( normalRangeFunc = func(kv KV, key, end []byte, ro RangeOptions) (*RangeResult, error) { return kv.Range(context.TODO(), key, end, ro) } txnRangeFunc = func(kv KV, key, end []byte, ro RangeOptions) (*RangeResult, error) { txn := kv.Read(ConcurrentReadTxMode, traceutil.TODO()) defer txn.End() return txn.Range(context.TODO(), key, end, ro) } normalPutFunc = func(kv KV, key, value []byte, lease lease.LeaseID) int64 { return kv.Put(key, value, lease) } txnPutFunc = func(kv KV, key, value []byte, lease lease.LeaseID) int64 { txn := kv.Write(traceutil.TODO()) defer txn.End() return txn.Put(key, value, lease) } normalDeleteRangeFunc = func(kv KV, key, end []byte) (n, rev int64) { return kv.DeleteRange(key, end) } txnDeleteRangeFunc = func(kv KV, key, end []byte) (n, rev int64) { txn := kv.Write(traceutil.TODO()) defer txn.End() return txn.DeleteRange(key, end) } ) func TestKVRange(t *testing.T) { testKVRange(t, normalRangeFunc) } func TestKVTxnRange(t *testing.T) { testKVRange(t, txnRangeFunc) } func testKVRange(t *testing.T, f rangeFunc) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) kvs := put3TestKVs(s) wrev := int64(4) tests := []struct { key, end []byte wkvs []mvccpb.KeyValue }{ // get no keys { []byte("doo"), []byte("foo"), nil, }, // get no keys when key == end { []byte("foo"), []byte("foo"), nil, }, // get no keys when ranging single key { []byte("doo"), nil, nil, }, // get all keys { []byte("foo"), []byte("foo3"), kvs, }, // get partial keys { []byte("foo"), []byte("foo1"), kvs[:1], }, // get single key { []byte("foo"), nil, kvs[:1], }, // get entire keyspace { []byte(""), []byte(""), kvs, }, } for i, tt := range tests { r, err := f(s, tt.key, tt.end, RangeOptions{}) if err != nil { t.Fatal(err) } if r.Rev != wrev { t.Errorf("#%d: rev = %d, want %d", i, r.Rev, wrev) } if !reflect.DeepEqual(r.KVs, tt.wkvs) { t.Errorf("#%d: kvs = %+v, want %+v", i, r.KVs, tt.wkvs) } } } func TestKVRangeRev(t *testing.T) { testKVRangeRev(t, normalRangeFunc) } func TestKVTxnRangeRev(t *testing.T) { testKVRangeRev(t, txnRangeFunc) } func testKVRangeRev(t *testing.T, f rangeFunc) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) kvs := put3TestKVs(s) tests := []struct { rev int64 wrev int64 wkvs []mvccpb.KeyValue }{ {-1, 4, kvs}, {0, 4, kvs}, {2, 4, kvs[:1]}, {3, 4, kvs[:2]}, {4, 4, kvs}, } for i, tt := range tests { r, err := f(s, []byte("foo"), []byte("foo3"), RangeOptions{Rev: tt.rev}) if err != nil { t.Fatal(err) } if r.Rev != tt.wrev { t.Errorf("#%d: rev = %d, want %d", i, r.Rev, tt.wrev) } if !reflect.DeepEqual(r.KVs, tt.wkvs) { t.Errorf("#%d: kvs = %+v, want %+v", i, r.KVs, tt.wkvs) } } } func TestKVRangeBadRev(t *testing.T) { testKVRangeBadRev(t, normalRangeFunc) } func TestKVTxnRangeBadRev(t *testing.T) { testKVRangeBadRev(t, txnRangeFunc) } func testKVRangeBadRev(t *testing.T, f rangeFunc) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) put3TestKVs(s) if _, err := s.Compact(traceutil.TODO(), 4); err != nil { t.Fatalf("compact error (%v)", err) } tests := []struct { rev int64 werr error }{ {-1, nil}, // <= 0 is most recent store {0, nil}, {1, ErrCompacted}, {2, ErrCompacted}, {4, nil}, {5, ErrFutureRev}, {100, ErrFutureRev}, } for i, tt := range tests { _, err := f(s, []byte("foo"), []byte("foo3"), RangeOptions{Rev: tt.rev}) if !errors.Is(err, tt.werr) { t.Errorf("#%d: error = %v, want %v", i, err, tt.werr) } } } func TestKVRangeLimit(t *testing.T) { testKVRangeLimit(t, normalRangeFunc) } func TestKVTxnRangeLimit(t *testing.T) { testKVRangeLimit(t, txnRangeFunc) } func testKVRangeLimit(t *testing.T, f rangeFunc) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) kvs := put3TestKVs(s) wrev := int64(4) tests := []struct { limit int64 wcounts int64 wkvs []mvccpb.KeyValue }{ // no limit {-1, 3, kvs}, // no limit {0, 3, kvs}, {1, 3, kvs[:1]}, {2, 3, kvs[:2]}, {3, 3, kvs}, {100, 3, kvs}, } for i, tt := range tests { r, err := f(s, []byte("foo"), []byte("foo3"), RangeOptions{Limit: tt.limit}) if err != nil { t.Fatalf("#%d: range error (%v)", i, err) } if !reflect.DeepEqual(r.KVs, tt.wkvs) { t.Errorf("#%d: kvs = %+v, want %+v", i, r.KVs, tt.wkvs) } if r.Rev != wrev { t.Errorf("#%d: rev = %d, want %d", i, r.Rev, wrev) } if tt.limit <= 0 || int(tt.limit) > len(kvs) { if r.Count != len(kvs) { t.Errorf("#%d: count = %d, want %d", i, r.Count, len(kvs)) } } else if r.Count != int(tt.wcounts) { t.Errorf("#%d: count = %d, want %d", i, r.Count, tt.limit) } } } func TestKVPutMultipleTimes(t *testing.T) { testKVPutMultipleTimes(t, normalPutFunc) } func TestKVTxnPutMultipleTimes(t *testing.T) { testKVPutMultipleTimes(t, txnPutFunc) } func testKVPutMultipleTimes(t *testing.T, f putFunc) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) for i := 0; i < 10; i++ { base := int64(i + 1) rev := f(s, []byte("foo"), []byte("bar"), lease.LeaseID(base)) if rev != base+1 { t.Errorf("#%d: rev = %d, want %d", i, rev, base+1) } r, err := s.Range(t.Context(), []byte("foo"), nil, RangeOptions{}) if err != nil { t.Fatal(err) } wkvs := []mvccpb.KeyValue{ {Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: base + 1, Version: base, Lease: base}, } if !reflect.DeepEqual(r.KVs, wkvs) { t.Errorf("#%d: kvs = %+v, want %+v", i, r.KVs, wkvs) } } } func TestKVDeleteRange(t *testing.T) { testKVDeleteRange(t, normalDeleteRangeFunc) } func TestKVTxnDeleteRange(t *testing.T) { testKVDeleteRange(t, txnDeleteRangeFunc) } func testKVDeleteRange(t *testing.T, f deleteRangeFunc) { tests := []struct { key, end []byte wrev int64 wN int64 }{ { []byte("foo"), nil, 5, 1, }, { []byte("foo"), []byte("foo1"), 5, 1, }, { []byte("foo"), []byte("foo2"), 5, 2, }, { []byte("foo"), []byte("foo3"), 5, 3, }, { []byte("foo3"), []byte("foo8"), 4, 0, }, { []byte("foo3"), nil, 4, 0, }, } for i, tt := range tests { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) s.Put([]byte("foo"), []byte("bar"), lease.NoLease) s.Put([]byte("foo1"), []byte("bar1"), lease.NoLease) s.Put([]byte("foo2"), []byte("bar2"), lease.NoLease) n, rev := f(s, tt.key, tt.end) if n != tt.wN || rev != tt.wrev { t.Errorf("#%d: n = %d, rev = %d, want (%d, %d)", i, n, rev, tt.wN, tt.wrev) } cleanup(s, b) } } func TestKVDeleteMultipleTimes(t *testing.T) { testKVDeleteMultipleTimes(t, normalDeleteRangeFunc) } func TestKVTxnDeleteMultipleTimes(t *testing.T) { testKVDeleteMultipleTimes(t, txnDeleteRangeFunc) } func testKVDeleteMultipleTimes(t *testing.T, f deleteRangeFunc) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) s.Put([]byte("foo"), []byte("bar"), lease.NoLease) n, rev := f(s, []byte("foo"), nil) if n != 1 || rev != 3 { t.Fatalf("n = %d, rev = %d, want (%d, %d)", n, rev, 1, 3) } for i := 0; i < 10; i++ { n, rev := f(s, []byte("foo"), nil) if n != 0 || rev != 3 { t.Fatalf("#%d: n = %d, rev = %d, want (%d, %d)", i, n, rev, 0, 3) } } } func TestKVPutWithSameLease(t *testing.T) { testKVPutWithSameLease(t, normalPutFunc) } func TestKVTxnPutWithSameLease(t *testing.T) { testKVPutWithSameLease(t, txnPutFunc) } func testKVPutWithSameLease(t *testing.T, f putFunc) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) leaseID := int64(1) // put foo rev := f(s, []byte("foo"), []byte("bar"), lease.LeaseID(leaseID)) if rev != 2 { t.Errorf("rev = %d, want %d", 2, rev) } // put foo with same lease again rev2 := f(s, []byte("foo"), []byte("bar"), lease.LeaseID(leaseID)) if rev2 != 3 { t.Errorf("rev = %d, want %d", 3, rev2) } // check leaseID r, err := s.Range(t.Context(), []byte("foo"), nil, RangeOptions{}) if err != nil { t.Fatal(err) } wkvs := []mvccpb.KeyValue{ {Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 3, Version: 2, Lease: leaseID}, } if !reflect.DeepEqual(r.KVs, wkvs) { t.Errorf("kvs = %+v, want %+v", r.KVs, wkvs) } } // TestKVOperationInSequence tests that range, put, delete on single key in // sequence repeatedly works correctly. func TestKVOperationInSequence(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) for i := 0; i < 10; i++ { base := int64(i*2 + 1) // put foo rev := s.Put([]byte("foo"), []byte("bar"), lease.NoLease) if rev != base+1 { t.Errorf("#%d: put rev = %d, want %d", i, rev, base+1) } r, err := s.Range(t.Context(), []byte("foo"), nil, RangeOptions{Rev: base + 1}) if err != nil { t.Fatal(err) } wkvs := []mvccpb.KeyValue{ {Key: []byte("foo"), Value: []byte("bar"), CreateRevision: base + 1, ModRevision: base + 1, Version: 1, Lease: int64(lease.NoLease)}, } if !reflect.DeepEqual(r.KVs, wkvs) { t.Errorf("#%d: kvs = %+v, want %+v", i, r.KVs, wkvs) } if r.Rev != base+1 { t.Errorf("#%d: range rev = %d, want %d", i, rev, base+1) } // delete foo n, rev := s.DeleteRange([]byte("foo"), nil) if n != 1 || rev != base+2 { t.Errorf("#%d: n = %d, rev = %d, want (%d, %d)", i, n, rev, 1, base+2) } r, err = s.Range(t.Context(), []byte("foo"), nil, RangeOptions{Rev: base + 2}) if err != nil { t.Fatal(err) } if r.KVs != nil { t.Errorf("#%d: kvs = %+v, want %+v", i, r.KVs, nil) } if r.Rev != base+2 { t.Errorf("#%d: range rev = %d, want %d", i, r.Rev, base+2) } } } func TestKVTxnBlockWriteOperations(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) tests := []func(){ func() { s.Put([]byte("foo"), nil, lease.NoLease) }, func() { s.DeleteRange([]byte("foo"), nil) }, } for i, tt := range tests { tf := tt txn := s.Write(traceutil.TODO()) done := make(chan struct{}, 1) go func() { tf() done <- struct{}{} }() select { case <-done: t.Fatalf("#%d: operation failed to be blocked", i) case <-time.After(10 * time.Millisecond): } txn.End() select { case <-done: case <-time.After(10 * time.Second): testutil.FatalStack(t, fmt.Sprintf("#%d: operation failed to be unblocked", i)) } } // only close backend when we know all the tx are finished cleanup(s, b) } func TestKVTxnNonBlockRange(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) txn := s.Write(traceutil.TODO()) defer txn.End() donec := make(chan struct{}) go func() { defer close(donec) s.Range(t.Context(), []byte("foo"), nil, RangeOptions{}) }() select { case <-donec: case <-time.After(100 * time.Millisecond): t.Fatalf("range operation blocked on write txn") } } // TestKVTxnOperationInSequence tests that txn range, put, delete on single key // in sequence repeatedly works correctly. func TestKVTxnOperationInSequence(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) for i := 0; i < 10; i++ { txn := s.Write(traceutil.TODO()) base := int64(i + 1) // put foo rev := txn.Put([]byte("foo"), []byte("bar"), lease.NoLease) if rev != base+1 { t.Errorf("#%d: put rev = %d, want %d", i, rev, base+1) } r, err := txn.Range(t.Context(), []byte("foo"), nil, RangeOptions{Rev: base + 1}) if err != nil { t.Fatal(err) } wkvs := []mvccpb.KeyValue{ {Key: []byte("foo"), Value: []byte("bar"), CreateRevision: base + 1, ModRevision: base + 1, Version: 1, Lease: int64(lease.NoLease)}, } if !reflect.DeepEqual(r.KVs, wkvs) { t.Errorf("#%d: kvs = %+v, want %+v", i, r.KVs, wkvs) } if r.Rev != base+1 { t.Errorf("#%d: range rev = %d, want %d", i, r.Rev, base+1) } // delete foo n, rev := txn.DeleteRange([]byte("foo"), nil) if n != 1 || rev != base+1 { t.Errorf("#%d: n = %d, rev = %d, want (%d, %d)", i, n, rev, 1, base+1) } r, err = txn.Range(t.Context(), []byte("foo"), nil, RangeOptions{Rev: base + 1}) if err != nil { t.Errorf("#%d: range error (%v)", i, err) } if r.KVs != nil { t.Errorf("#%d: kvs = %+v, want %+v", i, r.KVs, nil) } if r.Rev != base+1 { t.Errorf("#%d: range rev = %d, want %d", i, r.Rev, base+1) } txn.End() } } func TestKVCompactReserveLastValue(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) s.Put([]byte("foo"), []byte("bar0"), 1) s.Put([]byte("foo"), []byte("bar1"), 2) s.DeleteRange([]byte("foo"), nil) s.Put([]byte("foo"), []byte("bar2"), 3) // rev in tests will be called in Compact() one by one on the same store tests := []struct { rev int64 // wanted kvs right after the compacted rev wkvs []mvccpb.KeyValue }{ { 1, []mvccpb.KeyValue{ {Key: []byte("foo"), Value: []byte("bar0"), CreateRevision: 2, ModRevision: 2, Version: 1, Lease: 1}, }, }, { 2, []mvccpb.KeyValue{ {Key: []byte("foo"), Value: []byte("bar1"), CreateRevision: 2, ModRevision: 3, Version: 2, Lease: 2}, }, }, { 3, nil, }, { 4, []mvccpb.KeyValue{ {Key: []byte("foo"), Value: []byte("bar2"), CreateRevision: 5, ModRevision: 5, Version: 1, Lease: 3}, }, }, } for i, tt := range tests { _, err := s.Compact(traceutil.TODO(), tt.rev) if err != nil { t.Errorf("#%d: unexpect compact error %v", i, err) } r, err := s.Range(t.Context(), []byte("foo"), nil, RangeOptions{Rev: tt.rev + 1}) if err != nil { t.Errorf("#%d: unexpect range error %v", i, err) } if !reflect.DeepEqual(r.KVs, tt.wkvs) { t.Errorf("#%d: kvs = %+v, want %+v", i, r.KVs, tt.wkvs) } } } func TestKVCompactBad(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) s.Put([]byte("foo"), []byte("bar0"), lease.NoLease) s.Put([]byte("foo"), []byte("bar1"), lease.NoLease) s.Put([]byte("foo"), []byte("bar2"), lease.NoLease) // rev in tests will be called in Compact() one by one on the same store tests := []struct { rev int64 werr error }{ {0, nil}, {1, nil}, {1, ErrCompacted}, {4, nil}, {5, ErrFutureRev}, {100, ErrFutureRev}, } for i, tt := range tests { _, err := s.Compact(traceutil.TODO(), tt.rev) if !errors.Is(err, tt.werr) { t.Errorf("#%d: compact error = %v, want %v", i, err, tt.werr) } } } func TestKVHash(t *testing.T) { hashes := make([]uint32, 3) for i := 0; i < len(hashes); i++ { var err error b, _ := betesting.NewDefaultTmpBackend(t) kv := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) kv.Put([]byte("foo0"), []byte("bar0"), lease.NoLease) kv.Put([]byte("foo1"), []byte("bar0"), lease.NoLease) hashes[i], _, err = kv.hash() if err != nil { t.Fatalf("failed to get hash: %v", err) } cleanup(kv, b) } for i := 1; i < len(hashes); i++ { if hashes[i-1] != hashes[i] { t.Errorf("hash[%d](%d) != hash[%d](%d)", i-1, hashes[i-1], i, hashes[i]) } } } func TestKVRestore(t *testing.T) { compactBatchLimit := 5 tests := []func(kv KV){ func(kv KV) { kv.Put([]byte("foo"), []byte("bar0"), 1) kv.Put([]byte("foo"), []byte("bar1"), 2) kv.Put([]byte("foo"), []byte("bar2"), 3) kv.Put([]byte("foo2"), []byte("bar0"), 1) }, func(kv KV) { kv.Put([]byte("foo"), []byte("bar0"), 1) kv.DeleteRange([]byte("foo"), nil) kv.Put([]byte("foo"), []byte("bar1"), 2) }, func(kv KV) { kv.Put([]byte("foo"), []byte("bar0"), 1) kv.Put([]byte("foo"), []byte("bar1"), 2) kv.Compact(traceutil.TODO(), 1) }, func(kv KV) { // after restore, foo1 key only has tombstone revision kv.Put([]byte("foo1"), []byte("bar1"), 0) kv.Put([]byte("foo2"), []byte("bar2"), 0) kv.Put([]byte("foo3"), []byte("bar3"), 0) kv.Put([]byte("foo4"), []byte("bar4"), 0) kv.Put([]byte("foo5"), []byte("bar5"), 0) _, delAtRev := kv.DeleteRange([]byte("foo1"), nil) assert.Equal(t, int64(7), delAtRev) // after compaction and restore, foo1 key only has tombstone revision ch, _ := kv.Compact(traceutil.TODO(), delAtRev) <-ch }, } for i, tt := range tests { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{CompactionBatchLimit: compactBatchLimit}) tt(s) var kvss [][]mvccpb.KeyValue for k := int64(0); k < 10; k++ { r, _ := s.Range(t.Context(), []byte("a"), []byte("z"), RangeOptions{Rev: k}) kvss = append(kvss, r.KVs) } keysBefore := readGaugeInt(keysGauge) s.Close() // ns should recover the previous state from backend. ns := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{CompactionBatchLimit: compactBatchLimit}) if keysRestore := readGaugeInt(keysGauge); keysBefore != keysRestore { t.Errorf("#%d: got %d key count, expected %d", i, keysRestore, keysBefore) } // wait for possible compaction to finish testutil.WaitSchedule() var nkvss [][]mvccpb.KeyValue for k := int64(0); k < 10; k++ { r, _ := ns.Range(t.Context(), []byte("a"), []byte("z"), RangeOptions{Rev: k}) nkvss = append(nkvss, r.KVs) } cleanup(ns, b) if !reflect.DeepEqual(nkvss, kvss) { t.Errorf("#%d: kvs history = %+v, want %+v", i, nkvss, kvss) } } } func readGaugeInt(g prometheus.Gauge) int { ch := make(chan prometheus.Metric, 1) g.Collect(ch) m := <-ch mm := &dto.Metric{} m.Write(mm) return int(mm.GetGauge().GetValue()) } func TestKVSnapshot(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) wkvs := put3TestKVs(s) newPath := "new_test" f, err := os.Create(newPath) if err != nil { t.Fatal(err) } defer os.Remove(newPath) snap := s.b.Snapshot() defer snap.Close() _, err = snap.WriteTo(f) if err != nil { t.Fatal(err) } f.Close() ns := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer ns.Close() r, err := ns.Range(t.Context(), []byte("a"), []byte("z"), RangeOptions{}) if err != nil { t.Errorf("unexpect range error (%v)", err) } if !reflect.DeepEqual(r.KVs, wkvs) { t.Errorf("kvs = %+v, want %+v", r.KVs, wkvs) } if r.Rev != 4 { t.Errorf("rev = %d, want %d", r.Rev, 4) } } func TestWatchableKVWatch(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) w := s.NewWatchStream() defer w.Close() wid, _ := w.Watch(t.Context(), 0, []byte("foo"), []byte("fop"), 0) wev := []mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1, Lease: 1, }, }, { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{ Key: []byte("foo1"), Value: []byte("bar1"), CreateRevision: 3, ModRevision: 3, Version: 1, Lease: 2, }, }, { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{ Key: []byte("foo1"), Value: []byte("bar11"), CreateRevision: 3, ModRevision: 4, Version: 2, Lease: 3, }, }, } s.Put([]byte("foo"), []byte("bar"), 1) select { case resp := <-w.Chan(): if resp.WatchID != wid { t.Errorf("resp.WatchID got = %d, want = %d", resp.WatchID, wid) } ev := resp.Events[0] if !reflect.DeepEqual(ev, wev[0]) { t.Errorf("watched event = %+v, want %+v", ev, wev[0]) } case <-time.After(5 * time.Second): // CPU might be too slow, and the routine is not able to switch around testutil.FatalStack(t, "failed to watch the event") } s.Put([]byte("foo1"), []byte("bar1"), 2) select { case resp := <-w.Chan(): if resp.WatchID != wid { t.Errorf("resp.WatchID got = %d, want = %d", resp.WatchID, wid) } ev := resp.Events[0] if !reflect.DeepEqual(ev, wev[1]) { t.Errorf("watched event = %+v, want %+v", ev, wev[1]) } case <-time.After(5 * time.Second): testutil.FatalStack(t, "failed to watch the event") } w = s.NewWatchStream() wid, _ = w.Watch(t.Context(), 0, []byte("foo1"), []byte("foo2"), 3) select { case resp := <-w.Chan(): if resp.WatchID != wid { t.Errorf("resp.WatchID got = %d, want = %d", resp.WatchID, wid) } ev := resp.Events[0] if !reflect.DeepEqual(ev, wev[1]) { t.Errorf("watched event = %+v, want %+v", ev, wev[1]) } case <-time.After(5 * time.Second): testutil.FatalStack(t, "failed to watch the event") } s.Put([]byte("foo1"), []byte("bar11"), 3) select { case resp := <-w.Chan(): if resp.WatchID != wid { t.Errorf("resp.WatchID got = %d, want = %d", resp.WatchID, wid) } ev := resp.Events[0] if !reflect.DeepEqual(ev, wev[2]) { t.Errorf("watched event = %+v, want %+v", ev, wev[2]) } case <-time.After(5 * time.Second): testutil.FatalStack(t, "failed to watch the event") } } func cleanup(s KV, b backend.Backend) { s.Close() b.Close() } func put3TestKVs(s KV) []mvccpb.KeyValue { s.Put([]byte("foo"), []byte("bar"), 1) s.Put([]byte("foo1"), []byte("bar1"), 2) s.Put([]byte("foo2"), []byte("bar2"), 3) return []mvccpb.KeyValue{ {Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1, Lease: 1}, {Key: []byte("foo1"), Value: []byte("bar1"), CreateRevision: 3, ModRevision: 3, Version: 1, Lease: 2}, {Key: []byte("foo2"), Value: []byte("bar2"), CreateRevision: 4, ModRevision: 4, Version: 1, Lease: 3}, } } ================================================ FILE: server/storage/mvcc/kv_view.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "context" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" ) type readView struct{ kv KV } func (rv *readView) FirstRev() int64 { tr := rv.kv.Read(SharedBufReadTxMode, traceutil.TODO()) defer tr.End() return tr.FirstRev() } func (rv *readView) Rev() int64 { tr := rv.kv.Read(SharedBufReadTxMode, traceutil.TODO()) defer tr.End() return tr.Rev() } func (rv *readView) Range(ctx context.Context, key, end []byte, ro RangeOptions) (r *RangeResult, err error) { tr := rv.kv.Read(ConcurrentReadTxMode, traceutil.TODO()) defer tr.End() return tr.Range(ctx, key, end, ro) } type writeView struct{ kv KV } func (wv *writeView) DeleteRange(key, end []byte) (n, rev int64) { tw := wv.kv.Write(traceutil.TODO()) defer tw.End() return tw.DeleteRange(key, end) } func (wv *writeView) Put(key, value []byte, lease lease.LeaseID) (rev int64) { tw := wv.kv.Write(traceutil.TODO()) defer tw.End() return tw.Put(key, value, lease) } ================================================ FILE: server/storage/mvcc/kvstore.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "context" "errors" "fmt" "math" "sync" "time" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/client/pkg/v3/verify" "go.etcd.io/etcd/pkg/v3/schedule" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" ) var ( ErrCompacted = errors.New("mvcc: required revision has been compacted") ErrFutureRev = errors.New("mvcc: required revision is a future revision") ) var ( restoreChunkKeys = 10000 // non-const for testing defaultCompactionBatchLimit = 1000 defaultCompactionSleepInterval = 10 * time.Millisecond ) type StoreConfig struct { CompactionBatchLimit int CompactionSleepInterval time.Duration } type store struct { ReadView WriteView cfg StoreConfig // mu read locks for txns and write locks for non-txn store changes. mu sync.RWMutex b backend.Backend kvindex index le lease.Lessor // revMuLock protects currentRev and compactMainRev. // Locked at end of write txn and released after write txn unlock lock. // Locked before locking read txn and released after locking. revMu sync.RWMutex // currentRev is the revision of the last completed transaction. currentRev int64 // compactMainRev is the main revision of the last compaction. compactMainRev int64 fifoSched schedule.Scheduler stopc chan struct{} lg *zap.Logger hashes HashStorage } // NewStore returns a new store. It is useful to create a store inside // mvcc pkg. It should only be used for testing externally. // revive:disable-next-line:unexported-return this is used internally in the mvcc pkg func NewStore(lg *zap.Logger, b backend.Backend, le lease.Lessor, cfg StoreConfig) *store { if lg == nil { lg = zap.NewNop() } if cfg.CompactionBatchLimit == 0 { cfg.CompactionBatchLimit = defaultCompactionBatchLimit } if cfg.CompactionSleepInterval == 0 { cfg.CompactionSleepInterval = defaultCompactionSleepInterval } s := &store{ cfg: cfg, b: b, kvindex: newTreeIndex(lg), le: le, currentRev: 1, compactMainRev: -1, fifoSched: schedule.NewFIFOScheduler(lg), stopc: make(chan struct{}), lg: lg, } s.hashes = NewHashStorage(lg, s) s.ReadView = &readView{s} s.WriteView = &writeView{s} if s.le != nil { s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write(traceutil.TODO()) }) } tx := s.b.BatchTx() tx.LockOutsideApply() tx.UnsafeCreateBucket(schema.Key) schema.UnsafeCreateMetaBucket(tx) tx.Unlock() s.b.ForceCommit() s.mu.Lock() defer s.mu.Unlock() if err := s.restore(); err != nil { // TODO: return the error instead of panic here? panic("failed to recover store from backend") } return s } func (s *store) compactBarrier(ctx context.Context, ch chan struct{}) { if ctx == nil || ctx.Err() != nil { select { case <-s.stopc: default: // fix deadlock in mvcc, for more information, please refer to pr 11817. // s.stopc is only updated in restore operation, which is called by apply // snapshot call, compaction and apply snapshot requests are serialized by // raft, and do not happen at the same time. s.mu.Lock() f := schedule.NewJob("kvstore_compactBarrier", func(ctx context.Context) { s.compactBarrier(ctx, ch) }) s.fifoSched.Schedule(f) s.mu.Unlock() } return } close(ch) } func (s *store) hash() (hash uint32, revision int64, err error) { // TODO: hash and revision could be inconsistent, one possible fix is to add s.revMu.RLock() at the beginning of function, which is costly start := time.Now() s.b.ForceCommit() h, err := s.b.Hash(schema.DefaultIgnores) hashSec.Observe(time.Since(start).Seconds()) return h, s.currentRev, err } func (s *store) hashByRev(rev int64) (hash KeyValueHash, currentRev int64, err error) { var compactRev int64 start := time.Now() s.mu.RLock() s.revMu.RLock() compactRev, currentRev = s.compactMainRev, s.currentRev s.revMu.RUnlock() if rev > 0 && rev < compactRev { s.mu.RUnlock() return KeyValueHash{}, 0, ErrCompacted } else if rev > 0 && rev > currentRev { s.mu.RUnlock() return KeyValueHash{}, currentRev, ErrFutureRev } if rev == 0 { rev = currentRev } keep := s.kvindex.Keep(rev) tx := s.b.ReadTx() tx.RLock() defer tx.RUnlock() s.mu.RUnlock() hash, err = unsafeHashByRev(tx, compactRev, rev, keep) hashRevSec.Observe(time.Since(start).Seconds()) return hash, currentRev, err } func (s *store) updateCompactRev(rev int64) (<-chan struct{}, int64, error) { s.revMu.Lock() if rev <= s.compactMainRev { ch := make(chan struct{}) f := schedule.NewJob("kvstore_updateCompactRev_compactBarrier", func(ctx context.Context) { s.compactBarrier(ctx, ch) }) s.fifoSched.Schedule(f) s.revMu.Unlock() return ch, 0, ErrCompacted } if rev > s.currentRev { s.revMu.Unlock() return nil, 0, ErrFutureRev } compactMainRev := s.compactMainRev s.compactMainRev = rev SetScheduledCompact(s.b.BatchTx(), rev) // ensure that desired compaction is persisted // gofail: var compactBeforeCommitScheduledCompact struct{} s.b.ForceCommit() // gofail: var compactAfterCommitScheduledCompact struct{} s.revMu.Unlock() return nil, compactMainRev, nil } // checkPrevCompactionCompleted checks whether the previous scheduled compaction is completed. func (s *store) checkPrevCompactionCompleted() bool { tx := s.b.ReadTx() tx.RLock() defer tx.RUnlock() scheduledCompact, scheduledCompactFound := UnsafeReadScheduledCompact(tx) finishedCompact, finishedCompactFound := UnsafeReadFinishedCompact(tx) return scheduledCompact == finishedCompact && scheduledCompactFound == finishedCompactFound } func (s *store) compact(trace *traceutil.Trace, rev, prevCompactRev int64, prevCompactionCompleted bool) <-chan struct{} { ch := make(chan struct{}) j := schedule.NewJob("kvstore_compact", func(ctx context.Context) { if ctx.Err() != nil { s.compactBarrier(ctx, ch) return } hash, err := s.scheduleCompaction(rev, prevCompactRev) if err != nil { s.lg.Warn("Failed compaction", zap.Error(err)) s.compactBarrier(context.TODO(), ch) return } // Only store the hash value if the previous hash is completed, i.e. this compaction // hashes every revision from last compaction. For more details, see #15919. if prevCompactionCompleted { s.hashes.Store(hash) } else { s.lg.Info("previous compaction was interrupted, skip storing compaction hash value") } close(ch) }) s.fifoSched.Schedule(j) trace.Step("schedule compaction") return ch } func (s *store) compactLockfree(rev int64) (<-chan struct{}, error) { prevCompactionCompleted := s.checkPrevCompactionCompleted() ch, prevCompactRev, err := s.updateCompactRev(rev) if err != nil { return ch, err } return s.compact(traceutil.TODO(), rev, prevCompactRev, prevCompactionCompleted), nil } func (s *store) Compact(trace *traceutil.Trace, rev int64) (<-chan struct{}, error) { s.mu.Lock() prevCompactionCompleted := s.checkPrevCompactionCompleted() ch, prevCompactRev, err := s.updateCompactRev(rev) trace.Step("check and update compact revision") if err != nil { s.mu.Unlock() return ch, err } s.mu.Unlock() return s.compact(trace, rev, prevCompactRev, prevCompactionCompleted), nil } func (s *store) Commit() { s.mu.Lock() defer s.mu.Unlock() s.b.ForceCommit() } func (s *store) Restore(b backend.Backend) error { s.mu.Lock() defer s.mu.Unlock() close(s.stopc) s.fifoSched.Stop() s.b = b s.kvindex = newTreeIndex(s.lg) { // During restore the metrics might report 'special' values s.revMu.Lock() s.currentRev = 1 s.compactMainRev = -1 s.revMu.Unlock() } s.fifoSched = schedule.NewFIFOScheduler(s.lg) s.stopc = make(chan struct{}) return s.restore() } //nolint:unparam func (s *store) restore() error { s.setupMetricsReporter() min, max := NewRevBytes(), NewRevBytes() min = RevToBytes(Revision{Main: 1}, min) max = RevToBytes(Revision{Main: math.MaxInt64, Sub: math.MaxInt64}, max) keyToLease := make(map[string]lease.LeaseID) // restore index tx := s.b.ReadTx() tx.RLock() finishedCompact, found := UnsafeReadFinishedCompact(tx) if found { s.revMu.Lock() s.compactMainRev = finishedCompact s.lg.Info( "restored last compact revision", zap.String("meta-bucket-name-key", string(schema.FinishedCompactKeyName)), zap.Int64("restored-compact-revision", s.compactMainRev), ) s.revMu.Unlock() } scheduledCompact, _ := UnsafeReadScheduledCompact(tx) // index keys concurrently as they're loaded in from tx keysGauge.Set(0) rkvc, revc := restoreIntoIndex(s.lg, s.kvindex) for { keys, vals := tx.UnsafeRange(schema.Key, min, max, int64(restoreChunkKeys)) if len(keys) == 0 { break } // rkvc blocks if the total pending keys exceeds the restore // chunk size to keep keys from consuming too much memory. restoreChunk(s.lg, rkvc, keys, vals, keyToLease) if len(keys) < restoreChunkKeys { // partial set implies final set break } // next set begins after where this one ended newMin := BytesToRev(keys[len(keys)-1][:revBytesLen]) newMin.Sub++ min = RevToBytes(newMin, min) } close(rkvc) { s.revMu.Lock() s.currentRev = <-revc // keys in the range [compacted revision -N, compaction] might all be deleted due to compaction. // the correct revision should be set to compaction revision in the case, not the largest revision // we have seen. if s.currentRev < s.compactMainRev { s.currentRev = s.compactMainRev } // If the latest revision was a tombstone revision and etcd just compacted // it, but crashed right before persisting the FinishedCompactRevision, // then it would lead to revision decreasing in bbolt db file. In such // a scenario, we should adjust the current revision using the scheduled // compact revision on bootstrap when etcd gets started again. // // See https://github.com/etcd-io/etcd/issues/17780#issuecomment-2061900231 if s.currentRev < scheduledCompact { s.currentRev = scheduledCompact } s.revMu.Unlock() } if scheduledCompact <= s.compactMainRev { scheduledCompact = 0 } for key, lid := range keyToLease { if s.le == nil { tx.RUnlock() panic("no lessor to attach lease") } err := s.le.Attach(lid, []lease.LeaseItem{{Key: key}}) if err != nil { s.lg.Error( "failed to attach a lease", zap.String("lease-id", fmt.Sprintf("%016x", lid)), zap.Error(err), ) } } tx.RUnlock() s.lg.Info("kvstore restored", zap.Int64("current-rev", s.currentRev)) if scheduledCompact != 0 { if _, err := s.compactLockfree(scheduledCompact); err != nil { s.lg.Warn("compaction encountered error", zap.Int64("scheduled-compact-revision", scheduledCompact), zap.Error(err), ) } else { s.lg.Info( "resume scheduled compaction", zap.Int64("scheduled-compact-revision", scheduledCompact), ) } } return nil } type revKeyValue struct { key []byte kv mvccpb.KeyValue kstr string } func restoreIntoIndex(lg *zap.Logger, idx index) (chan<- revKeyValue, <-chan int64) { rkvc, revc := make(chan revKeyValue, restoreChunkKeys), make(chan int64, 1) go func() { currentRev := int64(1) defer func() { revc <- currentRev }() // restore the tree index from streaming the unordered index. kiCache := make(map[string]*keyIndex, restoreChunkKeys) for rkv := range rkvc { ki, ok := kiCache[rkv.kstr] // purge kiCache if many keys but still missing in the cache if !ok && len(kiCache) >= restoreChunkKeys { i := 10 for k := range kiCache { delete(kiCache, k) if i--; i == 0 { break } } } // cache miss, fetch from tree index if there if !ok { ki = &keyIndex{key: rkv.kv.Key} if idxKey := idx.KeyIndex(ki); idxKey != nil { kiCache[rkv.kstr], ki = idxKey, idxKey ok = true } } rev := BytesToRev(rkv.key) verify.Verify("revision shouldn't be less than the previous revision", func() (bool, map[string]any) { return rev.Main >= currentRev, map[string]any{ "revision": rev.Main, "previous revision": currentRev, } }) currentRev = rev.Main if ok { if isTombstone(rkv.key) { if err := ki.tombstone(lg, rev.Main, rev.Sub); err != nil { lg.Warn("tombstone encountered error", zap.Error(err)) } continue } ki.put(lg, rev.Main, rev.Sub) } else { if isTombstone(rkv.key) { ki.restoreTombstone(lg, rev.Main, rev.Sub) } else { ki.restore(lg, Revision{Main: rkv.kv.CreateRevision}, rev, rkv.kv.Version) } idx.Insert(ki) kiCache[rkv.kstr] = ki } } }() return rkvc, revc } func restoreChunk(lg *zap.Logger, kvc chan<- revKeyValue, keys, vals [][]byte, keyToLease map[string]lease.LeaseID) { for i, key := range keys { rkv := revKeyValue{key: key} if err := rkv.kv.Unmarshal(vals[i]); err != nil { lg.Fatal("failed to unmarshal mvccpb.KeyValue", zap.Error(err)) } rkv.kstr = string(rkv.kv.Key) if isTombstone(key) { delete(keyToLease, rkv.kstr) } else if lid := lease.LeaseID(rkv.kv.Lease); lid != lease.NoLease { keyToLease[rkv.kstr] = lid } else { delete(keyToLease, rkv.kstr) } kvc <- rkv } } func (s *store) Close() error { close(s.stopc) s.fifoSched.Stop() return nil } func (s *store) setupMetricsReporter() { b := s.b reportDbTotalSizeInBytesMu.Lock() reportDbTotalSizeInBytes = func() float64 { return float64(b.Size()) } reportDbTotalSizeInBytesMu.Unlock() reportDbTotalSizeInUseInBytesMu.Lock() reportDbTotalSizeInUseInBytes = func() float64 { return float64(b.SizeInUse()) } reportDbTotalSizeInUseInBytesMu.Unlock() reportDbOpenReadTxNMu.Lock() reportDbOpenReadTxN = func() float64 { return float64(b.OpenReadTxN()) } reportDbOpenReadTxNMu.Unlock() reportCurrentRevMu.Lock() reportCurrentRev = func() float64 { s.revMu.RLock() defer s.revMu.RUnlock() return float64(s.currentRev) } reportCurrentRevMu.Unlock() reportCompactRevMu.Lock() reportCompactRev = func() float64 { s.revMu.RLock() defer s.revMu.RUnlock() return float64(s.compactMainRev) } reportCompactRevMu.Unlock() } func (s *store) HashStorage() HashStorage { return s.hashes } ================================================ FILE: server/storage/mvcc/kvstore_bench_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "testing" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/etcdserver/cindex" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/schema" ) func BenchmarkStorePut(b *testing.B) { be, _ := betesting.NewDefaultTmpBackend(b) s := NewStore(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, be) // arbitrary number of bytes bytesN := 64 keys := createBytesSlice(bytesN, b.N) vals := createBytesSlice(bytesN, b.N) b.ResetTimer() for i := 0; i < b.N; i++ { s.Put(keys[i], vals[i], lease.NoLease) } } func BenchmarkStoreRangeKey1(b *testing.B) { benchmarkStoreRange(b, 1) } func BenchmarkStoreRangeKey100(b *testing.B) { benchmarkStoreRange(b, 100) } func benchmarkStoreRange(b *testing.B, n int) { be, _ := betesting.NewDefaultTmpBackend(b) s := NewStore(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, be) // 64 byte key/val keys, val := createBytesSlice(64, n), createBytesSlice(64, 1) for i := range keys { s.Put(keys[i], val[0], lease.NoLease) } // Force into boltdb tx instead of backend read tx. s.Commit() var begin, end []byte if n == 1 { begin, end = keys[0], nil } else { begin, end = []byte{}, []byte{} } b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { s.Range(b.Context(), begin, end, RangeOptions{}) } } func BenchmarkConsistentIndex(b *testing.B) { be, _ := betesting.NewDefaultTmpBackend(b) ci := cindex.NewConsistentIndex(be) defer betesting.Close(b, be) // This will force the index to be reread from scratch on each call. ci.SetConsistentIndex(0, 0) tx := be.BatchTx() tx.Lock() schema.UnsafeCreateMetaBucket(tx) ci.UnsafeSave(tx) tx.Unlock() b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { ci.ConsistentIndex() } } // BenchmarkStorePutUpdate is same as above, but instead updates single key func BenchmarkStorePutUpdate(b *testing.B) { be, _ := betesting.NewDefaultTmpBackend(b) s := NewStore(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, be) // arbitrary number of bytes keys := createBytesSlice(64, 1) vals := createBytesSlice(1024, 1) b.ResetTimer() for i := 0; i < b.N; i++ { s.Put(keys[0], vals[0], lease.NoLease) } } // BenchmarkStoreTxnPut benchmarks the Put operation // with transaction begin and end, where transaction involves // some synchronization operations, such as mutex locking. func BenchmarkStoreTxnPut(b *testing.B) { be, _ := betesting.NewDefaultTmpBackend(b) s := NewStore(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, be) // arbitrary number of bytes bytesN := 64 keys := createBytesSlice(bytesN, b.N) vals := createBytesSlice(bytesN, b.N) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { txn := s.Write(traceutil.TODO()) txn.Put(keys[i], vals[i], lease.NoLease) txn.End() } } // benchmarkStoreRestore benchmarks the restore operation func benchmarkStoreRestore(revsPerKey int, b *testing.B) { be, _ := betesting.NewDefaultTmpBackend(b) s := NewStore(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) // use closure to capture 's' to pick up the reassignment defer func() { cleanup(s, be) }() // arbitrary number of bytes bytesN := 64 keys := createBytesSlice(bytesN, b.N) vals := createBytesSlice(bytesN, b.N) for i := 0; i < b.N; i++ { for j := 0; j < revsPerKey; j++ { txn := s.Write(traceutil.TODO()) txn.Put(keys[i], vals[i], lease.NoLease) txn.End() } } require.NoError(b, s.Close()) b.ReportAllocs() b.ResetTimer() s = NewStore(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) } func BenchmarkStoreRestoreRevs1(b *testing.B) { benchmarkStoreRestore(1, b) } func BenchmarkStoreRestoreRevs10(b *testing.B) { benchmarkStoreRestore(10, b) } func BenchmarkStoreRestoreRevs20(b *testing.B) { benchmarkStoreRestore(20, b) } ================================================ FILE: server/storage/mvcc/kvstore_compaction.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "encoding/binary" "fmt" "time" humanize "github.com/dustin/go-humanize" "go.uber.org/zap" "go.etcd.io/etcd/server/v3/storage/schema" ) func (s *store) scheduleCompaction(compactMainRev, prevCompactRev int64) (KeyValueHash, error) { totalStart := time.Now() keep := s.kvindex.Compact(compactMainRev) indexCompactionPauseMs.Observe(float64(time.Since(totalStart) / time.Millisecond)) totalStart = time.Now() defer func() { dbCompactionTotalMs.Observe(float64(time.Since(totalStart) / time.Millisecond)) }() keyCompactions := 0 defer func() { dbCompactionKeysCounter.Add(float64(keyCompactions)) }() defer func() { dbCompactionLast.Set(float64(time.Now().Unix())) }() end := make([]byte, 8) binary.BigEndian.PutUint64(end, uint64(compactMainRev+1)) batchNum := s.cfg.CompactionBatchLimit h := newKVHasher(prevCompactRev, compactMainRev, keep) last := make([]byte, 8+1+8) for { var rev Revision start := time.Now() tx := s.b.BatchTx() tx.LockOutsideApply() // gofail: var compactAfterAcquiredBatchTxLock struct{} keys, values := tx.UnsafeRange(schema.Key, last, end, int64(batchNum)) for i := range keys { rev = BytesToRev(keys[i]) if _, ok := keep[rev]; !ok { tx.UnsafeDelete(schema.Key, keys[i]) keyCompactions++ } h.WriteKeyValue(keys[i], values[i]) } if len(keys) < batchNum { // gofail: var compactBeforeSetFinishedCompact struct{} UnsafeSetFinishedCompact(tx, compactMainRev) tx.Unlock() dbCompactionPauseMs.Observe(float64(time.Since(start) / time.Millisecond)) // gofail: var compactAfterSetFinishedCompact struct{} hash := h.Hash() size, sizeInUse := s.b.Size(), s.b.SizeInUse() s.lg.Info( "finished scheduled compaction", zap.Int64("compact-revision", compactMainRev), zap.Duration("took", time.Since(totalStart)), zap.Int("number-of-keys-compacted", keyCompactions), zap.Uint32("hash", hash.Hash), zap.Int64("current-db-size-bytes", size), zap.String("current-db-size", humanize.Bytes(uint64(size))), zap.Int64("current-db-size-in-use-bytes", sizeInUse), zap.String("current-db-size-in-use", humanize.Bytes(uint64(sizeInUse))), ) return hash, nil } tx.Unlock() // update last last = RevToBytes(Revision{Main: rev.Main, Sub: rev.Sub + 1}, last) // Immediately commit the compaction deletes instead of letting them accumulate in the write buffer // gofail: var compactBeforeCommitBatch struct{} s.b.ForceCommit() // gofail: var compactAfterCommitBatch struct{} dbCompactionPauseMs.Observe(float64(time.Since(start) / time.Millisecond)) select { case <-time.After(s.cfg.CompactionSleepInterval): case <-s.stopc: return KeyValueHash{}, fmt.Errorf("interrupted due to stop signal") } } } ================================================ FILE: server/storage/mvcc/kvstore_compaction_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "reflect" "testing" "time" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/schema" ) func TestScheduleCompaction(t *testing.T) { revs := []Revision{{Main: 1}, {Main: 2}, {Main: 3}} tests := []struct { rev int64 keep map[Revision]struct{} wrevs []Revision }{ // compact at 1 and discard all history { 1, nil, revs[1:], }, // compact at 3 and discard all history { 3, nil, nil, }, // compact at 1 and keeps history one step earlier { 1, map[Revision]struct{}{ {Main: 1}: {}, }, revs, }, // compact at 1 and keeps history two steps earlier { 3, map[Revision]struct{}{ {Main: 2}: {}, {Main: 3}: {}, }, revs[1:], }, } for i, tt := range tests { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) fi := newFakeIndex() fi.indexCompactRespc <- tt.keep s.kvindex = fi tx := s.b.BatchTx() tx.Lock() for _, rev := range revs { ibytes := NewRevBytes() ibytes = RevToBytes(rev, ibytes) tx.UnsafePut(schema.Key, ibytes, []byte("bar")) } tx.Unlock() _, err := s.scheduleCompaction(tt.rev, 0) if err != nil { t.Error(err) } tx.Lock() for _, rev := range tt.wrevs { ibytes := NewRevBytes() ibytes = RevToBytes(rev, ibytes) keys, _ := tx.UnsafeRange(schema.Key, ibytes, nil, 0) if len(keys) != 1 { t.Errorf("#%d: range on %v = %d, want 1", i, rev, len(keys)) } } vals, _ := UnsafeReadFinishedCompact(tx) if !reflect.DeepEqual(vals, tt.rev) { t.Errorf("#%d: finished compact equal %+v, want %+v", i, vals, tt.rev) } tx.Unlock() cleanup(s, b) } } func TestCompactAllAndRestore(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s0 := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer b.Close() s0.Put([]byte("foo"), []byte("bar"), lease.NoLease) s0.Put([]byte("foo"), []byte("bar1"), lease.NoLease) s0.Put([]byte("foo"), []byte("bar2"), lease.NoLease) s0.DeleteRange([]byte("foo"), nil) rev := s0.Rev() // compact all keys done, err := s0.Compact(traceutil.TODO(), rev) if err != nil { t.Fatal(err) } select { case <-done: case <-time.After(10 * time.Second): t.Fatal("timeout waiting for compaction to finish") } err = s0.Close() if err != nil { t.Fatal(err) } s1 := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) if s1.Rev() != rev { t.Errorf("rev = %v, want %v", s1.Rev(), rev) } _, err = s1.Range(t.Context(), []byte("foo"), nil, RangeOptions{}) if err != nil { t.Errorf("unexpect range error %v", err) } err = s1.Close() if err != nil { t.Fatal(err) } } ================================================ FILE: server/storage/mvcc/kvstore_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "bytes" "crypto/rand" "encoding/binary" "errors" "fmt" "math" mrand "math/rand" "reflect" "sort" "strconv" "sync" "testing" "time" "go.uber.org/zap" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/pkg/v3/schedule" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/schema" ) func TestStoreRev(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer s.Close() for i := 1; i <= 3; i++ { s.Put([]byte("foo"), []byte("bar"), lease.NoLease) if r := s.Rev(); r != int64(i+1) { t.Errorf("#%d: rev = %d, want %d", i, r, i+1) } } } func TestStorePut(t *testing.T) { lg := zaptest.NewLogger(t) kv := mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 1, ModRevision: 2, Version: 1, } kvb, err := kv.Marshal() if err != nil { t.Fatal(err) } tests := []struct { rev Revision r indexGetResp rr *rangeResp wrev Revision wkey []byte wkv mvccpb.KeyValue wputrev Revision }{ { Revision{Main: 1}, indexGetResp{Revision{}, Revision{}, 0, ErrRevisionNotFound}, nil, Revision{Main: 2}, newTestRevBytes(Revision{Main: 2}), mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1, Lease: 1, }, Revision{Main: 2}, }, { Revision{Main: 1, Sub: 1}, indexGetResp{Revision{Main: 2}, Revision{Main: 2}, 1, nil}, &rangeResp{[][]byte{newTestRevBytes(Revision{Main: 2, Sub: 1})}, [][]byte{kvb}}, Revision{Main: 2}, newTestRevBytes(Revision{Main: 2}), mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 2, Lease: 2, }, Revision{Main: 2}, }, { Revision{Main: 2}, indexGetResp{Revision{Main: 2, Sub: 1}, Revision{Main: 2}, 2, nil}, &rangeResp{[][]byte{newTestRevBytes(Revision{Main: 2, Sub: 1})}, [][]byte{kvb}}, Revision{Main: 3}, newTestRevBytes(Revision{Main: 3}), mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 3, Version: 3, Lease: 3, }, Revision{Main: 3}, }, } for i, tt := range tests { s := newFakeStore(lg) b := s.b.(*fakeBackend) fi := s.kvindex.(*fakeIndex) s.currentRev = tt.rev.Main fi.indexGetRespc <- tt.r if tt.rr != nil { b.tx.rangeRespc <- *tt.rr } s.Put([]byte("foo"), []byte("bar"), lease.LeaseID(i+1)) data, err := tt.wkv.Marshal() if err != nil { t.Errorf("#%d: marshal err = %v, want nil", i, err) } wact := []testutil.Action{ {Name: "seqput", Params: []any{schema.Key, tt.wkey, data}}, } if tt.rr != nil { wact = []testutil.Action{ {Name: "seqput", Params: []any{schema.Key, tt.wkey, data}}, } } if g := b.tx.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: tx action = %+v, want %+v", i, g, wact) } wact = []testutil.Action{ {Name: "get", Params: []any{[]byte("foo"), tt.wputrev.Main}}, {Name: "put", Params: []any{[]byte("foo"), tt.wputrev}}, } if g := fi.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: index action = %+v, want %+v", i, g, wact) } if s.currentRev != tt.wrev.Main { t.Errorf("#%d: rev = %+v, want %+v", i, s.currentRev, tt.wrev) } s.Close() } } func TestStoreRange(t *testing.T) { lg := zaptest.NewLogger(t) key := newTestRevBytes(Revision{Main: 2}) kv := mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 1, ModRevision: 2, Version: 1, } kvb, err := kv.Marshal() if err != nil { t.Fatal(err) } wrev := int64(2) tests := []struct { idxr indexRangeResp r rangeResp }{ { indexRangeResp{[][]byte{[]byte("foo")}, []Revision{{Main: 2}}}, rangeResp{[][]byte{key}, [][]byte{kvb}}, }, { indexRangeResp{[][]byte{[]byte("foo"), []byte("foo1")}, []Revision{{Main: 2}, {Main: 3}}}, rangeResp{[][]byte{key}, [][]byte{kvb}}, }, } ro := RangeOptions{Limit: 1, Rev: 0, Count: false} for i, tt := range tests { s := newFakeStore(lg) b := s.b.(*fakeBackend) fi := s.kvindex.(*fakeIndex) s.currentRev = 2 b.tx.rangeRespc <- tt.r fi.indexRangeRespc <- tt.idxr ret, err := s.Range(t.Context(), []byte("foo"), []byte("goo"), ro) if err != nil { t.Errorf("#%d: err = %v, want nil", i, err) } if w := []mvccpb.KeyValue{kv}; !reflect.DeepEqual(ret.KVs, w) { t.Errorf("#%d: kvs = %+v, want %+v", i, ret.KVs, w) } if ret.Rev != wrev { t.Errorf("#%d: rev = %d, want %d", i, ret.Rev, wrev) } wstart := NewRevBytes() wstart = RevToBytes(tt.idxr.revs[0], wstart) wact := []testutil.Action{ {Name: "range", Params: []any{schema.Key, wstart, []byte(nil), int64(0)}}, } if g := b.tx.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: tx action = %+v, want %+v", i, g, wact) } wact = []testutil.Action{ {Name: "range", Params: []any{[]byte("foo"), []byte("goo"), wrev}}, } if g := fi.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: index action = %+v, want %+v", i, g, wact) } if s.currentRev != 2 { t.Errorf("#%d: current rev = %+v, want %+v", i, s.currentRev, 2) } s.Close() } } func TestStoreDeleteRange(t *testing.T) { lg := zaptest.NewLogger(t) key := newTestRevBytes(Revision{Main: 2}) kv := mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 1, ModRevision: 2, Version: 1, } kvb, err := kv.Marshal() if err != nil { t.Fatal(err) } tests := []struct { rev Revision r indexRangeResp rr rangeResp wkey []byte wrev Revision wrrev int64 wdelrev Revision }{ { Revision{Main: 2}, indexRangeResp{[][]byte{[]byte("foo")}, []Revision{{Main: 2}}}, rangeResp{[][]byte{key}, [][]byte{kvb}}, newTestBucketKeyBytes(newBucketKey(3, 0, true)), Revision{Main: 3}, 2, Revision{Main: 3}, }, } for i, tt := range tests { s := newFakeStore(lg) b := s.b.(*fakeBackend) fi := s.kvindex.(*fakeIndex) s.currentRev = tt.rev.Main fi.indexRangeRespc <- tt.r b.tx.rangeRespc <- tt.rr n, _ := s.DeleteRange([]byte("foo"), []byte("goo")) if n != 1 { t.Errorf("#%d: n = %d, want 1", i, n) } data, err := (&mvccpb.KeyValue{ Key: []byte("foo"), }).Marshal() if err != nil { t.Errorf("#%d: marshal err = %v, want nil", i, err) } wact := []testutil.Action{ {Name: "seqput", Params: []any{schema.Key, tt.wkey, data}}, } if g := b.tx.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: tx action = %+v, want %+v", i, g, wact) } wact = []testutil.Action{ {Name: "range", Params: []any{[]byte("foo"), []byte("goo"), tt.wrrev}}, {Name: "tombstone", Params: []any{[]byte("foo"), tt.wdelrev}}, } if g := fi.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("#%d: index action = %+v, want %+v", i, g, wact) } if s.currentRev != tt.wrev.Main { t.Errorf("#%d: rev = %+v, want %+v", i, s.currentRev, tt.wrev) } s.Close() } } func TestStoreCompact(t *testing.T) { lg := zaptest.NewLogger(t) s := newFakeStore(lg) defer s.Close() b := s.b.(*fakeBackend) fi := s.kvindex.(*fakeIndex) s.currentRev = 3 fi.indexCompactRespc <- map[Revision]struct{}{{Main: 1}: {}} key1 := newTestRevBytes(Revision{Main: 1}) key2 := newTestRevBytes(Revision{Main: 2}) b.tx.rangeRespc <- rangeResp{[][]byte{}, [][]byte{}} b.tx.rangeRespc <- rangeResp{[][]byte{}, [][]byte{}} b.tx.rangeRespc <- rangeResp{[][]byte{key1, key2}, [][]byte{[]byte("alice"), []byte("bob")}} s.Compact(traceutil.TODO(), 3) s.fifoSched.WaitFinish(1) if s.compactMainRev != 3 { t.Errorf("compact main rev = %d, want 3", s.compactMainRev) } end := make([]byte, 8) binary.BigEndian.PutUint64(end, uint64(4)) wact := []testutil.Action{ {Name: "range", Params: []any{schema.Meta, schema.ScheduledCompactKeyName, []uint8(nil), int64(0)}}, {Name: "range", Params: []any{schema.Meta, schema.FinishedCompactKeyName, []uint8(nil), int64(0)}}, {Name: "put", Params: []any{schema.Meta, schema.ScheduledCompactKeyName, newTestRevBytes(Revision{Main: 3})}}, {Name: "range", Params: []any{schema.Key, make([]byte, 17), end, int64(10000)}}, {Name: "delete", Params: []any{schema.Key, key2}}, {Name: "put", Params: []any{schema.Meta, schema.FinishedCompactKeyName, newTestRevBytes(Revision{Main: 3})}}, } if g := b.tx.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("tx actions = %+v, want %+v", g, wact) } wact = []testutil.Action{ {Name: "compact", Params: []any{int64(3)}}, } if g := fi.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("index action = %+v, want %+v", g, wact) } } func TestStoreRestore(t *testing.T) { lg := zaptest.NewLogger(t) s := newFakeStore(lg) b := s.b.(*fakeBackend) fi := s.kvindex.(*fakeIndex) defer s.Close() putkey := newTestRevBytes(Revision{Main: 3}) putkv := mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 4, ModRevision: 4, Version: 1, } putkvb, err := putkv.Marshal() if err != nil { t.Fatal(err) } delkey := newTestBucketKeyBytes(newBucketKey(5, 0, true)) delkv := mvccpb.KeyValue{ Key: []byte("foo"), } delkvb, err := delkv.Marshal() if err != nil { t.Fatal(err) } b.tx.rangeRespc <- rangeResp{[][]byte{schema.FinishedCompactKeyName}, [][]byte{newTestRevBytes(Revision{Main: 3})}} b.tx.rangeRespc <- rangeResp{[][]byte{schema.ScheduledCompactKeyName}, [][]byte{newTestRevBytes(Revision{Main: 3})}} b.tx.rangeRespc <- rangeResp{[][]byte{putkey, delkey}, [][]byte{putkvb, delkvb}} b.tx.rangeRespc <- rangeResp{nil, nil} s.restore() if s.compactMainRev != 3 { t.Errorf("compact rev = %d, want 3", s.compactMainRev) } if s.currentRev != 5 { t.Errorf("current rev = %v, want 5", s.currentRev) } wact := []testutil.Action{ {Name: "range", Params: []any{schema.Meta, schema.FinishedCompactKeyName, []byte(nil), int64(0)}}, {Name: "range", Params: []any{schema.Meta, schema.ScheduledCompactKeyName, []byte(nil), int64(0)}}, {Name: "range", Params: []any{schema.Key, newTestRevBytes(Revision{Main: 1}), newTestRevBytes(Revision{Main: math.MaxInt64, Sub: math.MaxInt64}), int64(restoreChunkKeys)}}, } if g := b.tx.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("tx actions = %+v, want %+v", g, wact) } gens := []generation{ {created: Revision{Main: 4}, ver: 2, revs: []Revision{{Main: 3}, {Main: 5}}}, {created: Revision{Main: 0}, ver: 0, revs: nil}, } ki := &keyIndex{key: []byte("foo"), modified: Revision{Main: 5}, generations: gens} wact = []testutil.Action{ {Name: "keyIndex", Params: []any{ki}}, {Name: "insert", Params: []any{ki}}, } if g := fi.Action(); !reflect.DeepEqual(g, wact) { t.Errorf("index action = %+v, want %+v", g, wact) } } func TestRestoreDelete(t *testing.T) { oldChunk := restoreChunkKeys restoreChunkKeys = mrand.Intn(3) + 2 defer func() { restoreChunkKeys = oldChunk }() b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer b.Close() keys := make(map[string]struct{}) for i := 0; i < 20; i++ { ks := fmt.Sprintf("foo-%d", i) k := []byte(ks) s.Put(k, []byte("bar"), lease.NoLease) keys[ks] = struct{}{} switch mrand.Intn(3) { case 0: // put random key from past via random range on map ks = fmt.Sprintf("foo-%d", mrand.Intn(i+1)) s.Put([]byte(ks), []byte("baz"), lease.NoLease) keys[ks] = struct{}{} case 1: // delete random key via random range on map for k := range keys { s.DeleteRange([]byte(k), nil) delete(keys, k) break } } } s.Close() s = NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer s.Close() for i := 0; i < 20; i++ { ks := fmt.Sprintf("foo-%d", i) r, err := s.Range(t.Context(), []byte(ks), nil, RangeOptions{}) if err != nil { t.Fatal(err) } if _, ok := keys[ks]; ok { if len(r.KVs) == 0 { t.Errorf("#%d: expected %q, got deleted", i, ks) } } else if len(r.KVs) != 0 { t.Errorf("#%d: expected deleted, got %q", i, ks) } } } func TestRestoreContinueUnfinishedCompaction(t *testing.T) { tests := []string{"recreate", "restore"} for _, test := range tests { test := test t.Run(test, func(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s0 := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) s0.Put([]byte("foo"), []byte("bar"), lease.NoLease) s0.Put([]byte("foo"), []byte("bar1"), lease.NoLease) s0.Put([]byte("foo"), []byte("bar2"), lease.NoLease) // write scheduled compaction, but not do compaction tx := s0.b.BatchTx() tx.Lock() UnsafeSetScheduledCompact(tx, 2) tx.Unlock() var s *store switch test { case "recreate": s0.Close() s = NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) case "restore": // TODO(fuweid): store doesn't support to restore // from a closed status because there is no lock // for `Close` or action to mark it is closed. s0.Restore(b) s = s0 } defer cleanup(s, b) // wait for scheduled compaction to be finished time.Sleep(100 * time.Millisecond) if _, err := s.Range(t.Context(), []byte("foo"), nil, RangeOptions{Rev: 1}); !errors.Is(err, ErrCompacted) { t.Errorf("range on compacted rev error = %v, want %v", err, ErrCompacted) } // check the key in backend is deleted revbytes := NewRevBytes() revbytes = BucketKeyToBytes(newBucketKey(1, 0, false), revbytes) // The disk compaction is done asynchronously and requires more time on slow disk. // try 5 times for CI with slow IO. for i := 0; i < 5; i++ { tx := s.b.BatchTx() tx.Lock() ks, _ := tx.UnsafeRange(schema.Key, revbytes, nil, 0) tx.Unlock() if len(ks) != 0 { time.Sleep(100 * time.Millisecond) continue } return } t.Errorf("key for rev %+v still exists, want deleted", BytesToBucketKey(revbytes)) }) } } type hashKVResult struct { hash uint32 compactRev int64 } // TestHashKVWhenCompacting ensures that HashKV returns correct hash when compacting. func TestHashKVWhenCompacting(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) rev := 10000 for i := 2; i <= rev; i++ { s.Put([]byte("foo"), []byte(fmt.Sprintf("bar%d", i)), lease.NoLease) } hashCompactc := make(chan hashKVResult, 1) var wg sync.WaitGroup donec := make(chan struct{}) stopc := make(chan struct{}) // Call HashByRev(10000) in multiple goroutines until donec is closed for i := 0; i < 10; i++ { wg.Add(1) go func() { defer wg.Done() for { hash, _, err := s.HashStorage().HashByRev(int64(rev)) if err != nil { t.Error(err) } select { case <-stopc: return case <-donec: return case hashCompactc <- hashKVResult{hash.Hash, hash.CompactRevision}: } } }() } // Check computed hashes by HashByRev are correct in a goroutine, until donec is closed wg.Add(1) go func() { defer wg.Done() revHash := make(map[int64]uint32) for { select { case r := <-hashCompactc: if revHash[r.compactRev] == 0 { revHash[r.compactRev] = r.hash } if r.hash != revHash[r.compactRev] { t.Errorf("Hashes differ (current %v) != (saved %v)", r.hash, revHash[r.compactRev]) } case <-stopc: return case <-donec: return } } }() // Compact the store in a goroutine, using RevisionTombstone 9900 to 10000 and close donec when finished wg.Add(1) go func() { defer func() { close(donec) wg.Done() }() for i := 100; i >= 0; i-- { select { case <-stopc: return default: } _, err := s.Compact(traceutil.TODO(), int64(rev-i)) if err != nil { t.Error(err) } // Wait for the compaction job to finish s.fifoSched.WaitFinish(1) // Leave time for calls to HashByRev to take place after each compaction time.Sleep(10 * time.Millisecond) } }() select { case <-donec: case <-time.After(20 * time.Second): close(stopc) wg.Wait() testutil.FatalStack(t, "timeout") } close(stopc) wg.Wait() } // TestHashKVWithCompactedAndFutureRevisions ensures that HashKV returns a correct hash when called // with a past RevisionTombstone (lower than compacted), a future RevisionTombstone, and the exact compacted RevisionTombstone func TestHashKVWithCompactedAndFutureRevisions(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) rev := 10000 compactRev := rev / 2 for i := 2; i <= rev; i++ { s.Put([]byte("foo"), []byte(fmt.Sprintf("bar%d", i)), lease.NoLease) } if _, err := s.Compact(traceutil.TODO(), int64(compactRev)); err != nil { t.Fatal(err) } _, _, errFutureRev := s.HashStorage().HashByRev(int64(rev + 1)) if !errors.Is(errFutureRev, ErrFutureRev) { t.Error(errFutureRev) } _, _, errPastRev := s.HashStorage().HashByRev(int64(compactRev - 1)) if !errors.Is(errPastRev, ErrCompacted) { t.Error(errPastRev) } _, _, errCompactRev := s.HashStorage().HashByRev(int64(compactRev)) if errCompactRev != nil { t.Error(errCompactRev) } } // TestHashKVZeroRevision ensures that "HashByRev(0)" computes // correct hash value with latest RevisionTombstone. func TestHashKVZeroRevision(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) rev := 10000 for i := 2; i <= rev; i++ { s.Put([]byte("foo"), []byte(fmt.Sprintf("bar%d", i)), lease.NoLease) } if _, err := s.Compact(traceutil.TODO(), int64(rev/2)); err != nil { t.Fatal(err) } hash1, _, err := s.HashStorage().HashByRev(int64(rev)) if err != nil { t.Fatal(err) } var hash2 KeyValueHash hash2, _, err = s.HashStorage().HashByRev(0) if err != nil { t.Fatal(err) } if hash1 != hash2 { t.Errorf("hash %d (rev %d) != hash %d (rev 0)", hash1, rev, hash2) } } func TestTxnPut(t *testing.T) { // assign arbitrary size bytesN := 30 sliceN := 100 keys := createBytesSlice(bytesN, sliceN) vals := createBytesSlice(bytesN, sliceN) b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) for i := 0; i < sliceN; i++ { txn := s.Write(traceutil.TODO()) base := int64(i + 2) if rev := txn.Put(keys[i], vals[i], lease.NoLease); rev != base { t.Errorf("#%d: rev = %d, want %d", i, rev, base) } txn.End() } } // TestConcurrentReadNotBlockingWrite ensures Read does not blocking Write after its creation func TestConcurrentReadNotBlockingWrite(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) // write something to read later s.Put([]byte("foo"), []byte("bar"), lease.NoLease) // readTx simulates a long read request readTx1 := s.Read(ConcurrentReadTxMode, traceutil.TODO()) // write should not be blocked by reads done := make(chan struct{}, 1) go func() { s.Put([]byte("foo"), []byte("newBar"), lease.NoLease) // this is a write Txn done <- struct{}{} }() select { case <-done: case <-time.After(1 * time.Second): t.Fatalf("write should not be blocked by read") } // readTx2 simulates a short read request readTx2 := s.Read(ConcurrentReadTxMode, traceutil.TODO()) ro := RangeOptions{Limit: 1, Rev: 0, Count: false} ret, err := readTx2.Range(t.Context(), []byte("foo"), nil, ro) if err != nil { t.Fatalf("failed to range: %v", err) } // readTx2 should see the result of new write w := mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("newBar"), CreateRevision: 2, ModRevision: 3, Version: 2, } if !reflect.DeepEqual(ret.KVs[0], w) { t.Fatalf("range result = %+v, want = %+v", ret.KVs[0], w) } readTx2.End() ret, err = readTx1.Range(t.Context(), []byte("foo"), nil, ro) if err != nil { t.Fatalf("failed to range: %v", err) } // readTx1 should not see the result of new write w = mvccpb.KeyValue{ Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1, } if !reflect.DeepEqual(ret.KVs[0], w) { t.Fatalf("range result = %+v, want = %+v", ret.KVs[0], w) } readTx1.End() } // TestConcurrentReadTxAndWrite creates random concurrent Reads and Writes, and ensures Reads always see latest Writes func TestConcurrentReadTxAndWrite(t *testing.T) { var ( numOfReads = 100 numOfWrites = 100 maxNumOfPutsPerWrite = 10 committedKVs kvs // committedKVs records the key-value pairs written by the finished Write Txns mu sync.Mutex // mu protects committedKVs ) b, _ := betesting.NewDefaultTmpBackend(t) s := NewStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) var wg sync.WaitGroup wg.Add(numOfWrites) for i := 0; i < numOfWrites; i++ { go func() { defer wg.Done() time.Sleep(time.Duration(mrand.Intn(100)) * time.Millisecond) // random starting time tx := s.Write(traceutil.TODO()) numOfPuts := mrand.Intn(maxNumOfPutsPerWrite) + 1 var pendingKvs kvs for j := 0; j < numOfPuts; j++ { k := []byte(strconv.Itoa(mrand.Int())) v := []byte(strconv.Itoa(mrand.Int())) tx.Put(k, v, lease.NoLease) pendingKvs = append(pendingKvs, kv{k, v}) } // reads should not see above Puts until write is finished mu.Lock() committedKVs = merge(committedKVs, pendingKvs) // update shared data structure tx.End() mu.Unlock() }() } wg.Add(numOfReads) for i := 0; i < numOfReads; i++ { go func() { defer wg.Done() time.Sleep(time.Duration(mrand.Intn(100)) * time.Millisecond) // random starting time mu.Lock() wKVs := make(kvs, len(committedKVs)) copy(wKVs, committedKVs) tx := s.Read(ConcurrentReadTxMode, traceutil.TODO()) mu.Unlock() // get all keys in backend store, and compare with wKVs ret, err := tx.Range(t.Context(), []byte("\x00000000"), []byte("\xffffffff"), RangeOptions{}) tx.End() if err != nil { t.Errorf("failed to range keys: %v", err) return } if len(wKVs) == 0 && len(ret.KVs) == 0 { // no committed KVs yet return } var result kvs for _, keyValue := range ret.KVs { result = append(result, kv{keyValue.Key, keyValue.Value}) } if !reflect.DeepEqual(wKVs, result) { t.Errorf("unexpected range result") // too many key value pairs, skip printing them } }() } // wait until goroutines finish or timeout doneC := make(chan struct{}) go func() { wg.Wait() close(doneC) }() select { case <-doneC: case <-time.After(5 * time.Minute): testutil.FatalStack(t, "timeout") } } type kv struct { key []byte val []byte } type kvs []kv func (kvs kvs) Len() int { return len(kvs) } func (kvs kvs) Less(i, j int) bool { return bytes.Compare(kvs[i].key, kvs[j].key) < 0 } func (kvs kvs) Swap(i, j int) { kvs[i], kvs[j] = kvs[j], kvs[i] } func merge(dst, src kvs) kvs { dst = append(dst, src...) sort.Stable(dst) // remove duplicates, using only the newest value // ref: tx_buffer.go widx := 0 for ridx := 1; ridx < len(dst); ridx++ { if !bytes.Equal(dst[widx].key, dst[ridx].key) { widx++ } dst[widx] = dst[ridx] } return dst[:widx+1] } // TODO: test attach key to lessor func newTestRevBytes(rev Revision) []byte { bytes := NewRevBytes() return RevToBytes(rev, bytes) } func newTestBucketKeyBytes(rev BucketKey) []byte { bytes := NewRevBytes() return BucketKeyToBytes(rev, bytes) } func newFakeStore(lg *zap.Logger) *store { b := &fakeBackend{&fakeBatchTx{ Recorder: &testutil.RecorderBuffered{}, rangeRespc: make(chan rangeResp, 5), }} s := &store{ cfg: StoreConfig{ CompactionBatchLimit: 10000, CompactionSleepInterval: defaultCompactionSleepInterval, }, b: b, le: &lease.FakeLessor{}, kvindex: newFakeIndex(), currentRev: 0, compactMainRev: -1, fifoSched: schedule.NewFIFOScheduler(lg), stopc: make(chan struct{}), lg: lg, } s.ReadView, s.WriteView = &readView{s}, &writeView{s} s.hashes = NewHashStorage(lg, s) return s } func newFakeIndex() *fakeIndex { return &fakeIndex{ Recorder: &testutil.RecorderBuffered{}, indexGetRespc: make(chan indexGetResp, 1), indexRangeRespc: make(chan indexRangeResp, 1), indexRangeEventsRespc: make(chan indexRangeEventsResp, 1), indexCompactRespc: make(chan map[Revision]struct{}, 1), } } type rangeResp struct { keys [][]byte vals [][]byte } type fakeBatchTx struct { testutil.Recorder rangeRespc chan rangeResp } func (b *fakeBatchTx) LockInsideApply() {} func (b *fakeBatchTx) LockOutsideApply() {} func (b *fakeBatchTx) Lock() {} func (b *fakeBatchTx) Unlock() {} func (b *fakeBatchTx) RLock() {} func (b *fakeBatchTx) RUnlock() {} func (b *fakeBatchTx) UnsafeCreateBucket(bucket backend.Bucket) {} func (b *fakeBatchTx) UnsafeDeleteBucket(bucket backend.Bucket) {} func (b *fakeBatchTx) UnsafePut(bucket backend.Bucket, key []byte, value []byte) { b.Recorder.Record(testutil.Action{Name: "put", Params: []any{bucket, key, value}}) } func (b *fakeBatchTx) UnsafeSeqPut(bucket backend.Bucket, key []byte, value []byte) { b.Recorder.Record(testutil.Action{Name: "seqput", Params: []any{bucket, key, value}}) } func (b *fakeBatchTx) UnsafeRange(bucket backend.Bucket, key, endKey []byte, limit int64) (keys [][]byte, vals [][]byte) { b.Recorder.Record(testutil.Action{Name: "range", Params: []any{bucket, key, endKey, limit}}) r := <-b.rangeRespc return r.keys, r.vals } func (b *fakeBatchTx) UnsafeDelete(bucket backend.Bucket, key []byte) { b.Recorder.Record(testutil.Action{Name: "delete", Params: []any{bucket, key}}) } func (b *fakeBatchTx) UnsafeForEach(bucket backend.Bucket, visitor func(k, v []byte) error) error { return nil } func (b *fakeBatchTx) Commit() {} func (b *fakeBatchTx) CommitAndStop() {} type fakeBackend struct { tx *fakeBatchTx } func (b *fakeBackend) BatchTx() backend.BatchTx { return b.tx } func (b *fakeBackend) ReadTx() backend.ReadTx { return b.tx } func (b *fakeBackend) ConcurrentReadTx() backend.ReadTx { return b.tx } func (b *fakeBackend) Hash(func(bucketName, keyName []byte) bool) (uint32, error) { return 0, nil } func (b *fakeBackend) Size() int64 { return 0 } func (b *fakeBackend) SizeInUse() int64 { return 0 } func (b *fakeBackend) OpenReadTxN() int64 { return 0 } func (b *fakeBackend) Snapshot() backend.Snapshot { return nil } func (b *fakeBackend) ForceCommit() {} func (b *fakeBackend) Defrag() error { return nil } func (b *fakeBackend) Close() error { return nil } func (b *fakeBackend) SetTxPostLockInsideApplyHook(func()) {} type indexGetResp struct { rev Revision created Revision ver int64 err error } type indexRangeResp struct { keys [][]byte revs []Revision } type indexRangeEventsResp struct { revs []Revision } type fakeIndex struct { testutil.Recorder indexGetRespc chan indexGetResp indexRangeRespc chan indexRangeResp indexRangeEventsRespc chan indexRangeEventsResp indexCompactRespc chan map[Revision]struct{} } func (i *fakeIndex) Revisions(key, end []byte, atRev int64, limit int) ([]Revision, int) { _, rev := i.Range(key, end, atRev) if len(rev) >= limit { rev = rev[:limit] } return rev, len(rev) } func (i *fakeIndex) CountRevisions(key, end []byte, atRev int64) int { _, rev := i.Range(key, end, atRev) return len(rev) } func (i *fakeIndex) Get(key []byte, atRev int64) (rev, created Revision, ver int64, err error) { i.Recorder.Record(testutil.Action{Name: "get", Params: []any{key, atRev}}) r := <-i.indexGetRespc return r.rev, r.created, r.ver, r.err } func (i *fakeIndex) Range(key, end []byte, atRev int64) ([][]byte, []Revision) { i.Recorder.Record(testutil.Action{Name: "range", Params: []any{key, end, atRev}}) r := <-i.indexRangeRespc return r.keys, r.revs } func (i *fakeIndex) Put(key []byte, rev Revision) { i.Recorder.Record(testutil.Action{Name: "put", Params: []any{key, rev}}) } func (i *fakeIndex) Tombstone(key []byte, rev Revision) error { i.Recorder.Record(testutil.Action{Name: "tombstone", Params: []any{key, rev}}) return nil } func (i *fakeIndex) RangeSince(key, end []byte, rev int64) []Revision { i.Recorder.Record(testutil.Action{Name: "rangeEvents", Params: []any{key, end, rev}}) r := <-i.indexRangeEventsRespc return r.revs } func (i *fakeIndex) Compact(rev int64) map[Revision]struct{} { i.Recorder.Record(testutil.Action{Name: "compact", Params: []any{rev}}) return <-i.indexCompactRespc } func (i *fakeIndex) Keep(rev int64) map[Revision]struct{} { i.Recorder.Record(testutil.Action{Name: "keep", Params: []any{rev}}) return <-i.indexCompactRespc } func (i *fakeIndex) Equal(b index) bool { return false } func (i *fakeIndex) Insert(ki *keyIndex) { i.Recorder.Record(testutil.Action{Name: "insert", Params: []any{ki}}) } func (i *fakeIndex) KeyIndex(ki *keyIndex) *keyIndex { i.Recorder.Record(testutil.Action{Name: "keyIndex", Params: []any{ki}}) return nil } func createBytesSlice(bytesN, sliceN int) [][]byte { var rs [][]byte for len(rs) != sliceN { v := make([]byte, bytesN) if _, err := rand.Read(v); err != nil { panic(err) } rs = append(rs, v) } return rs } ================================================ FILE: server/storage/mvcc/kvstore_txn.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "context" "fmt" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" ) type storeTxnRead struct { storeTxnCommon tx backend.ReadTx } type storeTxnCommon struct { s *store tx backend.UnsafeReader firstRev int64 rev int64 trace *traceutil.Trace } func (s *store) Read(mode ReadTxMode, trace *traceutil.Trace) TxnRead { s.mu.RLock() s.revMu.RLock() // For read-only workloads, we use shared buffer by copying transaction read buffer // for higher concurrency with ongoing blocking writes. // For write/write-read transactions, we use the shared buffer // rather than duplicating transaction read buffer to avoid transaction overhead. var tx backend.ReadTx if mode == ConcurrentReadTxMode { tx = s.b.ConcurrentReadTx() } else { tx = s.b.ReadTx() } tx.RLock() // RLock is no-op. concurrentReadTx does not need to be locked after it is created. firstRev, rev := s.compactMainRev, s.currentRev s.revMu.RUnlock() return newMetricsTxnRead(&storeTxnRead{storeTxnCommon{s, tx, firstRev, rev, trace}, tx}) } func (tr *storeTxnCommon) FirstRev() int64 { return tr.firstRev } func (tr *storeTxnCommon) Rev() int64 { return tr.rev } func (tr *storeTxnCommon) Range(ctx context.Context, key, end []byte, ro RangeOptions) (r *RangeResult, err error) { return tr.rangeKeys(ctx, key, end, tr.Rev(), ro) } func (tr *storeTxnCommon) rangeKeys(ctx context.Context, key, end []byte, curRev int64, ro RangeOptions) (*RangeResult, error) { rev := ro.Rev if rev > curRev { return &RangeResult{KVs: nil, Count: -1, Rev: curRev}, ErrFutureRev } if rev <= 0 { rev = curRev } if rev < tr.s.compactMainRev { return &RangeResult{KVs: nil, Count: -1, Rev: 0}, ErrCompacted } if ro.Count { total := tr.s.kvindex.CountRevisions(key, end, rev) tr.trace.Step("count revisions from in-memory index tree") return &RangeResult{KVs: nil, Count: total, Rev: curRev}, nil } revpairs, total := tr.s.kvindex.Revisions(key, end, rev, int(ro.Limit)) tr.trace.Step("range keys from in-memory index tree") if len(revpairs) == 0 { return &RangeResult{KVs: nil, Count: total, Rev: curRev}, nil } limit := int(ro.Limit) if limit <= 0 || limit > len(revpairs) { limit = len(revpairs) } kvs := make([]mvccpb.KeyValue, limit) revBytes := NewRevBytes() for i, revpair := range revpairs[:len(kvs)] { select { case <-ctx.Done(): return nil, fmt.Errorf("rangeKeys: context cancelled: %w", ctx.Err()) default: } revBytes = RevToBytes(revpair, revBytes) _, vs := tr.tx.UnsafeRange(schema.Key, revBytes, nil, 0) if len(vs) != 1 { tr.s.lg.Fatal( "range failed to find revision pair", zap.Int64("revision-main", revpair.Main), zap.Int64("revision-sub", revpair.Sub), zap.Int64("revision-current", curRev), zap.Int64("range-option-rev", ro.Rev), zap.Int64("range-option-limit", ro.Limit), zap.Binary("key", key), zap.Binary("end", end), zap.Int("len-revpairs", len(revpairs)), zap.Int("len-values", len(vs)), ) } if err := kvs[i].Unmarshal(vs[0]); err != nil { tr.s.lg.Fatal( "failed to unmarshal mvccpb.KeyValue", zap.Error(err), ) } } tr.trace.Step("range keys from bolt db") return &RangeResult{KVs: kvs, Count: total, Rev: curRev}, nil } func (tr *storeTxnRead) End() { tr.tx.RUnlock() // RUnlock signals the end of concurrentReadTx. tr.s.mu.RUnlock() } type storeTxnWrite struct { storeTxnCommon tx backend.BatchTx // beginRev is the revision where the txn begins; it will write to the next revision. beginRev int64 changes []mvccpb.KeyValue } func (s *store) Write(trace *traceutil.Trace) TxnWrite { s.mu.RLock() tx := s.b.BatchTx() tx.LockInsideApply() tw := &storeTxnWrite{ storeTxnCommon: storeTxnCommon{s, tx, 0, 0, trace}, tx: tx, beginRev: s.currentRev, changes: make([]mvccpb.KeyValue, 0, 4), } return newMetricsTxnWrite(tw) } func (tw *storeTxnWrite) Rev() int64 { return tw.beginRev } func (tw *storeTxnWrite) Range(ctx context.Context, key, end []byte, ro RangeOptions) (r *RangeResult, err error) { rev := tw.beginRev if len(tw.changes) > 0 { rev++ } return tw.rangeKeys(ctx, key, end, rev, ro) } func (tw *storeTxnWrite) DeleteRange(key, end []byte) (int64, int64) { if n := tw.deleteRange(key, end); n != 0 || len(tw.changes) > 0 { return n, tw.beginRev + 1 } return 0, tw.beginRev } func (tw *storeTxnWrite) Put(key, value []byte, lease lease.LeaseID) int64 { tw.put(key, value, lease) return tw.beginRev + 1 } func (tw *storeTxnWrite) End() { // only update index if the txn modifies the mvcc state. if len(tw.changes) != 0 { // hold revMu lock to prevent new read txns from opening until writeback. tw.s.revMu.Lock() tw.s.currentRev++ } tw.tx.Unlock() if len(tw.changes) != 0 { tw.s.revMu.Unlock() } tw.s.mu.RUnlock() } func (tw *storeTxnWrite) put(key, value []byte, leaseID lease.LeaseID) { rev := tw.beginRev + 1 c := rev oldLease := lease.NoLease // if the key exists before, use its previous created and // get its previous leaseID _, created, ver, err := tw.s.kvindex.Get(key, rev) if err == nil { c = created.Main oldLease = tw.s.le.GetLease(lease.LeaseItem{Key: string(key)}) tw.trace.Step("get key's previous created_revision and leaseID") } ibytes := NewRevBytes() idxRev := Revision{Main: rev, Sub: int64(len(tw.changes))} ibytes = RevToBytes(idxRev, ibytes) ver = ver + 1 kv := mvccpb.KeyValue{ Key: key, Value: value, CreateRevision: c, ModRevision: rev, Version: ver, Lease: int64(leaseID), } d, err := kv.Marshal() if err != nil { tw.storeTxnCommon.s.lg.Fatal( "failed to marshal mvccpb.KeyValue", zap.Error(err), ) } tw.trace.Step("marshal mvccpb.KeyValue") tw.tx.UnsafeSeqPut(schema.Key, ibytes, d) tw.s.kvindex.Put(key, idxRev) tw.changes = append(tw.changes, kv) tw.trace.Step("store kv pair into bolt db") if oldLease == leaseID { tw.trace.Step("attach lease to kv pair") return } if oldLease != lease.NoLease { if tw.s.le == nil { panic("no lessor to detach lease") } err = tw.s.le.Detach(oldLease, []lease.LeaseItem{{Key: string(key)}}) if err != nil { tw.storeTxnCommon.s.lg.Error( "failed to detach old lease from a key", zap.Error(err), ) } } if leaseID != lease.NoLease { if tw.s.le == nil { panic("no lessor to attach lease") } err = tw.s.le.Attach(leaseID, []lease.LeaseItem{{Key: string(key)}}) if err != nil { panic("unexpected error from lease Attach") } } tw.trace.Step("attach lease to kv pair") } func (tw *storeTxnWrite) deleteRange(key, end []byte) int64 { rrev := tw.beginRev if len(tw.changes) > 0 { rrev++ } keys, _ := tw.s.kvindex.Range(key, end, rrev) if len(keys) == 0 { return 0 } for _, key := range keys { tw.delete(key) } return int64(len(keys)) } func (tw *storeTxnWrite) delete(key []byte) { ibytes := NewRevBytes() idxRev := newBucketKey(tw.beginRev+1, int64(len(tw.changes)), true) ibytes = BucketKeyToBytes(idxRev, ibytes) kv := mvccpb.KeyValue{Key: key} d, err := kv.Marshal() if err != nil { tw.storeTxnCommon.s.lg.Fatal( "failed to marshal mvccpb.KeyValue", zap.Error(err), ) } tw.tx.UnsafeSeqPut(schema.Key, ibytes, d) err = tw.s.kvindex.Tombstone(key, idxRev.Revision) if err != nil { tw.storeTxnCommon.s.lg.Fatal( "failed to tombstone an existing key", zap.String("key", string(key)), zap.Error(err), ) } tw.changes = append(tw.changes, kv) item := lease.LeaseItem{Key: string(key)} leaseID := tw.s.le.GetLease(item) if leaseID != lease.NoLease { err = tw.s.le.Detach(leaseID, []lease.LeaseItem{item}) if err != nil { tw.storeTxnCommon.s.lg.Error( "failed to detach old lease from a key", zap.Error(err), ) } } } func (tw *storeTxnWrite) Changes() []mvccpb.KeyValue { return tw.changes } ================================================ FILE: server/storage/mvcc/metrics.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "sync" "github.com/prometheus/client_golang/prometheus" ) var ( rangeCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "mvcc", Name: "range_total", Help: "Total number of ranges seen by this member.", }, ) putCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "mvcc", Name: "put_total", Help: "Total number of puts seen by this member.", }, ) deleteCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "mvcc", Name: "delete_total", Help: "Total number of deletes seen by this member.", }, ) txnCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "mvcc", Name: "txn_total", Help: "Total number of txns seen by this member.", }, ) keysGauge = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "keys_total", Help: "Total number of keys.", }, ) watchStreamGauge = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "watch_stream_total", Help: "Total number of watch streams.", }, ) watcherGauge = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "watcher_total", Help: "Total number of watchers.", }, ) slowWatcherGauge = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "slow_watcher_total", Help: "Total number of unsynced slow watchers.", }, ) totalEventsCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "events_total", Help: "Total number of events sent by this member.", }, ) pendingEventsGauge = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "pending_events_total", Help: "Total number of pending events to be sent.", }, ) indexCompactionPauseMs = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "index_compaction_pause_duration_milliseconds", Help: "Bucketed histogram of index compaction pause duration.", // lowest bucket start of upper bound 0.5 ms with factor 2 // highest bucket start of 0.5 ms * 2^13 == 4.096 sec Buckets: prometheus.ExponentialBuckets(0.5, 2, 14), }, ) dbCompactionPauseMs = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "db_compaction_pause_duration_milliseconds", Help: "Bucketed histogram of db compaction pause duration.", // lowest bucket start of upper bound 1 ms with factor 2 // highest bucket start of 1 ms * 2^12 == 4.096 sec Buckets: prometheus.ExponentialBuckets(1, 2, 13), }, ) dbCompactionTotalMs = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "db_compaction_total_duration_milliseconds", Help: "Bucketed histogram of db compaction total duration.", // lowest bucket start of upper bound 100 ms with factor 2 // highest bucket start of 100 ms * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(100, 2, 14), }, ) dbCompactionLast = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "db_compaction_last", Help: "The unix time of the last db compaction. Resets to 0 on start.", }, ) dbCompactionKeysCounter = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "db_compaction_keys_total", Help: "Total number of db keys compacted.", }, ) dbTotalSize = prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "mvcc", Name: "db_total_size_in_bytes", Help: "Total size of the underlying database physically allocated in bytes.", }, func() float64 { reportDbTotalSizeInBytesMu.RLock() defer reportDbTotalSizeInBytesMu.RUnlock() return reportDbTotalSizeInBytes() }, ) // overridden by mvcc initialization reportDbTotalSizeInBytesMu sync.RWMutex reportDbTotalSizeInBytes = func() float64 { return 0 } dbTotalSizeInUse = prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "mvcc", Name: "db_total_size_in_use_in_bytes", Help: "Total size of the underlying database logically in use in bytes.", }, func() float64 { reportDbTotalSizeInUseInBytesMu.RLock() defer reportDbTotalSizeInUseInBytesMu.RUnlock() return reportDbTotalSizeInUseInBytes() }, ) // overridden by mvcc initialization reportDbTotalSizeInUseInBytesMu sync.RWMutex reportDbTotalSizeInUseInBytes = func() float64 { return 0 } dbOpenReadTxN = prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "mvcc", Name: "db_open_read_transactions", Help: "The number of currently open read transactions", }, func() float64 { reportDbOpenReadTxNMu.RLock() defer reportDbOpenReadTxNMu.RUnlock() return reportDbOpenReadTxN() }, ) // overridden by mvcc initialization reportDbOpenReadTxNMu sync.RWMutex reportDbOpenReadTxN = func() float64 { return 0 } hashSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "mvcc", Name: "hash_duration_seconds", Help: "The latency distribution of storage hash operation.", // 100 MB usually takes 100 ms, so start with 10 MB of 10 ms // lowest bucket start of upper bound 0.01 sec (10 ms) with factor 2 // highest bucket start of 0.01 sec * 2^14 == 163.84 sec Buckets: prometheus.ExponentialBuckets(.01, 2, 15), }) hashRevSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "mvcc", Name: "hash_rev_duration_seconds", Help: "The latency distribution of storage hash by revision operation.", // 100 MB usually takes 100 ms, so start with 10 MB of 10 ms // lowest bucket start of upper bound 0.01 sec (10 ms) with factor 2 // highest bucket start of 0.01 sec * 2^14 == 163.84 sec Buckets: prometheus.ExponentialBuckets(.01, 2, 15), }) currentRev = prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "current_revision", Help: "The current revision of store.", }, func() float64 { reportCurrentRevMu.RLock() defer reportCurrentRevMu.RUnlock() return reportCurrentRev() }, ) // overridden by mvcc initialization reportCurrentRevMu sync.RWMutex reportCurrentRev = func() float64 { return 0 } compactRev = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "compact_revision", Help: "The revision of the last compaction in store.", }, func() float64 { reportCompactRevMu.RLock() defer reportCompactRevMu.RUnlock() return reportCompactRev() }, ) // overridden by mvcc initialization reportCompactRevMu sync.RWMutex reportCompactRev = func() float64 { return 0 } totalPutSizeGauge = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "total_put_size_in_bytes", Help: "The total size of put kv pairs seen by this member.", }) ) func init() { prometheus.MustRegister(rangeCounter) prometheus.MustRegister(putCounter) prometheus.MustRegister(deleteCounter) prometheus.MustRegister(txnCounter) prometheus.MustRegister(keysGauge) prometheus.MustRegister(watchStreamGauge) prometheus.MustRegister(watcherGauge) prometheus.MustRegister(slowWatcherGauge) prometheus.MustRegister(totalEventsCounter) prometheus.MustRegister(pendingEventsGauge) prometheus.MustRegister(indexCompactionPauseMs) prometheus.MustRegister(dbCompactionPauseMs) prometheus.MustRegister(dbCompactionTotalMs) prometheus.MustRegister(dbCompactionLast) prometheus.MustRegister(dbCompactionKeysCounter) prometheus.MustRegister(dbTotalSize) prometheus.MustRegister(dbTotalSizeInUse) prometheus.MustRegister(dbOpenReadTxN) prometheus.MustRegister(hashSec) prometheus.MustRegister(hashRevSec) prometheus.MustRegister(currentRev) prometheus.MustRegister(compactRev) prometheus.MustRegister(totalPutSizeGauge) } // ReportEventReceived reports that an event is received. // This function should be called when the external systems received an // event from mvcc.Watcher. func ReportEventReceived(n int) { pendingEventsGauge.Sub(float64(n)) totalEventsCounter.Add(float64(n)) } ================================================ FILE: server/storage/mvcc/metrics_txn.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "context" "go.etcd.io/etcd/server/v3/lease" ) type metricsTxnWrite struct { TxnWrite ranges uint puts uint deletes uint putSize int64 } func newMetricsTxnRead(tr TxnRead) TxnRead { return &metricsTxnWrite{&txnReadWrite{tr}, 0, 0, 0, 0} } func newMetricsTxnWrite(tw TxnWrite) TxnWrite { return &metricsTxnWrite{tw, 0, 0, 0, 0} } func (tw *metricsTxnWrite) Range(ctx context.Context, key, end []byte, ro RangeOptions) (*RangeResult, error) { tw.ranges++ return tw.TxnWrite.Range(ctx, key, end, ro) } func (tw *metricsTxnWrite) DeleteRange(key, end []byte) (n, rev int64) { tw.deletes++ return tw.TxnWrite.DeleteRange(key, end) } func (tw *metricsTxnWrite) Put(key, value []byte, lease lease.LeaseID) (rev int64) { tw.puts++ size := int64(len(key) + len(value)) tw.putSize += size return tw.TxnWrite.Put(key, value, lease) } func (tw *metricsTxnWrite) End() { defer tw.TxnWrite.End() if sum := tw.ranges + tw.puts + tw.deletes; sum > 1 { txnCounter.Inc() } ranges := float64(tw.ranges) rangeCounter.Add(ranges) puts := float64(tw.puts) putCounter.Add(puts) totalPutSizeGauge.Add(float64(tw.putSize)) deletes := float64(tw.deletes) deleteCounter.Add(deletes) } ================================================ FILE: server/storage/mvcc/revision.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "encoding/binary" "fmt" ) const ( // revBytesLen is the byte length of a normal revision. // First 8 bytes is the revision.main in big-endian format. The 9th byte // is a '_'. The last 8 bytes is the revision.sub in big-endian format. revBytesLen = 8 + 1 + 8 // markedRevBytesLen is the byte length of marked revision. // The first `revBytesLen` bytes represents a normal revision. The last // one byte is the mark. markedRevBytesLen = revBytesLen + 1 markBytePosition = markedRevBytesLen - 1 markTombstone byte = 't' ) type Revision struct { // Main is the main revision of a set of changes that happen atomically. Main int64 // Sub is the sub revision of a change in a set of changes that happen // atomically. Each change has different increasing sub revision in that // set. Sub int64 } func (a Revision) GreaterThan(b Revision) bool { if a.Main > b.Main { return true } if a.Main < b.Main { return false } return a.Sub > b.Sub } func RevToBytes(rev Revision, bytes []byte) []byte { return BucketKeyToBytes(newBucketKey(rev.Main, rev.Sub, false), bytes) } func BytesToRev(bytes []byte) Revision { return BytesToBucketKey(bytes).Revision } // BucketKey indicates modification of the key-value space. // The set of changes that share same main revision changes the key-value space atomically. type BucketKey struct { Revision tombstone bool } func newBucketKey(main, sub int64, isTombstone bool) BucketKey { return BucketKey{ Revision: Revision{ Main: main, Sub: sub, }, tombstone: isTombstone, } } func NewRevBytes() []byte { return make([]byte, revBytesLen, markedRevBytesLen) } func BucketKeyToBytes(rev BucketKey, bytes []byte) []byte { binary.BigEndian.PutUint64(bytes, uint64(rev.Main)) bytes[8] = '_' binary.BigEndian.PutUint64(bytes[9:], uint64(rev.Sub)) if rev.tombstone { switch len(bytes) { case revBytesLen: bytes = append(bytes, markTombstone) case markedRevBytesLen: bytes[markBytePosition] = markTombstone } } return bytes } func BytesToBucketKey(bytes []byte) BucketKey { if (len(bytes) != revBytesLen) && (len(bytes) != markedRevBytesLen) { panic(fmt.Sprintf("invalid revision length: %d", len(bytes))) } if bytes[8] != '_' { panic(fmt.Sprintf("invalid separator in bucket key: %q", bytes[8])) } main := int64(binary.BigEndian.Uint64(bytes[0:8])) sub := int64(binary.BigEndian.Uint64(bytes[9:])) if main < 0 || sub < 0 { panic(fmt.Sprintf("negative revision: main=%d sub=%d", main, sub)) } return BucketKey{ Revision: Revision{ Main: main, Sub: sub, }, tombstone: isTombstone(bytes), } } // isTombstone checks whether the revision bytes is a tombstone. func isTombstone(b []byte) bool { return len(b) == markedRevBytesLen && b[markBytePosition] == markTombstone } func IsTombstone(b []byte) bool { return isTombstone(b) } ================================================ FILE: server/storage/mvcc/store.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" ) func UnsafeReadFinishedCompact(tx backend.UnsafeReader) (int64, bool) { _, finishedCompactBytes := tx.UnsafeRange(schema.Meta, schema.FinishedCompactKeyName, nil, 0) if len(finishedCompactBytes) != 0 { return BytesToRev(finishedCompactBytes[0]).Main, true } return 0, false } func UnsafeReadScheduledCompact(tx backend.UnsafeReader) (int64, bool) { _, scheduledCompactBytes := tx.UnsafeRange(schema.Meta, schema.ScheduledCompactKeyName, nil, 0) if len(scheduledCompactBytes) != 0 { return BytesToRev(scheduledCompactBytes[0]).Main, true } return 0, false } func SetScheduledCompact(tx backend.BatchTx, value int64) { tx.LockInsideApply() defer tx.Unlock() UnsafeSetScheduledCompact(tx, value) } func UnsafeSetScheduledCompact(tx backend.UnsafeWriter, value int64) { rbytes := NewRevBytes() rbytes = RevToBytes(Revision{Main: value}, rbytes) tx.UnsafePut(schema.Meta, schema.ScheduledCompactKeyName, rbytes) } func SetFinishedCompact(tx backend.BatchTx, value int64) { tx.LockInsideApply() defer tx.Unlock() UnsafeSetFinishedCompact(tx, value) } func UnsafeSetFinishedCompact(tx backend.UnsafeWriter, value int64) { rbytes := NewRevBytes() rbytes = RevToBytes(Revision{Main: value}, rbytes) tx.UnsafePut(schema.Meta, schema.FinishedCompactKeyName, rbytes) } ================================================ FILE: server/storage/mvcc/store_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "fmt" "math" "testing" "time" "github.com/stretchr/testify/assert" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/schema" ) // TestScheduledCompact ensures that UnsafeSetScheduledCompact&UnsafeReadScheduledCompact work well together. func TestScheduledCompact(t *testing.T) { tcs := []struct { value int64 }{ { value: 1, }, { value: 0, }, { value: math.MaxInt64, }, } for _, tc := range tcs { t.Run(fmt.Sprint(tc.value), func(t *testing.T) { lg := zaptest.NewLogger(t) be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) tx := be.BatchTx() if tx == nil { t.Fatal("batch tx is nil") } tx.Lock() tx.UnsafeCreateBucket(schema.Meta) UnsafeSetScheduledCompact(tx, tc.value) tx.Unlock() be.ForceCommit() be.Close() b := backend.NewDefaultBackend(lg, tmpPath) defer b.Close() v, found := UnsafeReadScheduledCompact(b.BatchTx()) assert.True(t, found) assert.Equal(t, tc.value, v) }) } } // TestFinishedCompact ensures that UnsafeSetFinishedCompact&UnsafeReadFinishedCompact work well together. func TestFinishedCompact(t *testing.T) { tcs := []struct { value int64 }{ { value: 1, }, { value: 0, }, { value: math.MaxInt64, }, } for _, tc := range tcs { t.Run(fmt.Sprint(tc.value), func(t *testing.T) { lg := zaptest.NewLogger(t) be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) tx := be.BatchTx() if tx == nil { t.Fatal("batch tx is nil") } tx.Lock() tx.UnsafeCreateBucket(schema.Meta) UnsafeSetFinishedCompact(tx, tc.value) tx.Unlock() be.ForceCommit() be.Close() b := backend.NewDefaultBackend(lg, tmpPath) defer b.Close() v, found := UnsafeReadFinishedCompact(b.BatchTx()) assert.True(t, found) assert.Equal(t, tc.value, v) }) } } ================================================ FILE: server/storage/mvcc/testutil/hash.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutil import ( "context" "errors" "fmt" "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" "go.etcd.io/etcd/api/v3/mvccpb" ) const ( // CompactionCycle is high prime used to test hash calculation between compactions. CompactionCycle = 71 ) func TestCompactionHash(ctx context.Context, t *testing.T, h CompactionHashTestCase, compactionBatchLimit int) { var totalRevisions int64 = 1210 assert.Less(t, int64(compactionBatchLimit), totalRevisions) assert.Less(t, int64(CompactionCycle*10), totalRevisions) var rev int64 for ; rev < totalRevisions; rev += CompactionCycle { testCompactionHash(ctx, t, h, rev, rev+CompactionCycle) } testCompactionHash(ctx, t, h, rev, rev+totalRevisions) } type CompactionHashTestCase interface { Put(ctx context.Context, key, value string) error Delete(ctx context.Context, key string) error HashByRev(ctx context.Context, rev int64) (KeyValueHash, error) Defrag(ctx context.Context) error Compact(ctx context.Context, rev int64) error } type KeyValueHash struct { Hash uint32 CompactRevision int64 Revision int64 } func testCompactionHash(ctx context.Context, t *testing.T, h CompactionHashTestCase, start, stop int64) { for i := start; i <= stop; i++ { if i%67 == 0 { err := h.Delete(ctx, PickKey(i+83)) require.NoErrorf(t, err, "error on delete") } else { err := h.Put(ctx, PickKey(i), fmt.Sprint(i)) require.NoErrorf(t, err, "error on put") } } hash1, err := h.HashByRev(ctx, stop) require.NoErrorf(t, err, "error on hash (rev %v)", stop) err = h.Compact(ctx, stop) require.NoErrorf(t, err, "error on compact (rev %v)", stop) err = h.Defrag(ctx) require.NoErrorf(t, err, "error on defrag") hash2, err := h.HashByRev(ctx, stop) require.NoErrorf(t, err, "error on hash (rev %v)", stop) assert.Equalf(t, hash1, hash2, "hashes do not match on rev %v", stop) } func PickKey(i int64) string { if i%(CompactionCycle*2) == 30 { return "zenek" } if i%CompactionCycle == 30 { return "xavery" } // Use low prime number to ensure repeats without alignment switch i % 7 { case 0: return "alice" case 1: return "bob" case 2: return "celine" case 3: return "dominik" case 4: return "eve" case 5: return "frederica" case 6: return "gorge" default: panic("Can't count") } } func CorruptBBolt(fpath string) error { db, derr := bbolt.Open(fpath, os.ModePerm, &bbolt.Options{}) if derr != nil { return derr } defer db.Close() return db.Update(func(tx *bbolt.Tx) error { b := tx.Bucket([]byte("key")) if b == nil { return errors.New("got nil bucket for 'key'") } var vals [][]byte var keys [][]byte c := b.Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { keys = append(keys, k) var kv mvccpb.KeyValue if uerr := kv.Unmarshal(v); uerr != nil { return uerr } kv.Key[0]++ kv.Value[0]++ v2, v2err := kv.Marshal() if v2err != nil { return v2err } vals = append(vals, v2) } for i := range keys { if perr := b.Put(keys[i], vals[i]); perr != nil { return perr } } return nil }) } ================================================ FILE: server/storage/mvcc/watchable_store.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "sync" "time" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/client/pkg/v3/verify" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" ) // non-const so modifiable by tests var ( // chanBufLen is the length of the buffered chan // for sending out watched events. // See https://github.com/etcd-io/etcd/issues/11906 for more detail. chanBufLen = 128 // maxWatchersPerSync is the number of watchers to sync in a single batch maxWatchersPerSync = 512 // maxResyncPeriod is the period of executing resync. watchResyncPeriod = 100 * time.Millisecond ) func ChanBufLen() int { return chanBufLen } type watchable interface { watch(key, end []byte, startRev int64, id WatchID, ch chan<- WatchResponse, fcs ...FilterFunc) (*watcher, cancelFunc) progress(w *watcher) progressAll(watchers map[WatchID]*watcher) bool rev() int64 } type watchableStore struct { *store // mu protects watcher groups and batches. It should never be locked // before locking store.mu to avoid deadlock. mu sync.RWMutex // victims are watcher batches that were blocked on the watch channel victims []watcherBatch victimc chan struct{} // contains all unsynced watchers that needs to sync with events that have happened unsynced watcherGroup // contains all synced watchers that are in sync with the progress of the store. // The key of the map is the key that the watcher watches on. synced watcherGroup stopc chan struct{} wg sync.WaitGroup } var _ WatchableKV = (*watchableStore)(nil) // cancelFunc updates unsynced and synced maps when running // cancel operations. type cancelFunc func() func New(lg *zap.Logger, b backend.Backend, le lease.Lessor, cfg StoreConfig) WatchableKV { s := newWatchableStore(lg, b, le, cfg) s.wg.Add(2) go s.syncWatchersLoop() go s.syncVictimsLoop() return s } func newWatchableStore(lg *zap.Logger, b backend.Backend, le lease.Lessor, cfg StoreConfig) *watchableStore { if lg == nil { lg = zap.NewNop() } s := &watchableStore{ store: NewStore(lg, b, le, cfg), victimc: make(chan struct{}, 1), unsynced: newWatcherGroup(), synced: newWatcherGroup(), stopc: make(chan struct{}), } s.store.ReadView = &readView{s} s.store.WriteView = &writeView{s} if s.le != nil { // use this store as the deleter so revokes trigger watch events s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write(traceutil.TODO()) }) } return s } func (s *watchableStore) Close() error { close(s.stopc) s.wg.Wait() return s.store.Close() } func (s *watchableStore) NewWatchStream() WatchStream { watchStreamGauge.Inc() return &watchStream{ watchable: s, ch: make(chan WatchResponse, chanBufLen), cancels: make(map[WatchID]cancelFunc), watchers: make(map[WatchID]*watcher), } } func (s *watchableStore) watch(key, end []byte, startRev int64, id WatchID, ch chan<- WatchResponse, fcs ...FilterFunc) (*watcher, cancelFunc) { wa := &watcher{ key: key, end: end, startRev: startRev, minRev: startRev, id: id, ch: ch, fcs: fcs, } s.mu.Lock() s.revMu.RLock() synced := startRev > s.store.currentRev || startRev == 0 if synced { wa.minRev = s.store.currentRev + 1 if startRev > wa.minRev { wa.minRev = startRev } s.synced.add(wa) } else { slowWatcherGauge.Inc() s.unsynced.add(wa) } s.revMu.RUnlock() s.mu.Unlock() watcherGauge.Inc() return wa, func() { s.cancelWatcher(wa) } } // cancelWatcher removes references of the watcher from the watchableStore func (s *watchableStore) cancelWatcher(wa *watcher) { for { s.mu.Lock() if s.unsynced.delete(wa) { slowWatcherGauge.Dec() watcherGauge.Dec() break } else if s.synced.delete(wa) { watcherGauge.Dec() break } else if wa.ch == nil { // already canceled (e.g., cancel/close race) break } else if wa.compacted { watcherGauge.Dec() break } if !wa.victim { s.mu.Unlock() panic("watcher not victim but not in watch groups") } var victimBatch watcherBatch for _, wb := range s.victims { if wb[wa] != nil { victimBatch = wb break } } if victimBatch != nil { slowWatcherGauge.Dec() watcherGauge.Dec() delete(victimBatch, wa) break } // victim being processed so not accessible; retry s.mu.Unlock() time.Sleep(time.Millisecond) } wa.ch = nil s.mu.Unlock() } func (s *watchableStore) Restore(b backend.Backend) error { s.mu.Lock() defer s.mu.Unlock() err := s.store.Restore(b) if err != nil { return err } for wa := range s.synced.watchers { wa.restore = true s.unsynced.add(wa) } s.synced = newWatcherGroup() return nil } // syncWatchersLoop syncs the watcher in the unsynced map every 100ms. func (s *watchableStore) syncWatchersLoop() { defer s.wg.Done() delayTicker := time.NewTicker(watchResyncPeriod) defer delayTicker.Stop() for { s.mu.RLock() st := time.Now() lastUnsyncedWatchers := s.unsynced.size() s.mu.RUnlock() unsyncedWatchers := 0 if lastUnsyncedWatchers > 0 { unsyncedWatchers = s.syncWatchers() } syncDuration := time.Since(st) delayTicker.Reset(watchResyncPeriod) // more work pending? if unsyncedWatchers != 0 && lastUnsyncedWatchers > unsyncedWatchers { // be fair to other store operations by yielding time taken delayTicker.Reset(syncDuration) } select { case <-delayTicker.C: case <-s.stopc: return } } } // syncVictimsLoop tries to write precomputed watcher responses to // watchers that had a blocked watcher channel func (s *watchableStore) syncVictimsLoop() { defer s.wg.Done() for { for s.moveVictims() != 0 { // try to update all victim watchers } s.mu.RLock() isEmpty := len(s.victims) == 0 s.mu.RUnlock() var tickc <-chan time.Time if !isEmpty { tickc = time.After(10 * time.Millisecond) } select { case <-tickc: case <-s.victimc: case <-s.stopc: return } } } // moveVictims tries to update watches with already pending event data func (s *watchableStore) moveVictims() (moved int) { s.mu.Lock() victims := s.victims s.victims = nil s.mu.Unlock() var newVictim watcherBatch for _, wb := range victims { // try to send responses again for w, eb := range wb { // watcher has observed the store up to, but not including, w.minRev rev := w.minRev - 1 if !w.send(WatchResponse{WatchID: w.id, Events: eb.evs, Revision: rev}) { if newVictim == nil { newVictim = make(watcherBatch) } newVictim[w] = eb continue } pendingEventsGauge.Add(float64(len(eb.evs))) moved++ } // assign completed victim watchers to unsync/sync s.mu.Lock() s.store.revMu.RLock() curRev := s.store.currentRev for w, eb := range wb { if newVictim != nil && newVictim[w] != nil { // couldn't send watch response; stays victim continue } w.victim = false if eb.moreRev != 0 { w.minRev = eb.moreRev } if w.minRev <= curRev { s.unsynced.add(w) } else { slowWatcherGauge.Dec() s.synced.add(w) } } s.store.revMu.RUnlock() s.mu.Unlock() } if len(newVictim) > 0 { s.mu.Lock() s.victims = append(s.victims, newVictim) s.mu.Unlock() } return moved } // syncWatchers syncs unsynced watchers by: // 1. choose a set of watchers from the unsynced watcher group // 2. iterate over the set to get the minimum revision and remove compacted watchers // 3. use minimum revision to get all key-value pairs and send those events to watchers // 4. remove synced watchers in set from unsynced group and move to synced group func (s *watchableStore) syncWatchers() int { s.mu.Lock() defer s.mu.Unlock() if s.unsynced.size() == 0 { return 0 } s.store.revMu.RLock() defer s.store.revMu.RUnlock() // in order to find key-value pairs from unsynced watchers, we need to // find min revision index, and these revisions can be used to // query the backend store of key-value pairs curRev := s.store.currentRev compactionRev := s.store.compactMainRev wg, minRev := s.unsynced.choose(maxWatchersPerSync, curRev, compactionRev) evs := rangeEvents(s.store.lg, s.store.b, minRev, curRev+1, wg) victims := make(watcherBatch) wb := newWatcherBatch(wg, evs) for w := range wg.watchers { if w.minRev < compactionRev { // Skip the watcher that failed to send compacted watch response due to w.ch is full. // Next retry of syncWatchers would try to resend the compacted watch response to w.ch continue } w.minRev = max(curRev+1, w.minRev) eb, ok := wb[w] if !ok { // bring un-notified watcher to synced s.synced.add(w) s.unsynced.delete(w) continue } if eb.moreRev != 0 { w.minRev = eb.moreRev } if w.send(WatchResponse{WatchID: w.id, Events: eb.evs, Revision: curRev}) { pendingEventsGauge.Add(float64(len(eb.evs))) } else { w.victim = true } if w.victim { victims[w] = eb } else { if eb.moreRev != 0 { // stay unsynced; more to read continue } s.synced.add(w) } s.unsynced.delete(w) } s.addVictim(victims) vsz := 0 for _, v := range s.victims { vsz += len(v) } slowWatcherGauge.Set(float64(s.unsynced.size() + vsz)) return s.unsynced.size() } // rangeEvents returns events in range [minRev, maxRev). func rangeEvents(lg *zap.Logger, b backend.Backend, minRev, maxRev int64, c contains) []mvccpb.Event { if minRev < 0 { lg.Warn("Unexpected negative revision range start", zap.Int64("minRev", minRev)) minRev = 0 } minBytes, maxBytes := NewRevBytes(), NewRevBytes() minBytes = RevToBytes(Revision{Main: minRev}, minBytes) maxBytes = RevToBytes(Revision{Main: maxRev}, maxBytes) // UnsafeRange returns keys and values. And in boltdb, keys are revisions. // values are actual key-value pairs in backend. tx := b.ReadTx() tx.RLock() revs, vs := tx.UnsafeRange(schema.Key, minBytes, maxBytes, 0) evs := kvsToEvents(lg, c, revs, vs) // Must unlock after kvsToEvents, because vs (come from boltdb memory) is not deep copy. // We can only unlock after Unmarshal, which will do deep copy. // Otherwise we will trigger SIGSEGV during boltdb re-mmap. tx.RUnlock() return evs } type contains interface { contains(string) bool } // kvsToEvents gets all events for the watchers from all key-value pairs func kvsToEvents(lg *zap.Logger, c contains, revs, vals [][]byte) (evs []mvccpb.Event) { for i, v := range vals { var kv mvccpb.KeyValue if err := kv.Unmarshal(v); err != nil { lg.Panic("failed to unmarshal mvccpb.KeyValue", zap.Error(err)) } if !c.contains(string(kv.Key)) { continue } ty := mvccpb.Event_PUT if isTombstone(revs[i]) { ty = mvccpb.Event_DELETE // patch in mod revision so watchers won't skip kv.ModRevision = BytesToRev(revs[i]).Main } evs = append(evs, mvccpb.Event{Kv: &kv, Type: ty}) } return evs } // notify notifies the fact that given event at the given rev just happened to // watchers that watch on the key of the event. func (s *watchableStore) notify(rev int64, evs []mvccpb.Event) { victim := make(watcherBatch) for w, eb := range newWatcherBatch(&s.synced, evs) { if eb.revs != 1 { s.store.lg.Panic( "unexpected multiple revisions in watch notification", zap.Int("number-of-revisions", eb.revs), ) } if w.send(WatchResponse{WatchID: w.id, Events: eb.evs, Revision: rev}) { pendingEventsGauge.Add(float64(len(eb.evs))) } else { // move slow watcher to victims w.victim = true victim[w] = eb s.synced.delete(w) slowWatcherGauge.Inc() } // always update minRev // in case 'send' returns true and watcher stays synced, this is needed for Restore when all watchers become unsynced // in case 'send' returns false, this is needed for syncWatchers w.minRev = rev + 1 } s.addVictim(victim) } func (s *watchableStore) addVictim(victim watcherBatch) { if len(victim) == 0 { return } s.victims = append(s.victims, victim) select { case s.victimc <- struct{}{}: default: } } func (s *watchableStore) rev() int64 { return s.store.Rev() } func (s *watchableStore) progress(w *watcher) { s.progressIfSync(map[WatchID]*watcher{w.id: w}, w.id) } func (s *watchableStore) progressAll(watchers map[WatchID]*watcher) bool { return s.progressIfSync(watchers, clientv3.InvalidWatchID) } func (s *watchableStore) progressIfSync(watchers map[WatchID]*watcher, responseWatchID WatchID) bool { s.mu.RLock() defer s.mu.RUnlock() rev := s.rev() // Any watcher unsynced? for _, w := range watchers { if _, ok := s.synced.watchers[w]; !ok { return false } if rev < w.startRev { return false } } // If all watchers are synchronised, send out progress // notification on first watcher. Note that all watchers // should have the same underlying stream, and the progress // notification will be broadcasted client-side if required // (see dispatchEvent in client/v3/watch.go) for _, w := range watchers { w.send(WatchResponse{WatchID: responseWatchID, Revision: rev}) return true } return true } type watcher struct { // the watcher key key []byte // end indicates the end of the range to watch. // If end is set, the watcher is on a range. end []byte // victim is set when ch is blocked and undergoing victim processing victim bool // compacted is set when the watcher is removed because of compaction compacted bool // restore is true when the watcher is being restored from leader snapshot // which means that this watcher has just been moved from "synced" to "unsynced" // watcher group, possibly with a future revision when it was first added // to the synced watcher // "unsynced" watcher revision must always be <= current revision, // except when the watcher were to be moved from "synced" watcher group restore bool startRev int64 // minRev is the minimum revision update the watcher will accept minRev int64 id WatchID fcs []FilterFunc // a chan to send out the watch response. // The chan might be shared with other watchers. ch chan<- WatchResponse } func (w *watcher) send(wr WatchResponse) bool { progressEvent := len(wr.Events) == 0 if len(w.fcs) != 0 { ne := make([]mvccpb.Event, 0, len(wr.Events)) for i := range wr.Events { filtered := false for _, filter := range w.fcs { if filter(wr.Events[i]) { filtered = true break } } if !filtered { ne = append(ne, wr.Events[i]) } } wr.Events = ne } verify.Verify("Event.ModRevision is less than the w.startRev for watchID", func() (bool, map[string]any) { if w.startRev > 0 { for _, ev := range wr.Events { if ev.Kv.ModRevision < w.startRev { return false, map[string]any{ "Event.ModRevision": ev.Kv.ModRevision, "w.startRev": w.startRev, "watchID": w.id, } } } } return true, nil }) // if all events are filtered out, we should send nothing. if !progressEvent && len(wr.Events) == 0 { return true } select { case w.ch <- wr: return true default: return false } } ================================================ FILE: server/storage/mvcc/watchable_store_bench_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "math/rand" "testing" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) func BenchmarkWatchableStorePut(b *testing.B) { be, _ := betesting.NewDefaultTmpBackend(b) s := New(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, be) // arbitrary number of bytes bytesN := 64 keys := createBytesSlice(bytesN, b.N) vals := createBytesSlice(bytesN, b.N) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { s.Put(keys[i], vals[i], lease.NoLease) } } // BenchmarkWatchableStoreTxnPut benchmarks the Put operation // with transaction begin and end, where transaction involves // some synchronization operations, such as mutex locking. func BenchmarkWatchableStoreTxnPut(b *testing.B) { be, _ := betesting.NewDefaultTmpBackend(b) s := New(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, be) // arbitrary number of bytes bytesN := 64 keys := createBytesSlice(bytesN, b.N) vals := createBytesSlice(bytesN, b.N) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { txn := s.Write(traceutil.TODO()) txn.Put(keys[i], vals[i], lease.NoLease) txn.End() } } // BenchmarkWatchableStoreWatchPutSync benchmarks the case of // many synced watchers receiving a Put notification. func BenchmarkWatchableStoreWatchPutSync(b *testing.B) { benchmarkWatchableStoreWatchPut(b, true) } // BenchmarkWatchableStoreWatchPutUnsync benchmarks the case of // many unsynced watchers receiving a Put notification. func BenchmarkWatchableStoreWatchPutUnsync(b *testing.B) { benchmarkWatchableStoreWatchPut(b, false) } func benchmarkWatchableStoreWatchPut(b *testing.B, synced bool) { be, _ := betesting.NewDefaultTmpBackend(b) s := New(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, be) k := []byte("testkey") v := []byte("testval") rev := int64(0) if !synced { // non-0 value to keep watchers in unsynced rev = 1 } w := s.NewWatchStream() defer w.Close() watchIDs := make([]WatchID, b.N) for i := range watchIDs { watchIDs[i], _ = w.Watch(b.Context(), 0, k, nil, rev) } b.ResetTimer() b.ReportAllocs() // trigger watchers s.Put(k, v, lease.NoLease) for range watchIDs { <-w.Chan() } select { case wc := <-w.Chan(): b.Fatalf("unexpected data %v", wc) default: } } // BenchmarkWatchableStoreUnsyncedCancel benchmarks on cancel function // performance for unsynced watchers in a WatchableStore. It creates // k*N watchers to populate unsynced with a reasonably large number of // watchers. And measures the time it takes to cancel N watchers out // of k*N watchers. The performance is expected to differ depending on // the unsynced member implementation. // TODO: k is an arbitrary constant. We need to figure out what factor // we should put to simulate the real-world use cases. func BenchmarkWatchableStoreUnsyncedCancel(b *testing.B) { be, _ := betesting.NewDefaultTmpBackend(b) ws := newWatchableStore(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(ws, be) // Put a key so that we can spawn watchers on that key // (testKey in this test). This increases the rev to 1, // and later we can we set the watcher's startRev to 1, // and force watchers to be in unsynced. testKey := []byte("foo") testValue := []byte("bar") ws.Put(testKey, testValue, lease.NoLease) w := ws.NewWatchStream() defer w.Close() const k int = 2 benchSampleN := b.N watcherN := k * benchSampleN watchIDs := make([]WatchID, watcherN) for i := 0; i < watcherN; i++ { // non-0 value to keep watchers in unsynced watchIDs[i], _ = w.Watch(b.Context(), 0, testKey, nil, 1) } // random-cancel N watchers to make it not biased towards // data structures with an order, such as slice. ix := rand.Perm(watcherN) b.ResetTimer() b.ReportAllocs() // cancel N watchers for _, idx := range ix[:benchSampleN] { if err := w.Cancel(watchIDs[idx]); err != nil { b.Error(err) } } } func BenchmarkWatchableStoreSyncedCancel(b *testing.B) { be, _ := betesting.NewDefaultTmpBackend(b) s := New(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, be) // Put a key so that we can spawn watchers on that key testKey := []byte("foo") testValue := []byte("bar") s.Put(testKey, testValue, lease.NoLease) w := s.NewWatchStream() defer w.Close() // put 1 million watchers on the same key const watcherN = 1000000 watchIDs := make([]WatchID, watcherN) for i := 0; i < watcherN; i++ { // 0 for startRev to keep watchers in synced watchIDs[i], _ = w.Watch(b.Context(), 0, testKey, nil, 0) } // randomly cancel watchers to make it not biased towards // data structures with an order, such as slice. ix := rand.Perm(watcherN) b.ResetTimer() b.ReportAllocs() for _, idx := range ix { if err := w.Cancel(watchIDs[idx]); err != nil { b.Error(err) } } } ================================================ FILE: server/storage/mvcc/watchable_store_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "fmt" "reflect" "strings" "sync" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) func TestWatch(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) testKey := []byte("foo") testValue := []byte("bar") s.Put(testKey, testValue, lease.NoLease) w := s.NewWatchStream() defer w.Close() w.Watch(t.Context(), 0, testKey, nil, 0) if !s.(*watchableStore).synced.contains(string(testKey)) { // the key must have had an entry in synced t.Errorf("existence = false, want true") } } func TestNewWatcherCancel(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) testKey := []byte("foo") testValue := []byte("bar") s.Put(testKey, testValue, lease.NoLease) w := s.NewWatchStream() defer w.Close() wt, _ := w.Watch(t.Context(), 0, testKey, nil, 0) if err := w.Cancel(wt); err != nil { t.Error(err) } if s.(*watchableStore).synced.contains(string(testKey)) { // the key shoud have been deleted t.Errorf("existence = true, want false") } } func TestNewWatcherCountGauge(t *testing.T) { expectWatchGauge := func(watchers int) { expected := fmt.Sprintf(`# HELP etcd_debugging_mvcc_watcher_total Total number of watchers. # TYPE etcd_debugging_mvcc_watcher_total gauge etcd_debugging_mvcc_watcher_total %d `, watchers) err := testutil.CollectAndCompare(watcherGauge, strings.NewReader(expected), "etcd_debugging_mvcc_watcher_total") if err != nil { t.Error(err) } } t.Run("regular watch", func(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) // watcherGauge is a package variable and its value may change depending on // the execution of other tests initialGaugeState := int(testutil.ToFloat64(watcherGauge)) testKey := []byte("foo") testValue := []byte("bar") s.Put(testKey, testValue, lease.NoLease) // we expect the gauge state to still be in its initial state expectWatchGauge(initialGaugeState) w := s.NewWatchStream() defer w.Close() wt, _ := w.Watch(t.Context(), 0, testKey, nil, 0) // after creating watch, the gauge state should have increased expectWatchGauge(initialGaugeState + 1) if err := w.Cancel(wt); err != nil { t.Error(err) } // after cancelling watch, the gauge state should have decreased expectWatchGauge(initialGaugeState) w.Cancel(wt) // cancelling the watch twice shouldn't decrement the counter twice expectWatchGauge(initialGaugeState) }) t.Run("compacted watch", func(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) // watcherGauge is a package variable and its value may change depending on // the execution of other tests initialGaugeState := int(testutil.ToFloat64(watcherGauge)) testKey := []byte("foo") testValue := []byte("bar") s.Put(testKey, testValue, lease.NoLease) rev := s.Put(testKey, testValue, lease.NoLease) // compact up to the revision of the key we just put _, err := s.Compact(traceutil.TODO(), rev) if err != nil { t.Error(err) } // we expect the gauge state to still be in its initial state expectWatchGauge(initialGaugeState) w := s.NewWatchStream() defer w.Close() wt, _ := w.Watch(t.Context(), 0, testKey, nil, rev-1) // wait for the watcher to be marked as compacted select { case resp := <-w.Chan(): if resp.CompactRevision == 0 { t.Errorf("resp.Compacted = %v, want %v", resp.CompactRevision, rev) } case <-time.After(time.Second): t.Fatalf("failed to receive response (timeout)") } // after creating watch, the gauge state should have increased expectWatchGauge(initialGaugeState + 1) if err := w.Cancel(wt); err != nil { t.Error(err) } // after cancelling watch, the gauge state should have decreased expectWatchGauge(initialGaugeState) w.Cancel(wt) // cancelling the watch twice shouldn't decrement the counter twice expectWatchGauge(initialGaugeState) }) t.Run("compacted watch, close/cancel race", func(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) // watcherGauge is a package variable and its value may change depending on // the execution of other tests initialGaugeState := int(testutil.ToFloat64(watcherGauge)) testKey := []byte("foo") testValue := []byte("bar") s.Put(testKey, testValue, lease.NoLease) rev := s.Put(testKey, testValue, lease.NoLease) // compact up to the revision of the key we just put _, err := s.Compact(traceutil.TODO(), rev) if err != nil { t.Error(err) } // we expect the gauge state to still be in its initial state expectWatchGauge(initialGaugeState) w := s.NewWatchStream() wt, _ := w.Watch(t.Context(), 0, testKey, nil, rev-1) // wait for the watcher to be marked as compacted select { case resp := <-w.Chan(): if resp.CompactRevision == 0 { t.Errorf("resp.Compacted = %v, want %v", resp.CompactRevision, rev) } case <-time.After(time.Second): t.Fatalf("failed to receive response (timeout)") } // after creating watch, the gauge state should have increased expectWatchGauge(initialGaugeState + 1) // now race cancelling and closing the watcher and watch stream. // in rare scenarios the watcher cancel function can be invoked // multiple times, leading to a potentially negative gauge state, // see: https://github.com/etcd-io/etcd/issues/19577 wg := sync.WaitGroup{} wg.Add(2) go func() { w.Cancel(wt) wg.Done() }() go func() { w.Close() wg.Done() }() wg.Wait() // the gauge should be decremented to its original state expectWatchGauge(initialGaugeState) }) } // TestCancelUnsynced tests if running CancelFunc removes watchers from unsynced. func TestCancelUnsynced(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) // manually create watchableStore instead of newWatchableStore // because newWatchableStore automatically calls syncWatchers // method to sync watchers in unsynced map. We want to keep watchers // in unsynced to test if syncWatchers works as expected. s := newWatchableStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) // Put a key so that we can spawn watchers on that key. // (testKey in this test). This increases the rev to 1, // and later we can we set the watcher's startRev to 1, // and force watchers to be in unsynced. testKey := []byte("foo") testValue := []byte("bar") s.Put(testKey, testValue, lease.NoLease) w := s.NewWatchStream() defer w.Close() // arbitrary number for watchers watcherN := 100 // create watcherN of watch ids to cancel watchIDs := make([]WatchID, watcherN) for i := 0; i < watcherN; i++ { // use 1 to keep watchers in unsynced watchIDs[i], _ = w.Watch(t.Context(), 0, testKey, nil, 1) } for _, idx := range watchIDs { if err := w.Cancel(idx); err != nil { t.Error(err) } } // After running CancelFunc // // unsynced should be empty // because cancel removes watcher from unsynced if size := s.unsynced.size(); size != 0 { t.Errorf("unsynced size = %d, want 0", size) } } // TestSyncWatchers populates unsynced watcher map and tests syncWatchers // method to see if it correctly sends events to channel of unsynced watchers // and moves these watchers to synced. func TestSyncWatchers(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := newWatchableStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) testKey := []byte("foo") testValue := []byte("bar") s.Put(testKey, testValue, lease.NoLease) w := s.NewWatchStream() defer w.Close() watcherN := 100 for i := 0; i < watcherN; i++ { _, err := w.Watch(t.Context(), 0, testKey, nil, 1) require.NoError(t, err) } assert.Empty(t, s.synced.watcherSetByKey(string(testKey))) assert.Len(t, s.unsynced.watcherSetByKey(string(testKey)), watcherN) s.syncWatchers() assert.Len(t, s.synced.watcherSetByKey(string(testKey)), watcherN) assert.Empty(t, s.unsynced.watcherSetByKey(string(testKey))) require.Len(t, w.(*watchStream).ch, watcherN) for i := 0; i < watcherN; i++ { events := (<-w.(*watchStream).ch).Events assert.Len(t, events, 1) assert.Equal(t, []mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{ Key: testKey, CreateRevision: 2, ModRevision: 2, Version: 1, Value: testValue, }, }, }, events) } } func TestRangeEvents(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) lg := zaptest.NewLogger(t) s := NewStore(lg, b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) foo1 := []byte("foo1") foo2 := []byte("foo2") foo3 := []byte("foo3") value := []byte("bar") s.Put(foo1, value, lease.NoLease) s.Put(foo2, value, lease.NoLease) s.Put(foo3, value, lease.NoLease) s.DeleteRange(foo1, foo3) // Deletes "foo1" and "foo2" generating 2 events expectEvents := []mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{ Key: foo1, CreateRevision: 2, ModRevision: 2, Version: 1, Value: value, }, }, { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{ Key: foo2, CreateRevision: 3, ModRevision: 3, Version: 1, Value: value, }, }, { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{ Key: foo3, CreateRevision: 4, ModRevision: 4, Version: 1, Value: value, }, }, { Type: mvccpb.Event_DELETE, Kv: &mvccpb.KeyValue{ Key: foo1, ModRevision: 5, }, }, { Type: mvccpb.Event_DELETE, Kv: &mvccpb.KeyValue{ Key: foo2, ModRevision: 5, }, }, } tcs := []struct { minRev int64 maxRev int64 expectEvents []mvccpb.Event }{ // maxRev, top to bottom {minRev: -1, maxRev: 6, expectEvents: expectEvents[0:5]}, {minRev: -1, maxRev: 5, expectEvents: expectEvents[0:3]}, {minRev: -1, maxRev: 4, expectEvents: expectEvents[0:2]}, {minRev: -1, maxRev: 3, expectEvents: expectEvents[0:1]}, {minRev: -1, maxRev: 2, expectEvents: expectEvents[0:0]}, // minRev, bottom to top {minRev: -1, maxRev: 6, expectEvents: expectEvents[0:5]}, {minRev: 2, maxRev: 6, expectEvents: expectEvents[0:5]}, {minRev: 3, maxRev: 6, expectEvents: expectEvents[1:5]}, {minRev: 4, maxRev: 6, expectEvents: expectEvents[2:5]}, {minRev: 5, maxRev: 6, expectEvents: expectEvents[3:5]}, {minRev: 6, maxRev: 6, expectEvents: expectEvents[0:0]}, // Moving window algorithm, first increase maxRev, then increase minRev, repeat. {minRev: 2, maxRev: 2, expectEvents: expectEvents[0:0]}, {minRev: 2, maxRev: 3, expectEvents: expectEvents[0:1]}, {minRev: 2, maxRev: 4, expectEvents: expectEvents[0:2]}, {minRev: 3, maxRev: 4, expectEvents: expectEvents[1:2]}, {minRev: 3, maxRev: 5, expectEvents: expectEvents[1:3]}, {minRev: 4, maxRev: 5, expectEvents: expectEvents[2:3]}, {minRev: 4, maxRev: 6, expectEvents: expectEvents[2:5]}, {minRev: 5, maxRev: 6, expectEvents: expectEvents[3:5]}, {minRev: 6, maxRev: 6, expectEvents: expectEvents[5:5]}, } for i, tc := range tcs { t.Run(fmt.Sprintf("%d rangeEvents(%d, %d)", i, tc.minRev, tc.maxRev), func(t *testing.T) { assert.ElementsMatch(t, tc.expectEvents, rangeEvents(lg, b, tc.minRev, tc.maxRev, fakeContains{})) }) } } type fakeContains struct{} func (f fakeContains) contains(string) bool { return true } // TestWatchCompacted tests a watcher that watches on a compacted revision. func TestWatchCompacted(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) testKey := []byte("foo") testValue := []byte("bar") maxRev := 10 compactRev := int64(5) for i := 0; i < maxRev; i++ { s.Put(testKey, testValue, lease.NoLease) } _, err := s.Compact(traceutil.TODO(), compactRev) if err != nil { t.Fatalf("failed to compact kv (%v)", err) } w := s.NewWatchStream() defer w.Close() wt, _ := w.Watch(t.Context(), 0, testKey, nil, compactRev-1) select { case resp := <-w.Chan(): if resp.WatchID != wt { t.Errorf("resp.WatchID = %x, want %x", resp.WatchID, wt) } if resp.CompactRevision == 0 { t.Errorf("resp.Compacted = %v, want %v", resp.CompactRevision, compactRev) } case <-time.After(1 * time.Second): t.Fatalf("failed to receive response (timeout)") } } func TestWatchNoEventLossOnCompact(t *testing.T) { oldChanBufLen, oldMaxWatchersPerSync := chanBufLen, maxWatchersPerSync b, _ := betesting.NewDefaultTmpBackend(t) lg := zaptest.NewLogger(t) s := New(lg, b, &lease.FakeLessor{}, StoreConfig{}) defer func() { cleanup(s, b) chanBufLen, maxWatchersPerSync = oldChanBufLen, oldMaxWatchersPerSync }() chanBufLen, maxWatchersPerSync = 1, 4 testKey, testValue := []byte("foo"), []byte("bar") maxRev := 10 compactRev := int64(5) for i := 0; i < maxRev; i++ { s.Put(testKey, testValue, lease.NoLease) } _, err := s.Compact(traceutil.TODO(), compactRev) require.NoErrorf(t, err, "failed to compact kv (%v)", err) w := s.NewWatchStream() defer w.Close() watchers := map[WatchID]int64{ 0: 1, 1: 1, // create unsyncd watchers with startRev < compactRev 2: 6, // create unsyncd watchers with compactRev < startRev < currentRev } for id, startRev := range watchers { _, err := w.Watch(t.Context(), id, testKey, nil, startRev) require.NoError(t, err) } // fill up w.Chan() with 1 buf via 2 compacted watch response sImpl, ok := s.(*watchableStore) require.Truef(t, ok, "TestWatchNoEventLossOnCompact: needs a WatchableKV implementation") sImpl.syncWatchers() for len(watchers) > 0 { resp := <-w.Chan() if resp.CompactRevision != 0 { require.Equal(t, resp.CompactRevision, compactRev) require.Contains(t, watchers, resp.WatchID) delete(watchers, resp.WatchID) continue } nextRev := watchers[resp.WatchID] for _, ev := range resp.Events { require.Equalf(t, nextRev, ev.Kv.ModRevision, "got event revision %d but want %d for watcher with watch ID %d", ev.Kv.ModRevision, nextRev, resp.WatchID) nextRev++ } if nextRev == sImpl.rev()+1 { delete(watchers, resp.WatchID) } } } func TestWatchFutureRev(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) testKey := []byte("foo") testValue := []byte("bar") w := s.NewWatchStream() defer w.Close() wrev := int64(10) w.Watch(t.Context(), 0, testKey, nil, wrev) for i := 0; i < 10; i++ { rev := s.Put(testKey, testValue, lease.NoLease) if rev >= wrev { break } } select { case resp := <-w.Chan(): if resp.Revision != wrev { t.Fatalf("rev = %d, want %d", resp.Revision, wrev) } if len(resp.Events) != 1 { t.Fatalf("failed to get events from the response") } if resp.Events[0].Kv.ModRevision != wrev { t.Fatalf("kv.rev = %d, want %d", resp.Events[0].Kv.ModRevision, wrev) } case <-time.After(time.Second): t.Fatal("failed to receive event in 1 second.") } } func TestWatchRestore(t *testing.T) { resyncDelay := watchResyncPeriod * 3 / 2 t.Run("NoResync", func(t *testing.T) { testWatchRestore(t, 0, 0) }) t.Run("ResyncBefore", func(t *testing.T) { testWatchRestore(t, resyncDelay, 0) }) t.Run("ResyncAfter", func(t *testing.T) { testWatchRestore(t, 0, resyncDelay) }) t.Run("ResyncBeforeAndAfter", func(t *testing.T) { testWatchRestore(t, resyncDelay, resyncDelay) }) } func testWatchRestore(t *testing.T, delayBeforeRestore, delayAfterRestore time.Duration) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) testKey := []byte("foo") testValue := []byte("bar") tcs := []struct { name string startRevision int64 wantEvents []mvccpb.Event }{ { name: "zero revision", startRevision: 0, wantEvents: []mvccpb.Event{ {Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: testKey, Value: testValue, CreateRevision: 2, ModRevision: 2, Version: 1}}, {Type: mvccpb.Event_DELETE, Kv: &mvccpb.KeyValue{Key: testKey, ModRevision: 3}}, }, }, { name: "revision before first write", startRevision: 1, wantEvents: []mvccpb.Event{ {Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: testKey, Value: testValue, CreateRevision: 2, ModRevision: 2, Version: 1}}, {Type: mvccpb.Event_DELETE, Kv: &mvccpb.KeyValue{Key: testKey, ModRevision: 3}}, }, }, { name: "revision of first write", startRevision: 2, wantEvents: []mvccpb.Event{ {Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: testKey, Value: testValue, CreateRevision: 2, ModRevision: 2, Version: 1}}, {Type: mvccpb.Event_DELETE, Kv: &mvccpb.KeyValue{Key: testKey, ModRevision: 3}}, }, }, { name: "current revision", startRevision: 3, wantEvents: []mvccpb.Event{ {Type: mvccpb.Event_DELETE, Kv: &mvccpb.KeyValue{Key: testKey, ModRevision: 3}}, }, }, { name: "future revision", startRevision: 4, wantEvents: []mvccpb.Event{}, }, } watchers := []WatchStream{} for i, tc := range tcs { w := s.NewWatchStream() defer w.Close() watchers = append(watchers, w) w.Watch(t.Context(), WatchID(i+1), testKey, nil, tc.startRevision) } s.Put(testKey, testValue, lease.NoLease) time.Sleep(delayBeforeRestore) s.Restore(b) time.Sleep(delayAfterRestore) s.DeleteRange(testKey, nil) for i, tc := range tcs { t.Run(tc.name, func(t *testing.T) { events := readEventsForSecond(t, watchers[i].Chan()) if diff := cmp.Diff(tc.wantEvents, events); diff != "" { t.Errorf("unexpected events (-want +got):\n%s", diff) } }) } } func readEventsForSecond(t *testing.T, ws <-chan WatchResponse) []mvccpb.Event { events := []mvccpb.Event{} deadline := time.After(time.Second) for { select { case resp := <-ws: if len(resp.Events) == 0 { t.Fatalf("Events should never be empty, resp: %+v", resp) } events = append(events, resp.Events...) case <-deadline: return events case <-time.After(watchResyncPeriod * 3 / 2): return events } } } // TestWatchBatchUnsynced tests batching on unsynced watchers func TestWatchBatchUnsynced(t *testing.T) { tcs := []struct { name string revisions int watchBatchMaxRevs int eventsPerRevision int expectRevisionBatches [][]int64 }{ { name: "3 revisions, 4 revs per batch, 1 events per revision", revisions: 12, watchBatchMaxRevs: 4, eventsPerRevision: 1, expectRevisionBatches: [][]int64{ {2, 3, 4, 5}, {6, 7, 8, 9}, {10, 11, 12, 13}, }, }, { name: "3 revisions, 4 revs per batch, 3 events per revision", revisions: 12, watchBatchMaxRevs: 4, eventsPerRevision: 3, expectRevisionBatches: [][]int64{ {2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5}, {6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9}, {10, 10, 10, 11, 11, 11, 12, 12, 12, 13, 13, 13}, }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) oldMaxRevs := watchBatchMaxRevs defer func() { watchBatchMaxRevs = oldMaxRevs cleanup(s, b) }() watchBatchMaxRevs = tc.watchBatchMaxRevs v := []byte("foo") for i := 0; i < tc.revisions; i++ { txn := s.Write(traceutil.TODO()) for j := 0; j < tc.eventsPerRevision; j++ { txn.Put(v, v, lease.NoLease) } txn.End() } w := s.NewWatchStream() defer w.Close() w.Watch(t.Context(), 0, v, nil, 1) var revisionBatches [][]int64 eventCount := 0 for eventCount < tc.revisions*tc.eventsPerRevision { var revisions []int64 for _, e := range (<-w.Chan()).Events { revisions = append(revisions, e.Kv.ModRevision) eventCount++ } revisionBatches = append(revisionBatches, revisions) } assert.Equal(t, tc.expectRevisionBatches, revisionBatches) sImpl, ok := s.(*watchableStore) require.Truef(t, ok, "TestWatchBatchUnsynced: needs a WatchableKV implementation") sImpl.store.revMu.Lock() defer sImpl.store.revMu.Unlock() assert.Equal(t, 1, sImpl.synced.size()) assert.Equal(t, 0, sImpl.unsynced.size()) }) } } func TestNewMapwatcherToEventMap(t *testing.T) { k0, k1, k2 := []byte("foo0"), []byte("foo1"), []byte("foo2") v0, v1, v2 := []byte("bar0"), []byte("bar1"), []byte("bar2") ws := []*watcher{{key: k0}, {key: k1}, {key: k2}} evs := []mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: k0, Value: v0}, }, { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: k1, Value: v1}, }, { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: k2, Value: v2}, }, } tests := []struct { sync []*watcher evs []mvccpb.Event wwe map[*watcher][]mvccpb.Event }{ // no watcher in sync, some events should return empty wwe { nil, evs, map[*watcher][]mvccpb.Event{}, }, // one watcher in sync, one event that does not match the key of that // watcher should return empty wwe { []*watcher{ws[2]}, evs[:1], map[*watcher][]mvccpb.Event{}, }, // one watcher in sync, one event that matches the key of that // watcher should return wwe with that matching watcher { []*watcher{ws[1]}, evs[1:2], map[*watcher][]mvccpb.Event{ ws[1]: evs[1:2], }, }, // two watchers in sync that watches two different keys, one event // that matches the key of only one of the watcher should return wwe // with the matching watcher { []*watcher{ws[0], ws[2]}, evs[2:], map[*watcher][]mvccpb.Event{ ws[2]: evs[2:], }, }, // two watchers in sync that watches the same key, two events that // match the keys should return wwe with those two watchers { []*watcher{ws[0], ws[1]}, evs[:2], map[*watcher][]mvccpb.Event{ ws[0]: evs[:1], ws[1]: evs[1:2], }, }, } for i, tt := range tests { wg := newWatcherGroup() for _, w := range tt.sync { wg.add(w) } gwe := newWatcherBatch(&wg, tt.evs) if len(gwe) != len(tt.wwe) { t.Errorf("#%d: len(gwe) got = %d, want = %d", i, len(gwe), len(tt.wwe)) } // compare gwe and tt.wwe for w, eb := range gwe { if len(eb.evs) != len(tt.wwe[w]) { t.Errorf("#%d: len(eb.evs) got = %d, want = %d", i, len(eb.evs), len(tt.wwe[w])) } if !reflect.DeepEqual(eb.evs, tt.wwe[w]) { t.Errorf("#%d: reflect.DeepEqual events got = %v, want = true", i, false) } } } } // TestWatchVictims tests that watchable store delivers watch events // when the watch channel is temporarily clogged with too many events. func TestWatchVictims(t *testing.T) { oldChanBufLen, oldMaxWatchersPerSync := chanBufLen, maxWatchersPerSync b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer func() { cleanup(s, b) chanBufLen, maxWatchersPerSync = oldChanBufLen, oldMaxWatchersPerSync }() chanBufLen, maxWatchersPerSync = 1, 2 numPuts := chanBufLen * 64 testKey, testValue := []byte("foo"), []byte("bar") var wg sync.WaitGroup numWatches := maxWatchersPerSync * 128 errc := make(chan error, numWatches) wg.Add(numWatches) for i := 0; i < numWatches; i++ { go func() { w := s.NewWatchStream() w.Watch(t.Context(), 0, testKey, nil, 1) defer func() { w.Close() wg.Done() }() tc := time.After(10 * time.Second) evs, nextRev := 0, int64(2) for evs < numPuts { select { case <-tc: errc <- fmt.Errorf("time out") return case wr := <-w.Chan(): evs += len(wr.Events) for _, ev := range wr.Events { if ev.Kv.ModRevision != nextRev { errc <- fmt.Errorf("expected rev=%d, got %d", nextRev, ev.Kv.ModRevision) return } nextRev++ } time.Sleep(time.Millisecond) } } if evs != numPuts { errc <- fmt.Errorf("expected %d events, got %d", numPuts, evs) return } select { case <-w.Chan(): errc <- fmt.Errorf("unexpected response") default: } }() time.Sleep(time.Millisecond) } var wgPut sync.WaitGroup wgPut.Add(numPuts) for i := 0; i < numPuts; i++ { go func() { defer wgPut.Done() s.Put(testKey, testValue, lease.NoLease) }() } wgPut.Wait() wg.Wait() select { case err := <-errc: t.Fatal(err) default: } } // TestStressWatchCancelClose tests closing a watch stream while // canceling its watches. func TestStressWatchCancelClose(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) testKey, testValue := []byte("foo"), []byte("bar") var wg sync.WaitGroup readyc := make(chan struct{}) wg.Add(100) for i := 0; i < 100; i++ { go func() { defer wg.Done() w := s.NewWatchStream() ids := make([]WatchID, 10) for i := range ids { ids[i], _ = w.Watch(t.Context(), 0, testKey, nil, 0) } <-readyc wg.Add(1 + len(ids)/2) for i := range ids[:len(ids)/2] { go func(n int) { defer wg.Done() w.Cancel(ids[n]) }(i) } go func() { defer wg.Done() w.Close() }() }() } close(readyc) for i := 0; i < 100; i++ { s.Put(testKey, testValue, lease.NoLease) } wg.Wait() } ================================================ FILE: server/storage/mvcc/watchable_store_txn.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/pkg/v3/traceutil" ) func (tw *watchableStoreTxnWrite) End() { changes := tw.Changes() if len(changes) == 0 { tw.TxnWrite.End() return } rev := tw.Rev() + 1 evs := make([]mvccpb.Event, len(changes)) for i, change := range changes { evs[i].Kv = &changes[i] if change.CreateRevision == 0 { evs[i].Type = mvccpb.Event_DELETE evs[i].Kv.ModRevision = rev } else { evs[i].Type = mvccpb.Event_PUT } } // end write txn under watchable store lock so the updates are visible // when asynchronous event posting checks the current store revision tw.s.mu.Lock() tw.s.notify(rev, evs) tw.TxnWrite.End() tw.s.mu.Unlock() } type watchableStoreTxnWrite struct { TxnWrite s *watchableStore } func (s *watchableStore) Write(trace *traceutil.Trace) TxnWrite { return &watchableStoreTxnWrite{s.store.Write(trace), s} } ================================================ FILE: server/storage/mvcc/watcher.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "bytes" "context" "errors" "sync" "go.opentelemetry.io/otel/trace" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" ) var ( ErrWatcherNotExist = errors.New("mvcc: watcher does not exist") ErrEmptyWatcherRange = errors.New("mvcc: watcher range is empty") ErrWatcherDuplicateID = errors.New("mvcc: duplicate watch ID provided on the WatchStream") ) type WatchID int64 // FilterFunc returns true if the given event should be filtered out. type FilterFunc func(e mvccpb.Event) bool type WatchStream interface { // Watch creates a watcher. The watcher watches the events happening or // happened on the given key or range [key, end) from the given startRev. // // The whole event history can be watched unless compacted. // If "startRev" <=0, watch observes events after currentRev. // // The returned "id" is the ID of this watcher. It appears as WatchID // in events that are sent to the created watcher through stream channel. // The watch ID is used when it's not equal to AutoWatchID. Otherwise, // an auto-generated watch ID is returned. Watch(ctx context.Context, id WatchID, key, end []byte, startRev int64, fcs ...FilterFunc) (WatchID, error) // Chan returns a chan. All watch response will be sent to the returned chan. Chan() <-chan WatchResponse // RequestProgress requests the progress of the watcher with given ID. The response // will only be sent if the watcher is currently synced. // The responses will be sent through the WatchRespone Chan attached // with this stream to ensure correct ordering. // The responses contains no events. The revision in the response is the progress // of the watchers since the watcher is currently synced. RequestProgress(id WatchID) // RequestProgressAll requests a progress notification for all // watchers sharing the stream. If all watchers are synced, a // progress notification with watch ID -1 will be sent to an // arbitrary watcher of this stream, and the function returns // true. RequestProgressAll() bool // Cancel cancels a watcher by giving its ID. If watcher does not exist, an error will be // returned. Cancel(id WatchID) error // Close closes Chan and release all related resources. Close() // Rev returns the current revision of the KV the stream watches on. Rev() int64 } type WatchResponse struct { // WatchID is the WatchID of the watcher this response sent to. WatchID WatchID // Events contains all the events that needs to send. Events []mvccpb.Event // Revision is the revision of the KV when the watchResponse is created. // For a normal response, the revision should be the same as the last // modified revision inside Events. For a delayed response to a unsynced // watcher, the revision is greater than the last modified revision // inside Events. Revision int64 // CompactRevision is set when the watcher is cancelled due to compaction. CompactRevision int64 } // watchStream contains a collection of watchers that share // one streaming chan to send out watched events and other control events. type watchStream struct { watchable watchable ch chan WatchResponse mu sync.Mutex // guards fields below it // nextID is the ID pre-allocated for next new watcher in this stream nextID WatchID closed bool cancels map[WatchID]cancelFunc watchers map[WatchID]*watcher } // Watch creates a new watcher in the stream and returns its WatchID. func (ws *watchStream) Watch(ctx context.Context, id WatchID, key, end []byte, startRev int64, fcs ...FilterFunc) (WatchID, error) { // prevent wrong range where key >= end lexicographically // watch request with 'WithFromKey' has empty-byte range end if len(end) != 0 && bytes.Compare(key, end) != -1 { return -1, ErrEmptyWatcherRange } ws.mu.Lock() defer ws.mu.Unlock() if ws.closed { return -1, ErrEmptyWatcherRange } if id == clientv3.AutoWatchID { for ws.watchers[ws.nextID] != nil { ws.nextID++ } id = ws.nextID ws.nextID++ } else if _, ok := ws.watchers[id]; ok { return -1, ErrWatcherDuplicateID } w, c := ws.watchable.watch(key, end, startRev, id, ws.ch, fcs...) span := trace.SpanFromContext(ctx) ws.cancels[id] = func() { defer span.End() c() } ws.watchers[id] = w return id, nil } func (ws *watchStream) Chan() <-chan WatchResponse { return ws.ch } func (ws *watchStream) Cancel(id WatchID) error { ws.mu.Lock() cancel, ok := ws.cancels[id] w := ws.watchers[id] ok = ok && !ws.closed ws.mu.Unlock() if !ok { return ErrWatcherNotExist } cancel() ws.mu.Lock() // The watch isn't removed until cancel so that if Close() is called, // it will wait for the cancel. Otherwise, Close() could close the // watch channel while the store is still posting events. if ww := ws.watchers[id]; ww == w { delete(ws.cancels, id) delete(ws.watchers, id) } ws.mu.Unlock() return nil } func (ws *watchStream) Close() { ws.mu.Lock() defer ws.mu.Unlock() for _, cancel := range ws.cancels { cancel() } ws.closed = true close(ws.ch) watchStreamGauge.Dec() } func (ws *watchStream) Rev() int64 { ws.mu.Lock() defer ws.mu.Unlock() return ws.watchable.rev() } func (ws *watchStream) RequestProgress(id WatchID) { ws.mu.Lock() w, ok := ws.watchers[id] ws.mu.Unlock() if !ok { return } ws.watchable.progress(w) } func (ws *watchStream) RequestProgressAll() bool { ws.mu.Lock() defer ws.mu.Unlock() return ws.watchable.progressAll(ws.watchers) } ================================================ FILE: server/storage/mvcc/watcher_bench_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "fmt" "testing" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) func BenchmarkKVWatcherMemoryUsage(b *testing.B) { be, _ := betesting.NewDefaultTmpBackend(b) watchable := New(zaptest.NewLogger(b), be, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(watchable, be) w := watchable.NewWatchStream() defer w.Close() b.ReportAllocs() b.StartTimer() for i := 0; i < b.N; i++ { w.Watch(b.Context(), 0, []byte(fmt.Sprint("foo", i)), nil, 0) } } ================================================ FILE: server/storage/mvcc/watcher_group.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "fmt" "math" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/pkg/v3/adt" ) // watchBatchMaxRevs is the maximum distinct revisions that // may be sent to an unsynced watcher at a time. Declared as // var instead of const for testing purposes. var watchBatchMaxRevs = 1000 type eventBatch struct { // evs is a batch of revision-ordered events evs []mvccpb.Event // revs is the minimum unique revisions observed for this batch revs int // moreRev is first revision with more events following this batch moreRev int64 } func (eb *eventBatch) add(ev mvccpb.Event) { if eb.revs > watchBatchMaxRevs { // maxed out batch size return } if len(eb.evs) == 0 { // base case eb.revs = 1 eb.evs = append(eb.evs, ev) return } // revision accounting ebRev := eb.evs[len(eb.evs)-1].Kv.ModRevision evRev := ev.Kv.ModRevision if evRev > ebRev { eb.revs++ if eb.revs > watchBatchMaxRevs { eb.moreRev = evRev return } } eb.evs = append(eb.evs, ev) } type watcherBatch map[*watcher]*eventBatch func (wb watcherBatch) add(w *watcher, ev mvccpb.Event) { eb := wb[w] if eb == nil { eb = &eventBatch{} wb[w] = eb } eb.add(ev) } // newWatcherBatch maps watchers to their matched events. It enables quick // events look up by watcher. func newWatcherBatch(wg *watcherGroup, evs []mvccpb.Event) watcherBatch { if len(wg.watchers) == 0 { return nil } wb := make(watcherBatch) for _, ev := range evs { for w := range wg.watcherSetByKey(string(ev.Kv.Key)) { if ev.Kv.ModRevision >= w.minRev { // don't double notify wb.add(w, ev) } } } return wb } type watcherSet map[*watcher]struct{} func (w watcherSet) add(wa *watcher) { if _, ok := w[wa]; ok { panic("add watcher twice!") } w[wa] = struct{}{} } func (w watcherSet) union(ws watcherSet) { for wa := range ws { w.add(wa) } } func (w watcherSet) delete(wa *watcher) { if _, ok := w[wa]; !ok { panic("removing missing watcher!") } delete(w, wa) } type watcherSetByKey map[string]watcherSet func (w watcherSetByKey) add(wa *watcher) { set := w[string(wa.key)] if set == nil { set = make(watcherSet) w[string(wa.key)] = set } set.add(wa) } func (w watcherSetByKey) delete(wa *watcher) bool { k := string(wa.key) if v, ok := w[k]; ok { if _, ok := v[wa]; ok { delete(v, wa) if len(v) == 0 { // remove the set; nothing left delete(w, k) } return true } } return false } // watcherGroup is a collection of watchers organized by their ranges type watcherGroup struct { // keyWatchers has the watchers that watch on a single key keyWatchers watcherSetByKey // ranges has the watchers that watch a range; it is sorted by interval ranges adt.IntervalTree // watchers is the set of all watchers watchers watcherSet } func newWatcherGroup() watcherGroup { return watcherGroup{ keyWatchers: make(watcherSetByKey), ranges: adt.NewIntervalTree(), watchers: make(watcherSet), } } // add puts a watcher in the group. func (wg *watcherGroup) add(wa *watcher) { wg.watchers.add(wa) if wa.end == nil { wg.keyWatchers.add(wa) return } // interval already registered? ivl := adt.NewStringAffineInterval(string(wa.key), string(wa.end)) if iv := wg.ranges.Find(ivl); iv != nil { iv.Val.(watcherSet).add(wa) return } // not registered, put in interval tree ws := make(watcherSet) ws.add(wa) wg.ranges.Insert(ivl, ws) } // contains is whether the given key has a watcher in the group. func (wg *watcherGroup) contains(key string) bool { _, ok := wg.keyWatchers[key] return ok || wg.ranges.Intersects(adt.NewStringAffinePoint(key)) } // size gives the number of unique watchers in the group. func (wg *watcherGroup) size() int { return len(wg.watchers) } // delete removes a watcher from the group. func (wg *watcherGroup) delete(wa *watcher) bool { if _, ok := wg.watchers[wa]; !ok { return false } wg.watchers.delete(wa) if wa.end == nil { wg.keyWatchers.delete(wa) return true } ivl := adt.NewStringAffineInterval(string(wa.key), string(wa.end)) iv := wg.ranges.Find(ivl) if iv == nil { return false } ws := iv.Val.(watcherSet) delete(ws, wa) if len(ws) == 0 { // remove interval missing watchers if ok := wg.ranges.Delete(ivl); !ok { panic("could not remove watcher from interval tree") } } return true } // choose selects watchers from the watcher group to update func (wg *watcherGroup) choose(maxWatchers int, curRev, compactRev int64) (*watcherGroup, int64) { if len(wg.watchers) < maxWatchers { return wg, wg.chooseAll(curRev, compactRev) } ret := newWatcherGroup() for w := range wg.watchers { if maxWatchers <= 0 { break } maxWatchers-- ret.add(w) } return &ret, ret.chooseAll(curRev, compactRev) } func (wg *watcherGroup) chooseAll(curRev, compactRev int64) int64 { minRev := int64(math.MaxInt64) for w := range wg.watchers { if w.minRev > curRev { // after network partition, possibly choosing future revision watcher from restore operation // with watch key "proxy-namespace__lostleader" and revision "math.MaxInt64 - 2" // do not panic when such watcher had been moved from "synced" watcher during restore operation if !w.restore { panic(fmt.Errorf("watcher minimum revision %d should not exceed current revision %d", w.minRev, curRev)) } // mark 'restore' done, since it's chosen w.restore = false } if w.minRev < compactRev { select { case w.ch <- WatchResponse{WatchID: w.id, CompactRevision: compactRev}: w.compacted = true wg.delete(w) default: // retry next time } continue } if minRev > w.minRev { minRev = w.minRev } } return minRev } // watcherSetByKey gets the set of watchers that receive events on the given key. func (wg *watcherGroup) watcherSetByKey(key string) watcherSet { wkeys := wg.keyWatchers[key] wranges := wg.ranges.Stab(adt.NewStringAffinePoint(key)) // zero-copy cases switch { case len(wranges) == 0: // no need to merge ranges or copy; reuse single-key set return wkeys case len(wranges) == 0 && len(wkeys) == 0: return nil case len(wranges) == 1 && len(wkeys) == 0: return wranges[0].Val.(watcherSet) } // copy case ret := make(watcherSet) ret.union(wg.keyWatchers[key]) for _, item := range wranges { ret.union(item.Val.(watcherSet)) } return ret } ================================================ FILE: server/storage/mvcc/watcher_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mvcc import ( "bytes" "errors" "fmt" "os" "reflect" "testing" "time" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/lease" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) // TestWatcherWatchID tests that each watcher provides unique watchID, // and the watched event attaches the correct watchID. func TestWatcherWatchID(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) w := s.NewWatchStream() defer w.Close() idm := make(map[WatchID]struct{}) for i := 0; i < 10; i++ { id, _ := w.Watch(t.Context(), 0, []byte("foo"), nil, 0) if _, ok := idm[id]; ok { t.Errorf("#%d: id %d exists", i, id) } idm[id] = struct{}{} s.Put([]byte("foo"), []byte("bar"), lease.NoLease) resp := <-w.Chan() if resp.WatchID != id { t.Errorf("#%d: watch id in event = %d, want %d", i, resp.WatchID, id) } if err := w.Cancel(id); err != nil { t.Error(err) } } s.Put([]byte("foo2"), []byte("bar"), lease.NoLease) // unsynced watchers for i := 10; i < 20; i++ { id, _ := w.Watch(t.Context(), 0, []byte("foo2"), nil, 1) if _, ok := idm[id]; ok { t.Errorf("#%d: id %d exists", i, id) } idm[id] = struct{}{} resp := <-w.Chan() if resp.WatchID != id { t.Errorf("#%d: watch id in event = %d, want %d", i, resp.WatchID, id) } if err := w.Cancel(id); err != nil { t.Error(err) } } } func TestWatcherRequestsCustomID(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) w := s.NewWatchStream() defer w.Close() // - Request specifically ID #1 // - Try to duplicate it, get an error // - Make sure the auto-assignment skips over things we manually assigned tt := []struct { givenID WatchID expectedID WatchID expectedErr error }{ {1, 1, nil}, {1, 0, ErrWatcherDuplicateID}, {0, 0, nil}, {0, 2, nil}, } for i, tcase := range tt { id, err := w.Watch(t.Context(), tcase.givenID, []byte("foo"), nil, 0) if tcase.expectedErr != nil || err != nil { if !errors.Is(err, tcase.expectedErr) { t.Errorf("expected get error %q in test case %d, got %q", tcase.expectedErr, i, err) } } else if tcase.expectedID != id { t.Errorf("expected to create ID %d, got %d in test case %d", tcase.expectedID, id, i) } } } // TestWatcherWatchPrefix tests if Watch operation correctly watches // and returns events with matching prefixes. func TestWatcherWatchPrefix(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) w := s.NewWatchStream() defer w.Close() idm := make(map[WatchID]struct{}) val := []byte("bar") keyWatch, keyEnd, keyPut := []byte("foo"), []byte("fop"), []byte("foobar") for i := 0; i < 10; i++ { id, _ := w.Watch(t.Context(), 0, keyWatch, keyEnd, 0) if _, ok := idm[id]; ok { t.Errorf("#%d: unexpected duplicated id %x", i, id) } idm[id] = struct{}{} s.Put(keyPut, val, lease.NoLease) resp := <-w.Chan() if resp.WatchID != id { t.Errorf("#%d: watch id in event = %d, want %d", i, resp.WatchID, id) } if err := w.Cancel(id); err != nil { t.Errorf("#%d: unexpected cancel error %v", i, err) } if len(resp.Events) != 1 { t.Errorf("#%d: len(resp.Events) got = %d, want = 1", i, len(resp.Events)) } if len(resp.Events) == 1 { if !bytes.Equal(resp.Events[0].Kv.Key, keyPut) { t.Errorf("#%d: resp.Events got = %s, want = %s", i, resp.Events[0].Kv.Key, keyPut) } } } keyWatch1, keyEnd1, keyPut1 := []byte("foo1"), []byte("foo2"), []byte("foo1bar") s.Put(keyPut1, val, lease.NoLease) // unsynced watchers for i := 10; i < 15; i++ { id, _ := w.Watch(t.Context(), 0, keyWatch1, keyEnd1, 1) if _, ok := idm[id]; ok { t.Errorf("#%d: id %d exists", i, id) } idm[id] = struct{}{} resp := <-w.Chan() if resp.WatchID != id { t.Errorf("#%d: watch id in event = %d, want %d", i, resp.WatchID, id) } if err := w.Cancel(id); err != nil { t.Error(err) } if len(resp.Events) != 1 { t.Errorf("#%d: len(resp.Events) got = %d, want = 1", i, len(resp.Events)) } if len(resp.Events) == 1 { if !bytes.Equal(resp.Events[0].Kv.Key, keyPut1) { t.Errorf("#%d: resp.Events got = %s, want = %s", i, resp.Events[0].Kv.Key, keyPut1) } } } } // TestWatcherWatchWrongRange ensures that watcher with wrong 'end' range // does not create watcher, which panics when canceling in range tree. func TestWatcherWatchWrongRange(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) w := s.NewWatchStream() defer w.Close() if _, err := w.Watch(t.Context(), 0, []byte("foa"), []byte("foa"), 1); !errors.Is(err, ErrEmptyWatcherRange) { t.Fatalf("key == end range given; expected ErrEmptyWatcherRange, got %+v", err) } if _, err := w.Watch(t.Context(), 0, []byte("fob"), []byte("foa"), 1); !errors.Is(err, ErrEmptyWatcherRange) { t.Fatalf("key > end range given; expected ErrEmptyWatcherRange, got %+v", err) } // watch request with 'WithFromKey' has empty-byte range end if id, _ := w.Watch(t.Context(), 0, []byte("foo"), []byte{}, 1); id != 0 { t.Fatalf("\x00 is range given; id expected 0, got %d", id) } } func TestWatchDeleteRange(t *testing.T) { b, tmpPath := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer func() { b.Close() s.Close() os.Remove(tmpPath) }() testKeyPrefix := []byte("foo") for i := 0; i < 3; i++ { s.Put([]byte(fmt.Sprintf("%s_%d", testKeyPrefix, i)), []byte("bar"), lease.NoLease) } w := s.NewWatchStream() from, to := testKeyPrefix, []byte(fmt.Sprintf("%s_%d", testKeyPrefix, 99)) w.Watch(t.Context(), 0, from, to, 0) s.DeleteRange(from, to) we := []mvccpb.Event{ {Type: mvccpb.Event_DELETE, Kv: &mvccpb.KeyValue{Key: []byte("foo_0"), ModRevision: 5}}, {Type: mvccpb.Event_DELETE, Kv: &mvccpb.KeyValue{Key: []byte("foo_1"), ModRevision: 5}}, {Type: mvccpb.Event_DELETE, Kv: &mvccpb.KeyValue{Key: []byte("foo_2"), ModRevision: 5}}, } select { case r := <-w.Chan(): if !reflect.DeepEqual(r.Events, we) { t.Errorf("event = %v, want %v", r.Events, we) } case <-time.After(10 * time.Second): t.Fatal("failed to receive event after 10 seconds!") } } // TestWatchStreamCancelWatcherByID ensures cancel calls the cancel func of the watcher // with given id inside watchStream. func TestWatchStreamCancelWatcherByID(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) w := s.NewWatchStream() defer w.Close() id, _ := w.Watch(t.Context(), 0, []byte("foo"), nil, 0) tests := []struct { cancelID WatchID werr error }{ // no error should be returned when cancel the created watcher. {id, nil}, // not exist error should be returned when cancel again. {id, ErrWatcherNotExist}, // not exist error should be returned when cancel a bad id. {id + 1, ErrWatcherNotExist}, } for i, tt := range tests { gerr := w.Cancel(tt.cancelID) if !errors.Is(gerr, tt.werr) { t.Errorf("#%d: err = %v, want %v", i, gerr, tt.werr) } } if l := len(w.(*watchStream).cancels); l != 0 { t.Errorf("cancels = %d, want 0", l) } } func TestWatcherRequestProgressBadId(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := newWatchableStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) w := s.NewWatchStream() badID := WatchID(1000) w.RequestProgress(badID) select { case resp := <-w.Chan(): t.Fatalf("unexpected %+v", resp) default: } } func TestWatcherRequestProgress(t *testing.T) { testKey := []byte("foo") notTestKey := []byte("bad") testValue := []byte("bar") tcs := []struct { name string startRev int64 expectProgressBeforeSync bool expectProgressAfterSync bool }{ { name: "Zero revision", startRev: 0, expectProgressBeforeSync: true, expectProgressAfterSync: true, }, { name: "Old revision", startRev: 1, expectProgressAfterSync: true, }, { name: "Current revision", startRev: 2, expectProgressAfterSync: true, }, { name: "Current revision plus one", startRev: 3, }, { name: "Current revision plus two", startRev: 4, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := newWatchableStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) s.Put(testKey, testValue, lease.NoLease) w := s.NewWatchStream() id, _ := w.Watch(t.Context(), 0, notTestKey, nil, tc.startRev) w.RequestProgress(id) asssertProgressSent(t, w, id, tc.expectProgressBeforeSync) s.syncWatchers() w.RequestProgress(id) asssertProgressSent(t, w, id, tc.expectProgressAfterSync) }) } } func asssertProgressSent(t *testing.T, stream WatchStream, id WatchID, expectProgress bool) { select { case resp := <-stream.Chan(): if expectProgress { wrs := WatchResponse{WatchID: id, Revision: 2} if !reflect.DeepEqual(resp, wrs) { t.Fatalf("got %+v, expect %+v", resp, wrs) } } else { t.Fatalf("unexpected response %+v", resp) } default: if expectProgress { t.Fatalf("failed to receive progress") } } } func TestWatcherRequestProgressAll(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := newWatchableStore(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) testKey := []byte("foo") notTestKey := []byte("bad") testValue := []byte("bar") s.Put(testKey, testValue, lease.NoLease) // Create watch stream with watcher. We will not actually get // any notifications on it specifically, but there needs to be // at least one Watch for progress notifications to get // generated. w := s.NewWatchStream() w.Watch(t.Context(), 0, notTestKey, nil, 1) w.RequestProgressAll() select { case resp := <-w.Chan(): t.Fatalf("unexpected %+v", resp) default: } s.syncWatchers() w.RequestProgressAll() wrs := WatchResponse{WatchID: clientv3.InvalidWatchID, Revision: 2} select { case resp := <-w.Chan(): if !reflect.DeepEqual(resp, wrs) { t.Fatalf("got %+v, expect %+v", resp, wrs) } case <-time.After(time.Second): t.Fatal("failed to receive progress") } } func TestWatcherWatchWithFilter(t *testing.T) { b, _ := betesting.NewDefaultTmpBackend(t) s := New(zaptest.NewLogger(t), b, &lease.FakeLessor{}, StoreConfig{}) defer cleanup(s, b) w := s.NewWatchStream() defer w.Close() filterPut := func(e mvccpb.Event) bool { return e.Type == mvccpb.Event_PUT } w.Watch(t.Context(), 0, []byte("foo"), nil, 0, filterPut) done := make(chan struct{}, 1) go func() { <-w.Chan() done <- struct{}{} }() s.Put([]byte("foo"), []byte("bar"), 0) select { case <-done: t.Fatal("failed to filter put request") case <-time.After(100 * time.Millisecond): } s.DeleteRange([]byte("foo"), nil) select { case <-done: case <-time.After(100 * time.Millisecond): t.Fatal("failed to receive delete request") } } ================================================ FILE: server/storage/quota.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage import ( "sync" humanize "github.com/dustin/go-humanize" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/storage/backend" ) const ( // DefaultQuotaBytes is the number of bytes the backend Size may // consume before exceeding the space quota. DefaultQuotaBytes = int64(2 * 1024 * 1024 * 1024) // 2GB // MaxQuotaBytes is the maximum number of bytes suggested for a backend // quota. A larger quota may lead to degraded performance. MaxQuotaBytes = int64(8 * 1024 * 1024 * 1024) // 8GB ) // Quota represents an arbitrary quota against arbitrary requests. Each request // costs some charge; if there is not enough remaining charge, then there are // too few resources available within the quota to apply the request. type Quota interface { // Available judges whether the given request fits within the quota. Available(req any) bool // Cost computes the charge against the quota for a given request. Cost(req any) int // Remaining is the amount of charge left for the quota. Remaining() int64 } type passthroughQuota struct{} func (*passthroughQuota) Available(any) bool { return true } func (*passthroughQuota) Cost(any) int { return 0 } func (*passthroughQuota) Remaining() int64 { return 1 } type BackendQuota struct { be backend.Backend maxBackendBytes int64 } const ( // leaseOverhead is an estimate for the cost of storing a lease leaseOverhead = 64 // kvOverhead is an estimate for the cost of storing a key's Metadata kvOverhead = 256 ) var ( // only log once quotaLogOnce sync.Once DefaultQuotaSize = humanize.Bytes(uint64(DefaultQuotaBytes)) maxQuotaSize = humanize.Bytes(uint64(MaxQuotaBytes)) ) // NewBackendQuota creates a quota layer with the given storage limit. func NewBackendQuota(lg *zap.Logger, quotaBackendBytesCfg int64, be backend.Backend, name string) Quota { quotaBackendBytes.Set(float64(quotaBackendBytesCfg)) if quotaBackendBytesCfg < 0 { // disable quotas if negative quotaLogOnce.Do(func() { lg.Info( "disabled backend quota", zap.String("quota-name", name), zap.Int64("quota-size-bytes", quotaBackendBytesCfg), ) }) return &passthroughQuota{} } if quotaBackendBytesCfg == 0 { // use default size if no quota size given quotaLogOnce.Do(func() { if lg != nil { lg.Info( "enabled backend quota with default value", zap.String("quota-name", name), zap.Int64("quota-size-bytes", DefaultQuotaBytes), zap.String("quota-size", DefaultQuotaSize), ) } }) quotaBackendBytes.Set(float64(DefaultQuotaBytes)) return &BackendQuota{be, DefaultQuotaBytes} } quotaLogOnce.Do(func() { if quotaBackendBytesCfg > MaxQuotaBytes { lg.Warn( "quota exceeds the maximum value", zap.String("quota-name", name), zap.Int64("quota-size-bytes", quotaBackendBytesCfg), zap.String("quota-size", humanize.Bytes(uint64(quotaBackendBytesCfg))), zap.Int64("quota-maximum-size-bytes", MaxQuotaBytes), zap.String("quota-maximum-size", maxQuotaSize), ) } lg.Info( "enabled backend quota", zap.String("quota-name", name), zap.Int64("quota-size-bytes", quotaBackendBytesCfg), zap.String("quota-size", humanize.Bytes(uint64(quotaBackendBytesCfg))), ) }) return &BackendQuota{be, quotaBackendBytesCfg} } func (b *BackendQuota) Available(v any) bool { cost := b.Cost(v) // if there are no mutating requests, it's safe to pass through if cost == 0 { return true } // TODO: maybe optimize Backend.Size() return b.be.Size()+int64(cost) < b.maxBackendBytes } func (b *BackendQuota) Cost(v any) int { switch r := v.(type) { case *pb.PutRequest: return costPut(r) case *pb.TxnRequest: return costTxn(r) case *pb.LeaseGrantRequest: return leaseOverhead default: panic("unexpected cost") } } func costPut(r *pb.PutRequest) int { return kvOverhead + len(r.Key) + len(r.Value) } func costTxnReq(u *pb.RequestOp) int { r := u.GetRequestPut() if r == nil { return 0 } return costPut(r) } func costTxn(r *pb.TxnRequest) int { sizeSuccess := 0 for _, u := range r.Success { sizeSuccess += costTxnReq(u) } sizeFailure := 0 for _, u := range r.Failure { sizeFailure += costTxnReq(u) } if sizeFailure > sizeSuccess { return sizeFailure } return sizeSuccess } func (b *BackendQuota) Remaining() int64 { return b.maxBackendBytes - b.be.Size() } ================================================ FILE: server/storage/schema/actions.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "go.uber.org/zap" "go.etcd.io/etcd/server/v3/storage/backend" ) type action interface { // unsafeDo executes the action and returns revert action, when executed // should restore the state from before. unsafeDo(tx backend.UnsafeReadWriter) (revert action, err error) } type setKeyAction struct { Bucket backend.Bucket FieldName []byte FieldValue []byte } func (a setKeyAction) unsafeDo(tx backend.UnsafeReadWriter) (action, error) { revert := restoreFieldValueAction(tx, a.Bucket, a.FieldName) tx.UnsafePut(a.Bucket, a.FieldName, a.FieldValue) return revert, nil } type deleteKeyAction struct { Bucket backend.Bucket FieldName []byte } func (a deleteKeyAction) unsafeDo(tx backend.UnsafeReadWriter) (action, error) { revert := restoreFieldValueAction(tx, a.Bucket, a.FieldName) tx.UnsafeDelete(a.Bucket, a.FieldName) return revert, nil } func restoreFieldValueAction(tx backend.UnsafeReader, bucket backend.Bucket, fieldName []byte) action { _, vs := tx.UnsafeRange(bucket, fieldName, nil, 1) if len(vs) == 1 { return &setKeyAction{ Bucket: bucket, FieldName: fieldName, FieldValue: vs[0], } } return &deleteKeyAction{ Bucket: bucket, FieldName: fieldName, } } type ActionList []action // unsafeExecute executes actions one by one. If one of actions returns error, // it will revert them. func (as ActionList) unsafeExecute(lg *zap.Logger, tx backend.UnsafeReadWriter) error { revertActions := make(ActionList, 0, len(as)) for _, a := range as { revert, err := a.unsafeDo(tx) if err != nil { revertActions.unsafeExecuteInReversedOrder(lg, tx) return err } revertActions = append(revertActions, revert) } return nil } // unsafeExecuteInReversedOrder executes actions in revered order. Will panic on // action error. Should be used when reverting. func (as ActionList) unsafeExecuteInReversedOrder(lg *zap.Logger, tx backend.UnsafeReadWriter) { for j := len(as) - 1; j >= 0; j-- { _, err := as[j].unsafeDo(tx) if err != nil { lg.Panic("Cannot recover from revert error", zap.Error(err)) } } } ================================================ FILE: server/storage/schema/actions_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "errors" "fmt" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) func TestActionIsReversible(t *testing.T) { tcs := []struct { name string action action state map[string]string }{ { name: "setKeyAction empty state", action: setKeyAction{ Bucket: Meta, FieldName: []byte("/test"), FieldValue: []byte("1"), }, }, { name: "setKeyAction with key", action: setKeyAction{ Bucket: Meta, FieldName: []byte("/test"), FieldValue: []byte("1"), }, state: map[string]string{"/test": "2"}, }, { name: "deleteKeyAction empty state", action: deleteKeyAction{ Bucket: Meta, FieldName: []byte("/test"), }, }, { name: "deleteKeyAction with key", action: deleteKeyAction{ Bucket: Meta, FieldName: []byte("/test"), }, state: map[string]string{"/test": "2"}, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { be, _ := betesting.NewTmpBackend(t, time.Microsecond, 10) defer be.Close() tx := be.BatchTx() require.NotNilf(t, tx, "batch tx is nil") tx.Lock() defer tx.Unlock() UnsafeCreateMetaBucket(tx) putKeyValues(tx, Meta, tc.state) assertBucketState(t, tx, Meta, tc.state) reverse, err := tc.action.unsafeDo(tx) if err != nil { t.Errorf("Failed to upgrade, err: %v", err) } _, err = reverse.unsafeDo(tx) if err != nil { t.Errorf("Failed to downgrade, err: %v", err) } assertBucketState(t, tx, Meta, tc.state) }) } } func TestActionListRevert(t *testing.T) { tcs := []struct { name string actions ActionList expectState map[string]string expectError error }{ { name: "Apply multiple actions", actions: ActionList{ setKeyAction{Meta, []byte("/testKey1"), []byte("testValue1")}, setKeyAction{Meta, []byte("/testKey2"), []byte("testValue2")}, }, expectState: map[string]string{"/testKey1": "testValue1", "/testKey2": "testValue2"}, }, { name: "Broken action should result in changes reverted", actions: ActionList{ setKeyAction{Meta, []byte("/testKey1"), []byte("testValue1")}, brokenAction{}, setKeyAction{Meta, []byte("/testKey2"), []byte("testValue2")}, }, expectState: map[string]string{}, expectError: errBrokenAction, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) be, _ := betesting.NewTmpBackend(t, time.Microsecond, 10) defer be.Close() tx := be.BatchTx() require.NotNilf(t, tx, "batch tx is nil") tx.Lock() defer tx.Unlock() UnsafeCreateMetaBucket(tx) err := tc.actions.unsafeExecute(lg, tx) if !errors.Is(err, tc.expectError) { t.Errorf("Unexpected error or lack thereof, expected: %v, got: %v", tc.expectError, err) } assertBucketState(t, tx, Meta, tc.expectState) }) } } type brokenAction struct{} var errBrokenAction = fmt.Errorf("broken action error") func (c brokenAction) unsafeDo(tx backend.UnsafeReadWriter) (action, error) { return nil, errBrokenAction } func putKeyValues(tx backend.UnsafeWriter, bucket backend.Bucket, kvs map[string]string) { for k, v := range kvs { tx.UnsafePut(bucket, []byte(k), []byte(v)) } } func assertBucketState(t *testing.T, tx backend.UnsafeReadWriter, bucket backend.Bucket, expect map[string]string) { t.Helper() got := map[string]string{} ks, vs := tx.UnsafeRange(bucket, []byte("\x00"), []byte("\xff"), 0) for i := 0; i < len(ks); i++ { got[string(ks[i])] = string(vs[i]) } if expect == nil { expect = map[string]string{} } assert.Equal(t, expect, got) } ================================================ FILE: server/storage/schema/alarm.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "go.uber.org/zap" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/storage/backend" ) type AlarmBackend interface { CreateAlarmBucket() MustPutAlarm(member *etcdserverpb.AlarmMember) MustDeleteAlarm(alarm *etcdserverpb.AlarmMember) GetAllAlarms() ([]*etcdserverpb.AlarmMember, error) ForceCommit() } type alarmBackend struct { lg *zap.Logger be backend.Backend } func NewAlarmBackend(lg *zap.Logger, be backend.Backend) AlarmBackend { return &alarmBackend{ lg: lg, be: be, } } func (s *alarmBackend) CreateAlarmBucket() { tx := s.be.BatchTx() tx.LockOutsideApply() defer tx.Unlock() tx.UnsafeCreateBucket(Alarm) } func (s *alarmBackend) MustPutAlarm(alarm *etcdserverpb.AlarmMember) { tx := s.be.BatchTx() tx.LockInsideApply() defer tx.Unlock() s.mustUnsafePutAlarm(tx, alarm) } func (s *alarmBackend) mustUnsafePutAlarm(tx backend.UnsafeWriter, alarm *etcdserverpb.AlarmMember) { v, err := alarm.Marshal() if err != nil { s.lg.Panic("failed to marshal alarm member", zap.Error(err)) } tx.UnsafePut(Alarm, v, nil) } func (s *alarmBackend) MustDeleteAlarm(alarm *etcdserverpb.AlarmMember) { tx := s.be.BatchTx() tx.LockInsideApply() defer tx.Unlock() s.mustUnsafeDeleteAlarm(tx, alarm) } func (s *alarmBackend) mustUnsafeDeleteAlarm(tx backend.UnsafeWriter, alarm *etcdserverpb.AlarmMember) { v, err := alarm.Marshal() if err != nil { s.lg.Panic("failed to marshal alarm member", zap.Error(err)) } tx.UnsafeDelete(Alarm, v) } func (s *alarmBackend) GetAllAlarms() ([]*etcdserverpb.AlarmMember, error) { tx := s.be.ReadTx() tx.RLock() defer tx.RUnlock() return s.unsafeGetAllAlarms(tx) } func (s *alarmBackend) unsafeGetAllAlarms(tx backend.UnsafeReader) ([]*etcdserverpb.AlarmMember, error) { var ms []*etcdserverpb.AlarmMember err := tx.UnsafeForEach(Alarm, func(k, v []byte) error { var m etcdserverpb.AlarmMember if err := m.Unmarshal(k); err != nil { return err } ms = append(ms, &m) return nil }) return ms, err } func (s alarmBackend) ForceCommit() { s.be.ForceCommit() } ================================================ FILE: server/storage/schema/auth.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "bytes" "encoding/binary" "go.uber.org/zap" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/storage/backend" ) const ( revBytesLen = 8 ) var ( authEnabled = []byte{1} authDisabled = []byte{0} ) type authBackend struct { be backend.Backend lg *zap.Logger } var _ auth.AuthBackend = (*authBackend)(nil) func NewAuthBackend(lg *zap.Logger, be backend.Backend) auth.AuthBackend { return &authBackend{ be: be, lg: lg, } } func (abe *authBackend) CreateAuthBuckets() { tx := abe.be.BatchTx() tx.LockOutsideApply() defer tx.Unlock() tx.UnsafeCreateBucket(Auth) tx.UnsafeCreateBucket(AuthUsers) tx.UnsafeCreateBucket(AuthRoles) } func (abe *authBackend) ForceCommit() { abe.be.ForceCommit() } func (abe *authBackend) ReadTx() auth.AuthReadTx { return &authReadTx{tx: abe.be.ReadTx(), lg: abe.lg} } func (abe *authBackend) BatchTx() auth.AuthBatchTx { return &authBatchTx{tx: abe.be.BatchTx(), lg: abe.lg} } type authReadTx struct { tx backend.ReadTx lg *zap.Logger } type authBatchTx struct { tx backend.BatchTx lg *zap.Logger } var ( _ auth.AuthReadTx = (*authReadTx)(nil) _ auth.AuthBatchTx = (*authBatchTx)(nil) ) func (atx *authBatchTx) UnsafeSaveAuthEnabled(enabled bool) { if enabled { atx.tx.UnsafePut(Auth, AuthEnabledKeyName, authEnabled) } else { atx.tx.UnsafePut(Auth, AuthEnabledKeyName, authDisabled) } } func (atx *authBatchTx) UnsafeSaveAuthRevision(rev uint64) { revBytes := make([]byte, revBytesLen) binary.BigEndian.PutUint64(revBytes, rev) atx.tx.UnsafePut(Auth, AuthRevisionKeyName, revBytes) } func (atx *authBatchTx) UnsafeReadAuthEnabled() bool { return unsafeReadAuthEnabled(atx.tx) } func (atx *authBatchTx) UnsafeReadAuthRevision() uint64 { return unsafeReadAuthRevision(atx.tx) } func (atx *authBatchTx) Lock() { atx.tx.LockInsideApply() } func (atx *authBatchTx) Unlock() { atx.tx.Unlock() // Calling Commit() for defensive purpose. If the number of pending writes doesn't exceed batchLimit, // ReadTx can miss some writes issued by its predecessor BatchTx. atx.tx.Commit() } func (atx *authReadTx) UnsafeReadAuthEnabled() bool { return unsafeReadAuthEnabled(atx.tx) } func unsafeReadAuthEnabled(tx backend.UnsafeReader) bool { _, vs := tx.UnsafeRange(Auth, AuthEnabledKeyName, nil, 0) if len(vs) == 1 { if bytes.Equal(vs[0], authEnabled) { return true } } return false } func (atx *authReadTx) UnsafeReadAuthRevision() uint64 { return unsafeReadAuthRevision(atx.tx) } func unsafeReadAuthRevision(tx backend.UnsafeReader) uint64 { _, vs := tx.UnsafeRange(Auth, AuthRevisionKeyName, nil, 0) if len(vs) != 1 { // this can happen in the initialization phase return 0 } return binary.BigEndian.Uint64(vs[0]) } func (atx *authReadTx) RLock() { atx.tx.RLock() } func (atx *authReadTx) RUnlock() { atx.tx.RUnlock() } ================================================ FILE: server/storage/schema/auth_roles.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "go.uber.org/zap" "go.etcd.io/etcd/api/v3/authpb" "go.etcd.io/etcd/server/v3/storage/backend" ) func UnsafeCreateAuthRolesBucket(tx backend.UnsafeWriter) { tx.UnsafeCreateBucket(AuthRoles) } func (abe *authBackend) GetRole(roleName string) *authpb.Role { tx := abe.ReadTx() tx.RLock() defer tx.RUnlock() return tx.UnsafeGetRole(roleName) } func (atx *authBatchTx) UnsafeGetRole(roleName string) *authpb.Role { return unsafeGetRole(atx.lg, atx.tx, roleName) } func (abe *authBackend) GetAllRoles() []*authpb.Role { tx := abe.BatchTx() tx.Lock() defer tx.Unlock() return tx.UnsafeGetAllRoles() } func (atx *authBatchTx) UnsafeGetAllRoles() []*authpb.Role { return unsafeGetAllRoles(atx.lg, atx.tx) } func (atx *authBatchTx) UnsafePutRole(role *authpb.Role) { b, err := role.Marshal() if err != nil { atx.lg.Panic( "failed to marshal 'authpb.Role'", zap.String("role-name", string(role.Name)), zap.Error(err), ) } atx.tx.UnsafePut(AuthRoles, role.Name, b) } func (atx *authBatchTx) UnsafeDeleteRole(rolename string) { atx.tx.UnsafeDelete(AuthRoles, []byte(rolename)) } func (atx *authReadTx) UnsafeGetRole(roleName string) *authpb.Role { return unsafeGetRole(atx.lg, atx.tx, roleName) } func unsafeGetRole(lg *zap.Logger, tx backend.UnsafeReader, roleName string) *authpb.Role { _, vs := tx.UnsafeRange(AuthRoles, []byte(roleName), nil, 0) if len(vs) == 0 { return nil } role := &authpb.Role{} err := role.Unmarshal(vs[0]) if err != nil { lg.Panic("failed to unmarshal 'authpb.Role'", zap.Error(err)) } return role } func (atx *authReadTx) UnsafeGetAllRoles() []*authpb.Role { return unsafeGetAllRoles(atx.lg, atx.tx) } func unsafeGetAllRoles(lg *zap.Logger, tx backend.UnsafeReader) []*authpb.Role { _, vs := tx.UnsafeRange(AuthRoles, []byte{0}, []byte{0xff}, -1) if len(vs) == 0 { return nil } roles := make([]*authpb.Role, len(vs)) for i := range vs { role := &authpb.Role{} err := role.Unmarshal(vs[i]) if err != nil { lg.Panic("failed to unmarshal 'authpb.Role'", zap.Error(err)) } roles[i] = role } return roles } ================================================ FILE: server/storage/schema/auth_roles_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "testing" "time" "github.com/stretchr/testify/assert" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/authpb" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) func TestGetAllRoles(t *testing.T) { tcs := []struct { name string setup func(tx auth.UnsafeAuthWriter) want []*authpb.Role }{ { name: "Empty by default", setup: func(tx auth.UnsafeAuthWriter) {}, want: nil, }, { name: "Returns data put before", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutRole(&authpb.Role{ Name: []byte("readKey"), KeyPermission: []*authpb.Permission{ { PermType: authpb.Permission_READ, Key: []byte("key"), RangeEnd: []byte("end"), }, }, }) }, want: []*authpb.Role{ { Name: []byte("readKey"), KeyPermission: []*authpb.Permission{ { PermType: authpb.Permission_READ, Key: []byte("key"), RangeEnd: []byte("end"), }, }, }, }, }, { name: "Skips deleted", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutRole(&authpb.Role{ Name: []byte("role1"), }) tx.UnsafePutRole(&authpb.Role{ Name: []byte("role2"), }) tx.UnsafeDeleteRole("role1") }, want: []*authpb.Role{{Name: []byte("role2")}}, }, { name: "Returns data overridden by put", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutRole(&authpb.Role{ Name: []byte("role1"), KeyPermission: []*authpb.Permission{ { PermType: authpb.Permission_READ, }, }, }) tx.UnsafePutRole(&authpb.Role{ Name: []byte("role2"), }) tx.UnsafePutRole(&authpb.Role{ Name: []byte("role1"), KeyPermission: []*authpb.Permission{ { PermType: authpb.Permission_READWRITE, }, }, }) }, want: []*authpb.Role{ {Name: []byte("role1"), KeyPermission: []*authpb.Permission{{PermType: authpb.Permission_READWRITE}}}, {Name: []byte("role2")}, }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) abe := NewAuthBackend(lg, be) abe.CreateAuthBuckets() tx := abe.BatchTx() tx.Lock() tc.setup(tx) tx.Unlock() abe.ForceCommit() be.Close() be2 := backend.NewDefaultBackend(lg, tmpPath) defer be2.Close() abe2 := NewAuthBackend(lg, be2) users := abe2.GetAllRoles() assert.Equal(t, tc.want, users) }) } } func TestGetRole(t *testing.T) { tcs := []struct { name string setup func(tx auth.UnsafeAuthWriter) want *authpb.Role }{ { name: "Returns nil for missing", setup: func(tx auth.UnsafeAuthWriter) {}, want: nil, }, { name: "Returns data put before", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutRole(&authpb.Role{ Name: []byte("role1"), KeyPermission: []*authpb.Permission{ { PermType: authpb.Permission_READ, Key: []byte("key"), RangeEnd: []byte("end"), }, }, }) }, want: &authpb.Role{ Name: []byte("role1"), KeyPermission: []*authpb.Permission{ { PermType: authpb.Permission_READ, Key: []byte("key"), RangeEnd: []byte("end"), }, }, }, }, { name: "Return nil for deleted", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutRole(&authpb.Role{ Name: []byte("role1"), }) tx.UnsafeDeleteRole("role1") }, want: nil, }, { name: "Returns data overridden by put", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutRole(&authpb.Role{ Name: []byte("role1"), KeyPermission: []*authpb.Permission{ { PermType: authpb.Permission_READ, }, }, }) tx.UnsafePutRole(&authpb.Role{ Name: []byte("role1"), KeyPermission: []*authpb.Permission{ { PermType: authpb.Permission_READWRITE, }, }, }) }, want: &authpb.Role{ Name: []byte("role1"), KeyPermission: []*authpb.Permission{{PermType: authpb.Permission_READWRITE}}, }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) abe := NewAuthBackend(lg, be) abe.CreateAuthBuckets() tx := abe.BatchTx() tx.Lock() tc.setup(tx) tx.Unlock() abe.ForceCommit() be.Close() be2 := backend.NewDefaultBackend(lg, tmpPath) defer be2.Close() abe2 := NewAuthBackend(lg, be2) users := abe2.GetRole("role1") assert.Equal(t, tc.want, users) }) } } ================================================ FILE: server/storage/schema/auth_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "math" "testing" "time" "github.com/stretchr/testify/assert" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) // TestAuthEnabled ensures that UnsafeSaveAuthEnabled&UnsafeReadAuthEnabled work well together. func TestAuthEnabled(t *testing.T) { tcs := []struct { name string skipSetting bool setEnabled bool wantEnabled bool }{ { name: "Returns true after setting true", setEnabled: true, wantEnabled: true, }, { name: "Returns false after setting false", setEnabled: false, wantEnabled: false, }, { name: "Returns false by default", skipSetting: true, wantEnabled: false, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) abe := NewAuthBackend(lg, be) tx := abe.BatchTx() abe.CreateAuthBuckets() tx.Lock() if !tc.skipSetting { tx.UnsafeSaveAuthEnabled(tc.setEnabled) } tx.Unlock() abe.ForceCommit() be.Close() be2 := backend.NewDefaultBackend(lg, tmpPath) defer be2.Close() abe2 := NewAuthBackend(lg, be2) tx = abe2.BatchTx() tx.Lock() defer tx.Unlock() v := tx.UnsafeReadAuthEnabled() assert.Equal(t, tc.wantEnabled, v) }) } } // TestAuthRevision ensures that UnsafeSaveAuthRevision&UnsafeReadAuthRevision work well together. func TestAuthRevision(t *testing.T) { tcs := []struct { name string setRevision uint64 wantRevision uint64 }{ { name: "Returns 0 by default", wantRevision: 0, }, { name: "Returns 1 after setting 1", setRevision: 1, wantRevision: 1, }, { name: "Returns max int after setting max int", setRevision: math.MaxUint64, wantRevision: math.MaxUint64, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) abe := NewAuthBackend(lg, be) abe.CreateAuthBuckets() if tc.setRevision != 0 { tx := abe.BatchTx() tx.Lock() tx.UnsafeSaveAuthRevision(tc.setRevision) tx.Unlock() } abe.ForceCommit() be.Close() be2 := backend.NewDefaultBackend(lg, tmpPath) defer be2.Close() abe2 := NewAuthBackend(lg, be2) tx := abe2.BatchTx() tx.Lock() defer tx.Unlock() v := tx.UnsafeReadAuthRevision() assert.Equal(t, tc.wantRevision, v) }) } } ================================================ FILE: server/storage/schema/auth_users.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "go.uber.org/zap" "go.etcd.io/etcd/api/v3/authpb" "go.etcd.io/etcd/server/v3/storage/backend" ) func (abe *authBackend) GetUser(username string) *authpb.User { tx := abe.ReadTx() tx.RLock() defer tx.RUnlock() return tx.UnsafeGetUser(username) } func (atx *authBatchTx) UnsafeGetUser(username string) *authpb.User { return unsafeGetUser(atx.lg, atx.tx, username) } func (atx *authBatchTx) UnsafeGetAllUsers() []*authpb.User { return unsafeGetAllUsers(atx.lg, atx.tx) } func (atx *authBatchTx) UnsafePutUser(user *authpb.User) { b, err := user.Marshal() if err != nil { atx.lg.Panic("failed to unmarshal 'authpb.User'", zap.Error(err)) } atx.tx.UnsafePut(AuthUsers, user.Name, b) } func (atx *authBatchTx) UnsafeDeleteUser(username string) { atx.tx.UnsafeDelete(AuthUsers, []byte(username)) } func (atx *authReadTx) UnsafeGetUser(username string) *authpb.User { return unsafeGetUser(atx.lg, atx.tx, username) } func unsafeGetUser(lg *zap.Logger, tx backend.UnsafeReader, username string) *authpb.User { _, vs := tx.UnsafeRange(AuthUsers, []byte(username), nil, 0) if len(vs) == 0 { return nil } user := &authpb.User{} err := user.Unmarshal(vs[0]) if err != nil { lg.Panic( "failed to unmarshal 'authpb.User'", zap.String("user-name", username), zap.Error(err), ) } return user } func (abe *authBackend) GetAllUsers() []*authpb.User { tx := abe.ReadTx() tx.RLock() defer tx.RUnlock() return tx.UnsafeGetAllUsers() } func (atx *authReadTx) UnsafeGetAllUsers() []*authpb.User { return unsafeGetAllUsers(atx.lg, atx.tx) } func unsafeGetAllUsers(lg *zap.Logger, tx backend.UnsafeReader) []*authpb.User { var vs [][]byte err := tx.UnsafeForEach(AuthUsers, func(k []byte, v []byte) error { vs = append(vs, v) return nil }) if err != nil { lg.Panic("failed to get users", zap.Error(err)) } if len(vs) == 0 { return nil } users := make([]*authpb.User, len(vs)) for i := range vs { user := &authpb.User{} err := user.Unmarshal(vs[i]) if err != nil { lg.Panic("failed to unmarshal 'authpb.User'", zap.Error(err)) } users[i] = user } return users } ================================================ FILE: server/storage/schema/auth_users_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "testing" "time" "github.com/stretchr/testify/assert" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/authpb" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) func TestGetAllUsers(t *testing.T) { tcs := []struct { name string setup func(tx auth.UnsafeAuthWriter) want []*authpb.User }{ { name: "Empty by default", setup: func(tx auth.UnsafeAuthWriter) {}, want: nil, }, { name: "Returns user put before", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutUser(&authpb.User{ Name: []byte("alice"), Password: []byte("alicePassword"), Roles: []string{"aliceRole1", "aliceRole2"}, Options: &authpb.UserAddOptions{ NoPassword: true, }, }) }, want: []*authpb.User{ { Name: []byte("alice"), Password: []byte("alicePassword"), Roles: []string{"aliceRole1", "aliceRole2"}, Options: &authpb.UserAddOptions{ NoPassword: true, }, }, }, }, { name: "Skips deleted user", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutUser(&authpb.User{ Name: []byte("alice"), }) tx.UnsafePutUser(&authpb.User{ Name: []byte("bob"), }) tx.UnsafeDeleteUser("alice") }, want: []*authpb.User{{Name: []byte("bob")}}, }, { name: "Returns data overridden by put", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutUser(&authpb.User{ Name: []byte("alice"), Password: []byte("oldPassword"), }) tx.UnsafePutUser(&authpb.User{ Name: []byte("bob"), }) tx.UnsafePutUser(&authpb.User{ Name: []byte("alice"), Password: []byte("newPassword"), }) }, want: []*authpb.User{ {Name: []byte("alice"), Password: []byte("newPassword")}, {Name: []byte("bob")}, }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) abe := NewAuthBackend(lg, be) abe.CreateAuthBuckets() tx := abe.BatchTx() tx.Lock() tc.setup(tx) tx.Unlock() abe.ForceCommit() be.Close() be2 := backend.NewDefaultBackend(lg, tmpPath) defer be2.Close() abe2 := NewAuthBackend(lg, be2) users := abe2.ReadTx().UnsafeGetAllUsers() assert.Equal(t, tc.want, users) }) } } func TestGetUser(t *testing.T) { tcs := []struct { name string setup func(tx auth.UnsafeAuthWriter) want *authpb.User }{ { name: "Returns nil for missing user", setup: func(tx auth.UnsafeAuthWriter) {}, want: nil, }, { name: "Returns data put before", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutUser(&authpb.User{ Name: []byte("alice"), Password: []byte("alicePassword"), Roles: []string{"aliceRole1", "aliceRole2"}, Options: &authpb.UserAddOptions{ NoPassword: true, }, }) }, want: &authpb.User{ Name: []byte("alice"), Password: []byte("alicePassword"), Roles: []string{"aliceRole1", "aliceRole2"}, Options: &authpb.UserAddOptions{ NoPassword: true, }, }, }, { name: "Skips deleted", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutUser(&authpb.User{ Name: []byte("alice"), }) tx.UnsafeDeleteUser("alice") }, want: nil, }, { name: "Returns data overridden by put", setup: func(tx auth.UnsafeAuthWriter) { tx.UnsafePutUser(&authpb.User{ Name: []byte("alice"), Password: []byte("oldPassword"), }) tx.UnsafePutUser(&authpb.User{ Name: []byte("alice"), Password: []byte("newPassword"), }) }, want: &authpb.User{ Name: []byte("alice"), Password: []byte("newPassword"), }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) abe := NewAuthBackend(lg, be) abe.CreateAuthBuckets() tx := abe.BatchTx() tx.Lock() tc.setup(tx) tx.Unlock() abe.ForceCommit() be.Close() be2 := backend.NewDefaultBackend(lg, tmpPath) defer be2.Close() abe2 := NewAuthBackend(lg, be2) users := abe2.GetUser("alice") assert.Equal(t, tc.want, users) }) } } ================================================ FILE: server/storage/schema/bucket.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "bytes" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/storage/backend" ) var ( keyBucketName = []byte("key") metaBucketName = []byte("meta") leaseBucketName = []byte("lease") alarmBucketName = []byte("alarm") clusterBucketName = []byte("cluster") membersBucketName = []byte("members") membersRemovedBucketName = []byte("members_removed") authBucketName = []byte("auth") authUsersBucketName = []byte("authUsers") authRolesBucketName = []byte("authRoles") testBucketName = []byte("test") ) var ( Key = backend.Bucket(bucket{id: 1, name: keyBucketName, safeRangeBucket: true}) Meta = backend.Bucket(bucket{id: 2, name: metaBucketName, safeRangeBucket: false}) Lease = backend.Bucket(bucket{id: 3, name: leaseBucketName, safeRangeBucket: false}) Alarm = backend.Bucket(bucket{id: 4, name: alarmBucketName, safeRangeBucket: false}) Cluster = backend.Bucket(bucket{id: 5, name: clusterBucketName, safeRangeBucket: false}) Members = backend.Bucket(bucket{id: 10, name: membersBucketName, safeRangeBucket: false}) MembersRemoved = backend.Bucket(bucket{id: 11, name: membersRemovedBucketName, safeRangeBucket: false}) Auth = backend.Bucket(bucket{id: 20, name: authBucketName, safeRangeBucket: false}) AuthUsers = backend.Bucket(bucket{id: 21, name: authUsersBucketName, safeRangeBucket: false}) AuthRoles = backend.Bucket(bucket{id: 22, name: authRolesBucketName, safeRangeBucket: false}) Test = backend.Bucket(bucket{id: 100, name: testBucketName, safeRangeBucket: false}) AllBuckets = []backend.Bucket{Key, Meta, Lease, Alarm, Cluster, Members, MembersRemoved, Auth, AuthUsers, AuthRoles} ) type bucket struct { id backend.BucketID name []byte safeRangeBucket bool } func (b bucket) ID() backend.BucketID { return b.id } func (b bucket) Name() []byte { return b.name } func (b bucket) String() string { return string(b.Name()) } func (b bucket) IsSafeRangeBucket() bool { return b.safeRangeBucket } var ( // Pre v3.5 ScheduledCompactKeyName = []byte("scheduledCompactRev") FinishedCompactKeyName = []byte("finishedCompactRev") MetaConsistentIndexKeyName = []byte("consistent_index") AuthEnabledKeyName = []byte("authEnabled") AuthRevisionKeyName = []byte("authRevision") // Since v3.5 MetaTermKeyName = []byte("term") MetaConfStateName = []byte("confState") ClusterClusterVersionKeyName = []byte("clusterVersion") ClusterDowngradeKeyName = []byte("downgrade") // Since v3.6 MetaStorageVersionName = []byte("storageVersion") // Before adding new meta key please update server/etcdserver/version ) // DefaultIgnores defines buckets & keys to ignore in hash checking. func DefaultIgnores(bucket, key []byte) bool { // consistent index & term might be changed due to v2 internal sync, which // is not controllable by the user. // storage version might change after wal snapshot and is not controller by user. return bytes.Equal(bucket, Meta.Name()) && (bytes.Equal(key, MetaTermKeyName) || bytes.Equal(key, MetaConsistentIndexKeyName) || bytes.Equal(key, MetaStorageVersionName)) } func BackendMemberKey(id types.ID) []byte { return []byte(id.String()) } ================================================ FILE: server/storage/schema/changes.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import "go.etcd.io/etcd/server/v3/storage/backend" type schemaChange interface { upgradeAction() action downgradeAction() action } // addNewField represents adding new field when upgrading. Downgrade will remove the field. func addNewField(bucket backend.Bucket, fieldName []byte, fieldValue []byte) schemaChange { return simpleSchemaChange{ upgrade: setKeyAction{ Bucket: bucket, FieldName: fieldName, FieldValue: fieldValue, }, downgrade: deleteKeyAction{ Bucket: bucket, FieldName: fieldName, }, } } type simpleSchemaChange struct { upgrade action downgrade action } func (c simpleSchemaChange) upgradeAction() action { return c.upgrade } func (c simpleSchemaChange) downgradeAction() action { return c.downgrade } ================================================ FILE: server/storage/schema/changes_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "testing" "time" "github.com/stretchr/testify/require" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) func TestUpgradeDowngrade(t *testing.T) { tcs := []struct { name string change schemaChange expectStateAfterUpgrade map[string]string expectStateAfterDowngrade map[string]string }{ { name: "addNewField empty", change: addNewField(Meta, []byte("/test"), []byte("1")), expectStateAfterUpgrade: map[string]string{"/test": "1"}, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { be, _ := betesting.NewTmpBackend(t, time.Microsecond, 10) defer be.Close() tx := be.BatchTx() require.NotNilf(t, tx, "batch tx is nil") tx.Lock() defer tx.Unlock() UnsafeCreateMetaBucket(tx) _, err := tc.change.upgradeAction().unsafeDo(tx) if err != nil { t.Errorf("Failed to upgrade, err: %v", err) } assertBucketState(t, tx, Meta, tc.expectStateAfterUpgrade) _, err = tc.change.downgradeAction().unsafeDo(tx) if err != nil { t.Errorf("Failed to downgrade, err: %v", err) } assertBucketState(t, tx, Meta, tc.expectStateAfterDowngrade) }) } } ================================================ FILE: server/storage/schema/cindex.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "encoding/binary" "go.etcd.io/etcd/client/pkg/v3/verify" "go.etcd.io/etcd/server/v3/storage/backend" ) // UnsafeCreateMetaBucket creates the `meta` bucket (if it does not exist yet). func UnsafeCreateMetaBucket(tx backend.UnsafeWriter) { tx.UnsafeCreateBucket(Meta) } // CreateMetaBucket creates the `meta` bucket (if it does not exist yet). func CreateMetaBucket(tx backend.BatchTx) { tx.LockOutsideApply() defer tx.Unlock() tx.UnsafeCreateBucket(Meta) } // UnsafeReadConsistentIndex loads consistent index & term from given transaction. // returns 0,0 if the data are not found. // Term is persisted since v3.5. func UnsafeReadConsistentIndex(tx backend.UnsafeReader) (uint64, uint64) { _, vs := tx.UnsafeRange(Meta, MetaConsistentIndexKeyName, nil, 0) if len(vs) == 0 { return 0, 0 } v := binary.BigEndian.Uint64(vs[0]) _, ts := tx.UnsafeRange(Meta, MetaTermKeyName, nil, 0) if len(ts) == 0 { return v, 0 } t := binary.BigEndian.Uint64(ts[0]) return v, t } // ReadConsistentIndex loads consistent index and term from given transaction. // returns 0 if the data are not found. func ReadConsistentIndex(tx backend.ReadTx) (uint64, uint64) { tx.RLock() defer tx.RUnlock() return UnsafeReadConsistentIndex(tx) } func UnsafeUpdateConsistentIndexForce(tx backend.UnsafeReadWriter, index uint64, term uint64) { unsafeUpdateConsistentIndex(tx, index, term, true) } func UnsafeUpdateConsistentIndex(tx backend.UnsafeReadWriter, index uint64, term uint64) { unsafeUpdateConsistentIndex(tx, index, term, false) } func unsafeUpdateConsistentIndex(tx backend.UnsafeReadWriter, index uint64, term uint64, allowDecreasing bool) { if index == 0 { // Never save 0 as it means that we didn't load the real index yet. return } bs1 := make([]byte, 8) binary.BigEndian.PutUint64(bs1, index) if !allowDecreasing { verify.Verify("update of consistent index not advancing", func() (bool, map[string]any) { previousIndex, _ := UnsafeReadConsistentIndex(tx) return index >= previousIndex, map[string]any{ "previousIndex": previousIndex, "currentIndex": index, } }) } // put the index into the underlying backend // tx has been locked in TxnBegin, so there is no need to lock it again tx.UnsafePut(Meta, MetaConsistentIndexKeyName, bs1) if term > 0 { bs2 := make([]byte, 8) binary.BigEndian.PutUint64(bs2, term) tx.UnsafePut(Meta, MetaTermKeyName, bs2) } } ================================================ FILE: server/storage/schema/confstate.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "encoding/json" "log" "go.uber.org/zap" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/raft/v3/raftpb" ) // MustUnsafeSaveConfStateToBackend persists confState using given transaction (tx). // confState in backend is persisted since etcd v3.5. func MustUnsafeSaveConfStateToBackend(lg *zap.Logger, tx backend.UnsafeWriter, confState *raftpb.ConfState) { confStateBytes, err := json.Marshal(confState) if err != nil { lg.Panic("Cannot marshal raftpb.ConfState", zap.Stringer("conf-state", confState), zap.Error(err)) } tx.UnsafePut(Meta, MetaConfStateName, confStateBytes) } // UnsafeConfStateFromBackend retrieves ConfState from the backend. // Returns nil if confState in backend is not persisted (e.g. backend written by = 3.6, so we don't need to use any other // fields to identify the etcd's storage version. _, term := UnsafeReadConsistentIndex(tx) if term == 0 { return v, fmt.Errorf("missing term information") } return version.V3_5, nil } func schemaChangesForVersion(v semver.Version, isUpgrade bool) ([]schemaChange, error) { // changes should be taken from higher version higherV := v if isUpgrade { higherV = semver.Version{Major: v.Major, Minor: v.Minor + 1} } actions, found := schemaChanges[higherV] if !found { if isUpgrade { return nil, fmt.Errorf("version %q is not supported", higherV.String()) } return nil, fmt.Errorf("version %q is not supported", v.String()) } return actions, nil } var ( // schemaChanges list changes that were introduced in a particular version. // schema was introduced in v3.6 as so its changes were not tracked before. schemaChanges = map[semver.Version][]schemaChange{ version.V3_6: { addNewField(Meta, MetaStorageVersionName, emptyStorageVersion), }, version.V3_7: {}, } // emptyStorageVersion is used for v3.6 Step for the first time, in all other version StoragetVersion should be set by migrator. // Adding a addNewField for StorageVersion we can reuse logic to remove it when downgrading to v3.5 emptyStorageVersion = []byte("") ) ================================================ FILE: server/storage/schema/schema_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "strings" "testing" "time" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" "go.etcd.io/etcd/server/v3/storage/wal" waltesting "go.etcd.io/etcd/server/v3/storage/wal/testing" "go.etcd.io/raft/v3/raftpb" ) func TestValidate(t *testing.T) { tcs := []struct { name string version semver.Version // Overrides which keys should be set (default based on version) overrideKeys func(tx backend.UnsafeReadWriter) expectError bool expectErrorMsg string }{ // As storage version field was added in v3.6, for v3.5 we will not set it. // For storage to be considered v3.5 it have both confstate and term key set. { name: `V3.4 schema is correct`, version: version.V3_4, }, { name: `V3.5 schema without confstate and term fields is correct`, version: version.V3_5, overrideKeys: func(tx backend.UnsafeReadWriter) {}, }, { name: `V3.5 schema without term field is correct`, version: version.V3_5, overrideKeys: func(tx backend.UnsafeReadWriter) { MustUnsafeSaveConfStateToBackend(zap.NewNop(), tx, &raftpb.ConfState{}) }, }, { name: `V3.5 schema with all fields is correct`, version: version.V3_5, overrideKeys: func(tx backend.UnsafeReadWriter) { MustUnsafeSaveConfStateToBackend(zap.NewNop(), tx, &raftpb.ConfState{}) UnsafeUpdateConsistentIndex(tx, 1, 1) }, }, { name: `V3.6 schema is correct`, version: version.V3_6, }, { name: `V3.7 is correct`, version: version.V3_7, }, { name: `V3.8 schema is unknown and should return error`, version: version.V3_8, expectError: true, expectErrorMsg: `version "3.8.0" is not supported`, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { lg := zap.NewNop() dataPath := setupBackendData(t, tc.version, tc.overrideKeys) b := backend.NewDefaultBackend(lg, dataPath) defer b.Close() err := Validate(lg, b.ReadTx()) if (err != nil) != tc.expectError { t.Errorf("Validate(lg, tx) = %+v, expected error: %v", err, tc.expectError) } if err != nil && err.Error() != tc.expectErrorMsg { t.Errorf("Validate(lg, tx) = %q, expected error message: %q", err, tc.expectErrorMsg) } }) } } func TestMigrate(t *testing.T) { tcs := []struct { name string version semver.Version // Overrides which keys should be set (default based on version) overrideKeys func(tx backend.UnsafeReadWriter) targetVersion semver.Version walEntries []etcdserverpb.InternalRaftRequest expectVersion *semver.Version expectError bool expectErrorMsg string }{ // As storage version field was added in v3.6, for v3.5 we will not set it. // For storage to be considered v3.5 it has term key set. { name: `Upgrading v3.5 to v3.6 should be rejected if term is not set`, version: version.V3_5, overrideKeys: func(tx backend.UnsafeReadWriter) { MustUnsafeSaveConfStateToBackend(zap.NewNop(), tx, &raftpb.ConfState{}) }, targetVersion: version.V3_6, expectVersion: nil, expectError: true, expectErrorMsg: `cannot detect storage schema version: missing term information`, }, { name: `Upgrading v3.5 to v3.6 should succeed; all required fields are set`, version: version.V3_5, targetVersion: version.V3_6, expectVersion: &version.V3_6, }, { name: `Migrate on same v3.5 version passes and doesn't set storage version'`, version: version.V3_5, targetVersion: version.V3_5, expectVersion: nil, }, { name: `Migrate on same v3.6 version passes`, version: version.V3_6, targetVersion: version.V3_6, expectVersion: &version.V3_6, }, { name: `Migrate on same v3.7 version passes`, version: version.V3_7, targetVersion: version.V3_7, expectVersion: &version.V3_7, }, { name: "Upgrading 3.6 to v3.7 should work", version: version.V3_6, targetVersion: version.V3_7, expectVersion: &version.V3_7, }, { name: "Upgrading 3.7 to v3.8 is not supported", version: version.V3_7, targetVersion: version.V3_8, expectVersion: &version.V3_7, expectError: true, expectErrorMsg: `cannot create migration plan: version "3.8.0" is not supported`, }, { name: "Downgrading v3.7 to v3.6 should work", version: version.V3_7, targetVersion: version.V3_6, expectVersion: &version.V3_6, }, { name: "Downgrading v3.8 to v3.7 is not supported", version: version.V3_8, targetVersion: version.V3_7, expectVersion: &version.V3_8, expectError: true, expectErrorMsg: `cannot create migration plan: version "3.8.0" is not supported`, }, { name: "Downgrading v3.6 to v3.5 works as there are no v3.6 wal entries", version: version.V3_6, targetVersion: version.V3_5, walEntries: []etcdserverpb.InternalRaftRequest{ {Range: &etcdserverpb.RangeRequest{Key: []byte("\x00"), RangeEnd: []byte("\xff")}}, }, expectVersion: nil, }, { name: "Downgrading v3.6 to v3.5 fails if there are newer WAL entries", version: version.V3_6, targetVersion: version.V3_5, walEntries: []etcdserverpb.InternalRaftRequest{ {DowngradeVersionTest: &etcdserverpb.DowngradeVersionTestRequest{Ver: "3.6.0"}}, }, expectVersion: &version.V3_6, expectError: true, expectErrorMsg: "cannot downgrade storage, WAL contains newer entries", }, { name: "Downgrading v3.5 to v3.4 is not supported as schema was introduced in v3.6", version: version.V3_5, targetVersion: version.V3_4, expectVersion: nil, expectError: true, expectErrorMsg: `cannot create migration plan: version "3.5.0" is not supported`, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { lg := zap.NewNop() dataPath := setupBackendData(t, tc.version, tc.overrideKeys) w, _ := waltesting.NewTmpWAL(t, tc.walEntries) defer w.Close() walVersion, err := wal.ReadWALVersion(w) if err != nil { t.Fatal(err) } b := backend.NewDefaultBackend(lg, dataPath) defer b.Close() err = Migrate(lg, b.BatchTx(), walVersion, tc.targetVersion) if (err != nil) != tc.expectError { t.Errorf("Migrate(lg, tx, %q) = %+v, expected error: %v", tc.targetVersion, err, tc.expectError) } if err != nil && !strings.Contains(err.Error(), tc.expectErrorMsg) { t.Errorf("Migrate(lg, tx, %q) = %q, expected error message: %q", tc.targetVersion, err, tc.expectErrorMsg) } v := UnsafeReadStorageVersion(b.BatchTx()) assert.Equal(t, tc.expectVersion, v) }) } } func TestMigrateIsReversible(t *testing.T) { tcs := []struct { initialVersion semver.Version state map[string]string }{ { initialVersion: version.V3_5, state: map[string]string{ "confState": `{"auto_leave":false}`, "consistent_index": "\x00\x00\x00\x00\x00\x00\x00\x01", "term": "\x00\x00\x00\x00\x00\x00\x00\x01", }, }, { initialVersion: version.V3_6, state: map[string]string{ "confState": `{"auto_leave":false}`, "consistent_index": "\x00\x00\x00\x00\x00\x00\x00\x01", "term": "\x00\x00\x00\x00\x00\x00\x00\x01", "storageVersion": "3.6.0", }, }, } for _, tc := range tcs { t.Run(tc.initialVersion.String(), func(t *testing.T) { lg := zap.NewNop() dataPath := setupBackendData(t, tc.initialVersion, nil) be := backend.NewDefaultBackend(lg, dataPath) defer be.Close() tx := be.BatchTx() tx.Lock() defer tx.Unlock() assertBucketState(t, tx, Meta, tc.state) w, walPath := waltesting.NewTmpWAL(t, nil) walVersion, err := wal.ReadWALVersion(w) if err != nil { t.Fatal(err) } // Upgrade to current version ver := localBinaryVersion() err = UnsafeMigrate(lg, tx, walVersion, ver) if err != nil { t.Errorf("Migrate(lg, tx, %q) returned error %+v", ver, err) } assert.Equal(t, &ver, UnsafeReadStorageVersion(tx)) // Downgrade back to initial version w.Close() w = waltesting.Reopen(t, walPath) defer w.Close() walVersion, err = wal.ReadWALVersion(w) if err != nil { t.Fatal(err) } err = UnsafeMigrate(lg, tx, walVersion, tc.initialVersion) if err != nil { t.Errorf("Migrate(lg, tx, %q) returned error %+v", tc.initialVersion, err) } // Assert that all changes were revered assertBucketState(t, tx, Meta, tc.state) }) } } func setupBackendData(t *testing.T, ver semver.Version, overrideKeys func(tx backend.UnsafeReadWriter)) string { t.Helper() be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) tx := be.BatchTx() require.NotNilf(t, tx, "batch tx is nil") tx.Lock() UnsafeCreateMetaBucket(tx) if overrideKeys != nil { overrideKeys(tx) } else { switch ver { case version.V3_4: case version.V3_5: MustUnsafeSaveConfStateToBackend(zap.NewNop(), tx, &raftpb.ConfState{}) UnsafeUpdateConsistentIndex(tx, 1, 1) case version.V3_6: MustUnsafeSaveConfStateToBackend(zap.NewNop(), tx, &raftpb.ConfState{}) UnsafeUpdateConsistentIndex(tx, 1, 1) UnsafeSetStorageVersion(tx, &version.V3_6) case version.V3_7: MustUnsafeSaveConfStateToBackend(zap.NewNop(), tx, &raftpb.ConfState{}) UnsafeUpdateConsistentIndex(tx, 1, 1) UnsafeSetStorageVersion(tx, &version.V3_7) tx.UnsafePut(Meta, []byte("future-key"), []byte("")) case version.V3_8: MustUnsafeSaveConfStateToBackend(zap.NewNop(), tx, &raftpb.ConfState{}) UnsafeUpdateConsistentIndex(tx, 1, 1) UnsafeSetStorageVersion(tx, &version.V3_8) tx.UnsafePut(Meta, []byte("future-key"), []byte("")) default: t.Fatalf("Unsupported storage version") } } tx.Unlock() be.ForceCommit() be.Close() return tmpPath } ================================================ FILE: server/storage/schema/version.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "github.com/coreos/go-semver/semver" "go.etcd.io/bbolt" "go.etcd.io/etcd/server/v3/storage/backend" ) // ReadStorageVersion loads storage version from given backend transaction. // Populated since v3.6 func ReadStorageVersion(tx backend.ReadTx) *semver.Version { tx.RLock() defer tx.RUnlock() return UnsafeReadStorageVersion(tx) } // UnsafeReadStorageVersion loads storage version from given backend transaction. // Populated since v3.6 func UnsafeReadStorageVersion(tx backend.UnsafeReader) *semver.Version { _, vs := tx.UnsafeRange(Meta, MetaStorageVersionName, nil, 1) if len(vs) == 0 { return nil } v, err := semver.NewVersion(string(vs[0])) if err != nil { return nil } return v } // ReadStorageVersionFromSnapshot loads storage version from given bbolt transaction. // Populated since v3.6 func ReadStorageVersionFromSnapshot(tx *bbolt.Tx) *semver.Version { v := tx.Bucket(Meta.Name()).Get(MetaStorageVersionName) version, err := semver.NewVersion(string(v)) if err != nil { return nil } return version } // UnsafeSetStorageVersion updates etcd storage version in backend. // Populated since v3.6 func UnsafeSetStorageVersion(tx backend.UnsafeWriter, v *semver.Version) { sv := semver.Version{Major: v.Major, Minor: v.Minor} tx.UnsafePut(Meta, MetaStorageVersionName, []byte(sv.String())) } // UnsafeClearStorageVersion removes etcd storage version in backend. func UnsafeClearStorageVersion(tx backend.UnsafeWriter) { tx.UnsafeDelete(Meta, MetaStorageVersionName) } ================================================ FILE: server/storage/schema/version_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schema import ( "testing" "time" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/bbolt" "go.etcd.io/etcd/server/v3/storage/backend" betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" ) // TestVersion ensures that UnsafeSetStorageVersion/UnsafeReadStorageVersion work well together. func TestVersion(t *testing.T) { tcs := []struct { version string expectVersion string }{ { version: "3.5.0", expectVersion: "3.5.0", }, { version: "3.5.0-alpha", expectVersion: "3.5.0", }, { version: "3.5.0-beta.0", expectVersion: "3.5.0", }, { version: "3.5.0-rc.1", expectVersion: "3.5.0", }, { version: "3.5.1", expectVersion: "3.5.0", }, } for _, tc := range tcs { t.Run(tc.version, func(t *testing.T) { lg := zaptest.NewLogger(t) be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) tx := be.BatchTx() require.NotNilf(t, tx, "batch tx is nil") tx.Lock() tx.UnsafeCreateBucket(Meta) UnsafeSetStorageVersion(tx, semver.New(tc.version)) tx.Unlock() be.ForceCommit() be.Close() b := backend.NewDefaultBackend(lg, tmpPath) defer b.Close() v := UnsafeReadStorageVersion(b.BatchTx()) assert.Equal(t, tc.expectVersion, v.String()) }) } } // TestVersionSnapshot ensures that UnsafeSetStorageVersion/unsafeReadStorageVersionFromSnapshot work well together. func TestVersionSnapshot(t *testing.T) { tcs := []struct { version string expectVersion string }{ { version: "3.5.0", expectVersion: "3.5.0", }, { version: "3.5.0-alpha", expectVersion: "3.5.0", }, { version: "3.5.0-beta.0", expectVersion: "3.5.0", }, { version: "3.5.0-rc.1", expectVersion: "3.5.0", }, } for _, tc := range tcs { t.Run(tc.version, func(t *testing.T) { be, tmpPath := betesting.NewTmpBackend(t, time.Microsecond, 10) tx := be.BatchTx() require.NotNilf(t, tx, "batch tx is nil") tx.Lock() tx.UnsafeCreateBucket(Meta) UnsafeSetStorageVersion(tx, semver.New(tc.version)) tx.Unlock() be.ForceCommit() be.Close() db, err := bbolt.Open(tmpPath, 0o400, &bbolt.Options{ReadOnly: true}) if err != nil { t.Fatal(err) } defer db.Close() var ver *semver.Version if err = db.View(func(tx *bbolt.Tx) error { ver = ReadStorageVersionFromSnapshot(tx) return nil }); err != nil { t.Fatal(err) } assert.Equal(t, tc.expectVersion, ver.String()) }) } } ================================================ FILE: server/storage/storage.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage import ( "errors" "sync" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) type Storage interface { // Save function saves ents and state to the underlying stable storage. // Save MUST block until st and ents are on stable storage. Save(st raftpb.HardState, ents []raftpb.Entry) error // SaveSnap function saves snapshot to the underlying stable storage. SaveSnap(snap raftpb.Snapshot) error // Close closes the Storage and performs finalization. Close() error // Release releases the locked wal files older than the provided snapshot. Release(snap raftpb.Snapshot) error // Sync WAL Sync() error // MinimalEtcdVersion returns minimal etcd storage able to interpret WAL log. MinimalEtcdVersion() *semver.Version } type storage struct { lg *zap.Logger s *snap.Snapshotter // Mutex protected variables mux sync.RWMutex w *wal.WAL } func NewStorage(lg *zap.Logger, w *wal.WAL, s *snap.Snapshotter) Storage { return &storage{lg: lg, w: w, s: s} } // SaveSnap saves the snapshot file to disk and writes the WAL snapshot entry. func (st *storage) SaveSnap(snap raftpb.Snapshot) error { st.mux.RLock() defer st.mux.RUnlock() walsnap := walpb.Snapshot{ Index: new(snap.Metadata.Index), Term: new(snap.Metadata.Term), ConfState: &snap.Metadata.ConfState, } // save the snapshot file before writing the snapshot to the wal. // This makes it possible for the snapshot file to become orphaned, but prevents // a WAL snapshot entry from having no corresponding snapshot file. err := st.s.SaveSnap(snap) if err != nil { return err } // gofail: var raftBeforeWALSaveSnaphot struct{} return st.w.SaveSnapshot(walsnap) } // Release releases resources older than the given snap and are no longer needed: // - releases the locks to the wal files that are older than the provided wal for the given snap. // - deletes any .snap.db files that are older than the given snap. func (st *storage) Release(snap raftpb.Snapshot) error { st.mux.RLock() defer st.mux.RUnlock() if err := st.w.ReleaseLockTo(snap.Metadata.Index); err != nil { return err } return st.s.ReleaseSnapDBs(snap) } func (st *storage) Save(s raftpb.HardState, ents []raftpb.Entry) error { st.mux.RLock() defer st.mux.RUnlock() return st.w.Save(s, ents) } func (st *storage) Close() error { st.mux.Lock() defer st.mux.Unlock() return st.w.Close() } func (st *storage) Sync() error { st.mux.RLock() defer st.mux.RUnlock() return st.w.Sync() } func (st *storage) MinimalEtcdVersion() *semver.Version { st.mux.Lock() defer st.mux.Unlock() walsnap := walpb.Snapshot{} sn, err := st.s.Load() if err != nil && !errors.Is(err, snap.ErrNoSnapshot) { panic(err) } if sn != nil { walsnap.Index = new(sn.Metadata.Index) walsnap.Term = new(sn.Metadata.Term) walsnap.ConfState = &sn.Metadata.ConfState } w, err := st.w.Reopen(st.lg, walsnap) if err != nil { panic(err) } _, _, ents, err := w.ReadAll() if err != nil { panic(err) } v := wal.MinimalEtcdVersion(ents) st.w = w return v } ================================================ FILE: server/storage/util.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage import ( "encoding/json" "fmt" "sort" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/raft/v3/raftpb" ) // AssertNoV2StoreContent -> depending on the deprecation stage, warns or report an error // if the v2store contains custom content. func AssertNoV2StoreContent(lg *zap.Logger, st v2store.Store, deprecationStage config.V2DeprecationEnum) error { metaOnly, err := membership.IsMetaStoreOnly(st) if err != nil { return err } if metaOnly { return nil } if deprecationStage.IsAtLeast(config.V2Depr1WriteOnly) { return fmt.Errorf("detected disallowed custom content in v2store for stage --v2-deprecation=%s", deprecationStage) } lg.Warn("detected custom v2store content. Etcd v3.5 is the last version allowing to access it using API v2. Please remove the content.") return nil } // CreateConfigChangeEnts creates a series of Raft entries (i.e. // EntryConfChange) to remove the set of given IDs from the cluster. The ID // `self` is _not_ removed, even if present in the set. // If `self` is not inside the given ids, it creates a Raft entry to add a // default member with the given `self`. func CreateConfigChangeEnts(lg *zap.Logger, ids []uint64, self uint64, term, index uint64) []raftpb.Entry { found := false for _, id := range ids { if id == self { found = true } } var ents []raftpb.Entry next := index + 1 // NB: always add self first, then remove other nodes. Raft will panic if the // set of voters ever becomes empty. if !found { m := membership.Member{ ID: types.ID(self), RaftAttributes: membership.RaftAttributes{PeerURLs: []string{"http://localhost:2380"}}, } ctx, err := json.Marshal(m) if err != nil { lg.Panic("failed to marshal member", zap.Error(err)) } cc := &raftpb.ConfChange{ Type: raftpb.ConfChangeAddNode, NodeID: self, Context: ctx, } e := raftpb.Entry{ Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(cc), Term: term, Index: next, } ents = append(ents, e) next++ } for _, id := range ids { if id == self { continue } cc := &raftpb.ConfChange{ Type: raftpb.ConfChangeRemoveNode, NodeID: id, } e := raftpb.Entry{ Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(cc), Term: term, Index: next, } ents = append(ents, e) next++ } return ents } // GetEffectiveNodeIDsFromWALEntries returns an ordered set of IDs included in the given snapshot and // the entries. The given snapshot/entries can contain three kinds of // ID-related entry: // - ConfChangeAddNode, in which case the contained ID will Be added into the set. // - ConfChangeRemoveNode, in which case the contained ID will Be removed from the set. // - ConfChangeAddLearnerNode, in which the contained ID will Be added into the set. func GetEffectiveNodeIDsFromWALEntries(lg *zap.Logger, snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 { ids := make(map[uint64]bool) if snap != nil { for _, id := range snap.Metadata.ConfState.Voters { ids[id] = true } for _, id := range snap.Metadata.ConfState.Learners { ids[id] = true } } for _, e := range ents { if e.Type != raftpb.EntryConfChange { continue } var cc raftpb.ConfChange pbutil.MustUnmarshal(&cc, e.Data) switch cc.Type { case raftpb.ConfChangeAddLearnerNode: ids[cc.NodeID] = true case raftpb.ConfChangeAddNode: ids[cc.NodeID] = true case raftpb.ConfChangeRemoveNode: delete(ids, cc.NodeID) case raftpb.ConfChangeUpdateNode: // do nothing default: lg.Panic("unknown ConfChange Type", zap.String("type", cc.Type.String())) } } sids := make(types.Uint64Slice, 0, len(ids)) for id := range ids { sids = append(sids, id) } sort.Sort(sids) return sids } ================================================ FILE: server/storage/wal/decoder.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "encoding/binary" "errors" "fmt" "hash" "io" "sync" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/pkg/v3/crc" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) const minSectorSize = 512 // frameSizeBytes is frame size in bytes, including record size and padding size. const frameSizeBytes = 8 type Decoder interface { Decode(rec *walpb.Record) error LastOffset() int64 LastCRC() uint32 UpdateCRC(prevCrc uint32) } type decoder struct { mu sync.Mutex brs []*fileutil.FileBufReader // lastValidOff file offset following the last valid decoded record lastValidOff int64 crc hash.Hash32 // continueOnCrcError - causes the decoder to continue working even in case of crc mismatch. // This is a desired mode for tools performing inspection of the corrupted WAL logs. // See comments on 'Decode' method for semantic. continueOnCrcError bool } func NewDecoderAdvanced(continueOnCrcError bool, r ...fileutil.FileReader) Decoder { readers := make([]*fileutil.FileBufReader, len(r)) for i := range r { readers[i] = fileutil.NewFileBufReader(r[i]) } return &decoder{ brs: readers, crc: crc.New(0, crcTable), continueOnCrcError: continueOnCrcError, } } func NewDecoder(r ...fileutil.FileReader) Decoder { return NewDecoderAdvanced(false, r...) } // Decode reads the next record out of the file. // In the success path, fills 'rec' and returns nil. // When it fails, it returns err and usually resets 'rec' to the defaults. // When continueOnCrcError is set, the method may return ErrUnexpectedEOF or ErrCRCMismatch, but preserve the read // (potentially corrupted) record content. func (d *decoder) Decode(rec *walpb.Record) error { rec.Reset() d.mu.Lock() defer d.mu.Unlock() return d.decodeRecord(rec) } func (d *decoder) decodeRecord(rec *walpb.Record) error { if len(d.brs) == 0 { return io.EOF } fileBufReader := d.brs[0] l, err := readInt64(fileBufReader) if errors.Is(err, io.EOF) || (err == nil && l == 0) { // hit end of file or preallocated space d.brs = d.brs[1:] if len(d.brs) == 0 { return io.EOF } d.lastValidOff = 0 return d.decodeRecord(rec) } if err != nil { return err } recBytes, padBytes := decodeFrameSize(l) // The length of current WAL entry must be less than the remaining file size. maxEntryLimit := fileBufReader.FileInfo().Size() - d.lastValidOff - padBytes if recBytes > maxEntryLimit { return fmt.Errorf("%w: [wal] max entry size limit exceeded when reading %q, recBytes: %d, fileSize(%d) - offset(%d) - padBytes(%d) = entryLimit(%d)", io.ErrUnexpectedEOF, fileBufReader.FileInfo().Name(), recBytes, fileBufReader.FileInfo().Size(), d.lastValidOff, padBytes, maxEntryLimit) } data := make([]byte, recBytes+padBytes) if _, err = io.ReadFull(fileBufReader, data); err != nil { // ReadFull returns io.EOF only if no bytes were read // the decoder should treat this as an ErrUnexpectedEOF instead. if errors.Is(err, io.EOF) { err = io.ErrUnexpectedEOF } return err } if err := rec.Unmarshal(data[:recBytes]); err != nil { if d.isTornEntry(data) { return io.ErrUnexpectedEOF } return err } // skip crc checking if the record type is CrcType if rec.GetType() != CrcType { _, err := d.crc.Write(rec.Data) if err != nil { return err } if err := rec.Validate(d.crc.Sum32()); err != nil { if !d.continueOnCrcError { rec.Reset() } else { // If we continue, we want to update lastValidOff, such that following errors are consistent defer func() { d.lastValidOff += frameSizeBytes + recBytes + padBytes }() } if d.isTornEntry(data) { return fmt.Errorf("%w: in file '%s' at position: %d", io.ErrUnexpectedEOF, fileBufReader.FileInfo().Name(), d.lastValidOff) } return fmt.Errorf("%w: in file '%s' at position: %d", err, fileBufReader.FileInfo().Name(), d.lastValidOff) } } // record decoded as valid; point last valid offset to end of record d.lastValidOff += frameSizeBytes + recBytes + padBytes return nil } func decodeFrameSize(lenField int64) (recBytes int64, padBytes int64) { // the record size is stored in the lower 56 bits of the 64-bit length recBytes = int64(uint64(lenField) & ^(uint64(0xff) << 56)) // non-zero padding is indicated by set MSb / a negative length if lenField < 0 { // padding is stored in lower 3 bits of length MSB padBytes = int64((uint64(lenField) >> 56) & 0x7) } return recBytes, padBytes } // isTornEntry determines whether the last entry of the WAL was partially written // and corrupted because of a torn write. func (d *decoder) isTornEntry(data []byte) bool { if len(d.brs) != 1 { return false } fileOff := d.lastValidOff + frameSizeBytes curOff := 0 var chunks [][]byte // split data on sector boundaries for curOff < len(data) { chunkLen := int(minSectorSize - (fileOff % minSectorSize)) if chunkLen > len(data)-curOff { chunkLen = len(data) - curOff } chunks = append(chunks, data[curOff:curOff+chunkLen]) fileOff += int64(chunkLen) curOff += chunkLen } // if any data for a sector chunk is all 0, it's a torn write for _, sect := range chunks { isZero := true for _, v := range sect { if v != 0 { isZero = false break } } if isZero { return true } } return false } func (d *decoder) UpdateCRC(prevCrc uint32) { d.crc = crc.New(prevCrc, crcTable) } func (d *decoder) LastCRC() uint32 { return d.crc.Sum32() } func (d *decoder) LastOffset() int64 { return d.lastValidOff } func MustUnmarshalEntry(d []byte) raftpb.Entry { var e raftpb.Entry pbutil.MustUnmarshal(&e, d) return e } func MustUnmarshalState(d []byte) raftpb.HardState { var s raftpb.HardState pbutil.MustUnmarshal(&s, d) return s } func readInt64(r io.Reader) (int64, error) { var n int64 err := binary.Read(r, binary.LittleEndian, &n) return n, err } ================================================ FILE: server/storage/wal/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /* Package wal provides an implementation of write ahead log that is used by etcd. A WAL is created at a particular directory and is made up of a number of segmented WAL files. Inside each file the raft state and entries are appended to it with the Save method: metadata := []byte{} w, err := wal.Create(zap.NewExample(), "/var/lib/etcd", metadata) ... err := w.Save(s, ents) After saving a raft snapshot to disk, SaveSnapshot method should be called to record it. So WAL can match with the saved snapshot when restarting. index := uint64(10) term := uint64(2) err := w.SaveSnapshot(walpb.Snapshot{Index: &index, Term: &term}) When a user has finished using a WAL it must be closed: w.Close() Each WAL file is a stream of WAL records. A WAL record is a length field and a wal record protobuf. The record protobuf contains a CRC, a type, and a data payload. The length field is a 64-bit packed structure holding the length of the remaining logical record data in its lower 56 bits and its physical padding in the first three bits of the most significant byte. Each record is 8-byte aligned so that the length field is never torn. The CRC contains the CRC32 value of all record protobufs preceding the current record. WAL files are placed inside the directory in the following format: $seq-$index.wal The first WAL file to be created will be 0000000000000000-0000000000000000.wal indicating an initial sequence of 0 and an initial raft index of 0. The first entry written to WAL MUST have raft index 0. WAL will cut its current tail wal file if its size exceeds 64 MB. This will increment an internal sequence number and cause a new file to be created. If the last raft index saved was 0x20 and this is the first time cut has been called on this WAL then the sequence will increment from 0x0 to 0x1. The new file will be: 0000000000000001-0000000000000021.wal. If a second cut issues 0x10 entries with incremental index later, then the file will be called: 0000000000000002-0000000000000031.wal. At a later time a WAL can be opened at a particular snapshot. If there is no snapshot, an empty snapshot should be passed in. index := uint64(10) term := uint64(2) w, err := wal.Open("/var/lib/etcd", walpb.Snapshot{Index: &index, Term: &term}) ... The snapshot must have been written to the WAL. Additional items cannot be Saved to this WAL until all the items from the given snapshot to the end of the WAL are read first: metadata, state, ents, err := w.ReadAll() This will give you the metadata, the last raft.State and the slice of raft.Entry items in the log. */ package wal ================================================ FILE: server/storage/wal/encoder.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "encoding/binary" "errors" "hash" "io" "os" "sync" "time" "go.etcd.io/etcd/pkg/v3/crc" "go.etcd.io/etcd/pkg/v3/ioutil" "go.etcd.io/etcd/server/v3/storage/wal/walpb" ) // walPageBytes is the alignment for flushing records to the backing Writer. // It should be a multiple of the minimum sector size so that WAL can safely // distinguish between torn writes and ordinary data corruption. const walPageBytes = 8 * minSectorSize type encoder struct { mu sync.Mutex bw *ioutil.PageWriter crc hash.Hash32 buf []byte uint64buf []byte } func newEncoder(w io.Writer, prevCrc uint32, pageOffset int) *encoder { return &encoder{ bw: ioutil.NewPageWriter(w, walPageBytes, pageOffset), crc: crc.New(prevCrc, crcTable), // 1MB buffer buf: make([]byte, 1024*1024), uint64buf: make([]byte, 8), } } // newFileEncoder creates a new encoder with current file offset for the page writer. func newFileEncoder(f *os.File, prevCrc uint32) (*encoder, error) { offset, err := f.Seek(0, io.SeekCurrent) if err != nil { return nil, err } return newEncoder(f, prevCrc, int(offset)), nil } func (e *encoder) encode(rec *walpb.Record) error { if rec.Type == nil { return errors.New("record is missing type") } e.mu.Lock() defer e.mu.Unlock() e.crc.Write(rec.Data) rec.Crc = new(e.crc.Sum32()) var ( data []byte err error n int ) if rec.Size() > len(e.buf) { data, err = rec.Marshal() if err != nil { return err } } else { n, err = rec.MarshalTo(e.buf) if err != nil { return err } data = e.buf[:n] } data, lenField := prepareDataWithPadding(data) return write(e.bw, e.uint64buf, data, lenField) } func encodeFrameSize(dataBytes int) (lenField uint64, padBytes int) { lenField = uint64(dataBytes) // force 8 byte alignment so length never gets a torn write padBytes = (8 - (dataBytes % 8)) % 8 if padBytes != 0 { lenField |= uint64(0x80|padBytes) << 56 } return lenField, padBytes } func (e *encoder) flush() error { e.mu.Lock() defer e.mu.Unlock() return e.bw.Flush() } func prepareDataWithPadding(data []byte) ([]byte, uint64) { lenField, padBytes := encodeFrameSize(len(data)) if padBytes != 0 { data = append(data, make([]byte, padBytes)...) } return data, lenField } func write(w io.Writer, uint64buf, data []byte, lenField uint64) error { // write padding info binary.LittleEndian.PutUint64(uint64buf, lenField) start := time.Now() nv, err := w.Write(uint64buf) walWriteBytes.Add(float64(nv)) if err != nil { return err } // write the record with padding n, err := w.Write(data) walWriteSec.Observe(time.Since(start).Seconds()) walWriteBytes.Add(float64(n)) return err } ================================================ FILE: server/storage/wal/file_pipeline.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "fmt" "os" "path/filepath" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/fileutil" ) // filePipeline pipelines allocating disk space type filePipeline struct { lg *zap.Logger // dir to put files dir string // size of files to make, in bytes size int64 // count number of files generated count int filec chan *fileutil.LockedFile errc chan error donec chan struct{} } func newFilePipeline(lg *zap.Logger, dir string, fileSize int64) *filePipeline { if lg == nil { lg = zap.NewNop() } fp := &filePipeline{ lg: lg, dir: dir, size: fileSize, filec: make(chan *fileutil.LockedFile), errc: make(chan error, 1), donec: make(chan struct{}), } go fp.run() return fp } // Open returns a fresh file for writing. Rename the file before calling // Open again or there will be file collisions. // it will 'block' if the tmp file lock is already taken. func (fp *filePipeline) Open() (f *fileutil.LockedFile, err error) { select { case f = <-fp.filec: case err = <-fp.errc: } return f, err } func (fp *filePipeline) Close() error { close(fp.donec) return <-fp.errc } func (fp *filePipeline) alloc() (f *fileutil.LockedFile, err error) { // count % 2 so this file isn't the same as the one last published fpath := filepath.Join(fp.dir, fmt.Sprintf("%d.tmp", fp.count%2)) if f, err = createNewWALFile[*fileutil.LockedFile](fpath, false); err != nil { return nil, err } if err = fileutil.Preallocate(f.File, fp.size, true); err != nil { fp.lg.Error("failed to preallocate space when creating a new WAL", zap.Int64("size", fp.size), zap.Error(err)) f.Close() return nil, err } fp.count++ return f, nil } func (fp *filePipeline) run() { defer close(fp.errc) for { f, err := fp.alloc() if err != nil { fp.errc <- err return } select { case fp.filec <- f: case <-fp.donec: os.Remove(f.Name()) f.Close() return } } } ================================================ FILE: server/storage/wal/file_pipeline_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "math" "testing" "go.uber.org/zap/zaptest" ) func TestFilePipeline(t *testing.T) { tdir := t.TempDir() fp := newFilePipeline(zaptest.NewLogger(t), tdir, SegmentSizeBytes) defer fp.Close() f, ferr := fp.Open() if ferr != nil { t.Fatal(ferr) } f.Close() } func TestFilePipelineFailPreallocate(t *testing.T) { tdir := t.TempDir() fp := newFilePipeline(zaptest.NewLogger(t), tdir, math.MaxInt64) defer fp.Close() f, ferr := fp.Open() if f != nil || ferr == nil { // no space left on device t.Fatal("expected error on invalid pre-allocate size, but no error") } } ================================================ FILE: server/storage/wal/metrics.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import "github.com/prometheus/client_golang/prometheus" var ( walFsyncSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "disk", Name: "wal_fsync_duration_seconds", Help: "The latency distributions of fsync called by WAL.", // lowest bucket start of upper bound 0.001 sec (1 ms) with factor 2 // highest bucket start of 0.001 sec * 2^13 == 8.192 sec Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) walWriteSec = prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "etcd", Subsystem: "disk", Name: "wal_write_duration_seconds", Help: "The latency distributions of write called by WAL.", Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), }) walWriteBytes = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "disk", Name: "wal_write_bytes_total", Help: "Total number of bytes written in WAL.", }) ) func init() { prometheus.MustRegister(walFsyncSec) prometheus.MustRegister(walWriteSec) prometheus.MustRegister(walWriteBytes) } ================================================ FILE: server/storage/wal/record_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "bytes" "errors" "hash/crc32" "io" "os" "reflect" "testing" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/server/v3/storage/wal/walpb" ) var ( infoData = []byte("\b\xef\xfd\x02") infoRecord = append([]byte("\x0e\x00\x00\x00\x00\x00\x00\x00\b\x01\x10\x99\xb5\xe4\xd0\x03\x1a\x04"), infoData...) ) func TestReadRecord(t *testing.T) { badInfoRecord := make([]byte, len(infoRecord)) copy(badInfoRecord, infoRecord) badInfoRecord[len(badInfoRecord)-1] = 'a' tests := []struct { data []byte wr *walpb.Record we error }{ {infoRecord, &walpb.Record{Type: new(int64(1)), Crc: new(crc32.Checksum(infoData, crcTable)), Data: infoData}, nil}, {[]byte(""), &walpb.Record{}, io.EOF}, {infoRecord[:14], &walpb.Record{}, io.ErrUnexpectedEOF}, {infoRecord[:len(infoRecord)-len(infoData)], &walpb.Record{}, io.ErrUnexpectedEOF}, {infoRecord[:len(infoRecord)-8], &walpb.Record{}, io.ErrUnexpectedEOF}, {badInfoRecord, &walpb.Record{}, walpb.ErrCRCMismatch}, } rec := &walpb.Record{} for i, tt := range tests { buf := bytes.NewBuffer(tt.data) f, err := createFileWithData(t, buf) if err != nil { t.Errorf("Unexpected error: %v", err) } decoder := NewDecoder(fileutil.NewFileReader(f)) e := decoder.Decode(rec) if !reflect.DeepEqual(rec, tt.wr) { t.Errorf("#%d: block = %v, want %v", i, rec, tt.wr) } if !errors.Is(e, tt.we) { t.Errorf("#%d: err = %v, want %v", i, e, tt.we) } rec = &walpb.Record{} } } func TestWriteRecord(t *testing.T) { b := &walpb.Record{} typ := int64(0xABCD) d := []byte("Hello world!") buf := new(bytes.Buffer) e := newEncoder(buf, 0, 0) e.encode(&walpb.Record{Type: new(typ), Data: d}) e.flush() f, err := createFileWithData(t, buf) if err != nil { t.Errorf("Unexpected error: %v", err) } decoder := NewDecoder(fileutil.NewFileReader(f)) err = decoder.Decode(b) if err != nil { t.Errorf("err = %v, want nil", err) } if b.GetType() != typ { t.Errorf("type = %d, want %d", b.Type, typ) } if !reflect.DeepEqual(b.Data, d) { t.Errorf("data = %v, want %v", b.Data, d) } } func createFileWithData(t *testing.T, bf *bytes.Buffer) (*os.File, error) { f, err := os.CreateTemp(t.TempDir(), "wal") if err != nil { return nil, err } if _, err := f.Write(bf.Bytes()); err != nil { return nil, err } f.Seek(0, 0) return f, nil } ================================================ FILE: server/storage/wal/repair.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "errors" "io" "os" "path/filepath" "time" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/server/v3/storage/wal/walpb" ) // Repair tries to repair ErrUnexpectedEOF in the // last wal file by truncating. func Repair(lg *zap.Logger, dirpath string) bool { if lg == nil { lg = zap.NewNop() } f, err := openLast(lg, dirpath) if err != nil { return false } defer f.Close() lg.Info("repairing", zap.String("path", f.Name())) rec := &walpb.Record{} decoder := NewDecoder(fileutil.NewFileReader(f.File)) for { lastOffset := decoder.LastOffset() err := decoder.Decode(rec) switch { case err == nil: // update crc of the decoder when necessary if rec.GetType() == CrcType { crc := decoder.LastCRC() // current crc of decoder must match the crc of the record. // do no need to match 0 crc, since the decoder is a new one at this case. if crc != 0 && rec.Validate(crc) != nil { return false } decoder.UpdateCRC(rec.GetCrc()) } continue case errors.Is(err, io.EOF): lg.Info("repaired", zap.String("path", f.Name()), zap.Error(io.EOF)) return true case errors.Is(err, io.ErrUnexpectedEOF): brokenName := f.Name() + ".broken" bf, bferr := createNewWALFile[*os.File](brokenName, true) if bferr != nil { lg.Warn("failed to create backup file", zap.String("path", brokenName), zap.Error(bferr)) return false } defer bf.Close() if _, err = f.Seek(0, io.SeekStart); err != nil { lg.Warn("failed to read file", zap.String("path", f.Name()), zap.Error(err)) return false } if _, err = io.Copy(bf, f); err != nil { lg.Warn("failed to copy", zap.String("from", f.Name()), zap.String("to", brokenName), zap.Error(err)) return false } if err = f.Truncate(lastOffset); err != nil { lg.Warn("failed to truncate", zap.String("path", f.Name()), zap.Error(err)) return false } start := time.Now() if err = fileutil.Fsync(f.File); err != nil { lg.Warn("failed to fsync", zap.String("path", f.Name()), zap.Error(err)) return false } walFsyncSec.Observe(time.Since(start).Seconds()) lg.Info("repaired", zap.String("path", f.Name()), zap.Error(io.ErrUnexpectedEOF)) return true default: lg.Warn("failed to repair", zap.String("path", f.Name()), zap.Error(err)) return false } } } // openLast opens the last wal file for read and write. func openLast(lg *zap.Logger, dirpath string) (*fileutil.LockedFile, error) { names, err := readWALNames(lg, dirpath) if err != nil { return nil, err } last := filepath.Join(dirpath, names[len(names)-1]) return fileutil.LockFile(last, os.O_RDWR, fileutil.PrivateFileMode) } ================================================ FILE: server/storage/wal/repair_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "fmt" "io" "os" "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) type corruptFunc func(string, int64) error // TestRepairTruncate ensures a truncated file can be repaired func TestRepairTruncate(t *testing.T) { corruptf := func(p string, offset int64) error { f, err := openLast(zaptest.NewLogger(t), p) if err != nil { return err } defer f.Close() return f.Truncate(offset - 4) } testRepair(t, makeEnts(10), corruptf, 9) } func testRepair(t *testing.T, ents [][]raftpb.Entry, corrupt corruptFunc, expectedEnts int) { lg := zaptest.NewLogger(t) p := t.TempDir() // create WAL w, err := Create(lg, p, nil) defer func() { // The Close might fail. _ = w.Close() }() require.NoError(t, err) for _, es := range ents { require.NoError(t, w.Save(raftpb.HardState{}, es)) } offset, err := w.tail().Seek(0, io.SeekCurrent) require.NoError(t, err) require.NoError(t, w.Close()) require.NoError(t, corrupt(p, offset)) // verify we broke the wal w, err = Open(zaptest.NewLogger(t), p, walpb.Snapshot{}) require.NoError(t, err) _, _, _, err = w.ReadAll() require.ErrorIs(t, err, io.ErrUnexpectedEOF) require.NoError(t, w.Close()) // repair the wal require.True(t, Repair(lg, p)) // verify the broken wal has correct permissions bf := filepath.Join(p, filepath.Base(w.tail().Name())+".broken") fi, err := os.Stat(bf) require.NoError(t, err) expectedPerms := fmt.Sprintf("%o", os.FileMode(fileutil.PrivateFileMode)) actualPerms := fmt.Sprintf("%o", fi.Mode().Perm()) require.Equalf(t, expectedPerms, actualPerms, "unexpected file permissions on .broken wal") // read it back w, err = Open(lg, p, walpb.Snapshot{}) require.NoError(t, err) _, _, walEnts, err := w.ReadAll() require.NoError(t, err) assert.Len(t, walEnts, expectedEnts) // write some more entries to repaired log for i := 1; i <= 10; i++ { es := []raftpb.Entry{{Index: uint64(expectedEnts + i)}} require.NoError(t, w.Save(raftpb.HardState{}, es)) } require.NoError(t, w.Close()) // read back entries following repair, ensure it's all there w, err = Open(lg, p, walpb.Snapshot{}) require.NoError(t, err) _, _, walEnts, err = w.ReadAll() require.NoError(t, err) assert.Len(t, walEnts, expectedEnts+10) } func makeEnts(ents int) (ret [][]raftpb.Entry) { for i := 1; i <= ents; i++ { ret = append(ret, []raftpb.Entry{{Index: uint64(i)}}) } return ret } // TestRepairWriteTearLast repairs the WAL in case the last record is a torn write // that straddled two sectors. func TestRepairWriteTearLast(t *testing.T) { corruptf := func(p string, offset int64) error { f, err := openLast(zaptest.NewLogger(t), p) if err != nil { return err } defer f.Close() // 512 bytes perfectly aligns the last record, so use 1024 if offset < 1024 { return fmt.Errorf("got offset %d, expected >1024", offset) } if terr := f.Truncate(1024); terr != nil { return terr } return f.Truncate(offset) } testRepair(t, makeEnts(50), corruptf, 40) } // TestRepairWriteTearMiddle repairs the WAL when there is write tearing // in the middle of a record. func TestRepairWriteTearMiddle(t *testing.T) { corruptf := func(p string, offset int64) error { f, err := openLast(zaptest.NewLogger(t), p) if err != nil { return err } defer f.Close() // corrupt middle of 2nd record _, werr := f.WriteAt(make([]byte, 512), 4096+512) return werr } ents := makeEnts(5) // 4096 bytes of data so a middle sector is easy to corrupt dat := make([]byte, 4096) for i := range dat { dat[i] = byte(i) } for i := range ents { ents[i][0].Data = dat } testRepair(t, ents, corruptf, 1) } func TestRepairFailDeleteDir(t *testing.T) { p := t.TempDir() w, err := Create(zaptest.NewLogger(t), p, nil) if err != nil { t.Fatal(err) } oldSegmentSizeBytes := SegmentSizeBytes SegmentSizeBytes = 64 defer func() { SegmentSizeBytes = oldSegmentSizeBytes }() for _, es := range makeEnts(50) { if err = w.Save(raftpb.HardState{}, es); err != nil { t.Fatal(err) } } _, serr := w.tail().Seek(0, io.SeekCurrent) if serr != nil { t.Fatal(serr) } w.Close() f, err := openLast(zaptest.NewLogger(t), p) if err != nil { t.Fatal(err) } if terr := f.Truncate(20); terr != nil { t.Fatal(err) } f.Close() w, err = Open(zaptest.NewLogger(t), p, walpb.Snapshot{}) if err != nil { t.Fatal(err) } _, _, _, err = w.ReadAll() require.ErrorIsf(t, err, io.ErrUnexpectedEOF, "err = %v, want error %v", err, io.ErrUnexpectedEOF) w.Close() os.RemoveAll(p) require.Falsef(t, Repair(zaptest.NewLogger(t), p), "expect 'Repair' fail on unexpected directory deletion") } ================================================ FILE: server/storage/wal/testing/waltesting.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testing import ( "os" "path/filepath" "testing" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) func NewTmpWAL(tb testing.TB, reqs []etcdserverpb.InternalRaftRequest) (*wal.WAL, string) { tb.Helper() dir, err := os.MkdirTemp(tb.TempDir(), "etcd_wal_test") if err != nil { panic(err) } tmpPath := filepath.Join(dir, "wal") lg := zaptest.NewLogger(tb) w, err := wal.Create(lg, tmpPath, nil) if err != nil { tb.Fatalf("Failed to create WAL: %v", err) } err = w.Close() if err != nil { tb.Fatalf("Failed to close WAL: %v", err) } if len(reqs) != 0 { w, err = wal.Open(lg, tmpPath, walpb.Snapshot{}) if err != nil { tb.Fatalf("Failed to open WAL: %v", err) } var state raftpb.HardState _, state, _, err = w.ReadAll() if err != nil { tb.Fatalf("Failed to read WAL: %v", err) } var entries []raftpb.Entry for _, req := range reqs { entries = append(entries, raftpb.Entry{ Term: 1, Index: 1, Type: raftpb.EntryNormal, Data: pbutil.MustMarshal(&req), }) } err = w.Save(state, entries) if err != nil { tb.Fatalf("Failed to save WAL: %v", err) } err = w.Close() if err != nil { tb.Fatalf("Failed to close WAL: %v", err) } } w, err = wal.OpenForRead(lg, tmpPath, walpb.Snapshot{}) if err != nil { tb.Fatalf("Failed to open WAL: %v", err) } return w, tmpPath } func Reopen(tb testing.TB, walPath string) *wal.WAL { tb.Helper() lg := zaptest.NewLogger(tb) w, err := wal.OpenForRead(lg, walPath, walpb.Snapshot{}) if err != nil { tb.Fatalf("Failed to open WAL: %v", err) } return w } ================================================ FILE: server/storage/wal/util.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "errors" "fmt" "strings" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/fileutil" ) var errBadWALName = errors.New("bad wal name") // Exist returns true if there are any files in a given directory. func Exist(dir string) bool { names, err := fileutil.ReadDir(dir, fileutil.WithExt(".wal")) if err != nil { return false } return len(names) != 0 } // searchIndex returns the last array index of names whose raft index section is // equal to or smaller than the given index. // The given names MUST be sorted. func searchIndex(lg *zap.Logger, names []string, index uint64) (int, bool) { for i := len(names) - 1; i >= 0; i-- { name := names[i] _, curIndex, err := parseWALName(name) if err != nil { lg.Panic("failed to parse WAL file name", zap.String("path", name), zap.Error(err)) } if index >= curIndex { return i, true } } return -1, false } // names should have been sorted based on sequence number. // isValidSeq checks whether seq increases continuously. func isValidSeq(lg *zap.Logger, names []string) bool { var lastSeq uint64 for _, name := range names { curSeq, _, err := parseWALName(name) if err != nil { lg.Panic("failed to parse WAL file name", zap.String("path", name), zap.Error(err)) } if lastSeq != 0 && lastSeq != curSeq-1 { return false } lastSeq = curSeq } return true } func readWALNames(lg *zap.Logger, dirpath string) ([]string, error) { names, err := fileutil.ReadDir(dirpath) if err != nil { return nil, fmt.Errorf("[readWALNames] fileutil.ReadDir failed: %w", err) } wnames := checkWALNames(lg, names) if len(wnames) == 0 { return nil, ErrFileNotFound } return wnames, nil } func checkWALNames(lg *zap.Logger, names []string) []string { wnames := make([]string, 0) for _, name := range names { if _, _, err := parseWALName(name); err != nil { // don't complain about left over tmp files if !strings.HasSuffix(name, ".tmp") { lg.Warn( "ignored file in WAL directory", zap.String("path", name), ) } continue } wnames = append(wnames, name) } return wnames } func parseWALName(str string) (seq, index uint64, err error) { if !strings.HasSuffix(str, ".wal") { return 0, 0, errBadWALName } _, err = fmt.Sscanf(str, "%016x-%016x.wal", &seq, &index) return seq, index, err } func walName(seq, index uint64) string { return fmt.Sprintf("%016x-%016x.wal", seq, index) } ================================================ FILE: server/storage/wal/version.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "fmt" "strings" "github.com/coreos/go-semver/semver" "github.com/golang/protobuf/proto" //nolint:staticcheck // TODO: remove for a supported version "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/raft/v3/raftpb" ) // Version defines the wal version interface. type Version interface { // MinimalEtcdVersion returns minimal etcd version able to interpret WAL log. MinimalEtcdVersion() *semver.Version } // ReadWALVersion reads remaining entries from opened WAL and returns struct // that implements schema.WAL interface. func ReadWALVersion(w *WAL) (Version, error) { _, _, ents, err := w.ReadAll() if err != nil { return nil, err } return &walVersion{entries: ents}, nil } type walVersion struct { entries []raftpb.Entry } // MinimalEtcdVersion returns minimal etcd able to interpret entries from WAL log, func (w *walVersion) MinimalEtcdVersion() *semver.Version { return MinimalEtcdVersion(w.entries) } // MinimalEtcdVersion returns minimal etcd able to interpret entries from WAL log, // determined by looking at entries since the last snapshot and returning the highest // etcd version annotation from used messages, fields, enums and their values. func MinimalEtcdVersion(ents []raftpb.Entry) *semver.Version { var maxVer *semver.Version for _, ent := range ents { err := visitEntry(ent, func(path protoreflect.FullName, ver *semver.Version) error { maxVer = maxVersion(maxVer, ver) return nil }) if err != nil { panic(err) } } return maxVer } type Visitor func(path protoreflect.FullName, ver *semver.Version) error // VisitFileDescriptor calls visitor on each field and enum value with etcd version read from proto definition. // If field/enum value is not annotated, visitor will be called with nil. // Upon encountering invalid annotation, will immediately exit with error. func VisitFileDescriptor(file protoreflect.FileDescriptor, visitor Visitor) error { msgs := file.Messages() for i := 0; i < msgs.Len(); i++ { err := visitMessageDescriptor(msgs.Get(i), visitor) if err != nil { return err } } enums := file.Enums() for i := 0; i < enums.Len(); i++ { err := visitEnumDescriptor(enums.Get(i), visitor) if err != nil { return err } } return nil } func visitEntry(ent raftpb.Entry, visitor Visitor) error { err := visitMessage(proto.MessageReflect(&ent), visitor) if err != nil { return err } return visitEntryData(ent.Type, ent.Data, visitor) } func visitEntryData(entryType raftpb.EntryType, data []byte, visitor Visitor) error { var msg protoreflect.Message switch entryType { case raftpb.EntryNormal: var raftReq etcdserverpb.InternalRaftRequest if err := pbutil.Unmarshaler(&raftReq).Unmarshal(data); err != nil { return err } msg = proto.MessageReflect(&raftReq) if raftReq.DowngradeVersionTest != nil { ver, err := semver.NewVersion(raftReq.DowngradeVersionTest.Ver) if err != nil { return err } err = visitor(msg.Descriptor().FullName(), ver) if err != nil { return err } } case raftpb.EntryConfChange: var confChange raftpb.ConfChange err := pbutil.Unmarshaler(&confChange).Unmarshal(data) if err != nil { return nil } msg = proto.MessageReflect(&confChange) return visitor(msg.Descriptor().FullName(), &version.V3_0) case raftpb.EntryConfChangeV2: var confChange raftpb.ConfChangeV2 err := pbutil.Unmarshaler(&confChange).Unmarshal(data) if err != nil { return nil } msg = proto.MessageReflect(&confChange) return visitor(msg.Descriptor().FullName(), &version.V3_4) default: panic("unhandled") } return visitMessage(msg, visitor) } func visitMessageDescriptor(md protoreflect.MessageDescriptor, visitor Visitor) error { err := visitDescriptor(md, visitor) if err != nil { return err } fields := md.Fields() for i := 0; i < fields.Len(); i++ { fd := fields.Get(i) err = visitDescriptor(fd, visitor) if err != nil { return err } } enums := md.Enums() for i := 0; i < enums.Len(); i++ { err = visitEnumDescriptor(enums.Get(i), visitor) if err != nil { return err } } return err } func visitMessage(m protoreflect.Message, visitor Visitor) error { md := m.Descriptor() err := visitDescriptor(md, visitor) if err != nil { return err } m.Range(func(field protoreflect.FieldDescriptor, value protoreflect.Value) bool { fd := md.Fields().Get(field.Index()) err = visitDescriptor(fd, visitor) if err != nil { return false } switch m := value.Interface().(type) { case protoreflect.Message: err = visitMessage(m, visitor) case protoreflect.EnumNumber: err = visitEnumNumber(fd.Enum(), m, visitor) } return err == nil }) return err } func visitEnumDescriptor(enum protoreflect.EnumDescriptor, visitor Visitor) error { err := visitDescriptor(enum, visitor) if err != nil { return err } fields := enum.Values() for i := 0; i < fields.Len(); i++ { fd := fields.Get(i) err = visitDescriptor(fd, visitor) if err != nil { return err } } return err } func visitEnumNumber(enum protoreflect.EnumDescriptor, number protoreflect.EnumNumber, visitor Visitor) error { err := visitDescriptor(enum, visitor) if err != nil { return err } intNumber := int(number) fields := enum.Values() if intNumber >= fields.Len() || intNumber < 0 { return fmt.Errorf("could not visit EnumNumber [%d]", intNumber) } return visitEnumValue(fields.Get(intNumber), visitor) } func visitEnumValue(enum protoreflect.EnumValueDescriptor, visitor Visitor) error { valueOpts := enum.Options().(*descriptorpb.EnumValueOptions) if valueOpts != nil { ver, _ := etcdVersionFromOptionsString(valueOpts.String()) err := visitor(enum.FullName(), ver) if err != nil { return err } } return nil } func visitDescriptor(md protoreflect.Descriptor, visitor Visitor) error { opts, ok := md.Options().(fmt.Stringer) if !ok { return nil } ver, err := etcdVersionFromOptionsString(opts.String()) if err != nil { return fmt.Errorf("%s: %w", md.FullName(), err) } return visitor(md.FullName(), ver) } func maxVersion(a *semver.Version, b *semver.Version) *semver.Version { if a != nil && (b == nil || b.LessThan(*a)) { return a } return b } func etcdVersionFromOptionsString(opts string) (*semver.Version, error) { // TODO: Use proto.GetExtention when gogo/protobuf is usable with protoreflect msgs := []string{"[versionpb.etcd_version_msg]:", "[versionpb.etcd_version_field]:", "[versionpb.etcd_version_enum]:", "[versionpb.etcd_version_enum_value]:"} var end, index int for _, msg := range msgs { index = strings.Index(opts, msg) end = index + len(msg) if index != -1 { break } } if index == -1 { return nil, nil } var verStr string _, err := fmt.Sscanf(opts[end:], "%q", &verStr) if err != nil { return nil, err } if strings.Count(verStr, ".") == 1 { verStr = verStr + ".0" } ver, err := semver.NewVersion(verStr) if err != nil { return nil, err } return ver, nil } ================================================ FILE: server/storage/wal/version_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "fmt" "testing" "github.com/coreos/go-semver/semver" "github.com/golang/protobuf/proto" //nolint:staticcheck // TODO: remove for a supported version "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/reflect/protoreflect" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/membershippb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/raft/v3/raftpb" ) func TestEtcdVersionFromEntry(t *testing.T) { raftReq := etcdserverpb.InternalRaftRequest{Header: &etcdserverpb.RequestHeader{AuthRevision: 1}} normalRequestData := pbutil.MustMarshal(&raftReq) downgradeVersionTestV3_6Req := etcdserverpb.InternalRaftRequest{DowngradeVersionTest: &etcdserverpb.DowngradeVersionTestRequest{Ver: "3.6.0"}} downgradeVersionTestV3_6Data := pbutil.MustMarshal(&downgradeVersionTestV3_6Req) downgradeVersionTestV3_7Req := etcdserverpb.InternalRaftRequest{DowngradeVersionTest: &etcdserverpb.DowngradeVersionTestRequest{Ver: "3.7.0"}} downgradeVersionTestV3_7Data := pbutil.MustMarshal(&downgradeVersionTestV3_7Req) confChange := raftpb.ConfChange{Type: raftpb.ConfChangeAddLearnerNode} confChangeData := pbutil.MustMarshal(&confChange) confChangeV2 := raftpb.ConfChangeV2{Transition: raftpb.ConfChangeTransitionJointExplicit} confChangeV2Data := pbutil.MustMarshal(&confChangeV2) tcs := []struct { name string input raftpb.Entry expect *semver.Version }{ { name: "Using RequestHeader AuthRevision in NormalEntry implies v3.1", input: raftpb.Entry{ Term: 1, Index: 2, Type: raftpb.EntryNormal, Data: normalRequestData, }, expect: &version.V3_1, }, { name: "Setting downgradeTest version to 3.6 implies version within WAL", input: raftpb.Entry{ Term: 1, Index: 2, Type: raftpb.EntryNormal, Data: downgradeVersionTestV3_6Data, }, expect: &version.V3_6, }, { name: "Setting downgradeTest version to 3.7 implies version within WAL", input: raftpb.Entry{ Term: 1, Index: 2, Type: raftpb.EntryNormal, Data: downgradeVersionTestV3_7Data, }, expect: &version.V3_7, }, { name: "Using ConfigChange implies v3.0", input: raftpb.Entry{ Term: 1, Index: 2, Type: raftpb.EntryConfChange, Data: confChangeData, }, expect: &version.V3_0, }, { name: "Using ConfigChangeV2 implies v3.4", input: raftpb.Entry{ Term: 1, Index: 2, Type: raftpb.EntryConfChangeV2, Data: confChangeV2Data, }, expect: &version.V3_4, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { var maxVer *semver.Version err := visitEntry(tc.input, func(path protoreflect.FullName, ver *semver.Version) error { maxVer = maxVersion(maxVer, ver) return nil }) require.NoError(t, err) assert.Equal(t, tc.expect, maxVer) }) } } func TestEtcdVersionFromMessage(t *testing.T) { tcs := []struct { name string input proto.Message expect *semver.Version }{ { name: "Empty RequestHeader impies v3.0", input: &etcdserverpb.RequestHeader{}, expect: &version.V3_0, }, { name: "RequestHeader AuthRevision field set implies v3.5", input: &etcdserverpb.RequestHeader{AuthRevision: 1}, expect: &version.V3_1, }, { name: "RequestHeader Username set implies v3.0", input: &etcdserverpb.RequestHeader{Username: "Alice"}, expect: &version.V3_0, }, { name: "When two fields are set take higher version", input: &etcdserverpb.RequestHeader{AuthRevision: 1, Username: "Alice"}, expect: &version.V3_1, }, { name: "Setting a RequestHeader AuthRevision in subfield implies v3.1", input: &etcdserverpb.InternalRaftRequest{Header: &etcdserverpb.RequestHeader{AuthRevision: 1}}, expect: &version.V3_1, }, { name: "Setting a DowngradeInfoSetRequest implies v3.5", input: &etcdserverpb.InternalRaftRequest{DowngradeInfoSet: &membershippb.DowngradeInfoSetRequest{}}, expect: &version.V3_5, }, { name: "Enum CompareResult set to EQUAL implies v3.0", input: &etcdserverpb.Compare{Result: etcdserverpb.Compare_EQUAL}, expect: &version.V3_0, }, { name: "Enum CompareResult set to NOT_EQUAL implies v3.1", input: &etcdserverpb.Compare{Result: etcdserverpb.Compare_NOT_EQUAL}, expect: &version.V3_1, }, { name: "Oneof Compare version set implies v3.1", input: &etcdserverpb.Compare{TargetUnion: &etcdserverpb.Compare_Version{}}, expect: &version.V3_0, }, { name: "Oneof Compare lease set implies v3.3", input: &etcdserverpb.Compare{TargetUnion: &etcdserverpb.Compare_Lease{}}, expect: &version.V3_3, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { var maxVer *semver.Version err := visitMessage(proto.MessageReflect(tc.input), func(path protoreflect.FullName, ver *semver.Version) error { maxVer = maxVersion(maxVer, ver) return nil }) require.NoError(t, err) assert.Equal(t, tc.expect, maxVer) }) } } func TestEtcdVersionFromFieldOptionsString(t *testing.T) { tcs := []struct { input string expect *semver.Version }{ { input: "65001:0", }, { input: `65001:0 65004:"NodeID"`, }, { input: `[versionpb.XXX]:"3.5"`, }, { input: `[versionpb.etcd_version_msg]:"3.5"`, expect: &version.V3_5, }, { input: `[versionpb.etcd_version_enum]:"3.5"`, expect: &version.V3_5, }, { input: `[versionpb.etcd_version_field]:"3.5"`, expect: &version.V3_5, }, { input: `[versionpb.etcd_version_enum_value]:"3.5"`, expect: &version.V3_5, }, { input: `65001:0 [versionpb.etcd_version_msg]:"3.5"`, expect: &version.V3_5, }, { input: `65004:"NodeID" [versionpb.etcd_version_msg]:"3.5"`, expect: &version.V3_5, }, { input: `65004:"NodeID" [versionpb.etcd_version_enum]:"3.5"`, expect: &version.V3_5, }, { input: `[versionpb.other_field]:"NodeID" [versionpb.etcd_version_msg]:"3.5"`, expect: &version.V3_5, }, { input: `[versionpb.etcd_version_msg]:"3.5" 65001:0`, expect: &version.V3_5, }, { input: `[versionpb.etcd_version_msg]:"3.5" 65004:"NodeID"`, expect: &version.V3_5, }, { input: `[versionpb.etcd_version_msg]:"3.5" [versionpb.other_field]:"NodeID"`, expect: &version.V3_5, }, { input: `[versionpb.other_field]:"NodeID" [versionpb.etcd_version_msg]:"3.5" [versionpb.another_field]:"NodeID"`, expect: &version.V3_5, }, { input: `65001:0 [versionpb.etcd_version_msg]:"3.5" 65001:0"`, expect: &version.V3_5, }, } for _, tc := range tcs { t.Run(tc.input, func(t *testing.T) { ver, err := etcdVersionFromOptionsString(tc.input) require.NoError(t, err) assert.Equal(t, ver, tc.expect) }) } } func TestMaxVersion(t *testing.T) { tcs := []struct { a, b, expect *semver.Version }{ { a: nil, b: nil, expect: nil, }, { a: &version.V3_5, b: nil, expect: &version.V3_5, }, { a: nil, b: &version.V3_5, expect: &version.V3_5, }, { a: &version.V3_6, b: &version.V3_5, expect: &version.V3_6, }, { a: &version.V3_5, b: &version.V3_6, expect: &version.V3_6, }, } for _, tc := range tcs { t.Run(fmt.Sprintf("%v %v %v", tc.a, tc.b, tc.expect), func(t *testing.T) { got := maxVersion(tc.a, tc.b) assert.Equal(t, got, tc.expect) }) } } ================================================ FILE: server/storage/wal/wal.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "bytes" "errors" "fmt" "hash/crc32" "io" "os" "path/filepath" "strings" "sync" "time" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3" "go.etcd.io/raft/v3/raftpb" ) const ( MetadataType int64 = iota + 1 EntryType StateType CrcType SnapshotType // warnSyncDuration is the amount of time allotted to an fsync before // logging a warning warnSyncDuration = time.Second ) var ( // SegmentSizeBytes is the preallocated size of each wal segment file. // The actual size might be larger than this. In general, the default // value should be used, but this is defined as an exported variable // so that tests can set a different segment size. SegmentSizeBytes int64 = 64 * 1000 * 1000 // 64MB ErrMetadataConflict = errors.New("wal: conflicting metadata found") ErrFileNotFound = errors.New("wal: file not found") ErrCRCMismatch = walpb.ErrCRCMismatch ErrSnapshotMismatch = errors.New("wal: snapshot mismatch") ErrSnapshotNotFound = errors.New("wal: snapshot not found") ErrSliceOutOfRange = errors.New("wal: slice bounds out of range") ErrDecoderNotFound = errors.New("wal: decoder not found") crcTable = crc32.MakeTable(crc32.Castagnoli) ) // WAL is a logical representation of the stable storage. // WAL is either in read mode or append mode but not both. // A newly created WAL is in append mode, and ready for appending records. // A just opened WAL is in read mode, and ready for reading records. // The WAL will be ready for appending after reading out all the previous records. type WAL struct { lg *zap.Logger dir string // the living directory of the underlay files // dirFile is a fd for the wal directory for syncing on Rename dirFile *os.File metadata []byte // metadata recorded at the head of each WAL state raftpb.HardState // hardstate recorded at the head of WAL start walpb.Snapshot // snapshot to start reading decoder Decoder // decoder to Decode records readClose func() error // closer for Decode reader unsafeNoSync bool // if set, do not fsync mu sync.Mutex enti uint64 // index of the last entry saved to the wal encoder *encoder // encoder to encode records locks []*fileutil.LockedFile // the locked files the WAL holds (the name is increasing) fp *filePipeline } // Create creates a WAL ready for appending records. The given metadata is // recorded at the head of each WAL file, and can be retrieved with ReadAll // after the file is Open. func Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error) { if Exist(dirpath) { return nil, os.ErrExist } if lg == nil { lg = zap.NewNop() } // keep temporary wal directory so WAL initialization appears atomic tmpdirpath := filepath.Clean(dirpath) + ".tmp" if fileutil.Exist(tmpdirpath) { if err := os.RemoveAll(tmpdirpath); err != nil { return nil, err } } defer os.RemoveAll(tmpdirpath) if err := fileutil.CreateDirAll(lg, tmpdirpath); err != nil { lg.Warn( "failed to create a temporary WAL directory", zap.String("tmp-dir-path", tmpdirpath), zap.String("dir-path", dirpath), zap.Error(err), ) return nil, err } p := filepath.Join(tmpdirpath, walName(0, 0)) f, err := createNewWALFile[*fileutil.LockedFile](p, false) if err != nil { lg.Warn( "failed to flock an initial WAL file", zap.String("path", p), zap.Error(err), ) return nil, err } if _, err = f.Seek(0, io.SeekEnd); err != nil { lg.Warn( "failed to seek an initial WAL file", zap.String("path", p), zap.Error(err), ) return nil, err } if err = fileutil.Preallocate(f.File, SegmentSizeBytes, true); err != nil { lg.Warn( "failed to preallocate an initial WAL file", zap.String("path", p), zap.Int64("segment-bytes", SegmentSizeBytes), zap.Error(err), ) return nil, err } w := &WAL{ lg: lg, dir: dirpath, metadata: metadata, } w.encoder, err = newFileEncoder(f.File, 0) if err != nil { return nil, err } w.locks = append(w.locks, f) if err = w.saveCrc(0); err != nil { return nil, err } if err = w.encoder.encode(&walpb.Record{Type: new(MetadataType), Data: metadata}); err != nil { return nil, err } // Create an empty snapshot record during the initial bootstrap only. if err = w.SaveSnapshot(walpb.Snapshot{Index: new(uint64(0)), Term: new(uint64(0))}); err != nil { return nil, err } logDirPath := w.dir if w, err = w.renameWAL(tmpdirpath); err != nil { lg.Warn( "failed to rename the temporary WAL directory", zap.String("tmp-dir-path", tmpdirpath), zap.String("dir-path", logDirPath), zap.Error(err), ) return nil, err } var perr error defer func() { if perr != nil { w.cleanupWAL(lg) } }() // directory was renamed; sync parent dir to persist rename pdir, perr := fileutil.OpenDir(filepath.Dir(w.dir)) if perr != nil { lg.Warn( "failed to open the parent data directory", zap.String("parent-dir-path", filepath.Dir(w.dir)), zap.String("dir-path", w.dir), zap.Error(perr), ) return nil, perr } dirCloser := func() error { if perr = pdir.Close(); perr != nil { lg.Warn( "failed to close the parent data directory file", zap.String("parent-dir-path", filepath.Dir(w.dir)), zap.String("dir-path", w.dir), zap.Error(perr), ) return perr } return nil } start := time.Now() if perr = fileutil.Fsync(pdir); perr != nil { dirCloser() lg.Warn( "failed to fsync the parent data directory file", zap.String("parent-dir-path", filepath.Dir(w.dir)), zap.String("dir-path", w.dir), zap.Error(perr), ) return nil, perr } walFsyncSec.Observe(time.Since(start).Seconds()) if err = dirCloser(); err != nil { return nil, err } return w, nil } // createNewWALFile creates a WAL file. // To create a locked file, use *fileutil.LockedFile type parameter. // To create a standard file, use *os.File type parameter. // If forceNew is true, the file will be truncated if it already exists. func createNewWALFile[T *os.File | *fileutil.LockedFile](path string, forceNew bool) (T, error) { flag := os.O_WRONLY | os.O_CREATE if forceNew { flag |= os.O_TRUNC } if _, isLockedFile := any(T(nil)).(*fileutil.LockedFile); isLockedFile { lockedFile, err := fileutil.LockFile(path, flag, fileutil.PrivateFileMode) if err != nil { return nil, err } return any(lockedFile).(T), nil } file, err := os.OpenFile(path, flag, fileutil.PrivateFileMode) if err != nil { return nil, err } return any(file).(T), nil } func (w *WAL) Reopen(lg *zap.Logger, snap walpb.Snapshot) (*WAL, error) { err := w.Close() if err != nil { lg.Panic("failed to close WAL during reopen", zap.Error(err)) } return Open(lg, w.dir, snap) } func (w *WAL) SetUnsafeNoFsync() { w.unsafeNoSync = true } func (w *WAL) cleanupWAL(lg *zap.Logger) { var err error if err = w.Close(); err != nil { lg.Panic("failed to close WAL during cleanup", zap.Error(err)) } brokenDirName := fmt.Sprintf("%s.broken.%v", w.dir, time.Now().Format("20060102.150405.999999")) if err = os.Rename(w.dir, brokenDirName); err != nil { lg.Panic( "failed to rename WAL during cleanup", zap.Error(err), zap.String("source-path", w.dir), zap.String("rename-path", brokenDirName), ) } } func (w *WAL) renameWAL(tmpdirpath string) (*WAL, error) { if err := os.RemoveAll(w.dir); err != nil { return nil, err } // On non-Windows platforms, hold the lock while renaming. Releasing // the lock and trying to reacquire it quickly can be flaky because // it's possible the process will fork to spawn a process while this is // happening. The fds are set up as close-on-exec by the Go runtime, // but there is a window between the fork and the exec where another // process holds the lock. if err := os.Rename(tmpdirpath, w.dir); err != nil { var linkErr *os.LinkError if errors.As(err, &linkErr) { return w.renameWALUnlock(tmpdirpath) } return nil, err } w.fp = newFilePipeline(w.lg, w.dir, SegmentSizeBytes) df, err := fileutil.OpenDir(w.dir) w.dirFile = df return w, err } func (w *WAL) renameWALUnlock(tmpdirpath string) (*WAL, error) { // rename of directory with locked files doesn't work on windows/cifs; // close the WAL to release the locks so the directory can be renamed. w.lg.Info( "closing WAL to release flock and retry directory renaming", zap.String("from", tmpdirpath), zap.String("to", w.dir), ) w.Close() if err := os.Rename(tmpdirpath, w.dir); err != nil { return nil, err } // reopen and relock newWAL, oerr := Open(w.lg, w.dir, walpb.Snapshot{}) if oerr != nil { return nil, oerr } if _, _, _, err := newWAL.ReadAll(); err != nil { newWAL.Close() return nil, err } return newWAL, nil } // Open opens the WAL at the given snap. // The snap SHOULD have been previously saved to the WAL, or the following // ReadAll will fail. // The returned WAL is ready to read and the first record will be the one after // the given snap. The WAL cannot be appended to before reading out all of its // previous records. func Open(lg *zap.Logger, dirpath string, snap walpb.Snapshot) (*WAL, error) { w, err := openAtIndex(lg, dirpath, snap, true) if err != nil { return nil, fmt.Errorf("openAtIndex failed: %w", err) } if w.dirFile, err = fileutil.OpenDir(w.dir); err != nil { return nil, fmt.Errorf("fileutil.OpenDir failed: %w", err) } return w, nil } // OpenForRead only opens the wal files for read. // Write on a read only wal panics. func OpenForRead(lg *zap.Logger, dirpath string, snap walpb.Snapshot) (*WAL, error) { return openAtIndex(lg, dirpath, snap, false) } func openAtIndex(lg *zap.Logger, dirpath string, snap walpb.Snapshot, write bool) (*WAL, error) { if lg == nil { lg = zap.NewNop() } names, nameIndex, err := selectWALFiles(lg, dirpath, snap) if err != nil { return nil, fmt.Errorf("[openAtIndex] selectWALFiles failed: %w", err) } rs, ls, closer, err := openWALFiles(lg, dirpath, names, nameIndex, write) if err != nil { return nil, fmt.Errorf("[openAtIndex] openWALFiles failed: %w", err) } // create a WAL ready for reading w := &WAL{ lg: lg, dir: dirpath, start: snap, decoder: NewDecoder(rs...), readClose: closer, locks: ls, } if write { // write reuses the file descriptors from read; don't close so // WAL can append without dropping the file lock w.readClose = nil if _, _, err := parseWALName(filepath.Base(w.tail().Name())); err != nil { closer() return nil, fmt.Errorf("[openAtIndex] parseWALName failed: %w", err) } w.fp = newFilePipeline(lg, w.dir, SegmentSizeBytes) } return w, nil } func selectWALFiles(lg *zap.Logger, dirpath string, snap walpb.Snapshot) ([]string, int, error) { names, err := readWALNames(lg, dirpath) if err != nil { return nil, -1, fmt.Errorf("readWALNames failed: %w", err) } nameIndex, ok := searchIndex(lg, names, snap.GetIndex()) if !ok { return nil, -1, fmt.Errorf("wal: file not found which matches the snapshot index '%d'", snap.Index) } if !isValidSeq(lg, names[nameIndex:]) { return nil, -1, fmt.Errorf("wal: file sequence numbers (starting from %d) do not increase continuously", nameIndex) } return names, nameIndex, nil } func openWALFiles(lg *zap.Logger, dirpath string, names []string, nameIndex int, write bool) ([]fileutil.FileReader, []*fileutil.LockedFile, func() error, error) { rcs := make([]io.ReadCloser, 0) rs := make([]fileutil.FileReader, 0) ls := make([]*fileutil.LockedFile, 0) for _, name := range names[nameIndex:] { p := filepath.Join(dirpath, name) var f *os.File if write { l, err := fileutil.TryLockFile(p, os.O_RDWR, fileutil.PrivateFileMode) if err != nil { closeAll(lg, rcs...) return nil, nil, nil, fmt.Errorf("[openWALFiles] fileutil.TryLockFile failed: %w", err) } ls = append(ls, l) rcs = append(rcs, l) f = l.File } else { rf, err := os.OpenFile(p, os.O_RDONLY, fileutil.PrivateFileMode) if err != nil { closeAll(lg, rcs...) return nil, nil, nil, fmt.Errorf("[openWALFiles] os.OpenFile failed (%q): %w", p, err) } ls = append(ls, nil) rcs = append(rcs, rf) f = rf } fileReader := fileutil.NewFileReader(f) rs = append(rs, fileReader) } closer := func() error { return closeAll(lg, rcs...) } return rs, ls, closer, nil } // ReadAll reads out records of the current WAL. // If opened in write mode, it must read out all records until EOF. Or an error // will be returned. // If opened in read mode, it will try to read all records if possible. // If it cannot read out the expected snap, it will return ErrSnapshotNotFound. // If loaded snap doesn't match with the expected one, it will return // all the records and error ErrSnapshotMismatch. // TODO: detect not-last-snap error. // TODO: maybe loose the checking of match. // After ReadAll, the WAL will be ready for appending new records. // // ReadAll suppresses WAL entries that got overridden (i.e. a newer entry with the same index // exists in the log). Such a situation can happen in cases described in figure 7. of the // RAFT paper (http://web.stanford.edu/~ouster/cgi-bin/papers/raft-atc14.pdf). // // ReadAll may return uncommitted yet entries, that are subject to be overridden. // Do not apply entries that have index > state.commit, as they are subject to change. func (w *WAL) ReadAll() (metadata []byte, state raftpb.HardState, ents []raftpb.Entry, err error) { w.mu.Lock() defer w.mu.Unlock() rec := &walpb.Record{} if w.decoder == nil { return nil, state, nil, ErrDecoderNotFound } decoder := w.decoder var match bool for err = decoder.Decode(rec); err == nil; err = decoder.Decode(rec) { switch rec.GetType() { case EntryType: e := MustUnmarshalEntry(rec.Data) // 0 <= e.Index-w.start.Index - 1 < len(ents) if e.Index > w.start.GetIndex() { // prevent "panic: runtime error: slice bounds out of range [:13038096702221461992] with capacity 0" offset := e.Index - w.start.GetIndex() - 1 if offset > uint64(len(ents)) { // return error before append call causes runtime panic. // We still return the continuous WAL entries that have already been read. // Refer to https://github.com/etcd-io/etcd/pull/19038#issuecomment-2557414292. return nil, state, ents, fmt.Errorf("%w, snapshot[Index: %d, Term: %d], current entry[Index: %d, Term: %d], len(ents): %d", ErrSliceOutOfRange, w.start.Index, w.start.Term, e.Index, e.Term, len(ents)) } // The line below is potentially overriding some 'uncommitted' entries. ents = append(ents[:offset], e) } w.enti = e.Index case StateType: state = MustUnmarshalState(rec.Data) case MetadataType: if metadata != nil && !bytes.Equal(metadata, rec.Data) { state.Reset() return nil, state, nil, ErrMetadataConflict } metadata = rec.Data case CrcType: crc := decoder.LastCRC() // current crc of decoder must match the crc of the record. // do no need to match 0 crc, since the decoder is a new one at this case. if crc != 0 && rec.Validate(crc) != nil { state.Reset() return nil, state, nil, ErrCRCMismatch } decoder.UpdateCRC(rec.GetCrc()) case SnapshotType: var snap walpb.Snapshot pbutil.MustUnmarshal(&snap, rec.Data) if snap.GetIndex() == w.start.GetIndex() { if snap.GetTerm() != w.start.GetTerm() { state.Reset() return nil, state, nil, ErrSnapshotMismatch } match = true } default: state.Reset() return nil, state, nil, fmt.Errorf("unexpected block type %d", rec.Type) } } switch w.tail() { case nil: // We do not have to read out all entries in read mode. // The last record maybe a partial written one, so // `io.ErrUnexpectedEOF` might be returned. if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { state.Reset() return nil, state, nil, err } default: // We must read all the entries if WAL is opened in write mode. if !errors.Is(err, io.EOF) { state.Reset() return nil, state, nil, err } // decodeRecord() will return io.EOF if it detects a zero record, // but this zero record may be followed by non-zero records from // a torn write. Overwriting some of these non-zero records, but // not all, will cause CRC errors on WAL open. Since the records // were never fully synced to disk in the first place, it's safe // to zero them out to avoid any CRC errors from new writes. if _, err = w.tail().Seek(w.decoder.LastOffset(), io.SeekStart); err != nil { return nil, state, nil, err } if err = fileutil.ZeroToEnd(w.tail().File); err != nil { return nil, state, nil, err } } err = nil if !match { err = ErrSnapshotNotFound } // close decoder, disable reading if w.readClose != nil { w.readClose() w.readClose = nil } w.start = walpb.Snapshot{} w.metadata = metadata if w.tail() != nil { // create encoder (chain crc with the decoder), enable appending w.encoder, err = newFileEncoder(w.tail().File, w.decoder.LastCRC()) if err != nil { return nil, state, nil, err } } w.decoder = nil return metadata, state, ents, err } // ValidSnapshotEntries returns all the valid snapshot entries in the wal logs in the given directory. // Snapshot entries are valid if their index is less than or equal to the most recent committed hardstate. func ValidSnapshotEntries(lg *zap.Logger, walDir string) ([]walpb.Snapshot, error) { var snaps []walpb.Snapshot var state raftpb.HardState var err error rec := &walpb.Record{} names, err := readWALNames(lg, walDir) if err != nil { return nil, err } // open wal files in read mode, so that there is no conflict // when the same WAL is opened elsewhere in write mode rs, _, closer, err := openWALFiles(lg, walDir, names, 0, false) if err != nil { return nil, err } defer func() { if closer != nil { closer() } }() // create a new decoder from the readers on the WAL files decoder := NewDecoder(rs...) for err = decoder.Decode(rec); err == nil; err = decoder.Decode(rec) { switch rec.GetType() { case SnapshotType: var loadedSnap walpb.Snapshot pbutil.MustUnmarshal(&loadedSnap, rec.Data) snaps = append(snaps, loadedSnap) case StateType: state = MustUnmarshalState(rec.Data) case CrcType: crc := decoder.LastCRC() // current crc of decoder must match the crc of the record. // do no need to match 0 crc, since the decoder is a new one at this case. if crc != 0 && rec.Validate(crc) != nil { return nil, ErrCRCMismatch } decoder.UpdateCRC(rec.GetCrc()) } } // We do not have to read out all the WAL entries // as the decoder is opened in read mode. if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { return nil, err } // filter out any snaps that are newer than the committed hardstate n := 0 for _, s := range snaps { if s.GetIndex() <= state.Commit { snaps[n] = s n++ } } snaps = snaps[:n:n] return snaps, nil } // Verify reads through the given WAL and verifies that it is not corrupted. // It creates a new decoder to read through the records of the given WAL. // It does not conflict with any open WAL, but it is recommended not to // call this function after opening the WAL for writing. // If it cannot read out the expected snap, it will return ErrSnapshotNotFound. // If the loaded snap doesn't match with the expected one, it will // return error ErrSnapshotMismatch. func Verify(lg *zap.Logger, walDir string, snap walpb.Snapshot) (*raftpb.HardState, error) { var metadata []byte var err error var match bool var state raftpb.HardState rec := &walpb.Record{} if lg == nil { lg = zap.NewNop() } names, nameIndex, err := selectWALFiles(lg, walDir, snap) if err != nil { return nil, err } // open wal files in read mode, so that there is no conflict // when the same WAL is opened elsewhere in write mode rs, _, closer, err := openWALFiles(lg, walDir, names, nameIndex, false) if err != nil { return nil, err } defer func() { if closer != nil { closer() } }() // create a new decoder from the readers on the WAL files decoder := NewDecoder(rs...) for err = decoder.Decode(rec); err == nil; err = decoder.Decode(rec) { switch rec.GetType() { case MetadataType: if metadata != nil && !bytes.Equal(metadata, rec.Data) { return nil, ErrMetadataConflict } metadata = rec.Data case CrcType: crc := decoder.LastCRC() // Current crc of decoder must match the crc of the record. // We need not match 0 crc, since the decoder is a new one at this point. if crc != 0 && rec.Validate(crc) != nil { return nil, ErrCRCMismatch } decoder.UpdateCRC(rec.GetCrc()) case SnapshotType: var loadedSnap walpb.Snapshot pbutil.MustUnmarshal(&loadedSnap, rec.Data) if loadedSnap.GetIndex() == snap.GetIndex() { if loadedSnap.GetTerm() != snap.GetTerm() { return nil, ErrSnapshotMismatch } match = true } // We ignore all entry and state type records as these // are not necessary for validating the WAL contents case EntryType: case StateType: pbutil.MustUnmarshal(&state, rec.Data) default: return nil, fmt.Errorf("unexpected block type %d", rec.Type) } } // We do not have to read out all the WAL entries // as the decoder is opened in read mode. if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { return nil, err } if !match { return nil, ErrSnapshotNotFound } return &state, nil } // cut closes current file written and creates a new one ready to append. // cut first creates a temp wal file and writes necessary headers into it. // Then cut atomically rename temp wal file to a wal file. func (w *WAL) cut() error { // close old wal file; truncate to avoid wasting space if an early cut off, serr := w.tail().Seek(0, io.SeekCurrent) if serr != nil { return serr } if err := w.tail().Truncate(off); err != nil { return err } if err := w.sync(); err != nil { return err } fpath := filepath.Join(w.dir, walName(w.seq()+1, w.enti+1)) // create a temp wal file with name sequence + 1, or truncate the existing one newTail, err := w.fp.Open() if err != nil { return err } // update writer and save the previous crc w.locks = append(w.locks, newTail) prevCrc := w.encoder.crc.Sum32() w.encoder, err = newFileEncoder(w.tail().File, prevCrc) if err != nil { return err } if err = w.saveCrc(prevCrc); err != nil { return err } if err = w.encoder.encode(&walpb.Record{Type: new(MetadataType), Data: w.metadata}); err != nil { return err } if err = w.saveState(&w.state); err != nil { return err } // atomically move temp wal file to wal file if err = w.sync(); err != nil { return err } off, err = w.tail().Seek(0, io.SeekCurrent) if err != nil { return err } if err = os.Rename(newTail.Name(), fpath); err != nil { return err } start := time.Now() if err = fileutil.Fsync(w.dirFile); err != nil { return err } walFsyncSec.Observe(time.Since(start).Seconds()) // reopen newTail with its new path so calls to Name() match the wal filename format newTail.Close() if newTail, err = fileutil.LockFile(fpath, os.O_WRONLY, fileutil.PrivateFileMode); err != nil { return err } if _, err = newTail.Seek(off, io.SeekStart); err != nil { return err } w.locks[len(w.locks)-1] = newTail prevCrc = w.encoder.crc.Sum32() w.encoder, err = newFileEncoder(w.tail().File, prevCrc) if err != nil { return err } w.lg.Info("created a new WAL segment", zap.String("path", fpath)) return nil } func (w *WAL) sync() error { if w.encoder != nil { if err := w.encoder.flush(); err != nil { return err } } if w.unsafeNoSync { return nil } start := time.Now() err := fileutil.Fdatasync(w.tail().File) took := time.Since(start) if took > warnSyncDuration { w.lg.Warn( "slow fdatasync", zap.Duration("took", took), zap.Duration("expected-duration", warnSyncDuration), ) } walFsyncSec.Observe(took.Seconds()) return err } func (w *WAL) Sync() error { return w.sync() } // ReleaseLockTo releases the locks, which has smaller index than the given index // except the largest one among them. // For example, if WAL is holding lock 1,2,3,4,5,6, ReleaseLockTo(4) will release // lock 1,2 but keep 3. ReleaseLockTo(5) will release 1,2,3 but keep 4. func (w *WAL) ReleaseLockTo(index uint64) error { w.mu.Lock() defer w.mu.Unlock() if len(w.locks) == 0 { return nil } var smaller int found := false for i, l := range w.locks { _, lockIndex, err := parseWALName(filepath.Base(l.Name())) if err != nil { return err } if lockIndex >= index { smaller = i - 1 found = true break } } // if no lock index is greater than the release index, we can // release lock up to the last one(excluding). if !found { smaller = len(w.locks) - 1 } if smaller <= 0 { return nil } for i := 0; i < smaller; i++ { if w.locks[i] == nil { continue } w.locks[i].Close() } w.locks = w.locks[smaller:] return nil } // Close closes the current WAL file and directory. func (w *WAL) Close() error { w.mu.Lock() defer w.mu.Unlock() if w.fp != nil { w.fp.Close() w.fp = nil } if w.tail() != nil { if err := w.sync(); err != nil { return err } } for _, l := range w.locks { if l == nil { continue } if err := l.Close(); err != nil { w.lg.Error("failed to close WAL", zap.Error(err)) } } return w.dirFile.Close() } func (w *WAL) saveEntry(e *raftpb.Entry) error { // TODO: add MustMarshalTo to reduce one allocation. b := pbutil.MustMarshal(e) rec := &walpb.Record{Type: new(EntryType), Data: b} if err := w.encoder.encode(rec); err != nil { return err } w.enti = e.Index return nil } func (w *WAL) saveState(s *raftpb.HardState) error { if raft.IsEmptyHardState(*s) { return nil } w.state = *s b := pbutil.MustMarshal(s) rec := &walpb.Record{Type: new(StateType), Data: b} return w.encoder.encode(rec) } func (w *WAL) Save(st raftpb.HardState, ents []raftpb.Entry) error { w.mu.Lock() defer w.mu.Unlock() // short cut, do not call sync if raft.IsEmptyHardState(st) && len(ents) == 0 { return nil } mustSync := raft.MustSync(st, w.state, len(ents)) // TODO(xiangli): no more reference operator for i := range ents { if err := w.saveEntry(&ents[i]); err != nil { return err } } if err := w.saveState(&st); err != nil { return err } curOff, err := w.tail().Seek(0, io.SeekCurrent) if err != nil { return err } if curOff < SegmentSizeBytes { if mustSync { // gofail: var walBeforeSync struct{} err = w.sync() // gofail: var walAfterSync struct{} return err } return nil } return w.cut() } func (w *WAL) SaveSnapshot(e walpb.Snapshot) error { if err := walpb.ValidateSnapshotForWrite(&e); err != nil { return err } b := pbutil.MustMarshal(&e) w.mu.Lock() defer w.mu.Unlock() rec := &walpb.Record{Type: new(SnapshotType), Data: b} if err := w.encoder.encode(rec); err != nil { return err } // update enti only when snapshot is ahead of last index if w.enti < e.GetIndex() { w.enti = e.GetIndex() } return w.sync() } func (w *WAL) saveCrc(prevCrc uint32) error { return w.encoder.encode(&walpb.Record{Type: new(CrcType), Crc: &prevCrc}) } func (w *WAL) tail() *fileutil.LockedFile { if len(w.locks) > 0 { return w.locks[len(w.locks)-1] } return nil } func (w *WAL) seq() uint64 { t := w.tail() if t == nil { return 0 } seq, _, err := parseWALName(filepath.Base(t.Name())) if err != nil { w.lg.Fatal("failed to parse WAL name", zap.String("name", t.Name()), zap.Error(err)) } return seq } func closeAll(lg *zap.Logger, rcs ...io.ReadCloser) error { stringArr := make([]string, 0) for _, f := range rcs { if err := f.Close(); err != nil { lg.Warn("failed to close: ", zap.Error(err)) stringArr = append(stringArr, err.Error()) } } if len(stringArr) == 0 { return nil } return errors.New(strings.Join(stringArr, ", ")) } ================================================ FILE: server/storage/wal/wal_bench_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "testing" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/raft/v3/raftpb" ) func BenchmarkWrite100EntryWithoutBatch(b *testing.B) { benchmarkWriteEntry(b, 100, 0) } func BenchmarkWrite100EntryBatch10(b *testing.B) { benchmarkWriteEntry(b, 100, 10) } func BenchmarkWrite100EntryBatch100(b *testing.B) { benchmarkWriteEntry(b, 100, 100) } func BenchmarkWrite100EntryBatch500(b *testing.B) { benchmarkWriteEntry(b, 100, 500) } func BenchmarkWrite100EntryBatch1000(b *testing.B) { benchmarkWriteEntry(b, 100, 1000) } func BenchmarkWrite1000EntryWithoutBatch(b *testing.B) { benchmarkWriteEntry(b, 1000, 0) } func BenchmarkWrite1000EntryBatch10(b *testing.B) { benchmarkWriteEntry(b, 1000, 10) } func BenchmarkWrite1000EntryBatch100(b *testing.B) { benchmarkWriteEntry(b, 1000, 100) } func BenchmarkWrite1000EntryBatch500(b *testing.B) { benchmarkWriteEntry(b, 1000, 500) } func BenchmarkWrite1000EntryBatch1000(b *testing.B) { benchmarkWriteEntry(b, 1000, 1000) } func benchmarkWriteEntry(b *testing.B, size int, batch int) { p := b.TempDir() w, err := Create(zaptest.NewLogger(b), p, []byte("somedata")) require.NoErrorf(b, err, "err = %v, want nil", err) data := make([]byte, size) for i := 0; i < size; i++ { data[i] = byte(i) } e := &raftpb.Entry{Data: data} b.ResetTimer() n := 0 b.SetBytes(int64(e.Size())) for i := 0; i < b.N; i++ { err := w.saveEntry(e) if err != nil { b.Fatal(err) } n++ if n > batch { w.sync() n = 0 } } } ================================================ FILE: server/storage/wal/wal_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package wal import ( "bytes" "crypto/rand" "errors" "fmt" "io" "math" "os" "path" "path/filepath" "reflect" "regexp" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) var confState = raftpb.ConfState{ Voters: []uint64{0x00ffca74}, AutoLeave: false, } func TestNew(t *testing.T) { p := t.TempDir() w, err := Create(zaptest.NewLogger(t), p, []byte("somedata")) require.NoErrorf(t, err, "err = %v, want nil", err) if g := filepath.Base(w.tail().Name()); g != walName(0, 0) { t.Errorf("name = %+v, want %+v", g, walName(0, 0)) } defer w.Close() // file is preallocated to segment size; only read data written by wal off, err := w.tail().Seek(0, io.SeekCurrent) if err != nil { t.Fatal(err) } gd := make([]byte, off) f, err := os.Open(filepath.Join(p, filepath.Base(w.tail().Name()))) if err != nil { t.Fatal(err) } defer f.Close() if _, err = io.ReadFull(f, gd); err != nil { t.Fatalf("err = %v, want nil", err) } if os.Getenv("ETCD_UPDATE_FIXTURE_DATA") == "true" { os.WriteFile("testdata/TestNew.wal", gd, os.FileMode(0644)) } fixtureData, err := os.ReadFile("testdata/TestNew.wal") if err != nil { t.Fatal(err) } if !bytes.Equal(gd, fixtureData) { t.Log("data did not match fixture, rerun with ETCD_UPDATE_FIXTURE_DATA=true to regenerate fixture") t.Errorf("data = %v, want %v", gd, fixtureData) } var wb bytes.Buffer e := newEncoder(&wb, 0, 0) err = e.encode(&walpb.Record{Type: new(CrcType), Crc: new(uint32(0))}) require.NoErrorf(t, err, "err = %v, want nil", err) err = e.encode(&walpb.Record{Type: new(MetadataType), Data: []byte("somedata")}) require.NoErrorf(t, err, "err = %v, want nil", err) r := &walpb.Record{ Type: new(SnapshotType), Data: pbutil.MustMarshal(&walpb.Snapshot{Index: new(uint64(0)), Term: new(uint64(0))}), } err = e.encode(r) require.NoErrorf(t, err, "err = %v, want nil", err) e.flush() if !bytes.Equal(gd, wb.Bytes()) { t.Errorf("data = %v, want %v", gd, wb.Bytes()) } } func TestCreateNewWALFile(t *testing.T) { tests := []struct { name string fileType any forceNew bool }{ { name: "creating standard file should succeed and not truncate file", fileType: &os.File{}, forceNew: false, }, { name: "creating locked file should succeed and not truncate file", fileType: &fileutil.LockedFile{}, forceNew: false, }, { name: "creating standard file with forceNew should truncate file", fileType: &os.File{}, forceNew: true, }, { name: "creating locked file with forceNew should truncate file", fileType: &fileutil.LockedFile{}, forceNew: true, }, } for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { p := filepath.Join(t.TempDir(), walName(0, uint64(i))) // create initial file with some data to verify truncate behavior err := os.WriteFile(p, []byte("test data"), fileutil.PrivateFileMode) require.NoError(t, err) var f any switch tt.fileType.(type) { case *os.File: f, err = createNewWALFile[*os.File](p, tt.forceNew) require.IsType(t, &os.File{}, f) case *fileutil.LockedFile: f, err = createNewWALFile[*fileutil.LockedFile](p, tt.forceNew) require.IsType(t, &fileutil.LockedFile{}, f) default: panic("unknown file type") } require.NoError(t, err) // validate the file permissions fi, err := os.Stat(p) require.NoError(t, err) expectedPerms := fmt.Sprintf("%o", os.FileMode(fileutil.PrivateFileMode)) actualPerms := fmt.Sprintf("%o", fi.Mode().Perm()) require.Equalf(t, expectedPerms, actualPerms, "unexpected file permissions on %q", p) content, err := os.ReadFile(p) require.NoError(t, err) if tt.forceNew { require.Emptyf(t, string(content), "file content should be truncated but it wasn't") } else { require.Equalf(t, "test data", string(content), "file content should not be truncated but it was") } }) } } func TestCreateFailFromPollutedDir(t *testing.T) { p := t.TempDir() os.WriteFile(filepath.Join(p, "test.wal"), []byte("data"), os.ModeTemporary) _, err := Create(zaptest.NewLogger(t), p, []byte("data")) require.ErrorIsf(t, err, os.ErrExist, "expected %v, got %v", os.ErrExist, err) } func TestWalCleanup(t *testing.T) { testRoot := t.TempDir() p, err := os.MkdirTemp(testRoot, "waltest") if err != nil { t.Fatal(err) } logger := zaptest.NewLogger(t) w, err := Create(logger, p, []byte("")) require.NoErrorf(t, err, "err = %v, want nil", err) w.cleanupWAL(logger) fnames, err := fileutil.ReadDir(testRoot) require.NoErrorf(t, err, "err = %v, want nil", err) require.Lenf(t, fnames, 1, "expected 1 file under %v, got %v", testRoot, len(fnames)) pattern := fmt.Sprintf(`%s.broken\.[\d]{8}\.[\d]{6}\.[\d]{1,6}?`, filepath.Base(p)) match, _ := regexp.MatchString(pattern, fnames[0]) if !match { t.Errorf("match = false, expected true for %v with pattern %v", fnames[0], pattern) } } func TestCreateFailFromNoSpaceLeft(t *testing.T) { p := t.TempDir() oldSegmentSizeBytes := SegmentSizeBytes defer func() { SegmentSizeBytes = oldSegmentSizeBytes }() SegmentSizeBytes = math.MaxInt64 _, err := Create(zaptest.NewLogger(t), p, []byte("data")) require.Errorf(t, err, "expected error 'no space left on device', got nil") // no space left on device } func TestNewForInitedDir(t *testing.T) { p := t.TempDir() os.Create(filepath.Join(p, walName(0, 0))) if _, err := Create(zaptest.NewLogger(t), p, nil); err == nil || !errors.Is(err, os.ErrExist) { t.Errorf("err = %v, want %v", err, os.ErrExist) } } func TestOpenAtIndex(t *testing.T) { dir := t.TempDir() f, err := os.Create(filepath.Join(dir, walName(0, 0))) if err != nil { t.Fatal(err) } f.Close() w, err := Open(zaptest.NewLogger(t), dir, walpb.Snapshot{}) require.NoErrorf(t, err, "err = %v, want nil", err) if g := filepath.Base(w.tail().Name()); g != walName(0, 0) { t.Errorf("name = %+v, want %+v", g, walName(0, 0)) } if w.seq() != 0 { t.Errorf("seq = %d, want %d", w.seq(), 0) } w.Close() wname := walName(2, 10) f, err = os.Create(filepath.Join(dir, wname)) if err != nil { t.Fatal(err) } f.Close() w, err = Open(zaptest.NewLogger(t), dir, walpb.Snapshot{Index: new(uint64(5))}) require.NoErrorf(t, err, "err = %v, want nil", err) if g := filepath.Base(w.tail().Name()); g != wname { t.Errorf("name = %+v, want %+v", g, wname) } if w.seq() != 2 { t.Errorf("seq = %d, want %d", w.seq(), 2) } w.Close() emptydir := t.TempDir() if _, err = Open(zaptest.NewLogger(t), emptydir, walpb.Snapshot{}); !errors.Is(err, ErrFileNotFound) { t.Errorf("err = %v, want %v", err, ErrFileNotFound) } } // TestVerify tests that Verify throws a non-nil error when the WAL is corrupted. // The test creates a WAL directory and cuts out multiple WAL files. Then // it corrupts one of the files by completely truncating it. func TestVerify(t *testing.T) { lg := zaptest.NewLogger(t) walDir := t.TempDir() // create WAL w, err := Create(lg, walDir, nil) if err != nil { t.Fatal(err) } defer w.Close() // make 5 separate files for i := 0; i < 5; i++ { es := []raftpb.Entry{{Index: uint64(i), Data: []byte(fmt.Sprintf("waldata%d", i+1))}} if err = w.Save(raftpb.HardState{}, es); err != nil { t.Fatal(err) } if err = w.cut(); err != nil { t.Fatal(err) } } hs := raftpb.HardState{Term: 1, Vote: 3, Commit: 5} require.NoError(t, w.Save(hs, nil)) // to verify the WAL is not corrupted at this point hardstate, err := Verify(lg, walDir, walpb.Snapshot{}) if err != nil { t.Errorf("expected a nil error, got %v", err) } assert.Equal(t, hs, *hardstate) walFiles, err := os.ReadDir(walDir) if err != nil { t.Fatal(err) } // corrupt the WAL by truncating one of the WAL files completely err = os.Truncate(path.Join(walDir, walFiles[2].Name()), 0) if err != nil { t.Fatal(err) } _, err = Verify(lg, walDir, walpb.Snapshot{}) if err == nil { t.Error("expected a non-nil error, got nil") } } // TestCut tests cut // TODO: split it into smaller tests for better readability func TestCut(t *testing.T) { p := t.TempDir() w, err := Create(zaptest.NewLogger(t), p, nil) if err != nil { t.Fatal(err) } defer w.Close() state := raftpb.HardState{Term: 1} if err = w.Save(state, nil); err != nil { t.Fatal(err) } if err = w.cut(); err != nil { t.Fatal(err) } wname := walName(1, 1) if g := filepath.Base(w.tail().Name()); g != wname { t.Errorf("name = %s, want %s", g, wname) } es := []raftpb.Entry{{Index: 1, Term: 1, Data: []byte{1}}} if err = w.Save(raftpb.HardState{}, es); err != nil { t.Fatal(err) } if err = w.cut(); err != nil { t.Fatal(err) } snap := walpb.Snapshot{Index: new(uint64(2)), Term: new(uint64(1)), ConfState: &confState} if err = w.SaveSnapshot(snap); err != nil { t.Fatal(err) } wname = walName(2, 2) if g := filepath.Base(w.tail().Name()); g != wname { t.Errorf("name = %s, want %s", g, wname) } // check the state in the last WAL // We do check before closing the WAL to ensure that Cut syncs the data // into the disk. f, err := os.Open(filepath.Join(p, wname)) if err != nil { t.Fatal(err) } defer f.Close() nw := &WAL{ decoder: NewDecoder(fileutil.NewFileReader(f)), start: snap, } _, gst, _, err := nw.ReadAll() if err != nil { t.Fatal(err) } if !reflect.DeepEqual(gst, state) { t.Errorf("state = %+v, want %+v", gst, state) } } func TestSaveWithCut(t *testing.T) { p := t.TempDir() w, err := Create(zaptest.NewLogger(t), p, []byte("metadata")) if err != nil { t.Fatal(err) } state := raftpb.HardState{Term: 1} if err = w.Save(state, nil); err != nil { t.Fatal(err) } bigData := make([]byte, 500) strdata := "Hello World!!" copy(bigData, strdata) // set a lower value for SegmentSizeBytes, else the test takes too long to complete restoreLater := SegmentSizeBytes const EntrySize int = 500 SegmentSizeBytes = 2 * 1024 defer func() { SegmentSizeBytes = restoreLater }() index := uint64(0) for totalSize := 0; totalSize < int(SegmentSizeBytes); totalSize += EntrySize { ents := []raftpb.Entry{{Index: index, Term: 1, Data: bigData}} if err = w.Save(state, ents); err != nil { t.Fatal(err) } index++ } w.Close() neww, err := Open(zaptest.NewLogger(t), p, walpb.Snapshot{}) require.NoErrorf(t, err, "err = %v, want nil", err) defer neww.Close() wname := walName(1, index) if g := filepath.Base(neww.tail().Name()); g != wname { t.Errorf("name = %s, want %s", g, wname) } _, newhardstate, entries, err := neww.ReadAll() if err != nil { t.Fatal(err) } if !reflect.DeepEqual(newhardstate, state) { t.Errorf("Hard State = %+v, want %+v", newhardstate, state) } if len(entries) != int(SegmentSizeBytes/int64(EntrySize)) { t.Errorf("Number of entries = %d, expected = %d", len(entries), int(SegmentSizeBytes/int64(EntrySize))) } for _, oneent := range entries { if !bytes.Equal(oneent.Data, bigData) { t.Errorf("the saved data does not match at Index %d : found: %s , want :%s", oneent.Index, oneent.Data, bigData) } } } func TestRecover(t *testing.T) { cases := []struct { name string size int }{ { name: "10MB", size: 10 * 1024 * 1024, }, { name: "20MB", size: 20 * 1024 * 1024, }, { name: "40MB", size: 40 * 1024 * 1024, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { p := t.TempDir() w, err := Create(zaptest.NewLogger(t), p, []byte("metadata")) if err != nil { t.Fatal(err) } if err = w.SaveSnapshot(walpb.Snapshot{Index: new(uint64(0)), Term: new(uint64(0))}); err != nil { t.Fatal(err) } data := make([]byte, tc.size) n, err := rand.Read(data) assert.Equal(t, tc.size, n) if err != nil { t.Errorf("Unexpected error: %v", err) } ents := []raftpb.Entry{{Index: 1, Term: 1, Data: data}, {Index: 2, Term: 2, Data: data}} if err = w.Save(raftpb.HardState{}, ents); err != nil { t.Fatal(err) } sts := []raftpb.HardState{{Term: 1, Vote: 1, Commit: 1}, {Term: 2, Vote: 2, Commit: 2}} for _, s := range sts { if err = w.Save(s, nil); err != nil { t.Fatal(err) } } w.Close() if w, err = Open(zaptest.NewLogger(t), p, walpb.Snapshot{}); err != nil { t.Fatal(err) } metadata, state, entries, err := w.ReadAll() if err != nil { t.Fatal(err) } if !bytes.Equal(metadata, []byte("metadata")) { t.Errorf("metadata = %s, want %s", metadata, "metadata") } if !reflect.DeepEqual(entries, ents) { t.Errorf("ents = %+v, want %+v", entries, ents) } // only the latest state is recorded s := sts[len(sts)-1] if !reflect.DeepEqual(state, s) { t.Errorf("state = %+v, want %+v", state, s) } w.Close() }) } } func TestSearchIndex(t *testing.T) { tests := []struct { names []string index uint64 widx int wok bool }{ { []string{ "0000000000000000-0000000000000000.wal", "0000000000000001-0000000000001000.wal", "0000000000000002-0000000000002000.wal", }, 0x1000, 1, true, }, { []string{ "0000000000000001-0000000000004000.wal", "0000000000000002-0000000000003000.wal", "0000000000000003-0000000000005000.wal", }, 0x4000, 1, true, }, { []string{ "0000000000000001-0000000000002000.wal", "0000000000000002-0000000000003000.wal", "0000000000000003-0000000000005000.wal", }, 0x1000, -1, false, }, } for i, tt := range tests { idx, ok := searchIndex(zaptest.NewLogger(t), tt.names, tt.index) if idx != tt.widx { t.Errorf("#%d: idx = %d, want %d", i, idx, tt.widx) } if ok != tt.wok { t.Errorf("#%d: ok = %v, want %v", i, ok, tt.wok) } } } func TestScanWalName(t *testing.T) { tests := []struct { str string wseq, windex uint64 wok bool }{ {"0000000000000000-0000000000000000.wal", 0, 0, true}, {"0000000000000000.wal", 0, 0, false}, {"0000000000000000-0000000000000000.snap", 0, 0, false}, } for i, tt := range tests { s, index, err := parseWALName(tt.str) if g := err == nil; g != tt.wok { t.Errorf("#%d: ok = %v, want %v", i, g, tt.wok) } if s != tt.wseq { t.Errorf("#%d: seq = %d, want %d", i, s, tt.wseq) } if index != tt.windex { t.Errorf("#%d: index = %d, want %d", i, index, tt.windex) } } } func TestRecoverAfterCut(t *testing.T) { p := t.TempDir() md, err := Create(zaptest.NewLogger(t), p, []byte("metadata")) if err != nil { t.Fatal(err) } for i := 0; i < 10; i++ { if err = md.SaveSnapshot(walpb.Snapshot{Index: new(uint64(i)), Term: new(uint64(1)), ConfState: &confState}); err != nil { t.Fatal(err) } es := []raftpb.Entry{{Index: uint64(i)}} if err = md.Save(raftpb.HardState{}, es); err != nil { t.Fatal(err) } if err = md.cut(); err != nil { t.Fatal(err) } } md.Close() if err := os.Remove(filepath.Join(p, walName(4, 4))); err != nil { t.Fatal(err) } for i := 0; i < 10; i++ { w, err := Open(zaptest.NewLogger(t), p, walpb.Snapshot{Index: new(uint64(i)), Term: new(uint64(1))}) if err != nil { if i <= 4 { if !strings.Contains(err.Error(), "do not increase continuously") { t.Errorf("#%d: err = %v isn't expected, want: '* do not increase continuously'", i, err) } } else { t.Errorf("#%d: err = %v, want nil", i, err) } continue } metadata, _, entries, err := w.ReadAll() if err != nil { t.Errorf("#%d: err = %v, want nil", i, err) continue } if !bytes.Equal(metadata, []byte("metadata")) { t.Errorf("#%d: metadata = %s, want %s", i, metadata, "metadata") } for j, e := range entries { if e.Index != uint64(j+i+1) { t.Errorf("#%d: ents[%d].Index = %+v, want %+v", i, j, e.Index, j+i+1) } } w.Close() } } func TestOpenAtUncommittedIndex(t *testing.T) { p := t.TempDir() w, err := Create(zaptest.NewLogger(t), p, nil) if err != nil { t.Fatal(err) } if err = w.SaveSnapshot(walpb.Snapshot{Index: new(uint64(0)), Term: new(uint64(0))}); err != nil { t.Fatal(err) } if err = w.Save(raftpb.HardState{}, []raftpb.Entry{{Index: 0}}); err != nil { t.Fatal(err) } w.Close() w, err = Open(zaptest.NewLogger(t), p, walpb.Snapshot{}) if err != nil { t.Fatal(err) } // commit up to index 0, try to read index 1 if _, _, _, err = w.ReadAll(); err != nil { t.Errorf("err = %v, want nil", err) } w.Close() } // TestOpenForRead tests that OpenForRead can load all files. // The tests creates WAL directory, and cut out multiple WAL files. Then // it releases the lock of part of data, and excepts that OpenForRead // can read out all files even if some are locked for write. func TestOpenForRead(t *testing.T) { p := t.TempDir() // create WAL w, err := Create(zaptest.NewLogger(t), p, nil) if err != nil { t.Fatal(err) } defer w.Close() // make 10 separate files for i := 0; i < 10; i++ { es := []raftpb.Entry{{Index: uint64(i)}} if err = w.Save(raftpb.HardState{}, es); err != nil { t.Fatal(err) } if err = w.cut(); err != nil { t.Fatal(err) } } // release the lock to 5 unlockIndex := uint64(5) w.ReleaseLockTo(unlockIndex) // All are available for read w2, err := OpenForRead(zaptest.NewLogger(t), p, walpb.Snapshot{}) if err != nil { t.Fatal(err) } defer w2.Close() _, _, ents, err := w2.ReadAll() require.NoErrorf(t, err, "err = %v, want nil", err) if g := ents[len(ents)-1].Index; g != 9 { t.Errorf("last index read = %d, want %d", g, 9) } } func TestOpenWithMaxIndex(t *testing.T) { p := t.TempDir() // create WAL w1, err := Create(zaptest.NewLogger(t), p, nil) if err != nil { t.Fatal(err) } defer func() { if w1 != nil { w1.Close() } }() es := []raftpb.Entry{{Index: uint64(math.MaxInt64)}} if err = w1.Save(raftpb.HardState{}, es); err != nil { t.Fatal(err) } w1.Close() w1 = nil w2, err := Open(zaptest.NewLogger(t), p, walpb.Snapshot{}) if err != nil { t.Fatal(err) } defer w2.Close() _, _, _, err = w2.ReadAll() require.ErrorIsf(t, err, ErrSliceOutOfRange, "err = %v, want ErrSliceOutOfRange", err) } func TestSaveEmpty(t *testing.T) { var buf bytes.Buffer var est raftpb.HardState w := WAL{ encoder: newEncoder(&buf, 0, 0), } if err := w.saveState(&est); err != nil { t.Errorf("err = %v, want nil", err) } if len(buf.Bytes()) != 0 { t.Errorf("buf.Bytes = %d, want 0", len(buf.Bytes())) } } func TestReleaseLockTo(t *testing.T) { p := t.TempDir() // create WAL w, err := Create(zaptest.NewLogger(t), p, nil) defer func() { if err = w.Close(); err != nil { t.Fatal(err) } }() if err != nil { t.Fatal(err) } // release nothing if no files err = w.ReleaseLockTo(10) if err != nil { t.Errorf("err = %v, want nil", err) } // make 10 separate files for i := 0; i < 10; i++ { es := []raftpb.Entry{{Index: uint64(i)}} if err = w.Save(raftpb.HardState{}, es); err != nil { t.Fatal(err) } if err = w.cut(); err != nil { t.Fatal(err) } } // release the lock to 5 unlockIndex := uint64(5) w.ReleaseLockTo(unlockIndex) // expected remaining are 4,5,6,7,8,9,10 if len(w.locks) != 7 { t.Errorf("len(w.locks) = %d, want %d", len(w.locks), 7) } for i, l := range w.locks { var lockIndex uint64 _, lockIndex, err = parseWALName(filepath.Base(l.Name())) if err != nil { t.Fatal(err) } if lockIndex != uint64(i+4) { t.Errorf("#%d: lockindex = %d, want %d", i, lockIndex, uint64(i+4)) } } // release the lock to 15 unlockIndex = uint64(15) w.ReleaseLockTo(unlockIndex) // expected remaining is 10 if len(w.locks) != 1 { t.Errorf("len(w.locks) = %d, want %d", len(w.locks), 1) } _, lockIndex, err := parseWALName(filepath.Base(w.locks[0].Name())) if err != nil { t.Fatal(err) } if lockIndex != uint64(10) { t.Errorf("lockindex = %d, want %d", lockIndex, 10) } } // TestTailWriteNoSlackSpace ensures that tail writes append if there's no preallocated space. func TestTailWriteNoSlackSpace(t *testing.T) { p := t.TempDir() // create initial WAL w, err := Create(zaptest.NewLogger(t), p, []byte("metadata")) if err != nil { t.Fatal(err) } // write some entries for i := 1; i <= 5; i++ { es := []raftpb.Entry{{Index: uint64(i), Term: 1, Data: []byte{byte(i)}}} if err = w.Save(raftpb.HardState{Term: 1}, es); err != nil { t.Fatal(err) } } // get rid of slack space by truncating file off, serr := w.tail().Seek(0, io.SeekCurrent) if serr != nil { t.Fatal(serr) } if terr := w.tail().Truncate(off); terr != nil { t.Fatal(terr) } w.Close() // open, write more w, err = Open(zaptest.NewLogger(t), p, walpb.Snapshot{}) if err != nil { t.Fatal(err) } _, _, ents, rerr := w.ReadAll() if rerr != nil { t.Fatal(rerr) } require.Lenf(t, ents, 5, "got entries %+v, expected 5 entries", ents) // write more entries for i := 6; i <= 10; i++ { es := []raftpb.Entry{{Index: uint64(i), Term: 1, Data: []byte{byte(i)}}} if err = w.Save(raftpb.HardState{Term: 1}, es); err != nil { t.Fatal(err) } } w.Close() // confirm all writes w, err = Open(zaptest.NewLogger(t), p, walpb.Snapshot{}) if err != nil { t.Fatal(err) } _, _, ents, rerr = w.ReadAll() if rerr != nil { t.Fatal(rerr) } if len(ents) != 10 { t.Fatalf("got entries %+v, expected 10 entries", ents) } w.Close() } // TestRestartCreateWal ensures that an interrupted WAL initialization is clobbered on restart func TestRestartCreateWal(t *testing.T) { p := t.TempDir() var err error // make temporary directory so it looks like initialization is interrupted tmpdir := filepath.Clean(p) + ".tmp" if err = os.Mkdir(tmpdir, fileutil.PrivateDirMode); err != nil { t.Fatal(err) } if _, err = os.OpenFile(filepath.Join(tmpdir, "test"), os.O_WRONLY|os.O_CREATE, fileutil.PrivateFileMode); err != nil { t.Fatal(err) } w, werr := Create(zaptest.NewLogger(t), p, []byte("abc")) if werr != nil { t.Fatal(werr) } w.Close() if Exist(tmpdir) { t.Fatalf("got %q exists, expected it to not exist", tmpdir) } if w, err = OpenForRead(zaptest.NewLogger(t), p, walpb.Snapshot{}); err != nil { t.Fatal(err) } defer w.Close() if meta, _, _, rerr := w.ReadAll(); rerr != nil || string(meta) != "abc" { t.Fatalf("got error %v and meta %q, expected nil and %q", rerr, meta, "abc") } } // TestOpenOnTornWrite ensures that entries past the torn write are truncated. func TestOpenOnTornWrite(t *testing.T) { maxEntries := 40 clobberIdx := 20 overwriteEntries := 5 p := t.TempDir() w, err := Create(zaptest.NewLogger(t), p, nil) defer func() { if err = w.Close(); err != nil && !errors.Is(err, os.ErrInvalid) { t.Fatal(err) } }() if err != nil { t.Fatal(err) } // get offset of end of each saved entry offsets := make([]int64, maxEntries) for i := range offsets { es := []raftpb.Entry{{Index: uint64(i)}} if err = w.Save(raftpb.HardState{}, es); err != nil { t.Fatal(err) } if offsets[i], err = w.tail().Seek(0, io.SeekCurrent); err != nil { t.Fatal(err) } } fn := filepath.Join(p, filepath.Base(w.tail().Name())) w.Close() // clobber some entry with 0's to simulate a torn write f, ferr := os.OpenFile(fn, os.O_WRONLY, fileutil.PrivateFileMode) if ferr != nil { t.Fatal(ferr) } defer f.Close() _, err = f.Seek(offsets[clobberIdx], io.SeekStart) if err != nil { t.Fatal(err) } zeros := make([]byte, offsets[clobberIdx+1]-offsets[clobberIdx]) _, err = f.Write(zeros) if err != nil { t.Fatal(err) } f.Close() w, err = Open(zaptest.NewLogger(t), p, walpb.Snapshot{}) if err != nil { t.Fatal(err) } // seek up to clobbered entry _, _, _, err = w.ReadAll() if err != nil { t.Fatal(err) } // write a few entries past the clobbered entry for i := 0; i < overwriteEntries; i++ { // Index is different from old, truncated entries es := []raftpb.Entry{{Index: uint64(i + clobberIdx), Data: []byte("new")}} if err = w.Save(raftpb.HardState{}, es); err != nil { t.Fatal(err) } } w.Close() // read back the entries, confirm number of entries matches expectation w, err = OpenForRead(zaptest.NewLogger(t), p, walpb.Snapshot{}) if err != nil { t.Fatal(err) } _, _, ents, rerr := w.ReadAll() if rerr != nil { // CRC error? the old entries were likely never truncated away t.Fatal(rerr) } wEntries := (clobberIdx - 1) + overwriteEntries require.Equalf(t, len(ents), wEntries, "expected len(ents) = %d, got %d", wEntries, len(ents)) } func TestRenameFail(t *testing.T) { p := t.TempDir() oldSegmentSizeBytes := SegmentSizeBytes defer func() { SegmentSizeBytes = oldSegmentSizeBytes }() SegmentSizeBytes = math.MaxInt64 tp := t.TempDir() os.RemoveAll(tp) w := &WAL{ lg: zaptest.NewLogger(t), dir: p, } w2, werr := w.renameWAL(tp) if w2 != nil || werr == nil { // os.Rename should fail from 'no such file or directory' t.Fatalf("expected error, got %v", werr) } } // TestReadAllFail ensure ReadAll error if used without opening the WAL func TestReadAllFail(t *testing.T) { dir := t.TempDir() // create initial WAL f, err := Create(zaptest.NewLogger(t), dir, []byte("metadata")) if err != nil { t.Fatal(err) } f.Close() // try to read without opening the WAL _, _, _, err = f.ReadAll() if err == nil || !errors.Is(err, ErrDecoderNotFound) { t.Fatalf("err = %v, want ErrDecoderNotFound", err) } } // TestValidSnapshotEntries ensures ValidSnapshotEntries returns all valid wal snapshot entries, accounting // for hardstate func TestValidSnapshotEntries(t *testing.T) { p := t.TempDir() snap0 := walpb.Snapshot{Index: new(uint64(0)), Term: new(uint64(0))} snap1 := walpb.Snapshot{Index: new(uint64(1)), Term: new(uint64(1)), ConfState: &confState} state1 := raftpb.HardState{Commit: 1, Term: 1} snap2 := walpb.Snapshot{Index: new(uint64(2)), Term: new(uint64(1)), ConfState: &confState} snap3 := walpb.Snapshot{Index: new(uint64(3)), Term: new(uint64(2)), ConfState: &confState} state2 := raftpb.HardState{Commit: 3, Term: 2} snap4 := walpb.Snapshot{Index: new(uint64(4)), Term: new(uint64(2)), ConfState: &confState} // will be orphaned since the last committed entry will be snap3 func() { w, err := Create(zaptest.NewLogger(t), p, nil) if err != nil { t.Fatal(err) } defer w.Close() // snap0 is implicitly created at index 0, term 0 if err = w.SaveSnapshot(snap1); err != nil { t.Fatal(err) } if err = w.Save(state1, nil); err != nil { t.Fatal(err) } if err = w.SaveSnapshot(snap2); err != nil { t.Fatal(err) } if err = w.SaveSnapshot(snap3); err != nil { t.Fatal(err) } if err = w.Save(state2, nil); err != nil { t.Fatal(err) } if err = w.SaveSnapshot(snap4); err != nil { t.Fatal(err) } }() walSnaps, err := ValidSnapshotEntries(zaptest.NewLogger(t), p) if err != nil { t.Fatal(err) } expected := []walpb.Snapshot{snap0, snap1, snap2, snap3} if !reflect.DeepEqual(walSnaps, expected) { t.Errorf("expected walSnaps %+v, got %+v", expected, walSnaps) } } // TestValidSnapshotEntriesAfterPurgeWal ensure that there are many wal files, and after cleaning the first wal file, // it can work well. func TestValidSnapshotEntriesAfterPurgeWal(t *testing.T) { oldSegmentSizeBytes := SegmentSizeBytes SegmentSizeBytes = 64 defer func() { SegmentSizeBytes = oldSegmentSizeBytes }() p := t.TempDir() snap0 := walpb.Snapshot{} snap1 := walpb.Snapshot{Index: new(uint64(1)), Term: new(uint64(1)), ConfState: &confState} state1 := raftpb.HardState{Commit: 1, Term: 1} snap2 := walpb.Snapshot{Index: new(uint64(2)), Term: new(uint64(1)), ConfState: &confState} snap3 := walpb.Snapshot{Index: new(uint64(3)), Term: new(uint64(2)), ConfState: &confState} state2 := raftpb.HardState{Commit: 3, Term: 2} func() { w, err := Create(zaptest.NewLogger(t), p, nil) if err != nil { t.Fatal(err) } defer w.Close() // snap0 is implicitly created at index 0, term 0 if err = w.SaveSnapshot(snap1); err != nil { t.Fatal(err) } if err = w.Save(state1, nil); err != nil { t.Fatal(err) } if err = w.SaveSnapshot(snap2); err != nil { t.Fatal(err) } if err = w.SaveSnapshot(snap3); err != nil { t.Fatal(err) } for i := 0; i < 128; i++ { if err = w.Save(state2, nil); err != nil { t.Fatal(err) } } }() files, _, err := selectWALFiles(nil, p, snap0) if err != nil { t.Fatal(err) } os.Remove(p + "/" + files[0]) _, err = ValidSnapshotEntries(zaptest.NewLogger(t), p) if err != nil { t.Fatal(err) } } func TestLastRecordLengthExceedFileEnd(t *testing.T) { /* The data below was generated by code something like below. The length * of the last record was intentionally changed to 1000 in order to make * sure it exceeds the end of the file. * * for i := 0; i < 3; i++ { * es := []raftpb.Entry{{Index: uint64(i + 1), Data: []byte(fmt.Sprintf("waldata%d", i+1))}} * if err = w.Save(raftpb.HardState{}, es); err != nil { * t.Fatal(err) * } * } * ...... * var sb strings.Builder * for _, ch := range buf { * sb.WriteString(fmt.Sprintf("\\x%02x", ch)) * } */ // Generate WAL file t.Log("Generate a WAL file with the last record's length modified.") data := []byte("\x04\x00\x00\x00\x00\x00\x00\x84\x08\x04\x10\x00\x00" + "\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x84\x08\x01\x10\x00\x00" + "\x00\x00\x00\x0e\x00\x00\x00\x00\x00\x00\x82\x08\x05\x10\xa0\xb3" + "\x9b\x8f\x08\x1a\x04\x08\x00\x10\x00\x00\x00\x1a\x00\x00\x00\x00" + "\x00\x00\x86\x08\x02\x10\xba\x8b\xdc\x85\x0f\x1a\x10\x08\x00\x10" + "\x00\x18\x01\x22\x08\x77\x61\x6c\x64\x61\x74\x61\x31\x00\x00\x00" + "\x00\x00\x00\x1a\x00\x00\x00\x00\x00\x00\x86\x08\x02\x10\xa1\xe8" + "\xff\x9c\x02\x1a\x10\x08\x00\x10\x00\x18\x02\x22\x08\x77\x61\x6c" + "\x64\x61\x74\x61\x32\x00\x00\x00\x00\x00\x00\xe8\x03\x00\x00\x00" + "\x00\x00\x86\x08\x02\x10\xa1\x9c\xa1\xaa\x04\x1a\x10\x08\x00\x10" + "\x00\x18\x03\x22\x08\x77\x61\x6c\x64\x61\x74\x61\x33\x00\x00\x00" + "\x00\x00\x00") buf := bytes.NewBuffer(data) f, err := createFileWithData(t, buf) fileName := f.Name() require.NoError(t, err) t.Logf("fileName: %v", fileName) // Verify low-level decoder directly t.Log("Verify all records can be parsed correctly.") rec := &walpb.Record{} decoder := NewDecoder(fileutil.NewFileReader(f)) for { if err = decoder.Decode(rec); err != nil { require.ErrorIs(t, err, io.ErrUnexpectedEOF) break } if rec.GetType() == EntryType { e := MustUnmarshalEntry(rec.Data) t.Logf("Validating normal entry: %v", e) recData := fmt.Sprintf("waldata%d", e.Index) require.Equal(t, raftpb.EntryNormal, e.Type) require.Equal(t, recData, string(e.Data)) } rec = &walpb.Record{} } require.NoError(t, f.Close()) // Verify w.ReadAll() returns io.ErrUnexpectedEOF in the error chain. t.Log("Verify the w.ReadAll returns io.ErrUnexpectedEOF in the error chain") newFileName := filepath.Join(filepath.Dir(fileName), "0000000000000000-0000000000000000.wal") require.NoError(t, os.Rename(fileName, newFileName)) w, err := Open(zaptest.NewLogger(t), filepath.Dir(fileName), walpb.Snapshot{ Index: new(uint64(0)), Term: new(uint64(0)), }) require.NoError(t, err) defer w.Close() _, _, _, err = w.ReadAll() // Note: The wal file will be repaired automatically in production // environment, but only once. require.ErrorIs(t, err, io.ErrUnexpectedEOF) } ================================================ FILE: server/storage/wal/walpb/record.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package walpb import ( "errors" "fmt" ) var ErrCRCMismatch = errors.New("walpb: crc mismatch") func (rec *Record) Validate(crc uint32) error { if rec.GetCrc() == crc { return nil } return fmt.Errorf("%w: expected: %x computed: %x", ErrCRCMismatch, rec.GetCrc(), crc) } // ValidateSnapshotForWrite ensures the Snapshot the newly written snapshot is valid. // // There might exist log-entries written by old etcd versions that does not conform // to the requirements. func ValidateSnapshotForWrite(e *Snapshot) error { if e.Index == nil { return errors.New("snapshot is missing index: " + e.String()) } if e.Term == nil { return errors.New("snapshot is missing term: " + e.String()) } // Since etcd>=3.5.0 if e.ConfState == nil && e.GetIndex() > 0 { return errors.New("Saved (not-initial) snapshot is missing ConfState: " + e.String()) } return nil } ================================================ FILE: server/storage/wal/walpb/record.pb.go ================================================ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: record.proto package walpb import ( fmt "fmt" io "io" math "math" math_bits "math/bits" proto "github.com/golang/protobuf/proto" raftpb "go.etcd.io/raft/v3/raftpb" ) // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Record struct { Type *int64 `protobuf:"varint,1,opt,name=type" json:"type,omitempty"` Crc *uint32 `protobuf:"varint,2,opt,name=crc" json:"crc,omitempty"` Data []byte `protobuf:"bytes,3,opt,name=data" json:"data,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Record) Reset() { *m = Record{} } func (m *Record) String() string { return proto.CompactTextString(m) } func (*Record) ProtoMessage() {} func (*Record) Descriptor() ([]byte, []int) { return fileDescriptor_bf94fd919e302a1d, []int{0} } func (m *Record) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Record) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Record.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Record) XXX_Merge(src proto.Message) { xxx_messageInfo_Record.Merge(m, src) } func (m *Record) XXX_Size() int { return m.Size() } func (m *Record) XXX_DiscardUnknown() { xxx_messageInfo_Record.DiscardUnknown(m) } var xxx_messageInfo_Record proto.InternalMessageInfo func (m *Record) GetType() int64 { if m != nil && m.Type != nil { return *m.Type } return 0 } func (m *Record) GetCrc() uint32 { if m != nil && m.Crc != nil { return *m.Crc } return 0 } func (m *Record) GetData() []byte { if m != nil { return m.Data } return nil } // Keep in sync with raftpb.SnapshotMetadata. type Snapshot struct { Index *uint64 `protobuf:"varint,1,opt,name=index" json:"index,omitempty"` Term *uint64 `protobuf:"varint,2,opt,name=term" json:"term,omitempty"` // Field populated since >=etcd-3.5.0. ConfState *raftpb.ConfState `protobuf:"bytes,3,opt,name=conf_state,json=confState" json:"conf_state,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } func (m *Snapshot) Reset() { *m = Snapshot{} } func (m *Snapshot) String() string { return proto.CompactTextString(m) } func (*Snapshot) ProtoMessage() {} func (*Snapshot) Descriptor() ([]byte, []int) { return fileDescriptor_bf94fd919e302a1d, []int{1} } func (m *Snapshot) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } func (m *Snapshot) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { return xxx_messageInfo_Snapshot.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } return b[:n], nil } } func (m *Snapshot) XXX_Merge(src proto.Message) { xxx_messageInfo_Snapshot.Merge(m, src) } func (m *Snapshot) XXX_Size() int { return m.Size() } func (m *Snapshot) XXX_DiscardUnknown() { xxx_messageInfo_Snapshot.DiscardUnknown(m) } var xxx_messageInfo_Snapshot proto.InternalMessageInfo func (m *Snapshot) GetIndex() uint64 { if m != nil && m.Index != nil { return *m.Index } return 0 } func (m *Snapshot) GetTerm() uint64 { if m != nil && m.Term != nil { return *m.Term } return 0 } func (m *Snapshot) GetConfState() *raftpb.ConfState { if m != nil { return m.ConfState } return nil } func init() { proto.RegisterType((*Record)(nil), "walpb.Record") proto.RegisterType((*Snapshot)(nil), "walpb.Snapshot") } func init() { proto.RegisterFile("record.proto", fileDescriptor_bf94fd919e302a1d) } var fileDescriptor_bf94fd919e302a1d = []byte{ // 239 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x34, 0x8f, 0x41, 0x4a, 0xc4, 0x30, 0x14, 0x86, 0x8d, 0xed, 0x88, 0xc6, 0x11, 0x9c, 0xe0, 0xa2, 0xb8, 0x28, 0x65, 0x56, 0x05, 0x21, 0x11, 0x5d, 0xbb, 0x19, 0x6f, 0x90, 0xd9, 0xb9, 0x91, 0x4c, 0x9a, 0x8e, 0x03, 0x63, 0x5f, 0x78, 0x7d, 0x4c, 0xf5, 0x26, 0x1e, 0xc9, 0xa5, 0x47, 0x90, 0x7a, 0x11, 0xc9, 0x2b, 0xb3, 0xfa, 0xbf, 0xf0, 0xff, 0x7c, 0xe4, 0xc9, 0x39, 0x06, 0x0f, 0xd8, 0xe8, 0x88, 0x40, 0xa0, 0x66, 0x83, 0xdb, 0xc7, 0xcd, 0xed, 0x02, 0x5d, 0x4b, 0x71, 0x63, 0x52, 0x4c, 0xcd, 0x72, 0x25, 0xcf, 0x2c, 0x2f, 0x95, 0x92, 0x39, 0x7d, 0xc6, 0x50, 0x88, 0x4a, 0xd4, 0x99, 0x65, 0x56, 0xd7, 0x32, 0xf3, 0xe8, 0x8b, 0xd3, 0x4a, 0xd4, 0x57, 0x36, 0x61, 0x5a, 0x35, 0x8e, 0x5c, 0x91, 0x55, 0xa2, 0x9e, 0x5b, 0xe6, 0x65, 0x2b, 0xcf, 0xd7, 0x9d, 0x8b, 0xfd, 0x1b, 0x90, 0xba, 0x91, 0xb3, 0x5d, 0xd7, 0x84, 0x0f, 0xd6, 0xe4, 0x76, 0x7a, 0xb0, 0x3b, 0xe0, 0x3b, 0x8b, 0x72, 0xcb, 0xac, 0xee, 0xa5, 0xf4, 0xd0, 0xb5, 0xaf, 0x3d, 0x39, 0x0a, 0xec, 0xbb, 0x7c, 0x58, 0xe8, 0xe9, 0x87, 0xfa, 0x19, 0xba, 0x76, 0x9d, 0x0a, 0x7b, 0xe1, 0x8f, 0xb8, 0x7a, 0xfa, 0x1e, 0x4b, 0xf1, 0x33, 0x96, 0xe2, 0x77, 0x2c, 0xc5, 0xd7, 0x5f, 0x79, 0xf2, 0x72, 0xb7, 0x05, 0x1d, 0xc8, 0x37, 0x7a, 0x07, 0x26, 0xa5, 0xe9, 0x03, 0x1e, 0x02, 0x9a, 0xc3, 0xa3, 0xe9, 0x09, 0xd0, 0x6d, 0x83, 0x19, 0xdc, 0xde, 0xf0, 0xf5, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x48, 0x29, 0xe6, 0x31, 0x13, 0x01, 0x00, 0x00, } func (m *Record) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Record) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Record) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.Data != nil { i -= len(m.Data) copy(dAtA[i:], m.Data) i = encodeVarintRecord(dAtA, i, uint64(len(m.Data))) i-- dAtA[i] = 0x1a } if m.Crc != nil { i = encodeVarintRecord(dAtA, i, uint64(*m.Crc)) i-- dAtA[i] = 0x10 } if m.Type != nil { i = encodeVarintRecord(dAtA, i, uint64(*m.Type)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func (m *Snapshot) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } return dAtA[:n], nil } func (m *Snapshot) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } func (m *Snapshot) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.XXX_unrecognized != nil { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } if m.ConfState != nil { { size, err := m.ConfState.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } i -= size i = encodeVarintRecord(dAtA, i, uint64(size)) } i-- dAtA[i] = 0x1a } if m.Term != nil { i = encodeVarintRecord(dAtA, i, uint64(*m.Term)) i-- dAtA[i] = 0x10 } if m.Index != nil { i = encodeVarintRecord(dAtA, i, uint64(*m.Index)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } func encodeVarintRecord(dAtA []byte, offset int, v uint64) int { offset -= sovRecord(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) return base } func (m *Record) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Type != nil { n += 1 + sovRecord(uint64(*m.Type)) } if m.Crc != nil { n += 1 + sovRecord(uint64(*m.Crc)) } if m.Data != nil { l = len(m.Data) n += 1 + l + sovRecord(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func (m *Snapshot) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Index != nil { n += 1 + sovRecord(uint64(*m.Index)) } if m.Term != nil { n += 1 + sovRecord(uint64(*m.Term)) } if m.ConfState != nil { l = m.ConfState.Size() n += 1 + l + sovRecord(uint64(l)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } return n } func sovRecord(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } func sozRecord(x uint64) (n int) { return sovRecord(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } func (m *Record) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRecord } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: Record: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: Record: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) } var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRecord } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= int64(b&0x7F) << shift if b < 0x80 { break } } m.Type = &v case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Crc", wireType) } var v uint32 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRecord } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= uint32(b&0x7F) << shift if b < 0x80 { break } } m.Crc = &v case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRecord } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } if byteLen < 0 { return ErrInvalidLengthRecord } postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthRecord } if postIndex > l { return io.ErrUnexpectedEOF } m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) if m.Data == nil { m.Data = []byte{} } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRecord(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRecord } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func (m *Snapshot) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { preIndex := iNdEx var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRecord } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= uint64(b&0x7F) << shift if b < 0x80 { break } } fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { return fmt.Errorf("proto: Snapshot: wiretype end group for non-group") } if fieldNum <= 0 { return fmt.Errorf("proto: Snapshot: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) } var v uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRecord } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= uint64(b&0x7F) << shift if b < 0x80 { break } } m.Index = &v case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Term", wireType) } var v uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRecord } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ v |= uint64(b&0x7F) << shift if b < 0x80 { break } } m.Term = &v case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ConfState", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowRecord } if iNdEx >= l { return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ msglen |= int(b&0x7F) << shift if b < 0x80 { break } } if msglen < 0 { return ErrInvalidLengthRecord } postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthRecord } if postIndex > l { return io.ErrUnexpectedEOF } if m.ConfState == nil { m.ConfState = &raftpb.ConfState{} } if err := m.ConfState.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRecord(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthRecord } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) iNdEx += skippy } } if iNdEx > l { return io.ErrUnexpectedEOF } return nil } func skipRecord(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowRecord } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ wire |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } wireType := int(wire & 0x7) switch wireType { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowRecord } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } iNdEx++ if dAtA[iNdEx-1] < 0x80 { break } } case 1: iNdEx += 8 case 2: var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { return 0, ErrIntOverflowRecord } if iNdEx >= l { return 0, io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ length |= (int(b) & 0x7F) << shift if b < 0x80 { break } } if length < 0 { return 0, ErrInvalidLengthRecord } iNdEx += length case 3: depth++ case 4: if depth == 0 { return 0, ErrUnexpectedEndOfGroupRecord } depth-- case 5: iNdEx += 4 default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { return 0, ErrInvalidLengthRecord } if depth == 0 { return iNdEx, nil } } return 0, io.ErrUnexpectedEOF } var ( ErrInvalidLengthRecord = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowRecord = fmt.Errorf("proto: integer overflow") ErrUnexpectedEndOfGroupRecord = fmt.Errorf("proto: unexpected end of group") ) ================================================ FILE: server/storage/wal/walpb/record.proto ================================================ syntax = "proto2"; package walpb; import "raftpb/raft.proto"; option go_package = "go.etcd.io/etcd/server/v3/storage/wal/walpb"; message Record { optional int64 type = 1; optional uint32 crc = 2; optional bytes data = 3; } // Keep in sync with raftpb.SnapshotMetadata. message Snapshot { optional uint64 index = 1; optional uint64 term = 2; // Field populated since >=etcd-3.5.0. optional raftpb.ConfState conf_state = 3; } ================================================ FILE: server/storage/wal/walpb/record_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package walpb import ( "testing" "github.com/golang/protobuf/descriptor" //nolint:staticcheck // TODO: remove for a supported version "go.etcd.io/raft/v3/raftpb" ) func TestSnapshotMetadataCompatibility(t *testing.T) { _, snapshotMetadataMd := descriptor.ForMessage(&raftpb.SnapshotMetadata{}) //nolint:staticcheck // TODO: remove for a supported version _, snapshotMd := descriptor.ForMessage(&Snapshot{}) //nolint:staticcheck // TODO: remove for a supported version if len(snapshotMetadataMd.GetField()) != len(snapshotMd.GetField()) { t.Errorf("Different number of fields in raftpb.SnapshotMetadata vs. walpb.Snapshot. " + "They are supposed to be in sync.") } } func TestValidateSnapshot(t *testing.T) { tests := []struct { name string snap *Snapshot wantErr bool }{ {name: "empty", snap: &Snapshot{}, wantErr: true}, // index and term must be explicitly set {name: "initial", snap: &Snapshot{Index: new(uint64(0)), Term: new(uint64(0))}, wantErr: false}, {name: "invalid", snap: &Snapshot{Index: new(uint64(5)), Term: new(uint64(3))}, wantErr: true}, {name: "valid", snap: &Snapshot{Index: new(uint64(5)), Term: new(uint64(3)), ConfState: &raftpb.ConfState{Voters: []uint64{0x00cad1}}}, wantErr: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := ValidateSnapshotForWrite(tt.snap); (err != nil) != tt.wantErr { t.Errorf("ValidateSnapshotForWrite() error = %v, wantErr %v", err, tt.wantErr) } }) } } ================================================ FILE: server/verify/doc.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package verify // verify package is analyzing persistent state of etcd to find potential // inconsistencies. // In particular it covers cross-checking between different aspacts of etcd // storage like WAL & Backend. ================================================ FILE: server/verify/verify.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package verify import ( "fmt" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/verify" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/server/v3/storage/schema" wal2 "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) const envVerifyValueStorageWAL verify.VerificationType = "storage_wal" type Config struct { // DataDir is a root directory where the data being verified are stored. DataDir string // ExactIndex requires consistent_index in backend exactly match the last committed WAL entry. // Usually backend's consistent_index needs to be <= WAL.commit, but for backups the match // is expected to be exact. ExactIndex bool Logger *zap.Logger } // Verify performs consistency checks of given etcd data-directory. // The errors are reported as the returned error, but for some situations // the function can also panic. // The function is expected to work on not-in-use data model, i.e. // no file-locks should be taken. Verify does not modified the data. func Verify(cfg Config) (retErr error) { lg := cfg.Logger if lg == nil { lg = zap.NewNop() } if !fileutil.Exist(datadir.ToBackendFileName(cfg.DataDir)) { lg.Info("verification skipped due to non exist db file") return nil } lg.Info("verification of persisted state", zap.String("data-dir", cfg.DataDir)) defer func() { if retErr != nil { lg.Error("verification of persisted state failed", zap.String("data-dir", cfg.DataDir), zap.Error(retErr)) } else if r := recover(); r != nil { lg.Error("verification of persisted state failed", zap.String("data-dir", cfg.DataDir)) panic(r) } else { lg.Info("verification of persisted state successful", zap.String("data-dir", cfg.DataDir)) } }() be := backend.NewDefaultBackend(lg, datadir.ToBackendFileName(cfg.DataDir)) defer be.Close() snapshot, hardstate, err := validateWAL(cfg) if err != nil { return err } // TODO: Perform validation of consistency of membership between // backend/members & WAL confstate (and maybe storev2 if still exists). return validateConsistentIndex(cfg, hardstate, snapshot, be) } // VerifyIfEnabled performs verification according to ETCD_VERIFY env settings. // See Verify for more information. func VerifyIfEnabled(cfg Config) error { if verify.IsVerificationEnabled(envVerifyValueStorageWAL) { return Verify(cfg) } return nil } // MustVerifyIfEnabled performs verification according to ETCD_VERIFY env settings // and exits in case of found problems. // See Verify for more information. func MustVerifyIfEnabled(cfg Config) { if err := VerifyIfEnabled(cfg); err != nil { cfg.Logger.Fatal("Verification failed", zap.String("data-dir", cfg.DataDir), zap.Error(err)) } } func validateConsistentIndex(cfg Config, hardstate *raftpb.HardState, snapshot *walpb.Snapshot, be backend.Backend) error { index, term := schema.ReadConsistentIndex(be.ReadTx()) if cfg.ExactIndex && index != hardstate.Commit { return fmt.Errorf("backend.ConsistentIndex (%v) expected == WAL.HardState.commit (%v)", index, hardstate.Commit) } if cfg.ExactIndex && term != hardstate.Term { return fmt.Errorf("backend.Term (%v) expected == WAL.HardState.term, (%v)", term, hardstate.Term) } if index > hardstate.Commit { return fmt.Errorf("backend.ConsistentIndex (%v) must be <= WAL.HardState.commit (%v)", index, hardstate.Commit) } if term > hardstate.Term { return fmt.Errorf("backend.Term (%v) must be <= WAL.HardState.term, (%v)", term, hardstate.Term) } if index < snapshot.GetIndex() { return fmt.Errorf("backend.ConsistentIndex (%v) must be >= last snapshot index (%v)", index, snapshot.GetIndex()) } cfg.Logger.Info("verification: consistentIndex OK", zap.Uint64("backend-consistent-index", index), zap.Uint64("hardstate-commit", hardstate.Commit)) return nil } func validateWAL(cfg Config) (*walpb.Snapshot, *raftpb.HardState, error) { walDir := datadir.ToWALDir(cfg.DataDir) walSnaps, err := wal2.ValidSnapshotEntries(cfg.Logger, walDir) if err != nil { return nil, nil, err } snapshot := walSnaps[len(walSnaps)-1] hardstate, err := wal2.Verify(cfg.Logger, walDir, snapshot) if err != nil { return nil, nil, err } return &snapshot, hardstate, nil } ================================================ FILE: tests/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: tests/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/testing ================================================ FILE: tests/antithesis/Makefile ================================================ ANTITHESIS_ROOT :=$(dir $(realpath $(lastword $(MAKEFILE_LIST)))) REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel) USER_ID := $(shell id -u) GROUP_ID := $(shell id -g) ARCH ?= $(shell go env GOARCH) REF = main IMAGE_TAG = latest CFG_NODE_COUNT ?= 3 .PHONY: antithesis-build-client-docker-image antithesis-build-client-docker-image: validate-node-count # This is a release image using go build without our tooling. Using the # version defined in .go-version is the correct image tag. docker build \ --build-arg GO_IMAGE_TAG=$(shell cat $(REPOSITORY_ROOT)/.go-version) \ --file $(ANTITHESIS_ROOT)/test-template/Dockerfile \ --tag etcd-client:latest \ $(REPOSITORY_ROOT) .PHONY: antithesis-build-etcd-image antithesis-build-etcd-image: # Use the Workspace Go version from the go directive. This is the base image # later in the Dockerfile, we'll set the correct toolchain. docker build \ --build-arg GO_IMAGE_TAG=$(shell go work edit -json | jq .Go) \ --build-arg REF=$(REF) \ --tag etcd-server:latest \ $(ANTITHESIS_ROOT)/server/ .PHONY: antithesis-build-etcd-image-release-3.4 antithesis-build-etcd-image-release-3.4: REF=release-3.4 antithesis-build-etcd-image-release-3.4: antithesis-build-etcd-image .PHONY: antithesis-build-etcd-image-release-3.5 antithesis-build-etcd-image-release-3.5: REF=release-3.5 antithesis-build-etcd-image-release-3.5: antithesis-build-etcd-image .PHONY: antithesis-build-etcd-image-release-3.6 antithesis-build-etcd-image-release-3.6: REF=release-3.6 antithesis-build-etcd-image-release-3.6: antithesis-build-etcd-image .PHONY: antithesis-build-etcd-image-main antithesis-build-etcd-image-main: REF=main antithesis-build-etcd-image-main: antithesis-build-etcd-image .PHONY: antithesis-build-config-image antithesis-build-config-image: validate-node-count docker build -f config/Dockerfile config -t etcd-config:latest \ --build-arg IMAGE_TAG=$(IMAGE_TAG) \ --build-arg NODE_COUNT=$(CFG_NODE_COUNT) .PHONY: antithesis-docker-compose-up antithesis-docker-compose-up: validate-node-count export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \ docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml up .PHONY: antithesis-run-container-traffic antithesis-run-container-traffic: validate-node-count export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \ docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml exec client /opt/antithesis/test/v1/robustness/singleton_driver_traffic .PHONY: antithesis-run-container-validation antithesis-run-container-validation: validate-node-count export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \ docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml exec client /opt/antithesis/test/v1/robustness/finally_validation .PHONY: antithesis-run-local-traffic antithesis-run-local-traffic: export ETCD_ROBUSTNESS_DATA_PATHS=/tmp/etcddata0,/tmp/etcddata1,/tmp/etcddata2 && export ETCD_ROBUSTNESS_REPORT_PATH=report && export ETCD_ROBUSTNESS_ENDPOINTS=127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379 && \ go run --race ./test-template/robustness/traffic/main.go .PHONY: antithesis-run-local-validation antithesis-run-local-validation: export ETCD_ROBUSTNESS_DATA_PATHS=/tmp/etcddata0,/tmp/etcddata1,/tmp/etcddata2 && export ETCD_ROBUSTNESS_REPORT_PATH=report && export ETCD_ROBUSTNESS_ENDPOINTS=127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379 && \ go run --race ./test-template/robustness/finally/main.go .PHONY: antithesis-clean antithesis-clean: validate-node-count export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \ docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml down --remove-orphans rm -rf /tmp/etcddata0 /tmp/etcddata1 /tmp/etcddata2 /tmp/etcdreport .PHONY: validate-node-count validate-node-count: @if [ "$(CFG_NODE_COUNT)" != "1" ] && [ "$(CFG_NODE_COUNT)" != "3" ]; then \ echo "CFG_NODE_COUNT must be either 1 or 3 (got $(CFG_NODE_COUNT))"; \ exit 1; \ fi REQUIRED_K8S_TOOLS := kubectl kind .PHONY: check-k8s-tools check-k8s-tools: @for tool in $(REQUIRED_K8S_TOOLS); do \ if ! command -v $$tool >/dev/null 2>&1; then \ echo "Error: '$$tool' is missing. Please install it before continuing." >&2; \ exit 1; \ fi \ done ================================================ FILE: tests/antithesis/README.md ================================================ # etcd Antithesis tests This document describes the etcd test integration with [Antithesis]. Antithesis provides a testing platform that allows you to explore edge cases, race conditions, and rare bugs that are difficult or impossible to reproduce in a normal environment. [Antithesis]: https://antithesis.com/ ## Robustness vs Antithesis tests [Antithesis] runs the robustness tests inside their [deterministic simulation testing](https://antithesis.com/resources/deterministic_simulation_testing/) environment and [fault injection](https://antithesis.com/docs/environment/fault_injection/). For more details on robustness tests, see the [robustness directory](../robustness). ## Antithesis Setup The setup consists of a 3-node etcd cluster and a client container, orchestrated via [Docker Compose](https://antithesis.com/docs/getting_started/setup/). During the etcd Antithesis test suite the etcd server is built with the following patches: * **Critical code locations**: We replace etcd `gofail` comments (which signify code locations important for failure injection in robustness tests) with Antithesis `assert.Reachable`. This guides Antithesis to explore the execution space around these points. * **Assertions**: We change etcd `verify` package assertions to Antithesis `assert.Always`, encouraging the platform to try and break those assertions. * **Instrumentation**: The etcd binary is instrumented using `antithesis-go-instrumentor` to enable coverage tracking and feedback for the Antithesis platform. The Antithesis etcd tests configure the [Test Composer](https://antithesis.com/docs/test_templates/test_composer_reference/) in the following way: * **`entrypoint`**: * Waits for all etcd nodes to be healthy. * Emits the `setup_complete` message to Antithesis to start the testing phase. * **`singleton_driver_traffic`**: * Generates robustness test traffic against the cluster while faults are injected. * Runs as a [Singleton Driver Command], meaning it is the only one generating traffic. * All generated traffic is saved as an operation history and stored on a shared volume. * **`finally_validation`**: * Runs as a [Finally Command], meaning it is the last to run, with failure injection disabled. * Reads the history of operations and validates them using the robustness test validation logic. * Results of robustness tests are executed as Antithesis `assert.Always` assertions. * Similar to robustness tests, it emits a visualization of the operations history to an HTML file that is uploaded to the Antithesis platform. [Singleton Driver Command]: https://antithesis.com/docs/test_templates/test_composer_reference/#singleton-driver [Finally Command]: https://antithesis.com/docs/test_templates/test_composer_reference/#finally-command # Running tests with docker compose ## Quickstart ### 1. Build and Tag the Docker Image Run this command from the `antithesis/test-template` directory: ```bash make antithesis-build-client-docker-image make antithesis-build-etcd-image ``` Both commands build etcd-server and etcd-client from the current branch. To build a different version of etcd you can use: ```bash make antithesis-build-etcd-image REF=${GIT_REF} ``` ### 2. (Optional) Check the Image Locally You can verify your new image is built: ```bash docker images | grep etcd-client ``` It should show something like: ``` etcd-client latest ``` ### 3. Use in Docker Compose Run the following command from the root directory for Antithesis tests (`tests/antithesis`): ```bash make antithesis-docker-compose-up ``` The command uses the etcd client and server images built from step 1. The client will continuously check the health of the etcd nodes and print logs similar to: ``` [+] Running 4/4 ✔ Container etcd0 Created 0.0s ✔ Container etcd2 Created 0.0s ✔ Container etcd1 Created 0.0s ✔ Container client Recreated 0.1s Attaching to client, etcd0, etcd1, etcd2 etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.134294Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_ADVERTISE_CLIENT_URLS","variable-value":"http://etcd2.etcd:2379"} etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.138501Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_ADVERTISE_PEER_URLS","variable-value":"http://etcd2:2380"} etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.138646Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_CLUSTER","variable-value":"etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380"} etcd0 | {"level":"info","ts":"2025-04-14T07:23:25.138434Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_ADVERTISE_CLIENT_URLS","variable-value":"http://etcd0.etcd:2379"} etcd0 | {"level":"info","ts":"2025-04-14T07:23:25.138582Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_ADVERTISE_PEER_URLS","variable-value":"http://etcd0:2380"} etcd0 | {"level":"info","ts":"2025-04-14T07:23:25.138592Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_CLUSTER","variable-value":"etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380"} ... ... (skipping some repeated logs for brevity) ... ... etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.484698Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} etcd1 | {"level":"info","ts":"2025-04-14T07:23:25.484092Z","caller":"embed/serve.go:210","msg":"serving client traffic insecurely; this is strongly discouraged!","traffic":"grpc+http","address":"[::]:2379"} etcd0 | {"level":"info","ts":"2025-04-14T07:23:25.484563Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.485101Z","caller":"v3rpc/health.go:61","msg":"grpc service status changed","service":"","status":"SERVING"} etcd1 | {"level":"info","ts":"2025-04-14T07:23:25.484130Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.485782Z","caller":"embed/serve.go:210","msg":"serving client traffic insecurely; this is strongly discouraged!","traffic":"grpc+http","address":"[::]:2379"} etcd1 | {"level":"info","ts":"2025-04-14T07:23:25.484198Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} client | Client [entrypoint]: starting... client | Client [entrypoint]: checking cluster health... client | Client [entrypoint]: connection successful with etcd0 client | Client [entrypoint]: connection successful with etcd1 client | Client [entrypoint]: connection successful with etcd2 client | Client [entrypoint]: cluster is healthy! ``` And it will stay running indefinitely. ### 4. Running the tests ```bash make antithesis-run-container-traffic make antithesis-run-container-validation ``` Alternatively, with the etcd cluster from step 3, to run the tests locally without rebuilding the client image: ```bash make antithesis-run-local-traffic make antithesis-run-local-validation ``` ### 5. Prepare for next run Unfortunatelly robustness tests don't support running on non empty database. So for now you need to cleanup the storage before repeating the run or you will get "non empty database at start, required by model used for linearizability validation" error. ```bash make antithesis-clean ``` ## Troubleshooting - **Image Pull Errors**: If Docker can’t pull `etcd-client:latest`, make sure you built it locally (see the “Build and Tag” step) or push it to a registry that Compose can access. # Running Tests with Kubernetes (WIP) ## Prerequisites Please make sure that you have the following tools installed on your local: - [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) - [kind](https://kind.sigs.k8s.io/docs/user/quick-start#installation) ## Testing locally ### Setting up the cluster and deploying the images #### 1. Ensure your access to a test kubernetes cluster You can use `kind` to create a local cluster to deploy the etcd-server and test client. Once you have `kind` installed, you can use the following command to create a local cluster: ```bash kind create cluster ``` Alternatively, you can use any existing kubernetes cluster you have access to. #### 2. Build and load the images Please [build the client and server images](#1-build-and-tag-the-docker-image) first. Then load the images into the `kind` cluster: If you use `kind`, the cluster will need to have access to the images using the following commands: ```bash kind load docker-image etcd-client:latest kind load docker-image etcd-server:latest ``` If you use something other than `kind`, please make sure the images are accessible to your cluster. This might involve pushing the images to a container registry that your cluster can pull from. #### 3. Deploy the kubernetes manifests ```bash kubectl apply -f ./config/manifests ``` ### Tearing down the cluster ```bash kind delete cluster --name kind ``` ================================================ FILE: tests/antithesis/config/Dockerfile ================================================ ARG GO_VERSION=1.25.5 FROM golang:$GO_VERSION AS build RUN go install github.com/a8m/envsubst/cmd/envsubst@v1.4.3 ARG IMAGE_TAG ARG NODE_COUNT COPY docker-compose-${NODE_COUNT}-node.yml /docker-compose.yml.template RUN IMAGE_TAG=${IMAGE_TAG} cat /docker-compose.yml.template | envsubst > /docker-compose.yml FROM scratch COPY --from=build /docker-compose.yml /docker-compose.yml ================================================ FILE: tests/antithesis/config/docker-compose-1-node.yml ================================================ --- services: # This is needed for creating non-root data folders on host. # By default, if the folders don't exist when mounting, compose creates them with root as owner. # With root owner, accessing the WAL files from local tests will fail due to an unauthorized access error. init: image: 'docker.io/library/ubuntu:latest' user: root group_add: - '${GROUP_ID:-root}' volumes: - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcddata0 - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report command: - /bin/sh - -c - | rm -rf /var/etcddata0/* /var/report/* chown -R ${USER_ID:-root}:${GROUP_ID:-root} /var/etcddata0 /var/report etcd0: image: 'etcd-server:${IMAGE_TAG:-latest}' container_name: etcd0 hostname: etcd0 environment: ETCD_NAME: "etcd0" ETCD_INITIAL_ADVERTISE_PEER_URLS: "http://etcd0:2380" ETCD_LISTEN_PEER_URLS: "http://0.0.0.0:2380" ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" ETCD_ADVERTISE_CLIENT_URLS: "http://etcd0.etcd:2379" ETCD_INITIAL_CLUSTER_TOKEN: "etcd-cluster-1" ETCD_INITIAL_CLUSTER: "etcd0=http://etcd0:2380" ETCD_INITIAL_CLUSTER_STATE: "new" ETCD_DATA_DIR: "/var/etcd/data" ETCD_SNAPSHOT_CATCHUP_ENTRIES: 100 ETCD_SNAPSHOT_COUNT: 50 ETCD_COMPACTION_BATCH_LIMIT: 10 ETCD_VERIFY: "all" user: "${USER_ID:-root}:${GROUP_ID:-root}" depends_on: init: condition: service_completed_successfully ports: - 12379:2379 volumes: - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcd/data client: image: 'etcd-client:${IMAGE_TAG:-latest}' container_name: client entrypoint: ["/opt/antithesis/entrypoint/entrypoint"] user: "${USER_ID:-root}:${GROUP_ID:-root}" environment: ETCD_ROBUSTNESS_ENDPOINTS: "etcd0:2379" ETCD_ROBUSTNESS_DATA_PATHS: "/var/etcddata0" ETCD_ROBUSTNESS_REPORT_PATH: "/var/report" depends_on: etcd0: condition: service_started volumes: - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcddata0 - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report ================================================ FILE: tests/antithesis/config/docker-compose-3-node.yml ================================================ --- services: # This is needed for creating non-root data folders on host. # By default, if the folders don't exist when mounting, compose creates them with root as owner. # With root owner, accessing the WAL files from local tests will fail due to an unauthorized access error. init: image: 'docker.io/library/ubuntu:latest' user: root group_add: - '${GROUP_ID:-root}' volumes: - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcddata0 - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}1:/var/etcddata1 - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}2:/var/etcddata2 - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report command: - /bin/sh - -c - | rm -rf /var/etcddata0/* /var/etcddata1/* /var/etcddata2/* /var/report/* chown -R ${USER_ID:-root}:${GROUP_ID:-root} /var/etcddata0 /var/etcddata1 /var/etcddata2 /var/report etcd0: image: 'etcd-server:${IMAGE_TAG:-latest}' container_name: etcd0 hostname: etcd0 environment: ETCD_NAME: "etcd0" ETCD_INITIAL_ADVERTISE_PEER_URLS: "http://etcd0:2380" ETCD_LISTEN_PEER_URLS: "http://0.0.0.0:2380" ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" ETCD_ADVERTISE_CLIENT_URLS: "http://etcd0.etcd:2379" ETCD_INITIAL_CLUSTER_TOKEN: "etcd-cluster-1" ETCD_INITIAL_CLUSTER: "etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380" ETCD_INITIAL_CLUSTER_STATE: "new" ETCD_DATA_DIR: "/var/etcd/data" ETCD_SNAPSHOT_CATCHUP_ENTRIES: 100 ETCD_SNAPSHOT_COUNT: 50 ETCD_COMPACTION_BATCH_LIMIT: 10 ETCD_VERIFY: "all" user: "${USER_ID:-root}:${GROUP_ID:-root}" depends_on: init: condition: service_completed_successfully ports: - 12379:2379 volumes: - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcd/data etcd1: image: 'etcd-server:${IMAGE_TAG:-latest}' container_name: etcd1 hostname: etcd1 environment: ETCD_NAME: "etcd1" ETCD_INITIAL_ADVERTISE_PEER_URLS: "http://etcd1:2380" ETCD_LISTEN_PEER_URLS: "http://0.0.0.0:2380" ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" ETCD_ADVERTISE_CLIENT_URLS: "http://etcd1.etcd:2379" ETCD_INITIAL_CLUSTER_TOKEN: "etcd-cluster-1" ETCD_INITIAL_CLUSTER: "etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380" ETCD_INITIAL_CLUSTER_STATE: "new" ETCD_DATA_DIR: "/var/etcd/data" ETCD_SNAPSHOT_CATCHUP_ENTRIES: 100 ETCD_SNAPSHOT_COUNT: 50 ETCD_COMPACTION_BATCH_LIMIT: 10 ETCD_VERIFY: "all" user: "${USER_ID:-root}:${GROUP_ID:-root}" depends_on: init: condition: service_completed_successfully ports: - 22379:2379 volumes: - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}1:/var/etcd/data etcd2: image: 'etcd-server:${IMAGE_TAG:-latest}' container_name: etcd2 hostname: etcd2 environment: ETCD_NAME: "etcd2" ETCD_INITIAL_ADVERTISE_PEER_URLS: "http://etcd2:2380" ETCD_LISTEN_PEER_URLS: "http://0.0.0.0:2380" ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" ETCD_ADVERTISE_CLIENT_URLS: "http://etcd2.etcd:2379" ETCD_INITIAL_CLUSTER_TOKEN: "etcd-cluster-1" ETCD_INITIAL_CLUSTER: "etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380" ETCD_INITIAL_CLUSTER_STATE: "new" ETCD_DATA_DIR: "/var/etcd/data" ETCD_SNAPSHOT_CATCHUP_ENTRIES: 100 ETCD_SNAPSHOT_COUNT: 50 ETCD_COMPACTION_BATCH_LIMIT: 10 ETCD_VERIFY: "all" user: "${USER_ID:-root}:${GROUP_ID:-root}" depends_on: init: condition: service_completed_successfully ports: - 32379:2379 volumes: - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}2:/var/etcd/data client: image: 'etcd-client:${IMAGE_TAG:-latest}' container_name: client entrypoint: ["/opt/antithesis/entrypoint/entrypoint"] user: "${USER_ID:-root}:${GROUP_ID:-root}" environment: ETCD_ROBUSTNESS_ENDPOINTS: "etcd0:2379,etcd1:2379,etcd2:2379" ETCD_ROBUSTNESS_DATA_PATHS: "/var/etcddata0,/var/etcddata1,/var/etcddata2" ETCD_ROBUSTNESS_REPORT_PATH: "/var/report" depends_on: etcd0: condition: service_started etcd1: condition: service_started etcd2: condition: service_started volumes: - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcddata0 - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}1:/var/etcddata1 - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}2:/var/etcddata2 - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report ================================================ FILE: tests/antithesis/config/manifests/default-etcd-3-replicas.yaml ================================================ --- apiVersion: v1 kind: Service metadata: name: etcd namespace: default spec: type: ClusterIP clusterIP: None publishNotReadyAddresses: true ports: - name: client port: 2379 targetPort: client - name: peer port: 2380 targetPort: peer selector: app: etcd --- apiVersion: apps/v1 kind: StatefulSet metadata: name: etcd namespace: default spec: selector: matchLabels: app: etcd serviceName: etcd replicas: 3 podManagementPolicy: Parallel template: metadata: labels: app: etcd spec: terminationGracePeriodSeconds: 10 containers: - name: etcd-server image: etcd-server:latest imagePullPolicy: Never env: - name: ETCD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: CLIENT_PORT value: "2379" - name: PEER_PORT value: "2380" - name: ETCD_INITIAL_ADVERTISE_PEER_URLS value: http://$(ETCD_NAME).etcd.default.svc.cluster.local:$(PEER_PORT) - name: ETCD_LISTEN_PEER_URLS value: "http://0.0.0.0:$(PEER_PORT)" - name: ETCD_LISTEN_CLIENT_URLS value: "http://0.0.0.0:$(CLIENT_PORT)" - name: ETCD_ADVERTISE_CLIENT_URLS value: "https://$(ETCD_NAME).etcd:$(CLIENT_PORT)" - name: ETCD_INITIAL_CLUSTER_TOKEN value: "etcd-cluster-1" - name: ETCD_INITIAL_CLUSTER value: "etcd-0=http://etcd-0.etcd.default.svc.cluster.local:$(PEER_PORT),etcd-1=http://etcd-1.etcd.default.svc.cluster.local:$(PEER_PORT),etcd-2=http://etcd-2.etcd.default.svc.cluster.local:$(PEER_PORT)" - name: ETCD_INITIAL_CLUSTER_STATE value: "new" - name: ETCD_DATA_DIR value: "/var/etcd/data" - name: ETCD_SNAPSHOT_CATCHUP_ENTRIES value: "100" - name: ETCD_SNAPSHOT_COUNT value: "50" - name: ETCD_COMPACTION_BATCH_LIMIT value: "10" - name: ETCD_VERIFY value: "all" ports: - containerPort: 2379 name: client protocol: TCP - containerPort: 2380 name: peer protocol: TCP readinessProbe: httpGet: path: /readyz port: client livenessProbe: httpGet: path: /livez port: client volumeMounts: - name: data mountPath: /var/etcd/data volumeClaimTemplates: - metadata: name: data spec: storageClassName: standard accessModes: - ReadWriteOnce resources: requests: storage: 200Mi ================================================ FILE: tests/antithesis/server/Dockerfile ================================================ ARG ARCH=amd64 ARG GO_IMAGE_TAG FROM golang:$GO_IMAGE_TAG AS build # cloning etcd ARG REF=main RUN git clone --depth=1 https://github.com/etcd-io/etcd.git --branch=${REF} /etcd RUN go env -w GOTOOLCHAIN="go$(cat .go-version)" # inject assertions in place of gofail WORKDIR /etcd/server RUN go install golang.org/x/tools/cmd/goimports@latest RUN go get github.com/antithesishq/antithesis-sdk-go@v0.4.4 RUN for file in $(grep -rl '// gofail'); do sed -i 's|\/\/ gofail.*var \([[:alnum:]]*\) .*|assert\.Reachable("\1", nil)|' $file; goimports -w $file; done RUN go mod tidy # replace verify with antithesis WORKDIR /etcd/client/pkg/verify COPY inject/verify.patch verify.patch RUN if [ "${REF}" = "main" ]; then git apply verify.patch; fi WORKDIR /etcd/client/pkg RUN go mod tidy WORKDIR /etcd # setup go mod RUN go mod download # install instrumentor RUN go get github.com/antithesishq/antithesis-sdk-go@v0.4.4 RUN go install github.com/antithesishq/antithesis-sdk-go/tools/antithesis-go-instrumentor@a802e8810442e01d16b3e9df77d7ce3875e36e55 # v0.4.3 RUN go mod tidy # compile etcd server with instrumentor RUN mkdir /etcd_instrumented RUN `go env GOPATH`/bin/antithesis-go-instrumentor /etcd /etcd_instrumented # Remove this once antithesis fixed the bug # The instrumentor's notifier library doesn't preserve the go and toolchain directives # This makes any subsequent `go mod tidy` run remove all the directives and replaced them with one generated by the notifier folder # Updating /etcd_instrumented/notifier/go.mod with the original directives would prevent all subsequent `go mod tidy` from removing the directives # However `go mod tidy` is already invoked by the instrumentor, so it needs to be fixed as well. RUN for d in /etcd_instrumented/customer /etcd_instrumented/notifier; do \ cd ${d}; \ go mod edit -go=$(grep '^go ' /etcd/go.mod | cut -f2 -d' '); \ go mod edit -toolchain=$(grep 'toolchain ' /etcd/go.mod | cut -f2 -d' '); \ done WORKDIR /etcd_instrumented/customer # Some previous versions hardcode CGO_ENABLED=0 RUN find . -type f -exec sed -i 's/CGO_ENABLED=0/CGO_ENABLED=${CGO_ENABLED}/' {} + # Some previous versions explicitly need gobin, which could no longer be installed with go get RUN go install github.com/myitcv/gobin@v0.0.14 # 3.4.0 has vendoring. need to do this after instrumentation or else build fails RUN if [ -d "vendor" ]; then go mod vendor; fi # The instrumentation adds code and packages. Need go mod tidy for all modules before building RUN for d in server etcdutl etcdctl; do \ (cd ${d} && go mod tidy || true); \ done # The instrumentation also adds a new main file which clashes with dummy.go found in non release-3.4 branches RUN if [ -f "dummy.go" ]; then sed -i 's/package main_test/package main/' dummy.go; fi RUN CGO_ENABLED=1 GO_GCFLAGS="all=-N -l" make build RUN go install github.com/go-delve/delve/cmd/dlv@latest FROM ubuntu:24.04 COPY --from=build /go/bin/dlv /bin/dlv COPY --from=build /etcd_instrumented/ /etcd # Move symbols to /symbols directory https://antithesis.com/docs/instrumentation/#symbolization RUN mv /etcd/symbols /symbols EXPOSE 2379 2380 CMD ["/etcd/customer/bin/etcd"] ================================================ FILE: tests/antithesis/server/inject/verify.patch ================================================ diff --git a/client/pkg/verify/verify.go b/client/pkg/verify/verify.go index cb48d8ff0..095f890ec 100644 --- a/client/pkg/verify/verify.go +++ b/client/pkg/verify/verify.go @@ -18,6 +18,8 @@ import ( "fmt" "os" "strings" + + "github.com/antithesishq/antithesis-sdk-go/assert" ) const envVerify = "ETCD_VERIFY" @@ -69,7 +71,7 @@ func DisableVerifications() func() { func Verify(msg string, f VerifyFunc) { if IsVerificationEnabled(envVerifyValueAssert) { ok, details := f() - verifier(ok, msg, details) + assert.Always(ok, msg, details) } } ================================================ FILE: tests/antithesis/test-template/Dockerfile ================================================ ARG GO_IMAGE_TAG ARG ARCH=amd64 FROM golang:$GO_IMAGE_TAG AS build WORKDIR /build COPY . . WORKDIR /build/tests RUN go build -o /opt/antithesis/entrypoint/entrypoint -race ./antithesis/test-template/entrypoint/main.go RUN go build -o /opt/antithesis/test/v1/robustness/singleton_driver_traffic -race ./antithesis/test-template/robustness/traffic/main.go RUN go build -o /opt/antithesis/test/v1/robustness/finally_validation -race ./antithesis/test-template/robustness/finally/main.go FROM ubuntu:24.04 COPY --from=build /opt/ /opt/ ================================================ FILE: tests/antithesis/test-template/entrypoint/main.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build cgo && amd64 package main import ( "context" "fmt" "time" "github.com/antithesishq/antithesis-sdk-go/lifecycle" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/antithesis/test-template/robustness/common" ) // Sleep duration const SLEEP = 10 // CheckHealth checks health of all etcd nodes func CheckHealth() bool { hosts, _, _ := common.GetPaths() // iterate over each host and check health for _, host := range hosts { cli, err := clientv3.New(clientv3.Config{ Endpoints: []string{fmt.Sprintf("http://%s", host)}, DialTimeout: 5 * time.Second, }) if err != nil { fmt.Printf("Client [entrypoint]: connection failed with %s\n", host) fmt.Printf("Client [entrypoint]: error: %v\n", err) return false } defer func() { cErr := cli.Close() if cErr != nil { fmt.Printf("Client [entrypoint]: error closing connection: %v\n", cErr) } }() // fetch the key setting-up to confirm that the node is available _, err = cli.Get(context.Background(), "setting-up") if err != nil { fmt.Printf("Client [entrypoint]: connection failed with %s\n", host) fmt.Printf("Client [entrypoint]: error: %v\n", err) return false } fmt.Printf("Client [entrypoint]: connection successful with %s\n", host) } return true } func main() { fmt.Println("Client [entrypoint]: starting...") // run loop until all nodes are healthy for { fmt.Println("Client [entrypoint]: checking cluster health...") if CheckHealth() { fmt.Println("Client [entrypoint]: cluster is healthy!") break } fmt.Printf("Client [entrypoint]: cluster is not healthy. retrying in %d seconds...\n", SLEEP) time.Sleep(SLEEP * time.Second) } // signal that the setup looks complete lifecycle.SetupComplete( map[string]string{ "Message": "ETCD cluster is healthy", }, ) select {} } ================================================ FILE: tests/antithesis/test-template/robustness/common/path.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build cgo && amd64 package common import ( "fmt" "os" "strings" ) const ( endpointsEnv = "ETCD_ROBUSTNESS_ENDPOINTS" dataPathsEnv = "ETCD_ROBUSTNESS_DATA_PATHS" reportPathEnv = "ETCD_ROBUSTNESS_REPORT_PATH" ) func GetPaths() (hosts []string, reportPath string, dataPaths map[string]string) { // Check for environment variable overrides first envDataPathsStr := os.Getenv(dataPathsEnv) envReportPath := os.Getenv(reportPathEnv) envEndpointsStr := os.Getenv(endpointsEnv) // Temporary disable to make PR simpler to review //revive:disable:early-return if envEndpointsStr != "" { hosts = strings.Split(envEndpointsStr, ",") for i, host := range hosts { hosts[i] = strings.TrimSpace(host) } } else { panic(fmt.Sprintf("No endpoints specified in %s", endpointsEnv)) } if envReportPath != "" { reportPath = envReportPath } else { panic(fmt.Sprintf("No report path specified in %s", reportPathEnv)) } if envDataPathsStr != "" { envDataPaths := strings.Split(envDataPathsStr, ",") if len(envDataPaths) != len(hosts) { panic(fmt.Sprintf("Mismatched number of endpoints and data paths: %d endpoints, %d data paths", len(hosts), len(envDataPaths))) } dataPaths = make(map[string]string) for i, endpoint := range hosts { dataPaths[endpoint] = strings.TrimSpace(envDataPaths[i]) } } else { panic(fmt.Sprintf("No data paths specified in %s", dataPathsEnv)) } //revive:enable:early-return return hosts, reportPath, dataPaths } ================================================ FILE: tests/antithesis/test-template/robustness/finally/main.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build cgo && amd64 package main import ( "maps" "os" "path/filepath" "slices" "time" "github.com/antithesishq/antithesis-sdk-go/assert" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/antithesis/test-template/robustness/common" "go.etcd.io/etcd/tests/v3/robustness/report" "go.etcd.io/etcd/tests/v3/robustness/validate" ) const ( reportFileName = "history.html" ) func main() { _, reportPath, dirs := common.GetPaths() lg, err := zap.NewProduction() if err != nil { panic(err) } reports, err := report.LoadClientReports(reportPath) assert.Always(err == nil, "Loaded client reports", map[string]any{"error": err}) tf, err := report.LoadTrafficDetail(reportPath) if err != nil && !os.IsNotExist(err) { panic(err) } result := validateReports(lg, dirs, reports, tf) if err := result.Linearization.Visualize(lg, filepath.Join(reportPath, reportFileName)); err != nil { panic(err) } } func validateReports(lg *zap.Logger, serversDataPath map[string]string, reports []report.ClientReport, tf report.TrafficDetail) validate.RobustnessResult { persistedRequests, err := report.PersistedRequests(lg, slices.Collect(maps.Values(serversDataPath))) assertResult(validate.ResultFromError(err), "Loaded persisted requests") validateConfig := validate.Config{ExpectRevisionUnique: tf.ExpectUniqueRevision} result := validate.ValidateAndReturnVisualize(lg, validateConfig, reports, persistedRequests, 5*time.Minute) assertResult(result.Assumptions, "Validation assumptions fulfilled") if result.Linearization.Timeout { assert.Unreachable("Linearization timeout", nil) } else { assertResult(result.Linearization.Result, "Linearization validation passes") } assertResult(result.Watch, "Watch validation passes") assertResult(result.Serializable, "Serializable validation passes") lg.Info("Completed robustness validation") return result } func assertResult(result validate.Result, name string) { switch result.Status { case validate.Success, validate.Failure: assert.Always(result.Status == validate.Success, name, map[string]any{"msg": result.Message}) case validate.Unknown: default: assert.Unreachable(name, map[string]any{"msg": result.Message}) } } ================================================ FILE: tests/antithesis/test-template/robustness/traffic/main.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build cgo && amd64 package main import ( "context" "math/rand/v2" "os" "slices" "sync" "time" "github.com/antithesishq/antithesis-sdk-go/assert" "go.uber.org/zap" "golang.org/x/sync/errgroup" "golang.org/x/time/rate" "go.etcd.io/etcd/tests/v3/antithesis/test-template/robustness/common" "go.etcd.io/etcd/tests/v3/robustness/client" "go.etcd.io/etcd/tests/v3/robustness/identity" robustnessrand "go.etcd.io/etcd/tests/v3/robustness/random" "go.etcd.io/etcd/tests/v3/robustness/report" "go.etcd.io/etcd/tests/v3/robustness/traffic" ) var ( DefaultWatchInterval = 250 * time.Millisecond profile = traffic.Profile{ KeyValue: &traffic.KeyValue{ MinimalQPS: 100, MaximalQPS: 1000, BurstableQPS: 1000, MemberClientCount: 3, ClusterClientCount: 1, MaxNonUniqueRequestConcurrency: 3, }, Watch: &traffic.WatchDefault, Compaction: &traffic.CompactionDefault, } trafficNames = []string{ "etcd", "kubernetes", } traffics = []traffic.Traffic{ traffic.EtcdPutDeleteLease, traffic.Kubernetes, } ) func main() { hosts, reportPath, etcdetcdDataPaths := common.GetPaths() ctx := context.Background() baseTime := time.Now() duration := time.Duration(robustnessrand.RandRange(5, 15) * int64(time.Second)) lg, err := zap.NewProduction() if err != nil { panic(err) } choice := rand.IntN(len(traffics)) tf := traffics[choice] lg.Info("Traffic", zap.String("Type", trafficNames[choice])) r := report.TestReport{Logger: lg, ServersDataPath: etcdetcdDataPaths, Traffic: &report.TrafficDetail{ExpectUniqueRevision: tf.ExpectUniqueRevision()}} defer func() { if err = r.Report(reportPath); err != nil { lg.Error("Failed to save traffic generation report", zap.Error(err)) } }() lg.Info("Start traffic generation", zap.Duration("duration", duration), zap.String("base-time", baseTime.UTC().Format("2006-01-02T15:04:05.000000Z0700"))) r.Client, err = runTraffic(ctx, lg, tf, hosts, baseTime, duration) if err != nil { lg.Error("Failed to generate traffic") panic(err) } } func runTraffic(ctx context.Context, lg *zap.Logger, tf traffic.Traffic, hosts []string, baseTime time.Time, duration time.Duration) ([]report.ClientReport, error) { ids := identity.NewIDProvider() trafficSet := client.NewSet(ids, baseTime) defer trafficSet.Close() err := traffic.CheckEmptyDatabaseAtStart(ctx, lg, hosts, trafficSet) if err != nil { lg.Fatal("Failed empty database at start check", zap.Error(err)) } maxRevisionChan := make(chan int64, 1) watchConfig := client.WatchConfig{ RequestProgress: true, } g := errgroup.Group{} startTime := time.Since(baseTime) g.Go(func() error { defer close(maxRevisionChan) simulateTraffic(ctx, lg, tf, hosts, trafficSet, duration) maxRevision := report.OperationsMaxRevision(trafficSet.Reports()) maxRevisionChan <- maxRevision lg.Info("Finished simulating Traffic", zap.Int64("max-revision", maxRevision)) return nil }) watchSet := client.NewSet(ids, baseTime) defer watchSet.Close() g.Go(func() error { err := client.CollectClusterWatchEvents(ctx, client.CollectClusterWatchEventsParam{ Lg: lg, Endpoints: hosts, MaxRevisionChan: maxRevisionChan, Cfg: watchConfig, ClientSet: watchSet, }) return err }) if err := g.Wait(); err != nil { return nil, err } endTime := time.Since(baseTime) reports := slices.Concat(trafficSet.Reports(), watchSet.Reports()) totalStats := traffic.CalculateStats(reports, startTime, endTime) lg.Info("Completed traffic generation", zap.Int("successes", totalStats.Successes), zap.Int("failures", totalStats.Failures), zap.Float64("successRate", totalStats.SuccessRate()), zap.Duration("period", totalStats.Period), zap.Float64("qps", totalStats.QPS()), ) return reports, nil } func simulateTraffic(ctx context.Context, lg *zap.Logger, tf traffic.Traffic, hosts []string, clientSet *client.ClientSet, duration time.Duration) { var wg sync.WaitGroup leaseStorage := identity.NewLeaseIDStorage() kubernetesStorage := traffic.NewKubernetesStorage() limiter := rate.NewLimiter(rate.Limit(profile.KeyValue.MaximalQPS), profile.KeyValue.BurstableQPS) concurrencyLimiter := traffic.NewConcurrencyLimiter(profile.KeyValue.MaxNonUniqueRequestConcurrency) finish := closeAfter(ctx, duration) keyStore := traffic.NewKeyStore(10, "key") for i := range profile.KeyValue.MemberClientCount { c := connect(clientSet, []string{hosts[i%len(hosts)]}) wg.Add(1) go func(c *client.RecordingClient) { defer wg.Done() defer c.Close() tf.RunKeyValueLoop(ctx, traffic.RunTrafficLoopParam{ Client: c, QPSLimiter: limiter, IDs: clientSet.IdentityProvider(), LeaseIDStorage: leaseStorage, NonUniqueRequestConcurrencyLimiter: concurrencyLimiter, KeyStore: keyStore, Storage: kubernetesStorage, Finish: finish, }) }(c) } for range profile.KeyValue.ClusterClientCount { c := connect(clientSet, hosts) wg.Add(1) go func(c *client.RecordingClient) { defer wg.Done() defer c.Close() tf.RunKeyValueLoop(ctx, traffic.RunTrafficLoopParam{ Client: c, QPSLimiter: limiter, IDs: clientSet.IdentityProvider(), LeaseIDStorage: leaseStorage, NonUniqueRequestConcurrencyLimiter: concurrencyLimiter, KeyStore: keyStore, Storage: kubernetesStorage, Finish: finish, }) }(c) } if profile.Watch != nil { for i := range profile.Watch.MemberClientCount { c := connect(clientSet, []string{hosts[i%len(hosts)]}) wg.Add(1) go func(c *client.RecordingClient) { defer wg.Done() defer c.Close() tf.RunWatchLoop(ctx, traffic.RunWatchLoopParam{ Config: *profile.Watch, Client: c, QPSLimiter: limiter, KeyStore: keyStore, Storage: kubernetesStorage, Finish: finish, Logger: lg, }) }(c) } for range profile.Watch.ClusterClientCount { c := connect(clientSet, hosts) wg.Add(1) go func(c *client.RecordingClient) { defer wg.Done() defer c.Close() tf.RunWatchLoop(ctx, traffic.RunWatchLoopParam{ Config: *profile.Watch, Client: c, QPSLimiter: limiter, KeyStore: keyStore, Storage: kubernetesStorage, Finish: finish, Logger: lg, }) }(c) } } if profile.Compaction != nil { wg.Add(1) compactClient := connect(clientSet, hosts) go func(c *client.RecordingClient) { defer wg.Done() defer c.Close() tf.RunCompactLoop(ctx, traffic.RunCompactLoopParam{ Client: c, Period: profile.Compaction.Period, Finish: finish, }) }(compactClient) } defragPeriod := profile.Compaction.Period * time.Duration(len(hosts)) for _, h := range hosts { c := connect(clientSet, []string{h}) wg.Add(1) go func(c *client.RecordingClient) { defer wg.Done() defer c.Close() runDefragLoop(ctx, c, defragPeriod, finish) }(c) } wg.Wait() } func runDefragLoop(ctx context.Context, c *client.RecordingClient, period time.Duration, finish <-chan struct{}) { jittered := time.Duration(robustnessrand.RandRange(int64(period-period/2), int64(period+period/2))) ticker := time.NewTicker(jittered) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-finish: return case <-ticker.C: } dctx, cancel := context.WithTimeout(ctx, traffic.RequestTimeout) _, err := c.Defragment(dctx) cancel() if err != nil { continue } } } func connect(cs *client.ClientSet, endpoints []string) *client.RecordingClient { cli, err := cs.NewClient(endpoints) if err != nil { // Antithesis Assertion: client should always be able to connect to an etcd host assert.Unreachable("Client failed to connect to an etcd host", map[string]any{"endpoints": endpoints, "error": err}) os.Exit(1) } return cli } func closeAfter(ctx context.Context, t time.Duration) <-chan struct{} { out := make(chan struct{}) go func() { select { case <-time.After(t): case <-ctx.Done(): } close(out) }() return out } ================================================ FILE: tests/common/alarm_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "os" "strings" "testing" "time" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestAlarm(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterSize(1), config.WithQuotaBackendBytes(int64(13*os.Getpagesize())), ) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { // test small put still works smallbuf := strings.Repeat("a", 64) _, err := cc.Put(ctx, "1st_test", smallbuf, config.PutOptions{}) require.NoErrorf(t, err, "alarmTest: put kv error") // write some chunks to fill up the database buf := strings.Repeat("b", os.Getpagesize()) for { if _, err = cc.Put(ctx, "2nd_test", buf, config.PutOptions{}); err != nil { require.ErrorContains(t, err, "etcdserver: mvcc: database space exceeded") break } } // quota alarm should now be on alarmResp, err := cc.AlarmList(ctx) require.NoErrorf(t, err, "alarmTest: Alarm error") // check that Put is rejected when alarm is on if _, err = cc.Put(ctx, "3rd_test", smallbuf, config.PutOptions{}); err != nil { require.ErrorContains(t, err, "etcdserver: mvcc: database space exceeded") } // get latest revision to compact sresp, err := cc.Status(ctx) require.NoErrorf(t, err, "get endpoint status error") var rvs int64 for _, resp := range sresp { if resp != nil && resp.Header != nil { rvs = resp.Header.Revision break } } // make some space _, err = cc.Compact(ctx, rvs, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) require.NoErrorf(t, err, "alarmTest: Compact error") err = cc.Defragment(ctx, config.DefragOption{Timeout: 10 * time.Second}) require.NoErrorf(t, err, "alarmTest: defrag error") // turn off alarm for _, alarm := range alarmResp.Alarms { alarmMember := &clientv3.AlarmMember{ MemberID: alarm.MemberID, Alarm: alarm.Alarm, } _, err = cc.AlarmDisarm(ctx, alarmMember) require.NoErrorf(t, err, "alarmTest: Alarm error") } // put one more key below quota _, err = cc.Put(ctx, "4th_test", smallbuf, config.PutOptions{}) require.NoError(t, err) }) } func TestAlarmlistOnMemberRestart(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterSize(1), config.WithQuotaBackendBytes(int64(13*os.Getpagesize())), config.WithSnapshotCount(5), ) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { for i := 0; i < 6; i++ { _, err := cc.AlarmList(ctx) require.NoError(t, err) } clus.Members()[0].Stop() err := clus.Members()[0].Start(ctx) require.NoErrorf(t, err, "failed to start etcdserver") }) } ================================================ FILE: tests/common/auth_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "fmt" "path/filepath" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/storage/mvcc/testutil" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) var ( tokenTTL = time.Second * 3 defaultAuthToken = fmt.Sprintf("jwt,pub-key=%s,priv-key=%s,sign-method=RS256,ttl=%s", mustAbsPath("../fixtures/server.crt"), mustAbsPath("../fixtures/server.key.insecure"), tokenTTL) defaultKeyPath = mustAbsPath("../fixtures/server.key.insecure") verifyJWTOnlyAuth = fmt.Sprintf("jwt,pub-key=%s,sign-method=RS256,ttl=%s", mustAbsPath("../fixtures/server.crt"), tokenTTL) ) const ( PermissionDenied = "etcdserver: permission denied" AuthenticationFailed = "etcdserver: authentication failed, invalid user ID or password" InvalidAuthManagement = "etcdserver: invalid auth management" testPeerURL = "http://localhost:20011" ) func TestAuthEnable(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{}, []authUser{rootUser}), "failed to enable auth") }) } func TestAuthDisable(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.Put(ctx, "hoo", "a", config.PutOptions{}) require.NoError(t, err) require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) // test-user doesn't have the permission, it must fail _, err = testUserAuthClient.Put(ctx, "hoo", "bar", config.PutOptions{}) require.Error(t, err) require.NoErrorf(t, rootAuthClient.AuthDisable(ctx), "failed to disable auth") // now ErrAuthNotEnabled of Authenticate() is simply ignored _, err = testUserAuthClient.Put(ctx, "hoo", "bar", config.PutOptions{}) require.NoError(t, err) // now the key can be accessed _, err = testUserAuthClient.Put(ctx, "hoo", "bar", config.PutOptions{}) require.NoError(t, err) // confirm put succeeded resp, err := cc.Get(ctx, "hoo", config.GetOptions{}) require.NoError(t, err) require.Lenf(t, resp.Kvs, 1, "want key value pair 'hoo', 'bar' but got %+v", resp.Kvs) require.Equalf(t, "hoo", string(resp.Kvs[0].Key), "want key value pair 'hoo', 'bar' but got %+v", resp.Kvs) require.Equalf(t, "bar", string(resp.Kvs[0].Value), "want key value pair 'hoo', 'bar' but got %+v", resp.Kvs) }) } func TestAuthGracefulDisable(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{}, []authUser{rootUser}), "failed to enable auth") donec := make(chan struct{}) rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) startedC := make(chan struct{}, 1) go func() { defer close(donec) defer close(startedC) // sleep a bit to let the watcher connects while auth is still enabled time.Sleep(time.Second) // now disable auth... if err := rootAuthClient.AuthDisable(ctx); err != nil { t.Errorf("failed to auth disable %v", err) return } // ...and restart the node clus.Members()[0].Stop() if err := clus.Members()[0].Start(ctx); err != nil { t.Errorf("failed to restart member %v", err) return } startedC <- struct{}{} // the watcher should still work after reconnecting _, err := rootAuthClient.Put(ctx, "key", "value", config.PutOptions{}) assert.NoErrorf(t, err, "failed to put key value") }() <-startedC wCtx, wCancel := context.WithCancel(ctx) defer wCancel() watchCh := rootAuthClient.Watch(wCtx, "key", config.WatchOptions{Revision: 1}) wantedLen := 1 watchTimeout := 10 * time.Second wanted := []testutils.KV{{Key: "key", Val: "value"}} kvs, err := testutils.KeyValuesFromWatchChan(watchCh, wantedLen, watchTimeout) require.NoErrorf(t, err, "failed to get key-values from watch channel %s", err) require.Equal(t, wanted, kvs) <-donec }) } func TestAuthStatus(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { resp, err := cc.AuthStatus(ctx) require.NoError(t, err) require.Falsef(t, resp.Enabled, "want auth not enabled but enabled") require.NoErrorf(t, setupAuth(cc, []authRole{}, []authUser{rootUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) resp, err = rootAuthClient.AuthStatus(ctx) require.NoError(t, err) require.Truef(t, resp.Enabled, "want enabled but got not enabled") }) } func TestAuthRoleUpdate(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) _, err = testUserAuthClient.Put(ctx, "hoo", "bar", config.PutOptions{}) require.ErrorContains(t, err, PermissionDenied) // grant a new key _, err = rootAuthClient.RoleGrantPermission(ctx, testRoleName, "hoo", "", clientv3.PermissionType(clientv3.PermReadWrite)) require.NoError(t, err) // try a newly granted key _, err = testUserAuthClient.Put(ctx, "hoo", "bar", config.PutOptions{}) require.NoError(t, err) // confirm put succeeded resp, err := testUserAuthClient.Get(ctx, "hoo", config.GetOptions{}) require.NoError(t, err) require.Lenf(t, resp.Kvs, 1, "want key value pair 'hoo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "hoo", string(resp.Kvs[0].Key), "want key value pair 'hoo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "bar", string(resp.Kvs[0].Value), "want key value pair 'hoo' 'bar' but got %+v", resp.Kvs) // revoke the newly granted key _, err = rootAuthClient.RoleRevokePermission(ctx, testRoleName, "hoo", "") require.NoError(t, err) // try put to the revoked key _, err = testUserAuthClient.Put(ctx, "hoo", "bar", config.PutOptions{}) require.ErrorContains(t, err, PermissionDenied) // confirm a key still granted can be accessed resp, err = testUserAuthClient.Get(ctx, "foo", config.GetOptions{}) require.NoError(t, err) require.Lenf(t, resp.Kvs, 1, "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "foo", string(resp.Kvs[0].Key), "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "bar", string(resp.Kvs[0].Value), "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) }) } func TestAuthUserDeleteDuringOps(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) // create a key _, err = testUserAuthClient.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) // confirm put succeeded resp, err := testUserAuthClient.Get(ctx, "foo", config.GetOptions{}) require.NoError(t, err) require.Lenf(t, resp.Kvs, 1, "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "foo", string(resp.Kvs[0].Key), "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "bar", string(resp.Kvs[0].Value), "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) // delete the user _, err = rootAuthClient.UserDelete(ctx, testUserName) require.NoError(t, err) // check the user is deleted _, err = testUserAuthClient.Put(ctx, "foo", "baz", config.PutOptions{}) require.ErrorContains(t, err, AuthenticationFailed) }) } func TestAuthRoleRevokeDuringOps(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) // create a key _, err = testUserAuthClient.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) // confirm put succeeded resp, err := testUserAuthClient.Get(ctx, "foo", config.GetOptions{}) require.NoError(t, err) require.Lenf(t, resp.Kvs, 1, "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "foo", string(resp.Kvs[0].Key), "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "bar", string(resp.Kvs[0].Value), "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) // create a new role _, err = rootAuthClient.RoleAdd(ctx, "test-role2") require.NoError(t, err) // grant a new key to the new role _, err = rootAuthClient.RoleGrantPermission(ctx, "test-role2", "hoo", "", clientv3.PermissionType(clientv3.PermReadWrite)) require.NoError(t, err) // grant the new role to the user _, err = rootAuthClient.UserGrantRole(ctx, testUserName, "test-role2") require.NoError(t, err) // try a newly granted key _, err = testUserAuthClient.Put(ctx, "hoo", "bar", config.PutOptions{}) require.NoError(t, err) // confirm put succeeded resp, err = testUserAuthClient.Get(ctx, "hoo", config.GetOptions{}) require.NoError(t, err) require.Lenf(t, resp.Kvs, 1, "want key value pair 'hoo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "hoo", string(resp.Kvs[0].Key), "want key value pair 'hoo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "bar", string(resp.Kvs[0].Value), "want key value pair 'hoo' 'bar' but got %+v", resp.Kvs) // revoke a role from the user _, err = rootAuthClient.UserRevokeRole(ctx, testUserName, testRoleName) require.NoError(t, err) // check the role is revoked and permission is lost from the user _, err = testUserAuthClient.Put(ctx, "foo", "baz", config.PutOptions{}) require.ErrorContains(t, err, PermissionDenied) // try a key that can be accessed from the remaining role _, err = testUserAuthClient.Put(ctx, "hoo", "bar2", config.PutOptions{}) require.NoError(t, err) // confirm put succeeded resp, err = testUserAuthClient.Get(ctx, "hoo", config.GetOptions{}) require.NoError(t, err) require.Lenf(t, resp.Kvs, 1, "want key value pair 'hoo' 'bar2' but got %+v", resp.Kvs) require.Equalf(t, "hoo", string(resp.Kvs[0].Key), "want key value pair 'hoo' 'bar2' but got %+v", resp.Kvs) require.Equalf(t, "bar2", string(resp.Kvs[0].Value), "want key value pair 'hoo' 'bar2' but got %+v", resp.Kvs) }) } func TestAuthWriteKey(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.Put(ctx, "foo", "a", config.PutOptions{}) require.NoError(t, err) require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) // confirm root role can access to all keys _, err = rootAuthClient.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) resp, err := rootAuthClient.Get(ctx, "foo", config.GetOptions{}) require.NoError(t, err) require.Lenf(t, resp.Kvs, 1, "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "foo", string(resp.Kvs[0].Key), "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) require.Equalf(t, "bar", string(resp.Kvs[0].Value), "want key value pair 'foo' 'bar' but got %+v", resp.Kvs) // try invalid user _, err = clus.Client(WithAuth("a", "b")) require.ErrorContains(t, err, AuthenticationFailed) // try good user _, err = testUserAuthClient.Put(ctx, "foo", "bar2", config.PutOptions{}) require.NoError(t, err) // confirm put succeeded resp, err = testUserAuthClient.Get(ctx, "foo", config.GetOptions{}) require.NoError(t, err) require.Lenf(t, resp.Kvs, 1, "want key value pair 'foo' 'bar2' but got %+v", resp.Kvs) require.Equalf(t, "foo", string(resp.Kvs[0].Key), "want key value pair 'foo' 'bar2' but got %+v", resp.Kvs) require.Equalf(t, "bar2", string(resp.Kvs[0].Value), "want key value pair 'foo' 'bar2' but got %+v", resp.Kvs) // try bad password _, err = clus.Client(WithAuth(testUserName, "badpass")) require.ErrorContains(t, err, AuthenticationFailed) }) } func TestAuthTxn(t *testing.T) { tcs := []struct { name string cfg config.ClusterConfig }{ { "NoJWT", config.ClusterConfig{ClusterSize: 1}, }, { "JWT", config.ClusterConfig{ClusterSize: 1, AuthToken: defaultAuthToken}, }, } reqs := []txnReq{ { compare: []string{`version("c2") = "1"`}, ifSuccess: []string{"get s2"}, ifFail: []string{"get f2"}, expectResults: []string{"SUCCESS", "s2", "v"}, expectError: false, }, // a key of compare case isn't granted { compare: []string{`version("c1") = "1"`}, ifSuccess: []string{"get s2"}, ifFail: []string{"get f2"}, expectResults: []string{PermissionDenied}, expectError: true, }, // a key of success case isn't granted { compare: []string{`version("c2") = "1"`}, ifSuccess: []string{"get s1"}, ifFail: []string{"get f2"}, expectResults: []string{PermissionDenied}, expectError: true, }, // a key of failure case isn't granted { compare: []string{`version("c2") = "1"`}, ifSuccess: []string{"get s2"}, ifFail: []string{"get f1"}, expectResults: []string{PermissionDenied}, expectError: true, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.cfg)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { // keys with 1 suffix aren't granted to test-user keys := []string{"c1", "s1", "f1"} // keys with 2 suffix are granted to test-user, see Line 399 grantedKeys := []string{"c2", "s2", "f2"} for _, key := range keys { _, err := cc.Put(ctx, key, "v", config.PutOptions{}) require.NoError(t, err) } for _, key := range grantedKeys { _, err := cc.Put(ctx, key, "v", config.PutOptions{}) require.NoError(t, err) } require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) // grant keys to test-user for _, key := range grantedKeys { _, err := rootAuthClient.RoleGrantPermission(ctx, testRoleName, key, "", clientv3.PermissionType(clientv3.PermReadWrite)) require.NoError(t, err) } for _, req := range reqs { resp, err := testUserAuthClient.Txn(ctx, req.compare, req.ifSuccess, req.ifFail, config.TxnOptions{ Interactive: true, }) if req.expectError { require.ErrorContains(t, err, req.expectResults[0]) } else { require.NoError(t, err) require.Equal(t, req.expectResults, getRespValues(resp)) } } }) }) } } func TestAuthPrefixPerm(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) prefix := "/prefix/" // directory like prefix // grant keys to test-user _, err := rootAuthClient.RoleGrantPermission(ctx, "test-role", prefix, clientv3.GetPrefixRangeEnd(prefix), clientv3.PermissionType(clientv3.PermReadWrite)) require.NoError(t, err) // try a prefix granted permission for i := 0; i < 10; i++ { key := fmt.Sprintf("%s%d", prefix, i) _, err = testUserAuthClient.Put(ctx, key, "val", config.PutOptions{}) require.NoError(t, err) } // expect put 'key with prefix end "/prefix0"' value failed _, err = testUserAuthClient.Put(ctx, clientv3.GetPrefixRangeEnd(prefix), "baz", config.PutOptions{}) require.ErrorContains(t, err, PermissionDenied) // grant the prefix2 keys to test-user prefix2 := "/prefix2/" _, err = rootAuthClient.RoleGrantPermission(ctx, "test-role", prefix2, clientv3.GetPrefixRangeEnd(prefix2), clientv3.PermissionType(clientv3.PermReadWrite)) require.NoError(t, err) for i := 0; i < 10; i++ { key := fmt.Sprintf("%s%d", prefix2, i) _, err = testUserAuthClient.Put(ctx, key, "val", config.PutOptions{}) require.NoError(t, err) } }) } func TestAuthLeaseKeepAlive(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{}, []authUser{rootUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) resp, err := rootAuthClient.Grant(ctx, 10) require.NoError(t, err) leaseID := resp.ID _, err = rootAuthClient.Put(ctx, "key", "value", config.PutOptions{LeaseID: leaseID}) require.NoError(t, err) _, err = rootAuthClient.KeepAliveOnce(ctx, leaseID) require.NoError(t, err) gresp, err := rootAuthClient.Get(ctx, "key", config.GetOptions{}) require.NoError(t, err) require.Lenf(t, gresp.Kvs, 1, "want kv pair ('key', 'value') but got %v", gresp.Kvs) require.Equalf(t, "key", string(gresp.Kvs[0].Key), "want kv pair ('key', 'value') but got %v", gresp.Kvs) require.Equalf(t, "value", string(gresp.Kvs[0].Value), "want kv pair ('key', 'value') but got %v", gresp.Kvs) }) } func TestAuthRevokeWithDelete(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) // create a new role newTestRoleName := "test-role2" _, err := rootAuthClient.RoleAdd(ctx, newTestRoleName) require.NoError(t, err) // grant the new role to the user _, err = rootAuthClient.UserGrantRole(ctx, testUserName, newTestRoleName) require.NoError(t, err) // check the result resp, err := rootAuthClient.UserGet(ctx, testUserName) require.NoError(t, err) require.ElementsMatch(t, resp.Roles, []string{testRoleName, newTestRoleName}) // delete the role, test-role2 must be revoked from test-user _, err = rootAuthClient.RoleDelete(ctx, newTestRoleName) require.NoError(t, err) // check the result resp, err = rootAuthClient.UserGet(ctx, testUserName) require.NoError(t, err) require.ElementsMatch(t, resp.Roles, []string{testRoleName}) }) } func TestAuthLeaseTimeToLiveExpired(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{}, []authUser{rootUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) resp, err := rootAuthClient.Grant(ctx, 2) require.NoError(t, err) leaseID := resp.ID _, err = rootAuthClient.Put(ctx, "key", "val", config.PutOptions{LeaseID: leaseID}) require.NoError(t, err) // eliminate false positive time.Sleep(3 * time.Second) tresp, err := rootAuthClient.TimeToLive(ctx, leaseID, config.LeaseOption{}) require.NoError(t, err) require.Equal(t, int64(-1), tresp.TTL) gresp, err := rootAuthClient.Get(ctx, "key", config.GetOptions{}) require.NoError(t, err) require.Empty(t, gresp.Kvs) }) } func TestAuthLeaseGrantLeases(t *testing.T) { testRunner.BeforeTest(t) tcs := []testCase{ { name: "NoJWT", config: config.ClusterConfig{ClusterSize: 1}, }, { name: "JWT", config: config.ClusterConfig{ClusterSize: 1, AuthToken: defaultAuthToken}, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) resp, err := rootAuthClient.Grant(ctx, 10) require.NoError(t, err) leaseID := resp.ID lresp, err := rootAuthClient.Leases(ctx) require.NoError(t, err) require.Lenf(t, lresp.Leases, 1, "want %v leaseID but got %v leases", leaseID, lresp.Leases) require.Equalf(t, lresp.Leases[0].ID, leaseID, "want %v leaseID but got %v leases", leaseID, lresp.Leases) anonAuthClient := testutils.MustClient(clus.Client()) _, err = anonAuthClient.Grant(ctx, 10) require.ErrorContains(t, err, "etcdserver: user name is empty") testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) _, err = testUserAuthClient.Grant(ctx, 10) require.NoError(t, err) }) }) } } func TestAuthMemberAdd(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) _, err := testUserAuthClient.MemberAdd(ctx, "newmember", []string{testPeerURL}) require.ErrorContains(t, err, PermissionDenied) _, err = rootAuthClient.MemberAdd(ctx, "newmember", []string{testPeerURL}) require.NoError(t, err) }) } func TestAuthCompact(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) _, err := rootAuthClient.Put(ctx, "key", "value", config.PutOptions{}) require.NoError(t, err) _, err = rootAuthClient.Put(ctx, "key", "value", config.PutOptions{}) require.NoError(t, err) _, err = testUserAuthClient.Compact(ctx, 1, config.CompactOption{Physical: true}) require.ErrorContains(t, err, PermissionDenied) _, err = rootAuthClient.Compact(ctx, 1, config.CompactOption{Physical: true}) require.NoError(t, err) }) } func TestAuthLeaseLeases(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) anonAuthClient := testutils.MustClient(clus.Client()) lresp, err := rootAuthClient.Grant(ctx, 90) require.NoError(t, err) firstLeaseID := lresp.ID _, err = rootAuthClient.Put(ctx, "foo", "value", config.PutOptions{LeaseID: firstLeaseID}) require.NoError(t, err) lresp, err = rootAuthClient.Grant(ctx, 90) require.NoError(t, err) secondLeaseID := lresp.ID _, err = rootAuthClient.Put(ctx, "foo1", "value", config.PutOptions{LeaseID: secondLeaseID}) require.NoError(t, err) _, err = testUserAuthClient.Leases(ctx) require.ErrorContains(t, err, PermissionDenied) _, err = anonAuthClient.Leases(ctx) require.ErrorContains(t, err, "etcdserver: user name is empty") resp, err := rootAuthClient.Leases(ctx) require.NoError(t, err) require.Lenf(t, resp.Leases, 2, "want 2 leases but got %v", resp.Leases) leaseIDs := []clientv3.LeaseID{firstLeaseID, secondLeaseID} for _, lease := range resp.Leases { require.Containsf(t, leaseIDs, lease.ID, "unexpected lease ID %v, want one of %v", lease.ID, leaseIDs) } _, err = rootAuthClient.Revoke(ctx, secondLeaseID) require.NoError(t, err) _, err = testUserAuthClient.Leases(ctx) require.NoError(t, err) }) } func TestAuthMemberList(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) _, err := testUserAuthClient.MemberList(ctx, false) require.ErrorContains(t, err, PermissionDenied) _, err = rootAuthClient.MemberList(ctx, false) require.NoError(t, err) }) } func TestAuthMemberRemove(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clusterSize := 3 clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: clusterSize})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { memberIDToEndpoints := getMemberIDToEndpoints(ctx, t, clus) require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) memberID, clusterID := memberToRemove(ctx, t, rootAuthClient, clusterSize) delete(memberIDToEndpoints, memberID) endpoints := make([]string, 0, len(memberIDToEndpoints)) for _, ep := range memberIDToEndpoints { endpoints = append(endpoints, ep) } testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) // ordinary user cannot remove a member _, err := testUserAuthClient.MemberRemove(ctx, memberID) require.ErrorContains(t, err, PermissionDenied) // root can remove a member, building a client excluding removed member endpoint rootAuthClient2 := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword), WithEndpoints(endpoints))) resp, err := rootAuthClient2.MemberRemove(ctx, memberID) require.NoError(t, err) require.Equal(t, resp.Header.ClusterId, clusterID) found := false for _, member := range resp.Members { if member.ID == memberID { found = true break } } require.Falsef(t, found, "expect removed member not found in member remove response") }) } func TestAuthTestInvalidMgmt(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{}, []authUser{rootUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) _, err := rootAuthClient.UserDelete(ctx, rootUserName) require.ErrorContains(t, err, InvalidAuthManagement) _, err = rootAuthClient.UserRevokeRole(ctx, rootUserName, rootRoleName) require.ErrorContains(t, err, InvalidAuthManagement) }) } func TestAuthLeaseRevoke(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) lresp, err := rootAuthClient.Grant(ctx, 10) require.NoError(t, err) _, err = rootAuthClient.Put(ctx, "key", "value", config.PutOptions{LeaseID: lresp.ID}) require.NoError(t, err) _, err = rootAuthClient.Revoke(ctx, lresp.ID) require.NoError(t, err) _, err = rootAuthClient.Get(ctx, "key", config.GetOptions{}) require.NoError(t, err) lresp, err = rootAuthClient.Grant(ctx, 10) require.NoError(t, err) annoAuthClient := testutils.MustClient(clus.Client()) _, err = annoAuthClient.Revoke(ctx, lresp.ID) require.ErrorContainsf(t, err, "etcdserver: user name is empty", "should fail to revoke lease with unauthenticated client") }) } func TestAuthRoleGet(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) resp, err := rootAuthClient.RoleGet(ctx, testRoleName) require.NoError(t, err) requireRolePermissionEqual(t, testRole, resp.Perm) // test-user can get the information of test-role because it belongs to the role resp, err = testUserAuthClient.RoleGet(ctx, testRoleName) require.NoError(t, err) requireRolePermissionEqual(t, testRole, resp.Perm) // test-user cannot get the information of root because it doesn't belong to the role _, err = testUserAuthClient.RoleGet(ctx, rootRoleName) require.ErrorContains(t, err, PermissionDenied) }) } func TestAuthUserGet(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) resp, err := rootAuthClient.UserGet(ctx, testUserName) require.NoError(t, err) requireUserRolesEqual(t, testUser, resp.Roles) // test-user can get the information of test-user itself resp, err = testUserAuthClient.UserGet(ctx, testUserName) require.NoError(t, err) requireUserRolesEqual(t, testUser, resp.Roles) // test-user cannot get the information of root _, err = testUserAuthClient.UserGet(ctx, rootUserName) require.ErrorContains(t, err, PermissionDenied) }) } func TestAuthRoleList(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) resp, err := rootAuthClient.RoleList(ctx) require.NoError(t, err) requireUserRolesEqual(t, testUser, resp.Roles) }) } func TestAuthJWTExpire(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1, AuthToken: defaultAuthToken})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) _, err := testUserAuthClient.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) // wait an expiration of my JWT token <-time.After(3 * tokenTTL) // e2e test will generate a new token while // integration test that re-uses the same etcd client will refresh the token on server failure. _, err = testUserAuthClient.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) }) } func TestAuthJWTOnly(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1, AuthToken: verifyJWTOnlyAuth})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { authRev, err := setupAuthAndGetRevision(cc, []authRole{testRole}, []authUser{rootUser, testUser}) require.NoErrorf(t, err, "failed to enable auth") token, err := createSignedJWT(defaultKeyPath, "RS256", testUserName, authRev) require.NoErrorf(t, err, "failed to create test user JWT") testUserAuthClient := testutils.MustClient(clus.Client(WithAuthToken(token))) _, err = testUserAuthClient.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) }) } // TestAuthRevisionConsistency ensures auth revision is the same after member restarts func TestAuthRevisionConsistency(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1, AuthToken: defaultAuthToken})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{}, []authUser{rootUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) // add user _, err := rootAuthClient.UserAdd(ctx, testUserName, testPassword, config.UserAddOptions{}) require.NoError(t, err) // delete the same user _, err = rootAuthClient.UserDelete(ctx, testUserName) require.NoError(t, err) // get node0 auth revision aresp, err := rootAuthClient.AuthStatus(ctx) require.NoError(t, err) oldAuthRevision := aresp.AuthRevision // restart the node clus.Members()[0].Stop() require.NoError(t, clus.Members()[0].Start(ctx)) aresp2, err := rootAuthClient.AuthStatus(ctx) require.NoError(t, err) newAuthRevision := aresp2.AuthRevision require.Equal(t, oldAuthRevision, newAuthRevision) }) } // TestAuthTestCacheReload ensures permissions are persisted and will be reloaded after member restarts func TestAuthTestCacheReload(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1, AuthToken: defaultAuthToken})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) // create foo since that is within the permission set // expectation is to succeed _, err := testUserAuthClient.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) // restart the node clus.Members()[0].Stop() require.NoError(t, clus.Members()[0].Start(ctx)) // nothing has changed, but it fails without refreshing cache after restart _, err = testUserAuthClient.Put(ctx, "foo", "bar2", config.PutOptions{}) require.NoError(t, err) }) } // TestAuthLeaseTimeToLive gated lease time to live with RBAC control func TestAuthLeaseTimeToLive(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1, AuthToken: defaultAuthToken})) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) anonAuthClient := testutils.MustClient(clus.Client()) gresp, err := testUserAuthClient.Grant(ctx, 10) require.NoError(t, err) leaseID := gresp.ID _, err = testUserAuthClient.Put(ctx, "foo", "bar", config.PutOptions{LeaseID: leaseID}) require.NoError(t, err) _, err = testUserAuthClient.TimeToLive(ctx, leaseID, config.LeaseOption{WithAttachedKeys: true}) require.NoError(t, err) _, err = anonAuthClient.TimeToLive(ctx, leaseID, config.LeaseOption{WithAttachedKeys: false}) require.ErrorContains(t, err, "etcdserver: user name is empty") _, err = anonAuthClient.TimeToLive(ctx, leaseID, config.LeaseOption{WithAttachedKeys: true}) require.ErrorContains(t, err, "etcdserver: user name is empty") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) _, err = rootAuthClient.Put(ctx, "bar", "foo", config.PutOptions{LeaseID: leaseID}) require.NoError(t, err) // the lease is attached to bar, which test-user cannot access _, err = testUserAuthClient.TimeToLive(ctx, leaseID, config.LeaseOption{WithAttachedKeys: true}) require.Errorf(t, err, "test-user must not be able to access to the lease, because it's attached to the key bar") // without --keys, access should be allowed _, err = testUserAuthClient.TimeToLive(ctx, leaseID, config.LeaseOption{WithAttachedKeys: false}) require.NoError(t, err) }) } func TestAuthAlarm(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ ClusterSize: 1, QuotaBackendBytes: 1024 * 5, })) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth") rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword))) testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword))) for i := 0; ; i++ { _, err := rootAuthClient.Put(ctx, testutil.PickKey(int64(i)), strings.Repeat("A", 1024), config.PutOptions{}) if err == nil { continue } require.ErrorContains(t, err, "etcdserver: mvcc: database space exceeded") break } _, err := testUserAuthClient.AlarmList(ctx) require.ErrorContains(t, err, PermissionDenied) memberID := uint64(0) for i := 0; i < 10; i++ { resp, rerr := rootAuthClient.AlarmList(ctx) require.NoError(t, rerr) if len(resp.Alarms) > 0 { memberID = resp.Header.MemberId break } time.Sleep(1 * time.Second) } require.NotEqualf(t, uint64(0), memberID, "expect to find alarm with non-zero member ID") _, err = testUserAuthClient.AlarmDisarm(ctx, &clientv3.AlarmMember{ MemberID: memberID, Alarm: pb.AlarmType_NOSPACE, }) require.ErrorContains(t, err, PermissionDenied) resp, err := rootAuthClient.AlarmDisarm(ctx, &clientv3.AlarmMember{ MemberID: memberID, Alarm: pb.AlarmType_NOSPACE, }) require.NoError(t, err) require.Lenf(t, resp.Alarms, 1, "expect 1 alarm from disarm but got %v", resp.Alarms) resp, err = rootAuthClient.AlarmList(ctx) require.NoError(t, err) require.Emptyf(t, resp.Alarms, "expect no alarm after disarm but got %v", resp.Alarms) }) } func mustAbsPath(path string) string { abs, err := filepath.Abs(path) if err != nil { panic(err) } return abs } ================================================ FILE: tests/common/auth_util.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "fmt" "os" "testing" "time" "github.com/golang-jwt/jwt/v5" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/authpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/interfaces" ) const ( rootUserName = "root" rootRoleName = "root" rootPassword = "rootPassword" testUserName = "test-user" testRoleName = "test-role" testPassword = "pass" ) var ( rootUser = authUser{user: rootUserName, pass: rootPassword, role: rootRoleName} testUser = authUser{user: testUserName, pass: testPassword, role: testRoleName} testRole = authRole{ role: testRoleName, permission: clientv3.PermissionType(clientv3.PermReadWrite), key: "foo", keyEnd: "", } ) type authRole struct { role string permission clientv3.PermissionType key string keyEnd string } type authUser struct { user string pass string role string } func createRoles(c interfaces.Client, roles []authRole) error { for _, r := range roles { // add role if _, err := c.RoleAdd(context.TODO(), r.role); err != nil { return fmt.Errorf("RoleAdd failed: %w", err) } // grant permission to role if _, err := c.RoleGrantPermission(context.TODO(), r.role, r.key, r.keyEnd, r.permission); err != nil { return fmt.Errorf("RoleGrantPermission failed: %w", err) } } return nil } func createUsers(c interfaces.Client, users []authUser) error { for _, u := range users { // add user if _, err := c.UserAdd(context.TODO(), u.user, u.pass, config.UserAddOptions{}); err != nil { return fmt.Errorf("UserAdd failed: %w", err) } // grant role to user if _, err := c.UserGrantRole(context.TODO(), u.user, u.role); err != nil { return fmt.Errorf("UserGrantRole failed: %w", err) } } return nil } func createSignedJWT(keyPath, alg, username string, authRevision uint64) (string, error) { signMethod := jwt.GetSigningMethod(alg) keyBytes, err := os.ReadFile(keyPath) if err != nil { return "", err } key, err := jwt.ParseRSAPrivateKeyFromPEM(keyBytes) if err != nil { return "", err } tk := jwt.NewWithClaims(signMethod, jwt.MapClaims{ "username": username, "revision": authRevision, "exp": time.Now().Add(time.Minute).Unix(), }) return tk.SignedString(key) } func setupAuth(c interfaces.Client, roles []authRole, users []authUser) error { // create roles if err := createRoles(c, roles); err != nil { return err } if err := createUsers(c, users); err != nil { return err } // enable auth return c.AuthEnable(context.TODO()) } func setupAuthAndGetRevision(c interfaces.Client, roles []authRole, users []authUser) (uint64, error) { // create roles if err := createRoles(c, roles); err != nil { return 0, err } if err := createUsers(c, users); err != nil { return 0, err } // This needs to happen before enabling auth for the TestAuthJWTOnly // test case because once auth is enabled we can no longer mint a valid // auth token without the revision, which we won't be able to obtain // without a valid auth token. authrev, err := c.AuthStatus(context.TODO()) if err != nil { return 0, err } // enable auth return authrev.AuthRevision, c.AuthEnable(context.TODO()) } func requireRolePermissionEqual(t *testing.T, expectRole authRole, actual []*authpb.Permission) { require.Len(t, actual, 1) require.Equal(t, expectRole.permission, clientv3.PermissionType(actual[0].PermType)) require.Equal(t, expectRole.key, string(actual[0].Key)) require.Equal(t, expectRole.keyEnd, string(actual[0].RangeEnd)) } func requireUserRolesEqual(t *testing.T, expectUser authUser, actual []string) { require.Len(t, actual, 1) require.Equal(t, expectUser.role, actual[0]) } ================================================ FILE: tests/common/compact_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestCompact(t *testing.T) { testRunner.BeforeTest(t) tcs := []struct { name string options config.CompactOption }{ { name: "NoPhysical", options: config.CompactOption{Physical: false, Timeout: 10 * time.Second}, }, { name: "Physical", options: config.CompactOption{Physical: true, Timeout: 10 * time.Second}, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { kvs := []testutils.KV{{Key: "key", Val: "val1"}, {Key: "key", Val: "val2"}, {Key: "key", Val: "val3"}} for i := range kvs { _, err := cc.Put(ctx, kvs[i].Key, kvs[i].Val, config.PutOptions{}) require.NoErrorf(t, err, "compactTest #%d: put kv error", i) } get, err := cc.Get(ctx, "key", config.GetOptions{Revision: 3}) require.NoErrorf(t, err, "compactTest: Get kv by revision error") getkvs := testutils.KeyValuesFromGetResponse(get) assert.Equal(t, kvs[1:2], getkvs) _, err = cc.Compact(ctx, 4, tc.options) require.NoErrorf(t, err, "compactTest: Compact error") _, err = cc.Get(ctx, "key", config.GetOptions{Revision: 3}) require.ErrorContainsf(t, err, "required revision has been compacted", "compactTest: Get compact key error (%v)", err) _, err = cc.Compact(ctx, 2, tc.options) require.ErrorContains(t, err, "required revision has been compacted") }) }) } } ================================================ FILE: tests/common/defrag_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestDefragOnline(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() options := config.DefragOption{Timeout: 10 * time.Second} clus := testRunner.NewCluster(ctx, t) cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { defer clus.Close() kvs := []testutils.KV{{Key: "key", Val: "val1"}, {Key: "key", Val: "val2"}, {Key: "key", Val: "val3"}} for i := range kvs { _, err := cc.Put(ctx, kvs[i].Key, kvs[i].Val, config.PutOptions{}) require.NoErrorf(t, err, "compactTest #%d: put kv error", i) } _, err := cc.Compact(ctx, 4, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) require.NoErrorf(t, err, "defrag_test: compact with revision error (%v)", err) require.NoErrorf(t, cc.Defragment(ctx, options), "defrag_test: defrag error (%v)", err) }) } ================================================ FILE: tests/common/e2e_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build e2e package common import ( "fmt" "os" "strconv" "time" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func init() { testRunner = framework.E2eTestRunner clusterTestCases = e2eClusterTestCases } const ( // minimalE2eEnabledEnvVarName is for reducing e2e test matrix, leading to faster CI runtimes in some cases(e.g., presubmits). // See https://github.com/etcd-io/etcd/issues/18983 for background. minimalE2eEnabledEnvVarName = "E2E_TEST_MINIMAL" ) func minimalE2eEnabled() bool { v, ok := os.LookupEnv(minimalE2eEnabledEnvVarName) if !ok { return false } parsed, err := strconv.ParseBool(v) if err != nil { fmt.Printf("Invalid %s value %q: %v\n", minimalE2eEnabledEnvVarName, v, err) return false } return parsed } func e2eClusterTestCases() []testCase { minimalTestCases := []testCase{ { name: "NoTLS", config: config.ClusterConfig{ClusterSize: 1}, }, { name: "PeerTLS and ClientTLS", config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.ManualTLS, ClientTLS: config.ManualTLS}, }, } if minimalE2eEnabled() { return minimalTestCases } tcs := append(minimalTestCases, testCase{ name: "PeerAutoTLS and ClientAutoTLS", config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.AutoTLS, ClientTLS: config.AutoTLS}, }, ) if fileutil.Exist(e2e.BinPath.EtcdLastRelease) { tcs = append(tcs, testCase{ name: "MinorityLastVersion", config: config.ClusterConfig{ ClusterSize: 3, ClusterContext: &e2e.ClusterContext{ Version: e2e.MinorityLastVersion, }, }, }, testCase{ name: "QuorumLastVersion", config: config.ClusterConfig{ ClusterSize: 3, ClusterContext: &e2e.ClusterContext{ Version: e2e.QuorumLastVersion, }, }, }) } return tcs } func WithAuth(userName, password string) config.ClientOption { return e2e.WithAuth(userName, password) } func WithAuthToken(token string) config.ClientOption { return e2e.WithAuthToken(token) } func WithEndpoints(endpoints []string) config.ClientOption { return e2e.WithEndpoints(endpoints) } func WithDialTimeout(tio time.Duration) config.ClientOption { return e2e.WithDialTimeout(tio) } func WithHTTP2Debug() config.ClusterOption { return e2e.WithHTTP2Debug() } func WithUnixClient() config.ClusterOption { return e2e.WithUnixClient() } func WithTCPClient() config.ClusterOption { return e2e.WithTCPClient() } ================================================ FILE: tests/common/endpoint_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestEndpointStatus(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.Status(ctx) require.NoErrorf(t, err, "get endpoint status error: %v", err) }) } func TestEndpointHashKV(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t) defer clus.Close() cc := testutils.MustClient(clus.Client()) t.Log("Add some entries") for i := 0; i < 10; i++ { key := fmt.Sprintf("key-%d", i) value := fmt.Sprintf("value-%d", i) _, err := cc.Put(ctx, key, value, config.PutOptions{}) require.NoErrorf(t, err, "count not put key %q", key) } t.Log("Check all members' Hash and HashRevision") require.Eventually(t, func() bool { resp, err := cc.HashKV(ctx, 0) require.NoErrorf(t, err, "failed to get endpoint hashkv") require.Len(t, resp, 3) if resp[0].HashRevision == resp[1].HashRevision && resp[0].HashRevision == resp[2].HashRevision { require.Equal(t, resp[0].Hash, resp[1].Hash) require.Equal(t, resp[0].Hash, resp[2].Hash) return true } t.Logf("HashRevisions are not equal: [%d, %d, %d], retry...", resp[0].HashRevision, resp[1].HashRevision, resp[2].HashRevision) return false }, 5*time.Second, 200*time.Millisecond) } func TestEndpointHealth(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { require.NoErrorf(t, cc.Health(ctx), "get endpoint health error") }) } ================================================ FILE: tests/common/grpc_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" intf "go.etcd.io/etcd/tests/v3/framework/interfaces" "go.etcd.io/etcd/tests/v3/framework/testutils" ) type clientSocket int const ( clientSocketTCP clientSocket = iota clientSocketUnix ) func TestAuthority(t *testing.T) { testRunner.BeforeTest(t) tcs := []struct { name string clientSocket clientSocket useTLS bool useInsecureTLS bool clientURLPattern string expectAuthorityPattern string }{ { name: "unix:path", clientSocket: clientSocketUnix, clientURLPattern: "unix:localhost:${MEMBER_PORT}", expectAuthorityPattern: "localhost:${MEMBER_PORT}", }, { name: "unix://absolute_path", clientSocket: clientSocketUnix, clientURLPattern: "unix://localhost:${MEMBER_PORT}", expectAuthorityPattern: "localhost:${MEMBER_PORT}", }, // "unixs" is not standard schema supported by etcd { name: "unixs:absolute_path", clientSocket: clientSocketUnix, useTLS: true, clientURLPattern: "unixs:localhost:${MEMBER_PORT}", expectAuthorityPattern: "localhost:${MEMBER_PORT}", }, { name: "unixs://absolute_path", clientSocket: clientSocketUnix, useTLS: true, clientURLPattern: "unixs://localhost:${MEMBER_PORT}", expectAuthorityPattern: "localhost:${MEMBER_PORT}", }, { name: "http://domain[:port]", clientSocket: clientSocketTCP, clientURLPattern: "http://localhost:${MEMBER_PORT}", expectAuthorityPattern: "localhost:${MEMBER_PORT}", }, { name: "http://address[:port]", clientSocket: clientSocketTCP, clientURLPattern: "http://127.0.0.1:${MEMBER_PORT}", expectAuthorityPattern: "127.0.0.1:${MEMBER_PORT}", }, { name: "https://domain[:port] insecure", clientSocket: clientSocketTCP, useTLS: true, useInsecureTLS: true, clientURLPattern: "https://localhost:${MEMBER_PORT}", expectAuthorityPattern: "localhost:${MEMBER_PORT}", }, { name: "https://address[:port] insecure", clientSocket: clientSocketTCP, useTLS: true, useInsecureTLS: true, clientURLPattern: "https://127.0.0.1:${MEMBER_PORT}", expectAuthorityPattern: "127.0.0.1:${MEMBER_PORT}", }, { name: "https://domain[:port]", clientSocket: clientSocketTCP, useTLS: true, clientURLPattern: "https://localhost:${MEMBER_PORT}", expectAuthorityPattern: "localhost:${MEMBER_PORT}", }, { name: "https://address[:port]", clientSocket: clientSocketTCP, useTLS: true, clientURLPattern: "https://127.0.0.1:${MEMBER_PORT}", expectAuthorityPattern: "127.0.0.1:${MEMBER_PORT}", }, } for _, tc := range tcs { for _, clusterSize := range []int{1, 3} { t.Run(fmt.Sprintf("Size: %d, Scenario: %q", clusterSize, tc.name), func(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() cfg := config.NewClusterConfig() cfg.ClusterSize = clusterSize switch { case tc.useInsecureTLS: cfg.ClientTLS = config.AutoTLS case tc.useTLS: cfg.ClientTLS = config.ManualTLS default: cfg.ClientTLS = config.NoTLS } opts := []config.ClusterOption{ config.WithClusterConfig(cfg), WithHTTP2Debug(), // enable http2 header logs only for e2e tests } switch tc.clientSocket { case clientSocketTCP: opts = append(opts, WithTCPClient()) case clientSocketUnix: opts = append(opts, WithUnixClient()) } clus := testRunner.NewCluster(ctx, t, opts...) defer clus.Close() tmpEndpoints, ok := clus.(intf.TemplateEndpoints) require.Truef(t, ok, "cluster does not implement TemplateEndpoints") endpoints := tmpEndpoints.TemplateEndpoints(t, tc.clientURLPattern) cc := testutils.MustClient(clus.Client(WithEndpoints(endpoints))) for i := 0; i < 100; i++ { _, err := cc.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) } testutils.ExecuteWithTimeout(t, 5*time.Second, func() { asserter, ok := clus.(intf.AssertAuthority) require.Truef(t, ok, "cluster does not implement AssertAuthority") asserter.AssertAuthority(t, tc.expectAuthorityPattern) }) }) } } } ================================================ FILE: tests/common/hashkv_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" intf "go.etcd.io/etcd/tests/v3/framework/interfaces" "go.etcd.io/etcd/tests/v3/framework/testutils" ) // TestVerifyHashKVAfterCompact tests that HashKV is consistent across all members after a physical compaction. // It tests both cases where the compaction is on a tombstone revision and where it is not. func TestVerifyHashKVAfterCompact(t *testing.T) { testRunner.BeforeTest(t) for _, keys := range [][]string{ {"key0"}, {"key0", "key1"}, } { for _, compactedOnTombstoneRev := range []bool{false, true} { t.Run(fmt.Sprintf("Keys=%v/CompactedOnTombstone=%v", keys, compactedOnTombstoneRev), func(t *testing.T) { for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { if tc.config.ClusterSize < 2 { t.Skip("Skipping test for single-member cluster") } ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) tombstoneRevs, latestRev := populateDataForHashKV(t, cc, keys) compactedOnRev := tombstoneRevs[0] // If compaction revision isn't a tombstone, select a revision in the middle of two tombstones. if !compactedOnTombstoneRev { require.Greater(t, len(tombstoneRevs), 1) compactedOnRev = (tombstoneRevs[0] + tombstoneRevs[1]) / 2 require.Greater(t, compactedOnRev, tombstoneRevs[0]) require.Greater(t, tombstoneRevs[1], compactedOnRev) } _, err := cc.Compact(ctx, compactedOnRev, config.CompactOption{Physical: true}) require.NoError(t, err) for rev := compactedOnRev; rev <= latestRev; rev++ { verifyConsistentHashKVAcrossAllMembers(t, cc, rev) } }) } }) } } } // TestVerifyHashKVAfterTwoCompactsOnTombstone tests that HashKV is consistent // across all members after two physical compactions on tombstone revisions. func TestVerifyHashKVAfterTwoCompactsOnTombstone(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { if tc.config.ClusterSize < 2 { t.Skip("Skipping test for single-member cluster") } ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) tombstoneRevs, latestRev := populateDataForHashKV(t, cc, []string{"key0"}) require.GreaterOrEqual(t, len(tombstoneRevs), 2) firstCompactOnRev := tombstoneRevs[0] t.Logf("COMPACT rev=%d", firstCompactOnRev) _, err := cc.Compact(ctx, firstCompactOnRev, config.CompactOption{Physical: true}) require.NoError(t, err) secondCompactOnRev := tombstoneRevs[1] t.Logf("COMPACT rev=%d", secondCompactOnRev) _, err = cc.Compact(ctx, secondCompactOnRev, config.CompactOption{Physical: true}) require.NoError(t, err) for rev := secondCompactOnRev; rev <= latestRev; rev++ { verifyConsistentHashKVAcrossAllMembers(t, cc, rev) } }) } } // TestVerifyHashKVAfterCompactOnLastTombstone tests that HashKV is consistent // across all members after a physical compaction on the last tombstone revision. func TestVerifyHashKVAfterCompactOnLastTombstone(t *testing.T) { testRunner.BeforeTest(t) for _, keys := range [][]string{ {"key0"}, {"key0", "key1"}, } { t.Run(fmt.Sprintf("Keys=%v", keys), func(t *testing.T) { for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { if tc.config.ClusterSize < 2 { t.Skip("Skipping test for single-member cluster") } ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) tombstoneRevs, latestRev := populateDataForHashKV(t, cc, keys) require.NotEmpty(t, tombstoneRevs) compactOnRev := tombstoneRevs[len(tombstoneRevs)-1] t.Logf("COMPACT rev=%d", compactOnRev) _, err := cc.Compact(ctx, compactOnRev, config.CompactOption{Physical: true}) require.NoError(t, err) for rev := compactOnRev; rev <= latestRev; rev++ { verifyConsistentHashKVAcrossAllMembers(t, cc, rev) } }) } }) } } // TestVerifyHashKVAfterCompactAndDefrag tests that HashKV is consistent // within a member before and after a physical compaction and defragmentation. func TestVerifyHashKVAfterCompactAndDefrag(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) tombstoneRevs, _ := populateDataForHashKV(t, cc, []string{"key0"}) require.NotEmpty(t, tombstoneRevs) compactOnRev := tombstoneRevs[0] t.Logf("COMPACT rev=%d", compactOnRev) before, err := cc.HashKV(ctx, compactOnRev) require.NoError(t, err) _, err = cc.Compact(ctx, compactOnRev, config.CompactOption{Physical: true}) require.NoError(t, err) err = cc.Defragment(ctx, config.DefragOption{}) require.NoError(t, err) after, err := cc.HashKV(ctx, compactOnRev) require.NoError(t, err) require.Len(t, before, len(after)) for i := range before { assert.Equal(t, before[i].Hash, after[i].Hash) } }) } } // populateDataForHashKV populates some sample data, and return a slice of tombstone // revisions and the latest revision. func populateDataForHashKV(t *testing.T, cc intf.Client, keys []string) ([]int64, int64) { t.Helper() ctx := t.Context() totalOperations := 40 var ( tombStoneRevs []int64 latestRev int64 ) deleteStep := 10 // submit a delete operation on every 10 operations for i := 1; i <= totalOperations; i++ { if i%deleteStep == 0 { t.Logf("Deleting key=%s", keys[0]) // Only delete the first key for simplicity resp, derr := cc.Delete(ctx, keys[0], config.DeleteOptions{}) require.NoError(t, derr) latestRev = resp.Header.Revision tombStoneRevs = append(tombStoneRevs, resp.Header.Revision) continue } value := fmt.Sprintf("%d", i) var ops []string for _, key := range keys { ops = append(ops, fmt.Sprintf("put %s %s", key, value)) } t.Logf("Writing keys: %v, value: %s", keys, value) resp, terr := cc.Txn(ctx, nil, ops, nil, config.TxnOptions{Interactive: true}) require.NoError(t, terr) require.True(t, resp.Succeeded) require.Len(t, resp.Responses, len(ops)) latestRev = resp.Header.Revision } return tombStoneRevs, latestRev } func verifyConsistentHashKVAcrossAllMembers(t *testing.T, cc intf.Client, hashKVOnRev int64) { t.Helper() ctx := t.Context() t.Logf("HashKV on rev=%d", hashKVOnRev) assert.Eventually(t, func() bool { resp, err := cc.HashKV(ctx, hashKVOnRev) if err != nil { t.Logf("HashKV failed: %v", err) return false } // Ensure that there are multiple members in the cluster. if len(resp) <= 1 { t.Logf("Expected multiple members, got %d", len(resp)) return false } // Check if all members have the same hash for i := 1; i < len(resp); i++ { if resp[i].Hash != resp[0].Hash { t.Logf("There is a chance that the physical compaction is not yet completed on the followers: Leader hash=%d, Follower hash=%d", resp[0].Hash, resp[i].Hash) return false } } return true }, 3*time.Second, 100*time.Millisecond) } ================================================ FILE: tests/common/integration_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build integration package common import ( "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/integration" ) func init() { testRunner = framework.IntegrationTestRunner clusterTestCases = integrationClusterTestCases } func integrationClusterTestCases() []testCase { return []testCase{ { name: "NoTLS", config: config.ClusterConfig{ClusterSize: 1}, }, { name: "PeerTLS and ClientTLS", config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.ManualTLS, ClientTLS: config.ManualTLS}, }, { name: "PeerAutoTLS and ClientAutoTLS", config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.AutoTLS, ClientTLS: config.AutoTLS}, }, } } func WithAuth(userName, password string) config.ClientOption { return integration.WithAuth(userName, password) } func WithAuthToken(token string) config.ClientOption { return integration.WithAuthToken(token) } func WithEndpoints(endpoints []string) config.ClientOption { return integration.WithEndpoints(endpoints) } func WithHTTP2Debug() config.ClusterOption { return integration.WithHTTP2Debug() } func WithUnixClient() config.ClusterOption { return integration.WithUnixClient() } func WithTCPClient() config.ClusterOption { return integration.WithTCPClient() } ================================================ FILE: tests/common/kv_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "fmt" "slices" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestKVPut(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { key, value := "foo", "bar" _, err := cc.Put(ctx, key, value, config.PutOptions{}) require.NoErrorf(t, err, "count not put key %q", key) resp, err := cc.Get(ctx, key, config.GetOptions{}) require.NoErrorf(t, err, "count not get key %q, err: %s", key, err) assert.Lenf(t, resp.Kvs, 1, "Unexpected length of response, got %d", len(resp.Kvs)) assert.Equalf(t, string(resp.Kvs[0].Key), key, "Unexpected key, want %q, got %q", key, resp.Kvs[0].Key) assert.Equalf(t, string(resp.Kvs[0].Value), value, "Unexpected value, want %q, got %q", value, resp.Kvs[0].Value) }) }) } } func TestKVGet(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { resp, err := cc.Get(ctx, "", config.GetOptions{Prefix: true}) require.NoError(t, err) firstRev := resp.Header.Revision kvA := createKV("a", "aa1", firstRev+1, firstRev+1, 1) kvB := createKV("b", "a", firstRev+2, firstRev+2, 1) kvCV1 := createKV("c", "ac1", firstRev+3, firstRev+3, 1) kvCV2 := createKV("c", "ac2", firstRev+3, firstRev+4, 2) kvC := createKV("c", "aac", firstRev+3, firstRev+5, 3) kvFoo := createKV("foo", "bar", firstRev+6, firstRev+6, 1) kvFooAbc := createKV("foo/abc", "0", firstRev+7, firstRev+7, 1) kvFop := createKV("fop", "s", firstRev+8, firstRev+8, 1) inputs := []*mvccpb.KeyValue{kvA, kvB, kvCV1, kvCV2, kvC, kvFoo, kvFooAbc, kvFop} for i := range inputs { _, putError := cc.Put(ctx, string(inputs[i].Key), string(inputs[i].Value), config.PutOptions{}) require.NoErrorf(t, putError, "count not put key value %q", inputs[i]) } allKvs := []*mvccpb.KeyValue{kvA, kvB, kvC, kvFoo, kvFooAbc, kvFop} kvsByVersion := []*mvccpb.KeyValue{kvA, kvB, kvFoo, kvFooAbc, kvFop, kvC} reversedKvs := []*mvccpb.KeyValue{kvFop, kvFooAbc, kvFoo, kvC, kvB, kvA} kvsByValue := []*mvccpb.KeyValue{kvFooAbc, kvB, kvA, kvC, kvFoo, kvFop} kvsByValueDesc := []*mvccpb.KeyValue{kvFop, kvFoo, kvC, kvA, kvB, kvFooAbc} currentResp, err := cc.Get(ctx, "", config.GetOptions{Prefix: true}) require.NoError(t, err) currentHeader := &etcdserverpb.ResponseHeader{ ClusterId: currentResp.Header.ClusterId, Revision: currentResp.Header.Revision, RaftTerm: currentResp.Header.RaftTerm, } type testcase struct { name string begin string options config.GetOptions wantResponse *clientv3.GetResponse } tests := []testcase{ {name: "Get one specific key (a)", begin: "a", wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 1, Kvs: []*mvccpb.KeyValue{kvA}}}, {name: "Get one specific key (a), serializable", begin: "a", options: config.GetOptions{Serializable: true}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 1, Kvs: []*mvccpb.KeyValue{kvA}}}, {name: "Get [a, c)", begin: "a", options: config.GetOptions{End: "c"}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 2, Kvs: []*mvccpb.KeyValue{kvA, kvB}}}, {name: "blank key with --prefix option -> all KVs", begin: "", options: config.GetOptions{Prefix: true}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: allKvs}}, {name: "blank key with --from-key option -> all KVs", begin: "", options: config.GetOptions{FromKey: true}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: allKvs}}, {name: "Range covering all keys -> all KVs", begin: "a", options: config.GetOptions{End: "x"}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: allKvs}}, {name: "blank key with --prefix and revision -> [first key, entry at specified revision]", begin: "", options: config.GetOptions{Prefix: true, Revision: int(firstRev + 3)}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 3, Kvs: []*mvccpb.KeyValue{kvA, kvB, kvCV1}}}, {name: "--count-only for one single key", begin: "a", options: config.GetOptions{CountOnly: true}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 1, Kvs: nil}}, {name: "--prefix of foo -> all entries with the prefix", begin: "foo", options: config.GetOptions{Prefix: true}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 2, Kvs: []*mvccpb.KeyValue{kvFoo, kvFooAbc}}}, {name: "--from-key of 'foo' -> [", begin: "foo", options: config.GetOptions{FromKey: true}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 3, Kvs: []*mvccpb.KeyValue{kvFoo, kvFooAbc, kvFop}}}, {name: "blank key with limit set", begin: "", options: config.GetOptions{Prefix: true, Limit: 2}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: []*mvccpb.KeyValue{kvA, kvB}, More: true}}, {name: "all kvs ordered by mod revision ascending", begin: "", options: config.GetOptions{Prefix: true, Order: clientv3.SortAscend, SortBy: clientv3.SortByModRevision}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: allKvs}}, {name: "all KVs ordered by version ascending", begin: "", options: config.GetOptions{Prefix: true, Order: clientv3.SortAscend, SortBy: clientv3.SortByVersion}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: kvsByVersion}}, {name: "all KVs ordered by create revision, unspecified sort order", begin: "", options: config.GetOptions{Prefix: true, Order: clientv3.SortNone, SortBy: clientv3.SortByCreateRevision}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: allKvs}}, {name: "all KVs ordered by create revision descending", begin: "", options: config.GetOptions{Prefix: true, Order: clientv3.SortDescend, SortBy: clientv3.SortByCreateRevision}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: reversedKvs}}, {name: "all KVs ordered by key descending", begin: "", options: config.GetOptions{Prefix: true, Order: clientv3.SortDescend, SortBy: clientv3.SortByKey}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: reversedKvs}}, {name: "all KVs ordered by value, unspecified sort order", begin: "", options: config.GetOptions{Prefix: true, Order: clientv3.SortNone, SortBy: clientv3.SortByValue}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: kvsByValue}}, {name: "all KVs ordered by value, ascending", begin: "", options: config.GetOptions{Prefix: true, Order: clientv3.SortAscend, SortBy: clientv3.SortByValue}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: kvsByValue}}, {name: "all KVs ordered by value descending", begin: "", options: config.GetOptions{Prefix: true, Order: clientv3.SortDescend, SortBy: clientv3.SortByValue}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: kvsByValueDesc}}, {name: "all KVs descending", begin: "", options: config.GetOptions{Prefix: true, Order: clientv3.SortDescend}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: reversedKvs}}, {name: "Get first version of 'c' by its revision", begin: "c", options: config.GetOptions{Revision: int(firstRev) + 3}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 1, Kvs: []*mvccpb.KeyValue{kvCV1}}}, {name: "Get second version of 'c' by its revision", begin: "c", options: config.GetOptions{Revision: int(firstRev) + 4}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 1, Kvs: []*mvccpb.KeyValue{kvCV2}}}, {name: "Get third version of 'c' by its revision", begin: "c", options: config.GetOptions{Revision: int(firstRev) + 5}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 1, Kvs: []*mvccpb.KeyValue{kvC}}}, {name: "Get the latest version of 'c'", begin: "c", wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 1, Kvs: []*mvccpb.KeyValue{kvC}}}, {name: "all KVs with mininum mod revision sorted by mod revision", begin: "", options: config.GetOptions{Prefix: true, MinModRevision: int(firstRev) + 3, SortBy: clientv3.SortByModRevision}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: []*mvccpb.KeyValue{kvC, kvFoo, kvFooAbc, kvFop}}}, {name: "all KVs with maximum mod revision, sorted by key descending", begin: "", options: config.GetOptions{Prefix: true, MaxModRevision: int(firstRev) + 4, Order: clientv3.SortDescend, SortBy: clientv3.SortByKey}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: []*mvccpb.KeyValue{kvB, kvA}}}, {name: "all KVs with minimum create revision, sorted by version, descending", begin: "", options: config.GetOptions{Prefix: true, MinCreateRevision: int(firstRev) + 3, Order: clientv3.SortDescend, SortBy: clientv3.SortByVersion}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: []*mvccpb.KeyValue{kvC, kvFoo, kvFooAbc, kvFop}}}, {name: "all KVs with maximimum create revision, sorted by value", begin: "", options: config.GetOptions{Prefix: true, MaxCreateRevision: int(firstRev) + 6, Order: clientv3.SortDescend, SortBy: clientv3.SortByValue}, wantResponse: &clientv3.GetResponse{Header: currentHeader, Count: 6, Kvs: []*mvccpb.KeyValue{kvFoo, kvC, kvA, kvB}}}, } testsWithKeysOnly := make([]testcase, 0, len(tests)) for _, otc := range tests { if otc.options.CountOnly { continue // can't use both --count-only and --keys-only at the same time } withKeysOnly := otc withKeysOnly.name = fmt.Sprintf("%s --keys-only", withKeysOnly.name) withKeysOnly.options.KeysOnly = true wantResponse := *otc.wantResponse wantResponse.Kvs = dropValue(withKeysOnly.wantResponse.Kvs) withKeysOnly.wantResponse = &wantResponse testsWithKeysOnly = append(testsWithKeysOnly, withKeysOnly) } for _, tt := range slices.Concat(tests, testsWithKeysOnly) { t.Run(tt.name, func(t *testing.T) { resp, err := cc.Get(ctx, tt.begin, tt.options) require.NoErrorf(t, err, "count not get key %q, err: %s", tt.begin, err) resp.Header.MemberId = 0 assert.Equal(t, tt.wantResponse, resp) }) } }) }) } } func createKV(key, val string, createRev, modRev, ver int64) *mvccpb.KeyValue { return &mvccpb.KeyValue{ Key: []byte(key), Value: []byte(val), CreateRevision: createRev, ModRevision: modRev, Version: ver, } } func dropValue(s []*mvccpb.KeyValue) []*mvccpb.KeyValue { ss := make([]*mvccpb.KeyValue, 0, len(s)) for _, kv := range s { clone := *kv clone.Value = nil ss = append(ss, &clone) } return ss } func TestKVDelete(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { kvs := []string{"a", "b", "c", "c/abc", "d"} tests := []struct { deleteKey string options config.DeleteOptions wantDeleted int wantKeys []string }{ { // delete all keys deleteKey: "", options: config.DeleteOptions{Prefix: true}, wantDeleted: 5, }, { // delete all keys deleteKey: "", options: config.DeleteOptions{FromKey: true}, wantDeleted: 5, }, { deleteKey: "a", options: config.DeleteOptions{End: "c"}, wantDeleted: 2, wantKeys: []string{"c", "c/abc", "d"}, }, { deleteKey: "c", wantDeleted: 1, wantKeys: []string{"a", "b", "c/abc", "d"}, }, { deleteKey: "c", options: config.DeleteOptions{Prefix: true}, wantDeleted: 2, wantKeys: []string{"a", "b", "d"}, }, { deleteKey: "c", options: config.DeleteOptions{FromKey: true}, wantDeleted: 3, wantKeys: []string{"a", "b"}, }, { deleteKey: "e", wantDeleted: 0, wantKeys: kvs, }, } for _, tt := range tests { for i := range kvs { _, err := cc.Put(ctx, kvs[i], "bar", config.PutOptions{}) require.NoErrorf(t, err, "count not put key %q", kvs[i]) } del, err := cc.Delete(ctx, tt.deleteKey, tt.options) require.NoErrorf(t, err, "count not get key %q, err", tt.deleteKey) assert.Equal(t, tt.wantDeleted, int(del.Deleted)) get, err := cc.Get(ctx, "", config.GetOptions{Prefix: true}) require.NoErrorf(t, err, "count not get key") kvs := testutils.KeysFromGetResponse(get) assert.Equal(t, tt.wantKeys, kvs) } }) }) } } func TestKVGetNoQuorum(t *testing.T) { testRunner.BeforeTest(t) tcs := []struct { name string options config.GetOptions wantError bool }{ { name: "Serializable", options: config.GetOptions{Serializable: true}, }, { name: "Linearizable", options: config.GetOptions{Serializable: false, Timeout: time.Second}, wantError: true, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t) defer clus.Close() clus.Members()[0].Stop() clus.Members()[1].Stop() cc := clus.Members()[2].Client() testutils.ExecuteUntil(ctx, t, func() { key := "foo" _, err := cc.Get(ctx, key, tc.options) if tc.wantError { require.Error(t, err) } else { require.NoError(t, err) } }) }) } } ================================================ FILE: tests/common/lease_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "testing" "time" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestLeaseGrantTimeToLive(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { ttl := int64(10) leaseResp, err := cc.Grant(ctx, ttl) require.NoError(t, err) ttlResp, err := cc.TimeToLive(ctx, leaseResp.ID, config.LeaseOption{}) require.NoError(t, err) require.Equal(t, ttl, ttlResp.GrantedTTL) }) }) } } func TestLeaseGrantAndList(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { nestedCases := []struct { name string leaseCount int }{ { name: "no_leases", leaseCount: 0, }, { name: "one_lease", leaseCount: 1, }, { name: "many_leases", leaseCount: 3, }, } for _, nc := range nestedCases { t.Run(tc.name+"/"+nc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() t.Logf("Creating cluster...") clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) t.Logf("Created cluster and client") testutils.ExecuteUntil(ctx, t, func() { var createdLeases []clientv3.LeaseID for i := 0; i < nc.leaseCount; i++ { leaseResp, err := cc.Grant(ctx, 10) t.Logf("Grant returned: resp:%s err:%v", leaseResp.String(), err) require.NoError(t, err) createdLeases = append(createdLeases, leaseResp.ID) } // Because we're not guaranteed to talk to the same member, wait for // listing to eventually return true, either by the result propagating // or by hitting an up to date member. var leases []clientv3.LeaseStatus require.Eventually(t, func() bool { resp, err := cc.Leases(ctx) if err != nil { return false } leases = resp.Leases // TODO: update this to use last Revision from leaseResp // after https://github.com/etcd-io/etcd/issues/13989 is fixed return len(leases) == len(createdLeases) }, 2*time.Second, 10*time.Millisecond) returnedLeases := make([]clientv3.LeaseID, 0, nc.leaseCount) for _, status := range leases { returnedLeases = append(returnedLeases, status.ID) } require.ElementsMatch(t, createdLeases, returnedLeases) }) }) } } } func TestLeaseGrantTimeToLiveExpired(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { leaseResp, err := cc.Grant(ctx, 2) require.NoError(t, err) _, err = cc.Put(ctx, "foo", "bar", config.PutOptions{LeaseID: leaseResp.ID}) require.NoError(t, err) getResp, err := cc.Get(ctx, "foo", config.GetOptions{}) require.NoError(t, err) require.Equal(t, int64(1), getResp.Count) // FIXME: When leader changes, old leader steps // back to follower and ignores the lease revoking. // The new leader will restart TTL counting. If so, // we should call time.Sleep again and wait for revoking. // It can't avoid flakey but reduce flakey possibility. for i := 0; i < 3; i++ { currentLeader := clus.WaitLeader(t) t.Logf("[%d] current leader index %d", i, currentLeader) time.Sleep(3 * time.Second) newLeader := clus.WaitLeader(t) if newLeader == currentLeader { break } t.Logf("[%d] leader changed, new leader index %d", i, newLeader) } ttlResp, err := cc.TimeToLive(ctx, leaseResp.ID, config.LeaseOption{}) require.NoError(t, err) require.Equal(t, int64(-1), ttlResp.TTL) getResp, err = cc.Get(ctx, "foo", config.GetOptions{}) require.NoError(t, err) // Value should expire with the lease require.Equal(t, int64(0), getResp.Count) }) }) } } func TestLeaseGrantKeepAliveOnce(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { leaseResp, err := cc.Grant(ctx, 2) require.NoError(t, err) _, err = cc.KeepAliveOnce(ctx, leaseResp.ID) require.NoError(t, err) // FIXME: When leader changes, old leader steps // back to follower and ignores the lease revoking. // The new leader will restart TTL counting. If so, // we should call time.Sleep again and wait for revoking. // It can't avoid flakey but reduce flakey possibility. for i := 0; i < 3; i++ { currentLeader := clus.WaitLeader(t) t.Logf("[%d] current leader index %d", i, currentLeader) time.Sleep(2 * time.Second) newLeader := clus.WaitLeader(t) if newLeader == currentLeader { break } t.Logf("[%d] leader changed, new leader index %d", i, newLeader) } ttlResp, err := cc.TimeToLive(ctx, leaseResp.ID, config.LeaseOption{}) require.NoError(t, err) // We still have a lease! require.Greater(t, int64(2), ttlResp.TTL) }) }) } } func TestLeaseGrantRevoke(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { leaseResp, err := cc.Grant(ctx, 20) require.NoError(t, err) _, err = cc.Put(ctx, "foo", "bar", config.PutOptions{LeaseID: leaseResp.ID}) require.NoError(t, err) getResp, err := cc.Get(ctx, "foo", config.GetOptions{}) require.NoError(t, err) require.Equal(t, int64(1), getResp.Count) _, err = cc.Revoke(ctx, leaseResp.ID) require.NoError(t, err) ttlResp, err := cc.TimeToLive(ctx, leaseResp.ID, config.LeaseOption{}) require.NoError(t, err) require.Equal(t, int64(-1), ttlResp.TTL) getResp, err = cc.Get(ctx, "foo", config.GetOptions{}) require.NoError(t, err) // Value should expire with the lease require.Equal(t, int64(0), getResp.Count) }) }) } } ================================================ FILE: tests/common/main_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "testing" "go.etcd.io/etcd/tests/v3/framework/config" intf "go.etcd.io/etcd/tests/v3/framework/interfaces" ) var ( testRunner intf.TestRunner clusterTestCases func() []testCase ) func TestMain(m *testing.M) { testRunner.TestMain(m) } type testCase struct { name string config config.ClusterConfig } ================================================ FILE: tests/common/maintenance_auth_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "testing" "time" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" intf "go.etcd.io/etcd/tests/v3/framework/interfaces" "go.etcd.io/etcd/tests/v3/framework/testutils" ) /* Test Defragment */ func TestDefragmentWithNoAuth(t *testing.T) { testDefragmentWithAuth(t, false, true) } func TestDefragmentWithInvalidAuth(t *testing.T) { testDefragmentWithAuth(t, true, true, WithAuth("invalid", "invalid")) } func TestDefragmentWithRootAuth(t *testing.T) { testDefragmentWithAuth(t, false, false, WithAuth("root", "rootPass")) } func TestDefragmentWithUserAuth(t *testing.T) { testDefragmentWithAuth(t, false, true, WithAuth("user0", "user0Pass")) } func testDefragmentWithAuth(t *testing.T, expectConnectionError, expectOperationError bool, opts ...config.ClientOption) { testMaintenanceOperationWithAuth(t, expectConnectionError, expectOperationError, func(ctx context.Context, cc intf.Client) error { return cc.Defragment(ctx, config.DefragOption{Timeout: 10 * time.Second}) }, opts...) } /* Test Downgrade */ func TestDowngradeWithNoAuth(t *testing.T) { testDowngradeWithAuth(t, false, true) } func TestDowngradeWithInvalidAuth(t *testing.T) { testDowngradeWithAuth(t, true, true, WithAuth("invalid", "invalid")) } func TestDowngradeWithRootAuth(t *testing.T) { testDowngradeWithAuth(t, false, false, WithAuth("root", "rootPass")) } func TestDowngradeWithUserAuth(t *testing.T) { testDowngradeWithAuth(t, false, true, WithAuth("user0", "user0Pass")) } func testDowngradeWithAuth(t *testing.T, _expectConnectionError, _expectOperationError bool, _opts ...config.ClientOption) { // TODO(ahrtr): finish this after we added interface methods `Downgrade` into `Client` t.Skip() } /* Test HashKV */ func TestHashKVWithNoAuth(t *testing.T) { testHashKVWithAuth(t, false, true) } func TestHashKVWithInvalidAuth(t *testing.T) { testHashKVWithAuth(t, true, true, WithAuth("invalid", "invalid")) } func TestHashKVWithRootAuth(t *testing.T) { testHashKVWithAuth(t, false, false, WithAuth("root", "rootPass")) } func TestHashKVWithUserAuth(t *testing.T) { testHashKVWithAuth(t, false, true, WithAuth("user0", "user0Pass")) } func testHashKVWithAuth(t *testing.T, expectConnectionError, expectOperationError bool, opts ...config.ClientOption) { testMaintenanceOperationWithAuth(t, expectConnectionError, expectOperationError, func(ctx context.Context, cc intf.Client) error { _, err := cc.HashKV(ctx, 0) return err }, opts...) } /* Test MoveLeader */ func TestMoveLeaderWithNoAuth(t *testing.T) { testMoveLeaderWithAuth(t, false, true) } func TestMoveLeaderWithInvalidAuth(t *testing.T) { testMoveLeaderWithAuth(t, true, true, WithAuth("invalid", "invalid")) } func TestMoveLeaderWithRootAuth(t *testing.T) { testMoveLeaderWithAuth(t, false, false, WithAuth("root", "rootPass")) } func TestMoveLeaderWithUserAuth(t *testing.T) { testMoveLeaderWithAuth(t, false, true, WithAuth("user0", "user0Pass")) } func testMoveLeaderWithAuth(t *testing.T, _expectConnectionError, _expectOperationError bool, _opts ...config.ClientOption) { // TODO(ahrtr): finish this after we added interface methods `MoveLeader` into `Client` t.Skip() } /* Test Snapshot */ func TestSnapshotWithNoAuth(t *testing.T) { testSnapshotWithAuth(t, false, true) } func TestSnapshotWithInvalidAuth(t *testing.T) { testSnapshotWithAuth(t, true, true, WithAuth("invalid", "invalid")) } func TestSnapshotWithRootAuth(t *testing.T) { testSnapshotWithAuth(t, false, false, WithAuth("root", "rootPass")) } func TestSnapshotWithUserAuth(t *testing.T) { testSnapshotWithAuth(t, false, true, WithAuth("user0", "user0Pass")) } func testSnapshotWithAuth(t *testing.T, _expectConnectionError, _expectOperationError bool, _opts ...config.ClientOption) { // TODO(ahrtr): finish this after we added interface methods `Snapshot` into `Client` t.Skip() } /* Test Status */ func TestStatusWithNoAuth(t *testing.T) { testStatusWithAuth(t, false, true) } func TestStatusWithInvalidAuth(t *testing.T) { testStatusWithAuth(t, true, true, WithAuth("invalid", "invalid")) } func TestStatusWithRootAuth(t *testing.T) { testStatusWithAuth(t, false, false, WithAuth("root", "rootPass")) } func TestStatusWithUserAuth(t *testing.T) { testStatusWithAuth(t, false, true, WithAuth("user0", "user0Pass")) } func testStatusWithAuth(t *testing.T, expectConnectionError, expectOperationError bool, opts ...config.ClientOption) { testMaintenanceOperationWithAuth(t, expectConnectionError, expectOperationError, func(ctx context.Context, cc intf.Client) error { _, err := cc.Status(ctx) return err }, opts...) } func setupAuthForMaintenanceTest(c intf.Client) error { roles := []authRole{ { role: "role0", permission: clientv3.PermissionType(clientv3.PermReadWrite), key: "foo", }, } users := []authUser{ { user: "root", pass: "rootPass", role: "root", }, { user: "user0", pass: "user0Pass", role: "role0", }, } return setupAuth(c, roles, users) } func testMaintenanceOperationWithAuth(t *testing.T, expectConnectError, expectOperationError bool, f func(context.Context, intf.Client) error, opts ...config.ClientOption) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t) defer clus.Close() cc := testutils.MustClient(clus.Client()) err := setupAuthForMaintenanceTest(cc) require.NoError(t, err) ccWithAuth, err := clus.Client(opts...) if expectConnectError { require.Errorf(t, err, "%s: expected connection error, but got successful response", t.Name()) t.Logf("%s: connection error: %v", t.Name(), err) return } if err != nil { require.NoErrorf(t, err, "%s: unexpected connection error", t.Name()) return } // sleep 1 second to wait for etcd cluster to finish the authentication process. // TODO(ahrtr): find a better way to do it. time.Sleep(1 * time.Second) testutils.ExecuteUntil(ctx, t, func() { err := f(ctx, ccWithAuth) if expectOperationError { require.Errorf(t, err, "%s: expected error, but got successful response", t.Name()) t.Logf("%s: operation error: %v", t.Name(), err) return } require.NoErrorf(t, err, "%s: unexpected operation error", t.Name()) }) } ================================================ FILE: tests/common/member_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/config" intf "go.etcd.io/etcd/tests/v3/framework/interfaces" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestMemberList(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { resp, err := cc.MemberList(ctx, false) require.NoErrorf(t, err, "could not get member list") expectNum := len(clus.Members()) require.Lenf(t, resp.Members, expectNum, "unexpected number of members") assert.Eventually(t, func() bool { resp, err := cc.MemberList(ctx, false) if err != nil { t.Logf("Failed to get member list, err: %v", err) return false } for _, m := range resp.Members { if len(m.ClientURLs) == 0 { t.Logf("member is not started, memberID:%d, memberName:%s", m.ID, m.Name) return false } } return true }, time.Second*5, time.Millisecond*100) }) }) } } func TestMemberAdd(t *testing.T) { testRunner.BeforeTest(t) learnerTcs := []struct { name string learner bool }{ { name: "NotLearner", learner: false, }, { name: "Learner", learner: true, }, } quorumTcs := []struct { name string strictReconfigCheck bool waitForQuorum bool expectError bool }{ { name: "StrictReconfigCheck/WaitForQuorum", strictReconfigCheck: true, waitForQuorum: true, }, { name: "StrictReconfigCheck/NoWaitForQuorum", strictReconfigCheck: true, expectError: true, }, { name: "DisableStrictReconfigCheck/WaitForQuorum", waitForQuorum: true, }, { name: "DisableStrictReconfigCheck/NoWaitForQuorum", }, } for _, learnerTc := range learnerTcs { for _, quorumTc := range quorumTcs { for _, clusterTc := range clusterTestCases() { t.Run(learnerTc.name+"/"+quorumTc.name+"/"+clusterTc.name, func(t *testing.T) { ctxTimeout := 10 * time.Second if quorumTc.waitForQuorum { ctxTimeout += etcdserver.HealthInterval } ctx, cancel := context.WithTimeout(t.Context(), ctxTimeout) defer cancel() c := clusterTc.config c.StrictReconfigCheck = quorumTc.strictReconfigCheck clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(c)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { var addResp *clientv3.MemberAddResponse var err error if quorumTc.waitForQuorum { time.Sleep(etcdserver.HealthInterval) } if learnerTc.learner { addResp, err = cc.MemberAddAsLearner(ctx, "newmember", []string{"http://localhost:123"}) } else { addResp, err = cc.MemberAdd(ctx, "newmember", []string{"http://localhost:123"}) } if quorumTc.expectError && c.ClusterSize > 1 { // calling MemberAdd/MemberAddAsLearner on a single node will not fail, // whether strictReconfigCheck or whether waitForQuorum require.ErrorContains(t, err, "etcdserver: unhealthy cluster") } else { require.NoErrorf(t, err, "MemberAdd failed") require.NotNilf(t, addResp.Member, "MemberAdd failed, expected: member != nil, got: member == nil") require.NotZerof(t, addResp.Member.ID, "MemberAdd failed, expected: ID != 0, got: ID == 0") require.NotEmptyf(t, addResp.Member.PeerURLs, "MemberAdd failed, expected: non-empty PeerURLs, got: empty PeerURLs") } }) }) } } } } func TestMemberRemove(t *testing.T) { testRunner.BeforeTest(t) tcs := []struct { name string strictReconfigCheck bool waitForQuorum bool expectSingleNodeError bool expectClusterError bool }{ { name: "StrictReconfigCheck/WaitForQuorum", strictReconfigCheck: true, waitForQuorum: true, expectSingleNodeError: true, }, { name: "StrictReconfigCheck/NoWaitForQuorum", strictReconfigCheck: true, expectSingleNodeError: true, expectClusterError: true, }, { name: "DisableStrictReconfigCheck/WaitForQuorum", waitForQuorum: true, }, { name: "DisableStrictReconfigCheck/NoWaitForQuorum", }, } for _, quorumTc := range tcs { for _, clusterTc := range clusterTestCases() { if !quorumTc.strictReconfigCheck && clusterTc.config.ClusterSize == 1 { // skip these test cases // when strictReconfigCheck is disabled, calling MemberRemove will cause the single node to panic continue } t.Run(quorumTc.name+"/"+clusterTc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 14*time.Second) defer cancel() c := clusterTc.config c.StrictReconfigCheck = quorumTc.strictReconfigCheck clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(c)) defer clus.Close() // client connects to a specific member which won't be removed from cluster cc := clus.Members()[0].Client() testutils.ExecuteUntil(ctx, t, func() { if quorumTc.waitForQuorum { // wait for health interval + leader election time.Sleep(etcdserver.HealthInterval + 2*time.Second) } memberID, clusterID := memberToRemove(ctx, t, cc, c.ClusterSize) removeResp, err := cc.MemberRemove(ctx, memberID) if c.ClusterSize == 1 && quorumTc.expectSingleNodeError { require.ErrorContains(t, err, "etcdserver: re-configuration failed due to not enough started members") return } if c.ClusterSize > 1 && quorumTc.expectClusterError { require.ErrorContains(t, err, "etcdserver: unhealthy cluster") return } require.NoErrorf(t, err, "MemberRemove failed") t.Logf("removeResp.Members:%v", removeResp.Members) require.Equalf(t, removeResp.Header.ClusterId, clusterID, "MemberRemove failed, expected ClusterID: %d, got: %d", clusterID, removeResp.Header.ClusterId) require.Lenf(t, removeResp.Members, c.ClusterSize-1, "MemberRemove failed, expected length of members: %d, got: %d", c.ClusterSize-1, len(removeResp.Members)) for _, m := range removeResp.Members { require.NotEqualf(t, m.ID, memberID, "MemberRemove failed, member(id=%d) is still in cluster", memberID) } }) }) } } } // memberToRemove chooses a member to remove. // If clusterSize == 1, return the only member. // Otherwise, return a member that client has not connected to. // It ensures that `MemberRemove` function does not return an "etcdserver: server stopped" error. func memberToRemove(ctx context.Context, t *testing.T, client intf.Client, clusterSize int) (memberID uint64, clusterID uint64) { listResp, err := client.MemberList(ctx, false) require.NoError(t, err) clusterID = listResp.Header.ClusterId if clusterSize == 1 { memberID = listResp.Members[0].ID } else { // get status of the specific member that client has connected to statusResp, err := client.Status(ctx) require.NoError(t, err) // choose a member that client has not connected to for _, m := range listResp.Members { if m.ID != statusResp[0].Header.MemberId { memberID = m.ID break } } require.NotZerof(t, memberID, "memberToRemove failed. listResp:%v, statusResp:%v", listResp, statusResp) } return memberID, clusterID } func getMemberIDToEndpoints(ctx context.Context, t *testing.T, clus intf.Cluster) (memberIDToEndpoints map[uint64]string) { memberIDToEndpoints = make(map[uint64]string, len(clus.Endpoints())) for _, ep := range clus.Endpoints() { cc := testutils.MustClient(clus.Client(WithEndpoints([]string{ep}))) gresp, err := cc.Get(ctx, "health", config.GetOptions{}) require.NoError(t, err) memberIDToEndpoints[gresp.Header.MemberId] = ep } return memberIDToEndpoints } ================================================ FILE: tests/common/role_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestRoleAdd_Simple(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.RoleAdd(ctx, "root") require.NoError(t, err) }) }) } } func TestRoleAdd_Error(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterSize(1)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.RoleAdd(ctx, "test-role") require.NoError(t, err) _, err = cc.RoleAdd(ctx, "test-role") require.ErrorContainsf(t, err, rpctypes.ErrRoleAlreadyExist.Error(), "want (%v) error, but got (%v)", rpctypes.ErrRoleAlreadyExist, err) _, err = cc.RoleAdd(ctx, "") require.ErrorContainsf(t, err, rpctypes.ErrRoleEmpty.Error(), "want (%v) error, but got (%v)", rpctypes.ErrRoleEmpty, err) }) } func TestRootRole(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterSize(1)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.RoleAdd(ctx, "root") require.NoError(t, err) resp, err := cc.RoleGet(ctx, "root") require.NoError(t, err) t.Logf("get role resp %+v", resp) // granting to root should be refused by server and a no-op _, err = cc.RoleGrantPermission(ctx, "root", "foo", "", clientv3.PermissionType(clientv3.PermReadWrite)) require.NoError(t, err) resp2, err := cc.RoleGet(ctx, "root") require.NoError(t, err) t.Logf("get role resp %+v", resp2) }) } func TestRoleGrantRevokePermission(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterSize(1)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.RoleAdd(ctx, "role1") require.NoError(t, err) _, err = cc.RoleGrantPermission(ctx, "role1", "bar", "", clientv3.PermissionType(clientv3.PermRead)) require.NoError(t, err) _, err = cc.RoleGrantPermission(ctx, "role1", "bar", "", clientv3.PermissionType(clientv3.PermWrite)) require.NoError(t, err) _, err = cc.RoleGrantPermission(ctx, "role1", "bar", "foo", clientv3.PermissionType(clientv3.PermReadWrite)) require.NoError(t, err) _, err = cc.RoleRevokePermission(ctx, "role1", "foo", "") require.ErrorContainsf(t, err, rpctypes.ErrPermissionNotGranted.Error(), "want error (%v), but got (%v)", rpctypes.ErrPermissionNotGranted, err) _, err = cc.RoleRevokePermission(ctx, "role1", "bar", "foo") require.NoError(t, err) }) } func TestRoleDelete(t *testing.T) { testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterSize(1)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.RoleAdd(ctx, "role1") require.NoError(t, err) _, err = cc.RoleDelete(ctx, "role1") require.NoError(t, err) }) } ================================================ FILE: tests/common/status_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestStatus(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { rs, err := cc.Status(ctx) require.NoErrorf(t, err, "could not get status") require.Lenf(t, rs, tc.config.ClusterSize, "wrong number of status responses. expected:%d, got:%d ", tc.config.ClusterSize, len(rs)) memberIDs := make(map[uint64]struct{}) for _, r := range rs { require.NotNilf(t, r, "status response is nil") memberIDs[r.Header.MemberId] = struct{}{} } require.Lenf(t, rs, len(memberIDs), "found duplicated members") }) }) } } ================================================ FILE: tests/common/txn_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) type txnReq struct { compare []string ifSuccess []string ifFail []string expectResults []string expectError bool } func TestTxnSucc(t *testing.T) { testRunner.BeforeTest(t) reqs := []txnReq{ { compare: []string{`value("key1") != "value2"`, `value("key2") != "value1"`}, ifSuccess: []string{"get key1", "get key2"}, expectResults: []string{"SUCCESS", "key1", "value1", "key2", "value2"}, }, { compare: []string{`version("key1") = "1"`, `version("key2") = "1"`}, ifSuccess: []string{"get key1", "get key2", `put "key \"with\" space" "value \x23"`}, ifFail: []string{`put key1 "fail"`, `put key2 "fail"`}, expectResults: []string{"SUCCESS", "key1", "value1", "key2", "value2", "OK"}, }, { compare: []string{`version("key \"with\" space") = "1"`}, ifSuccess: []string{`get "key \"with\" space"`}, expectResults: []string{"SUCCESS", `key "with" space`, "value \x23"}, }, } for _, cfg := range clusterTestCases() { t.Run(cfg.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(cfg.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.Put(ctx, "key1", "value1", config.PutOptions{}) require.NoErrorf(t, err, "could not create key:%s, value:%s", "key1", "value1") _, err = cc.Put(ctx, "key2", "value2", config.PutOptions{}) require.NoErrorf(t, err, "could not create key:%s, value:%s", "key2", "value2") for _, req := range reqs { resp, err := cc.Txn(ctx, req.compare, req.ifSuccess, req.ifFail, config.TxnOptions{ Interactive: true, }) require.NoErrorf(t, err, "Txn returned error: %s", err) assert.Equal(t, req.expectResults, getRespValues(resp)) } }) }) } } func TestTxnFail(t *testing.T) { testRunner.BeforeTest(t) reqs := []txnReq{ { compare: []string{`version("key") < "0"`}, ifSuccess: []string{`put key "success"`}, ifFail: []string{`put key "fail"`}, expectResults: []string{"FAILURE", "OK"}, }, { compare: []string{`value("key1") != "value1"`}, ifSuccess: []string{`put key1 "success"`}, ifFail: []string{`put key1 "fail"`}, expectResults: []string{"FAILURE", "OK"}, }, } for _, cfg := range clusterTestCases() { t.Run(cfg.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(cfg.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { _, err := cc.Put(ctx, "key1", "value1", config.PutOptions{}) require.NoErrorf(t, err, "could not create key:%s, value:%s", "key1", "value1") for _, req := range reqs { resp, err := cc.Txn(ctx, req.compare, req.ifSuccess, req.ifFail, config.TxnOptions{ Interactive: true, }) require.NoErrorf(t, err, "Txn returned error: %s", err) assert.Equal(t, req.expectResults, getRespValues(resp)) } }) }) } } func getRespValues(r *clientv3.TxnResponse) []string { var ss []string if r.Succeeded { ss = append(ss, "SUCCESS") } else { ss = append(ss, "FAILURE") } for _, resp := range r.Responses { switch v := resp.Response.(type) { case *pb.ResponseOp_ResponseDeleteRange: r := (clientv3.DeleteResponse)(*v.ResponseDeleteRange) ss = append(ss, fmt.Sprintf("%d", r.Deleted)) case *pb.ResponseOp_ResponsePut: r := (clientv3.PutResponse)(*v.ResponsePut) ss = append(ss, "OK") if r.PrevKv != nil { ss = append(ss, string(r.PrevKv.Key), string(r.PrevKv.Value)) } case *pb.ResponseOp_ResponseRange: r := (clientv3.GetResponse)(*v.ResponseRange) for _, kv := range r.Kvs { ss = append(ss, string(kv.Key), string(kv.Value)) } default: ss = append(ss, fmt.Sprintf("\"Unknown\" : %q\n", fmt.Sprintf("%+v", v))) } } return ss } ================================================ FILE: tests/common/unit_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !(e2e || integration) package common import ( "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" ) func init() { testRunner = framework.UnitTestRunner clusterTestCases = unitClusterTestCases } func unitClusterTestCases() []testCase { return nil } // WithAuth is when a build tag (e.g. e2e or integration) isn't configured // in IDE, then IDE may complain "Unresolved reference 'WithAuth'". // So we need to define a default WithAuth to resolve such case. func WithAuth(userName, password string) config.ClientOption { return func(any) {} } func WithAuthToken(token string) config.ClientOption { return func(any) {} } func WithEndpoints(endpoints []string) config.ClientOption { return func(any) {} } func WithHTTP2Debug() config.ClusterOption { return func(c *config.ClusterConfig) {} } func WithTCPClient() config.ClusterOption { return func(c *config.ClusterConfig) {} } func WithUnixClient() config.ClusterOption { return func(c *config.ClusterConfig) {} } ================================================ FILE: tests/common/user_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestUserAdd_Simple(t *testing.T) { testRunner.BeforeTest(t) tcs := []struct { name string username string password string noPassword bool expectedError string }{ { name: "empty_username_not_allowed", username: "", password: "foobar", // Very Vague error expectation because the CLI and the API return very // different error structures. expectedError: "user name", }, { // Can create a user with no password, restricted to CN auth name: "no_password_with_noPassword_set", username: "foo", password: "", noPassword: true, }, { // Can create a user with no password, but not restricted to CN auth name: "no_password_without_noPassword_set", username: "foo", password: "", noPassword: false, }, { name: "regular_user_with_password", username: "foo", password: "bar", }, } for _, tc := range clusterTestCases() { for _, nc := range tcs { t.Run(tc.name+"/"+nc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { resp, err := cc.UserAdd(ctx, nc.username, nc.password, config.UserAddOptions{NoPassword: nc.noPassword}) if nc.expectedError != "" { require.ErrorContainsf(t, err, nc.expectedError, "expected user creation to fail") } else { require.NoError(t, err) require.NotNilf(t, resp, "unexpected nil response to successful user creation") } }) }) } } } func TestUserAdd_DuplicateUserNotAllowed(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { user := "barb" password := "rhubarb" _, err := cc.UserAdd(ctx, user, password, config.UserAddOptions{}) require.NoErrorf(t, err, "first user creation should succeed") _, err = cc.UserAdd(ctx, user, password, config.UserAddOptions{}) assert.ErrorContains(t, err, "etcdserver: user name already exists") }) }) } } func TestUserList(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { // No Users Yet resp, err := cc.UserList(ctx) require.NoErrorf(t, err, "user listing should succeed") require.Emptyf(t, resp.Users, "expected no pre-existing users, found: %q", resp.Users) user := "barb" password := "rhubarb" _, err = cc.UserAdd(ctx, user, password, config.UserAddOptions{}) require.NoErrorf(t, err, "user creation should succeed") // Users! resp, err = cc.UserList(ctx) require.NoErrorf(t, err, "user listing should succeed") require.Lenf(t, resp.Users, 1, "expected one user, found: %q", resp.Users) }) }) } } func TestUserDelete(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { user := "barb" password := "rhubarb" _, err := cc.UserAdd(ctx, user, password, config.UserAddOptions{}) require.NoErrorf(t, err, "user creation should succeed") resp, err := cc.UserList(ctx) require.NoErrorf(t, err, "user listing should succeed") require.Lenf(t, resp.Users, 1, "expected one user, found: %q", resp.Users) // Delete barb, sorry barb! _, err = cc.UserDelete(ctx, user) require.NoErrorf(t, err, "user deletion should succeed at first") resp, err = cc.UserList(ctx) require.NoErrorf(t, err, "user listing should succeed") require.Emptyf(t, resp.Users, "expected no users after deletion, found: %q", resp.Users) // Try to delete barb again _, err = cc.UserDelete(ctx, user) assert.ErrorContains(t, err, "user name not found") }) }) } } func TestUserChangePassword(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { user := "barb" password := "rhubarb" newPassword := "potato" _, err := cc.UserAdd(ctx, user, password, config.UserAddOptions{}) require.NoErrorf(t, err, "user creation should succeed") err = cc.UserChangePass(ctx, user, newPassword) require.NoErrorf(t, err, "user password change should succeed") err = cc.UserChangePass(ctx, "non-existent-user", newPassword) assert.ErrorContains(t, err, "user name not found") }) }) } } ================================================ FILE: tests/common/wait_leader_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" ) func TestWaitLeader(t *testing.T) { testRunner.BeforeTest(t) for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() leader := clus.WaitLeader(t) require.GreaterOrEqualf(t, leader, 0, "WaitLeader failed for the leader index (%d) is out of range, cluster member count: %d", leader, len(clus.Members())) require.Lessf(t, leader, len(clus.Members()), "WaitLeader failed for the leader index (%d) is out of range, cluster member count: %d", leader, len(clus.Members())) }) } } func TestWaitLeader_MemberStop(t *testing.T) { testRunner.BeforeTest(t) tcs := []testCase{ { name: "PeerTLS", config: config.NewClusterConfig(config.WithPeerTLS(config.ManualTLS)), }, { name: "PeerAutoTLS", config: config.NewClusterConfig(config.WithPeerTLS(config.AutoTLS)), }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() lead1 := clus.WaitLeader(t) require.GreaterOrEqualf(t, lead1, 0, "WaitLeader failed for the leader index (%d) is out of range, cluster member count: %d", lead1, len(clus.Members())) require.Lessf(t, lead1, len(clus.Members()), "WaitLeader failed for the leader index (%d) is out of range, cluster member count: %d", lead1, len(clus.Members())) clus.Members()[lead1].Stop() lead2 := clus.WaitLeader(t) require.GreaterOrEqualf(t, lead2, 0, "WaitLeader failed for the leader index (%d) is out of range, cluster member count: %d", lead2, len(clus.Members())) require.Lessf(t, lead2, len(clus.Members()), "WaitLeader failed for the leader index (%d) is out of range, cluster member count: %d", lead2, len(clus.Members())) require.NotEqualf(t, lead1, lead2, "WaitLeader failed for the leader(index=%d) did not change as expected after a member stopped", lead1) }) } } ================================================ FILE: tests/common/watch_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package common import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestWatch(t *testing.T) { testRunner.BeforeTest(t) watchTimeout := 1 * time.Second for _, tc := range clusterTestCases() { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(tc.config)) defer clus.Close() cc := testutils.MustClient(clus.Client()) testutils.ExecuteUntil(ctx, t, func() { tests := []struct { puts []testutils.KV watchKey string opts config.WatchOptions wanted []testutils.KV }{ { // watch by revision puts: []testutils.KV{{Key: "bar", Val: "revision_1"}, {Key: "bar", Val: "revision_2"}, {Key: "bar", Val: "revision_3"}}, watchKey: "bar", opts: config.WatchOptions{Revision: 3}, wanted: []testutils.KV{{Key: "bar", Val: "revision_2"}, {Key: "bar", Val: "revision_3"}}, }, { // watch 1 key puts: []testutils.KV{{Key: "sample", Val: "value"}}, watchKey: "sample", opts: config.WatchOptions{Revision: 1}, wanted: []testutils.KV{{Key: "sample", Val: "value"}}, }, { // watch 3 keys by prefix puts: []testutils.KV{{Key: "foo1", Val: "val1"}, {Key: "foo2", Val: "val2"}, {Key: "foo3", Val: "val3"}}, watchKey: "foo", opts: config.WatchOptions{Revision: 1, Prefix: true}, wanted: []testutils.KV{{Key: "foo1", Val: "val1"}, {Key: "foo2", Val: "val2"}, {Key: "foo3", Val: "val3"}}, }, { // watch 3 keys by range puts: []testutils.KV{{Key: "key1", Val: "val1"}, {Key: "key3", Val: "val3"}, {Key: "key2", Val: "val2"}}, watchKey: "key", opts: config.WatchOptions{Revision: 1, RangeEnd: "key3"}, wanted: []testutils.KV{{Key: "key1", Val: "val1"}, {Key: "key2", Val: "val2"}}, }, } for _, tt := range tests { wCtx, wCancel := context.WithCancel(ctx) wch := cc.Watch(wCtx, tt.watchKey, tt.opts) require.NotNilf(t, wch, "failed to watch %s", tt.watchKey) for j := range tt.puts { _, err := cc.Put(ctx, tt.puts[j].Key, tt.puts[j].Val, config.PutOptions{}) require.NoErrorf(t, err, "can't not put key %q, err: %s", tt.puts[j].Key, err) } kvs, err := testutils.KeyValuesFromWatchChan(wch, len(tt.wanted), watchTimeout) if err != nil { wCancel() require.NoErrorf(t, err, "failed to get key-values from watch channel %s", err) } wCancel() assert.Equal(t, tt.wanted, kvs) } }) }) } } ================================================ FILE: tests/e2e/cluster_downgrade_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "math/rand" "testing" "time" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/types" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) type CancellationState int const ( noCancellation CancellationState = iota cancelRightBeforeEnable cancelRightAfterEnable cancelAfterDowngrading ) func TestDowngradeUpgradeClusterOf1(t *testing.T) { testDowngradeUpgrade(t, 1, 1, false, noCancellation) } func TestDowngradeUpgrade2InClusterOf3(t *testing.T) { testDowngradeUpgrade(t, 2, 3, false, noCancellation) } func TestDowngradeUpgradeClusterOf3(t *testing.T) { testDowngradeUpgrade(t, 3, 3, false, noCancellation) } func TestDowngradeUpgradeClusterOf1WithSnapshot(t *testing.T) { testDowngradeUpgrade(t, 1, 1, true, noCancellation) } func TestDowngradeUpgradeClusterOf3WithSnapshot(t *testing.T) { testDowngradeUpgrade(t, 3, 3, true, noCancellation) } func TestDowngradeCancellationWithoutEnablingClusterOf1(t *testing.T) { testDowngradeUpgrade(t, 0, 1, false, cancelRightBeforeEnable) } func TestDowngradeCancellationRightAfterEnablingClusterOf1(t *testing.T) { testDowngradeUpgrade(t, 0, 1, false, cancelRightAfterEnable) } func TestDowngradeCancellationWithoutEnablingClusterOf3(t *testing.T) { testDowngradeUpgrade(t, 0, 3, false, cancelRightBeforeEnable) } func TestDowngradeCancellationRightAfterEnablingClusterOf3(t *testing.T) { testDowngradeUpgrade(t, 0, 3, false, cancelRightAfterEnable) } func TestDowngradeCancellationAfterDowngrading1InClusterOf3(t *testing.T) { testDowngradeUpgrade(t, 1, 3, false, cancelAfterDowngrading) } func TestDowngradeCancellationAfterDowngrading2InClusterOf3(t *testing.T) { testDowngradeUpgrade(t, 2, 3, false, cancelAfterDowngrading) } func testDowngradeUpgrade(t *testing.T, numberOfMembersToDowngrade int, clusterSize int, triggerSnapshot bool, triggerCancellation CancellationState) { currentEtcdBinary := e2e.BinPath.Etcd lastReleaseBinary := e2e.BinPath.EtcdLastRelease if !fileutil.Exist(lastReleaseBinary) { t.Skipf("%q does not exist", lastReleaseBinary) } currentVersion, err := e2e.GetVersionFromBinary(currentEtcdBinary) require.NoError(t, err) // wipe any pre-release suffix like -alpha.0 we see commonly in builds currentVersion.PreRelease = "" lastVersion, err := e2e.GetVersionFromBinary(lastReleaseBinary) require.NoError(t, err) require.Equalf(t, lastVersion.Minor, currentVersion.Minor-1, "unexpected minor version difference") currentVersionStr := currentVersion.String() lastVersionStr := lastVersion.String() lastClusterVersion := semver.New(lastVersionStr) lastClusterVersion.Patch = 0 e2e.BeforeTest(t) t.Logf("Create cluster with version %s", currentVersionStr) var snapshotCount uint64 = 10 epc := newCluster(t, clusterSize, snapshotCount) for i := 0; i < len(epc.Procs); i++ { e2e.ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{ Cluster: currentVersionStr, Server: version.Version, Storage: currentVersionStr, }) } cc := epc.Etcdctl() t.Logf("Cluster created") if len(epc.Procs) > 1 { t.Log("Waiting health interval to required to make membership changes") time.Sleep(etcdserver.HealthInterval) } t.Log("Downgrade should be disabled") e2e.ValidateDowngradeInfo(t, epc, &pb.DowngradeInfo{Enabled: false}) t.Log("Adding member to test membership, but a learner avoid breaking quorum") resp, err := cc.MemberAddAsLearner(t.Context(), "fake1", []string{"http://127.0.0.1:1001"}) require.NoError(t, err) if triggerSnapshot { t.Logf("Generating snapshot") generateSnapshot(t, snapshotCount, cc) verifySnapshot(t, epc) } t.Log("Removing learner to test membership") _, err = cc.MemberRemove(t.Context(), resp.Member.ID) require.NoError(t, err) beforeMembers, beforeKV := getMembersAndKeys(t, cc) if triggerCancellation == cancelRightBeforeEnable { t.Logf("Cancelling downgrade before enabling") e2e.DowngradeCancel(t, epc) t.Log("Downgrade cancelled, validating if cluster is in the right state") e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersion)) return // No need to perform downgrading, end the test here } e2e.DowngradeEnable(t, epc, lastVersion) t.Log("Downgrade should be enabled") e2e.ValidateDowngradeInfo(t, epc, &pb.DowngradeInfo{Enabled: true, TargetVersion: lastClusterVersion.String()}) if triggerCancellation == cancelRightAfterEnable { t.Logf("Cancelling downgrade right after enabling (no node is downgraded yet)") e2e.DowngradeCancel(t, epc) t.Log("Downgrade cancelled, validating if cluster is in the right state") e2e.ValidateMemberVersions(t, epc, generateIdenticalVersions(clusterSize, currentVersion)) return // No need to perform downgrading, end the test here } membersToChange := rand.Perm(len(epc.Procs))[:numberOfMembersToDowngrade] t.Logf("Elect members for operations on members: %v", membersToChange) t.Logf("Starting downgrade process to %q", lastVersionStr) err = e2e.DowngradeUpgradeMembersByID(t, nil, epc, membersToChange, true, currentVersion, lastClusterVersion) require.NoError(t, err) if len(membersToChange) == len(epc.Procs) { e2e.AssertProcessLogs(t, epc.Procs[epc.WaitLeader(t)], "the cluster has been downgraded") } t.Log("Downgrade complete") afterMembers, afterKV := getMembersAndKeys(t, cc) assert.Equal(t, beforeKV.Kvs, afterKV.Kvs) assert.Equal(t, beforeMembers.Members, afterMembers.Members) if len(epc.Procs) > 1 { t.Log("Waiting health interval to required to make membership changes") time.Sleep(etcdserver.HealthInterval) } if triggerCancellation == cancelAfterDowngrading { e2e.DowngradeCancel(t, epc) t.Log("Downgrade cancelled, validating if cluster is in the right state") e2e.ValidateMemberVersions(t, epc, generatePartialCancellationVersions(clusterSize, membersToChange, lastClusterVersion)) } t.Log("Adding learner to test membership, but avoid breaking quorum") resp, err = cc.MemberAddAsLearner(t.Context(), "fake2", []string{"http://127.0.0.1:1002"}) require.NoError(t, err) if triggerSnapshot { t.Logf("Generating snapshot") generateSnapshot(t, snapshotCount, cc) verifySnapshot(t, epc) } t.Log("Removing learner to test membership") _, err = cc.MemberRemove(t.Context(), resp.Member.ID) require.NoError(t, err) beforeMembers, beforeKV = getMembersAndKeys(t, cc) t.Logf("Starting upgrade process to %q", currentVersionStr) downgradeEnabled := triggerCancellation == noCancellation && numberOfMembersToDowngrade < clusterSize err = e2e.DowngradeUpgradeMembersByID(t, nil, epc, membersToChange, downgradeEnabled, lastClusterVersion, currentVersion) require.NoError(t, err) t.Log("Upgrade complete") if downgradeEnabled { t.Log("Downgrade should be still enabled") e2e.ValidateDowngradeInfo(t, epc, &pb.DowngradeInfo{Enabled: true, TargetVersion: lastClusterVersion.String()}) } else { t.Log("Downgrade should be disabled") e2e.ValidateDowngradeInfo(t, epc, &pb.DowngradeInfo{Enabled: false}) } afterMembers, afterKV = getMembersAndKeys(t, cc) assert.Equal(t, beforeKV.Kvs, afterKV.Kvs) assert.Equal(t, beforeMembers.Members, afterMembers.Members) } func newCluster(t *testing.T, clusterSize int, snapshotCount uint64) *e2e.EtcdProcessCluster { epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithClusterSize(clusterSize), e2e.WithSnapshotCount(snapshotCount), e2e.WithKeepDataDir(true), ) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } t.Cleanup(func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }) return epc } func generateSnapshot(t *testing.T, snapshotCount uint64, cc *e2e.EtcdctlV3) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() var i uint64 t.Logf("Adding keys") for i = 0; i < snapshotCount*3; i++ { _, err := cc.Put(ctx, fmt.Sprintf("%d", i), "1", config.PutOptions{}) assert.NoError(t, err) } } func verifySnapshot(t *testing.T, epc *e2e.EtcdProcessCluster) { for i := range epc.Procs { t.Logf("Verifying snapshot for member %d", i) ss := snap.New(epc.Cfg.Logger, datadir.ToSnapDir(epc.Procs[i].Config().DataDirPath)) _, err := ss.Load() require.NoError(t, err) } t.Logf("All members have a valid snapshot") } func verifySnapshotMembers(t *testing.T, epc *e2e.EtcdProcessCluster, expectedMembers *clientv3.MemberListResponse) { for i := range epc.Procs { t.Logf("Verifying snapshot for member %d", i) ss := snap.New(epc.Cfg.Logger, datadir.ToSnapDir(epc.Procs[i].Config().DataDirPath)) snap, err := ss.Load() require.NoError(t, err) st := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix) err = st.Recovery(snap.Data) require.NoError(t, err) for _, m := range expectedMembers.Members { _, err := st.Get(membership.MemberStoreKey(types.ID(m.ID)), true, true) require.NoError(t, err) } t.Logf("Verifed snapshot for member %d", i) } t.Log("All members have a valid snapshot") } func getMembersAndKeys(t *testing.T, cc *e2e.EtcdctlV3) (*clientv3.MemberListResponse, *clientv3.GetResponse) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() kvs, err := cc.Get(ctx, "", config.GetOptions{Prefix: true}) require.NoError(t, err) members, err := cc.MemberList(ctx, false) assert.NoError(t, err) return members, kvs } func generateIdenticalVersions(clusterSize int, ver *semver.Version) []*version.Versions { ret := make([]*version.Versions, clusterSize) for i := range clusterSize { ret[i] = &version.Versions{ Cluster: ver.String(), Server: ver.String(), Storage: ver.String(), } } return ret } func generatePartialCancellationVersions(clusterSize int, membersToChange []int, ver *semver.Version) []*version.Versions { ret := make([]*version.Versions, clusterSize) for i := range clusterSize { ret[i] = &version.Versions{ Cluster: ver.String(), Server: e2e.OffsetMinor(ver, 1).String(), Storage: "", } } for i := range membersToChange { ret[membersToChange[i]].Server = ver.String() } return ret } ================================================ FILE: tests/e2e/cmux_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // These tests are directly validating etcd connection multiplexing. //go:build !cluster_proxy package e2e import ( "context" "encoding/json" "fmt" "os" "path/filepath" "strings" "testing" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestConnectionMultiplexing(t *testing.T) { e2e.BeforeTest(t) for _, tc := range []struct { name string serverTLS e2e.ClientConnType separateHTTPPort bool }{ { name: "ServerTLS", serverTLS: e2e.ClientTLS, }, { name: "ServerNonTLS", serverTLS: e2e.ClientNonTLS, }, { name: "ServerTLSAndNonTLS", serverTLS: e2e.ClientTLSAndNonTLS, }, { name: "SeparateHTTP/ServerTLS", serverTLS: e2e.ClientTLS, separateHTTPPort: true, }, { name: "SeparateHTTP/ServerNonTLS", serverTLS: e2e.ClientNonTLS, separateHTTPPort: true, }, } { t.Run(tc.name, func(t *testing.T) { ctx := t.Context() cfg := e2e.NewConfig(e2e.WithClusterSize(1)) cfg.Client.ConnectionType = tc.serverTLS cfg.ClientHTTPSeparate = tc.separateHTTPPort clus, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithConfig(cfg)) require.NoError(t, err) defer clus.Close() var clientScenarios []e2e.ClientConnType switch tc.serverTLS { case e2e.ClientTLS: clientScenarios = []e2e.ClientConnType{e2e.ClientTLS} case e2e.ClientNonTLS: clientScenarios = []e2e.ClientConnType{e2e.ClientNonTLS} case e2e.ClientTLSAndNonTLS: clientScenarios = []e2e.ClientConnType{e2e.ClientTLS, e2e.ClientNonTLS} } for _, clientTLS := range clientScenarios { name := "ClientNonTLS" if clientTLS == e2e.ClientTLS { name = "ClientTLS" } t.Run(name, func(t *testing.T) { testConnectionMultiplexing(ctx, t, clus.Procs[0], clientTLS) }) } }) } } func testConnectionMultiplexing(ctx context.Context, t *testing.T, member e2e.EtcdProcess, connType e2e.ClientConnType) { httpEndpoint := member.EndpointsHTTP()[0] grpcEndpoint := member.EndpointsGRPC()[0] switch connType { case e2e.ClientTLS: httpEndpoint = e2e.ToTLS(httpEndpoint) grpcEndpoint = e2e.ToTLS(grpcEndpoint) case e2e.ClientNonTLS: default: panic(fmt.Sprintf("Unsupported conn type %v", connType)) } t.Run("etcdctl", func(t *testing.T) { etcdctl, err := e2e.NewEtcdctl(e2e.ClientConfig{ConnectionType: connType}, []string{grpcEndpoint}) require.NoError(t, err) _, err = etcdctl.Get(ctx, "a", config.GetOptions{}) assert.NoError(t, err) }) t.Run("clientv3", func(t *testing.T) { c := newClient(t, []string{grpcEndpoint}, e2e.ClientConfig{ConnectionType: connType}) _, err := c.Get(ctx, "a") assert.NoError(t, err) }) t.Run("curl", func(t *testing.T) { for _, httpVersion := range []string{"2", "1.1", ""} { tname := "http" + httpVersion if httpVersion == "" { tname = "default" } t.Run(tname, func(t *testing.T) { assert.NoError(t, fetchGRPCGateway(httpEndpoint, httpVersion, connType)) assert.NoError(t, fetchMetrics(t, httpEndpoint, httpVersion, connType)) assert.NoError(t, fetchVersion(httpEndpoint, httpVersion, connType)) assert.NoError(t, fetchHealth(httpEndpoint, httpVersion, connType)) assert.NoError(t, fetchDebugVars(httpEndpoint, httpVersion, connType)) }) } }) } func fetchGRPCGateway(endpoint string, httpVersion string, connType e2e.ClientConnType) error { rangeData, err := json.Marshal(&pb.RangeRequest{ Key: []byte("a"), }) if err != nil { return err } req := e2e.CURLReq{Endpoint: "/v3/kv/range", Value: string(rangeData), Timeout: 5, HTTPVersion: httpVersion} respData, err := curl(endpoint, "POST", req, connType) if err != nil { return err } return validateGrpcgatewayRangeReponse([]byte(respData)) } func validateGrpcgatewayRangeReponse(respData []byte) error { // Modified json annotation so ResponseHeader fields are stored in string. type responseHeader struct { //revive:disable:var-naming ClusterId uint64 `json:"cluster_id,string,omitempty"` MemberId uint64 `json:"member_id,string,omitempty"` //revive:enable:var-naming Revision int64 `json:"revision,string,omitempty"` RaftTerm uint64 `json:"raft_term,string,omitempty"` } type rangeResponse struct { Header *responseHeader `json:"header,omitempty"` Kvs []*mvccpb.KeyValue `json:"kvs,omitempty"` More bool `json:"more,omitempty"` Count int64 `json:"count,omitempty"` } var resp rangeResponse return json.Unmarshal(respData, &resp) } func fetchMetrics(t *testing.T, endpoint string, httpVersion string, connType e2e.ClientConnType) error { tmpDir := t.TempDir() metricFile := filepath.Join(tmpDir, "metrics") req := e2e.CURLReq{Endpoint: "/metrics", Timeout: 5, HTTPVersion: httpVersion, OutputFile: metricFile} if _, err := curl(endpoint, "GET", req, connType); err != nil { return err } rawData, err := os.ReadFile(metricFile) if err != nil { return fmt.Errorf("failed to read the metric: %w", err) } respData := string(rawData) parser := expfmt.NewTextParser(model.LegacyValidation) _, err = parser.TextToMetricFamilies(strings.NewReader(strings.ReplaceAll(respData, "\r\n", "\n"))) return err } func fetchVersion(endpoint string, httpVersion string, connType e2e.ClientConnType) error { req := e2e.CURLReq{Endpoint: "/version", Timeout: 5, HTTPVersion: httpVersion} respData, err := curl(endpoint, "GET", req, connType) if err != nil { return err } var resp version.Versions return json.Unmarshal([]byte(respData), &resp) } func fetchHealth(endpoint string, httpVersion string, connType e2e.ClientConnType) error { req := e2e.CURLReq{Endpoint: "/health", Timeout: 5, HTTPVersion: httpVersion} respData, err := curl(endpoint, "GET", req, connType) if err != nil { return err } var resp etcdhttp.Health return json.Unmarshal([]byte(respData), &resp) } func fetchDebugVars(endpoint string, httpVersion string, connType e2e.ClientConnType) error { req := e2e.CURLReq{Endpoint: "/debug/vars", Timeout: 5, HTTPVersion: httpVersion} respData, err := curl(endpoint, "GET", req, connType) if err != nil { return err } var resp map[string]any return json.Unmarshal([]byte(respData), &resp) } ================================================ FILE: tests/e2e/corrupt_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "context" "fmt" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/server/v3/storage/mvcc/testutil" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestEtcdCorruptHash(t *testing.T) { // oldenv := os.Getenv("EXPECT_DEBUG") // defer os.Setenv("EXPECT_DEBUG", oldenv) // os.Setenv("EXPECT_DEBUG", "1") cfg := e2e.NewConfigNoTLS() // trigger snapshot so that restart member can load peers from disk cfg.ServerConfig.SnapshotCount = 3 testCtl(t, corruptTest, withQuorum(), withCfg(*cfg), withInitialCorruptCheck(), withCorruptFunc(testutil.CorruptBBolt), ) } func corruptTest(cx ctlCtx) { cx.t.Log("putting 10 keys...") for i := 0; i < 10; i++ { if err := ctlV3Put(cx, fmt.Sprintf("foo%05d", i), fmt.Sprintf("v%05d", i), ""); err != nil { if cx.dialTimeout > 0 && !isGRPCTimedout(err) { cx.t.Fatalf("putTest ctlV3Put error (%v)", err) } } } // enough time for all nodes sync on the same data cx.t.Log("sleeping 3sec to let nodes sync...") time.Sleep(3 * time.Second) cx.t.Log("connecting clientv3...") eps := cx.epc.EndpointsGRPC() cli1, err := clientv3.New(clientv3.Config{Endpoints: []string{eps[1]}, DialTimeout: 3 * time.Second}) require.NoError(cx.t, err) defer cli1.Close() sresp, err := cli1.Status(context.TODO(), eps[0]) cx.t.Logf("checked status sresp:%v err:%v", sresp, err) require.NoError(cx.t, err) id0 := sresp.Header.GetMemberId() cx.t.Log("stopping etcd[0]...") cx.epc.Procs[0].Stop() // corrupting first member by modifying backend offline. fp := datadir.ToBackendFileName(cx.epc.Procs[0].Config().DataDirPath) cx.t.Logf("corrupting backend: %v", fp) err = cx.corruptFunc(fp) require.NoError(cx.t, err) cx.t.Log("restarting etcd[0]") ep := cx.epc.Procs[0] proc, err := e2e.SpawnCmd(append([]string{ep.Config().ExecPath}, ep.Config().Args...), cx.envMap) require.NoError(cx.t, err) defer proc.Stop() cx.t.Log("waiting for etcd[0] failure...") // restarting corrupted member should fail e2e.WaitReadyExpectProc(context.TODO(), proc, []string{fmt.Sprintf("etcdmain: %016x found data inconsistency with peers", id0)}) } func TestInPlaceRecovery(t *testing.T) { basePort := 20000 e2e.BeforeTest(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() // Initialize the cluster. epcOld, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithInitialClusterToken("old"), e2e.WithKeepDataDir(false), e2e.WithCorruptCheckTime(time.Second), e2e.WithBasePort(basePort), ) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } t.Cleanup(func() { if errC := epcOld.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }) t.Log("old cluster started.") // Put some data into the old cluster, so that after recovering from a blank db, the hash diverges. t.Log("putting 10 keys...") oldCc, err := e2e.NewEtcdctl(epcOld.Cfg.Client, epcOld.EndpointsGRPC()) require.NoError(t, err) for i := 0; i < 10; i++ { _, err = oldCc.Put(ctx, testutil.PickKey(int64(i)), fmt.Sprint(i), config.PutOptions{}) require.NoErrorf(t, err, "error on put") } // Create a new cluster config, but with the same port numbers. In this way the new servers can stay in // contact with the old ones. epcNewConfig := e2e.NewConfig( e2e.WithInitialClusterToken("new"), e2e.WithKeepDataDir(false), e2e.WithCorruptCheckTime(time.Second), e2e.WithBasePort(basePort), e2e.WithInitialCorruptCheck(true), ) epcNew, err := e2e.InitEtcdProcessCluster(t, epcNewConfig) if err != nil { t.Fatalf("could not init etcd process cluster (%v)", err) } t.Cleanup(func() { if errC := epcNew.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }) newCc, err := e2e.NewEtcdctl(epcNew.Cfg.Client, epcNew.EndpointsGRPC()) require.NoError(t, err) // Rolling recovery of the servers. wg := sync.WaitGroup{} t.Log("rolling updating servers in place...") for i := range epcNew.Procs { oldProc := epcOld.Procs[i] err = oldProc.Close() if err != nil { t.Fatalf("could not stop etcd process (%v)", err) } t.Logf("old cluster server %d: %s stopped.", i, oldProc.Config().Name) wg.Add(1) // Start servers in background to avoid blocking on server start. // EtcdProcess.Start waits until etcd becomes healthy, which will not happen here until we restart at least 2 members. go func(proc e2e.EtcdProcess) { defer wg.Done() err = proc.Start(ctx) if err != nil { t.Errorf("could not start etcd process (%v)", err) } t.Logf("new cluster server: %s started in-place with blank db.", proc.Config().Name) }(epcNew.Procs[i]) t.Log("sleeping 5 sec to let nodes do periodical check...") time.Sleep(5 * time.Second) } wg.Wait() t.Log("new cluster started.") alarmResponse, err := newCc.AlarmList(ctx) require.NoErrorf(t, err, "error on alarm list") for _, alarm := range alarmResponse.Alarms { if alarm.Alarm == etcdserverpb.AlarmType_CORRUPT { t.Fatalf("there is no corruption after in-place recovery, but corruption reported.") } } t.Log("no corruption detected.") } func TestPeriodicCheckDetectsCorruption(t *testing.T) { checkTime := time.Second e2e.BeforeTest(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() corruptCheckTime := e2e.WithCorruptCheckTime(time.Second) epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithKeepDataDir(true), corruptCheckTime, ) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } t.Cleanup(func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }) cc := epc.Etcdctl() for i := 0; i < 10; i++ { _, err = cc.Put(ctx, testutil.PickKey(int64(i)), fmt.Sprint(i), config.PutOptions{}) require.NoErrorf(t, err, "error on put") } memberID, found, err := getMemberIDByName(ctx, cc, epc.Procs[0].Config().Name) require.NoErrorf(t, err, "error on member list") assert.Truef(t, found, "member not found") epc.Procs[0].Stop() err = testutil.CorruptBBolt(datadir.ToBackendFileName(epc.Procs[0].Config().DataDirPath)) require.NoError(t, err) err = epc.Procs[0].Restart(t.Context()) require.NoError(t, err) time.Sleep(checkTime * 11 / 10) alarmResponse, err := cc.AlarmList(ctx) require.NoErrorf(t, err, "error on alarm list") assert.Equal(t, []*etcdserverpb.AlarmMember{{Alarm: etcdserverpb.AlarmType_CORRUPT, MemberID: memberID}}, alarmResponse.Alarms) } func TestCompactHashCheckDetectCorruption(t *testing.T) { testCompactHashCheckDetectCorruption(t, false) } func TestCompactHashCheckDetectCorruptionWithFeatureGate(t *testing.T) { testCompactHashCheckDetectCorruption(t, true) } func testCompactHashCheckDetectCorruption(t *testing.T, useFeatureGate bool) { checkTime := time.Second e2e.BeforeTest(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() opts := []e2e.EPClusterOption{e2e.WithKeepDataDir(true), e2e.WithCompactHashCheckTime(checkTime)} if useFeatureGate { opts = append(opts, e2e.WithServerFeatureGate("CompactHashCheck", true)) } else { opts = append(opts, e2e.WithCompactHashCheckEnabled(true)) } epc, err := e2e.NewEtcdProcessCluster(ctx, t, opts...) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } t.Cleanup(func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }) cc := epc.Etcdctl() for i := 0; i < 10; i++ { _, err = cc.Put(ctx, testutil.PickKey(int64(i)), fmt.Sprint(i), config.PutOptions{}) require.NoErrorf(t, err, "error on put") } memberID, found, err := getMemberIDByName(ctx, cc, epc.Procs[0].Config().Name) require.NoErrorf(t, err, "error on member list") assert.Truef(t, found, "member not found") epc.Procs[0].Stop() err = testutil.CorruptBBolt(datadir.ToBackendFileName(epc.Procs[0].Config().DataDirPath)) require.NoError(t, err) err = epc.Procs[0].Restart(ctx) require.NoError(t, err) _, err = cc.Compact(ctx, 5, config.CompactOption{}) require.NoError(t, err) time.Sleep(checkTime * 11 / 10) alarmResponse, err := cc.AlarmList(ctx) require.NoErrorf(t, err, "error on alarm list") assert.Equal(t, []*etcdserverpb.AlarmMember{{Alarm: etcdserverpb.AlarmType_CORRUPT, MemberID: memberID}}, alarmResponse.Alarms) } func TestCompactHashCheckDetectCorruptionInterrupt(t *testing.T) { testCompactHashCheckDetectCorruptionInterrupt(t, false) } func TestCompactHashCheckDetectCorruptionInterruptWithFeatureGate(t *testing.T) { testCompactHashCheckDetectCorruptionInterrupt(t, true) } func testCompactHashCheckDetectCorruptionInterrupt(t *testing.T, useFeatureGate bool) { checkTime := time.Second e2e.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) defer cancel() slowCompactionNodeIndex := 1 // Start a new cluster, with compact hash check enabled. t.Log("creating a new cluster with 3 nodes...") dataDirPath := t.TempDir() opts := []e2e.EPClusterOption{ e2e.WithKeepDataDir(true), e2e.WithCompactHashCheckTime(checkTime), e2e.WithClusterSize(3), e2e.WithDataDirPath(dataDirPath), e2e.WithLogLevel("info"), } if useFeatureGate { opts = append(opts, e2e.WithServerFeatureGate("CompactHashCheck", true)) } else { opts = append(opts, e2e.WithCompactHashCheckEnabled(true)) } compactionBatchLimit := e2e.WithCompactionBatchLimit(1) cfg := e2e.NewConfig(opts...) epc, err := e2e.InitEtcdProcessCluster(t, cfg) require.NoError(t, err) // Assign a node a very slow compaction speed, so that its compaction can be interrupted. err = epc.UpdateProcOptions(slowCompactionNodeIndex, t, compactionBatchLimit, e2e.WithCompactionSleepInterval(1*time.Hour), ) require.NoError(t, err) epc, err = e2e.StartEtcdProcessCluster(ctx, t, epc, cfg) require.NoError(t, err) t.Cleanup(func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }) // Put 10 identical keys to the cluster, so that the compaction will drop some stale values. t.Log("putting 10 values to the identical key...") cc := epc.Etcdctl() for i := 0; i < 10; i++ { _, err = cc.Put(ctx, "key", fmt.Sprint(i), config.PutOptions{}) require.NoErrorf(t, err, "error on put") } t.Log("compaction started...") _, err = cc.Compact(ctx, 5, config.CompactOption{}) require.NoError(t, err) err = epc.Procs[slowCompactionNodeIndex].Close() require.NoError(t, err) err = epc.UpdateProcOptions(slowCompactionNodeIndex, t) require.NoError(t, err) t.Logf("restart proc %d to interrupt its compaction...", slowCompactionNodeIndex) err = epc.Procs[slowCompactionNodeIndex].Restart(ctx) require.NoError(t, err) // Wait until the node finished compaction and the leader finished compaction hash check _, err = epc.Procs[slowCompactionNodeIndex].Logs().ExpectWithContext(ctx, expect.ExpectedResponse{Value: "finished scheduled compaction"}) require.NoErrorf(t, err, "can't get log indicating finished scheduled compaction") leaderIndex := epc.WaitLeader(t) _, err = epc.Procs[leaderIndex].Logs().ExpectWithContext(ctx, expect.ExpectedResponse{Value: "finished compaction hash check"}) require.NoErrorf(t, err, "can't get log indicating finished compaction hash check") alarmResponse, err := cc.AlarmList(ctx) require.NoErrorf(t, err, "error on alarm list") for _, alarm := range alarmResponse.Alarms { if alarm.Alarm == etcdserverpb.AlarmType_CORRUPT { t.Fatal("there should be no corruption after resuming the compaction, but corruption detected") } } t.Log("no corruption detected.") } func TestCtlV3SerializableRead(t *testing.T) { testCtlV3ReadAfterWrite(t, clientv3.WithSerializable()) } func TestCtlV3LinearizableRead(t *testing.T) { testCtlV3ReadAfterWrite(t) } func testCtlV3ReadAfterWrite(t *testing.T, ops ...clientv3.OpOption) { e2e.BeforeTest(t) ctx := t.Context() epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(1), e2e.WithEnvVars(map[string]string{"GOFAIL_FAILPOINTS": `raftBeforeSave=sleep("200ms");beforeCommit=sleep("200ms")`}), ) require.NoErrorf(t, err, "failed to start etcd cluster") defer func() { derr := epc.Close() require.NoErrorf(t, derr, "failed to close etcd cluster") }() cc, err := clientv3.New(clientv3.Config{ Endpoints: epc.EndpointsGRPC(), DialKeepAliveTime: 5 * time.Second, DialKeepAliveTimeout: 1 * time.Second, }) require.NoError(t, err) defer func() { derr := cc.Close() require.NoError(t, derr) }() _, err = cc.Put(ctx, "foo", "bar") require.NoError(t, err) // Refer to https://github.com/etcd-io/etcd/pull/16658#discussion_r1341346778 t.Log("Restarting the etcd process to ensure all data is persisted") err = epc.Procs[0].Restart(ctx) require.NoError(t, err) epc.WaitLeader(t) _, err = cc.Put(ctx, "foo", "bar2") require.NoError(t, err) t.Log("Killing the etcd process right after successfully writing a new key/value") err = epc.Procs[0].Kill() require.NoError(t, err) err = epc.Procs[0].Wait(ctx) require.NoError(t, err) stopc := make(chan struct{}, 1) donec := make(chan struct{}, 1) t.Log("Starting a goroutine to repeatedly read the key/value") count := 0 go func() { defer func() { donec <- struct{}{} }() for { select { case <-stopc: return default: } rctx, cancel := context.WithTimeout(ctx, 2*time.Second) resp, rerr := cc.Get(rctx, "foo", ops...) cancel() if rerr != nil { continue } count++ assert.Equal(t, "bar2", string(resp.Kvs[0].Value)) } }() t.Log("Starting the etcd process again") err = epc.Procs[0].Start(ctx) require.NoError(t, err) time.Sleep(3 * time.Second) stopc <- struct{}{} <-donec assert.Positive(t, count) t.Logf("Checked the key/value %d times", count) } ================================================ FILE: tests/e2e/ctl_v3_auth_cluster_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "path/filepath" "testing" "time" "github.com/stretchr/testify/assert" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestAuthCluster(t *testing.T) { e2e.BeforeTest(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(1), e2e.WithSnapshotCount(2), ) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } defer func() { if err := epc.Close(); err != nil { t.Fatalf("could not close test cluster (%v)", err) } }() epcClient := epc.Etcdctl() createUsers(ctx, t, epcClient) if err := epcClient.AuthEnable(ctx); err != nil { t.Fatalf("could not enable Auth: (%v)", err) } testUserClientOpts := e2e.WithAuth("test", "testPassword") rootUserClientOpts := e2e.WithAuth("root", "rootPassword") // write more than SnapshotCount keys to single leader to make sure snapshot is created for i := 0; i <= 10; i++ { if _, err := epc.Etcdctl(testUserClientOpts).Put(ctx, fmt.Sprintf("/test/%d", i), "test", config.PutOptions{}); err != nil { t.Fatalf("failed to Put (%v)", err) } } // start second process if _, err := epc.StartNewProc(ctx, nil, t, false /* addAsLearner */, rootUserClientOpts); err != nil { t.Fatalf("could not start second etcd process (%v)", err) } // make sure writes to both endpoints are successful endpoints := epc.EndpointsGRPC() assert.Len(t, endpoints, 2) for _, endpoint := range epc.EndpointsGRPC() { if _, err := epc.Etcdctl(testUserClientOpts, e2e.WithEndpoints([]string{endpoint})).Put(ctx, "/test/key", endpoint, config.PutOptions{}); err != nil { t.Fatalf("failed to write to Put to %q (%v)", endpoint, err) } } // verify all nodes have exact same revision and hash assert.Eventually(t, func() bool { hashKvs, err := epc.Etcdctl(rootUserClientOpts).HashKV(ctx, 0) if err != nil { t.Logf("failed to get HashKV: %v", err) return false } if len(hashKvs) != 2 { t.Logf("not exactly 2 hashkv responses returned: %d", len(hashKvs)) return false } if hashKvs[0].Header.Revision != hashKvs[1].Header.Revision { t.Logf("The two members' revision (%d, %d) are not equal", hashKvs[0].Header.Revision, hashKvs[1].Header.Revision) return false } assert.Equal(t, hashKvs[0].Hash, hashKvs[1].Hash) return true }, time.Second*5, time.Millisecond*100) } func applyTLSWithRootCommonName() func() { var ( oldCertPath = e2e.CertPath oldPrivateKeyPath = e2e.PrivateKeyPath oldCaPath = e2e.CaPath newCertPath = filepath.Join(e2e.FixturesDir, "CommonName-root.crt") newPrivateKeyPath = filepath.Join(e2e.FixturesDir, "CommonName-root.key") newCaPath = filepath.Join(e2e.FixturesDir, "CommonName-root.crt") ) e2e.CertPath = newCertPath e2e.PrivateKeyPath = newPrivateKeyPath e2e.CaPath = newCaPath return func() { e2e.CertPath = oldCertPath e2e.PrivateKeyPath = oldPrivateKeyPath e2e.CaPath = oldCaPath } } func createUsers(ctx context.Context, t *testing.T, client *e2e.EtcdctlV3) { if _, err := client.UserAdd(ctx, "root", "rootPassword", config.UserAddOptions{}); err != nil { t.Fatalf("could not add root user (%v)", err) } if _, err := client.RoleAdd(ctx, "root"); err != nil { t.Fatalf("could not create 'root' role (%v)", err) } if _, err := client.UserGrantRole(ctx, "root", "root"); err != nil { t.Fatalf("could not grant root role to root user (%v)", err) } if _, err := client.RoleAdd(ctx, "test"); err != nil { t.Fatalf("could not create 'test' role (%v)", err) } if _, err := client.RoleGrantPermission(ctx, "test", "/test/", "/test0", clientv3.PermissionType(clientv3.PermReadWrite)); err != nil { t.Fatalf("could not RoleGrantPermission (%v)", err) } if _, err := client.UserAdd(ctx, "test", "testPassword", config.UserAddOptions{}); err != nil { t.Fatalf("could not add user test (%v)", err) } if _, err := client.UserGrantRole(ctx, "test", "test"); err != nil { t.Fatalf("could not grant test role user (%v)", err) } } ================================================ FILE: tests/e2e/ctl_v3_auth_no_proxy_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // These tests depend on certificate-based authentication that is NOT supported // by gRPC proxy. //go:build !cluster_proxy package e2e import ( "context" "fmt" "sync" "testing" "time" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3AuthCertCN(t *testing.T) { testCtl(t, authTestCertCN, withCfg(*e2e.NewConfigClientTLSCertAuth())) } func TestCtlV3AuthCertCNAndUsername(t *testing.T) { testCtl(t, authTestCertCNAndUsername, withCfg(*e2e.NewConfigClientTLSCertAuth())) } func TestCtlV3AuthCertCNAndUsernameNoPassword(t *testing.T) { testCtl(t, authTestCertCNAndUsernameNoPassword, withCfg(*e2e.NewConfigClientTLSCertAuth())) } func TestCtlV3AuthCertCNWithWithConcurrentOperation(t *testing.T) { e2e.BeforeTest(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() // apply the certificate which has `root` CommonName, // and reset the setting when the test case finishes. // TODO(ahrtr): enhance the e2e test framework to support // certificates with CommonName. t.Log("Apply certificate with root CommonName") resetCert := applyTLSWithRootCommonName() defer resetCert() t.Log("Create etcd cluster") epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(1), e2e.WithClientConnType(e2e.ClientTLS), e2e.WithClientCertAuthority(true), ) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } defer func() { if err := epc.Close(); err != nil { t.Fatalf("could not close test cluster (%v)", err) } }() epcClient := epc.Etcdctl() t.Log("Create users") createUsers(ctx, t, epcClient) t.Log("Enable auth") if err := epcClient.AuthEnable(ctx); err != nil { t.Fatalf("could not enable Auth: (%v)", err) } // Create two goroutines, one goroutine keeps creating & deleting users, // and the other goroutine keeps writing & deleting K/V entries. var wg sync.WaitGroup wg.Add(2) errs := make(chan error, 2) donec := make(chan struct{}) // Create the first goroutine to create & delete users t.Log("Create the first goroutine to create & delete users") go func() { defer wg.Done() for i := 0; i < 100; i++ { user := fmt.Sprintf("testuser-%d", i) pass := fmt.Sprintf("testpass-%d", i) if _, err := epcClient.UserAdd(ctx, user, pass, config.UserAddOptions{}); err != nil { errs <- fmt.Errorf("failed to create user %q: %w", user, err) break } if _, err := epcClient.UserDelete(ctx, user); err != nil { errs <- fmt.Errorf("failed to delete user %q: %w", user, err) break } } t.Log("The first goroutine finished") }() // Create the second goroutine to write & delete K/V entries t.Log("Create the second goroutine to write & delete K/V entries") go func() { defer wg.Done() for i := 0; i < 100; i++ { key := fmt.Sprintf("key-%d", i) value := fmt.Sprintf("value-%d", i) if _, err := epcClient.Put(ctx, key, value, config.PutOptions{}); err != nil { errs <- fmt.Errorf("failed to put key %q: %w", key, err) break } if _, err := epcClient.Delete(ctx, key, config.DeleteOptions{}); err != nil { errs <- fmt.Errorf("failed to delete key %q: %w", key, err) break } } t.Log("The second goroutine finished") }() t.Log("Waiting for the two goroutines to complete") go func() { wg.Wait() close(donec) }() t.Log("Waiting for test result") select { case err := <-errs: t.Fatalf("Unexpected error: %v", err) case <-donec: t.Log("All done!") case <-time.After(40 * time.Second): t.Fatal("Test case timeout after 40 seconds") } } ================================================ FILE: tests/e2e/ctl_v3_auth_security_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "strings" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/e2e" ) // TestAuth_CVE_2021_28235 verifies https://nvd.nist.gov/vuln/detail/CVE-2021-28235 func TestAuth_CVE_2021_28235(t *testing.T) { testCtl(t, authTestCVE2021_28235, withCfg(*e2e.NewConfigNoTLS()), withLogLevel("debug")) } func authTestCVE2021_28235(cx ctlCtx) { // create root user with root role rootPass := "changeme123" err := ctlV3User(cx, []string{"add", "root", "--interactive=false"}, "User root created", []string{rootPass}) require.NoError(cx.t, err) err = ctlV3User(cx, []string{"grant-role", "root", "root"}, "Role root is granted to user root", nil) require.NoError(cx.t, err) err = ctlV3AuthEnable(cx) require.NoError(cx.t, err) // issue a put request cx.user, cx.pass = "root", rootPass err = ctlV3Put(cx, "foo", "bar", "") require.NoError(cx.t, err) // GET /debug/requests httpEndpoint := cx.epc.Procs[0].EndpointsHTTP()[0] req := e2e.CURLReq{Endpoint: "/debug/requests?fam=grpc.Recv.etcdserverpb.Auth&b=0&exp=1", Timeout: 5} respData, err := curl(httpEndpoint, "GET", req, e2e.ClientNonTLS) require.NoError(cx.t, err) if strings.Contains(respData, rootPass) { cx.t.Errorf("The root password is included in the request.\n %s", respData) } } ================================================ FILE: tests/e2e/ctl_v3_auth_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "os" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3AuthMemberUpdate(t *testing.T) { testCtl(t, authTestMemberUpdate) } func TestCtlV3AuthFromKeyPerm(t *testing.T) { testCtl(t, authTestFromKeyPerm) } // TestCtlV3AuthAndWatch TODO https://github.com/etcd-io/etcd/issues/7988 is the blocker of migration to common/auth_test.go func TestCtlV3AuthAndWatch(t *testing.T) { testCtl(t, authTestWatch) } func TestCtlV3AuthAndWatchJWT(t *testing.T) { testCtl(t, authTestWatch, withCfg(*e2e.NewConfigJWT())) } // TestCtlV3AuthEndpointHealth https://github.com/etcd-io/etcd/pull/13774#discussion_r1189118815 is the blocker of migration to common/auth_test.go func TestCtlV3AuthEndpointHealth(t *testing.T) { testCtl(t, authTestEndpointHealth, withQuorum()) } // TestCtlV3AuthSnapshot TODO fill up common/maintenance_auth_test.go when Snapshot API is added in interfaces.Client func TestCtlV3AuthSnapshot(t *testing.T) { testCtl(t, authTestSnapshot) } func TestCtlV3AuthSnapshotJWT(t *testing.T) { testCtl(t, authTestSnapshot, withCfg(*e2e.NewConfigJWT())) } func TestCtlV3GetAuthStatus(t *testing.T) { testCtl(t, authTestGetAuthStatus) } func ctlV3AuthStatus(cx ctlCtx, expected string) error { cmd := append(cx.PrefixArgs(), "auth", "status") return e2e.SpawnWithExpectWithEnv(cmd, cx.envMap, expect.ExpectedResponse{Value: expected}) } func authTestGetAuthStatus(cx ctlCtx) { require.NoError(cx.t, ctlV3AuthStatus(cx, "Authentication Status: false")) require.NoError(cx.t, authEnable(cx)) require.NoError(cx.t, ctlV3AuthStatus(cx, "Authentication Status: true")) } func authEnable(cx ctlCtx) error { // create root user with root role if err := ctlV3User(cx, []string{"add", "root", "--interactive=false"}, "User root created", []string{"root"}); err != nil { return fmt.Errorf("failed to create root user %w", err) } if err := ctlV3User(cx, []string{"grant-role", "root", "root"}, "Role root is granted to user root", nil); err != nil { return fmt.Errorf("failed to grant root user root role %w", err) } if err := ctlV3AuthEnable(cx); err != nil { return fmt.Errorf("authEnableTest ctlV3AuthEnable error (%w)", err) } return nil } func ctlV3AuthEnable(cx ctlCtx) error { cmdArgs := append(cx.PrefixArgs(), "auth", "enable") return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: "Authentication Enabled"}) } func ctlV3PutFailPerm(cx ctlCtx, key, val string) error { return e2e.SpawnWithExpectWithEnv(append(cx.PrefixArgs(), "put", key, val), cx.envMap, expect.ExpectedResponse{Value: "permission denied"}) } func authSetupTestUser(cx ctlCtx) { err := ctlV3User(cx, []string{"add", "test-user", "--interactive=false"}, "User test-user created", []string{"pass"}) require.NoError(cx.t, err) err = e2e.SpawnWithExpectWithEnv(append(cx.PrefixArgs(), "role", "add", "test-role"), cx.envMap, expect.ExpectedResponse{Value: "Role test-role created"}) require.NoError(cx.t, err) err = ctlV3User(cx, []string{"grant-role", "test-user", "test-role"}, "Role test-role is granted to user test-user", nil) require.NoError(cx.t, err) cmd := append(cx.PrefixArgs(), "role", "grant-permission", "test-role", "readwrite", "foo") err = e2e.SpawnWithExpectWithEnv(cmd, cx.envMap, expect.ExpectedResponse{Value: "Role test-role updated"}) require.NoError(cx.t, err) } func authTestMemberUpdate(cx ctlCtx) { require.NoError(cx.t, authEnable(cx)) cx.user, cx.pass = "root", "root" authSetupTestUser(cx) mr, err := getMemberList(cx, false) require.NoError(cx.t, err) // ordinary user cannot update a member cx.user, cx.pass = "test-user", "pass" peerURL := fmt.Sprintf("http://localhost:%d", e2e.EtcdProcessBasePort+11) memberID := fmt.Sprintf("%x", mr.Members[0].ID) if err = ctlV3MemberUpdate(cx, memberID, peerURL); err == nil { cx.t.Fatalf("ordinary user must not be allowed to update a member") } // root can update a member cx.user, cx.pass = "root", "root" err = ctlV3MemberUpdate(cx, memberID, peerURL) require.NoError(cx.t, err) } func authTestCertCN(cx ctlCtx) { require.NoError(cx.t, authEnable(cx)) cx.user, cx.pass = "root", "root" require.NoError(cx.t, ctlV3User(cx, []string{"add", "example.com", "--interactive=false"}, "User example.com created", []string{""})) require.NoError(cx.t, e2e.SpawnWithExpectWithEnv(append(cx.PrefixArgs(), "role", "add", "test-role"), cx.envMap, expect.ExpectedResponse{Value: "Role test-role created"})) require.NoError(cx.t, ctlV3User(cx, []string{"grant-role", "example.com", "test-role"}, "Role test-role is granted to user example.com", nil)) // grant a new key require.NoError(cx.t, ctlV3RoleGrantPermission(cx, "test-role", grantingPerm{true, true, "hoo", "", false})) // try a granted key cx.user, cx.pass = "", "" if err := ctlV3Put(cx, "hoo", "bar", ""); err != nil { cx.t.Error(err) } // try a non-granted key cx.user, cx.pass = "", "" require.ErrorContains(cx.t, ctlV3PutFailPerm(cx, "baz", "bar"), "permission denied") } func authTestFromKeyPerm(cx ctlCtx) { require.NoError(cx.t, authEnable(cx)) cx.user, cx.pass = "root", "root" authSetupTestUser(cx) // grant keys after z to test-user cx.user, cx.pass = "root", "root" require.NoError(cx.t, ctlV3RoleGrantPermission(cx, "test-role", grantingPerm{true, true, "z", "\x00", false})) // try the granted open ended permission cx.user, cx.pass = "test-user", "pass" for i := 0; i < 10; i++ { key := fmt.Sprintf("z%d", i) require.NoError(cx.t, ctlV3Put(cx, key, "val", "")) } largeKey := "" for i := 0; i < 10; i++ { largeKey += "\xff" require.NoError(cx.t, ctlV3Put(cx, largeKey, "val", "")) } // try a non granted key require.ErrorContains(cx.t, ctlV3PutFailPerm(cx, "x", "baz"), "permission denied") // revoke the open ended permission cx.user, cx.pass = "root", "root" require.NoError(cx.t, ctlV3RoleRevokePermission(cx, "test-role", "z", "", true)) // try the revoked open ended permission cx.user, cx.pass = "test-user", "pass" for i := 0; i < 10; i++ { key := fmt.Sprintf("z%d", i) require.ErrorContains(cx.t, ctlV3PutFailPerm(cx, key, "val"), "permission denied") } // grant the entire keys cx.user, cx.pass = "root", "root" require.NoError(cx.t, ctlV3RoleGrantPermission(cx, "test-role", grantingPerm{true, true, "", "\x00", false})) // try keys, of course it must be allowed because test-role has a permission of the entire keys cx.user, cx.pass = "test-user", "pass" for i := 0; i < 10; i++ { key := fmt.Sprintf("z%d", i) require.NoError(cx.t, ctlV3Put(cx, key, "val", "")) } // revoke the entire keys cx.user, cx.pass = "root", "root" require.NoError(cx.t, ctlV3RoleRevokePermission(cx, "test-role", "", "", true)) // try the revoked entire key permission cx.user, cx.pass = "test-user", "pass" for i := 0; i < 10; i++ { key := fmt.Sprintf("z%d", i) err := ctlV3PutFailPerm(cx, key, "val") require.ErrorContains(cx.t, err, "permission denied") } } func authTestWatch(cx ctlCtx) { require.NoError(cx.t, authEnable(cx)) cx.user, cx.pass = "root", "root" authSetupTestUser(cx) // grant a key range require.NoError(cx.t, ctlV3RoleGrantPermission(cx, "test-role", grantingPerm{true, true, "key", "key4", false})) tests := []struct { puts []kv args []string wkv []kvExec want bool }{ { // watch 1 key, should be successful []kv{{"key", "value"}}, []string{"key", "--rev", "1"}, []kvExec{{key: "key", val: "value"}}, true, }, { // watch 3 keys by range, should be successful []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}}, []string{"key", "key3", "--rev", "1"}, []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}}, true, }, { // watch 1 key, should not be successful []kv{}, []string{"key5", "--rev", "1"}, []kvExec{}, false, }, { // watch 3 keys by range, should not be successful []kv{}, []string{"key", "key6", "--rev", "1"}, []kvExec{}, false, }, } cx.user, cx.pass = "test-user", "pass" for i, tt := range tests { donec := make(chan struct{}) go func(i int, puts []kv) { defer close(donec) for j := range puts { if err := ctlV3Put(cx, puts[j].key, puts[j].val, ""); err != nil { cx.t.Errorf("watchTest #%d-%d: ctlV3Put error (%v)", i, j, err) } } }(i, tt.puts) var err error if tt.want { err = ctlV3Watch(cx, tt.args, tt.wkv...) if err != nil && cx.dialTimeout > 0 && !isGRPCTimedout(err) { cx.t.Errorf("watchTest #%d: ctlV3Watch error (%v)", i, err) } } else { err = ctlV3WatchFailPerm(cx, tt.args) // this will not have any meaningful error output, but the process fails due to the cancellation require.ErrorContains(cx.t, err, "unexpected exit code") } <-donec } } func authTestSnapshot(cx ctlCtx) { maintenanceInitKeys(cx) require.NoError(cx.t, authEnable(cx)) cx.user, cx.pass = "root", "root" authSetupTestUser(cx) fpath := "test-auth.snapshot" defer os.RemoveAll(fpath) // ordinary user cannot save a snapshot cx.user, cx.pass = "test-user", "pass" require.Errorf(cx.t, ctlV3SnapshotSave(cx, fpath), "ordinary user should not be able to save a snapshot") // root can save a snapshot cx.user, cx.pass = "root", "root" require.NoErrorf(cx.t, ctlV3SnapshotSave(cx, fpath), "snapshotTest ctlV3SnapshotSave error") st, err := getSnapshotStatus(cx, fpath) require.NoErrorf(cx.t, err, "snapshotTest getSnapshotStatus error") if st.Revision != 4 { cx.t.Fatalf("expected 4, got %d", st.Revision) } if st.TotalKey < 1 { cx.t.Fatalf("expected at least 1, got %d", st.TotalKey) } } func authTestEndpointHealth(cx ctlCtx) { require.NoError(cx.t, authEnable(cx)) cx.user, cx.pass = "root", "root" authSetupTestUser(cx) require.NoErrorf(cx.t, ctlV3EndpointHealth(cx), "endpointStatusTest ctlV3EndpointHealth error") require.NoError(cx.t, ctlV3RoleGrantPermission(cx, "test-role", grantingPerm{true, true, "health", "", false})) cx.user, cx.pass = "test-user", "pass" func(cx ctlCtx) { cmdArgs := append(cx.PrefixArgs(), "endpoint", "health") lines := make([]expect.ExpectedResponse, cx.epc.Cfg.ClusterSize) for i := range lines { lines[i] = expect.ExpectedResponse{ Value: cx.epc.Procs[i].EndpointsGRPC()[0] + " is unhealthy: failed to commit proposal: Unable to fetch the alarm list", } } proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) require.NoErrorf(cx.t, err, "failed to spawn endpoint health command") defer func() { require.Errorf(cx.t, proc.Close(), "endpoint health command should reject all non-root users") }() for _, line := range lines { _, lerr := proc.ExpectWithContext(context.TODO(), line) require.NoErrorf(cx.t, lerr, "endpoint health should fail with permission denied error") } }(cx) } func certCNAndUsername(cx ctlCtx, noPassword bool) { require.NoError(cx.t, authEnable(cx)) cx.user, cx.pass = "root", "root" authSetupTestUser(cx) if noPassword { require.NoError(cx.t, ctlV3User(cx, []string{"add", "example.com", "--no-password"}, "User example.com created", []string{""})) } else { require.NoError(cx.t, ctlV3User(cx, []string{"add", "example.com", "--interactive=false"}, "User example.com created", []string{""})) } require.NoError(cx.t, e2e.SpawnWithExpectWithEnv(append(cx.PrefixArgs(), "role", "add", "test-role-cn"), cx.envMap, expect.ExpectedResponse{Value: "Role test-role-cn created"})) require.NoError(cx.t, ctlV3User(cx, []string{"grant-role", "example.com", "test-role-cn"}, "Role test-role-cn is granted to user example.com", nil)) // grant a new key for CN based user require.NoError(cx.t, ctlV3RoleGrantPermission(cx, "test-role-cn", grantingPerm{true, true, "hoo", "", false})) // grant a new key for username based user require.NoError(cx.t, ctlV3RoleGrantPermission(cx, "test-role", grantingPerm{true, true, "bar", "", false})) // try a granted key for CN based user cx.user, cx.pass = "", "" require.NoError(cx.t, ctlV3Put(cx, "hoo", "bar", "")) // try a granted key for username based user cx.user, cx.pass = "test-user", "pass" require.NoError(cx.t, ctlV3Put(cx, "bar", "bar", "")) // try a non-granted key for both of them cx.user, cx.pass = "", "" require.ErrorContains(cx.t, ctlV3PutFailPerm(cx, "baz", "bar"), "permission denied") cx.user, cx.pass = "test-user", "pass" require.ErrorContains(cx.t, ctlV3PutFailPerm(cx, "baz", "bar"), "permission denied") } func authTestCertCNAndUsername(cx ctlCtx) { certCNAndUsername(cx, false) } func authTestCertCNAndUsernameNoPassword(cx ctlCtx) { certCNAndUsername(cx, true) } func ctlV3EndpointHealth(cx ctlCtx) error { cmdArgs := append(cx.PrefixArgs(), "endpoint", "health") lines := make([]expect.ExpectedResponse, cx.epc.Cfg.ClusterSize) for i := range lines { lines[i] = expect.ExpectedResponse{Value: "is healthy"} } return e2e.SpawnWithExpects(cmdArgs, cx.envMap, lines...) } func ctlV3User(cx ctlCtx, args []string, expStr string, stdIn []string) error { cmdArgs := append(cx.PrefixArgs(), "user") cmdArgs = append(cmdArgs, args...) proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) if err != nil { return err } defer proc.Close() // Send 'stdIn' strings as input. for _, s := range stdIn { if err = proc.Send(s + "\r"); err != nil { return err } } _, err = proc.Expect(expStr) return err } ================================================ FILE: tests/e2e/ctl_v3_completion_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "bytes" "fmt" "os" "os/exec" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3CompletionBash(t *testing.T) { testShellCompletion(t, e2e.BinPath.Etcdctl, "bash") } func TestUtlV3CompletionBash(t *testing.T) { testShellCompletion(t, e2e.BinPath.Etcdutl, "bash") } // testShellCompletion can only run in non-coverage mode. The etcdctl and etcdutl // built with `-tags cov` mode will show go-test result after each execution, like // // PASS // coverage: 0.0% of statements in ./... // // Since the PASS is not real command, the `source completion" fails with // command-not-found error. func testShellCompletion(t *testing.T, binPath, shellName string) { e2e.BeforeTest(t) stdout := new(bytes.Buffer) completionCmd := exec.Command(binPath, "completion", shellName) completionCmd.Stdout = stdout completionCmd.Stderr = os.Stderr require.NoError(t, completionCmd.Run()) filename := fmt.Sprintf("etcdctl-%s.completion", shellName) require.NoError(t, os.WriteFile(filename, stdout.Bytes(), 0o644)) shellCmd := exec.Command(shellName, "-c", "source "+filename) require.NoError(t, shellCmd.Run()) } ================================================ FILE: tests/e2e/ctl_v3_defrag_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3DefragOffline(t *testing.T) { testCtlWithOffline(t, maintenanceInitKeys, defragOfflineTest) } func maintenanceInitKeys(cx ctlCtx) { kvs := []kv{{"key", "val1"}, {"key", "val2"}, {"key", "val3"}} for i := range kvs { require.NoError(cx.t, ctlV3Put(cx, kvs[i].key, kvs[i].val, "")) } } func ctlV3OfflineDefrag(cx ctlCtx) error { cmdArgs := append(cx.PrefixArgsUtl(), "defrag", "--data-dir", cx.dataDir) lines := []expect.ExpectedResponse{{Value: "finished defragmenting directory"}} return e2e.SpawnWithExpects(cmdArgs, cx.envMap, lines...) } func defragOfflineTest(cx ctlCtx) { if err := ctlV3OfflineDefrag(cx); err != nil { cx.t.Fatalf("defragTest ctlV3Defrag error (%v)", err) } } ================================================ FILE: tests/e2e/ctl_v3_elect_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "os" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3Elect(t *testing.T) { testCtl(t, testElect) } func testElect(cx ctlCtx) { name := "a" holder, ch, err := ctlV3Elect(cx, name, "p1", false) require.NoError(cx.t, err) l1 := "" select { case <-time.After(2 * time.Second): cx.t.Fatalf("timed out electing") case l1 = <-ch: if !strings.HasPrefix(l1, name) { cx.t.Errorf("got %q, expected %q prefix", l1, name) } } // blocked process that won't win the election blocked, ch, err := ctlV3Elect(cx, name, "p2", true) require.NoError(cx.t, err) select { case <-time.After(100 * time.Millisecond): case <-ch: cx.t.Fatalf("should block") } // overlap with a blocker that will win the election blockAcquire, ch, err := ctlV3Elect(cx, name, "p2", false) require.NoError(cx.t, err) defer func(blockAcquire *expect.ExpectProcess) { err = blockAcquire.Stop() require.NoError(cx.t, err) blockAcquire.Wait() }(blockAcquire) select { case <-time.After(100 * time.Millisecond): case <-ch: cx.t.Fatalf("should block") } // kill blocked process with clean shutdown require.NoError(cx.t, blocked.Signal(os.Interrupt)) err = e2e.CloseWithTimeout(blocked, time.Second) if err != nil { // due to being blocked, this can potentially get killed and thus exit non-zero sometimes require.ErrorContains(cx.t, err, "unexpected exit code") } // kill the holder with clean shutdown require.NoError(cx.t, holder.Signal(os.Interrupt)) require.NoError(cx.t, e2e.CloseWithTimeout(holder, time.Second)) // blockAcquire should win the election select { case <-time.After(time.Second): cx.t.Fatalf("timed out from waiting to holding") case l2 := <-ch: if l1 == l2 || !strings.HasPrefix(l2, name) { cx.t.Fatalf("expected different elect name, got l1=%q, l2=%q", l1, l2) } } } // ctlV3Elect creates a elect process with a channel listening for when it wins the election. func ctlV3Elect(cx ctlCtx, name, proposal string, expectFailure bool) (*expect.ExpectProcess, <-chan string, error) { cmdArgs := append(cx.PrefixArgs(), "elect", name, proposal) proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) outc := make(chan string, 1) if err != nil { close(outc) return proc, outc, err } go func() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() s, xerr := proc.ExpectFunc(ctx, func(string) bool { return true }) if xerr != nil { if !expectFailure { cx.t.Errorf("expect failed (%v)", xerr) } } outc <- s }() return proc, outc, err } ================================================ FILE: tests/e2e/ctl_v3_kv_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "net" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3PutTimeout(t *testing.T) { testCtl(t, putTest, withDefaultDialTimeout()) } func TestCtlV3PutClientTLSFlagByEnv(t *testing.T) { testCtl(t, putTest, withCfg(*e2e.NewConfigClientTLS()), withFlagByEnv()) } func TestCtlV3PutIgnoreValue(t *testing.T) { testCtl(t, putTestIgnoreValue) } func TestCtlV3PutIgnoreLease(t *testing.T) { testCtl(t, putTestIgnoreLease) } func TestCtlV3GetTimeout(t *testing.T) { testCtl(t, getTest, withDefaultDialTimeout()) } func TestCtlV3GetFormat(t *testing.T) { testCtl(t, getFormatTest) } func TestCtlV3GetRev(t *testing.T) { testCtl(t, getRevTest) } func TestCtlV3GetMinMaxCreateModRev(t *testing.T) { testCtl(t, getMinMaxCreateModRevTest) } func TestCtlV3GetKeysOnly(t *testing.T) { testCtl(t, getKeysOnlyTest) } func TestCtlV3GetCountOnly(t *testing.T) { testCtl(t, getCountOnlyTest) } func TestCtlV3DelTimeout(t *testing.T) { testCtl(t, delTest, withDefaultDialTimeout()) } func TestCtlV3TimeoutWhenNoProcessListensOnEndpoint(t *testing.T) { e2e.BeforeTest(t) endpoint := unusedLocalTCPAddr(t) cmdArgs := []string{ e2e.BinPath.Etcdctl, "--debug", "--endpoints", endpoint, "--dial-timeout", "5s", "get", "foo", } assertEtcdctlDialTimedout(t, cmdArgs, nil) } func TestCtlV3TimeoutWhenTLSClientCertMissing(t *testing.T) { testCtl(t, timeoutWhenTLSClientCertMissingTest, withCfg(*e2e.NewConfigClientTLS())) } func TestCtlV3GetRevokedCRL(t *testing.T) { cfg := e2e.NewConfig( e2e.WithClusterSize(1), e2e.WithClientConnType(e2e.ClientTLS), e2e.WithClientRevokeCerts(true), e2e.WithClientCertAuthority(true), ) testCtl(t, testGetRevokedCRL, withCfg(*cfg)) } func testGetRevokedCRL(cx ctlCtx) { // test reject require.ErrorContains(cx.t, ctlV3Put(cx, "k", "v", ""), "context deadline exceeded") // test accept cx.epc.Cfg.Client.RevokeCerts = false require.NoError(cx.t, ctlV3Put(cx, "k", "v", "")) } func timeoutWhenTLSClientCertMissingTest(cx ctlCtx) { cmdArgs := []string{ e2e.BinPath.Etcdctl, "--debug", "--endpoints", strings.Join(cx.epc.EndpointsGRPC(), ","), "--dial-timeout", "5s", "get", "foo", } assertEtcdctlDialTimedout(cx.t, cmdArgs, nil) } func unusedLocalTCPAddr(t *testing.T) string { t.Helper() l, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) defer l.Close() return l.Addr().String() } func assertEtcdctlDialTimedout(t *testing.T, cmdArgs []string, envVars map[string]string) { t.Helper() proc, err := e2e.SpawnCmd(cmdArgs, envVars) require.NoError(t, err) proc.Wait() err = proc.Close() require.Error(t, err) out := strings.Join(proc.Lines(), "\n") require.Containsf(t, out, context.DeadlineExceeded.Error(), "expected timeout output, got close error: %v, output: %q", err, out, ) } func putTest(cx ctlCtx) { key, value := "foo", "bar" if err := ctlV3Put(cx, key, value, ""); err != nil { if cx.dialTimeout > 0 && !isGRPCTimedout(err) { cx.t.Fatalf("putTest ctlV3Put error (%v)", err) } } if err := ctlV3Get(cx, []string{key}, kv{key, value}); err != nil { if cx.dialTimeout > 0 && !isGRPCTimedout(err) { cx.t.Fatalf("putTest ctlV3Get error (%v)", err) } } } func putTestIgnoreValue(cx ctlCtx) { require.NoError(cx.t, ctlV3Put(cx, "foo", "bar", "")) require.NoError(cx.t, ctlV3Get(cx, []string{"foo"}, kv{"foo", "bar"})) require.NoError(cx.t, ctlV3Put(cx, "foo", "", "", "--ignore-value")) require.NoError(cx.t, ctlV3Get(cx, []string{"foo"}, kv{"foo", "bar"})) } func putTestIgnoreLease(cx ctlCtx) { leaseID, err := ctlV3LeaseGrant(cx, 10) if err != nil { cx.t.Fatalf("putTestIgnoreLease: ctlV3LeaseGrant error (%v)", err) } if err := ctlV3Put(cx, "foo", "bar", leaseID); err != nil { cx.t.Fatalf("putTestIgnoreLease: ctlV3Put error (%v)", err) } if err := ctlV3Get(cx, []string{"foo"}, kv{"foo", "bar"}); err != nil { cx.t.Fatalf("putTestIgnoreLease: ctlV3Get error (%v)", err) } if err := ctlV3Put(cx, "foo", "bar1", "", "--ignore-lease"); err != nil { cx.t.Fatalf("putTestIgnoreLease: ctlV3Put error (%v)", err) } if err := ctlV3Get(cx, []string{"foo"}, kv{"foo", "bar1"}); err != nil { cx.t.Fatalf("putTestIgnoreLease: ctlV3Get error (%v)", err) } if err := ctlV3LeaseRevoke(cx, leaseID); err != nil { cx.t.Fatalf("putTestIgnoreLease: ctlV3LeaseRevok error (%v)", err) } if err := ctlV3Get(cx, []string{"key"}); err != nil { // expect no output cx.t.Fatalf("putTestIgnoreLease: ctlV3Get error (%v)", err) } } func getTest(cx ctlCtx) { var ( kvs = []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}} revkvs = []kv{{"key3", "val3"}, {"key2", "val2"}, {"key1", "val1"}} ) for i := range kvs { if err := ctlV3Put(cx, kvs[i].key, kvs[i].val, ""); err != nil { cx.t.Fatalf("getTest #%d: ctlV3Put error (%v)", i, err) } } tests := []struct { args []string wkv []kv }{ {[]string{"key1"}, []kv{{"key1", "val1"}}}, {[]string{"", "--prefix"}, kvs}, {[]string{"", "--from-key"}, kvs}, {[]string{"key", "--prefix"}, kvs}, {[]string{"key", "--prefix", "--limit=2"}, kvs[:2]}, {[]string{"key", "--prefix", "--order=ASCEND", "--sort-by=MODIFY"}, kvs}, {[]string{"key", "--prefix", "--order=ASCEND", "--sort-by=VERSION"}, kvs}, {[]string{"key", "--prefix", "--sort-by=CREATE"}, kvs}, // ASCEND by default {[]string{"key", "--prefix", "--order=DESCEND", "--sort-by=CREATE"}, revkvs}, {[]string{"key", "--prefix", "--order=DESCEND", "--sort-by=KEY"}, revkvs}, } for i, tt := range tests { if err := ctlV3Get(cx, tt.args, tt.wkv...); err != nil { if cx.dialTimeout > 0 && !isGRPCTimedout(err) { cx.t.Errorf("getTest #%d: ctlV3Get error (%v)", i, err) } } } } func getFormatTest(cx ctlCtx) { require.NoError(cx.t, ctlV3Put(cx, "abc", "123", "")) tests := []struct { format string valueOnly bool wstr string }{ {"simple", false, "abc"}, {"simple", true, "123"}, {"json", false, `"kvs":[{"key":"YWJj"`}, {"protobuf", false, "\x17\b\x93\xe7\xf6\x93\xd4ņ\xe14\x10\xed"}, } for i, tt := range tests { cmdArgs := append(cx.PrefixArgs(), "get") cmdArgs = append(cmdArgs, "--write-out="+tt.format) if tt.valueOnly { cmdArgs = append(cmdArgs, "--print-value-only") } cmdArgs = append(cmdArgs, "abc") lines, err := e2e.RunUtilCompletion(cmdArgs, cx.envMap) if err != nil { cx.t.Errorf("#%d: error (%v)", i, err) } assert.Contains(cx.t, strings.Join(lines, "\n"), tt.wstr) } } func getRevTest(cx ctlCtx) { kvs := []kv{{"key", "val1"}, {"key", "val2"}, {"key", "val3"}} for i := range kvs { if err := ctlV3Put(cx, kvs[i].key, kvs[i].val, ""); err != nil { cx.t.Fatalf("getRevTest #%d: ctlV3Put error (%v)", i, err) } } tests := []struct { args []string wkv []kv }{ {[]string{"key", "--rev", "2"}, kvs[:1]}, {[]string{"key", "--rev", "3"}, kvs[1:2]}, {[]string{"key", "--rev", "4"}, kvs[2:]}, } for i, tt := range tests { if err := ctlV3Get(cx, tt.args, tt.wkv...); err != nil { cx.t.Errorf("getTest #%d: ctlV3Get error (%v)", i, err) } } } func getMinMaxCreateModRevTest(cx ctlCtx) { kvs := []kv{ // revision: store | key create | key modify {"key1", "val1"}, // 2 2 2 {"key2", "val2"}, // 3 3 3 {"key1", "val3"}, // 4 2 4 {"key4", "val4"}, // 5 5 5 } for i := range kvs { if err := ctlV3Put(cx, kvs[i].key, kvs[i].val, ""); err != nil { cx.t.Fatalf("getRevTest #%d: ctlV3Put error (%v)", i, err) } } tests := []struct { args []string wkv []kv }{ {[]string{"key", "--prefix", "--max-create-rev", "3"}, []kv{kvs[1], kvs[2]}}, {[]string{"key", "--prefix", "--min-create-rev", "3"}, []kv{kvs[1], kvs[3]}}, {[]string{"key", "--prefix", "--max-mod-rev", "3"}, []kv{kvs[1]}}, {[]string{"key", "--prefix", "--min-mod-rev", "4"}, kvs[2:]}, } for i, tt := range tests { if err := ctlV3Get(cx, tt.args, tt.wkv...); err != nil { cx.t.Errorf("getMinModRevTest #%d: ctlV3Get error (%v)", i, err) } } } func getKeysOnlyTest(cx ctlCtx) { require.NoError(cx.t, ctlV3Put(cx, "key", "val", "")) cmdArgs := append(cx.PrefixArgs(), []string{"get", "--keys-only", "key"}...) require.NoError(cx.t, e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: "key"})) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() lines, err := e2e.SpawnWithExpectLines(ctx, cmdArgs, cx.envMap, expect.ExpectedResponse{Value: "key"}) require.NoError(cx.t, err) require.NotContainsf(cx.t, lines, "val", "got value but passed --keys-only") } func getCountOnlyTest(cx ctlCtx) { cmdArgs := append(cx.PrefixArgs(), []string{"get", "--count-only", "key", "--prefix", "--write-out=fields"}...) require.NoError(cx.t, e2e.SpawnWithExpects(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: "\"Count\" : 0"})) require.NoError(cx.t, ctlV3Put(cx, "key", "val", "")) cmdArgs = append(cx.PrefixArgs(), []string{"get", "--count-only", "key", "--prefix", "--write-out=fields"}...) require.NoError(cx.t, e2e.SpawnWithExpects(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: "\"Count\" : 1"})) require.NoError(cx.t, ctlV3Put(cx, "key1", "val", "")) require.NoError(cx.t, ctlV3Put(cx, "key1", "val", "")) cmdArgs = append(cx.PrefixArgs(), []string{"get", "--count-only", "key", "--prefix", "--write-out=fields"}...) require.NoError(cx.t, e2e.SpawnWithExpects(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: "\"Count\" : 2"})) require.NoError(cx.t, ctlV3Put(cx, "key2", "val", "")) cmdArgs = append(cx.PrefixArgs(), []string{"get", "--count-only", "key", "--prefix", "--write-out=fields"}...) require.NoError(cx.t, e2e.SpawnWithExpects(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: "\"Count\" : 3"})) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() cmdArgs = append(cx.PrefixArgs(), []string{"get", "--count-only", "key3", "--prefix", "--write-out=fields"}...) lines, err := e2e.SpawnWithExpectLines(ctx, cmdArgs, cx.envMap, expect.ExpectedResponse{Value: "\"Count\""}) require.NoError(cx.t, err) require.NotContains(cx.t, lines, "\"Count\" : 3") } func delTest(cx ctlCtx) { tests := []struct { puts []kv args []string deletedNum int }{ { // delete all keys []kv{{"foo1", "bar"}, {"foo2", "bar"}, {"foo3", "bar"}}, []string{"", "--prefix"}, 3, }, { // delete all keys []kv{{"foo1", "bar"}, {"foo2", "bar"}, {"foo3", "bar"}}, []string{"", "--from-key"}, 3, }, { []kv{{"this", "value"}}, []string{"that"}, 0, }, { []kv{{"sample", "value"}}, []string{"sample"}, 1, }, { []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}}, []string{"key", "--prefix"}, 3, }, { []kv{{"zoo1", "bar"}, {"zoo2", "bar2"}, {"zoo3", "bar3"}}, []string{"zoo1", "--from-key"}, 3, }, } for i, tt := range tests { for j := range tt.puts { if err := ctlV3Put(cx, tt.puts[j].key, tt.puts[j].val, ""); err != nil { cx.t.Fatalf("delTest #%d-%d: ctlV3Put error (%v)", i, j, err) } } if err := ctlV3Del(cx, tt.args, tt.deletedNum); err != nil { if cx.dialTimeout > 0 && !isGRPCTimedout(err) { cx.t.Fatalf("delTest #%d: ctlV3Del error (%v)", i, err) } } } } func ctlV3Put(cx ctlCtx, key, value, leaseID string, flags ...string) error { skipValue := false skipLease := false for _, f := range flags { if f == "--ignore-value" { skipValue = true } if f == "--ignore-lease" { skipLease = true } } cmdArgs := append(cx.PrefixArgs(), "put", key) if !skipValue { cmdArgs = append(cmdArgs, value) } if leaseID != "" && !skipLease { cmdArgs = append(cmdArgs, "--lease", leaseID) } if len(flags) != 0 { cmdArgs = append(cmdArgs, flags...) } return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: "OK"}) } type kv struct { key, val string } func ctlV3Get(cx ctlCtx, args []string, kvs ...kv) error { cmdArgs := append(cx.PrefixArgs(), "get") cmdArgs = append(cmdArgs, args...) if !cx.quorum { cmdArgs = append(cmdArgs, "--consistency", "s") } var lines []expect.ExpectedResponse for _, elem := range kvs { lines = append(lines, expect.ExpectedResponse{Value: elem.key}, expect.ExpectedResponse{Value: elem.val}) } return e2e.SpawnWithExpects(cmdArgs, cx.envMap, lines...) } func ctlV3Del(cx ctlCtx, args []string, num int) error { cmdArgs := append(cx.PrefixArgs(), "del") cmdArgs = append(cmdArgs, args...) return e2e.SpawnWithExpects(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: fmt.Sprintf("%d", num)}) } ================================================ FILE: tests/e2e/ctl_v3_lease_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "strconv" "strings" "testing" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3LeaseKeepAlive(t *testing.T) { cfg := e2e.NewConfigAutoTLS() enableFastLeaseKeepAlive(cfg) testCtl(t, leaseTestKeepAlive, withCfg(*cfg)) } func TestCtlV3LeaseKeepAliveDisableFastKeepAlive(t *testing.T) { cfg := e2e.NewConfigAutoTLS() disableFastLeaseKeepAlive(cfg) testCtl(t, leaseTestKeepAlive, withCfg(*cfg)) } func TestCtlV3LeaseKeepAliveNoTLS(t *testing.T) { cfg := e2e.NewConfigNoTLS() enableFastLeaseKeepAlive(cfg) testCtl(t, leaseTestKeepAlive, withCfg(*cfg)) } func TestCtlV3LeaseKeepAliveNoTLSDisableFastKeepAlive(t *testing.T) { cfg := e2e.NewConfigNoTLS() disableFastLeaseKeepAlive(cfg) testCtl(t, leaseTestKeepAlive, withCfg(*cfg)) } func TestCtlV3LeaseKeepAliveClientTLS(t *testing.T) { cfg := e2e.NewConfigClientTLS() enableFastLeaseKeepAlive(cfg) testCtl(t, leaseTestKeepAlive, withCfg(*cfg)) } func TestCtlV3LeaseKeepAliveClientTLSDisableFastKeepAlive(t *testing.T) { cfg := e2e.NewConfigClientTLS() disableFastLeaseKeepAlive(cfg) testCtl(t, leaseTestKeepAlive, withCfg(*cfg)) } func TestCtlV3LeaseKeepAliveClientAutoTLS(t *testing.T) { cfg := e2e.NewConfigClientAutoTLS() enableFastLeaseKeepAlive(cfg) testCtl(t, leaseTestKeepAlive, withCfg(*cfg)) } func TestCtlV3LeaseKeepAliveClientAutoTLSDisableFastKeepAlive(t *testing.T) { cfg := e2e.NewConfigClientAutoTLS() disableFastLeaseKeepAlive(cfg) testCtl(t, leaseTestKeepAlive, withCfg(*cfg)) } func TestCtlV3LeaseKeepAlivePeerTLS(t *testing.T) { cfg := e2e.NewConfigPeerTLS() enableFastLeaseKeepAlive(cfg) testCtl(t, leaseTestKeepAlive, withCfg(*cfg)) } func TestCtlV3LeaseKeepAlivePeerTLSDisableFastKeepAlive(t *testing.T) { cfg := e2e.NewConfigPeerTLS() disableFastLeaseKeepAlive(cfg) testCtl(t, leaseTestKeepAlive, withCfg(*cfg)) } func enableFastLeaseKeepAlive(cfg *e2e.EtcdProcessClusterConfig) { e2e.WithServerFeatureGate("FastLeaseKeepAlive", true)(cfg) } func disableFastLeaseKeepAlive(cfg *e2e.EtcdProcessClusterConfig) { e2e.WithServerFeatureGate("FastLeaseKeepAlive", false)(cfg) } func leaseTestKeepAlive(cx ctlCtx) { // put with TTL 10 seconds and keep-alive leaseID, err := ctlV3LeaseGrant(cx, 10) if err != nil { cx.t.Fatalf("leaseTestKeepAlive: ctlV3LeaseGrant error (%v)", err) } if err := ctlV3Put(cx, "key", "val", leaseID); err != nil { cx.t.Fatalf("leaseTestKeepAlive: ctlV3Put error (%v)", err) } if err := ctlV3LeaseKeepAlive(cx, leaseID); err != nil { cx.t.Fatalf("leaseTestKeepAlive: ctlV3LeaseKeepAlive error (%v)", err) } if err := ctlV3Get(cx, []string{"key"}, kv{"key", "val"}); err != nil { cx.t.Fatalf("leaseTestKeepAlive: ctlV3Get error (%v)", err) } } func ctlV3LeaseGrant(cx ctlCtx, ttl int) (string, error) { cmdArgs := append(cx.PrefixArgs(), "lease", "grant", strconv.Itoa(ttl)) proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) if err != nil { return "", err } line, err := proc.Expect(" granted with TTL(") if err != nil { return "", err } if err = proc.Close(); err != nil { return "", err } // parse 'line LEASE_ID granted with TTL(5s)' to get lease ID hs := strings.Split(line, " ") if len(hs) < 2 { return "", fmt.Errorf("lease grant failed with %q", line) } return hs[1], nil } func ctlV3LeaseKeepAlive(cx ctlCtx, leaseID string) error { cmdArgs := append(cx.PrefixArgs(), "lease", "keep-alive", leaseID) proc, err := e2e.SpawnCmd(cmdArgs, nil) if err != nil { return err } if _, err = proc.Expect(fmt.Sprintf("lease %s keepalived with TTL(", leaseID)); err != nil { return err } return proc.Stop() } func ctlV3LeaseRevoke(cx ctlCtx, leaseID string) error { cmdArgs := append(cx.PrefixArgs(), "lease", "revoke", leaseID) return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: fmt.Sprintf("lease %s revoked", leaseID)}) } ================================================ FILE: tests/e2e/ctl_v3_lock_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "os" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3Lock(t *testing.T) { testCtl(t, testLock) } func TestCtlV3LockWithCmd(t *testing.T) { testCtl(t, testLockWithCmd) } func testLock(cx ctlCtx) { name := "a" holder, ch, err := ctlV3Lock(cx, name) require.NoError(cx.t, err) l1 := "" select { case <-time.After(2 * time.Second): cx.t.Fatalf("timed out locking") case l1 = <-ch: if !strings.HasPrefix(l1, name) { cx.t.Errorf("got %q, expected %q prefix", l1, name) } } // blocked process that won't acquire the lock blocked, ch, err := ctlV3Lock(cx, name) require.NoError(cx.t, err) select { case <-time.After(100 * time.Millisecond): case <-ch: cx.t.Fatalf("should block") } // overlap with a blocker that will acquire the lock blockAcquire, ch, err := ctlV3Lock(cx, name) require.NoError(cx.t, err) defer func(blockAcquire *expect.ExpectProcess) { err = blockAcquire.Stop() require.NoError(cx.t, err) blockAcquire.Wait() }(blockAcquire) select { case <-time.After(100 * time.Millisecond): case <-ch: cx.t.Fatalf("should block") } // kill blocked process with clean shutdown require.NoError(cx.t, blocked.Signal(os.Interrupt)) err = e2e.CloseWithTimeout(blocked, time.Second) if err != nil { // due to being blocked, this can potentially get killed and thus exit non-zero sometimes require.ErrorContains(cx.t, err, "unexpected exit code") } // kill the holder with clean shutdown require.NoError(cx.t, holder.Signal(os.Interrupt)) require.NoError(cx.t, e2e.CloseWithTimeout(holder, 200*time.Millisecond+time.Second)) // blockAcquire should acquire the lock select { case <-time.After(time.Second): cx.t.Fatalf("timed out from waiting to holding") case l2 := <-ch: if l1 == l2 || !strings.HasPrefix(l2, name) { cx.t.Fatalf("expected different lock name, got l1=%q, l2=%q", l1, l2) } } } func testLockWithCmd(cx ctlCtx) { // exec command with zero exit code echoCmd := []string{"echo"} require.NoError(cx.t, ctlV3LockWithCmd(cx, echoCmd, expect.ExpectedResponse{Value: ""})) // exec command with non-zero exit code code := 3 awkCmd := []string{"awk", fmt.Sprintf("BEGIN{exit %d}", code)} expect := expect.ExpectedResponse{Value: fmt.Sprintf("Error: exit status %d", code)} require.ErrorContains(cx.t, ctlV3LockWithCmd(cx, awkCmd, expect), expect.Value) } // ctlV3Lock creates a lock process with a channel listening for when it acquires the lock. func ctlV3Lock(cx ctlCtx, name string) (*expect.ExpectProcess, <-chan string, error) { cmdArgs := append(cx.PrefixArgs(), "lock", name) proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) outc := make(chan string, 1) if err != nil { close(outc) return proc, outc, err } go func() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() s, xerr := proc.ExpectFunc(ctx, func(string) bool { return true }) if xerr != nil { require.ErrorContains(cx.t, xerr, "Error: context canceled") } outc <- s }() return proc, outc, err } // ctlV3LockWithCmd creates a lock process to exec command. func ctlV3LockWithCmd(cx ctlCtx, execCmd []string, as ...expect.ExpectedResponse) error { // use command as lock name cmdArgs := append(cx.PrefixArgs(), "lock", execCmd[0]) cmdArgs = append(cmdArgs, execCmd...) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() return e2e.SpawnWithExpectsContext(ctx, cmdArgs, cx.envMap, as...) } ================================================ FILE: tests/e2e/ctl_v3_make_mirror_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3MakeMirror(t *testing.T) { testCtl(t, makeMirrorTest) } func TestCtlV3MakeMirrorModifyDestPrefix(t *testing.T) { testCtl(t, makeMirrorModifyDestPrefixTest) } func TestCtlV3MakeMirrorNoDestPrefix(t *testing.T) { testCtl(t, makeMirrorNoDestPrefixTest) } func TestCtlV3MakeMirrorWithWatchRev(t *testing.T) { testCtl(t, makeMirrorWithWatchRev) } func makeMirrorTest(cx ctlCtx) { var ( flags []string kvs = []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}} kvs2 = []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}} prefix = "key" ) testMirrorCommand(cx, flags, kvs, kvs2, prefix, prefix) } func makeMirrorModifyDestPrefixTest(cx ctlCtx) { var ( flags = []string{"--prefix", "o_", "--dest-prefix", "d_"} kvs = []kv{{"o_key1", "val1"}, {"o_key2", "val2"}, {"o_key3", "val3"}} kvs2 = []kvExec{{key: "d_key1", val: "val1"}, {key: "d_key2", val: "val2"}, {key: "d_key3", val: "val3"}} srcprefix = "o_" destprefix = "d_" ) testMirrorCommand(cx, flags, kvs, kvs2, srcprefix, destprefix) } func makeMirrorNoDestPrefixTest(cx ctlCtx) { var ( flags = []string{"--prefix", "o_", "--no-dest-prefix"} kvs = []kv{{"o_key1", "val1"}, {"o_key2", "val2"}, {"o_key3", "val3"}} kvs2 = []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}} srcprefix = "o_" destprefix = "key" ) testMirrorCommand(cx, flags, kvs, kvs2, srcprefix, destprefix) } func makeMirrorWithWatchRev(cx ctlCtx) { var ( flags = []string{"--prefix", "o_", "--no-dest-prefix", "--rev", "4"} kvs = []kv{{"o_key1", "val1"}, {"o_key2", "val2"}, {"o_key3", "val3"}, {"o_key4", "val4"}} kvs2 = []kvExec{{key: "key3", val: "val3"}, {key: "key4", val: "val4"}} srcprefix = "o_" destprefix = "key" ) testMirrorCommand(cx, flags, kvs, kvs2, srcprefix, destprefix) } func testMirrorCommand(cx ctlCtx, flags []string, sourcekvs []kv, destkvs []kvExec, srcprefix, destprefix string) { // set up another cluster to mirror with mirrorcfg := e2e.NewConfigAutoTLS() mirrorcfg.ClusterSize = 1 mirrorcfg.BasePort = 10000 mirrorctx := ctlCtx{ t: cx.t, cfg: *mirrorcfg, dialTimeout: 7 * time.Second, } mirrorepc, err := e2e.NewEtcdProcessCluster(context.TODO(), cx.t, e2e.WithConfig(&mirrorctx.cfg)) if err != nil { cx.t.Fatalf("could not start etcd process cluster (%v)", err) } mirrorctx.epc = mirrorepc defer func() { if err = mirrorctx.epc.Close(); err != nil { cx.t.Fatalf("error closing etcd processes (%v)", err) } }() cmdArgs := append(cx.PrefixArgs(), "make-mirror") cmdArgs = append(cmdArgs, flags...) cmdArgs = append(cmdArgs, fmt.Sprintf("localhost:%d", mirrorcfg.BasePort)) proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) require.NoError(cx.t, err) defer func() { require.NoError(cx.t, proc.Stop()) }() for i := range sourcekvs { require.NoError(cx.t, ctlV3Put(cx, sourcekvs[i].key, sourcekvs[i].val, "")) } require.NoError(cx.t, ctlV3Get(cx, []string{srcprefix, "--prefix"}, sourcekvs...)) require.NoError(cx.t, ctlV3Watch(mirrorctx, []string{destprefix, "--rev", "1", "--prefix"}, destkvs...)) } ================================================ FILE: tests/e2e/ctl_v3_member_no_proxy_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "context" "math/rand" "os" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestMemberReplace(t *testing.T) { e2e.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) defer cancel() epc, err := e2e.NewEtcdProcessCluster(ctx, t) require.NoError(t, err) defer epc.Close() memberIdx := rand.Int() % len(epc.Procs) member := epc.Procs[memberIdx] memberName := member.Config().Name var endpoints []string for i := 1; i < len(epc.Procs); i++ { endpoints = append(endpoints, epc.Procs[(memberIdx+i)%len(epc.Procs)].EndpointsGRPC()...) } cc, err := e2e.NewEtcdctl(epc.Cfg.Client, endpoints) require.NoError(t, err) memberID, found, err := getMemberIDByName(ctx, cc, memberName) require.NoError(t, err) require.Truef(t, found, "Member not found") // Need to wait health interval for cluster to accept member changes time.Sleep(etcdserver.HealthInterval) t.Logf("Removing member %s", memberName) _, err = cc.MemberRemove(ctx, memberID) require.NoError(t, err) _, found, err = getMemberIDByName(ctx, cc, memberName) require.NoError(t, err) require.Falsef(t, found, "Expected member to be removed") for member.IsRunning() { err = member.Wait(ctx) if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { t.Fatalf("member didn't exit as expected: %v", err) } } t.Logf("Removing member %s data", memberName) err = os.RemoveAll(member.Config().DataDirPath) require.NoError(t, err) t.Logf("Adding member %s back", memberName) removedMemberPeerURL := member.Config().PeerURL.String() _, err = cc.MemberAdd(ctx, memberName, []string{removedMemberPeerURL}) require.NoError(t, err) err = e2e.PatchArgs(member.Config().Args, "initial-cluster-state", "existing") require.NoError(t, err) // Sleep 100ms to bypass the known issue https://github.com/etcd-io/etcd/issues/16687. time.Sleep(100 * time.Millisecond) t.Logf("Starting member %s", memberName) err = member.Start(ctx) require.NoError(t, err) testutils.ExecuteUntil(ctx, t, func() { for { _, found, err := getMemberIDByName(ctx, cc, memberName) if err != nil || !found { time.Sleep(10 * time.Millisecond) continue } break } }) } func TestMemberReplaceWithLearner(t *testing.T) { e2e.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) defer cancel() epc, err := e2e.NewEtcdProcessCluster(ctx, t) require.NoError(t, err) defer epc.Close() memberIdx := rand.Int() % len(epc.Procs) member := epc.Procs[memberIdx] memberName := member.Config().Name var endpoints []string for i := 1; i < len(epc.Procs); i++ { endpoints = append(endpoints, epc.Procs[(memberIdx+i)%len(epc.Procs)].EndpointsGRPC()...) } cc, err := e2e.NewEtcdctl(epc.Cfg.Client, endpoints) require.NoError(t, err) memberID, found, err := getMemberIDByName(ctx, cc, memberName) require.NoError(t, err) require.Truef(t, found, "Member not found") // Need to wait health interval for cluster to accept member changes time.Sleep(etcdserver.HealthInterval) t.Logf("Removing member %s", memberName) _, err = cc.MemberRemove(ctx, memberID) require.NoError(t, err) _, found, err = getMemberIDByName(ctx, cc, memberName) require.NoError(t, err) require.Falsef(t, found, "Expected member to be removed") for member.IsRunning() { err = member.Wait(ctx) if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { t.Fatalf("member didn't exit as expected: %v", err) } } t.Logf("Removing member %s data", memberName) err = os.RemoveAll(member.Config().DataDirPath) require.NoError(t, err) t.Logf("Adding member %s back as Learner", memberName) removedMemberPeerURL := member.Config().PeerURL.String() _, err = cc.MemberAddAsLearner(ctx, memberName, []string{removedMemberPeerURL}) require.NoError(t, err) err = e2e.PatchArgs(member.Config().Args, "initial-cluster-state", "existing") require.NoError(t, err) // Sleep 100ms to bypass the known issue https://github.com/etcd-io/etcd/issues/16687. time.Sleep(100 * time.Millisecond) t.Logf("Starting member %s", memberName) err = member.Start(ctx) require.NoError(t, err) var learnMemberID uint64 testutils.ExecuteUntil(ctx, t, func() { for { learnMemberID, found, err = getMemberIDByName(ctx, cc, memberName) if err != nil || !found { time.Sleep(10 * time.Millisecond) continue } break } }) learnMemberID, found, err = getMemberIDByName(ctx, cc, memberName) require.NoError(t, err) require.Truef(t, found, "Member not found") _, err = cc.MemberPromote(ctx, learnMemberID) require.NoError(t, err) } ================================================ FILE: tests/e2e/ctl_v3_member_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "encoding/json" "errors" "fmt" "io" "reflect" "strings" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3MemberList(t *testing.T) { testCtl(t, memberListTest) } func TestCtlV3MemberListWithHex(t *testing.T) { testCtl(t, memberListWithHexTest) } func TestCtlV3MemberListSerializable(t *testing.T) { cfg := e2e.NewConfig( e2e.WithClusterSize(1), ) testCtl(t, memberListSerializableTest, withCfg(*cfg)) } func TestCtlV3MemberAdd(t *testing.T) { testCtl(t, memberAddTest) } func TestCtlV3MemberAddAsLearner(t *testing.T) { testCtl(t, memberAddAsLearnerTest) } func TestCtlV3MemberUpdate(t *testing.T) { testCtl(t, memberUpdateTest) } func TestCtlV3MemberPromoteWithAuthFromLeader(t *testing.T) { testCtl(t, memberPromoteWithAuth(false), withTestTimeout(30*time.Second)) } func TestCtlV3MemberPromoteWithAuthFromFollower(t *testing.T) { testCtl(t, memberPromoteWithAuth(true), withTestTimeout(30*time.Second)) } func TestCtlV3MemberUpdateNoTLS(t *testing.T) { testCtl(t, memberUpdateTest, withCfg(*e2e.NewConfigNoTLS())) } func TestCtlV3MemberUpdateClientTLS(t *testing.T) { testCtl(t, memberUpdateTest, withCfg(*e2e.NewConfigClientTLS())) } func TestCtlV3MemberUpdateClientAutoTLS(t *testing.T) { testCtl(t, memberUpdateTest, withCfg(*e2e.NewConfigClientAutoTLS())) } func TestCtlV3MemberUpdatePeerTLS(t *testing.T) { testCtl(t, memberUpdateTest, withCfg(*e2e.NewConfigPeerTLS())) } // TestCtlV3ConsistentMemberList requires the gofailpoint to be enabled. // If you execute this case locally, please do not forget to execute // `make gofail-enable`. func TestCtlV3ConsistentMemberList(t *testing.T) { e2e.BeforeTest(t) ctx := t.Context() epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(1), e2e.WithEnvVars(map[string]string{"GOFAIL_FAILPOINTS": `beforeApplyOneConfChange=sleep("2s")`}), ) require.NoErrorf(t, err, "failed to start etcd cluster") defer func() { derr := epc.Close() require.NoErrorf(t, derr, "failed to close etcd cluster") }() t.Log("Adding and then removing a learner") resp, err := epc.Etcdctl().MemberAddAsLearner(ctx, "newLearner", []string{fmt.Sprintf("http://localhost:%d", e2e.EtcdProcessBasePort+11)}) require.NoError(t, err) _, err = epc.Etcdctl().MemberRemove(ctx, resp.Member.ID) require.NoError(t, err) t.Logf("Added and then removed a learner with ID: %x", resp.Member.ID) t.Log("Restarting the etcd process to ensure all data is persisted") err = epc.Procs[0].Restart(ctx) require.NoError(t, err) var wg sync.WaitGroup wg.Add(2) stopc := make(chan struct{}, 2) t.Log("Starting a goroutine to repeatedly restart etcdserver") go func() { defer func() { stopc <- struct{}{} wg.Done() }() for i := 0; i < 3; i++ { select { case <-stopc: return default: } merr := epc.Procs[0].Restart(ctx) assert.NoError(t, merr) epc.WaitLeader(t) time.Sleep(100 * time.Millisecond) } }() t.Log("Starting a goroutine to repeated check the member list") count := 0 go func() { defer func() { stopc <- struct{}{} wg.Done() }() for { select { case <-stopc: return default: } // Defailt timeout is 2s. We need to set a longer // timeout here to make sure we can get the response. mresp, merr := epc.Etcdctl(e2e.WithDialTimeout(5*time.Second)).MemberList(ctx, true) if merr != nil { continue } count++ assert.Len(t, mresp.Members, 1) } }() wg.Wait() assert.Positive(t, count) t.Logf("Checked the member list %d times", count) } func memberListTest(cx ctlCtx) { if err := ctlV3MemberList(cx); err != nil { cx.t.Fatalf("memberListTest ctlV3MemberList error (%v)", err) } } func memberListSerializableTest(cx ctlCtx) { resp, err := getMemberList(cx, false) require.NoError(cx.t, err) require.Len(cx.t, resp.Members, 1) peerURL := fmt.Sprintf("http://localhost:%d", e2e.EtcdProcessBasePort+11) err = ctlV3MemberAdd(cx, peerURL, false) require.NoError(cx.t, err) resp, err = getMemberList(cx, true) require.NoError(cx.t, err) require.Len(cx.t, resp.Members, 2) } func ctlV3MemberList(cx ctlCtx) error { cmdArgs := append(cx.PrefixArgs(), "member", "list") lines := make([]expect.ExpectedResponse, cx.cfg.ClusterSize) for i := range lines { lines[i] = expect.ExpectedResponse{Value: "started"} } return e2e.SpawnWithExpects(cmdArgs, cx.envMap, lines...) } func getMemberList(cx ctlCtx, serializable bool) (etcdserverpb.MemberListResponse, error) { cmdArgs := append(cx.PrefixArgs(), "--write-out", "json", "member", "list") if serializable { cmdArgs = append(cmdArgs, "--consistency", "s") } proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) if err != nil { return etcdserverpb.MemberListResponse{}, err } var txt string txt, err = proc.Expect("members") if err != nil { return etcdserverpb.MemberListResponse{}, err } if err = proc.Close(); err != nil { return etcdserverpb.MemberListResponse{}, err } resp := etcdserverpb.MemberListResponse{} dec := json.NewDecoder(strings.NewReader(txt)) if err := dec.Decode(&resp); errors.Is(err, io.EOF) { return etcdserverpb.MemberListResponse{}, err } return resp, nil } func memberListWithHexTest(cx ctlCtx) { resp, err := getMemberList(cx, false) if err != nil { cx.t.Fatalf("getMemberList error (%v)", err) } cmdArgs := append(cx.PrefixArgs(), "--write-out", "json", "--hex", "member", "list") proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) if err != nil { cx.t.Fatalf("memberListWithHexTest error (%v)", err) } var txt string txt, err = proc.Expect("members") if err != nil { cx.t.Fatalf("memberListWithHexTest error (%v)", err) } if err = proc.Close(); err != nil { cx.t.Fatalf("memberListWithHexTest error (%v)", err) } hexResp := etcdserverpb.MemberListResponse{} dec := json.NewDecoder(strings.NewReader(txt)) if err := dec.Decode(&hexResp); errors.Is(err, io.EOF) { cx.t.Fatalf("memberListWithHexTest error (%v)", err) } num := len(resp.Members) hexNum := len(hexResp.Members) if num != hexNum { cx.t.Fatalf("member number,expected %d,got %d", num, hexNum) } if num == 0 { cx.t.Fatal("member number is 0") } if resp.Header.RaftTerm != hexResp.Header.RaftTerm { cx.t.Fatalf("Unexpected raft_term, expected %d, got %d", resp.Header.RaftTerm, hexResp.Header.RaftTerm) } for i := 0; i < num; i++ { if resp.Members[i].Name != hexResp.Members[i].Name { cx.t.Fatalf("Unexpected member name,expected %v, got %v", resp.Members[i].Name, hexResp.Members[i].Name) } if !reflect.DeepEqual(resp.Members[i].PeerURLs, hexResp.Members[i].PeerURLs) { cx.t.Fatalf("Unexpected member peerURLs, expected %v, got %v", resp.Members[i].PeerURLs, hexResp.Members[i].PeerURLs) } if !reflect.DeepEqual(resp.Members[i].ClientURLs, hexResp.Members[i].ClientURLs) { cx.t.Fatalf("Unexpected member clientURLs, expected %v, got %v", resp.Members[i].ClientURLs, hexResp.Members[i].ClientURLs) } } } func memberAddTest(cx ctlCtx) { peerURL := fmt.Sprintf("http://localhost:%d", e2e.EtcdProcessBasePort+11) require.NoError(cx.t, ctlV3MemberAdd(cx, peerURL, false)) } func memberAddAsLearnerTest(cx ctlCtx) { peerURL := fmt.Sprintf("http://localhost:%d", e2e.EtcdProcessBasePort+11) require.NoError(cx.t, ctlV3MemberAdd(cx, peerURL, true)) } func memberPromoteWithAuth(fromFollower bool) func(cx ctlCtx) { return func(cx ctlCtx) { ctx := context.Background() // Add a regular member _, err := cx.epc.StartNewProc(ctx, nil, cx.t, false) require.NoError(cx.t, err) var learnerID uint64 var addErr error for { // Add a learner once the cluster is healthy learnerID, addErr = cx.epc.StartNewProc(ctx, nil, cx.t, true) if addErr != nil { if strings.Contains(addErr.Error(), "etcdserver: unhealthy cluster") { time.Sleep(1 * time.Second) continue } } break } require.NoError(cx.t, addErr) leaderIdx := cx.epc.WaitLeader(cx.t) followerIdx := (leaderIdx + 1) % len(cx.epc.Procs) require.NoError(cx.t, authEnable(cx)) cx.user, cx.pass = "root", "root" if fromFollower { _, err = cx.epc.Procs[followerIdx]. Etcdctl(e2e.WithAuth("root", "root")). MemberPromote(ctx, learnerID) } else { _, err = cx.epc.Procs[leaderIdx]. Etcdctl(e2e.WithAuth("root", "root")). MemberPromote(ctx, learnerID) } require.NoError(cx.t, err) } } func ctlV3MemberAdd(cx ctlCtx, peerURL string, isLearner bool) error { cmdArgs := append(cx.PrefixArgs(), "member", "add", "newmember", fmt.Sprintf("--peer-urls=%s", peerURL)) asLearner := " " if isLearner { cmdArgs = append(cmdArgs, "--learner") asLearner = " as learner " } return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: fmt.Sprintf(" added%sto cluster ", asLearner)}) } func memberUpdateTest(cx ctlCtx) { mr, err := getMemberList(cx, false) require.NoError(cx.t, err) peerURL := fmt.Sprintf("http://localhost:%d", e2e.EtcdProcessBasePort+11) memberID := fmt.Sprintf("%x", mr.Members[0].ID) require.NoError(cx.t, ctlV3MemberUpdate(cx, memberID, peerURL)) } func ctlV3MemberUpdate(cx ctlCtx, memberID, peerURL string) error { cmdArgs := append(cx.PrefixArgs(), "member", "update", memberID, fmt.Sprintf("--peer-urls=%s", peerURL)) return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: " updated in cluster "}) } func TestRemoveNonExistingMember(t *testing.T) { e2e.BeforeTest(t) ctx := t.Context() cfg := e2e.ConfigStandalone(*e2e.NewConfig()) epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithConfig(cfg)) require.NoError(t, err) defer epc.Close() c := epc.Etcdctl() _, err = c.MemberRemove(ctx, 1) require.Error(t, err) // Ensure that membership is properly bootstrapped. assert.NoError(t, epc.Restart(ctx)) } ================================================ FILE: tests/e2e/ctl_v3_move_leader_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "crypto/tls" "fmt" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3MoveLeaderScenarios(t *testing.T) { securityParent := map[string]struct { cfg e2e.EtcdProcessClusterConfig }{ "Secure": {cfg: *e2e.NewConfigTLS()}, "Insecure": {cfg: *e2e.NewConfigNoTLS()}, } tests := map[string]struct { env map[string]string }{ "happy path": {env: map[string]string{}}, "with env": {env: map[string]string{"ETCDCTL_ENDPOINTS": "something-else-is-set"}}, } for testName, tc := range securityParent { for subTestName, tx := range tests { t.Run(testName+" "+subTestName, func(t *testing.T) { testCtlV3MoveLeader(t, tc.cfg, tx.env) }) } } } func testCtlV3MoveLeader(t *testing.T, cfg e2e.EtcdProcessClusterConfig, envVars map[string]string) { epc := setupEtcdctlTest(t, &cfg, true) defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }() var tcfg *tls.Config if cfg.Client.ConnectionType == e2e.ClientTLS { tinfo := transport.TLSInfo{ CertFile: e2e.CertPath, KeyFile: e2e.PrivateKeyPath, TrustedCAFile: e2e.CaPath, } var err error tcfg, err = tinfo.ClientConfig() require.NoError(t, err) } var leadIdx int var leaderID uint64 var transferee uint64 for i, ep := range epc.EndpointsGRPC() { cli, err := clientv3.New(clientv3.Config{ Endpoints: []string{ep}, DialTimeout: 3 * time.Second, TLS: tcfg, }) require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) resp, err := cli.Status(ctx, ep) if err != nil { t.Fatalf("failed to get status from endpoint %s: %v", ep, err) } cancel() cli.Close() if resp.Header.GetMemberId() == resp.Leader { leadIdx = i leaderID = resp.Leader } else { transferee = resp.Header.GetMemberId() } } cx := ctlCtx{ t: t, cfg: *e2e.NewConfigNoTLS(), dialTimeout: 7 * time.Second, epc: epc, envMap: envVars, } tests := []struct { eps []string expect string expectErr bool }{ { // request to non-leader []string{cx.epc.EndpointsGRPC()[(leadIdx+1)%3]}, "no leader endpoint given at ", true, }, { // request to leader []string{cx.epc.EndpointsGRPC()[leadIdx]}, fmt.Sprintf("Leadership transferred from %s to %s", types.ID(leaderID), types.ID(transferee)), false, }, { // request to all endpoints cx.epc.EndpointsGRPC(), "Leadership transferred", false, }, } for i, tc := range tests { prefix := cx.prefixArgs(tc.eps) cmdArgs := append(prefix, "move-leader", types.ID(transferee).String()) err := e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: tc.expect}) if tc.expectErr { require.ErrorContains(t, err, tc.expect) } else { require.NoErrorf(t, err, "#%d: %v", i, err) } } } func setupEtcdctlTest(t *testing.T, cfg *e2e.EtcdProcessClusterConfig, quorum bool) *e2e.EtcdProcessCluster { if !quorum { cfg = e2e.ConfigStandalone(*cfg) } epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg)) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } return epc } ================================================ FILE: tests/e2e/ctl_v3_role_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "testing" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) // TestCtlV3RoleAddTimeout tests add role with 0 grpc dial timeout while it tolerates dial timeout error. // This is unique in e2e test func TestCtlV3RoleAddTimeout(t *testing.T) { testCtl(t, roleAddTest, withDefaultDialTimeout()) } func roleAddTest(cx ctlCtx) { cmdSet := []struct { args []string expectedStr string }{ // Add a role. { args: []string{"add", "root"}, expectedStr: "Role root created", }, // Try adding the same role. { args: []string{"add", "root"}, expectedStr: "role name already exists", }, } for i, cmd := range cmdSet { if err := ctlV3Role(cx, cmd.args, cmd.expectedStr); err != nil { if cx.dialTimeout > 0 && !isGRPCTimedout(err) { cx.t.Fatalf("roleAddTest #%d: ctlV3Role error (%v)", i, err) } } } } func ctlV3Role(cx ctlCtx, args []string, expStr string) error { cmdArgs := append(cx.PrefixArgs(), "role") cmdArgs = append(cmdArgs, args...) return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: expStr}) } func ctlV3RoleGrantPermission(cx ctlCtx, rolename string, perm grantingPerm) error { cmdArgs := append(cx.PrefixArgs(), "role", "grant-permission") if perm.prefix { cmdArgs = append(cmdArgs, "--prefix") } else if len(perm.rangeEnd) == 1 && perm.rangeEnd[0] == '\x00' { cmdArgs = append(cmdArgs, "--from-key") } cmdArgs = append(cmdArgs, rolename) cmdArgs = append(cmdArgs, grantingPermToArgs(perm)...) proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) if err != nil { return err } defer proc.Close() expStr := fmt.Sprintf("Role %s updated", rolename) _, err = proc.Expect(expStr) return err } func ctlV3RoleRevokePermission(cx ctlCtx, rolename string, key, rangeEnd string, fromKey bool) error { cmdArgs := append(cx.PrefixArgs(), "role", "revoke-permission") cmdArgs = append(cmdArgs, rolename) cmdArgs = append(cmdArgs, key) var expStr string if len(rangeEnd) != 0 { cmdArgs = append(cmdArgs, rangeEnd) expStr = fmt.Sprintf("Permission of range [%s, %s) is revoked from role %s", key, rangeEnd, rolename) } else if fromKey { cmdArgs = append(cmdArgs, "--from-key") expStr = fmt.Sprintf("Permission of range [%s, is revoked from role %s", key, rolename) } else { expStr = fmt.Sprintf("Permission of key %s is revoked from role %s", key, rolename) } proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) if err != nil { return err } defer proc.Close() _, err = proc.Expect(expStr) return err } type grantingPerm struct { read bool write bool key string rangeEnd string prefix bool } func grantingPermToArgs(perm grantingPerm) []string { permstr := "" if perm.read { permstr += "read" } if perm.write { permstr += "write" } if len(permstr) == 0 { panic("invalid granting permission") } if len(perm.rangeEnd) == 0 { return []string{permstr, perm.key} } if len(perm.rangeEnd) == 1 && perm.rangeEnd[0] == '\x00' { return []string{permstr, perm.key} } return []string{permstr, perm.key, perm.rangeEnd} } ================================================ FILE: tests/e2e/ctl_v3_snapshot_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "encoding/json" "errors" "fmt" "io" "os" "path" "path/filepath" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v3rpc "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/etcdutl/v3/snapshot" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestCtlV3Snapshot(t *testing.T) { testCtl(t, snapshotTest) } func snapshotTest(cx ctlCtx) { maintenanceInitKeys(cx) leaseID, err := ctlV3LeaseGrant(cx, 100) if err != nil { cx.t.Fatalf("snapshot: ctlV3LeaseGrant error (%v)", err) } if err = ctlV3Put(cx, "withlease", "withlease", leaseID); err != nil { cx.t.Fatalf("snapshot: ctlV3Put error (%v)", err) } fpath := filepath.Join(cx.t.TempDir(), "snapshot") defer os.RemoveAll(fpath) if err = ctlV3SnapshotSave(cx, fpath); err != nil { cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err) } st, err := getSnapshotStatus(cx, fpath) if err != nil { cx.t.Fatalf("snapshotTest getSnapshotStatus error (%v)", err) } if st.Revision != 5 { cx.t.Fatalf("expected 4, got %d", st.Revision) } if st.TotalKey < 2 { cx.t.Fatalf("expected at least 2, got %d", st.TotalKey) } } func TestCtlV3SnapshotCorrupt(t *testing.T) { testCtl(t, snapshotCorruptTest) } func snapshotCorruptTest(cx ctlCtx) { fpath := filepath.Join(cx.t.TempDir(), "snapshot") defer os.RemoveAll(fpath) if err := ctlV3SnapshotSave(cx, fpath); err != nil { cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err) } // corrupt file f, oerr := os.OpenFile(fpath, os.O_WRONLY, 0) require.NoError(cx.t, oerr) _, err := f.Write(make([]byte, 512)) require.NoError(cx.t, err) f.Close() datadir := cx.t.TempDir() serr := e2e.SpawnWithExpectWithEnv( append(cx.PrefixArgsUtl(), "snapshot", "restore", "--data-dir", datadir, fpath), cx.envMap, expect.ExpectedResponse{Value: "expected sha256"}) require.ErrorContains(cx.t, serr, "Error: expected sha256") } // TestCtlV3SnapshotStatusBeforeRestore ensures that the snapshot // status does not modify the snapshot file func TestCtlV3SnapshotStatusBeforeRestore(t *testing.T) { testCtl(t, snapshotStatusBeforeRestoreTest) } func snapshotStatusBeforeRestoreTest(cx ctlCtx) { fpath := filepath.Join(cx.t.TempDir(), "snapshot") defer os.RemoveAll(fpath) if err := ctlV3SnapshotSave(cx, fpath); err != nil { cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err) } // snapshot status on the fresh snapshot file _, err := getSnapshotStatus(cx, fpath) if err != nil { cx.t.Fatalf("snapshotTest getSnapshotStatus error (%v)", err) } dataDir := cx.t.TempDir() defer os.RemoveAll(dataDir) serr := e2e.SpawnWithExpectWithEnv( append(cx.PrefixArgsUtl(), "snapshot", "restore", "--data-dir", dataDir, fpath), cx.envMap, expect.ExpectedResponse{Value: "added member"}) require.NoError(cx.t, serr) } func ctlV3SnapshotSave(cx ctlCtx, fpath string) error { cmdArgs := append(cx.PrefixArgs(), "snapshot", "save", fpath) return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: fmt.Sprintf("Snapshot saved at %s", fpath)}) } func getSnapshotStatus(cx ctlCtx, fpath string) (snapshot.Status, error) { cmdArgs := append(cx.PrefixArgsUtl(), "--write-out", "json", "snapshot", "status", fpath) proc, err := e2e.SpawnCmd(cmdArgs, nil) if err != nil { return snapshot.Status{}, err } var txt string txt, err = proc.Expect("totalKey") if err != nil { return snapshot.Status{}, err } if err = proc.Close(); err != nil { return snapshot.Status{}, err } resp := snapshot.Status{} dec := json.NewDecoder(strings.NewReader(txt)) if err := dec.Decode(&resp); errors.Is(err, io.EOF) { return snapshot.Status{}, err } return resp, nil } func TestIssue6361(t *testing.T) { testIssue6361(t) } // TestIssue6361 ensures new member that starts with snapshot correctly // syncs up with other members and serve correct data. func testIssue6361(t *testing.T) { // This tests is pretty flaky on semaphoreci as of 2021-01-10. // TODO: Remove when the flakiness source is identified. t.Setenv("EXPECT_DEBUG", "1") e2e.BeforeTest(t) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithClusterSize(1), e2e.WithKeepDataDir(true), ) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }() dialTimeout := 10 * time.Second prefixArgs := []string{e2e.BinPath.Etcdctl, "--endpoints", strings.Join(epc.EndpointsGRPC(), ","), "--dial-timeout", dialTimeout.String()} t.Log("Writing some keys...") kvs := []kv{{"foo1", "val1"}, {"foo2", "val2"}, {"foo3", "val3"}} for i := range kvs { err = e2e.SpawnWithExpect(append(prefixArgs, "put", kvs[i].key, kvs[i].val), expect.ExpectedResponse{Value: "OK"}) require.NoError(t, err) } fpath := filepath.Join(t.TempDir(), "test.snapshot") t.Log("etcdctl saving snapshot...") require.NoError(t, e2e.SpawnWithExpects(append(prefixArgs, "snapshot", "save", fpath), nil, expect.ExpectedResponse{Value: fmt.Sprintf("Snapshot saved at %s", fpath)}, )) t.Log("Stopping the original server...") require.NoError(t, epc.Procs[0].Stop()) newDataDir := filepath.Join(t.TempDir(), "test.data") t.Log("etcdctl restoring the snapshot...") require.NoError(t, e2e.SpawnWithExpect([]string{ e2e.BinPath.Etcdutl, "snapshot", "restore", fpath, "--name", epc.Procs[0].Config().Name, "--initial-cluster", epc.Procs[0].Config().InitialCluster, "--initial-cluster-token", epc.Procs[0].Config().InitialToken, "--initial-advertise-peer-urls", epc.Procs[0].Config().PeerURL.String(), "--data-dir", newDataDir, }, expect.ExpectedResponse{Value: "added member"})) t.Log("(Re)starting the etcd member using the restored snapshot...") epc.Procs[0].Config().DataDirPath = newDataDir for i := range epc.Procs[0].Config().Args { if epc.Procs[0].Config().Args[i] == "--data-dir" { epc.Procs[0].Config().Args[i+1] = newDataDir } } require.NoError(t, epc.Procs[0].Restart(t.Context())) t.Log("Ensuring the restored member has the correct data...") for i := range kvs { require.NoError(t, e2e.SpawnWithExpect(append(prefixArgs, "get", kvs[i].key), expect.ExpectedResponse{Value: kvs[i].val})) } t.Log("Adding new member into the cluster") clientURL := fmt.Sprintf("http://localhost:%d", e2e.EtcdProcessBasePort+30) peerURL := fmt.Sprintf("http://localhost:%d", e2e.EtcdProcessBasePort+31) require.NoError(t, e2e.SpawnWithExpect(append(prefixArgs, "member", "add", "newmember", fmt.Sprintf("--peer-urls=%s", peerURL)), expect.ExpectedResponse{Value: " added to cluster "})) newDataDir2 := t.TempDir() defer os.RemoveAll(newDataDir2) name2 := "infra2" initialCluster2 := epc.Procs[0].Config().InitialCluster + fmt.Sprintf(",%s=%s", name2, peerURL) t.Log("Starting the new member") // start the new member var nepc *expect.ExpectProcess nepc, err = e2e.SpawnCmd([]string{ epc.Procs[0].Config().ExecPath, "--name", name2, "--listen-client-urls", clientURL, "--advertise-client-urls", clientURL, "--listen-peer-urls", peerURL, "--initial-advertise-peer-urls", peerURL, "--initial-cluster", initialCluster2, "--initial-cluster-state", "existing", "--data-dir", newDataDir2, }, nil) require.NoError(t, err) _, err = nepc.Expect("ready to serve client requests") require.NoError(t, err) prefixArgs = []string{e2e.BinPath.Etcdctl, "--endpoints", clientURL, "--dial-timeout", dialTimeout.String()} t.Log("Ensuring added member has data from incoming snapshot...") for i := range kvs { require.NoError(t, e2e.SpawnWithExpect(append(prefixArgs, "get", kvs[i].key), expect.ExpectedResponse{Value: kvs[i].val})) } t.Log("Stopping the second member") require.NoError(t, nepc.Stop()) t.Log("Test logic done") } // TestCtlV3SnapshotVersion is for storageVersion to be stored, all fields // expected 3.6 fields need to be set. This happens after first WAL snapshot. // In this test we lower SnapshotCount to 1 to ensure WAL snapshot is triggered. func TestCtlV3SnapshotVersion(t *testing.T) { testCtl(t, snapshotVersionTest, withCfg(*e2e.NewConfig(e2e.WithSnapshotCount(1)))) } func snapshotVersionTest(cx ctlCtx) { maintenanceInitKeys(cx) fpath := filepath.Join(cx.t.TempDir(), "snapshot") defer os.RemoveAll(fpath) if err := ctlV3SnapshotSave(cx, fpath); err != nil { cx.t.Fatalf("snapshotVersionTest ctlV3SnapshotSave error (%v)", err) } st, err := getSnapshotStatus(cx, fpath) if err != nil { cx.t.Fatalf("snapshotVersionTest getSnapshotStatus error (%v)", err) } if st.Version != "3.7.0" { cx.t.Fatalf("expected %q, got %q", "3.7.0", st.Version) } } func TestRestoreCompactionRevBump(t *testing.T) { e2e.BeforeTest(t) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithClusterSize(1), e2e.WithKeepDataDir(true), ) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }() ctl := epc.Etcdctl() watchCh := ctl.Watch(t.Context(), "foo", config.WatchOptions{Prefix: true}) // flake-fix: the watch can sometimes miss the first put below causing test failure time.Sleep(100 * time.Millisecond) kvs := []testutils.KV{{Key: "foo1", Val: "val1"}, {Key: "foo2", Val: "val2"}, {Key: "foo3", Val: "val3"}} for i := range kvs { _, err = ctl.Put(t.Context(), kvs[i].Key, kvs[i].Val, config.PutOptions{}) require.NoError(t, err) } watchTimeout := 1 * time.Second watchRes, err := testutils.KeyValuesFromWatchChan(watchCh, len(kvs), watchTimeout) require.NoErrorf(t, err, "failed to get key-values from watch channel %s", err) require.Equal(t, kvs, watchRes) // ensure we get the right revision back for each of the keys currentRev := 4 baseRev := 2 hasKVs(t, ctl, kvs, currentRev, baseRev) fpath := filepath.Join(t.TempDir(), "test.snapshot") t.Log("etcdctl saving snapshot...") cmdPrefix := []string{e2e.BinPath.Etcdctl, "--endpoints", strings.Join(epc.EndpointsGRPC(), ",")} require.NoError(t, e2e.SpawnWithExpects(append(cmdPrefix, "snapshot", "save", fpath), nil, expect.ExpectedResponse{Value: fmt.Sprintf("Snapshot saved at %s", fpath)})) // add some more kvs that are not in the snapshot that will be lost after restore unsnappedKVs := []testutils.KV{{Key: "unsnapped1", Val: "one"}, {Key: "unsnapped2", Val: "two"}, {Key: "unsnapped3", Val: "three"}} for i := range unsnappedKVs { _, err = ctl.Put(t.Context(), unsnappedKVs[i].Key, unsnappedKVs[i].Val, config.PutOptions{}) require.NoError(t, err) } membersBefore, err := ctl.MemberList(t.Context(), false) require.NoError(t, err) t.Log("Stopping the original server...") require.NoError(t, epc.Stop()) newDataDir := filepath.Join(t.TempDir(), "test.data") t.Log("etcdctl restoring the snapshot...") bumpAmount := 10000 require.NoError(t, e2e.SpawnWithExpect([]string{ e2e.BinPath.Etcdutl, "snapshot", "restore", fpath, "--name", epc.Procs[0].Config().Name, "--initial-cluster", epc.Procs[0].Config().InitialCluster, "--initial-cluster-token", epc.Procs[0].Config().InitialToken, "--initial-advertise-peer-urls", epc.Procs[0].Config().PeerURL.String(), "--bump-revision", fmt.Sprintf("%d", bumpAmount), "--mark-compacted", "--data-dir", newDataDir, }, expect.ExpectedResponse{Value: "added member"})) t.Log("(Re)starting the etcd member using the restored snapshot...") epc.Procs[0].Config().DataDirPath = newDataDir err = e2e.PatchArgs(epc.Procs[0].Config().Args, "data-dir", newDataDir) require.NoError(t, err) // Verify that initial snapshot is created by the restore operation verifySnapshotMembers(t, epc, membersBefore) require.NoError(t, epc.Restart(t.Context())) t.Log("Ensuring the restored member has the correct data...") hasKVs(t, ctl, kvs, currentRev, baseRev) for i := range unsnappedKVs { v, gerr := ctl.Get(t.Context(), unsnappedKVs[i].Key, config.GetOptions{}) require.NoError(t, gerr) require.Equal(t, int64(0), v.Count) } cancelResult, ok := <-watchCh require.Truef(t, ok, "watchChannel should be open") require.Equal(t, v3rpc.ErrCompacted, cancelResult.Err()) require.Truef(t, cancelResult.Canceled, "expected ongoing watch to be cancelled after restoring with --mark-compacted") require.Equal(t, int64(bumpAmount+currentRev), cancelResult.CompactRevision) _, ok = <-watchCh require.Falsef(t, ok, "watchChannel should be closed after restoring with --mark-compacted") // clients might restart the watch at the old base revision, that should not yield any new data // everything up until bumpAmount+currentRev should return "already compacted" for i := bumpAmount - 2; i < bumpAmount+currentRev; i++ { watchCh = ctl.Watch(t.Context(), "foo", config.WatchOptions{Prefix: true, Revision: int64(i)}) cancelResult := <-watchCh require.Equal(t, v3rpc.ErrCompacted, cancelResult.Err()) require.Truef(t, cancelResult.Canceled, "expected ongoing watch to be cancelled after restoring with --mark-compacted") require.Equal(t, int64(bumpAmount+currentRev), cancelResult.CompactRevision) } // a watch after that revision should yield successful results when a new put arrives ctx, cancel := context.WithTimeout(t.Context(), watchTimeout*5) defer cancel() watchCh = ctl.Watch(ctx, "foo", config.WatchOptions{Prefix: true, Revision: int64(bumpAmount + currentRev + 1)}) _, err = ctl.Put(ctx, "foo4", "val4", config.PutOptions{}) require.NoError(t, err) watchRes, err = testutils.KeyValuesFromWatchChan(watchCh, 1, watchTimeout) require.NoErrorf(t, err, "failed to get key-values from watch channel %s", err) require.Equal(t, []testutils.KV{{Key: "foo4", Val: "val4"}}, watchRes) } func hasKVs(t *testing.T, ctl *e2e.EtcdctlV3, kvs []testutils.KV, currentRev int, baseRev int) { for i := range kvs { v, err := ctl.Get(t.Context(), kvs[i].Key, config.GetOptions{}) require.NoError(t, err) require.Equal(t, int64(1), v.Count) require.Equal(t, kvs[i].Val, string(v.Kvs[0].Value)) require.Equal(t, int64(baseRev+i), v.Kvs[0].CreateRevision) require.Equal(t, int64(baseRev+i), v.Kvs[0].ModRevision) require.Equal(t, int64(1), v.Kvs[0].Version) require.GreaterOrEqual(t, int64(currentRev), v.Kvs[0].ModRevision) } } func TestBreakConsistentIndexNewerThanSnapshot(t *testing.T) { e2e.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() var snapshotCount uint64 = 50 epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(1), e2e.WithKeepDataDir(true), e2e.WithSnapshotCount(snapshotCount), ) require.NoError(t, err) defer epc.Close() member := epc.Procs[0] t.Log("Stop member and copy out the db file to tmp directory") err = member.Stop() require.NoError(t, err) dbPath := path.Join(member.Config().DataDirPath, "member", "snap", "db") tmpFile := path.Join(t.TempDir(), "db") err = copyFile(dbPath, tmpFile) require.NoError(t, err) t.Log("Ensure snapshot there is a newer snapshot") err = member.Start(ctx) require.NoError(t, err) generateSnapshot(t, snapshotCount, member.Etcdctl()) _, err = member.Logs().ExpectWithContext(ctx, expect.ExpectedResponse{Value: "saved snapshot"}) require.NoError(t, err) err = member.Stop() require.NoError(t, err) t.Log("Start etcd with older db file") err = copyFile(tmpFile, dbPath) require.NoError(t, err) err = member.Start(ctx) require.Error(t, err) _, err = member.Logs().ExpectWithContext(ctx, expect.ExpectedResponse{Value: "failed to find database snapshot file (snap: snapshot file doesn't exist)"}) assert.NoError(t, err) } func copyFile(src, dst string) error { f, err := os.Open(src) if err != nil { return err } defer f.Close() w, err := os.Create(dst) if err != nil { return err } defer w.Close() if _, err = io.Copy(w, f); err != nil { return err } return w.Sync() } ================================================ FILE: tests/e2e/ctl_v3_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "os" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/pkg/v3/featuregate" "go.etcd.io/etcd/pkg/v3/flags" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCtlV3Version(t *testing.T) { testCtl(t, versionTest) } func TestClusterVersion(t *testing.T) { e2e.BeforeTest(t) tests := []struct { name string rollingStart bool }{ { name: "When start servers at the same time", rollingStart: false, }, { name: "When start servers one by one", rollingStart: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { e2e.BeforeTest(t) cfg := e2e.NewConfig( e2e.WithSnapshotCount(3), e2e.WithBasePeerScheme("unix"), // to avoid port conflict) e2e.WithRollingStart(tt.rollingStart), ) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg)) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }() ctx := ctlCtx{ t: t, cfg: *cfg, epc: epc, } cv := version.Cluster(version.Version) clusterVersionTest(ctx, `"etcdcluster":"`+cv) }) } } func versionTest(cx ctlCtx) { if err := ctlV3Version(cx); err != nil { cx.t.Fatalf("versionTest ctlV3Version error (%v)", err) } } func clusterVersionTest(cx ctlCtx, expected string) { var err error for i := 0; i < 35; i++ { if err = e2e.CURLGet(cx.epc, e2e.CURLReq{Endpoint: "/version", Expected: expect.ExpectedResponse{Value: expected}}); err != nil { cx.t.Logf("#%d: v3 is not ready yet (%v)", i, err) time.Sleep(200 * time.Millisecond) continue } break } if err != nil { cx.t.Fatalf("failed cluster version test expected %v got (%v)", expected, err) } } func ctlV3Version(cx ctlCtx) error { cmdArgs := append(cx.PrefixArgs(), "version") return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: version.Version}) } // TestCtlV3DialWithHTTPScheme ensures that client handles Endpoints with HTTPS scheme. func TestCtlV3DialWithHTTPScheme(t *testing.T) { testCtl(t, dialWithSchemeTest, withCfg(*e2e.NewConfigClientTLS())) } func dialWithSchemeTest(cx ctlCtx) { cmdArgs := append(cx.prefixArgs(cx.epc.EndpointsGRPC()), "put", "foo", "bar") require.NoError(cx.t, e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, expect.ExpectedResponse{Value: "OK"})) } type ctlCtx struct { t *testing.T cfg e2e.EtcdProcessClusterConfig corruptFunc func(string) error disableStrictReconfigCheck bool epc *e2e.EtcdProcessCluster envMap map[string]string dialTimeout time.Duration testTimeout time.Duration quorum bool // if true, set up 3-node cluster and linearizable read interactive bool user string pass string initialCorruptCheck bool // dir that was used during the test dataDir string } type ctlOption func(*ctlCtx) func (cx *ctlCtx) applyOpts(opts []ctlOption) { for _, opt := range opts { opt(cx) } cx.initialCorruptCheck = true } func withCfg(cfg e2e.EtcdProcessClusterConfig) ctlOption { return func(cx *ctlCtx) { cx.cfg = cfg } } func withDefaultDialTimeout() ctlOption { return withDialTimeout(0) } func withDialTimeout(timeout time.Duration) ctlOption { return func(cx *ctlCtx) { cx.dialTimeout = timeout } } func withTestTimeout(timeout time.Duration) ctlOption { return func(cx *ctlCtx) { cx.testTimeout = timeout } } func withQuorum() ctlOption { return func(cx *ctlCtx) { cx.quorum = true } } func withInteractive() ctlOption { return func(cx *ctlCtx) { cx.interactive = true } } func withInitialCorruptCheck() ctlOption { return func(cx *ctlCtx) { cx.initialCorruptCheck = true } } func withCorruptFunc(f func(string) error) ctlOption { return func(cx *ctlCtx) { cx.corruptFunc = f } } func withFlagByEnv() ctlOption { return func(cx *ctlCtx) { cx.envMap = make(map[string]string) } } // This function must be called after the `withCfg`, otherwise its value // may be overwritten by `withCfg`. func withMaxConcurrentStreams(streams uint32) ctlOption { return func(cx *ctlCtx) { cx.cfg.ServerConfig.MaxConcurrentStreams = streams } } func withLogLevel(logLevel string) ctlOption { return func(cx *ctlCtx) { cx.cfg.ServerConfig.LogLevel = logLevel } } func testCtl(t *testing.T, testFunc func(ctlCtx), opts ...ctlOption) { testCtlWithOffline(t, testFunc, nil, opts...) } func getDefaultCtlCtx(t *testing.T) ctlCtx { return ctlCtx{ t: t, cfg: *e2e.NewConfigAutoTLS(), dialTimeout: 7 * time.Second, } } func testCtlWithOffline(t *testing.T, testFunc func(ctlCtx), testOfflineFunc func(ctlCtx), opts ...ctlOption) { e2e.BeforeTest(t) ret := getDefaultCtlCtx(t) ret.applyOpts(opts) if !ret.quorum { ret.cfg = *e2e.ConfigStandalone(ret.cfg) } ret.cfg.ServerConfig.StrictReconfigCheck = !ret.disableStrictReconfigCheck if ret.initialCorruptCheck { ret.cfg.ServerConfig.ServerFeatureGate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("InitialCorruptCheck=%t", ret.initialCorruptCheck)) } if testOfflineFunc != nil { ret.cfg.KeepDataDir = true } epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(&ret.cfg)) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } ret.epc = epc ret.dataDir = epc.Procs[0].Config().DataDirPath runCtlTest(t, testFunc, testOfflineFunc, ret) } func runCtlTest(t *testing.T, testFunc func(ctlCtx), testOfflineFunc func(ctlCtx), cx ctlCtx) { defer func() { if cx.envMap != nil { for k := range cx.envMap { os.Unsetenv(k) } cx.envMap = make(map[string]string) } if cx.epc != nil { cx.epc.Stop() cx.epc.Close() } }() donec := make(chan struct{}) go func() { defer close(donec) testFunc(cx) t.Log("---testFunc logic DONE") }() timeout := cx.getTestTimeout() select { case <-time.After(timeout): testutil.FatalStack(t, fmt.Sprintf("test timed out after %v", timeout)) case <-donec: } t.Log("closing test cluster...") assert.NoError(t, cx.epc.Stop()) assert.NoError(t, cx.epc.Close()) cx.epc = nil t.Log("closed test cluster...") if testOfflineFunc != nil { testOfflineFunc(cx) } } func (cx *ctlCtx) getTestTimeout() time.Duration { timeout := cx.testTimeout if timeout == 0 { timeout = 2*cx.dialTimeout + time.Second if cx.dialTimeout == 0 { timeout = 30 * time.Second } } return timeout } func (cx *ctlCtx) prefixArgs(eps []string) []string { fmap := make(map[string]string) fmap["endpoints"] = strings.Join(eps, ",") fmap["dial-timeout"] = cx.dialTimeout.String() if cx.epc.Cfg.Client.ConnectionType == e2e.ClientTLS { if cx.epc.Cfg.Client.AutoTLS { fmap["insecure-transport"] = "false" fmap["insecure-skip-tls-verify"] = "true" } else if cx.epc.Cfg.Client.RevokeCerts { fmap["cacert"] = e2e.CaPath fmap["cert"] = e2e.RevokedCertPath fmap["key"] = e2e.RevokedPrivateKeyPath } else { fmap["cacert"] = e2e.CaPath fmap["cert"] = e2e.CertPath fmap["key"] = e2e.PrivateKeyPath } } if cx.user != "" { fmap["user"] = cx.user + ":" + cx.pass } useEnv := cx.envMap != nil cmdArgs := []string{e2e.BinPath.Etcdctl} for k, v := range fmap { if useEnv { ek := flags.FlagToEnv("ETCDCTL", k) cx.envMap[ek] = v } else { cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", k, v)) } } return cmdArgs } // PrefixArgs prefixes etcdctl command. // Make sure to unset environment variables after tests. func (cx *ctlCtx) PrefixArgs() []string { return cx.prefixArgs(cx.epc.EndpointsGRPC()) } // PrefixArgsUtl returns prefix of the command that is etcdutl // Please not thet 'utl' compatible commands does not consume --endpoints flag. func (cx *ctlCtx) PrefixArgsUtl() []string { return []string{e2e.BinPath.Etcdutl} } func isGRPCTimedout(err error) bool { return strings.Contains(err.Error(), "grpc: timed out trying to connect") } ================================================ FILE: tests/e2e/ctl_v3_watch_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "os" "strings" "testing" "go.etcd.io/etcd/tests/v3/framework/e2e" ) type kvExec struct { key, val string execOutput string } func setupWatchArgs(cx ctlCtx, args []string) []string { cmdArgs := append(cx.PrefixArgs(), "watch") if cx.interactive { cmdArgs = append(cmdArgs, "--interactive") } else { cmdArgs = append(cmdArgs, args...) } return cmdArgs } func ctlV3Watch(cx ctlCtx, args []string, kvs ...kvExec) error { cmdArgs := setupWatchArgs(cx, args) proc, err := e2e.SpawnCmd(cmdArgs, nil) if err != nil { return err } if cx.interactive { wl := strings.Join(append([]string{"watch"}, args...), " ") + "\r" if err = proc.Send(wl); err != nil { return err } } for _, elem := range kvs { if _, err = proc.Expect(elem.key); err != nil { return err } if _, err = proc.Expect(elem.val); err != nil { return err } if elem.execOutput != "" { if _, err = proc.Expect(elem.execOutput); err != nil { return err } } } return proc.Stop() } func ctlV3WatchFailPerm(cx ctlCtx, args []string) error { cmdArgs := setupWatchArgs(cx, args) proc, err := e2e.SpawnCmd(cmdArgs, nil) if err != nil { return err } if cx.interactive { wl := strings.Join(append([]string{"watch"}, args...), " ") + "\r" if err = proc.Send(wl); err != nil { return err } } // TODO(mitake): after printing accurate error message that includes // "permission denied", the above string argument of proc.Expect() // should be updated. _, err = proc.Expect("watch is canceled by the server") if err != nil { return err } return proc.Close() } func TestCtlV3Watch(t *testing.T) { testCtl(t, watchTest) } func TestCtlV3WatchNoTLS(t *testing.T) { testCtl(t, watchTest, withCfg(*e2e.NewConfigNoTLS())) } func TestCtlV3WatchClientTLS(t *testing.T) { testCtl(t, watchTest, withCfg(*e2e.NewConfigClientTLS())) } func TestCtlV3WatchPeerTLS(t *testing.T) { testCtl(t, watchTest, withCfg(*e2e.NewConfigPeerTLS())) } func TestCtlV3WatchTimeout(t *testing.T) { testCtl(t, watchTest, withDefaultDialTimeout()) } func TestCtlV3WatchInteractive(t *testing.T) { testCtl(t, watchTest, withInteractive()) } func TestCtlV3WatchInteractiveNoTLS(t *testing.T) { testCtl(t, watchTest, withInteractive(), withCfg(*e2e.NewConfigNoTLS())) } func TestCtlV3WatchInteractiveClientTLS(t *testing.T) { testCtl(t, watchTest, withInteractive(), withCfg(*e2e.NewConfigClientTLS())) } func TestCtlV3WatchInteractivePeerTLS(t *testing.T) { testCtl(t, watchTest, withInteractive(), withCfg(*e2e.NewConfigPeerTLS())) } func watchTest(cx ctlCtx) { tests := []struct { puts []kv envKey string envRange string args []string wkv []kvExec }{ { // watch 1 key with env puts: []kv{{"sample", "value"}}, envKey: "sample", args: []string{"--rev", "1"}, wkv: []kvExec{{key: "sample", val: "value"}}, }, { // watch 1 key with ${ETCD_WATCH_VALUE} puts: []kv{{"sample", "value"}}, args: []string{"sample", "--rev", "1", "--", "env"}, wkv: []kvExec{{key: "sample", val: "value", execOutput: `ETCD_WATCH_VALUE="value"`}}, }, { // watch 1 key with "echo watch event received", with env puts: []kv{{"sample", "value"}}, envKey: "sample", args: []string{"--rev", "1", "--", "echo", "watch event received"}, wkv: []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}}, }, { // watch 1 key with "echo watch event received" puts: []kv{{"sample", "value"}}, args: []string{"--rev", "1", "sample", "--", "echo", "watch event received"}, wkv: []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}}, }, { // watch 1 key with "echo \"Hello World!\"" puts: []kv{{"sample", "value"}}, args: []string{"--rev", "1", "sample", "--", "echo", "\"Hello World!\""}, wkv: []kvExec{{key: "sample", val: "value", execOutput: "Hello World!"}}, }, { // watch 1 key with "echo watch event received" puts: []kv{{"sample", "value"}}, args: []string{"sample", "samplx", "--rev", "1", "--", "echo", "watch event received"}, wkv: []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}}, }, { // watch 1 key with "echo watch event received" puts: []kv{{"sample", "value"}}, envKey: "sample", envRange: "samplx", args: []string{"--rev", "1", "--", "echo", "watch event received"}, wkv: []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}}, }, { // watch 1 key with "echo watch event received" puts: []kv{{"sample", "value"}}, args: []string{"sample", "--rev", "1", "samplx", "--", "echo", "watch event received"}, wkv: []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}}, }, { // watch 3 keys by prefix, with env puts: []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}}, envKey: "key", args: []string{"--rev", "1", "--prefix"}, wkv: []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}}, }, { // watch 3 keys by range, with env puts: []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}}, envKey: "key", envRange: "key3", args: []string{"--rev", "1"}, wkv: []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}}, }, } for i, tt := range tests { donec := make(chan struct{}) go func(i int, puts []kv) { for j := range puts { if err := ctlV3Put(cx, puts[j].key, puts[j].val, ""); err != nil { cx.t.Errorf("watchTest #%d-%d: ctlV3Put error (%v)", i, j, err) } } close(donec) }(i, tt.puts) unsetEnv := func() {} if tt.envKey != "" || tt.envRange != "" { if tt.envKey != "" { os.Setenv("ETCDCTL_WATCH_KEY", tt.envKey) unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_KEY") } } if tt.envRange != "" { os.Setenv("ETCDCTL_WATCH_RANGE_END", tt.envRange) unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_RANGE_END") } } if tt.envKey != "" && tt.envRange != "" { unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_KEY") os.Unsetenv("ETCDCTL_WATCH_RANGE_END") } } } if err := ctlV3Watch(cx, tt.args, tt.wkv...); err != nil { if cx.dialTimeout > 0 && !isGRPCTimedout(err) { cx.t.Errorf("watchTest #%d: ctlV3Watch error (%v)", i, err) } } unsetEnv() <-donec } } ================================================ FILE: tests/e2e/defrag_no_space_test.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestDefragNoSpace(t *testing.T) { tests := []struct { name string failpoint string err string }{ { name: "no space (#18810) - can't open/create new bbolt db", failpoint: "defragOpenFileError", err: "no space", }, { name: "defragdb failure", failpoint: "defragdbFail", err: "some random error", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { e2e.BeforeTest(t) clus, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), ) require.NoError(t, err) t.Cleanup(func() { clus.Stop() }) member := clus.Procs[0] require.NoError(t, member.Failpoints().SetupHTTP(t.Context(), tc.failpoint, fmt.Sprintf(`return("%s")`, tc.err))) require.ErrorContains(t, member.Etcdctl().Defragment(t.Context(), config.DefragOption{Timeout: time.Minute}), tc.err) // Make sure etcd continues to run even after the failed defrag attempt _, err = member.Etcdctl().Put(t.Context(), "foo", "bar", config.PutOptions{}) require.NoError(t, err) value, err := member.Etcdctl().Get(t.Context(), "foo", config.GetOptions{}) require.NoError(t, err) require.Len(t, value.Kvs, 1) require.Equal(t, "bar", string(value.Kvs[0].Value)) }) } } ================================================ FILE: tests/e2e/discovery_v3_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "strconv" "strings" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestClusterOf1UsingV3Discovery_1endpoint(t *testing.T) { testClusterUsingV3Discovery(t, 1, 1, e2e.ClientNonTLS, false) } func TestClusterOf3UsingV3Discovery_1endpoint(t *testing.T) { testClusterUsingV3Discovery(t, 1, 3, e2e.ClientTLS, true) } func TestTLSClusterOf5UsingV3Discovery_1endpoint(t *testing.T) { testClusterUsingV3Discovery(t, 1, 5, e2e.ClientTLS, false) } func TestClusterOf1UsingV3Discovery_3endpoints(t *testing.T) { testClusterUsingV3Discovery(t, 3, 1, e2e.ClientNonTLS, false) } func TestClusterOf3UsingV3Discovery_3endpoints(t *testing.T) { testClusterUsingV3Discovery(t, 3, 3, e2e.ClientTLS, true) } func TestTLSClusterOf5UsingV3Discovery_3endpoints(t *testing.T) { testClusterUsingV3Discovery(t, 3, 5, e2e.ClientTLS, false) } func testClusterUsingV3Discovery(t *testing.T, discoveryClusterSize, targetClusterSize int, clientTLSType e2e.ClientConnType, isClientAutoTLS bool) { e2e.BeforeTest(t) // step 1: start the discovery service ds, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithBasePort(2000), e2e.WithClusterSize(discoveryClusterSize), e2e.WithClientConnType(clientTLSType), e2e.WithClientAutoTLS(isClientAutoTLS), ) if err != nil { t.Fatalf("could not start discovery etcd cluster (%v)", err) } defer ds.Close() // step 2: configure the cluster size discoveryToken := "8A591FAB-1D72-41FA-BDF2-A27162FDA1E0" configSizeKey := fmt.Sprintf("/_etcd/registry/%s/_config/size", discoveryToken) configSizeValStr := strconv.Itoa(targetClusterSize) if err = ctlV3Put(ctlCtx{epc: ds}, configSizeKey, configSizeValStr, ""); err != nil { t.Errorf("failed to configure cluster size to discovery serivce, error: %v", err) } // step 3: start the etcd cluster epc, err := bootstrapEtcdClusterUsingV3Discovery(t, ds.EndpointsGRPC(), discoveryToken, targetClusterSize, clientTLSType, isClientAutoTLS) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } defer epc.Close() // step 4: sanity test on the etcd cluster etcdctl := []string{e2e.BinPath.Etcdctl, "--endpoints", strings.Join(epc.EndpointsGRPC(), ",")} require.NoError(t, e2e.SpawnWithExpect(append(etcdctl, "put", "key", "value"), expect.ExpectedResponse{Value: "OK"})) require.NoError(t, e2e.SpawnWithExpect(append(etcdctl, "get", "key"), expect.ExpectedResponse{Value: "value"})) } func bootstrapEtcdClusterUsingV3Discovery(t *testing.T, discoveryEndpoints []string, discoveryToken string, clusterSize int, clientTLSType e2e.ClientConnType, isClientAutoTLS bool) (*e2e.EtcdProcessCluster, error) { // cluster configuration cfg := e2e.NewConfig( e2e.WithBasePort(3000), e2e.WithClusterSize(clusterSize), e2e.WithIsPeerTLS(true), e2e.WithIsPeerAutoTLS(true), e2e.WithDiscoveryToken(discoveryToken), e2e.WithDiscoveryEndpoints(discoveryEndpoints), ) // initialize the cluster epc, err := e2e.InitEtcdProcessCluster(t, cfg) if err != nil { t.Fatalf("could not initialize etcd cluster (%v)", err) return epc, err } // populate discovery related security configuration for _, ep := range epc.Procs { epCfg := ep.Config() if clientTLSType == e2e.ClientTLS { if isClientAutoTLS { epCfg.Args = append(epCfg.Args, "--discovery-insecure-transport=false") epCfg.Args = append(epCfg.Args, "--discovery-insecure-skip-tls-verify=true") } else { epCfg.Args = append(epCfg.Args, "--discovery-cacert="+e2e.CaPath) epCfg.Args = append(epCfg.Args, "--discovery-cert="+e2e.CertPath) epCfg.Args = append(epCfg.Args, "--discovery-key="+e2e.PrivateKeyPath) } } } // start the cluster return e2e.StartEtcdProcessCluster(t.Context(), t, epc, cfg) } ================================================ FILE: tests/e2e/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /* Package e2e implements tests built upon etcd binaries, and focus on end-to-end testing. Features/goals of the end-to-end tests: 1. test command-line parsing and arguments. 2. test user-facing command-line API. 3. launch full processes and check for expected outputs. */ package e2e ================================================ FILE: tests/e2e/etcd_config_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "net" "os" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) const exampleConfigFile = "../../etcd.conf.yml.sample" func TestEtcdExampleConfig(t *testing.T) { e2e.SkipInShortMode(t) proc, err := e2e.SpawnCmd([]string{e2e.BinPath.Etcd, "--config-file", exampleConfigFile}, nil) require.NoError(t, err) require.NoError(t, e2e.WaitReadyExpectProc(t.Context(), proc, e2e.EtcdServerReadyLines)) require.NoError(t, proc.Stop()) } func TestEtcdMultiPeer(t *testing.T) { e2e.SkipInShortMode(t) peers, tmpdirs := make([]string, 3), make([]string, 3) for i := range peers { peers[i] = fmt.Sprintf("e%d=http://127.0.0.1:%d", i, e2e.EtcdProcessBasePort+i) tmpdirs[i] = t.TempDir() } ic := strings.Join(peers, ",") procs := make([]*expect.ExpectProcess, len(peers)) defer func() { for i := range procs { if procs[i] != nil { procs[i].Stop() procs[i].Close() } } }() for i := range procs { args := []string{ e2e.BinPath.Etcd, "--name", fmt.Sprintf("e%d", i), "--listen-client-urls", "http://0.0.0.0:0", "--data-dir", tmpdirs[i], "--advertise-client-urls", "http://0.0.0.0:0", "--listen-peer-urls", fmt.Sprintf("http://127.0.0.1:%d,http://127.0.0.1:%d", e2e.EtcdProcessBasePort+i, e2e.EtcdProcessBasePort+len(peers)+i), "--initial-advertise-peer-urls", fmt.Sprintf("http://127.0.0.1:%d", e2e.EtcdProcessBasePort+i), "--initial-cluster", ic, } p, err := e2e.SpawnCmd(args, nil) require.NoError(t, err) procs[i] = p } for _, p := range procs { err := e2e.WaitReadyExpectProc(t.Context(), p, e2e.EtcdServerReadyLines) require.NoError(t, err) } } // TestEtcdUnixPeers checks that etcd will boot with unix socket peers. func TestEtcdUnixPeers(t *testing.T) { e2e.SkipInShortMode(t) d := t.TempDir() proc, err := e2e.SpawnCmd( []string{ e2e.BinPath.Etcd, "--data-dir", d, "--name", "e1", "--listen-peer-urls", "unix://etcd.unix:1", "--initial-advertise-peer-urls", "unix://etcd.unix:1", "--initial-cluster", "e1=unix://etcd.unix:1", }, nil, ) defer os.Remove("etcd.unix:1") require.NoError(t, err) require.NoError(t, e2e.WaitReadyExpectProc(t.Context(), proc, e2e.EtcdServerReadyLines)) require.NoError(t, proc.Stop()) } // TestEtcdListenMetricsURLsWithMissingClientTLSInfo checks that the HTTPs listen metrics URL // but without the client TLS info will fail its verification. func TestEtcdListenMetricsURLsWithMissingClientTLSInfo(t *testing.T) { e2e.SkipInShortMode(t) tempDir := t.TempDir() defer os.RemoveAll(tempDir) caFile, certFiles, keyFiles, err := generateCertsForIPs(tempDir, []net.IP{net.ParseIP("127.0.0.1")}) require.NoError(t, err) // non HTTP but metrics URL is HTTPS, invalid when the client TLS info is not provided clientURL := fmt.Sprintf("http://localhost:%d", e2e.EtcdProcessBasePort) peerURL := fmt.Sprintf("https://localhost:%d", e2e.EtcdProcessBasePort+1) listenMetricsURL := fmt.Sprintf("https://localhost:%d", e2e.EtcdProcessBasePort+2) commonArgs := []string{ e2e.BinPath.Etcd, "--name", "e0", "--data-dir", tempDir, "--listen-client-urls", clientURL, "--advertise-client-urls", clientURL, "--initial-advertise-peer-urls", peerURL, "--listen-peer-urls", peerURL, "--initial-cluster", "e0=" + peerURL, "--listen-metrics-urls", listenMetricsURL, "--peer-cert-file", certFiles[0], "--peer-key-file", keyFiles[0], "--peer-trusted-ca-file", caFile, "--peer-client-cert-auth", } proc, err := e2e.SpawnCmd(commonArgs, nil) require.NoError(t, err) defer func() { require.NoError(t, proc.Stop()) _ = proc.Close() }() require.NoError(t, e2e.WaitReadyExpectProc(t.Context(), proc, []string{embed.ErrMissingClientTLSInfoForMetricsURL.Error()})) } // TestEtcdPeerCNAuth checks that the inter peer auth based on CN of cert is working correctly. func TestEtcdPeerCNAuth(t *testing.T) { e2e.SkipInShortMode(t) peers, tmpdirs := make([]string, 3), make([]string, 3) for i := range peers { peers[i] = fmt.Sprintf("e%d=https://127.0.0.1:%d", i, e2e.EtcdProcessBasePort+i) tmpdirs[i] = t.TempDir() } ic := strings.Join(peers, ",") procs := make([]*expect.ExpectProcess, len(peers)) defer func() { for i := range procs { if procs[i] != nil { procs[i].Stop() procs[i].Close() } } }() // node 0 and 1 have a cert with the correct CN, node 2 doesn't for i := range procs { commonArgs := []string{ e2e.BinPath.Etcd, "--name", fmt.Sprintf("e%d", i), "--listen-client-urls", "http://0.0.0.0:0", "--data-dir", tmpdirs[i], "--advertise-client-urls", "http://0.0.0.0:0", "--listen-peer-urls", fmt.Sprintf("https://127.0.0.1:%d,https://127.0.0.1:%d", e2e.EtcdProcessBasePort+i, e2e.EtcdProcessBasePort+len(peers)+i), "--initial-advertise-peer-urls", fmt.Sprintf("https://127.0.0.1:%d", e2e.EtcdProcessBasePort+i), "--initial-cluster", ic, } var args []string if i <= 1 { args = []string{ "--peer-cert-file", e2e.CertPath, "--peer-key-file", e2e.PrivateKeyPath, "--peer-client-cert-file", e2e.CertPath, "--peer-client-key-file", e2e.PrivateKeyPath, "--peer-trusted-ca-file", e2e.CaPath, "--peer-client-cert-auth", "--peer-cert-allowed-cn", "example.com", } } else { args = []string{ "--peer-cert-file", e2e.CertPath2, "--peer-key-file", e2e.PrivateKeyPath2, "--peer-client-cert-file", e2e.CertPath2, "--peer-client-key-file", e2e.PrivateKeyPath2, "--peer-trusted-ca-file", e2e.CaPath, "--peer-client-cert-auth", "--peer-cert-allowed-cn", "example2.com", } } commonArgs = append(commonArgs, args...) p, err := e2e.SpawnCmd(commonArgs, nil) require.NoError(t, err) procs[i] = p } for i, p := range procs { var expect []string if i <= 1 { expect = e2e.EtcdServerReadyLines } else { expect = []string{"remote error: tls: bad certificate"} } err := e2e.WaitReadyExpectProc(t.Context(), p, expect) require.NoError(t, err) } } // TestEtcdPeerMultiCNAuth checks that the inter peer auth based on CN of cert is working correctly // when there are multiple allowed values for the CN. func TestEtcdPeerMultiCNAuth(t *testing.T) { e2e.SkipInShortMode(t) peers, tmpdirs := make([]string, 3), make([]string, 3) for i := range peers { peers[i] = fmt.Sprintf("e%d=https://127.0.0.1:%d", i, e2e.EtcdProcessBasePort+i) tmpdirs[i] = t.TempDir() } ic := strings.Join(peers, ",") procs := make([]*expect.ExpectProcess, len(peers)) defer func() { for i := range procs { if procs[i] != nil { procs[i].Stop() procs[i].Close() } } }() // all nodes have unique certs with different CNs // node 0 and 1 have a cert with one of the correct CNs, node 2 doesn't for i := range procs { commonArgs := []string{ e2e.BinPath.Etcd, "--name", fmt.Sprintf("e%d", i), "--listen-client-urls", "http://0.0.0.0:0", "--data-dir", tmpdirs[i], "--advertise-client-urls", "http://0.0.0.0:0", "--listen-peer-urls", fmt.Sprintf("https://127.0.0.1:%d,https://127.0.0.1:%d", e2e.EtcdProcessBasePort+i, e2e.EtcdProcessBasePort+len(peers)+i), "--initial-advertise-peer-urls", fmt.Sprintf("https://127.0.0.1:%d", e2e.EtcdProcessBasePort+i), "--initial-cluster", ic, } var args []string switch i { case 0: args = []string{ "--peer-cert-file", e2e.CertPath, // server.crt has CN "example.com". "--peer-key-file", e2e.PrivateKeyPath, "--peer-client-cert-file", e2e.CertPath, "--peer-client-key-file", e2e.PrivateKeyPath, "--peer-trusted-ca-file", e2e.CaPath, "--peer-client-cert-auth", "--peer-cert-allowed-cn", "example.com,example2.com", } case 1: args = []string{ "--peer-cert-file", e2e.CertPath2, // server2.crt has CN "example2.com". "--peer-key-file", e2e.PrivateKeyPath2, "--peer-client-cert-file", e2e.CertPath2, "--peer-client-key-file", e2e.PrivateKeyPath2, "--peer-trusted-ca-file", e2e.CaPath, "--peer-client-cert-auth", "--peer-cert-allowed-cn", "example.com,example2.com", } default: args = []string{ "--peer-cert-file", e2e.CertPath3, // server3.crt has CN "ca". "--peer-key-file", e2e.PrivateKeyPath3, "--peer-client-cert-file", e2e.CertPath3, "--peer-client-key-file", e2e.PrivateKeyPath3, "--peer-trusted-ca-file", e2e.CaPath, "--peer-client-cert-auth", "--peer-cert-allowed-cn", "example.com,example2.com", } } commonArgs = append(commonArgs, args...) p, err := e2e.SpawnCmd(commonArgs, nil) require.NoError(t, err) procs[i] = p } for i, p := range procs { var expect []string if i <= 1 { expect = e2e.EtcdServerReadyLines } else { expect = []string{"remote error: tls: bad certificate"} } err := e2e.WaitReadyExpectProc(t.Context(), p, expect) require.NoError(t, err) } } // TestEtcdPeerNameAuth checks that the inter peer auth based on cert name validation is working correctly. func TestEtcdPeerNameAuth(t *testing.T) { e2e.SkipInShortMode(t) peers, tmpdirs := make([]string, 3), make([]string, 3) for i := range peers { peers[i] = fmt.Sprintf("e%d=https://127.0.0.1:%d", i, e2e.EtcdProcessBasePort+i) tmpdirs[i] = t.TempDir() } ic := strings.Join(peers, ",") procs := make([]*expect.ExpectProcess, len(peers)) defer func() { for i := range procs { if procs[i] != nil { procs[i].Stop() procs[i].Close() } os.RemoveAll(tmpdirs[i]) } }() // node 0 and 1 have a cert with the correct certificate name, node 2 doesn't for i := range procs { commonArgs := []string{ e2e.BinPath.Etcd, "--name", fmt.Sprintf("e%d", i), "--listen-client-urls", "http://0.0.0.0:0", "--data-dir", tmpdirs[i], "--advertise-client-urls", "http://0.0.0.0:0", "--listen-peer-urls", fmt.Sprintf("https://127.0.0.1:%d,https://127.0.0.1:%d", e2e.EtcdProcessBasePort+i, e2e.EtcdProcessBasePort+len(peers)+i), "--initial-advertise-peer-urls", fmt.Sprintf("https://127.0.0.1:%d", e2e.EtcdProcessBasePort+i), "--initial-cluster", ic, } var args []string if i <= 1 { args = []string{ "--peer-cert-file", e2e.CertPath, "--peer-key-file", e2e.PrivateKeyPath, "--peer-trusted-ca-file", e2e.CaPath, "--peer-client-cert-auth", "--peer-cert-allowed-hostname", "localhost", } } else { args = []string{ "--peer-cert-file", e2e.CertPath2, "--peer-key-file", e2e.PrivateKeyPath2, "--peer-trusted-ca-file", e2e.CaPath, "--peer-client-cert-auth", "--peer-cert-allowed-hostname", "example2.com", } } commonArgs = append(commonArgs, args...) p, err := e2e.SpawnCmd(commonArgs, nil) require.NoError(t, err) procs[i] = p } for i, p := range procs { var expect []string if i <= 1 { expect = e2e.EtcdServerReadyLines } else { expect = []string{"client certificate authentication failed"} } err := e2e.WaitReadyExpectProc(t.Context(), p, expect) require.NoError(t, err) } } // TestEtcdPeerLocalAddr checks that the inter peer auth works with when // the member LocalAddr is set. func TestEtcdPeerLocalAddr(t *testing.T) { e2e.SkipInShortMode(t) nodeIP, err := getLocalIP() t.Log("Using node IP", nodeIP) require.NoError(t, err) peers, tmpdirs := make([]string, 3), make([]string, 3) for i := range peers { peerIP := nodeIP if i == 0 { peerIP = "127.0.0.1" } peers[i] = fmt.Sprintf("e%d=https://%s:%d", i, peerIP, e2e.EtcdProcessBasePort+i) tmpdirs[i] = t.TempDir() } procs := make([]*expect.ExpectProcess, len(peers)) defer func() { for i := range procs { if procs[i] != nil { procs[i].Stop() procs[i].Close() } os.RemoveAll(tmpdirs[i]) } }() tempDir := t.TempDir() caFile, certFiles, keyFiles, err := generateCertsForIPs(tempDir, []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP(nodeIP)}) require.NoError(t, err) defer func() { os.RemoveAll(tempDir) }() // node 0 (127.0.0.1) does not set `--feature-gates=SetMemberLocalAddr=true`, // while nodes 1 and nodes 2 do. // // node 0's peer certificate is signed for 127.0.0.1, but it uses the host // IP (by default) to communicate with peers, so they don't match. // Accordingly, other peers will reject connections from node 0. // // Both node 1 and node 2's peer certificates are signed for the host IP, // and they also communicate with peers using the host IP (explicitly set // with --initial-advertise-peer-urls and // --feature-gates=SetMemberLocalAddr=true), so node 0 has no issue connecting // to them. // // Refer to https://github.com/etcd-io/etcd/issues/17068. for i := range procs { peerIP := nodeIP if i == 0 { peerIP = "127.0.0.1" } ic := strings.Join(peers, ",") commonArgs := []string{ e2e.BinPath.Etcd, "--name", fmt.Sprintf("e%d", i), "--listen-client-urls", "http://0.0.0.0:0", "--data-dir", tmpdirs[i], "--advertise-client-urls", "http://0.0.0.0:0", "--initial-advertise-peer-urls", fmt.Sprintf("https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i), "--listen-peer-urls", fmt.Sprintf("https://%s:%d,https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i, peerIP, e2e.EtcdProcessBasePort+len(peers)+i), "--initial-cluster", ic, } var args []string if i == 0 { args = []string{ "--peer-cert-file", certFiles[0], "--peer-key-file", keyFiles[0], "--peer-trusted-ca-file", caFile, "--peer-client-cert-auth", } } else { args = []string{ "--peer-cert-file", certFiles[1], "--peer-key-file", keyFiles[1], "--peer-trusted-ca-file", caFile, "--peer-client-cert-auth", "--feature-gates=SetMemberLocalAddr=true", } } commonArgs = append(commonArgs, args...) p, err := e2e.SpawnCmd(commonArgs, nil) require.NoError(t, err) procs[i] = p } for i, p := range procs { var expect []string if i == 0 { expect = e2e.EtcdServerReadyLines } else { expect = []string{"x509: certificate is valid for 127.0.0.1, not "} } err := e2e.WaitReadyExpectProc(t.Context(), p, expect) require.NoError(t, err) } } func TestGrpcproxyAndCommonName(t *testing.T) { e2e.SkipInShortMode(t) argsWithNonEmptyCN := []string{ e2e.BinPath.Etcd, "grpc-proxy", "start", "--cert", e2e.CertPath2, "--key", e2e.PrivateKeyPath2, "--cacert", e2e.CaPath, } argsWithEmptyCN := []string{ e2e.BinPath.Etcd, "grpc-proxy", "start", "--cert", e2e.CertPath3, "--key", e2e.PrivateKeyPath3, "--cacert", e2e.CaPath, } err := e2e.SpawnWithExpect(argsWithNonEmptyCN, expect.ExpectedResponse{Value: "cert has non empty Common Name"}) require.ErrorContains(t, err, "cert has non empty Common Name") p, err := e2e.SpawnCmd(argsWithEmptyCN, nil) defer func() { if p != nil { p.Stop() } }() require.NoError(t, err) } func TestGrpcproxyAndListenCipherSuite(t *testing.T) { e2e.SkipInShortMode(t) cases := []struct { name string args []string }{ { name: "ArgsWithCipherSuites", args: []string{ e2e.BinPath.Etcd, "grpc-proxy", "start", "--listen-cipher-suites", "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", }, }, { name: "ArgsWithoutCipherSuites", args: []string{ e2e.BinPath.Etcd, "grpc-proxy", "start", "--listen-cipher-suites", "", }, }, } for _, test := range cases { t.Run(test.name, func(t *testing.T) { pw, err := e2e.SpawnCmd(test.args, nil) require.NoError(t, err) require.NoError(t, pw.Stop()) }) } } func TestBootstrapDefragFlag(t *testing.T) { e2e.SkipInShortMode(t) proc, err := e2e.SpawnCmd([]string{e2e.BinPath.Etcd, "--bootstrap-defrag-threshold-megabytes", "1000"}, nil) require.NoError(t, err) require.NoError(t, e2e.WaitReadyExpectProc(t.Context(), proc, []string{"Skipping defragmentation"})) require.NoError(t, proc.Stop()) // wait for the process to exit, otherwise test will have leaked goroutine if err := proc.Close(); err != nil { t.Logf("etcd process closed with error %v", err) } } func TestSnapshotCatchupEntriesFlag(t *testing.T) { e2e.SkipInShortMode(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() proc, err := e2e.SpawnCmd([]string{e2e.BinPath.Etcd, "--snapshot-catchup-entries", "1000"}, nil) require.NoError(t, err) require.NoError(t, e2e.WaitReadyExpectProc(ctx, proc, []string{"\"snapshot-catchup-entries\":1000"})) require.NoError(t, e2e.WaitReadyExpectProc(ctx, proc, []string{"serving client traffic"})) require.NoError(t, proc.Stop()) // wait for the process to exit, otherwise test will have leaked goroutine if err := proc.Close(); err != nil { t.Logf("etcd process closed with error %v", err) } } // TestEtcdHealthyWithTinySnapshotCatchupEntries ensures multi-node etcd cluster remains healthy with 1 snapshot catch up entry func TestEtcdHealthyWithTinySnapshotCatchupEntries(t *testing.T) { e2e.BeforeTest(t) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithClusterSize(3), e2e.WithSnapshotCount(1), e2e.WithSnapshotCatchUpEntries(1), ) require.NoErrorf(t, err, "could not start etcd process cluster (%v)", err) t.Cleanup(func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }) // simulate 10 clients keep writing to etcd in parallel with no error ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() g, ctx := errgroup.WithContext(ctx) for i := 0; i < 10; i++ { clientID := i g.Go(func() error { cc := epc.Etcdctl() for j := 0; j < 100; j++ { if _, err := cc.Put(ctx, "foo", fmt.Sprintf("bar%d", clientID), config.PutOptions{}); err != nil { return err } } return nil }) } require.NoError(t, g.Wait()) } func TestEtcdTLSVersion(t *testing.T) { e2e.SkipInShortMode(t) d := t.TempDir() proc, err := e2e.SpawnCmd( []string{ e2e.BinPath.Etcd, "--data-dir", d, "--name", "e1", "--listen-client-urls", "https://0.0.0.0:0", "--advertise-client-urls", "https://0.0.0.0:0", "--listen-peer-urls", fmt.Sprintf("https://127.0.0.1:%d", e2e.EtcdProcessBasePort), "--initial-advertise-peer-urls", fmt.Sprintf("https://127.0.0.1:%d", e2e.EtcdProcessBasePort), "--initial-cluster", fmt.Sprintf("e1=https://127.0.0.1:%d", e2e.EtcdProcessBasePort), "--peer-cert-file", e2e.CertPath, "--peer-key-file", e2e.PrivateKeyPath, "--cert-file", e2e.CertPath2, "--key-file", e2e.PrivateKeyPath2, "--tls-min-version", "TLS1.2", "--tls-max-version", "TLS1.3", }, nil, ) assert.NoError(t, err) assert.NoErrorf(t, e2e.WaitReadyExpectProc(t.Context(), proc, e2e.EtcdServerReadyLines), "did not receive expected output from etcd process") assert.NoError(t, proc.Stop()) proc.Wait() // ensure the port has been released proc.Close() } // TestEtcdDeprecatedFlags checks that etcd will print warning messages if deprecated flags are set. func TestEtcdDeprecatedFlags(t *testing.T) { e2e.SkipInShortMode(t) commonArgs := []string{ e2e.BinPath.Etcd, "--name", "e1", } testCases := []struct { name string args []string expectedMsg string }{ { name: "max-snapshots", args: append(commonArgs, "--max-snapshots=10"), expectedMsg: "--max-snapshots is deprecated in 3.6 and will be decommissioned in 3.8", }, { name: "v2-deprecation", args: append(commonArgs, "--v2-deprecation", "write-only-drop-data"), expectedMsg: "--v2-deprecation is deprecated and scheduled for removal in v3.8. The default value is enforced, ignoring user input", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { proc, err := e2e.SpawnCmd( tc.args, nil, ) require.NoError(t, err) require.NoError(t, e2e.WaitReadyExpectProc(t.Context(), proc, []string{tc.expectedMsg})) require.NoError(t, proc.Stop()) proc.Wait() // ensure the port has been released proc.Close() }) } } // TestV2DeprecationEnforceDefaultValue verifies that etcd enforces the default V2Deprecation level // and ignores users input. func TestV2DeprecationEnforceDefaultValue(t *testing.T) { e2e.SkipInShortMode(t) commonArgs := []string{ e2e.BinPath.Etcd, "--name", "e1", } validV2DeprecationLevels := []string{"write-only", "write-only-drop-data", "gone"} expectedDeprecationLevelMsg := `"v2-deprecation":"write-only"` for _, optionLevel := range validV2DeprecationLevels { t.Run(optionLevel, func(t *testing.T) { proc, err := e2e.SpawnCmd( append(commonArgs, "--v2-deprecation", optionLevel), nil, ) require.NoError(t, err) require.NoError(t, e2e.WaitReadyExpectProc(t.Context(), proc, []string{expectedDeprecationLevelMsg})) require.NoError(t, proc.Stop()) proc.Wait() // ensure the port has been released proc.Close() }) } } func TestEtcdAdvertiseClientUnix(t *testing.T) { e2e.SkipInShortMode(t) // Create a temporary directory for the data directory dataDir := t.TempDir() socketDir := t.TempDir() unixSocket := fmt.Sprintf("unix://%s/etcd-client.sock", socketDir) // Start etcd with AdvertiseClientUrls set to a unix socket proc, err := e2e.SpawnCmd( []string{ e2e.BinPath.Etcd, "--data-dir", dataDir, "--name", "etcd1", "--listen-client-urls", unixSocket, "--advertise-client-urls", unixSocket, }, nil, ) require.NoError(t, err) defer func() { _ = proc.Stop() _ = proc.Close() }() // Wait for the process to be ready require.NoError(t, e2e.WaitReadyExpectProc(t.Context(), proc, e2e.EtcdServerReadyLines)) // Write a key/value pair using etcdctl with unix socket putArgs := []string{ e2e.BinPath.Etcdctl, "--endpoints", unixSocket, "put", "foo", "bar", } putProc, err := e2e.SpawnCmd(putArgs, nil) require.NoError(t, err) require.NoError(t, e2e.WaitReadyExpectProc(t.Context(), putProc, []string{"OK"})) require.NoError(t, putProc.Stop()) _ = putProc.Close() // Read the key back using etcdctl with unix socket getArgs := []string{ e2e.BinPath.Etcdctl, "--endpoints", unixSocket, "get", "foo", } getProc, err := e2e.SpawnCmd(getArgs, nil) require.NoError(t, err) require.NoError(t, e2e.WaitReadyExpectProc(t.Context(), getProc, []string{"foo", "bar"})) require.NoError(t, getProc.Stop()) _ = getProc.Close() } ================================================ FILE: tests/e2e/etcd_grpcproxy_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestGrpcProxyAutoSync(t *testing.T) { e2e.SkipInShortMode(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(1)) require.NoError(t, err) defer func() { assert.NoError(t, epc.Close()) }() var ( node1ClientURL = epc.Procs[0].Config().ClientURL proxyClientURL = "127.0.0.1:32379" ) // Run independent grpc-proxy instance proxyProc, err := e2e.SpawnCmd([]string{ e2e.BinPath.Etcd, "grpc-proxy", "start", "--advertise-client-url", proxyClientURL, "--listen-addr", proxyClientURL, "--endpoints", node1ClientURL, "--endpoints-auto-sync-interval", "1s", }, nil) require.NoError(t, err) defer func() { assert.NoError(t, proxyProc.Stop()) }() proxyCtl, err := e2e.NewEtcdctl(e2e.ClientConfig{}, []string{proxyClientURL}) require.NoError(t, err) _, err = proxyCtl.Put(ctx, "k1", "v1", config.PutOptions{}) require.NoError(t, err) // Add and start second member _, err = epc.StartNewProc(ctx, nil, t, false /* addAsLearner */) require.NoError(t, err) // Wait for auto sync of endpoints err = waitForEndpointInLog(ctx, proxyProc, epc.Procs[1].Config().ClientURL) require.NoError(t, err) err = epc.CloseProc(ctx, func(proc e2e.EtcdProcess) bool { return proc.Config().ClientURL == node1ClientURL }) require.NoError(t, err) var resp *clientv3.GetResponse for i := 0; i < 10; i++ { resp, err = proxyCtl.Get(ctx, "k1", config.GetOptions{}) if err != nil && strings.Contains(err.Error(), rpctypes.ErrGRPCLeaderChanged.Error()) { time.Sleep(500 * time.Millisecond) continue } } require.NoError(t, err) kvs := testutils.KeyValuesFromGetResponse(resp) assert.Equal(t, []testutils.KV{{Key: "k1", Val: "v1"}}, kvs) } func TestGrpcProxyTLSVersions(t *testing.T) { e2e.SkipInShortMode(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(1)) require.NoError(t, err) defer func() { assert.NoError(t, epc.Close()) }() var ( node1ClientURL = epc.Procs[0].Config().ClientURL proxyClientURL = "127.0.0.1:42379" ) // Run independent grpc-proxy instance proxyProc, err := e2e.SpawnCmd([]string{ e2e.BinPath.Etcd, "grpc-proxy", "start", "--advertise-client-url", proxyClientURL, "--listen-addr", proxyClientURL, "--endpoints", node1ClientURL, "--endpoints-auto-sync-interval", "1s", "--cert-file", e2e.CertPath2, "--key-file", e2e.PrivateKeyPath2, "--trusted-ca-file", e2e.CaPath, "--tls-min-version", "TLS1.2", "--tls-max-version", "TLS1.3", }, nil) require.NoError(t, err) defer func() { assert.NoError(t, proxyProc.Stop()) }() _, err = proxyProc.ExpectFunc(ctx, func(s string) bool { return strings.Contains(s, "started gRPC proxy") }) require.NoError(t, err) ctx, cancel = context.WithTimeout(ctx, 10*time.Second) defer cancel() healthErr := e2e.SpawnWithExpectsContext(ctx, []string{ "curl", "--http1.1", "--fail", "--verbose", "--cacert", e2e.CaPath, "--cert", e2e.CertPath2, "--key", e2e.PrivateKeyPath2, "https://" + proxyClientURL + "/proxy/health", }, nil, expect.ExpectedResponse{Value: `"health":"true"`}) require.NoError(t, healthErr) } func waitForEndpointInLog(ctx context.Context, proxyProc *expect.ExpectProcess, endpoint string) error { endpoint = strings.Replace(endpoint, "http://", "", 1) ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() _, err := proxyProc.ExpectFunc(ctx, func(s string) bool { return strings.Contains(s, endpoint) }) return err } ================================================ FILE: tests/e2e/etcd_mix_versions_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) type clusterTestCase struct { name string config *e2e.EtcdProcessClusterConfig } func clusterTestCases(size int) []clusterTestCase { tcs := []clusterTestCase{ { name: "CurrentVersion", config: e2e.NewConfig(e2e.WithClusterSize(size)), }, } if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { return tcs } tcs = append(tcs, clusterTestCase{ name: "LastVersion", config: e2e.NewConfig(e2e.WithClusterSize(size), e2e.WithVersion(e2e.LastVersion)), }, ) if size > 2 { tcs = append(tcs, clusterTestCase{ name: "MinorityLastVersion", config: e2e.NewConfig(e2e.WithClusterSize(size), e2e.WithVersion(e2e.MinorityLastVersion)), }, clusterTestCase{ name: "QuorumLastVersion", config: e2e.NewConfig(e2e.WithClusterSize(size), e2e.WithVersion(e2e.QuorumLastVersion)), }, ) } return tcs } // TestMixVersionsSnapshotByAddingMember tests the mix version send snapshots by adding member func TestMixVersionsSnapshotByAddingMember(t *testing.T) { for _, tc := range clusterTestCases(1) { t.Run(tc.name+"-adding-new-member-of-current-version", func(t *testing.T) { mixVersionsSnapshotTestByAddingMember(t, tc.config, e2e.CurrentVersion) }) // etcd doesn't support adding a new member of old version into // a cluster with higher version. For example, etcd cluster // version is 3.6.x, then a new member of 3.5.x can't join the // cluster. Please refer to link below, // https://github.com/etcd-io/etcd/blob/3e903d0b12e399519a4013c52d4635ec8bdd6863/server/etcdserver/cluster_util.go#L222-L230 /*t.Run(tc.name+"-adding-new-member-of-last-version", func(t *testing.T) { mixVersionsSnapshotTestByAddingMember(t, tc.config, e2e.LastVersion) })*/ } } func mixVersionsSnapshotTestByAddingMember(t *testing.T, cfg *e2e.EtcdProcessClusterConfig, newInstanceVersion e2e.ClusterVersion) { e2e.BeforeTest(t) if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease) } t.Logf("Create an etcd cluster with %d member", cfg.ClusterSize) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg), e2e.WithSnapshotCount(10), ) require.NoErrorf(t, err, "failed to start etcd cluster") defer func() { derr := epc.Close() require.NoErrorf(t, derr, "failed to close etcd cluster") }() t.Log("Writing 20 keys to the cluster (more than SnapshotCount entries to trigger at least a snapshot)") writeKVs(t, epc.Etcdctl(), 0, 20) t.Log("start a new etcd instance, which will receive a snapshot from the leader.") newCfg := *epc.Cfg newCfg.Version = newInstanceVersion newCfg.ServerConfig.SnapshotCatchUpEntries = 10 t.Log("Starting a new etcd instance") _, err = epc.StartNewProc(t.Context(), &newCfg, t, false /* addAsLearner */) require.NoErrorf(t, err, "failed to start the new etcd instance") defer epc.CloseProc(t.Context(), nil) assertKVHash(t, epc) } func TestMixVersionsSnapshotByMockingPartition(t *testing.T) { mockPartitionNodeIndex := 2 for _, tc := range clusterTestCases(3) { t.Run(tc.name, func(t *testing.T) { mixVersionsSnapshotTestByMockPartition(t, tc.config, mockPartitionNodeIndex) }) } } func mixVersionsSnapshotTestByMockPartition(t *testing.T, cfg *e2e.EtcdProcessClusterConfig, mockPartitionNodeIndex int) { e2e.BeforeTest(t) if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease) } clusterOptions := []e2e.EPClusterOption{ e2e.WithConfig(cfg), e2e.WithSnapshotCount(10), e2e.WithSnapshotCatchUpEntries(10), } t.Logf("Create an etcd cluster with %d member", cfg.ClusterSize) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, clusterOptions...) require.NoErrorf(t, err, "failed to start etcd cluster") defer func() { derr := epc.Close() require.NoErrorf(t, derr, "failed to close etcd cluster") }() toPartitionedMember := epc.Procs[mockPartitionNodeIndex] t.Log("Stop and restart the partitioned member") err = toPartitionedMember.Stop() require.NoError(t, err) t.Log("Writing 30 keys to the cluster (more than SnapshotCount entries to trigger at least a snapshot)") writeKVs(t, epc.Etcdctl(), 0, 30) t.Log("Verify logs to check leader has saved snapshot") leaderEPC := epc.Procs[epc.WaitLeader(t)] e2e.AssertProcessLogs(t, leaderEPC, "saved snapshot") t.Log("Restart the partitioned member") err = toPartitionedMember.Restart(t.Context()) require.NoError(t, err) assertKVHash(t, epc) leaderEPC = epc.Procs[epc.WaitLeader(t)] t.Log("Verify logs to check snapshot be sent from leader to follower") e2e.AssertProcessLogs(t, leaderEPC, "sent database snapshot") } func writeKVs(t *testing.T, etcdctl *e2e.EtcdctlV3, startIdx, endIdx int) { for i := startIdx; i < endIdx; i++ { key := fmt.Sprintf("key-%d", i) value := fmt.Sprintf("value-%d", i) _, err := etcdctl.Put(t.Context(), key, value, config.PutOptions{}) require.NoErrorf(t, err, "failed to put %q", key) } } func assertKVHash(t *testing.T, epc *e2e.EtcdProcessCluster) { clusterSize := len(epc.Procs) if clusterSize < 2 { return } t.Log("Verify all nodes have exact same revision and hash") assert.Eventually(t, func() bool { hashKvs, err := epc.Etcdctl().HashKV(t.Context(), 0) if err != nil { t.Logf("failed to get HashKV: %v", err) return false } if len(hashKvs) != clusterSize { t.Logf("expected %d hashkv responses, but got: %d", clusterSize, len(hashKvs)) return false } for i := 1; i < clusterSize; i++ { if hashKvs[0].Header.Revision != hashKvs[i].Header.Revision { t.Logf("Got different revisions, [%d, %d]", hashKvs[0].Header.Revision, hashKvs[i].Header.Revision) return false } assert.Equal(t, hashKvs[0].Hash, hashKvs[i].Hash) } return true }, 10*time.Second, 500*time.Millisecond) } ================================================ FILE: tests/e2e/etcd_release_upgrade_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "strings" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/testutils" ) // TestReleaseUpgrade ensures that changes to master branch does not affect // upgrade from latest etcd releases. func TestReleaseUpgrade(t *testing.T) { if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease) } e2e.BeforeTest(t) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithVersion(e2e.LastVersion), e2e.WithSnapshotCount(3), e2e.WithBasePeerScheme("unix"), // to avoid port conflict ) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }() cx := ctlCtx{ t: t, cfg: *e2e.NewConfigNoTLS(), dialTimeout: 7 * time.Second, quorum: true, epc: epc, } var kvs []kv for i := 0; i < 5; i++ { kvs = append(kvs, kv{key: fmt.Sprintf("foo%d", i), val: "bar"}) } for i := range kvs { if err = ctlV3Put(cx, kvs[i].key, kvs[i].val, ""); err != nil { cx.t.Fatalf("#%d: ctlV3Put error (%v)", i, err) } } t.Log("Cluster of etcd in old version running") for i := range epc.Procs { t.Logf("Stopping node: %v", i) if err = epc.Procs[i].Stop(); err != nil { t.Fatalf("#%d: error closing etcd process (%v)", i, err) } t.Logf("Stopped node: %v", i) epc.Procs[i].Config().ExecPath = e2e.BinPath.Etcd epc.Procs[i].Config().KeepDataDir = true t.Logf("Restarting node in the new version: %v", i) if err = epc.Procs[i].Restart(t.Context()); err != nil { t.Fatalf("error restarting etcd process (%v)", err) } t.Logf("Testing reads after node restarts: %v", i) for j := range kvs { require.NoErrorf(cx.t, ctlV3Get(cx, []string{kvs[j].key}, []kv{kvs[j]}...), "#%d-%d: ctlV3Get error", i, j) } t.Logf("Tested reads after node restarts: %v", i) } t.Log("Waiting for full upgrade...") // TODO: update after release candidate // expect upgraded cluster version // new cluster version needs more time to upgrade ver := version.Cluster(version.Version) for i := 0; i < 7; i++ { if err = e2e.CURLGet(epc, e2e.CURLReq{Endpoint: "/version", Expected: expect.ExpectedResponse{Value: `"etcdcluster":"` + ver}}); err != nil { t.Logf("#%d: %v is not ready yet (%v)", i, ver, err) time.Sleep(time.Second) continue } break } if err != nil { t.Fatalf("cluster version is not upgraded (%v)", err) } t.Log("TestReleaseUpgrade businessLogic DONE") } func TestReleaseUpgradeWithRestart(t *testing.T) { if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease) } e2e.BeforeTest(t) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithVersion(e2e.LastVersion), e2e.WithSnapshotCount(10), e2e.WithBasePeerScheme("unix"), ) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }() cx := ctlCtx{ t: t, cfg: *e2e.NewConfigNoTLS(), dialTimeout: 7 * time.Second, quorum: true, epc: epc, } var kvs []kv for i := 0; i < 50; i++ { kvs = append(kvs, kv{key: fmt.Sprintf("foo%d", i), val: "bar"}) } for i := range kvs { require.NoErrorf(cx.t, ctlV3Put(cx, kvs[i].key, kvs[i].val, ""), "#%d: ctlV3Put error", i) } for i := range epc.Procs { require.NoErrorf(t, epc.Procs[i].Stop(), "#%d: error closing etcd process", i) } var wg sync.WaitGroup wg.Add(len(epc.Procs)) for i := range epc.Procs { go func(i int) { epc.Procs[i].Config().ExecPath = e2e.BinPath.Etcd epc.Procs[i].Config().KeepDataDir = true assert.NoErrorf(t, epc.Procs[i].Restart(t.Context()), "error restarting etcd process") wg.Done() }(i) } wg.Wait() require.NoError(t, ctlV3Get(cx, []string{kvs[0].key}, []kv{kvs[0]}...)) } func TestClusterUpgradeAfterPromotingMembers(t *testing.T) { if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease) } e2e.BeforeTest(t) currentVersion, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd) require.NoErrorf(t, err, "failed to get version from binary") lastClusterVersion, err := e2e.GetVersionFromBinary(e2e.BinPath.EtcdLastRelease) require.NoErrorf(t, err, "failed to get version from last release binary") clusterSize := 3 for _, tc := range []struct { name string snapshot int }{ { name: "create snapshot after promoted", snapshot: 10, }, { name: "no snapshot after promoted", }, } { t.Run(tc.name, func(t *testing.T) { ctx := t.Context() epc, _ := mustCreateNewClusterByPromotingMembers(t, e2e.LastVersion, clusterSize, e2e.WithSnapshotCount(uint64(tc.snapshot))) defer func() { require.NoError(t, epc.Close()) }() for i := 0; i < tc.snapshot; i++ { _, err = epc.Etcdctl().Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) } err = e2e.DowngradeUpgradeMembers(t, nil, epc, clusterSize, false, lastClusterVersion, currentVersion) require.NoError(t, err) t.Logf("Checking all members' status after upgrading") ensureAllMembersAreVotingMembers(t, epc) t.Logf("Checking all members are ready to serve client requests") for i := 0; i < clusterSize; i++ { _, err = epc.Procs[i].Etcdctl().Put(t.Context(), "foo", "bar", config.PutOptions{}) require.NoError(t, err) } }) } } func mustCreateNewClusterByPromotingMembers(t *testing.T, clusterVersion e2e.ClusterVersion, clusterSize int, opts ...e2e.EPClusterOption) (*e2e.EtcdProcessCluster, []*etcdserverpb.Member) { require.GreaterOrEqualf(t, clusterSize, 1, "clusterSize must be at least 1") ctx := t.Context() t.Logf("Creating new etcd cluster - version: %s, clusterSize: %v", clusterVersion, clusterSize) opts = append(opts, e2e.WithVersion(clusterVersion), e2e.WithClusterSize(1)) epc, err := e2e.NewEtcdProcessCluster(ctx, t, opts...) require.NoErrorf(t, err, "failed to start first etcd process") defer func() { if t.Failed() { epc.Close() } }() var promotedMembers []*etcdserverpb.Member for i := 1; i < clusterSize; i++ { var ( memberID uint64 aerr error ) // NOTE: New promoted member needs time to get connected. t.Logf("[%d] Adding new member as learner", i) testutils.ExecuteWithTimeout(t, 1*time.Minute, func() { for { memberID, aerr = epc.StartNewProc(ctx, nil, t, true) if aerr != nil { if strings.Contains(aerr.Error(), "etcdserver: unhealthy cluster") { time.Sleep(1 * time.Second) continue } } break } }) require.NoError(t, aerr) t.Logf("[%d] Promoting member (%x)", i, memberID) etcdctl := epc.Procs[0].Etcdctl() resp, merr := etcdctl.MemberPromote(ctx, memberID) require.NoError(t, merr) for _, m := range resp.Members { if m.ID == memberID { promotedMembers = append(promotedMembers, m) } } } t.Log("Ensure all members are voting members from user perspective") ensureAllMembersAreVotingMembers(t, epc) return epc, promotedMembers } func ensureAllMembersAreVotingMembers(t *testing.T, epc *e2e.EtcdProcessCluster) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() resp, err := epc.Etcdctl().MemberList(ctx, false) require.NoError(t, err) require.Len(t, resp.Members, len(epc.Procs)) for _, m := range resp.Members { require.Falsef(t, m.IsLearner, "node(%x)", m.ID) } } ================================================ FILE: tests/e2e/failover_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "context" "testing" "time" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" "google.golang.org/grpc" _ "google.golang.org/grpc/health" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) const ( // in sync with how kubernetes uses etcd // https://github.com/kubernetes/kubernetes/blob/release-1.28/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go#L59-L71 keepaliveTime = 30 * time.Second keepaliveTimeout = 10 * time.Second dialTimeout = 20 * time.Second clientRuntime = 10 * time.Second requestTimeout = 100 * time.Millisecond ) func TestFailoverOnDefrag(t *testing.T) { tcs := []struct { name string clusterOptions []e2e.EPClusterOption gRPCDialOptions []grpc.DialOption // common assertion expectedMinQPS float64 // happy case assertion expectedMaxFailureRate float64 // negative case assertion expectedMinFailureRate float64 }{ { name: "defrag failover happy case", clusterOptions: []e2e.EPClusterOption{ e2e.WithClusterSize(3), e2e.WithServerFeatureGate("StopGRPCServiceOnDefrag", true), e2e.WithGoFailEnabled(true), }, gRPCDialOptions: []grpc.DialOption{ grpc.WithDisableServiceConfig(), grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy": "round_robin", "healthCheckConfig": {"serviceName": ""}}`), }, expectedMinQPS: 20, expectedMaxFailureRate: 0.01, }, { name: "defrag blocks one-third of requests with stopGRPCServiceOnDefrag set to false", clusterOptions: []e2e.EPClusterOption{ e2e.WithClusterSize(3), e2e.WithServerFeatureGate("StopGRPCServiceOnDefrag", false), e2e.WithGoFailEnabled(true), }, gRPCDialOptions: []grpc.DialOption{ grpc.WithDisableServiceConfig(), grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy": "round_robin", "healthCheckConfig": {"serviceName": ""}}`), }, expectedMinQPS: 20, expectedMinFailureRate: 0.25, }, { name: "defrag blocks one-third of requests with stopGRPCServiceOnDefrag set to true and client health check disabled", clusterOptions: []e2e.EPClusterOption{ e2e.WithClusterSize(3), e2e.WithServerFeatureGate("StopGRPCServiceOnDefrag", true), e2e.WithGoFailEnabled(true), }, expectedMinQPS: 20, expectedMinFailureRate: 0.25, }, { name: "defrag failover happy case with feature gate", clusterOptions: []e2e.EPClusterOption{ e2e.WithClusterSize(3), e2e.WithServerFeatureGate("StopGRPCServiceOnDefrag", true), e2e.WithGoFailEnabled(true), }, gRPCDialOptions: []grpc.DialOption{ grpc.WithDisableServiceConfig(), grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy": "round_robin", "healthCheckConfig": {"serviceName": ""}}`), }, expectedMinQPS: 20, expectedMaxFailureRate: 0.01, }, { name: "defrag blocks one-third of requests with StopGRPCServiceOnDefrag feature gate set to false", clusterOptions: []e2e.EPClusterOption{ e2e.WithClusterSize(3), e2e.WithServerFeatureGate("StopGRPCServiceOnDefrag", false), e2e.WithGoFailEnabled(true), }, gRPCDialOptions: []grpc.DialOption{ grpc.WithDisableServiceConfig(), grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy": "round_robin", "healthCheckConfig": {"serviceName": ""}}`), }, expectedMinQPS: 20, expectedMinFailureRate: 0.25, }, { name: "defrag blocks one-third of requests with StopGRPCServiceOnDefrag feature gate set to true and client health check disabled", clusterOptions: []e2e.EPClusterOption{ e2e.WithClusterSize(3), e2e.WithServerFeatureGate("StopGRPCServiceOnDefrag", true), e2e.WithGoFailEnabled(true), }, expectedMinQPS: 20, expectedMinFailureRate: 0.25, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { e2e.BeforeTest(t) clus, cerr := e2e.NewEtcdProcessCluster(t.Context(), t, tc.clusterOptions...) require.NoError(t, cerr) t.Cleanup(func() { clus.Stop() }) endpoints := clus.EndpointsGRPC() requestVolume, successfulRequestCount := 0, 0 start := time.Now() g := new(errgroup.Group) g.Go(func() (lastErr error) { clusterClient, cerr := clientv3.New(clientv3.Config{ DialTimeout: dialTimeout, DialKeepAliveTime: keepaliveTime, DialKeepAliveTimeout: keepaliveTimeout, Endpoints: endpoints, DialOptions: tc.gRPCDialOptions, }) if cerr != nil { return cerr } defer clusterClient.Close() timeout := time.After(clientRuntime) for { select { case <-timeout: return lastErr default: } getContext, cancel := context.WithTimeout(t.Context(), requestTimeout) _, err := clusterClient.Get(getContext, "health") cancel() requestVolume++ if err != nil { lastErr = err continue } successfulRequestCount++ } }) triggerDefrag(t, clus.Procs[0]) err := g.Wait() if err != nil { t.Logf("etcd client failed to fail over, error (%v)", err) } qps := float64(requestVolume) / float64(time.Since(start)) * float64(time.Second) failureRate := 1 - float64(successfulRequestCount)/float64(requestVolume) t.Logf("request failure rate is %.2f%%, qps is %.2f requests/second", failureRate*100, qps) require.GreaterOrEqual(t, qps, tc.expectedMinQPS) if tc.expectedMaxFailureRate != 0.0 { require.LessOrEqual(t, failureRate, tc.expectedMaxFailureRate) } if tc.expectedMinFailureRate != 0.0 { require.GreaterOrEqual(t, failureRate, tc.expectedMinFailureRate) } }) } } func triggerDefrag(t *testing.T, member e2e.EtcdProcess) { require.NoError(t, member.Failpoints().SetupHTTP(t.Context(), "defragBeforeCopy", `sleep("10s")`)) require.NoError(t, member.Etcdctl().Defragment(t.Context(), config.DefragOption{Timeout: time.Minute})) } ================================================ FILE: tests/e2e/force_new_cluster_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "encoding/json" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/testutils" ) // TestForceNewCluster verified that etcd works as expected when --force-new-cluster. // Refer to discussion in https://github.com/etcd-io/etcd/issues/20009. func TestForceNewCluster(t *testing.T) { e2e.BeforeTest(t) testCases := []struct { name string snapcount int }{ { name: "create a snapshot after promotion", snapcount: 10, }, { name: "no snapshot after promotion", snapcount: 0, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { epc, promotedMembers := mustCreateNewClusterByPromotingMembers(t, e2e.CurrentVersion, 5, e2e.WithKeepDataDir(true), e2e.WithSnapshotCount(uint64(tc.snapcount))) require.Len(t, promotedMembers, 4) for i := 0; i < tc.snapcount; i++ { _, err := epc.Etcdctl().Put(t.Context(), "foo", "bar", config.PutOptions{}) require.NoError(t, err) } require.NoError(t, epc.Close()) m := epc.Procs[0] t.Logf("Forcibly create a one-member cluster with member: %s", m.Config().Name) m.Config().Args = append(m.Config().Args, "--force-new-cluster") require.NoError(t, m.Start(t.Context())) t.Log("Restarting the member") require.NoError(t, m.Restart(t.Context())) t.Log("Closing the member") require.NoError(t, m.Close()) }) } } func TestForceNewCluster_MemberCount(t *testing.T) { e2e.BeforeTest(t) epc, promotedMembers := mustCreateNewClusterByPromotingMembers(t, e2e.CurrentVersion, 3, e2e.WithKeepDataDir(true)) require.Len(t, promotedMembers, 2) // Wait for the backend TXN to sync/commit the data to disk, to ensure // the consistent-index is persisted. Another way is to issue a snapshot // command to forcibly commit the backend TXN. time.Sleep(time.Second) t.Log("Killing all the members") require.NoError(t, epc.Kill()) require.NoError(t, epc.Wait(t.Context())) m := epc.Procs[0] t.Logf("Forcibly create a one-member cluster with member: %s", m.Config().Name) m.Config().Args = append(m.Config().Args, "--force-new-cluster") require.NoError(t, m.Start(t.Context())) t.Log("Online checking the member count") mresp, merr := m.Etcdctl().MemberList(t.Context(), false) require.NoError(t, merr) require.Len(t, mresp.Members, 1) t.Log("Closing the member") require.NoError(t, m.Close()) require.NoError(t, m.Wait(t.Context())) t.Log("Offline checking the member count") members := mustReadMembersFromBoltDB(t, m.Config().DataDirPath) require.Len(t, members, 1) } // TestForceNewCluster_AddLearner_MemberCount verifies that `--force-new-cluster` // should always be able to clean up all other members, including learners. func TestForceNewCluster_AddLearner_MemberCount(t *testing.T) { e2e.BeforeTest(t) testCases := []struct { name string snapcount int }{ { name: "no snapshot after adding learner", snapcount: 0, }, { name: "create a snapshot after adding learner", snapcount: 5, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { cfg := e2e.NewConfig(e2e.WithClusterSize(3)) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg), e2e.WithSnapshotCount(uint64(tc.snapcount)), e2e.WithKeepDataDir(true)) require.NoError(t, err) t.Log("Adding a learner member") testutils.ExecuteWithTimeout(t, 1*time.Minute, func() { for { _, aerr := epc.StartNewProc(t.Context(), nil, t, true) if aerr != nil { if strings.Contains(aerr.Error(), "etcdserver: unhealthy cluster") { time.Sleep(1 * time.Second) continue } } break } }) for i := 0; i < tc.snapcount; i++ { _, werr := epc.Etcdctl().Put(t.Context(), "foo", "bar", config.PutOptions{}) require.NoError(t, werr) } require.NoError(t, epc.Close()) m := epc.Procs[0] t.Logf("Forcibly create a one-member cluster with member: %s", m.Config().Name) m.Config().Args = append(m.Config().Args, "--force-new-cluster") require.NoError(t, m.Start(t.Context())) t.Log("Restarting the member") require.NoError(t, m.Restart(t.Context())) defer func() { t.Log("Closing the member") require.NoError(t, m.Close()) }() t.Log("Checking member count") resp, merr := m.Etcdctl().MemberList(t.Context(), false) require.NoError(t, merr) require.Len(t, resp.Members, 1) }) } } func mustReadMembersFromBoltDB(t *testing.T, dataDir string) []*membership.Member { dbPath := datadir.ToBackendFileName(dataDir) db, err := bbolt.Open(dbPath, 0o400, &bbolt.Options{ReadOnly: true}) require.NoError(t, err) defer func() { require.NoError(t, db.Close()) }() var members []*membership.Member _ = db.View(func(tx *bbolt.Tx) error { b := tx.Bucket(schema.Members.Name()) _ = b.ForEach(func(k, v []byte) error { m := membership.Member{} err := json.Unmarshal(v, &m) require.NoError(t, err) members = append(members, &m) return nil }) return nil }) return members } ================================================ FILE: tests/e2e/gateway_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "strings" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) var defaultGatewayEndpoint = "127.0.0.1:23790" func TestGateway(t *testing.T) { ec, err := e2e.NewEtcdProcessCluster(t.Context(), t) require.NoError(t, err) defer ec.Stop() eps := strings.Join(ec.EndpointsGRPC(), ",") p := startGateway(t, eps) defer func() { p.Stop() p.Close() }() err = e2e.SpawnWithExpect([]string{e2e.BinPath.Etcdctl, "--endpoints=" + defaultGatewayEndpoint, "put", "foo", "bar"}, expect.ExpectedResponse{Value: "OK\r\n"}) if err != nil { t.Errorf("failed to finish put request through gateway: %v", err) } } func startGateway(t *testing.T, endpoints string) *expect.ExpectProcess { p, err := expect.NewExpect(e2e.BinPath.Etcd, "gateway", "--endpoints="+endpoints, "start") require.NoError(t, err) _, err = p.Expect("ready to proxy client requests") require.NoError(t, err) return p } ================================================ FILE: tests/e2e/graceful_shutdown_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/interfaces" "go.etcd.io/raft/v3" ) func TestGracefulShutdown(t *testing.T) { tcs := []struct { name string clusterSize int }{ { name: "clusterSize3", clusterSize: 3, }, { name: "clusterSize5", clusterSize: 5, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { testRunner := e2e.NewE2eRunner() testRunner.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() clus := testRunner.NewCluster(ctx, t, config.WithClusterSize(tc.clusterSize)) // clean up orphaned resources like closing member client. defer clus.Close() // shutdown each etcd member process sequentially // and start from old leader, (new leader), (follower) tryShutdownLeader(ctx, t, clus.Members()) }) } } // tryShutdownLeader tries stop etcd member if it is leader. // it also asserts stop leader should not take longer than 1.5 seconds and leaderID has been changed within 500ms. func tryShutdownLeader(ctx context.Context, t *testing.T, members []interfaces.Member) { quorum := len(members)/2 + 1 for len(members) > quorum { leader, leaderID, term, followers := getLeader(ctx, t, members) stopped := make(chan error, 1) go func() { // each etcd server will wait up to 1 seconds to close all idle connections in peer handler. start := time.Now() leader.Stop() took := time.Since(start) if took > 1500*time.Millisecond { stopped <- fmt.Errorf("leader stop took %v longer than 1.5 seconds", took) return } stopped <- nil }() // etcd election timeout could range from 1s to 2s without explicit leadership transfer. // assert leader ID has been changed within 500ms time.Sleep(500 * time.Millisecond) resps, err := followers[0].Client().Status(ctx) require.NoError(t, err) require.NotEqual(t, raft.None, leaderID) require.Equal(t, resps[0].RaftTerm, term+1) require.NotEqualf(t, resps[0].Leader, leaderID, "expect old leaderID %x changed to new leader ID %x", leaderID, resps[0].Leader) err = <-stopped require.NoError(t, err) members = followers } } func getLeader(ctx context.Context, t *testing.T, members []interfaces.Member) (leader interfaces.Member, leaderID, term uint64, followers []interfaces.Member) { leaderIdx := -1 for i, m := range members { mc := m.Client() sresps, err := mc.Status(ctx) require.NoError(t, err) if sresps[0].Leader == sresps[0].Header.MemberId { leaderIdx = i leaderID = sresps[0].Leader term = sresps[0].RaftTerm break } } if leaderIdx == -1 { return nil, 0, 0, members } leader = members[leaderIdx] return leader, leaderID, term, append(members[:leaderIdx], members[leaderIdx+1:]...) } ================================================ FILE: tests/e2e/http_health_check_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "context" "fmt" "io" "net/http" "os" "path" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/server/v3/storage/mvcc/testutil" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/testutils" ) const ( healthCheckTimeout = 2 * time.Second putCommandTimeout = 200 * time.Millisecond ) type healthCheckConfig struct { url string expectedStatusCode int expectedTimeoutError bool expectedRespSubStrings []string } type injectFailure func(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessCluster, duration time.Duration) func TestHTTPHealthHandler(t *testing.T) { e2e.BeforeTest(t) client := &http.Client{} tcs := []struct { name string injectFailure injectFailure clusterOptions []e2e.EPClusterOption healthChecks []healthCheckConfig }{ { name: "no failures", // happy case clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(1)}, healthChecks: []healthCheckConfig{ { url: "/health", expectedStatusCode: http.StatusOK, }, }, }, { name: "activated no space alarm", injectFailure: triggerNoSpaceAlarm, clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(1), e2e.WithQuotaBackendBytes(int64(13 * os.Getpagesize()))}, healthChecks: []healthCheckConfig{ { url: "/health", expectedStatusCode: http.StatusServiceUnavailable, }, { url: "/health?exclude=NOSPACE", expectedStatusCode: http.StatusOK, }, }, }, { name: "overloaded server slow apply", injectFailure: triggerSlowApply, clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(3), e2e.WithGoFailEnabled(true)}, healthChecks: []healthCheckConfig{ { url: "/health?serializable=true", expectedStatusCode: http.StatusOK, }, { url: "/health?serializable=false", expectedTimeoutError: true, }, }, }, { name: "network partitioned", injectFailure: blackhole, clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(3), e2e.WithIsPeerTLS(true), e2e.WithPeerProxy(true)}, healthChecks: []healthCheckConfig{ { url: "/health?serializable=true", expectedStatusCode: http.StatusOK, }, { url: "/health?serializable=false", expectedTimeoutError: true, expectedStatusCode: http.StatusServiceUnavailable, // old leader may return "etcdserver: leader changed" error with 503 in ReadIndex leaderChangedNotifier }, }, }, { name: "raft loop deadlock", injectFailure: triggerRaftLoopDeadLock, clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true)}, healthChecks: []healthCheckConfig{ { // current kubeadm etcd liveness check failed to detect raft loop deadlock in steady state // ref. https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/phases/etcd/local.go#L225-L226 // current liveness probe depends on the etcd /health check has a flaw that new /livez check should resolve. url: "/health?serializable=true", expectedStatusCode: http.StatusOK, }, { url: "/health?serializable=false", expectedTimeoutError: true, }, }, }, // verify that auth enabled serializable read must go through mvcc { name: "slow buffer write back with auth enabled", injectFailure: triggerSlowBufferWriteBackWithAuth, clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true)}, healthChecks: []healthCheckConfig{ { url: "/health?serializable=true", expectedTimeoutError: true, }, }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) defer cancel() clus, err := e2e.NewEtcdProcessCluster(ctx, t, tc.clusterOptions...) require.NoError(t, err) defer clus.Close() testutils.ExecuteUntil(ctx, t, func() { if tc.injectFailure != nil { // guaranteed that failure point is active until all the health checks timeout. duration := time.Duration(len(tc.healthChecks)+1) * healthCheckTimeout tc.injectFailure(ctx, t, clus, duration) } for _, hc := range tc.healthChecks { requestURL := clus.Procs[0].EndpointsHTTP()[0] + hc.url t.Logf("health check URL is %s", requestURL) doHealthCheckAndVerify(t, client, requestURL, hc.expectedTimeoutError, hc.expectedStatusCode, hc.expectedRespSubStrings) } }) }) } } var defaultHealthCheckConfigs = []healthCheckConfig{ { url: "/livez", expectedStatusCode: http.StatusOK, expectedRespSubStrings: []string{`ok`}, }, { url: "/readyz", expectedStatusCode: http.StatusOK, expectedRespSubStrings: []string{`ok`}, }, { url: "/livez?verbose=true", expectedStatusCode: http.StatusOK, expectedRespSubStrings: []string{`[+]serializable_read ok`}, }, { url: "/readyz?verbose=true", expectedStatusCode: http.StatusOK, expectedRespSubStrings: []string{ `[+]serializable_read ok`, `[+]data_corruption ok`, }, }, } func TestHTTPLivezReadyzHandler(t *testing.T) { e2e.BeforeTest(t) client := &http.Client{} tcs := []struct { name string injectFailure injectFailure clusterOptions []e2e.EPClusterOption healthChecks []healthCheckConfig }{ { name: "no failures", // happy case clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(1)}, healthChecks: defaultHealthCheckConfigs, }, { name: "activated no space alarm", injectFailure: triggerNoSpaceAlarm, clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(1), e2e.WithQuotaBackendBytes(int64(13 * os.Getpagesize()))}, healthChecks: defaultHealthCheckConfigs, }, // Readiness is not an indicator of performance. Slow response is not covered by readiness. // refer to https://tinyurl.com/livez-readyz-design-doc or https://github.com/etcd-io/etcd/issues/16007#issuecomment-1726541091 in case tinyurl is down. { name: "overloaded server slow apply", injectFailure: triggerSlowApply, clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(3), e2e.WithGoFailEnabled(true)}, // TODO expected behavior of readyz check should be 200 after ReadIndex check is implemented to replace linearizable read. healthChecks: []healthCheckConfig{ { url: "/livez", expectedStatusCode: http.StatusOK, }, { url: "/readyz", expectedTimeoutError: true, expectedStatusCode: http.StatusServiceUnavailable, }, }, }, { name: "network partitioned", injectFailure: blackhole, clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(3), e2e.WithIsPeerTLS(true), e2e.WithPeerProxy(true)}, healthChecks: []healthCheckConfig{ { url: "/livez", expectedStatusCode: http.StatusOK, }, { url: "/readyz", expectedTimeoutError: true, expectedStatusCode: http.StatusServiceUnavailable, expectedRespSubStrings: []string{ `[-]linearizable_read failed: etcdserver: leader changed`, }, }, }, }, { name: "raft loop deadlock", injectFailure: triggerRaftLoopDeadLock, clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true)}, // TODO expected behavior of livez check should be 503 or timeout after RaftLoopDeadLock check is implemented. healthChecks: []healthCheckConfig{ { url: "/livez", expectedStatusCode: http.StatusOK, }, { url: "/readyz", expectedTimeoutError: true, expectedStatusCode: http.StatusServiceUnavailable, }, }, }, // verify that auth enabled serializable read must go through mvcc { name: "slow buffer write back with auth enabled", injectFailure: triggerSlowBufferWriteBackWithAuth, clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true)}, healthChecks: []healthCheckConfig{ { url: "/livez", expectedTimeoutError: true, }, { url: "/readyz", expectedTimeoutError: true, }, }, }, { name: "corrupt", injectFailure: triggerCorrupt, clusterOptions: []e2e.EPClusterOption{e2e.WithClusterSize(3), e2e.WithCorruptCheckTime(time.Second)}, healthChecks: []healthCheckConfig{ { url: "/livez?verbose=true", expectedStatusCode: http.StatusOK, expectedRespSubStrings: []string{`[+]serializable_read ok`}, }, { url: "/readyz", expectedStatusCode: http.StatusServiceUnavailable, expectedRespSubStrings: []string{ `[+]serializable_read ok`, `[-]data_corruption failed: alarm activated: CORRUPT`, }, }, }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) defer cancel() clus, err := e2e.NewEtcdProcessCluster(ctx, t, tc.clusterOptions...) require.NoError(t, err) defer clus.Close() testutils.ExecuteUntil(ctx, t, func() { if tc.injectFailure != nil { // guaranteed that failure point is active until all the health checks timeout. duration := time.Duration(len(tc.healthChecks)+1) * healthCheckTimeout tc.injectFailure(ctx, t, clus, duration) } for _, hc := range tc.healthChecks { requestURL := clus.Procs[0].EndpointsHTTP()[0] + hc.url t.Logf("health check URL is %s", requestURL) doHealthCheckAndVerify(t, client, requestURL, hc.expectedTimeoutError, hc.expectedStatusCode, hc.expectedRespSubStrings) } }) }) } } func doHealthCheckAndVerify(t *testing.T, client *http.Client, url string, expectTimeoutError bool, expectStatusCode int, expectRespSubStrings []string) { ctx, cancel := context.WithTimeout(t.Context(), healthCheckTimeout) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) require.NoErrorf(t, err, "failed to creat request %+v", err) resp, herr := client.Do(req) cancel() if expectTimeoutError { if herr != nil && strings.Contains(herr.Error(), context.DeadlineExceeded.Error()) { return } } require.NoErrorf(t, herr, "failed to get response %+v", err) defer resp.Body.Close() body, err := io.ReadAll(resp.Body) resp.Body.Close() require.NoErrorf(t, err, "failed to read response %+v", err) t.Logf("health check response body is:\n%s", body) require.Equal(t, expectStatusCode, resp.StatusCode) for _, expectRespSubString := range expectRespSubStrings { require.Contains(t, string(body), expectRespSubString) } } func triggerNoSpaceAlarm(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessCluster, _ time.Duration) { buf := strings.Repeat("b", os.Getpagesize()) etcdctl := clus.Etcdctl() for { if _, err := etcdctl.Put(ctx, "foo", buf, config.PutOptions{}); err != nil { require.ErrorContains(t, err, "etcdserver: mvcc: database space exceeded") break } } } func triggerSlowApply(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessCluster, duration time.Duration) { // the following proposal will be blocked at applying stage // because when apply index < committed index, linearizable read would time out. require.NoError(t, clus.Procs[0].Failpoints().SetupHTTP(ctx, "beforeApplyOneEntryNormal", fmt.Sprintf(`sleep("%s")`, duration))) _, err := clus.Procs[1].Etcdctl().Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) } func blackhole(_ context.Context, t *testing.T, clus *e2e.EtcdProcessCluster, _ time.Duration) { member := clus.Procs[0] proxy := member.PeerProxy() t.Logf("Blackholing traffic from and to member %q", member.Config().Name) proxy.BlackholeTx() proxy.BlackholeRx() } func triggerRaftLoopDeadLock(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessCluster, duration time.Duration) { require.NoError(t, clus.Procs[0].Failpoints().SetupHTTP(ctx, "raftBeforeSave", fmt.Sprintf(`sleep("%s")`, duration))) clus.Procs[0].Etcdctl().Put(context.Background(), "foo", "bar", config.PutOptions{Timeout: putCommandTimeout}) } func triggerSlowBufferWriteBackWithAuth(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessCluster, duration time.Duration) { etcdctl := clus.Etcdctl() _, err := etcdctl.UserAdd(ctx, "root", "root", config.UserAddOptions{}) require.NoError(t, err) _, err = etcdctl.UserGrantRole(ctx, "root", "root") require.NoError(t, err) require.NoError(t, etcdctl.AuthEnable(ctx)) require.NoError(t, clus.Procs[0].Failpoints().SetupHTTP(ctx, "beforeWritebackBuf", fmt.Sprintf(`sleep("%s")`, duration))) clus.Procs[0].Etcdctl(e2e.WithAuth("root", "root")).Put(context.Background(), "foo", "bar", config.PutOptions{Timeout: putCommandTimeout}) } func triggerCorrupt(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessCluster, _ time.Duration) { etcdctl := clus.Procs[0].Etcdctl() for i := 0; i < 10; i++ { _, err := etcdctl.Put(ctx, "foo", "bar", config.PutOptions{}) require.NoError(t, err) } err := clus.Procs[0].Kill() require.NoError(t, err) err = clus.Procs[0].Wait(ctx) require.NoError(t, err) err = testutil.CorruptBBolt(path.Join(clus.Procs[0].Config().DataDirPath, "member", "snap", "db")) require.NoError(t, err) err = clus.Procs[0].Start(ctx) for { time.Sleep(time.Second) select { case <-ctx.Done(): require.NoError(t, err) default: } response, err := etcdctl.AlarmList(ctx) if err != nil { continue } if len(response.Alarms) == 0 { continue } require.Len(t, response.Alarms, 1) if response.Alarms[0].Alarm == etcdserverpb.AlarmType_CORRUPT { break } } } ================================================ FILE: tests/e2e/leader_snapshot_no_proxy_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "context" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/failpoint" ) func TestRecoverSnapshotBackend(t *testing.T) { e2e.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(3), e2e.WithKeepDataDir(true), e2e.WithPeerProxy(true), e2e.WithSnapshotCatchUpEntries(50), e2e.WithSnapshotCount(50), e2e.WithGoFailEnabled(true), e2e.WithIsPeerTLS(true), ) require.NoError(t, err) defer epc.Close() blackholedMember := epc.Procs[0] otherMember := epc.Procs[1] wg := sync.WaitGroup{} trafficCtx, trafficCancel := context.WithCancel(ctx) c, err := clientv3.New(clientv3.Config{ Endpoints: otherMember.EndpointsGRPC(), Logger: zap.NewNop(), DialKeepAliveTime: 10 * time.Second, DialKeepAliveTimeout: 100 * time.Millisecond, }) require.NoError(t, err) defer c.Close() wg.Add(1) go func() { defer wg.Done() for { select { case <-trafficCtx.Done(): return default: } putCtx, putCancel := context.WithTimeout(trafficCtx, 50*time.Millisecond) c.Put(putCtx, "a", "b") putCancel() time.Sleep(10 * time.Millisecond) } }() err = blackholedMember.Failpoints().SetupHTTP(ctx, "applyBeforeOpenSnapshot", "panic") require.NoError(t, err) err = failpoint.Blackhole(ctx, t, blackholedMember, epc, true) require.NoError(t, err) err = blackholedMember.Wait(ctx) require.NoError(t, err) trafficCancel() wg.Wait() err = blackholedMember.Start(ctx) require.NoError(t, err) _, err = blackholedMember.Logs().ExpectWithContext(ctx, expect.ExpectedResponse{Value: "Recovering from snapshot backend"}) require.NoError(t, err) _, err = blackholedMember.Etcdctl().Put(ctx, "a", "1", config.PutOptions{}) assert.NoError(t, err) } ================================================ FILE: tests/e2e/logging_test.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "encoding/json" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestNoErrorLogsDuringNormalOperations(t *testing.T) { tests := []struct { name string options []e2e.EPClusterOption allowedErrors map[string]bool }{ { name: "single node cluster", options: []e2e.EPClusterOption{ e2e.WithClusterSize(1), e2e.WithLogLevel("debug"), }, allowedErrors: map[string]bool{ "setting up serving from embedded etcd failed.": true, // See https://github.com/etcd-io/etcd/pull/19040#issuecomment-2539173800 // TODO: Remove with etcd 3.7 "cannot detect storage schema version: missing term information": true, }, }, { name: "three node cluster", options: []e2e.EPClusterOption{ e2e.WithClusterSize(3), e2e.WithLogLevel("debug"), }, allowedErrors: map[string]bool{ "setting up serving from embedded etcd failed.": true, // See https://github.com/etcd-io/etcd/pull/19040#issuecomment-2539173800 // TODO: Remove with etcd 3.7 "cannot detect storage schema version: missing term information": true, }, }, { name: "three node cluster with auto tls (all)", options: []e2e.EPClusterOption{ e2e.WithClusterSize(3), e2e.WithLogLevel("debug"), e2e.WithIsPeerTLS(true), e2e.WithIsPeerAutoTLS(true), e2e.WithClientAutoTLS(true), e2e.WithClientConnType(e2e.ClientTLS), }, allowedErrors: map[string]bool{ "setting up serving from embedded etcd failed.": true, // See https://github.com/etcd-io/etcd/pull/19040#issuecomment-2539173800 // TODO: Remove with etcd 3.7 "cannot detect storage schema version: missing term information": true, }, }, { name: "three node cluster with auto tls (peers)", options: []e2e.EPClusterOption{ e2e.WithClusterSize(3), e2e.WithLogLevel("debug"), e2e.WithIsPeerTLS(true), e2e.WithIsPeerAutoTLS(true), }, allowedErrors: map[string]bool{ "setting up serving from embedded etcd failed.": true, // See https://github.com/etcd-io/etcd/pull/19040#issuecomment-2539173800 // TODO: Remove with etcd 3.7 "cannot detect storage schema version: missing term information": true, }, }, { name: "three node cluster with auto tls (client)", options: []e2e.EPClusterOption{ e2e.WithClusterSize(3), e2e.WithLogLevel("debug"), e2e.WithClientAutoTLS(true), e2e.WithClientConnType(e2e.ClientTLS), }, allowedErrors: map[string]bool{ "setting up serving from embedded etcd failed.": true, // See https://github.com/etcd-io/etcd/pull/19040#issuecomment-2539173800 // TODO: Remove with etcd 3.7 "cannot detect storage schema version: missing term information": true, }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { e2e.BeforeTest(t) ctx := t.Context() epc, err := e2e.NewEtcdProcessCluster(ctx, t, tc.options...) require.NoError(t, err) defer epc.Close() require.Lenf(t, epc.Procs, epc.Cfg.ClusterSize, "embedded etcd cluster process count is not as expected") // Collect the handle of logs before closing the processes. var logHandles []e2e.LogsExpect for i := range epc.Cfg.ClusterSize { logHandles = append(logHandles, epc.Procs[i].Logs()) } time.Sleep(time.Second) require.NoErrorf(t, epc.Close(), "closing etcd processes") // Now that the processes are closed we can collect all log lines. This must happen after closing, else we // might not get all log lines. var lines []string for _, h := range logHandles { lines = append(lines, h.Lines()...) } require.NotEmptyf(t, lines, "expected at least one log line") var entry logEntry for _, line := range lines { err := json.Unmarshal([]byte(line), &entry) require.NoErrorf(t, err, "parse log line as json, line: %s", line) if tc.allowedErrors[entry.Message] || tc.allowedErrors[entry.Error] { continue } require.NotEqualf(t, "error", entry.Level, "error level log message found: %s", line) } }) } } ================================================ FILE: tests/e2e/main_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "os" "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestMain(m *testing.M) { e2e.InitFlags() v := m.Run() if v == 0 && testutil.CheckLeakedGoroutine() { os.Exit(1) } os.Exit(v) } ================================================ FILE: tests/e2e/member_no_proxy_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "strings" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/testutils" ) // TestReproduce20340 reproduces the issue https://github.com/etcd-io/etcd/issues/20340. // Refer to https://github.com/etcd-io/etcd/issues/20340#issuecomment-3105037914. func TestReproduce20340(t *testing.T) { e2e.BeforeTest(t) ctx := t.Context() epc, members := mustCreateNewClusterByPromotingMembers(t, e2e.CurrentVersion, 3) defer func() { require.NoError(t, epc.Close()) }() t.Logf("Removing the second member (%x)", members[0].ID) etcdctl := epc.Procs[0].Etcdctl() testutils.ExecuteWithTimeout(t, 10*time.Second, func() { for { _, merr := etcdctl.MemberRemove(ctx, members[0].ID) if merr != nil { if strings.Contains(merr.Error(), "etcdserver: unhealthy cluster") { time.Sleep(1 * time.Second) continue } t.Fatalf("failed to remove member: %s", merr.Error()) } break } }) epc.Procs = append(epc.Procs[:1], epc.Procs[2:]...) t.Logf("Restarting member: %s", epc.Procs[0].Config().Name) require.NoError(t, epc.Procs[0].Restart(ctx)) } ================================================ FILE: tests/e2e/metrics_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "net/url" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestV3MetricsSecure(t *testing.T) { cfg := e2e.NewConfigTLS() cfg.ClusterSize = 1 cfg.MetricsURLScheme = "https" testCtl(t, metricsTest) } func TestV3MetricsInsecure(t *testing.T) { cfg := e2e.NewConfigTLS() cfg.ClusterSize = 1 cfg.MetricsURLScheme = "http" testCtl(t, metricsTest) } func TestV3LearnerMetricRecover(t *testing.T) { cfg := e2e.NewConfigTLS() cfg.ServerConfig.SnapshotCount = 10 testCtl(t, learnerMetricRecoverTest, withCfg(*cfg)) } func TestV3LearnerMetricApplyFromSnapshotTest(t *testing.T) { cfg := e2e.NewConfigTLS() cfg.ServerConfig.SnapshotCount = 10 testCtl(t, learnerMetricApplyFromSnapshotTest, withCfg(*cfg)) } func metricsTest(cx ctlCtx) { require.NoError(cx.t, ctlV3Put(cx, "k", "v", "")) i := 0 for _, test := range []struct { endpoint, expected string }{ {"/metrics", "etcd_mvcc_put_total 2"}, {"/metrics", "etcd_debugging_mvcc_keys_total 1"}, {"/metrics", "etcd_mvcc_delete_total 3"}, {"/metrics", fmt.Sprintf(`etcd_server_version{server_version="%s"} 1`, version.Version)}, {"/metrics", fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version))}, {"/metrics", `grpc_server_handled_total{grpc_code="Canceled",grpc_method="Watch",grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"} 6`}, {"/health", `{"health":"true","reason":""}`}, } { i++ require.NoError(cx.t, ctlV3Put(cx, fmt.Sprintf("%d", i), "v", "")) require.NoError(cx.t, ctlV3Del(cx, []string{fmt.Sprintf("%d", i)}, 1)) require.NoError(cx.t, ctlV3Watch(cx, []string{"k", "--rev", "1"}, []kvExec{{key: "k", val: "v"}}...)) require.NoErrorf(cx.t, e2e.CURLGet(cx.epc, e2e.CURLReq{Endpoint: test.endpoint, Expected: expect.ExpectedResponse{Value: test.expected}}), "failed get with curl") } } func learnerMetricRecoverTest(cx ctlCtx) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() _, err := cx.epc.StartNewProc(ctx, nil, cx.t, true /* addAsLearner */) require.NoError(cx.t, err) expectLearnerMetrics(cx) triggerSnapshot(ctx, cx) // Restart cluster require.NoError(cx.t, cx.epc.Restart(ctx)) expectLearnerMetrics(cx) } func learnerMetricApplyFromSnapshotTest(cx ctlCtx) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Add learner but do not start it _, learnerCfg, err := cx.epc.AddMember(ctx, nil, cx.t, true /* addAsLearner */) require.NoError(cx.t, err) triggerSnapshot(ctx, cx) // Start the learner require.NoError(cx.t, cx.epc.StartNewProcFromConfig(ctx, cx.t, learnerCfg)) expectLearnerMetrics(cx) } func triggerSnapshot(ctx context.Context, cx ctlCtx) { etcdctl := cx.epc.Procs[0].Etcdctl() for i := 0; i < int(cx.epc.Cfg.ServerConfig.SnapshotCount); i++ { _, err := etcdctl.Put(ctx, "k", "v", config.PutOptions{}) require.NoError(cx.t, err) } } func expectLearnerMetrics(cx ctlCtx) { expectLearnerMetric(cx, 0, "etcd_server_is_learner 0") expectLearnerMetric(cx, 1, "etcd_server_is_learner 1") } func expectLearnerMetric(cx ctlCtx, procIdx int, expectMetric string) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() args := e2e.CURLPrefixArgsCluster(cx.epc.Cfg, cx.epc.Procs[procIdx], "GET", e2e.CURLReq{Endpoint: "/metrics"}) require.NoError(cx.t, e2e.SpawnWithExpectsContext(ctx, args, nil, expect.ExpectedResponse{Value: expectMetric})) } func TestNoMetricsMissing(t *testing.T) { var ( // Note the list doesn't contain all the metrics, because the // labelled metrics won't be exposed by prometheus by default. // They are only exposed when at least one value with labels // is set. basicMetrics = []string{ "etcd_cluster_version", "etcd_debugging_auth_revision", "etcd_debugging_disk_backend_commit_rebalance_duration_seconds", "etcd_debugging_disk_backend_commit_spill_duration_seconds", "etcd_debugging_disk_backend_commit_write_duration_seconds", "etcd_debugging_lease_granted_total", "etcd_debugging_lease_renewed_total", "etcd_debugging_lease_revoked_total", "etcd_debugging_lease_ttl_total", "etcd_debugging_mvcc_compact_revision", "etcd_debugging_mvcc_current_revision", "etcd_debugging_mvcc_db_compaction_keys_total", "etcd_debugging_mvcc_db_compaction_last", "etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds", "etcd_debugging_mvcc_db_compaction_total_duration_milliseconds", "etcd_debugging_mvcc_events_total", "etcd_debugging_mvcc_index_compaction_pause_duration_milliseconds", "etcd_debugging_mvcc_keys_total", "etcd_debugging_mvcc_pending_events_total", "etcd_debugging_mvcc_slow_watcher_total", "etcd_debugging_mvcc_total_put_size_in_bytes", "etcd_debugging_mvcc_watch_stream_total", "etcd_debugging_mvcc_watcher_total", "etcd_debugging_server_lease_expired_total", "etcd_debugging_server_watch_send_loop_watch_stream_duration_seconds", "etcd_debugging_server_watch_send_loop_watch_stream_duration_per_event_seconds", "etcd_debugging_server_watch_send_loop_control_stream_duration_seconds", "etcd_debugging_server_watch_send_loop_progress_duration_seconds", "etcd_debugging_snap_save_marshalling_duration_seconds", "etcd_debugging_snap_save_total_duration_seconds", "etcd_debugging_store_expires_total", "etcd_debugging_store_watch_requests_total", "etcd_debugging_store_watchers", "etcd_disk_backend_commit_duration_seconds", "etcd_disk_backend_defrag_duration_seconds", "etcd_disk_backend_snapshot_duration_seconds", "etcd_disk_defrag_inflight", "etcd_disk_wal_fsync_duration_seconds", "etcd_disk_wal_write_bytes_total", "etcd_disk_wal_write_duration_seconds", "etcd_grpc_proxy_cache_hits_total", "etcd_grpc_proxy_cache_keys_total", "etcd_grpc_proxy_cache_misses_total", "etcd_grpc_proxy_events_coalescing_total", "etcd_grpc_proxy_watchers_coalescing_total", "etcd_mvcc_db_open_read_transactions", "etcd_mvcc_db_total_size_in_bytes", "etcd_mvcc_db_total_size_in_use_in_bytes", "etcd_mvcc_delete_total", "etcd_mvcc_hash_duration_seconds", "etcd_mvcc_hash_rev_duration_seconds", "etcd_mvcc_put_total", "etcd_mvcc_range_total", "etcd_mvcc_txn_total", "etcd_network_client_grpc_received_bytes_total", "etcd_network_client_grpc_sent_bytes_total", "etcd_network_known_peers", "etcd_server_apply_duration_seconds", "etcd_server_client_requests_total", "etcd_server_go_version", "etcd_server_has_leader", "etcd_server_health_failures", "etcd_server_health_success", "etcd_server_heartbeat_send_failures_total", "etcd_server_id", "etcd_server_is_leader", "etcd_server_is_learner", "etcd_server_leader_changes_seen_total", "etcd_server_learner_promote_successes", "etcd_server_proposals_applied_total", "etcd_server_proposals_committed_total", "etcd_server_proposals_failed_total", "etcd_server_proposals_pending", "etcd_server_quota_backend_bytes", "etcd_server_range_duration_seconds", "etcd_server_read_indexes_failed_total", "etcd_server_request_duration_seconds", "etcd_server_slow_apply_total", "etcd_server_slow_read_indexes_total", "etcd_server_snapshot_apply_in_progress_total", "etcd_server_version", "etcd_snap_db_fsync_duration_seconds", "etcd_snap_db_save_total_duration_seconds", "etcd_snap_fsync_duration_seconds", "go_gc_duration_seconds", "go_gc_gogc_percent", "go_gc_gomemlimit_bytes", "go_goroutines", "go_info", "go_memstats_alloc_bytes", "go_memstats_alloc_bytes_total", "go_memstats_buck_hash_sys_bytes", "go_memstats_frees_total", "go_memstats_gc_sys_bytes", "go_memstats_heap_alloc_bytes", "go_memstats_heap_idle_bytes", "go_memstats_heap_inuse_bytes", "go_memstats_heap_objects", "go_memstats_heap_released_bytes", "go_memstats_heap_sys_bytes", "go_memstats_last_gc_time_seconds", "go_memstats_mallocs_total", "go_memstats_mcache_inuse_bytes", "go_memstats_mcache_sys_bytes", "go_memstats_mspan_inuse_bytes", "go_memstats_mspan_sys_bytes", "go_memstats_next_gc_bytes", "go_memstats_other_sys_bytes", "go_memstats_stack_inuse_bytes", "go_memstats_stack_sys_bytes", "go_memstats_sys_bytes", "go_sched_gomaxprocs_threads", "go_threads", "grpc_server_handled_total", "grpc_server_msg_received_total", "grpc_server_msg_sent_total", "grpc_server_started_total", "os_fd_limit", "os_fd_used", "promhttp_metric_handler_requests_in_flight", "promhttp_metric_handler_requests_total", } extraMultipleMemberClusterMetrics = []string{ "etcd_network_active_peers", "etcd_network_peer_received_bytes_total", "etcd_network_peer_sent_bytes_total", } extraExtensiveMetrics = []string{"grpc_server_handling_seconds"} ) testCases := []struct { name string options []e2e.EPClusterOption expectedMetrics []string }{ { name: "basic metrics of 1 member cluster", options: []e2e.EPClusterOption{ e2e.WithClusterSize(1), }, expectedMetrics: basicMetrics, }, { name: "basic metrics of 3 member cluster", options: []e2e.EPClusterOption{ e2e.WithClusterSize(3), }, expectedMetrics: append(basicMetrics, extraMultipleMemberClusterMetrics...), }, { name: "extensive metrics of 1 member cluster", options: []e2e.EPClusterOption{ e2e.WithClusterSize(1), e2e.WithExtensiveMetrics(), }, expectedMetrics: append(basicMetrics, extraExtensiveMetrics...), }, { name: "extensive metrics of 3 member cluster", options: []e2e.EPClusterOption{ e2e.WithClusterSize(3), e2e.WithExtensiveMetrics(), }, expectedMetrics: append(append(basicMetrics, extraExtensiveMetrics...), extraMultipleMemberClusterMetrics...), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { e2e.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() epc, err := e2e.NewEtcdProcessCluster(ctx, t, tc.options...) require.NoError(t, err) defer epc.Close() c := epc.Procs[0].Etcdctl() for i := 0; i < 3; i++ { _, err = c.Put(ctx, fmt.Sprintf("key_%d", i), fmt.Sprintf("value_%d", i), config.PutOptions{}) require.NoError(t, err) } _, err = c.Get(ctx, "k", config.GetOptions{}) require.NoError(t, err) metricsURL, err := url.JoinPath(epc.Procs[0].Config().ClientURL, "metrics") require.NoError(t, err) mfs, err := e2e.GetMetrics(metricsURL) require.NoError(t, err) var missingMetrics []string for _, metrics := range tc.expectedMetrics { if _, ok := mfs[metrics]; !ok { missingMetrics = append(missingMetrics, metrics) } } require.Emptyf(t, missingMetrics, "Some metrics are missing: %v", missingMetrics) // Please keep the log below to generate the expected metrics. // t.Logf("All metrics: %v", formatMetrics(slices.Sorted(maps.Keys(mfs)))) }) } } // formatMetrics is only for test purpose /*func formatMetrics(metrics []string) string { quoted := make([]string, len(metrics)) for i, s := range metrics { quoted[i] = fmt.Sprintf(`"%s",`, s) } return fmt.Sprintf("[]string{\n%s\n}", strings.Join(quoted, "\n")) }*/ ================================================ FILE: tests/e2e/promote_experimental_flag_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestWarningApplyDuration(t *testing.T) { e2e.BeforeTest(t) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithClusterSize(1), e2e.WithWarningUnaryRequestDuration(time.Microsecond), ) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } t.Cleanup(func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }) cc := epc.Etcdctl() _, err = cc.Put(t.Context(), "foo", "bar", config.PutOptions{}) require.NoErrorf(t, err, "error on put") // verify warning e2e.AssertProcessLogs(t, epc.Procs[0], "request stats") } ================================================ FILE: tests/e2e/reproduce_17780_test.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/stringutil" "go.etcd.io/etcd/tests/v3/framework/e2e" ) // TestReproduce17780 reproduces the issue: https://github.com/etcd-io/etcd/issues/17780. func TestReproduce17780(t *testing.T) { e2e.BeforeTest(t) compactionBatchLimit := 10 ctx := t.Context() clus, cerr := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(3), e2e.WithGoFailEnabled(true), e2e.WithSnapshotCount(1000), e2e.WithCompactionBatchLimit(compactionBatchLimit), e2e.WithWatchProcessNotifyInterval(100*time.Millisecond), ) require.NoError(t, cerr) t.Cleanup(func() { clus.Stop() }) leaderIdx := clus.WaitLeader(t) targetIdx := (leaderIdx + 1) % clus.Cfg.ClusterSize cli := newClient(t, clus.Procs[targetIdx].EndpointsGRPC(), e2e.ClientConfig{}) // Revision: 2 -> 8 for new keys n := compactionBatchLimit - 2 valueSize := 16 for i := 2; i <= n; i++ { _, err := cli.Put(ctx, fmt.Sprintf("%d", i), stringutil.RandString(uint(valueSize))) require.NoError(t, err) } // Revision: 9 -> 11 for delete keys with compared revision // // We need last compaction batch is no-op and all the tombstones should // be deleted in previous compaction batch. So that we just lost the // finishedCompactRev after panic. for i := 9; i <= compactionBatchLimit+1; i++ { rev := i - 5 key := fmt.Sprintf("%d", rev) _, err := cli.Delete(ctx, key) require.NoError(t, err) } require.NoError(t, clus.Procs[targetIdx].Failpoints().SetupHTTP(ctx, "compactBeforeSetFinishedCompact", `panic`)) _, err := cli.Compact(ctx, 11, clientv3.WithCompactPhysical()) require.Error(t, err) require.NoError(t, clus.Procs[targetIdx].Restart(ctx)) // NOTE: We should not decrease the revision if there is no record // about finished compact operation. resp, err := cli.Get(ctx, fmt.Sprintf("%d", n)) require.NoError(t, err) assert.GreaterOrEqual(t, resp.Header.Revision, int64(11)) // Revision 4 should be deleted by compaction. resp, err = cli.Get(ctx, fmt.Sprintf("%d", 4)) require.NoError(t, err) require.Equal(t, int64(0), resp.Count) next := 20 for i := 12; i <= next; i++ { _, err := cli.Put(ctx, fmt.Sprintf("%d", i), stringutil.RandString(uint(valueSize))) require.NoError(t, err) } expectedRevision := next for procIdx, proc := range clus.Procs { cli = newClient(t, proc.EndpointsGRPC(), e2e.ClientConfig{}) resp, err := cli.Get(ctx, fmt.Sprintf("%d", next)) require.NoError(t, err) assert.GreaterOrEqualf(t, resp.Header.Revision, int64(expectedRevision), "LeaderIdx: %d, Current: %d", leaderIdx, procIdx) } } ================================================ FILE: tests/e2e/reproduce_18667_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "math/rand" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" ) // TestReproduce18667 reproduces the issue: https://github.com/etcd-io/etcd/issues/18667. func TestReproduce18667(t *testing.T) { e2e.BeforeTest(t) ctx := t.Context() clus, cerr := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(3), e2e.WithSnapshotCount(1000), ) require.NoError(t, cerr) t.Cleanup(func() { clus.Stop() }) targetIdx := rand.Intn(3) cli := newClient(t, clus.Procs[targetIdx].EndpointsGRPC(), e2e.ClientConfig{}) t.Log("Put key-value [k1...k20]") var revision int64 for i := 1; i <= 20; i++ { resp, err := cli.Put(ctx, fmt.Sprintf("k%d", i), fmt.Sprintf("v%d", i)) require.NoError(t, err) revision = resp.Header.Revision } t.Logf("Compact on the latest revision %d", revision) _, err := cli.Compact(ctx, revision) require.NoError(t, err) t.Log("Send txn to make data inconsistent among etcdservers") cmp := clientv3.Compare(clientv3.Value("k1"), "=", "v1") then := []clientv3.Op{ clientv3.OpPut("k2", "foo"), clientv3.OpGet("k1", clientv3.WithRev(revision-1)), } _, err = cli.Txn(ctx).If(cmp).Then(then...).Commit() require.Error(t, err) require.Contains(t, err.Error(), rpctypes.ErrCompacted.Error()) t.Log("Verify all members have consistent data on key 'k2'") for i := 0; i < clus.Cfg.ClusterSize; i++ { idx := (targetIdx + i) % clus.Cfg.ClusterSize cli = newClient(t, clus.Procs[idx].EndpointsGRPC(), e2e.ClientConfig{}) resp, err := cli.Get(ctx, "k2") require.NoError(t, err) // TODO: All members are supposed to have the same key/value pair k2:v2. // However, the issue https://github.com/etcd-io/etcd/issues/18667 // hasn't been resolved yet, so some members hold the wrong data // "k2:foo". Once the issue is fixed, we should remove the if-else // branch below, and all member should have consistent data k2:v2. if i == 0 { assert.Equal(t, "v2", string(resp.Kvs[0].Value)) } else { assert.Equal(t, "foo", string(resp.Kvs[0].Value)) } } } ================================================ FILE: tests/e2e/reproduce_19406_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "net/url" "testing" "time" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/stringutil" "go.etcd.io/etcd/tests/v3/framework/e2e" ) // TestReproduce19406 reproduces the issue: https://github.com/etcd-io/etcd/issues/19406 func TestReproduce19406(t *testing.T) { e2e.BeforeTest(t) compactionSleepInterval := 100 * time.Millisecond ctx := t.Context() clus, cerr := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), e2e.WithCompactionBatchLimit(1), e2e.WithCompactionSleepInterval(compactionSleepInterval), ) require.NoError(t, cerr) t.Cleanup(func() { require.NoError(t, clus.Stop()) }) // Produce some data cli := newClient(t, clus.EndpointsGRPC(), e2e.ClientConfig{}) valueSize := 10 var latestRevision int64 produceKeyNum := 20 for i := 0; i <= produceKeyNum; i++ { resp, err := cli.Put(ctx, fmt.Sprintf("%d", i), stringutil.RandString(uint(valueSize))) require.NoError(t, err) latestRevision = resp.Header.Revision } // Sleep for PerCompactionInterationInterval to simulate a single iteration of compaction lasting at least this duration. PerCompactionInterationInterval := compactionSleepInterval require.NoError(t, clus.Procs[0].Failpoints().SetupHTTP(ctx, "compactAfterAcquiredBatchTxLock", fmt.Sprintf(`sleep("%s")`, PerCompactionInterationInterval))) // start compaction t.Log("start compaction...") _, err := cli.Compact(ctx, latestRevision, clientv3.WithCompactPhysical()) require.NoError(t, err) t.Log("finished compaction...") // Validate that total compaction sleep interval // Compaction runs in batches. During each batch, it acquires a lock, releases it at the end, // and then waits for a compactionSleepInterval before starting the next batch. This pause // allows PUT requests to be processed. // Therefore, the total compaction sleep interval larger or equal to // (compaction iteration number - 1) * compactionSleepInterval httpEndpoint := clus.EndpointsHTTP()[0] totalKeys := produceKeyNum + 1 pauseDuration, totalDuration := getEtcdCompactionMetrics(t, httpEndpoint) require.NoError(t, err) actualSleepInterval := time.Duration(totalDuration-pauseDuration) * time.Millisecond expectSleepInterval := compactionSleepInterval * time.Duration(totalKeys) t.Logf("db_compaction_pause_duration: %.2f db_compaction_total_duration: %.2f, totalKeys: %d", pauseDuration, totalDuration, totalKeys) require.GreaterOrEqualf(t, actualSleepInterval, expectSleepInterval, "expect total compact sleep interval larger than (%v) but got (%v)", expectSleepInterval, actualSleepInterval) } func getEtcdCompactionMetrics(t *testing.T, httpEndpoint string) (pauseDuration, totalDuration float64) { metricsURL, err := url.JoinPath(httpEndpoint, "metrics") require.NoError(t, err) // Fetch metrics from the endpoint metricFamilies, err := e2e.GetMetrics(metricsURL) require.NoError(t, err) // Extract sum from histogram metric getHistogramSum := func(name string) float64 { mf, ok := metricFamilies[name] require.Truef(t, ok, "metric %q not found", name) require.NotEmptyf(t, mf.Metric, "metric %q has no data", name) hist := mf.Metric[0].GetHistogram() require.NotEmptyf(t, hist, "metric %q is not a histogram", name) return hist.GetSampleSum() } pauseDuration = getHistogramSum("etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds") totalDuration = getHistogramSum("etcd_debugging_mvcc_db_compaction_total_duration_milliseconds") return pauseDuration, totalDuration } ================================================ FILE: tests/e2e/reproduce_20271_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "fmt" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) // TestIssue20271 reproduces the issue: https://github.com/etcd-io/etcd/issues/20271. func TestIssue20271(t *testing.T) { e2e.BeforeTest(t) ctx := t.Context() snapCount := 10 cfg := e2e.NewConfig( e2e.WithSnapshotCount(uint64(snapCount)), e2e.WithSnapshotCatchUpEntries(uint64(snapCount)), e2e.WithClusterSize(3), e2e.WithKeepDataDir(true), e2e.WithGoFailEnabled(true), e2e.WithLogLevel("debug"), e2e.WithRollingStart(true), e2e.WithInitialLeaderIndex(0), ) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg)) require.NoError(t, err) defer func() { require.NoError(t, epc.Close()) }() t.Log("Step 1: Write some data to the cluster") for i := 0; i < snapCount*5; i++ { _, err = epc.Procs[0].Etcdctl().Put(ctx, fmt.Sprintf("foo%d", i), strings.Repeat("Oops", 1024), config.PutOptions{}) require.NoError(t, err) } t.Log(`Step 2: Config the third member to sleep 15s after OpenSnapshotBackend and use SIGSTOP to pause it.`) require.NoError(t, epc.Procs[2].Failpoints().SetupHTTP(ctx, "applyAfterOpenSnapshot", `sleep("15s")`)) epc.Procs[2].Pause() t.Log("Step 3: Delete some key values to trigger new snapshot on the first two members") for i := 0; i < snapCount+20; i++ { _, err = epc.Procs[0].Etcdctl().Delete(ctx, fmt.Sprintf("foo%d", i), config.DeleteOptions{}) require.NoError(t, err) } t.Log("Step 4: Restarting the first two members to re-connect to the paused member, so the inflight messages will be dropped. This will trigger new leader to send snapshot file to the third member.") for _, proc := range epc.Procs[:2] { require.NoError(t, proc.Restart(ctx)) } epc.Procs[2].Resume() t.Log(`Step 5: After opening snapshot file from new leader, invoke defragment\n to override boltdb file. So, for the following changes, the third member will commit them into deleted boltdb file.`) e2e.AssertProcessLogs(t, epc.Procs[2], "applySnapshot: opened snapshot backend") err = epc.Procs[2].Etcdctl().Defragment(ctx, config.DefragOption{Timeout: 30 * time.Second}) require.NoError(t, err) e2e.AssertProcessLogs(t, epc.Procs[2], "applied snapshot") require.NoError(t, epc.Procs[2].Failpoints().DeactivateHTTP(ctx, "applyAfterOpenSnapshot")) t.Log("Step 6: Write some data to the cluster") for i := 0; i < snapCount/2; i++ { _, err = epc.Procs[0].Etcdctl().Put(ctx, fmt.Sprintf("foo%d", i), strings.Repeat("Oops", 1), config.PutOptions{}) require.NoError(t, err) } t.Log("Step 7: Restart the third member. It recovers from the new boltdb file. Therefore, data written in Step 6 is lost.") require.NoError(t, epc.Procs[2].Restart(ctx)) t.Log("Step 8: Check hashkv of each member") assertKVHash(t, epc) } ================================================ FILE: tests/e2e/runtime_reconfiguration_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "context" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/e2e" ) // TestRuntimeReconfigGrowClusterSize ensures growing cluster size with two phases // Phase 1 - Inform cluster of new configuration // Phase 2 - Start new member func TestRuntimeReconfigGrowClusterSize(t *testing.T) { e2e.BeforeTest(t) tcs := []struct { name string clusterSize int asLearner bool }{ { name: "grow cluster size from 1 to 3", clusterSize: 1, }, { name: "grow cluster size from 3 to 5", clusterSize: 3, }, { name: "grow cluster size from 1 to 3 with learner", clusterSize: 1, asLearner: true, }, { name: "grow cluster size from 3 to 5 with learner", clusterSize: 3, asLearner: true, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(tc.clusterSize)) require.NoError(t, err) require.NoError(t, epc.Procs[0].Etcdctl().Health(ctx)) defer func() { err := epc.Close() require.NoErrorf(t, err, "failed to close etcd cluster") }() for i := 0; i < 2; i++ { time.Sleep(etcdserver.HealthInterval) if !tc.asLearner { addMember(ctx, t, epc) } else { addMemberAsLearnerAndPromote(ctx, t, epc) } } }) } } func TestRuntimeReconfigDecreaseClusterSize(t *testing.T) { e2e.BeforeTest(t) tcs := []struct { name string clusterSize int asLearner bool }{ { name: "decrease cluster size from 3 to 1", clusterSize: 3, }, { name: "decrease cluster size from 5 to 3", clusterSize: 5, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(tc.clusterSize)) require.NoError(t, err) require.NoError(t, epc.Procs[0].Etcdctl().Health(ctx)) defer func() { err := epc.Close() require.NoErrorf(t, err, "failed to close etcd cluster") }() for i := 0; i < 2; i++ { time.Sleep(etcdserver.HealthInterval) removeFirstMember(ctx, t, epc) } }) } } func TestRuntimeReconfigRollingUpgrade(t *testing.T) { e2e.BeforeTest(t) tcs := []struct { name string withLearner bool }{ { name: "with learner", withLearner: true, }, { name: "without learner", withLearner: false, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(3)) require.NoError(t, err) require.NoError(t, epc.Procs[0].Etcdctl().Health(ctx)) defer func() { err := epc.Close() require.NoErrorf(t, err, "failed to close etcd cluster") }() for i := 0; i < 2; i++ { time.Sleep(etcdserver.HealthInterval) removeFirstMember(ctx, t, epc) epc.WaitLeader(t) // if we do not wait for leader, without the fix of notify raft Advance, // have to wait 1 sec to pass the test stably. if tc.withLearner { addMemberAsLearnerAndPromote(ctx, t, epc) } else { addMember(ctx, t, epc) } } }) } } func addMember(ctx context.Context, t *testing.T, epc *e2e.EtcdProcessCluster) { _, err := epc.StartNewProc(ctx, nil, t, false /* addAsLearner */) require.NoError(t, err) require.NoError(t, epc.Procs[len(epc.Procs)-1].Etcdctl().Health(ctx)) } func addMemberAsLearnerAndPromote(ctx context.Context, t *testing.T, epc *e2e.EtcdProcessCluster) { endpoints := epc.EndpointsGRPC() id, err := epc.StartNewProc(ctx, nil, t, true /* addAsLearner */) require.NoError(t, err) attempt := 0 for attempt < 3 { _, err = epc.Etcdctl(e2e.WithEndpoints(endpoints)).MemberPromote(ctx, id) if err == nil || !strings.Contains(err.Error(), "can only promote a learner member which is in sync with leader") { break } time.Sleep(100 * time.Millisecond) attempt++ } require.NoError(t, err) newLearnerMemberProc := epc.Procs[len(epc.Procs)-1] require.NoError(t, newLearnerMemberProc.Etcdctl().Health(ctx)) } func removeFirstMember(ctx context.Context, t *testing.T, epc *e2e.EtcdProcessCluster) { // avoid tearing down the last etcd process if len(epc.Procs) == 1 { return } firstProc := epc.Procs[0] sts, err := firstProc.Etcdctl().Status(ctx) require.NoError(t, err) memberIDToRemove := sts[0].Header.MemberId epc.Procs = epc.Procs[1:] _, err = epc.Etcdctl().MemberRemove(ctx, memberIDToRemove) require.NoError(t, err) require.NoError(t, firstProc.Stop()) require.NoError(t, firstProc.Close()) } ================================================ FILE: tests/e2e/utils.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "bytes" "context" "crypto/rand" "crypto/rsa" "crypto/x509" "crypto/x509/pkix" "encoding/json" "encoding/pem" "fmt" "math/big" "net" "os" "strings" "testing" "time" "go.uber.org/zap" "golang.org/x/sync/errgroup" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/stringutil" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/integration" ) func newClient(t *testing.T, entpoints []string, cfg e2e.ClientConfig) *clientv3.Client { tlscfg, err := tlsInfo(t, cfg) if err != nil { t.Fatal(err) } ccfg := clientv3.Config{ Endpoints: entpoints, DialTimeout: 5 * time.Second, } if tlscfg != nil { ccfg.TLS, err = tlscfg.ClientConfig() if err != nil { t.Fatal(err) } } c, err := clientv3.New(ccfg) if err != nil { t.Fatal(err) } t.Cleanup(func() { c.Close() }) return c } // tlsInfo follows the Client-to-server communication in https://etcd.io/docs/v3.6/op-guide/security/#basic-setup func tlsInfo(tb testing.TB, cfg e2e.ClientConfig) (*transport.TLSInfo, error) { switch cfg.ConnectionType { case e2e.ClientNonTLS, e2e.ClientTLSAndNonTLS: return nil, nil case e2e.ClientTLS: if cfg.AutoTLS { tls, err := transport.SelfCert(zap.NewNop(), tb.TempDir(), []string{"localhost"}, 1) if err != nil { return nil, fmt.Errorf("failed to generate cert: %w", err) } return &tls, nil } return &integration.TestTLSInfo, nil default: return nil, fmt.Errorf("config %v not supported", cfg) } } func fillEtcdWithData(ctx context.Context, c *clientv3.Client, dbSize int) error { g := errgroup.Group{} concurrency := 10 keyCount := 100 keysPerRoutine := keyCount / concurrency valueSize := dbSize / keyCount for i := 0; i < concurrency; i++ { i := i g.Go(func() error { for j := 0; j < keysPerRoutine; j++ { _, err := c.Put(ctx, fmt.Sprintf("%d", i*keysPerRoutine+j), stringutil.RandString(uint(valueSize))) if err != nil { return err } } return nil }) } return g.Wait() } func curl(endpoint string, method string, curlReq e2e.CURLReq, connType e2e.ClientConnType) (string, error) { args := e2e.CURLPrefixArgs(endpoint, e2e.ClientConfig{ConnectionType: connType}, false, method, curlReq) lines, err := e2e.RunUtilCompletion(args, nil) if err != nil { return "", err } return strings.Join(lines, "\n"), nil } func runCommandAndReadJSONOutput(args []string) (map[string]any, error) { lines, err := e2e.RunUtilCompletion(args, nil) if err != nil { return nil, err } var resp map[string]any err = json.Unmarshal([]byte(strings.Join(lines, "\n")), &resp) if err != nil { return nil, err } return resp, nil } func getMemberIDByName(ctx context.Context, c *e2e.EtcdctlV3, name string) (id uint64, found bool, err error) { resp, err := c.MemberList(ctx, false) if err != nil { return 0, false, err } for _, member := range resp.Members { if name == member.Name { return member.ID, true, nil } } return 0, false, nil } func generateCertsForIPs(tempDir string, ips []net.IP) (caFile string, certFiles []string, keyFiles []string, err error) { ca := &x509.Certificate{ SerialNumber: big.NewInt(1001), Subject: pkix.Name{ Organization: []string{"etcd"}, OrganizationalUnit: []string{"etcd Security"}, Locality: []string{"San Francisco"}, Province: []string{"California"}, Country: []string{"USA"}, }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(0, 0, 1), IsCA: true, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, BasicConstraintsValid: true, } caKey, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { return "", nil, nil, err } caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) if err != nil { return "", nil, nil, err } caFile, _, err = saveCertToFile(tempDir, caBytes, nil) if err != nil { return "", nil, nil, err } for i, ip := range ips { cert := &x509.Certificate{ SerialNumber: big.NewInt(1001 + int64(i)), Subject: pkix.Name{ Organization: []string{"etcd"}, OrganizationalUnit: []string{"etcd Security"}, Locality: []string{"San Francisco"}, Province: []string{"California"}, Country: []string{"USA"}, }, IPAddresses: []net.IP{ip}, NotBefore: time.Now(), NotAfter: time.Now().AddDate(0, 0, 1), SubjectKeyId: []byte{1, 2, 3, 4, 5}, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, KeyUsage: x509.KeyUsageDigitalSignature, } certKey, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { return "", nil, nil, err } certBytes, err := x509.CreateCertificate(rand.Reader, cert, ca, &certKey.PublicKey, caKey) if err != nil { return "", nil, nil, err } certFile, keyFile, err := saveCertToFile(tempDir, certBytes, certKey) if err != nil { return "", nil, nil, err } certFiles = append(certFiles, certFile) keyFiles = append(keyFiles, keyFile) } return caFile, certFiles, keyFiles, nil } func saveCertToFile(tempDir string, certBytes []byte, key *rsa.PrivateKey) (certFile string, keyFile string, err error) { certPEM := new(bytes.Buffer) pem.Encode(certPEM, &pem.Block{ Type: "CERTIFICATE", Bytes: certBytes, }) cf, err := os.CreateTemp(tempDir, "*.crt") if err != nil { return "", "", err } defer cf.Close() if _, err := cf.Write(certPEM.Bytes()); err != nil { return "", "", err } if key != nil { certKeyPEM := new(bytes.Buffer) pem.Encode(certKeyPEM, &pem.Block{ Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key), }) kf, err := os.CreateTemp(tempDir, "*.key.insecure") if err != nil { return "", "", err } defer kf.Close() if _, err := kf.Write(certKeyPEM.Bytes()); err != nil { return "", "", err } return cf.Name(), kf.Name(), nil } return cf.Name(), "", nil } func getLocalIP() (string, error) { conn, err := net.Dial("udp", "8.8.8.8:80") if err != nil { return "", err } defer conn.Close() localAddress := conn.LocalAddr().(*net.UDPAddr) return localAddress.IP.String(), nil } ================================================ FILE: tests/e2e/utl_migrate_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "fmt" "path/filepath" "strings" "testing" "time" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestEtctlutlMigrate(t *testing.T) { lastReleaseBinary := e2e.BinPath.EtcdLastRelease tcs := []struct { name string targetVersion string clusterVersion e2e.ClusterVersion clusterSize int force bool expectLogsSubString string expectStorageVersion *semver.Version }{ { name: "Invalid target version string", targetVersion: "abc", clusterSize: 1, expectLogsSubString: `Error: wrong target version format, expected "X.Y", got "abc"`, expectStorageVersion: &version.V3_7, }, { name: "Invalid target version", targetVersion: "3.a", clusterSize: 1, expectLogsSubString: `Error: failed to parse target version: strconv.ParseInt: parsing "a": invalid syntax`, expectStorageVersion: &version.V3_7, }, { name: "Target with only major version is invalid", targetVersion: "3", clusterSize: 1, expectLogsSubString: `Error: wrong target version format, expected "X.Y", got "3"`, expectStorageVersion: &version.V3_7, }, { name: "Target with patch version is invalid", targetVersion: "3.6.0", clusterSize: 1, expectLogsSubString: `Error: wrong target version format, expected "X.Y", got "3.6.0"`, expectStorageVersion: &version.V3_7, }, { name: "Migrate v3.6 to v3.6 is no-op", clusterVersion: e2e.LastVersion, clusterSize: 1, targetVersion: "3.6", expectStorageVersion: &version.V3_6, expectLogsSubString: "storage version up-to-date\t" + `{"storage-version": "3.6"}`, }, { name: "Upgrade 1 member cluster from v3.5 to v3.6 should work", clusterVersion: e2e.LastVersion, clusterSize: 1, targetVersion: "3.7", expectStorageVersion: &version.V3_7, }, { name: "Upgrade 3 member cluster from v3.6 to v3.7 should work", clusterVersion: e2e.LastVersion, clusterSize: 3, targetVersion: "3.7", expectStorageVersion: &version.V3_7, }, { name: "Migrate v3.7 to v3.7 is no-op", targetVersion: "3.7", clusterSize: 1, expectLogsSubString: "storage version up-to-date\t" + `{"storage-version": "3.7"}`, expectStorageVersion: &version.V3_7, }, { name: "Downgrade 1 member cluster from v3.7 to v3.6 should work", targetVersion: "3.6", clusterSize: 1, expectLogsSubString: "updated storage version", expectStorageVersion: &version.V3_6, }, { name: "Downgrade 3 member cluster from v3.7 to v3.6 should work", targetVersion: "3.6", clusterSize: 3, expectLogsSubString: "updated storage version", expectStorageVersion: &version.V3_6, }, { name: "Upgrade v3.7 to v3.8 with force should work", targetVersion: "3.8", clusterSize: 1, force: true, expectLogsSubString: "forcefully set storage version\t" + `{"storage-version": "3.8"}`, expectStorageVersion: &semver.Version{Major: 3, Minor: 8}, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { e2e.BeforeTest(t) lg := zaptest.NewLogger(t) if tc.clusterVersion != e2e.CurrentVersion && !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { t.Skipf("%q does not exist", lastReleaseBinary) } dataDirPath := t.TempDir() epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithVersion(tc.clusterVersion), e2e.WithDataDirPath(dataDirPath), e2e.WithClusterSize(1), e2e.WithKeepDataDir(true), // Set low SnapshotCount to ensure wal snapshot is done e2e.WithSnapshotCount(1), ) if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } defer func() { if errC := epc.Close(); errC != nil { t.Fatalf("error closing etcd processes (%v)", errC) } }() dialTimeout := 10 * time.Second prefixArgs := []string{e2e.BinPath.Etcdctl, "--endpoints", strings.Join(epc.EndpointsGRPC(), ","), "--dial-timeout", dialTimeout.String()} t.Log("Write keys to ensure wal snapshot is created and all v3.5 fields are set...") for i := 0; i < 10; i++ { require.NoError(t, e2e.SpawnWithExpect(append(prefixArgs, "put", fmt.Sprintf("%d", i), "value"), expect.ExpectedResponse{Value: "OK"})) } t.Log("Stopping the members") for i := 0; i < len(epc.Procs); i++ { t.Logf("Stopping server %d: %v", i, epc.Procs[i].EndpointsGRPC()) err = epc.Procs[i].Stop() require.NoError(t, err) } t.Log("etcdutl migrate all members") for i := 0; i < len(epc.Procs); i++ { t.Logf("etcdutl migrate member %d: %v", i, epc.Procs[i].EndpointsGRPC()) memberDataDir := epc.Procs[i].Config().DataDirPath args := []string{e2e.BinPath.Etcdutl, "migrate", "--data-dir", memberDataDir, "--target-version", tc.targetVersion} if tc.force { args = append(args, "--force") } err = e2e.SpawnWithExpect(args, expect.ExpectedResponse{Value: tc.expectLogsSubString}) if err != nil && tc.expectLogsSubString != "" { require.ErrorContains(t, err, tc.expectLogsSubString) } else { require.NoError(t, err) } be := backend.NewDefaultBackend(lg, filepath.Join(memberDataDir, "member/snap/db")) ver := schema.ReadStorageVersion(be.ReadTx()) assert.Equal(t, tc.expectStorageVersion, ver) be.Close() } }) } } ================================================ FILE: tests/e2e/v2store_deprecation_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "sort" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestV2DeprecationNotYet(t *testing.T) { e2e.BeforeTest(t) t.Log("Verify its infeasible to start etcd with --v2-deprecation=not-yet mode") proc, err := e2e.SpawnCmd([]string{e2e.BinPath.Etcd, "--v2-deprecation=not-yet"}, nil) require.NoError(t, err) _, err = proc.Expect(`invalid value "not-yet" for flag -v2-deprecation: invalid value "not-yet"`) assert.NoError(t, err) } // TestV2DeprecationSnapshotMatches ensures that etcd v3.7 still commits v2 store // changes to the snapshot for backwards compatibility. func TestV2DeprecationSnapshotMatches(t *testing.T) { e2e.BeforeTest(t) lastReleaseData := t.TempDir() currentReleaseData := t.TempDir() if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease) } ctx, cancel := context.WithCancel(t.Context()) defer cancel() var snapshotCount uint64 = 10 epc := runEtcdAndCreateSnapshot(t, e2e.LastVersion, lastReleaseData, snapshotCount) oldMemberDataDir := epc.Procs[0].Config().DataDirPath cc1 := epc.Etcdctl() addAndRemoveKeysAndMembers(ctx, t, cc1, snapshotCount) require.NoError(t, epc.Close()) epc = runEtcdAndCreateSnapshot(t, e2e.CurrentVersion, currentReleaseData, snapshotCount) newMemberDataDir := epc.Procs[0].Config().DataDirPath cc2 := epc.Etcdctl() addAndRemoveKeysAndMembers(ctx, t, cc2, snapshotCount) require.NoError(t, epc.Close()) assertSnapshotsMatch(t, oldMemberDataDir, newMemberDataDir) } func addAndRemoveKeysAndMembers(ctx context.Context, tb testing.TB, cc *e2e.EtcdctlV3, snapshotCount uint64) { // Execute some non-trivial key&member operation var i uint64 for i = 0; i < snapshotCount*3; i++ { _, err := cc.Put(ctx, fmt.Sprintf("%d", i), "1", config.PutOptions{}) require.NoError(tb, err) } member1, err := cc.MemberAddAsLearner(ctx, "member1", []string{"http://127.0.0.1:2000"}) require.NoError(tb, err) for i = 0; i < snapshotCount*2; i++ { _, err = cc.Delete(ctx, fmt.Sprintf("%d", i), config.DeleteOptions{}) require.NoError(tb, err) } _, err = cc.MemberRemove(ctx, member1.Member.ID) require.NoError(tb, err) for i = 0; i < snapshotCount; i++ { _, err = cc.Put(ctx, fmt.Sprintf("%d", i), "2", config.PutOptions{}) require.NoError(tb, err) } _, err = cc.MemberAddAsLearner(ctx, "member2", []string{"http://127.0.0.1:2001"}) require.NoError(tb, err) for i = 0; i < snapshotCount/2; i++ { _, err = cc.Put(ctx, fmt.Sprintf("%d", i), "3", config.PutOptions{}) assert.NoError(tb, err) } } func TestV2DeprecationSnapshotRecover(t *testing.T) { e2e.BeforeTest(t) dataDir := t.TempDir() ctx, cancel := context.WithCancel(t.Context()) defer cancel() if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease) } epc := runEtcdAndCreateSnapshot(t, e2e.LastVersion, dataDir, 10) cc := epc.Etcdctl() lastReleaseGetResponse, err := cc.Get(ctx, "", config.GetOptions{Prefix: true}) require.NoError(t, err) lastReleaseMemberListResponse, err := cc.MemberList(ctx, false) assert.NoError(t, err) assert.NoError(t, epc.Close()) cfg := e2e.ConfigStandalone(*e2e.NewConfig( e2e.WithVersion(e2e.CurrentVersion), e2e.WithDataDirPath(dataDir), )) epc, err = e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg)) require.NoError(t, err) cc = epc.Etcdctl() currentReleaseGetResponse, err := cc.Get(ctx, "", config.GetOptions{Prefix: true}) require.NoError(t, err) currentReleaseMemberListResponse, err := cc.MemberList(ctx, false) require.NoError(t, err) assert.Equal(t, lastReleaseGetResponse.Kvs, currentReleaseGetResponse.Kvs) assert.Equal(t, lastReleaseMemberListResponse.Members, currentReleaseMemberListResponse.Members) assert.NoError(t, epc.Close()) } func runEtcdAndCreateSnapshot(tb testing.TB, serverVersion e2e.ClusterVersion, dataDir string, snapshotCount uint64) *e2e.EtcdProcessCluster { cfg := e2e.ConfigStandalone(*e2e.NewConfig( e2e.WithVersion(serverVersion), e2e.WithDataDirPath(dataDir), e2e.WithSnapshotCount(snapshotCount), e2e.WithKeepDataDir(true), )) epc, err := e2e.NewEtcdProcessCluster(tb.Context(), tb, e2e.WithConfig(cfg)) assert.NoError(tb, err) return epc } func filterSnapshotFiles(path string) bool { return strings.HasSuffix(path, ".snap") } func assertSnapshotsMatch(tb testing.TB, firstDataDir, secondDataDir string) { lg := zaptest.NewLogger(tb) firstFiles, err := fileutil.ListFiles(firstDataDir, filterSnapshotFiles) require.NoError(tb, err) secondFiles, err := fileutil.ListFiles(secondDataDir, filterSnapshotFiles) require.NoError(tb, err) assert.NotEmpty(tb, firstFiles) assert.NotEmpty(tb, secondFiles) assert.Len(tb, secondFiles, len(firstFiles)) sort.Strings(firstFiles) sort.Strings(secondFiles) for i := 0; i < len(firstFiles); i++ { assertV2StoreMembershipEqual(tb, lg, firstFiles[i], secondFiles[i]) } } func assertV2StoreMembershipEqual(tb testing.TB, lg *zap.Logger, firstSnapPath, secondSnapPath string) { st1 := loadV2StoreData(tb, lg, firstSnapPath) st2 := loadV2StoreData(tb, lg, secondSnapPath) st1Members, st1Deleted := membership.MembersFromStore(lg, st1) st2Members, st2Deleted := membership.MembersFromStore(lg, st2) require.Lenf(tb, st1Members, len(st2Members), "number of members in v2 store do not match") require.NotEmptyf(tb, st1Members, "no members found in v2 store") require.Lenf(tb, st1Deleted, len(st2Deleted), "number of deleted members in v2 store do not match") // remove ID because original ID was generated from hash of peerURLs + clusterName + time require.Equal(tb, rebuildMembers(tb, st1Members), rebuildMembers(tb, st2Members)) } // loadV2StoreData reads v2 store from the snapshot file at fpath. func loadV2StoreData(tb testing.TB, lg *zap.Logger, fpath string) v2store.Store { sn, err := snap.Read(lg, fpath) require.NoError(tb, err) v2data := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix) v2data.Recovery(sn.Data) return v2data } // rebuildMembers rebuilds the members map with zeroed IDs and peerURLs as keys. func rebuildMembers(tb testing.TB, members map[types.ID]*membership.Member) map[string]*membership.Member { newMembers := make(map[string]*membership.Member) for _, m := range members { peerURLs, err := types.NewURLs(m.PeerURLs) require.NoError(tb, err) m.ID = 0 newMembers[peerURLs.String()] = m } return newMembers } ================================================ FILE: tests/e2e/v3_cipher_suite_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "fmt" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCurlV3CipherSuitesValid(t *testing.T) { testCurlV3CipherSuites(t, true) } func TestCurlV3CipherSuitesMismatch(t *testing.T) { testCurlV3CipherSuites(t, false) } func testCurlV3CipherSuites(t *testing.T, valid bool) { cc := e2e.NewConfigClientTLS() cc.ClusterSize = 1 cc.ServerConfig.CipherSuites = []string{ "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", } testFunc := cipherSuiteTestValid if !valid { testFunc = cipherSuiteTestMismatch } testCtl(t, testFunc, withCfg(*cc)) } func cipherSuiteTestValid(cx ctlCtx) { if err := e2e.CURLGet(cx.epc, e2e.CURLReq{ Endpoint: "/metrics", Expected: expect.ExpectedResponse{Value: fmt.Sprintf(`etcd_server_version{server_version="%s"} 1`, version.Version)}, Ciphers: "ECDHE-RSA-AES128-GCM-SHA256", // TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 }); err != nil { require.ErrorContains(cx.t, err, fmt.Sprintf(`etcd_server_version{server_version="%s"} 1`, version.Version)) } } func cipherSuiteTestMismatch(cx ctlCtx) { err := e2e.CURLGet(cx.epc, e2e.CURLReq{ Endpoint: "/metrics", Expected: expect.ExpectedResponse{Value: "failed setting cipher list"}, Ciphers: "ECDHE-RSA-DES-CBC3-SHA", // TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA }) require.ErrorContains(cx.t, err, "curl: (59) failed setting cipher list") } ================================================ FILE: tests/e2e/v3_curl_auth_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "encoding/json" "fmt" "math/rand" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/authpb" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCurlV3Auth(t *testing.T) { testCtl(t, testCurlV3Auth) } func TestCurlV3AuthClientTLSCertAuth(t *testing.T) { testCtl(t, testCurlV3Auth, withCfg(*e2e.NewConfigClientTLSCertAuthWithNoCN())) } func TestCurlV3AuthUserBasicOperations(t *testing.T) { testCtl(t, testCurlV3AuthUserBasicOperations) } func TestCurlV3AuthUserGrantRevokeRoles(t *testing.T) { testCtl(t, testCurlV3AuthUserGrantRevokeRoles) } func TestCurlV3AuthRoleBasicOperations(t *testing.T) { testCtl(t, testCurlV3AuthRoleBasicOperations) } func TestCurlV3AuthRoleManagePermission(t *testing.T) { testCtl(t, testCurlV3AuthRoleManagePermission) } func TestCurlV3AuthEnableDisableStatus(t *testing.T) { testCtl(t, testCurlV3AuthEnableDisableStatus) } func testCurlV3Auth(cx ctlCtx) { usernames := []string{"root", "nonroot", "nooption"} pwds := []string{"toor", "pass", "pass"} options := []*authpb.UserAddOptions{{NoPassword: false}, {NoPassword: false}, nil} // create users for i := 0; i < len(usernames); i++ { user, err := json.Marshal(&pb.AuthUserAddRequest{Name: usernames[i], Password: pwds[i], Options: options[i]}) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/user/add", Value: string(user), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3Auth failed to add user %v", usernames[i]) } // create root role rolereq, err := json.Marshal(&pb.AuthRoleAddRequest{Name: "root"}) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/role/add", Value: string(rolereq), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3Auth failed to create role") // grant root role for i := 0; i < len(usernames); i++ { grantroleroot, merr := json.Marshal(&pb.AuthUserGrantRoleRequest{User: usernames[i], Role: "root"}) require.NoError(cx.t, merr) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/user/grant", Value: string(grantroleroot), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3Auth failed to grant role") } // enable auth require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/enable", Value: "{}", Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3Auth failed to enable auth") for i := 0; i < len(usernames); i++ { // put "bar[i]" into "foo[i]" putreq, err := json.Marshal(&pb.PutRequest{Key: []byte(fmt.Sprintf("foo%d", i)), Value: []byte(fmt.Sprintf("bar%d", i))}) require.NoError(cx.t, err) // fail put no auth require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/kv/put", Value: string(putreq), Expected: expect.ExpectedResponse{Value: "etcdserver: user name is empty"}, }), "testCurlV3Auth failed to put without token") // auth request authreq, err := json.Marshal(&pb.AuthenticateRequest{Name: usernames[i], Password: pwds[i]}) require.NoError(cx.t, err) var ( authHeader string cmdArgs []string lineFunc = func(txt string) bool { return true } ) cmdArgs = e2e.CURLPrefixArgsCluster(cx.epc.Cfg, cx.epc.Procs[rand.Intn(cx.epc.Cfg.ClusterSize)], "POST", e2e.CURLReq{ Endpoint: "/v3/auth/authenticate", Value: string(authreq), }) proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) require.NoError(cx.t, err) defer proc.Close() cURLRes, err := proc.ExpectFunc(context.Background(), lineFunc) require.NoError(cx.t, err) authRes := make(map[string]any) require.NoError(cx.t, json.Unmarshal([]byte(cURLRes), &authRes)) token, ok := authRes[rpctypes.TokenFieldNameGRPC].(string) if !ok { cx.t.Fatalf("failed invalid token in authenticate response using user (%v)", usernames[i]) } authHeader = "Authorization: " + token // put with auth require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/kv/put", Value: string(putreq), Header: authHeader, Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3Auth failed to auth put with user (%v)", usernames[i]) } } func testCurlV3AuthUserBasicOperations(cx ctlCtx) { usernames := []string{"user1", "user2", "user3"} // create users for i := 0; i < len(usernames); i++ { user, err := json.Marshal(&pb.AuthUserAddRequest{Name: usernames[i], Password: "123"}) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/user/add", Value: string(user), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthUserBasicOperations failed to add user %v", usernames[i]) } // change password user, err := json.Marshal(&pb.AuthUserChangePasswordRequest{Name: "user1", Password: "456"}) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/user/changepw", Value: string(user), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthUserBasicOperations failed to change user's password") // get users usernames = []string{"user1", "userX"} expectedResponse := []string{"revision", "etcdserver: user name not found"} for i := 0; i < len(usernames); i++ { user, err = json.Marshal(&pb.AuthUserGetRequest{ Name: usernames[i], }) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/user/get", Value: string(user), Expected: expect.ExpectedResponse{Value: expectedResponse[i]}, }), "testCurlV3AuthUserBasicOperations failed to get user %v", usernames[i]) } // delete users usernames = []string{"user2", "userX"} expectedResponse = []string{"revision", "etcdserver: user name not found"} for i := 0; i < len(usernames); i++ { user, err = json.Marshal(&pb.AuthUserDeleteRequest{ Name: usernames[i], }) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/user/delete", Value: string(user), Expected: expect.ExpectedResponse{Value: expectedResponse[i]}, }), "testCurlV3AuthUserBasicOperations failed to delete user %v", usernames[i]) } // list users clus := cx.epc args := e2e.CURLPrefixArgsCluster(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "POST", e2e.CURLReq{ Endpoint: "/v3/auth/user/list", Value: "{}", }) resp, err := runCommandAndReadJSONOutput(args) require.NoError(cx.t, err) users, ok := resp["users"] require.True(cx.t, ok) userSlice := users.([]any) require.Len(cx.t, userSlice, 2) require.Equal(cx.t, "user1", userSlice[0]) require.Equal(cx.t, "user3", userSlice[1]) } func testCurlV3AuthUserGrantRevokeRoles(cx ctlCtx) { var ( username = "user1" rolename = "role1" ) // create user user, err := json.Marshal(&pb.AuthUserAddRequest{Name: username, Password: "123"}) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/user/add", Value: string(user), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthUserGrantRevokeRoles failed to add user %v", username) // create role role, err := json.Marshal(&pb.AuthRoleAddRequest{Name: rolename}) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/role/add", Value: string(role), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthUserGrantRevokeRoles failed to add role %v", rolename) // grant role to user grantRoleReq, err := json.Marshal(&pb.AuthUserGrantRoleRequest{ User: username, Role: rolename, }) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/user/grant", Value: string(grantRoleReq), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthUserGrantRevokeRoles failed to grant role to user") // revoke role from user revokeRoleReq, err := json.Marshal(&pb.AuthUserRevokeRoleRequest{ Name: username, Role: rolename, }) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/user/revoke", Value: string(revokeRoleReq), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthUserGrantRevokeRoles failed to revoke role from user") } func testCurlV3AuthRoleBasicOperations(cx ctlCtx) { rolenames := []string{"role1", "role2", "role3"} // create roles for i := 0; i < len(rolenames); i++ { role, err := json.Marshal(&pb.AuthRoleAddRequest{Name: rolenames[i]}) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/role/add", Value: string(role), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthRoleBasicOperations failed to add role %v", rolenames[i]) } // get roles rolenames = []string{"role1", "roleX"} expectedResponse := []string{"revision", "etcdserver: role name not found"} for i := 0; i < len(rolenames); i++ { role, err := json.Marshal(&pb.AuthRoleGetRequest{ Role: rolenames[i], }) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/role/get", Value: string(role), Expected: expect.ExpectedResponse{Value: expectedResponse[i]}, }), "testCurlV3AuthRoleBasicOperations failed to get role %v", rolenames[i]) } // delete roles rolenames = []string{"role2", "roleX"} expectedResponse = []string{"revision", "etcdserver: role name not found"} for i := 0; i < len(rolenames); i++ { role, err := json.Marshal(&pb.AuthRoleDeleteRequest{ Role: rolenames[i], }) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/role/delete", Value: string(role), Expected: expect.ExpectedResponse{Value: expectedResponse[i]}, }), "testCurlV3AuthRoleBasicOperations failed to delete role %v", rolenames[i]) } // list roles clus := cx.epc args := e2e.CURLPrefixArgsCluster(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "POST", e2e.CURLReq{ Endpoint: "/v3/auth/role/list", Value: "{}", }) resp, err := runCommandAndReadJSONOutput(args) require.NoError(cx.t, err) roles, ok := resp["roles"] require.True(cx.t, ok) roleSlice := roles.([]any) require.Len(cx.t, roleSlice, 2) require.Equal(cx.t, "role1", roleSlice[0]) require.Equal(cx.t, "role3", roleSlice[1]) } func testCurlV3AuthRoleManagePermission(cx ctlCtx) { rolename := "role1" // create a role role, err := json.Marshal(&pb.AuthRoleAddRequest{Name: rolename}) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/role/add", Value: string(role), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthRoleManagePermission failed to add role %v", rolename) // grant permission grantPermissionReq, err := json.Marshal(&pb.AuthRoleGrantPermissionRequest{ Name: rolename, Perm: &authpb.Permission{ PermType: authpb.Permission_READ, Key: []byte("fakeKey"), }, }) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/role/grant", Value: string(grantPermissionReq), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthRoleManagePermission failed to grant permission to role %v", rolename) // revoke permission revokePermissionReq, err := json.Marshal(&pb.AuthRoleRevokePermissionRequest{ Role: rolename, Key: []byte("fakeKey"), }) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/role/revoke", Value: string(revokePermissionReq), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthRoleManagePermission failed to revoke permission from role %v", rolename) } func testCurlV3AuthEnableDisableStatus(cx ctlCtx) { // enable auth require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/enable", Value: "{}", Expected: expect.ExpectedResponse{Value: "etcdserver: root user does not exist"}, }), "testCurlV3AuthEnableDisableStatus failed to enable auth") // disable auth require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/disable", Value: "{}", Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthEnableDisableStatus failed to disable auth") // auth status require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/auth/status", Value: "{}", Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3AuthEnableDisableStatus failed to get auth status") } ================================================ FILE: tests/e2e/v3_curl_cluster_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "encoding/json" "fmt" "strconv" "testing" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCurlV3ClusterOperations(t *testing.T) { testCtl(t, testCurlV3ClusterOperations, withCfg(*e2e.NewConfig(e2e.WithClusterSize(1)))) } func testCurlV3ClusterOperations(cx ctlCtx) { var ( peerURL = "http://127.0.0.1:22380" updatedPeerURL = "http://127.0.0.1:32380" ) // add member cx.t.Logf("Adding member %q", peerURL) addMemberReq, err := json.Marshal(&pb.MemberAddRequest{PeerURLs: []string{peerURL}, IsLearner: true}) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/cluster/member/add", Value: string(addMemberReq), Expected: expect.ExpectedResponse{Value: peerURL}, }), "testCurlV3ClusterOperations failed to add member") // list members and get the new member's ID cx.t.Log("Listing members after adding a member") members := mustListMembers(cx) require.Len(cx.t, members, 2) cx.t.Logf("members: %+v", members) var newMemberIDStr string for _, m := range members { mObj := m.(map[string]any) pURL := mObj["peerURLs"].([]any)[0].(string) if pURL == peerURL { newMemberIDStr = mObj["ID"].(string) break } } require.Positive(cx.t, newMemberIDStr) // update member cx.t.Logf("Update peerURL from %q to %q for member %q", peerURL, updatedPeerURL, newMemberIDStr) newMemberID, err := strconv.ParseUint(newMemberIDStr, 10, 64) require.NoError(cx.t, err) updateMemberReq, err := json.Marshal(&pb.MemberUpdateRequest{ID: newMemberID, PeerURLs: []string{updatedPeerURL}}) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/cluster/member/update", Value: string(updateMemberReq), Expected: expect.ExpectedResponse{Value: updatedPeerURL}, }), "testCurlV3ClusterOperations failed to update member") // promote member cx.t.Logf("Promoting the member %d", newMemberID) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/cluster/member/promote", Value: fmt.Sprintf(`{"ID": %d}`, newMemberID), Expected: expect.ExpectedResponse{Value: "etcdserver: can only promote a learner member which is in sync with leader"}, }), "testCurlV3ClusterOperations failed to promote member") // remove member cx.t.Logf("Removing the member %d", newMemberID) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/cluster/member/remove", Value: fmt.Sprintf(`{"ID": %d}`, newMemberID), Expected: expect.ExpectedResponse{Value: "members"}, }), "testCurlV3ClusterOperations failed to remove member") // list members again after deleting a member cx.t.Log("Listing members again after deleting a member") members = mustListMembers(cx) require.Len(cx.t, members, 1) } func mustListMembers(cx ctlCtx) []any { clus := cx.epc args := e2e.CURLPrefixArgsCluster(clus.Cfg, clus.Procs[0], "POST", e2e.CURLReq{ Endpoint: "/v3/cluster/member/list", Value: "{}", }) resp, err := runCommandAndReadJSONOutput(args) require.NoError(cx.t, err) members, ok := resp["members"] require.True(cx.t, ok) return members.([]any) } ================================================ FILE: tests/e2e/v3_curl_election_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "encoding/base64" "encoding/json" "fmt" "math/rand" "strconv" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/pkg/v3/expect" epb "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCurlV3CampaignNoTLS(t *testing.T) { testCtl(t, testCurlV3Campaign, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3Campaign(cx ctlCtx) { // campaign cdata, err := json.Marshal(&epb.CampaignRequest{ Name: []byte("/election-prefix"), Value: []byte("v1"), }) require.NoError(cx.t, err) cargs := e2e.CURLPrefixArgsCluster(cx.epc.Cfg, cx.epc.Procs[rand.Intn(cx.epc.Cfg.ClusterSize)], "POST", e2e.CURLReq{ Endpoint: "/v3/election/campaign", Value: string(cdata), }) lines, err := e2e.SpawnWithExpectLines(context.TODO(), cargs, cx.envMap, expect.ExpectedResponse{Value: `"leader":{"name":"`}) require.NoErrorf(cx.t, err, "failed post campaign request") if len(lines) != 1 { cx.t.Fatalf("len(lines) expected 1, got %+v", lines) } var cresp campaignResponse require.NoErrorf(cx.t, json.Unmarshal([]byte(lines[0]), &cresp), "failed to unmarshal campaign response") ndata, err := base64.StdEncoding.DecodeString(cresp.Leader.Name) require.NoErrorf(cx.t, err, "failed to decode leader key") kdata, err := base64.StdEncoding.DecodeString(cresp.Leader.Key) require.NoErrorf(cx.t, err, "failed to decode leader key") // observe observeReq, err := json.Marshal(&epb.LeaderRequest{ Name: []byte("/election-prefix"), }) require.NoError(cx.t, err) clus := cx.epc args := e2e.CURLPrefixArgsCluster(clus.Cfg, clus.Procs[0], "POST", e2e.CURLReq{ Endpoint: "/v3/election/observe", Value: string(observeReq), }) proc, err := e2e.SpawnCmd(args, nil) require.NoError(cx.t, err) proc.ExpectWithContext(context.TODO(), expect.ExpectedResponse{ Value: fmt.Sprintf(`"key":"%s"`, cresp.Leader.Key), }) require.NoError(cx.t, proc.Stop()) // proclaim rev, _ := strconv.ParseInt(cresp.Leader.Rev, 10, 64) lease, _ := strconv.ParseInt(cresp.Leader.Lease, 10, 64) pdata, err := json.Marshal(&epb.ProclaimRequest{ Leader: &epb.LeaderKey{ Name: ndata, Key: kdata, Rev: rev, Lease: lease, }, Value: []byte("v2"), }) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/election/proclaim", Value: string(pdata), Expected: expect.ExpectedResponse{Value: `"revision":`}, }), "failed post proclaim request") } func TestCurlV3ProclaimMissiongLeaderKeyNoTLS(t *testing.T) { testCtl(t, testCurlV3ProclaimMissiongLeaderKey, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3ProclaimMissiongLeaderKey(cx ctlCtx) { pdata, err := json.Marshal(&epb.ProclaimRequest{Value: []byte("v2")}) require.NoError(cx.t, err) require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/election/proclaim", Value: string(pdata), Expected: expect.ExpectedResponse{Value: `"message":"\"leader\" field must be provided"`}, }), "failed post proclaim request") } func TestCurlV3ResignMissiongLeaderKeyNoTLS(t *testing.T) { testCtl(t, testCurlV3ResignMissiongLeaderKey, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3ResignMissiongLeaderKey(cx ctlCtx) { require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/election/resign", Value: `{}`, Expected: expect.ExpectedResponse{Value: `"message":"\"leader\" field must be provided"`}, }), "failed post resign request") } func TestCurlV3ElectionLeader(t *testing.T) { testCtl(t, testCurlV3ElectionLeader, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3ElectionLeader(cx ctlCtx) { require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/election/leader", Value: `{"name": "aGVsbG8="}`, // base64 encoded string "hello" Expected: expect.ExpectedResponse{Value: `election: no leader`}, }), "testCurlV3ElectionLeader failed to get leader") } // to manually decode; JSON marshals integer fields with // string types, so can't unmarshal with epb.CampaignResponse type campaignResponse struct { Leader struct { Name string `json:"name,omitempty"` Key string `json:"key,omitempty"` Rev string `json:"rev,omitempty"` Lease string `json:"lease,omitempty"` } `json:"leader,omitempty"` } func CURLWithExpected(cx ctlCtx, tests []v3cURLTest) error { for _, t := range tests { value := fmt.Sprintf("%v", t.value) if err := e2e.CURLPost(cx.epc, e2e.CURLReq{Endpoint: t.endpoint, Value: value, Expected: expect.ExpectedResponse{Value: t.expected}}); err != nil { return fmt.Errorf("endpoint (%s): error (%w), wanted %v", t.endpoint, err, t.expected) } } return nil } ================================================ FILE: tests/e2e/v3_curl_kv_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "encoding/json" "testing" protov1 "github.com/golang/protobuf/proto" //nolint:staticcheck // TODO: remove for a supported version gw "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCurlV3KVBasicOperation(t *testing.T) { testCurlV3KV(t, testCurlV3KVBasicOperation) } func TestCurlV3KVTxn(t *testing.T) { testCurlV3KV(t, testCurlV3KVTxn) } func TestCurlV3KVCompact(t *testing.T) { testCurlV3KV(t, testCurlV3KVCompact) } func testCurlV3KV(t *testing.T, f func(ctlCtx)) { testCases := []struct { name string cfg ctlOption }{ { name: "noTLS", cfg: withCfg(*e2e.NewConfigNoTLS()), }, { name: "autoTLS", cfg: withCfg(*e2e.NewConfigAutoTLS()), }, { name: "allTLS", cfg: withCfg(*e2e.NewConfigTLS()), }, { name: "peerTLS", cfg: withCfg(*e2e.NewConfigPeerTLS()), }, { name: "clientTLS", cfg: withCfg(*e2e.NewConfigClientTLS()), }, } for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { testCtl(t, f, tc.cfg) }) } } func testCurlV3KVBasicOperation(cx ctlCtx) { var ( key = []byte("foo") value = []byte("bar") // this will be automatically base64-encoded by Go expectedPutResponse = `"revision":"` expectedGetResponse = `"value":"` expectedDeleteResponse = `"deleted":"1"` ) putData, err := json.Marshal(&pb.PutRequest{ Key: key, Value: value, }) require.NoError(cx.t, err) rangeData, err := json.Marshal(&pb.RangeRequest{ Key: key, }) require.NoError(cx.t, err) deleteData, err := json.Marshal(&pb.DeleteRangeRequest{ Key: key, }) require.NoError(cx.t, err) err = e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/kv/put", Value: string(putData), Expected: expect.ExpectedResponse{Value: expectedPutResponse}, }) require.NoErrorf(cx.t, err, "testCurlV3KVBasicOperation put failed") err = e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/kv/range", Value: string(rangeData), Expected: expect.ExpectedResponse{Value: expectedGetResponse}, }) require.NoErrorf(cx.t, err, "testCurlV3KVBasicOperation get failed") if cx.cfg.Client.ConnectionType == e2e.ClientTLSAndNonTLS { err = e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/kv/range", Value: string(rangeData), Expected: expect.ExpectedResponse{Value: expectedGetResponse}, IsTLS: true, }) require.NoErrorf(cx.t, err, "testCurlV3KVBasicOperation get failed") } err = e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/kv/deleterange", Value: string(deleteData), Expected: expect.ExpectedResponse{Value: expectedDeleteResponse}, }) require.NoErrorf(cx.t, err, "testCurlV3KVBasicOperation delete failed") } func testCurlV3KVTxn(cx ctlCtx) { txn := &pb.TxnRequest{ Compare: []*pb.Compare{ { Key: []byte("foo"), Result: pb.Compare_EQUAL, Target: pb.Compare_CREATE, TargetUnion: &pb.Compare_CreateRevision{CreateRevision: 0}, }, }, Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Key: []byte("foo"), Value: []byte("bar"), }, }, }, }, } m := gw.JSONPb{ MarshalOptions: protojson.MarshalOptions{ UseProtoNames: true, EmitUnpopulated: false, }, } jsonDat, jerr := m.Marshal(protov1.MessageV2(txn)) require.NoError(cx.t, jerr) succeeded, responses := mustExecuteTxn(cx, string(jsonDat)) require.True(cx.t, succeeded) require.Len(cx.t, responses, 1) putResponse := responses[0].(map[string]any) _, ok := putResponse["response_put"] require.True(cx.t, ok) // was crashing etcd server malformed := `{"compare":[{"result":0,"target":1,"key":"Zm9v","TargetUnion":null}],"success":[{"Request":{"RequestPut":{"key":"Zm9v","value":"YmFy"}}}]}` err := e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/kv/txn", Value: malformed, Expected: expect.ExpectedResponse{Value: "etcdserver: key not found"}, }) require.NoErrorf(cx.t, err, "testCurlV3Txn with malformed request failed") } func mustExecuteTxn(cx ctlCtx, reqData string) (bool, []any) { clus := cx.epc args := e2e.CURLPrefixArgsCluster(clus.Cfg, clus.Procs[0], "POST", e2e.CURLReq{ Endpoint: "/v3/kv/txn", Value: reqData, }) resp, err := runCommandAndReadJSONOutput(args) require.NoError(cx.t, err) succeeded, ok := resp["succeeded"] require.True(cx.t, ok) responses, ok := resp["responses"] require.True(cx.t, ok) return succeeded.(bool), responses.([]any) } func testCurlV3KVCompact(cx ctlCtx) { compactRequest, err := json.Marshal(&pb.CompactionRequest{ Revision: 10000, }) require.NoError(cx.t, err) err = e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/kv/compaction", Value: string(compactRequest), Expected: expect.ExpectedResponse{ Value: `"message":"etcdserver: mvcc: required revision is a future revision"`, }, }) require.NoErrorf(cx.t, err, "testCurlV3KVCompact failed") } ================================================ FILE: tests/e2e/v3_curl_lease_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "testing" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCurlV3LeaseGrantNoTLS(t *testing.T) { testCtl(t, testCurlV3LeaseGrant, withCfg(*e2e.NewConfigNoTLS())) } func TestCurlV3LeaseRevokeNoTLS(t *testing.T) { testCtl(t, testCurlV3LeaseRevoke, withCfg(*e2e.NewConfigNoTLS())) } func TestCurlV3LeaseLeasesNoTLS(t *testing.T) { testCtl(t, testCurlV3LeaseLeases, withCfg(*e2e.NewConfigNoTLS())) } func TestCurlV3LeaseKeepAliveNoTLS(t *testing.T) { testCtl(t, testCurlV3LeaseKeepAlive, withCfg(*e2e.NewConfigNoTLS())) } type v3cURLTest struct { endpoint string value string expected string } func testCurlV3LeaseGrant(cx ctlCtx) { leaseID := e2e.RandomLeaseID() tests := []v3cURLTest{ { endpoint: "/v3/lease/grant", value: gwLeaseGrant(cx, leaseID, 0), expected: gwLeaseIDExpected(leaseID), }, { endpoint: "/v3/lease/grant", value: gwLeaseGrant(cx, 0, 20), expected: `"TTL":"20"`, }, { endpoint: "/v3/kv/put", value: gwKVPutLease(cx, "foo", "bar", leaseID), expected: `"revision":"`, }, { endpoint: "/v3/lease/timetolive", value: gwLeaseTTLWithKeys(cx, leaseID), expected: `"grantedTTL"`, }, } require.NoErrorf(cx.t, CURLWithExpected(cx, tests), "testCurlV3LeaseGrant") } func testCurlV3LeaseRevoke(cx ctlCtx) { leaseID := e2e.RandomLeaseID() tests := []v3cURLTest{ { endpoint: "/v3/lease/grant", value: gwLeaseGrant(cx, leaseID, 0), expected: gwLeaseIDExpected(leaseID), }, { endpoint: "/v3/lease/revoke", value: gwLeaseRevoke(cx, leaseID), expected: `"revision":"`, }, } require.NoErrorf(cx.t, CURLWithExpected(cx, tests), "testCurlV3LeaseRevoke") } func testCurlV3LeaseLeases(cx ctlCtx) { leaseID := e2e.RandomLeaseID() tests := []v3cURLTest{ { endpoint: "/v3/lease/grant", value: gwLeaseGrant(cx, leaseID, 0), expected: gwLeaseIDExpected(leaseID), }, { endpoint: "/v3/lease/leases", value: "{}", expected: gwLeaseIDExpected(leaseID), }, } require.NoErrorf(cx.t, CURLWithExpected(cx, tests), "testCurlV3LeaseGrant") } func testCurlV3LeaseKeepAlive(cx ctlCtx) { leaseID := e2e.RandomLeaseID() tests := []v3cURLTest{ { endpoint: "/v3/lease/grant", value: gwLeaseGrant(cx, leaseID, 0), expected: gwLeaseIDExpected(leaseID), }, { endpoint: "/v3/lease/keepalive", value: gwLeaseKeepAlive(cx, leaseID), expected: gwLeaseIDExpected(leaseID), }, } require.NoErrorf(cx.t, CURLWithExpected(cx, tests), "testCurlV3LeaseGrant") } func gwLeaseIDExpected(leaseID int64) string { return fmt.Sprintf(`"ID":"%d"`, leaseID) } func gwLeaseTTLWithKeys(cx ctlCtx, leaseID int64) string { d := &pb.LeaseTimeToLiveRequest{ID: leaseID, Keys: true} s, err := e2e.DataMarshal(d) require.NoErrorf(cx.t, err, "gwLeaseTTLWithKeys: error") return s } func gwLeaseKeepAlive(cx ctlCtx, leaseID int64) string { d := &pb.LeaseKeepAliveRequest{ID: leaseID} s, err := e2e.DataMarshal(d) require.NoErrorf(cx.t, err, "gwLeaseKeepAlive: Marshal error") return s } func gwLeaseGrant(cx ctlCtx, leaseID int64, ttl int64) string { d := &pb.LeaseGrantRequest{ID: leaseID, TTL: ttl} s, err := e2e.DataMarshal(d) require.NoErrorf(cx.t, err, "gwLeaseGrant: Marshal error") return s } func gwLeaseRevoke(cx ctlCtx, leaseID int64) string { d := &pb.LeaseRevokeRequest{ID: leaseID} s, err := e2e.DataMarshal(d) require.NoErrorf(cx.t, err, "gwLeaseRevoke: Marshal error") return s } func gwKVPutLease(cx ctlCtx, k string, v string, leaseID int64) string { d := pb.PutRequest{Key: []byte(k), Value: []byte(v), Lease: leaseID} s, err := e2e.DataMarshal(d) require.NoErrorf(cx.t, err, "gwKVPutLease: Marshal error") return s } ================================================ FILE: tests/e2e/v3_curl_lock_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "encoding/json" "fmt" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCurlV3LockOperations(t *testing.T) { testCtl(t, testCurlV3LockOperations, withCfg(*e2e.NewConfig(e2e.WithClusterSize(1)))) } func testCurlV3LockOperations(cx ctlCtx) { // lock lockReq, err := json.Marshal(&v3lockpb.LockRequest{Name: []byte("lock1")}) require.NoError(cx.t, err) clus := cx.epc args := e2e.CURLPrefixArgsCluster(clus.Cfg, clus.Procs[0], "POST", e2e.CURLReq{ Endpoint: "/v3/lock/lock", Value: string(lockReq), }) resp, err := runCommandAndReadJSONOutput(args) require.NoError(cx.t, err) key, ok := resp["key"] require.True(cx.t, ok) // unlock require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/lock/unlock", Value: fmt.Sprintf(`{"key": "%v"}`, key), Expected: expect.ExpectedResponse{Value: "revision"}, }), "testCurlV3LockOperations failed to execute unlock") } ================================================ FILE: tests/e2e/v3_curl_maintenance_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "math/rand" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCurlV3MaintenanceAlarmMissiongAlarm(t *testing.T) { testCtl(t, testCurlV3MaintenanceAlarmMissiongAlarm, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3MaintenanceAlarmMissiongAlarm(cx ctlCtx) { require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/maintenance/alarm", Value: `{"action": "ACTIVATE"}`, }), "failed post maintenance alarm") } func TestCurlV3MaintenanceStatus(t *testing.T) { testCtl(t, testCurlV3MaintenanceStatus, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3MaintenanceStatus(cx ctlCtx) { clus := cx.epc args := e2e.CURLPrefixArgsCluster(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "POST", e2e.CURLReq{ Endpoint: "/v3/maintenance/status", Value: "{}", }) resp, err := runCommandAndReadJSONOutput(args) require.NoError(cx.t, err) requiredFields := []string{"version", "dbSize", "leader", "raftIndex", "raftTerm", "raftAppliedIndex", "dbSizeInUse", "storageVersion"} for _, field := range requiredFields { if _, ok := resp[field]; !ok { cx.t.Fatalf("Field %q not found in (%v)", field, resp) } } require.Equal(cx.t, version.Version, resp["version"]) } func TestCurlV3MaintenanceDefragment(t *testing.T) { testCtl(t, testCurlV3MaintenanceDefragment, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3MaintenanceDefragment(cx ctlCtx) { require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/maintenance/defragment", Value: "{}", Expected: expect.ExpectedResponse{ Value: "{}", }, }), "failed post maintenance defragment request") } func TestCurlV3MaintenanceHash(t *testing.T) { testCtl(t, testCurlV3MaintenanceHash, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3MaintenanceHash(cx ctlCtx) { clus := cx.epc args := e2e.CURLPrefixArgsCluster(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "POST", e2e.CURLReq{ Endpoint: "/v3/maintenance/hash", Value: "{}", }) resp, err := runCommandAndReadJSONOutput(args) require.NoError(cx.t, err) requiredFields := []string{"header", "hash"} for _, field := range requiredFields { if _, ok := resp[field]; !ok { cx.t.Fatalf("Field %q not found in (%v)", field, resp) } } } func TestCurlV3MaintenanceHashKV(t *testing.T) { testCtl(t, testCurlV3MaintenanceHashKV, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3MaintenanceHashKV(cx ctlCtx) { clus := cx.epc args := e2e.CURLPrefixArgsCluster(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "POST", e2e.CURLReq{ Endpoint: "/v3/maintenance/hashkv", Value: "{}", }) resp, err := runCommandAndReadJSONOutput(args) require.NoError(cx.t, err) requiredFields := []string{"header", "hash", "compact_revision", "hash_revision"} for _, field := range requiredFields { if _, ok := resp[field]; !ok { cx.t.Fatalf("Field %q not found in (%v)", field, resp) } } } func TestCurlV3MaintenanceSnapshot(t *testing.T) { testCtl(t, testCurlV3MaintenanceSnapshot, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3MaintenanceSnapshot(cx ctlCtx) { require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/maintenance/snapshot", Value: "{}", Expected: expect.ExpectedResponse{ Value: `"result":{"blob":`, }, }), "failed post maintenance snapshot request") } func TestCurlV3MaintenanceMoveleader(t *testing.T) { testCtl(t, testCurlV3MaintenanceMoveleader, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3MaintenanceMoveleader(cx ctlCtx) { require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/maintenance/transfer-leadership", Value: `{"targetID": 123}`, Expected: expect.ExpectedResponse{ Value: `"message":"etcdserver: bad leader transferee"`, }, }), "failed post maintenance moveleader request") } func TestCurlV3MaintenanceDowngrade(t *testing.T) { testCtl(t, testCurlV3MaintenanceDowngrade, withCfg(*e2e.NewConfigNoTLS())) } func testCurlV3MaintenanceDowngrade(cx ctlCtx) { require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{ Endpoint: "/v3/maintenance/downgrade", Value: `{"action": 0, "version": "3.0"}`, Expected: expect.ExpectedResponse{ Value: `"message":"etcdserver: invalid downgrade target version"`, }, }), "failed post maintenance downgrade request") } ================================================ FILE: tests/e2e/v3_curl_maxstream_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "encoding/json" "fmt" "math/rand" "sync" "syscall" "testing" "time" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/testutils" ) // TestCurlV3_MaxStreams_BelowLimit_NoTLS_Small tests no TLS func TestCurlV3_MaxStreams_BelowLimit_NoTLS_Small(t *testing.T) { testCurlV3MaxStream(t, false, withCfg(*e2e.NewConfigNoTLS()), withMaxConcurrentStreams(3)) } func TestCurlV3_MaxStreams_BelowLimit_NoTLS_Medium(t *testing.T) { testCurlV3MaxStream(t, false, withCfg(*e2e.NewConfigNoTLS()), withMaxConcurrentStreams(100), withTestTimeout(20*time.Second)) } func TestCurlV3_MaxStreamsNoTLS_BelowLimit_Large(t *testing.T) { f, err := setRLimit(10240) require.NoError(t, err) defer f() testCurlV3MaxStream(t, false, withCfg(*e2e.NewConfigNoTLS()), withMaxConcurrentStreams(1000), withTestTimeout(200*time.Second)) } func TestCurlV3_MaxStreams_ReachLimit_NoTLS_Small(t *testing.T) { testCurlV3MaxStream(t, true, withCfg(*e2e.NewConfigNoTLS()), withMaxConcurrentStreams(3)) } func TestCurlV3_MaxStreams_ReachLimit_NoTLS_Medium(t *testing.T) { testCurlV3MaxStream(t, true, withCfg(*e2e.NewConfigNoTLS()), withMaxConcurrentStreams(100), withTestTimeout(20*time.Second)) } // TestCurlV3_MaxStreams_BelowLimit_TLS_Small tests with TLS func TestCurlV3_MaxStreams_BelowLimit_TLS_Small(t *testing.T) { testCurlV3MaxStream(t, false, withCfg(*e2e.NewConfigTLS()), withMaxConcurrentStreams(3)) } func TestCurlV3_MaxStreams_BelowLimit_TLS_Medium(t *testing.T) { testCurlV3MaxStream(t, false, withCfg(*e2e.NewConfigTLS()), withMaxConcurrentStreams(100), withTestTimeout(20*time.Second)) } func TestCurlV3_MaxStreams_ReachLimit_TLS_Small(t *testing.T) { testCurlV3MaxStream(t, true, withCfg(*e2e.NewConfigTLS()), withMaxConcurrentStreams(3)) } func TestCurlV3_MaxStreams_ReachLimit_TLS_Medium(t *testing.T) { testCurlV3MaxStream(t, true, withCfg(*e2e.NewConfigTLS()), withMaxConcurrentStreams(100), withTestTimeout(20*time.Second)) } func testCurlV3MaxStream(t *testing.T, reachLimit bool, opts ...ctlOption) { e2e.BeforeTest(t) // Step 1: generate configuration for creating cluster t.Log("Generating configuration for creating cluster.") cx := getDefaultCtlCtx(t) cx.applyOpts(opts) // We must set the `ClusterSize` to 1, otherwise different streams may // connect to different members, accordingly it's difficult to test the // behavior. cx.cfg.ClusterSize = 1 // Step 2: create the cluster t.Log("Creating an etcd cluster") epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(&cx.cfg)) require.NoErrorf(t, err, "Failed to start etcd cluster") cx.epc = epc cx.dataDir = epc.Procs[0].Config().DataDirPath // Step 3: run test // (a) generate ${concurrentNumber} concurrent watch streams; // (b) submit a range request. var wg sync.WaitGroup concurrentNumber := cx.cfg.ServerConfig.MaxConcurrentStreams - 1 expectedResponse := `"revision":"` if reachLimit { concurrentNumber = cx.cfg.ServerConfig.MaxConcurrentStreams expectedResponse = "Operation timed out" } wg.Add(int(concurrentNumber)) t.Logf("Running the test, MaxConcurrentStreams: %d, concurrentNumber: %d, expected range's response: %s\n", cx.cfg.ServerConfig.MaxConcurrentStreams, concurrentNumber, expectedResponse) closeServerCh := make(chan struct{}) submitConcurrentWatch(cx, int(concurrentNumber), &wg, closeServerCh) submitRangeAfterConcurrentWatch(cx, expectedResponse) // Step 4: Close the cluster t.Log("Closing test cluster...") close(closeServerCh) require.NoError(t, epc.Close()) t.Log("Closed test cluster") // Step 5: Waiting all watch goroutines to exit. doneCh := make(chan struct{}) go func() { defer close(doneCh) wg.Wait() }() timeout := cx.getTestTimeout() t.Logf("Waiting test case to finish, timeout: %s", timeout) select { case <-time.After(timeout): testutil.FatalStack(t, fmt.Sprintf("test timed out after %v", timeout)) case <-doneCh: t.Log("All watch goroutines exited.") } t.Log("testCurlV3MaxStream done!") } func submitConcurrentWatch(cx ctlCtx, number int, wgDone *sync.WaitGroup, closeCh chan struct{}) { watchData, err := json.Marshal(&pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), }, }) require.NoError(cx.t, err) var wgSchedule sync.WaitGroup createWatchConnection := func() error { cluster := cx.epc member := cluster.Procs[rand.Intn(cluster.Cfg.ClusterSize)] curlReq := e2e.CURLReq{Endpoint: "/v3/watch", Value: string(watchData)} args := e2e.CURLPrefixArgsCluster(cluster.Cfg, member, "POST", curlReq) proc, err := e2e.SpawnCmd(args, nil) if err != nil { return fmt.Errorf("failed to spawn: %w", err) } defer proc.Stop() // make sure that watch request has been created expectedLine := `"created":true}}` _, lerr := proc.ExpectWithContext(context.TODO(), expect.ExpectedResponse{Value: expectedLine}) if lerr != nil { return fmt.Errorf("%v %w (expected %q). Try EXPECT_DEBUG=TRUE", args, lerr, expectedLine) } wgSchedule.Done() // hold the connection and wait for server shutdown perr := proc.Close() // curl process will return select { case <-closeCh: default: // perr could be nil. return fmt.Errorf("unexpected connection close before server closes: %w", perr) } return nil } testutils.ExecuteWithTimeout(cx.t, cx.getTestTimeout(), func() { wgSchedule.Add(number) for i := 0; i < number; i++ { go func(i int) { defer wgDone.Done() require.NoErrorf(cx.t, createWatchConnection(), "testCurlV3MaxStream watch failed: %d", i) }(i) } // make sure all goroutines have already been scheduled. wgSchedule.Wait() }) } func submitRangeAfterConcurrentWatch(cx ctlCtx, expectedValue string) { rangeData, err := json.Marshal(&pb.RangeRequest{ Key: []byte("foo"), }) require.NoError(cx.t, err) cx.t.Log("Submitting range request...") if err := e2e.CURLPost(cx.epc, e2e.CURLReq{Endpoint: "/v3/kv/range", Value: string(rangeData), Expected: expect.ExpectedResponse{Value: expectedValue}, Timeout: 5}); err != nil { require.ErrorContains(cx.t, err, expectedValue) } cx.t.Log("range request done") } // setRLimit sets the open file limitation, and return a function which // is used to reset the limitation. func setRLimit(nofile uint64) (func() error, error) { var rLimit syscall.Rlimit if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil { return nil, fmt.Errorf("failed to get open file limit, error: %w", err) } var wLimit syscall.Rlimit wLimit.Max = nofile wLimit.Cur = nofile if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &wLimit); err != nil { return nil, fmt.Errorf("failed to set max open file limit, %w", err) } return func() error { if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil { return fmt.Errorf("failed reset max open file limit, %w", err) } return nil }, nil } ================================================ FILE: tests/e2e/v3_curl_watch_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "encoding/json" "syscall" "testing" "time" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestCurlV3Watch(t *testing.T) { testCtl(t, testCurlV3Watch) } func testCurlV3Watch(cx ctlCtx) { // store "bar" into "foo" putreq, err := json.Marshal(&pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}) require.NoError(cx.t, err) // watch for first update to "foo" wcr := &pb.WatchCreateRequest{Key: []byte("foo"), StartRevision: 1} wreq, err := json.Marshal(wcr) require.NoError(cx.t, err) // marshaling the grpc to json gives: // "{"RequestUnion":{"CreateRequest":{"key":"Zm9v","start_revision":1}}}" // but the gprc-gateway expects a different format.. wstr := `{"create_request" : ` + string(wreq) + "}" require.NoErrorf(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{Endpoint: "/v3/kv/put", Value: string(putreq), Expected: expect.ExpectedResponse{Value: "revision"}}), "failed testCurlV3Watch put with curl") // expects "bar", timeout after 2 seconds since stream waits forever require.ErrorContains(cx.t, e2e.CURLPost(cx.epc, e2e.CURLReq{Endpoint: "/v3/watch", Value: wstr, Expected: expect.ExpectedResponse{Value: `"YmFy"`}, Timeout: 2}), "unexpected exit code") } // TestCurlWatchIssue19509 tries to reproduce https://github.com/etcd-io/etcd/issues/19509 func TestCurlWatchIssue19509(t *testing.T) { e2e.BeforeTest(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithConfig(e2e.NewConfigClientTLS()), e2e.WithClusterSize(1)) require.NoError(t, err) defer epc.Close() curlCmdAndArgs := e2e.CURLPrefixArgsCluster(epc.Cfg, epc.Procs[0], "POST", e2e.CURLReq{Endpoint: "/v3/watch", Timeout: 3}) for i := 0; i < 10; i++ { curlProc, err := e2e.SpawnCmd(curlCmdAndArgs, nil) require.NoError(t, err) time.Sleep(100 * time.Millisecond) _ = curlProc.Signal(syscall.SIGKILL) _ = curlProc.Close() require.Truef(t, epc.Procs[0].IsRunning(), "etcdserver already exited after %d curl watch requests", i+1) } } ================================================ FILE: tests/e2e/v3_lease_no_proxy_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/testutils" ) // TestLeaseRevoke_IgnoreOldLeader verifies that leases shouldn't be revoked // by old leader. // See the case 1 in https://github.com/etcd-io/etcd/issues/15247#issuecomment-1777862093. func TestLeaseRevoke_IgnoreOldLeader(t *testing.T) { t.Run("3 members", func(t *testing.T) { testLeaseRevokeIssue(t, 3, true) }) t.Run("5 members", func(t *testing.T) { testLeaseRevokeIssue(t, 5, true) }) } // TestLeaseRevoke_ClientSwitchToOtherMember verifies that leases shouldn't // be revoked by new leader. // See the case 2 in https://github.com/etcd-io/etcd/issues/15247#issuecomment-1777862093. func TestLeaseRevoke_ClientSwitchToOtherMember(t *testing.T) { t.Run("3 members", func(t *testing.T) { testLeaseRevokeIssue(t, 3, false) }) t.Run("5 members", func(t *testing.T) { testLeaseRevokeIssue(t, 5, false) }) } func testLeaseRevokeIssue(t *testing.T, clusterSize int, connectToOneFollower bool) { e2e.BeforeTest(t) ctx := t.Context() t.Log("Starting a new etcd cluster") epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(clusterSize), e2e.WithGoFailEnabled(true), e2e.WithGoFailClientTimeout(40*time.Second), ) require.NoError(t, err) defer func() { require.NoErrorf(t, epc.Close(), "error closing etcd processes") }() leaderIdx := epc.WaitLeader(t) t.Logf("Leader index: %d", leaderIdx) epsForNormalOperations := epc.Procs[(leaderIdx+2)%clusterSize].EndpointsGRPC() t.Logf("Creating a client for normal operations: %v", epsForNormalOperations) client, err := clientv3.New(clientv3.Config{Endpoints: epsForNormalOperations, DialTimeout: 3 * time.Second}) require.NoError(t, err) defer client.Close() var epsForLeaseKeepAlive []string if connectToOneFollower { epsForLeaseKeepAlive = epc.Procs[(leaderIdx+1)%clusterSize].EndpointsGRPC() } else { epsForLeaseKeepAlive = epc.EndpointsGRPC() } t.Logf("Creating a client for the leaseKeepAlive operation: %v", epsForLeaseKeepAlive) clientForKeepAlive, err := clientv3.New(clientv3.Config{Endpoints: epsForLeaseKeepAlive, DialTimeout: 3 * time.Second}) require.NoError(t, err) defer clientForKeepAlive.Close() resp, err := client.Status(ctx, epsForNormalOperations[0]) require.NoError(t, err) oldLeaderID := resp.Leader t.Log("Creating a new lease") leaseRsp, err := client.Grant(ctx, 20) require.NoError(t, err) t.Log("Starting a goroutine to keep alive the lease") doneC := make(chan struct{}) stopC := make(chan struct{}) startC := make(chan struct{}, 1) go func() { defer close(doneC) respC, kerr := clientForKeepAlive.KeepAlive(ctx, leaseRsp.ID) assert.NoError(t, kerr) // ensure we have received the first response from the server <-respC startC <- struct{}{} for { select { case <-stopC: return case <-respC: } } }() t.Log("Wait for the keepAlive goroutine to get started") <-startC t.Log("Trigger the failpoint to simulate stalled writing") err = epc.Procs[leaderIdx].Failpoints().SetupHTTP(ctx, "raftBeforeSave", `sleep("30s")`) require.NoError(t, err) cctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) t.Logf("Waiting for a new leader to be elected, old leader index: %d, old leader ID: %d", leaderIdx, oldLeaderID) testutils.ExecuteUntil(cctx, t, func() { for { resp, err = client.Status(ctx, epsForNormalOperations[0]) if err == nil && resp.Leader != oldLeaderID { t.Logf("A new leader has already been elected, new leader index: %d", resp.Leader) return } time.Sleep(100 * time.Millisecond) } }) cancel() t.Log("Writing a key/value pair") _, err = client.Put(ctx, "foo", "bar") require.NoError(t, err) t.Log("Sleeping 30 seconds") time.Sleep(30 * time.Second) t.Log("Remove the failpoint 'raftBeforeSave'") err = epc.Procs[leaderIdx].Failpoints().DeactivateHTTP(ctx, "raftBeforeSave") require.NoError(t, err) // By default, etcd tries to revoke leases every 7 seconds. t.Log("Sleeping 10 seconds") time.Sleep(10 * time.Second) t.Log("Confirming the lease isn't revoked") leases, err := client.Leases(ctx) require.NoError(t, err) require.Len(t, leases.Leases, 1) t.Log("Waiting for the keepAlive goroutine to exit") close(stopC) <-doneC } func TestLeaseRevokeDuringRenew(t *testing.T) { e2e.BeforeTest(t) ctx := t.Context() t.Log("Starting a new etcd cluster") epc, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), e2e.WithGoFailClientTimeout(40*time.Second), ) require.NoError(t, err) defer func() { require.NoErrorf(t, epc.Close(), "error closing etcd processes") }() eps := epc.Procs[0].EndpointsGRPC() t.Logf("Creating two clients for lease operations: %v", eps) clientForRenew, err := clientv3.New(clientv3.Config{Endpoints: eps, DialTimeout: 3 * time.Second}) require.NoError(t, err) defer clientForRenew.Close() clientForRevoke, err := clientv3.New(clientv3.Config{Endpoints: eps, DialTimeout: 3 * time.Second}) require.NoError(t, err) defer clientForRevoke.Close() t.Log("Creating a new lease") leaseRsp, err := clientForRenew.Grant(ctx, 60) require.NoError(t, err) t.Log("Activate the 'beforeCheckpointInLeaseRenew' failpoint") require.NoError(t, epc.Procs[0].Failpoints().SetupHTTP(ctx, "beforeCheckpointInLeaseRenew", `sleep("3s")`)) t.Logf("Starting a goroutine to keep alive the lease: %d", leaseRsp.ID) doneC := make(chan struct{}) startC := make(chan struct{}, 1) var renewError error go func() { defer close(doneC) startC <- struct{}{} _, renewError = clientForRenew.KeepAliveOnce(ctx, leaseRsp.ID) }() t.Log("Wait for the KeepAliveOnce goroutine to get started") <-startC time.Sleep(200 * time.Millisecond) t.Logf("Revoke the lease: %d", leaseRsp.ID) _, lerr := clientForRevoke.Revoke(ctx, leaseRsp.ID) require.NoError(t, lerr) t.Log("Waiting for the keepAlive goroutine to exit") <-doneC require.ErrorIs(t, rpctypes.ErrLeaseNotFound, renewError) } ================================================ FILE: tests/e2e/watch_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // These tests are performance sensitive, addition of cluster proxy makes them unstable. //go:build !cluster_proxy package e2e import ( "context" "fmt" "strings" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" v3rpc "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" ) const ( watchResponsePeriod = 100 * time.Millisecond watchTestDuration = 5 * time.Second readLoadConcurrency = 10 ) type testCase struct { name string client e2e.ClientConfig clientHTTPSeparate bool maxWatchDelay time.Duration dbSizeBytes int } const ( Kilo = 1000 Mega = 1000 * Kilo ) // 10 MB is not a bottleneck of grpc server, but filling up etcd with data. // Keeping it lower so tests don't take too long. // If we implement reuse of db we could increase the dbSize. var tcs = []testCase{ { name: "NoTLS", maxWatchDelay: 150 * time.Millisecond, dbSizeBytes: 5 * Mega, }, { name: "TLS", client: e2e.ClientConfig{ConnectionType: e2e.ClientTLS}, maxWatchDelay: 150 * time.Millisecond, dbSizeBytes: 5 * Mega, }, { name: "SeparateHTTPNoTLS", clientHTTPSeparate: true, maxWatchDelay: 150 * time.Millisecond, dbSizeBytes: 5 * Mega, }, { name: "SeparateHTTPTLS", client: e2e.ClientConfig{ConnectionType: e2e.ClientTLS}, clientHTTPSeparate: true, maxWatchDelay: 150 * time.Millisecond, dbSizeBytes: 5 * Mega, }, } func TestWatchDelayForPeriodicProgressNotification(t *testing.T) { e2e.BeforeTest(t) for _, tc := range tcs { tc := tc cfg := e2e.DefaultConfig() cfg.ClusterSize = 1 cfg.ServerConfig.WatchProgressNotifyInterval = watchResponsePeriod cfg.Client = tc.client cfg.ClientHTTPSeparate = tc.clientHTTPSeparate t.Run(tc.name, func(t *testing.T) { clus, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg)) require.NoError(t, err) defer clus.Close() c := newClient(t, clus.EndpointsGRPC(), tc.client) require.NoError(t, fillEtcdWithData(t.Context(), c, tc.dbSizeBytes)) ctx, cancel := context.WithTimeout(t.Context(), watchTestDuration) defer cancel() g := errgroup.Group{} continuouslyExecuteGetAll(ctx, t, &g, c) validateWatchDelay(t, c.Watch(ctx, "fake-key", clientv3.WithProgressNotify()), tc.maxWatchDelay) require.NoError(t, g.Wait()) }) } } func TestWatchDelayForManualProgressNotification(t *testing.T) { e2e.BeforeTest(t) for _, tc := range tcs { tc := tc cfg := e2e.DefaultConfig() cfg.ClusterSize = 1 cfg.Client = tc.client cfg.ClientHTTPSeparate = tc.clientHTTPSeparate t.Run(tc.name, func(t *testing.T) { clus, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg)) require.NoError(t, err) defer clus.Close() c := newClient(t, clus.EndpointsGRPC(), tc.client) require.NoError(t, fillEtcdWithData(t.Context(), c, tc.dbSizeBytes)) ctx, cancel := context.WithTimeout(t.Context(), watchTestDuration) defer cancel() g := errgroup.Group{} continuouslyExecuteGetAll(ctx, t, &g, c) g.Go(func() error { for { err := c.RequestProgress(ctx) if err != nil { if strings.Contains(err.Error(), "context deadline exceeded") { return nil } return err } time.Sleep(watchResponsePeriod) } }) validateWatchDelay(t, c.Watch(ctx, "fake-key"), tc.maxWatchDelay) require.NoError(t, g.Wait()) }) } } func TestWatchDelayForEvent(t *testing.T) { e2e.BeforeTest(t) for _, tc := range tcs { tc := tc cfg := e2e.DefaultConfig() cfg.ClusterSize = 1 cfg.Client = tc.client cfg.ClientHTTPSeparate = tc.clientHTTPSeparate t.Run(tc.name, func(t *testing.T) { clus, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg)) require.NoError(t, err) defer clus.Close() c := newClient(t, clus.EndpointsGRPC(), tc.client) require.NoError(t, fillEtcdWithData(t.Context(), c, tc.dbSizeBytes)) ctx, cancel := context.WithTimeout(t.Context(), watchTestDuration) defer cancel() g := errgroup.Group{} g.Go(func() error { i := 0 for { _, err := c.Put(ctx, "key", fmt.Sprintf("%d", i)) if err != nil { if strings.Contains(err.Error(), "context deadline exceeded") { return nil } return err } time.Sleep(watchResponsePeriod) } }) continuouslyExecuteGetAll(ctx, t, &g, c) validateWatchDelay(t, c.Watch(ctx, "key"), tc.maxWatchDelay) require.NoError(t, g.Wait()) }) } } func validateWatchDelay(t *testing.T, watch clientv3.WatchChan, maxWatchDelay time.Duration) { start := time.Now() var maxDelay time.Duration for range watch { sinceLast := time.Since(start) if sinceLast > watchResponsePeriod+maxWatchDelay { t.Errorf("Unexpected watch response delayed over allowed threshold %s, delay: %s", maxWatchDelay, sinceLast-watchResponsePeriod) } else { t.Logf("Got watch response, since last: %s", sinceLast) } if sinceLast > maxDelay { maxDelay = sinceLast } start = time.Now() } sinceLast := time.Since(start) if sinceLast > maxDelay && sinceLast > watchResponsePeriod+maxWatchDelay { t.Errorf("Unexpected watch response delayed over allowed threshold %s, delay: unknown", maxWatchDelay) t.Errorf("Test finished while in middle of delayed response, measured delay: %s", sinceLast-watchResponsePeriod) t.Logf("Please increase the test duration to measure delay") } else { t.Logf("Max delay: %s", maxDelay-watchResponsePeriod) } } func continuouslyExecuteGetAll(ctx context.Context, t *testing.T, g *errgroup.Group, c *clientv3.Client) { mux := sync.RWMutex{} size := 0 for i := 0; i < readLoadConcurrency; i++ { g.Go(func() error { for { resp, err := c.Get(ctx, "", clientv3.WithPrefix()) if err != nil { if strings.Contains(err.Error(), "context deadline exceeded") { return nil } return err } respSize := 0 for _, kv := range resp.Kvs { respSize += kv.Size() } mux.Lock() size += respSize mux.Unlock() } }) } g.Go(func() error { lastSize := size for range time.Tick(time.Second) { select { case <-ctx.Done(): return nil default: } mux.RLock() t.Logf("Generating read load around %.1f MB/s", float64(size-lastSize)/1000/1000) lastSize = size mux.RUnlock() } return nil }) } // TestDeleteEventDrop_Issue18089 is an e2e test to reproduce the issue reported in: https://github.com/etcd-io/etcd/issues/18089 // // The goal is to reproduce a DELETE event being dropped in a watch after a compaction // occurs on the revision where the deletion took place. In order to reproduce this, we // perform the following sequence (steps for reproduction thanks to @ahrtr): // - PUT k v2 (assume returned revision = r2) // - PUT k v3 (assume returned revision = r3) // - PUT k v4 (assume returned revision = r4) // - DELETE k (assume returned revision = r5) // - PUT k v6 (assume returned revision = r6) // - COMPACT r5 // - WATCH rev=r5 // // We should get the DELETE event (r5) followed by the PUT event (r6). func TestDeleteEventDrop_Issue18089(t *testing.T) { e2e.BeforeTest(t) cfg := e2e.DefaultConfig() cfg.ClusterSize = 1 cfg.Client = e2e.ClientConfig{ConnectionType: e2e.ClientTLS} clus, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg)) require.NoError(t, err) defer clus.Close() c := newClient(t, clus.EndpointsGRPC(), cfg.Client) defer c.Close() ctx := t.Context() const ( key = "k" v2 = "v2" v3 = "v3" v4 = "v4" v6 = "v6" ) t.Logf("PUT key=%s, val=%s", key, v2) _, err = c.KV.Put(ctx, key, v2) require.NoError(t, err) t.Logf("PUT key=%s, val=%s", key, v3) _, err = c.KV.Put(ctx, key, v3) require.NoError(t, err) t.Logf("PUT key=%s, val=%s", key, v4) _, err = c.KV.Put(ctx, key, v4) require.NoError(t, err) t.Logf("DELTE key=%s", key) deleteResp, err := c.KV.Delete(ctx, key) require.NoError(t, err) t.Logf("PUT key=%s, val=%s", key, v6) _, err = c.KV.Put(ctx, key, v6) require.NoError(t, err) t.Logf("COMPACT rev=%d", deleteResp.Header.Revision) _, err = c.KV.Compact(ctx, deleteResp.Header.Revision, clientv3.WithCompactPhysical()) require.NoError(t, err) watchChan := c.Watch(ctx, key, clientv3.WithRev(deleteResp.Header.Revision)) select { case watchResp := <-watchChan: require.Len(t, watchResp.Events, 2) require.Equal(t, mvccpb.Event_DELETE, watchResp.Events[0].Type) deletedKey := string(watchResp.Events[0].Kv.Key) require.Equal(t, key, deletedKey) require.Equal(t, mvccpb.Event_PUT, watchResp.Events[1].Type) updatedKey := string(watchResp.Events[1].Kv.Key) require.Equal(t, key, updatedKey) require.Equal(t, v6, string(watchResp.Events[1].Kv.Value)) case <-time.After(100 * time.Millisecond): // we care only about the first response, but have an // escape hatch in case the watch response is delayed. t.Fatal("timed out getting watch response") } } func TestStartWatcherFromCompactedRevision(t *testing.T) { t.Run("compaction on tombstone revision", func(t *testing.T) { testStartWatcherFromCompactedRevision(t, true) }) t.Run("compaction on normal revision", func(t *testing.T) { testStartWatcherFromCompactedRevision(t, false) }) } func testStartWatcherFromCompactedRevision(t *testing.T, performCompactOnTombstone bool) { e2e.BeforeTest(t) cfg := e2e.DefaultConfig() cfg.Client = e2e.ClientConfig{ConnectionType: e2e.ClientTLS} clus, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg), e2e.WithClusterSize(1)) require.NoError(t, err) defer clus.Close() c := newClient(t, clus.EndpointsGRPC(), cfg.Client) defer c.Close() ctx := t.Context() key := "foo" totalRev := 100 type valueEvent struct { value string typ mvccpb.Event_EventType } var ( // requestedValues records all requested change requestedValues = make([]valueEvent, 0) // revisionChan sends each compacted revision via this channel compactionRevChan = make(chan int64) // compactionStep means that client performs a compaction on every 7 operations compactionStep = 7 ) // This goroutine will submit changes on $key $totalRev times. It will // perform compaction after every $compactedAfterChanges changes. // Except for first time, the watcher always receives the compacted // revision as start. go func() { defer close(compactionRevChan) lastRevision := int64(1) compactionRevChan <- lastRevision for vi := 1; vi <= totalRev; vi++ { var respHeader *etcdserverpb.ResponseHeader if vi%compactionStep == 0 && performCompactOnTombstone { t.Logf("DELETE key=%s", key) resp, derr := c.KV.Delete(ctx, key) assert.NoError(t, derr) respHeader = resp.Header requestedValues = append(requestedValues, valueEvent{value: "", typ: mvccpb.Event_DELETE}) } else { value := fmt.Sprintf("%d", vi) t.Logf("PUT key=%s, val=%s", key, value) resp, perr := c.KV.Put(ctx, key, value) assert.NoError(t, perr) respHeader = resp.Header requestedValues = append(requestedValues, valueEvent{value: value, typ: mvccpb.Event_PUT}) } lastRevision = respHeader.Revision if vi%compactionStep == 0 { compactionRevChan <- lastRevision t.Logf("COMPACT rev=%d", lastRevision) _, err = c.KV.Compact(ctx, lastRevision, clientv3.WithCompactPhysical()) assert.NoError(t, err) } } }() receivedEvents := make([]*clientv3.Event, 0) fromCompactedRev := false for fromRev := range compactionRevChan { watchChan := c.Watch(ctx, key, clientv3.WithRev(fromRev)) prevEventCount := len(receivedEvents) // firstReceived represents this is first watch response. // Just in case that ETCD sends event one by one. firstReceived := true t.Logf("Start to watch key %s starting from revision %d", key, fromRev) watchLoop: for { currentEventCount := len(receivedEvents) if currentEventCount-prevEventCount == compactionStep || currentEventCount == totalRev { break } select { case watchResp := <-watchChan: t.Logf("Receive the number of events: %d", len(watchResp.Events)) for i := range watchResp.Events { ev := watchResp.Events[i] // If the $fromRev is the compacted revision, // the first event should be the same as the last event receives in last watch response. if firstReceived && fromCompactedRev { firstReceived = false last := receivedEvents[prevEventCount-1] assert.Equalf(t, last.Type, ev.Type, "last received event type %s, but got event type %s", last.Type, ev.Type) assert.Equalf(t, string(last.Kv.Key), string(ev.Kv.Key), "last received event key %s, but got event key %s", string(last.Kv.Key), string(ev.Kv.Key)) assert.Equalf(t, string(last.Kv.Value), string(ev.Kv.Value), "last received event value %s, but got event value %s", string(last.Kv.Value), string(ev.Kv.Value)) continue } receivedEvents = append(receivedEvents, ev) } if len(watchResp.Events) == 0 { require.Equal(t, v3rpc.ErrCompacted, watchResp.Err()) break watchLoop } case <-time.After(10 * time.Second): t.Fatal("timed out getting watch response") } } fromCompactedRev = true } t.Logf("Received total number of events: %d", len(receivedEvents)) require.Len(t, requestedValues, totalRev) require.Lenf(t, receivedEvents, totalRev, "should receive %d events", totalRev) for idx, expected := range requestedValues { ev := receivedEvents[idx] require.Equalf(t, expected.typ, ev.Type, "#%d expected event %s", idx, expected.typ) updatedKey := string(ev.Kv.Key) require.Equal(t, key, updatedKey) if expected.typ == mvccpb.Event_PUT { updatedValue := string(ev.Kv.Value) require.Equal(t, expected.value, updatedValue) } } } // TestResumeCompactionOnTombstone verifies whether a deletion event is preserved // when etcd restarts and resumes compaction on a key that only has a tombstone revision. func TestResumeCompactionOnTombstone(t *testing.T) { e2e.BeforeTest(t) ctx := t.Context() compactBatchLimit := 5 cfg := e2e.DefaultConfig() clus, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithConfig(cfg), e2e.WithClusterSize(1), e2e.WithCompactionBatchLimit(compactBatchLimit), e2e.WithGoFailEnabled(true), e2e.WithWatchProcessNotifyInterval(100*time.Millisecond), ) require.NoError(t, err) defer clus.Close() c1 := newClient(t, clus.EndpointsGRPC(), cfg.Client) defer c1.Close() keyPrefix := "/key-" for i := 0; i < compactBatchLimit; i++ { key := fmt.Sprintf("%s%d", keyPrefix, i) value := fmt.Sprintf("%d", i) t.Logf("PUT key=%s, val=%s", key, value) _, err = c1.KV.Put(ctx, key, value) require.NoError(t, err) } firstKey := keyPrefix + "0" t.Logf("DELETE key=%s", firstKey) deleteResp, err := c1.KV.Delete(ctx, firstKey) require.NoError(t, err) var deleteEvent *clientv3.Event select { case watchResp := <-c1.Watch(ctx, firstKey, clientv3.WithRev(deleteResp.Header.Revision)): require.Len(t, watchResp.Events, 1) require.Equal(t, mvccpb.Event_DELETE, watchResp.Events[0].Type) deletedKey := string(watchResp.Events[0].Kv.Key) require.Equal(t, firstKey, deletedKey) deleteEvent = watchResp.Events[0] case <-time.After(100 * time.Millisecond): t.Fatal("timed out getting watch response") } require.NoError(t, clus.Procs[0].Failpoints().SetupHTTP(ctx, "compactBeforeSetFinishedCompact", `panic`)) t.Logf("COMPACT rev=%d", deleteResp.Header.Revision) _, err = c1.KV.Compact(ctx, deleteResp.Header.Revision, clientv3.WithCompactPhysical()) require.Error(t, err) require.NoError(t, clus.Restart(ctx)) c2 := newClient(t, clus.EndpointsGRPC(), cfg.Client) defer c2.Close() watchChan := c2.Watch(ctx, firstKey, clientv3.WithRev(deleteResp.Header.Revision)) select { case watchResp := <-watchChan: require.Equal(t, []*clientv3.Event{deleteEvent}, watchResp.Events) case <-time.After(100 * time.Millisecond): // we care only about the first response, but have an // escape hatch in case the watch response is delayed. t.Fatal("timed out getting watch response") } } ================================================ FILE: tests/e2e/zap_logging_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "encoding/json" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func TestServerJsonLogging(t *testing.T) { e2e.BeforeTest(t) epc, err := e2e.NewEtcdProcessCluster(t.Context(), t, e2e.WithClusterSize(1), e2e.WithLogLevel("debug"), ) require.NoErrorf(t, err, "could not start etcd process cluster") logs := epc.Procs[0].Logs() time.Sleep(time.Second) require.NoErrorf(t, epc.Close(), "error closing etcd processes") var entry logEntry lines := logs.Lines() if len(lines) == 0 { t.Errorf("Expected at least one log line") } for _, line := range lines { err := json.Unmarshal([]byte(line), &entry) if err != nil { t.Errorf("Failed to parse log line as json, err: %q, line: %s", err, line) continue } if entry.Level == "" { t.Errorf(`Missing "level" key, line: %s`, line) } if entry.Timestamp == "" { t.Errorf(`Missing "ts" key, line: %s`, line) } if _, err := time.Parse("2006-01-02T15:04:05.999999Z0700", entry.Timestamp); entry.Timestamp != "" && err != nil { t.Errorf(`Unexpected "ts" key format, err: %s`, err) } if entry.Caller == "" { t.Errorf(`Missing "caller" key, line: %s`, line) } if entry.Message == "" { t.Errorf(`Missing "message" key, line: %s`, line) } } } type logEntry struct { Level string `json:"level"` Timestamp string `json:"ts"` Caller string `json:"caller"` Message string `json:"msg"` Error string `json:"error"` } func TestConnectionRejectMessage(t *testing.T) { e2e.SkipInShortMode(t) testCases := []struct { name string url string expectedErrMsg string }{ { name: "reject client connection", url: "https://127.0.0.1:2379/version", expectedErrMsg: "rejected connection on client endpoint", }, { name: "reject peer connection", url: "https://127.0.0.1:2380/members", expectedErrMsg: "rejected connection on peer endpoint", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { commonArgs := []string{ e2e.BinPath.Etcd, "--name", "etcd1", "--listen-client-urls", "https://127.0.0.1:2379", "--advertise-client-urls", "https://127.0.0.1:2379", "--cert-file", e2e.CertPath, "--key-file", e2e.PrivateKeyPath, "--trusted-ca-file", e2e.CaPath, "--listen-peer-urls", "https://127.0.0.1:2380", "--initial-advertise-peer-urls", "https://127.0.0.1:2380", "--initial-cluster", "etcd1=https://127.0.0.1:2380", "--peer-cert-file", e2e.CertPath, "--peer-key-file", e2e.PrivateKeyPath, "--peer-trusted-ca-file", e2e.CaPath, } t.Log("Starting an etcd process and wait for it to get ready.") p, err := e2e.SpawnCmd(commonArgs, nil) require.NoError(t, err) err = e2e.WaitReadyExpectProc(t.Context(), p, e2e.EtcdServerReadyLines) require.NoError(t, err) defer func() { p.Stop() p.Close() }() t.Log("Starting a separate goroutine to verify the expected output.") startedCh := make(chan struct{}, 1) doneCh := make(chan struct{}, 1) go func() { startedCh <- struct{}{} verr := e2e.WaitReadyExpectProc(t.Context(), p, []string{tc.expectedErrMsg}) assert.NoError(t, verr) doneCh <- struct{}{} }() // wait for the goroutine to get started <-startedCh t.Log("Running curl command to trigger the corresponding warning message.") curlCmdArgs := []string{"curl", "--connect-timeout", "1", "-k", tc.url} curlCmd, err := e2e.SpawnCmd(curlCmdArgs, nil) require.NoError(t, err) defer func() { curlCmd.Stop() curlCmd.Close() }() t.Log("Waiting for the result.") select { case <-doneCh: case <-time.After(5 * time.Second): t.Fatal("Timed out waiting for the result") } }) } } ================================================ FILE: tests/fixtures/CommonName-root.crt ================================================ -----BEGIN CERTIFICATE----- MIIE5zCCA8+gAwIBAgIJAKooGDZuR2mMMA0GCSqGSIb3DQEBCwUAMH8xCzAJBgNV BAYTAkNOMRAwDgYDVQQIDAdCZWlqaW5nMRAwDgYDVQQHDAdCZWlqaW5nMQ0wCwYD VQQKDAREZW1vMQ0wCwYDVQQLDAREZW1vMQ0wCwYDVQQDDARyb290MR8wHQYJKoZI hvcNAQkBFhB0ZXN0QGV4YW1wbGUuY29tMB4XDTIyMTExNjA2NTI1M1oXDTMyMTEx MzA2NTI1M1owfzELMAkGA1UEBhMCQ04xEDAOBgNVBAgMB0JlaWppbmcxEDAOBgNV BAcMB0JlaWppbmcxDTALBgNVBAoMBERlbW8xDTALBgNVBAsMBERlbW8xDTALBgNV BAMMBHJvb3QxHzAdBgkqhkiG9w0BCQEWEHRlc3RAZXhhbXBsZS5jb20wggEiMA0G CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEAKcjzhtOG3hWbAUCbudE1gPOeteT 0INk2ngN2uCMYjYSZmaGhW/GZk3EvV7wKVuhdTyrh36E5Iajng9d2t1iOU/8jROU +uAyrS3C/S5P/urq8VBUrt3VG/44bhwTEdafNnAWQ6ojYfmK0tRqoQn1Ftm30l8I nWof5Jm3loNA2WdNdvAp/D+6OpjUdqGdMkFd0NhkuQODMnycBMw6btUTj5SnmrMk q7V1aasx4BqN5C4DciZF0pyyR/TT8MoQ5Vcit8rHvQUyz42Lj8+28RkDoi4prJ1i tLaCt2egDp58vXlYQZTd50inMhnBIapKNdGpg3flW/8AFul1tCTqd8NfAgMBAAGj ggFkMIIBYDAdBgNVHQ4EFgQUpwwvEqXjA/ArJu1Jnpw7+/sttOAwgbMGA1UdIwSB qzCBqIAUpwwvEqXjA/ArJu1Jnpw7+/sttOChgYSkgYEwfzELMAkGA1UEBhMCQ04x EDAOBgNVBAgMB0JlaWppbmcxEDAOBgNVBAcMB0JlaWppbmcxDTALBgNVBAoMBERl bW8xDTALBgNVBAsMBERlbW8xDTALBgNVBAMMBHJvb3QxHzAdBgkqhkiG9w0BCQEW EHRlc3RAZXhhbXBsZS5jb22CCQCqKBg2bkdpjDAMBgNVHRMEBTADAQH/MAsGA1Ud DwQEAwIC/DA2BgNVHREELzAtggtleGFtcGxlLmNvbYINKi5leGFtcGxlLmNvbYIJ bG9jYWxob3N0hwR/AAABMDYGA1UdEgQvMC2CC2V4YW1wbGUuY29tgg0qLmV4YW1w bGUuY29tgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZIhvcNAQELBQADggEBAGi48ntm 8cn08FrsCDWapsck7a56/dyFyzLg10c0blu396tzC3ZDCAwQYzHjeXVdeWHyGO+f KSFlmR6IA0jq6pFhUyJtgaAUJ91jW6s68GTVhlLoFhtYjy6EvhQ0lo+7GWh4qB2s LI0mJPjaLZY1teAC4TswzwMDVD8QsB06/aFBlA65VjgZiVH+aMwWJ88gKfVGp0Pv AApsy5MvwQn8WZ2L6foSY04OzXtmAg2gCl0PyDNgieqFDcM1g7mklHNgWl2Gvtte G6+TiB3gGUUlTsdy0+LS2psL71RS5Jv7g/7XGmSKBPqRmYyQ2t7m2kLPwWKtL5tE 63c0FPtpV0FzKdU= -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/CommonName-root.key ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAxACnI84bTht4VmwFAm7nRNYDznrXk9CDZNp4DdrgjGI2EmZm hoVvxmZNxL1e8ClboXU8q4d+hOSGo54PXdrdYjlP/I0TlPrgMq0twv0uT/7q6vFQ VK7d1Rv+OG4cExHWnzZwFkOqI2H5itLUaqEJ9RbZt9JfCJ1qH+SZt5aDQNlnTXbw Kfw/ujqY1HahnTJBXdDYZLkDgzJ8nATMOm7VE4+Up5qzJKu1dWmrMeAajeQuA3Im RdKcskf00/DKEOVXIrfKx70FMs+Ni4/PtvEZA6IuKaydYrS2grdnoA6efL15WEGU 3edIpzIZwSGqSjXRqYN35Vv/ABbpdbQk6nfDXwIDAQABAoIBAA5AMebTjH6wVp6J +g9EOwJxQROZMOVparRBgisXt+3dEitiUKAFQaw+MfdVAXsatrPVj1S1ZEiLSRLK YjmjuSb0HdGx/DN/zh9BIiukNuLQGQp+AyY1FKHzCBfYQahNSrqGvb2Qq+UosXkb fSBHly6/u5K28/vvXhD1kQudIOvtAc9tOg8LZnM6N3J4E0GtLqWimRZ4jNK4APu1 YsLIg87Eam+7x25+phz9xc22tZ1H4WY9FnOGprPnievqiV7mgcNGAklTB93C6yX1 EI+QxQnPg0P732C4EJZFDPqhVRA4E7BUb5uTIXCJBA/FFuRIx9ppyLZKt9vjTchM 8YWIEsECgYEA/5DRR9FkIWJZb0Pv3SCc53PMPT/xpYB6lH2lGtG+u+L71dJNDiPt da3dPXSBy+aF7BbmRDawRvyOLGArlWiSsoEUVlES8BYzQ1MmfDf+MJooJoBE6/g6 2OyyNnPde1GqyxsxgNTITvJCTjYH64lxKVRYfMgMAASK49SjYiEgGn8CgYEAxFXs Oe0sUcc3P1cQ9pJfSVKpSczZq/OGAxqlniqRHvoWgFfKOWB6F9PN0rd8G2aMlfGS BjyiPe770gtpX8Z4G4lrtkJD8NvGoVC8yX78HbrXL2RA4lPjQfrveUnwXIRbRKWa 6D/GAYPOuNvJmwF4hY/orWyIqvpNczIjTjs1JyECgYEAvhuNAn6JnKfbXYBM+tIa xbWHFXzula2IAdOhMN0bpApKSZmBxmYFa0elTuTO9M2Li77RFacU5AlU/T+gzCiZ D34jkb4Hd18cTRWaiEbiqGbUPSennVzu8ZTJUOZJuEVc5m9ZGLuwMcHWfvWEWLrJ 2fOrS09IVe8LHkV8MC/yAKMCgYBmDUdhgK9Fvqgv60Cs+b4/rZDDBJCsOUOSP3qQ sQ2HrXSet4MsucIcuoJEog0HbRFsKwm85i1qxdrs/fOCzfXGUnLDZMRN4N7pIL9Q eQnxJhoNzy2Otw3sUNPDFrSyUjXig7X2PJfeV7XPDqdHQ8dynS/TXRPY04wIcao6 Uro5IQKBgFUz2GjAxI6uc7ihmRv/GYTuXYOlO0IN7MFwQDd0pVnWHkLNZscO9L9/ ALV4g1p/75CewlQfyC8ynOJJWcDeHHFNsSMsOzAxUOVtVenaF/dgwk95wpXj6Rx6 4kvQqnJg97fRBbyzvQcdL36kL8+pbmHNoqHPwxbuigYShB74d6/h -----END RSA PRIVATE KEY----- ================================================ FILE: tests/fixtures/ca-csr.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "ca", "ca": { "expiry": "87600h" } } ================================================ FILE: tests/fixtures/ca.crt ================================================ -----BEGIN CERTIFICATE----- MIIDrjCCApagAwIBAgIUNkN+TZ3hgHno+H9j56nWkmb4dBEwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQDZwQPFZB+Kt6RIzYvTgbNlRIX/cLVknIy4ZqhLYDQNOdosJn04jjkCfS3k F5JZuabkUs6d6JcLTbLWV5hCrwZVlCFf3PDn6DvK12GZpybhuqMPZ2T8P2U17AFP mUj/Rm+25t8Er5r+8ijZmqVi1X1Ef041CFGESr3KjaMjec2kYf38cfEOp2Yq1JWO 0wpVfLElnyDQY9XILdnBepCRZYPq1eW1OSkRk+dZQnJP6BO95IoyREDuBUeTrteR 7dHHTF9AAgR5tnyZ+eLuVUZ2kskcWLxH3y9RyjvVJ+1uCzbdydVPf0H1pBoqWcuA PYjYkLKMOKBWfYJhSzykhf+QMC7xAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQpJiv07dkY9WB0zgB6wOb/HMi8oDAN BgkqhkiG9w0BAQsFAAOCAQEA0TQ8rRmLt4wYjz0BKh+jElMIg6LBPsCPpfmGDLmK fdj4Jp7QFlLmXlQSmm8zKz3ftKoOFPYGQYHUkIirIrQB/tdHoXJLgxCzI0SrdCiM m/DPVjfOTa9Mm5rPcUR79rGDLj2BgzDB+NTETVDXo8mAL5MjFdUyh6jOGBctkCG/ TWdUaN33ZLwUl488NLaw98fIZ/F4d/dsyCJvHEaoo++dgjduoQxmH9Scr2Frmd8G zYxOoZHG3ARBDp2mpr+I3UCR1/KTITF/NXL6gDcNY3wyZzoaGua7Bd/ysMSi1w3j CyvClSvRPJRLQemGUP7B/Y8FUkbJ2i/7tz6ozn8sLi3V2Q== -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/client-ca-csr-nocn.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "", "hosts": [ "127.0.0.1", "localhost" ] } ================================================ FILE: tests/fixtures/client-clientusage.crt ================================================ -----BEGIN CERTIFICATE----- MIIECDCCAvCgAwIBAgIULbzkAv8zbkJzZIRDPnBwXl0/BH0wDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQDWBNo9tYRoQKv76xabz0EPXGJKHIrUjf0NbXz3d9jbP2sH 3hutXr/A221pULfZYIZdaUtmEuEr1905nYwJ2gnO9Y/iSc6fQ/4EjoT+VZLdINQw I1dG2rtv2ZuYL5oYfgCjLkV1LzYuyfY/zJ93WoJW0YA0t50MEQNGEqD7pYlhsPej iGyjagSi7zsoAkAagNprULH6RyAqDG7db+MfJOUzHUv4PWGBXPb0PHY3xA+WayFB nP5AZO16oDh/UnzvfEAJULXeIOLs4eOmtzKMwZwrWzgCB+jBeVlc1FOwXQcmBamN eYUs75GoO9aSSLROvnQiw2P0z0xVNmDokDXGsSRxAgMBAAGjgZIwgY8wDgYDVR0P AQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYD VR0OBBYEFCB4ysDF81d6lkKIvebj08BcRWNoMB8GA1UdIwQYMBaAFCkmK/Tt2Rj1 YHTOAHrA5v8cyLygMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG 9w0BAQsFAAOCAQEAo2B+piCBTjdpCLFj/kc+A0alZTbNdr0+BTsN+5aBE9k4JlZS smkIQL0vyzjKw/W/o2EyPVcVKJX52/GQsC3bQrBb2lH1jRYgt5pRo24kKHy4Nlc3 IaYg++ssfT2ZdpYiL3lzLyOHEumcynz3nI5M81e5CCIdEennxaM8FuiYN5OXDOR3 j+bCYHLYPaWYZopfiSrnq+Z4gRUS2sMI1yqtiPSUdIJLnTfyEEdexvs/KUtFWvFO 4AcecKvT6HA8oNDiWfE6e854uDLTkbXW1rK+FWPU9pv5NR50+GBCvxvmDGtGXxQu yu+kOsx2gfgNc4idIv1pjZF/1YzrrKGAhChN2A== -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/client-clientusage.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEogIBAAKCAQEA1gTaPbWEaECr++sWm89BD1xiShyK1I39DW1893fY2z9rB94b rV6/wNttaVC32WCGXWlLZhLhK9fdOZ2MCdoJzvWP4knOn0P+BI6E/lWS3SDUMCNX Rtq7b9mbmC+aGH4Aoy5FdS82Lsn2P8yfd1qCVtGANLedDBEDRhKg+6WJYbD3o4hs o2oEou87KAJAGoDaa1Cx+kcgKgxu3W/jHyTlMx1L+D1hgVz29Dx2N8QPlmshQZz+ QGTteqA4f1J873xACVC13iDi7OHjprcyjMGcK1s4AgfowXlZXNRTsF0HJgWpjXmF LO+RqDvWkki0Tr50IsNj9M9MVTZg6JA1xrEkcQIDAQABAoIBAAGBZTub5EOLeOo7 vBv6eD2wa6yTyNI38Xi/tWpUOH1KU+lpQY6VpQmpQXrFK5Xm3OsZS4N7TIQvb4nx NsP2+aywA4QW+tIZ+1Zy3jKfzXmqunNgPEPuU/U0dai7ZP0ZHc4IDEsHuvzXRNks Ck8fnt0XeixkwkEMeZZrmSBMCMxcHAWxiv+oXF+olN3vTD2aDC8T6YwahMyQUQfW IA9fuO8Dzzmk2I7mDHa29cbB+PW4E5tkJmHVZqEu8jPgMjCJGc2IR1YpLAXF8YBB vgh6ZgI6JOg1OiNETuQekamAMOblFVOdPUjPSxuyJzEE8VpIdD3Z9UMNq+FDQh/F j1lEEEECgYEA9nYwUh+e0H9c9IRBLNYAbq2PV4SpFKvFrHOTQpylMPisUTgdHKLT CvO1wbNprElBAulOWobCyKshWGd5ECFsCvsWS6xmGi442q3ov5xtAMmvSmtW8s+8 tUeVRQGS/Yn5Uxj2msUPe6vJEniLgsxmbFbDYqvr65COrAsCDEY3DkkCgYEA3k09 EGhiO1joDtJPI21vUzzecBuep32oKiwip3OgS/mct04/QR+6lp1x4sPMYlyxbyk9 jPdkzU07d8r+mES9RweE5lc1aCaF5eA8y6qtL9vBgsXRiEXlpYLxb0TOQaYNU0qM aYumYPWjsjwYDvRKaVzThFUkYwapKFqtMV98BOkCgYAkIOkucLIwMCtpMKX5M5m2 n7yegLTkcdW1VO/mWN4iUqG3+jjSRNAZD+a58VnxRn/ANIEm5hBRqDxoICrwAWY8 Kdh32VrSRapR7CJtTDnyXp5Sk2+YgnlQPaEVD4kDn6Er3EHyKCb/4wvDqGYTE3GE OifEJB2eV3+Cms5/DB/v+QKBgFzV8r9saEGSkm7GI2iPJiOj0t0Mm8gksNrTzbES l4nC91CR69adkoWdwNbLoAof3bWnil3ZXw5hx4jyjDo40rbcDANJvjL9i4OBjsIb R/Ipmvmq9SMs1Ye2VG98U4qU9xGmm1bkjBoH21HuyLlOCdlQe8DS8bwtJu2EWLm6 v4cpAoGAP3pqi6iIZAbJVqw6p5Xd/BLzn3iS+lwQjwF/IWp5UnFCGovnVSJG2wqP kxL9jy4asMDDuMKzEzO3+UT6WBRI+idV8PgDNEYkXcnVAA5lZ+2kCJwRICsC6MYH 1nIHJtPngUrwT3TUhMp/WfpYUjTdiOC3aJmKq/NGZxE8/Sb3G6U= -----END RSA PRIVATE KEY----- ================================================ FILE: tests/fixtures/client-nocn.crt ================================================ -----BEGIN CERTIFICATE----- MIID/DCCAuSgAwIBAgIUCzIuVb3586z5C2rQ65jeo4wfbfowDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMGIxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKmOrIfZ9mH9 O3wLgGinUXDAG+XAP6P6NG9VkWaCUfOkY8x8RKSeuOri31EgYGmFYmQXCtS/WlHD GCLrUhTnIrC1/WqvuPJIoMMTw7JLh59IuIWdlxds7FWjyuLmi4oUHvCG6aXiT/Z3 ylp4r/HBL+R6KKqQpRjFfwhb1bIWpxZe5ghUtx4AuAW7ayQgpC7FJ3aVW/SS5p0m IxyKqGvl45IsZuZY59Sa/X2AWSRpr+qe0tM4n1R+1bDhjcV6EuhyfubdSkZHfUJp PaoUdynHT/VuI5xMF4OXbiwXP36XvHiHd9LIrPOyubrRYvn8dKweBJkvNCnlQo09 zVH5zb9p0DsCAwEAAaOBnDCBmTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYI KwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFG5evtY/ UIPMBcah3B/1BWDI14nUMB8GA1UdIwQYMBaAFCkmK/Tt2Rj1YHTOAHrA5v8cyLyg MBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEA VBjy5UtSe/f66d7dKgZVVfKDiOeSb1knATSy7/JyubxVgq64yTN6fqIYRQg4gyVW IPf8W4BbhEXeA7VumVuTTKjILoufGecjrjA1Skb4lWGfV21A51Fs9TcMLPiQYZ1b e2J2Trtd0CsteQj4BDrbgiSxahJBaj+4PfXM1tef51DJs+gEg16DGxdzFBtlY+ih SwOX6YcUyxYzYX2szafPpVRuQqU0B63FkvBbsNMX1KamtAsLtvf/JxYpPY9eg5t/ b5L6pXQkp6bK3q8Gv1WApjD8tcwqBkcJrbjgJ6gfW9h3zEbLmxkAv46sJodVLInL SYrHgrQ7TRd29DybB6cPAQ== -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/client-nocn.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAqY6sh9n2Yf07fAuAaKdRcMAb5cA/o/o0b1WRZoJR86RjzHxE pJ646uLfUSBgaYViZBcK1L9aUcMYIutSFOcisLX9aq+48kigwxPDskuHn0i4hZ2X F2zsVaPK4uaLihQe8IbppeJP9nfKWniv8cEv5HooqpClGMV/CFvVshanFl7mCFS3 HgC4BbtrJCCkLsUndpVb9JLmnSYjHIqoa+Xjkixm5ljn1Jr9fYBZJGmv6p7S0zif VH7VsOGNxXoS6HJ+5t1KRkd9Qmk9qhR3KcdP9W4jnEwXg5duLBc/fpe8eId30sis 87K5utFi+fx0rB4EmS80KeVCjT3NUfnNv2nQOwIDAQABAoIBAECPnM4VhiUFgTLY RkqS+wWNgJHYw+KyEGkcEcMQeBfnTkC8SH7OGOcG/7UqOMu1CCPISk17lu5u9K/H HnfrEmBqy1VmF2vZj6z3x5oJ/FgAHpJx0OgQh2SMe2IuGo+23ZkEJc8N/xh/wEL2 lTfeMVgz02wuq05lVNtf7FxlF7YCSaxxxDtQQTDR3BSq6l12tB81TQvAD+yh35Gs 1jGhPeKHWc1jny309vczpJq4eIK2xhE+MT8YZAiuHCLGOHUlBBpleo5knyMueVE/ /Ezbz6eFiIFYpoHA3d3pv3Dy+5WVnhD0YDQPe+jCQrzxyFGDiN488JQ2tVeRM85b q0naaZECgYEA1T8XWPqRkhjMy0vJxTVux+wdD3u9DIvgBfHxjBUS2xlZdOiLLmBD CDVLKe+Twn0KiTb0eU+zNn4g1qnxLXmAH7xYWPLtqoI4mM0O89SWxr06ExplamHp w5k5O3eJr0veKyCUqVbZRZsOQLi1zqEbaOqpA7TrsQOOT5io+0vVoV0CgYEAy407 JRaGBTBNOPayBVFY+7PRsSRPtcjzbOHriCe4rDn8aIPPmzHyWEIL0pXk5I1eW978 veC/2oZMsxO2vaKta1bSSOrNA8UJQ+t5Ipp6Fj6yAI5dMDcgOIctE8ctxDUfccQM kS5DDw0W3zYMI7ixyOe6ydX4OAlcpZgqFpNIJncCgYEAuB1pAyIUXZeb+krNQsAH jgWGcb/cUeDS408pxlDLnvAcFJxSzw+90HBzHRoE8X8UgbQ5ECSIDxyHLdA8s46b 2Mq9XM8h9H3Kb+NcbZm3NJBce/Hmbhtrwb2hdH6ZGgjfIU1YDX02yqo9fBP+pRDk oYk5tEGY3ZS8YmzkOVQYduECgYACgnNAOc7dMYNCOIhpWF9oewcS0AfLjfayWPa2 bwbv2KcsArQEjdEXFXlf10lDKBsJtu4WyTaUUyOO8adHH0JUGHXvQDXW3g8HL1gG /TCUJaG8MAUmGwfiqof7vnDqAl2o4WnmQFPDU738coYjypsmhvTemCy/RB5ITF/4 d0hkcQKBgAWpzCnPAh4tPWw1OGE2QSsbRR15hR+67BltiZ+nxJnDcXXS2i08QBkA 3VR0ywWsos+Sox6jm8LpH8RiKqZ5laUjHHUUuX1Tgfxn4EmHo6bBffw7k9vkY7xr w5Nw/gMRevkRrDQ4Z66z2HspyCHfmdPzWX9zsaSc4nzNs7fw2/uf -----END RSA PRIVATE KEY----- ================================================ FILE: tests/fixtures/ed25519-private-key.pem ================================================ -----BEGIN PRIVATE KEY----- MC4CAQAwBQYDK2VwBCIEIAtiwQ7KeS1I0otY9gw1Ox4av/zQ+wvs/8AIaTkawQ73 -----END PRIVATE KEY----- ================================================ FILE: tests/fixtures/ed25519-public-key.pem ================================================ -----BEGIN PUBLIC KEY----- MCowBQYDK2VwAyEAuOUxC8Bbn1KqYctlim/MHaP5JrtmeK5xcs+9w506btA= -----END PUBLIC KEY----- ================================================ FILE: tests/fixtures/gencert.json ================================================ { "signing": { "default": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "87600h" }, "profiles": { "client-only": { "usages": [ "signing", "key encipherment", "client auth" ], "expiry": "87600h" }, "server-only": { "usages": [ "signing", "key encipherment", "server auth" ], "expiry": "87600h" } } } } ================================================ FILE: tests/fixtures/gencerts.sh ================================================ #!/bin/bash set -euo pipefail if ! [[ "$0" =~ "./gencerts.sh" ]]; then echo "must be run from 'fixtures'" exit 255 fi if ! command -v cfssl; then echo "cfssl is not installed" echo 'use: bash -c "cd ../../tools/mod; go install github.com/cloudflare/cfssl/cmd/cfssl"' exit 255 fi if ! command -v cfssljson; then echo "cfssljson is not installed" echo 'use: bash -c "cd ../../tools/mod; go install github.com/cloudflare/cfssl/cmd/cfssljson"' exit 255 fi cfssl gencert --initca=true ./ca-csr.json | cfssljson --bare ./ca mv ca.pem ca.crt if command -v openssl >/dev/null; then openssl x509 -in ca.crt -noout -text fi # gencert [config_file.json] [cert-name] function gencert { cfssl gencert \ --ca ./ca.crt \ --ca-key ./ca-key.pem \ --config ./gencert.json \ $1 | cfssljson --bare ./$2 mv $2.pem $2.crt mv $2-key.pem $2.key.insecure } # generate DNS: localhost, IP: 127.0.0.1, CN: example.com certificates, with dual usage gencert ./server-ca-csr.json server #generates certificate that only has the 'server auth' usage gencert "--profile=server-only ./server-ca-csr.json" server-serverusage #generates certificate that only has the 'client auth' usage gencert "--profile=client-only ./server-ca-csr.json" client-clientusage #generates certificate that does not contain CN, to be used for proxy -> server connections. gencert ./client-ca-csr-nocn.json client-nocn # generate DNS: localhost, IP: 127.0.0.1, CN: example.com certificates (ECDSA) gencert ./server-ca-csr-ecdsa.json server-ecdsa # generate IP: 127.0.0.1, CN: example.com certificates gencert ./server-ca-csr-ip.json server-ip # generate IPv6: [::1], CN: example.com certificates gencert ./server-ca-csr-ipv6.json server-ipv6 # generate DNS: localhost, IP: 127.0.0.1, CN: example2.com certificates gencert ./server-ca-csr2.json server2 # generate DNS: localhost, IP: 127.0.0.1, CN: "" certificates gencert ./server-ca-csr3.json server3 # generate wildcard certificates DNS: *.etcd.local gencert ./server-ca-csr-wildcard.json server-wildcard # generate revoked certificates and crl cfssl gencert --ca ./ca.crt \ --ca-key ./ca-key.pem \ --config ./gencert.json \ ./server-ca-csr.json 2>revoked.stderr | cfssljson --bare ./server-revoked mv server-revoked.pem server-revoked.crt mv server-revoked-key.pem server-revoked.key.insecure grep serial revoked.stderr | awk ' { print $9 } ' >revoke.txt cfssl gencrl revoke.txt ca.crt ca-key.pem | base64 --decode >revoke.crl rm -f *.csr *.pem *.stderr *.txt ================================================ FILE: tests/fixtures/server-ca-csr-ecdsa.json ================================================ { "key": { "algo": "ecdsa", "size": 256 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "example.com", "hosts": [ "127.0.0.1", "localhost" ] } ================================================ FILE: tests/fixtures/server-ca-csr-ip.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "example.com", "hosts": [ "127.0.0.1" ] } ================================================ FILE: tests/fixtures/server-ca-csr-ipv6.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "example.com", "hosts": [ "::1" ] } ================================================ FILE: tests/fixtures/server-ca-csr-wildcard.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "example.com", "hosts": [ "*.etcd.local", "etcd.local", "127.0.0.1", "localhost" ] } ================================================ FILE: tests/fixtures/server-ca-csr.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "example.com", "hosts": [ "127.0.0.1", "localhost" ] } ================================================ FILE: tests/fixtures/server-ca-csr2.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "example2.com", "hosts": [ "127.0.0.1", "localhost" ] } ================================================ FILE: tests/fixtures/server-ca-csr3.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "", "hosts": [ "127.0.0.1", "localhost" ] } ================================================ FILE: tests/fixtures/server-ecdsa.crt ================================================ -----BEGIN CERTIFICATE----- MIIDRzCCAi+gAwIBAgIUVE0fLzH6W4M2gJVJhmQdkQdKpO0wDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjO PQMBBwNCAAREhCklwbvzFozNPkr3Y5PGrQr1ygfL5Q+XhvPOTTEjEN/zwjw9L0Qa jfhE8Md89qED0j8xHAKeQRrulgv/FWXXo4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAd BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNV HQ4EFgQUXJTZKpg0EYo+wtiYmacwd1OSKfgwHwYDVR0jBBgwFoAUKSYr9O3ZGPVg dM4AesDm/xzIvKAwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3 DQEBCwUAA4IBAQBd3RuqNsDxYS/RRc9Df8gLaXn/QQhATx6s3+pKHYplIH9sGPCh ybI4MpwLnuoqxew8dxy7oi/BBXPWSUuVznRV/vLKAIuULoKg2Eb06d17OmqOaakl asGnJ7z9e6mxHPVDzjkORNlJShY4YOG0tUg4hC5/9Qxh6EGNUKtRC3x4Tm8Jl6me uGLUjsQV7YhQNRDFECUQmKwolEbwXbAi2SN3I37CBFDFwDT/0BxtfGSn0ZiXRHze k1dmg9V3r9UPcucb3Djoad/N5YClfFtX/ANC8bufkkdfQLQwIBCUwcPlGxrBAVoD BoqpmQdpQ/yINKesAD/r5dF2SmUEhZhn6GSK -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/server-ecdsa.key.insecure ================================================ -----BEGIN EC PRIVATE KEY----- MHcCAQEEIEmvbcwNyqDHWXBG2IHZffLme5Ti8oHYzaapBvwkRSWWoAoGCCqGSM49 AwEHoUQDQgAERIQpJcG78xaMzT5K92OTxq0K9coHy+UPl4bzzk0xIxDf88I8PS9E Go34RPDHfPahA9I/MRwCnkEa7pYL/xVl1w== -----END EC PRIVATE KEY----- ================================================ FILE: tests/fixtures/server-ip.crt ================================================ -----BEGIN CERTIFICATE----- MIIEBzCCAu+gAwIBAgIUIqRH3sc1siaGkZVpWKSoDAIEMucwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQDUTSITIyfyfyE+fdQGM8sali6fk4pFCpuODbljr/ywVI81 /MrLGhX/zySgRHKKG5f23CWbImgtIsPbScKxNNQcvDRXmbsLtlH/o7Eoun/e0aGp rzr0p0QNGGeKFfUBTnaB+Z7+V92oxjNAuyeMZstqJxjOWDGpCS+yFVvP/ElRsL2H JVZHWOykwKdLznRUjlw//PcvJrNsO9DzYluZ6tDqlN5nyB6aW0h9ZkcCskGDo+1V 94tjh5rGTscREVIWxVxHgLMFlvaEJlz64pVgc8VWD6famaiqP5nC0WOx5BJtSrmF 3WH97DkfVcXWpqUbEAey6G4sLU0a/08iKoWbJmbvAgMBAAGjgZEwgY4wDgYDVR0P AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB Af8EAjAAMB0GA1UdDgQWBBQiwt5ZVBlE2nrYnp5z4R6ADUnCwTAfBgNVHSMEGDAW gBQpJiv07dkY9WB0zgB6wOb/HMi8oDAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3 DQEBCwUAA4IBAQAh5Jxw4TbDnQJMzj53KxEgNxbd++p7LMhZkN5X8jtuDe81rzeQ CyvJlrLEVPKbXiQF6cFV3TOvrZY/PM8UoAcXv0noEtrlRrrjbk9e4My3Zu4O1IHB MvfXOuF8JN5L3kCrcUcjhMrx8XTLyathSTQxG6TCh7X4+/vXufbZkkzg8nhtSSWB t7hWYo3KA25TgjP4E68BpnddNe9ad2sMIpIM1ovhhW6v8Uzux+eKkBkeb2Cdmrhd CEzK33WrDPCLsxa8hOByCWFj76qQ+Q5kgJvK3F7kBLWijhRKLhJBQzCJDCvx4yE1 w/e+aG74m1tAID1XjTuCZIcTZTEHZi31ogVM -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/server-ip.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA1E0iEyMn8n8hPn3UBjPLGpYun5OKRQqbjg25Y6/8sFSPNfzK yxoV/88koERyihuX9twlmyJoLSLD20nCsTTUHLw0V5m7C7ZR/6OxKLp/3tGhqa86 9KdEDRhnihX1AU52gfme/lfdqMYzQLsnjGbLaicYzlgxqQkvshVbz/xJUbC9hyVW R1jspMCnS850VI5cP/z3LyazbDvQ82JbmerQ6pTeZ8gemltIfWZHArJBg6PtVfeL Y4eaxk7HERFSFsVcR4CzBZb2hCZc+uKVYHPFVg+n2pmoqj+ZwtFjseQSbUq5hd1h /ew5H1XF1qalGxAHsuhuLC1NGv9PIiqFmyZm7wIDAQABAoIBAQDP8BSV5fM0guxO xvOqd4RRMBPOXLYrVW5yvmJ8j1zSYKA8YrNGJvCxM3ROPXxqZQh807dJsXOT8d8f o6k74+B1nKkvu/UGTbcWyn+0wqaH2Y+cIXN/OW1f3i1bhJIKi41rVNEzkWAb9LUy i5z62ZwXBuA3Cw7o34SFyoG4vwQZK1efygUXGIKSHdwAW7mwPD3MLIuIJgDrj2P4 2aLb1jcRyUKCY06QD7tsOH6pn59qkjVnoYMJl7B8DiCRFXjtt1yRZ5RU9sMT8PxV Px29qIouHvKcLt5cNX/Z1VjoopTHCOvhKMmQzP0ubPp7/Ytu2tPQcc/8DoZ/3+aw Zr+27SSRAoGBANva1d0wSR17QTjfcKLDup9+ERztyW2fxi7hGgKIqzPCY4E+vGTX KC5eToNpyo89COa/Z3rHKzLlSYpDaQiB/kWqEm3HPEW2Yq1YHaI7nZsuBPUkMtH+ xOBFyZUYG2aaqQiBuvvSJRxP5puAXGlIWAQp9qOLwtbQJ0gGRMhq4yutAoGBAPc0 Y0xzPNpTkjcRGDN7srcZw12tqy5bpi/TW+Ynlfxg2OnO5e4W9TEgFxcsNeqAB1IB QBd1QhVnpHFANIThX4XNQ59FJ1jOYuRwKWpjBNOol3YWhLlBwPrRLxJNxYXbZha4 zafhrvv3VMatU9Tc+a4gnZ+ooSM1m/rTAQqfunCLAoGAIAHC4tm1u0IHY8U7u6Zt E+0hhqmjin8ZNhf1VmsZKYbiP52nhbLBGccG/SC4qZPEKPuyj/BQ/K7evu9Dakaq gu/YkPzRbIC56uyKG+U786yGcj3b3DCP7uqaB0ekLZLUivWACEs2teF3/Cl6yqUK k0icrICbU/Sn01d+SgMtoV0CgYBzUx9YFRK4j/BQfEscCYMwZHZ+B30qnVsESMhA sQsJuGy5dupRjqhIiL3884UbpyrDGQ47Y1q2/aj7pIZbz4BuvXnknbBjf7Um+SR5 G0SvMaGnV44HlyNeX6RkF6AkeFxCEWjv/xtRNOt53HaVgZmBoHmoeFTkRihEdZew yx+BTQKBgHMs/wLEaC0zLeYZV90s8cR+LqrRl1IkjEtpHg09ESDLX5IyobivSmbB wOkkFI0N9KzcDjwj3qmPeyXiFIAOX5ToXAWM1tljUOfpniwxv69bzUkSZqwopI/0 OK6gMIYt2GakcSIQrqHBuvGEKEM8I7Aa4QdbO55J4T6qYMqO4xyV -----END RSA PRIVATE KEY----- ================================================ FILE: tests/fixtures/server-ipv6.crt ================================================ -----BEGIN CERTIFICATE----- MIIEEzCCAvugAwIBAgIUPVE8WTSDgzGco/kjCilmiddHMvwwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQDmszDivhUchvYxLdlHvRonIlsQ9VRVO5C4iQwrTzVTjb9y q0oXGx1uQGNZyjMMStsZ4Vgi7UqBaEe9z3DIQxe4rzEn8B9NqWS6VgGBI3N9mjiI BBNzUdJOoaelFqFbIyzL6cxCndovL6hURTxJTLI80dM5pdhfddPckfXmjD6qrZ4k 9bhIyQX+TZViHs6il5HWMJi9XdEW+VCBZ+Zaqjb0vMbBh5mEZIpYCdz7WeoowRUl kcP7AbFg/PzP6Tg5xe5sNmrSWZSB4QuGfTV9EMTFVA5WqI2Z+T388IOuh5DEw6NL swHME4eMsCwbZYees7xZh0tERDZUeOJFAifNrd0tAgMBAAGjgZ0wgZowDgYDVR0P AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB Af8EAjAAMB0GA1UdDgQWBBRhyokLlyprk+9tLhouzyinfDZuHDAfBgNVHSMEGDAW gBQpJiv07dkY9WB0zgB6wOb/HMi8oDAbBgNVHREEFDAShxAAAAAAAAAAAAAAAAAA AAABMA0GCSqGSIb3DQEBCwUAA4IBAQDBHcGP2z7UVPCh9Tj0M79mLPB76E9BtTNB 5cadSemk/itnGol4K5x+BILqRvQpKbUm7Yif4XVKtBiPnZothEg5mxcTGO5n3EVi Y7KmeVZxUkPESQQVnM36ymG+jzSf00KeGhras71ddbAKZBVm+nsL3j1pLz+MGksV m8xzIW/ilM/zL8ivlSy5XBu4JqJET9O5vs4RBYmzNNC9D2WxxNMm2bAxCd+1Kg82 6TLAFGGla0e8fG39TMfLeQYqHf8FdmGqkhhVStjvgRPnVvEK6Uv6ZESZZBgOb97O m4BnW5gp9abS3mTYdDMo+TzgcKSlTKbcRV3SHA/9Cjih8IkOdIKW -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/server-ipv6.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEA5rMw4r4VHIb2MS3ZR70aJyJbEPVUVTuQuIkMK081U42/cqtK FxsdbkBjWcozDErbGeFYIu1KgWhHvc9wyEMXuK8xJ/AfTalkulYBgSNzfZo4iAQT c1HSTqGnpRahWyMsy+nMQp3aLy+oVEU8SUyyPNHTOaXYX3XT3JH15ow+qq2eJPW4 SMkF/k2VYh7OopeR1jCYvV3RFvlQgWfmWqo29LzGwYeZhGSKWAnc+1nqKMEVJZHD +wGxYPz8z+k4OcXubDZq0lmUgeELhn01fRDExVQOVqiNmfk9/PCDroeQxMOjS7MB zBOHjLAsG2WHnrO8WYdLREQ2VHjiRQInza3dLQIDAQABAoIBABAXMXKvJVPPCf7W HtCFHPzbxZRCODaVp/tm+6VNqf+A5Hh///PqnTviW8uYccUKt4tvjzEoccji2BYi ENC29UGZXolVkylchj0E4Kf8LAL3rbe26RBjBZMcbU/zax+rLWWvkeKXle8ymL// 8DuAkPHzBJOBwLyvwC4jNA53e6t1vKp/j5bpR0VZHFvsGSvGGx3P0l34DwHYjdL7 +Q6jwaoIYDA8rcZHXzT/UCHa2dsk/m1OzFKUdyJVkZ0JIqRBhBmG7Z16T9xfoEZP Ycv8TudQeH4sbiedwCYSmkfU+CtPzGWhZ4jjBegujJwrQJwF5TbRsqwHv3JOPE6K hhJTs0ECgYEA8v2JMqFTwKhmh6EASI0YfLvrP1LNR6K6Hhdoe3/RYP2Qg12Cm7rv 7Eq1kpptu9eSnFRPdS16bTyRzTa1/eEPPjvTxCalnEOVdPSbkw6zr6QFwQh6HMrh fbLQJy1jY/Gj5JkVLMK9l+iLLY9vJ5ZZPfJYdmlZ1USwbcsbF0u37hECgYEA8w0z ZE34FsKWXdcMNG97OWYMqXWhOlLyxKtsWGUUEoTdK+PjDTDVQVe7KjjhOFehazD1 mfDS1VGwBPHQgjxBDqreBc0HvA4o1B+Js0SiQVbhxkvyDHHnxI5MeUROPCeGljzn lPM9BnrX1KurSNS+j9YwtUiM4TLcMMdBXdv5UV0CgYEAgudHRDlZD08pfSOlLXCl onzyLPkEkfT+YzulE/M17xRrB/oWZKL+ocNVshbzyuBFoWZiL/RCIhshSPaScKUQ Oyyr1t4jFd3q5Ejqjvy6nIK2ftl8P4qkk70DGjf/dVY2Pu6hU63NycqDQBYngaIj jZXDRndW5+fLTDrA63nlKqECgYBxTj4fDJoTQjOHG7F84Fu5rnFIrqWy4uh59tBT hQuOdpIE3AAFLja8d4GxdULJWVDO/8v/L92ZxLMiGvjxPdW2WMGYQrTQXml6Ohmf kOdzPmWSY+U7F/7MCuprvgQa1vJPJ6VuMtbIJoxngIAhO8x6kYeze1bxxRwRQVKf xuS7oQKBgQDrsuXe9dKZU4lLTO+mPCHZPgSwY+3rFvpAFjoM9YT2/H0EwBrBubtn NYyN3ih9SyaJdO6RKSEn822sz1Vc8A0xqpjyhUWDxyRCtc+GyVKr2WIC0DraGUjn flBpRAox7c3DiPQLQXV9WxFVscM1WaNOVEuuHwmUmaM6UXJRXT7jOQ== -----END RSA PRIVATE KEY----- ================================================ FILE: tests/fixtures/server-revoked.crt ================================================ -----BEGIN CERTIFICATE----- MIIEEjCCAvqgAwIBAgIUSjz5+dzCGgG0oO+u91Rqruj70ZQwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQDs90tJyp+StNGHa6spxT4Eeuok40ghqxDwukfOqCe7rIJj 4pOYD2Tk6kFIlCJr+ug1fihWtTVVo0ZKUb4pwhXVDX82HiCyLd7OV7cfEKtnec3q dUoJ/SR2iyjnVvRubL5szw9hx+XpRC1jo7HweBEFtCBPHPYdM67k9L6yfOoSkqCF 0/aqwbmJop0QSZa4RPEhkvyDLPaNtPnqBcoEF6SRxMSlJl1lVORIsGR5Q4/eZ62k 2ZMy8IwSqFeDtAWSSl9Gqi6cD7WWZsSqYWV06g/8mRj2zRubJuuaZx2a6QMv+Cpz e32EpX2uZjpT2emCpRFDsy2RBWwLDJrk2+W7Piw5AgMBAAGjgZwwgZkwDgYDVR0P AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB Af8EAjAAMB0GA1UdDgQWBBTEZsOjdDCzMXxW7aVXJsWd8Wii/DAfBgNVHSMEGDAW gBQpJiv07dkY9WB0zgB6wOb/HMi8oDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A AAEwDQYJKoZIhvcNAQELBQADggEBAGKuQ05hpgwhR2Rdd82mHFc9ItnlW/G88V/p 49TuPpDWJwSidOTvEkzUdwVPtIWoZp7GNB8/ZkHIf6t69UxEiFqltkDCIo/VJimk 2Zk5cgPKgdBaZaufwAzSZWfpnX9k7IIi+gVjGBhYqw8a159AdP75kNjj4oYjcYAE 8FS0K6srsZVf2ER625psFsJG8ZVOjJOqs7fk32aAlCXSwCnOovf9qVlJA40nWi1u pPIA3vW8JZ1UaYB3GKFkzdDaIGm9R7STEMx4I1gba3UewzN+a/h2Y7gQAWHs9VuR /zTcENPQuLafN/+DNDVO/DmCgNzO9H1cdvlyUoh5VaKhmpRrw2M= -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/server-revoked.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA7PdLScqfkrTRh2urKcU+BHrqJONIIasQ8LpHzqgnu6yCY+KT mA9k5OpBSJQia/roNX4oVrU1VaNGSlG+KcIV1Q1/Nh4gsi3ezle3HxCrZ3nN6nVK Cf0kdoso51b0bmy+bM8PYcfl6UQtY6Ox8HgRBbQgTxz2HTOu5PS+snzqEpKghdP2 qsG5iaKdEEmWuETxIZL8gyz2jbT56gXKBBekkcTEpSZdZVTkSLBkeUOP3metpNmT MvCMEqhXg7QFkkpfRqounA+1lmbEqmFldOoP/JkY9s0bmybrmmcdmukDL/gqc3t9 hKV9rmY6U9npgqURQ7MtkQVsCwya5Nvluz4sOQIDAQABAoIBAHcRIxFm8Jtko8up vA13AFx77l6unTXdoNt0nlQmhiB04+eQl5zWT1n+ouL3G/ypzDfkthwrXSs0qUL6 o9SToyi0aXEl3kPpbIS96lN/qsCJoX/ng1ZVjhbKgbkMJjG+DkjaGd6F9O4qxavF OsmbauI0ye82nCu8JmsA1zkULwE5GENavHsAnHx7xKLZROWlhO1RH22S6TK78cjJ Hu0wBoUZQFDAErN1GqM6o3VnL/GGCS1hAqvs4BR8YJPxukIjSL3zG93QH8fy2jXG suRNgczwfW0f5qxn9CDBA68vIPWsFWYpGgWIyEqNf3LllQ7bSApjXNX+PFGAxFjW nQdzW2kCgYEA8ZsIBbRraUYrydvfPVPIJzbBWGim408MWY2bsW7Cy5o19VXr4UPA llFJDAkXmJMDjHKkYu3rHcBrqu3/CvDe0YpqImdCz8CjP5dzy2oJMaHRRa6qNzjm fbE5GR29ZEofxraaTSZCBicXXu/vpG7W3zqqlDc+sq6+zM+hrcX0p18CgYEA+xV/ 5YlSGp0sxq8HWzBh5syXy9LcGNRM81nrcXQ80/OoNFdba4uCoCa459aZCo2YYanJ eH+8BzYDetgEcacNVzbJQum38StGB5R/y64NjZgZowIA/68V7qDbRFf1F59zZWpE vgHa+KE+dCXK1PTUt/wqFIcrIajSyhMXAMr6S2cCgYBR+WPza4+2HFTnHG7WBAM5 Kt7W/EsDfOKXz/Avd4EoS55bK1fpCm/hkJrUNpGG9vqRQKR93HOVmJ/vUujh8W/o cKoqGhcVHitFfEGRltyftmOm3Ohr7CZoJyVUXD7SNEQry/D2lDB6nfDUCVyp0eGd w+30c/oV7ixWmWwl5bBoyQKBgQDlGsERGTQpxLFOufbkZklu59C60zSyE0YD51DG vWGjpPkeiXeJsksHB05Bfbc3wewBcYO8yBEyIz8ZoHKtodiyc/NBczG8hdfoor/Z goAra1Y5P2LZ61D/5RcuTXP+kighqc3/8oFzzO3H3ZQurRhMqXNcN9pLZFiyuqiK uKuakQKBgCXr164GYD3iq8LUNRZ/wHWnDEGvL9u+0ZKviC3SZA1wRdcn5m/GpmAP fzV6NnPwoUkXA2ees5G2aYGuWxU2laqFFflqzuECs8A4XQUwGqoSs29lt5fFm1Y7 sNxQwwyEU7u4n2pAGzFjILAIUmUGrw6dpiUtHFDEhxzuHIA1uHxR -----END RSA PRIVATE KEY----- ================================================ FILE: tests/fixtures/server-serverusage.crt ================================================ -----BEGIN CERTIFICATE----- MIIECDCCAvCgAwIBAgIUDVfELNb4NV52PJjENU2DNOIFx6IwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQDvx5mE6ZwV61si7QK18X+K6lJSWr8rE+0l4YbvbLh/bCdP bnPzTsiK+FN4FA9fur73L/RvQDxO9XG99pKIhKPGxiit6DSJ9rL24RpK7tJpgnnK Kdry6E+6/ZvVXBP3LtdJyX3kpdw+TSivlwi50CeEQA5argOXnyDNGIOWC8iMpeg7 z3pE0DfpDEgLMjGE/I/9YHRCOiK/8kQchXqVfWHpALFakf9+QPNpNrEIPjV7IPja GEUeDG9MI0PYbBl45NkKIi0nelV/Nay9kyPSPKfvngJU2dTqsGSVXW+DlZo7OJBh RLEih+CWGfotGzFzXWdQRvD/VleaxYYUDKkyHVx1AgMBAAGjgZIwgY8wDgYDVR0P AQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYD VR0OBBYEFKdzgCE8K0tYt9LaF5N2aQrrHru0MB8GA1UdIwQYMBaAFCkmK/Tt2Rj1 YHTOAHrA5v8cyLygMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG 9w0BAQsFAAOCAQEAYcD+1ebiJna7dfydgw/yox+b6/KO16evaU4c5Spu5O+VtcZG rKMi8MT8V0U7kL9Xo9TszbzwpPWr1It0wcmM1cZrwykkT/baVJADaLtfSFUtlCDX HNB04C2UUBPPosFr1d5YtwyN55qxgyMg+IDeMubYZ4qwDWCYBTIiz9yHoQ6LuuV8 Tkkpa6X5n4+fO2iUgA6SZUkwZGdbQLOz9VMa1qgyOz3ejuDeMc4sa08iADs4wG/X ohRGg0Df5THeXhR+Pn0HBf3T0eTAeZzLL5xtlIn9o6o9CEU573uEYQI1BG1kcDeQ Rs9J/2iuLqr8GAjr7k5aUW3FFYRtqC3YR0G5Eg== -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/server-serverusage.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEpQIBAAKCAQEA78eZhOmcFetbIu0CtfF/iupSUlq/KxPtJeGG72y4f2wnT25z 807IivhTeBQPX7q+9y/0b0A8TvVxvfaSiISjxsYoreg0ifay9uEaSu7SaYJ5yina 8uhPuv2b1VwT9y7XScl95KXcPk0or5cIudAnhEAOWq4Dl58gzRiDlgvIjKXoO896 RNA36QxICzIxhPyP/WB0Qjoiv/JEHIV6lX1h6QCxWpH/fkDzaTaxCD41eyD42hhF HgxvTCND2GwZeOTZCiItJ3pVfzWsvZMj0jyn754CVNnU6rBklV1vg5WaOziQYUSx Iofglhn6LRsxc11nUEbw/1ZXmsWGFAypMh1cdQIDAQABAoIBAQCnpgkisyuc78fy 7YAdslKY0Cjqx+QtvGrtN3he4sdE4FvD39hWX9k7wVCq/muZZTqsHe1r85+3HUl/ pmzh4suX6Wj73wUNCV4r20vE5KJdfwqkXQtnFyLX/QX98blL9IY2YxkQyx7ouI4f 5xwEvxNCFn9yy4RbeLk4bVFjka2RF/x6qEUCHq5Q74vWvyC1i3kGKgYruM39RQw3 D5fG8xdUexBc32nfzynP+0NcFAiy+yUQWOLcE4i8XaegFvg+QvWOx1iwjqU3FDeC JzKrtw9SLBWf7AGraxA59K4WJ63xqGqFugWcFaYh923X8zES/s0wrtV2T14Lgj3Q aWJ0DfQBAoGBAPNd1Aph0Z9PwZ1zivJprj8XDnFP2XIqlODYuiVddjbvIV13d6F/ PE/ViW0MVduF0ejkPs9+hSxYOH58EWIt44Li/Nre1U42ny+fJrY1P5Mq5nriM4L4 lx2YFaWzAoxzpMbbQ14kEMcQSicziDbBx62aaQYu4UwrvqXYdSYp+D+BAoGBAPw6 Gtv6hytg19GtH6sQG9/4K4cLGX4lJE3pTL3eUicfoEI+hviZdG8FS77Uv05ga6fQ OlyqvpmmXp6fgTrSlHBeKO75A3sT7fP1v1foq1y+CdMGytOnJENUc80bN0L1dFI1 zwYm7eLDP0KdUYpf+Rpgcap4StQbotpc6oy705b1AoGBAO9z26VXd+ybifKE9Cru Zp727aP6IAaf9RqCxCztl9oXUanoWVISoeIfRfeA0p2LPu06Xr7ESv5F01hIdMY4 RonLE2W7KP+q6NfvbSSMogAIjvxLwslUFUPuFyaRSqmtQ2zR4qgnLkbfNUb7AkR2 SCT9L+cAi3bp98ywfRvO4c6BAoGANkAJJudry1i5EtA5z4FXfYTTV+h7QzaZ6GgV qYD4CpIy1gy82xumf3qUICeCPkle3mlbJDNVa5btIxELqqtAYiregwfsR7yxoZdp 4G6a7Qey9UCwv3Vjx1eS0LrZ1/0TV9ta++fDotJ7+Mf9kdWyromv6QqWjaikDnON v1dm20ECgYEA6i+uvBuomUzqJYjUTNgMtmfYjuXv8D58wLkY10si7T2ayAlFKBG2 Dno/dojOcixO1dtaq7dA+KNXtESekjT1Y4VleGHWpEglRScE629iow4ASrluP/Ec F2DvTRW4daFDWQV4je1u0+wDj5B8KZjO/e759BztiRyRqTCzpxTa8Ms= -----END RSA PRIVATE KEY----- ================================================ FILE: tests/fixtures/server-wildcard.crt ================================================ -----BEGIN CERTIFICATE----- MIIELDCCAxSgAwIBAgIUYcN3lj3pyfwV4xtQPKH6l61/GHgwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQDpVKzMA1J6nfusMHTkdLzTh25BPsP+LI9qVhGVBnT3Xik3 yZ5QkbpH3kRRj7krRvydO09FccZ4807rX+pP2V7NNz9k9wuOatZl62/1uQFgtpC6 IEj0Y22w4BI6RbmWsoXG8k0/vr3r3J+X/RdRI4Zj9qjDD8YE0SDapnmpTZ/s8GKY fTkv+vadEYvRS2ZD1UivNlMqHjL+YUw24slTYY8vlzgBc9sB8nC4bnXKBbBDCvUd iHBHqM5SFYzHEdViDRUT6SwBE2QmODsasYJRGUPhqabNFaxpP0xa6D/uQoQZ4UTi 4Ljw0N+j5wpbogJ/Pf8rfewTKobQSO0jnZi5nmwHAgMBAAGjgbYwgbMwDgYDVR0P AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB Af8EAjAAMB0GA1UdDgQWBBQxIm4m73svrwC7gqsRhW8wVL5CyjAfBgNVHSMEGDAW gBQpJiv07dkY9WB0zgB6wOb/HMi8oDA0BgNVHREELTArggwqLmV0Y2QubG9jYWyC CmV0Y2QubG9jYWyCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEA yDzgXQZHgP5uFuCCsoGGCSfOiUxEcFpr19al08mlL88Hw69QExEJK2wNmGoVY/v2 fv58gnUvDzJ1+V2MTJt7NkT28d05pJ62ud6auH1I3SwREN1mizXAx23P9sftSGln gdTFnyS/9U15BkRJSYe6w1Bm08GRsosVM5poKbYOol+Cx5WyVQHedf+GFoTSu5lB 16mKjYscvezeIgWCW/X8MGju8HbCOOV/SKPJ8MCK4fBxdESTuTsmM5rZj3n9ROY9 mvAMcLEDa8xhRHkdls7pd/ZsypN/blbVMBskCldiLLRb6FwI6sarBzihofW4nwef S8e6/xuD4RxpUMOcMBuGSg== -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/server-wildcard.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA6VSszANSep37rDB05HS804duQT7D/iyPalYRlQZ0914pN8me UJG6R95EUY+5K0b8nTtPRXHGePNO61/qT9lezTc/ZPcLjmrWZetv9bkBYLaQuiBI 9GNtsOASOkW5lrKFxvJNP76969yfl/0XUSOGY/aoww/GBNEg2qZ5qU2f7PBimH05 L/r2nRGL0UtmQ9VIrzZTKh4y/mFMNuLJU2GPL5c4AXPbAfJwuG51ygWwQwr1HYhw R6jOUhWMxxHVYg0VE+ksARNkJjg7GrGCURlD4ammzRWsaT9MWug/7kKEGeFE4uC4 8NDfo+cKW6ICfz3/K33sEyqG0EjtI52YuZ5sBwIDAQABAoIBAC3SKRTvWhUmTTQl V+89VY+cuvQpJUgW7BsPx+giGnoxjZqdB2//Djvq1DPIK67qA9XEve5/R2CdN1RV w6fmog1e2h4zvZs8M9pT/+qbaD/b2lQS3wDPPc1MU4gKBUYozMii8LSh+p4E93pb g2a1uUCMQdv8jwCHKRKHOsEas1tOAaZheeauni0e/RIczsgfvGuzgBRO8c5PLmA5 vh8MGxoKCvXQRrG+l3m77Ni0e6jj+p/3rA41Y1iNhj+jAgcsi5N1l6GsBKr5rFMp w5pmfkLWnaN1a7Z25KWrVEsvSnxQpoIRXJt9mrnc5IztEtL0Yr49VQRWGSCzrXBr xZrnyXECgYEA+6ClMRZCubvk4xZewVYGL4we3L3VpwbHkYgIA/wBttZYm2oBl09a eERj87gC2Qzkpa8HcS20//KFxVNYzW1ExBqicndZfd4I8M06eMsjLnsDP1wLnxQZ dyroxoqiEkIVWxautq/VrIO2dgAgVDHi2AJsdc0n7qnx9U026vqxx0UCgYEA7WKj J7lvk/fyVZAe84VAG/kLDgmkcSD+kGf1vIie24dABiMjPxUb8cNrP1e5KAWpe40r lUhPWroW0xXPrIhS+6PrcTA4/2ebJ4Qf0aAtDL3gHnB2E/BTuDb12PnkrZqg2jBK YctXIfNIcNXYVP1y6Lq10WvwviBG2nniAPUKZNsCgYAJsdXLf10QxOF7slfyQPs6 B78EqDe8GLHFtKUCakoyni2Jx1rKVp9YtOHY+QT7EdkZXRX/UVCA7/ohcSWhvI0C tTf/CwQiqlRT2sRe9Qyk9M5aOZSlC2QzyC5xv9OgunUSLlyK41lrLSPxhe248LcZ tXYyT7YzJs8QsWnlQcVptQKBgQDPwJ+hyHyKN1ly4Kr13Qx6br7qDi5Ig+PGZfV+ huLgpcG2nVHfh43pTGm0CgYVrL7jTm1yPNKWSH5pRpF2IejeKluHt/hqLjZvowZl 45UJrbNTcIEmehILCq6msi0cclOMIO84H0mmgNBJUB4AY8AJRj6RhbIv8vePhVPy GoJ6OQKBgFWdHMY6qm8avVslWekWH5zA4acqg1jUR4t/8HCqEFMrBhxCzWcJJhsH kqzUxxZ0dceqaYlieG1CayImbONpqhlVtUpjZ1HhCQ3cO1qK8fX2raxdow2BWAaX QxnME2lB3Uxn673IAF2HyjLKpKwupRpg5QUD7Zw5s7P7FdpeHLlE -----END RSA PRIVATE KEY----- ================================================ FILE: tests/fixtures/server.crt ================================================ -----BEGIN CERTIFICATE----- MIIEEjCCAvqgAwIBAgIUPMH88Ekdi6JPq+5703+qGFH7VmMwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQDD3vhO7LT77scWKnsozEg1DiQQsAbgGFfAoQJOvgrRv7V7 I5+9n7hlpqKEIYkOuX0LSqpLBJ+9ORxXPBNZFKsytryOc3ZWTAoozUkufUOKfUFa 1QpExA5u/FpwtNXGRGC35wus/JVTtcIiifeml2PIdyoxdXfev6y4yJvhO38Osqru GoySDORGsPrmLdpoUieofhmHWEgONpoY3fVsqAwiP1NMDNuqbVHvDjMykxj9AmQa WBdspsRXcgcl5Dp7mf1KVPRbvnCOjLuDDiBCwTTgpU3sDFyhHm/Bq29lPwlHWi7Z WKQYGIwqQOfOjfjhH009/+z11Z/1ovj+FWLbcXP9AgMBAAGjgZwwgZkwDgYDVR0P AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB Af8EAjAAMB0GA1UdDgQWBBT0sUW5xBildDtaySEQYE9vX+8SZDAfBgNVHSMEGDAW gBQpJiv07dkY9WB0zgB6wOb/HMi8oDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A AAEwDQYJKoZIhvcNAQELBQADggEBAATMlUra5N3Z4KB++xmGM6h9OhYmbKqn6IEV o4mqrTimyzgtvVsHh/v3XvBzxAdma6QjkZygfg+EIHSLbJVmZzxzr3YeENu4EyDc l+FfNPCyFHX9CH2Rk1ZThkQbmqVrzInXmG47G/PbTC2l8+kAZwvp37QfIJNCYIku XTp9R72sEkfNXxZxsZjwM7Z++LaB+cVEuLNJG0OpMhouTuxoN6pemzBmmFBP2mOr SeZnuEVtvDIbklJDdgcB/mPd7FE2xCVfa+p5Ol9Fcw5aPxTXAAQ+aVDtzh7jcFWk SV4K/ZYFqYN4E4H1UXlkHKj/qCmryvPxe8DVzu6Pd0ZyA0H4Lxk= -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/server.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEAw974Tuy0++7HFip7KMxINQ4kELAG4BhXwKECTr4K0b+1eyOf vZ+4ZaaihCGJDrl9C0qqSwSfvTkcVzwTWRSrMra8jnN2VkwKKM1JLn1Din1BWtUK RMQObvxacLTVxkRgt+cLrPyVU7XCIon3ppdjyHcqMXV33r+suMib4Tt/DrKq7hqM kgzkRrD65i3aaFInqH4Zh1hIDjaaGN31bKgMIj9TTAzbqm1R7w4zMpMY/QJkGlgX bKbEV3IHJeQ6e5n9SlT0W75wjoy7gw4gQsE04KVN7AxcoR5vwatvZT8JR1ou2Vik GBiMKkDnzo344R9NPf/s9dWf9aL4/hVi23Fz/QIDAQABAoIBAQCKvee5UCYqxkoz Q0gV8A29txSI1YcpOVT/V41g5XCYfmk4nlVKZlahelVnrrF8wpr2Yp8ZoF7eFBQl HqK92Mwjkhkh9lt+aUJRAIiz63rqICspAfrSFuX6a7pMV2uNk2XHHlvA3vGPaBHp kTzgvh+qIe67NfAA0liwUzlHY3NunpXW6UQm2OWtabOfc1zZ78E58It0VTaAWKZq KDOxQGdwS4BGIUbsCvktPgncDDzi8wIZY/6YDTXkYKUJWqVEWf3SregXbX6/cjQa x8v2v95LiMk0vp9E5GG9QKdTEC+fZyQsq983G9t4O9VPw2JBR2TUxIRrPH0gkvhQ F1n3yYABAoGBAM1qHX1wQoHiOpVsgaXrAX2QWrBeNpbGAgVh67MsD+FlUH/ZikcK dsz/pTg+TmUuURKadXWJx43E/IVb6Bw8io0uF30aXeWRSIDK99FAmx2GxxTBge5f MtAQSTctr//yYLaNZWSdTpmaQYRtenK8zN6OTQ251M0sWyWsv3UcpBABAoGBAPQb NFqaw8C8JgJtHa7wrYCU9ShNxzQzo/jgu5ZuO6CxKNYyFGR8ZOpWQx8vTrNkjDXN iq9oZ3gIcm7c4TQOg3ydISNyoAEYrhRgQC6+rDkl4Zgby6ssvDHMqCclMp5Ztn3Y bfXG2gk3ULLH8TkQ2KWRJrpPT3iSdvLVXmNHHaP9AoGADAxhVm4zOHMQhJssr5Kt L7Q73YRpJ0bN74rizEuVUt8ibZ1Q4wHWHggQpM/iwUSKNNEiepZuQf5/4UKWxrE2 XzmI3ymgwEpZOlStXHSxpHW3T5xaBqVG0bVi1f20CQsqaQq6G8CuT4wgs6fIOtqg GZ23H0r7FF25qugLAs9/QAECgYBsi6ROHb+p9oAYWBj474DXSmVxVJSd+9CQHK6N h9rv65czF/XFcSMWqOET/t9KGg3W5t0ifpRz5Z2s+n8RvNpvERfpQVEw656M5Pfl UVgX2WZlUwbPyQauRkkHjxzhGRdzAkhzH8dYjcZOmWYEcB9GEDNeaWH3RXmrJYHh N4BQqQKBgQCtdULs1ju0Y5zwSdkxhBWJHxCpLjuCpxfpWmnVFlp/Fj9S7rBfzWrt hYjsNyfMUm13jg4+6CJNdf3pQbdczXWTt/E0t/uoo1m8Azcd1qFPKV8wVOSiZBXv mMWHW8fw55ygoFyBlQpzwI6M+Zpwv3Smli/H9gQClsS2RmdIjUhqtg== -----END RSA PRIVATE KEY----- ================================================ FILE: tests/fixtures/server2.crt ================================================ -----BEGIN CERTIFICATE----- MIIEEzCCAvugAwIBAgIUJFakdIrS8b6xWLAXMmEpn+awNIgwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMHkxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEVMBMGA1UEAxMMZXhhbXBsZTIuY29tMIIBIjANBgkqhkiG9w0BAQEF AAOCAQ8AMIIBCgKCAQEA5QaGXSEZtAlH/JFbe6lZWl9WrbVJ2A1ieZroszfBF4u5 +LidfM/E7t9GrTq35pb5Ie7V1kvoo547r0yNubQPgU/DNGDIV1CXe7tSrkMLGNuF hVHr2IAnRJL9DZUFDqlkiimBz7m/rcHtwX+5v54YBh8L2A9p0Cd7EfgPEWBQBXBL XbwOY3ZpdIn6jAtyAr3uME9RkH1veDomI/qlgD6t5/IrucFkhLSqzNYysMYhlUxG JABPLWJI2qalY6f25o7JtbErjrDphosQdSLw187KvgDaNi++KnOk1HxRBWbR+3va o3tr5YT8h0yK9ZlsAQwxwGELHtFBSmjfneoF54YuYwIDAQABo4GcMIGZMA4GA1Ud DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0T AQH/BAIwADAdBgNVHQ4EFgQUzW7RhcPbhjbvEryk5QOQNSBdTPswHwYDVR0jBBgw FoAUKSYr9O3ZGPVgdM4AesDm/xzIvKAwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/ AAABMA0GCSqGSIb3DQEBCwUAA4IBAQARWyOreuBLCuo5W4bFFkgcUA7xFwSGqtgN 9LxYpLT1ADipdDvH9/XGA51plGwpNT8/99ipgMjhpGZTS1iK7pHwaBAdAqRRqT1k MZ5X6KFeQOp0uKkWMJiyYaQmJR26tYDeWzwWxYD/VXI3fb5ZOmNRjDpx2uiqDTPX kKY3U0vHGzbOWSRKEqTGUJ84yErSdPRK5gD48oqtUWczRbnFl6KV5pFb0Fq1Dhka GwzBOWLuQTSxBU12LYAcKCLIweE21i6GJkxLDqtzI1RS7BRDsaE2cRWJV/6VqqFg bs7FtNqLm0rdxdp7jnq9GCAXq5tFghTP56I9YkJLaYkdW7vjXkc9 -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/server2.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEpAIBAAKCAQEA5QaGXSEZtAlH/JFbe6lZWl9WrbVJ2A1ieZroszfBF4u5+Lid fM/E7t9GrTq35pb5Ie7V1kvoo547r0yNubQPgU/DNGDIV1CXe7tSrkMLGNuFhVHr 2IAnRJL9DZUFDqlkiimBz7m/rcHtwX+5v54YBh8L2A9p0Cd7EfgPEWBQBXBLXbwO Y3ZpdIn6jAtyAr3uME9RkH1veDomI/qlgD6t5/IrucFkhLSqzNYysMYhlUxGJABP LWJI2qalY6f25o7JtbErjrDphosQdSLw187KvgDaNi++KnOk1HxRBWbR+3vao3tr 5YT8h0yK9ZlsAQwxwGELHtFBSmjfneoF54YuYwIDAQABAoIBAQCPO6xutBPaJ+/Q gqv/Q+NxBK02CGo9Z+mNehdMdnMZobZWWkeMVniomBUgo9d9rC/1S+SKmIDPS1ey g6MjX/xOeC7yJBFHokyLApVsDNv02N3BioGArm1gkrkWdHtsNv589gaMfnPlXKKw YIwvzdTihyomH0Wi+/4ZN9VcnaqOKwTmTFzuWZ8JdjRQZnXfdyL7opaWahG9P9UI X2nhre73fxeF1OIO0lOeDtxM21/cZnzlclD5mj3OncXC+dqqqmxyMnynaZ2Wzlsb CPZA9JaNPCMjWwJsd1xvKH2DcI2hGgSwvsho/mgsMWruHBHiSkIxwOgYYTRDrfvi pUL1f+55AoGBAPBVR9A/mzRkVIcMIw4efP/DWJHkz68L40+MNsH5qwqGdcru4vpg HEdi7LyHDn3j1ij9UgJTg9b2+6BX/KlQwXw9cG9Xao8Aqge2kAmjlUIwvsqhU9DN ZrAIj+B2s4yksCXRqb3IqO3cFjoMPzw9FHV3NMojRMrmwOtiSUQtI9TtAoGBAPP0 imIJdS2pUzoIo0ceByqn9xQPCLwGK/b5+QCOVDnZAbcPoilJnLn6sEQKoJs7P+ds pvPRBf3CsUHJ0jrJvf5GOUjpSxTsJkOaan8pHULpSwcSToWUM/5eRxAwrKGMlmSM iwNjoGh35gfc1eYZbElxVVCyCfz57jKs4jW3MXaPAoGAVNfGYl4SDIzeyk4ekf1x Y1kzC04bg1BPDuYQ7qmVGEIfk2SB/KGxWgIyUNvc4dRs5kuHiAqzoE/QxOpK5/r6 U0HdT3EszQ8O92obr0twhc1vjVkmna/lcH+VS0icWipJhRBfPAB6on3v2s44BKwL bOyIVlPdFUQhFve7pbXJ0IECgYBHeVEF8iFrtF1W9mroDispWzavoMv9Uo2U+Z3z hL+2hxbSjHkFQbTyZDk6Ziax9ET/x7yOWKI5u831KW03nh3VHrvv2bIOujVnvxkO knwpO3Ko6rsotcgZ8YM+ghRB7I+ve+HKp2i60s4JZbEhjjdEuTi2wMLeZFdeb3qD JF4QjwKBgQDiS7vic8gwLq3Txhcos8FJ7SfyjqAT0yHXk2Mmw3YedsjsyGTEqRTQ LAvp/B4f2jJzc5OurFg0qJSomkQTru0EbVhFAnD5G8HjpcuvHVunamZLT7mlAsg/ A1DZFvlU5GyiL4DdtRj4NemjiBRVVKRZlgGCmZxQabZHfuk9qVzTeQ== -----END RSA PRIVATE KEY----- ================================================ FILE: tests/fixtures/server3.crt ================================================ -----BEGIN CERTIFICATE----- MIID/DCCAuSgAwIBAgIUQZUvcjm8Dc/5ehERIxy496qys2kwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4 MDBaMGIxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ4GPkb+yxbC eKNxmv9OT+mpdDW55yfvPTQiVo4iU5pWl6/m91rbcpg0NGfOtZhFLpKnpEs/tjMY 0BRTGsL/szApZ6O/vdsS4D6kzSySuLP8a6eNEQcj5eL1g79yQZQUbs6iAnJXXV0Z 9v3YJ3UW1hU9VahcYb2/myIjIvP8El2ciXehwHENt+YZGMkP7DRYimCQBUYdQ7Of sY3ojwarrqvjvd05VXgWcfloLujxCJVwwrUHXvvifdByVsU+DVCkZGuhIwO8FKa0 YaJbp2b/PXHt9AnLNsphyy20Tu7SL2QF8lO59w9044mRvdQ9YT+E5jgUk6CCwwpf yXVcbRUltbUCAwEAAaOBnDCBmTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYI KwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFHLNtdcN WCzXJr5GFWZRQOX0rCMQMB8GA1UdIwQYMBaAFCkmK/Tt2Rj1YHTOAHrA5v8cyLyg MBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEA FCfeTDh1DzpNlcxbublMCbar8ZoWHXd7k5BmWJBqA6qmRPGa6FBZ+8J1jUs+7ZFo h7Cd4cOfn77+VxGo9Rt5bZ4t/luZrMNGVDO52CD8eSR9+iZ1OMTtYod9cjuX24aQ oXTGvuwVnSXBxqcettCBfNOCPT8oWIdDY6gTUzl3GjiFlX1Jl9nntCAqxO67rcDZ KzX33NUewsOHuA6B7Plzhkw+WNUTHvrlrJBolWWYZWArwR9PrmmsmynXB7IXU9vA HvFpOZ5vkqq2lV0+Sk4Rwg0EQX0Wf9LyEEDBcCNTOqVv+Dbj0+i+cNPz/T05vkA+ gxtYmjZz+g7NIVBSMyqSQg== -----END CERTIFICATE----- ================================================ FILE: tests/fixtures/server3.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEoQIBAAKCAQEAngY+Rv7LFsJ4o3Ga/05P6al0NbnnJ+89NCJWjiJTmlaXr+b3 WttymDQ0Z861mEUukqekSz+2MxjQFFMawv+zMClno7+92xLgPqTNLJK4s/xrp40R ByPl4vWDv3JBlBRuzqICclddXRn2/dgndRbWFT1VqFxhvb+bIiMi8/wSXZyJd6HA cQ235hkYyQ/sNFiKYJAFRh1Ds5+xjeiPBquuq+O93TlVeBZx+Wgu6PEIlXDCtQde ++J90HJWxT4NUKRka6EjA7wUprRholunZv89ce30Ccs2ymHLLbRO7tIvZAXyU7n3 D3TjiZG91D1hP4TmOBSToILDCl/JdVxtFSW1tQIDAQABAoIBAByHlAbNSW06fv1D LXCaeuL8rPZmMc2L68jVyjqvB9j9eTVQxaeppu7DvhJfx3lORDJGAetz/TkMacTB nDtIXtl7IDL4ExbSOZoVttUtSBt2nxkI5uIbIQ3wtXCC+EP7zGWR6k8qZrjAT09V DwqcrNn40NYsl5jiVue64DycbdRofeE/GGyp55LkrIavI8ROPDHmXIDleHr/fnS+ /3cwmRl0YWJjdFEuDsP8SIfH87UC5dlEq0q+6XV+Rm/sYWKnCMDERXLzIUfr6OcE G/dC8OnjhkLfAykM9runu+QQJUtylqwVZbM71Gr0GxsjU0b+cWsR2H2gd0wChoJN Au89+wECgYEAw3GZ7VwOn5d4bmYt4t7tVg3XXw1EAurG4mQeuLBxP+bThLCBydEL e6Zg5xfJlJNqTAGEsTqqFB/36ecLNuf+nm0lSwssTchn0xU3/0kDAw+QOS/6b/zq xxabtTa0GDOGMTu1aVbM6uKIziEgiPqOTYyJKaAA2ITSCmwiZ/5OxZkCgYEAzvyT eNIiI/kzzfhadDqL6FVvd4D8A6x18YQySt2VTHQcn9f4p8QSD2DPRlgnYLACxFQV XD34c6IeFVLM2/f0+ZRFW+Hf03AZ/ytoe2MSYTmYDtWMGjsHNF5xO3nkE3oQla1O bKXBmJ0oTr4NIL5vZ3rV8FTIFYY05YfaMBi5Sn0CgYEAsQyzPZPcZ3SHE7Oas9/x LrihNylESEQ44RODxRmJrjLDwHtJR/MIrP3+4Lnq0Z5td+cUNp0HP+3p3sl/nkCx pwEG/KFlhB0c+NpK/Qc+JEKwCy5Md7CtWqc/bPzeTuI2GVmWsJOCVPHcrqbR22Tn DpdWFhAtU/eWcvyceoqk/1kCgYBv9L3vg/lja89RgRur8l7qdAuun92wPwAsekyZ ofC3QbaZ3r9oPu1l0/9JFTV3XrygZLqJAhv4r5+F+RtFf4DJ3iEF6c6fFut40YnZ 82RlojlVDLyTE4p6EPs+KFftEQEXdH4O1jk4ywiaTsHbDCZF2nMNY042FjlWTXz+ tuDCIQJ/InGW7eqaTUGxOC7XmvtSyuhP93pzEXY5hm+WQSWnkQIKfM5c64fsbnps +rJgOmykj+QyvEiVgoPdlbd3FDAgK9YR/3vYMsLzQ4OIZLx3b6/cvt1bn9VQg6Sl rFTa9FlOGBzNF0MazAEGygml8AcGdAkz3ztrWr5qXhJXaSMDvg== -----END RSA PRIVATE KEY----- ================================================ FILE: tests/framework/config/client.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "time" clientv3 "go.etcd.io/etcd/client/v3" ) // ClientOption configures the client with additional parameter. // For example, if Auth is enabled, the common test cases just need to // use `WithAuth` to return a ClientOption. Note that the common `WithAuth` // function calls `e2e.WithAuth` or `integration.WithAuth`, depending on the // build tag (either "e2e" or "integration"). type ClientOption func(any) type GetOptions struct { Revision int End string CountOnly bool Serializable bool Prefix bool FromKey bool Limit int Order clientv3.SortOrder SortBy clientv3.SortTarget Timeout time.Duration KeysOnly bool MinModRevision int MaxModRevision int MinCreateRevision int MaxCreateRevision int } type PutOptions struct { LeaseID clientv3.LeaseID Timeout time.Duration } type DeleteOptions struct { Prefix bool FromKey bool End string } type TxnOptions struct { Interactive bool } type CompactOption struct { Physical bool Timeout time.Duration } type DefragOption struct { Timeout time.Duration } type LeaseOption struct { WithAttachedKeys bool } type UserAddOptions struct { NoPassword bool } type WatchOptions struct { Prefix bool Revision int64 RangeEnd string } ================================================ FILE: tests/framework/config/cluster.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "time" ) type TLSConfig string const ( NoTLS TLSConfig = "" AutoTLS TLSConfig = "auto-tls" ManualTLS TLSConfig = "manual-tls" TickDuration = 10 * time.Millisecond ) type ClusterConfig struct { ClusterSize int PeerTLS TLSConfig ClientTLS TLSConfig QuotaBackendBytes int64 StrictReconfigCheck bool AuthToken string SnapshotCount uint64 // ClusterContext is used by "e2e" or "integration" to extend the // ClusterConfig. The common test cases shouldn't care about what // data is encoded or included; instead "e2e" or "integration" // framework should decode or parse it separately. ClusterContext any } func DefaultClusterConfig() ClusterConfig { return ClusterConfig{ ClusterSize: 3, StrictReconfigCheck: true, } } func NewClusterConfig(opts ...ClusterOption) ClusterConfig { c := DefaultClusterConfig() for _, opt := range opts { opt(&c) } return c } type ClusterOption func(*ClusterConfig) func WithClusterConfig(cfg ClusterConfig) ClusterOption { return func(c *ClusterConfig) { *c = cfg } } func WithClusterSize(size int) ClusterOption { return func(c *ClusterConfig) { c.ClusterSize = size } } func WithPeerTLS(tls TLSConfig) ClusterOption { return func(c *ClusterConfig) { c.PeerTLS = tls } } func WithClientTLS(tls TLSConfig) ClusterOption { return func(c *ClusterConfig) { c.ClientTLS = tls } } func WithQuotaBackendBytes(bytes int64) ClusterOption { return func(c *ClusterConfig) { c.QuotaBackendBytes = bytes } } func WithSnapshotCount(count uint64) ClusterOption { return func(c *ClusterConfig) { c.SnapshotCount = count } } func WithStrictReconfigCheck(strict bool) ClusterOption { return func(c *ClusterConfig) { c.StrictReconfigCheck = strict } } ================================================ FILE: tests/framework/e2e/cluster.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "flag" "fmt" "maps" "net/url" "path" "path/filepath" "regexp" "strings" "testing" "time" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/featuregate" "go.etcd.io/etcd/pkg/v3/proxy" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/config" ) const EtcdProcessBasePort = 20000 type ClientConnType int const ( ClientNonTLS ClientConnType = iota ClientTLS ClientTLSAndNonTLS ) type ClientConfig struct { ConnectionType ClientConnType CertAuthority bool AutoTLS bool RevokeCerts bool DialTimeout time.Duration } // allow alphanumerics, underscores and dashes var testNameCleanRegex = regexp.MustCompile(`[^a-zA-Z0-9 \-_]+`) func NewConfigNoTLS() *EtcdProcessClusterConfig { return DefaultConfig() } func NewConfigAutoTLS() *EtcdProcessClusterConfig { return NewConfig( WithIsPeerTLS(true), WithIsPeerAutoTLS(true), ) } func NewConfigTLS() *EtcdProcessClusterConfig { return NewConfig( WithClientConnType(ClientTLS), WithIsPeerTLS(true), ) } func NewConfigClientTLS() *EtcdProcessClusterConfig { return NewConfig(WithClientConnType(ClientTLS)) } func NewConfigClientAutoTLS() *EtcdProcessClusterConfig { return NewConfig( WithClusterSize(1), WithClientAutoTLS(true), WithClientConnType(ClientTLS), ) } func NewConfigPeerTLS() *EtcdProcessClusterConfig { return NewConfig( WithIsPeerTLS(true), ) } func NewConfigClientTLSCertAuth() *EtcdProcessClusterConfig { return NewConfig( WithClusterSize(1), WithClientConnType(ClientTLS), WithClientCertAuthority(true), ) } func NewConfigClientTLSCertAuthWithNoCN() *EtcdProcessClusterConfig { return NewConfig( WithClusterSize(1), WithClientConnType(ClientTLS), WithClientCertAuthority(true), WithCN(false), ) } func NewConfigJWT() *EtcdProcessClusterConfig { return NewConfig( WithClusterSize(1), WithAuthTokenOpts("jwt,pub-key="+path.Join(FixturesDir, "server.crt")+ ",priv-key="+path.Join(FixturesDir, "server.key.insecure")+",sign-method=RS256,ttl=5s"), ) } func ConfigStandalone(cfg EtcdProcessClusterConfig) *EtcdProcessClusterConfig { ret := cfg ret.ClusterSize = 1 return &ret } type EtcdProcessCluster struct { lg *zap.Logger Cfg *EtcdProcessClusterConfig Procs []EtcdProcess nextSeq int // sequence number of the next etcd process (if it will be required) } type EtcdProcessClusterConfig struct { ServerConfig embed.Config // Test config KeepDataDir bool Logger *zap.Logger GoFailEnabled bool GoFailClientTimeout time.Duration LazyFSEnabled bool PeerProxy bool // Process config EnvVars map[string]string Version ClusterVersion // Cluster setup config ClusterSize int // InitialLeaderIndex makes sure the leader is the ith proc // when the cluster starts if it is specified (>=0). InitialLeaderIndex int RollingStart bool // BaseDataDirPath specifies the data-dir for the members. If test cases // do not specify `BaseDataDirPath`, then e2e framework creates a // temporary directory for each member; otherwise, it creates a // subdirectory (e.g. member-0, member-1 and member-2) under the given // `BaseDataDirPath` for each member. BaseDataDirPath string // Dynamic per member configuration BasePeerScheme string BasePort int BaseClientScheme string MetricsURLScheme string Client ClientConfig ClientHTTPSeparate bool IsPeerTLS bool IsPeerAutoTLS bool CN bool } func DefaultConfig() *EtcdProcessClusterConfig { cfg := &EtcdProcessClusterConfig{ ClusterSize: 3, CN: true, InitialLeaderIndex: -1, ServerConfig: *embed.NewConfig(), } cfg.ServerConfig.InitialClusterToken = "new" return cfg } func NewConfig(opts ...EPClusterOption) *EtcdProcessClusterConfig { c := DefaultConfig() for _, opt := range opts { opt(c) } return c } type EPClusterOption func(*EtcdProcessClusterConfig) func WithConfig(cfg *EtcdProcessClusterConfig) EPClusterOption { return func(c *EtcdProcessClusterConfig) { *c = *cfg } } func WithVersion(version ClusterVersion) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.Version = version } } func WithInitialLeaderIndex(i int) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.InitialLeaderIndex = i } } func WithDataDirPath(path string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.BaseDataDirPath = path } } func WithKeepDataDir(keep bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.KeepDataDir = keep } } func WithSnapshotCount(count uint64) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.SnapshotCount = count } } func WithSnapshotCatchUpEntries(count uint64) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.SnapshotCatchUpEntries = count } } func WithClusterSize(size int) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ClusterSize = size } } func WithBasePeerScheme(scheme string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.BasePeerScheme = scheme } } func WithBasePort(port int) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.BasePort = port } } func WithBaseClientScheme(scheme string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.BaseClientScheme = scheme } } func WithClientConnType(clientConnType ClientConnType) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.Client.ConnectionType = clientConnType } } func WithClientCertAuthority(enabled bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.Client.CertAuthority = enabled } } func WithIsPeerTLS(isPeerTLS bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.IsPeerTLS = isPeerTLS } } func WithIsPeerAutoTLS(isPeerAutoTLS bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.IsPeerAutoTLS = isPeerAutoTLS } } func WithClientAutoTLS(isClientAutoTLS bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.Client.AutoTLS = isClientAutoTLS } } func WithClientRevokeCerts(isClientCRL bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.Client.RevokeCerts = isClientCRL } } func WithCN(cn bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.CN = cn } } func WithQuotaBackendBytes(bytes int64) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.QuotaBackendBytes = bytes } } func WithStrictReconfigCheck(strict bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.StrictReconfigCheck = strict } } func WithAuthTokenOpts(token string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.AuthToken = token } } func WithRollingStart(rolling bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.RollingStart = rolling } } func WithDiscoveryEndpoints(endpoints []string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.DiscoveryCfg.Endpoints = endpoints } } func WithDiscoveryToken(token string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.DiscoveryCfg.Token = token } } func WithLogLevel(level string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.LogLevel = level } } func WithCorruptCheckTime(time time.Duration) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.CorruptCheckTime = time } } func WithInitialClusterToken(token string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.InitialClusterToken = token } } func WithInitialCorruptCheck(enabled bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.ServerFeatureGate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("InitialCorruptCheck=%t", enabled)) } } func WithCompactHashCheckEnabled(enabled bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.ServerFeatureGate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("CompactHashCheck=%t", enabled)) } } func WithCompactHashCheckTime(time time.Duration) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.CompactHashCheckTime = time } } func WithGoFailEnabled(enabled bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.GoFailEnabled = enabled } } func WithGoFailClientTimeout(dur time.Duration) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.GoFailClientTimeout = dur } } func WithLazyFSEnabled(enabled bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.LazyFSEnabled = enabled } } func WithWarningUnaryRequestDuration(time time.Duration) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.WarningUnaryRequestDuration = time } } func WithServerFeatureGate(featureName string, val bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { if err := c.ServerConfig.ServerFeatureGate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", featureName, val)); err != nil { panic(err) } } } func WithCompactionBatchLimit(limit int) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.CompactionBatchLimit = limit } } func WithCompactionSleepInterval(time time.Duration) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.CompactionSleepInterval = time } } func WithWatchProcessNotifyInterval(interval time.Duration) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.WatchProgressNotifyInterval = interval } } func WithEnvVars(ev map[string]string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.EnvVars = ev } } func WithPeerProxy(enabled bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.PeerProxy = enabled } } func WithClientHTTPSeparate(enabled bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ClientHTTPSeparate = enabled } } func WithForceNewCluster(enabled bool) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.ForceNewCluster = enabled } } func WithMetricsURLScheme(scheme string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.MetricsURLScheme = scheme } } func WithCipherSuites(suites []string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.CipherSuites = suites } } func WithExtensiveMetrics() EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.Metrics = "extensive" } } func WithEnableDistributedTracing(addr string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.ServerConfig.EnableDistributedTracing = true c.ServerConfig.DistributedTracingServiceName = "etcd" c.ServerConfig.DistributedTracingAddress = addr c.ServerConfig.DistributedTracingSamplingRatePerMillion = 1_000_000 } } // NewEtcdProcessCluster launches a new cluster from etcd processes, returning // a new EtcdProcessCluster once all nodes are ready to accept client requests. func NewEtcdProcessCluster(ctx context.Context, tb testing.TB, opts ...EPClusterOption) (*EtcdProcessCluster, error) { cfg := NewConfig(opts...) epc, err := InitEtcdProcessCluster(tb, cfg) if err != nil { return nil, err } return StartEtcdProcessCluster(ctx, tb, epc, cfg) } // InitEtcdProcessCluster initializes a new cluster based on the given config. // It doesn't start the cluster. func InitEtcdProcessCluster(tb testing.TB, cfg *EtcdProcessClusterConfig) (*EtcdProcessCluster, error) { SkipInShortMode(tb) if cfg.Logger == nil { cfg.Logger = zaptest.NewLogger(tb) } if cfg.BasePort == 0 { cfg.BasePort = EtcdProcessBasePort } if cfg.ServerConfig.SnapshotCount == 0 { cfg.ServerConfig.SnapshotCount = etcdserver.DefaultSnapshotCount } // validate SnapshotCatchUpEntries could be set for at least one member if cfg.ServerConfig.SnapshotCatchUpEntries != etcdserver.DefaultSnapshotCatchUpEntries { if !CouldSetSnapshotCatchupEntries(BinPath.Etcd) { return nil, fmt.Errorf("cannot set SnapshotCatchUpEntries for current etcd version: %s", BinPath.Etcd) } if cfg.Version == LastVersion && !CouldSetSnapshotCatchupEntries(BinPath.EtcdLastRelease) { return nil, fmt.Errorf("cannot set SnapshotCatchUpEntries for last etcd version: %s", BinPath.EtcdLastRelease) } } etcdCfgs := cfg.EtcdAllServerProcessConfigs(tb) epc := &EtcdProcessCluster{ Cfg: cfg, lg: zaptest.NewLogger(tb), Procs: make([]EtcdProcess, cfg.ClusterSize), nextSeq: cfg.ClusterSize, } // launch etcd processes for i := range etcdCfgs { proc, err := NewEtcdProcess(tb, etcdCfgs[i]) if err != nil { epc.Close() return nil, fmt.Errorf("cannot configure: %w", err) } epc.Procs[i] = proc } return epc, nil } // StartEtcdProcessCluster launches a new cluster from etcd processes. func StartEtcdProcessCluster(ctx context.Context, tb testing.TB, epc *EtcdProcessCluster, cfg *EtcdProcessClusterConfig) (*EtcdProcessCluster, error) { if cfg.RollingStart { if err := epc.RollingStart(ctx); err != nil { return nil, fmt.Errorf("cannot rolling-start: %w", err) } } else { if err := epc.Start(ctx); err != nil { return nil, fmt.Errorf("cannot start: %w", err) } } for _, proc := range epc.Procs { if cfg.GoFailEnabled && !proc.Failpoints().Enabled() { epc.Close() tb.Skip("please run 'make gofail-enable && make build' before running the test") } } if cfg.InitialLeaderIndex >= 0 { if err := epc.MoveLeader(ctx, tb, cfg.InitialLeaderIndex); err != nil { return nil, fmt.Errorf("failed to move leader: %w", err) } } return epc, nil } func (cfg *EtcdProcessClusterConfig) ClientScheme() string { return setupScheme(cfg.BaseClientScheme, cfg.Client.ConnectionType == ClientTLS) } func (cfg *EtcdProcessClusterConfig) PeerScheme() string { return setupScheme(cfg.BasePeerScheme, cfg.IsPeerTLS) } func (cfg *EtcdProcessClusterConfig) EtcdAllServerProcessConfigs(tb testing.TB) []*EtcdServerProcessConfig { etcdCfgs := make([]*EtcdServerProcessConfig, cfg.ClusterSize) initialCluster := make([]string, cfg.ClusterSize) for i := 0; i < cfg.ClusterSize; i++ { etcdCfgs[i] = cfg.EtcdServerProcessConfig(tb, i) initialCluster[i] = fmt.Sprintf("%s=%s", etcdCfgs[i].Name, etcdCfgs[i].PeerURL.String()) } for i := range etcdCfgs { cfg.SetInitialOrDiscovery(etcdCfgs[i], initialCluster, "new") } return etcdCfgs } func (cfg *EtcdProcessClusterConfig) SetInitialOrDiscovery(serverCfg *EtcdServerProcessConfig, initialCluster []string, initialClusterState string) { if len(cfg.ServerConfig.DiscoveryCfg.Endpoints) == 0 { serverCfg.InitialCluster = strings.Join(initialCluster, ",") serverCfg.Args = append(serverCfg.Args, "--initial-cluster="+serverCfg.InitialCluster) serverCfg.Args = append(serverCfg.Args, "--initial-cluster-state="+initialClusterState) } if len(cfg.ServerConfig.DiscoveryCfg.Endpoints) > 0 { serverCfg.Args = append(serverCfg.Args, fmt.Sprintf("--discovery-token=%s", cfg.ServerConfig.DiscoveryCfg.Token)) serverCfg.Args = append(serverCfg.Args, fmt.Sprintf("--discovery-endpoints=%s", strings.Join(cfg.ServerConfig.DiscoveryCfg.Endpoints, ","))) } } func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i int) *EtcdServerProcessConfig { var curls []string var curl string port := cfg.BasePort + 5*i clientPort := port peerPort := port + 1 metricsPort := port + 2 peer2Port := port + 3 clientHTTPPort := port + 4 if cfg.Client.ConnectionType == ClientTLSAndNonTLS { curl = clientURL(cfg.ClientScheme(), clientPort, ClientNonTLS) curls = []string{curl, clientURL(cfg.ClientScheme(), clientPort, ClientTLS)} } else { curl = clientURL(cfg.ClientScheme(), clientPort, cfg.Client.ConnectionType) curls = []string{curl} } peerListenURL := url.URL{Scheme: cfg.PeerScheme(), Host: fmt.Sprintf("localhost:%d", peerPort)} peerAdvertiseURL := url.URL{Scheme: cfg.PeerScheme(), Host: fmt.Sprintf("localhost:%d", peerPort)} var proxyCfg *proxy.ServerConfig if cfg.PeerProxy { if !cfg.IsPeerTLS { panic("Can't use peer proxy without peer TLS as it can result in malformed packets") } peerAdvertiseURL.Host = fmt.Sprintf("localhost:%d", peer2Port) proxyCfg = &proxy.ServerConfig{ Logger: zap.NewNop(), To: peerListenURL, From: peerAdvertiseURL, } } name := fmt.Sprintf("%s-test-%d", testNameCleanRegex.ReplaceAllString(tb.Name(), ""), i) var dataDirPath string if cfg.BaseDataDirPath == "" { dataDirPath = tb.TempDir() } else { // When test cases specify the BaseDataDirPath and there are more than // one member in the cluster, we need to create a subdirectory for // each member to avoid conflict. // We also create a subdirectory for one-member cluster, because we // support dynamically adding new member. dataDirPath = filepath.Join(cfg.BaseDataDirPath, fmt.Sprintf("member-%d", i)) } args := []string{ "--name=" + name, "--listen-client-urls=" + strings.Join(curls, ","), "--advertise-client-urls=" + strings.Join(curls, ","), "--listen-peer-urls=" + peerListenURL.String(), "--initial-advertise-peer-urls=" + peerAdvertiseURL.String(), "--initial-cluster-token=" + cfg.ServerConfig.InitialClusterToken, "--data-dir=" + dataDirPath, "--snapshot-count=" + fmt.Sprintf("%d", cfg.ServerConfig.SnapshotCount), } var clientHTTPURL string if cfg.ClientHTTPSeparate { clientHTTPURL = clientURL(cfg.ClientScheme(), clientHTTPPort, cfg.Client.ConnectionType) args = append(args, "--listen-client-http-urls="+clientHTTPURL) } if cfg.ServerConfig.ForceNewCluster { args = append(args, "--force-new-cluster=true") } if cfg.ServerConfig.QuotaBackendBytes > 0 { args = append(args, "--quota-backend-bytes="+fmt.Sprintf("%d", cfg.ServerConfig.QuotaBackendBytes), ) } if !cfg.ServerConfig.StrictReconfigCheck { args = append(args, "--strict-reconfig-check=false") } if cfg.ServerConfig.EnableDistributedTracing { args = append(args, "--enable-distributed-tracing", fmt.Sprintf("--distributed-tracing-address=%s", cfg.ServerConfig.DistributedTracingAddress), fmt.Sprintf("--distributed-tracing-service-name=%s", cfg.ServerConfig.DistributedTracingServiceName), fmt.Sprintf("--distributed-tracing-sampling-rate=%d", cfg.ServerConfig.DistributedTracingSamplingRatePerMillion), ) } var murl string if cfg.MetricsURLScheme != "" { murl = (&url.URL{ Scheme: cfg.MetricsURLScheme, Host: fmt.Sprintf("localhost:%d", metricsPort), }).String() args = append(args, "--listen-metrics-urls="+murl) } args = append(args, cfg.TLSArgs()...) execPath := cfg.binaryPath(i) if cfg.ServerConfig.SnapshotCatchUpEntries != etcdserver.DefaultSnapshotCatchUpEntries { if !IsSnapshotCatchupEntriesFlagAvailable(execPath) { cfg.ServerConfig.SnapshotCatchUpEntries = etcdserver.DefaultSnapshotCatchUpEntries } } var ( binaryVersion *semver.Version err error ) if execPath != "" { binaryVersion, err = GetVersionFromBinary(execPath) if err != nil { tb.Logf("Failed to get binary version from %s: %v", execPath, err) } } defaultValues := values(*embed.NewConfig()) overrideValues := values(cfg.ServerConfig) for flag, value := range overrideValues { if defaultValue := defaultValues[flag]; value == "" || value == defaultValue { continue } if strings.HasSuffix(flag, "snapshot-catchup-entries") && !CouldSetSnapshotCatchupEntries(execPath) { continue } args = append(args, convertFlag(flag, value, binaryVersion)) } envVars := map[string]string{} maps.Copy(envVars, cfg.EnvVars) var gofailPort int if cfg.GoFailEnabled { gofailPort = (i+1)*10000 + 2381 envVars["GOFAIL_HTTP"] = fmt.Sprintf("127.0.0.1:%d", gofailPort) } return &EtcdServerProcessConfig{ lg: cfg.Logger, ExecPath: execPath, Args: args, EnvVars: envVars, TLSArgs: cfg.TLSArgs(), Client: cfg.Client, DataDirPath: dataDirPath, KeepDataDir: cfg.KeepDataDir, Name: name, PeerURL: peerAdvertiseURL, ClientURL: curl, ClientHTTPURL: clientHTTPURL, MetricsURL: murl, InitialToken: cfg.ServerConfig.InitialClusterToken, GoFailPort: gofailPort, GoFailClientTimeout: cfg.GoFailClientTimeout, Proxy: proxyCfg, LazyFSEnabled: cfg.LazyFSEnabled, } } func (cfg *EtcdProcessClusterConfig) binaryPath(i int) string { var execPath string switch cfg.Version { case CurrentVersion: execPath = BinPath.Etcd case MinorityLastVersion: if i <= cfg.ClusterSize/2 { execPath = BinPath.Etcd } else { execPath = BinPath.EtcdLastRelease } case QuorumLastVersion: if i <= cfg.ClusterSize/2 { execPath = BinPath.EtcdLastRelease } else { execPath = BinPath.Etcd } case LastVersion: execPath = BinPath.EtcdLastRelease default: panic(fmt.Sprintf("Unknown cluster version %v", cfg.Version)) } return execPath } func (epc *EtcdProcessCluster) MinServerVersion() (*semver.Version, error) { var minVersion *semver.Version for _, member := range epc.Procs { ver, err := GetVersionFromBinary(member.Config().ExecPath) if err != nil { return nil, fmt.Errorf("failed to get version from member %s binary: %w", member.Config().Name, err) } if minVersion == nil || ver.LessThan(*minVersion) { minVersion = ver } } return minVersion, nil } func values(cfg embed.Config) map[string]string { fs := flag.NewFlagSet("etcd", flag.ContinueOnError) cfg.AddFlags(fs) values := map[string]string{} fs.VisitAll(func(f *flag.Flag) { value := f.Value.String() if value == "false" || value == "0" { value = "" } values[f.Name] = value }) return values } func clientURL(scheme string, port int, connType ClientConnType) string { curlHost := fmt.Sprintf("localhost:%d", port) switch connType { case ClientNonTLS: return (&url.URL{Scheme: scheme, Host: curlHost}).String() case ClientTLS: return (&url.URL{Scheme: ToTLS(scheme), Host: curlHost}).String() default: panic(fmt.Sprintf("Unsupported connection type %v", connType)) } } func (cfg *EtcdProcessClusterConfig) TLSArgs() (args []string) { if cfg.Client.ConnectionType != ClientNonTLS { if cfg.Client.AutoTLS { args = append(args, "--auto-tls") } else { tlsClientArgs := []string{ "--cert-file", CertPath, "--key-file", PrivateKeyPath, "--trusted-ca-file", CaPath, } args = append(args, tlsClientArgs...) if cfg.Client.CertAuthority { args = append(args, "--client-cert-auth") } } } if cfg.IsPeerTLS { if cfg.IsPeerAutoTLS { args = append(args, "--peer-auto-tls") } else { tlsPeerArgs := []string{ "--peer-cert-file", CertPath, "--peer-key-file", PrivateKeyPath, "--peer-trusted-ca-file", CaPath, } args = append(args, tlsPeerArgs...) } } if cfg.Client.RevokeCerts { args = append(args, "--client-crl-file", CrlPath, "--client-cert-auth") } if len(cfg.ServerConfig.CipherSuites) > 0 { args = append(args, "--cipher-suites", strings.Join(cfg.ServerConfig.CipherSuites, ",")) } return args } func (epc *EtcdProcessCluster) EndpointsGRPC() []string { return epc.Endpoints(func(ep EtcdProcess) []string { return ep.EndpointsGRPC() }) } func (epc *EtcdProcessCluster) EndpointsHTTP() []string { return epc.Endpoints(func(ep EtcdProcess) []string { return ep.EndpointsHTTP() }) } func (epc *EtcdProcessCluster) Endpoints(f func(ep EtcdProcess) []string) (ret []string) { for _, p := range epc.Procs { ret = append(ret, f(p)...) } return ret } func (epc *EtcdProcessCluster) CloseProc(ctx context.Context, finder func(EtcdProcess) bool, opts ...config.ClientOption) error { procIndex := -1 if finder != nil { for i := range epc.Procs { if finder(epc.Procs[i]) { procIndex = i break } } } else { procIndex = len(epc.Procs) - 1 } if procIndex == -1 { return fmt.Errorf("no process found to stop") } proc := epc.Procs[procIndex] epc.Procs = append(epc.Procs[:procIndex], epc.Procs[procIndex+1:]...) if proc == nil { return nil } // First remove member from the cluster memberCtl := epc.Etcdctl(opts...) memberList, err := memberCtl.MemberList(ctx, false) if err != nil { return fmt.Errorf("failed to get member list: %w", err) } memberID, err := findMemberIDByEndpoint(memberList.Members, proc.Config().ClientURL) if err != nil { return fmt.Errorf("failed to find member ID: %w", err) } sleepDuration := 500 * time.Millisecond maxRetries := int((2 * etcdserver.HealthInterval) / sleepDuration) memberRemoved := false for i := 0; i < maxRetries; i++ { _, err := memberCtl.MemberRemove(ctx, memberID) if err != nil && strings.Contains(err.Error(), "member not found") { memberRemoved = true break } time.Sleep(sleepDuration) } if !memberRemoved { return fmt.Errorf("failed to remove member after %d tries", maxRetries) } epc.lg.Info("successfully removed member", zap.String("acurl", proc.Config().ClientURL)) // Then stop process return proc.Close() } // StartNewProc grows cluster size by one with two phases // Phase 1 - Inform cluster of new configuration // Phase 2 - Start new member func (epc *EtcdProcessCluster) StartNewProc(ctx context.Context, cfg *EtcdProcessClusterConfig, tb testing.TB, addAsLearner bool, opts ...config.ClientOption) (memberID uint64, err error) { memberID, serverCfg, err := epc.AddMember(ctx, cfg, tb, addAsLearner, opts...) if err != nil { return 0, err } // Then start process if err = epc.StartNewProcFromConfig(ctx, tb, serverCfg); err != nil { return 0, err } return memberID, nil } // AddMember adds a new member to the cluster without starting it. func (epc *EtcdProcessCluster) AddMember(ctx context.Context, cfg *EtcdProcessClusterConfig, tb testing.TB, addAsLearner bool, opts ...config.ClientOption) (memberID uint64, serverCfg *EtcdServerProcessConfig, err error) { if cfg != nil { serverCfg = cfg.EtcdServerProcessConfig(tb, epc.nextSeq) } else { serverCfg = epc.Cfg.EtcdServerProcessConfig(tb, epc.nextSeq) } epc.nextSeq++ initialCluster := []string{ fmt.Sprintf("%s=%s", serverCfg.Name, serverCfg.PeerURL.String()), } for _, p := range epc.Procs { initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", p.Config().Name, p.Config().PeerURL.String())) } epc.Cfg.SetInitialOrDiscovery(serverCfg, initialCluster, "existing") // First add new member to cluster tb.Logf("add new member to cluster; member-name %s, member-peer-url %s", serverCfg.Name, serverCfg.PeerURL.String()) memberCtl := epc.Etcdctl(opts...) var resp *clientv3.MemberAddResponse if addAsLearner { resp, err = memberCtl.MemberAddAsLearner(ctx, serverCfg.Name, []string{serverCfg.PeerURL.String()}) } else { resp, err = memberCtl.MemberAdd(ctx, serverCfg.Name, []string{serverCfg.PeerURL.String()}) } if err != nil { return 0, nil, fmt.Errorf("failed to add new member: %w", err) } return resp.Member.ID, serverCfg, nil } // StartNewProcFromConfig starts a new member process from the given config. func (epc *EtcdProcessCluster) StartNewProcFromConfig(ctx context.Context, tb testing.TB, serverCfg *EtcdServerProcessConfig) error { tb.Log("start new member") proc, err := NewEtcdProcess(tb, serverCfg) if err != nil { epc.Close() return fmt.Errorf("cannot configure: %w", err) } epc.Procs = append(epc.Procs, proc) return proc.Start(ctx) } // UpdateProcOptions updates the options for a specific process. If no opt is set, then the config is identical // to the cluster. func (epc *EtcdProcessCluster) UpdateProcOptions(i int, tb testing.TB, opts ...EPClusterOption) error { if epc.Procs[i].IsRunning() { return fmt.Errorf("process %d is still running, please close it before updating its options", i) } cfg := *epc.Cfg for _, opt := range opts { opt(&cfg) } serverCfg := cfg.EtcdServerProcessConfig(tb, i) var initialCluster []string for _, p := range epc.Procs { initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", p.Config().Name, p.Config().PeerURL.String())) } epc.Cfg.SetInitialOrDiscovery(serverCfg, initialCluster, "new") proc, err := NewEtcdProcess(tb, serverCfg) if err != nil { return err } epc.Procs[i] = proc return nil } func PatchArgs(args []string, flag, newValue string) error { for i, arg := range args { if strings.Contains(arg, flag) { args[i] = fmt.Sprintf("--%s=%s", flag, newValue) return nil } } return fmt.Errorf("--%s flag not found", flag) } func (epc *EtcdProcessCluster) Start(ctx context.Context) error { return epc.start(func(ep EtcdProcess) error { return ep.Start(ctx) }) } func (epc *EtcdProcessCluster) RollingStart(ctx context.Context) error { return epc.rollingStart(func(ep EtcdProcess) error { return ep.Start(ctx) }) } func (epc *EtcdProcessCluster) Restart(ctx context.Context) error { return epc.start(func(ep EtcdProcess) error { return ep.Restart(ctx) }) } func (epc *EtcdProcessCluster) start(f func(ep EtcdProcess) error) error { readyC := make(chan error, len(epc.Procs)) for i := range epc.Procs { go func(n int) { readyC <- f(epc.Procs[n]) }(i) } for range epc.Procs { if err := <-readyC; err != nil { epc.Close() return err } } return nil } func (epc *EtcdProcessCluster) rollingStart(f func(ep EtcdProcess) error) error { readyC := make(chan error, len(epc.Procs)) for i := range epc.Procs { go func(n int) { readyC <- f(epc.Procs[n]) }(i) // make sure the servers do not start at the same time time.Sleep(time.Second) } for range epc.Procs { if err := <-readyC; err != nil { epc.Close() return err } } return nil } func (epc *EtcdProcessCluster) Kill() (err error) { for _, p := range epc.Procs { if p == nil { continue } if curErr := p.Kill(); curErr != nil { if err != nil { err = fmt.Errorf("%w; %w", err, curErr) } else { err = curErr } } } return err } func (epc *EtcdProcessCluster) Wait(ctx context.Context) error { closedC := make(chan error, len(epc.Procs)) for i := range epc.Procs { go func(n int) { epc.Procs[n].Wait(ctx) closedC <- epc.Procs[n].Wait(ctx) }(i) } for range epc.Procs { if err := <-closedC; err != nil { return err } } return nil } func (epc *EtcdProcessCluster) Stop() (err error) { for _, p := range epc.Procs { if p == nil { continue } if curErr := p.Stop(); curErr != nil { if err != nil { err = fmt.Errorf("%w; %w", err, curErr) } else { err = curErr } } } return err } func (epc *EtcdProcessCluster) ConcurrentStop() (err error) { errCh := make(chan error, len(epc.Procs)) for i := range epc.Procs { if epc.Procs[i] == nil { errCh <- nil continue } go func(n int) { errCh <- epc.Procs[n].Stop() }(i) } for range epc.Procs { if curErr := <-errCh; curErr != nil { if err != nil { err = fmt.Errorf("%w; %w", err, curErr) } else { err = curErr } } } close(errCh) return err } func (epc *EtcdProcessCluster) Etcdctl(opts ...config.ClientOption) *EtcdctlV3 { etcdctl, err := NewEtcdctl(epc.Cfg.Client, epc.EndpointsGRPC(), opts...) if err != nil { panic(err) } return etcdctl } func (epc *EtcdProcessCluster) Close() error { epc.lg.Info("closing test cluster...") err := epc.Stop() for _, p := range epc.Procs { // p is nil when NewEtcdProcess fails in the middle // Close still gets called to clean up test data if p == nil { continue } if cerr := p.Close(); cerr != nil { err = cerr } } epc.lg.Info("closed test cluster.") return err } func findMemberIDByEndpoint(members []*etcdserverpb.Member, endpoint string) (uint64, error) { for _, m := range members { if m.ClientURLs[0] == endpoint { return m.ID, nil } } return 0, fmt.Errorf("member not found") } // WaitLeader returns index of the member in c.Members() that is leader // or fails the test (if not established in 30s). func (epc *EtcdProcessCluster) WaitLeader(tb testing.TB) int { ctx, cancel := context.WithTimeout(tb.Context(), 30*time.Second) defer cancel() return epc.WaitMembersForLeader(ctx, tb, epc.Procs) } // WaitMembersForLeader waits until given members agree on the same leader, // and returns its 'index' in the 'membs' list func (epc *EtcdProcessCluster) WaitMembersForLeader(ctx context.Context, tb testing.TB, membs []EtcdProcess) int { cc := epc.Etcdctl() // ensure leader is up via linearizable get for { select { case <-ctx.Done(): tb.Fatal("WaitMembersForLeader timeout") default: } _, err := cc.Get(ctx, "0", config.GetOptions{Timeout: 10*config.TickDuration + time.Second}) if err == nil || strings.Contains(err.Error(), "Key not found") { break } tb.Logf("WaitMembersForLeader Get err: %v", err) } leaders := make(map[uint64]struct{}) members := make(map[uint64]int) for { select { case <-ctx.Done(): tb.Fatal("WaitMembersForLeader timeout") default: } for i := range membs { if !membs[i].IsRunning() { // if member[i] has stopped continue } resp, err := membs[i].Etcdctl().Status(ctx) if err != nil { if strings.Contains(err.Error(), "connection refused") { // if member[i] has stopped continue } tb.Fatal(err) } members[resp[0].Header.MemberId] = i leaders[resp[0].Leader] = struct{}{} } // members agree on the same leader if len(leaders) == 1 { break } leaders = make(map[uint64]struct{}) members = make(map[uint64]int) time.Sleep(10 * config.TickDuration) } for l := range leaders { if index, ok := members[l]; ok { tb.Logf("members agree on a leader, members:%v , leader:%v", members, l) return index } tb.Fatalf("members agree on a leader which is not one of members, members:%v , leader:%v", members, l) } tb.Fatal("impossible path of execution") return -1 } // MoveLeader moves the leader to the ith process. func (epc *EtcdProcessCluster) MoveLeader(ctx context.Context, tb testing.TB, i int) error { if i < 0 || i >= len(epc.Procs) { return fmt.Errorf("invalid index: %d, must between 0 and %d", i, len(epc.Procs)-1) } tb.Logf("moving leader to Procs[%d]", i) oldLeader := epc.WaitMembersForLeader(ctx, tb, epc.Procs) if oldLeader == i { tb.Logf("Procs[%d] is already the leader", i) return nil } resp, err := epc.Procs[i].Etcdctl().Status(ctx) if err != nil { return err } memberID := resp[0].Header.MemberId err = epc.Procs[oldLeader].Etcdctl().MoveLeader(ctx, memberID) if err != nil { return err } newLeader := epc.WaitMembersForLeader(ctx, tb, epc.Procs) if newLeader != i { tb.Fatalf("expect new leader to be Procs[%d] but got Procs[%d]", i, newLeader) } tb.Logf("moved leader from Procs[%d] to Procs[%d]", oldLeader, i) return nil } ================================================ FILE: tests/framework/e2e/cluster_direct.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package e2e import "testing" func NewEtcdProcess(tb testing.TB, cfg *EtcdServerProcessConfig) (EtcdProcess, error) { return NewEtcdServerProcess(tb, cfg) } ================================================ FILE: tests/framework/e2e/cluster_proxy.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build cluster_proxy package e2e import ( "context" "fmt" "net" "net/url" "path" "strconv" "strings" "testing" "go.uber.org/zap" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/config" ) type proxyEtcdProcess struct { *EtcdServerProcess // TODO(ahrtr): We need to remove `proxyV2` and v2discovery when the v2client is removed. proxyV2 *proxyV2Proc proxyV3 *proxyV3Proc } func NewEtcdProcess(t testing.TB, cfg *EtcdServerProcessConfig) (EtcdProcess, error) { return NewProxyEtcdProcess(t, cfg) } func NewProxyEtcdProcess(t testing.TB, cfg *EtcdServerProcessConfig) (*proxyEtcdProcess, error) { ep, err := NewEtcdServerProcess(t, cfg) if err != nil { return nil, err } pep := &proxyEtcdProcess{ EtcdServerProcess: ep, proxyV2: newProxyV2Proc(cfg), proxyV3: newProxyV3Proc(cfg), } return pep, nil } func (p *proxyEtcdProcess) EndpointsHTTP() []string { return p.proxyV2.endpoints() } func (p *proxyEtcdProcess) EndpointsGRPC() []string { return p.proxyV3.endpoints() } func (p *proxyEtcdProcess) EndpointsMetrics() []string { panic("not implemented; proxy doesn't provide health information") } func (p *proxyEtcdProcess) Start(ctx context.Context) error { if err := p.EtcdServerProcess.Start(ctx); err != nil { return err } return p.proxyV3.Start(ctx) } func (p *proxyEtcdProcess) Restart(ctx context.Context) error { if err := p.EtcdServerProcess.Restart(ctx); err != nil { return err } return p.proxyV3.Restart(ctx) } func (p *proxyEtcdProcess) Stop() error { err := p.proxyV3.Stop() if eerr := p.EtcdServerProcess.Stop(); eerr != nil && err == nil { // fails on go-grpc issue #1384 if !strings.Contains(eerr.Error(), "exit status 2") { err = eerr } } return err } func (p *proxyEtcdProcess) Close() error { err := p.proxyV3.Close() if eerr := p.EtcdServerProcess.Close(); eerr != nil && err == nil { // fails on go-grpc issue #1384 if !strings.Contains(eerr.Error(), "exit status 2") { err = eerr } } return err } func (p *proxyEtcdProcess) Etcdctl(opts ...config.ClientOption) *EtcdctlV3 { etcdctl, err := NewEtcdctl(p.EtcdServerProcess.Config().Client, p.EtcdServerProcess.EndpointsGRPC(), opts...) if err != nil { panic(err) } return etcdctl } type proxyProc struct { lg *zap.Logger name string execPath string args []string ep string murl string donec chan struct{} proc *expect.ExpectProcess } func (pp *proxyProc) endpoints() []string { return []string{pp.ep} } func (pp *proxyProc) start() error { if pp.proc != nil { panic("already started") } proc, err := SpawnCmdWithLogger(pp.lg, append([]string{pp.execPath}, pp.args...), nil, pp.name) if err != nil { return err } pp.proc = proc return nil } func (pp *proxyProc) waitReady(ctx context.Context, readyStr string) error { defer close(pp.donec) return WaitReadyExpectProc(ctx, pp.proc, []string{readyStr}) } func (pp *proxyProc) Stop() error { if pp.proc == nil { return nil } err := pp.proc.Stop() if err != nil { return err } err = pp.proc.Close() if err != nil { // proxy received SIGTERM signal if !(strings.Contains(err.Error(), "unexpected exit code") || // v2proxy exits with status 1 on auto tls; not sure why strings.Contains(err.Error(), "exit status 1")) { return err } } pp.proc = nil <-pp.donec pp.donec = make(chan struct{}) return nil } func (pp *proxyProc) Close() error { return pp.Stop() } type proxyV2Proc struct { proxyProc dataDir string } func proxyListenURL(cfg *EtcdServerProcessConfig, portOffset int) string { u, err := url.Parse(cfg.ClientURL) if err != nil { panic(err) } host, port, _ := net.SplitHostPort(u.Host) p, _ := strconv.ParseInt(port, 10, 16) u.Host = fmt.Sprintf("%s:%d", host, int(p)+portOffset) return u.String() } func newProxyV2Proc(cfg *EtcdServerProcessConfig) *proxyV2Proc { listenAddr := proxyListenURL(cfg, 2) name := fmt.Sprintf("testname-proxy-%p", cfg) dataDir := path.Join(cfg.DataDirPath, name+".etcd") args := []string{ "--name", name, "--proxy", "on", "--listen-client-urls", listenAddr, "--initial-cluster", cfg.Name + "=" + cfg.PeerURL.String(), "--data-dir", dataDir, } return &proxyV2Proc{ proxyProc: proxyProc{ name: cfg.Name, lg: cfg.lg, execPath: cfg.ExecPath, args: append(args, cfg.TLSArgs...), ep: listenAddr, donec: make(chan struct{}), }, dataDir: dataDir, } } type proxyV3Proc struct { proxyProc } func newProxyV3Proc(cfg *EtcdServerProcessConfig) *proxyV3Proc { listenAddr := proxyListenURL(cfg, 3) args := []string{ "grpc-proxy", "start", "--listen-addr", strings.Split(listenAddr, "/")[2], "--endpoints", cfg.ClientURL, // pass-through member RPCs "--advertise-client-url", "", "--data-dir", cfg.DataDirPath, } murl := "" if cfg.MetricsURL != "" { murl = proxyListenURL(cfg, 4) args = append(args, "--metrics-addr", murl) } tlsArgs := []string{} for i := 0; i < len(cfg.TLSArgs); i++ { switch cfg.TLSArgs[i] { case "--cert-file": tlsArgs = append(tlsArgs, "--cert-file", cfg.TLSArgs[i+1]) i++ case "--key-file": tlsArgs = append(tlsArgs, "--key-file", cfg.TLSArgs[i+1]) i++ case "--trusted-ca-file": tlsArgs = append(tlsArgs, "--trusted-ca-file", cfg.TLSArgs[i+1]) i++ case "--auto-tls": tlsArgs = append(tlsArgs, "--auto-tls", "--insecure-skip-tls-verify") case "--peer-trusted-ca-file", "--peer-cert-file", "--peer-key-file": i++ // skip arg case "--client-cert-auth", "--peer-auto-tls": default: tlsArgs = append(tlsArgs, cfg.TLSArgs[i]) } } if len(cfg.TLSArgs) > 0 { // Configure certificates for connection proxy ---> server. // This certificate must NOT have CN set. tlsArgs = append(tlsArgs, "--cert", path.Join(FixturesDir, "client-nocn.crt"), "--key", path.Join(FixturesDir, "client-nocn.key.insecure"), "--cacert", path.Join(FixturesDir, "ca.crt"), "--client-crl-file", path.Join(FixturesDir, "revoke.crl")) } return &proxyV3Proc{ proxyProc{ name: cfg.Name, lg: cfg.lg, execPath: cfg.ExecPath, args: append(args, tlsArgs...), ep: listenAddr, murl: murl, donec: make(chan struct{}), }, } } func (v3p *proxyV3Proc) Restart(ctx context.Context) error { if err := v3p.Stop(); err != nil { return err } return v3p.Start(ctx) } func (v3p *proxyV3Proc) Start(ctx context.Context) error { if err := v3p.start(); err != nil { return err } return v3p.waitReady(ctx, "started gRPC proxy") } ================================================ FILE: tests/framework/e2e/cluster_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "testing" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" ) func TestEtcdServerProcessConfig(t *testing.T) { v3_6_0 := semver.Version{Major: 3, Minor: 6, Patch: 0} v3_7_0 := semver.Version{Major: 3, Minor: 7, Patch: 0} tcs := []struct { name string config *EtcdProcessClusterConfig expectArgsEquals []string expectArgsNotContain []string expectArgsContain []string mockBinaryVersion *semver.Version }{ { name: "Default", config: NewConfig(WithDataDirPath("/tmp/a")), expectArgsEquals: []string{ "--name=TestEtcdServerProcessConfigDefault-test-0", "--listen-client-urls=http://localhost:0", "--advertise-client-urls=http://localhost:0", "--listen-peer-urls=http://localhost:1", "--initial-advertise-peer-urls=http://localhost:1", "--initial-cluster-token=new", "--data-dir=/tmp/a/member-0", "--snapshot-count=10000", "--initial-cluster-token=new", }, }, { name: "SnapshotCount", config: NewConfig(WithSnapshotCount(42)), expectArgsContain: []string{ "--snapshot-count=42", }, }, { name: "QuotaBackendBytes", config: NewConfig(WithQuotaBackendBytes(123)), expectArgsContain: []string{ "--quota-backend-bytes=123", }, }, { name: "CorruptCheck", config: NewConfig(WithInitialCorruptCheck(true)), expectArgsContain: []string{ "--feature-gates=InitialCorruptCheck=true", }, }, { name: "StrictReconfigCheck", config: NewConfig(WithStrictReconfigCheck(false)), expectArgsContain: []string{ "--strict-reconfig-check=false", }, }, { name: "CatchUpEntries", config: NewConfig(WithSnapshotCatchUpEntries(100)), expectArgsContain: []string{ "--snapshot-catchup-entries=100", }, mockBinaryVersion: &v3_7_0, }, { name: "CatchUpEntriesNoVersion", config: NewConfig(WithSnapshotCatchUpEntries(100), WithVersion(LastVersion)), expectArgsNotContain: []string{ "--snapshot-catchup-entries=100", }, }, { name: "CatchUpEntriesOldVersion", config: NewConfig(WithSnapshotCatchUpEntries(100), WithVersion(LastVersion)), expectArgsContain: []string{ "--snapshot-catchup-entries=100", }, mockBinaryVersion: &v3_6_0, }, { name: "ClientHTTPSeparate", config: NewConfig(WithClientHTTPSeparate(true)), expectArgsContain: []string{ "--listen-client-http-urls=http://localhost:4", }, }, { name: "ForceNewCluster", config: NewConfig(WithForceNewCluster(true)), expectArgsContain: []string{ "--force-new-cluster=true", }, }, { name: "MetricsURL", config: NewConfig(WithMetricsURLScheme("http")), expectArgsContain: []string{ "--listen-metrics-urls=http://localhost:2", }, }, { name: "ClientTLS", config: NewConfig(WithClientConnType(ClientTLS)), expectArgsContain: []string{ "--cert-file", "--key-file", "--trusted-ca-file", }, expectArgsNotContain: []string{ "--auto-tls", "--client-cert-auth", }, }, { name: "ClientTLSCA", config: NewConfig(WithClientConnType(ClientTLS), WithClientCertAuthority(true)), expectArgsContain: []string{ "--cert-file", "--key-file", "--trusted-ca-file", "--client-cert-auth", }, expectArgsNotContain: []string{ "--auto-tls", }, }, { name: "ClientAutoTLS", config: NewConfig(WithClientConnType(ClientTLS), WithClientAutoTLS(true)), expectArgsContain: []string{ "--auto-tls", }, expectArgsNotContain: []string{ "--cert-file", "--key-file", "--trusted-ca-file", "--client-cert-auth", }, }, { name: "PeerTLS", config: NewConfig(WithIsPeerTLS(true)), expectArgsContain: []string{ "--peer-cert-file", "--peer-key-file", "--peer-trusted-ca-file", }, expectArgsNotContain: []string{ "--peer-auto-tls", "--peer-client-cert-auth", }, }, { name: "PeerAutoTLS", config: NewConfig(WithIsPeerTLS(true), WithIsPeerAutoTLS(true)), expectArgsContain: []string{ "--peer-auto-tls", }, expectArgsNotContain: []string{ "--peer-cert-file", "--peer-key-file", "--peer-trusted-ca-file", "--peer-client-cert-auth", }, }, { name: "RevokeCerts", config: NewConfig(WithClientRevokeCerts(true)), expectArgsContain: []string{ "--client-crl-file", "--client-cert-auth", }, }, { name: "CipherSuites", config: NewConfig(WithCipherSuites([]string{"a", "b"})), expectArgsContain: []string{ "--cipher-suites", }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { var mockGetVersionFromBinary func(binaryPath string) (*semver.Version, error) if tc.mockBinaryVersion == nil { mockGetVersionFromBinary = func(binaryPath string) (*semver.Version, error) { return nil, fmt.Errorf("could not get binary version") } } else { mockGetVersionFromBinary = func(binaryPath string) (*semver.Version, error) { return tc.mockBinaryVersion, nil } } setGetVersionFromBinary(t, mockGetVersionFromBinary) args := tc.config.EtcdServerProcessConfig(t, 0).Args if len(tc.expectArgsEquals) != 0 { assert.Equal(t, args, tc.expectArgsEquals) } if len(tc.expectArgsContain) != 0 { assert.Subset(t, args, tc.expectArgsContain) } if len(tc.expectArgsNotContain) != 0 { assert.NotSubset(t, args, tc.expectArgsNotContain) } }) } } ================================================ FILE: tests/framework/e2e/config.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "fmt" "strings" "github.com/coreos/go-semver/semver" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/tests/v3/framework/config" ) type ClusterVersion string const ( CurrentVersion ClusterVersion = "" MinorityLastVersion ClusterVersion = "minority-last-version" QuorumLastVersion ClusterVersion = "quorum-last-version" LastVersion ClusterVersion = "last-version" ) func (cv ClusterVersion) String() string { if cv == CurrentVersion { return "current-version" } return string(cv) } type ClusterContext struct { Version ClusterVersion EnvVars map[string]string UseUnix bool } func WithHTTP2Debug() config.ClusterOption { return func(c *config.ClusterConfig) { ctx := ensureE2EClusterContext(c) if ctx.EnvVars == nil { ctx.EnvVars = map[string]string{} } // Enable debug mode to get logs with http2 headers (including authority) ctx.EnvVars["GODEBUG"] = "http2debug=2" c.ClusterContext = ctx } } func WithUnixClient() config.ClusterOption { return func(c *config.ClusterConfig) { ctx := ensureE2EClusterContext(c) ctx.UseUnix = true c.ClusterContext = ctx } } func WithTCPClient() config.ClusterOption { return func(c *config.ClusterConfig) { ctx := ensureE2EClusterContext(c) ctx.UseUnix = false c.ClusterContext = ctx } } func ensureE2EClusterContext(c *config.ClusterConfig) *ClusterContext { ctx, _ := c.ClusterContext.(*ClusterContext) if ctx == nil { ctx = &ClusterContext{} } return ctx } var experimentalFlags = map[string]struct{}{ "compact-hash-check-time": {}, "corrupt-check-time": {}, "compaction-batch-limit": {}, "watch-progress-notify-interval": {}, "warning-apply-duration": {}, "bootstrap-defrag-threshold-megabytes": {}, "memory-mlock": {}, "snapshot-catchup-entries": {}, "compaction-sleep-interval": {}, "downgrade-check-time": {}, "peer-skip-client-san-verification": {}, "enable-distributed-tracing": {}, "distributed-tracing-address": {}, "distributed-tracing-service-name": {}, "distributed-tracing-instance-id": {}, "distributed-tracing-sampling-rate": {}, } // convertFlag converts between experimental and non-experimental flags. // All experimental flags have been removed in version 3.7, but versions prior // to 3.6 only support experimental flags. The robustness tests use the same // code from the main branch to test all previously supported releases, so we // need to convert flags accordingly based on the binary version. func convertFlag(name, value string, ver *semver.Version) string { // For versions >= 3.6, use the normal (non-experimental) flag. if ver == nil || !ver.LessThan(version.V3_6) { return fmt.Sprintf("--%s=%s", name, value) } // For versions < 3.6, use the experimental flag if it exists in `experimentalFlags` if _, ok := experimentalFlags[name]; ok { return fmt.Sprintf("--experimental-%s=%s", name, value) } return fmt.Sprintf("--%s=%s", name, value) } func convertFlags(args []string, ver *semver.Version) []string { var retArgs []string for _, arg := range args { kv := strings.Split(arg, "=") if len(kv) != 2 { retArgs = append(retArgs, arg) continue } name := strings.TrimPrefix(kv[0], "--") name = strings.TrimPrefix(name, "experimental-") retArgs = append(retArgs, convertFlag(name, kv[1], ver)) } return retArgs } ================================================ FILE: tests/framework/e2e/curl.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "math/rand" "strings" "time" "go.etcd.io/etcd/pkg/v3/expect" ) type CURLReq struct { Username string Password string IsTLS bool Timeout int Endpoint string Value string Expected expect.ExpectedResponse Header string Ciphers string HTTPVersion string OutputFile string } func (r CURLReq) timeoutDuration() time.Duration { if r.Timeout != 0 { return time.Duration(r.Timeout) * time.Second } // assume a sane default to finish a curl request return 5 * time.Second } // CURLPrefixArgsCluster builds the beginning of a curl command for a given key // addressed to a random URL in the given cluster. func CURLPrefixArgsCluster(cfg *EtcdProcessClusterConfig, member EtcdProcess, method string, req CURLReq) []string { return CURLPrefixArgs(member.Config().ClientURL, cfg.Client, cfg.CN, method, req) } func CURLPrefixArgs(clientURL string, cfg ClientConfig, CN bool, method string, req CURLReq) []string { cmdArgs := []string{"curl"} if req.HTTPVersion != "" { cmdArgs = append(cmdArgs, "--http"+req.HTTPVersion) } if req.IsTLS { if cfg.ConnectionType != ClientTLSAndNonTLS { panic("should not use cURLPrefixArgsUseTLS when serving only TLS or non-TLS") } cmdArgs = append(cmdArgs, "--cacert", CaPath, "--cert", CertPath, "--key", PrivateKeyPath) clientURL = ToTLS(clientURL) } else if cfg.ConnectionType == ClientTLS { if CN { cmdArgs = append(cmdArgs, "--cacert", CaPath, "--cert", CertPath, "--key", PrivateKeyPath) } else { cmdArgs = append(cmdArgs, "--cacert", CaPath, "--cert", CertPath3, "--key", PrivateKeyPath3) } } ep := clientURL + req.Endpoint if req.Username != "" || req.Password != "" { cmdArgs = append(cmdArgs, "-L", "-u", fmt.Sprintf("%s:%s", req.Username, req.Password), ep) } else { cmdArgs = append(cmdArgs, "-L", ep) } if req.Timeout != 0 { cmdArgs = append(cmdArgs, "-m", fmt.Sprintf("%d", req.Timeout)) } if req.Header != "" { cmdArgs = append(cmdArgs, "-H", req.Header) } if req.Ciphers != "" { cmdArgs = append(cmdArgs, "--ciphers", req.Ciphers) } if req.OutputFile != "" { cmdArgs = append(cmdArgs, "--output", req.OutputFile) } switch method { case "POST", "PUT": dt := req.Value if !strings.HasPrefix(dt, "{") { // for non-JSON value dt = "value=" + dt } cmdArgs = append(cmdArgs, "-X", method, "-d", dt) } return cmdArgs } func CURLPost(clus *EtcdProcessCluster, req CURLReq) error { ctx, cancel := context.WithTimeout(context.Background(), req.timeoutDuration()) defer cancel() return SpawnWithExpectsContext(ctx, CURLPrefixArgsCluster(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "POST", req), nil, req.Expected) } func CURLPut(clus *EtcdProcessCluster, req CURLReq) error { ctx, cancel := context.WithTimeout(context.Background(), req.timeoutDuration()) defer cancel() return SpawnWithExpectsContext(ctx, CURLPrefixArgsCluster(clus.Cfg, clus.Procs[rand.Intn(clus.Cfg.ClusterSize)], "PUT", req), nil, req.Expected) } func CURLGet(clus *EtcdProcessCluster, req CURLReq) error { member := clus.Procs[rand.Intn(clus.Cfg.ClusterSize)] return CURLGetFromMember(clus, member, req) } func CURLGetFromMember(clus *EtcdProcessCluster, member EtcdProcess, req CURLReq) error { ctx, cancel := context.WithTimeout(context.Background(), req.timeoutDuration()) defer cancel() return SpawnWithExpectsContext(ctx, CURLPrefixArgsCluster(clus.Cfg, member, "GET", req), nil, req.Expected) } ================================================ FILE: tests/framework/e2e/downgrade.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "encoding/json" "fmt" "math/rand" "strings" "testing" "time" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/require" "go.uber.org/zap" "golang.org/x/sync/errgroup" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/testutils" ) func DowngradeEnable(t *testing.T, epc *EtcdProcessCluster, ver *semver.Version) { t.Logf("etcdctl downgrade enable %s", ver.String()) c := epc.Etcdctl() testutils.ExecuteWithTimeout(t, 20*time.Second, func() { err := c.DowngradeEnable(t.Context(), ver.String()) require.NoError(t, err) }) t.Log("Downgrade enabled, validating if cluster is ready for downgrade") for i := 0; i < len(epc.Procs); i++ { ValidateVersion(t, epc.Cfg, epc.Procs[i], version.Versions{ Cluster: ver.String(), Server: OffsetMinor(ver, 1).String(), Storage: ver.String(), }) } t.Log("Cluster is ready for downgrade") } func DowngradeCancel(t *testing.T, epc *EtcdProcessCluster) { c := epc.Etcdctl() var err error testutils.ExecuteWithTimeout(t, 1*time.Minute, func() { for { t.Logf("etcdctl downgrade cancel") err = c.DowngradeCancel(t.Context()) if err != nil { if strings.Contains(err.Error(), "no inflight downgrade job") { // cancellation has been performed successfully t.Log(err) err = nil break } t.Logf("etcdctl downgrade error: %v, retrying", err) continue } t.Logf("etcdctl downgrade cancel executed successfully") break } }) require.NoError(t, err) t.Log("Cluster downgrade cancellation is completed") } func ValidateDowngradeInfo(t *testing.T, clus *EtcdProcessCluster, expected *pb.DowngradeInfo) { cfg := clus.Cfg for i := 0; i < len(clus.Procs); i++ { member := clus.Procs[i] mc := member.Etcdctl() mName := member.Config().Name testutils.ExecuteWithTimeout(t, 1*time.Minute, func() { for { statuses, err := mc.Status(t.Context()) if err != nil { cfg.Logger.Warn("failed to get member status and retrying", zap.Error(err), zap.String("member", mName)) time.Sleep(time.Second) continue } require.Lenf(t, statuses, 1, "member %s", mName) got := (*pb.StatusResponse)(statuses[0]).GetDowngradeInfo() if got.GetEnabled() == expected.GetEnabled() && got.GetTargetVersion() == expected.GetTargetVersion() { cfg.Logger.Info("DowngradeInfo match", zap.String("member", mName)) break } cfg.Logger.Warn("DowngradeInfo didn't match retrying", zap.String("member", mName), zap.Dict("expected", zap.Bool("Enabled", expected.GetEnabled()), zap.String("TargetVersion", expected.GetTargetVersion()), ), zap.Dict("got", zap.Bool("Enabled", got.GetEnabled()), zap.String("TargetVersion", got.GetTargetVersion()), ), ) time.Sleep(time.Second) } }) } } func DowngradeUpgradeMembers(t *testing.T, lg *zap.Logger, clus *EtcdProcessCluster, numberOfMembersToChange int, downgradeEnabled bool, currentVersion, targetVersion *semver.Version) error { membersToChange := rand.Perm(len(clus.Procs))[:numberOfMembersToChange] t.Logf("Elect members for operations on members: %v", membersToChange) return DowngradeUpgradeMembersByID(t, lg, clus, membersToChange, downgradeEnabled, currentVersion, targetVersion) } func DowngradeUpgradeMembersByID(t *testing.T, lg *zap.Logger, clus *EtcdProcessCluster, membersToChange []int, downgradeEnabled bool, currentVersion, targetVersion *semver.Version) error { if lg == nil { lg = clus.lg } isDowngrade := targetVersion.LessThan(*currentVersion) opString := "upgrading" newExecPath := BinPath.Etcd if isDowngrade { opString = "downgrading" newExecPath = BinPath.EtcdLastRelease } binaryVersion, err := GetVersionFromBinary(newExecPath) if err != nil { return fmt.Errorf("failed to get binary version from %s: %w", newExecPath, err) } g := new(errgroup.Group) for _, memberID := range membersToChange { member := clus.Procs[memberID] if member.Config().ExecPath == newExecPath { return fmt.Errorf("member:%s is already running with the %s target binary - %s", member.Config().Name, opString, member.Config().ExecPath) } lg.Info(fmt.Sprintf("%s member", opString), zap.String("member", member.Config().Name)) if err := member.Stop(); err != nil { return err } // When we downgrade or upgrade a member, we need to re-generate the flags, to convert some non-experimental // flags to experimental flags, or vice verse. member.Config().Args = convertFlags(member.Config().Args, binaryVersion) member.Config().ExecPath = newExecPath lg.Info("Restarting member", zap.String("member", member.Config().Name)) // We shouldn't block on waiting for the member to be ready, // otherwise it will be blocked forever if other members are // not started yet. g.Go(func() error { return member.Start(t.Context()) }) } if err := g.Wait(); err != nil { return err } t.Log("Waiting health interval to make sure the leader propagates version to new processes") time.Sleep(etcdserver.HealthInterval) lg.Info("Validating versions") clusterVersion := targetVersion if !isDowngrade { if downgradeEnabled { // If the downgrade isn't cancelled yet, then the cluster // version will always stay at the lower version, no matter // what's the binary version of each member. clusterVersion = currentVersion } else { // If the downgrade has already been cancelled, then the // cluster version is the minimal server version. minVer, err := clus.MinServerVersion() if err != nil { return fmt.Errorf("failed to get min server version: %w", err) } clusterVersion = minVer } } for _, memberID := range membersToChange { member := clus.Procs[memberID] ValidateVersion(t, clus.Cfg, member, version.Versions{ Cluster: clusterVersion.String(), Server: targetVersion.String(), }) } return nil } func ValidateMemberVersions(t *testing.T, epc *EtcdProcessCluster, expect []*version.Versions) { for i := 0; i < len(epc.Procs); i++ { ValidateVersion(t, epc.Cfg, epc.Procs[i], *expect[i]) } t.Log("Cluster member version validation after downgrade cancellation is completed") } func ValidateVersion(t *testing.T, cfg *EtcdProcessClusterConfig, member EtcdProcess, expect version.Versions) { testutils.ExecuteWithTimeout(t, 1*time.Minute, func() { for { result, err := getMemberVersionByCurl(cfg, member) if err != nil { cfg.Logger.Warn("failed to get member version and retrying", zap.Error(err), zap.String("member", member.Config().Name)) time.Sleep(time.Second) continue } cfg.Logger.Info("Comparing versions", zap.String("member", member.Config().Name), zap.Any("got", result), zap.Any("want", expect)) if err := compareMemberVersion(expect, result); err != nil { cfg.Logger.Warn("Versions didn't match retrying", zap.Error(err), zap.String("member", member.Config().Name)) time.Sleep(time.Second) continue } cfg.Logger.Info("Versions match", zap.String("member", member.Config().Name)) break } }) } // OffsetMinor returns the version with offset from the original minor, with the same major. func OffsetMinor(v *semver.Version, offset int) *semver.Version { var minor int64 if offset >= 0 { minor = v.Minor + int64(offset) } else { diff := int64(-offset) if diff < v.Minor { minor = v.Minor - diff } } return &semver.Version{Major: v.Major, Minor: minor} } func majorMinorVersionsEqual(v1, v2 string) (bool, error) { ver1, err := semver.NewVersion(v1) if err != nil { return false, err } ver2, err := semver.NewVersion(v2) if err != nil { return false, err } return ver1.Major == ver2.Major && ver1.Minor == ver2.Minor, nil } func compareMemberVersion(expect version.Versions, target version.Versions) error { if expect.Server != "" { result, err := majorMinorVersionsEqual(expect.Server, target.Server) if err != nil { return err } if !result { return fmt.Errorf("expect etcdserver version %v, but got %v", expect.Server, target.Server) } } if expect.Cluster != "" { result, err := majorMinorVersionsEqual(expect.Cluster, target.Cluster) if err != nil { return err } if !result { return fmt.Errorf("expect etcdcluster version %v, but got %v", expect.Cluster, target.Cluster) } } if expect.Storage != "" { result, err := majorMinorVersionsEqual(expect.Storage, target.Storage) if err != nil { return err } if !result { return fmt.Errorf("expect storage version %v, but got %v", expect.Storage, target.Storage) } } return nil } func getMemberVersionByCurl(cfg *EtcdProcessClusterConfig, member EtcdProcess) (version.Versions, error) { args := CURLPrefixArgsCluster(cfg, member, "GET", CURLReq{Endpoint: "/version"}) lines, err := RunUtilCompletion(args, nil) if err != nil { return version.Versions{}, err } data := strings.Join(lines, "\n") result := version.Versions{} if err := json.Unmarshal([]byte(data), &result); err != nil { return version.Versions{}, fmt.Errorf("failed to unmarshal (%v): %w", data, err) } return result, nil } ================================================ FILE: tests/framework/e2e/e2e.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "net/url" "os" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/config" intf "go.etcd.io/etcd/tests/v3/framework/interfaces" ) type e2eRunner struct{} func NewE2eRunner() intf.TestRunner { return &e2eRunner{} } func (e e2eRunner) TestMain(m *testing.M) { InitFlags() v := m.Run() if v == 0 && testutil.CheckLeakedGoroutine() { os.Exit(1) } os.Exit(v) } func (e e2eRunner) BeforeTest(tb testing.TB) { BeforeTest(tb) } func (e e2eRunner) NewCluster(ctx context.Context, tb testing.TB, opts ...config.ClusterOption) intf.Cluster { cfg := config.NewClusterConfig(opts...) e2eConfig := NewConfig( WithClusterSize(cfg.ClusterSize), WithQuotaBackendBytes(cfg.QuotaBackendBytes), WithStrictReconfigCheck(cfg.StrictReconfigCheck), WithAuthTokenOpts(cfg.AuthToken), WithSnapshotCount(cfg.SnapshotCount), ) if ctx, ok := cfg.ClusterContext.(*ClusterContext); ok && ctx != nil { e2eConfig.Version = ctx.Version e2eConfig.EnvVars = ctx.EnvVars if ctx.UseUnix { e2eConfig.BaseClientScheme = "unix" } } switch cfg.ClientTLS { case config.NoTLS: e2eConfig.Client.ConnectionType = ClientNonTLS case config.AutoTLS: e2eConfig.Client.AutoTLS = true e2eConfig.Client.ConnectionType = ClientTLS case config.ManualTLS: e2eConfig.Client.AutoTLS = false e2eConfig.Client.ConnectionType = ClientTLS default: tb.Fatalf("ClientTLS config %q not supported", cfg.ClientTLS) } switch cfg.PeerTLS { case config.NoTLS: e2eConfig.IsPeerTLS = false e2eConfig.IsPeerAutoTLS = false case config.AutoTLS: e2eConfig.IsPeerTLS = true e2eConfig.IsPeerAutoTLS = true case config.ManualTLS: e2eConfig.IsPeerTLS = true e2eConfig.IsPeerAutoTLS = false default: tb.Fatalf("PeerTLS config %q not supported", cfg.PeerTLS) } epc, err := NewEtcdProcessCluster(ctx, tb, WithConfig(e2eConfig)) if err != nil { tb.Fatalf("could not start etcd integrationCluster: %s", err) } return &e2eCluster{tb, *epc} } type e2eCluster struct { t testing.TB EtcdProcessCluster } func (c *e2eCluster) Client(opts ...config.ClientOption) (intf.Client, error) { etcdctl, err := NewEtcdctl(c.Cfg.Client, c.EndpointsGRPC(), opts...) return e2eClient{etcdctl}, err } func (c *e2eCluster) Endpoints() []string { return c.EndpointsGRPC() } func (c *e2eCluster) Members() (ms []intf.Member) { for _, proc := range c.EtcdProcessCluster.Procs { ms = append(ms, e2eMember{EtcdProcess: proc, Cfg: c.Cfg}) } return ms } func (c *e2eCluster) TemplateEndpoints(tb testing.TB, pattern string) []string { tb.Helper() var endpoints []string for i := 0; i < c.Cfg.ClusterSize; i++ { ent := pattern ent = strings.ReplaceAll(ent, "${MEMBER_PORT}", fmt.Sprintf("%d", EtcdProcessBasePort+i*5)) endpoints = append(endpoints, ent) } return endpoints } func (c *e2eCluster) AssertAuthority(tb testing.TB, expectAuthorityPattern string) { for i := range c.Procs { line, _ := c.Procs[i].Logs().ExpectWithContext(tb.Context(), expect.ExpectedResponse{ Value: `http2: decoded hpack field header field ":authority"`, }) line = strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r") u, err := url.Parse(c.Procs[i].EndpointsGRPC()[0]) require.NoError(tb, err) expectAuthority := strings.ReplaceAll(expectAuthorityPattern, "${MEMBER_PORT}", u.Port()) expectLine := fmt.Sprintf(`http2: decoded hpack field header field ":authority" = %q`, expectAuthority) assert.Truef(tb, strings.HasSuffix(line, expectLine), "Got %q expected suffix %q", line, expectLine) } } type e2eClient struct { *EtcdctlV3 } type e2eMember struct { EtcdProcess Cfg *EtcdProcessClusterConfig } func (m e2eMember) Client() intf.Client { etcdctl, err := NewEtcdctl(m.Cfg.Client, m.EndpointsGRPC()) if err != nil { panic(err) } return e2eClient{etcdctl} } func (m e2eMember) Start(ctx context.Context) error { return m.EtcdProcess.Start(ctx) } func (m e2eMember) Stop() { m.EtcdProcess.Stop() } ================================================ FILE: tests/framework/e2e/etcd_process.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "bytes" "context" "errors" "fmt" "io" "net/http" "net/url" "os" "strings" "syscall" "testing" "time" "github.com/coreos/go-semver/semver" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/pkg/v3/proxy" "go.etcd.io/etcd/tests/v3/framework/config" ) var EtcdServerReadyLines = []string{"ready to serve client requests"} // EtcdProcess is a process that serves etcd requests. type EtcdProcess interface { EndpointsGRPC() []string EndpointsHTTP() []string EndpointsMetrics() []string Etcdctl(opts ...config.ClientOption) *EtcdctlV3 IsRunning() bool Wait(ctx context.Context) error Start(ctx context.Context) error Restart(ctx context.Context) error Stop() error Close() error Config() *EtcdServerProcessConfig PeerProxy() proxy.Server Failpoints() *BinaryFailpoints LazyFS() *LazyFS Logs() LogsExpect Kill() error Pause() error Resume() error } type LogsExpect interface { ExpectWithContext(context.Context, expect.ExpectedResponse) (string, error) Lines() []string LineCount() int } type EtcdServerProcess struct { cfg *EtcdServerProcessConfig proc *expect.ExpectProcess proxy proxy.Server lazyfs *LazyFS failpoints *BinaryFailpoints donec chan struct{} // closed when Interact() terminates } type EtcdServerProcessConfig struct { lg *zap.Logger ExecPath string Args []string TLSArgs []string EnvVars map[string]string Client ClientConfig DataDirPath string KeepDataDir bool Name string PeerURL url.URL ClientURL string ClientHTTPURL string MetricsURL string InitialToken string InitialCluster string GoFailPort int GoFailClientTimeout time.Duration LazyFSEnabled bool Proxy *proxy.ServerConfig } func NewEtcdServerProcess(tb testing.TB, cfg *EtcdServerProcessConfig) (*EtcdServerProcess, error) { if !fileutil.Exist(cfg.ExecPath) { return nil, fmt.Errorf("could not find etcd binary: %s", cfg.ExecPath) } if !cfg.KeepDataDir { if err := os.RemoveAll(cfg.DataDirPath); err != nil { return nil, err } if err := os.Mkdir(cfg.DataDirPath, 0o700); err != nil { return nil, err } } ep := &EtcdServerProcess{cfg: cfg, donec: make(chan struct{})} if cfg.GoFailPort != 0 { ep.failpoints = &BinaryFailpoints{ member: ep, clientTimeout: cfg.GoFailClientTimeout, } } if cfg.LazyFSEnabled { ep.lazyfs = newLazyFS(cfg.lg, cfg.DataDirPath, tb) } return ep, nil } func (ep *EtcdServerProcess) EndpointsGRPC() []string { return []string{ep.cfg.ClientURL} } func (ep *EtcdServerProcess) EndpointsHTTP() []string { if ep.cfg.ClientHTTPURL == "" { return []string{ep.cfg.ClientURL} } return []string{ep.cfg.ClientHTTPURL} } func (ep *EtcdServerProcess) EndpointsMetrics() []string { return []string{ep.cfg.MetricsURL} } func (ep *EtcdServerProcess) Etcdctl(opts ...config.ClientOption) *EtcdctlV3 { etcdctl, err := NewEtcdctl(ep.Config().Client, ep.EndpointsGRPC(), opts...) if err != nil { panic(err) } return etcdctl } func (ep *EtcdServerProcess) Start(ctx context.Context) error { ep.donec = make(chan struct{}) if ep.proc != nil { panic("already started") } if ep.cfg.Proxy != nil && ep.proxy == nil { ep.cfg.lg.Info("starting proxy...", zap.String("name", ep.cfg.Name), zap.String("from", ep.cfg.Proxy.From.String()), zap.String("to", ep.cfg.Proxy.To.String())) ep.proxy = proxy.NewServer(*ep.cfg.Proxy) select { case <-ep.proxy.Ready(): case err := <-ep.proxy.Error(): return err } } if ep.lazyfs != nil { ep.cfg.lg.Info("starting lazyfs...", zap.String("name", ep.cfg.Name)) err := ep.lazyfs.Start(ctx) if err != nil { return err } } ep.cfg.lg.Info("starting server...", zap.String("name", ep.cfg.Name)) proc, err := SpawnCmdWithLogger(ep.cfg.lg, append([]string{ep.cfg.ExecPath}, ep.cfg.Args...), ep.cfg.EnvVars, ep.cfg.Name) if err != nil { return err } ep.proc = proc err = ep.waitReady(ctx) if err == nil { ep.cfg.lg.Info("started server.", zap.String("name", ep.cfg.Name), zap.Int("pid", ep.proc.Pid())) } return err } func (ep *EtcdServerProcess) Restart(ctx context.Context) error { ep.cfg.lg.Info("restarting server...", zap.String("name", ep.cfg.Name)) if err := ep.Stop(); err != nil { return err } err := ep.Start(ctx) if err == nil { ep.cfg.lg.Info("restarted server", zap.String("name", ep.cfg.Name)) } return err } func (ep *EtcdServerProcess) Stop() (err error) { if ep == nil || ep.proc == nil { return nil } ep.cfg.lg.Info("stopping server...", zap.String("name", ep.cfg.Name)) defer func() { ep.proc = nil }() err = ep.proc.Stop() if err != nil { return err } err = ep.proc.Close() if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { return err } <-ep.donec ep.donec = make(chan struct{}) if ep.cfg.PeerURL.Scheme == "unix" || ep.cfg.PeerURL.Scheme == "unixs" { err = os.Remove(ep.cfg.PeerURL.Host + ep.cfg.PeerURL.Path) if err != nil && !os.IsNotExist(err) { return err } } ep.cfg.lg.Info("stopped server.", zap.String("name", ep.cfg.Name)) if ep.proxy != nil { ep.cfg.lg.Info("stopping proxy...", zap.String("name", ep.cfg.Name)) err = ep.proxy.Close() ep.proxy = nil if err != nil { return err } } if ep.lazyfs != nil { ep.cfg.lg.Info("stopping lazyfs...", zap.String("name", ep.cfg.Name)) err = ep.lazyfs.Stop() ep.lazyfs = nil if err != nil { return err } } return nil } func (ep *EtcdServerProcess) Close() error { ep.cfg.lg.Info("closing server...", zap.String("name", ep.cfg.Name)) if err := ep.Stop(); err != nil { return err } if !ep.cfg.KeepDataDir { ep.cfg.lg.Info("removing directory", zap.String("data-dir", ep.cfg.DataDirPath)) return os.RemoveAll(ep.cfg.DataDirPath) } return nil } func (ep *EtcdServerProcess) waitReady(ctx context.Context) error { defer close(ep.donec) err := WaitReadyExpectProc(ctx, ep.proc, EtcdServerReadyLines) if err != nil { return fmt.Errorf("failed to find etcd ready lines %q, err: %w", EtcdServerReadyLines, err) } return nil } func (ep *EtcdServerProcess) Config() *EtcdServerProcessConfig { return ep.cfg } func (ep *EtcdServerProcess) Logs() LogsExpect { if ep.proc == nil { ep.cfg.lg.Panic("Please grab logs before process is stopped") } return ep.proc } func (ep *EtcdServerProcess) Kill() error { ep.cfg.lg.Info("killing server...", zap.String("name", ep.cfg.Name)) if ep.proc == nil { return nil } return ep.proc.Signal(syscall.SIGKILL) } func (ep *EtcdServerProcess) Pause() error { ep.cfg.lg.Info("Pausing server...", zap.String("name", ep.cfg.Name)) return ep.proc.Signal(syscall.SIGSTOP) } func (ep *EtcdServerProcess) Resume() error { ep.cfg.lg.Info("Resuming server...", zap.String("name", ep.cfg.Name)) return ep.proc.Signal(syscall.SIGCONT) } func (ep *EtcdServerProcess) Wait(ctx context.Context) error { ch := make(chan struct{}) go func() { defer close(ch) if ep.proc != nil { ep.proc.Wait() exitCode, exitErr := ep.proc.ExitCode() ep.cfg.lg.Info("server exited", zap.String("name", ep.cfg.Name), zap.Int("code", exitCode), zap.Error(exitErr), ) } }() select { case <-ch: ep.proc = nil return nil case <-ctx.Done(): return ctx.Err() } } func (ep *EtcdServerProcess) IsRunning() bool { if ep.proc == nil { return false } exitCode, err := ep.proc.ExitCode() if errors.Is(err, expect.ErrProcessRunning) { return true } ep.cfg.lg.Info("server exited", zap.String("name", ep.cfg.Name), zap.Int("code", exitCode), zap.Error(err)) ep.proc = nil return false } func AssertProcessLogs(t *testing.T, ep EtcdProcess, expectLog string) { t.Helper() var err error ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() _, err = ep.Logs().ExpectWithContext(ctx, expect.ExpectedResponse{Value: expectLog}) if err != nil { t.Fatal(err) } } func (ep *EtcdServerProcess) PeerProxy() proxy.Server { return ep.proxy } func (ep *EtcdServerProcess) LazyFS() *LazyFS { return ep.lazyfs } func (ep *EtcdServerProcess) Failpoints() *BinaryFailpoints { return ep.failpoints } type BinaryFailpoints struct { member EtcdProcess availableCache map[string]string clientTimeout time.Duration } func (f *BinaryFailpoints) SetupEnv(failpoint, payload string) error { if f.member.IsRunning() { return errors.New("cannot setup environment variable while process is running") } f.member.Config().EnvVars["GOFAIL_FAILPOINTS"] = fmt.Sprintf("%s=%s", failpoint, payload) return nil } func (f *BinaryFailpoints) SetupHTTP(ctx context.Context, failpoint, payload string) error { host := fmt.Sprintf("127.0.0.1:%d", f.member.Config().GoFailPort) failpointURL := url.URL{ Scheme: "http", Host: host, Path: failpoint, } r, err := http.NewRequestWithContext(ctx, http.MethodPut, failpointURL.String(), bytes.NewBuffer([]byte(payload))) if err != nil { return err } httpClient := http.Client{ Timeout: 1 * time.Second, } if f.clientTimeout != 0 { httpClient.Timeout = f.clientTimeout } resp, err := httpClient.Do(r) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusNoContent { errMsg, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("bad status code: %d, err: %w", resp.StatusCode, err) } return fmt.Errorf("bad status code: %d, err: %s", resp.StatusCode, errMsg) } return nil } func (f *BinaryFailpoints) DeactivateHTTP(ctx context.Context, failpoint string) error { host := fmt.Sprintf("127.0.0.1:%d", f.member.Config().GoFailPort) failpointURL := url.URL{ Scheme: "http", Host: host, Path: failpoint, } r, err := http.NewRequestWithContext(ctx, http.MethodDelete, failpointURL.String(), nil) if err != nil { return err } httpClient := http.Client{ Timeout: time.Second, } if f.clientTimeout != 0 { httpClient.Timeout = f.clientTimeout } resp, err := httpClient.Do(r) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusNoContent { errMsg, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("bad status code: %d, err: %w", resp.StatusCode, err) } return fmt.Errorf("bad status code: %d, err: %s", resp.StatusCode, errMsg) } return nil } func (f *BinaryFailpoints) Enabled() bool { _, err := failpoints(f.member) return err == nil } func (f *BinaryFailpoints) Available(failpoint string) bool { if f.availableCache == nil { fs, err := failpoints(f.member) if err != nil { panic(err) } f.availableCache = fs } _, found := f.availableCache[failpoint] return found } func failpoints(member EtcdProcess) (map[string]string, error) { body, err := fetchFailpointsBody(member) if err != nil { return nil, err } defer body.Close() return parseFailpointsBody(body) } func fetchFailpointsBody(member EtcdProcess) (io.ReadCloser, error) { address := fmt.Sprintf("127.0.0.1:%d", member.Config().GoFailPort) failpointURL := url.URL{ Scheme: "http", Host: address, } resp, err := http.Get(failpointURL.String()) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { defer resp.Body.Close() errMsg, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("invalid status code: %d, err: %w", resp.StatusCode, err) } return nil, fmt.Errorf("invalid status code: %d, err:%s", resp.StatusCode, errMsg) } return resp.Body, nil } func parseFailpointsBody(body io.Reader) (map[string]string, error) { data, err := io.ReadAll(body) if err != nil { return nil, err } lines := strings.Split(string(data), "\n") failpoints := map[string]string{} for _, line := range lines { // Format: // failpoint=value parts := strings.SplitN(line, "=", 2) failpoint := parts[0] var value string if len(parts) == 2 { value = parts[1] } failpoints[failpoint] = value } return failpoints, nil } var GetVersionFromBinary = func(binaryPath string) (*semver.Version, error) { if !fileutil.Exist(binaryPath) { return nil, fmt.Errorf("binary path does not exist: %s", binaryPath) } lines, err := RunUtilCompletion([]string{binaryPath, "--version"}, nil) if err != nil { return nil, fmt.Errorf("could not find binary version from %s, err: %w", binaryPath, err) } for _, line := range lines { if strings.HasPrefix(line, "etcd Version:") { versionString := strings.TrimSpace(strings.SplitAfter(line, ":")[1]) version, err := semver.NewVersion(versionString) if err != nil { return nil, err } return &semver.Version{ Major: version.Major, Minor: version.Minor, Patch: version.Patch, }, nil } } return nil, fmt.Errorf("could not find version in binary output of %s, lines outputted were %v", binaryPath, lines) } // setGetVersionFromBinary changes the GetVersionFromBinary function to a mock in testing. func setGetVersionFromBinary(tb testing.TB, f func(binaryPath string) (*semver.Version, error)) { origGetVersionFromBinary := GetVersionFromBinary GetVersionFromBinary = f tb.Cleanup(func() { GetVersionFromBinary = origGetVersionFromBinary }) } func CouldSetSnapshotCatchupEntries(execPath string) bool { v, err := GetVersionFromBinary(execPath) if err != nil { return false } // snapshot-catchup-entries flag was backported in https://github.com/etcd-io/etcd/pull/17808 v3_5_14 := semver.Version{Major: 3, Minor: 5, Patch: 14} return v.Compare(v3_5_14) >= 0 } func IsSnapshotCatchupEntriesFlagAvailable(execPath string) bool { v, err := GetVersionFromBinary(execPath) if err != nil { return false } return !v.LessThan(version.V3_6) } ================================================ FILE: tests/framework/e2e/etcd_spawn.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "os" "strings" "go.uber.org/zap" "go.etcd.io/etcd/pkg/v3/expect" ) func SpawnCmd(args []string, envVars map[string]string) (*expect.ExpectProcess, error) { return SpawnNamedCmd(strings.Join(args, "_"), args, envVars) } func SpawnNamedCmd(processName string, args []string, envVars map[string]string) (*expect.ExpectProcess, error) { return SpawnCmdWithLogger(zap.NewNop(), args, envVars, processName) } func SpawnCmdWithLogger(lg *zap.Logger, args []string, envVars map[string]string, name string) (*expect.ExpectProcess, error) { wd, err := os.Getwd() if err != nil { return nil, err } env := mergeEnvVariables(envVars) lg.Info("spawning process", zap.Strings("args", args), zap.String("working-dir", wd), zap.String("name", name), zap.Strings("environment-variables", env)) return expect.NewExpectWithEnv(args[0], args[1:], env, name) } ================================================ FILE: tests/framework/e2e/etcdctl.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "encoding/json" "errors" "fmt" "io" "strconv" "strings" "time" "go.etcd.io/etcd/api/v3/authpb" "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/config" ) type EtcdctlV3 struct { cfg ClientConfig endpoints []string authConfig clientv3.AuthConfig } func NewEtcdctl(cfg ClientConfig, endpoints []string, opts ...config.ClientOption) (*EtcdctlV3, error) { ctl := &EtcdctlV3{ cfg: cfg, endpoints: endpoints, } for _, opt := range opts { opt(ctl) } if !ctl.authConfig.Empty() { client, err := clientv3.New(clientv3.Config{ Endpoints: ctl.endpoints, DialTimeout: 5 * time.Second, Username: ctl.authConfig.Username, Password: ctl.authConfig.Password, Token: ctl.authConfig.Token, }) if err != nil { return nil, err } client.Close() } return ctl, nil } func WithAuth(userName, password string) config.ClientOption { return func(c any) { ctl := c.(*EtcdctlV3) ctl.authConfig.Username = userName ctl.authConfig.Password = password } } func WithAuthToken(token string) config.ClientOption { return func(c any) { ctl := c.(*EtcdctlV3) ctl.authConfig.Token = token } } func WithEndpoints(endpoints []string) config.ClientOption { return func(c any) { ctl := c.(*EtcdctlV3) ctl.endpoints = endpoints } } func WithDialTimeout(tio time.Duration) config.ClientOption { return func(c any) { ctl := c.(*EtcdctlV3) ctl.cfg.DialTimeout = tio } } func (ctl *EtcdctlV3) DowngradeEnable(ctx context.Context, version string) error { _, err := SpawnWithExpectLines(ctx, ctl.cmdArgs("downgrade", "enable", version), nil, expect.ExpectedResponse{Value: "Downgrade enable success"}) return err } func (ctl *EtcdctlV3) DowngradeCancel(ctx context.Context) error { _, err := SpawnWithExpectLines(ctx, ctl.cmdArgs("downgrade", "cancel"), nil, expect.ExpectedResponse{Value: "Downgrade cancel success"}) return err } func (ctl *EtcdctlV3) Get(ctx context.Context, key string, o config.GetOptions) (*clientv3.GetResponse, error) { var args []string if o.Timeout != 0 { args = append(args, fmt.Sprintf("--command-timeout=%s", o.Timeout)) } if o.Serializable { args = append(args, "--consistency", "s") } args = append(args, "get", key, "-w", "json") if o.End != "" { args = append(args, o.End) } if o.Revision != 0 { args = append(args, fmt.Sprintf("--rev=%d", o.Revision)) } if o.Prefix { args = append(args, "--prefix") } if o.Limit != 0 { args = append(args, fmt.Sprintf("--limit=%d", o.Limit)) } if o.FromKey { args = append(args, "--from-key") } writeOut := "json" if o.CountOnly || o.KeysOnly { writeOut = "fields" } args = append(args, "-w", writeOut) if o.CountOnly { args = append(args, "--count-only") } if o.KeysOnly { args = append(args, "--keys-only") } if o.MaxCreateRevision != 0 { args = append(args, fmt.Sprintf("--max-create-rev=%d", o.MaxCreateRevision)) } if o.MinCreateRevision != 0 { args = append(args, fmt.Sprintf("--min-create-rev=%d", o.MinCreateRevision)) } if o.MaxModRevision != 0 { args = append(args, fmt.Sprintf("--max-mod-rev=%d", o.MaxModRevision)) } if o.MinModRevision != 0 { args = append(args, fmt.Sprintf("--min-mod-rev=%d", o.MinModRevision)) } switch o.SortBy { case clientv3.SortByCreateRevision: args = append(args, "--sort-by=CREATE") case clientv3.SortByModRevision: args = append(args, "--sort-by=MODIFY") case clientv3.SortByValue: args = append(args, "--sort-by=VALUE") case clientv3.SortByVersion: args = append(args, "--sort-by=VERSION") case clientv3.SortByKey: // nothing default: return nil, fmt.Errorf("bad sort target %v", o.SortBy) } switch o.Order { case clientv3.SortAscend: args = append(args, "--order=ASCEND") case clientv3.SortDescend: args = append(args, "--order=DESCEND") case clientv3.SortNone: // nothing default: return nil, fmt.Errorf("bad sort order %v", o.Order) } if o.CountOnly { cmd, err := SpawnCmd(ctl.cmdArgs(args...), nil) if err != nil { return nil, err } defer cmd.Close() // Relying on finding 'Count' as the last line of the output to get all the lines from cmd.Lines() _, err = cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "Count"}) if err != nil { return nil, err } return parseFieldsGetResponse(cmd.Lines()) } resp := clientv3.GetResponse{} err := ctl.spawnJSONCmd(ctx, &resp, args...) return &resp, err } func parseFieldsGetResponse(lines []string) (*clientv3.GetResponse, error) { resp := &clientv3.GetResponse{Header: &etcdserverpb.ResponseHeader{}} for _, l := range lines { fields := strings.Split(l, ":") key, value := strings.TrimSpace(fields[0]), strings.TrimSpace(fields[1]) var err error if key, err = strconv.Unquote(key); err != nil { return resp, err } switch key { case "ClusterID": resp.Header.ClusterId, err = strconv.ParseUint(value, 10, 64) case "MemberID": resp.Header.MemberId, err = strconv.ParseUint(value, 10, 64) case "Revision": resp.Header.Revision, err = strconv.ParseInt(value, 10, 64) case "RaftTerm": resp.Header.RaftTerm, err = strconv.ParseUint(value, 10, 64) case "More": resp.More, err = strconv.ParseBool(value) case "Count": resp.Count, err = strconv.ParseInt(value, 10, 64) default: return resp, fmt.Errorf("unexpected field %q:%s", key, value) } if err != nil { return resp, err } } return resp, nil } func (ctl *EtcdctlV3) Put(ctx context.Context, key, value string, opts config.PutOptions) (*clientv3.PutResponse, error) { resp := clientv3.PutResponse{} args := []string{} args = append(args, "put", key, value) if opts.LeaseID != 0 { args = append(args, "--lease", strconv.FormatInt(int64(opts.LeaseID), 16)) } if opts.Timeout != 0 { args = append(args, fmt.Sprintf("--command-timeout=%s", opts.Timeout)) } err := ctl.spawnJSONCmd(ctx, &resp, args...) return &resp, err } func (ctl *EtcdctlV3) Delete(ctx context.Context, key string, o config.DeleteOptions) (*clientv3.DeleteResponse, error) { args := []string{"del", key} if o.End != "" { args = append(args, o.End) } if o.Prefix { args = append(args, "--prefix") } if o.FromKey { args = append(args, "--from-key") } var resp clientv3.DeleteResponse err := ctl.spawnJSONCmd(ctx, &resp, args...) return &resp, err } func (ctl *EtcdctlV3) Txn(ctx context.Context, compares, ifSucess, ifFail []string, o config.TxnOptions) (*clientv3.TxnResponse, error) { args := ctl.cmdArgs() args = append(args, "txn") if o.Interactive { args = append(args, "--interactive") } args = append(args, "-w", "json", "--hex=true") cmd, err := SpawnCmd(args, nil) if err != nil { return nil, err } defer cmd.Close() _, err = cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "compares:"}) if err != nil { return nil, err } for _, cmp := range compares { if err = cmd.Send(cmp + "\r"); err != nil { return nil, err } } if err = cmd.Send("\r"); err != nil { return nil, err } _, err = cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "success requests (get, put, del):"}) if err != nil { return nil, err } for _, req := range ifSucess { if err = cmd.Send(req + "\r"); err != nil { return nil, err } } if err = cmd.Send("\r"); err != nil { return nil, err } _, err = cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "failure requests (get, put, del):"}) if err != nil { return nil, err } for _, req := range ifFail { if err = cmd.Send(req + "\r"); err != nil { return nil, err } } if err = cmd.Send("\r"); err != nil { return nil, err } var line string line, err = cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "header"}) if err != nil { return nil, err } var resp clientv3.TxnResponse addTxnResponse(&resp, line) err = json.Unmarshal([]byte(line), &resp) return &resp, err } // addTxnResponse looks for ResponseOp json tags and adds the objects for json decoding func addTxnResponse(resp *clientv3.TxnResponse, jsonData string) { if resp == nil { return } if resp.Responses == nil { resp.Responses = []*etcdserverpb.ResponseOp{} } jd := json.NewDecoder(strings.NewReader(jsonData)) for { t, e := jd.Token() if errors.Is(e, io.EOF) { break } if t == "response_range" { resp.Responses = append(resp.Responses, &etcdserverpb.ResponseOp{ Response: &etcdserverpb.ResponseOp_ResponseRange{}, }) } if t == "response_put" { resp.Responses = append(resp.Responses, &etcdserverpb.ResponseOp{ Response: &etcdserverpb.ResponseOp_ResponsePut{}, }) } if t == "response_delete_range" { resp.Responses = append(resp.Responses, &etcdserverpb.ResponseOp{ Response: &etcdserverpb.ResponseOp_ResponseDeleteRange{}, }) } if t == "response_txn" { resp.Responses = append(resp.Responses, &etcdserverpb.ResponseOp{ Response: &etcdserverpb.ResponseOp_ResponseTxn{}, }) } } } func (ctl *EtcdctlV3) MemberList(ctx context.Context, serializable bool) (*clientv3.MemberListResponse, error) { var resp clientv3.MemberListResponse args := []string{"member", "list"} if serializable { args = append(args, "--consistency", "s") } err := ctl.spawnJSONCmd(ctx, &resp, args...) return &resp, err } func (ctl *EtcdctlV3) MemberAdd(ctx context.Context, name string, peerAddrs []string) (*clientv3.MemberAddResponse, error) { var resp clientv3.MemberAddResponse err := ctl.spawnJSONCmd(ctx, &resp, "member", "add", name, "--peer-urls", strings.Join(peerAddrs, ",")) return &resp, err } func (ctl *EtcdctlV3) MemberAddAsLearner(ctx context.Context, name string, peerAddrs []string) (*clientv3.MemberAddResponse, error) { var resp clientv3.MemberAddResponse err := ctl.spawnJSONCmd(ctx, &resp, "member", "add", name, "--learner", "--peer-urls", strings.Join(peerAddrs, ",")) return &resp, err } func (ctl *EtcdctlV3) MemberRemove(ctx context.Context, id uint64) (*clientv3.MemberRemoveResponse, error) { var resp clientv3.MemberRemoveResponse err := ctl.spawnJSONCmd(ctx, &resp, "member", "remove", fmt.Sprintf("%x", id)) return &resp, err } func (ctl *EtcdctlV3) MemberPromote(ctx context.Context, id uint64) (*clientv3.MemberPromoteResponse, error) { var resp clientv3.MemberPromoteResponse err := ctl.spawnJSONCmd(ctx, &resp, "member", "promote", fmt.Sprintf("%x", id)) return &resp, err } // MoveLeader requests current leader to transfer its leadership to the transferee. // Request must be made to the leader. func (ctl *EtcdctlV3) MoveLeader(ctx context.Context, transfereeID uint64) error { _, err := SpawnWithExpectLines(ctx, ctl.cmdArgs("move-leader", fmt.Sprintf("%x", transfereeID)), nil, expect.ExpectedResponse{Value: "Leadership transferred"}) return err } func (ctl *EtcdctlV3) cmdArgs(args ...string) []string { cmdArgs := []string{BinPath.Etcdctl} for k, v := range ctl.flags() { cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", k, v)) } return append(cmdArgs, args...) } func (ctl *EtcdctlV3) flags() map[string]string { fmap := make(map[string]string) if ctl.cfg.ConnectionType == ClientTLS { if ctl.cfg.AutoTLS { fmap["insecure-transport"] = "false" fmap["insecure-skip-tls-verify"] = "true" } else if ctl.cfg.RevokeCerts { fmap["cacert"] = CaPath fmap["cert"] = RevokedCertPath fmap["key"] = RevokedPrivateKeyPath } else { fmap["cacert"] = CaPath fmap["cert"] = CertPath fmap["key"] = PrivateKeyPath } } fmap["endpoints"] = strings.Join(ctl.endpoints, ",") if ctl.authConfig.Token != "" { fmap["auth-jwt-token"] = ctl.authConfig.Token } else if !ctl.authConfig.Empty() { fmap["user"] = ctl.authConfig.Username + ":" + ctl.authConfig.Password } if ctl.cfg.DialTimeout != 0 { fmap["dial-timeout"] = ctl.cfg.DialTimeout.String() } return fmap } func (ctl *EtcdctlV3) Compact(ctx context.Context, rev int64, o config.CompactOption) (*clientv3.CompactResponse, error) { args := ctl.cmdArgs("compact", fmt.Sprint(rev)) if o.Timeout != 0 { args = append(args, fmt.Sprintf("--command-timeout=%s", o.Timeout)) } if o.Physical { args = append(args, "--physical") } _, err := SpawnWithExpectLines(ctx, args, nil, expect.ExpectedResponse{Value: fmt.Sprintf("compacted revision %v", rev)}) return nil, err } func (ctl *EtcdctlV3) Status(ctx context.Context) ([]*clientv3.StatusResponse, error) { var epStatus []*struct { Endpoint string Status *clientv3.StatusResponse } err := ctl.spawnJSONCmd(ctx, &epStatus, "endpoint", "status") if err != nil { return nil, err } resp := make([]*clientv3.StatusResponse, len(epStatus)) for i, e := range epStatus { resp[i] = e.Status } return resp, err } func (ctl *EtcdctlV3) HashKV(ctx context.Context, rev int64) ([]*clientv3.HashKVResponse, error) { var epHashKVs []*struct { Endpoint string HashKV *clientv3.HashKVResponse } err := ctl.spawnJSONCmd(ctx, &epHashKVs, "endpoint", "hashkv", "--rev", fmt.Sprint(rev)) if err != nil { return nil, err } resp := make([]*clientv3.HashKVResponse, len(epHashKVs)) for i, e := range epHashKVs { resp[i] = e.HashKV } return resp, err } func (ctl *EtcdctlV3) Health(ctx context.Context) error { args := ctl.cmdArgs() args = append(args, "endpoint", "health") lines := make([]expect.ExpectedResponse, len(ctl.endpoints)) for i := range lines { lines[i] = expect.ExpectedResponse{Value: "is healthy"} } _, err := SpawnWithExpectLines(ctx, args, nil, lines...) return err } func (ctl *EtcdctlV3) Grant(ctx context.Context, ttl int64) (*clientv3.LeaseGrantResponse, error) { args := ctl.cmdArgs() args = append(args, "lease", "grant", strconv.FormatInt(ttl, 10), "-w", "json") cmd, err := SpawnCmd(args, nil) if err != nil { return nil, err } defer cmd.Close() var resp clientv3.LeaseGrantResponse line, err := cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "ID"}) if err != nil { return nil, err } err = json.Unmarshal([]byte(line), &resp) return &resp, err } func (ctl *EtcdctlV3) TimeToLive(ctx context.Context, id clientv3.LeaseID, o config.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) { args := ctl.cmdArgs() args = append(args, "lease", "timetolive", strconv.FormatInt(int64(id), 16), "-w", "json") if o.WithAttachedKeys { args = append(args, "--keys") } cmd, err := SpawnCmd(args, nil) if err != nil { return nil, err } defer cmd.Close() var resp clientv3.LeaseTimeToLiveResponse line, err := cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "member_id"}) if err != nil { return nil, err } err = json.Unmarshal([]byte(line), &resp) return &resp, err } func (ctl *EtcdctlV3) Defragment(ctx context.Context, o config.DefragOption) error { args := append(ctl.cmdArgs(), "defrag") if o.Timeout != 0 { args = append(args, fmt.Sprintf("--command-timeout=%s", o.Timeout)) } lines := make([]expect.ExpectedResponse, len(ctl.endpoints)) for i := range lines { lines[i] = expect.ExpectedResponse{Value: "Finished defragmenting etcd member"} } _, err := SpawnWithExpectLines(ctx, args, map[string]string{}, lines...) return err } func (ctl *EtcdctlV3) Leases(ctx context.Context) (*clientv3.LeaseLeasesResponse, error) { args := ctl.cmdArgs("lease", "list", "-w", "json") cmd, err := SpawnCmd(args, nil) if err != nil { return nil, err } defer cmd.Close() var resp clientv3.LeaseLeasesResponse line, err := cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "member_id"}) if err != nil { return nil, err } err = json.Unmarshal([]byte(line), &resp) return &resp, err } func (ctl *EtcdctlV3) KeepAliveOnce(ctx context.Context, id clientv3.LeaseID) (*clientv3.LeaseKeepAliveResponse, error) { args := ctl.cmdArgs("lease", "keep-alive", strconv.FormatInt(int64(id), 16), "--once", "-w", "json") cmd, err := SpawnCmd(args, nil) if err != nil { return nil, err } defer cmd.Close() var resp clientv3.LeaseKeepAliveResponse line, err := cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "ID"}) if err != nil { return nil, err } err = json.Unmarshal([]byte(line), &resp) return &resp, err } func (ctl *EtcdctlV3) Revoke(ctx context.Context, id clientv3.LeaseID) (*clientv3.LeaseRevokeResponse, error) { var resp clientv3.LeaseRevokeResponse err := ctl.spawnJSONCmd(ctx, &resp, "lease", "revoke", strconv.FormatInt(int64(id), 16)) return &resp, err } func (ctl *EtcdctlV3) AlarmList(ctx context.Context) (*clientv3.AlarmResponse, error) { var resp clientv3.AlarmResponse err := ctl.spawnJSONCmd(ctx, &resp, "alarm", "list") return &resp, err } func (ctl *EtcdctlV3) AlarmDisarm(ctx context.Context, _ *clientv3.AlarmMember) (*clientv3.AlarmResponse, error) { args := ctl.cmdArgs() args = append(args, "alarm", "disarm", "-w", "json") ep, err := SpawnCmd(args, nil) if err != nil { return nil, err } defer ep.Close() var resp clientv3.AlarmResponse line, err := ep.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "alarm"}) if err != nil { return nil, err } err = json.Unmarshal([]byte(line), &resp) return &resp, err } func (ctl *EtcdctlV3) AuthEnable(ctx context.Context) error { args := []string{"auth", "enable"} cmd, err := SpawnCmd(ctl.cmdArgs(args...), nil) if err != nil { return err } defer cmd.Close() _, err = cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "Authentication Enabled"}) return err } func (ctl *EtcdctlV3) AuthDisable(ctx context.Context) error { args := []string{"auth", "disable"} cmd, err := SpawnCmd(ctl.cmdArgs(args...), nil) if err != nil { return err } defer cmd.Close() _, err = cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "Authentication Disabled"}) return err } func (ctl *EtcdctlV3) AuthStatus(ctx context.Context) (*clientv3.AuthStatusResponse, error) { var resp clientv3.AuthStatusResponse err := ctl.spawnJSONCmd(ctx, &resp, "auth", "status") return &resp, err } func (ctl *EtcdctlV3) UserAdd(ctx context.Context, name, password string, opts config.UserAddOptions) (*clientv3.AuthUserAddResponse, error) { args := ctl.cmdArgs() args = append(args, "user", "add") if password == "" { args = append(args, name) } else { args = append(args, fmt.Sprintf("%s:%s", name, password)) } if opts.NoPassword { args = append(args, "--no-password") } args = append(args, "--interactive=false", "-w", "json") cmd, err := SpawnCmd(args, nil) if err != nil { return nil, err } defer cmd.Close() // If no password is provided, and NoPassword isn't set, the CLI will always // wait for a password, send an enter in this case for an "empty" password. if !opts.NoPassword && password == "" { err = cmd.Send("\n") if err != nil { return nil, err } } var resp clientv3.AuthUserAddResponse line, err := cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "header"}) if err != nil { return nil, err } err = json.Unmarshal([]byte(line), &resp) return &resp, err } func (ctl *EtcdctlV3) UserGet(ctx context.Context, name string) (*clientv3.AuthUserGetResponse, error) { var resp clientv3.AuthUserGetResponse err := ctl.spawnJSONCmd(ctx, &resp, "user", "get", name) return &resp, err } func (ctl *EtcdctlV3) UserList(ctx context.Context) (*clientv3.AuthUserListResponse, error) { var resp clientv3.AuthUserListResponse err := ctl.spawnJSONCmd(ctx, &resp, "user", "list") return &resp, err } func (ctl *EtcdctlV3) UserDelete(ctx context.Context, name string) (*clientv3.AuthUserDeleteResponse, error) { var resp clientv3.AuthUserDeleteResponse err := ctl.spawnJSONCmd(ctx, &resp, "user", "delete", name) return &resp, err } func (ctl *EtcdctlV3) UserChangePass(ctx context.Context, user, newPass string) error { args := ctl.cmdArgs() args = append(args, "user", "passwd", user, "--interactive=false") cmd, err := SpawnCmd(args, nil) if err != nil { return err } defer cmd.Close() err = cmd.Send(newPass + "\n") if err != nil { return err } _, err = cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "Password updated"}) return err } func (ctl *EtcdctlV3) UserGrantRole(ctx context.Context, user string, role string) (*clientv3.AuthUserGrantRoleResponse, error) { var resp clientv3.AuthUserGrantRoleResponse err := ctl.spawnJSONCmd(ctx, &resp, "user", "grant-role", user, role) return &resp, err } func (ctl *EtcdctlV3) UserRevokeRole(ctx context.Context, user string, role string) (*clientv3.AuthUserRevokeRoleResponse, error) { var resp clientv3.AuthUserRevokeRoleResponse err := ctl.spawnJSONCmd(ctx, &resp, "user", "revoke-role", user, role) return &resp, err } func (ctl *EtcdctlV3) RoleAdd(ctx context.Context, name string) (*clientv3.AuthRoleAddResponse, error) { var resp clientv3.AuthRoleAddResponse err := ctl.spawnJSONCmd(ctx, &resp, "role", "add", name) return &resp, err } func (ctl *EtcdctlV3) RoleGrantPermission(ctx context.Context, name string, key, rangeEnd string, permType clientv3.PermissionType) (*clientv3.AuthRoleGrantPermissionResponse, error) { permissionType := authpb.Permission_Type_name[int32(permType)] var resp clientv3.AuthRoleGrantPermissionResponse err := ctl.spawnJSONCmd(ctx, &resp, "role", "grant-permission", name, permissionType, key, rangeEnd) return &resp, err } func (ctl *EtcdctlV3) RoleGet(ctx context.Context, role string) (*clientv3.AuthRoleGetResponse, error) { var resp clientv3.AuthRoleGetResponse err := ctl.spawnJSONCmd(ctx, &resp, "role", "get", role) return &resp, err } func (ctl *EtcdctlV3) RoleList(ctx context.Context) (*clientv3.AuthRoleListResponse, error) { var resp clientv3.AuthRoleListResponse err := ctl.spawnJSONCmd(ctx, &resp, "role", "list") return &resp, err } func (ctl *EtcdctlV3) RoleRevokePermission(ctx context.Context, role string, key, rangeEnd string) (*clientv3.AuthRoleRevokePermissionResponse, error) { var resp clientv3.AuthRoleRevokePermissionResponse err := ctl.spawnJSONCmd(ctx, &resp, "role", "revoke-permission", role, key, rangeEnd) return &resp, err } func (ctl *EtcdctlV3) RoleDelete(ctx context.Context, role string) (*clientv3.AuthRoleDeleteResponse, error) { var resp clientv3.AuthRoleDeleteResponse err := ctl.spawnJSONCmd(ctx, &resp, "role", "delete", role) return &resp, err } func (ctl *EtcdctlV3) spawnJSONCmd(ctx context.Context, output any, args ...string) error { args = append(args, "-w", "json") cmd, err := SpawnCmd(append(ctl.cmdArgs(), args...), nil) if err != nil { return err } defer cmd.Close() line, err := cmd.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "header"}) if err != nil { return err } return json.Unmarshal([]byte(line), output) } func (ctl *EtcdctlV3) Watch(ctx context.Context, key string, opts config.WatchOptions) clientv3.WatchChan { args := ctl.cmdArgs() args = append(args, "watch", key) if opts.RangeEnd != "" { args = append(args, opts.RangeEnd) } args = append(args, "-w", "json") if opts.Prefix { args = append(args, "--prefix") } if opts.Revision != 0 { args = append(args, "--rev", fmt.Sprint(opts.Revision)) } proc, err := SpawnCmd(args, nil) if err != nil { return nil } ch := make(chan clientv3.WatchResponse) go func() { defer proc.Stop() for { select { case <-ctx.Done(): close(ch) return default: if line := proc.ReadLine(); line != "" { var resp clientv3.WatchResponse json.Unmarshal([]byte(line), &resp) if resp.Canceled { ch <- resp close(ch) return } if len(resp.Events) > 0 { ch <- resp } } } } }() return ch } ================================================ FILE: tests/framework/e2e/etcdctl_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "encoding/json" "testing" clientv3 "go.etcd.io/etcd/client/v3" ) func Test_addTxnResponse(t *testing.T) { jsonData := `{"header":{"cluster_id":238453183653593855,"member_id":14578408409545168728,"revision":3,"raft_term":2},"succeeded":true,"responses":[{"Response":{"response_range":{"header":{"revision":3},"kvs":[{"key":"a2V5MQ==","create_revision":2,"mod_revision":2,"version":1,"value":"dmFsdWUx"}],"count":1}}},{"Response":{"response_range":{"header":{"revision":3},"kvs":[{"key":"a2V5Mg==","create_revision":3,"mod_revision":3,"version":1,"value":"dmFsdWUy"}],"count":1}}}]}` var resp clientv3.TxnResponse addTxnResponse(&resp, jsonData) err := json.Unmarshal([]byte(jsonData), &resp) if err != nil { t.Errorf("json Unmarshal failed. err: %s", err) } enc, err := json.Marshal(resp) if err != nil { t.Errorf("json Marshal failed. err: %s", err) } if string(enc) != jsonData { t.Error("could not get original message after encoding") } } ================================================ FILE: tests/framework/e2e/flags.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "flag" "os" "runtime" "go.etcd.io/etcd/tests/v3/framework/testutils" ) var ( CertDir string CertPath string PrivateKeyPath string CaPath string CertPath2 string PrivateKeyPath2 string CertPath3 string PrivateKeyPath3 string CrlPath string RevokedCertPath string RevokedPrivateKeyPath string BinPath binPath FixturesDir = testutils.MustAbsPath("../fixtures") ) type binPath struct { Etcd string EtcdLastRelease string Etcdctl string Etcdutl string LazyFS string } func (bp *binPath) LazyFSAvailable() bool { _, err := os.Stat(bp.LazyFS) if err != nil { if !os.IsNotExist(err) { panic(err) } return false } return true } func InitFlags() { os.Setenv("ETCD_UNSUPPORTED_ARCH", runtime.GOARCH) binDirDef := testutils.MustAbsPath("../../bin") certDirDef := FixturesDir binDir := flag.String("bin-dir", binDirDef, "The directory for store etcd and etcdctl binaries.") binLastRelease := flag.String("bin-last-release", "", "The path for the last release etcd binary.") flag.StringVar(&CertDir, "cert-dir", certDirDef, "The directory for store certificate files.") flag.Parse() BinPath = binPath{ Etcd: *binDir + "/etcd", EtcdLastRelease: *binDir + "/etcd-last-release", Etcdctl: *binDir + "/etcdctl", Etcdutl: *binDir + "/etcdutl", LazyFS: *binDir + "/lazyfs", } if *binLastRelease != "" { BinPath.EtcdLastRelease = *binLastRelease } CertPath = CertDir + "/server.crt" PrivateKeyPath = CertDir + "/server.key.insecure" CaPath = CertDir + "/ca.crt" RevokedCertPath = CertDir + "/server-revoked.crt" RevokedPrivateKeyPath = CertDir + "/server-revoked.key.insecure" CrlPath = CertDir + "/revoke.crl" CertPath2 = CertDir + "/server2.crt" PrivateKeyPath2 = CertDir + "/server2.key.insecure" CertPath3 = CertDir + "/server3.crt" PrivateKeyPath3 = CertDir + "/server3.key.insecure" } ================================================ FILE: tests/framework/e2e/lazyfs.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "fmt" "os" "path/filepath" "go.uber.org/zap" "go.etcd.io/etcd/pkg/v3/expect" ) func newLazyFS(lg *zap.Logger, dataDir string, tmp TempDirProvider) *LazyFS { return &LazyFS{ lg: lg, DataDir: dataDir, LazyFSDir: tmp.TempDir(), } } type TempDirProvider interface { TempDir() string } type LazyFS struct { lg *zap.Logger DataDir string LazyFSDir string ep *expect.ExpectProcess } func (fs *LazyFS) Start(ctx context.Context) (err error) { if fs.ep != nil { return nil } err = os.WriteFile(fs.configPath(), fs.config(), 0o666) if err != nil { return err } dataPath := filepath.Join(fs.LazyFSDir, "data") err = os.Mkdir(dataPath, 0o700) if err != nil { return err } flags := []string{fs.DataDir, "--config-path", fs.configPath(), "-o", "modules=subdir", "-o", "subdir=" + dataPath, "-f"} fs.lg.Info("Started lazyfs", zap.Strings("flags", flags)) fs.ep, err = expect.NewExpect(BinPath.LazyFS, flags...) if err != nil { return err } _, err = fs.ep.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "waiting for fault commands"}) return err } func (fs *LazyFS) configPath() string { return filepath.Join(fs.LazyFSDir, "config.toml") } func (fs *LazyFS) socketPath() string { return filepath.Join(fs.LazyFSDir, "sock.fifo") } func (fs *LazyFS) config() []byte { return []byte(fmt.Sprintf(`[faults] fifo_path=%q [cache] apply_eviction=false [cache.simple] custom_size="1gb" blocks_per_page=1 [filesystem] log_all_operations=false `, fs.socketPath())) } func (fs *LazyFS) Stop() error { if fs.ep == nil { return nil } defer func() { fs.ep = nil }() err := fs.ep.Stop() if err != nil { return err } return fs.ep.Close() } func (fs *LazyFS) ClearCache(ctx context.Context) error { err := os.WriteFile(fs.socketPath(), []byte("lazyfs::clear-cache\n"), 0o666) if err != nil { return err } // TODO: Wait for response on socket instead of reading logs to get command completion. // Set `fifo_path_completed` config for LazyFS to create separate socket to write when it has completed command. _, err = fs.ep.ExpectWithContext(ctx, expect.ExpectedResponse{Value: "cache is cleared"}) return err } ================================================ FILE: tests/framework/e2e/metrics.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "bytes" "io" "net/http" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" ) func GetMetrics(metricsURL string) (map[string]*dto.MetricFamily, error) { httpClient := http.Client{Transport: &http.Transport{}} resp, err := httpClient.Get(metricsURL) if err != nil { return nil, err } defer resp.Body.Close() data, err := io.ReadAll(resp.Body) if err != nil { return nil, err } parser := expfmt.NewTextParser(model.LegacyValidation) return parser.TextToMetricFamilies(bytes.NewReader(data)) } ================================================ FILE: tests/framework/e2e/testing.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func BeforeTest(tb testing.TB) { SkipInShortMode(tb) testutil.BeforeTest(tb) } ================================================ FILE: tests/framework/e2e/util.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package e2e import ( "context" "encoding/json" "fmt" "math/rand" "os" "strings" "testing" "time" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/pkg/v3/expect" ) func WaitReadyExpectProc(ctx context.Context, exproc *expect.ExpectProcess, readyStrs []string) error { matchSet := func(l string) bool { for _, s := range readyStrs { if strings.Contains(l, s) { return true } } return false } _, err := exproc.ExpectFunc(ctx, matchSet) return err } func SpawnWithExpect(args []string, expected expect.ExpectedResponse) error { return SpawnWithExpects(args, nil, []expect.ExpectedResponse{expected}...) } func SpawnWithExpectWithEnv(args []string, envVars map[string]string, expected expect.ExpectedResponse) error { return SpawnWithExpects(args, envVars, []expect.ExpectedResponse{expected}...) } func SpawnWithExpects(args []string, envVars map[string]string, xs ...expect.ExpectedResponse) error { return SpawnWithExpectsContext(context.TODO(), args, envVars, xs...) } func SpawnWithExpectsContext(ctx context.Context, args []string, envVars map[string]string, xs ...expect.ExpectedResponse) error { _, err := SpawnWithExpectLines(ctx, args, envVars, xs...) return err } func SpawnWithExpectLines(ctx context.Context, args []string, envVars map[string]string, xs ...expect.ExpectedResponse) ([]string, error) { proc, err := SpawnCmd(args, envVars) if err != nil { return nil, err } defer proc.Close() // process until either stdout or stderr contains // the expected string var ( lines []string ) for _, txt := range xs { l, lerr := proc.ExpectWithContext(ctx, txt) if lerr != nil { proc.Close() return nil, fmt.Errorf("%v %w (expected %q, got %q). Try EXPECT_DEBUG=TRUE", args, lerr, txt.Value, lines) } lines = append(lines, l) } perr := proc.Close() if perr != nil { return lines, fmt.Errorf("err: %w, with output lines %v", perr, proc.Lines()) } l := proc.LineCount() if len(xs) == 0 && l != 0 { // expect no output return nil, fmt.Errorf("unexpected output from %v (got lines %q, line count %d). Try EXPECT_DEBUG=TRUE", args, lines, l) } return lines, nil } func RunUtilCompletion(args []string, envVars map[string]string) ([]string, error) { proc, err := SpawnCmd(args, envVars) if err != nil { return nil, fmt.Errorf("failed to spawn command %v with error: %w", args, err) } proc.Wait() err = proc.Close() if err != nil { return nil, fmt.Errorf("failed to close command %v with error: %w", args, err) } return proc.Lines(), nil } func RandomLeaseID() int64 { return rand.New(rand.NewSource(time.Now().UnixNano())).Int63() } func DataMarshal(data any) (d string, e error) { m, err := json.Marshal(data) if err != nil { return "", err } return string(m), nil } func CloseWithTimeout(p *expect.ExpectProcess, d time.Duration) error { errc := make(chan error, 1) go func() { errc <- p.Close() }() select { case err := <-errc: return err case <-time.After(d): p.Stop() // retry close after stopping to collect SIGQUIT data, if any CloseWithTimeout(p, time.Second) } return fmt.Errorf("took longer than %v to Close process %+v", d, p) } func setupScheme(s string, isTLS bool) string { if s == "" { s = "http" } if isTLS { s = ToTLS(s) } return s } func ToTLS(s string) string { if strings.Contains(s, "http") && !strings.Contains(s, "https") { return strings.Replace(s, "http", "https", 1) } if strings.Contains(s, "unix") && !strings.Contains(s, "unixs") { return strings.Replace(s, "unix", "unixs", 1) } return s } func SkipInShortMode(tb testing.TB) { testutil.SkipTestIfShortMode(tb, "e2e tests are not running in --short mode") } func mergeEnvVariables(envVars map[string]string) []string { var env []string // Environment variables are passed as parameter have higher priority // than os environment variables. for k, v := range envVars { env = append(env, fmt.Sprintf("%s=%s", k, v)) } // Now, we can set os environment variables not passed as parameter. currVars := os.Environ() for _, v := range currVars { p := strings.Split(v, "=") // TODO: Remove PATH when we stop using system binaries (`awk`, `echo`) if !strings.HasPrefix(p[0], "ETCD_") && !strings.HasPrefix(p[0], "ETCDCTL_") && !strings.HasPrefix(p[0], "EXPECT_") && p[0] != "PATH" { continue } if _, ok := envVars[p[0]]; !ok { env = append(env, fmt.Sprintf("%s=%s", p[0], p[1])) } } return env } ================================================ FILE: tests/framework/integration/bridge.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "io" "net" "sync" ) type Dialer interface { Dial() (net.Conn, error) } // Bridge interface exposing methods of the bridge type Bridge interface { Close() DropConnections() PauseConnections() UnpauseConnections() Blackhole() Unblackhole() } // bridge proxies connections between listener and dialer, making it possible // to disconnect grpc network connections without closing the logical grpc connection. type bridge struct { dialer Dialer l net.Listener conns map[*bridgeConn]struct{} stopc chan struct{} pausec chan struct{} blackholec chan struct{} wg sync.WaitGroup mu sync.Mutex } func newBridge(dialer Dialer, listener net.Listener) *bridge { b := &bridge{ // bridge "port" is ("%05d%05d0", port, pid) since go1.8 expects the port to be a number dialer: dialer, l: listener, conns: make(map[*bridgeConn]struct{}), stopc: make(chan struct{}), pausec: make(chan struct{}), blackholec: make(chan struct{}), } close(b.pausec) b.wg.Add(1) go b.serveListen() return b } func (b *bridge) Close() { b.l.Close() b.mu.Lock() select { case <-b.stopc: default: close(b.stopc) } b.mu.Unlock() b.wg.Wait() } func (b *bridge) DropConnections() { b.mu.Lock() defer b.mu.Unlock() for bc := range b.conns { bc.Close() } b.conns = make(map[*bridgeConn]struct{}) } func (b *bridge) PauseConnections() { b.mu.Lock() b.pausec = make(chan struct{}) b.mu.Unlock() } func (b *bridge) UnpauseConnections() { b.mu.Lock() select { case <-b.pausec: default: close(b.pausec) } b.mu.Unlock() } func (b *bridge) serveListen() { defer func() { b.l.Close() b.mu.Lock() for bc := range b.conns { bc.Close() } b.mu.Unlock() b.wg.Done() }() for { inc, ierr := b.l.Accept() if ierr != nil { return } b.mu.Lock() pausec := b.pausec b.mu.Unlock() select { case <-b.stopc: inc.Close() return case <-pausec: } outc, oerr := b.dialer.Dial() if oerr != nil { inc.Close() return } bc := &bridgeConn{inc, outc, make(chan struct{})} b.wg.Add(1) b.mu.Lock() b.conns[bc] = struct{}{} go b.serveConn(bc) b.mu.Unlock() } } func (b *bridge) serveConn(bc *bridgeConn) { defer func() { close(bc.donec) bc.Close() b.mu.Lock() delete(b.conns, bc) b.mu.Unlock() b.wg.Done() }() var wg sync.WaitGroup wg.Add(2) go func() { b.ioCopy(bc.out, bc.in) bc.close() wg.Done() }() go func() { b.ioCopy(bc.in, bc.out) bc.close() wg.Done() }() wg.Wait() } type bridgeConn struct { in net.Conn out net.Conn donec chan struct{} } func (bc *bridgeConn) Close() { bc.close() <-bc.donec } func (bc *bridgeConn) close() { bc.in.Close() bc.out.Close() } func (b *bridge) Blackhole() { b.mu.Lock() close(b.blackholec) b.mu.Unlock() } func (b *bridge) Unblackhole() { b.mu.Lock() for bc := range b.conns { bc.Close() } b.conns = make(map[*bridgeConn]struct{}) b.blackholec = make(chan struct{}) b.mu.Unlock() } // ref. https://github.com/golang/go/blob/master/src/io/io.go copyBuffer func (b *bridge) ioCopy(dst io.Writer, src io.Reader) (err error) { buf := make([]byte, 32*1024) for { select { case <-b.blackholec: io.Copy(io.Discard, src) return nil default: } nr, er := src.Read(buf) if nr > 0 { nw, ew := dst.Write(buf[0:nr]) if ew != nil { return ew } if nr != nw { return io.ErrShortWrite } } if er != nil { err = er break } } return err } ================================================ FILE: tests/framework/integration/cluster.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "crypto/tls" "errors" "fmt" "io" "log" "math/rand" "net" "net/http" "net/http/httptest" "os" "reflect" "sort" "strings" "sync" "sync/atomic" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/soheilhy/cmux" "go.uber.org/zap" "go.uber.org/zap/zapcore" "go.uber.org/zap/zaptest" "golang.org/x/crypto/bcrypt" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/tlsutil" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/client/pkg/v3/types" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/featuregate" "go.etcd.io/etcd/pkg/v3/grpctesting" "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp" "go.etcd.io/etcd/server/v3/etcdserver/api/membership" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" "go.etcd.io/etcd/server/v3/etcdserver/api/v3election" epb "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb" "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock" lockpb "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb" "go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc" "go.etcd.io/etcd/server/v3/features" "go.etcd.io/etcd/server/v3/verify" framecfg "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" "go.etcd.io/raft/v3" ) const ( // RequestWaitTimeout is the time duration to wait for a request to go through or detect leader loss. RequestWaitTimeout = 5 * time.Second RequestTimeout = 20 * time.Second ClusterName = "etcd" BasePort = 21000 URLScheme = "unix" URLSchemeTLS = "unixs" BaseGRPCPort = 30000 ) var ( ElectionTicks = 10 // UniqueCount integration test is used to set unique member ids UniqueCount = int32(0) TestTLSInfo = transport.TLSInfo{ KeyFile: testutils.MustAbsPath("../fixtures/server.key.insecure"), CertFile: testutils.MustAbsPath("../fixtures/server.crt"), TrustedCAFile: testutils.MustAbsPath("../fixtures/ca.crt"), ClientCertAuth: true, } TestTLSInfoWithSpecificUsage = transport.TLSInfo{ KeyFile: testutils.MustAbsPath("../fixtures/server-serverusage.key.insecure"), CertFile: testutils.MustAbsPath("../fixtures/server-serverusage.crt"), ClientKeyFile: testutils.MustAbsPath("../fixtures/client-clientusage.key.insecure"), ClientCertFile: testutils.MustAbsPath("../fixtures/client-clientusage.crt"), TrustedCAFile: testutils.MustAbsPath("../fixtures/ca.crt"), ClientCertAuth: true, } TestTLSInfoIP = transport.TLSInfo{ KeyFile: testutils.MustAbsPath("../fixtures/server-ip.key.insecure"), CertFile: testutils.MustAbsPath("../fixtures/server-ip.crt"), TrustedCAFile: testutils.MustAbsPath("../fixtures/ca.crt"), ClientCertAuth: true, } TestTLSInfoExpired = transport.TLSInfo{ KeyFile: testutils.MustAbsPath("./fixtures-expired/server.key.insecure"), CertFile: testutils.MustAbsPath("./fixtures-expired/server.crt"), TrustedCAFile: testutils.MustAbsPath("./fixtures-expired/ca.crt"), ClientCertAuth: true, } TestTLSInfoExpiredIP = transport.TLSInfo{ KeyFile: testutils.MustAbsPath("./fixtures-expired/server-ip.key.insecure"), CertFile: testutils.MustAbsPath("./fixtures-expired/server-ip.crt"), TrustedCAFile: testutils.MustAbsPath("./fixtures-expired/ca.crt"), ClientCertAuth: true, } DefaultTokenJWT = fmt.Sprintf("jwt,pub-key=%s,priv-key=%s,sign-method=RS256,ttl=2s", testutils.MustAbsPath("../fixtures/server.crt"), testutils.MustAbsPath("../fixtures/server.key.insecure")) // UniqueNumber is used to generate unique port numbers // Should only be accessed via atomic package methods. UniqueNumber int32 ) type ClusterConfig struct { Size int PeerTLS *transport.TLSInfo ClientTLS *transport.TLSInfo AuthToken string QuotaBackendBytes int64 BackendBatchInterval time.Duration MaxTxnOps uint MaxRequestBytes uint SnapshotCount uint64 SnapshotCatchUpEntries uint64 GRPCKeepAliveMinTime time.Duration GRPCKeepAliveInterval time.Duration GRPCKeepAliveTimeout time.Duration GRPCAdditionalServerOptions []grpc.ServerOption ClientMaxCallSendMsgSize int ClientMaxCallRecvMsgSize int // UseIP is true to use only IP for gRPC requests. UseIP bool // UseBridge adds bridge between client and grpc server. Should be used in tests that // want to manipulate connection or require connection not breaking despite server stop/restart. UseBridge bool // UseTCP configures server listen on tcp socket. If disabled unix socket is used. UseTCP bool EnableLeaseCheckpoint bool LeaseCheckpointInterval time.Duration LeaseCheckpointPersist bool WatchProgressNotifyInterval time.Duration MaxLearners int DisableStrictReconfigCheck bool CorruptCheckTime time.Duration Metrics string } type Cluster struct { Cfg *ClusterConfig Members []*Member LastMemberNum int mu sync.Mutex } func SchemeFromTLSInfo(tls *transport.TLSInfo) string { if tls == nil { return URLScheme } return URLSchemeTLS } // fillClusterForMembers fills up Member.InitialPeerURLsMap from each member's [name, scheme and PeerListeners address] func (c *Cluster) fillClusterForMembers() error { addrs := make([]string, 0) for _, m := range c.Members { scheme := SchemeFromTLSInfo(m.PeerTLSInfo) for _, l := range m.PeerListeners { addrs = append(addrs, fmt.Sprintf("%s=%s://%s", m.Name, scheme, l.Addr().String())) } } clusterStr := strings.Join(addrs, ",") var err error for _, m := range c.Members { m.InitialPeerURLsMap, err = types.NewURLsMap(clusterStr) if err != nil { return err } } return nil } func (c *Cluster) Launch(t testutil.TB) { t.Logf("Launching new cluster...") errc := make(chan error) for _, m := range c.Members { // Members are launched in separate goroutines because if they boot // using discovery url, they have to wait for others to register to continue. go func(m *Member) { errc <- m.Launch() }(m) } for range c.Members { if err := <-errc; err != nil { c.Terminate(t) t.Fatalf("error setting up member: %v", err) } } // wait Cluster to be stable to receive future client requests c.WaitMembersMatch(t, c.ProtoMembers()) c.waitVersion() for _, m := range c.Members { t.Logf(" - %v -> %v (%v)", m.Name, m.ID(), m.GRPCURL) } } // ProtoMembers returns a list of all active members as etcdserverpb.Member func (c *Cluster) ProtoMembers() []*pb.Member { var ms []*pb.Member for _, m := range c.Members { pScheme := SchemeFromTLSInfo(m.PeerTLSInfo) cScheme := SchemeFromTLSInfo(m.ClientTLSInfo) cm := &pb.Member{Name: m.Name} for _, ln := range m.PeerListeners { cm.PeerURLs = append(cm.PeerURLs, pScheme+"://"+ln.Addr().String()) } for _, ln := range m.ClientListeners { cm.ClientURLs = append(cm.ClientURLs, cScheme+"://"+ln.Addr().String()) } ms = append(ms, cm) } return ms } func (c *Cluster) MustNewMember(t testutil.TB) *Member { memberNumber := c.LastMemberNum c.LastMemberNum++ m := MustNewMember(t, MemberConfig{ Name: fmt.Sprintf("m%v", memberNumber), MemberNumber: memberNumber, AuthToken: c.Cfg.AuthToken, PeerTLS: c.Cfg.PeerTLS, ClientTLS: c.Cfg.ClientTLS, QuotaBackendBytes: c.Cfg.QuotaBackendBytes, BackendBatchInterval: c.Cfg.BackendBatchInterval, MaxTxnOps: c.Cfg.MaxTxnOps, MaxRequestBytes: c.Cfg.MaxRequestBytes, SnapshotCount: c.Cfg.SnapshotCount, SnapshotCatchUpEntries: c.Cfg.SnapshotCatchUpEntries, GRPCKeepAliveMinTime: c.Cfg.GRPCKeepAliveMinTime, GRPCKeepAliveInterval: c.Cfg.GRPCKeepAliveInterval, GRPCKeepAliveTimeout: c.Cfg.GRPCKeepAliveTimeout, GRPCAdditionalServerOptions: c.Cfg.GRPCAdditionalServerOptions, ClientMaxCallSendMsgSize: c.Cfg.ClientMaxCallSendMsgSize, ClientMaxCallRecvMsgSize: c.Cfg.ClientMaxCallRecvMsgSize, UseIP: c.Cfg.UseIP, UseBridge: c.Cfg.UseBridge, UseTCP: c.Cfg.UseTCP, EnableLeaseCheckpoint: c.Cfg.EnableLeaseCheckpoint, LeaseCheckpointInterval: c.Cfg.LeaseCheckpointInterval, LeaseCheckpointPersist: c.Cfg.LeaseCheckpointPersist, WatchProgressNotifyInterval: c.Cfg.WatchProgressNotifyInterval, MaxLearners: c.Cfg.MaxLearners, DisableStrictReconfigCheck: c.Cfg.DisableStrictReconfigCheck, CorruptCheckTime: c.Cfg.CorruptCheckTime, Metrics: c.Cfg.Metrics, }) return m } // addMember return PeerURLs of the added member. func (c *Cluster) addMember(t testutil.TB) types.URLs { m := c.MustNewMember(t) scheme := SchemeFromTLSInfo(c.Cfg.PeerTLS) // send add request to the Cluster var err error for i := 0; i < len(c.Members); i++ { peerURL := scheme + "://" + m.PeerListeners[0].Addr().String() if err = c.AddMemberByURL(t, c.Members[i].Client, peerURL); err == nil { break } } if err != nil { t.Fatalf("add member failed on all members error: %v", err) } m.InitialPeerURLsMap = types.URLsMap{} for _, mm := range c.Members { m.InitialPeerURLsMap[mm.Name] = mm.PeerURLs } m.InitialPeerURLsMap[m.Name] = m.PeerURLs m.NewCluster = false if err := m.Launch(); err != nil { t.Fatal(err) } c.Members = append(c.Members, m) // wait Cluster to be stable to receive future client requests c.WaitMembersMatch(t, c.ProtoMembers()) return m.PeerURLs } func (c *Cluster) AddMemberByURL(t testutil.TB, cc *clientv3.Client, peerURL string) error { ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout) _, err := cc.MemberAdd(ctx, []string{peerURL}) cancel() if err != nil { return err } // wait for the add node entry applied in the Cluster members := append(c.ProtoMembers(), &pb.Member{PeerURLs: []string{peerURL}, ClientURLs: []string{}}) c.WaitMembersMatch(t, members) return nil } // AddMember return PeerURLs of the added member. func (c *Cluster) AddMember(t testutil.TB) types.URLs { return c.addMember(t) } func (c *Cluster) RemoveMember(t testutil.TB, cc *clientv3.Client, id uint64) error { // send remove request to the Cluster ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout) _, err := cc.MemberRemove(ctx, id) cancel() if err != nil { return err } newMembers := make([]*Member, 0) for _, m := range c.Members { if uint64(m.Server.MemberID()) != id { newMembers = append(newMembers, m) } else { m.Client.Close() select { case <-m.Server.StopNotify(): m.Terminate(t) // 1s stop delay + election timeout + 1s disk and network delay + connection write timeout // TODO: remove connection write timeout by selecting on http response closeNotifier // blocking on https://github.com/golang/go/issues/9524 case <-time.After(time.Second + time.Duration(ElectionTicks)*framecfg.TickDuration + time.Second + rafthttp.ConnWriteTimeout): t.Fatalf("failed to remove member %s in time", m.Server.MemberID()) } } } c.Members = newMembers c.WaitMembersMatch(t, c.ProtoMembers()) return nil } func (c *Cluster) WaitMembersMatch(t testutil.TB, membs []*pb.Member) { ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout) defer cancel() for _, m := range c.Members { cc := ToGRPC(m.Client) select { case <-m.Server.StopNotify(): continue default: } for { resp, err := cc.Cluster.MemberList(ctx, &pb.MemberListRequest{Linearizable: false}) if errors.Is(err, context.DeadlineExceeded) { t.Fatal(err) } if err != nil { continue } if isMembersEqual(resp.Members, membs) { break } time.Sleep(framecfg.TickDuration) } } } // WaitLeader returns index of the member in c.Members that is leader // or fails the test (if not established in 30s). func (c *Cluster) WaitLeader(tb testing.TB) int { return c.WaitMembersForLeader(tb, c.Members) } // WaitMembersForLeader waits until given members agree on the same leader, // and returns its 'index' in the 'membs' list func (c *Cluster) WaitMembersForLeader(tb testing.TB, membs []*Member) int { tb.Logf("WaitMembersForLeader") ctx, cancel := context.WithTimeout(tb.Context(), 30*time.Second) defer cancel() l := 0 for l = c.waitMembersForLeader(ctx, tb, membs); l < 0; { if ctx.Err() != nil { tb.Fatalf("WaitLeader FAILED: %v", ctx.Err()) } } tb.Logf("WaitMembersForLeader succeeded. Cluster leader index: %v", l) // TODO: Consider second pass check as sometimes leadership is lost // soon after election: // // We perform multiple attempts, as some-times just after successful WaitLLeader // there is a race and leadership is quickly lost: // - MsgAppResp message with higher term from 2acc3d3b521981 [term: 3] {"member": "m0"} // - 9903a56eaf96afac became follower at term 3 {"member": "m0"} // - 9903a56eaf96afac lost leader 9903a56eaf96afac at term 3 {"member": "m0"} return l } // WaitMembersForLeader waits until given members agree on the same leader, // and returns its 'index' in the 'membs' list func (c *Cluster) waitMembersForLeader(ctx context.Context, tb testing.TB, membs []*Member) int { possibleLead := make(map[uint64]bool) var lead uint64 for _, m := range membs { possibleLead[uint64(m.Server.MemberID())] = true } cc, err := c.ClusterClient(tb) if err != nil { tb.Fatal(err) } // ensure leader is up via linearizable get for { fctx, fcancel := context.WithTimeout(ctx, 10*framecfg.TickDuration+time.Second) _, err := cc.Get(fctx, "0") fcancel() if err == nil || strings.Contains(err.Error(), "Key not found") { break } } for lead == 0 || !possibleLead[lead] { lead = 0 for _, m := range membs { select { case <-m.Server.StopNotify(): continue default: } if lead != 0 && lead != m.Server.Lead() { lead = 0 time.Sleep(10 * framecfg.TickDuration) break } lead = m.Server.Lead() } } for i, m := range membs { if uint64(m.Server.MemberID()) == lead { tb.Logf("waitMembersForLeader found leader. Member: %v lead: %x", i, lead) return i } } tb.Logf("waitMembersForLeader failed (-1)") return -1 } func (c *Cluster) WaitNoLeader() { c.WaitMembersNoLeader(c.Members) } // WaitMembersNoLeader waits until given members lose leader. func (c *Cluster) WaitMembersNoLeader(membs []*Member) { noLeader := false for !noLeader { noLeader = true for _, m := range membs { select { case <-m.Server.StopNotify(): continue default: } if m.Server.Lead() != 0 { noLeader = false time.Sleep(10 * framecfg.TickDuration) break } } } } func (c *Cluster) waitVersion() { for _, m := range c.Members { for { if m.Server.ClusterVersion() != nil { break } time.Sleep(framecfg.TickDuration) } } } // isMembersEqual checks whether two members equal except ID field. // The given wmembs should always set ID field to empty string. func isMembersEqual(membs []*pb.Member, wmembs []*pb.Member) bool { sort.Sort(SortableMemberSliceByPeerURLs(membs)) sort.Sort(SortableMemberSliceByPeerURLs(wmembs)) return cmp.Equal(membs, wmembs, cmpopts.IgnoreFields(pb.Member{}, "ID", "PeerURLs", "ClientURLs")) } func NewLocalListener(t testutil.TB) net.Listener { c := atomic.AddInt32(&UniqueCount, 1) // Go 1.8+ allows only numbers in port addr := fmt.Sprintf("127.0.0.1:%05d%05d", c+BasePort, os.Getpid()) return NewListenerWithAddr(t, addr) } func NewListenerWithAddr(t testutil.TB, addr string) net.Listener { t.Logf("Creating listener with addr: %v", addr) l, err := transport.NewUnixListener(addr) if err != nil { t.Fatal(err) } return l } type Member struct { config.ServerConfig UniqNumber int MemberNumber int Port string PeerListeners, ClientListeners []net.Listener GRPCListener net.Listener // PeerTLSInfo enables peer TLS when set PeerTLSInfo *transport.TLSInfo // ClientTLSInfo enables client TLS when set ClientTLSInfo *transport.TLSInfo DialOptions []grpc.DialOption RaftHandler *testutil.PauseableHandler Server *etcdserver.EtcdServer ServerClosers []func() GRPCServerOpts []grpc.ServerOption GRPCServer *grpc.Server GRPCURL string GRPCBridge Bridge // ServerClient is a clientv3 that directly calls the etcdserver. ServerClient *clientv3.Client // Client is a clientv3 that communicates via socket, either UNIX or TCP. Client *clientv3.Client KeepDataDirTerminate bool ClientMaxCallSendMsgSize int ClientMaxCallRecvMsgSize int UseIP bool UseBridge bool UseTCP bool IsLearner bool Closed bool GRPCServerRecorder *grpctesting.GRPCRecorder LogObserver *testutils.LogObserver } type MemberConfig struct { Name string UniqNumber int64 MemberNumber int PeerTLS *transport.TLSInfo ClientTLS *transport.TLSInfo AuthToken string QuotaBackendBytes int64 BackendBatchInterval time.Duration MaxTxnOps uint MaxRequestBytes uint SnapshotCount uint64 SnapshotCatchUpEntries uint64 GRPCKeepAliveMinTime time.Duration GRPCKeepAliveInterval time.Duration GRPCKeepAliveTimeout time.Duration GRPCAdditionalServerOptions []grpc.ServerOption ClientMaxCallSendMsgSize int ClientMaxCallRecvMsgSize int UseIP bool UseBridge bool UseTCP bool EnableLeaseCheckpoint bool LeaseCheckpointInterval time.Duration LeaseCheckpointPersist bool WatchProgressNotifyInterval time.Duration MaxLearners int DisableStrictReconfigCheck bool CorruptCheckTime time.Duration Metrics string } // MustNewMember return an inited member with the given name. If peerTLS is // set, it will use https scheme to communicate between peers. func MustNewMember(t testutil.TB, mcfg MemberConfig) *Member { var err error m := &Member{ MemberNumber: mcfg.MemberNumber, UniqNumber: int(atomic.AddInt32(&UniqueCount, 1)), } peerScheme := SchemeFromTLSInfo(mcfg.PeerTLS) clientScheme := SchemeFromTLSInfo(mcfg.ClientTLS) pln := NewLocalListener(t) m.PeerListeners = []net.Listener{pln} m.PeerURLs, err = types.NewURLs([]string{peerScheme + "://" + pln.Addr().String()}) if err != nil { t.Fatal(err) } m.PeerTLSInfo = mcfg.PeerTLS cln := NewLocalListener(t) m.ClientListeners = []net.Listener{cln} m.ClientURLs, err = types.NewURLs([]string{clientScheme + "://" + cln.Addr().String()}) if err != nil { t.Fatal(err) } m.ClientTLSInfo = mcfg.ClientTLS m.Name = mcfg.Name m.DataDir, err = os.MkdirTemp(t.TempDir(), "etcd") if err != nil { t.Fatal(err) } clusterStr := fmt.Sprintf("%s=%s://%s", mcfg.Name, peerScheme, pln.Addr().String()) m.InitialPeerURLsMap, err = types.NewURLsMap(clusterStr) if err != nil { t.Fatal(err) } m.InitialClusterToken = ClusterName m.NewCluster = true m.BootstrapTimeout = 10 * time.Millisecond if m.PeerTLSInfo != nil { m.ServerConfig.PeerTLSInfo = *m.PeerTLSInfo } m.ElectionTicks = ElectionTicks m.InitialElectionTickAdvance = true m.TickMs = uint(framecfg.TickDuration / time.Millisecond) m.PreVote = true m.QuotaBackendBytes = mcfg.QuotaBackendBytes m.BackendBatchInterval = mcfg.BackendBatchInterval m.MaxTxnOps = mcfg.MaxTxnOps if m.MaxTxnOps == 0 { m.MaxTxnOps = embed.DefaultMaxTxnOps } m.MaxRequestBytes = mcfg.MaxRequestBytes if m.MaxRequestBytes == 0 { m.MaxRequestBytes = embed.DefaultMaxRequestBytes } m.SnapshotCount = etcdserver.DefaultSnapshotCount if mcfg.SnapshotCount != 0 { m.SnapshotCount = mcfg.SnapshotCount } m.SnapshotCatchUpEntries = etcdserver.DefaultSnapshotCatchUpEntries if mcfg.SnapshotCatchUpEntries != 0 { m.SnapshotCatchUpEntries = mcfg.SnapshotCatchUpEntries } // for the purpose of integration testing, simple token is enough m.AuthToken = "simple" if mcfg.AuthToken != "" { m.AuthToken = mcfg.AuthToken } m.BcryptCost = uint(bcrypt.MinCost) // use min bcrypt cost to speedy up integration testing m.GRPCServerOpts = []grpc.ServerOption{} if mcfg.GRPCKeepAliveMinTime > time.Duration(0) { m.GRPCServerOpts = append(m.GRPCServerOpts, grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ MinTime: mcfg.GRPCKeepAliveMinTime, PermitWithoutStream: false, })) } if mcfg.GRPCKeepAliveInterval > time.Duration(0) && mcfg.GRPCKeepAliveTimeout > time.Duration(0) { m.GRPCServerOpts = append(m.GRPCServerOpts, grpc.KeepaliveParams(keepalive.ServerParameters{ Time: mcfg.GRPCKeepAliveInterval, Timeout: mcfg.GRPCKeepAliveTimeout, })) } m.GRPCServerOpts = append(m.GRPCServerOpts, mcfg.GRPCAdditionalServerOptions...) m.ClientMaxCallSendMsgSize = mcfg.ClientMaxCallSendMsgSize m.ClientMaxCallRecvMsgSize = mcfg.ClientMaxCallRecvMsgSize m.UseIP = mcfg.UseIP m.UseBridge = mcfg.UseBridge m.UseTCP = mcfg.UseTCP m.LeaseCheckpointInterval = mcfg.LeaseCheckpointInterval m.WatchProgressNotifyInterval = mcfg.WatchProgressNotifyInterval m.InitialCorruptCheck = true if mcfg.CorruptCheckTime > time.Duration(0) { m.CorruptCheckTime = mcfg.CorruptCheckTime } m.WarningApplyDuration = embed.DefaultWarningApplyDuration m.WarningUnaryRequestDuration = embed.DefaultWarningUnaryRequestDuration m.MaxLearners = membership.DefaultMaxLearners if mcfg.MaxLearners != 0 { m.MaxLearners = mcfg.MaxLearners } m.Metrics = mcfg.Metrics m.V2Deprecation = config.V2_DEPR_DEFAULT //nolint:staticcheck // TODO: remove for a supported version m.GRPCServerRecorder = &grpctesting.GRPCRecorder{} m.Logger, m.LogObserver = memberLogger(t, mcfg.Name) m.ServerFeatureGate = features.NewDefaultServerFeatureGate(m.Name, m.Logger) featureGates := fmt.Sprintf("LeaseCheckpoint=%v,LeaseCheckpointPersist=%v", mcfg.EnableLeaseCheckpoint, mcfg.LeaseCheckpointPersist) if err := m.ServerFeatureGate.(featuregate.MutableFeatureGate).Set(featureGates); err != nil { t.Fatalf("Set FeatureGate FAILED: %v", err) } m.StrictReconfigCheck = !mcfg.DisableStrictReconfigCheck if err := m.listenGRPC(); err != nil { t.Fatalf("listenGRPC FAILED: %v", err) } t.Cleanup(func() { // if we didn't cleanup the logger, the consecutive test // might reuse this (t). raft.ResetDefaultLogger() }) return m } func memberLogger(t testutil.TB, name string) (*zap.Logger, *testutils.LogObserver) { level := zapcore.InfoLevel if os.Getenv("CLUSTER_DEBUG") != "" { level = zapcore.DebugLevel } obCore, logOb := testutils.NewLogObserver(level) options := zaptest.WrapOptions( zap.Fields(zap.String("member", name)), // copy logged entities to log observer zap.WrapCore(func(oldCore zapcore.Core) zapcore.Core { return zapcore.NewTee(oldCore, obCore) }), ) return zaptest.NewLogger(t, zaptest.Level(level), options).Named(name), logOb } // listenGRPC starts a grpc server over a unix domain socket on the member func (m *Member) listenGRPC() error { // prefix with localhost so cert has right domain network, host, port := m.grpcAddr() grpcAddr := net.JoinHostPort(host, port) wd, err := os.Getwd() if err != nil { return err } m.Logger.Info("LISTEN GRPC", zap.String("grpcAddr", grpcAddr), zap.String("m.Name", m.Name), zap.String("workdir", wd)) grpcListener, err := net.Listen(network, grpcAddr) if err != nil { return fmt.Errorf("listen failed on grpc socket %s (%w)", grpcAddr, err) } addr := grpcListener.Addr().String() _, port, err = net.SplitHostPort(addr) if err != nil { return fmt.Errorf("failed to parse grpc listen port from address %s (%w)", addr, err) } m.Port = port m.GRPCURL = fmt.Sprintf("%s://%s", m.clientScheme(), addr) m.Logger.Info("LISTEN GRPC SUCCESS", zap.String("grpcAddr", m.GRPCURL), zap.String("m.Name", m.Name), zap.String("workdir", wd), zap.String("port", m.Port)) if m.UseBridge { _, err = m.addBridge() if err != nil { grpcListener.Close() return err } } m.GRPCListener = grpcListener return nil } func (m *Member) clientScheme() string { switch { case m.UseTCP && m.ClientTLSInfo != nil: return "https" case m.UseTCP && m.ClientTLSInfo == nil: return "http" case !m.UseTCP && m.ClientTLSInfo != nil: return "unixs" case !m.UseTCP && m.ClientTLSInfo == nil: return "unix" } m.Logger.Panic("Failed to determine client schema") return "" } func (m *Member) addBridge() (Bridge, error) { network, host, port := m.grpcAddr() grpcAddr := net.JoinHostPort(host, m.Port) bridgePort := fmt.Sprintf("%s%s", port, "0") if m.UseTCP { bridgePort = "0" } bridgeAddr := net.JoinHostPort(host, bridgePort) m.Logger.Info("LISTEN BRIDGE", zap.String("grpc-address", bridgeAddr), zap.String("member", m.Name)) bridgeListener, err := transport.NewUnixListener(bridgeAddr) if err != nil { return nil, fmt.Errorf("listen failed on bridge socket %s (%w)", bridgeAddr, err) } m.GRPCBridge = newBridge(dialer{network: network, addr: grpcAddr}, bridgeListener) addr := bridgeListener.Addr().String() m.Logger.Info("LISTEN BRIDGE SUCCESS", zap.String("grpc-address", addr), zap.String("member", m.Name)) m.GRPCURL = m.clientScheme() + "://" + addr return m.GRPCBridge, nil } func (m *Member) Bridge() Bridge { if !m.UseBridge { m.Logger.Panic("Bridge not available. Please configure using bridge before creating Cluster.") } return m.GRPCBridge } func (m *Member) grpcAddr() (network, host, port string) { // prefix with localhost so cert has right domain host = "localhost" if m.UseIP { // for IP-only TLS certs host = "127.0.0.1" } network = "unix" if m.UseTCP { network = "tcp" } if m.Port != "" { return network, host, m.Port } port = m.Name if m.UseTCP { // let net.Listen choose the port automatically port = fmt.Sprintf("%d", 0) } return network, host, port } func (m *Member) GRPCPortNumber() string { return m.Port } type dialer struct { network string addr string } func (d dialer) Dial() (net.Conn, error) { return net.Dial(d.network, d.addr) } func (m *Member) ElectionTimeout() time.Duration { return time.Duration(m.Server.Cfg.ElectionTicks*int(m.Server.Cfg.TickMs)) * time.Millisecond } func (m *Member) ID() types.ID { return m.Server.MemberID() } // NewClientV3 creates a new grpc client connection to the member func NewClientV3(m *Member) (*clientv3.Client, error) { if m.GRPCURL == "" { return nil, fmt.Errorf("member not configured for grpc") } cfg := clientv3.Config{ Endpoints: []string{m.GRPCURL}, DialTimeout: 5 * time.Second, MaxCallSendMsgSize: m.ClientMaxCallSendMsgSize, MaxCallRecvMsgSize: m.ClientMaxCallRecvMsgSize, Logger: m.Logger.Named("client"), } if m.ClientTLSInfo != nil { tls, err := m.ClientTLSInfo.ClientConfig() if err != nil { return nil, err } cfg.TLS = tls } if m.DialOptions != nil { cfg.DialOptions = append(cfg.DialOptions, m.DialOptions...) } return newClientV3(cfg) } // Clone returns a member with the same server configuration. The returned // member will not set PeerListeners and ClientListeners. func (m *Member) Clone(t testutil.TB) *Member { mm := &Member{} mm.ServerConfig = m.ServerConfig var err error clientURLStrs := m.ClientURLs.StringSlice() mm.ClientURLs, err = types.NewURLs(clientURLStrs) if err != nil { // this should never fail panic(err) } peerURLStrs := m.PeerURLs.StringSlice() mm.PeerURLs, err = types.NewURLs(peerURLStrs) if err != nil { // this should never fail panic(err) } clusterStr := m.InitialPeerURLsMap.String() mm.InitialPeerURLsMap, err = types.NewURLsMap(clusterStr) if err != nil { // this should never fail panic(err) } mm.InitialClusterToken = m.InitialClusterToken mm.ElectionTicks = m.ElectionTicks mm.PeerTLSInfo = m.PeerTLSInfo mm.ClientTLSInfo = m.ClientTLSInfo mm.Logger, mm.LogObserver = memberLogger(t, mm.Name+"c") return mm } // Launch starts a member based on ServerConfig, PeerListeners // and ClientListeners. func (m *Member) Launch() error { m.Logger.Info( "launching a member", zap.String("name", m.Name), zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), zap.String("grpc-url", m.GRPCURL), ) var err error if m.Server, err = etcdserver.NewServer(m.ServerConfig); err != nil { return fmt.Errorf("failed to initialize the etcd server: %w", err) } m.Server.Start() var peerTLScfg *tls.Config if m.PeerTLSInfo != nil && !m.PeerTLSInfo.Empty() { if peerTLScfg, err = m.PeerTLSInfo.ServerConfig(); err != nil { return err } } if m.GRPCListener != nil { var tlscfg *tls.Config if m.ClientTLSInfo != nil && !m.ClientTLSInfo.Empty() { tlscfg, err = m.ClientTLSInfo.ServerConfig() if err != nil { return err } } m.GRPCServer = v3rpc.Server(m.Server, tlscfg, m.GRPCServerRecorder.UnaryInterceptor(), m.GRPCServerOpts...) m.ServerClient = v3client.New(m.Server) lockpb.RegisterLockServer(m.GRPCServer, v3lock.NewLockServer(m.ServerClient)) epb.RegisterElectionServer(m.GRPCServer, v3election.NewElectionServer(m.ServerClient)) go m.GRPCServer.Serve(m.GRPCListener) } m.RaftHandler = &testutil.PauseableHandler{Next: etcdhttp.NewPeerHandler(m.Logger, m.Server)} h := (http.Handler)(m.RaftHandler) if m.GRPCListener != nil { h = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { m.RaftHandler.ServeHTTP(w, r) }) } for _, ln := range m.PeerListeners { cm := cmux.New(ln) // don't hang on matcher after closing listener cm.SetReadTimeout(time.Second) // serve http1/http2 rafthttp/grpc ll := cm.Match(cmux.Any()) if peerTLScfg != nil { if ll, err = transport.NewTLSListener(ll, m.PeerTLSInfo); err != nil { return err } } hs := &httptest.Server{ Listener: ll, Config: &http.Server{ Handler: h, TLSConfig: peerTLScfg, ErrorLog: log.New(io.Discard, "net/http", 0), }, TLS: peerTLScfg, } hs.Start() donec := make(chan struct{}) go func() { defer close(donec) cm.Serve() }() closer := func() { ll.Close() hs.CloseClientConnections() hs.Close() <-donec } m.ServerClosers = append(m.ServerClosers, closer) } for _, ln := range m.ClientListeners { handler := http.NewServeMux() etcdhttp.HandleDebug(handler) etcdhttp.HandleVersion(handler, m.Server) etcdhttp.HandleMetrics(handler) etcdhttp.HandleHealth(m.Logger, handler, m.Server) hs := &httptest.Server{ Listener: ln, Config: &http.Server{ Handler: handler, ErrorLog: log.New(io.Discard, "net/http", 0), }, } if m.ClientTLSInfo == nil { hs.Start() } else { info := m.ClientTLSInfo hs.TLS, err = info.ServerConfig() if err != nil { return err } // baseConfig is called on initial TLS handshake start. // // Previously, // 1. Server has non-empty (*tls.Config).Certificates on client hello // 2. Server calls (*tls.Config).GetCertificate iff: // - Server'Server (*tls.Config).Certificates is not empty, or // - Client supplies SNI; non-empty (*tls.ClientHelloInfo).ServerName // // When (*tls.Config).Certificates is always populated on initial handshake, // client is expected to provide a valid matching SNI to pass the TLS // verification, thus trigger server (*tls.Config).GetCertificate to reload // TLS assets. However, a cert whose SAN field does not include domain names // but only IP addresses, has empty (*tls.ClientHelloInfo).ServerName, thus // it was never able to trigger TLS reload on initial handshake; first // ceritifcate object was being used, never being updated. // // Now, (*tls.Config).Certificates is created empty on initial TLS client // handshake, in order to trigger (*tls.Config).GetCertificate and populate // rest of the certificates on every new TLS connection, even when client // SNI is empty (e.g. cert only includes IPs). // // This introduces another problem with "httptest.Server": // when server initial certificates are empty, certificates // are overwritten by Go'Server internal test certs, which have // different SAN fields (e.g. example.com). To work around, // re-overwrite (*tls.Config).Certificates before starting // test server. tlsCert, nerr := tlsutil.NewCert(info.CertFile, info.KeyFile, nil) if nerr != nil { return nerr } hs.TLS.Certificates = []tls.Certificate{*tlsCert} hs.StartTLS() } closer := func() { ln.Close() hs.CloseClientConnections() hs.Close() } m.ServerClosers = append(m.ServerClosers, closer) } if m.GRPCURL != "" && m.Client == nil { m.Client, err = NewClientV3(m) if err != nil { return err } } m.Logger.Info( "launched a member", zap.String("name", m.Name), zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), zap.String("grpc-url", m.GRPCURL), ) return nil } func (m *Member) RecordedRequests() []grpctesting.RequestInfo { return m.GRPCServerRecorder.RecordedRequests() } func (m *Member) WaitOK(t testutil.TB) { m.WaitStarted(t) for m.Server.Leader() == 0 { time.Sleep(framecfg.TickDuration) } } func (m *Member) WaitStarted(t testutil.TB) { for { ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout) _, err := m.Client.Get(ctx, "/", clientv3.WithSerializable()) if err != nil { time.Sleep(framecfg.TickDuration) continue } cancel() break } } func WaitClientV3(t testutil.TB, kv clientv3.KV) { WaitClientV3WithKey(t, kv, "/") } func WaitClientV3WithKey(t testutil.TB, kv clientv3.KV, key string) { timeout := time.Now().Add(RequestTimeout) var err error for time.Now().Before(timeout) { ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout) _, err = kv.Get(ctx, key) cancel() if err == nil { return } time.Sleep(framecfg.TickDuration) } if err != nil { t.Fatalf("timed out waiting for client: %v", err) } } func (m *Member) URL() string { return m.ClientURLs[0].String() } func (m *Member) Pause() { m.RaftHandler.Pause() m.Server.PauseSending() } func (m *Member) Resume() { m.RaftHandler.Resume() m.Server.ResumeSending() } // Close stops the member'Server etcdserver and closes its connections func (m *Member) Close() { if m.GRPCBridge != nil { m.GRPCBridge.Close() m.GRPCBridge = nil } if m.ServerClient != nil { m.ServerClient.Close() m.ServerClient = nil } if m.GRPCServer != nil { ch := make(chan struct{}) go func() { defer close(ch) // close listeners to stop accepting new connections, // will block on any existing transports m.GRPCServer.GracefulStop() }() // wait until all pending RPCs are finished select { case <-ch: case <-time.After(2 * time.Second): // took too long, manually close open transports // e.g. watch streams m.GRPCServer.Stop() <-ch } m.GRPCServer = nil } if m.Server != nil { m.Server.HardStop() } for _, f := range m.ServerClosers { f() } if !m.Closed { // Avoid verification of the same file multiple times // (that might not exist any longer) verify.MustVerifyIfEnabled(verify.Config{ Logger: m.Logger, DataDir: m.DataDir, ExactIndex: false, }) } m.Closed = true } // Stop stops the member, but the data dir of the member is preserved. func (m *Member) Stop(_ testutil.TB) { m.Logger.Info( "stopping a member", zap.String("name", m.Name), zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), zap.String("grpc-url", m.GRPCURL), ) m.Close() m.ServerClosers = nil m.Logger.Info( "stopped a member", zap.String("name", m.Name), zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), zap.String("grpc-url", m.GRPCURL), ) } // CheckLeaderTransition waits for leader transition, returning the new leader ID. func CheckLeaderTransition(m *Member, oldLead uint64) uint64 { interval := time.Duration(m.Server.Cfg.TickMs) * time.Millisecond for m.Server.Lead() == 0 || (m.Server.Lead() == oldLead) { time.Sleep(interval) } return m.Server.Lead() } // StopNotify unblocks when a member stop completes func (m *Member) StopNotify() <-chan struct{} { return m.Server.StopNotify() } // Restart starts the member using the preserved data dir. func (m *Member) Restart(t testutil.TB) error { m.Logger.Info( "restarting a member", zap.String("name", m.Name), zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), zap.String("grpc-url", m.GRPCURL), ) newPeerListeners := make([]net.Listener, 0) for _, ln := range m.PeerListeners { newPeerListeners = append(newPeerListeners, NewListenerWithAddr(t, ln.Addr().String())) } m.PeerListeners = newPeerListeners newClientListeners := make([]net.Listener, 0) for _, ln := range m.ClientListeners { newClientListeners = append(newClientListeners, NewListenerWithAddr(t, ln.Addr().String())) } m.ClientListeners = newClientListeners if m.GRPCListener != nil { if err := m.listenGRPC(); err != nil { t.Fatal(err) } } err := m.Launch() m.Logger.Info( "restarted a member", zap.String("name", m.Name), zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), zap.String("grpc-url", m.GRPCURL), zap.Error(err), ) return err } // Terminate stops the member and removes the data dir. func (m *Member) Terminate(t testutil.TB) { m.Logger.Info( "terminating a member", zap.String("name", m.Name), zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), zap.String("grpc-url", m.GRPCURL), ) m.Close() if !m.KeepDataDirTerminate { if err := os.RemoveAll(m.ServerConfig.DataDir); err != nil { t.Fatal(err) } } m.Logger.Info( "terminated a member", zap.String("name", m.Name), zap.Strings("advertise-peer-urls", m.PeerURLs.StringSlice()), zap.Strings("listen-client-urls", m.ClientURLs.StringSlice()), zap.String("grpc-url", m.GRPCURL), ) } // Metric gets the metric value for a member func (m *Member) Metric(metricName string, expectLabels ...string) (string, error) { cfgtls := transport.TLSInfo{} tr, err := transport.NewTimeoutTransport(cfgtls, time.Second, time.Second, time.Second) if err != nil { return "", err } cli := &http.Client{Transport: tr} resp, err := cli.Get(m.ClientURLs[0].String() + "/metrics") if err != nil { return "", err } defer resp.Body.Close() b, rerr := io.ReadAll(resp.Body) if rerr != nil { return "", rerr } lines := strings.Split(string(b), "\n") for _, l := range lines { if !strings.HasPrefix(l, metricName) { continue } ok := true for _, lv := range expectLabels { if !strings.Contains(l, lv) { ok = false break } } if !ok { continue } return strings.Split(l, " ")[1], nil } return "", nil } // InjectPartition drops connections from m to others, vice versa. func (m *Member) InjectPartition(t testutil.TB, others ...*Member) { for _, other := range others { m.Server.CutPeer(other.Server.MemberID()) other.Server.CutPeer(m.Server.MemberID()) t.Logf("network partition injected between: %v <-> %v", m.Server.MemberID(), other.Server.MemberID()) } } // RecoverPartition recovers connections from m to others, vice versa. func (m *Member) RecoverPartition(t testutil.TB, others ...*Member) { for _, other := range others { m.Server.MendPeer(other.Server.MemberID()) other.Server.MendPeer(m.Server.MemberID()) t.Logf("network partition between: %v <-> %v", m.Server.MemberID(), other.Server.MemberID()) } } func (m *Member) ReadyNotify() <-chan struct{} { return m.Server.ReadyNotify() } type SortableMemberSliceByPeerURLs []*pb.Member func (p SortableMemberSliceByPeerURLs) Len() int { return len(p) } func (p SortableMemberSliceByPeerURLs) Less(i, j int) bool { return p[i].PeerURLs[0] < p[j].PeerURLs[0] } func (p SortableMemberSliceByPeerURLs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } // NewCluster returns a launched Cluster with a grpc client connection // for each Cluster member. func NewCluster(t testutil.TB, cfg *ClusterConfig) *Cluster { t.Helper() assertInTestContext(t) testutil.SkipTestIfShortMode(t, "Cannot start etcd Cluster in --short tests") c := &Cluster{Cfg: cfg} ms := make([]*Member, cfg.Size) for i := 0; i < cfg.Size; i++ { ms[i] = c.MustNewMember(t) } c.Members = ms if err := c.fillClusterForMembers(); err != nil { t.Fatalf("fillClusterForMembers failed: %v", err) } c.Launch(t) return c } func (c *Cluster) TakeClient(idx int) { c.mu.Lock() c.Members[idx].Client = nil c.mu.Unlock() } func (c *Cluster) Terminate(t testutil.TB) { if t != nil { t.Logf("========= Cluster termination started =====================") } for _, m := range c.Members { if m.Client != nil { m.Client.Close() } } var wg sync.WaitGroup wg.Add(len(c.Members)) for _, m := range c.Members { go func(mm *Member) { defer wg.Done() mm.Terminate(t) }(m) } wg.Wait() if t != nil { t.Logf("========= Cluster termination succeeded ===================") } } func (c *Cluster) RandClient() *clientv3.Client { return c.Members[rand.Intn(len(c.Members))].Client } func (c *Cluster) Client(i int) *clientv3.Client { return c.Members[i].Client } func (c *Cluster) Endpoints() []string { var endpoints []string for _, m := range c.Members { endpoints = append(endpoints, m.GRPCURL) } return endpoints } func (c *Cluster) ClusterClient(tb testing.TB, opts ...framecfg.ClientOption) (client *clientv3.Client, err error) { cfg, err := c.newClientCfg() if err != nil { return nil, err } for _, opt := range opts { opt(cfg) } client, err = newClientV3(*cfg) if err != nil { return nil, err } tb.Cleanup(func() { client.Close() }) return client, nil } func WithAuth(userName, password string) framecfg.ClientOption { return func(c any) { cfg := c.(*clientv3.Config) cfg.Username = userName cfg.Password = password } } func WithAuthToken(token string) framecfg.ClientOption { return func(c any) { cfg := c.(*clientv3.Config) cfg.Token = token } } func WithEndpoints(endpoints []string) framecfg.ClientOption { return func(c any) { cfg := c.(*clientv3.Config) cfg.Endpoints = endpoints } } func WithDialTimeout(tio time.Duration) framecfg.ClientOption { return func(c any) { cfg := c.(*clientv3.Config) cfg.DialTimeout = tio } } func (c *Cluster) newClientCfg() (*clientv3.Config, error) { cfg := &clientv3.Config{ Endpoints: c.Endpoints(), DialTimeout: 5 * time.Second, MaxCallSendMsgSize: c.Cfg.ClientMaxCallSendMsgSize, MaxCallRecvMsgSize: c.Cfg.ClientMaxCallRecvMsgSize, } if c.Cfg.ClientTLS != nil { tls, err := c.Cfg.ClientTLS.ClientConfig() if err != nil { return nil, err } cfg.TLS = tls } return cfg, nil } // NewClientV3 creates a new grpc client connection to the member func (c *Cluster) NewClientV3(memberIndex int) (*clientv3.Client, error) { return NewClientV3(c.Members[memberIndex]) } func makeClients(t testutil.TB, clus *Cluster, clients *[]*clientv3.Client, chooseMemberIndex func() int) func() *clientv3.Client { var mu sync.Mutex *clients = nil return func() *clientv3.Client { cli, err := clus.NewClientV3(chooseMemberIndex()) if err != nil { t.Fatalf("cannot create client: %v", err) } mu.Lock() *clients = append(*clients, cli) mu.Unlock() return cli } } // MakeSingleNodeClients creates factory of clients that all connect to member 0. // All the created clients are put on the 'clients' list. The factory is thread-safe. func MakeSingleNodeClients(t testutil.TB, clus *Cluster, clients *[]*clientv3.Client) func() *clientv3.Client { return makeClients(t, clus, clients, func() int { return 0 }) } // MakeMultiNodeClients creates factory of clients that all connect to random members. // All the created clients are put on the 'clients' list. The factory is thread-safe. func MakeMultiNodeClients(t testutil.TB, clus *Cluster, clients *[]*clientv3.Client) func() *clientv3.Client { return makeClients(t, clus, clients, func() int { return rand.Intn(len(clus.Members)) }) } // CloseClients closes all the clients from the 'clients' list. func CloseClients(t testutil.TB, clients []*clientv3.Client) { for _, cli := range clients { if err := cli.Close(); err != nil { t.Fatal(err) } } } type GRPCAPI struct { // Cluster is the Cluster API for the client'Server connection. Cluster pb.ClusterClient // KV is the keyvalue API for the client'Server connection. KV pb.KVClient // Lease is the lease API for the client'Server connection. Lease pb.LeaseClient // Watch is the watch API for the client'Server connection. Watch pb.WatchClient // Maintenance is the maintenance API for the client'Server connection. Maintenance pb.MaintenanceClient // Auth is the authentication API for the client'Server connection. Auth pb.AuthClient // Lock is the lock API for the client'Server connection. Lock lockpb.LockClient // Election is the election API for the client'Server connection. Election epb.ElectionClient } // GetLearnerMembers returns the list of learner members in Cluster using MemberList API. func (c *Cluster) GetLearnerMembers() ([]*pb.Member, error) { cli := c.Client(0) resp, err := cli.MemberList(context.Background()) if err != nil { return nil, fmt.Errorf("failed to list member %w", err) } var learners []*pb.Member for _, m := range resp.Members { if m.IsLearner { learners = append(learners, m) } } return learners, nil } // AddAndLaunchLearnerMember creates a learner member, adds it to Cluster // via v3 MemberAdd API, and then launches the new member. func (c *Cluster) AddAndLaunchLearnerMember(t testutil.TB) { m := c.MustNewMember(t) m.IsLearner = true scheme := SchemeFromTLSInfo(c.Cfg.PeerTLS) peerURLs := []string{scheme + "://" + m.PeerListeners[0].Addr().String()} cli := c.Client(0) _, err := cli.MemberAddAsLearner(context.Background(), peerURLs) if err != nil { t.Fatalf("failed to add learner member %v", err) } m.InitialPeerURLsMap = types.URLsMap{} for _, mm := range c.Members { m.InitialPeerURLsMap[mm.Name] = mm.PeerURLs } m.InitialPeerURLsMap[m.Name] = m.PeerURLs m.NewCluster = false if err := m.Launch(); err != nil { t.Fatal(err) } c.Members = append(c.Members, m) c.waitMembersMatch(t) } // getMembers returns a list of members in Cluster, in format of etcdserverpb.Member func (c *Cluster) getMembers() []*pb.Member { var mems []*pb.Member for _, m := range c.Members { mem := &pb.Member{ Name: m.Name, PeerURLs: m.PeerURLs.StringSlice(), ClientURLs: m.ClientURLs.StringSlice(), IsLearner: m.IsLearner, } mems = append(mems, mem) } return mems } // waitMembersMatch waits until v3rpc MemberList returns the 'same' members info as the // local 'c.Members', which is the local recording of members in the testing Cluster. With // the exception that the local recording c.Members does not have info on Member.ID, which // is generated when the member is been added to Cluster. // // Note: // A successful match means the Member.clientURLs are matched. This means member has already // finished publishing its server attributes to Cluster. Publishing attributes is a Cluster-wide // write request (in v2 server). Therefore, at this point, any raft log entries prior to this // would have already been applied. // // If a new member was added to an existing Cluster, at this point, it has finished publishing // its own server attributes to the Cluster. And therefore by the same argument, it has already // applied the raft log entries (especially those of type raftpb.ConfChangeType). At this point, // the new member has the correct view of the Cluster configuration. // // Special note on learner member: // Learner member is only added to a Cluster via v3rpc MemberAdd API (as of v3.4). When starting // the learner member, its initial view of the Cluster created by peerURLs map does not have info // on whether or not the new member itself is learner. But at this point, a successful match does // indicate that the new learner member has applied the raftpb.ConfChangeAddLearnerNode entry // which was used to add the learner itself to the Cluster, and therefore it has the correct info // on learner. func (c *Cluster) waitMembersMatch(t testutil.TB) { wMembers := c.getMembers() sort.Sort(SortableProtoMemberSliceByPeerURLs(wMembers)) cli := c.Client(0) for { resp, err := cli.MemberList(context.Background()) if err != nil { t.Fatalf("failed to list member %v", err) } if len(resp.Members) != len(wMembers) { continue } sort.Sort(SortableProtoMemberSliceByPeerURLs(resp.Members)) for _, m := range resp.Members { m.ID = 0 } if reflect.DeepEqual(resp.Members, wMembers) { return } time.Sleep(framecfg.TickDuration) } } type SortableProtoMemberSliceByPeerURLs []*pb.Member func (p SortableProtoMemberSliceByPeerURLs) Len() int { return len(p) } func (p SortableProtoMemberSliceByPeerURLs) Less(i, j int) bool { return p[i].PeerURLs[0] < p[j].PeerURLs[0] } func (p SortableProtoMemberSliceByPeerURLs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } // InitializeMemberWithResponse initializes a member with the response func (c *Cluster) InitializeMemberWithResponse(t testutil.TB, m *Member, resp *clientv3.MemberAddResponse) { m.IsLearner = resp.Member.IsLearner m.NewCluster = false m.InitialPeerURLsMap = types.URLsMap{} for _, mm := range c.Members { m.InitialPeerURLsMap[mm.Name] = mm.PeerURLs } m.InitialPeerURLsMap[m.Name] = types.MustNewURLs(resp.Member.PeerURLs) c.Members = append(c.Members, m) } ================================================ FILE: tests/framework/integration/cluster_direct.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package integration import ( pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb" "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb" ) const ThroughProxy = false func ToGRPC(c *clientv3.Client) GRPCAPI { return GRPCAPI{ pb.NewClusterClient(c.ActiveConnection()), pb.NewKVClient(c.ActiveConnection()), pb.NewLeaseClient(c.ActiveConnection()), pb.NewWatchClient(c.ActiveConnection()), pb.NewMaintenanceClient(c.ActiveConnection()), pb.NewAuthClient(c.ActiveConnection()), v3lockpb.NewLockClient(c.ActiveConnection()), v3electionpb.NewElectionClient(c.ActiveConnection()), } } func newClientV3(cfg clientv3.Config) (*clientv3.Client, error) { return clientv3.New(cfg) } ================================================ FILE: tests/framework/integration/cluster_proxy.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build cluster_proxy package integration import ( "context" "sync" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/namespace" "go.etcd.io/etcd/server/v3/proxy/grpcproxy" "go.etcd.io/etcd/server/v3/proxy/grpcproxy/adapter" ) const ThroughProxy = true var ( pmu sync.Mutex proxies map[*clientv3.Client]grpcClientProxy = make(map[*clientv3.Client]grpcClientProxy) ) const proxyNamespace = "proxy-namespace" type grpcClientProxy struct { ctx context.Context ctxCancel func() grpc GRPCAPI wdonec <-chan struct{} kvdonec <-chan struct{} lpdonec <-chan struct{} } func ToGRPC(c *clientv3.Client) GRPCAPI { pmu.Lock() defer pmu.Unlock() // dedicated context bound to 'grpc-proxy' lifetype // (so in practice lifetime of the client connection to the proxy). // TODO: Refactor to a separate clientv3.Client instance instead of the context alone. ctx, ctxCancel := context.WithCancel(context.WithValue(context.TODO(), "_name", "grpcProxyContext")) lg := c.GetLogger() if v, ok := proxies[c]; ok { return v.grpc } // test namespacing proxy c.KV = namespace.NewKV(c.KV, proxyNamespace) c.Watcher = namespace.NewWatcher(c.Watcher, proxyNamespace) c.Lease = namespace.NewLease(c.Lease, proxyNamespace) // test coalescing/caching proxy kvp, kvpch := grpcproxy.NewKvProxy(c) wp, wpch := grpcproxy.NewWatchProxy(ctx, lg, c) lp, lpch := grpcproxy.NewLeaseProxy(ctx, c) mp := grpcproxy.NewMaintenanceProxy(c) clp, _ := grpcproxy.NewClusterProxy(lg, c, "", "") // without registering proxy URLs authp := grpcproxy.NewAuthProxy(c) lockp := grpcproxy.NewLockProxy(c) electp := grpcproxy.NewElectionProxy(c) grpc := GRPCAPI{ adapter.ClusterServerToClusterClient(clp), adapter.KvServerToKvClient(kvp), adapter.LeaseServerToLeaseClient(lp), adapter.WatchServerToWatchClient(wp), adapter.MaintenanceServerToMaintenanceClient(mp), adapter.AuthServerToAuthClient(authp), adapter.LockServerToLockClient(lockp), adapter.ElectionServerToElectionClient(electp), } proxies[c] = grpcClientProxy{ctx: ctx, ctxCancel: ctxCancel, grpc: grpc, wdonec: wpch, kvdonec: kvpch, lpdonec: lpch} return grpc } type proxyCloser struct { clientv3.Watcher proxyCtxCancel func() wdonec <-chan struct{} kvdonec <-chan struct{} lclose func() lpdonec <-chan struct{} } func (pc *proxyCloser) Close() error { pc.proxyCtxCancel() <-pc.kvdonec err := pc.Watcher.Close() <-pc.wdonec pc.lclose() <-pc.lpdonec return err } func newClientV3(cfg clientv3.Config) (*clientv3.Client, error) { c, err := clientv3.New(cfg) if err != nil { return nil, err } rpc := ToGRPC(c) c.KV = clientv3.NewKVFromKVClient(rpc.KV, c) pmu.Lock() lc := c.Lease c.Lease = clientv3.NewLeaseFromLeaseClient(rpc.Lease, c, cfg.DialTimeout) c.Watcher = &proxyCloser{ Watcher: clientv3.NewWatchFromWatchClient(rpc.Watch, c), wdonec: proxies[c].wdonec, kvdonec: proxies[c].kvdonec, lclose: func() { lc.Close() }, lpdonec: proxies[c].lpdonec, proxyCtxCancel: proxies[c].ctxCancel, } pmu.Unlock() return c, nil } ================================================ FILE: tests/framework/integration/config.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import "go.etcd.io/etcd/tests/v3/framework/config" type ClusterContext struct { UseUnix bool } func WithHTTP2Debug() config.ClusterOption { return func(c *config.ClusterConfig) {} } func WithUnixClient() config.ClusterOption { return func(c *config.ClusterConfig) { ctx := ensureIntegrationClusterContext(c) ctx.UseUnix = true c.ClusterContext = ctx } } func WithTCPClient() config.ClusterOption { return func(c *config.ClusterConfig) { ctx := ensureIntegrationClusterContext(c) ctx.UseUnix = false c.ClusterContext = ctx } } func ensureIntegrationClusterContext(c *config.ClusterConfig) *ClusterContext { ctx, _ := c.ClusterContext.(*ClusterContext) if ctx == nil { ctx = &ClusterContext{} } return ctx } ================================================ FILE: tests/framework/integration/integration.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "fmt" "strings" "testing" "go.uber.org/zap" healthpb "google.golang.org/grpc/health/grpc_health_v1" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" etcdctlcmd "go.etcd.io/etcd/etcdctl/v3/ctlv3/command" "go.etcd.io/etcd/tests/v3/framework/config" intf "go.etcd.io/etcd/tests/v3/framework/interfaces" ) type integrationRunner struct{} func NewIntegrationRunner() intf.TestRunner { return &integrationRunner{} } func (e integrationRunner) TestMain(m *testing.M) { testutil.MustTestMainWithLeakDetection(m) } func (e integrationRunner) BeforeTest(tb testing.TB) { BeforeTest(tb) } func (e integrationRunner) NewCluster(ctx context.Context, tb testing.TB, opts ...config.ClusterOption) intf.Cluster { var err error cfg := config.NewClusterConfig(opts...) integrationCfg := ClusterConfig{ Size: cfg.ClusterSize, QuotaBackendBytes: cfg.QuotaBackendBytes, DisableStrictReconfigCheck: !cfg.StrictReconfigCheck, AuthToken: cfg.AuthToken, SnapshotCount: cfg.SnapshotCount, } integrationCfg.ClientTLS, err = tlsInfo(tb, cfg.ClientTLS) if err != nil { tb.Fatalf("ClientTLS: %s", err) } integrationCfg.PeerTLS, err = tlsInfo(tb, cfg.PeerTLS) if err != nil { tb.Fatalf("PeerTLS: %s", err) } if cfg.ClusterContext != nil { if ctx, ok := cfg.ClusterContext.(*ClusterContext); ok && ctx != nil { integrationCfg.UseTCP = !ctx.UseUnix integrationCfg.UseIP = !ctx.UseUnix } } return &integrationCluster{ Cluster: NewCluster(tb, &integrationCfg), t: tb, ctx: ctx, } } func tlsInfo(tb testing.TB, cfg config.TLSConfig) (*transport.TLSInfo, error) { switch cfg { case config.NoTLS: return nil, nil case config.AutoTLS: tls, err := transport.SelfCert(zap.NewNop(), tb.TempDir(), []string{"localhost"}, 1) if err != nil { return nil, fmt.Errorf("failed to generate cert: %w", err) } return &tls, nil case config.ManualTLS: return &TestTLSInfo, nil default: return nil, fmt.Errorf("config %q not supported", cfg) } } type integrationCluster struct { *Cluster t testing.TB ctx context.Context } func (c *integrationCluster) Members() (ms []intf.Member) { for _, m := range c.Cluster.Members { ms = append(ms, integrationMember{Member: m, t: c.t}) } return ms } func (c *integrationCluster) TemplateEndpoints(tb testing.TB, pattern string) []string { tb.Helper() var endpoints []string for _, m := range c.Cluster.Members { ent := pattern ent = strings.ReplaceAll(ent, "${MEMBER_PORT}", m.GRPCPortNumber()) ent = strings.ReplaceAll(ent, "${MEMBER_NAME}", m.Name) endpoints = append(endpoints, ent) } return endpoints } func templateAuthority(tb testing.TB, pattern string, m *Member) string { tb.Helper() authority := pattern authority = strings.ReplaceAll(authority, "${MEMBER_PORT}", m.GRPCPortNumber()) authority = strings.ReplaceAll(authority, "${MEMBER_NAME}", m.Name) return authority } func (c *integrationCluster) AssertAuthority(tb testing.TB, expectedAuthorityPattern string) { tb.Helper() const filterMethod = "/etcdserverpb.KV/Put" for _, m := range c.Cluster.Members { expectedAuthority := templateAuthority(tb, expectedAuthorityPattern, m) requestsFound := 0 for _, r := range m.RecordedRequests() { if r.FullMethod != filterMethod { continue } if r.Authority == expectedAuthority { requestsFound++ } else { tb.Errorf("Got unexpected authority header, member %q, request %q, got %q, expected %q", m.Name, r.FullMethod, r.Authority, expectedAuthority) } } if requestsFound == 0 { tb.Errorf("Expect at least one request with matched authority header value was recorded by the server intercepter on member %s but got 0", m.Name) } } } type integrationMember struct { *Member t testing.TB } func (m integrationMember) Client() intf.Client { return integrationClient{Client: m.Member.Client} } func (m integrationMember) Start(ctx context.Context) error { return m.Member.Restart(m.t) } func (m integrationMember) Stop() { m.Member.Stop(m.t) } func (c *integrationCluster) Close() error { c.Terminate(c.t) return nil } func (c *integrationCluster) Client(opts ...config.ClientOption) (intf.Client, error) { cc, err := c.ClusterClient(c.t, opts...) if err != nil { return nil, err } return integrationClient{Client: cc}, nil } type integrationClient struct { *clientv3.Client } func (c integrationClient) Get(ctx context.Context, key string, o config.GetOptions) (*clientv3.GetResponse, error) { if o.Timeout != 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, o.Timeout) defer cancel() } var clientOpts []clientv3.OpOption if o.Revision != 0 { clientOpts = append(clientOpts, clientv3.WithRev(int64(o.Revision))) } if o.End != "" { clientOpts = append(clientOpts, clientv3.WithRange(o.End)) } if o.Serializable { clientOpts = append(clientOpts, clientv3.WithSerializable()) } if o.Prefix { clientOpts = append(clientOpts, clientv3.WithPrefix()) } if o.Limit != 0 { clientOpts = append(clientOpts, clientv3.WithLimit(int64(o.Limit))) } if o.FromKey { clientOpts = append(clientOpts, clientv3.WithFromKey()) } if o.CountOnly { clientOpts = append(clientOpts, clientv3.WithCountOnly()) } if o.KeysOnly { clientOpts = append(clientOpts, clientv3.WithKeysOnly()) } if o.SortBy != clientv3.SortByKey || o.Order != clientv3.SortNone { clientOpts = append(clientOpts, clientv3.WithSort(o.SortBy, o.Order)) } if o.MaxCreateRevision != 0 { clientOpts = append(clientOpts, clientv3.WithMaxCreateRev(int64(o.MaxCreateRevision))) } if o.MinCreateRevision != 0 { clientOpts = append(clientOpts, clientv3.WithMinCreateRev(int64(o.MinCreateRevision))) } if o.MaxModRevision != 0 { clientOpts = append(clientOpts, clientv3.WithMaxModRev(int64(o.MaxModRevision))) } if o.MinModRevision != 0 { clientOpts = append(clientOpts, clientv3.WithMinModRev(int64(o.MinModRevision))) } return c.Client.Get(ctx, key, clientOpts...) } func (c integrationClient) Put(ctx context.Context, key, value string, opts config.PutOptions) (*clientv3.PutResponse, error) { if opts.Timeout != 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, opts.Timeout) defer cancel() } var clientOpts []clientv3.OpOption if opts.LeaseID != 0 { clientOpts = append(clientOpts, clientv3.WithLease(opts.LeaseID)) } return c.Client.Put(ctx, key, value, clientOpts...) } func (c integrationClient) Delete(ctx context.Context, key string, o config.DeleteOptions) (*clientv3.DeleteResponse, error) { var clientOpts []clientv3.OpOption if o.Prefix { clientOpts = append(clientOpts, clientv3.WithPrefix()) } if o.FromKey { clientOpts = append(clientOpts, clientv3.WithFromKey()) } if o.End != "" { clientOpts = append(clientOpts, clientv3.WithRange(o.End)) } return c.Client.Delete(ctx, key, clientOpts...) } func (c integrationClient) Compact(ctx context.Context, rev int64, o config.CompactOption) (*clientv3.CompactResponse, error) { if o.Timeout != 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, o.Timeout) defer cancel() } var clientOpts []clientv3.CompactOption if o.Physical { clientOpts = append(clientOpts, clientv3.WithCompactPhysical()) } return c.Client.Compact(ctx, rev, clientOpts...) } func (c integrationClient) Status(ctx context.Context) ([]*clientv3.StatusResponse, error) { endpoints := c.Client.Endpoints() var resp []*clientv3.StatusResponse for _, ep := range endpoints { status, err := c.Client.Status(ctx, ep) if err != nil { return nil, err } resp = append(resp, status) } return resp, nil } func (c integrationClient) HashKV(ctx context.Context, rev int64) ([]*clientv3.HashKVResponse, error) { endpoints := c.Client.Endpoints() var resp []*clientv3.HashKVResponse for _, ep := range endpoints { hashKV, err := c.Client.HashKV(ctx, ep, rev) if err != nil { return nil, err } resp = append(resp, hashKV) } return resp, nil } func (c integrationClient) Health(ctx context.Context) error { cli := healthpb.NewHealthClient(c.Client.ActiveConnection()) resp, err := cli.Check(ctx, &healthpb.HealthCheckRequest{}) if err != nil { return err } if resp.Status != healthpb.HealthCheckResponse_SERVING { return fmt.Errorf("status expected %s, got %s", healthpb.HealthCheckResponse_SERVING, resp.Status) } return nil } func (c integrationClient) Defragment(ctx context.Context, o config.DefragOption) error { if o.Timeout != 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, o.Timeout) defer cancel() } for _, ep := range c.Endpoints() { _, err := c.Client.Defragment(ctx, ep) if err != nil { return err } } return nil } func (c integrationClient) TimeToLive(ctx context.Context, id clientv3.LeaseID, o config.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) { var leaseOpts []clientv3.LeaseOption if o.WithAttachedKeys { leaseOpts = append(leaseOpts, clientv3.WithAttachedKeys()) } return c.Client.TimeToLive(ctx, id, leaseOpts...) } func (c integrationClient) Leases(ctx context.Context) (*clientv3.LeaseLeasesResponse, error) { return c.Client.Leases(ctx) } func (c integrationClient) KeepAliveOnce(ctx context.Context, id clientv3.LeaseID) (*clientv3.LeaseKeepAliveResponse, error) { return c.Client.KeepAliveOnce(ctx, id) } func (c integrationClient) Revoke(ctx context.Context, id clientv3.LeaseID) (*clientv3.LeaseRevokeResponse, error) { return c.Client.Revoke(ctx, id) } func (c integrationClient) AuthEnable(ctx context.Context) error { _, err := c.Client.AuthEnable(ctx) return err } func (c integrationClient) AuthDisable(ctx context.Context) error { _, err := c.Client.AuthDisable(ctx) return err } func (c integrationClient) AuthStatus(ctx context.Context) (*clientv3.AuthStatusResponse, error) { return c.Client.AuthStatus(ctx) } func (c integrationClient) UserAdd(ctx context.Context, name, password string, opts config.UserAddOptions) (*clientv3.AuthUserAddResponse, error) { return c.Client.UserAddWithOptions(ctx, name, password, &clientv3.UserAddOptions{ NoPassword: opts.NoPassword, }) } func (c integrationClient) UserGet(ctx context.Context, name string) (*clientv3.AuthUserGetResponse, error) { return c.Client.UserGet(ctx, name) } func (c integrationClient) UserList(ctx context.Context) (*clientv3.AuthUserListResponse, error) { return c.Client.UserList(ctx) } func (c integrationClient) UserDelete(ctx context.Context, name string) (*clientv3.AuthUserDeleteResponse, error) { return c.Client.UserDelete(ctx, name) } func (c integrationClient) UserChangePass(ctx context.Context, user, newPass string) error { _, err := c.Client.UserChangePassword(ctx, user, newPass) return err } func (c integrationClient) UserGrantRole(ctx context.Context, user string, role string) (*clientv3.AuthUserGrantRoleResponse, error) { return c.Client.UserGrantRole(ctx, user, role) } func (c integrationClient) UserRevokeRole(ctx context.Context, user string, role string) (*clientv3.AuthUserRevokeRoleResponse, error) { return c.Client.UserRevokeRole(ctx, user, role) } func (c integrationClient) RoleAdd(ctx context.Context, name string) (*clientv3.AuthRoleAddResponse, error) { return c.Client.RoleAdd(ctx, name) } func (c integrationClient) RoleGrantPermission(ctx context.Context, name string, key, rangeEnd string, permType clientv3.PermissionType) (*clientv3.AuthRoleGrantPermissionResponse, error) { return c.Client.RoleGrantPermission(ctx, name, key, rangeEnd, permType) } func (c integrationClient) RoleGet(ctx context.Context, role string) (*clientv3.AuthRoleGetResponse, error) { return c.Client.RoleGet(ctx, role) } func (c integrationClient) RoleList(ctx context.Context) (*clientv3.AuthRoleListResponse, error) { return c.Client.RoleList(ctx) } func (c integrationClient) RoleRevokePermission(ctx context.Context, role string, key, rangeEnd string) (*clientv3.AuthRoleRevokePermissionResponse, error) { return c.Client.RoleRevokePermission(ctx, role, key, rangeEnd) } func (c integrationClient) RoleDelete(ctx context.Context, role string) (*clientv3.AuthRoleDeleteResponse, error) { return c.Client.RoleDelete(ctx, role) } func (c integrationClient) Txn(ctx context.Context, compares, ifSucess, ifFail []string, o config.TxnOptions) (*clientv3.TxnResponse, error) { txn := c.Client.Txn(ctx) var cmps []clientv3.Cmp for _, c := range compares { cmp, err := etcdctlcmd.ParseCompare(c) if err != nil { return nil, err } cmps = append(cmps, *cmp) } succOps := getOps(ifSucess) failOps := getOps(ifFail) txnrsp, err := txn. If(cmps...). Then(succOps...). Else(failOps...). Commit() return txnrsp, err } func getOps(ss []string) []clientv3.Op { var ops []clientv3.Op for _, s := range ss { s = strings.TrimSpace(s) args := etcdctlcmd.Argify(s) switch args[0] { case "get": ops = append(ops, clientv3.OpGet(args[1])) case "put": ops = append(ops, clientv3.OpPut(args[1], args[2])) case "del": ops = append(ops, clientv3.OpDelete(args[1])) } } return ops } func (c integrationClient) Watch(ctx context.Context, key string, opts config.WatchOptions) clientv3.WatchChan { var opOpts []clientv3.OpOption if opts.Prefix { opOpts = append(opOpts, clientv3.WithPrefix()) } if opts.Revision != 0 { opOpts = append(opOpts, clientv3.WithRev(opts.Revision)) } if opts.RangeEnd != "" { opOpts = append(opOpts, clientv3.WithRange(opts.RangeEnd)) } return c.Client.Watch(ctx, key, opOpts...) } func (c integrationClient) MemberAdd(ctx context.Context, _ string, peerAddrs []string) (*clientv3.MemberAddResponse, error) { return c.Client.MemberAdd(ctx, peerAddrs) } func (c integrationClient) MemberAddAsLearner(ctx context.Context, _ string, peerAddrs []string) (*clientv3.MemberAddResponse, error) { return c.Client.MemberAddAsLearner(ctx, peerAddrs) } func (c integrationClient) MemberRemove(ctx context.Context, id uint64) (*clientv3.MemberRemoveResponse, error) { return c.Client.MemberRemove(ctx, id) } func (c integrationClient) MemberList(ctx context.Context, serializable bool) (*clientv3.MemberListResponse, error) { if serializable { return c.Client.MemberList(ctx, clientv3.WithSerializable()) } return c.Client.MemberList(ctx) } ================================================ FILE: tests/framework/integration/testing.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "os" "testing" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/verify" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/embed" gofail "go.etcd.io/gofail/runtime" ) var ( insideTestContext bool ) type testOptions struct { goLeakDetection bool skipInShort bool failpoint *failpoint } type failpoint struct { name string payload string } func newTestOptions(opts ...TestOption) *testOptions { o := &testOptions{goLeakDetection: true, skipInShort: true} for _, opt := range opts { opt(o) } return o } type TestOption func(opt *testOptions) // WithoutGoLeakDetection disables checking whether a testcase leaked a goroutine. func WithoutGoLeakDetection() TestOption { return func(opt *testOptions) { opt.goLeakDetection = false } } func WithoutSkipInShort() TestOption { return func(opt *testOptions) { opt.skipInShort = false } } // WithFailpoint registers a go fail point func WithFailpoint(name, payload string) TestOption { return func(opt *testOptions) { opt.failpoint = &failpoint{name: name, payload: payload} } } // BeforeTestExternal initializes test context and is targeted for external APIs. // In general the `integration` package is not targeted to be used outside of // etcd project, but till the dedicated package is developed, this is // the best entry point so far (without backward compatibility promise). func BeforeTestExternal(t testutil.TB) { BeforeTest(t, WithoutSkipInShort(), WithoutGoLeakDetection()) } func SkipIfNoGoFail(t testutil.TB) { if len(gofail.List()) == 0 { t.Skip("please run 'make gofail-enable' before running the test") } } func BeforeTest(t testutil.TB, opts ...TestOption) { t.Helper() options := newTestOptions(opts...) if insideTestContext { t.Fatal("already in test context. BeforeTest was likely already called") } if options.skipInShort { testutil.SkipTestIfShortMode(t, "Cannot create clusters in --short tests") } if options.goLeakDetection { testutil.RegisterLeakDetection(t) } if options.failpoint != nil && len(options.failpoint.name) != 0 { SkipIfNoGoFail(t) require.NoError(t, gofail.Enable(options.failpoint.name, options.failpoint.payload)) t.Cleanup(func() { require.NoError(t, gofail.Disable(options.failpoint.name)) }) } previousWD, err := os.Getwd() if err != nil { t.Fatal(err) } previousInsideTestContext := insideTestContext // Integration tests should verify written state as much as possible. revertFunc := verify.EnableAllVerifications() // Registering cleanup early, such it will get executed even if the helper fails. t.Cleanup(func() { insideTestContext = previousInsideTestContext os.Chdir(previousWD) revertFunc() }) insideTestContext = true os.Chdir(t.TempDir()) } func assertInTestContext(t testutil.TB) { if !insideTestContext { t.Errorf("the function can be called only in the test context. Was integration.BeforeTest() called ?") } } func NewEmbedConfig(tb testing.TB, name string) *embed.Config { cfg := embed.NewConfig() cfg.Name = name lg := zaptest.NewLogger(tb, zaptest.Level(zapcore.InfoLevel)).Named(cfg.Name) cfg.ZapLoggerBuilder = embed.NewZapLoggerBuilder(lg) cfg.Dir = tb.TempDir() return cfg } func NewClient(tb testing.TB, cfg clientv3.Config) (*clientv3.Client, error) { if cfg.Logger == nil { cfg.Logger = zaptest.NewLogger(tb).Named("client") } return clientv3.New(cfg) } ================================================ FILE: tests/framework/interfaces/interface.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package interfaces import ( "context" "testing" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" ) type TestRunner interface { TestMain(m *testing.M) BeforeTest(testing.TB) NewCluster(context.Context, testing.TB, ...config.ClusterOption) Cluster } type Cluster interface { Members() []Member Client(opts ...config.ClientOption) (Client, error) WaitLeader(t testing.TB) int Close() error Endpoints() []string } type Member interface { Client() Client Start(ctx context.Context) error Stop() } type Client interface { Put(context context.Context, key, value string, opts config.PutOptions) (*clientv3.PutResponse, error) Get(context context.Context, key string, opts config.GetOptions) (*clientv3.GetResponse, error) Delete(context context.Context, key string, opts config.DeleteOptions) (*clientv3.DeleteResponse, error) Compact(context context.Context, rev int64, opts config.CompactOption) (*clientv3.CompactResponse, error) Status(context context.Context) ([]*clientv3.StatusResponse, error) HashKV(context context.Context, rev int64) ([]*clientv3.HashKVResponse, error) Health(context context.Context) error Defragment(context context.Context, opts config.DefragOption) error AlarmList(context context.Context) (*clientv3.AlarmResponse, error) AlarmDisarm(context context.Context, alarmMember *clientv3.AlarmMember) (*clientv3.AlarmResponse, error) Grant(context context.Context, ttl int64) (*clientv3.LeaseGrantResponse, error) TimeToLive(context context.Context, id clientv3.LeaseID, opts config.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) Leases(context context.Context) (*clientv3.LeaseLeasesResponse, error) KeepAliveOnce(context context.Context, id clientv3.LeaseID) (*clientv3.LeaseKeepAliveResponse, error) Revoke(context context.Context, id clientv3.LeaseID) (*clientv3.LeaseRevokeResponse, error) AuthEnable(context context.Context) error AuthDisable(context context.Context) error AuthStatus(context context.Context) (*clientv3.AuthStatusResponse, error) UserAdd(context context.Context, name, password string, opts config.UserAddOptions) (*clientv3.AuthUserAddResponse, error) UserGet(context context.Context, name string) (*clientv3.AuthUserGetResponse, error) UserList(context context.Context) (*clientv3.AuthUserListResponse, error) UserDelete(context context.Context, name string) (*clientv3.AuthUserDeleteResponse, error) UserChangePass(context context.Context, user, newPass string) error UserGrantRole(context context.Context, user string, role string) (*clientv3.AuthUserGrantRoleResponse, error) UserRevokeRole(context context.Context, user string, role string) (*clientv3.AuthUserRevokeRoleResponse, error) RoleAdd(context context.Context, name string) (*clientv3.AuthRoleAddResponse, error) RoleGrantPermission(context context.Context, name string, key, rangeEnd string, permType clientv3.PermissionType) (*clientv3.AuthRoleGrantPermissionResponse, error) RoleGet(context context.Context, role string) (*clientv3.AuthRoleGetResponse, error) RoleList(context context.Context) (*clientv3.AuthRoleListResponse, error) RoleRevokePermission(context context.Context, role string, key, rangeEnd string) (*clientv3.AuthRoleRevokePermissionResponse, error) RoleDelete(context context.Context, role string) (*clientv3.AuthRoleDeleteResponse, error) Txn(context context.Context, compares, ifSucess, ifFail []string, o config.TxnOptions) (*clientv3.TxnResponse, error) MemberList(context context.Context, serializable bool) (*clientv3.MemberListResponse, error) MemberAdd(context context.Context, name string, peerAddrs []string) (*clientv3.MemberAddResponse, error) MemberAddAsLearner(context context.Context, name string, peerAddrs []string) (*clientv3.MemberAddResponse, error) MemberRemove(ctx context.Context, id uint64) (*clientv3.MemberRemoveResponse, error) Watch(ctx context.Context, key string, opts config.WatchOptions) clientv3.WatchChan } type TemplateEndpoints interface { TemplateEndpoints(tb testing.TB, pattern string) []string } type AssertAuthority interface { AssertAuthority(tb testing.TB, expectedAuthorityPattern string) } ================================================ FILE: tests/framework/testrunner.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package framework import ( "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/integration" intf "go.etcd.io/etcd/tests/v3/framework/interfaces" "go.etcd.io/etcd/tests/v3/framework/unit" ) var ( // UnitTestRunner only runs in `--short` mode, will fail otherwise. Attempts in cluster creation will result in tests being skipped. UnitTestRunner intf.TestRunner = unit.NewUnitRunner() // E2eTestRunner runs etcd and etcdctl binaries in a separate process. E2eTestRunner = e2e.NewE2eRunner() // IntegrationTestRunner runs etcdserver.EtcdServer in separate goroutine and uses client libraries to communicate. IntegrationTestRunner = integration.NewIntegrationRunner() ) ================================================ FILE: tests/framework/testutils/execute.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutils import ( "context" "fmt" "testing" "time" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func ExecuteWithTimeout(t *testing.T, timeout time.Duration, f func()) { ctx, cancel := context.WithTimeout(t.Context(), timeout) defer cancel() ExecuteUntil(ctx, t, f) } func ExecuteUntil(ctx context.Context, t *testing.T, f func()) { deadline, deadlineSet := ctx.Deadline() timeout := time.Until(deadline) donec := make(chan struct{}) go func() { defer close(donec) f() }() select { case <-ctx.Done(): msg := ctx.Err().Error() if deadlineSet { msg = fmt.Sprintf("test timed out after %v, err: %v", timeout, msg) } testutil.FatalStack(t, msg) case <-donec: } } ================================================ FILE: tests/framework/testutils/helpters.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutils import ( "errors" "time" clientv3 "go.etcd.io/etcd/client/v3" intf "go.etcd.io/etcd/tests/v3/framework/interfaces" ) type KV struct { Key, Val string } func KeysFromGetResponse(resp *clientv3.GetResponse) (kvs []string) { for _, kv := range resp.Kvs { kvs = append(kvs, string(kv.Key)) } return kvs } func KeyValuesFromGetResponse(resp *clientv3.GetResponse) (kvs []KV) { for _, kv := range resp.Kvs { kvs = append(kvs, KV{Key: string(kv.Key), Val: string(kv.Value)}) } return kvs } func KeyValuesFromWatchResponse(resp clientv3.WatchResponse) (kvs []KV) { for _, event := range resp.Events { kvs = append(kvs, KV{Key: string(event.Kv.Key), Val: string(event.Kv.Value)}) } return kvs } func KeyValuesFromWatchChan(wch clientv3.WatchChan, wantedLen int, timeout time.Duration) (kvs []KV, err error) { for { select { case watchResp, ok := <-wch: if ok { kvs = append(kvs, KeyValuesFromWatchResponse(watchResp)...) if len(kvs) == wantedLen { return kvs, nil } } case <-time.After(timeout): return nil, errors.New("closed watcher channel should not block") } } } func MustClient(c intf.Client, err error) intf.Client { if err != nil { panic(err) } return c } ================================================ FILE: tests/framework/testutils/log_observer.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutils import ( "context" "fmt" "strings" "sync" "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" zapobserver "go.uber.org/zap/zaptest/observer" ) type LogObserver struct { ob *zapobserver.ObservedLogs enc zapcore.Encoder mu sync.Mutex // entries stores all the logged entries after syncLogs. entries []zapobserver.LoggedEntry } func NewLogObserver(level zapcore.LevelEnabler) (zapcore.Core, *LogObserver) { // align with zaptest enc := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) co, ob := zapobserver.New(level) return co, &LogObserver{ ob: ob, enc: enc, } } // Expect returns the first N lines containing the given string. func (logOb *LogObserver) Expect(ctx context.Context, s string, count int) ([]string, error) { return logOb.ExpectFunc(ctx, func(log string) bool { return strings.Contains(log, s) }, count) } // ExpectFunc returns the first N line satisfying the function f. func (logOb *LogObserver) ExpectFunc(ctx context.Context, filter func(string) bool, count int) ([]string, error) { i := 0 res := make([]string, 0, count) for { select { case <-ctx.Done(): return nil, ctx.Err() default: } entries := logOb.syncLogs() // The order of entries won't be changed because of append-only. // It's safe to skip scanned entries by reusing `i`. for ; i < len(entries); i++ { buf, err := logOb.enc.EncodeEntry(entries[i].Entry, entries[i].Context) if err != nil { return nil, fmt.Errorf("failed to encode entry: %w", err) } logInStr := buf.String() if filter(logInStr) { res = append(res, logInStr) } if len(res) >= count { return res, nil } } time.Sleep(10 * time.Millisecond) } } // syncLogs is to take all the existing logged entries from zapobserver and // truncate zapobserver's entries slice. func (logOb *LogObserver) syncLogs() []zapobserver.LoggedEntry { logOb.mu.Lock() defer logOb.mu.Unlock() logOb.entries = append(logOb.entries, logOb.ob.TakeAll()...) return logOb.entries } ================================================ FILE: tests/framework/testutils/log_observer_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutils import ( "context" "fmt" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" ) func TestLogObserver_Timeout(t *testing.T) { logCore, logOb := NewLogObserver(zap.InfoLevel) logger := zap.New(logCore) logger.Info(t.Name()) ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) _, err := logOb.Expect(ctx, "unknown", 1) cancel() require.ErrorIs(t, err, context.DeadlineExceeded) assert.Len(t, logOb.entries, 1) } func TestLogObserver_Expect(t *testing.T) { logCore, logOb := NewLogObserver(zap.InfoLevel) logger := zap.New(logCore) ctx, cancel := context.WithCancel(t.Context()) defer cancel() resCh := make(chan []string, 1) go func() { defer close(resCh) res, err := logOb.Expect(ctx, t.Name(), 2) assert.NoError(t, err) resCh <- res }() msgs := []string{"Hello " + t.Name(), t.Name() + ", World"} for _, msg := range msgs { logger.Info(msg) time.Sleep(40 * time.Millisecond) } res := <-resCh assert.Len(t, res, 2) // The logged message should be like // // 2023-04-16T11:46:19.367+0800 INFO Hello TestLogObserver_Expect // 2023-04-16T11:46:19.408+0800 INFO TestLogObserver_Expect, World // // The prefix timestamp is unpredictable so we should assert the suffix // only. for idx := range msgs { expected := fmt.Sprintf("\tINFO\t%s\n", msgs[idx]) assert.True(t, strings.HasSuffix(res[idx], expected)) } assert.Len(t, logOb.entries, 2) } ================================================ FILE: tests/framework/testutils/path.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testutils import "path/filepath" func MustAbsPath(path string) string { abs, err := filepath.Abs(path) if err != nil { panic(err) } return abs } ================================================ FILE: tests/framework/unit/unit.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package unit import ( "context" "flag" "fmt" "os" "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/tests/v3/framework/config" intf "go.etcd.io/etcd/tests/v3/framework/interfaces" ) type unitRunner struct{} var _ intf.TestRunner = (*unitRunner)(nil) func NewUnitRunner() intf.TestRunner { return &unitRunner{} } func (e unitRunner) TestMain(m *testing.M) { flag.Parse() if !testing.Short() { fmt.Println(`No test mode selected, please selected either e2e mode with "--tags e2e" or integration mode with "--tags integration"`) os.Exit(1) } } func (e unitRunner) BeforeTest(tb testing.TB) { } func (e unitRunner) NewCluster(ctx context.Context, tb testing.TB, opts ...config.ClusterOption) intf.Cluster { testutil.SkipTestIfShortMode(tb, "Cannot create clusters in --short tests") return nil } ================================================ FILE: tests/go.mod ================================================ module go.etcd.io/etcd/tests/v3 go 1.26 toolchain go1.26.1 replace ( go.etcd.io/etcd/api/v3 => ../api go.etcd.io/etcd/cache/v3 => ../cache go.etcd.io/etcd/client/pkg/v3 => ../client/pkg go.etcd.io/etcd/client/v3 => ../client/v3 go.etcd.io/etcd/etcdctl/v3 => ../etcdctl go.etcd.io/etcd/etcdutl/v3 => ../etcdutl go.etcd.io/etcd/pkg/v3 => ../pkg go.etcd.io/etcd/server/v3 => ../server ) require ( github.com/anishathalye/porcupine v1.1.0 github.com/antithesishq/antithesis-sdk-go v0.4.3 github.com/coreos/go-semver v0.3.1 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/golang/protobuf v1.5.4 github.com/google/go-cmp v0.7.0 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 github.com/olekukonko/tablewriter v1.1.3 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.67.5 github.com/soheilhy/cmux v0.1.5 github.com/stretchr/testify v1.11.1 go.etcd.io/bbolt v1.4.3 go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 go.etcd.io/etcd/cache/v3 v3.6.1 go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/v3 v3.6.0-alpha.0 go.etcd.io/etcd/etcdctl/v3 v3.6.0-alpha.0 go.etcd.io/etcd/etcdutl/v3 v3.6.0-alpha.0 go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0 go.etcd.io/etcd/server/v3 v3.6.0-alpha.0 go.etcd.io/gofail v0.2.0 go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 go.opentelemetry.io/otel v1.42.0 go.opentelemetry.io/otel/sdk v1.42.0 go.opentelemetry.io/proto/otlp v1.9.0 go.uber.org/zap v1.27.1 golang.org/x/crypto v0.48.0 golang.org/x/sync v0.19.0 golang.org/x/time v0.14.0 google.golang.org/grpc v1.79.2 google.golang.org/protobuf v1.36.11 ) require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.2.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cheggaaa/pb/v3 v3.1.7 // indirect github.com/clipperhouse/displaywidth v0.6.2 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/creack/pty v1.1.18 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.18.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/errors v1.1.0 // indirect github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect go.opentelemetry.io/otel/metric v1.42.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/utils v0.0.0-20260108192941-914a6e750570 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) ================================================ FILE: tests/go.sum ================================================ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/anishathalye/porcupine v1.1.0 h1:jkMLqDejaWqvhvjxYKyqwQO3d1Jw+/08wHiIw0O4wcU= github.com/anishathalye/porcupine v1.1.0/go.mod h1:WM0SsFjWNl2Y4BqHr/E/ll2yY1GY1jqn+W7Z/84Zoog= github.com/antithesishq/antithesis-sdk-go v0.4.3 h1:a2hGdDogClzHzFu20r1z0tzD6zwSWUipiaerAjZVP90= github.com/antithesishq/antithesis-sdk-go v0.4.3/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.2.0 h1:tgObeVOf8WAvtuAX6DhJ4xks4CFNwPDZiqzGqIHE51E= github.com/bgentry/speakeasy v0.2.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb/v3 v3.1.7 h1:2FsIW307kt7A/rz/ZI2lvPO+v3wKazzE4K/0LtTWsOI= github.com/cheggaaa/pb/v3 v3.1.7/go.mod h1:/Ji89zfVPeC/u5j8ukD0MBPHt2bzTYp74lQ7KlgFWTQ= github.com/clipperhouse/displaywidth v0.6.2 h1:ZDpTkFfpHOKte4RG5O/BOyf3ysnvFswpyYrV7z2uAKo= github.com/clipperhouse/displaywidth v0.6.2/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM= github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0 h1:jrYnow5+hy3WRDCBypUFvVKNSPPCdqgSXIE9eJDD8LM= github.com/olekukonko/ll v0.1.4-0.20260115111900-9e59c2286df0/go.mod h1:b52bVQRRPObe+yyBl0TxNfhesL0nedD4Cht0/zx55Ew= github.com/olekukonko/tablewriter v1.1.3 h1:VSHhghXxrP0JHl+0NnKid7WoEmd9/urKRJLysb70nnA= github.com/olekukonko/tablewriter v1.1.3/go.mod h1:9VU0knjhmMkXjnMKrZ3+L2JhhtsQ/L38BbL3CRNE8tM= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.etcd.io/gofail v0.2.0 h1:p19drv16FKK345a09a1iubchlw/vmRuksmRzgBIGjcA= go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o= go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee h1:9s5V0M58uCy51LgP6SUjROx7Ofqf8lGmeD/cCLaoagI= go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee/go.mod h1:VteWcRz3UV3TOpfex1x8jgPKAyjRXLKw3j8RdK3UAps= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 h1:XmiuHzgJt067+a6kwyAzkhXooYVv3/TOw9cM2VfJgUM= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/utils v0.0.0-20260108192941-914a6e750570 h1:JT4W8lsdrGENg9W+YwwdLJxklIuKWdRm+BC+xt33FOY= k8s.io/utils v0.0.0-20260108192941-914a6e750570/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= ================================================ FILE: tests/integration/cache_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "errors" "fmt" "strings" "testing" "time" "github.com/google/go-cmp/cmp" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" cache "go.etcd.io/etcd/cache/v3" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestCacheWithoutPrefixWatch(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) client := clus.Client(0) c, err := cache.New(client, "", cache.WithHistoryWindowSize(32)) if err != nil { t.Fatalf("New(...): %v", err) } t.Cleanup(c.Close) if err := c.WaitReady(t.Context()); err != nil { t.Fatalf("cache not ready: %v", err) } testWatch(t, client.KV, c) } func TestWatch(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) client := clus.Client(0) testWatch(t, client.KV, client.Watcher) } func testWatch(t *testing.T, kv clientv3.KV, watcher Watcher) { ctx := t.Context() rev2PutFooA := &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/a"), Value: []byte("1"), CreateRevision: 2, ModRevision: 2, Version: 1, }, } rev3PutFooB := &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/b"), Value: []byte("2"), CreateRevision: 3, ModRevision: 3, Version: 1, }, } rev4DeleteFooA := &clientv3.Event{ Type: clientv3.EventTypeDelete, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/a"), ModRevision: 4, }, } rev5PutFooA := &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/a"), Value: []byte("3"), CreateRevision: 5, ModRevision: 5, Version: 1, }, } rev5DeleteFooB := &clientv3.Event{ Type: clientv3.EventTypeDelete, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/b"), ModRevision: 5, }, } rev6PutFooC := &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/c"), Value: []byte("x"), CreateRevision: 6, ModRevision: 6, Version: 1, }, } rev7PutFooBar := &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/bar"), Value: []byte("y"), CreateRevision: 7, ModRevision: 7, Version: 1, }, } rev8PutFooBaz := &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/baz"), Value: []byte("z"), CreateRevision: 8, ModRevision: 8, Version: 1, }, } rev9PutFooYoo := &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/yoo"), Value: []byte("1"), CreateRevision: 9, ModRevision: 9, Version: 1, }, } rev10PutZoo := &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/zoo"), Value: []byte("1"), CreateRevision: 10, ModRevision: 10, Version: 1, }, } rev11PutFooFuture := &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/future"), Value: []byte("42"), CreateRevision: 11, ModRevision: 11, Version: 1, }, } rev12PutFooTx1 := &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/tx1"), Value: []byte("a"), CreateRevision: 12, ModRevision: 12, Version: 1, }, } rev12DeleteFooFuture := &clientv3.Event{ Type: clientv3.EventTypeDelete, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/future"), ModRevision: 12, }, } rev12PutFooTx2 := &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/tx2"), Value: []byte("b"), CreateRevision: 12, ModRevision: 12, Version: 1, }, } tcs := []struct { name string key string opts []clientv3.OpOption wantEvents []*clientv3.Event }{ { name: "Watch single key existing /foo/c", key: "/foo/c", opts: []clientv3.OpOption{clientv3.WithRev(2)}, wantEvents: []*clientv3.Event{rev6PutFooC}, }, { name: "Watch single key non‑existent /doesnotexist", key: "/doesnotexist", opts: []clientv3.OpOption{clientv3.WithRev(2)}, wantEvents: nil, }, { name: "Watch range empty", key: "", opts: []clientv3.OpOption{clientv3.WithRange(""), clientv3.WithRev(2)}, wantEvents: nil, }, { name: "Watch range [/foo/a, /foo/b)", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithRange("/foo/b"), clientv3.WithRev(2)}, wantEvents: []*clientv3.Event{rev2PutFooA, rev4DeleteFooA, rev5PutFooA}, }, { name: "Watch with prefix /foo/b", key: "/foo/b", opts: []clientv3.OpOption{clientv3.WithPrefix(), clientv3.WithRev(2)}, wantEvents: []*clientv3.Event{rev3PutFooB, rev5DeleteFooB, rev7PutFooBar, rev8PutFooBaz}, }, { name: "Watch with prefix non-existent /doesnotexist", key: "/doesnotexist", opts: []clientv3.OpOption{clientv3.WithPrefix(), clientv3.WithRev(2)}, wantEvents: nil, }, { name: "Watch with prefix empty string", key: "", opts: []clientv3.OpOption{clientv3.WithPrefix(), clientv3.WithRev(2)}, wantEvents: []*clientv3.Event{rev2PutFooA, rev3PutFooB, rev4DeleteFooA, rev5PutFooA, rev5DeleteFooB, rev6PutFooC, rev7PutFooBar, rev8PutFooBaz, rev9PutFooYoo, rev10PutZoo, rev11PutFooFuture, rev12PutFooTx1, rev12DeleteFooFuture, rev12PutFooTx2}, }, { name: "Watch from key /foo/b", key: "/foo/b", opts: []clientv3.OpOption{clientv3.WithFromKey(), clientv3.WithRev(2)}, wantEvents: []*clientv3.Event{rev3PutFooB, rev5DeleteFooB, rev6PutFooC, rev7PutFooBar, rev8PutFooBaz, rev9PutFooYoo, rev10PutZoo, rev11PutFooFuture, rev12PutFooTx1, rev12DeleteFooFuture, rev12PutFooTx2}, }, { name: "Watch from empty key", key: "", opts: []clientv3.OpOption{clientv3.WithFromKey(), clientv3.WithRev(2)}, wantEvents: []*clientv3.Event{rev2PutFooA, rev3PutFooB, rev4DeleteFooA, rev5PutFooA, rev5DeleteFooB, rev6PutFooC, rev7PutFooBar, rev8PutFooBaz, rev9PutFooYoo, rev10PutZoo, rev11PutFooFuture, rev12PutFooTx1, rev12DeleteFooFuture, rev12PutFooTx2}, }, { name: "Watch from non-existent key /doesnotexist", key: "/doesnotexist", opts: []clientv3.OpOption{clientv3.WithFromKey(), clientv3.WithRev(2)}, wantEvents: []*clientv3.Event{rev2PutFooA, rev3PutFooB, rev4DeleteFooA, rev5PutFooA, rev5DeleteFooB, rev6PutFooC, rev7PutFooBar, rev8PutFooBaz, rev9PutFooYoo, rev10PutZoo, rev11PutFooFuture, rev12PutFooTx1, rev12DeleteFooFuture, rev12PutFooTx2}, }, { name: "Watch from rev 4 with single key /foo/a", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithRev(4)}, wantEvents: []*clientv3.Event{rev4DeleteFooA, rev5PutFooA}, }, { name: "Watch from rev 6 with single key /foo/a", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithRev(6)}, wantEvents: nil, }, { name: "Watch from rev 5 with prefix /foo", key: "/foo", opts: []clientv3.OpOption{clientv3.WithPrefix(), clientv3.WithRev(5)}, wantEvents: []*clientv3.Event{ rev5PutFooA, rev5DeleteFooB, rev6PutFooC, rev7PutFooBar, rev8PutFooBaz, rev9PutFooYoo, rev11PutFooFuture, rev12PutFooTx1, rev12DeleteFooFuture, rev12PutFooTx2, }, }, { name: "Watch from rev 10 with prefix /foo", key: "/foo", opts: []clientv3.OpOption{clientv3.WithPrefix(), clientv3.WithRev(10)}, wantEvents: []*clientv3.Event{ rev11PutFooFuture, rev12PutFooTx1, rev12DeleteFooFuture, rev12PutFooTx2, }, }, { name: "Watch from rev 4 with range [/foo/a, /foo/c)", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithRange("/foo/c"), clientv3.WithRev(4)}, wantEvents: []*clientv3.Event{ rev4DeleteFooA, rev5PutFooA, rev5DeleteFooB, rev7PutFooBar, rev8PutFooBaz, }, }, { name: "Latest‑revision watcher for /foo", key: "/foo", opts: []clientv3.OpOption{clientv3.WithPrefix()}, wantEvents: []*clientv3.Event{rev11PutFooFuture, rev12PutFooTx1, rev12DeleteFooFuture, rev12PutFooTx2}, }, { name: "Watch from rev 11 with single key /foo/future", key: "/foo", opts: []clientv3.OpOption{clientv3.WithRev(11), clientv3.WithPrefix()}, wantEvents: []*clientv3.Event{rev11PutFooFuture, rev12PutFooTx1, rev12DeleteFooFuture, rev12PutFooTx2}, }, { name: "Watch from rev 12 with txn prefix /foo", key: "/foo", opts: []clientv3.OpOption{clientv3.WithRev(12), clientv3.WithPrefix()}, wantEvents: []*clientv3.Event{rev12PutFooTx1, rev12DeleteFooFuture, rev12PutFooTx2}, }, } t.Log("Write the first batch of events rev 2-10") if _, err := kv.Put(ctx, string(rev2PutFooA.Kv.Key), string(rev2PutFooA.Kv.Value)); err != nil { t.Fatalf("Put: %v", err) } if _, err := kv.Put(ctx, string(rev3PutFooB.Kv.Key), string(rev3PutFooB.Kv.Value)); err != nil { t.Fatalf("Put: %v", err) } if _, err := kv.Delete(ctx, string(rev4DeleteFooA.Kv.Key)); err != nil { t.Fatalf("Delete: %v", err) } if _, err := kv.Txn(ctx).Then(clientv3.OpPut(string(rev5PutFooA.Kv.Key), string(rev5PutFooA.Kv.Value)), clientv3.OpDelete(string(rev5DeleteFooB.Kv.Key))).Commit(); err != nil { t.Fatalf("Txn: %v", err) } if _, err := kv.Put(ctx, string(rev6PutFooC.Kv.Key), string(rev6PutFooC.Kv.Value)); err != nil { t.Fatalf("Put: %v", err) } if _, err := kv.Put(ctx, string(rev7PutFooBar.Kv.Key), string(rev7PutFooBar.Kv.Value)); err != nil { t.Fatalf("Put: %v", err) } if _, err := kv.Put(ctx, string(rev8PutFooBaz.Kv.Key), string(rev8PutFooBaz.Kv.Value)); err != nil { t.Fatalf("Put: %v", err) } if _, err := kv.Put(ctx, string(rev9PutFooYoo.Kv.Key), string(rev9PutFooYoo.Kv.Value)); err != nil { t.Fatalf("Put: %v", err) } if _, err := kv.Put(ctx, string(rev10PutZoo.Kv.Key), string(rev10PutZoo.Kv.Value)); err != nil { t.Fatalf("Put: %v", err) } t.Log("Open watches") watches := make([]clientv3.WatchChan, len(tcs)) for i, tc := range tcs { watches[i] = watcher.Watch(ctx, tc.key, tc.opts...) } time.Sleep(50 * time.Millisecond) t.Log("Write the second batch of events rev 11‑12") if _, err := kv.Put(ctx, string(rev11PutFooFuture.Kv.Key), string(rev11PutFooFuture.Kv.Value)); err != nil { t.Fatalf("Put /foo/future: %v", err) } if _, err := kv.Txn(ctx).Then( clientv3.OpPut(string(rev12PutFooTx1.Kv.Key), string(rev12PutFooTx1.Kv.Value)), clientv3.OpDelete(string(rev12DeleteFooFuture.Kv.Key)), clientv3.OpPut(string(rev12PutFooTx2.Kv.Key), string(rev12PutFooTx2.Kv.Value)), ).Commit(); err != nil { t.Fatalf("Txn rev12: %v", err) } t.Log("Validate") for i, tc := range tcs { i, tc := i, tc t.Run(tc.name, func(t *testing.T) { t.Parallel() events, _ := collectAndAssertAtomicEvents(t, watches[i]) if diff := cmp.Diff(tc.wantEvents, events); diff != "" { t.Errorf("unexpected events (-want +got):\n%s", diff) } }) } } func TestCacheWithPrefixWatch(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) client := clus.Client(0) ctx := t.Context() tests := []struct { name string key string opts []clientv3.OpOption expectCanceled bool }{ { name: "single key within prefix", key: "/foo/a", opts: nil, expectCanceled: false, }, { name: "single key outside prefix returns error", key: "/bar/a", opts: nil, expectCanceled: true, }, { name: "prefix() within cache prefix", key: "/foo", opts: []clientv3.OpOption{clientv3.WithPrefix()}, expectCanceled: false, }, { name: "prefix() outside cache prefix returns error", key: "/bar", opts: []clientv3.OpOption{clientv3.WithPrefix()}, expectCanceled: true, }, { name: "range within prefix", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithRange("/foo/b")}, expectCanceled: false, }, { name: "range crosses cache prefix boundary returns error", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithRange("/zzz")}, expectCanceled: true, }, { name: "fromKey not allowed when cache has prefix returns error", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithFromKey()}, expectCanceled: true, }, } const testPutKey = "/foo/a" for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { c, err := cache.New(client, "/foo") if err != nil { t.Fatalf("New(...): %v", err) } defer c.Close() if err := c.WaitReady(ctx); err != nil { t.Fatal(err) } watchCtx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() ch := c.Watch(watchCtx, tc.key, tc.opts...) if !tc.expectCanceled { if _, err := client.Put(ctx, testPutKey, "val"); err != nil { t.Fatalf("Put(%q): %v", testPutKey, err) } } select { case resp, ok := <-ch: if tc.expectCanceled { if !ok || !resp.Canceled { t.Fatalf("expected canceled watch, got %+v (closed=%v)", resp, !ok) } return } if !ok || resp.Canceled { t.Fatalf("expected active watch (not canceled), got %+v (closed=%v)", resp, !ok) } if len(resp.Events) == 0 { t.Fatalf("watch returned no events, expected at least the test event") } if string(resp.Events[0].Kv.Key) != testPutKey { t.Fatalf("got event for key %q, want %q", resp.Events[0].Kv.Key, testPutKey) } case <-watchCtx.Done(): if tc.expectCanceled { t.Fatalf("watch did not cancel within timeout") } else { t.Fatalf("active watch did not deliver event within timeout") } } }) } } func TestCacheWatchPrefixProgressNotify(t *testing.T) { if integration.ThroughProxy { t.Skip("grpc proxy currently does not support requesting progress notifications") } integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) client := clus.Client(0) ctx := t.Context() c, err := cache.New(client, "/foo") if err != nil { t.Fatalf("cache.New: %v", err) } t.Cleanup(c.Close) if err := c.WaitReady(ctx); err != nil { t.Fatalf("cache.WaitReady: %v", err) } wctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() watchCh := c.Watch(wctx, "/foo", clientv3.WithPrefix()) var latestRev int64 for i := 0; i < 5; i++ { resp, err := client.Put(ctx, fmt.Sprintf("/bar/out-%d", i), "v") if err != nil { t.Fatalf("Put(/bar/out-%d): %v", i, err) } latestRev = resp.Header.Revision } if err := client.RequestProgress(ctx); err != nil { t.Fatalf("RequestProgress: %v", err) } var progressRev int64 select { case resp, ok := <-watchCh: if !ok || resp.Canceled { t.Fatalf("expected active watch (not canceled), got %+v (closed=%v)", resp, !ok) } if len(resp.Events) != 0 { t.Fatalf("expected a progress notification (no events), got %d event(s)", len(resp.Events)) } if !resp.IsProgressNotify() { t.Fatalf("expected IsProgressNotify()==true, got false (resp: %+v)", resp) } progressRev = resp.Header.Revision if progressRev < latestRev { t.Fatalf("progress revision %d < latest outside-prefix rev %d", progressRev, latestRev) } case <-wctx.Done(): t.Fatalf("timed out waiting for progress notification: %v", wctx.Err()) } } func TestCacheWithoutPrefixGet(t *testing.T) { tcs := []struct { name string initialEvents, followupEvents []*clientv3.Event }{ {"watch-early (no pre-events)", nil, TestGetEvents}, {"watch-mid (partial pre-events)", filterEvents(TestGetEvents, revLessThan(4)), filterEvents(TestGetEvents, revGreaterEqual(4))}, {"watch-late (all pre-events)", TestGetEvents, nil}, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) client, kv := clus.Client(0), clus.Client(0).KV testGet(t, kv, func() Getter { c, err := cache.New(client, "") if err != nil { t.Fatalf("cache.New: %v", err) } t.Cleanup(c.Close) if err := c.WaitReady(t.Context()); err != nil { t.Fatalf("cache not ready: %v", err) } return c }, tc.initialEvents, tc.followupEvents) }) } } func TestGet(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) client := clus.Client(0) kv := client.KV testGet(t, kv, func() Getter { return kv }, TestGetEvents, nil) } func testGet(t *testing.T, kv clientv3.KV, getReader func() Getter, initialEvents, followupEvents []*clientv3.Event) { ctx := t.Context() t.Log("Setup") initialRev := applyEvents(ctx, t, kv, initialEvents) reader := getReader() if c, ok := reader.(*cache.Cache); ok { if err := c.WaitForRevision(ctx, initialRev); err != nil { t.Fatalf("cache never caught up to rev %d: %v", initialRev, err) } } followupRev := applyEvents(ctx, t, kv, followupEvents) if c, ok := reader.(*cache.Cache); ok { if err := c.WaitForRevision(ctx, followupRev); err != nil { t.Fatalf("cache never caught up to rev %d: %v", followupRev, err) } } t.Log("Validate") for _, tc := range getTestCases { tc := tc t.Run(tc.name, func(t *testing.T) { op := clientv3.OpGet(tc.key, tc.opts...) requestedRev := op.Rev() resp, err := reader.Get(ctx, tc.key, tc.opts...) if tc.expectErr != nil { if !errors.Is(err, tc.expectErr) { t.Fatalf("expected %v for Get %q; got %v", tc.expectErr, tc.key, err) } return } if err != nil { if _, ok := reader.(*cache.Cache); ok && requestedRev > 0 && requestedRev < initialRev && errors.Is(err, rpctypes.ErrCompacted) { t.Logf("expected ErrCompacted: requestedRev=%d < initialCompleteRev=%d", requestedRev, initialRev) return } t.Fatalf("Get %q failed: %v", tc.key, err) } if diff := cmp.Diff(tc.wantKVs, resp.Kvs); diff != "" { t.Fatalf("unexpected KVs (-want +got):\n%s", diff) } if resp.Header.Revision != tc.wantRevision { t.Fatalf("revision: got %d, want %d", resp.Header.Revision, tc.wantRevision) } }) } } var TestGetEvents = []*clientv3.Event{ Rev2PutFooA, Rev3PutFooB, Rev4PutFooC, Rev5PutFooD, Rev6DeleteFooD, Rev7TxnPutFooA, Rev7TxnPutFooB, Rev8PutFooA, } var ( Rev2PutFooA = &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/a"), Value: []byte("a1"), CreateRevision: 2, ModRevision: 2, Version: 1, }, } Rev3PutFooB = &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/b"), Value: []byte("b1"), CreateRevision: 3, ModRevision: 3, Version: 1, }, } Rev4PutFooC = &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/c"), Value: []byte("c1"), CreateRevision: 4, ModRevision: 4, Version: 1, }, } Rev5PutFooD = &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/d"), Value: []byte("d1"), CreateRevision: 5, ModRevision: 5, Version: 1, }, } Rev6DeleteFooD = &clientv3.Event{ Type: clientv3.EventTypeDelete, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/d"), ModRevision: 6, }, } Rev7TxnPutFooA = &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/a"), Value: []byte("a2"), CreateRevision: 2, ModRevision: 7, Version: 2, }, } Rev7TxnPutFooB = &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/b"), Value: []byte("b2"), CreateRevision: 3, ModRevision: 7, Version: 2, }, } Rev8PutFooA = &clientv3.Event{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{ Key: []byte("/foo/a"), Value: []byte("a3"), CreateRevision: 2, ModRevision: 8, Version: 3, }, } ) type getTestCase struct { name string key string opts []clientv3.OpOption wantKVs []*mvccpb.KeyValue wantRevision int64 expectErr error } var getTestCases = []getTestCase{ { name: "single key /foo/a", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable()}, wantKVs: []*mvccpb.KeyValue{Rev8PutFooA.Kv}, wantRevision: 8, }, { name: "single key /foo/a at rev=2", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRev(2)}, wantKVs: []*mvccpb.KeyValue{Rev2PutFooA.Kv}, wantRevision: 8, }, { name: "single key /foo/a at rev=7", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRev(7)}, wantKVs: []*mvccpb.KeyValue{Rev7TxnPutFooA.Kv}, wantRevision: 8, }, { name: "single key /foo/a at rev=8", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRev(8)}, wantKVs: []*mvccpb.KeyValue{Rev8PutFooA.Kv}, wantRevision: 8, }, { name: "single key /foo/a at rev=9 (future), returns error", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRev(9)}, expectErr: rpctypes.ErrFutureRev, }, { name: "non-existing key", key: "/doesnotexist", opts: []clientv3.OpOption{clientv3.WithSerializable()}, wantKVs: nil, wantRevision: 8, }, { name: "non-existing key at rev=4", key: "/doesnotexist", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRev(4)}, wantKVs: nil, wantRevision: 8, }, { name: "non-existing key at rev=9 (future), returns error", key: "/doesnotexist", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRev(9)}, expectErr: rpctypes.ErrFutureRev, }, { name: "prefix /foo", key: "/foo", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithPrefix()}, wantKVs: []*mvccpb.KeyValue{Rev8PutFooA.Kv, Rev7TxnPutFooB.Kv, Rev4PutFooC.Kv}, wantRevision: 8, }, { name: "prefix /foo at rev=5", key: "/foo", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithPrefix(), clientv3.WithRev(5)}, wantKVs: []*mvccpb.KeyValue{Rev2PutFooA.Kv, Rev3PutFooB.Kv, Rev4PutFooC.Kv, Rev5PutFooD.Kv}, wantRevision: 8, }, { name: "prefix /foo/b at rev=4", key: "/foo/b", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithPrefix(), clientv3.WithRev(4)}, wantKVs: []*mvccpb.KeyValue{Rev3PutFooB.Kv}, wantRevision: 8, }, { name: "prefix /foo/b at rev=7", key: "/foo/b", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithPrefix(), clientv3.WithRev(7)}, wantKVs: []*mvccpb.KeyValue{Rev7TxnPutFooB.Kv}, wantRevision: 8, }, { name: "prefix /foo at rev=9 (future), returns error", key: "/foo", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithPrefix(), clientv3.WithRev(9)}, wantKVs: []*mvccpb.KeyValue{Rev2PutFooA.Kv, Rev3PutFooB.Kv, Rev4PutFooC.Kv, Rev5PutFooD.Kv}, expectErr: rpctypes.ErrFutureRev, }, { name: "range [/foo/a, /foo/c)", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRange("/foo/c")}, wantKVs: []*mvccpb.KeyValue{Rev8PutFooA.Kv, Rev7TxnPutFooB.Kv}, wantRevision: 8, }, { name: "range [/foo/a, /foo/d) at rev=5", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRange("/foo/d"), clientv3.WithRev(5)}, wantKVs: []*mvccpb.KeyValue{Rev2PutFooA.Kv, Rev3PutFooB.Kv, Rev4PutFooC.Kv}, wantRevision: 8, }, { name: "range [/foo/a, /foo/c) at rev=9 (future), returns error", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRange("/foo/c"), clientv3.WithRev(9)}, expectErr: rpctypes.ErrFutureRev, }, { name: "fromKey /foo/b", key: "/foo/b", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithFromKey()}, wantKVs: []*mvccpb.KeyValue{Rev7TxnPutFooB.Kv, Rev4PutFooC.Kv}, wantRevision: 8, }, { name: "fromKey /foo/b at rev=7", key: "/foo/b", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithFromKey(), clientv3.WithRev(7)}, wantKVs: []*mvccpb.KeyValue{Rev7TxnPutFooB.Kv, Rev4PutFooC.Kv}, wantRevision: 8, }, { name: "fromKey /foo/b at rev=9 (future), returns error", key: "/foo/b", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithFromKey(), clientv3.WithRev(9)}, expectErr: rpctypes.ErrFutureRev, }, } func TestCacheWithPrefixGetInScope(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) cli := clus.Client(0) testWithPrefixGet(t, cli, func() Getter { c, err := cache.New(cli, "/foo") if err != nil { t.Fatalf("cache.New: %v", err) } t.Cleanup(c.Close) if err := c.WaitReady(t.Context()); err != nil { t.Fatalf("cache.WaitReady: %v", err) } return c }) } func TestWithPrefixGet(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) cli := clus.Client(0) testWithPrefixGet(t, cli, func() Getter { return cli.KV }) } func testWithPrefixGet(t *testing.T, cli *clientv3.Client, getReader func() Getter) { ctx := t.Context() seedResp, err := cli.Put(ctx, "/foo/a", "val") if err != nil { t.Fatalf("seed put: %v", err) } seedRev := seedResp.Header.Revision reader := getReader() var latestRev int64 for i := 0; i < 5; i++ { r, err := cli.Put(ctx, fmt.Sprintf("/bar/x%d", i), fmt.Sprintf("%d", i)) if err != nil { t.Fatalf("advance put: %v", err) } latestRev = r.Header.Revision } if err := cli.RequestProgress(ctx); err != nil { t.Fatalf("RequestProgress: %v", err) } if c, ok := reader.(*cache.Cache); ok { if err := c.WaitForRevision(ctx, latestRev); err != nil { t.Fatalf("cache didn’t observe progress to rev %d: %v", latestRev, err) } } expectedFooA := &mvccpb.KeyValue{ Key: []byte("/foo/a"), Value: []byte("val"), CreateRevision: seedRev, ModRevision: seedRev, Version: 1, } testCases := []struct { name string key string opts []clientv3.OpOption wantKVs []*mvccpb.KeyValue wantRevision int64 }{ { name: "single key within cache prefix", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable()}, wantKVs: []*mvccpb.KeyValue{expectedFooA}, wantRevision: latestRev, }, { name: "single key within cache prefix at latest/progress rev", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRev(latestRev)}, wantKVs: []*mvccpb.KeyValue{expectedFooA}, wantRevision: latestRev, }, { name: "prefix query within cache prefix", key: "/foo", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithPrefix()}, wantKVs: []*mvccpb.KeyValue{expectedFooA}, wantRevision: latestRev, }, { name: "prefix query within cache prefix at latest/progress rev", key: "/foo", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithPrefix(), clientv3.WithRev(latestRev)}, wantKVs: []*mvccpb.KeyValue{expectedFooA}, wantRevision: latestRev, }, { name: "range query within cache prefix", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRange("/foo/b")}, wantKVs: []*mvccpb.KeyValue{expectedFooA}, wantRevision: latestRev, }, { name: "range query within cache prefix at latest/progress rev", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRange("/foo/z"), clientv3.WithRev(latestRev)}, wantKVs: []*mvccpb.KeyValue{expectedFooA}, wantRevision: latestRev, }, } for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { resp, err := reader.Get(ctx, tc.key, tc.opts...) if err != nil { t.Fatalf("Get(%q): %v", tc.key, err) } if diff := cmp.Diff(tc.wantKVs, resp.Kvs); diff != "" { t.Errorf("unexpected KVs (-want +got):\n%s", diff) } if resp.Header.Revision != tc.wantRevision { t.Errorf("Header.Revision=%d; want: %d", resp.Header.Revision, tc.wantRevision) } }) } } func TestCacheWithPrefixGetOutOfScope(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) cli := clus.Client(0) c, err := cache.New(cli, "/foo") if err != nil { t.Fatalf("cache.New: %v", err) } defer c.Close() ctx := t.Context() if err := c.WaitReady(ctx); err != nil { t.Fatalf("cache.WaitReady: %v", err) } cases := []struct { name string key string opts []clientv3.OpOption }{ { name: "single key outside prefix", key: "/bar/a", opts: []clientv3.OpOption{clientv3.WithSerializable()}, }, { name: "prefix() outside cache prefix", key: "/bar", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithPrefix()}, }, { name: "range crossing cache boundary", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithRange("/zzz")}, }, { name: "fromKey disallowed with cache prefix", key: "/foo/a", opts: []clientv3.OpOption{clientv3.WithSerializable(), clientv3.WithFromKey()}, }, } for _, tc := range cases { tc := tc t.Run(tc.name, func(t *testing.T) { _, err := c.Get(ctx, tc.key, tc.opts...) if !errors.Is(err, cache.ErrKeyRangeInvalid) { t.Fatalf("expected ErrKeyRangeInvalid; got %v", err) } }) } } func TestCacheLaggingWatcher(t *testing.T) { const prefix = "/test/" integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) client := clus.Client(0) tests := []struct { name string window int eventCount int wantExactEventCount int wantAtMaxEventCount int wantClosed bool }{ { name: "all event fit", window: 10, eventCount: 9, wantExactEventCount: 9, wantClosed: false, }, { name: "events fill window", window: 10, eventCount: 10, wantExactEventCount: 10, wantClosed: false, }, { name: "event fill pipeline", window: 10, eventCount: 11, wantExactEventCount: 11, wantClosed: false, }, { name: "pipeline overflow", window: 10, eventCount: 12, wantAtMaxEventCount: 1, // Either 0 or 1. wantClosed: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctx := t.Context() c, err := cache.New( client, prefix, cache.WithHistoryWindowSize(tt.window), cache.WithPerWatcherBufferSize(0), cache.WithResyncInterval(10*time.Millisecond), ) if err != nil { t.Fatalf("New(...): %v", err) } defer c.Close() if err := c.WaitReady(ctx); err != nil { t.Fatalf("cache not ready: %v", err) } ch := c.Watch(ctx, prefix, clientv3.WithPrefix()) if err := c.WaitForNextResync(ctx); err != nil { t.Fatalf("cache not synced: %v", err) } generateEvents(t, client, prefix, tt.eventCount) if err := c.WaitForNextResync(ctx); err != nil { t.Fatalf("cache not synced: %v", err) } gotEvents, ok := collectAndAssertAtomicEvents(t, ch) closed := !ok if tt.wantExactEventCount != 0 && tt.wantExactEventCount != len(gotEvents) { t.Errorf("gotEvents=%v, wantEvents=%v", len(gotEvents), tt.wantExactEventCount) } if tt.wantAtMaxEventCount != 0 && len(gotEvents) > tt.wantAtMaxEventCount { t.Errorf("gotEvents=%v, wantEvents<%v", len(gotEvents), tt.wantAtMaxEventCount) } if closed != tt.wantClosed { t.Errorf("closed=%v, wantClosed=%v", closed, tt.wantClosed) } }) } } func TestCacheUnsupportedWatchOptions(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) client := clus.Client(0) c, err := cache.New(client, "", cache.WithHistoryWindowSize(1)) if err != nil { t.Fatalf("cache.New: %v", err) } defer c.Close() if err := c.WaitReady(t.Context()); err != nil { t.Fatalf("cache not ready: %v", err) } unsupported := []struct { name string opt clientv3.OpOption }{ {"WithPrevKV", clientv3.WithPrevKV()}, {"WithFragment", clientv3.WithFragment()}, {"WithProgressNotify", clientv3.WithProgressNotify()}, {"WithCreatedNotify", clientv3.WithCreatedNotify()}, {"WithFilterPut", clientv3.WithFilterPut()}, {"WithFilterDelete", clientv3.WithFilterDelete()}, } for _, tc := range unsupported { tc := tc t.Run(tc.name, func(t *testing.T) { ch := c.Watch(t.Context(), "foo", tc.opt) resp, ok := <-ch if !ok { t.Fatalf("channel closed without yielding a response") } if !resp.Canceled { t.Errorf("expected Canceled=true, got %+v", resp) } if !strings.Contains(resp.Err().Error(), cache.ErrUnsupportedRequest.Error()) { t.Errorf("expected ErrUnsupportedWatch text %q, got %v", cache.ErrUnsupportedRequest.Error(), resp.Err()) } }) } } func TestCacheUnsupportedGetOptions(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) client := clus.Client(0) c, err := cache.New(client, "", cache.WithHistoryWindowSize(1)) if err != nil { t.Fatalf("cache.New: %v", err) } defer c.Close() if err := c.WaitReady(t.Context()); err != nil { t.Fatalf("cache not ready: %v", err) } unsupported := []struct { name string opts []clientv3.OpOption }{ {"WithCountOnly", []clientv3.OpOption{clientv3.WithCountOnly()}}, {"WithLimit", []clientv3.OpOption{clientv3.WithLimit(1)}}, {"WithSort", []clientv3.OpOption{clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)}}, {"WithPrevKV", []clientv3.OpOption{clientv3.WithPrevKV()}}, {"WithMinModRevision", []clientv3.OpOption{clientv3.WithMinModRev(2)}}, {"WithMaxModRevision", []clientv3.OpOption{clientv3.WithMaxModRev(10)}}, {"WithMinCreateRevision", []clientv3.OpOption{clientv3.WithMinCreateRev(3)}}, {"WithMaxCreateRevision", []clientv3.OpOption{clientv3.WithMaxCreateRev(5)}}, {"NoSerializable", nil}, } for _, tc := range unsupported { tc := tc t.Run(tc.name, func(t *testing.T) { _, err := c.Get(t.Context(), "foo", tc.opts...) if !errors.Is(err, cache.ErrUnsupportedRequest) { t.Errorf("Get with %s: expected ErrUnsupportedRequest, got %v", tc.name, err) } }) } } func generateEvents(t *testing.T, client *clientv3.Client, prefix string, n int) { t.Helper() for i := 0; i < n; i++ { key := fmt.Sprintf("%s%d", prefix, i) if _, err := client.Put(t.Context(), key, fmt.Sprintf("%d", i)); err != nil { t.Fatalf("Put(%q): %v", key, err) } } } type Watcher interface { Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan } type Getter interface { Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) } func collectAndAssertAtomicEvents(t *testing.T, watch clientv3.WatchChan) (events []*clientv3.Event, ok bool) { deadline := time.After(time.Second) var lastRevision int64 for { select { case resp, ok := <-watch: if !ok { return events, false } if len(resp.Events) != 0 && resp.Events[0].Kv.ModRevision == lastRevision { t.Fatalf("same revision found as in previous response: %d", lastRevision) } for _, ev := range resp.Events { if ev.Kv.ModRevision < lastRevision { t.Fatalf("revision went backwards: last %d, now %d", lastRevision, ev.Kv.ModRevision) } events = append(events, ev) lastRevision = ev.Kv.ModRevision } case <-deadline: return events, true case <-time.After(100 * time.Millisecond): return events, true } } } func applyEvents(ctx context.Context, t *testing.T, kv clientv3.KV, evs []*clientv3.Event) int64 { var lastRev int64 for _, batches := range batchEventsByRevision(evs) { lastRev = applyEventBatch(ctx, t, kv, batches) } return lastRev } func batchEventsByRevision(events []*clientv3.Event) [][]*clientv3.Event { var batches [][]*clientv3.Event if len(events) == 0 { return batches } start := 0 for end := 1; end < len(events); end++ { if events[end].Kv.ModRevision != events[start].Kv.ModRevision { batches = append(batches, events[start:end]) start = end } } batches = append(batches, events[start:]) return batches } func applyEventBatch(ctx context.Context, t *testing.T, kv clientv3.KV, batch []*clientv3.Event) int64 { ops := make([]clientv3.Op, 0, len(batch)) for _, event := range batch { switch event.Type { case clientv3.EventTypePut: ops = append(ops, clientv3.OpPut(string(event.Kv.Key), string(event.Kv.Value))) case clientv3.EventTypeDelete: ops = append(ops, clientv3.OpDelete(string(event.Kv.Key))) default: t.Fatalf("unsupported event type: %v", event.Type) } } resp, err := kv.Txn(ctx).Then(ops...).Commit() if err != nil { t.Fatalf("Txn failed: %v", err) } return resp.Header.Revision } func filterEvents(evs []*clientv3.Event, pred func(int64) bool) []*clientv3.Event { var out []*clientv3.Event for _, ev := range evs { if pred(ev.Kv.ModRevision) { out = append(out, ev) } } return out } func revLessThan(n int64) func(int64) bool { return func(r int64) bool { return r < n } } func revGreaterEqual(n int64) func(int64) bool { return func(r int64) bool { return r >= n } } ================================================ FILE: tests/integration/clientv3/cluster_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "context" "fmt" "math/rand" "reflect" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestMemberList(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) capi := clus.RandClient() resp, err := capi.MemberList(t.Context()) if err != nil { t.Fatalf("failed to list member %v", err) } if len(resp.Members) != 3 { t.Errorf("number of members = %d, want %d", len(resp.Members), 3) } } func TestMemberAdd(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) defer clus.Terminate(t) capi := clus.RandClient() urls := []string{"http://127.0.0.1:1234"} resp, err := capi.MemberAdd(t.Context(), urls) if err != nil { t.Fatalf("failed to add member %v", err) } if !reflect.DeepEqual(resp.Member.PeerURLs, urls) { t.Errorf("urls = %v, want %v", urls, resp.Member.PeerURLs) } } func TestMemberAddWithExistingURLs(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) defer clus.Terminate(t) capi := clus.RandClient() resp, err := capi.MemberList(t.Context()) if err != nil { t.Fatalf("failed to list member %v", err) } existingURL := resp.Members[0].PeerURLs[0] _, err = capi.MemberAdd(t.Context(), []string{existingURL}) expectedErrKeywords := "Peer URLs already exists" if err == nil { t.Fatalf("expecting add member to fail, got no error") } if !strings.Contains(err.Error(), expectedErrKeywords) { t.Errorf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error()) } } func TestMemberRemove(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) defer clus.Terminate(t) capi := clus.Client(1) resp, err := capi.MemberList(t.Context()) if err != nil { t.Fatalf("failed to list member %v", err) } rmvID := resp.Members[0].ID // indexes in capi member list don't necessarily match cluster member list; // find member that is not the client to remove for _, m := range resp.Members { mURLs, _ := types.NewURLs(m.PeerURLs) if !reflect.DeepEqual(mURLs, clus.Members[1].ServerConfig.PeerURLs) { rmvID = m.ID break } } _, err = capi.MemberRemove(t.Context(), rmvID) if err != nil { t.Fatalf("failed to remove member %v", err) } resp, err = capi.MemberList(t.Context()) if err != nil { t.Fatalf("failed to list member %v", err) } if len(resp.Members) != 2 { t.Errorf("number of members = %d, want %d", len(resp.Members), 2) } } func TestMemberUpdate(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) capi := clus.RandClient() resp, err := capi.MemberList(t.Context()) if err != nil { t.Fatalf("failed to list member %v", err) } urls := []string{"http://127.0.0.1:1234"} _, err = capi.MemberUpdate(t.Context(), resp.Members[0].ID, urls) if err != nil { t.Fatalf("failed to update member %v", err) } resp, err = capi.MemberList(t.Context()) if err != nil { t.Fatalf("failed to list member %v", err) } if !reflect.DeepEqual(resp.Members[0].PeerURLs, urls) { t.Errorf("urls = %v, want %v", urls, resp.Members[0].PeerURLs) } } func TestMemberAddUpdateWrongURLs(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) capi := clus.RandClient() tt := [][]string{ // missing protocol scheme {"://127.0.0.1:2379"}, // unsupported scheme {"mailto://127.0.0.1:2379"}, // not conform to host:port {"http://127.0.0.1"}, // contain a path {"http://127.0.0.1:2379/path"}, // first path segment in URL cannot contain colon {"127.0.0.1:1234"}, // URL scheme must be http, https, unix, or unixs {"localhost:1234"}, } for i := range tt { _, err := capi.MemberAdd(t.Context(), tt[i]) if err == nil { t.Errorf("#%d: MemberAdd err = nil, but error", i) } _, err = capi.MemberUpdate(t.Context(), 0, tt[i]) if err == nil { t.Errorf("#%d: MemberUpdate err = nil, but error", i) } } } func TestMemberAddForLearner(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) defer clus.Terminate(t) capi := clus.RandClient() urls := []string{"http://127.0.0.1:1234"} resp, err := capi.MemberAddAsLearner(t.Context(), urls) if err != nil { t.Fatalf("failed to add member %v", err) } if !resp.Member.IsLearner { t.Errorf("Added a member as learner, got resp.Member.IsLearner = %v", resp.Member.IsLearner) } numberOfLearners := 0 for _, m := range resp.Members { if m.IsLearner { numberOfLearners++ } } if numberOfLearners != 1 { t.Errorf("Added 1 learner node to cluster, got %d", numberOfLearners) } } func TestMemberPromote(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) defer clus.Terminate(t) // member promote request can be sent to any server in cluster, // the request will be auto-forwarded to leader on server-side. // This test explicitly includes the server-side forwarding by // sending the request to follower. leaderIdx := clus.WaitLeader(t) followerIdx := (leaderIdx + 1) % 3 capi := clus.Client(followerIdx) learnerMember := clus.MustNewMember(t) urls := learnerMember.PeerURLs.StringSlice() memberAddResp, err := capi.MemberAddAsLearner(t.Context(), urls) if err != nil { t.Fatalf("failed to add member %v", err) } if !memberAddResp.Member.IsLearner { t.Fatalf("Added a member as learner, got resp.Member.IsLearner = %v", memberAddResp.Member.IsLearner) } learnerID := memberAddResp.Member.ID numberOfLearners := 0 for _, m := range memberAddResp.Members { if m.IsLearner { numberOfLearners++ } } if numberOfLearners != 1 { t.Fatalf("Added 1 learner node to cluster, got %d", numberOfLearners) } // learner is not started yet. Expect learner progress check to fail. // As the result, member promote request will fail. _, err = capi.MemberPromote(t.Context(), learnerID) expectedErrKeywords := "can only promote a learner member which is in sync with leader" if err == nil { t.Fatalf("expecting promote not ready learner to fail, got no error") } if !strings.Contains(err.Error(), expectedErrKeywords) { t.Fatalf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error()) } // Initialize and launch learner member based on the response of V3 Member Add API. // (the response has information on peer urls of the existing members in cluster) clus.InitializeMemberWithResponse(t, learnerMember, memberAddResp) require.NoError(t, learnerMember.Launch()) // retry until promote succeed or timeout timeout := time.After(5 * time.Second) for { select { case <-time.After(500 * time.Millisecond): case <-timeout: t.Fatalf("failed all attempts to promote learner member, last error: %v", err) } _, err = capi.MemberPromote(t.Context(), learnerID) // successfully promoted learner if err == nil { break } // if member promote fails due to learner not ready, retry. // otherwise fails the test. if !strings.Contains(err.Error(), expectedErrKeywords) { t.Fatalf("unexpected error when promoting learner member: %v", err) } } } // TestMemberPromoteMemberNotLearner ensures that promoting a voting member fails. func TestMemberPromoteMemberNotLearner(t *testing.T) { integration.BeforeTest(t, integration.WithFailpoint("raftBeforeAdvance", `sleep(100)`)) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) // member promote request can be sent to any server in cluster, // the request will be auto-forwarded to leader on server-side. // This test explicitly includes the server-side forwarding by // sending the request to follower. leaderIdx := clus.WaitLeader(t) followerIdx := (leaderIdx + 1) % 3 cli := clus.Client(followerIdx) resp, err := cli.MemberList(t.Context()) if err != nil { t.Fatalf("failed to list member %v", err) } if len(resp.Members) != 3 { t.Fatalf("number of members = %d, want %d", len(resp.Members), 3) } // promoting any of the voting members in cluster should fail expectedErrKeywords := "can only promote a learner member" for _, m := range resp.Members { _, err = cli.MemberPromote(t.Context(), m.ID) if err == nil { t.Fatalf("expect promoting voting member to fail, got no error") } if !strings.Contains(err.Error(), expectedErrKeywords) { t.Fatalf("expect error to contain %s, got %s", expectedErrKeywords, err.Error()) } } } // TestMemberPromoteMemberNotExist ensures that promoting a member that does not exist in cluster fails. func TestMemberPromoteMemberNotExist(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) // member promote request can be sent to any server in cluster, // the request will be auto-forwarded to leader on server-side. // This test explicitly includes the server-side forwarding by // sending the request to follower. leaderIdx := clus.WaitLeader(t) followerIdx := (leaderIdx + 1) % 3 cli := clus.Client(followerIdx) resp, err := cli.MemberList(t.Context()) if err != nil { t.Fatalf("failed to list member %v", err) } if len(resp.Members) != 3 { t.Fatalf("number of members = %d, want %d", len(resp.Members), 3) } // generate an random ID that does not exist in cluster var randID uint64 for { randID = rand.Uint64() notExist := true for _, m := range resp.Members { if m.ID == randID { notExist = false break } } if notExist { break } } expectedErrKeywords := "member not found" _, err = cli.MemberPromote(t.Context(), randID) if err == nil { t.Fatalf("expect promoting voting member to fail, got no error") } if !strings.Contains(err.Error(), expectedErrKeywords) { t.Errorf("expect error to contain %s, got %s", expectedErrKeywords, err.Error()) } } // TestMaxLearnerInCluster verifies that the maximum number of learners allowed in a cluster func TestMaxLearnerInCluster(t *testing.T) { integration.BeforeTest(t, integration.WithFailpoint("raftBeforeAdvance", `sleep(100)`)) // 1. start with a cluster with 3 voting member and max learner 2 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, MaxLearners: 2, DisableStrictReconfigCheck: true}) defer clus.Terminate(t) // 2. adding 2 learner members should succeed for i := 0; i < 2; i++ { ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) _, err := clus.Client(0).MemberAddAsLearner(ctx, []string{fmt.Sprintf("http://127.0.0.1:123%d", i)}) cancel() if err != nil { t.Fatalf("failed to add learner member %v", err) } } // ensure client endpoint is voting member leaderIdx := clus.WaitLeader(t) capi := clus.Client(leaderIdx) resp1, err := capi.MemberList(t.Context()) if err != nil { t.Fatalf("failed to get member list") } numberOfLearners := 0 for _, m := range resp1.Members { if m.IsLearner { numberOfLearners++ } } if numberOfLearners != 2 { t.Fatalf("added 2 learner node to cluster, got %d", numberOfLearners) } // 3. cluster has 3 voting member and 2 learner, adding another learner should fail _, err = clus.Client(0).MemberAddAsLearner(t.Context(), []string{"http://127.0.0.1:2342"}) if err == nil { t.Fatalf("expect member add to fail, got no error") } expectedErrKeywords := "too many learner members in cluster" if !strings.Contains(err.Error(), expectedErrKeywords) { t.Fatalf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error()) } // 4. cluster has 3 voting member and 1 learner, adding a voting member should succeed _, err = clus.Client(0).MemberAdd(t.Context(), []string{"http://127.0.0.1:3453"}) if err != nil { t.Errorf("failed to add member %v", err) } } ================================================ FILE: tests/integration/clientv3/concurrency/election_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency_test import ( "context" "log" "strings" "testing" "time" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestResumeElection(t *testing.T) { const prefix = "/resume-election/" cli, err := integration.NewClient(t, clientv3.Config{Endpoints: exampleEndpoints()}) if err != nil { log.Fatal(err) } defer cli.Close() var s *concurrency.Session s, err = concurrency.NewSession(cli) if err != nil { log.Fatal(err) } defer s.Close() e := concurrency.NewElection(s, prefix) // entire test should never take more than 10 seconds ctx, cancel := context.WithTimeout(t.Context(), time.Second*10) defer cancel() // become leader require.NoErrorf(t, e.Campaign(ctx, "candidate1"), "Campaign() returned non nil err") // get the leadership details of the current election var leader *clientv3.GetResponse leader, err = e.Leader(ctx) require.NoErrorf(t, err, "Leader() returned non nil err") // Recreate the election e = concurrency.ResumeElection(s, prefix, string(leader.Kvs[0].Key), leader.Kvs[0].CreateRevision) respChan := make(chan *clientv3.GetResponse) go func() { defer close(respChan) o := e.Observe(ctx) respChan <- nil for resp := range o { // Ignore any observations that candidate1 was elected if string(resp.Kvs[0].Value) == "candidate1" { continue } respChan <- &resp return } t.Error("Observe() channel closed prematurely") }() // wait until observe goroutine is running <-respChan // put some random data to generate a change event, this put should be // ignored by Observe() because it is not under the election prefix. _, err = cli.Put(ctx, "foo", "bar") require.NoErrorf(t, err, "Put('foo') returned non nil err") // resign as leader require.NoErrorf(t, e.Resign(ctx), "Resign() returned non nil err") // elect a different candidate require.NoErrorf(t, e.Campaign(ctx, "candidate2"), "Campaign() returned non nil err") // wait for observed leader change resp := <-respChan kv := resp.Kvs[0] if !strings.HasPrefix(string(kv.Key), prefix) { t.Errorf("expected observed election to have prefix '%s' got %q", prefix, string(kv.Key)) } if string(kv.Value) != "candidate2" { t.Errorf("expected new leader to be 'candidate1' got %q", string(kv.Value)) } } ================================================ FILE: tests/integration/clientv3/concurrency/example_election_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency_test import ( "context" "fmt" "log" "sync" "time" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" ) func mockElectionCampaign() { fmt.Println("completed first election with e2") fmt.Println("completed second election with e1") } func ExampleElection_Campaign() { forUnitTestsRunInMockedContext( mockElectionCampaign, func() { cli, err := clientv3.New(clientv3.Config{Endpoints: exampleEndpoints()}) if err != nil { log.Fatal(err) } defer cli.Close() // create two separate sessions for election competition s1, err := concurrency.NewSession(cli) if err != nil { log.Fatal(err) } defer s1.Close() e1 := concurrency.NewElection(s1, "/my-election/") s2, err := concurrency.NewSession(cli) if err != nil { log.Fatal(err) } defer s2.Close() e2 := concurrency.NewElection(s2, "/my-election/") // create competing candidates, with e1 initially losing to e2 var wg sync.WaitGroup wg.Add(2) electc := make(chan *concurrency.Election, 2) go func() { defer wg.Done() // delay candidacy so e2 wins first time.Sleep(3 * time.Second) if err := e1.Campaign(context.Background(), "e1"); err != nil { log.Fatal(err) } electc <- e1 }() go func() { defer wg.Done() if err := e2.Campaign(context.Background(), "e2"); err != nil { log.Fatal(err) } electc <- e2 }() cctx, cancel := context.WithCancel(context.TODO()) defer cancel() e := <-electc fmt.Println("completed first election with", string((<-e.Observe(cctx)).Kvs[0].Value)) // resign so next candidate can be elected if err := e.Resign(context.TODO()); err != nil { log.Fatal(err) } e = <-electc fmt.Println("completed second election with", string((<-e.Observe(cctx)).Kvs[0].Value)) wg.Wait() }) // Output: // completed first election with e2 // completed second election with e1 } ================================================ FILE: tests/integration/clientv3/concurrency/example_mutex_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency_test import ( "context" "errors" "fmt" "log" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" ) func mockMutexTryLock() { fmt.Println("acquired lock for s1") fmt.Println("cannot acquire lock for s2, as already locked in another session") fmt.Println("released lock for s1") fmt.Println("acquired lock for s2") } func ExampleMutex_TryLock() { forUnitTestsRunInMockedContext( mockMutexTryLock, func() { cli, err := clientv3.New(clientv3.Config{Endpoints: exampleEndpoints()}) if err != nil { log.Fatal(err) } defer cli.Close() // create two separate sessions for lock competition s1, err := concurrency.NewSession(cli) if err != nil { log.Fatal(err) } defer s1.Close() m1 := concurrency.NewMutex(s1, "/my-lock") s2, err := concurrency.NewSession(cli) if err != nil { log.Fatal(err) } defer s2.Close() m2 := concurrency.NewMutex(s2, "/my-lock") // acquire lock for s1 if err = m1.Lock(context.TODO()); err != nil { log.Fatal(err) } fmt.Println("acquired lock for s1") if err = m2.TryLock(context.TODO()); err == nil { log.Fatal("should not acquire lock") } if errors.Is(err, concurrency.ErrLocked) { fmt.Println("cannot acquire lock for s2, as already locked in another session") } if err = m1.Unlock(context.TODO()); err != nil { log.Fatal(err) } fmt.Println("released lock for s1") if err = m2.TryLock(context.TODO()); err != nil { log.Fatal(err) } fmt.Println("acquired lock for s2") }) // Output: // acquired lock for s1 // cannot acquire lock for s2, as already locked in another session // released lock for s1 // acquired lock for s2 } func mockMutexLock() { fmt.Println("acquired lock for s1") fmt.Println("released lock for s1") fmt.Println("acquired lock for s2") } func ExampleMutex_Lock() { forUnitTestsRunInMockedContext( mockMutexLock, func() { cli, err := clientv3.New(clientv3.Config{Endpoints: exampleEndpoints()}) if err != nil { log.Fatal(err) } defer cli.Close() // create two separate sessions for lock competition s1, err := concurrency.NewSession(cli) if err != nil { log.Fatal(err) } defer s1.Close() m1 := concurrency.NewMutex(s1, "/my-lock/") s2, err := concurrency.NewSession(cli) if err != nil { log.Fatal(err) } defer s2.Close() m2 := concurrency.NewMutex(s2, "/my-lock/") // acquire lock for s1 if err := m1.Lock(context.TODO()); err != nil { log.Fatal(err) } fmt.Println("acquired lock for s1") m2Locked := make(chan struct{}) go func() { defer close(m2Locked) // wait until s1 is locks /my-lock/ if err := m2.Lock(context.TODO()); err != nil { log.Fatal(err) } }() if err := m1.Unlock(context.TODO()); err != nil { log.Fatal(err) } fmt.Println("released lock for s1") <-m2Locked fmt.Println("acquired lock for s2") }) // Output: // acquired lock for s1 // released lock for s1 // acquired lock for s2 } ================================================ FILE: tests/integration/clientv3/concurrency/example_stm_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency_test import ( "context" "fmt" "log" "math/rand" "sync" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" ) func mockSTMApply() { fmt.Println("account sum is 500") } // ExampleSTM_apply shows how to use STM with a transactional // transfer between balances. func ExampleSTM_apply() { forUnitTestsRunInMockedContext( mockSTMApply, func() { cli, err := clientv3.New(clientv3.Config{Endpoints: exampleEndpoints()}) if err != nil { log.Fatal(err) } defer cli.Close() // set up "accounts" totalAccounts := 5 for i := 0; i < totalAccounts; i++ { k := fmt.Sprintf("accts/%d", i) if _, err = cli.Put(context.TODO(), k, "100"); err != nil { log.Fatal(err) } } exchange := func(stm concurrency.STM) { from, to := rand.Intn(totalAccounts), rand.Intn(totalAccounts) if from == to { // nothing to do return } // read values fromK, toK := fmt.Sprintf("accts/%d", from), fmt.Sprintf("accts/%d", to) fromV, toV := stm.Get(fromK), stm.Get(toK) fromInt, toInt := 0, 0 fmt.Sscanf(fromV, "%d", &fromInt) fmt.Sscanf(toV, "%d", &toInt) // transfer amount xfer := fromInt / 2 fromInt, toInt = fromInt-xfer, toInt+xfer // write back stm.Put(fromK, fmt.Sprintf("%d", fromInt)) stm.Put(toK, fmt.Sprintf("%d", toInt)) } // concurrently exchange values between accounts var wg sync.WaitGroup wg.Add(10) for i := 0; i < 10; i++ { go func() { defer wg.Done() if _, serr := concurrency.NewSTM(cli, func(stm concurrency.STM) error { exchange(stm) return nil }); serr != nil { log.Fatal(serr) } }() } wg.Wait() // confirm account sum matches sum from beginning. sum := 0 accts, err := cli.Get(context.TODO(), "accts/", clientv3.WithPrefix()) if err != nil { log.Fatal(err) } for _, kv := range accts.Kvs { v := 0 fmt.Sscanf(string(kv.Value), "%d", &v) sum += v } fmt.Println("account sum is", sum) }) // Output: // account sum is 500 } ================================================ FILE: tests/integration/clientv3/concurrency/main_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency_test import ( "os" "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/tests/v3/integration" ) var lazyCluster = integration.NewLazyCluster() func exampleEndpoints() []string { return lazyCluster.EndpointsGRPC() } func forUnitTestsRunInMockedContext(_mocking func(), example func()) { // For integration tests runs in the provided environment example() } // TestMain sets up an etcd cluster if running the examples. func TestMain(m *testing.M) { cleanup := testutil.BeforeIntegrationExamples(m) v := m.Run() lazyCluster.Terminate() if v == 0 { testutil.MustCheckLeakedGoroutine() } cleanup() os.Exit(v) } ================================================ FILE: tests/integration/clientv3/concurrency/mutex_test.go ================================================ // Copyright 2019 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency_test import ( "errors" "testing" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestMutexLockSessionExpired(t *testing.T) { cli, err := integration.NewClient(t, clientv3.Config{Endpoints: exampleEndpoints()}) require.NoError(t, err) defer cli.Close() // create two separate sessions for lock competition s1, err := concurrency.NewSession(cli) require.NoError(t, err) defer s1.Close() m1 := concurrency.NewMutex(s1, "/my-lock/") s2, err := concurrency.NewSession(cli) require.NoError(t, err) m2 := concurrency.NewMutex(s2, "/my-lock/") // acquire lock for s1 require.NoError(t, m1.Lock(t.Context())) m2Locked := make(chan struct{}) var err2 error go func() { defer close(m2Locked) // m2 blocks since m1 already acquired lock /my-lock/ if err2 = m2.Lock(t.Context()); err2 == nil { t.Error("expect session expired error") } }() // revoke the session of m2 before unlock m1 require.NoError(t, s2.Close()) require.NoError(t, m1.Unlock(t.Context())) <-m2Locked } func TestMutexUnlock(t *testing.T) { cli, err := integration.NewClient(t, clientv3.Config{Endpoints: exampleEndpoints()}) require.NoError(t, err) defer cli.Close() s1, err := concurrency.NewSession(cli) require.NoError(t, err) defer s1.Close() m1 := concurrency.NewMutex(s1, "/my-lock/") err = m1.Unlock(t.Context()) require.Errorf(t, err, "expect lock released error") if !errors.Is(err, concurrency.ErrLockReleased) { t.Fatal(err) } require.NoError(t, m1.Lock(t.Context())) require.NoError(t, m1.Unlock(t.Context())) err = m1.Unlock(t.Context()) if err == nil { t.Fatal("expect lock released error") } if !errors.Is(err, concurrency.ErrLockReleased) { t.Fatal(err) } } ================================================ FILE: tests/integration/clientv3/concurrency/session_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package concurrency_test import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestSessionOptions(t *testing.T) { cli, err := integration.NewClient(t, clientv3.Config{Endpoints: exampleEndpoints()}) require.NoError(t, err) defer cli.Close() lease, err := cli.Grant(t.Context(), 100) require.NoError(t, err) s, err := concurrency.NewSession(cli, concurrency.WithLease(lease.ID)) require.NoError(t, err) defer s.Close() assert.Equal(t, s.Lease(), lease.ID) go s.Orphan() select { case <-s.Done(): case <-time.After(time.Millisecond * 100): t.Fatal("session did not get orphaned as expected") } } func TestSessionTTLOptions(t *testing.T) { cli, err := integration.NewClient(t, clientv3.Config{Endpoints: exampleEndpoints()}) require.NoError(t, err) defer cli.Close() setTTL := 90 s, err := concurrency.NewSession(cli, concurrency.WithTTL(setTTL)) require.NoError(t, err) defer s.Close() leaseID := s.Lease() // TTL retrieved should be less than the set TTL, but not equal to default:60 or exprired:-1 resp, err := cli.Lease.TimeToLive(t.Context(), leaseID) if err != nil { t.Log(err) } if resp.TTL == -1 { t.Errorf("client lease should not be expired: %d", resp.TTL) } if resp.TTL == 60 { t.Errorf("default TTL value is used in the session, instead of set TTL: %d", setTTL) } if resp.TTL >= int64(setTTL) || resp.TTL < int64(setTTL)-20 { t.Errorf("Session TTL from lease should be less, but close to set TTL %d, have: %d", setTTL, resp.TTL) } } func TestSessionCtx(t *testing.T) { cli, err := integration.NewClient(t, clientv3.Config{Endpoints: exampleEndpoints()}) require.NoError(t, err) defer cli.Close() lease, err := cli.Grant(t.Context(), 100) require.NoError(t, err) s, err := concurrency.NewSession(cli, concurrency.WithLease(lease.ID)) require.NoError(t, err) defer s.Close() assert.Equal(t, s.Lease(), lease.ID) childCtx, cancel := context.WithCancel(s.Ctx()) defer cancel() go s.Orphan() select { case <-childCtx.Done(): case <-time.After(time.Millisecond * 100): t.Fatal("child context of session context is not canceled") } assert.Equal(t, childCtx.Err(), context.Canceled) } ================================================ FILE: tests/integration/clientv3/connectivity/black_hole_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package connectivity_test import ( "context" "errors" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" clientv3test "go.etcd.io/etcd/tests/v3/integration/clientv3" ) // TestBalancerUnderBlackholeKeepAliveWatch tests when watch discovers it cannot talk to // blackholed endpoint, client balancer switches to healthy one. // TODO: test server-to-client keepalive ping func TestBalancerUnderBlackholeKeepAliveWatch(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 2, GRPCKeepAliveMinTime: time.Millisecond, // avoid too_many_pings UseBridge: true, }) defer clus.Terminate(t) eps := []string{clus.Members[0].GRPCURL, clus.Members[1].GRPCURL} ccfg := clientv3.Config{ Endpoints: []string{eps[0]}, DialTimeout: time.Second, DialKeepAliveTime: time.Second, DialKeepAliveTimeout: 500 * time.Millisecond, } // gRPC internal implementation related. pingInterval := ccfg.DialKeepAliveTime + ccfg.DialKeepAliveTimeout // 3s for slow machine to process watch and reset connections // TODO: only send healthy endpoint to gRPC so gRPC wont waste time to // dial for unhealthy endpoint. // then we can reduce 3s to 1s. timeout := pingInterval + integration.RequestWaitTimeout cli, err := integration.NewClient(t, ccfg) require.NoError(t, err) defer cli.Close() wch := cli.Watch(t.Context(), "foo", clientv3.WithCreatedNotify()) _, ok := <-wch require.Truef(t, ok, "watch failed on creation") // endpoint can switch to eps[1] when it detects the failure of eps[0] cli.SetEndpoints(eps...) // give enough time for balancer resolution time.Sleep(5 * time.Second) clus.Members[0].Bridge().Blackhole() _, err = clus.Client(1).Put(t.Context(), "foo", "bar") require.NoError(t, err) select { case <-wch: case <-time.After(timeout): t.Error("took too long to receive watch events") } clus.Members[0].Bridge().Unblackhole() // waiting for moving eps[0] out of unhealthy, so that it can be re-pined. time.Sleep(ccfg.DialTimeout) clus.Members[1].Bridge().Blackhole() // make sure client[0] can connect to eps[0] after remove the blackhole. _, err = clus.Client(0).Get(t.Context(), "foo") require.NoError(t, err) _, err = clus.Client(0).Put(t.Context(), "foo", "bar1") require.NoError(t, err) select { case <-wch: case <-time.After(timeout): t.Error("took too long to receive watch events") } } func TestBalancerUnderBlackholeNoKeepAlivePut(t *testing.T) { testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Put(ctx, "foo", "bar") if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || errors.Is(err, rpctypes.ErrTimeout) { return errExpected } return err }) } func TestBalancerUnderBlackholeNoKeepAliveDelete(t *testing.T) { testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Delete(ctx, "foo") if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || errors.Is(err, rpctypes.ErrTimeout) { return errExpected } return err }) } func TestBalancerUnderBlackholeNoKeepAliveTxn(t *testing.T) { testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Txn(ctx). If(clientv3.Compare(clientv3.Version("foo"), "=", 0)). Then(clientv3.OpPut("foo", "bar")). Else(clientv3.OpPut("foo", "baz")).Commit() if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || errors.Is(err, rpctypes.ErrTimeout) { return errExpected } return err }) } func TestBalancerUnderBlackholeNoKeepAliveLinearizableGet(t *testing.T) { testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Get(ctx, "a") if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || errors.Is(err, rpctypes.ErrTimeout) { return errExpected } return err }) } func TestBalancerUnderBlackholeNoKeepAliveSerializableGet(t *testing.T) { testBalancerUnderBlackholeNoKeepAlive(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Get(ctx, "a", clientv3.WithSerializable()) if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) { return errExpected } return err }) } // testBalancerUnderBlackholeNoKeepAlive ensures that first request to blackholed endpoint // fails due to context timeout, but succeeds on next try, with endpoint switch. func testBalancerUnderBlackholeNoKeepAlive(t *testing.T, op func(*clientv3.Client, context.Context) error) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 2, UseBridge: true, }) defer clus.Terminate(t) eps := []string{clus.Members[0].GRPCURL, clus.Members[1].GRPCURL} ccfg := clientv3.Config{ Endpoints: []string{eps[0]}, DialTimeout: 1 * time.Second, } cli, err := integration.NewClient(t, ccfg) require.NoError(t, err) defer cli.Close() // wait for eps[0] to be pinned clientv3test.MustWaitPinReady(t, cli) // add all eps to list, so that when the original pined one fails // the client can switch to other available eps cli.SetEndpoints(eps...) // blackhole eps[0] clus.Members[0].Bridge().Blackhole() // With round robin balancer, client will make a request to a healthy endpoint // within a few requests. // TODO: first operation can succeed // when gRPC supports better retry on non-delivered request for i := 0; i < 5; i++ { ctx, cancel := context.WithTimeout(t.Context(), time.Second*5) err = op(cli, ctx) cancel() if err == nil { break } else if errors.Is(err, errExpected) { t.Logf("#%d: current error %v", i, err) } else { t.Errorf("#%d: failed with error %v", i, err) } } require.NoError(t, err) } ================================================ FILE: tests/integration/clientv3/connectivity/dial_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package connectivity_test import ( "context" "math/rand" "strings" "testing" "time" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" "go.etcd.io/etcd/tests/v3/framework/testutils" clientv3test "go.etcd.io/etcd/tests/v3/integration/clientv3" ) var ( testTLSInfo = transport.TLSInfo{ KeyFile: testutils.MustAbsPath("../../../fixtures/server.key.insecure"), CertFile: testutils.MustAbsPath("../../../fixtures/server.crt"), TrustedCAFile: testutils.MustAbsPath("../../../fixtures/ca.crt"), ClientCertAuth: true, } testTLSInfoExpired = transport.TLSInfo{ KeyFile: testutils.MustAbsPath("../../fixtures-expired/server.key.insecure"), CertFile: testutils.MustAbsPath("../../fixtures-expired/server.crt"), TrustedCAFile: testutils.MustAbsPath("../../fixtures-expired/ca.crt"), ClientCertAuth: true, } ) // TestDialTLSExpired tests client with expired certs fails to dial. func TestDialTLSExpired(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, PeerTLS: &testTLSInfo, ClientTLS: &testTLSInfo}) defer clus.Terminate(t) tls, err := testTLSInfoExpired.ClientConfig() require.NoError(t, err) // expect remote errors "tls: bad certificate" _, err = integration.NewClient(t, clientv3.Config{ Endpoints: []string{clus.Members[0].GRPCURL}, DialTimeout: 3 * time.Second, TLS: tls, }) require.Truef(t, clientv3test.IsClientTimeout(err), "expected dial timeout error") } // TestDialTLSNoConfig ensures the client fails to dial / times out // when TLS endpoints (https, unixs) are given but no tls config. func TestDialTLSNoConfig(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, ClientTLS: &testTLSInfo}) defer clus.Terminate(t) // expect "signed by unknown authority" c, err := integration.NewClient(t, clientv3.Config{ Endpoints: []string{clus.Members[0].GRPCURL}, DialTimeout: time.Second, }) defer func() { if c != nil { c.Close() } }() require.Truef(t, clientv3test.IsClientTimeout(err), "expected dial timeout error") } // TestDialSetEndpointsBeforeFail ensures SetEndpoints can replace unavailable // endpoints with available ones. func TestDialSetEndpointsBeforeFail(t *testing.T) { testDialSetEndpoints(t, true) } func TestDialSetEndpointsAfterFail(t *testing.T) { testDialSetEndpoints(t, false) } // testDialSetEndpoints ensures SetEndpoints can replace unavailable endpoints with available ones. func testDialSetEndpoints(t *testing.T, setBefore bool) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) // get endpoint list eps := make([]string, 3) for i := range eps { eps[i] = clus.Members[i].GRPCURL } toKill := rand.Intn(len(eps)) cfg := clientv3.Config{ Endpoints: []string{eps[toKill]}, DialTimeout: 1 * time.Second, } cli, err := integration.NewClient(t, cfg) require.NoError(t, err) defer cli.Close() if setBefore { cli.SetEndpoints(eps[toKill%3], eps[(toKill+1)%3]) } // make a dead node clus.Members[toKill].Stop(t) clus.WaitLeader(t) if !setBefore { cli.SetEndpoints(eps[toKill%3], eps[(toKill+1)%3]) } time.Sleep(time.Second * 2) ctx, cancel := context.WithTimeout(t.Context(), integration.RequestWaitTimeout) _, err = cli.Get(ctx, "foo", clientv3.WithSerializable()) require.NoError(t, err) cancel() } // TestSwitchSetEndpoints ensures SetEndpoints can switch one endpoint // with a new one that doesn't include original endpoint. func TestSwitchSetEndpoints(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) // get non partitioned members endpoints eps := []string{clus.Members[1].GRPCURL, clus.Members[2].GRPCURL} cli := clus.Client(0) clus.Members[0].InjectPartition(t, clus.Members[1:]...) cli.SetEndpoints(eps...) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() _, err := cli.Get(ctx, "foo") require.NoError(t, err) } func TestRejectOldCluster(t *testing.T) { integration.BeforeTest(t) // 2 endpoints to test multi-endpoint Status clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 2}) defer clus.Terminate(t) cfg := clientv3.Config{ Endpoints: []string{clus.Members[0].GRPCURL, clus.Members[1].GRPCURL}, DialTimeout: 5 * time.Second, RejectOldCluster: true, } cli, err := integration.NewClient(t, cfg) require.NoError(t, err) cli.Close() } // TestDialForeignEndpoint checks an endpoint that is not registered // with the balancer can be dialed. func TestDialForeignEndpoint(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 2}) defer clus.Terminate(t) conn, err := clus.Client(0).Dial(clus.Client(1).Endpoints()[0]) require.NoError(t, err) defer conn.Close() // grpc can return a lazy connection that's not connected yet; confirm // that it can communicate with the cluster. kvc := clientv3.NewKVFromKVClient(pb.NewKVClient(conn), clus.Client(0)) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() _, gerr := kvc.Get(ctx, "abc") require.NoError(t, gerr) } // TestSetEndpointAndPut checks that a Put following a SetEndpoints // to a working endpoint will always succeed. func TestSetEndpointAndPut(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 2}) defer clus.Terminate(t) clus.Client(1).SetEndpoints(clus.Members[0].GRPCURL) _, err := clus.Client(1).Put(t.Context(), "foo", "bar") if err != nil && !strings.Contains(err.Error(), "closing") { t.Fatal(err) } } ================================================ FILE: tests/integration/clientv3/connectivity/doc.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package connectivity ================================================ FILE: tests/integration/clientv3/connectivity/main_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package connectivity import ( "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func TestMain(m *testing.M) { testutil.MustTestMainWithLeakDetection(m) } ================================================ FILE: tests/integration/clientv3/connectivity/network_partition_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package connectivity_test import ( "context" "errors" "testing" "time" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" clientv3test "go.etcd.io/etcd/tests/v3/integration/clientv3" ) var errExpected = errors.New("expected error") func isErrorExpected(err error) bool { return clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || errors.Is(err, rpctypes.ErrTimeout) || errors.Is(err, rpctypes.ErrTimeoutDueToLeaderFail) } // TestBalancerUnderNetworkPartitionPut tests when one member becomes isolated, // first Put request fails, and following retry succeeds with client balancer // switching to others. func TestBalancerUnderNetworkPartitionPut(t *testing.T) { testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Put(ctx, "a", "b") if isErrorExpected(err) { return errExpected } return err }, time.Second) } func TestBalancerUnderNetworkPartitionDelete(t *testing.T) { testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Delete(ctx, "a") if isErrorExpected(err) { return errExpected } return err }, time.Second) } func TestBalancerUnderNetworkPartitionTxn(t *testing.T) { testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Txn(ctx). If(clientv3.Compare(clientv3.Version("foo"), "=", 0)). Then(clientv3.OpPut("foo", "bar")). Else(clientv3.OpPut("foo", "baz")).Commit() if isErrorExpected(err) { return errExpected } return err }, time.Second) } // TestBalancerUnderNetworkPartitionLinearizableGetWithLongTimeout tests // when one member becomes isolated, first quorum Get request succeeds // by switching endpoints within the timeout (long enough to cover endpoint switch). func TestBalancerUnderNetworkPartitionLinearizableGetWithLongTimeout(t *testing.T) { testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Get(ctx, "a") if isErrorExpected(err) { return errExpected } return err }, 7*time.Second) } // TestBalancerUnderNetworkPartitionLinearizableGetWithShortTimeout tests // when one member becomes isolated, first quorum Get request fails, // and following retry succeeds with client balancer switching to others. func TestBalancerUnderNetworkPartitionLinearizableGetWithShortTimeout(t *testing.T) { testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Get(ctx, "a") if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) { return errExpected } return err }, time.Second) } func TestBalancerUnderNetworkPartitionSerializableGet(t *testing.T) { testBalancerUnderNetworkPartition(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Get(ctx, "a", clientv3.WithSerializable()) return err }, time.Second) } func testBalancerUnderNetworkPartition(t *testing.T, op func(*clientv3.Client, context.Context) error, timeout time.Duration) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 3, }) defer clus.Terminate(t) eps := []string{clus.Members[0].GRPCURL, clus.Members[1].GRPCURL, clus.Members[2].GRPCURL} // expect pin eps[0] ccfg := clientv3.Config{ Endpoints: []string{eps[0]}, DialTimeout: 3 * time.Second, } cli, err := integration.NewClient(t, ccfg) require.NoError(t, err) defer cli.Close() // wait for eps[0] to be pinned clientv3test.MustWaitPinReady(t, cli) // add other endpoints for later endpoint switch cli.SetEndpoints(eps...) time.Sleep(time.Second * 2) clus.Members[0].InjectPartition(t, clus.Members[1:]...) for i := 0; i < 5; i++ { ctx, cancel := context.WithTimeout(t.Context(), timeout) err = op(cli, ctx) t.Logf("Op returned error: %v", err) t.Log("Cancelling...") cancel() if err == nil { break } if !errors.Is(err, errExpected) { t.Errorf("#%d: expected '%v', got '%v'", i, errExpected, err) } // give enough time for endpoint switch // TODO: remove random sleep by syncing directly with balancer if i == 0 { time.Sleep(5 * time.Second) } } if err != nil { t.Errorf("balancer did not switch in time (%v)", err) } } // TestBalancerUnderNetworkPartitionLinearizableGetLeaderElection ensures balancer // switches endpoint when leader fails and linearizable get requests returns // "etcdserver: request timed out". func TestBalancerUnderNetworkPartitionLinearizableGetLeaderElection(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 3, }) defer clus.Terminate(t) eps := []string{clus.Members[0].GRPCURL, clus.Members[1].GRPCURL, clus.Members[2].GRPCURL} lead := clus.WaitLeader(t) timeout := 3 * clus.Members[(lead+1)%2].ServerConfig.ReqTimeout() cli, err := integration.NewClient(t, clientv3.Config{ Endpoints: []string{eps[(lead+1)%2]}, DialTimeout: 2 * time.Second, }) require.NoError(t, err) defer cli.Close() // add all eps to list, so that when the original pined one fails // the client can switch to other available eps cli.SetEndpoints(eps[lead], eps[(lead+1)%2]) // isolate leader clus.Members[lead].InjectPartition(t, clus.Members[(lead+1)%3], clus.Members[(lead+2)%3]) // expects balancer to round robin to leader within two attempts for i := 0; i < 2; i++ { ctx, cancel := context.WithTimeout(t.Context(), timeout) _, err = cli.Get(ctx, "a") cancel() if err == nil { break } } require.NoError(t, err) } func TestBalancerUnderNetworkPartitionWatchLeader(t *testing.T) { testBalancerUnderNetworkPartitionWatch(t, true) } func TestBalancerUnderNetworkPartitionWatchFollower(t *testing.T) { testBalancerUnderNetworkPartitionWatch(t, false) } // testBalancerUnderNetworkPartitionWatch ensures watch stream // to a partitioned node be closed when context requires leader. func testBalancerUnderNetworkPartitionWatch(t *testing.T, isolateLeader bool) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 3, }) defer clus.Terminate(t) eps := []string{clus.Members[0].GRPCURL, clus.Members[1].GRPCURL, clus.Members[2].GRPCURL} target := clus.WaitLeader(t) if !isolateLeader { target = (target + 1) % 3 } // pin eps[target] watchCli, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{eps[target]}}) require.NoError(t, err) t.Logf("watchCli created to: %v", target) defer watchCli.Close() // wait for eps[target] to be connected clientv3test.MustWaitPinReady(t, watchCli) t.Logf("successful connection with server: %v", target) // We stick to the original endpoint, so when the one fails we don't switch // under the cover to other available eps, but expose the failure to the // caller (test assertion). wch := watchCli.Watch(clientv3.WithRequireLeader(t.Context()), "foo", clientv3.WithCreatedNotify()) select { case <-wch: case <-time.After(integration.RequestWaitTimeout): t.Fatal("took too long to create watch") } t.Logf("watch established") // isolate eps[target] clus.Members[target].InjectPartition(t, clus.Members[(target+1)%3], clus.Members[(target+2)%3], ) select { case ev := <-wch: if len(ev.Events) != 0 { t.Fatal("expected no event") } require.ErrorIs(t, ev.Err(), rpctypes.ErrNoLeader) case <-time.After(integration.RequestWaitTimeout): // enough time to detect leader lost t.Fatal("took too long to detect leader lost") } } func TestDropReadUnderNetworkPartition(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 3, }) defer clus.Terminate(t) leaderIndex := clus.WaitLeader(t) // get a follower endpoint eps := []string{clus.Members[(leaderIndex+1)%3].GRPCURL} ccfg := clientv3.Config{ Endpoints: eps, DialTimeout: 10 * time.Second, } cli, err := integration.NewClient(t, ccfg) require.NoError(t, err) defer cli.Close() // wait for eps[0] to be pinned clientv3test.MustWaitPinReady(t, cli) // add other endpoints for later endpoint switch cli.SetEndpoints(eps...) time.Sleep(time.Second * 2) conn, err := cli.Dial(clus.Members[(leaderIndex+1)%3].GRPCURL) require.NoError(t, err) defer conn.Close() clus.Members[leaderIndex].InjectPartition(t, clus.Members[(leaderIndex+1)%3], clus.Members[(leaderIndex+2)%3]) kvc := clientv3.NewKVFromKVClient(pb.NewKVClient(conn), nil) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) _, err = kvc.Get(ctx, "a") cancel() require.ErrorIsf(t, err, rpctypes.ErrLeaderChanged, "expected %v, got %v", rpctypes.ErrLeaderChanged, err) for i := 0; i < 5; i++ { ctx, cancel = context.WithTimeout(t.Context(), 10*time.Second) _, err = kvc.Get(ctx, "a") cancel() if err != nil { if errors.Is(err, rpctypes.ErrTimeout) { <-time.After(time.Second) i++ continue } t.Fatalf("expected nil or timeout, got %v", err) } // No error returned and no retry required break } } ================================================ FILE: tests/integration/clientv3/connectivity/server_shutdown_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package connectivity_test import ( "bytes" "context" "errors" "fmt" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" clientv3test "go.etcd.io/etcd/tests/v3/integration/clientv3" ) // TestBalancerUnderServerShutdownWatch expects that watch client // switch its endpoints when the member of the pinned endpoint fails. func TestBalancerUnderServerShutdownWatch(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 3, UseBridge: true, }) defer clus.Terminate(t) eps := []string{clus.Members[0].GRPCURL, clus.Members[1].GRPCURL, clus.Members[2].GRPCURL} lead := clus.WaitLeader(t) // pin eps[lead] watchCli, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{eps[lead]}}) require.NoError(t, err) defer watchCli.Close() // wait for eps[lead] to be pinned clientv3test.MustWaitPinReady(t, watchCli) // add all eps to list, so that when the original pined one fails // the client can switch to other available eps watchCli.SetEndpoints(eps...) key, val := "foo", "bar" wch := watchCli.Watch(t.Context(), key, clientv3.WithCreatedNotify()) select { case <-wch: case <-time.After(integration.RequestWaitTimeout): t.Fatal("took too long to create watch") } donec := make(chan struct{}) go func() { defer close(donec) // switch to others when eps[lead] is shut down select { case ev := <-wch: if werr := ev.Err(); werr != nil { t.Error(werr) } if len(ev.Events) != 1 { t.Errorf("expected one event, got %+v", ev) } if !bytes.Equal(ev.Events[0].Kv.Value, []byte(val)) { t.Errorf("expected %q, got %+v", val, ev.Events[0].Kv) } case <-time.After(7 * time.Second): t.Error("took too long to receive events") } }() // shut down eps[lead] clus.Members[lead].Terminate(t) // writes to eps[lead+1] putCli, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{eps[(lead+1)%3]}}) require.NoError(t, err) defer putCli.Close() for { ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) _, err = putCli.Put(ctx, key, val) cancel() if err == nil { break } if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || errors.Is(err, rpctypes.ErrTimeout) || errors.Is(err, rpctypes.ErrTimeoutDueToLeaderFail) { continue } t.Fatal(err) } select { case <-donec: case <-time.After(5 * time.Second): // enough time for balancer switch t.Fatal("took too long to receive events") } } func TestBalancerUnderServerShutdownPut(t *testing.T) { testBalancerUnderServerShutdownMutable(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Put(ctx, "foo", "bar") return err }) } func TestBalancerUnderServerShutdownDelete(t *testing.T) { testBalancerUnderServerShutdownMutable(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Delete(ctx, "foo") return err }) } func TestBalancerUnderServerShutdownTxn(t *testing.T) { testBalancerUnderServerShutdownMutable(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Txn(ctx). If(clientv3.Compare(clientv3.Version("foo"), "=", 0)). Then(clientv3.OpPut("foo", "bar")). Else(clientv3.OpPut("foo", "baz")).Commit() return err }) } // testBalancerUnderServerShutdownMutable expects that when the member of // the pinned endpoint is shut down, the balancer switches its endpoints // and all subsequent put/delete/txn requests succeed with new endpoints. func testBalancerUnderServerShutdownMutable(t *testing.T, op func(*clientv3.Client, context.Context) error) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 3, }) defer clus.Terminate(t) eps := []string{clus.Members[0].GRPCURL, clus.Members[1].GRPCURL, clus.Members[2].GRPCURL} // pin eps[0] cli, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{eps[0]}}) require.NoError(t, err) defer cli.Close() // wait for eps[0] to be pinned clientv3test.MustWaitPinReady(t, cli) // add all eps to list, so that when the original pined one fails // the client can switch to other available eps cli.SetEndpoints(eps...) // shut down eps[0] clus.Members[0].Terminate(t) // switched to others when eps[0] was explicitly shut down // and following request should succeed // TODO: remove this (expose client connection state?) time.Sleep(time.Second) cctx, ccancel := context.WithTimeout(t.Context(), time.Second) err = op(cli, cctx) ccancel() require.NoError(t, err) } func TestBalancerUnderServerShutdownGetLinearizable(t *testing.T) { testBalancerUnderServerShutdownImmutable(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Get(ctx, "foo") return err }, 7*time.Second) // give enough time for leader election, balancer switch } func TestBalancerUnderServerShutdownGetSerializable(t *testing.T) { testBalancerUnderServerShutdownImmutable(t, func(cli *clientv3.Client, ctx context.Context) error { _, err := cli.Get(ctx, "foo", clientv3.WithSerializable()) return err }, 2*time.Second) } // testBalancerUnderServerShutdownImmutable expects that when the member of // the pinned endpoint is shut down, the balancer switches its endpoints // and all subsequent range requests succeed with new endpoints. func testBalancerUnderServerShutdownImmutable(t *testing.T, op func(*clientv3.Client, context.Context) error, timeout time.Duration) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 3, }) defer clus.Terminate(t) eps := []string{clus.Members[0].GRPCURL, clus.Members[1].GRPCURL, clus.Members[2].GRPCURL} // pin eps[0] cli, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{eps[0]}}) if err != nil { t.Errorf("failed to create client: %v", err) } defer cli.Close() // wait for eps[0] to be pinned clientv3test.MustWaitPinReady(t, cli) // add all eps to list, so that when the original pined one fails // the client can switch to other available eps cli.SetEndpoints(eps...) // shut down eps[0] clus.Members[0].Terminate(t) // switched to others when eps[0] was explicitly shut down // and following request should succeed cctx, ccancel := context.WithTimeout(t.Context(), timeout) err = op(cli, cctx) ccancel() if err != nil { t.Errorf("failed to finish range request in time %v (timeout %v)", err, timeout) } } func TestBalancerUnderServerStopInflightLinearizableGetOnRestart(t *testing.T) { tt := []pinTestOpt{ {pinLeader: true, stopPinFirst: true}, {pinLeader: true, stopPinFirst: false}, {pinLeader: false, stopPinFirst: true}, {pinLeader: false, stopPinFirst: false}, } for _, w := range tt { t.Run(fmt.Sprintf("%#v", w), func(t *testing.T) { testBalancerUnderServerStopInflightRangeOnRestart(t, true, w) }) } } func TestBalancerUnderServerStopInflightSerializableGetOnRestart(t *testing.T) { tt := []pinTestOpt{ {pinLeader: true, stopPinFirst: true}, {pinLeader: true, stopPinFirst: false}, {pinLeader: false, stopPinFirst: true}, {pinLeader: false, stopPinFirst: false}, } for _, w := range tt { t.Run(fmt.Sprintf("%#v", w), func(t *testing.T) { testBalancerUnderServerStopInflightRangeOnRestart(t, false, w) }) } } type pinTestOpt struct { pinLeader bool stopPinFirst bool } // testBalancerUnderServerStopInflightRangeOnRestart expects // inflight range request reconnects on server restart. func testBalancerUnderServerStopInflightRangeOnRestart(t *testing.T, linearizable bool, opt pinTestOpt) { integration.BeforeTest(t) cfg := &integration.ClusterConfig{ Size: 2, UseBridge: true, } if linearizable { cfg.Size = 3 } clus := integration.NewCluster(t, cfg) defer clus.Terminate(t) eps := []string{clus.Members[0].GRPCURL, clus.Members[1].GRPCURL} if linearizable { eps = append(eps, clus.Members[2].GRPCURL) } lead := clus.WaitLeader(t) target := lead if !opt.pinLeader { target = (target + 1) % 2 } // pin eps[target] cli, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{eps[target]}}) if err != nil { t.Errorf("failed to create client: %v", err) } defer cli.Close() // wait for eps[target] to be pinned clientv3test.MustWaitPinReady(t, cli) // add all eps to list, so that when the original pined one fails // the client can switch to other available eps cli.SetEndpoints(eps...) if opt.stopPinFirst { clus.Members[target].Stop(t) // give some time for balancer switch before stopping the other time.Sleep(time.Second) clus.Members[(target+1)%2].Stop(t) } else { clus.Members[(target+1)%2].Stop(t) // balancer cannot pin other member since it's already stopped clus.Members[target].Stop(t) } // 3-second is the minimum interval between endpoint being marked // as unhealthy and being removed from unhealthy, so possibly // takes >5-second to unpin and repin an endpoint // TODO: decrease timeout when balancer switch rewrite clientTimeout := 7 * time.Second var gops []clientv3.OpOption if !linearizable { gops = append(gops, clientv3.WithSerializable()) } donec, readyc := make(chan struct{}), make(chan struct{}, 1) go func() { defer close(donec) ctx, cancel := context.WithTimeout(t.Context(), clientTimeout) readyc <- struct{}{} // TODO: The new grpc load balancer will not pin to an endpoint // as intended by this test. But it will round robin member within // two attempts. // Remove retry loop once the new grpc load balancer provides retry. for i := 0; i < 2; i++ { _, err = cli.Get(ctx, "abc", gops...) if err == nil { break } } cancel() if err != nil { t.Errorf("unexpected error: %v", err) } }() <-readyc clus.Members[target].Restart(t) select { case <-time.After(clientTimeout + integration.RequestWaitTimeout): t.Fatalf("timed out waiting for Get [linearizable: %v, opt: %+v]", linearizable, opt) case <-donec: } } ================================================ FILE: tests/integration/clientv3/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package clientv3test implements tests built upon embedded etcd, and focuses on // correctness of etcd client. package clientv3test ================================================ FILE: tests/integration/clientv3/examples/example_auth_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3_test import ( "context" "fmt" "log" clientv3 "go.etcd.io/etcd/client/v3" ) func mockAuth() { fmt.Println(`etcdserver: permission denied`) fmt.Println(`user u permission: key "foo", range end "zoo"`) } func ExampleAuth() { forUnitTestsRunInMockedContext( mockAuth, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() if _, err = cli.RoleAdd(context.TODO(), "root"); err != nil { log.Fatal(err) } if _, err = cli.UserAdd(context.TODO(), "root", "123"); err != nil { log.Fatal(err) } if _, err = cli.UserGrantRole(context.TODO(), "root", "root"); err != nil { log.Fatal(err) } if _, err = cli.RoleAdd(context.TODO(), "r"); err != nil { log.Fatal(err) } if _, err = cli.RoleGrantPermission( context.TODO(), "r", // role name "foo", // key "zoo", // range end clientv3.PermissionType(clientv3.PermReadWrite), ); err != nil { log.Fatal(err) } if _, err = cli.UserAdd(context.TODO(), "u", "123"); err != nil { log.Fatal(err) } if _, err = cli.UserGrantRole(context.TODO(), "u", "r"); err != nil { log.Fatal(err) } if _, err = cli.AuthEnable(context.TODO()); err != nil { log.Fatal(err) } cliAuth, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, Username: "u", Password: "123", }) if err != nil { log.Fatal(err) } defer cliAuth.Close() if _, err = cliAuth.Put(context.TODO(), "foo1", "bar"); err != nil { log.Fatal(err) } _, err = cliAuth.Txn(context.TODO()). If(clientv3.Compare(clientv3.Value("zoo1"), ">", "abc")). Then(clientv3.OpPut("zoo1", "XYZ")). Else(clientv3.OpPut("zoo1", "ABC")). Commit() fmt.Println(err) // now check the permission with the root account rootCli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, Username: "root", Password: "123", }) if err != nil { log.Fatal(err) } defer rootCli.Close() resp, err := rootCli.RoleGet(context.TODO(), "r") if err != nil { log.Fatal(err) } fmt.Printf("user u permission: key %q, range end %q\n", resp.Perm[0].Key, resp.Perm[0].RangeEnd) if _, err = rootCli.AuthDisable(context.TODO()); err != nil { log.Fatal(err) } }) // Output: etcdserver: permission denied // user u permission: key "foo", range end "zoo" } ================================================ FILE: tests/integration/clientv3/examples/example_cluster_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3_test import ( "context" "fmt" "log" clientv3 "go.etcd.io/etcd/client/v3" ) func mockClusterMemberList() { fmt.Println("members: 3") } func ExampleCluster_memberList() { forUnitTestsRunInMockedContext(mockClusterMemberList, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() resp, err := cli.MemberList(context.Background()) if err != nil { log.Fatal(err) } fmt.Println("members:", len(resp.Members)) }) // Output: members: 3 } func mockClusterMemberAdd() { fmt.Println("added member.PeerURLs: [http://localhost:32380]") fmt.Println("members count: 4") } func ExampleCluster_memberAdd() { forUnitTestsRunInMockedContext(mockClusterMemberAdd, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() // Add member 1: mresp, err := cli.MemberAdd(context.Background(), []string{"http://localhost:32380"}) if err != nil { log.Fatal(err) } fmt.Println("added member.PeerURLs:", mresp.Member.PeerURLs) fmt.Println("members count:", len(mresp.Members)) // Restore original cluster state _, err = cli.MemberRemove(context.Background(), mresp.Member.ID) if err != nil { log.Fatal(err) } }) // Output: // added member.PeerURLs: [http://localhost:32380] // members count: 4 } func mockClusterMemberAddAsLearner() { fmt.Println("members count: 4") fmt.Println("added member.IsLearner: true") } func ExampleCluster_memberAddAsLearner() { forUnitTestsRunInMockedContext(mockClusterMemberAddAsLearner, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() mresp, err := cli.MemberAddAsLearner(context.Background(), []string{"http://localhost:32381"}) if err != nil { log.Fatal(err) } // Restore original cluster state _, err = cli.MemberRemove(context.Background(), mresp.Member.ID) if err != nil { log.Fatal(err) } fmt.Println("members count:", len(mresp.Members)) fmt.Println("added member.IsLearner:", mresp.Member.IsLearner) }) // Output: // members count: 4 // added member.IsLearner: true } func mockClusterMemberRemove() {} func ExampleCluster_memberRemove() { forUnitTestsRunInMockedContext(mockClusterMemberRemove, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() resp, err := cli.MemberList(context.Background()) if err != nil { log.Fatal(err) } _, err = cli.MemberRemove(context.Background(), resp.Members[0].ID) if err != nil { log.Fatal(err) } // Restore original cluster: _, err = cli.MemberAdd(context.Background(), resp.Members[0].PeerURLs) if err != nil { log.Fatal(err) } }) } func mockClusterMemberUpdate() {} func ExampleCluster_memberUpdate() { forUnitTestsRunInMockedContext(mockClusterMemberUpdate, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() resp, err := cli.MemberList(context.Background()) if err != nil { log.Fatal(err) } peerURLs := []string{"http://localhost:12380"} _, err = cli.MemberUpdate(context.Background(), resp.Members[0].ID, peerURLs) if err != nil { log.Fatal(err) } // Restore to mitigate impact on other tests: _, err = cli.MemberUpdate(context.Background(), resp.Members[0].ID, resp.Members[0].PeerURLs) if err != nil { log.Fatal(err) } }) // Output: } ================================================ FILE: tests/integration/clientv3/examples/example_kv_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3_test import ( "context" "errors" "fmt" "log" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" ) func mockKVPut() {} func ExampleKV_put() { forUnitTestsRunInMockedContext(mockKVPut, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) _, err = cli.Put(ctx, "sample_key", "sample_value") cancel() if err != nil { log.Fatal(err) } }) // Output: } func mockKVPutErrorHandling() { fmt.Println("client-side error: etcdserver: key is not provided") } func ExampleKV_putErrorHandling() { forUnitTestsRunInMockedContext(mockKVPutErrorHandling, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) _, err = cli.Put(ctx, "", "sample_value") cancel() if err != nil { if errors.Is(err, context.Canceled) { fmt.Printf("ctx is canceled by another routine: %v\n", err) } else if errors.Is(err, context.DeadlineExceeded) { fmt.Printf("ctx is attached with a deadline is exceeded: %v\n", err) } else if errors.Is(err, rpctypes.ErrEmptyKey) { fmt.Printf("client-side error: %v\n", err) } else { fmt.Printf("bad cluster endpoints, which are not etcd servers: %v\n", err) } } }) // Output: client-side error: etcdserver: key is not provided } func mockKVGet() { fmt.Println("foo : bar") } func ExampleKV_get() { forUnitTestsRunInMockedContext(mockKVGet, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() _, err = cli.Put(context.TODO(), "foo", "bar") if err != nil { log.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) resp, err := cli.Get(ctx, "foo") cancel() if err != nil { log.Fatal(err) } for _, ev := range resp.Kvs { fmt.Printf("%s : %s\n", ev.Key, ev.Value) } }) // Output: foo : bar } func mockKVGetWithRev() { fmt.Println("foo : bar1") } func ExampleKV_getWithRev() { forUnitTestsRunInMockedContext(mockKVGetWithRev, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() presp, err := cli.Put(context.TODO(), "foo", "bar1") if err != nil { log.Fatal(err) } _, err = cli.Put(context.TODO(), "foo", "bar2") if err != nil { log.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) resp, err := cli.Get(ctx, "foo", clientv3.WithRev(presp.Header.Revision)) cancel() if err != nil { log.Fatal(err) } for _, ev := range resp.Kvs { fmt.Printf("%s : %s\n", ev.Key, ev.Value) } }) // Output: foo : bar1 } func mockKVGetSortedPrefix() { fmt.Println(`key_2 : value`) fmt.Println(`key_1 : value`) fmt.Println(`key_0 : value`) } func ExampleKV_getSortedPrefix() { forUnitTestsRunInMockedContext(mockKVGetSortedPrefix, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() for i := range make([]int, 3) { ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) _, err = cli.Put(ctx, fmt.Sprintf("key_%d", i), "value") cancel() if err != nil { log.Fatal(err) } } ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) resp, err := cli.Get(ctx, "key", clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortDescend)) cancel() if err != nil { log.Fatal(err) } for _, ev := range resp.Kvs { fmt.Printf("%s : %s\n", ev.Key, ev.Value) } }) // Output: // key_2 : value // key_1 : value // key_0 : value } func mockKVDelete() { fmt.Println("Deleted all keys: true") } func ExampleKV_delete() { forUnitTestsRunInMockedContext(mockKVDelete, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) defer cancel() // count keys about to be deleted gresp, err := cli.Get(ctx, "key", clientv3.WithPrefix()) if err != nil { log.Fatal(err) } // delete the keys dresp, err := cli.Delete(ctx, "key", clientv3.WithPrefix()) if err != nil { log.Fatal(err) } fmt.Println("Deleted all keys:", int64(len(gresp.Kvs)) == dresp.Deleted) }) // Output: // Deleted all keys: true } func mockKVCompact() {} func ExampleKV_compact() { forUnitTestsRunInMockedContext(mockKVCompact, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) resp, err := cli.Get(ctx, "foo") cancel() if err != nil { log.Fatal(err) } compRev := resp.Header.Revision // specify compact revision of your choice ctx, cancel = context.WithTimeout(context.Background(), requestTimeout) _, err = cli.Compact(ctx, compRev) cancel() if err != nil { log.Fatal(err) } }) // Output: } func mockKVTxn() { fmt.Println("key : XYZ") } func ExampleKV_txn() { forUnitTestsRunInMockedContext(mockKVTxn, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() kvc := clientv3.NewKV(cli) _, err = kvc.Put(context.TODO(), "key", "xyz") if err != nil { log.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) _, err = kvc.Txn(ctx). // txn value comparisons are lexical If(clientv3.Compare(clientv3.Value("key"), ">", "abc")). // the "Then" runs, since "xyz" > "abc" Then(clientv3.OpPut("key", "XYZ")). // the "Else" does not run Else(clientv3.OpPut("key", "ABC")). Commit() cancel() if err != nil { log.Fatal(err) } gresp, err := kvc.Get(context.TODO(), "key") if err != nil { log.Fatal(err) } for _, ev := range gresp.Kvs { fmt.Printf("%s : %s\n", ev.Key, ev.Value) } }) // Output: key : XYZ } func mockKVDo() {} func ExampleKV_do() { forUnitTestsRunInMockedContext(mockKVDo, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() ops := []clientv3.Op{ clientv3.OpPut("put-key", "123"), clientv3.OpGet("put-key"), clientv3.OpPut("put-key", "456"), } for _, op := range ops { if _, err := cli.Do(context.TODO(), op); err != nil { log.Fatal(err) } } }) // Output: } ================================================ FILE: tests/integration/clientv3/examples/example_lease_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3_test import ( "context" "fmt" "log" clientv3 "go.etcd.io/etcd/client/v3" ) func mockLeaseGrant() { } func ExampleLease_grant() { forUnitTestsRunInMockedContext(mockLeaseGrant, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() // minimum lease TTL is 5-second resp, err := cli.Grant(context.TODO(), 5) if err != nil { log.Fatal(err) } // after 5 seconds, the key 'foo' will be removed _, err = cli.Put(context.TODO(), "foo", "bar", clientv3.WithLease(resp.ID)) if err != nil { log.Fatal(err) } }) // Output: } func mockLeaseRevoke() { fmt.Println("number of keys: 0") } func ExampleLease_revoke() { forUnitTestsRunInMockedContext(mockLeaseRevoke, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() resp, err := cli.Grant(context.TODO(), 5) if err != nil { log.Fatal(err) } _, err = cli.Put(context.TODO(), "foo", "bar", clientv3.WithLease(resp.ID)) if err != nil { log.Fatal(err) } // revoking lease expires the key attached to its lease ID _, err = cli.Revoke(context.TODO(), resp.ID) if err != nil { log.Fatal(err) } gresp, err := cli.Get(context.TODO(), "foo") if err != nil { log.Fatal(err) } fmt.Println("number of keys:", len(gresp.Kvs)) }) // Output: number of keys: 0 } func mockLeaseKeepAlive() { fmt.Println("ttl: 5") } func ExampleLease_keepAlive() { forUnitTestsRunInMockedContext(mockLeaseKeepAlive, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() resp, err := cli.Grant(context.TODO(), 5) if err != nil { log.Fatal(err) } _, err = cli.Put(context.TODO(), "foo", "bar", clientv3.WithLease(resp.ID)) if err != nil { log.Fatal(err) } // the key 'foo' will be kept forever ch, kaerr := cli.KeepAlive(context.TODO(), resp.ID) if kaerr != nil { log.Fatal(kaerr) } ka := <-ch if ka != nil { fmt.Println("ttl:", ka.TTL) } else { fmt.Println("Unexpected NULL") } }) // Output: ttl: 5 } func mockLeaseKeepAliveOnce() { fmt.Println("ttl: 5") } func ExampleLease_keepAliveOnce() { forUnitTestsRunInMockedContext(mockLeaseKeepAliveOnce, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() resp, err := cli.Grant(context.TODO(), 5) if err != nil { log.Fatal(err) } _, err = cli.Put(context.TODO(), "foo", "bar", clientv3.WithLease(resp.ID)) if err != nil { log.Fatal(err) } // to renew the lease only once ka, kaerr := cli.KeepAliveOnce(context.TODO(), resp.ID) if kaerr != nil { log.Fatal(kaerr) } fmt.Println("ttl:", ka.TTL) }) // Output: ttl: 5 } ================================================ FILE: tests/integration/clientv3/examples/example_maintenance_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3_test import ( "context" "log" clientv3 "go.etcd.io/etcd/client/v3" ) func mockMaintenanceStatus() {} func ExampleMaintenance_status() { forUnitTestsRunInMockedContext(mockMaintenanceStatus, func() { for _, ep := range exampleEndpoints() { cli, err := clientv3.New(clientv3.Config{ Endpoints: []string{ep}, DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() _, err = cli.Status(context.Background(), ep) if err != nil { log.Fatal(err) } } }) // Output: } func mockMaintenanceDefragment() {} func ExampleMaintenance_defragment() { forUnitTestsRunInMockedContext(mockMaintenanceDefragment, func() { for _, ep := range exampleEndpoints() { cli, err := clientv3.New(clientv3.Config{ Endpoints: []string{ep}, DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() if _, err = cli.Defragment(context.TODO(), ep); err != nil { log.Fatal(err) } } }) // Output: } ================================================ FILE: tests/integration/clientv3/examples/example_metrics_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3_test import ( "context" "fmt" "io" "log" "net" "net/http" "strings" grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" clientv3 "go.etcd.io/etcd/client/v3" ) func mockClientMetrics() { fmt.Println(`grpc_client_started_total{grpc_method="Range",grpc_service="etcdserverpb.KV",grpc_type="unary"} 1`) } func ExampleClient_metrics() { forUnitTestsRunInMockedContext(mockClientMetrics, func() { clientMetrics := grpcprom.NewClientMetrics() prometheus.Register(clientMetrics) cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialOptions: []grpc.DialOption{ grpc.WithUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), grpc.WithStreamInterceptor(clientMetrics.StreamClientInterceptor()), }, }) if err != nil { log.Fatal(err) } defer cli.Close() // get a key so it shows up in the metrics as a range RPC cli.Get(context.TODO(), "test_key") // listen for all Prometheus metrics ln, err := net.Listen("tcp", ":0") if err != nil { log.Fatal(err) } donec := make(chan struct{}) go func() { defer close(donec) http.Serve(ln, promhttp.Handler()) }() defer func() { ln.Close() <-donec }() // make an http request to fetch all Prometheus metrics url := "http://" + ln.Addr().String() + "/metrics" resp, err := http.Get(url) if err != nil { log.Fatalf("fetch error: %v", err) } b, err := io.ReadAll(resp.Body) resp.Body.Close() if err != nil { log.Fatalf("fetch error: reading %s: %v", url, err) } // confirm range request in metrics for _, l := range strings.Split(string(b), "\n") { if strings.Contains(l, `grpc_client_started_total{grpc_method="Range"`) { fmt.Println(l) break } } }) // Output: // grpc_client_started_total{grpc_method="Range",grpc_service="etcdserverpb.KV",grpc_type="unary"} 1 } ================================================ FILE: tests/integration/clientv3/examples/example_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3_test import ( "context" "log" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" ) func mockConfigInsecure() {} func ExampleConfig_insecure() { forUnitTestsRunInMockedContext(mockConfigInsecure, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() // make sure to close the client _, err = cli.Put(context.TODO(), "foo", "bar") if err != nil { log.Fatal(err) } }) // Without the line below the test is not being executed // Output: } func mockConfigWithTLS() {} func ExampleConfig_withTLS() { forUnitTestsRunInMockedContext(mockConfigWithTLS, func() { tlsInfo := transport.TLSInfo{ CertFile: "/tmp/test-certs/test-name-1.pem", KeyFile: "/tmp/test-certs/test-name-1-key.pem", TrustedCAFile: "/tmp/test-certs/trusted-ca.pem", } tlsConfig, err := tlsInfo.ClientConfig() if err != nil { log.Fatal(err) } cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, TLS: tlsConfig, }) if err != nil { log.Fatal(err) } defer cli.Close() // make sure to close the client _, err = cli.Put(context.TODO(), "foo", "bar") if err != nil { log.Fatal(err) } }) // Without the line below the test is not being executed // Output: } ================================================ FILE: tests/integration/clientv3/examples/example_watch_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3_test import ( "context" "fmt" "log" "time" clientv3 "go.etcd.io/etcd/client/v3" ) func mockWatcherWatch() { fmt.Println(`PUT "foo" : "bar"`) } func ExampleWatcher_watch() { forUnitTestsRunInMockedContext(mockWatcherWatch, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() rch := cli.Watch(context.Background(), "foo") for wresp := range rch { for _, ev := range wresp.Events { fmt.Printf("%s %q : %q\n", ev.Type, ev.Kv.Key, ev.Kv.Value) } } }) // PUT "foo" : "bar" } func mockWatcherWatchWithPrefix() { fmt.Println(`PUT "foo1" : "bar"`) } func ExampleWatcher_watchWithPrefix() { forUnitTestsRunInMockedContext(mockWatcherWatchWithPrefix, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() rch := cli.Watch(context.Background(), "foo", clientv3.WithPrefix()) for wresp := range rch { for _, ev := range wresp.Events { fmt.Printf("%s %q : %q\n", ev.Type, ev.Kv.Key, ev.Kv.Value) } } }) // PUT "foo1" : "bar" } func mockWatcherWatchWithRange() { fmt.Println(`PUT "foo1" : "bar1"`) fmt.Println(`PUT "foo2" : "bar2"`) fmt.Println(`PUT "foo3" : "bar3"`) } func ExampleWatcher_watchWithRange() { forUnitTestsRunInMockedContext(mockWatcherWatchWithRange, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } defer cli.Close() // watches within ['foo1', 'foo4'), in lexicographical order rch := cli.Watch(context.Background(), "foo1", clientv3.WithRange("foo4")) go func() { cli.Put(context.Background(), "foo1", "bar1") cli.Put(context.Background(), "foo5", "bar5") cli.Put(context.Background(), "foo2", "bar2") cli.Put(context.Background(), "foo3", "bar3") }() i := 0 for wresp := range rch { for _, ev := range wresp.Events { fmt.Printf("%s %q : %q\n", ev.Type, ev.Kv.Key, ev.Kv.Value) i++ if i == 3 { // After 3 messages we are done. cli.Delete(context.Background(), "foo", clientv3.WithPrefix()) cli.Close() return } } } }) // Output: // PUT "foo1" : "bar1" // PUT "foo2" : "bar2" // PUT "foo3" : "bar3" } func mockWatcherWatchWithProgressNotify() { fmt.Println(`wresp.IsProgressNotify: true`) } func ExampleWatcher_watchWithProgressNotify() { forUnitTestsRunInMockedContext(mockWatcherWatchWithProgressNotify, func() { cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialTimeout: dialTimeout, }) if err != nil { log.Fatal(err) } rch := cli.Watch(context.Background(), "foo", clientv3.WithProgressNotify()) closedch := make(chan bool) go func() { // This assumes that cluster is configured with frequent WatchProgressNotifyInterval // e.g. WatchProgressNotifyInterval: 200 * time.Millisecond. time.Sleep(time.Second) err := cli.Close() if err != nil { log.Fatal(err) } close(closedch) }() wresp := <-rch fmt.Println("wresp.IsProgressNotify:", wresp.IsProgressNotify()) <-closedch }) // TODO: Rather wresp.IsProgressNotify: true should be expected // Output: // wresp.IsProgressNotify: true } ================================================ FILE: tests/integration/clientv3/examples/main_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3_test import ( "log" "os" "testing" "time" "go.etcd.io/etcd/client/pkg/v3/testutil" framework "go.etcd.io/etcd/tests/v3/framework/integration" "go.etcd.io/etcd/tests/v3/integration" ) const ( dialTimeout = 5 * time.Second requestTimeout = 10 * time.Second ) var lazyCluster = integration.NewLazyClusterWithConfig( framework.ClusterConfig{ Size: 3, WatchProgressNotifyInterval: 200 * time.Millisecond, DisableStrictReconfigCheck: true, }) func exampleEndpoints() []string { return lazyCluster.EndpointsGRPC() } func forUnitTestsRunInMockedContext(_ func(), example func()) { // For integration tests runs in the provided environment example() } // TestMain sets up an etcd cluster if running the examples. func TestMain(m *testing.M) { testutil.ExitInShortMode("Skipping: the tests require real cluster") tempDir, err := os.MkdirTemp(os.TempDir(), "etcd-integration") if err != nil { log.Printf("Failed to obtain tempDir: %v", tempDir) os.Exit(1) } defer os.RemoveAll(tempDir) err = os.Chdir(tempDir) if err != nil { log.Printf("Failed to change working dir to: %s: %v", tempDir, err) os.Exit(1) } log.Printf("Running tests (examples) in dir(%v): ...", tempDir) v := m.Run() lazyCluster.Terminate() if v == 0 { testutil.MustCheckLeakedGoroutine() } os.Exit(v) } ================================================ FILE: tests/integration/clientv3/experimental/recipes/v3_barrier_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipes_test import ( "testing" "time" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" recipe "go.etcd.io/etcd/client/v3/experimental/recipes" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestBarrierSingleNode(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) testBarrier(t, 5, func() *clientv3.Client { return clus.Client(0) }) } func TestBarrierMultiNode(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) testBarrier(t, 5, func() *clientv3.Client { return clus.RandClient() }) } func testBarrier(t *testing.T, waiters int, chooseClient func() *clientv3.Client) { b := recipe.NewBarrier(chooseClient(), "test-barrier") require.NoErrorf(t, b.Hold(), "could not hold barrier") require.Errorf(t, b.Hold(), "able to double-hold barrier") // put a random key to move the revision forward if _, err := chooseClient().Put(t.Context(), "x", ""); err != nil { t.Errorf("could not put x (%v)", err) } donec := make(chan struct{}) stopc := make(chan struct{}) defer close(stopc) for i := 0; i < waiters; i++ { go func() { br := recipe.NewBarrier(chooseClient(), "test-barrier") if err := br.Wait(); err != nil { t.Errorf("could not wait on barrier (%v)", err) } select { case donec <- struct{}{}: case <-stopc: } }() } select { case <-donec: t.Fatalf("barrier did not wait") default: } require.NoErrorf(t, b.Release(), "could not release barrier") timerC := time.After(time.Duration(waiters*100) * time.Millisecond) for i := 0; i < waiters; i++ { select { case <-timerC: t.Fatalf("barrier timed out") case <-donec: } } } func TestBarrierWaitNonexistentKey(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) if _, err := cli.Put(cli.Ctx(), "test-barrier-0", ""); err != nil { t.Errorf("could not put test-barrier0, err:%v", err) } donec := make(chan struct{}) stopc := make(chan struct{}) defer close(stopc) waiters := 5 for i := 0; i < waiters; i++ { go func() { br := recipe.NewBarrier(cli, "test-barrier") if err := br.Wait(); err != nil { t.Errorf("could not wait on barrier (%v)", err) } select { case donec <- struct{}{}: case <-stopc: } }() } // all waiters should return immediately if waiting on a nonexistent key "test-barrier" even if key "test-barrier-0" exists timerC := time.After(time.Duration(waiters*100) * time.Millisecond) for i := 0; i < waiters; i++ { select { case <-timerC: t.Fatal("barrier timed out") case <-donec: } } } ================================================ FILE: tests/integration/clientv3/experimental/recipes/v3_double_barrier_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipes_test import ( "errors" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" recipe "go.etcd.io/etcd/client/v3/experimental/recipes" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestDoubleBarrier(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) waiters := 10 session, err := concurrency.NewSession(clus.RandClient()) if err != nil { t.Error(err) } defer session.Orphan() b := recipe.NewDoubleBarrier(session, "test-barrier", waiters) donec := make(chan struct{}) defer close(donec) for i := 0; i < waiters-1; i++ { go func() { session, err := concurrency.NewSession(clus.RandClient()) if err != nil { t.Error(err) } defer session.Orphan() bb := recipe.NewDoubleBarrier(session, "test-barrier", waiters) if err := bb.Enter(); err != nil { t.Errorf("could not enter on barrier (%v)", err) } <-donec if err := bb.Leave(); err != nil { t.Errorf("could not leave on barrier (%v)", err) } <-donec }() } time.Sleep(10 * time.Millisecond) select { case donec <- struct{}{}: t.Fatalf("barrier did not enter-wait") default: } require.NoErrorf(t, b.Enter(), "could not enter last barrier") timerC := time.After(time.Duration(waiters*100) * time.Millisecond) for i := 0; i < waiters-1; i++ { select { case <-timerC: t.Fatalf("barrier enter timed out") case donec <- struct{}{}: } } time.Sleep(10 * time.Millisecond) select { case donec <- struct{}{}: t.Fatalf("barrier did not leave-wait") default: } b.Leave() timerC = time.After(time.Duration(waiters*100) * time.Millisecond) for i := 0; i < waiters-1; i++ { select { case <-timerC: t.Fatalf("barrier leave timed out") case donec <- struct{}{}: } } } func TestDoubleBarrierTooManyClients(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) waiters := 10 session, err := concurrency.NewSession(clus.RandClient()) if err != nil { t.Error(err) } defer session.Orphan() b := recipe.NewDoubleBarrier(session, "test-barrier", waiters) donec := make(chan struct{}) var ( wgDone sync.WaitGroup // make sure all clients have finished the tasks wgEntered sync.WaitGroup // make sure all clients have entered the double barrier ) wgDone.Add(waiters) wgEntered.Add(waiters) for i := 0; i < waiters; i++ { go func() { defer wgDone.Done() gsession, gerr := concurrency.NewSession(clus.RandClient()) if gerr != nil { t.Error(gerr) } defer gsession.Orphan() bb := recipe.NewDoubleBarrier(session, "test-barrier", waiters) if gerr = bb.Enter(); gerr != nil { t.Errorf("could not enter on barrier (%v)", gerr) } wgEntered.Done() <-donec if gerr = bb.Leave(); gerr != nil { t.Errorf("could not leave on barrier (%v)", gerr) } }() } // Wait until all clients have already entered the double barrier, so // no any other client can enter the barrier. wgEntered.Wait() t.Log("Try to enter into double barrier") if err = b.Enter(); !errors.Is(err, recipe.ErrTooManyClients) { t.Errorf("Unexcepted error, expected: ErrTooManyClients, got: %v", err) } resp, err := clus.RandClient().Get(t.Context(), "test-barrier/waiters", clientv3.WithPrefix()) if err != nil { t.Errorf("Unexpected error: %v", err) } // Make sure the extra `b.Enter()` did not create a new ephemeral key assert.Len(t, resp.Kvs, waiters) close(donec) wgDone.Wait() } func TestDoubleBarrierFailover(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) waiters := 10 donec := make(chan struct{}) defer close(donec) s0, err := concurrency.NewSession(clus.Client(0)) if err != nil { t.Error(err) } defer s0.Orphan() s1, err := concurrency.NewSession(clus.Client(0)) if err != nil { t.Error(err) } defer s1.Orphan() // sacrificial barrier holder; lease will be revoked go func() { b := recipe.NewDoubleBarrier(s0, "test-barrier", waiters) if berr := b.Enter(); berr != nil { t.Errorf("could not enter on barrier (%v)", berr) } <-donec }() for i := 0; i < waiters-1; i++ { go func() { b := recipe.NewDoubleBarrier(s1, "test-barrier", waiters) if berr := b.Enter(); berr != nil { t.Errorf("could not enter on barrier (%v)", berr) } <-donec b.Leave() <-donec }() } // wait for barrier enter to unblock for i := 0; i < waiters; i++ { select { case donec <- struct{}{}: case <-time.After(10 * time.Second): t.Fatalf("timed out waiting for enter, %d", i) } } require.NoError(t, s0.Close()) // join on rest of waiters for i := 0; i < waiters-1; i++ { select { case donec <- struct{}{}: case <-time.After(10 * time.Second): t.Fatalf("timed out waiting for leave, %d", i) } } } ================================================ FILE: tests/integration/clientv3/experimental/recipes/v3_lock_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipes_test import ( "context" "errors" "fmt" "math/rand" "sync" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" recipe "go.etcd.io/etcd/client/v3/experimental/recipes" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestMutexLockSingleNode(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) var clients []*clientv3.Client testMutexLock(t, 5, integration.MakeSingleNodeClients(t, clus, &clients)) integration.CloseClients(t, clients) } func TestMutexLockMultiNode(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) var clients []*clientv3.Client testMutexLock(t, 5, integration.MakeMultiNodeClients(t, clus, &clients)) integration.CloseClients(t, clients) } func testMutexLock(t *testing.T, waiters int, chooseClient func() *clientv3.Client) { // stream lock acquisitions lockedC := make(chan *concurrency.Mutex, waiters) errC := make(chan error, waiters) var wg sync.WaitGroup wg.Add(waiters) for i := 0; i < waiters; i++ { go func(i int) { defer wg.Done() session, err := concurrency.NewSession(chooseClient()) if err != nil { errC <- fmt.Errorf("#%d: failed to create new session: %w", i, err) return } m := concurrency.NewMutex(session, "test-mutex") if err := m.Lock(t.Context()); err != nil { errC <- fmt.Errorf("#%d: failed to wait on lock: %w", i, err) return } lockedC <- m }(i) } // unlock locked mutexes timerC := time.After(time.Duration(waiters) * time.Second) for i := 0; i < waiters; i++ { select { case <-timerC: t.Fatalf("timed out waiting for lock %d", i) case err := <-errC: t.Fatalf("Unexpected error: %v", err) case m := <-lockedC: // lock acquired with m select { case <-lockedC: t.Fatalf("lock %d followers did not wait", i) default: } require.NoErrorf(t, m.Unlock(t.Context()), "could not release lock") } } wg.Wait() } func TestMutexTryLockSingleNode(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) t.Logf("3 nodes cluster created...") var clients []*clientv3.Client testMutexTryLock(t, 5, integration.MakeSingleNodeClients(t, clus, &clients)) integration.CloseClients(t, clients) } func TestMutexTryLockMultiNode(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) var clients []*clientv3.Client testMutexTryLock(t, 5, integration.MakeMultiNodeClients(t, clus, &clients)) integration.CloseClients(t, clients) } func testMutexTryLock(t *testing.T, lockers int, chooseClient func() *clientv3.Client) { ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() lockedC := make(chan *concurrency.Mutex) notlockedC := make(chan *concurrency.Mutex) for i := 0; i < lockers; i++ { go func(i int) { session, err := concurrency.NewSession(chooseClient()) if err != nil { t.Error(err) } m := concurrency.NewMutex(session, "test-mutex-try-lock") err = m.TryLock(ctx) if err == nil { select { case lockedC <- m: case <-ctx.Done(): t.Errorf("Thread: %v, Context failed: %v", i, err) } } else if errors.Is(err, concurrency.ErrLocked) { select { case notlockedC <- m: case <-ctx.Done(): t.Errorf("Thread: %v, Context failed: %v", i, err) } } else { t.Errorf("Thread: %v; Unexpected Error %v", i, err) } }(i) } timerC := time.After(30 * time.Second) select { case <-lockedC: for i := 0; i < lockers-1; i++ { select { case <-lockedC: t.Fatalf("Multiple Mutes locked on same key") case <-notlockedC: case <-timerC: t.Errorf("timed out waiting for lock") } } case <-timerC: t.Errorf("timed out waiting for lock (30s)") } } // TestMutexSessionRelock ensures that acquiring the same lock with the same // session will not result in deadlock. func TestMutexSessionRelock(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) session, err := concurrency.NewSession(clus.RandClient()) if err != nil { t.Error(err) } m := concurrency.NewMutex(session, "test-mutex") require.NoError(t, m.Lock(t.Context())) m2 := concurrency.NewMutex(session, "test-mutex") require.NoError(t, m2.Lock(t.Context())) } // TestMutexWaitsOnCurrentHolder ensures a mutex is only acquired once all // waiters older than the new owner are gone by testing the case where // the waiter prior to the acquirer expires before the current holder. func TestMutexWaitsOnCurrentHolder(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cctx := t.Context() cli := clus.Client(0) firstOwnerSession, err := concurrency.NewSession(cli) if err != nil { t.Error(err) } defer firstOwnerSession.Close() firstOwnerMutex := concurrency.NewMutex(firstOwnerSession, "test-mutex") require.NoError(t, firstOwnerMutex.Lock(cctx)) victimSession, err := concurrency.NewSession(cli) if err != nil { t.Error(err) } defer victimSession.Close() victimDonec := make(chan struct{}) go func() { defer close(victimDonec) concurrency.NewMutex(victimSession, "test-mutex").Lock(cctx) }() // ensure mutexes associated with firstOwnerSession and victimSession waits before new owner wch := cli.Watch(cctx, "test-mutex", clientv3.WithPrefix(), clientv3.WithRev(1)) putCounts := 0 for putCounts < 2 { select { case wrp := <-wch: putCounts += len(wrp.Events) case <-time.After(time.Second): t.Fatal("failed to receive watch response") } } require.Equalf(t, 2, putCounts, "expect 2 put events, but got %v", putCounts) newOwnerSession, err := concurrency.NewSession(cli) if err != nil { t.Error(err) } defer newOwnerSession.Close() newOwnerDonec := make(chan struct{}) go func() { defer close(newOwnerDonec) concurrency.NewMutex(newOwnerSession, "test-mutex").Lock(cctx) }() select { case wrp := <-wch: require.Lenf(t, wrp.Events, 1, "expect a event, but got %v events", len(wrp.Events)) e := wrp.Events[0] require.Equalf(t, mvccpb.Event_PUT, e.Type, "expect a put event on prefix test-mutex, but got event type %v", e.Type) case <-time.After(time.Second): t.Fatalf("failed to receive a watch response") } // simulate losing the client that's next in line to acquire the lock victimSession.Close() // ensures the deletion of victim waiter from server side. select { case wrp := <-wch: require.Lenf(t, wrp.Events, 1, "expect a event, but got %v events", len(wrp.Events)) e := wrp.Events[0] require.Equalf(t, mvccpb.Event_DELETE, e.Type, "expect a delete event on prefix test-mutex, but got event type %v", e.Type) case <-time.After(time.Second): t.Fatal("failed to receive a watch response") } select { case <-newOwnerDonec: t.Fatal("new owner obtained lock before first owner unlocked") default: } require.NoError(t, firstOwnerMutex.Unlock(cctx)) select { case <-newOwnerDonec: case <-time.After(time.Second): t.Fatal("new owner failed to obtain lock") } select { case <-victimDonec: case <-time.After(time.Second): t.Fatal("victim mutex failed to exit after first owner releases lock") } } func BenchmarkMutex4Waiters(b *testing.B) { integration.BeforeTest(b) // XXX switch tests to use TB interface clus := integration.NewCluster(nil, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(nil) for i := 0; i < b.N; i++ { testMutexLock(nil, 4, func() *clientv3.Client { return clus.RandClient() }) } } func TestRWMutexSingleNode(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) testRWMutex(t, 5, func() *clientv3.Client { return clus.Client(0) }) } func TestRWMutexMultiNode(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) testRWMutex(t, 5, func() *clientv3.Client { return clus.RandClient() }) } func testRWMutex(t *testing.T, waiters int, chooseClient func() *clientv3.Client) { // stream rwlock acquistions rlockedC := make(chan *recipe.RWMutex, 1) wlockedC := make(chan *recipe.RWMutex, 1) for i := 0; i < waiters; i++ { go func() { session, err := concurrency.NewSession(chooseClient()) if err != nil { t.Error(err) } rwm := recipe.NewRWMutex(session, "test-rwmutex") if rand.Intn(2) == 0 { if err := rwm.RLock(); err != nil { t.Errorf("could not rlock (%v)", err) } rlockedC <- rwm } else { if err := rwm.Lock(); err != nil { t.Errorf("could not lock (%v)", err) } wlockedC <- rwm } }() } // unlock locked rwmutexes timerC := time.After(time.Duration(waiters) * time.Second) for i := 0; i < waiters; i++ { select { case <-timerC: t.Fatalf("timed out waiting for lock %d", i) case wl := <-wlockedC: select { case <-rlockedC: t.Fatalf("rlock %d readers did not wait", i) default: } require.NoErrorf(t, wl.Unlock(), "could not release lock") case rl := <-rlockedC: select { case <-wlockedC: t.Fatalf("rlock %d writers did not wait", i) default: } require.NoErrorf(t, rl.RUnlock(), "could not release rlock") } } } ================================================ FILE: tests/integration/clientv3/experimental/recipes/v3_queue_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package recipes_test import ( "fmt" "math/rand" "sync/atomic" "testing" "github.com/stretchr/testify/require" recipe "go.etcd.io/etcd/client/v3/experimental/recipes" "go.etcd.io/etcd/tests/v3/framework/integration" ) const ( manyQueueClients = 3 queueItemsPerClient = 2 ) // TestQueueOneReaderOneWriter confirms the queue is FIFO func TestQueueOneReaderOneWriter(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) done := make(chan struct{}) defer func() { <-done }() go func() { defer func() { done <- struct{}{} }() etcdc := clus.RandClient() q := recipe.NewQueue(etcdc, "testq") for i := 0; i < 5; i++ { if err := q.Enqueue(fmt.Sprintf("%d", i)); err != nil { t.Errorf("error enqueuing (%v)", err) } } }() etcdc := clus.RandClient() q := recipe.NewQueue(etcdc, "testq") for i := 0; i < 5; i++ { s, err := q.Dequeue() require.NoErrorf(t, err, "error dequeueing (%v)", err) require.Equalf(t, s, fmt.Sprintf("%d", i), "expected dequeue value %v, got %v", s, i) } } func TestQueueManyReaderOneWriter(t *testing.T) { testQueueNReaderMWriter(t, manyQueueClients, 1) } func TestQueueOneReaderManyWriter(t *testing.T) { testQueueNReaderMWriter(t, 1, manyQueueClients) } func TestQueueManyReaderManyWriter(t *testing.T) { testQueueNReaderMWriter(t, manyQueueClients, manyQueueClients) } // BenchmarkQueue benchmarks Queues using many/many readers/writers func BenchmarkQueue(b *testing.B) { integration.BeforeTest(b) // XXX switch tests to use TB interface clus := integration.NewCluster(nil, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(nil) for i := 0; i < b.N; i++ { testQueueNReaderMWriter(nil, manyQueueClients, manyQueueClients) } } // TestPrQueueOneReaderOneWriter tests whether priority queues respect priorities. func TestPrQueueOneReaderOneWriter(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) // write out five items with random priority etcdc := clus.RandClient() q := recipe.NewPriorityQueue(etcdc, "testprq") for i := 0; i < 5; i++ { // [0, 2] priority for priority collision to test seq keys pr := uint16(rand.Intn(3)) require.NoErrorf(t, q.Enqueue(fmt.Sprintf("%d", pr), pr), "error enqueuing") } // read back items; confirm priority order is respected lastPr := -1 for i := 0; i < 5; i++ { s, err := q.Dequeue() require.NoErrorf(t, err, "error dequeueing (%v)", err) curPr := 0 _, err = fmt.Sscanf(s, "%d", &curPr) require.NoErrorf(t, err, `error parsing item "%s" (%v)`, s, err) require.LessOrEqualf(t, lastPr, curPr, "expected priority %v > %v", curPr, lastPr) } } func TestPrQueueManyReaderManyWriter(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) rqs := newPriorityQueues(clus, manyQueueClients) wqs := newPriorityQueues(clus, manyQueueClients) testReadersWriters(t, rqs, wqs) } // BenchmarkPrQueueOneReaderOneWriter benchmarks Queues using n/n readers/writers func BenchmarkPrQueueOneReaderOneWriter(b *testing.B) { integration.BeforeTest(b) // XXX switch tests to use TB interface clus := integration.NewCluster(nil, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(nil) rqs := newPriorityQueues(clus, 1) wqs := newPriorityQueues(clus, 1) for i := 0; i < b.N; i++ { testReadersWriters(nil, rqs, wqs) } } func testQueueNReaderMWriter(t *testing.T, n int, m int) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) testReadersWriters(t, newQueues(clus, n), newQueues(clus, m)) } func newQueues(clus *integration.Cluster, n int) (qs []testQueue) { for i := 0; i < n; i++ { etcdc := clus.RandClient() qs = append(qs, recipe.NewQueue(etcdc, "q")) } return qs } func newPriorityQueues(clus *integration.Cluster, n int) (qs []testQueue) { for i := 0; i < n; i++ { etcdc := clus.RandClient() q := &flatPriorityQueue{recipe.NewPriorityQueue(etcdc, "prq")} qs = append(qs, q) } return qs } func testReadersWriters(t *testing.T, rqs []testQueue, wqs []testQueue) { rerrc := make(chan error) werrc := make(chan error) manyWriters(wqs, queueItemsPerClient, werrc) manyReaders(rqs, len(wqs)*queueItemsPerClient, rerrc) for range wqs { if err := <-werrc; err != nil { t.Errorf("error writing (%v)", err) } } for range rqs { if err := <-rerrc; err != nil { t.Errorf("error reading (%v)", err) } } } func manyReaders(qs []testQueue, totalReads int, errc chan<- error) { var rxReads int32 for _, q := range qs { go func(q testQueue) { for { total := atomic.AddInt32(&rxReads, 1) if int(total) > totalReads { break } if _, err := q.Dequeue(); err != nil { errc <- err return } } errc <- nil }(q) } } func manyWriters(qs []testQueue, writesEach int, errc chan<- error) { for _, q := range qs { go func(q testQueue) { for j := 0; j < writesEach; j++ { if err := q.Enqueue("foo"); err != nil { errc <- err return } } errc <- nil }(q) } } type testQueue interface { Enqueue(val string) error Dequeue() (string, error) } type flatPriorityQueue struct{ *recipe.PriorityQueue } func (q *flatPriorityQueue) Enqueue(val string) error { // randomized to stress dequeuing logic; order isn't important return q.PriorityQueue.Enqueue(val, uint16(rand.Intn(2))) } func (q *flatPriorityQueue) Dequeue() (string, error) { return q.PriorityQueue.Dequeue() } ================================================ FILE: tests/integration/clientv3/kv_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "bytes" "context" "errors" "fmt" "os" "reflect" "strconv" "strings" "testing" "time" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/api/v3/version" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestKVPutError(t *testing.T) { integration.BeforeTest(t) var ( maxReqBytes = 1.5 * 1024 * 1024 // hard coded max in v3_server.go quota = int64(int(maxReqBytes*1.2) + 8*os.Getpagesize()) // make sure we have enough overhead in backend quota. See discussion in #6486. ) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, QuotaBackendBytes: quota, ClientMaxCallSendMsgSize: 100 * 1024 * 1024}) defer clus.Terminate(t) kv := clus.RandClient() ctx := t.Context() _, err := kv.Put(ctx, "", "bar") if !errors.Is(err, rpctypes.ErrEmptyKey) { t.Fatalf("expected %v, got %v", rpctypes.ErrEmptyKey, err) } _, err = kv.Put(ctx, "key", strings.Repeat("a", int(maxReqBytes+100))) if !errors.Is(err, rpctypes.ErrRequestTooLarge) { t.Fatalf("expected %v, got %v", rpctypes.ErrRequestTooLarge, err) } _, err = kv.Put(ctx, "foo1", strings.Repeat("a", int(maxReqBytes-50))) require.NoError(t, err) // below quota time.Sleep(1 * time.Second) // give enough time for commit _, err = kv.Put(ctx, "foo2", strings.Repeat("a", int(maxReqBytes-50))) if !errors.Is(err, rpctypes.ErrNoSpace) { // over quota t.Fatalf("expected %v, got %v", rpctypes.ErrNoSpace, err) } } func TestKVPutWithLease(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) lapi := clus.RandClient() kv := clus.RandClient() ctx := t.Context() lease, err := lapi.Grant(t.Context(), 10) if err != nil { t.Fatalf("failed to create lease %v", err) } key := "hello" val := "world" if _, err = kv.Put(ctx, key, val, clientv3.WithLease(lease.ID)); err != nil { t.Fatalf("couldn't put %q (%v)", key, err) } resp, err := kv.Get(ctx, key) if err != nil { t.Fatalf("couldn't get key (%v)", err) } if len(resp.Kvs) != 1 { t.Fatalf("expected 1 key, got %d", len(resp.Kvs)) } if !bytes.Equal([]byte(val), resp.Kvs[0].Value) { t.Errorf("val = %s, want %s", val, resp.Kvs[0].Value) } if lease.ID != clientv3.LeaseID(resp.Kvs[0].Lease) { t.Errorf("val = %d, want %d", lease.ID, resp.Kvs[0].Lease) } } // TestKVPutWithIgnoreValue ensures that Put with WithIgnoreValue does not clobber the old value. func TestKVPutWithIgnoreValue(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kv := clus.RandClient() _, err := kv.Put(t.Context(), "foo", "", clientv3.WithIgnoreValue()) if !errors.Is(err, rpctypes.ErrKeyNotFound) { t.Fatalf("err expected %v, got %v", rpctypes.ErrKeyNotFound, err) } _, err = kv.Put(t.Context(), "foo", "bar") require.NoError(t, err) _, err = kv.Put(t.Context(), "foo", "", clientv3.WithIgnoreValue()) require.NoError(t, err) rr, rerr := kv.Get(t.Context(), "foo") require.NoError(t, rerr) if len(rr.Kvs) != 1 { t.Fatalf("len(rr.Kvs) expected 1, got %d", len(rr.Kvs)) } if !bytes.Equal(rr.Kvs[0].Value, []byte("bar")) { t.Fatalf("value expected 'bar', got %q", rr.Kvs[0].Value) } } // TestKVPutWithIgnoreLease ensures that Put with WithIgnoreLease does not affect the existing lease for the key. func TestKVPutWithIgnoreLease(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kv := clus.RandClient() lapi := clus.RandClient() resp, err := lapi.Grant(t.Context(), 10) if err != nil { t.Errorf("failed to create lease %v", err) } if _, err = kv.Put(t.Context(), "zoo", "bar", clientv3.WithIgnoreLease()); !errors.Is(err, rpctypes.ErrKeyNotFound) { t.Fatalf("err expected %v, got %v", rpctypes.ErrKeyNotFound, err) } _, err = kv.Put(t.Context(), "zoo", "bar", clientv3.WithLease(resp.ID)) require.NoError(t, err) _, err = kv.Put(t.Context(), "zoo", "bar1", clientv3.WithIgnoreLease()) require.NoError(t, err) rr, rerr := kv.Get(t.Context(), "zoo") require.NoError(t, rerr) if len(rr.Kvs) != 1 { t.Fatalf("len(rr.Kvs) expected 1, got %d", len(rr.Kvs)) } if rr.Kvs[0].Lease != int64(resp.ID) { t.Fatalf("lease expected %v, got %v", resp.ID, rr.Kvs[0].Lease) } } func TestKVPutWithRequireLeader(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) clus.Members[1].Stop(t) clus.Members[2].Stop(t) // wait for election timeout, then member[0] will not have a leader. var ( electionTicks = 10 tickDuration = 10 * time.Millisecond ) time.Sleep(time.Duration(3*electionTicks) * tickDuration) kv := clus.Client(0) _, err := kv.Put(clientv3.WithRequireLeader(t.Context()), "foo", "bar") if !errors.Is(err, rpctypes.ErrNoLeader) { t.Fatal(err) } cnt, err := clus.Members[0].Metric( "etcd_server_client_requests_total", `type="unary"`, fmt.Sprintf(`client_api_version="%v"`, version.APIVersion), ) require.NoError(t, err) cv, err := strconv.ParseInt(cnt, 10, 32) require.NoError(t, err) if cv < 1 { // >1 when retried t.Fatalf("expected at least 1, got %q", cnt) } // clients may give timeout errors since the members are stopped; take // the clients so that terminating the cluster won't complain clus.Client(1).Close() clus.Client(2).Close() clus.TakeClient(1) clus.TakeClient(2) } func TestKVRange(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kv := clus.RandClient() ctx := t.Context() keySet := []string{"a", "b", "c", "c", "c", "foo", "foo/abc", "fop"} for i, key := range keySet { if _, err := kv.Put(ctx, key, ""); err != nil { t.Fatalf("#%d: couldn't put %q (%v)", i, key, err) } } resp, err := kv.Get(ctx, keySet[0]) if err != nil { t.Fatalf("couldn't get key (%v)", err) } wheader := resp.Header tests := []struct { begin, end string rev int64 opts []clientv3.OpOption wantSet []*mvccpb.KeyValue }{ // fetch entire keyspace using WithFromKey { "\x00", "", 0, []clientv3.OpOption{clientv3.WithFromKey(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)}, []*mvccpb.KeyValue{ {Key: []byte("a"), Value: nil, CreateRevision: 2, ModRevision: 2, Version: 1}, {Key: []byte("b"), Value: nil, CreateRevision: 3, ModRevision: 3, Version: 1}, {Key: []byte("c"), Value: nil, CreateRevision: 4, ModRevision: 6, Version: 3}, {Key: []byte("foo"), Value: nil, CreateRevision: 7, ModRevision: 7, Version: 1}, {Key: []byte("foo/abc"), Value: nil, CreateRevision: 8, ModRevision: 8, Version: 1}, {Key: []byte("fop"), Value: nil, CreateRevision: 9, ModRevision: 9, Version: 1}, }, }, } for i, tt := range tests { opts := []clientv3.OpOption{clientv3.WithRange(tt.end), clientv3.WithRev(tt.rev)} opts = append(opts, tt.opts...) resp, err := kv.Get(ctx, tt.begin, opts...) if err != nil { t.Fatalf("#%d: couldn't range (%v)", i, err) } if !reflect.DeepEqual(wheader, resp.Header) { t.Fatalf("#%d: wheader expected %+v, got %+v", i, wheader, resp.Header) } if !reflect.DeepEqual(tt.wantSet, resp.Kvs) { t.Fatalf("#%d: resp.Kvs expected %+v, got %+v", i, tt.wantSet, resp.Kvs) } } } func TestKVGetErrConnClosed(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) donec := make(chan struct{}) require.NoError(t, cli.Close()) clus.TakeClient(0) go func() { defer close(donec) _, err := cli.Get(t.Context(), "foo") if !clientv3.IsConnCanceled(err) { t.Errorf("expected %v, got %v", context.Canceled, err) } }() select { case <-time.After(integration.RequestWaitTimeout): t.Fatal("kv.Get took too long") case <-donec: } } func TestKVNewAfterClose(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) clus.TakeClient(0) require.NoError(t, cli.Close()) donec := make(chan struct{}) go func() { _, err := cli.Get(t.Context(), "foo") if !clientv3.IsConnCanceled(err) { t.Errorf("expected %v, got %v", context.Canceled, err) } close(donec) }() select { case <-time.After(integration.RequestWaitTimeout): t.Fatal("kv.Get took too long") case <-donec: } } func TestKVDeleteRange(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kv := clus.RandClient() ctx := t.Context() tests := []struct { key string opts []clientv3.OpOption wkeys []string }{ // * { key: "\x00", opts: []clientv3.OpOption{clientv3.WithFromKey()}, wkeys: nil, }, } for i, tt := range tests { keySet := []string{"a", "b", "c", "c/abc", "d"} for j, key := range keySet { if _, err := kv.Put(ctx, key, ""); err != nil { t.Fatalf("#%d: couldn't put %q (%v)", j, key, err) } } _, err := kv.Delete(ctx, tt.key, tt.opts...) if err != nil { t.Fatalf("#%d: couldn't delete range (%v)", i, err) } resp, err := kv.Get(ctx, "a", clientv3.WithFromKey()) if err != nil { t.Fatalf("#%d: couldn't get keys (%v)", i, err) } var keys []string for _, kv := range resp.Kvs { keys = append(keys, string(kv.Key)) } if !reflect.DeepEqual(tt.wkeys, keys) { t.Errorf("#%d: resp.Kvs got %v, expected %v", i, keys, tt.wkeys) } } } func TestKVCompactError(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kv := clus.RandClient() ctx := t.Context() for i := 0; i < 5; i++ { if _, err := kv.Put(ctx, "foo", "bar"); err != nil { t.Fatalf("couldn't put 'foo' (%v)", err) } } _, err := kv.Compact(ctx, 6) if err != nil { t.Fatalf("couldn't compact 6 (%v)", err) } _, err = kv.Compact(ctx, 6) if !errors.Is(err, rpctypes.ErrCompacted) { t.Fatalf("expected %v, got %v", rpctypes.ErrCompacted, err) } _, err = kv.Compact(ctx, 100) if !errors.Is(err, rpctypes.ErrFutureRev) { t.Fatalf("expected %v, got %v", rpctypes.ErrFutureRev, err) } } func TestKVCompact(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kv := clus.RandClient() ctx := t.Context() for i := 0; i < 10; i++ { if _, err := kv.Put(ctx, "foo", "bar"); err != nil { t.Fatalf("couldn't put 'foo' (%v)", err) } } _, err := kv.Compact(ctx, 7) if err != nil { t.Fatalf("couldn't compact kv space (%v)", err) } _, err = kv.Compact(ctx, 7) if err == nil || !errors.Is(err, rpctypes.ErrCompacted) { t.Fatalf("error got %v, want %v", err, rpctypes.ErrCompacted) } wcli := clus.RandClient() // new watcher could precede receiving the compaction without quorum first wcli.Get(ctx, "quorum-get") wchan := wcli.Watch(ctx, "foo", clientv3.WithRev(3)) wr := <-wchan if wr.CompactRevision != 7 { t.Fatalf("wchan CompactRevision got %v, want 7", wr.CompactRevision) } if !wr.Canceled { t.Fatalf("expected canceled watcher on compacted revision, got %v", wr.Canceled) } if !errors.Is(wr.Err(), rpctypes.ErrCompacted) { t.Fatalf("watch response error expected %v, got %v", rpctypes.ErrCompacted, wr.Err()) } wr, ok := <-wchan if ok { t.Fatalf("wchan got %v, expected closed", wr) } if wr.Err() != nil { t.Fatalf("watch response error expected nil, got %v", wr.Err()) } _, err = kv.Compact(ctx, 1000) if err == nil || !errors.Is(err, rpctypes.ErrFutureRev) { t.Fatalf("error got %v, want %v", err, rpctypes.ErrFutureRev) } } // TestKVGetRetry ensures get will retry on disconnect. func TestKVGetRetry(t *testing.T) { integration.BeforeTest(t) clusterSize := 3 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: clusterSize, UseBridge: true}) defer clus.Terminate(t) // because killing leader and following election // could give no other endpoints for client reconnection fIdx := (clus.WaitLeader(t) + 1) % clusterSize kv := clus.Client(fIdx) ctx := t.Context() _, err := kv.Put(ctx, "foo", "bar") require.NoError(t, err) clus.Members[fIdx].Stop(t) donec := make(chan struct{}, 1) go func() { // Get will fail, but reconnect will trigger gresp, gerr := kv.Get(ctx, "foo") if gerr != nil { t.Error(gerr) } wkvs := []*mvccpb.KeyValue{ { Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1, }, } if !reflect.DeepEqual(gresp.Kvs, wkvs) { t.Errorf("bad get: got %v, want %v", gresp.Kvs, wkvs) } donec <- struct{}{} }() time.Sleep(100 * time.Millisecond) clus.Members[fIdx].Restart(t) clus.Members[fIdx].WaitOK(t) select { case <-time.After(20 * time.Second): t.Fatalf("timed out waiting for get") case <-donec: } } // TestKVPutFailGetRetry ensures a get will retry following a failed put. func TestKVPutFailGetRetry(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) kv := clus.Client(0) clus.Members[0].Stop(t) ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() _, err := kv.Put(ctx, "foo", "bar") if err == nil { t.Fatalf("got success on disconnected put, wanted error") } donec := make(chan struct{}, 1) go func() { // Get will fail, but reconnect will trigger gresp, gerr := kv.Get(t.Context(), "foo") if gerr != nil { t.Error(gerr) } if len(gresp.Kvs) != 0 { t.Errorf("bad get kvs: got %+v, want empty", gresp.Kvs) } donec <- struct{}{} }() time.Sleep(100 * time.Millisecond) clus.Members[0].Restart(t) select { case <-time.After(20 * time.Second): t.Fatalf("timed out waiting for get") case <-donec: } } // TestKVGetCancel tests that a context cancel on a Get terminates as expected. func TestKVGetCancel(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) oldconn := clus.Client(0).ActiveConnection() kv := clus.Client(0) ctx, cancel := context.WithCancel(t.Context()) cancel() resp, err := kv.Get(ctx, "abc") if err == nil { t.Fatalf("cancel on get response %v, expected context error", resp) } newconn := clus.Client(0).ActiveConnection() if oldconn != newconn { t.Fatalf("cancel on get broke client connection") } } // TestKVGetStoppedServerAndClose ensures closing after a failed Get works. func TestKVGetStoppedServerAndClose(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) clus.Members[0].Stop(t) ctx, cancel := context.WithTimeout(t.Context(), time.Second) // this Get fails and triggers an asynchronous connection retry _, err := cli.Get(ctx, "abc") cancel() if err != nil && !(IsCanceled(err) || IsClientTimeout(err)) { t.Fatal(err) } } // TestKVPutStoppedServerAndClose ensures closing after a failed Put works. func TestKVPutStoppedServerAndClose(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) clus.Members[0].Stop(t) ctx, cancel := context.WithTimeout(t.Context(), time.Second) // get retries on all errors. // so here we use it to eat the potential broken pipe error for the next put. // grpc client might see a broken pipe error when we issue the get request before // grpc finds out the original connection is down due to the member shutdown. _, err := cli.Get(ctx, "abc") cancel() if err != nil && !(IsCanceled(err) || IsClientTimeout(err)) { t.Fatal(err) } ctx, cancel = context.WithTimeout(t.Context(), time.Second) // this Put fails and triggers an asynchronous connection retry _, err = cli.Put(ctx, "abc", "123") cancel() if err != nil && !(IsCanceled(err) || IsClientTimeout(err) || IsUnavailable(err)) { t.Fatal(err) } } // TestKVPutAtMostOnce ensures that a Put will only occur at most once // in the presence of network errors. func TestKVPutAtMostOnce(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) _, err := clus.Client(0).Put(t.Context(), "k", "1") require.NoError(t, err) for i := 0; i < 10; i++ { clus.Members[0].Bridge().DropConnections() donec := make(chan struct{}) go func() { defer close(donec) for i := 0; i < 10; i++ { clus.Members[0].Bridge().DropConnections() time.Sleep(5 * time.Millisecond) } }() _, err = clus.Client(0).Put(t.Context(), "k", "v") <-donec if err != nil { break } } resp, err := clus.Client(0).Get(t.Context(), "k") require.NoError(t, err) if resp.Kvs[0].Version > 11 { t.Fatalf("expected version <= 10, got %+v", resp.Kvs[0]) } } // TestKVLargeRequests tests various client/server side request limits. func TestKVLargeRequests(t *testing.T) { integration.BeforeTest(t) tests := []struct { // make sure that "MaxCallSendMsgSize" < server-side default send/recv limit maxRequestBytesServer uint maxCallSendBytesClient int maxCallRecvBytesClient int valueSize int expectError error }{ { maxRequestBytesServer: 256, maxCallSendBytesClient: 0, maxCallRecvBytesClient: 0, valueSize: 1024, expectError: rpctypes.ErrRequestTooLarge, }, // without proper client-side receive size limit // "code = ResourceExhausted desc = grpc: received message larger than max (5242929 vs. 4194304)" { maxRequestBytesServer: 7*1024*1024 + 512*1024, maxCallSendBytesClient: 7 * 1024 * 1024, maxCallRecvBytesClient: 0, valueSize: 5 * 1024 * 1024, expectError: nil, }, { maxRequestBytesServer: 10 * 1024 * 1024, maxCallSendBytesClient: 100 * 1024 * 1024, maxCallRecvBytesClient: 0, valueSize: 10 * 1024 * 1024, expectError: rpctypes.ErrRequestTooLarge, }, { maxRequestBytesServer: 10 * 1024 * 1024, maxCallSendBytesClient: 10 * 1024 * 1024, maxCallRecvBytesClient: 0, valueSize: 10 * 1024 * 1024, expectError: status.Errorf(codes.ResourceExhausted, "trying to send message larger than max "), }, { maxRequestBytesServer: 10 * 1024 * 1024, maxCallSendBytesClient: 100 * 1024 * 1024, maxCallRecvBytesClient: 0, valueSize: 10*1024*1024 + 5, expectError: rpctypes.ErrRequestTooLarge, }, { maxRequestBytesServer: 10 * 1024 * 1024, maxCallSendBytesClient: 10 * 1024 * 1024, maxCallRecvBytesClient: 0, valueSize: 10*1024*1024 + 5, expectError: status.Errorf(codes.ResourceExhausted, "trying to send message larger than max "), }, } for i, test := range tests { clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 1, MaxRequestBytes: test.maxRequestBytesServer, ClientMaxCallSendMsgSize: test.maxCallSendBytesClient, ClientMaxCallRecvMsgSize: test.maxCallRecvBytesClient, }, ) cli := clus.Client(0) _, err := cli.Put(t.Context(), "foo", strings.Repeat("a", test.valueSize)) var etcdErr rpctypes.EtcdError if errors.As(err, &etcdErr) { if !errors.Is(err, test.expectError) { t.Errorf("#%d: expected %v, got %v", i, test.expectError, err) } } else if err != nil && !strings.HasPrefix(err.Error(), test.expectError.Error()) { t.Errorf("#%d: expected error starting with '%s', got '%s'", i, test.expectError.Error(), err.Error()) } // put request went through, now expects large response back if err == nil { _, err = cli.Get(t.Context(), "foo") if err != nil { t.Errorf("#%d: get expected no error, got %v", i, err) } } clus.Terminate(t) } } // TestKVForLearner ensures learner member only accepts serializable read request. func TestKVForLearner(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) defer clus.Terminate(t) // we have to add and launch learner member after initial cluster was created, because // bootstrapping a cluster with learner member is not supported. clus.AddAndLaunchLearnerMember(t) learners, err := clus.GetLearnerMembers() if err != nil { t.Fatalf("failed to get the learner members in cluster: %v", err) } if len(learners) != 1 { t.Fatalf("added 1 learner to cluster, got %d", len(learners)) } if len(clus.Members) != 4 { t.Fatalf("expecting 4 members in cluster after adding the learner member, got %d", len(clus.Members)) } // note: // 1. clus.Members[3] is the newly added learner member, which was appended to clus.Members // 2. we are using member's grpcAddr instead of clientURLs as the endpoint for clientv3.Config, // because the implementation of integration test has diverged from embed/etcd.go. learnerEp := clus.Members[3].GRPCURL cfg := clientv3.Config{ Endpoints: []string{learnerEp}, DialTimeout: 5 * time.Second, } // this client only has endpoint of the learner member cli, err := integration.NewClient(t, cfg) if err != nil { t.Fatalf("failed to create clientv3: %v", err) } defer cli.Close() // wait until learner member is ready <-clus.Members[3].ReadyNotify() tests := []struct { op clientv3.Op wErr bool }{ { op: clientv3.OpGet("foo", clientv3.WithSerializable()), wErr: false, }, { op: clientv3.OpGet("foo"), wErr: true, }, { op: clientv3.OpPut("foo", "bar"), wErr: true, }, { op: clientv3.OpDelete("foo"), wErr: true, }, { op: clientv3.OpTxn([]clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("foo"), "=", 0)}, nil, nil), wErr: true, }, } for idx, test := range tests { _, err := cli.Do(t.Context(), test.op) if err != nil && !test.wErr { t.Errorf("%d: expect no error, got %v", idx, err) } if err == nil && test.wErr { t.Errorf("%d: expect error, got nil", idx) } } } // TestBalancerSupportLearner verifies that balancer's retry and failover mechanism supports cluster with learner member func TestBalancerSupportLearner(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) defer clus.Terminate(t) // we have to add and launch learner member after initial cluster was created, because // bootstrapping a cluster with learner member is not supported. clus.AddAndLaunchLearnerMember(t) learners, err := clus.GetLearnerMembers() if err != nil { t.Fatalf("failed to get the learner members in cluster: %v", err) } if len(learners) != 1 { t.Fatalf("added 1 learner to cluster, got %d", len(learners)) } // clus.Members[3] is the newly added learner member, which was appended to clus.Members learnerEp := clus.Members[3].GRPCURL cfg := clientv3.Config{ Endpoints: []string{learnerEp}, DialTimeout: 5 * time.Second, } cli, err := integration.NewClient(t, cfg) if err != nil { t.Fatalf("failed to create clientv3: %v", err) } defer cli.Close() // wait until learner member is ready <-clus.Members[3].ReadyNotify() if _, err = cli.Get(t.Context(), "foo"); err == nil { t.Fatalf("expect Get request to learner to fail, got no error") } t.Logf("Expected: Read from learner error: %v", err) eps := []string{learnerEp, clus.Members[0].GRPCURL} cli.SetEndpoints(eps...) if _, err := cli.Get(t.Context(), "foo"); err != nil { t.Errorf("expect no error (balancer should retry when request to learner fails), got error: %v", err) } } ================================================ FILE: tests/integration/clientv3/lease/doc.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lease ================================================ FILE: tests/integration/clientv3/lease/lease_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lease_test import ( "context" "errors" "fmt" "reflect" "sort" "strconv" "sync" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestLeaseNotFoundError(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kv := clus.RandClient() _, err := kv.Put(t.Context(), "foo", "bar", clientv3.WithLease(clientv3.LeaseID(500))) require.ErrorIsf(t, err, rpctypes.ErrLeaseNotFound, "expected %v, got %v", rpctypes.ErrLeaseNotFound, err) } func TestLeaseGrant(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) lapi := clus.RandClient() kv := clus.RandClient() _, merr := lapi.Grant(t.Context(), clientv3.MaxLeaseTTL+1) require.ErrorIsf(t, merr, rpctypes.ErrLeaseTTLTooLarge, "err = %v, want %v", merr, rpctypes.ErrLeaseTTLTooLarge) resp, err := lapi.Grant(t.Context(), 10) if err != nil { t.Errorf("failed to create lease %v", err) } _, err = kv.Put(t.Context(), "foo", "bar", clientv3.WithLease(resp.ID)) require.NoErrorf(t, err, "failed to create key with lease %v", err) } func TestLeaseRevoke(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) lapi := clus.RandClient() kv := clus.RandClient() resp, err := lapi.Grant(t.Context(), 10) if err != nil { t.Errorf("failed to create lease %v", err) } _, err = lapi.Revoke(t.Context(), resp.ID) if err != nil { t.Errorf("failed to revoke lease %v", err) } _, err = kv.Put(t.Context(), "foo", "bar", clientv3.WithLease(resp.ID)) require.ErrorIsf(t, err, rpctypes.ErrLeaseNotFound, "err = %v, want %v", err, rpctypes.ErrLeaseNotFound) } func TestLeaseKeepAliveOnce(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) lapi := clus.RandClient() resp, err := lapi.Grant(t.Context(), 10) if err != nil { t.Errorf("failed to create lease %v", err) } _, err = lapi.KeepAliveOnce(t.Context(), resp.ID) if err != nil { t.Errorf("failed to keepalive lease %v", err) } _, err = lapi.KeepAliveOnce(t.Context(), clientv3.LeaseID(0)) if !errors.Is(err, rpctypes.ErrLeaseNotFound) { t.Errorf("expected %v, got %v", rpctypes.ErrLeaseNotFound, err) } } func TestLeaseKeepAlive(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) lapi := clus.Client(0) clus.TakeClient(0) resp, err := lapi.Grant(t.Context(), 10) if err != nil { t.Errorf("failed to create lease %v", err) } type uncomparableCtx struct { context.Context _ func() } ctx, cancel := context.WithCancel(t.Context()) defer cancel() rc, kerr := lapi.KeepAlive(uncomparableCtx{Context: ctx}, resp.ID) if kerr != nil { t.Errorf("failed to keepalive lease %v", kerr) } kresp, ok := <-rc if !ok { t.Errorf("chan is closed, want not closed") } require.NotNilf(t, kresp, "unexpected null response") if kresp.ID != resp.ID { t.Errorf("ID = %x, want %x", kresp.ID, resp.ID) } ctx2, cancel2 := context.WithCancel(t.Context()) rc2, kerr2 := lapi.KeepAlive(uncomparableCtx{Context: ctx2}, resp.ID) if kerr2 != nil { t.Errorf("failed to keepalive lease %v", kerr2) } cancel2() _, ok = <-rc2 if ok { t.Errorf("chan is not closed, want cancel stop keepalive") } select { case <-rc: // cancel2() should not affect first keepalive t.Errorf("chan is closed, want keepalive continue") default: } lapi.Close() _, ok = <-rc if ok { t.Errorf("chan is not closed, want lease Close() closes chan") } } func TestLeaseKeepAliveSeconds(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) resp, err := cli.Grant(t.Context(), 3) if err != nil { t.Errorf("failed to create lease %v", err) } rc, kerr := cli.KeepAlive(t.Context(), resp.ID) if kerr != nil { t.Errorf("failed to keepalive lease %v", kerr) } for i := 0; i < 3; i++ { if _, ok := <-rc; !ok { t.Errorf("[%d] chan is closed, want not closed", i) } } } // TestLeaseKeepAliveHandleFailure tests lease keep alive handling faillure // TODO: add a client that can connect to all the members of cluster via unix sock. // TODO: test handle more complicated failures. func TestLeaseKeepAliveHandleFailure(t *testing.T) { t.Skip("test it when we have a cluster client") integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) // TODO: change this line to get a cluster client lapi := clus.RandClient() resp, err := lapi.Grant(t.Context(), 10) if err != nil { t.Errorf("failed to create lease %v", err) } rc, kerr := lapi.KeepAlive(t.Context(), resp.ID) if kerr != nil { t.Errorf("failed to keepalive lease %v", kerr) } kresp := <-rc if kresp.ID != resp.ID { t.Errorf("ID = %x, want %x", kresp.ID, resp.ID) } // restart the connected member. clus.Members[0].Stop(t) select { case <-rc: t.Fatalf("unexpected keepalive") case <-time.After(10*time.Second/3 + 1): } // recover the member. clus.Members[0].Restart(t) kresp = <-rc if kresp.ID != resp.ID { t.Errorf("ID = %x, want %x", kresp.ID, resp.ID) } lapi.Close() _, ok := <-rc if ok { t.Errorf("chan is not closed, want lease Close() closes chan") } } type leaseCh struct { lid clientv3.LeaseID ch <-chan *clientv3.LeaseKeepAliveResponse } // TestLeaseKeepAliveNotFound ensures a revoked lease won't halt other leases. func TestLeaseKeepAliveNotFound(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.RandClient() var lchs []leaseCh for i := 0; i < 3; i++ { resp, rerr := cli.Grant(t.Context(), 5) require.NoError(t, rerr) kach, kaerr := cli.KeepAlive(t.Context(), resp.ID) require.NoError(t, kaerr) lchs = append(lchs, leaseCh{resp.ID, kach}) } _, err := cli.Revoke(t.Context(), lchs[1].lid) require.NoError(t, err) <-lchs[0].ch if _, ok := <-lchs[0].ch; !ok { t.Fatal("closed keepalive on wrong lease") } if _, ok := <-lchs[1].ch; ok { t.Fatal("expected closed keepalive") } } func TestLeaseGrantErrConnClosed(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) clus.TakeClient(0) require.NoError(t, cli.Close()) donec := make(chan struct{}) go func() { defer close(donec) _, err := cli.Grant(t.Context(), 5) if !clientv3.IsConnCanceled(err) { // context.Canceled if grpc-go balancer calls 'Get' with an inflight client.Close. t.Errorf("expected %v, or server unavailable, got %v", context.Canceled, err) } }() select { case <-time.After(integration.RequestWaitTimeout): t.Fatal("le.Grant took too long") case <-donec: } } // TestLeaseKeepAliveFullResponseQueue ensures when response // queue is full thus dropping keepalive response sends, // keepalive request is sent with the same rate of TTL / 3. func TestLeaseKeepAliveFullResponseQueue(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lapi := clus.Client(0) // expect lease keepalive every 10-second lresp, err := lapi.Grant(t.Context(), 30) require.NoErrorf(t, err, "failed to create lease %v", err) id := lresp.ID old := clientv3.LeaseResponseChSize defer func() { clientv3.LeaseResponseChSize = old }() clientv3.LeaseResponseChSize = 0 // never fetch from response queue, and let it become full _, err = lapi.KeepAlive(t.Context(), id) require.NoErrorf(t, err, "failed to keepalive lease %v", err) // TTL should not be refreshed after 3 seconds // expect keepalive to be triggered after TTL/3 time.Sleep(3 * time.Second) tr, terr := lapi.TimeToLive(t.Context(), id) require.NoErrorf(t, terr, "failed to get lease information %v", terr) if tr.TTL >= 29 { t.Errorf("unexpected kept-alive lease TTL %d", tr.TTL) } } func TestLeaseGrantNewAfterClose(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) clus.TakeClient(0) require.NoError(t, cli.Close()) donec := make(chan struct{}) go func() { _, err := cli.Grant(t.Context(), 5) if !clientv3.IsConnCanceled(err) { t.Errorf("expected %v or server unavailable, got %v", context.Canceled, err) } close(donec) }() select { case <-time.After(integration.RequestWaitTimeout): t.Fatal("le.Grant took too long") case <-donec: } } func TestLeaseRevokeNewAfterClose(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) resp, err := cli.Grant(t.Context(), 5) require.NoError(t, err) leaseID := resp.ID clus.TakeClient(0) require.NoError(t, cli.Close()) errMsgCh := make(chan string, 1) go func() { _, err := cli.Revoke(t.Context(), leaseID) if !clientv3.IsConnCanceled(err) { errMsgCh <- fmt.Sprintf("expected %v or server unavailable, got %v", context.Canceled, err) } else { errMsgCh <- "" } }() select { case <-time.After(integration.RequestWaitTimeout): t.Fatal("le.Revoke took too long") case errMsg := <-errMsgCh: require.Empty(t, errMsg) } } // TestLeaseKeepAliveCloseAfterDisconnectRevoke ensures the keep alive channel is closed // following a disconnection, lease revoke, then reconnect. func TestLeaseKeepAliveCloseAfterDisconnectRevoke(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) cli := clus.Client(0) // setup lease and do a keepalive resp, err := cli.Grant(t.Context(), 10) require.NoError(t, err) rc, kerr := cli.KeepAlive(t.Context(), resp.ID) require.NoError(t, kerr) kresp := <-rc require.Equalf(t, kresp.ID, resp.ID, "ID = %x, want %x", kresp.ID, resp.ID) // keep client disconnected clus.Members[0].Stop(t) time.Sleep(time.Second) clus.WaitLeader(t) _, err = clus.Client(1).Revoke(t.Context(), resp.ID) require.NoError(t, err) clus.Members[0].Restart(t) // some responses may still be buffered; drain until close timer := time.After(time.Duration(kresp.TTL) * time.Second) for kresp != nil { select { case kresp = <-rc: case <-timer: t.Fatalf("keepalive channel did not close") } } } // TestLeaseKeepAliveInitTimeout ensures the keep alive channel closes if // the initial keep alive request never gets a response. func TestLeaseKeepAliveInitTimeout(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) cli := clus.Client(0) // setup lease and do a keepalive resp, err := cli.Grant(t.Context(), 5) require.NoError(t, err) // keep client disconnected clus.Members[0].Stop(t) rc, kerr := cli.KeepAlive(t.Context(), resp.ID) require.NoError(t, kerr) select { case ka, ok := <-rc: require.Falsef(t, ok, "unexpected keepalive %v, expected closed channel", ka) case <-time.After(10 * time.Second): t.Fatalf("keepalive channel did not close") } clus.Members[0].Restart(t) } // TestLeaseKeepAliveTTLTimeout ensures the keep alive channel closes if // a keep alive request after the first never gets a response. func TestLeaseKeepAliveTTLTimeout(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) cli := clus.Client(0) // setup lease and do a keepalive resp, err := cli.Grant(t.Context(), 5) require.NoError(t, err) rc, kerr := cli.KeepAlive(t.Context(), resp.ID) require.NoError(t, kerr) kresp := <-rc require.Equalf(t, kresp.ID, resp.ID, "ID = %x, want %x", kresp.ID, resp.ID) // keep client disconnected clus.Members[0].Stop(t) select { case ka, ok := <-rc: require.Falsef(t, ok, "unexpected keepalive %v, expected closed channel", ka) case <-time.After(10 * time.Second): t.Fatalf("keepalive channel did not close") } clus.Members[0].Restart(t) } func TestLeaseTimeToLive(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) c := clus.RandClient() lapi := c resp, err := lapi.Grant(t.Context(), 10) if err != nil { t.Errorf("failed to create lease %v", err) } kv := clus.RandClient() keys := []string{"foo1", "foo2"} for i := range keys { _, err = kv.Put(t.Context(), keys[i], "bar", clientv3.WithLease(resp.ID)) require.NoError(t, err) } // linearized read to ensure Puts propagated to server backing lapi _, err = c.Get(t.Context(), "abc") require.NoError(t, err) lresp, lerr := lapi.TimeToLive(t.Context(), resp.ID, clientv3.WithAttachedKeys()) require.NoError(t, lerr) require.Equalf(t, lresp.ID, resp.ID, "leaseID expected %d, got %d", resp.ID, lresp.ID) require.Equalf(t, int64(10), lresp.GrantedTTL, "GrantedTTL expected %d, got %d", 10, lresp.GrantedTTL) if lresp.TTL == 0 || lresp.TTL > lresp.GrantedTTL { t.Fatalf("unexpected TTL %d (granted %d)", lresp.TTL, lresp.GrantedTTL) } ks := make([]string, len(lresp.Keys)) for i := range lresp.Keys { ks[i] = string(lresp.Keys[i]) } sort.Strings(ks) require.Truef(t, reflect.DeepEqual(ks, keys), "keys expected %v, got %v", keys, ks) lresp, lerr = lapi.TimeToLive(t.Context(), resp.ID) require.NoError(t, lerr) require.Emptyf(t, lresp.Keys, "unexpected keys %+v", lresp.Keys) } func TestLeaseTimeToLiveLeaseNotFound(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.RandClient() resp, err := cli.Grant(t.Context(), 10) if err != nil { t.Errorf("failed to create lease %v", err) } _, err = cli.Revoke(t.Context(), resp.ID) if err != nil { t.Errorf("failed to Revoke lease %v", err) } lresp, err := cli.TimeToLive(t.Context(), resp.ID) // TimeToLive() should return a response with TTL=-1. require.NoErrorf(t, err, "expected err to be nil") require.NotNilf(t, lresp, "expected lresp not to be nil") require.NotNilf(t, lresp.ResponseHeader, "expected ResponseHeader not to be nil") require.Equalf(t, lresp.ID, resp.ID, "expected Lease ID %v, but got %v", resp.ID, lresp.ID) require.Equalf(t, int64(-1), lresp.TTL, "expected TTL %v, but got %v", int64(-1), lresp.TTL) } func TestLeaseLeases(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.RandClient() var ids []clientv3.LeaseID for i := 0; i < 5; i++ { resp, err := cli.Grant(t.Context(), 10) if err != nil { t.Errorf("failed to create lease %v", err) } ids = append(ids, resp.ID) } resp, err := cli.Leases(t.Context()) require.NoError(t, err) require.Lenf(t, resp.Leases, 5, "len(resp.Leases) expected 5, got %d", len(resp.Leases)) for i := range resp.Leases { require.Equalf(t, ids[i], resp.Leases[i].ID, "#%d: lease ID expected %d, got %d", i, ids[i], resp.Leases[i].ID) } } // TestLeaseRenewLostQuorum ensures keepalives work after losing quorum // for a while. func TestLeaseRenewLostQuorum(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) cli := clus.Client(0) r, err := cli.Grant(t.Context(), 4) require.NoError(t, err) kctx, kcancel := context.WithCancel(t.Context()) defer kcancel() ka, err := cli.KeepAlive(kctx, r.ID) require.NoError(t, err) // consume first keepalive so next message sends when cluster is down <-ka lastKa := time.Now() // force keepalive stream message to timeout clus.Members[1].Stop(t) clus.Members[2].Stop(t) // Use TTL-2 since the client closes the keepalive channel if no // keepalive arrives before the lease deadline; the client will // try to resend a keepalive after TTL/3 seconds, so for a TTL of 4, // sleeping for 2s should be sufficient time for issuing a retry. // The cluster has two seconds to recover and reply to the keepalive. time.Sleep(time.Duration(r.TTL-2) * time.Second) clus.Members[1].Restart(t) clus.Members[2].Restart(t) if time.Since(lastKa) > time.Duration(r.TTL)*time.Second { t.Skip("waited too long for server stop and restart") } select { case _, ok := <-ka: require.Truef(t, ok, "keepalive closed") case <-time.After(time.Duration(r.TTL) * time.Second): t.Fatalf("timed out waiting for keepalive") } } func TestLeaseKeepAliveLoopExit(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) ctx := t.Context() cli := clus.Client(0) clus.TakeClient(0) resp, err := cli.Grant(ctx, 5) require.NoError(t, err) cli.Close() _, err = cli.KeepAlive(ctx, resp.ID) var keepAliveHaltedErr clientv3.ErrKeepAliveHalted require.ErrorAsf(t, err, &keepAliveHaltedErr, "expected %T, got %v(%T)", clientv3.ErrKeepAliveHalted{}, err, err) } // TestV3LeaseFailureOverlap issues Grant and KeepAlive requests to a cluster // before, during, and after quorum loss to confirm Grant/KeepAlive tolerates // transient cluster failure. func TestV3LeaseFailureOverlap(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 2, UseBridge: true}) defer clus.Terminate(t) numReqs := 5 cli := clus.Client(0) // bring up a session, tear it down updown := func(i int) error { sess, err := concurrency.NewSession(cli) if err != nil { return err } ch := make(chan struct{}) go func() { defer close(ch) sess.Close() }() select { case <-ch: case <-time.After(time.Minute / 4): t.Fatalf("timeout %d", i) } return nil } var wg sync.WaitGroup mkReqs := func(n int) { wg.Add(numReqs) for i := 0; i < numReqs; i++ { go func() { defer wg.Done() err := updown(n) if err == nil || errors.Is(err, rpctypes.ErrTimeoutDueToConnectionLost) { return } t.Error(err) }() } } mkReqs(1) clus.Members[1].Stop(t) mkReqs(2) time.Sleep(time.Second) mkReqs(3) clus.Members[1].Restart(t) mkReqs(4) wg.Wait() } // TestLeaseWithRequireLeader checks keep-alive channel close when no leader. func TestLeaseWithRequireLeader(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 2, UseBridge: true}) defer clus.Terminate(t) c := clus.Client(0) lid1, err1 := c.Grant(t.Context(), 60) require.NoError(t, err1) lid2, err2 := c.Grant(t.Context(), 60) require.NoError(t, err2) // kaReqLeader close if the leader is lost kaReqLeader, kerr1 := c.KeepAlive(clientv3.WithRequireLeader(t.Context()), lid1.ID) require.NoError(t, kerr1) // kaWait will wait even if the leader is lost kaWait, kerr2 := c.KeepAlive(t.Context(), lid2.ID) require.NoError(t, kerr2) select { case <-kaReqLeader: case <-time.After(5 * time.Second): t.Fatalf("require leader first keep-alive timed out") } select { case <-kaWait: case <-time.After(5 * time.Second): t.Fatalf("leader not required first keep-alive timed out") } prevUnavailableCount := getLeaseKeepAliveMetric(t, clus.Members[0], "Unavailable") prevCanceledCount := getLeaseKeepAliveMetric(t, clus.Members[0], "Canceled") clus.Members[1].Stop(t) // kaReqLeader may issue multiple requests while waiting for the first // response from proxy server; drain any stray keepalive responses time.Sleep(100 * time.Millisecond) for { <-kaReqLeader if len(kaReqLeader) == 0 { break } } select { case resp, ok := <-kaReqLeader: require.Falsef(t, ok, "expected closed require leader, got response %+v", resp) case <-time.After(5 * time.Second): t.Fatal("keepalive with require leader took too long to close") } require.Eventuallyf(t, func() bool { return getLeaseKeepAliveMetric(t, clus.Members[0], "Unavailable") > prevUnavailableCount }, 3*time.Second, 100*time.Millisecond, "expected Unavailable metric to increase after leader loss, prev count: %d", prevUnavailableCount) // Ensure the error is Unavailable, not Canceled currentCanceledCount := getLeaseKeepAliveMetric(t, clus.Members[0], "Canceled") require.Equalf(t, prevCanceledCount, currentCanceledCount, "Canceled metric should not change, expected %d, got %d", prevCanceledCount, currentCanceledCount) select { case _, ok := <-kaWait: require.Truef(t, ok, "got closed channel with no require leader, expected non-closed") case <-time.After(10 * time.Millisecond): // wait some to detect any closes happening soon after kaReqLeader closing } } func getLeaseKeepAliveMetric(t *testing.T, member *integration.Member, grpcCode string) int64 { t.Helper() metricVal, err := member.Metric( "grpc_server_handled_total", `grpc_method="LeaseKeepAlive"`, fmt.Sprintf(`grpc_code="%v"`, grpcCode), ) require.NoError(t, err) count, err := strconv.ParseInt(metricVal, 10, 64) require.NoError(t, err) return count } ================================================ FILE: tests/integration/clientv3/lease/leasing_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lease_test import ( "context" "errors" "fmt" "math/rand" "reflect" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/client/v3/leasing" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestLeasingPutGet(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) lKV1, closeLKV1, err := leasing.NewKV(clus.Client(0), "foo/") require.NoError(t, err) defer closeLKV1() lKV2, closeLKV2, err := leasing.NewKV(clus.Client(1), "foo/") require.NoError(t, err) defer closeLKV2() lKV3, closeLKV3, err := leasing.NewKV(clus.Client(2), "foo/") require.NoError(t, err) defer closeLKV3() resp, err := lKV1.Get(t.Context(), "abc") require.NoError(t, err) if len(resp.Kvs) != 0 { t.Errorf("expected nil, got %q", resp.Kvs[0].Key) } _, err = lKV1.Put(t.Context(), "abc", "def") require.NoError(t, err) resp, err = lKV2.Get(t.Context(), "abc") require.NoError(t, err) if string(resp.Kvs[0].Key) != "abc" { t.Errorf("expected key=%q, got key=%q", "abc", resp.Kvs[0].Key) } if string(resp.Kvs[0].Value) != "def" { t.Errorf("expected value=%q, got value=%q", "def", resp.Kvs[0].Value) } _, err = lKV3.Get(t.Context(), "abc") require.NoError(t, err) _, err = lKV2.Put(t.Context(), "abc", "ghi") require.NoError(t, err) resp, err = lKV3.Get(t.Context(), "abc") require.NoError(t, err) if string(resp.Kvs[0].Key) != "abc" { t.Errorf("expected key=%q, got key=%q", "abc", resp.Kvs[0].Key) } if string(resp.Kvs[0].Value) != "ghi" { t.Errorf("expected value=%q, got value=%q", "ghi", resp.Kvs[0].Value) } } // TestLeasingInterval checks the leasing KV fetches key intervals. func TestLeasingInterval(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() keys := []string{"abc/a", "abc/b", "abc/a/a"} for _, k := range keys { _, err = clus.Client(0).Put(t.Context(), k, "v") require.NoError(t, err) } resp, err := lkv.Get(t.Context(), "abc/", clientv3.WithPrefix()) require.NoError(t, err) require.Lenf(t, resp.Kvs, 3, "expected keys %+v, got response keys %+v", keys, resp.Kvs) // load into cache _, err = lkv.Get(t.Context(), "abc/a") require.NoError(t, err) // get when prefix is also a cached key resp, err = lkv.Get(t.Context(), "abc/a", clientv3.WithPrefix()) require.NoError(t, err) require.Lenf(t, resp.Kvs, 2, "expected keys %+v, got response keys %+v", keys, resp.Kvs) } // TestLeasingPutInvalidateNew checks the leasing KV updates its cache on a Put to a new key. func TestLeasingPutInvalidateNew(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) _, err = lkv.Put(t.Context(), "k", "v") require.NoError(t, err) lkvResp, err := lkv.Get(t.Context(), "k") require.NoError(t, err) cResp, cerr := clus.Client(0).Get(t.Context(), "k") require.NoError(t, cerr) require.Truef(t, reflect.DeepEqual(lkvResp, cResp), `expected %+v, got response %+v`, cResp, lkvResp) } // TestLeasingPutInvalidateExisting checks the leasing KV updates its cache on a Put to an existing key. func TestLeasingPutInvalidateExisting(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) _, err := clus.Client(0).Put(t.Context(), "k", "abc") require.NoError(t, err) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) _, err = lkv.Put(t.Context(), "k", "v") require.NoError(t, err) lkvResp, err := lkv.Get(t.Context(), "k") require.NoError(t, err) cResp, cerr := clus.Client(0).Get(t.Context(), "k") require.NoError(t, cerr) require.Truef(t, reflect.DeepEqual(lkvResp, cResp), `expected %+v, got response %+v`, cResp, lkvResp) } // TestLeasingGetNoLeaseTTL checks a key with a TTL is not leased. func TestLeasingGetNoLeaseTTL(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() lresp, err := clus.Client(0).Grant(t.Context(), 60) require.NoError(t, err) _, err = clus.Client(0).Put(t.Context(), "k", "v", clientv3.WithLease(lresp.ID)) require.NoError(t, err) gresp, err := lkv.Get(t.Context(), "k") require.NoError(t, err) assert.Len(t, gresp.Kvs, 1) clus.Members[0].Stop(t) ctx, cancel := context.WithTimeout(t.Context(), time.Second) _, err = lkv.Get(ctx, "k") cancel() assert.Equal(t, err, ctx.Err()) } // TestLeasingGetSerializable checks the leasing KV can make serialized requests // when the etcd cluster is partitioned. func TestLeasingGetSerializable(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 2, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = clus.Client(0).Put(t.Context(), "cached", "abc") require.NoError(t, err) _, err = lkv.Get(t.Context(), "cached") require.NoError(t, err) clus.Members[1].Stop(t) // don't necessarily try to acquire leasing key ownership for new key resp, err := lkv.Get(t.Context(), "uncached", clientv3.WithSerializable()) require.NoError(t, err) require.Emptyf(t, resp.Kvs, `expected no keys, got response %+v`, resp) clus.Members[0].Stop(t) // leasing key ownership should have "cached" locally served cachedResp, err := lkv.Get(t.Context(), "cached", clientv3.WithSerializable()) require.NoError(t, err) if len(cachedResp.Kvs) != 1 || string(cachedResp.Kvs[0].Value) != "abc" { t.Fatalf(`expected "cached"->"abc", got response %+v`, cachedResp) } } // TestLeasingPrevKey checks the cache respects WithPrevKV on puts. func TestLeasingPrevKey(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 2}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = clus.Client(0).Put(t.Context(), "k", "abc") require.NoError(t, err) // acquire leasing key _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) resp, err := lkv.Put(t.Context(), "k", "def", clientv3.WithPrevKV()) require.NoError(t, err) if resp.PrevKv == nil || string(resp.PrevKv.Value) != "abc" { t.Fatalf(`expected PrevKV.Value="abc", got response %+v`, resp) } } // TestLeasingRevGet checks the cache respects Get by Revision. func TestLeasingRevGet(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() putResp, err := clus.Client(0).Put(t.Context(), "k", "abc") require.NoError(t, err) _, err = clus.Client(0).Put(t.Context(), "k", "def") require.NoError(t, err) // check historic revision getResp, gerr := lkv.Get(t.Context(), "k", clientv3.WithRev(putResp.Header.Revision)) require.NoError(t, gerr) if len(getResp.Kvs) != 1 || string(getResp.Kvs[0].Value) != "abc" { t.Fatalf(`expected "k"->"abc" at rev=%d, got response %+v`, putResp.Header.Revision, getResp) } // check current revision getResp, gerr = lkv.Get(t.Context(), "k") require.NoError(t, gerr) if len(getResp.Kvs) != 1 || string(getResp.Kvs[0].Value) != "def" { t.Fatalf(`expected "k"->"def" at rev=%d, got response %+v`, putResp.Header.Revision, getResp) } } // TestLeasingGetWithOpts checks options that can be served through the cache do not depend on the server. func TestLeasingGetWithOpts(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = clus.Client(0).Put(t.Context(), "k", "abc") require.NoError(t, err) // in cache _, err = lkv.Get(t.Context(), "k", clientv3.WithKeysOnly()) require.NoError(t, err) clus.Members[0].Stop(t) opts := []clientv3.OpOption{ clientv3.WithKeysOnly(), clientv3.WithLimit(1), clientv3.WithMinCreateRev(1), clientv3.WithMinModRev(1), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), clientv3.WithSerializable(), } for _, opt := range opts { _, err = lkv.Get(t.Context(), "k", opt) require.NoError(t, err) } var getOpts []clientv3.OpOption for i := 0; i < len(opts); i++ { getOpts = append(getOpts, opts[rand.Intn(len(opts))]) } getOpts = getOpts[:rand.Intn(len(opts))] _, err = lkv.Get(t.Context(), "k", getOpts...) require.NoError(t, err) } // TestLeasingConcurrentPut ensures that a get after concurrent puts returns // the recently put data. func TestLeasingConcurrentPut(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() // force key into leasing key cache _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) // concurrently put through leasing client numPuts := 16 putc := make(chan *clientv3.PutResponse, numPuts) for i := 0; i < numPuts; i++ { go func() { resp, perr := lkv.Put(t.Context(), "k", "abc") if perr != nil { t.Error(perr) } putc <- resp }() } // record maximum revision from puts maxRev := int64(0) for i := 0; i < numPuts; i++ { if resp := <-putc; resp.Header.Revision > maxRev { maxRev = resp.Header.Revision } } // confirm Get gives most recently put revisions getResp, gerr := lkv.Get(t.Context(), "k") require.NoError(t, gerr) if mr := getResp.Kvs[0].ModRevision; mr != maxRev { t.Errorf("expected ModRevision %d, got %d", maxRev, mr) } if ver := getResp.Kvs[0].Version; ver != int64(numPuts) { t.Errorf("expected Version %d, got %d", numPuts, ver) } } func TestLeasingDisconnectedGet(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = clus.Client(0).Put(t.Context(), "cached", "abc") require.NoError(t, err) // get key so it's cached _, err = lkv.Get(t.Context(), "cached") require.NoError(t, err) clus.Members[0].Stop(t) // leasing key ownership should have "cached" locally served cachedResp, err := lkv.Get(t.Context(), "cached") require.NoError(t, err) if len(cachedResp.Kvs) != 1 || string(cachedResp.Kvs[0].Value) != "abc" { t.Fatalf(`expected "cached"->"abc", got response %+v`, cachedResp) } } func TestLeasingDeleteOwner(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = clus.Client(0).Put(t.Context(), "k", "abc") require.NoError(t, err) // get+own / delete / get _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) _, err = lkv.Delete(t.Context(), "k") require.NoError(t, err) resp, err := lkv.Get(t.Context(), "k") require.NoError(t, err) require.Emptyf(t, resp.Kvs, `expected "k" to be deleted, got response %+v`, resp) // try to double delete _, err = lkv.Delete(t.Context(), "k") require.NoError(t, err) } func TestLeasingDeleteNonOwner(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv1, closeLKV1, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV1() lkv2, closeLKV2, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV2() _, err = clus.Client(0).Put(t.Context(), "k", "abc") require.NoError(t, err) // acquire ownership _, err = lkv1.Get(t.Context(), "k") require.NoError(t, err) // delete via non-owner _, err = lkv2.Delete(t.Context(), "k") require.NoError(t, err) // key should be removed from lkv1 resp, err := lkv1.Get(t.Context(), "k") require.NoError(t, err) require.Emptyf(t, resp.Kvs, `expected "k" to be deleted, got response %+v`, resp) } func TestLeasingOverwriteResponse(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = clus.Client(0).Put(t.Context(), "k", "abc") require.NoError(t, err) resp, err := lkv.Get(t.Context(), "k") require.NoError(t, err) resp.Kvs[0].Key[0] = 'z' resp.Kvs[0].Value[0] = 'z' resp, err = lkv.Get(t.Context(), "k") require.NoError(t, err) if string(resp.Kvs[0].Key) != "k" { t.Errorf(`expected key "k", got %q`, string(resp.Kvs[0].Key)) } if string(resp.Kvs[0].Value) != "abc" { t.Errorf(`expected value "abc", got %q`, string(resp.Kvs[0].Value)) } } func TestLeasingOwnerPutResponse(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = clus.Client(0).Put(t.Context(), "k", "abc") require.NoError(t, err) _, gerr := lkv.Get(t.Context(), "k") require.NoError(t, gerr) presp, err := lkv.Put(t.Context(), "k", "def") require.NoError(t, err) if presp == nil { t.Fatal("expected put response, got nil") } clus.Members[0].Stop(t) gresp, gerr := lkv.Get(t.Context(), "k") require.NoError(t, gerr) if gresp.Kvs[0].ModRevision != presp.Header.Revision { t.Errorf("expected mod revision %d, got %d", presp.Header.Revision, gresp.Kvs[0].ModRevision) } if gresp.Kvs[0].Version != 2 { t.Errorf("expected version 2, got version %d", gresp.Kvs[0].Version) } } func TestLeasingTxnOwnerGetRange(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() keyCount := rand.Intn(10) + 1 for i := 0; i < keyCount; i++ { k := fmt.Sprintf("k-%d", i) _, err = clus.Client(0).Put(t.Context(), k, k+k) require.NoError(t, err) } _, err = lkv.Get(t.Context(), "k-") require.NoError(t, err) tresp, terr := lkv.Txn(t.Context()).Then(clientv3.OpGet("k-", clientv3.WithPrefix())).Commit() require.NoError(t, terr) resp := tresp.Responses[0].GetResponseRange() require.Equalf(t, len(resp.Kvs), keyCount, "expected %d keys, got response %+v", keyCount, resp.Kvs) } func TestLeasingTxnOwnerGet(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) client := clus.Client(0) lkv, closeLKV, err := leasing.NewKV(client, "pfx/") require.NoError(t, err) defer func() { // In '--tags cluster_proxy' mode the client need to be closed before // closeLKV(). This interrupts all outstanding watches. Closing by closeLKV() // is not sufficient as (unfortunately) context close does not interrupts Watches. // See ./clientv3/watch.go: // >> Currently, client contexts are overwritten with "valCtx" that never closes. << clus.TakeClient(0) // avoid double Close() of the client. client.Close() closeLKV() }() // TODO: Randomization in tests is a bad practice (except explicitly exploratory). keyCount := rand.Intn(10) + 1 var ops []clientv3.Op presps := make([]*clientv3.PutResponse, keyCount) for i := range presps { k := fmt.Sprintf("k-%d", i) presp, err := client.Put(t.Context(), k, k+k) require.NoError(t, err) presps[i] = presp _, err = lkv.Get(t.Context(), k) require.NoError(t, err) ops = append(ops, clientv3.OpGet(k)) } // TODO: Randomization in unit tests is a bad practice (except explicitly exploratory). ops = ops[:rand.Intn(len(ops))] // served through cache clus.Members[0].Stop(t) var thenOps, elseOps []clientv3.Op cmps, useThen := randCmps("k-", presps) if useThen { thenOps = ops elseOps = []clientv3.Op{clientv3.OpPut("k", "1")} } else { thenOps = []clientv3.Op{clientv3.OpPut("k", "1")} elseOps = ops } tresp, terr := lkv.Txn(t.Context()). If(cmps...). Then(thenOps...). Else(elseOps...).Commit() require.NoError(t, terr) require.Equalf(t, tresp.Succeeded, useThen, "expected succeeded=%v, got tresp=%+v", useThen, tresp) require.Lenf(t, ops, len(tresp.Responses), "expected %d responses, got %d", len(ops), len(tresp.Responses)) wrev := presps[len(presps)-1].Header.Revision require.GreaterOrEqualf(t, tresp.Header.Revision, wrev, "expected header revision >= %d, got %d", wrev, tresp.Header.Revision) for i := range ops { k := fmt.Sprintf("k-%d", i) rr := tresp.Responses[i].GetResponseRange() require.NotNilf(t, rr, "expected get response, got %+v", tresp.Responses[i]) if string(rr.Kvs[0].Key) != k || string(rr.Kvs[0].Value) != k+k { t.Errorf(`expected key for %q, got %+v`, k, rr.Kvs) } } } func TestLeasingTxnOwnerDeleteRange(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() keyCount := rand.Intn(10) + 1 for i := 0; i < keyCount; i++ { k := fmt.Sprintf("k-%d", i) _, perr := clus.Client(0).Put(t.Context(), k, k+k) require.NoError(t, perr) } // cache in lkv resp, err := lkv.Get(t.Context(), "k-", clientv3.WithPrefix()) require.NoError(t, err) require.Equalf(t, len(resp.Kvs), keyCount, "expected %d keys, got %d", keyCount, len(resp.Kvs)) _, terr := lkv.Txn(t.Context()).Then(clientv3.OpDelete("k-", clientv3.WithPrefix())).Commit() require.NoError(t, terr) resp, err = lkv.Get(t.Context(), "k-", clientv3.WithPrefix()) require.NoError(t, err) require.Emptyf(t, resp.Kvs, "expected no keys, got %d", len(resp.Kvs)) } func TestLeasingTxnOwnerDelete(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = clus.Client(0).Put(t.Context(), "k", "abc") require.NoError(t, err) // cache in lkv _, gerr := lkv.Get(t.Context(), "k") require.NoError(t, gerr) _, terr := lkv.Txn(t.Context()).Then(clientv3.OpDelete("k")).Commit() require.NoError(t, terr) resp, err := lkv.Get(t.Context(), "k") require.NoError(t, err) require.Emptyf(t, resp.Kvs, "expected no keys, got %d", len(resp.Kvs)) } func TestLeasingTxnOwnerIf(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = clus.Client(0).Put(t.Context(), "k", "abc") require.NoError(t, err) _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) // served through cache clus.Members[0].Stop(t) tests := []struct { cmps []clientv3.Cmp wSucceeded bool wResponses int }{ // success { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Value("k"), "=", "abc")}, wSucceeded: true, wResponses: 1, }, { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("k"), "=", 2)}, wSucceeded: true, wResponses: 1, }, { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision("k"), "=", 2)}, wSucceeded: true, wResponses: 1, }, { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Version("k"), "=", 1)}, wSucceeded: true, wResponses: 1, }, // failure { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Value("k"), ">", "abc")}, }, { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("k"), ">", 2)}, }, { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision("k"), "=", 2)}, wSucceeded: true, wResponses: 1, }, { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Version("k"), ">", 1)}, }, { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Value("k"), "<", "abc")}, }, { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("k"), "<", 2)}, }, { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision("k"), "<", 2)}, }, { cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Version("k"), "<", 1)}, }, { cmps: []clientv3.Cmp{ clientv3.Compare(clientv3.Version("k"), "=", 1), clientv3.Compare(clientv3.Version("k"), "<", 1), }, }, } for i, tt := range tests { tresp, terr := lkv.Txn(t.Context()).If(tt.cmps...).Then(clientv3.OpGet("k")).Commit() require.NoError(t, terr) if tresp.Succeeded != tt.wSucceeded { t.Errorf("#%d: expected succeeded %v, got %v", i, tt.wSucceeded, tresp.Succeeded) } if len(tresp.Responses) != tt.wResponses { t.Errorf("#%d: expected %d responses, got %d", i, tt.wResponses, len(tresp.Responses)) } } } func TestLeasingTxnCancel(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) lkv1, closeLKV1, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV1() lkv2, closeLKV2, err := leasing.NewKV(clus.Client(1), "pfx/") require.NoError(t, err) defer closeLKV2() // acquire lease but disconnect so no revoke in time _, err = lkv1.Get(t.Context(), "k") require.NoError(t, err) clus.Members[0].Stop(t) // wait for leader election, if any _, err = clus.Client(1).Get(t.Context(), "abc") require.NoError(t, err) ctx, cancel := context.WithCancel(t.Context()) go func() { time.Sleep(100 * time.Millisecond) cancel() }() _, err = lkv2.Txn(ctx).Then(clientv3.OpPut("k", "v")).Commit() require.ErrorIsf(t, err, context.Canceled, "expected %v, got %v", context.Canceled, err) } func TestLeasingTxnNonOwnerPut(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() lkv2, closeLKV2, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV2() _, err = clus.Client(0).Put(t.Context(), "k", "abc") require.NoError(t, err) _, err = clus.Client(0).Put(t.Context(), "k2", "123") require.NoError(t, err) // cache in lkv _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) _, err = lkv.Get(t.Context(), "k2") require.NoError(t, err) // invalidate via lkv2 txn opArray := make([]clientv3.Op, 0) opArray = append(opArray, clientv3.OpPut("k2", "456")) tresp, terr := lkv2.Txn(t.Context()).Then( clientv3.OpTxn(nil, opArray, nil), clientv3.OpPut("k", "def"), clientv3.OpPut("k3", "999"), // + a key not in any cache ).Commit() require.NoError(t, terr) if !tresp.Succeeded || len(tresp.Responses) != 3 { t.Fatalf("expected txn success, got %+v", tresp) } // check cache was invalidated gresp, gerr := lkv.Get(t.Context(), "k") require.NoError(t, gerr) if len(gresp.Kvs) != 1 || string(gresp.Kvs[0].Value) != "def" { t.Errorf(`expected value "def", got %+v`, gresp) } gresp, gerr = lkv.Get(t.Context(), "k2") require.NoError(t, gerr) if len(gresp.Kvs) != 1 || string(gresp.Kvs[0].Value) != "456" { t.Errorf(`expected value "def", got %+v`, gresp) } // check puts were applied and are all in the same revision w := clus.Client(0).Watch( clus.Client(0).Ctx(), "k", clientv3.WithRev(tresp.Header.Revision), clientv3.WithPrefix()) wresp := <-w c := 0 var evs []clientv3.Event for _, ev := range wresp.Events { evs = append(evs, *ev) if ev.Kv.ModRevision == tresp.Header.Revision { c++ } } require.Equalf(t, 3, c, "expected 3 put events, got %+v", evs) } // TestLeasingTxnRandIfThenOrElse randomly leases keys two separate clients, then // issues a random If/{Then,Else} transaction on those keys to one client. func TestLeasingTxnRandIfThenOrElse(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv1, closeLKV1, err1 := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err1) defer closeLKV1() lkv2, closeLKV2, err2 := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err2) defer closeLKV2() keyCount := 16 dat := make([]*clientv3.PutResponse, keyCount) for i := 0; i < keyCount; i++ { k, v := fmt.Sprintf("k-%d", i), fmt.Sprintf("%d", i) dat[i], err1 = clus.Client(0).Put(t.Context(), k, v) require.NoError(t, err1) } // nondeterministically populate leasing caches var wg sync.WaitGroup getc := make(chan struct{}, keyCount) getRandom := func(kv clientv3.KV) { defer wg.Done() for i := 0; i < keyCount/2; i++ { k := fmt.Sprintf("k-%d", rand.Intn(keyCount)) if _, err := kv.Get(t.Context(), k); err != nil { t.Error(err) } getc <- struct{}{} } } wg.Add(2) defer wg.Wait() go getRandom(lkv1) go getRandom(lkv2) // random list of comparisons, all true cmps, useThen := randCmps("k-", dat) // random list of puts/gets; unique keys var ops []clientv3.Op usedIdx := make(map[int]struct{}) for i := 0; i < keyCount; i++ { idx := rand.Intn(keyCount) if _, ok := usedIdx[idx]; ok { continue } usedIdx[idx] = struct{}{} k := fmt.Sprintf("k-%d", idx) switch rand.Intn(2) { case 0: ops = append(ops, clientv3.OpGet(k)) case 1: ops = append(ops, clientv3.OpPut(k, "a")) // TODO: add delete } } // random lengths ops = ops[:rand.Intn(len(ops))] // wait for some gets to populate the leasing caches before committing for i := 0; i < keyCount/2; i++ { <-getc } // randomly choose between then and else blocks var thenOps, elseOps []clientv3.Op if useThen { thenOps = ops } else { // force failure elseOps = ops } tresp, terr := lkv1.Txn(t.Context()).If(cmps...).Then(thenOps...).Else(elseOps...).Commit() require.NoError(t, terr) // cmps always succeed require.Equalf(t, tresp.Succeeded, useThen, "expected succeeded=%v, got tresp=%+v", useThen, tresp) // get should match what was put checkPuts := func(s string, kv clientv3.KV) { for _, op := range ops { if !op.IsPut() { continue } resp, rerr := kv.Get(t.Context(), string(op.KeyBytes())) require.NoError(t, rerr) if len(resp.Kvs) != 1 || string(resp.Kvs[0].Value) != "a" { t.Fatalf(`%s: expected value="a", got %+v`, s, resp.Kvs) } } } checkPuts("client(0)", clus.Client(0)) checkPuts("lkv1", lkv1) checkPuts("lkv2", lkv2) } func TestLeasingOwnerPutError(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) clus.Members[0].Stop(t) ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() resp, err := lkv.Put(ctx, "k", "v") require.Errorf(t, err, "expected error, got response %+v", resp) } func TestLeasingOwnerDeleteError(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) clus.Members[0].Stop(t) ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() resp, err := lkv.Delete(ctx, "k") require.Errorf(t, err, "expected error, got response %+v", resp) } func TestLeasingNonOwnerPutError(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() clus.Members[0].Stop(t) ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) defer cancel() resp, err := lkv.Put(ctx, "k", "v") require.Errorf(t, err, "expected error, got response %+v", resp) } func TestLeasingOwnerDeletePrefix(t *testing.T) { testLeasingOwnerDelete(t, clientv3.OpDelete("key/", clientv3.WithPrefix())) } func TestLeasingOwnerDeleteFrom(t *testing.T) { testLeasingOwnerDelete(t, clientv3.OpDelete("kd", clientv3.WithFromKey())) } func testLeasingOwnerDelete(t *testing.T, del clientv3.Op) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "0/") require.NoError(t, err) defer closeLKV() for i := 0; i < 8; i++ { _, err = clus.Client(0).Put(t.Context(), fmt.Sprintf("key/%d", i), "123") require.NoError(t, err) } _, err = lkv.Get(t.Context(), "key/1") require.NoError(t, err) opResp, delErr := lkv.Do(t.Context(), del) require.NoError(t, delErr) delResp := opResp.Del() // confirm keys are invalidated from cache and deleted on etcd for i := 0; i < 8; i++ { resp, err := lkv.Get(t.Context(), fmt.Sprintf("key/%d", i)) require.NoError(t, err) require.Emptyf(t, resp.Kvs, "expected no keys on key/%d, got %+v", i, resp) } // confirm keys were deleted atomically w := clus.Client(0).Watch( clus.Client(0).Ctx(), "key/", clientv3.WithRev(delResp.Header.Revision), clientv3.WithPrefix()) wresp := <-w require.Lenf(t, wresp.Events, 8, "expected %d delete events,got %d", 8, len(wresp.Events)) } func TestLeasingDeleteRangeBounds(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) delkv, closeDelKV, err := leasing.NewKV(clus.Client(0), "0/") require.NoError(t, err) defer closeDelKV() getkv, closeGetKv, err := leasing.NewKV(clus.Client(0), "0/") require.NoError(t, err) defer closeGetKv() for _, k := range []string{"j", "m"} { _, err = clus.Client(0).Put(t.Context(), k, "123") require.NoError(t, err) _, err = getkv.Get(t.Context(), k) require.NoError(t, err) } _, err = delkv.Delete(t.Context(), "k", clientv3.WithPrefix()) require.NoError(t, err) // leases still on server? for _, k := range []string{"j", "m"} { resp, geterr := clus.Client(0).Get(t.Context(), "0/"+k, clientv3.WithPrefix()) require.NoError(t, geterr) require.Lenf(t, resp.Kvs, 1, "expected leasing key, got %+v", resp) } // j and m should still have leases registered since not under k* clus.Members[0].Stop(t) _, err = getkv.Get(t.Context(), "j") require.NoError(t, err) _, err = getkv.Get(t.Context(), "m") require.NoError(t, err) } func TestLeasingDeleteRangeContendTxn(t *testing.T) { then := []clientv3.Op{clientv3.OpDelete("key/", clientv3.WithPrefix())} testLeasingDeleteRangeContend(t, clientv3.OpTxn(nil, then, nil)) } func TestLeaseDeleteRangeContendDel(t *testing.T) { op := clientv3.OpDelete("key/", clientv3.WithPrefix()) testLeasingDeleteRangeContend(t, op) } func testLeasingDeleteRangeContend(t *testing.T, op clientv3.Op) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) delkv, closeDelKV, err := leasing.NewKV(clus.Client(0), "0/") require.NoError(t, err) defer closeDelKV() putkv, closePutKV, err := leasing.NewKV(clus.Client(0), "0/") require.NoError(t, err) defer closePutKV() const maxKey = 8 for i := 0; i < maxKey; i++ { key := fmt.Sprintf("key/%d", i) _, err = clus.Client(0).Put(t.Context(), key, "123") require.NoError(t, err) _, err = putkv.Get(t.Context(), key) require.NoError(t, err) } ctx, cancel := context.WithCancel(t.Context()) donec := make(chan struct{}) go func(t *testing.T) { defer close(donec) for i := 0; ctx.Err() == nil; i++ { key := fmt.Sprintf("key/%d", i%maxKey) if _, err = putkv.Put(t.Context(), key, "123"); err != nil { t.Errorf("fail putting key %s: %v", key, err) } if _, err = putkv.Get(t.Context(), key); err != nil { t.Errorf("fail getting key %s: %v", key, err) } } }(t) _, delErr := delkv.Do(t.Context(), op) cancel() <-donec require.NoError(t, delErr) // confirm keys on non-deleter match etcd for i := 0; i < maxKey; i++ { key := fmt.Sprintf("key/%d", i) resp, err := putkv.Get(t.Context(), key) require.NoError(t, err) servResp, err := clus.Client(0).Get(t.Context(), key) require.NoError(t, err) if !reflect.DeepEqual(resp.Kvs, servResp.Kvs) { t.Errorf("#%d: expected %+v, got %+v", i, servResp.Kvs, resp.Kvs) } } } func TestLeasingPutGetDeleteConcurrent(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkvs := make([]clientv3.KV, 16) for i := range lkvs { lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") require.NoError(t, err) defer closeLKV() lkvs[i] = lkv } getdel := func(kv clientv3.KV) { _, err := kv.Put(t.Context(), "k", "abc") require.NoError(t, err) time.Sleep(time.Millisecond) _, err = kv.Get(t.Context(), "k") require.NoError(t, err) _, err = kv.Delete(t.Context(), "k") require.NoError(t, err) time.Sleep(2 * time.Millisecond) } var wg sync.WaitGroup wg.Add(16) for i := 0; i < 16; i++ { go func() { defer wg.Done() for _, kv := range lkvs { getdel(kv) } }() } wg.Wait() resp, err := lkvs[0].Get(t.Context(), "k") require.NoError(t, err) require.Emptyf(t, resp.Kvs, "expected no kvs, got %+v", resp.Kvs) resp, err = clus.Client(0).Get(t.Context(), "k") require.NoError(t, err) require.Emptyf(t, resp.Kvs, "expected no kvs, got %+v", resp.Kvs) } // TestLeasingReconnectOwnerRevoke checks that revocation works if // disconnected when trying to submit revoke txn. func TestLeasingReconnectOwnerRevoke(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) lkv1, closeLKV1, err1 := leasing.NewKV(clus.Client(0), "foo/") require.NoError(t, err1) defer closeLKV1() lkv2, closeLKV2, err2 := leasing.NewKV(clus.Client(1), "foo/") require.NoError(t, err2) defer closeLKV2() _, err := lkv1.Get(t.Context(), "k") require.NoError(t, err) // force leader away from member 0 clus.Members[0].Stop(t) clus.WaitLeader(t) clus.Members[0].Restart(t) cctx, cancel := context.WithCancel(t.Context()) sdonec, pdonec := make(chan struct{}), make(chan struct{}) // make lkv1 connection choppy so Txn fails go func() { defer close(sdonec) for i := 0; i < 3 && cctx.Err() == nil; i++ { clus.Members[0].Stop(t) time.Sleep(10 * time.Millisecond) clus.Members[0].Restart(t) } }() go func() { defer close(pdonec) if _, err := lkv2.Put(cctx, "k", "v"); err != nil { t.Log(err) } // blocks until lkv1 connection comes back resp, err := lkv1.Get(cctx, "k") if err != nil { t.Error(err) } if string(resp.Kvs[0].Value) != "v" { t.Errorf(`expected "v" value, got %+v`, resp) } }() select { case <-pdonec: cancel() <-sdonec case <-time.After(15 * time.Second): cancel() <-sdonec <-pdonec t.Fatal("took too long to revoke and put") } } // TestLeasingReconnectOwnerRevokeCompact checks that revocation works if // disconnected and the watch is compacted. func TestLeasingReconnectOwnerRevokeCompact(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) lkv1, closeLKV1, err1 := leasing.NewKV(clus.Client(0), "foo/") require.NoError(t, err1) defer closeLKV1() lkv2, closeLKV2, err2 := leasing.NewKV(clus.Client(1), "foo/") require.NoError(t, err2) defer closeLKV2() _, err := lkv1.Get(t.Context(), "k") require.NoError(t, err) clus.Members[0].Stop(t) clus.WaitLeader(t) // put some more revisions for compaction _, err = clus.Client(1).Put(t.Context(), "a", "123") require.NoError(t, err) presp, err := clus.Client(1).Put(t.Context(), "a", "123") require.NoError(t, err) // compact while lkv1 is disconnected rev := presp.Header.Revision _, err = clus.Client(1).Compact(t.Context(), rev) require.NoError(t, err) clus.Members[0].Restart(t) cctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() _, err = lkv2.Put(cctx, "k", "v") require.NoError(t, err) resp, err := lkv1.Get(cctx, "k") require.NoError(t, err) require.Equalf(t, "v", string(resp.Kvs[0].Value), `expected "v" value, got %+v`, resp) } // TestLeasingReconnectOwnerConsistency checks a write error on an owner will // not cause inconsistency between the server and the client. func TestLeasingReconnectOwnerConsistency(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") defer closeLKV() require.NoError(t, err) _, err = lkv.Put(t.Context(), "k", "x") require.NoError(t, err) _, err = lkv.Put(t.Context(), "kk", "y") require.NoError(t, err) _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) for i := 0; i < 10; i++ { v := fmt.Sprintf("%d", i) donec := make(chan struct{}) clus.Members[0].Bridge().DropConnections() go func() { defer close(donec) for i := 0; i < 20; i++ { clus.Members[0].Bridge().DropConnections() time.Sleep(time.Millisecond) } }() switch rand.Intn(7) { case 0: _, err = lkv.Put(t.Context(), "k", v) case 1: _, err = lkv.Delete(t.Context(), "k") case 2: txn := lkv.Txn(t.Context()).Then( clientv3.OpGet("k"), clientv3.OpDelete("k"), ) _, err = txn.Commit() case 3: txn := lkv.Txn(t.Context()).Then( clientv3.OpGet("k"), clientv3.OpPut("k", v), ) _, err = txn.Commit() case 4: _, err = lkv.Do(t.Context(), clientv3.OpPut("k", v)) case 5: _, err = lkv.Do(t.Context(), clientv3.OpDelete("k")) case 6: _, err = lkv.Delete(t.Context(), "k", clientv3.WithPrefix()) } <-donec if err != nil { // TODO wrap input client to generate errors break } } lresp, lerr := lkv.Get(t.Context(), "k") require.NoError(t, lerr) cresp, cerr := clus.Client(0).Get(t.Context(), "k") require.NoError(t, cerr) require.Truef(t, reflect.DeepEqual(lresp.Kvs, cresp.Kvs), "expected %+v, got %+v", cresp, lresp) } func TestLeasingTxnAtomicCache(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") require.NoError(t, err) defer closeLKV() puts, gets := make([]clientv3.Op, 16), make([]clientv3.Op, 16) for i := range puts { k := fmt.Sprintf("k-%d", i) puts[i], gets[i] = clientv3.OpPut(k, k), clientv3.OpGet(k) } _, err = clus.Client(0).Txn(t.Context()).Then(puts...).Commit() require.NoError(t, err) for i := range gets { _, err = lkv.Do(t.Context(), gets[i]) require.NoError(t, err) } numPutters, numGetters := 16, 16 var wgPutters, wgGetters sync.WaitGroup wgPutters.Add(numPutters) wgGetters.Add(numGetters) txnerrCh := make(chan error, 1) f := func() { defer wgPutters.Done() for i := 0; i < 10; i++ { if _, txnerr := lkv.Txn(t.Context()).Then(puts...).Commit(); txnerr != nil { select { case txnerrCh <- txnerr: default: } } } } donec := make(chan struct{}, numPutters) g := func() { defer wgGetters.Done() for { select { case <-donec: return default: } tresp, err := lkv.Txn(t.Context()).Then(gets...).Commit() if err != nil { t.Error(err) } revs := make([]int64, len(gets)) for i, resp := range tresp.Responses { rr := resp.GetResponseRange() revs[i] = rr.Kvs[0].ModRevision } for i := 1; i < len(revs); i++ { if revs[i] != revs[i-1] { t.Errorf("expected matching revisions, got %+v", revs) } } } } for i := 0; i < numGetters; i++ { go g() } for i := 0; i < numPutters; i++ { go f() } wgPutters.Wait() select { case txnerr := <-txnerrCh: t.Fatal(txnerr) default: } close(donec) wgGetters.Wait() } // TestLeasingReconnectTxn checks that Txn is resilient to disconnects. func TestLeasingReconnectTxn(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") require.NoError(t, err) defer closeLKV() _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) donec := make(chan struct{}) go func() { defer close(donec) clus.Members[0].Bridge().DropConnections() for i := 0; i < 10; i++ { clus.Members[0].Bridge().DropConnections() time.Sleep(time.Millisecond) } time.Sleep(10 * time.Millisecond) }() _, lerr := lkv.Txn(t.Context()). If(clientv3.Compare(clientv3.Version("k"), "=", 0)). Then(clientv3.OpGet("k")). Commit() <-donec require.NoError(t, lerr) } // TestLeasingReconnectNonOwnerGet checks a get error on an owner will // not cause inconsistency between the server and the client. func TestLeasingReconnectNonOwnerGet(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") require.NoError(t, err) defer closeLKV() // populate a few keys so some leasing gets have keys for i := 0; i < 4; i++ { k := fmt.Sprintf("k-%d", i*2) _, err = lkv.Put(t.Context(), k, k[2:]) require.NoError(t, err) } n := 0 for i := 0; i < 10; i++ { donec := make(chan struct{}) clus.Members[0].Bridge().DropConnections() go func() { defer close(donec) for j := 0; j < 10; j++ { clus.Members[0].Bridge().DropConnections() time.Sleep(time.Millisecond) } }() _, err = lkv.Get(t.Context(), fmt.Sprintf("k-%d", i)) <-donec n++ if err != nil { break } } for i := 0; i < n; i++ { k := fmt.Sprintf("k-%d", i) lresp, lerr := lkv.Get(t.Context(), k) require.NoError(t, lerr) cresp, cerr := clus.Client(0).Get(t.Context(), k) require.NoError(t, cerr) require.Truef(t, reflect.DeepEqual(lresp.Kvs, cresp.Kvs), "expected %+v, got %+v", cresp, lresp) } } func TestLeasingTxnRangeCmp(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") require.NoError(t, err) defer closeLKV() _, err = clus.Client(0).Put(t.Context(), "k", "a") require.NoError(t, err) // k2 version = 2 _, err = clus.Client(0).Put(t.Context(), "k2", "a") require.NoError(t, err) _, err = clus.Client(0).Put(t.Context(), "k2", "a") require.NoError(t, err) // cache k _, err = lkv.Get(t.Context(), "k") require.NoError(t, err) cmp := clientv3.Compare(clientv3.Version("k").WithPrefix(), "=", 1) tresp, terr := lkv.Txn(t.Context()).If(cmp).Commit() require.NoError(t, terr) require.Falsef(t, tresp.Succeeded, "expected Succeeded=false, got %+v", tresp) } func TestLeasingDo(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") require.NoError(t, err) defer closeLKV() ops := []clientv3.Op{ clientv3.OpTxn(nil, nil, nil), clientv3.OpGet("a"), clientv3.OpPut("a/abc", "v"), clientv3.OpDelete("a", clientv3.WithPrefix()), clientv3.OpTxn(nil, nil, nil), } for i, op := range ops { resp, resperr := lkv.Do(t.Context(), op) if resperr != nil { t.Errorf("#%d: failed (%v)", i, resperr) } switch { case op.IsGet() && resp.Get() == nil: t.Errorf("#%d: get but nil get response", i) case op.IsPut() && resp.Put() == nil: t.Errorf("#%d: put op but nil get response", i) case op.IsDelete() && resp.Del() == nil: t.Errorf("#%d: delete op but nil delete response", i) case op.IsTxn() && resp.Txn() == nil: t.Errorf("#%d: txn op but nil txn response", i) } } gresp, err := clus.Client(0).Get(t.Context(), "a", clientv3.WithPrefix()) require.NoError(t, err) require.Emptyf(t, gresp.Kvs, "expected no keys, got %+v", gresp.Kvs) } func TestLeasingTxnOwnerPutBranch(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") require.NoError(t, err) defer closeLKV() n := 0 treeOp := makePutTreeOp("tree", &n, 4) for i := 0; i < n; i++ { k := fmt.Sprintf("tree/%d", i) _, err = clus.Client(0).Put(t.Context(), k, "a") require.NoError(t, err) _, err = lkv.Get(t.Context(), k) require.NoError(t, err) } _, err = lkv.Do(t.Context(), treeOp) require.NoError(t, err) // lkv shouldn't need to call out to server for updated leased keys clus.Members[0].Stop(t) for i := 0; i < n; i++ { k := fmt.Sprintf("tree/%d", i) lkvResp, err := lkv.Get(t.Context(), k) require.NoError(t, err) clusResp, err := clus.Client(1).Get(t.Context(), k) require.NoError(t, err) require.Truef(t, reflect.DeepEqual(clusResp.Kvs, lkvResp.Kvs), "expected %+v, got %+v", clusResp.Kvs, lkvResp.Kvs) } } func makePutTreeOp(pfx string, v *int, depth int) clientv3.Op { key := fmt.Sprintf("%s/%d", pfx, *v) *v = *v + 1 if depth == 0 { return clientv3.OpPut(key, "leaf") } t, e := makePutTreeOp(pfx, v, depth-1), makePutTreeOp(pfx, v, depth-1) tPut, ePut := clientv3.OpPut(key, "then"), clientv3.OpPut(key, "else") cmps := make([]clientv3.Cmp, 1) if rand.Intn(2) == 0 { // follow then path cmps[0] = clientv3.Compare(clientv3.Version("nokey"), "=", 0) } else { // follow else path cmps[0] = clientv3.Compare(clientv3.Version("nokey"), ">", 0) } return clientv3.OpTxn(cmps, []clientv3.Op{t, tPut}, []clientv3.Op{e, ePut}) } func randCmps(pfx string, dat []*clientv3.PutResponse) (cmps []clientv3.Cmp, then bool) { for i := 0; i < len(dat); i++ { idx := rand.Intn(len(dat)) k := fmt.Sprintf("%s%d", pfx, idx) rev := dat[idx].Header.Revision var cmp clientv3.Cmp switch rand.Intn(4) { case 0: cmp = clientv3.Compare(clientv3.CreateRevision(k), ">", rev-1) case 1: cmp = clientv3.Compare(clientv3.Version(k), "=", 1) case 2: cmp = clientv3.Compare(clientv3.CreateRevision(k), "=", rev) case 3: cmp = clientv3.Compare(clientv3.CreateRevision(k), "!=", rev+1) } cmps = append(cmps, cmp) } cmps = cmps[:rand.Intn(len(dat))] if rand.Intn(2) == 0 { return cmps, true } i := rand.Intn(len(dat)) cmps = append(cmps, clientv3.Compare(clientv3.Version(fmt.Sprintf("k-%d", i)), "=", 0)) return cmps, false } func TestLeasingSessionExpire(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/", concurrency.WithTTL(1)) require.NoError(t, err) defer closeLKV() lkv2, closeLKV2, err := leasing.NewKV(clus.Client(0), "foo/") require.NoError(t, err) defer closeLKV2() // acquire lease on abc _, err = lkv.Get(t.Context(), "abc") require.NoError(t, err) // down endpoint lkv uses for keepalives clus.Members[0].Stop(t) err = waitForLeasingExpire(clus.Client(1), "foo/abc") require.NoError(t, err) waitForExpireAck(t, lkv) clus.Members[0].Restart(t) integration.WaitClientV3(t, lkv2) _, err = lkv2.Put(t.Context(), "abc", "def") require.NoError(t, err) resp, err := lkv.Get(t.Context(), "abc") require.NoError(t, err) require.Equal(t, "def", string(resp.Kvs[0].Value)) } func TestLeasingSessionExpireCancel(t *testing.T) { tests := []func(context.Context, clientv3.KV) error{ func(ctx context.Context, kv clientv3.KV) error { _, err := kv.Get(ctx, "abc") return err }, func(ctx context.Context, kv clientv3.KV) error { _, err := kv.Delete(ctx, "abc") return err }, func(ctx context.Context, kv clientv3.KV) error { _, err := kv.Put(ctx, "abc", "v") return err }, func(ctx context.Context, kv clientv3.KV) error { _, err := kv.Txn(ctx).Then(clientv3.OpGet("abc")).Commit() return err }, func(ctx context.Context, kv clientv3.KV) error { _, err := kv.Do(ctx, clientv3.OpPut("abc", "v")) return err }, func(ctx context.Context, kv clientv3.KV) error { _, err := kv.Do(ctx, clientv3.OpDelete("abc")) return err }, func(ctx context.Context, kv clientv3.KV) error { _, err := kv.Do(ctx, clientv3.OpGet("abc")) return err }, func(ctx context.Context, kv clientv3.KV) error { op := clientv3.OpTxn(nil, []clientv3.Op{clientv3.OpGet("abc")}, nil) _, err := kv.Do(ctx, op) return err }, } for i := range tests { t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/", concurrency.WithTTL(1)) require.NoError(t, err) defer closeLKV() _, err = lkv.Get(t.Context(), "abc") require.NoError(t, err) // down endpoint lkv uses for keepalives clus.Members[0].Stop(t) err = waitForLeasingExpire(clus.Client(1), "foo/abc") require.NoError(t, err) waitForExpireAck(t, lkv) ctx, cancel := context.WithCancel(t.Context()) errc := make(chan error, 1) go func() { errc <- tests[i](ctx, lkv) }() // some delay to get past for ctx.Err() != nil {} loops time.Sleep(100 * time.Millisecond) cancel() select { case err := <-errc: if !errors.Is(err, ctx.Err()) { t.Errorf("#%d: expected %v of server unavailable, got %v", i, ctx.Err(), err) } case <-time.After(5 * time.Second): t.Errorf("#%d: timed out waiting for cancel", i) } clus.Members[0].Restart(t) }) } } func waitForLeasingExpire(kv clientv3.KV, lkey string) error { for { time.Sleep(1 * time.Second) resp, err := kv.Get(context.TODO(), lkey, clientv3.WithPrefix()) if err != nil { return err } if len(resp.Kvs) == 0 { // server expired the leasing key return nil } } } func waitForExpireAck(t *testing.T, kv clientv3.KV) { // wait for leasing client to acknowledge lost lease for i := 0; i < 10; i++ { ctx, cancel := context.WithTimeout(t.Context(), time.Second) _, err := kv.Get(ctx, "abc") cancel() if errors.Is(err, ctx.Err()) { return } else if err != nil { t.Logf("current error: %v", err) } time.Sleep(time.Second) } t.Fatalf("waited too long to acknlowedge lease expiration") } ================================================ FILE: tests/integration/clientv3/lease/main_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package lease_test import ( "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func TestMain(m *testing.M) { testutil.MustTestMainWithLeakDetection(m) } ================================================ FILE: tests/integration/clientv3/main_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func TestMain(m *testing.M) { testutil.MustTestMainWithLeakDetection(m) } ================================================ FILE: tests/integration/clientv3/maintenance_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "bytes" "context" "crypto/sha256" "errors" "fmt" "io" "math" "os" "path/filepath" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/api/v3/version" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/mvcc/testutil" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestMaintenanceHashKV(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) for i := 0; i < 3; i++ { _, err := clus.RandClient().Put(t.Context(), "foo", "bar") require.NoError(t, err) } var hv uint32 for i := 0; i < 3; i++ { cli := clus.Client(i) // ensure writes are replicated _, err := cli.Get(t.Context(), "foo") require.NoError(t, err) hresp, err := cli.HashKV(t.Context(), clus.Members[i].GRPCURL, 0) require.NoError(t, err) if hv == 0 { hv = hresp.Hash continue } if hv != hresp.Hash { t.Fatalf("#%d: hash expected %d, got %d", i, hv, hresp.Hash) } } } // TestCompactionHash tests compaction hash // TODO: Change this to fuzz test func TestCompactionHash(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cc, err := clus.ClusterClient(t) require.NoError(t, err) testutil.TestCompactionHash(t.Context(), t, hashTestCase{cc, clus.Members[0].GRPCURL}, 1000) } type hashTestCase struct { *clientv3.Client url string } func (tc hashTestCase) Put(ctx context.Context, key, value string) error { _, err := tc.Client.Put(ctx, key, value) return err } func (tc hashTestCase) Delete(ctx context.Context, key string) error { _, err := tc.Client.Delete(ctx, key) return err } func (tc hashTestCase) HashByRev(ctx context.Context, rev int64) (testutil.KeyValueHash, error) { resp, err := tc.Client.HashKV(ctx, tc.url, rev) return testutil.KeyValueHash{Hash: resp.Hash, CompactRevision: resp.CompactRevision, Revision: resp.Header.Revision}, err } func (tc hashTestCase) Defrag(ctx context.Context) error { _, err := tc.Client.Defragment(ctx, tc.url) return err } func (tc hashTestCase) Compact(ctx context.Context, rev int64) error { _, err := tc.Client.Compact(ctx, rev) // Wait for compaction to be compacted time.Sleep(50 * time.Millisecond) return err } func TestMaintenanceMoveLeader(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) oldLeadIdx := clus.WaitLeader(t) targetIdx := (oldLeadIdx + 1) % 3 target := uint64(clus.Members[targetIdx].ID()) cli := clus.Client(targetIdx) _, err := cli.MoveLeader(t.Context(), target) if !errors.Is(err, rpctypes.ErrNotLeader) { t.Fatalf("error expected %v, got %v", rpctypes.ErrNotLeader, err) } cli = clus.Client(oldLeadIdx) _, err = cli.MoveLeader(t.Context(), target) require.NoError(t, err) leadIdx := clus.WaitLeader(t) lead := uint64(clus.Members[leadIdx].ID()) if target != lead { t.Fatalf("new leader expected %d, got %d", target, lead) } } // TestMaintenanceSnapshotCancel ensures that context cancel // before snapshot reading returns corresponding context errors. func TestMaintenanceSnapshotCancel(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) // reading snapshot with canceled context should error out ctx, cancel := context.WithCancel(t.Context()) // Since http2 spec defines the receive windows's size and max size of // frame in the stream, the underlayer - gRPC client can pre-read data // from server even if the application layer hasn't read it yet. // // And the initialized cluster has 20KiB snapshot, which can be // pre-read by underlayer. We should increase the snapshot's size here, // just in case that io.Copy won't return the canceled error. populateDataIntoCluster(t, clus, 3, 1024*1024) rc1, err := clus.RandClient().Snapshot(ctx) require.NoError(t, err) defer rc1.Close() // read 16 bytes to ensure that server opens snapshot buf := make([]byte, 16) n, err := rc1.Read(buf) require.NoError(t, err) assert.Equal(t, 16, n) cancel() _, err = io.Copy(io.Discard, rc1) if !errors.Is(err, context.Canceled) { t.Errorf("expected %v, got %v", context.Canceled, err) } } // TestMaintenanceSnapshotWithVersionTimeout ensures that SnapshotWithVersion function // returns corresponding context errors when context timeout happened before snapshot reading func TestMaintenanceSnapshotWithVersionTimeout(t *testing.T) { testMaintenanceSnapshotTimeout(t, func(ctx context.Context, client *clientv3.Client) (io.ReadCloser, error) { resp, err := client.SnapshotWithVersion(ctx) if err != nil { return nil, err } return resp.Snapshot, nil }) } // TestMaintenanceSnapshotTimeout ensures that Snapshot function // returns corresponding context errors when context timeout happened before snapshot reading func TestMaintenanceSnapshotTimeout(t *testing.T) { testMaintenanceSnapshotTimeout(t, func(ctx context.Context, client *clientv3.Client) (io.ReadCloser, error) { return client.Snapshot(ctx) }) } // testMaintenanceSnapshotTimeout given snapshot function ensures that it // returns corresponding context errors when context timeout happened before snapshot reading func testMaintenanceSnapshotTimeout(t *testing.T, snapshot func(context.Context, *clientv3.Client) (io.ReadCloser, error)) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) // reading snapshot with deadline exceeded should error out ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() // Since http2 spec defines the receive windows's size and max size of // frame in the stream, the underlayer - gRPC client can pre-read data // from server even if the application layer hasn't read it yet. // // And the initialized cluster has 20KiB snapshot, which can be // pre-read by underlayer. We should increase the snapshot's size here, // just in case that io.Copy won't return the timeout error. populateDataIntoCluster(t, clus, 3, 1024*1024) rc2, err := snapshot(ctx, clus.RandClient()) require.NoError(t, err) defer rc2.Close() time.Sleep(2 * time.Second) _, err = io.Copy(io.Discard, rc2) if IsClientTimeout(err) { return } // Assumes the client receives a single message header and then // waits for the payload body. If the context is canceled before // the payload arrives, the client will read io.EOF. However, the // grpc-go client converts this into io.ErrUnexpectedEOF with an // internal error code. Ideally, grpc-go might return context.Canceled // instead, but it's unclear if that's feasible. Let's explicitly // check for this error in the test code. // // REF: https://github.com/grpc/grpc-go/blob/6821606f351799b026fda1e6ba143315e6c1e620/rpc_util.go#L644 // // Once https://github.com/grpc/grpc-go/issues/8281 is fixed, we should // revert this change. See more discussion in https://github.com/etcd-io/etcd/pull/19833. assert.ErrorIs(t, status.Error(codes.Internal, io.ErrUnexpectedEOF.Error()), err) } // TestMaintenanceSnapshotWithVersionErrorInflight ensures that ReaderCloser returned by SnapshotWithVersion function // will fail to read with corresponding context errors on inflight context cancel timeout. func TestMaintenanceSnapshotWithVersionErrorInflight(t *testing.T) { testMaintenanceSnapshotErrorInflight(t, func(ctx context.Context, client *clientv3.Client) (io.ReadCloser, error) { resp, err := client.SnapshotWithVersion(ctx) if err != nil { return nil, err } return resp.Snapshot, nil }) } // TestMaintenanceSnapshotErrorInflight ensures that ReaderCloser returned by Snapshot function // will fail to read with corresponding context errors on inflight context cancel timeout. func TestMaintenanceSnapshotErrorInflight(t *testing.T) { testMaintenanceSnapshotErrorInflight(t, func(ctx context.Context, client *clientv3.Client) (io.ReadCloser, error) { return client.Snapshot(ctx) }) } // testMaintenanceSnapshotErrorInflight given snapshot function ensures that ReaderCloser returned by it // will fail to read with corresponding context errors on inflight context cancel timeout. func testMaintenanceSnapshotErrorInflight(t *testing.T, snapshot func(context.Context, *clientv3.Client) (io.ReadCloser, error)) { integration.BeforeTest(t) lg := zaptest.NewLogger(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) // take about 1-second to read snapshot clus.Members[0].Stop(t) dpath := filepath.Join(clus.Members[0].DataDir, "member", "snap", "db") b := backend.NewDefaultBackend(lg, dpath) s := mvcc.NewStore(lg, b, &lease.FakeLessor{}, mvcc.StoreConfig{CompactionBatchLimit: math.MaxInt32}) rev := 100000 for i := 2; i <= rev; i++ { s.Put([]byte(fmt.Sprintf("%10d", i)), bytes.Repeat([]byte("a"), 1024), lease.NoLease) } s.Close() b.Close() clus.Members[0].Restart(t) // reading snapshot with canceled context should error out ctx, cancel := context.WithCancel(t.Context()) rc1, err := snapshot(ctx, clus.RandClient()) require.NoError(t, err) defer rc1.Close() donec := make(chan struct{}) go func() { time.Sleep(300 * time.Millisecond) cancel() close(donec) }() _, err = io.Copy(io.Discard, rc1) if err != nil && !errors.Is(err, context.Canceled) { t.Errorf("expected %v, got %v", context.Canceled, err) } <-donec // reading snapshot with deadline exceeded should error out ctx, cancel = context.WithTimeout(t.Context(), time.Second) defer cancel() rc2, err := snapshot(ctx, clus.RandClient()) require.NoError(t, err) defer rc2.Close() // 300ms left and expect timeout while snapshot reading is in progress time.Sleep(700 * time.Millisecond) _, err = io.Copy(io.Discard, rc2) if err != nil && !IsClientTimeout(err) { t.Errorf("expected client timeout, got %v", err) } } // TestMaintenanceSnapshotWithVersionVersion ensures that SnapshotWithVersion returns correct version value. func TestMaintenanceSnapshotWithVersionVersion(t *testing.T) { integration.BeforeTest(t) // Set SnapshotCount to 1 to force raft snapshot to ensure that storage version is set clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, SnapshotCount: 1}) defer clus.Terminate(t) // Put some keys to ensure that wal snapshot is triggered for i := 0; i < 10; i++ { clus.RandClient().Put(t.Context(), fmt.Sprintf("%d", i), "1") } // reading snapshot with canceled context should error out resp, err := clus.RandClient().SnapshotWithVersion(t.Context()) require.NoError(t, err) defer resp.Snapshot.Close() if resp.Version != "3.7.0" { t.Errorf("unexpected version, expected %q, got %q", version.Version, resp.Version) } } func TestMaintenanceSnapshotContentDigest(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) populateDataIntoCluster(t, clus, 3, 1024*1024) // reading snapshot with canceled context should error out resp, err := clus.RandClient().SnapshotWithVersion(t.Context()) require.NoError(t, err) defer resp.Snapshot.Close() tmpDir := t.TempDir() snapFile, err := os.Create(filepath.Join(tmpDir, t.Name())) require.NoError(t, err) defer snapFile.Close() snapSize, err := io.Copy(snapFile, resp.Snapshot) require.NoError(t, err) // read the checksum checksumSize := int64(sha256.Size) _, err = snapFile.Seek(-checksumSize, io.SeekEnd) require.NoError(t, err) checksumInBytes, err := io.ReadAll(snapFile) require.NoError(t, err) require.Len(t, checksumInBytes, int(checksumSize)) // remove the checksum part and rehash err = snapFile.Truncate(snapSize - checksumSize) require.NoError(t, err) _, err = snapFile.Seek(0, io.SeekStart) require.NoError(t, err) hashWriter := sha256.New() _, err = io.Copy(hashWriter, snapFile) require.NoError(t, err) // compare the checksum actualChecksum := hashWriter.Sum(nil) require.Equal(t, checksumInBytes, actualChecksum) } func TestMaintenanceStatus(t *testing.T) { testCases := []struct { name string quotaCfg int64 expectedQuota int64 }{ { name: "0 quota", quotaCfg: 0, expectedQuota: storage.DefaultQuotaBytes, }, { name: "default quota", quotaCfg: storage.DefaultQuotaBytes, expectedQuota: storage.DefaultQuotaBytes, }, { name: "customized quota", quotaCfg: 300010002000, expectedQuota: 300010002000, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, QuotaBackendBytes: tc.quotaCfg}) defer clus.Terminate(t) t.Logf("Waiting for leader...") clus.WaitLeader(t) t.Logf("Leader established.") eps := make([]string, 3) for i := 0; i < 3; i++ { eps[i] = clus.Members[i].GRPCURL } t.Logf("Creating client...") cli, err := integration.NewClient(t, clientv3.Config{Endpoints: eps}) require.NoError(t, err) defer cli.Close() t.Logf("Creating client [DONE]") prevID, leaderFound := uint64(0), false for i := 0; i < 3; i++ { resp, err := cli.Status(t.Context(), eps[i]) require.NoError(t, err) t.Logf("Response from %v: %v", i, resp) require.Equal(t, tc.expectedQuota, resp.DbSizeQuota) if prevID == 0 { prevID, leaderFound = resp.Header.MemberId, resp.Header.MemberId == resp.Leader continue } if prevID == resp.Header.MemberId { t.Errorf("#%d: status returned duplicate member ID with %016x", i, prevID) } if leaderFound && resp.Header.MemberId == resp.Leader { t.Errorf("#%d: leader already found, but found another %016x", i, resp.Header.MemberId) } if !leaderFound { leaderFound = resp.Header.MemberId == resp.Leader } } if !leaderFound { t.Fatal("no leader found") } }) } } ================================================ FILE: tests/integration/clientv3/metrics_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "bufio" "bytes" "errors" "io" "net" "net/http" "strconv" "strings" "testing" "time" grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/stretchr/testify/require" "google.golang.org/grpc" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestV3ClientMetrics(t *testing.T) { integration.BeforeTest(t) var ( addr = "localhost:27989" ln net.Listener ) srv := &http.Server{Handler: promhttp.Handler()} srv.SetKeepAlivesEnabled(false) ln, err := transport.NewUnixListener(addr) if err != nil { t.Errorf("Error: %v occurred while listening on addr: %v", err, addr) } donec := make(chan struct{}) defer func() { ln.Close() <-donec }() // listen for all Prometheus metrics go func() { defer close(donec) serr := srv.Serve(ln) if serr != nil && !transport.IsClosedConnError(serr) { t.Errorf("Err serving http requests: %v", serr) } }() url := "unix://" + addr + "/metrics" clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) clientMetrics := grpcprom.NewClientMetrics() prometheus.Register(clientMetrics) cfg := clientv3.Config{ Endpoints: []string{clus.Members[0].GRPCURL}, DialOptions: []grpc.DialOption{ grpc.WithUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), grpc.WithStreamInterceptor(clientMetrics.StreamClientInterceptor()), }, } cli, cerr := integration.NewClient(t, cfg) require.NoError(t, cerr) defer cli.Close() wc := cli.Watch(t.Context(), "foo") wBefore := sumCountersForMetricAndLabels(t, url, "grpc_client_msg_received_total", "Watch", "bidi_stream") pBefore := sumCountersForMetricAndLabels(t, url, "grpc_client_started_total", "Put", "unary") _, err = cli.Put(t.Context(), "foo", "bar") if err != nil { t.Errorf("Error putting value in key store") } pAfter := sumCountersForMetricAndLabels(t, url, "grpc_client_started_total", "Put", "unary") if pBefore+1 != pAfter { t.Errorf("grpc_client_started_total expected %d, got %d", 1, pAfter-pBefore) } // consume watch response select { case <-wc: case <-time.After(10 * time.Second): t.Error("Timeout occurred for getting watch response") } wAfter := sumCountersForMetricAndLabels(t, url, "grpc_client_msg_received_total", "Watch", "bidi_stream") if wBefore+1 != wAfter { t.Errorf("grpc_client_msg_received_total expected %d, got %d", 1, wAfter-wBefore) } } func sumCountersForMetricAndLabels(t *testing.T, url string, metricName string, matchingLabelValues ...string) int { count := 0 for _, line := range getHTTPBodyAsLines(t, url) { ok := true if !strings.HasPrefix(line, metricName) { continue } for _, labelValue := range matchingLabelValues { if !strings.Contains(line, `"`+labelValue+`"`) { ok = false break } } if !ok { continue } valueString := line[strings.LastIndex(line, " ")+1 : len(line)-1] valueFloat, err := strconv.ParseFloat(valueString, 32) if err != nil { t.Fatalf("failed parsing value for line: %v and matchingLabelValues: %v", line, matchingLabelValues) } count += int(valueFloat) } return count } func getHTTPBodyAsLines(t *testing.T, url string) []string { data := getHTTPBodyAsBytes(t, url) reader := bufio.NewReader(bytes.NewReader(data)) var lines []string for { line, err := reader.ReadString('\n') if err != nil { if errors.Is(err, io.EOF) { break } t.Fatalf("error reading: %v", err) } lines = append(lines, line) } return lines } func getHTTPBodyAsBytes(t *testing.T, url string) []byte { cfgtls := transport.TLSInfo{} tr, err := transport.NewTransport(cfgtls, time.Second) if err != nil { t.Fatalf("Error getting transport: %v", err) } tr.MaxIdleConns = -1 tr.DisableKeepAlives = true cli := &http.Client{Transport: tr} resp, err := cli.Get(url) if err != nil { t.Fatalf("Error fetching: %v", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { t.Fatalf("Error reading http body: %v", err) } return body } ================================================ FILE: tests/integration/clientv3/mirror_auth_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package clientv3test import ( "reflect" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/mirror" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestMirrorSync_Authenticated(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) initialClient := clus.Client(0) // Create a user to run the mirror process that only has access to /syncpath initialClient.RoleAdd(t.Context(), "syncer") initialClient.RoleGrantPermission(t.Context(), "syncer", "/syncpath", clientv3.GetPrefixRangeEnd("/syncpath"), clientv3.PermissionType(clientv3.PermReadWrite)) initialClient.UserAdd(t.Context(), "syncer", "syncfoo") initialClient.UserGrantRole(t.Context(), "syncer", "syncer") // Seed /syncpath with some initial data _, err := initialClient.KV.Put(t.Context(), "/syncpath/foo", "bar") require.NoError(t, err) // Require authentication authSetupRoot(t, initialClient.Auth) // Create a client as the `syncer` user. cfg := clientv3.Config{ Endpoints: initialClient.Endpoints(), DialTimeout: 5 * time.Second, Username: "syncer", Password: "syncfoo", } syncClient, err := integration.NewClient(t, cfg) require.NoError(t, err) defer syncClient.Close() // Now run the sync process, create changes, and get the initial sync state syncer := mirror.NewSyncer(syncClient, "/syncpath", 0) gch, ech := syncer.SyncBase(t.Context()) wkvs := []*mvccpb.KeyValue{{Key: []byte("/syncpath/foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}} for g := range gch { if !reflect.DeepEqual(g.Kvs, wkvs) { t.Fatalf("kv = %v, want %v", g.Kvs, wkvs) } } for e := range ech { t.Fatalf("unexpected error %v", e) } // Start a continuous sync wch := syncer.SyncUpdates(t.Context()) // Update state _, err = syncClient.KV.Put(t.Context(), "/syncpath/foo", "baz") require.NoError(t, err) // Wait for the updated state to sync select { case r := <-wch: wkv := &mvccpb.KeyValue{Key: []byte("/syncpath/foo"), Value: []byte("baz"), CreateRevision: 2, ModRevision: 3, Version: 2} if !reflect.DeepEqual(r.Events[0].Kv, wkv) { t.Fatalf("kv = %v, want %v", r.Events[0].Kv, wkv) } case <-time.After(time.Second): t.Fatal("failed to receive update in one second") } } ================================================ FILE: tests/integration/clientv3/mirror_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "fmt" "reflect" "sync" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/client/v3/mirror" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestMirrorSync(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) c := clus.Client(0) _, err := c.KV.Put(t.Context(), "foo", "bar") require.NoError(t, err) syncer := mirror.NewSyncer(c, "", 0) gch, ech := syncer.SyncBase(t.Context()) wkvs := []*mvccpb.KeyValue{{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}} for g := range gch { if !reflect.DeepEqual(g.Kvs, wkvs) { t.Fatalf("kv = %v, want %v", g.Kvs, wkvs) } } for e := range ech { t.Fatalf("unexpected error %v", e) } wch := syncer.SyncUpdates(t.Context()) _, err = c.KV.Put(t.Context(), "foo", "bar") require.NoError(t, err) select { case r := <-wch: wkv := &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 3, Version: 2} if !reflect.DeepEqual(r.Events[0].Kv, wkv) { t.Fatalf("kv = %v, want %v", r.Events[0].Kv, wkv) } case <-time.After(time.Second): t.Fatal("failed to receive update in one second") } } func TestMirrorSyncBase(t *testing.T) { integration.BeforeTest(t) cluster := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer cluster.Terminate(t) cli := cluster.Client(0) ctx := t.Context() keyCh := make(chan string) var wg sync.WaitGroup for i := 0; i < 50; i++ { wg.Add(1) go func() { defer wg.Done() for key := range keyCh { if _, err := cli.Put(ctx, key, "test"); err != nil { t.Error(err) } } }() } for i := 0; i < 2000; i++ { keyCh <- fmt.Sprintf("test%d", i) } close(keyCh) wg.Wait() syncer := mirror.NewSyncer(cli, "test", 0) respCh, errCh := syncer.SyncBase(ctx) count := 0 for resp := range respCh { count = count + len(resp.Kvs) if !resp.More { break } } for err := range errCh { t.Fatalf("unexpected error %v", err) } if count != 2000 { t.Errorf("unexpected kv count: %d", count) } } ================================================ FILE: tests/integration/clientv3/namespace_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "reflect" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/namespace" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestNamespacePutGet(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) c := clus.Client(0) nsKV := namespace.NewKV(c.KV, "foo/") _, err := nsKV.Put(t.Context(), "abc", "bar") require.NoError(t, err) resp, err := nsKV.Get(t.Context(), "abc") require.NoError(t, err) if string(resp.Kvs[0].Key) != "abc" { t.Errorf("expected key=%q, got key=%q", "abc", resp.Kvs[0].Key) } resp, err = c.Get(t.Context(), "foo/abc") require.NoError(t, err) if string(resp.Kvs[0].Value) != "bar" { t.Errorf("expected value=%q, got value=%q", "bar", resp.Kvs[0].Value) } } func TestNamespaceWatch(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) c := clus.Client(0) nsKV := namespace.NewKV(c.KV, "foo/") nsWatcher := namespace.NewWatcher(c.Watcher, "foo/") _, err := nsKV.Put(t.Context(), "abc", "bar") require.NoError(t, err) nsWch := nsWatcher.Watch(t.Context(), "abc", clientv3.WithRev(1)) wkv := &mvccpb.KeyValue{Key: []byte("abc"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1} if wr := <-nsWch; len(wr.Events) != 1 || !reflect.DeepEqual(wr.Events[0].Kv, wkv) { t.Errorf("expected namespaced event %+v, got %+v", wkv, wr.Events[0].Kv) } wch := c.Watch(t.Context(), "foo/abc", clientv3.WithRev(1)) wkv = &mvccpb.KeyValue{Key: []byte("foo/abc"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1} if wr := <-wch; len(wr.Events) != 1 || !reflect.DeepEqual(wr.Events[0].Kv, wkv) { t.Errorf("expected unnamespaced event %+v, got %+v", wkv, wr) } // let client close teardown namespace watch c.Watcher = nsWatcher } ================================================ FILE: tests/integration/clientv3/naming/endpoints_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package naming_test import ( "context" "reflect" "testing" "github.com/stretchr/testify/require" etcd "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/naming/endpoints" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestEndpointManager(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) em, err := endpoints.NewManager(clus.RandClient(), "foo") if err != nil { t.Fatal("failed to create EndpointManager", err) } ctx, watchCancel := context.WithCancel(t.Context()) defer watchCancel() w, err := em.NewWatchChannel(ctx) if err != nil { t.Fatal("failed to establish watch", err) } e1 := endpoints.Endpoint{Addr: "127.0.0.1", Metadata: "metadata"} err = em.AddEndpoint(t.Context(), "foo/a1", e1) if err != nil { t.Fatal("failed to add foo", err) } us := <-w if us == nil { t.Fatal("failed to get update") } wu := &endpoints.Update{ Op: endpoints.Add, Key: "foo/a1", Endpoint: e1, } require.Truef(t, reflect.DeepEqual(us[0], wu), "up = %#v, want %#v", us[0], wu) err = em.DeleteEndpoint(t.Context(), "foo/a1") require.NoErrorf(t, err, "failed to udpate %v", err) us = <-w if us == nil { t.Fatal("failed to get udpate") } wu = &endpoints.Update{ Op: endpoints.Delete, Key: "foo/a1", } require.Truef(t, reflect.DeepEqual(us[0], wu), "up = %#v, want %#v", us[0], wu) } // TestEndpointManagerAtomicity ensures the resolver will initialize // correctly with multiple hosts and correctly receive multiple // updates in a single revision. func TestEndpointManagerAtomicity(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) c := clus.RandClient() em, err := endpoints.NewManager(c, "foo") if err != nil { t.Fatal("failed to create EndpointManager", err) } err = em.Update(t.Context(), []*endpoints.UpdateWithOpts{ endpoints.NewAddUpdateOpts("foo/host", endpoints.Endpoint{Addr: "127.0.0.1:2000"}), endpoints.NewAddUpdateOpts("foo/host2", endpoints.Endpoint{Addr: "127.0.0.1:2001"}), }) require.NoError(t, err) ctx, watchCancel := context.WithCancel(t.Context()) defer watchCancel() w, err := em.NewWatchChannel(ctx) require.NoError(t, err) updates := <-w require.Lenf(t, updates, 2, "expected two updates, got %+v", updates) _, err = c.Txn(t.Context()).Then(etcd.OpDelete("foo/host"), etcd.OpDelete("foo/host2")).Commit() require.NoError(t, err) updates = <-w if len(updates) != 2 || (updates[0].Op != endpoints.Delete && updates[1].Op != endpoints.Delete) { t.Fatalf("expected two delete updates, got %+v", updates) } } func TestEndpointManagerCRUD(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) em, err := endpoints.NewManager(clus.RandClient(), "foo") if err != nil { t.Fatal("failed to create EndpointManager", err) } // Add k1 := "foo/a1" e1 := endpoints.Endpoint{Addr: "127.0.0.1", Metadata: "metadata1"} err = em.AddEndpoint(t.Context(), k1, e1) if err != nil { t.Fatal("failed to add", k1, err) } k2 := "foo/a2" e2 := endpoints.Endpoint{Addr: "127.0.0.2", Metadata: "metadata2"} err = em.AddEndpoint(t.Context(), k2, e2) if err != nil { t.Fatal("failed to add", k2, err) } eps, err := em.List(t.Context()) if err != nil { t.Fatal("failed to list foo") } require.Lenf(t, eps, 2, "unexpected the number of endpoints: %d", len(eps)) require.Truef(t, reflect.DeepEqual(eps[k1], e1), "unexpected endpoints: %s", k1) require.Truef(t, reflect.DeepEqual(eps[k2], e2), "unexpected endpoints: %s", k2) // Delete err = em.DeleteEndpoint(t.Context(), k1) if err != nil { t.Fatal("failed to delete", k2, err) } eps, err = em.List(t.Context()) if err != nil { t.Fatal("failed to list foo") } require.Lenf(t, eps, 1, "unexpected the number of endpoints: %d", len(eps)) require.Truef(t, reflect.DeepEqual(eps[k2], e2), "unexpected endpoints: %s", k2) // Update k3 := "foo/a3" e3 := endpoints.Endpoint{Addr: "127.0.0.3", Metadata: "metadata3"} updates := []*endpoints.UpdateWithOpts{ {Update: endpoints.Update{Op: endpoints.Add, Key: k3, Endpoint: e3}}, {Update: endpoints.Update{Op: endpoints.Delete, Key: k2}}, } err = em.Update(t.Context(), updates) if err != nil { t.Fatal("failed to update", err) } eps, err = em.List(t.Context()) if err != nil { t.Fatal("failed to list foo") } require.Lenf(t, eps, 1, "unexpected the number of endpoints: %d", len(eps)) require.Truef(t, reflect.DeepEqual(eps[k3], e3), "unexpected endpoints: %s", k3) } ================================================ FILE: tests/integration/clientv3/naming/main_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package naming_test import ( "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func TestMain(m *testing.M) { testutil.MustTestMainWithLeakDetection(m) } ================================================ FILE: tests/integration/clientv3/naming/resolver_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package naming_test import ( "bytes" "fmt" "strconv" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" testpb "google.golang.org/grpc/interop/grpc_testing" "go.etcd.io/etcd/client/v3/naming/endpoints" "go.etcd.io/etcd/client/v3/naming/resolver" "go.etcd.io/etcd/pkg/v3/grpctesting" "go.etcd.io/etcd/tests/v3/framework/integration" ) func testEtcdGRPCResolver(t *testing.T, lbPolicy string) { // Setup two new dummy stub servers payloadBody := []byte{'1'} s1 := grpctesting.NewDummyStubServer(payloadBody) if err := s1.Start(nil); err != nil { t.Fatal("failed to start dummy grpc server (s1)", err) } defer s1.Stop() s2 := grpctesting.NewDummyStubServer(payloadBody) if err := s2.Start(nil); err != nil { t.Fatal("failed to start dummy grpc server (s2)", err) } defer s2.Stop() // Create new cluster with endpoint manager with two endpoints clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) em, err := endpoints.NewManager(clus.Client(0), "foo") if err != nil { t.Fatal("failed to create EndpointManager", err) } e1 := endpoints.Endpoint{Addr: s1.Addr()} e2 := endpoints.Endpoint{Addr: s2.Addr()} err = em.AddEndpoint(t.Context(), "foo/e1", e1) if err != nil { t.Fatal("failed to add foo", err) } err = em.AddEndpoint(t.Context(), "foo/e2", e2) if err != nil { t.Fatal("failed to add foo", err) } b, err := resolver.NewBuilder(clus.Client(1)) if err != nil { t.Fatal("failed to new resolver builder", err) } // Create connection with provided lb policy conn, err := grpc.NewClient("etcd:///foo", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(b), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingPolicy":"%s"}`, lbPolicy))) if err != nil { t.Fatal("failed to connect to foo", err) } defer conn.Close() // Send an initial request that should go to e1 c := testpb.NewTestServiceClient(conn) resp, err := c.UnaryCall(t.Context(), &testpb.SimpleRequest{}, grpc.WaitForReady(true)) if err != nil { t.Fatal("failed to invoke rpc to foo (e1)", err) } if resp.GetPayload() == nil || !bytes.Equal(resp.GetPayload().GetBody(), payloadBody) { t.Fatalf("unexpected response from foo (e1): %s", resp.GetPayload().GetBody()) } // Send more requests lastResponse := []byte{'1'} totalRequests := 3500 for i := 1; i < totalRequests; i++ { resp, err := c.UnaryCall(t.Context(), &testpb.SimpleRequest{}, grpc.WaitForReady(true)) if err != nil { t.Fatal("failed to invoke rpc to foo", err) } t.Logf("Response: %v", string(resp.GetPayload().GetBody())) require.NotNilf(t, resp.GetPayload(), "unexpected response from foo: %s", resp.GetPayload().GetBody()) lastResponse = resp.GetPayload().GetBody() } // If the load balancing policy is pick first then return payload should equal number of requests t.Logf("Last response: %v", string(lastResponse)) if lbPolicy == "pick_first" { require.Equalf(t, "3500", string(lastResponse), "unexpected total responses from foo: %s", lastResponse) } // If the load balancing policy is round robin we should see roughly half total requests served by each server if lbPolicy == "round_robin" { responses, err := strconv.Atoi(string(lastResponse)) require.NoErrorf(t, err, "couldn't convert to int: %s", lastResponse) // Allow 25% tolerance as round robin is not perfect and we don't want the test to flake expected := float64(totalRequests) * 0.5 assert.InEpsilonf(t, expected, float64(responses), 0.25, "unexpected total responses from foo: %s", lastResponse) } } // TestEtcdGrpcResolverPickFirst mimics scenarios described in grpc_naming.md doc. func TestEtcdGrpcResolverPickFirst(t *testing.T) { integration.BeforeTest(t) // Pick first is the default load balancer policy for grpc-go testEtcdGRPCResolver(t, "pick_first") } // TestEtcdGrpcResolverRoundRobin mimics scenarios described in grpc_naming.md doc. func TestEtcdGrpcResolverRoundRobin(t *testing.T) { integration.BeforeTest(t) // Round robin is a common alternative for more production oriented scenarios testEtcdGRPCResolver(t, "round_robin") } func TestEtcdEndpointManager(t *testing.T) { integration.BeforeTest(t) s1PayloadBody := []byte{'1'} s1 := grpctesting.NewDummyStubServer(s1PayloadBody) err := s1.Start(nil) require.NoError(t, err) defer s1.Stop() s2PayloadBody := []byte{'2'} s2 := grpctesting.NewDummyStubServer(s2PayloadBody) err = s2.Start(nil) require.NoError(t, err) defer s2.Stop() clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) // Check if any endpoint with the same prefix "foo" will not break the logic with multiple endpoints em, err := endpoints.NewManager(clus.Client(0), "foo") require.NoError(t, err) emOther, err := endpoints.NewManager(clus.Client(1), "foo_other") require.NoError(t, err) e1 := endpoints.Endpoint{Addr: s1.Addr()} e2 := endpoints.Endpoint{Addr: s2.Addr()} em.AddEndpoint(t.Context(), "foo/e1", e1) emOther.AddEndpoint(t.Context(), "foo_other/e2", e2) epts, err := em.List(t.Context()) require.NoError(t, err) eptsOther, err := emOther.List(t.Context()) require.NoError(t, err) assert.Len(t, epts, 1) assert.Len(t, eptsOther, 1) } ================================================ FILE: tests/integration/clientv3/ordering_kv_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "errors" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/ordering" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestDetectKvOrderViolation(t *testing.T) { errOrderViolation := errors.New("DetectedOrderViolation") integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) cfg := clientv3.Config{ Endpoints: []string{ clus.Members[0].GRPCURL, clus.Members[1].GRPCURL, clus.Members[2].GRPCURL, }, } cli, err := integration.NewClient(t, cfg) require.NoError(t, err) defer func() { assert.NoError(t, cli.Close()) }() ctx := t.Context() _, err = clus.Client(0).Put(ctx, "foo", "bar") require.NoError(t, err) // ensure that the second member has the current revision for the key foo _, err = clus.Client(1).Get(ctx, "foo") require.NoError(t, err) // stop third member in order to force the member to have an outdated revision clus.Members[2].Stop(t) time.Sleep(1 * time.Second) // give enough time for operation _, err = cli.Put(ctx, "foo", "buzz") require.NoError(t, err) // perform get request against the first member, in order to // set up kvOrdering to expect "foo" revisions greater than that of // the third member. orderingKv := ordering.NewKV(cli.KV, func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error { return errOrderViolation }) v, err := orderingKv.Get(ctx, "foo") require.NoError(t, err) t.Logf("Read from the first member: v:%v err:%v", v, err) assert.Equal(t, []byte("buzz"), v.Kvs[0].Value) // ensure that only the third member is queried during requests clus.Members[0].Stop(t) clus.Members[1].Stop(t) require.NoError(t, clus.Members[2].Restart(t)) // force OrderingKv to query the third member cli.SetEndpoints(clus.Members[2].GRPCURL) time.Sleep(2 * time.Second) // FIXME: Figure out how pause SetEndpoints sufficiently that this is not needed t.Logf("Quering m2 after restart") v, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable()) t.Logf("Quering m2 returned: v:%v err:%v ", v, err) if !errors.Is(err, errOrderViolation) { t.Fatalf("expected %v, got err:%v v:%v", errOrderViolation, err, v) } } func TestDetectTxnOrderViolation(t *testing.T) { errOrderViolation := errors.New("DetectedOrderViolation") integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) cfg := clientv3.Config{ Endpoints: []string{ clus.Members[0].GRPCURL, clus.Members[1].GRPCURL, clus.Members[2].GRPCURL, }, } cli, err := integration.NewClient(t, cfg) require.NoError(t, err) defer func() { assert.NoError(t, cli.Close()) }() ctx := t.Context() _, err = clus.Client(0).Put(ctx, "foo", "bar") require.NoError(t, err) // ensure that the second member has the current revision for the key foo _, err = clus.Client(1).Get(ctx, "foo") require.NoError(t, err) // stop third member in order to force the member to have an outdated revision clus.Members[2].Stop(t) time.Sleep(1 * time.Second) // give enough time for operation _, err = clus.Client(1).Put(ctx, "foo", "buzz") require.NoError(t, err) // perform get request against the first member, in order to // set up kvOrdering to expect "foo" revisions greater than that of // the third member. orderingKv := ordering.NewKV(cli.KV, func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error { return errOrderViolation }) orderingTxn := orderingKv.Txn(ctx) _, err = orderingTxn.If( clientv3.Compare(clientv3.Value("b"), ">", "a"), ).Then( clientv3.OpGet("foo"), ).Commit() require.NoError(t, err) // ensure that only the third member is queried during requests clus.Members[0].Stop(t) clus.Members[1].Stop(t) require.NoError(t, clus.Members[2].Restart(t)) // force OrderingKv to query the third member cli.SetEndpoints(clus.Members[2].GRPCURL) time.Sleep(2 * time.Second) // FIXME: Figure out how pause SetEndpoints sufficiently that this is not needed _, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable()) if !errors.Is(err, errOrderViolation) { t.Fatalf("expected %v, got %v", errOrderViolation, err) } orderingTxn = orderingKv.Txn(ctx) _, err = orderingTxn.If( clientv3.Compare(clientv3.Value("b"), ">", "a"), ).Then( clientv3.OpGet("foo", clientv3.WithSerializable()), ).Commit() if !errors.Is(err, errOrderViolation) { t.Fatalf("expected %v, got %v", errOrderViolation, err) } } ================================================ FILE: tests/integration/clientv3/ordering_util_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "errors" "testing" "time" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/ordering" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestEndpointSwitchResolvesViolation ensures // - ErrNoGreaterRev error is returned from partitioned member when it has stale revision // - no more error after partition recovers func TestEndpointSwitchResolvesViolation(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) eps := []string{ clus.Members[0].GRPCURL, clus.Members[1].GRPCURL, clus.Members[2].GRPCURL, } cfg := clientv3.Config{Endpoints: []string{clus.Members[0].GRPCURL}} cli, err := integration.NewClient(t, cfg) require.NoError(t, err) defer cli.Close() ctx := t.Context() _, err = clus.Client(0).Put(ctx, "foo", "bar") require.NoError(t, err) // ensure that the second member has current revision for key "foo" _, err = clus.Client(1).Get(ctx, "foo") require.NoError(t, err) // create partition between third members and the first two members // in order to guarantee that the third member's revision of "foo" // falls behind as updates to "foo" are issued to the first two members. clus.Members[2].InjectPartition(t, clus.Members[:2]...) time.Sleep(1 * time.Second) // give enough time for the operation // update to "foo" will not be replicated to the third member due to the partition _, err = clus.Client(1).Put(ctx, "foo", "buzz") require.NoError(t, err) cli.SetEndpoints(eps...) time.Sleep(1 * time.Second) // give enough time for the operation orderingKv := ordering.NewKV(cli.KV, ordering.NewOrderViolationSwitchEndpointClosure(cli)) // set prevRev to the second member's revision of "foo" such that // the revision is higher than the third member's revision of "foo" _, err = orderingKv.Get(ctx, "foo") require.NoError(t, err) t.Logf("Reconfigure client to speak only to the 'partitioned' member") cli.SetEndpoints(clus.Members[2].GRPCURL) time.Sleep(1 * time.Second) // give enough time for the operation _, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable()) if !errors.Is(err, ordering.ErrNoGreaterRev) { t.Fatal("While speaking to partitioned leader, we should get ErrNoGreaterRev error") } clus.Members[2].RecoverPartition(t, clus.Members[:2]...) time.Sleep(1 * time.Second) // give enough time for the operation _, err = orderingKv.Get(ctx, "foo") if err != nil { t.Fatal("After partition recovered, third member should recover and return no error") } } // TestUnresolvableOrderViolation ensures ErrNoGreaterRev error is returned when available members only have stale revisions func TestUnresolvableOrderViolation(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 5, UseBridge: true}) defer clus.Terminate(t) cfg := clientv3.Config{ Endpoints: []string{ clus.Members[0].GRPCURL, clus.Members[1].GRPCURL, clus.Members[2].GRPCURL, clus.Members[3].GRPCURL, clus.Members[4].GRPCURL, }, } cli, err := integration.NewClient(t, cfg) require.NoError(t, err) defer cli.Close() eps := cli.Endpoints() ctx := t.Context() cli.SetEndpoints(clus.Members[0].GRPCURL) time.Sleep(1 * time.Second) _, err = cli.Put(ctx, "foo", "bar") require.NoError(t, err) // stop fourth member in order to force the member to have an outdated revision clus.Members[3].Stop(t) time.Sleep(1 * time.Second) // give enough time for operation // stop fifth member in order to force the member to have an outdated revision clus.Members[4].Stop(t) time.Sleep(1 * time.Second) // give enough time for operation _, err = cli.Put(ctx, "foo", "buzz") require.NoError(t, err) cli.SetEndpoints(eps...) time.Sleep(1 * time.Second) // give enough time for operation OrderingKv := ordering.NewKV(cli.KV, ordering.NewOrderViolationSwitchEndpointClosure(cli)) // set prevRev to the first member's revision of "foo" such that // the revision is higher than the fourth and fifth members' revision of "foo" _, err = OrderingKv.Get(ctx, "foo") require.NoError(t, err) clus.Members[0].Stop(t) clus.Members[1].Stop(t) clus.Members[2].Stop(t) require.NoError(t, clus.Members[3].Restart(t)) require.NoError(t, clus.Members[4].Restart(t)) clus.Members[3].WaitStarted(t) cli.SetEndpoints(clus.Members[3].GRPCURL) time.Sleep(1 * time.Second) // give enough time for operation _, err = OrderingKv.Get(ctx, "foo", clientv3.WithSerializable()) require.ErrorIsf(t, err, ordering.ErrNoGreaterRev, "expected %v, got %v", ordering.ErrNoGreaterRev, err) } ================================================ FILE: tests/integration/clientv3/snapshot/v3_snapshot_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package snapshot_test import ( "context" "fmt" "math/rand" "net/url" "os" "path/filepath" "testing" "time" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/client/pkg/v3/testutil" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/snapshot" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestSaveSnapshotFilePermissions ensures that the snapshot is saved with // the correct file permissions. func TestSaveSnapshotFilePermissions(t *testing.T) { expectedFileMode := os.FileMode(fileutil.PrivateFileMode) kvs := []kv{{"foo1", "bar1"}, {"foo2", "bar2"}, {"foo3", "bar3"}} _, dbPath := createSnapshotFile(t, newEmbedConfig(t), kvs) defer os.RemoveAll(dbPath) dbInfo, err := os.Stat(dbPath) require.NoErrorf(t, err, "failed to get test snapshot file status: %v", err) actualFileMode := dbInfo.Mode() require.Equalf(t, expectedFileMode, actualFileMode, "expected test snapshot file mode %s, got %s:", expectedFileMode, actualFileMode) } // TestSaveSnapshotVersion ensures that the snapshot returns proper storage version. func TestSaveSnapshotVersion(t *testing.T) { // Put some keys to ensure that wal snapshot is triggered var kvs []kv for i := 0; i < 10; i++ { kvs = append(kvs, kv{fmt.Sprintf("%d", i), "test"}) } cfg := newEmbedConfig(t) // Force raft snapshot to ensure that storage version is set cfg.SnapshotCount = 1 ver, dbPath := createSnapshotFile(t, cfg, kvs) defer os.RemoveAll(dbPath) require.Equalf(t, "3.7.0", ver, "expected snapshot version %s, got %s:", "3.7.0", ver) } type kv struct { k, v string } func newEmbedConfig(t *testing.T) *embed.Config { clusterN := 1 urls := newEmbedURLs(clusterN * 2) cURLs, pURLs := urls[:clusterN], urls[clusterN:] cfg := integration.NewEmbedConfig(t, "default") cfg.ClusterState = "new" cfg.ListenClientUrls, cfg.AdvertiseClientUrls = cURLs, cURLs cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = pURLs, pURLs cfg.InitialCluster = fmt.Sprintf("%s=%s", cfg.Name, pURLs[0].String()) return cfg } // creates a snapshot file and returns the file path. func createSnapshotFile(t *testing.T, cfg *embed.Config, kvs []kv) (version string, dbPath string) { testutil.SkipTestIfShortMode(t, "Snapshot creation tests are depending on embedded etcd server so are integration-level tests.") srv, err := embed.StartEtcd(cfg) require.NoError(t, err) defer func() { srv.Close() }() select { case <-srv.Server.ReadyNotify(): case <-time.After(3 * time.Second): t.Fatalf("failed to start embed.Etcd for creating snapshots") } ccfg := clientv3.Config{Endpoints: []string{cfg.AdvertiseClientUrls[0].String()}} cli, err := integration.NewClient(t, ccfg) require.NoError(t, err) defer cli.Close() for i := range kvs { ctx, cancel := context.WithTimeout(t.Context(), testutil.RequestTimeout) _, err = cli.Put(ctx, kvs[i].k, kvs[i].v) cancel() require.NoError(t, err) } dbPath = filepath.Join(t.TempDir(), fmt.Sprintf("snapshot%d.db", time.Now().Nanosecond())) version, err = snapshot.SaveWithVersion(t.Context(), zaptest.NewLogger(t), ccfg, dbPath) require.NoError(t, err) return version, dbPath } func newEmbedURLs(n int) (urls []url.URL) { urls = make([]url.URL, n) for i := 0; i < n; i++ { u, _ := url.Parse(fmt.Sprintf("unix://localhost:%d", rand.Intn(45000))) urls[i] = *u } return urls } ================================================ FILE: tests/integration/clientv3/txn_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "context" "errors" "fmt" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestTxnError(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kv := clus.RandClient() ctx := t.Context() _, err := kv.Txn(ctx).Then(clientv3.OpPut("foo", "bar1"), clientv3.OpPut("foo", "bar2")).Commit() if !errors.Is(err, rpctypes.ErrDuplicateKey) { t.Fatalf("expected %v, got %v", rpctypes.ErrDuplicateKey, err) } ops := make([]clientv3.Op, int(embed.DefaultMaxTxnOps+10)) for i := range ops { ops[i] = clientv3.OpPut(fmt.Sprintf("foo%d", i), "") } _, err = kv.Txn(ctx).Then(ops...).Commit() if !errors.Is(err, rpctypes.ErrTooManyOps) { t.Fatalf("expected %v, got %v", rpctypes.ErrTooManyOps, err) } } func TestTxnWriteFail(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) kv := clus.Client(0) clus.Members[0].Stop(t) txnc, getc := make(chan struct{}), make(chan struct{}) go func() { ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() resp, err := kv.Txn(ctx).Then(clientv3.OpPut("foo", "bar")).Commit() if err == nil { t.Errorf("expected error, got response %v", resp) } close(txnc) }() go func() { defer close(getc) select { case <-time.After(5 * time.Second): t.Errorf("timed out waiting for txn fail") case <-txnc: } // and ensure the put didn't take gresp, gerr := clus.Client(1).Get(t.Context(), "foo") if gerr != nil { t.Error(gerr) } if len(gresp.Kvs) != 0 { t.Errorf("expected no keys, got %v", gresp.Kvs) } }() select { case <-time.After(5 * clus.Members[1].ServerConfig.ReqTimeout()): t.Fatalf("timed out waiting for get") case <-getc: } // reconnect so terminate doesn't complain about double-close clus.Members[0].Restart(t) } func TestTxnReadRetry(t *testing.T) { t.Skipf("skipping txn read retry test: re-enable after we do retry on txn read request") integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) kv := clus.Client(0) thenOps := [][]clientv3.Op{ {clientv3.OpGet("foo")}, {clientv3.OpTxn(nil, []clientv3.Op{clientv3.OpGet("foo")}, nil)}, {clientv3.OpTxn(nil, nil, nil)}, {}, } for i := range thenOps { clus.Members[0].Stop(t) <-clus.Members[0].StopNotify() donec := make(chan struct{}, 1) go func() { _, err := kv.Txn(t.Context()).Then(thenOps[i]...).Commit() if err != nil { t.Errorf("expected response, got error %v", err) } donec <- struct{}{} }() // wait for txn to fail on disconnect time.Sleep(100 * time.Millisecond) // restart node; client should resume clus.Members[0].Restart(t) select { case <-donec: case <-time.After(2 * clus.Members[1].ServerConfig.ReqTimeout()): t.Fatalf("waited too long") } } } func TestTxnSuccess(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kv := clus.Client(0) ctx := t.Context() _, err := kv.Txn(ctx).Then(clientv3.OpPut("foo", "bar")).Commit() require.NoError(t, err) resp, err := kv.Get(ctx, "foo") require.NoError(t, err) if len(resp.Kvs) != 1 || string(resp.Kvs[0].Key) != "foo" { t.Fatalf("unexpected Get response %v", resp) } } func TestTxnCompareRange(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kv := clus.Client(0) fooResp, err := kv.Put(t.Context(), "foo/", "bar") require.NoError(t, err) _, err = kv.Put(t.Context(), "foo/a", "baz") require.NoError(t, err) tresp, terr := kv.Txn(t.Context()).If( clientv3.Compare( clientv3.CreateRevision("foo/"), "=", fooResp.Header.Revision). WithPrefix(), ).Commit() require.NoError(t, terr) if tresp.Succeeded { t.Fatal("expected prefix compare to false, got compares as true") } } func TestTxnNested(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kv := clus.Client(0) tresp, err := kv.Txn(t.Context()). If(clientv3.Compare(clientv3.Version("foo"), "=", 0)). Then( clientv3.OpPut("foo", "bar"), clientv3.OpTxn(nil, []clientv3.Op{clientv3.OpPut("abc", "123")}, nil)). Else(clientv3.OpPut("foo", "baz")).Commit() require.NoError(t, err) if len(tresp.Responses) != 2 { t.Errorf("expected 2 top-level txn responses, got %+v", tresp.Responses) } // check txn writes were applied resp, err := kv.Get(t.Context(), "foo") require.NoError(t, err) if len(resp.Kvs) != 1 || string(resp.Kvs[0].Value) != "bar" { t.Errorf("unexpected Get response %+v", resp) } resp, err = kv.Get(t.Context(), "abc") require.NoError(t, err) if len(resp.Kvs) != 1 || string(resp.Kvs[0].Value) != "123" { t.Errorf("unexpected Get response %+v", resp) } } ================================================ FILE: tests/integration/clientv3/user_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "context" "errors" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestUserError(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) authapi := clus.RandClient() _, err := authapi.UserAdd(t.Context(), "foo", "bar") require.NoError(t, err) _, err = authapi.UserAdd(t.Context(), "foo", "bar") require.ErrorIsf(t, err, rpctypes.ErrUserAlreadyExist, "expected %v, got %v", rpctypes.ErrUserAlreadyExist, err) _, err = authapi.UserDelete(t.Context(), "not-exist-user") require.ErrorIsf(t, err, rpctypes.ErrUserNotFound, "expected %v, got %v", rpctypes.ErrUserNotFound, err) _, err = authapi.UserGrantRole(t.Context(), "foo", "test-role-does-not-exist") require.ErrorIsf(t, err, rpctypes.ErrRoleNotFound, "expected %v, got %v", rpctypes.ErrRoleNotFound, err) } func TestAddUserAfterDelete(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) authapi := clus.RandClient() authSetupRoot(t, authapi.Auth) cfg := clientv3.Config{ Endpoints: authapi.Endpoints(), DialTimeout: 5 * time.Second, } cfg.Username, cfg.Password = "root", "123" authed, err := integration.NewClient(t, cfg) require.NoError(t, err) defer authed.Close() // add user _, err = authed.UserAdd(t.Context(), "foo", "bar") require.NoError(t, err) _, err = authapi.Authenticate(t.Context(), "foo", "bar") require.NoError(t, err) // delete user _, err = authed.UserDelete(t.Context(), "foo") require.NoError(t, err) if _, err = authed.Authenticate(t.Context(), "foo", "bar"); err == nil { t.Errorf("expect Authenticate error for old password") } // add user back _, err = authed.UserAdd(t.Context(), "foo", "bar") require.NoError(t, err) _, err = authed.Authenticate(t.Context(), "foo", "bar") require.NoError(t, err) // change password _, err = authed.UserChangePassword(t.Context(), "foo", "bar2") require.NoError(t, err) _, err = authed.UserChangePassword(t.Context(), "foo", "bar1") require.NoError(t, err) if _, err = authed.Authenticate(t.Context(), "foo", "bar"); err == nil { t.Errorf("expect Authenticate error for old password") } if _, err = authed.Authenticate(t.Context(), "foo", "bar2"); err == nil { t.Errorf("expect Authenticate error for old password") } _, err = authed.Authenticate(t.Context(), "foo", "bar1") require.NoError(t, err) } func TestUserErrorAuth(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) authapi := clus.RandClient() authSetupRoot(t, authapi.Auth) // unauthenticated client _, err := authapi.UserAdd(t.Context(), "foo", "bar") require.ErrorIsf(t, err, rpctypes.ErrUserEmpty, "expected %v, got %v", rpctypes.ErrUserEmpty, err) // wrong id or password cfg := clientv3.Config{ Endpoints: authapi.Endpoints(), DialTimeout: 5 * time.Second, } cfg.Username, cfg.Password = "wrong-id", "123" _, err = integration.NewClient(t, cfg) require.ErrorIsf(t, err, rpctypes.ErrAuthFailed, "expected %v, got %v", rpctypes.ErrAuthFailed, err) cfg.Username, cfg.Password = "root", "wrong-pass" _, err = integration.NewClient(t, cfg) require.ErrorIsf(t, err, rpctypes.ErrAuthFailed, "expected %v, got %v", rpctypes.ErrAuthFailed, err) cfg.Username, cfg.Password = "root", "123" authed, err := integration.NewClient(t, cfg) require.NoError(t, err) defer authed.Close() _, err = authed.UserList(t.Context()) require.NoError(t, err) } func authSetupRoot(t *testing.T, auth clientv3.Auth) { _, err := auth.UserAdd(t.Context(), "root", "123") require.NoError(t, err) _, err = auth.RoleAdd(t.Context(), "root") require.NoError(t, err) _, err = auth.UserGrantRole(t.Context(), "root", "root") require.NoError(t, err) _, err = auth.AuthEnable(t.Context()) require.NoError(t, err) } // TestGetTokenWithoutAuth is when Client can connect to etcd even if they // supply credentials and the server is in AuthDisable mode. func TestGetTokenWithoutAuth(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 2}) defer clus.Terminate(t) authapi := clus.RandClient() var err error var client *clientv3.Client // make sure "auth" was disabled _, err = authapi.AuthDisable(t.Context()) require.NoError(t, err) // "Username" and "Password" must be used cfg := clientv3.Config{ Endpoints: authapi.Endpoints(), DialTimeout: 5 * time.Second, Username: "root", Password: "123", } client, err = integration.NewClient(t, cfg) if err == nil { defer client.Close() } switch { case err == nil: t.Log("passes as expected") case errors.Is(err, context.DeadlineExceeded): t.Errorf("not expected result:%v with endpoint:%s", err, authapi.Endpoints()) default: t.Errorf("other errors:%v", err) } } ================================================ FILE: tests/integration/clientv3/util.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package clientv3test import ( "context" "errors" "fmt" "strings" "testing" "time" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" ) // MustWaitPinReady waits up to 3-second until connection is up (pin endpoint). // Fatal on time-out. func MustWaitPinReady(t *testing.T, cli *clientv3.Client) { // TODO: decrease timeout after balancer rewrite!!! ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) _, err := cli.Get(ctx, "foo") cancel() if err != nil { t.Fatal(err) } } // IsServerCtxTimeout checks reason of the error. // e.g. due to clock drifts in server-side, // client context times out first in server-side // while original client-side context is not timed out yet func IsServerCtxTimeout(err error) bool { if err == nil { return false } ev, ok := status.FromError(err) if !ok { return false } code := ev.Code() return (code == codes.DeadlineExceeded /*3.5+"*/ || code == codes.Unknown /*<=3.4*/) && strings.Contains(err.Error(), "context deadline exceeded") } // IsClientTimeout checks reason of the error. // In grpc v1.11.3+ dial timeouts can error out with transport.ErrConnClosing. Previously dial timeouts // would always error out with context.DeadlineExceeded. func IsClientTimeout(err error) bool { if err == nil { return false } if errors.Is(err, context.DeadlineExceeded) { return true } ev, ok := status.FromError(err) if !ok { return false } code := ev.Code() return code == codes.DeadlineExceeded } func IsCanceled(err error) bool { if err == nil { return false } if errors.Is(err, context.Canceled) { return true } ev, ok := status.FromError(err) if !ok { return false } code := ev.Code() return code == codes.Canceled } func IsUnavailable(err error) bool { if err == nil { return false } if errors.Is(err, context.Canceled) { return true } ev, ok := status.FromError(err) if !ok { return false } code := ev.Code() return code == codes.Unavailable } // populateDataIntoCluster populates the key-value pairs into cluster and the // key will be named by testing.T.Name()-index. func populateDataIntoCluster(t *testing.T, cluster *integration.Cluster, numKeys int, valueSize int) { ctx := t.Context() for i := 0; i < numKeys; i++ { _, err := cluster.RandClient().Put(ctx, fmt.Sprintf("%s-%v", t.Name(), i), strings.Repeat("a", valueSize)) if err != nil { t.Errorf("populating data expected no error, but got %v", err) } } } ================================================ FILE: tests/integration/clientv3/watch/v3_watch_restore_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package watch import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/integration" ) // MustFetchNotEmptyMetric attempts to fetch given 'metric' from 'member', // waiting for not-empty value or 'timeout'. func MustFetchNotEmptyMetric(tb testing.TB, member *integration.Member, metric string, timeout <-chan time.Time) string { metricValue := "" tick := time.Tick(config.TickDuration) for metricValue == "" { tb.Logf("Waiting for metric: %v", metric) select { case <-timeout: tb.Fatalf("Failed to fetch metric %v", metric) return "" case <-tick: var err error metricValue, err = member.Metric(metric) if err != nil { tb.Fatal(err) } } } return metricValue } // TestV3WatchRestoreSnapshotUnsync tests whether slow follower can restore // from leader snapshot, and still notify on watchers from an old revision // that were created in synced watcher group in the first place. // TODO: fix panic with gRPC proxy "panic: watcher current revision should not exceed current revision" func TestV3WatchRestoreSnapshotUnsync(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 3, SnapshotCount: 10, SnapshotCatchUpEntries: 5, }) defer clus.Terminate(t) // spawn a watcher before shutdown, and put it in synced watcher ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() wStream, errW := integration.ToGRPC(clus.Client(0)).Watch.Watch(ctx) require.NoError(t, errW) if err := wStream.Send(&pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{Key: []byte("foo"), StartRevision: 5}, }}); err != nil { t.Fatalf("wStream.Send error: %v", err) } wresp, errR := wStream.Recv() if errR != nil { t.Errorf("wStream.Recv error: %v", errR) } if !wresp.Created { t.Errorf("wresp.Created got = %v, want = true", wresp.Created) } clus.Members[0].InjectPartition(t, clus.Members[1:]...) initialLead := clus.WaitMembersForLeader(t, clus.Members[1:]) + 1 t.Logf("elected lead: %v", clus.Members[initialLead].Server.MemberID()) t.Logf("sleeping for 2 seconds") time.Sleep(2 * time.Second) t.Logf("sleeping for 2 seconds DONE") kvc := integration.ToGRPC(clus.Client(1)).KV // to trigger snapshot from the leader to the stopped follower for i := 0; i < 15; i++ { _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}) if err != nil { t.Errorf("#%d: couldn't put key (%v)", i, err) } } // NOTE: When starting a new cluster with 3 members, each member will // apply 3 ConfChange directly at the beginning before a leader is // elected. Leader will apply 3 MemberAttrSet and 1 ClusterVersionSet // changes. So member 0 has index 8 in raft log before network // partition. We need to trigger EtcdServer.snapshot() at least twice. // // SnapshotCount: 10, SnapshotCatchUpEntries: 5 // // T1: L(snapshot-index: 11, compacted-index: 6), F_m0(index:8) // T2: L(snapshot-index: 22, compacted-index: 17), F_m0(index:8, out of date) // // Since there is no way to confirm server has compacted the log, we // use log monitor to watch and expect "compacted Raft logs" content. // In v3.6 we no longer generates "compacted Raft logs" log as raft compaction happens independently to snapshot. // For now let's use snapshot log which should be equivalent to compaction. expectMemberLog(t, clus.Members[initialLead], 5*time.Second, "saved snapshot to disk", 2) // After RecoverPartition, leader L will send snapshot to slow F_m0 // follower, because F_m0(index:8) is 'out of date' compared to // L(compacted-index:17). clus.Members[0].RecoverPartition(t, clus.Members[1:]...) // We don't expect leadership change here, just recompute the leader'Server index // within clus.Members list. lead := clus.WaitLeader(t) // Sending is scheduled on fifo 'sched' within EtcdServer::run, // so it can start delayed after recovery. send := MustFetchNotEmptyMetric(t, clus.Members[lead], "etcd_network_snapshot_send_inflights_total", time.After(5*time.Second)) if send != "0" && send != "1" { // 0 if already sent, 1 if sending t.Fatalf("inflight snapshot snapshot_send_inflights_total expected 0 or 1, got %q", send) } receives := MustFetchNotEmptyMetric(t, clus.Members[(lead+1)%3], "etcd_network_snapshot_receive_inflights_total", time.After(5*time.Second)) if receives != "0" && receives != "1" { // 0 if already received, 1 if receiving t.Fatalf("inflight snapshot receives expected 0 or 1, got %q", receives) } expectMemberLog(t, clus.Members[0], 5*time.Second, "received and saved database snapshot", 1) t.Logf("sleeping for 2 seconds") time.Sleep(2 * time.Second) t.Logf("sleeping for 2 seconds DONE") // slow follower now applies leader snapshot // should be able to notify on old-revision watchers in unsynced // make sure restore watch operation correctly moves watchers // between synced and unsynced watchers errc := make(chan error, 1) go func() { cresp, cerr := wStream.Recv() if cerr != nil { errc <- cerr return } // from start revision 5 to latest revision 16 if len(cresp.Events) != 12 { errc <- fmt.Errorf("expected 12 events, got %+v", cresp.Events) return } errc <- nil }() select { case <-time.After(10 * time.Second): t.Fatal("took too long to receive events from restored watcher") case err := <-errc: if err != nil { t.Fatalf("wStream.Recv error: %v", err) } } } func expectMemberLog(t *testing.T, m *integration.Member, timeout time.Duration, s string, count int) { ctx, cancel := context.WithTimeout(t.Context(), timeout) defer cancel() lines, err := m.LogObserver.Expect(ctx, s, count) if err != nil { t.Fatalf("failed to expect (log:%s, count:%v): %v", s, count, err) } for _, line := range lines { t.Logf("[expected line]: %v", line) } } ================================================ FILE: tests/integration/clientv3/watch/v3_watch_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package watch import ( "bytes" "context" "errors" "fmt" "reflect" "sort" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/tests/v3/framework/integration" gofail "go.etcd.io/gofail/runtime" ) // TestV3WatchFromCurrentRevision tests Watch APIs from current revision. func TestV3WatchFromCurrentRevision(t *testing.T) { integration.BeforeTest(t) tests := []struct { name string putKeys []string watchRequest *pb.WatchRequest wresps []*pb.WatchResponse }{ { "watch the key, matching", []string{"foo"}, &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), }, }}, []*pb.WatchResponse{ { Header: &pb.ResponseHeader{Revision: 2}, Created: false, Events: []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, }, }, }, }, }, { "watch the key, non-matching", []string{"foo"}, &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("helloworld"), }, }}, []*pb.WatchResponse{}, }, { "watch the prefix, matching", []string{"fooLong"}, &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), RangeEnd: []byte("fop"), }, }}, []*pb.WatchResponse{ { Header: &pb.ResponseHeader{Revision: 2}, Created: false, Events: []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("fooLong"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, }, }, }, }, }, { "watch the prefix, non-matching", []string{"foo"}, &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("helloworld"), RangeEnd: []byte("helloworle"), }, }}, []*pb.WatchResponse{}, }, { "watch full range, matching", []string{"fooLong"}, &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte(""), RangeEnd: []byte("\x00"), }, }}, []*pb.WatchResponse{ { Header: &pb.ResponseHeader{Revision: 2}, Created: false, Events: []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("fooLong"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, }, }, }, }, }, { "multiple puts, one watcher with matching key", []string{"foo", "foo", "foo"}, &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), }, }}, []*pb.WatchResponse{ { Header: &pb.ResponseHeader{Revision: 2}, Created: false, Events: []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, }, }, }, { Header: &pb.ResponseHeader{Revision: 3}, Created: false, Events: []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 3, Version: 2}, }, }, }, { Header: &pb.ResponseHeader{Revision: 4}, Created: false, Events: []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 4, Version: 3}, }, }, }, }, }, { "multiple puts, one watcher with matching prefix", []string{"foo", "foo", "foo"}, &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), RangeEnd: []byte("fop"), }, }}, []*pb.WatchResponse{ { Header: &pb.ResponseHeader{Revision: 2}, Created: false, Events: []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, }, }, }, { Header: &pb.ResponseHeader{Revision: 3}, Created: false, Events: []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 3, Version: 2}, }, }, }, { Header: &pb.ResponseHeader{Revision: 4}, Created: false, Events: []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 4, Version: 3}, }, }, }, }, }, } for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) wAPI := integration.ToGRPC(clus.RandClient()).Watch ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() wStream, err := wAPI.Watch(ctx) if err != nil { t.Fatalf("#%d: wAPI.Watch error: %v", i, err) } err = wStream.Send(tt.watchRequest) if err != nil { t.Fatalf("#%d: wStream.Send error: %v", i, err) } // ensure watcher request created a new watcher cresp, err := wStream.Recv() if err != nil { t.Fatalf("#%d: wStream.Recv error: %v", i, err) } if !cresp.Created { t.Fatalf("#%d: did not create watchid, got %+v", i, cresp) } if cresp.Canceled { t.Fatalf("#%d: canceled watcher on create %+v", i, cresp) } createdWatchID := cresp.WatchId if cresp.Header == nil || cresp.Header.Revision != 1 { t.Fatalf("#%d: header revision got +%v, wanted revison 1", i, cresp) } // asynchronously create keys ch := make(chan struct{}, 1) go func() { for _, k := range tt.putKeys { kvc := integration.ToGRPC(clus.RandClient()).KV req := &pb.PutRequest{Key: []byte(k), Value: []byte("bar")} if _, err := kvc.Put(t.Context(), req); err != nil { t.Errorf("#%d: couldn't put key (%v)", i, err) } } ch <- struct{}{} }() // check stream results for j, wresp := range tt.wresps { resp, err := wStream.Recv() if err != nil { t.Errorf("#%d.%d: wStream.Recv error: %v", i, j, err) } if resp.Header == nil { t.Fatalf("#%d.%d: unexpected nil resp.Header", i, j) } if resp.Header.Revision != wresp.Header.Revision { t.Errorf("#%d.%d: resp.Header.Revision got = %d, want = %d", i, j, resp.Header.Revision, wresp.Header.Revision) } if wresp.Created != resp.Created { t.Errorf("#%d.%d: resp.Created got = %v, want = %v", i, j, resp.Created, wresp.Created) } if resp.WatchId != createdWatchID { t.Errorf("#%d.%d: resp.WatchId got = %d, want = %d", i, j, resp.WatchId, createdWatchID) } if !reflect.DeepEqual(resp.Events, wresp.Events) { t.Errorf("#%d.%d: resp.Events got = %+v, want = %+v", i, j, resp.Events, wresp.Events) } } rok, nr := waitResponse(wStream, 1*time.Second) if !rok { t.Errorf("unexpected pb.WatchResponse is received %+v", nr) } // wait for the client to finish sending the keys before terminating the cluster <-ch }) } } // TestV3WatchFutureRevision tests Watch APIs from a future revision. func TestV3WatchFutureRevision(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) wAPI := integration.ToGRPC(clus.RandClient()).Watch ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() wStream, err := wAPI.Watch(ctx) if err != nil { t.Fatalf("wAPI.Watch error: %v", err) } wkey := []byte("foo") wrev := int64(10) req := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{Key: wkey, StartRevision: wrev}, }} err = wStream.Send(req) if err != nil { t.Fatalf("wStream.Send error: %v", err) } // ensure watcher request created a new watcher cresp, err := wStream.Recv() if err != nil { t.Fatalf("wStream.Recv error: %v", err) } if !cresp.Created { t.Fatalf("create %v, want %v", cresp.Created, true) } kvc := integration.ToGRPC(clus.RandClient()).KV for { req := &pb.PutRequest{Key: wkey, Value: []byte("bar")} resp, rerr := kvc.Put(t.Context(), req) if rerr != nil { t.Fatalf("couldn't put key (%v)", rerr) } if resp.Header.Revision == wrev { break } } // ensure watcher request created a new watcher cresp, err = wStream.Recv() if err != nil { t.Fatalf("wStream.Recv error: %v", err) } if cresp.Header.Revision != wrev { t.Fatalf("revision = %d, want %d", cresp.Header.Revision, wrev) } if len(cresp.Events) != 1 { t.Fatalf("failed to receive events") } if cresp.Events[0].Kv.ModRevision != wrev { t.Errorf("mod revision = %d, want %d", cresp.Events[0].Kv.ModRevision, wrev) } } // TestV3WatchWrongRange tests wrong range does not create watchers. func TestV3WatchWrongRange(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) wAPI := integration.ToGRPC(clus.RandClient()).Watch ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() wStream, err := wAPI.Watch(ctx) if err != nil { t.Fatalf("wAPI.Watch error: %v", err) } tests := []struct { key []byte end []byte canceled bool }{ {[]byte("a"), []byte("a"), true}, // wrong range end {[]byte("b"), []byte("a"), true}, // wrong range end {[]byte("foo"), []byte{0}, false}, // watch request with 'WithFromKey' } for i, tt := range tests { if err := wStream.Send(&pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{Key: tt.key, RangeEnd: tt.end, StartRevision: 1}, }}); err != nil { t.Fatalf("#%d: wStream.Send error: %v", i, err) } cresp, err := wStream.Recv() if err != nil { t.Fatalf("#%d: wStream.Recv error: %v", i, err) } if !cresp.Created { t.Fatalf("#%d: create %v, want %v", i, cresp.Created, true) } if cresp.Canceled != tt.canceled { t.Fatalf("#%d: canceled %v, want %v", i, tt.canceled, cresp.Canceled) } if tt.canceled && cresp.WatchId != clientv3.InvalidWatchID { t.Fatalf("#%d: canceled watch ID %d, want %d", i, cresp.WatchId, clientv3.InvalidWatchID) } } } // TestV3WatchCancelSynced tests Watch APIs cancellation from synced map. func TestV3WatchCancelSynced(t *testing.T) { integration.BeforeTest(t) testV3WatchCancel(t, 0) } // TestV3WatchCancelUnsynced tests Watch APIs cancellation from unsynced map. func TestV3WatchCancelUnsynced(t *testing.T) { integration.BeforeTest(t) testV3WatchCancel(t, 1) } func testV3WatchCancel(t *testing.T, startRev int64) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() wStream, errW := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) if errW != nil { t.Fatalf("wAPI.Watch error: %v", errW) } wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), StartRevision: startRev, }, }} if err := wStream.Send(wreq); err != nil { t.Fatalf("wStream.Send error: %v", err) } wresp, errR := wStream.Recv() if errR != nil { t.Errorf("wStream.Recv error: %v", errR) } if !wresp.Created { t.Errorf("wresp.Created got = %v, want = true", wresp.Created) } creq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CancelRequest{ CancelRequest: &pb.WatchCancelRequest{ WatchId: wresp.WatchId, }, }} if err := wStream.Send(creq); err != nil { t.Fatalf("wStream.Send error: %v", err) } cresp, err := wStream.Recv() if err != nil { t.Errorf("wStream.Recv error: %v", err) } if !cresp.Canceled { t.Errorf("cresp.Canceled got = %v, want = true", cresp.Canceled) } kvc := integration.ToGRPC(clus.RandClient()).KV if _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}); err != nil { t.Errorf("couldn't put key (%v)", err) } // watch got canceled, so this should block rok, nr := waitResponse(wStream, 1*time.Second) if !rok { t.Errorf("unexpected pb.WatchResponse is received %+v", nr) } } // TestV3WatchCurrentPutOverlap ensures current watchers receive all events with // overlapping puts. func TestV3WatchCurrentPutOverlap(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() wStream, wErr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) if wErr != nil { t.Fatalf("wAPI.Watch error: %v", wErr) } // last mod_revision that will be observed nrRevisions := 32 // first revision already allocated as empty revision var wg sync.WaitGroup for i := 1; i < nrRevisions; i++ { wg.Add(1) go func() { defer wg.Done() kvc := integration.ToGRPC(clus.RandClient()).KV req := &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")} if _, err := kvc.Put(t.Context(), req); err != nil { t.Errorf("couldn't put key (%v)", err) } }() } // maps watcher to current expected revision progress := make(map[int64]int64) wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{Key: []byte("foo"), RangeEnd: []byte("fop")}, }} if err := wStream.Send(wreq); err != nil { t.Fatalf("first watch request failed (%v)", err) } more := true progress[-1] = 0 // watcher creation pending for more { resp, err := wStream.Recv() if err != nil { t.Fatalf("wStream.Recv error: %v", err) } if resp.Created { // accept events > header revision progress[resp.WatchId] = resp.Header.Revision + 1 if resp.Header.Revision == int64(nrRevisions) { // covered all revisions; create no more watchers progress[-1] = int64(nrRevisions) + 1 } else if err := wStream.Send(wreq); err != nil { t.Fatalf("watch request failed (%v)", err) } } else if len(resp.Events) == 0 { t.Fatalf("got events %v, want non-empty", resp.Events) } else { wRev, ok := progress[resp.WatchId] if !ok { t.Fatalf("got %+v, but watch id shouldn't exist ", resp) } if resp.Events[0].Kv.ModRevision != wRev { t.Fatalf("got %+v, wanted first revision %d", resp, wRev) } lastRev := resp.Events[len(resp.Events)-1].Kv.ModRevision progress[resp.WatchId] = lastRev + 1 } more = false for _, v := range progress { if v <= int64(nrRevisions) { more = true break } } } if rok, nr := waitResponse(wStream, time.Second); !rok { t.Errorf("unexpected pb.WatchResponse is received %+v", nr) } wg.Wait() } // TestV3WatchEmptyKey ensures synced watchers see empty key PUTs as PUT events func TestV3WatchEmptyKey(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() ws, werr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) require.NoError(t, werr) req := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), }, }} require.NoError(t, ws.Send(req)) _, err := ws.Recv() require.NoError(t, err) // put a key with empty value kvc := integration.ToGRPC(clus.RandClient()).KV preq := &pb.PutRequest{Key: []byte("foo")} _, err = kvc.Put(t.Context(), preq) require.NoError(t, err) // check received PUT resp, rerr := ws.Recv() require.NoError(t, rerr) wevs := []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo"), CreateRevision: 2, ModRevision: 2, Version: 1}, }, } if !reflect.DeepEqual(resp.Events, wevs) { t.Fatalf("got %v, expected %v", resp.Events, wevs) } } func TestV3WatchMultipleWatchersSynced(t *testing.T) { integration.BeforeTest(t) testV3WatchMultipleWatchers(t, 0) } func TestV3WatchMultipleWatchersUnsynced(t *testing.T) { integration.BeforeTest(t) testV3WatchMultipleWatchers(t, 1) } // testV3WatchMultipleWatchers tests multiple watchers on the same key // and one watcher with matching prefix. It first puts the key // that matches all watchers, and another key that matches only // one watcher to test if it receives expected events. func testV3WatchMultipleWatchers(t *testing.T, startRev int64) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() wStream, errW := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) if errW != nil { t.Fatalf("wAPI.Watch error: %v", errW) } watchKeyN := 4 for i := 0; i < watchKeyN+1; i++ { var wreq *pb.WatchRequest if i < watchKeyN { wreq = &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), StartRevision: startRev, }, }} } else { wreq = &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("fo"), RangeEnd: []byte("fp"), StartRevision: startRev, }, }} } if err := wStream.Send(wreq); err != nil { t.Fatalf("wStream.Send error: %v", err) } } ids := make(map[int64]struct{}) for i := 0; i < watchKeyN+1; i++ { wresp, err := wStream.Recv() if err != nil { t.Fatalf("wStream.Recv error: %v", err) } if !wresp.Created { t.Fatalf("wresp.Created got = %v, want = true", wresp.Created) } ids[wresp.WatchId] = struct{}{} } if _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}); err != nil { t.Fatalf("couldn't put key (%v)", err) } for i := 0; i < watchKeyN+1; i++ { wresp, err := wStream.Recv() if err != nil { t.Fatalf("wStream.Recv error: %v", err) } if _, ok := ids[wresp.WatchId]; !ok { t.Errorf("watchId %d is not created!", wresp.WatchId) } else { delete(ids, wresp.WatchId) } if len(wresp.Events) == 0 { t.Errorf("#%d: no events received", i) } for _, ev := range wresp.Events { if string(ev.Kv.Key) != "foo" { t.Errorf("ev.Kv.Key got = %s, want = foo", ev.Kv.Key) } if string(ev.Kv.Value) != "bar" { t.Errorf("ev.Kv.Value got = %s, want = bar", ev.Kv.Value) } } } // now put one key that has only one matching watcher if _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("fo"), Value: []byte("bar")}); err != nil { t.Fatalf("couldn't put key (%v)", err) } wresp, err := wStream.Recv() if err != nil { t.Errorf("wStream.Recv error: %v", err) } if len(wresp.Events) != 1 { t.Fatalf("len(wresp.Events) got = %d, want = 1", len(wresp.Events)) } if string(wresp.Events[0].Kv.Key) != "fo" { t.Errorf("wresp.Events[0].Kv.Key got = %s, want = fo", wresp.Events[0].Kv.Key) } // now Recv should block because there is no more events coming rok, nr := waitResponse(wStream, 1*time.Second) if !rok { t.Errorf("unexpected pb.WatchResponse is received %+v", nr) } } func TestV3WatchMultipleEventsTxnSynced(t *testing.T) { integration.BeforeTest(t) testV3WatchMultipleEventsTxn(t, 0) } func TestV3WatchMultipleEventsTxnUnsynced(t *testing.T) { integration.BeforeTest(t) testV3WatchMultipleEventsTxn(t, 1) } // testV3WatchMultipleEventsTxn tests Watch APIs when it receives multiple events. func testV3WatchMultipleEventsTxn(t *testing.T, startRev int64) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() wStream, wErr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) if wErr != nil { t.Fatalf("wAPI.Watch error: %v", wErr) } wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), RangeEnd: []byte("fop"), StartRevision: startRev, }, }} if err := wStream.Send(wreq); err != nil { t.Fatalf("wStream.Send error: %v", err) } if resp, err := wStream.Recv(); err != nil || !resp.Created { t.Fatalf("create response failed: resp=%v, err=%v", resp, err) } kvc := integration.ToGRPC(clus.RandClient()).KV txn := pb.TxnRequest{} for i := 0; i < 3; i++ { ru := &pb.RequestOp{} ru.Request = &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Key: []byte(fmt.Sprintf("foo%d", i)), Value: []byte("bar"), }, } txn.Success = append(txn.Success, ru) } tresp, err := kvc.Txn(t.Context(), &txn) if err != nil { t.Fatalf("kvc.Txn error: %v", err) } if !tresp.Succeeded { t.Fatalf("kvc.Txn failed: %+v", tresp) } var events []*mvccpb.Event for len(events) < 3 { resp, err := wStream.Recv() if err != nil { t.Errorf("wStream.Recv error: %v", err) } events = append(events, resp.Events...) } sort.Sort(eventsSortByKey(events)) wevents := []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo0"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, }, { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo1"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, }, { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo2"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, }, } if !reflect.DeepEqual(events, wevents) { t.Errorf("events got = %+v, want = %+v", events, wevents) } rok, nr := waitResponse(wStream, 1*time.Second) if !rok { t.Errorf("unexpected pb.WatchResponse is received %+v", nr) } } type eventsSortByKey []*mvccpb.Event func (evs eventsSortByKey) Len() int { return len(evs) } func (evs eventsSortByKey) Swap(i, j int) { evs[i], evs[j] = evs[j], evs[i] } func (evs eventsSortByKey) Less(i, j int) bool { return bytes.Compare(evs[i].Kv.Key, evs[j].Kv.Key) < 0 } func TestV3WatchMultipleEventsPutUnsynced(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV if _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo0"), Value: []byte("bar")}); err != nil { t.Fatalf("couldn't put key (%v)", err) } if _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo1"), Value: []byte("bar")}); err != nil { t.Fatalf("couldn't put key (%v)", err) } ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() wStream, wErr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) if wErr != nil { t.Fatalf("wAPI.Watch error: %v", wErr) } wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), RangeEnd: []byte("fop"), StartRevision: 1, }, }} if err := wStream.Send(wreq); err != nil { t.Fatalf("wStream.Send error: %v", err) } if _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo0"), Value: []byte("bar")}); err != nil { t.Fatalf("couldn't put key (%v)", err) } if _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo1"), Value: []byte("bar")}); err != nil { t.Fatalf("couldn't put key (%v)", err) } allWevents := []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo0"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, }, { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo1"), Value: []byte("bar"), CreateRevision: 3, ModRevision: 3, Version: 1}, }, { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo0"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 4, Version: 2}, }, { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo1"), Value: []byte("bar"), CreateRevision: 3, ModRevision: 5, Version: 2}, }, } var events []*mvccpb.Event for len(events) < 4 { resp, err := wStream.Recv() if err != nil { t.Errorf("wStream.Recv error: %v", err) } if resp.Created { continue } events = append(events, resp.Events...) // if PUT requests are committed by now, first receive would return // multiple events, but if not, it returns a single event. In SSD, // it should return 4 events at once. } if !reflect.DeepEqual(events, allWevents) { t.Errorf("events got = %+v, want = %+v", events, allWevents) } rok, nr := waitResponse(wStream, 1*time.Second) if !rok { t.Errorf("unexpected pb.WatchResponse is received %+v", nr) } } // TestV3WatchProgressOnMemberRestart verifies the client side doesn't // receive duplicated events. // Refer to https://github.com/etcd-io/etcd/pull/15248#issuecomment-1423225742. func TestV3WatchProgressOnMemberRestart(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 1, WatchProgressNotifyInterval: time.Second, }) defer clus.Terminate(t) client := clus.RandClient() ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() errC := make(chan error, 1) watchReady := make(chan struct{}, 1) doneC := make(chan struct{}, 1) progressNotifyC := make(chan struct{}, 1) go func() { defer close(doneC) var ( lastWatchedModRevision int64 gotProgressNotification bool ) wch := client.Watch(ctx, "foo", clientv3.WithProgressNotify()) watchReady <- struct{}{} for wr := range wch { if wr.Err() != nil { errC <- fmt.Errorf("watch error: %w", wr.Err()) return } if len(wr.Events) == 0 { // We need to make sure at least one progress notification // is received after receiving the normal watch response // and before restarting the member. if lastWatchedModRevision > 0 { gotProgressNotification = true progressNotifyC <- struct{}{} } continue } for _, event := range wr.Events { if event.Kv.ModRevision <= lastWatchedModRevision { errC <- fmt.Errorf("got an unexpected revision: %d, lastWatchedModRevision: %d", event.Kv.ModRevision, lastWatchedModRevision) return } lastWatchedModRevision = event.Kv.ModRevision } if gotProgressNotification { return } } }() // waiting for the watcher ready t.Log("Waiting for the watcher to be ready.") <-watchReady time.Sleep(time.Second) // write a K/V firstly t.Log("Writing key 'foo' firstly") _, err := client.Put(ctx, "foo", "bar1") require.NoError(t, err) // make sure at least one progress notification is received // before restarting the member t.Log("Waiting for the progress notification") select { case <-progressNotifyC: case <-time.After(5 * time.Second): t.Log("Do not receive the progress notification in 5 seconds, move forward anyway.") } // restart the member t.Log("Restarting the member") clus.Members[0].Stop(t) clus.Members[0].Restart(t) clus.Members[0].WaitOK(t) // write the same key again after the member restarted t.Log("Writing the same key 'foo' again after restarting the member") _, err = client.Put(ctx, "foo", "bar2") require.NoError(t, err) t.Log("Waiting for result") select { case <-progressNotifyC: t.Log("Progress notification received") case err := <-errC: t.Fatal(err) case <-doneC: t.Log("Done") case <-time.After(15 * time.Second): t.Fatal("Timed out waiting for the response") } } func TestV3WatchMultipleStreamsSynced(t *testing.T) { integration.BeforeTest(t) testV3WatchMultipleStreams(t, 0) } func TestV3WatchMultipleStreamsUnsynced(t *testing.T) { integration.BeforeTest(t) testV3WatchMultipleStreams(t, 1) } // testV3WatchMultipleStreams tests multiple watchers on the same key on multiple streams. func testV3WatchMultipleStreams(t *testing.T, startRev int64) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) wAPI := integration.ToGRPC(clus.RandClient()).Watch kvc := integration.ToGRPC(clus.RandClient()).KV streams := make([]pb.Watch_WatchClient, 5) for i := range streams { ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() wStream, errW := wAPI.Watch(ctx) if errW != nil { t.Fatalf("wAPI.Watch error: %v", errW) } wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), StartRevision: startRev, }, }} if err := wStream.Send(wreq); err != nil { t.Fatalf("wStream.Send error: %v", err) } streams[i] = wStream } for _, wStream := range streams { wresp, err := wStream.Recv() if err != nil { t.Fatalf("wStream.Recv error: %v", err) } if !wresp.Created { t.Fatalf("wresp.Created got = %v, want = true", wresp.Created) } } if _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}); err != nil { t.Fatalf("couldn't put key (%v)", err) } var wg sync.WaitGroup wg.Add(len(streams)) wevents := []*mvccpb.Event{ { Type: mvccpb.Event_PUT, Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, }, } for i := range streams { go func(i int) { defer wg.Done() wStream := streams[i] wresp, err := wStream.Recv() if err != nil { t.Errorf("wStream.Recv error: %v", err) } if wresp.WatchId != 0 { t.Errorf("watchId got = %d, want = 0", wresp.WatchId) } if !reflect.DeepEqual(wresp.Events, wevents) { t.Errorf("wresp.Events got = %+v, want = %+v", wresp.Events, wevents) } // now Recv should block because there is no more events coming rok, nr := waitResponse(wStream, 1*time.Second) if !rok { t.Errorf("unexpected pb.WatchResponse is received %+v", nr) } }(i) } wg.Wait() } // waitResponse waits on the given stream for given duration. // If there is no more events, true and a nil response will be // returned closing the WatchClient stream. Or the response will // be returned. func waitResponse(wc pb.Watch_WatchClient, timeout time.Duration) (bool, *pb.WatchResponse) { rCh := make(chan *pb.WatchResponse, 1) donec := make(chan struct{}) defer close(donec) go func() { resp, _ := wc.Recv() select { case rCh <- resp: case <-donec: } }() select { case nr := <-rCh: return false, nr case <-time.After(timeout): } // didn't get response wc.CloseSend() return true, nil } func TestWatchWithProgressNotify(t *testing.T) { // accelerate report interval so test terminates quickly oldpi := v3rpc.GetProgressReportInterval() // using atomics to avoid race warnings v3rpc.SetProgressReportInterval(3 * time.Second) testInterval := 3 * time.Second defer func() { v3rpc.SetProgressReportInterval(oldpi) }() integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() wStream, wErr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) if wErr != nil { t.Fatalf("wAPI.Watch error: %v", wErr) } // create two watchers, one with progressNotify set. wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{Key: []byte("foo"), StartRevision: 1, ProgressNotify: true}, }} if err := wStream.Send(wreq); err != nil { t.Fatalf("watch request failed (%v)", err) } wreq = &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{Key: []byte("foo"), StartRevision: 1}, }} if err := wStream.Send(wreq); err != nil { t.Fatalf("watch request failed (%v)", err) } // two creation + one notification for i := 0; i < 3; i++ { rok, resp := waitResponse(wStream, testInterval+time.Second) if resp.Created { continue } if rok { t.Errorf("failed to receive response from watch stream") } if resp.Header.Revision != 1 { t.Errorf("revision = %d, want 1", resp.Header.Revision) } if len(resp.Events) != 0 { t.Errorf("len(resp.Events) = %d, want 0", len(resp.Events)) } } // no more notification rok, resp := waitResponse(wStream, time.Second) if !rok { t.Errorf("unexpected pb.WatchResponse is received %+v", resp) } } // TestV3WatchClose opens many watchers concurrently on multiple streams. func TestV3WatchClose(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) c := clus.Client(0) wapi := integration.ToGRPC(c).Watch var wg sync.WaitGroup wg.Add(100) for i := 0; i < 100; i++ { go func() { ctx, cancel := context.WithCancel(t.Context()) defer func() { wg.Done() cancel() }() ws, err := wapi.Watch(ctx) if err != nil { return } cr := &pb.WatchCreateRequest{Key: []byte("a")} req := &pb.WatchRequest{ RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: cr, }, } ws.Send(req) ws.Recv() }() } clus.Members[0].Bridge().DropConnections() wg.Wait() } // TestV3WatchWithFilter ensures watcher filters out the events correctly. func TestV3WatchWithFilter(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() ws, werr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) require.NoError(t, werr) req := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), Filters: []pb.WatchCreateRequest_FilterType{pb.WatchCreateRequest_NOPUT}, }, }} require.NoError(t, ws.Send(req)) _, err := ws.Recv() require.NoError(t, err) recv := make(chan *pb.WatchResponse, 1) go func() { // check received PUT resp, rerr := ws.Recv() if rerr != nil { t.Error(rerr) } recv <- resp }() // put a key with empty value kvc := integration.ToGRPC(clus.RandClient()).KV preq := &pb.PutRequest{Key: []byte("foo")} _, err = kvc.Put(t.Context(), preq) require.NoError(t, err) select { case <-recv: t.Fatal("failed to filter out put event") case <-time.After(100 * time.Millisecond): } dreq := &pb.DeleteRangeRequest{Key: []byte("foo")} _, err = kvc.DeleteRange(t.Context(), dreq) require.NoError(t, err) select { case resp := <-recv: wevs := []*mvccpb.Event{ { Type: mvccpb.Event_DELETE, Kv: &mvccpb.KeyValue{Key: []byte("foo"), ModRevision: 3}, }, } if !reflect.DeepEqual(resp.Events, wevs) { t.Fatalf("got %v, expected %v", resp.Events, wevs) } case <-time.After(100 * time.Millisecond): t.Fatal("failed to receive delete event") } } func TestV3WatchWithPrevKV(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) wctx, wcancel := context.WithCancel(t.Context()) defer wcancel() tests := []struct { key string end string vals []string }{{ key: "foo", end: "fop", vals: []string{"bar1", "bar2"}, }, { key: "/abc", end: "/abd", vals: []string{"first", "second"}, }} for i, tt := range tests { kvc := integration.ToGRPC(clus.RandClient()).KV _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte(tt.key), Value: []byte(tt.vals[0])}) require.NoError(t, err) ws, werr := integration.ToGRPC(clus.RandClient()).Watch.Watch(wctx) require.NoError(t, werr) req := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte(tt.key), RangeEnd: []byte(tt.end), PrevKv: true, }, }} err = ws.Send(req) require.NoError(t, err) _, err = ws.Recv() require.NoError(t, err) _, err = kvc.Put(t.Context(), &pb.PutRequest{Key: []byte(tt.key), Value: []byte(tt.vals[1])}) require.NoError(t, err) recv := make(chan *pb.WatchResponse, 1) go func() { // check received PUT resp, rerr := ws.Recv() if rerr != nil { t.Error(rerr) } recv <- resp }() select { case resp := <-recv: if tt.vals[1] != string(resp.Events[0].Kv.Value) { t.Errorf("#%d: unequal value: want=%s, get=%s", i, tt.vals[1], resp.Events[0].Kv.Value) } if tt.vals[0] != string(resp.Events[0].PrevKv.Value) { t.Errorf("#%d: unequal value: want=%s, get=%s", i, tt.vals[0], resp.Events[0].PrevKv.Value) } case <-time.After(30 * time.Second): t.Error("timeout waiting for watch response") } } } // TestV3WatchCancellation ensures that watch cancellation frees up server resources. func TestV3WatchCancellation(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() cli := clus.RandClient() // increment watcher total count and keep a stream open cli.Watch(ctx, "/foo") for i := 0; i < 1000; i++ { wctx, wcancel := context.WithCancel(ctx) cli.Watch(wctx, "/foo") wcancel() } // Wait a little for cancellations to take hold time.Sleep(3 * time.Second) minWatches, err := clus.Members[0].Metric("etcd_debugging_mvcc_watcher_total") require.NoError(t, err) var expected string if integration.ThroughProxy { // grpc proxy has additional 2 watches open expected = "3" } else { expected = "1" } if minWatches != expected { t.Fatalf("expected %s watch, got %s", expected, minWatches) } } // TestV3WatchCloseCancelRace ensures that watch close doesn't decrement the watcher total too far. func TestV3WatchCloseCancelRace(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() cli := clus.RandClient() for i := 0; i < 1000; i++ { wctx, wcancel := context.WithCancel(ctx) cli.Watch(wctx, "/foo") wcancel() } // Wait a little for cancellations to take hold time.Sleep(3 * time.Second) minWatches, err := clus.Members[0].Metric("etcd_debugging_mvcc_watcher_total") require.NoError(t, err) var expected string if integration.ThroughProxy { // grpc proxy has additional 2 watches open expected = "2" } else { expected = "0" } if minWatches != expected { t.Fatalf("expected %s watch, got %s", expected, minWatches) } } // TestV3WatchProgressWaitsForSync checks that progress notifications // don't get sent until the watcher is synchronised func TestV3WatchProgressWaitsForSync(t *testing.T) { // Disable for gRPC proxy, as it does not support requesting // progress notifications if integration.ThroughProxy { t.Skip("grpc proxy currently does not support requesting progress notifications") } integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) client := clus.RandClient() ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() // Write a couple values into key to make sure there's a // non-trivial amount of history. count := 1001 t.Logf("Writing key 'foo' %d times", count) for i := 0; i < count; i++ { _, err := client.Put(ctx, "foo", fmt.Sprintf("bar%d", i)) require.NoError(t, err) } // Create watch channel starting at revision 1 (i.e. it starts // unsynced because of the update above) wch := client.Watch(ctx, "foo", clientv3.WithRev(1)) // Immediately request a progress notification. As the client // is unsynchronised, the server will not sent any notification, // as client can infer progress from events. err := client.RequestProgress(ctx) require.NoError(t, err) // Verify that we get the watch responses first. Note that // events might be spread across multiple packets. eventCount := 0 for eventCount < count { wr := <-wch if wr.Err() != nil { t.Fatal(fmt.Errorf("watch error: %w", wr.Err())) } if wr.IsProgressNotify() { t.Fatal("Progress notification from unsynced client!") } if wr.Header.Revision != int64(count+1) { t.Fatal("Incomplete watch response!") } eventCount += len(wr.Events) } // client needs to request progress notification again err = client.RequestProgress(ctx) require.NoError(t, err) wr2 := <-wch if wr2.Err() != nil { t.Fatal(fmt.Errorf("watch error: %w", wr2.Err())) } if !wr2.IsProgressNotify() { t.Fatal("Did not receive progress notification!") } if wr2.Header.Revision != int64(count+1) { t.Fatal("Wrong revision in progress notification!") } } func TestV3WatchProgressWaitsForSyncNoEvents(t *testing.T) { if integration.ThroughProxy { t.Skip("grpc proxy currently does not support requesting progress notifications") } integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) client := clus.RandClient() ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() resp, err := client.Put(ctx, "bar", "1") require.NoError(t, err) wch := client.Watch(ctx, "foo", clientv3.WithRev(resp.Header.Revision)) // Request the progress notification on newly created watch that was not yet synced. err = client.RequestProgress(ctx) ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() require.NoError(t, err) gotProgressNotification := false for { select { case <-ticker.C: err := client.RequestProgress(ctx) require.NoError(t, err) case resp := <-wch: if resp.Err() != nil { t.Fatal(fmt.Errorf("watch error: %w", resp.Err())) } if resp.IsProgressNotify() { gotProgressNotification = true } } if gotProgressNotification { break } } require.Truef(t, gotProgressNotification, "Expected to get progress notification") } // TestV3NoEventsLostOnCompact verifies that slow watchers exit with compacted watch response // if its next revision of events are compacted and no lost events sent to client. func TestV3NoEventsLostOnCompact(t *testing.T) { if integration.ThroughProxy { t.Skip("grpc proxy currently does not support requesting progress notifications") } integration.BeforeTest(t) integration.SkipIfNoGoFail(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) client := clus.RandClient() ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() // sendLoop throughput is rate-limited to 1 event per second require.NoError(t, gofail.Enable("beforeSendWatchResponse", `sleep("1s")`)) wch := client.Watch(ctx, "foo") var rev int64 writeCount := mvcc.ChanBufLen() * 11 / 10 for i := 0; i < writeCount; i++ { resp, err := client.Put(ctx, "foo", "bar") require.NoError(t, err) rev = resp.Header.Revision } _, err := client.Compact(ctx, rev) require.NoError(t, err) time.Sleep(time.Second) require.NoError(t, gofail.Disable("beforeSendWatchResponse")) eventCount := 0 compacted := false for resp := range wch { err = resp.Err() if err != nil { if !errors.Is(err, rpctypes.ErrCompacted) { t.Fatalf("want watch response err %v but got %v", rpctypes.ErrCompacted, err) } compacted = true break } eventCount += len(resp.Events) if eventCount == writeCount { break } } assert.Truef(t, compacted, "Expected stream to get compacted, instead we got %d events out of %d events", eventCount, writeCount) } ================================================ FILE: tests/integration/clientv3/watch/watch_fragment_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy package watch import ( "fmt" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/testutil" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestWatchFragmentDisable ensures that large watch // response exceeding server-side request limit can // arrive even without watch response fragmentation. func TestWatchFragmentDisable(t *testing.T) { testWatchFragment(t, false, false) } // TestWatchFragmentDisableWithGRPCLimit verifies // large watch response exceeding server-side request // limit and client-side gRPC response receive limit // cannot arrive without watch events fragmentation, // because multiple events exceed client-side gRPC // response receive limit. func TestWatchFragmentDisableWithGRPCLimit(t *testing.T) { testWatchFragment(t, false, true) } // TestWatchFragmentEnable ensures that large watch // response exceeding server-side request limit arrive // with watch response fragmentation. func TestWatchFragmentEnable(t *testing.T) { testWatchFragment(t, true, false) } // TestWatchFragmentEnableWithGRPCLimit verifies // large watch response exceeding server-side request // limit and client-side gRPC response receive limit // can arrive only when watch events are fragmented. func TestWatchFragmentEnableWithGRPCLimit(t *testing.T) { testWatchFragment(t, true, true) } // testWatchFragment triggers watch response that spans over multiple // revisions exceeding server request limits when combined. func testWatchFragment(t *testing.T, fragment, exceedRecvLimit bool) { integration.BeforeTest(t) cfg := &integration.ClusterConfig{ Size: 1, MaxRequestBytes: 1.5 * 1024 * 1024, } if exceedRecvLimit { cfg.ClientMaxCallRecvMsgSize = 1.5 * 1024 * 1024 } clus := integration.NewCluster(t, cfg) defer clus.Terminate(t) cli := clus.Client(0) errc := make(chan error) for i := 0; i < 10; i++ { go func(i int) { _, err := cli.Put(t.Context(), fmt.Sprint("foo", i), strings.Repeat("a", 1024*1024), ) errc <- err }(i) } for i := 0; i < 10; i++ { err := <-errc require.NoErrorf(t, err, "failed to put") } opts := []clientv3.OpOption{clientv3.WithPrefix(), clientv3.WithRev(1)} if fragment { opts = append(opts, clientv3.WithFragment()) } wch := cli.Watch(t.Context(), "foo", opts...) // expect 10 MiB watch response select { case ws := <-wch: // without fragment, should exceed gRPC client receive limit if !fragment && exceedRecvLimit { require.Emptyf(t, ws.Events, "expected 0 events with watch fragmentation") exp := "code = ResourceExhausted desc = grpc: received message larger than max (" require.Containsf(t, ws.Err().Error(), exp, "expected 'ResourceExhausted' error") return } // still expect merged watch events require.Lenf(t, ws.Events, 10, "expected 10 events with watch fragmentation") require.NoErrorf(t, ws.Err(), "unexpected error") case <-time.After(testutil.RequestTimeout): t.Fatalf("took too long to receive events") } } ================================================ FILE: tests/integration/clientv3/watch/watch_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package watch import ( "context" "errors" "fmt" "math/rand" "reflect" "sort" "strconv" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" mvccpb "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/api/v3/version" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc" "go.etcd.io/etcd/tests/v3/framework/integration" ) type watcherTest func(*testing.T, *watchctx) type watchctx struct { clus *integration.Cluster w clientv3.Watcher kv clientv3.KV wclientMember int kvMember int ch clientv3.WatchChan } func runWatchTest(t *testing.T, f watcherTest) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) wclientMember := rand.Intn(3) w := clus.Client(wclientMember).Watcher // select a different client for KV operations so puts succeed if // a test knocks out the watcher client. kvMember := rand.Intn(3) for kvMember == wclientMember { kvMember = rand.Intn(3) } kv := clus.Client(kvMember).KV wctx := &watchctx{clus, w, kv, wclientMember, kvMember, nil} f(t, wctx) } // TestWatchMultiWatcher modifies multiple keys and observes the changes. func TestWatchMultiWatcher(t *testing.T) { runWatchTest(t, testWatchMultiWatcher) } func testWatchMultiWatcher(t *testing.T, wctx *watchctx) { numKeyUpdates := 4 keys := []string{"foo", "bar", "baz"} donec := make(chan struct{}) // wait for watcher shutdown defer func() { for i := 0; i < len(keys)+1; i++ { <-donec } }() readyc := make(chan struct{}) for _, k := range keys { // key watcher go func(key string) { ch := wctx.w.Watch(t.Context(), key) if ch == nil { t.Errorf("expected watcher channel, got nil") } readyc <- struct{}{} for i := 0; i < numKeyUpdates; i++ { resp, ok := <-ch if !ok { t.Errorf("watcher unexpectedly closed") } v := fmt.Sprintf("%s-%d", key, i) gotv := string(resp.Events[0].Kv.Value) if gotv != v { t.Errorf("#%d: got %s, wanted %s", i, gotv, v) } } donec <- struct{}{} }(k) } // prefix watcher on "b" (bar and baz) go func() { prefixc := wctx.w.Watch(t.Context(), "b", clientv3.WithPrefix()) if prefixc == nil { t.Errorf("expected watcher channel, got nil") } readyc <- struct{}{} var evs []*clientv3.Event for i := 0; i < numKeyUpdates*2; i++ { resp, ok := <-prefixc if !ok { t.Errorf("watcher unexpectedly closed") } evs = append(evs, resp.Events...) } // check response var expected []string bkeys := []string{"bar", "baz"} for _, k := range bkeys { for i := 0; i < numKeyUpdates; i++ { expected = append(expected, fmt.Sprintf("%s-%d", k, i)) } } var got []string for _, ev := range evs { got = append(got, string(ev.Kv.Value)) } sort.Strings(got) if !reflect.DeepEqual(expected, got) { t.Errorf("got %v, expected %v", got, expected) } // ensure no extra data select { case resp, ok := <-prefixc: if !ok { t.Errorf("watcher unexpectedly closed") } t.Errorf("unexpected event %+v", resp) case <-time.After(time.Second): } donec <- struct{}{} }() // wait for watcher bring up for i := 0; i < len(keys)+1; i++ { <-readyc } // generate events ctx := t.Context() for i := 0; i < numKeyUpdates; i++ { for _, k := range keys { v := fmt.Sprintf("%s-%d", k, i) _, err := wctx.kv.Put(ctx, k, v) require.NoError(t, err) } } } // TestWatchRange tests watcher creates ranges func TestWatchRange(t *testing.T) { runWatchTest(t, testWatchRange) } func testWatchRange(t *testing.T, wctx *watchctx) { wctx.ch = wctx.w.Watch(t.Context(), "a", clientv3.WithRange("c")) require.NotNilf(t, wctx.ch, "expected non-nil channel") putAndWatch(t, wctx, "a", "a") putAndWatch(t, wctx, "b", "b") putAndWatch(t, wctx, "bar", "bar") } // TestWatchReconnRequest tests the send failure path when requesting a watcher. func TestWatchReconnRequest(t *testing.T) { runWatchTest(t, testWatchReconnRequest) } func testWatchReconnRequest(t *testing.T, wctx *watchctx) { donec, stopc := make(chan struct{}), make(chan struct{}, 1) go func() { timer := time.After(2 * time.Second) defer close(donec) // take down watcher connection for { wctx.clus.Members[wctx.wclientMember].Bridge().DropConnections() select { case <-timer: // spinning on close may live lock reconnection return case <-stopc: return default: } } }() // should reconnect when requesting watch wctx.ch = wctx.w.Watch(t.Context(), "a") require.NotNilf(t, wctx.ch, "expected non-nil channel") // wait for disconnections to stop stopc <- struct{}{} <-donec // spinning on dropping connections may trigger a leader election // due to resource starvation; l-read to ensure the cluster is stable ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) _, err := wctx.kv.Get(ctx, "_") require.NoError(t, err) cancel() // ensure watcher works putAndWatch(t, wctx, "a", "a") } // TestWatchReconnInit tests watcher resumes correctly if connection lost // before any data was sent. func TestWatchReconnInit(t *testing.T) { runWatchTest(t, testWatchReconnInit) } func testWatchReconnInit(t *testing.T, wctx *watchctx) { wctx.ch = wctx.w.Watch(t.Context(), "a") require.NotNilf(t, wctx.ch, "expected non-nil channel") wctx.clus.Members[wctx.wclientMember].Bridge().DropConnections() // watcher should recover putAndWatch(t, wctx, "a", "a") } // TestWatchReconnRunning tests watcher resumes correctly if connection lost // after data was sent. func TestWatchReconnRunning(t *testing.T) { runWatchTest(t, testWatchReconnRunning) } func testWatchReconnRunning(t *testing.T, wctx *watchctx) { wctx.ch = wctx.w.Watch(t.Context(), "a") require.NotNilf(t, wctx.ch, "expected non-nil channel") putAndWatch(t, wctx, "a", "a") // take down watcher connection wctx.clus.Members[wctx.wclientMember].Bridge().DropConnections() // watcher should recover putAndWatch(t, wctx, "a", "b") } // TestWatchCancelImmediate ensures a closed channel is returned // if the context is cancelled. func TestWatchCancelImmediate(t *testing.T) { runWatchTest(t, testWatchCancelImmediate) } func testWatchCancelImmediate(t *testing.T, wctx *watchctx) { ctx, cancel := context.WithCancel(t.Context()) cancel() wch := wctx.w.Watch(ctx, "a") select { case wresp, ok := <-wch: require.Falsef(t, ok, "read wch got %v; expected closed channel", wresp) default: t.Fatalf("closed watcher channel should not block") } } // TestWatchCancelInit tests watcher closes correctly after no events. func TestWatchCancelInit(t *testing.T) { runWatchTest(t, testWatchCancelInit) } func testWatchCancelInit(t *testing.T, wctx *watchctx) { ctx, cancel := context.WithCancel(t.Context()) wctx.ch = wctx.w.Watch(ctx, "a") require.NotNilf(t, wctx.ch, "expected non-nil watcher channel") cancel() select { case <-time.After(time.Second): t.Fatalf("took too long to cancel") case _, ok := <-wctx.ch: require.Falsef(t, ok, "expected watcher channel to close") } } // TestWatchCancelRunning tests watcher closes correctly after events. func TestWatchCancelRunning(t *testing.T) { runWatchTest(t, testWatchCancelRunning) } func testWatchCancelRunning(t *testing.T, wctx *watchctx) { ctx, cancel := context.WithCancel(t.Context()) wctx.ch = wctx.w.Watch(ctx, "a") require.NotNilf(t, wctx.ch, "expected non-nil watcher channel") _, err := wctx.kv.Put(ctx, "a", "a") require.NoError(t, err) cancel() select { case <-time.After(time.Second): t.Fatalf("took too long to cancel") case _, ok := <-wctx.ch: if !ok { // closed before getting put; OK break } // got the PUT; should close next select { case <-time.After(time.Second): t.Fatalf("took too long to close") case v, ok2 := <-wctx.ch: require.Falsef(t, ok2, "expected watcher channel to close, got %v", v) } } } func putAndWatch(t *testing.T, wctx *watchctx, key, val string) { _, err := wctx.kv.Put(t.Context(), key, val) require.NoError(t, err) select { case <-time.After(5 * time.Second): t.Fatalf("watch timed out") case v, ok := <-wctx.ch: require.Truef(t, ok, "unexpected watch close") err := v.Err() require.NoErrorf(t, err, "unexpected watch response error") require.Equalf(t, string(v.Events[0].Kv.Value), val, "bad value got %v, wanted %v", v.Events[0].Kv.Value, val) } } // TestWatchResumeAfterDisconnect tests watch resume after member disconnects then connects. // It ensures that correct events are returned corresponding to the start revision. func TestWatchResumeAfterDisconnect(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) cli := clus.Client(0) _, err := cli.Put(t.Context(), "b", "2") require.NoError(t, err) _, err = cli.Put(t.Context(), "a", "3") require.NoError(t, err) // if resume is broken, it'll pick up this key first instead of a=3 _, err = cli.Put(t.Context(), "a", "4") require.NoError(t, err) // watch from revision 1 wch := clus.Client(0).Watch(t.Context(), "a", clientv3.WithRev(1), clientv3.WithCreatedNotify()) // response for the create watch request, no events are in this response // the current revision of etcd should be 4 if resp, ok := <-wch; !ok || resp.Header.Revision != 4 { t.Fatalf("got (%v, %v), expected create notification rev=4", resp, ok) } // pause wch clus.Members[0].Bridge().DropConnections() clus.Members[0].Bridge().PauseConnections() select { case resp, ok := <-wch: t.Skipf("wch should block, got (%+v, %v); drop not fast enough", resp, ok) case <-time.After(100 * time.Millisecond): } // resume wch clus.Members[0].Bridge().UnpauseConnections() select { case resp, ok := <-wch: if !ok { t.Fatal("unexpected watch close") } // Events should be put(a, 3) and put(a, 4) if len(resp.Events) != 2 { t.Fatal("expected two events on watch") } require.Equalf(t, "3", string(resp.Events[0].Kv.Value), "expected value=3, got event %+v", resp.Events[0]) require.Equalf(t, "4", string(resp.Events[1].Kv.Value), "expected value=4, got event %+v", resp.Events[1]) case <-time.After(5 * time.Second): t.Fatal("watch timed out") } } // TestWatchResumeCompacted checks that the watcher gracefully closes in case // that it tries to resume to a revision that's been compacted out of the store. // Since the watcher's server restarts with stale data, the watcher will receive // either a compaction error or all keys by staying in sync before the compaction // is finally applied. func TestWatchResumeCompacted(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) // create a waiting watcher at rev 1 w := clus.Client(0) wch := w.Watch(t.Context(), "foo", clientv3.WithRev(1)) select { case w := <-wch: t.Errorf("unexpected message from wch %v", w) default: } clus.Members[0].Stop(t) clus.WaitLeader(t) // put some data and compact away numPuts := 5 kv := clus.Client(1) for i := 0; i < numPuts; i++ { _, err := kv.Put(t.Context(), "foo", "bar") require.NoError(t, err) } _, err := kv.Compact(t.Context(), 3) require.NoError(t, err) clus.Members[0].Restart(t) // since watch's server isn't guaranteed to be synced with the cluster when // the watch resumes, there is a window where the watch can stay synced and // read off all events; if the watcher misses the window, it will go out of // sync and get a compaction error. wRev := int64(2) for int(wRev) <= numPuts+1 { var wresp clientv3.WatchResponse var ok bool select { case wresp, ok = <-wch: require.Truef(t, ok, "expected wresp, but got closed channel") case <-time.After(5 * time.Second): t.Fatalf("compacted watch timed out") } for _, ev := range wresp.Events { require.Equalf(t, ev.Kv.ModRevision, wRev, "expected modRev %v, got %+v", wRev, ev) wRev++ } if wresp.Err() == nil { continue } if !errors.Is(wresp.Err(), rpctypes.ErrCompacted) { t.Fatalf("wresp.Err() expected %v, got %+v", rpctypes.ErrCompacted, wresp.Err()) } break } if int(wRev) > numPuts+1 { // got data faster than the compaction return } // received compaction error; ensure the channel closes select { case wresp, ok := <-wch: if ok { t.Fatalf("expected closed channel, but got %v", wresp) } case <-time.After(5 * time.Second): t.Fatalf("timed out waiting for channel close") } } // TestWatchCompactRevision ensures the CompactRevision error is given on a // compaction event ahead of a watcher. func TestWatchCompactRevision(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) // set some keys kv := clus.RandClient() for i := 0; i < 5; i++ { _, err := kv.Put(t.Context(), "foo", "bar") require.NoError(t, err) } w := clus.RandClient() _, err := kv.Compact(t.Context(), 4) require.NoError(t, err) wch := w.Watch(t.Context(), "foo", clientv3.WithRev(2)) // get compacted error message wresp, ok := <-wch if !ok { t.Fatalf("expected wresp, but got closed channel") } if !errors.Is(wresp.Err(), rpctypes.ErrCompacted) { t.Fatalf("wresp.Err() expected %v, but got %v", rpctypes.ErrCompacted, wresp.Err()) } if !wresp.Canceled { t.Fatalf("wresp.Canceled expected true, got %+v", wresp) } // ensure the channel is closed if wresp, ok = <-wch; ok { t.Fatalf("expected closed channel, but got %v", wresp) } } func TestWatchWithProgressNotify2(t *testing.T) { testWatchWithProgressNotify(t, true) } func TestWatchWithProgressNotifyNoEvent(t *testing.T) { testWatchWithProgressNotify(t, false) } func testWatchWithProgressNotify(t *testing.T, watchOnPut bool) { integration.BeforeTest(t) // accelerate report interval so test terminates quickly oldpi := v3rpc.GetProgressReportInterval() // using atomics to avoid race warnings v3rpc.SetProgressReportInterval(3 * time.Second) pi := 3 * time.Second defer func() { v3rpc.SetProgressReportInterval(oldpi) }() clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) wc := clus.RandClient() opts := []clientv3.OpOption{clientv3.WithProgressNotify()} if watchOnPut { opts = append(opts, clientv3.WithPrefix()) } rch := wc.Watch(t.Context(), "foo", opts...) select { case resp := <-rch: // wait for notification if len(resp.Events) != 0 { t.Fatalf("resp.Events expected none, got %+v", resp.Events) } case <-time.After(2 * pi): t.Fatalf("watch response expected in %v, but timed out", pi) } kvc := clus.RandClient() _, err := kvc.Put(t.Context(), "foox", "bar") require.NoError(t, err) select { case resp := <-rch: if resp.Header.Revision != 2 { t.Fatalf("resp.Header.Revision expected 2, got %d", resp.Header.Revision) } if watchOnPut { // wait for put if watch on the put key ev := []*clientv3.Event{{ Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{Key: []byte("foox"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, }} if !reflect.DeepEqual(ev, resp.Events) { t.Fatalf("expected %+v, got %+v", ev, resp.Events) } } else if len(resp.Events) != 0 { // wait for notification otherwise t.Fatalf("expected no events, but got %+v", resp.Events) } case <-time.After(time.Duration(1.5 * float64(pi))): t.Fatalf("watch response expected in %v, but timed out", pi) } } func TestConfigurableWatchProgressNotifyInterval(t *testing.T) { integration.BeforeTest(t) progressInterval := 200 * time.Millisecond clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 3, WatchProgressNotifyInterval: progressInterval, }) defer clus.Terminate(t) opts := []clientv3.OpOption{clientv3.WithProgressNotify()} rch := clus.RandClient().Watch(t.Context(), "foo", opts...) timeout := 1 * time.Second // we expect to receive watch progress notify in 2 * progressInterval, // but for CPU-starved situation it may take longer. So we use 1 second here for timeout. select { case resp := <-rch: // waiting for a watch progress notify response if !resp.IsProgressNotify() { t.Fatalf("expected resp.IsProgressNotify() == true") } case <-time.After(timeout): t.Fatalf("timed out waiting for watch progress notify response in %v", timeout) } } func TestWatchRequestProgress(t *testing.T) { if integration.ThroughProxy { t.Skipf("grpc-proxy does not support WatchProgress yet") } testCases := []struct { name string watchers []string }{ {"0-watcher", []string{}}, {"1-watcher", []string{"/"}}, {"2-watcher", []string{"/", "/"}}, } for _, c := range testCases { t.Run(c.name, func(t *testing.T) { integration.BeforeTest(t) watchTimeout := 3 * time.Second clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) wc := clus.RandClient() var watchChans []clientv3.WatchChan for _, prefix := range c.watchers { watchChans = append(watchChans, wc.Watch(t.Context(), prefix, clientv3.WithPrefix())) } _, err := wc.Put(t.Context(), "/a", "1") require.NoError(t, err) for _, rch := range watchChans { select { case resp := <-rch: // wait for notification require.Lenf(t, resp.Events, 1, "resp.Events expected 1, got %d", len(resp.Events)) case <-time.After(watchTimeout): t.Fatalf("watch response expected in %v, but timed out", watchTimeout) } } // put a value not being watched to increment revision _, err = wc.Put(t.Context(), "x", "1") require.NoError(t, err) require.NoError(t, wc.RequestProgress(t.Context())) // verify all watch channels receive a progress notify for _, rch := range watchChans { select { case resp := <-rch: require.Truef(t, resp.IsProgressNotify(), "expected resp.IsProgressNotify() == true") require.Equalf(t, int64(3), resp.Header.Revision, "resp.Header.Revision expected 3, got %d", resp.Header.Revision) case <-time.After(watchTimeout): t.Fatalf("progress response expected in %v, but timed out", watchTimeout) } } }) } } func TestWatchEventType(t *testing.T) { integration.BeforeTest(t) cluster := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer cluster.Terminate(t) client := cluster.RandClient() ctx := t.Context() watchChan := client.Watch(ctx, "/", clientv3.WithPrefix()) _, err := client.Put(ctx, "/toDelete", "foo") require.NoErrorf(t, err, "Put failed: %v", err) _, err = client.Put(ctx, "/toDelete", "bar") require.NoErrorf(t, err, "Put failed: %v", err) _, err = client.Delete(ctx, "/toDelete") require.NoErrorf(t, err, "Delete failed: %v", err) lcr, err := client.Lease.Grant(ctx, 1) require.NoErrorf(t, err, "lease create failed: %v", err) _, err = client.Put(ctx, "/toExpire", "foo", clientv3.WithLease(lcr.ID)) require.NoErrorf(t, err, "Put failed: %v", err) tests := []struct { et mvccpb.Event_EventType isCreate bool isModify bool }{{ et: clientv3.EventTypePut, isCreate: true, }, { et: clientv3.EventTypePut, isModify: true, }, { et: clientv3.EventTypeDelete, }, { et: clientv3.EventTypePut, isCreate: true, }, { et: clientv3.EventTypeDelete, }} var res []*clientv3.Event for { select { case wres := <-watchChan: res = append(res, wres.Events...) case <-time.After(10 * time.Second): t.Fatalf("Should receive %d events and then break out loop", len(tests)) } if len(res) == len(tests) { break } } for i, tt := range tests { ev := res[i] if tt.et != ev.Type { t.Errorf("#%d: event type want=%s, get=%s", i, tt.et, ev.Type) } if tt.isCreate && !ev.IsCreate() { t.Errorf("#%d: event should be CreateEvent", i) } if tt.isModify && !ev.IsModify() { t.Errorf("#%d: event should be ModifyEvent", i) } } } func TestWatchErrConnClosed(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) donec := make(chan struct{}) go func() { defer close(donec) ch := cli.Watch(t.Context(), "foo") if wr := <-ch; !IsCanceled(wr.Err()) { t.Errorf("expected context canceled, got %v", wr.Err()) } }() require.NoError(t, cli.ActiveConnection().Close()) clus.TakeClient(0) select { case <-time.After(integration.RequestWaitTimeout): t.Fatal("wc.Watch took too long") case <-donec: } } func IsCanceled(err error) bool { if err == nil { return false } if errors.Is(err, context.Canceled) { return true } ev, ok := status.FromError(err) if !ok { return false } code := ev.Code() return code == codes.Canceled } func TestWatchAfterClose(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) clus.TakeClient(0) require.NoError(t, cli.Close()) donec := make(chan struct{}) go func() { cli.Watch(t.Context(), "foo") if err := cli.Close(); err != nil && !errors.Is(err, context.Canceled) { t.Errorf("expected %v, got %v", context.Canceled, err) } close(donec) }() select { case <-time.After(integration.RequestWaitTimeout): t.Fatal("wc.Watch took too long") case <-donec: } } // TestWatchWithRequireLeader checks the watch channel closes when no leader. func TestWatchWithRequireLeader(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) // Put a key for the non-require leader watch to read as an event. // The watchers will be on member[0]; put key through member[0] to // ensure that it receives the update so watching after killing quorum // is guaranteed to have the key. liveClient := clus.Client(0) _, err := liveClient.Put(t.Context(), "foo", "bar") require.NoError(t, err) clus.Members[1].Stop(t) clus.Members[2].Stop(t) clus.Client(1).Close() clus.Client(2).Close() clus.TakeClient(1) clus.TakeClient(2) // wait for election timeout, then member[0] will not have a leader. tickDuration := 10 * time.Millisecond // existing streams need three elections before they're torn down; wait until 5 elections cycle // so proxy tests receive a leader loss event on its existing watch before creating a new watch. time.Sleep(time.Duration(5*clus.Members[0].ElectionTicks) * tickDuration) chLeader := liveClient.Watch(clientv3.WithRequireLeader(t.Context()), "foo", clientv3.WithRev(1)) chNoLeader := liveClient.Watch(t.Context(), "foo", clientv3.WithRev(1)) select { case resp, ok := <-chLeader: require.Truef(t, ok, "expected %v watch channel, got closed channel", rpctypes.ErrNoLeader) require.ErrorIsf(t, resp.Err(), rpctypes.ErrNoLeader, "expected %v watch response error, got %+v", rpctypes.ErrNoLeader, resp) case <-time.After(integration.RequestWaitTimeout): t.Fatal("watch without leader took too long to close") } select { case resp, ok := <-chLeader: require.Falsef(t, ok, "expected closed channel, got response %v", resp) case <-time.After(integration.RequestWaitTimeout): t.Fatal("waited too long for channel to close") } _, ok := <-chNoLeader require.Truef(t, ok, "expected response, got closed channel") cnt, err := clus.Members[0].Metric( "etcd_server_client_requests_total", `type="stream"`, fmt.Sprintf(`client_api_version="%v"`, version.APIVersion), ) require.NoError(t, err) cv, err := strconv.ParseInt(cnt, 10, 32) require.NoError(t, err) require.GreaterOrEqualf(t, cv, int64(2), "expected at least 2, got %q", cnt) } // TestWatchWithFilter checks that watch filtering works. func TestWatchWithFilter(t *testing.T) { integration.BeforeTest(t) cluster := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer cluster.Terminate(t) client := cluster.RandClient() ctx := t.Context() wcNoPut := client.Watch(ctx, "a", clientv3.WithFilterPut()) wcNoDel := client.Watch(ctx, "a", clientv3.WithFilterDelete()) _, err := client.Put(ctx, "a", "abc") require.NoError(t, err) _, err = client.Delete(ctx, "a") require.NoError(t, err) npResp := <-wcNoPut if len(npResp.Events) != 1 || npResp.Events[0].Type != clientv3.EventTypeDelete { t.Fatalf("expected delete event, got %+v", npResp.Events) } ndResp := <-wcNoDel if len(ndResp.Events) != 1 || ndResp.Events[0].Type != clientv3.EventTypePut { t.Fatalf("expected put event, got %+v", ndResp.Events) } select { case resp := <-wcNoPut: t.Fatalf("unexpected event on filtered put (%+v)", resp) case resp := <-wcNoDel: t.Fatalf("unexpected event on filtered delete (%+v)", resp) case <-time.After(100 * time.Millisecond): } } // TestWatchWithCreatedNotification checks that WithCreatedNotify returns a // Created watch response. func TestWatchWithCreatedNotification(t *testing.T) { integration.BeforeTest(t) cluster := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer cluster.Terminate(t) client := cluster.RandClient() ctx := t.Context() createC := client.Watch(ctx, "a", clientv3.WithCreatedNotify()) resp := <-createC require.Truef(t, resp.Created, "expected created event, got %v", resp) } // TestWatchWithCreatedNotificationDropConn ensures that // a watcher with created notify does not post duplicate // created events from disconnect. func TestWatchWithCreatedNotificationDropConn(t *testing.T) { integration.BeforeTest(t) cluster := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer cluster.Terminate(t) client := cluster.RandClient() wch := client.Watch(t.Context(), "a", clientv3.WithCreatedNotify()) resp := <-wch require.Truef(t, resp.Created, "expected created event, got %v", resp) cluster.Members[0].Bridge().DropConnections() // check watch channel doesn't post another watch response. select { case wresp := <-wch: t.Fatalf("got unexpected watch response: %+v\n", wresp) case <-time.After(time.Second): // watcher may not reconnect by the time it hits the select, // so it wouldn't have a chance to filter out the second create event } } // TestWatchCancelOnServer ensures client watcher cancels propagate back to the server. func TestWatchCancelOnServer(t *testing.T) { integration.BeforeTest(t) cluster := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer cluster.Terminate(t) client := cluster.RandClient() numWatches := 10 // The grpc proxy starts watches to detect leadership after the proxy server // returns as started; to avoid racing on the proxy's internal watches, wait // until require leader watches get create responses to ensure the leadership // watches have started. for { ctx, cancel := context.WithCancel(clientv3.WithRequireLeader(t.Context())) ww := client.Watch(ctx, "a", clientv3.WithCreatedNotify()) wresp := <-ww cancel() if wresp.Err() == nil { break } } cancels := make([]context.CancelFunc, numWatches) for i := 0; i < numWatches; i++ { // force separate streams in client md := metadata.Pairs("some-key", fmt.Sprintf("%d", i)) mctx := metadata.NewOutgoingContext(t.Context(), md) ctx, cancel := context.WithCancel(mctx) cancels[i] = cancel w := client.Watch(ctx, fmt.Sprintf("%d", i), clientv3.WithCreatedNotify()) <-w } // get max watches; proxy tests have leadership watches, so total may be >numWatches maxWatches, _ := cluster.Members[0].Metric("etcd_debugging_mvcc_watcher_total") // cancel all and wait for cancels to propagate to etcd server for i := 0; i < numWatches; i++ { cancels[i]() } time.Sleep(time.Second) minWatches, err := cluster.Members[0].Metric("etcd_debugging_mvcc_watcher_total") require.NoError(t, err) maxWatchV, minWatchV := 0, 0 n, serr := fmt.Sscanf(maxWatches+" "+minWatches, "%d %d", &maxWatchV, &minWatchV) if n != 2 || serr != nil { t.Fatalf("expected n=2 and err=nil, got n=%d and err=%v", n, serr) } require.GreaterOrEqualf(t, maxWatchV-minWatchV, numWatches, "expected %d canceled watchers, got %d", numWatches, maxWatchV-minWatchV) } // TestWatchOverlapContextCancel stresses the watcher stream teardown path by // creating/canceling watchers to ensure that new watchers are not taken down // by a torn down watch stream. The sort of race that's being detected: // 1. create w1 using a cancelable ctx with %v as "ctx" // 2. cancel ctx // 3. watcher client begins tearing down watcher grpc stream since no more watchers // 3. start creating watcher w2 using a new "ctx" (not canceled), attaches to old grpc stream // 4. watcher client finishes tearing down stream on "ctx" // 5. w2 comes back canceled func TestWatchOverlapContextCancel(t *testing.T) { f := func(clus *integration.Cluster) {} testWatchOverlapContextCancel(t, f) } func TestWatchOverlapDropConnContextCancel(t *testing.T) { f := func(clus *integration.Cluster) { clus.Members[0].Bridge().DropConnections() } testWatchOverlapContextCancel(t, f) } func testWatchOverlapContextCancel(t *testing.T, f func(*integration.Cluster)) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) n := 100 ctxs, ctxc := make([]context.Context, 5), make([]chan struct{}, 5) for i := range ctxs { // make unique stream md := metadata.Pairs("some-key", fmt.Sprintf("%d", i)) ctxs[i] = metadata.NewOutgoingContext(t.Context(), md) // limits the maximum number of outstanding watchers per stream ctxc[i] = make(chan struct{}, 2) } // issue concurrent watches on "abc" with cancel cli := clus.RandClient() _, err := cli.Put(t.Context(), "abc", "def") require.NoError(t, err) ch := make(chan struct{}, n) tCtx, cancelFunc := context.WithCancel(t.Context()) defer cancelFunc() for i := 0; i < n; i++ { go func() { defer func() { ch <- struct{}{} }() idx := rand.Intn(len(ctxs)) ctx, cancel := context.WithCancel(ctxs[idx]) ctxc[idx] <- struct{}{} wch := cli.Watch(ctx, "abc", clientv3.WithRev(1)) select { case <-tCtx.Done(): cancel() return default: } f(clus) select { case _, ok := <-wch: if !ok { t.Errorf("unexpected closed channel %p", wch) } // may take a second or two to reestablish a watcher because of // grpc back off policies for disconnects case <-time.After(5 * time.Second): t.Errorf("timed out waiting for watch on %p", wch) } // randomize how cancel overlaps with watch creation if rand.Intn(2) == 0 { <-ctxc[idx] cancel() } else { cancel() <-ctxc[idx] } }() } // join on watches for i := 0; i < n; i++ { select { case <-ch: case <-time.After(5 * time.Second): t.Fatalf("timed out waiting for completed watch") } } } // TestWatchCancelAndCloseClient ensures that canceling a watcher then immediately // closing the client does not return a client closing error. func TestWatchCancelAndCloseClient(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) ctx, cancel := context.WithCancel(t.Context()) wch := cli.Watch(ctx, "abc") donec := make(chan struct{}) go func() { defer close(donec) select { case wr, ok := <-wch: if ok { t.Errorf("expected closed watch after cancel(), got resp=%+v err=%v", wr, wr.Err()) } case <-time.After(5 * time.Second): t.Error("timed out waiting for closed channel") } }() cancel() require.NoError(t, cli.Close()) <-donec clus.TakeClient(0) } // TestWatchStressResumeClose establishes a bunch of watchers, disconnects // to put them in resuming mode, cancels them so some resumes by cancel fail, // then closes the watcher interface to ensure correct clean up. func TestWatchStressResumeClose(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) cli := clus.Client(0) ctx, cancel := context.WithCancel(t.Context()) // add more watches than can be resumed before the cancel wchs := make([]clientv3.WatchChan, 2000) for i := range wchs { wchs[i] = cli.Watch(ctx, "abc") } clus.Members[0].Bridge().DropConnections() cancel() require.NoError(t, cli.Close()) clus.TakeClient(0) } // TestWatchCancelDisconnected ensures canceling a watcher works when // its grpc stream is disconnected / reconnecting. func TestWatchCancelDisconnected(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) ctx, cancel := context.WithCancel(t.Context()) // add more watches than can be resumed before the cancel wch := cli.Watch(ctx, "abc") clus.Members[0].Stop(t) cancel() select { case <-wch: case <-time.After(time.Second): t.Fatal("took too long to cancel disconnected watcher") } } // TestWatchClose ensures that close does not return error func TestWatchClose(t *testing.T) { runWatchTest(t, testWatchClose) } func testWatchClose(t *testing.T, wctx *watchctx) { ctx, cancel := context.WithCancel(t.Context()) wch := wctx.w.Watch(ctx, "a") cancel() require.NotNilf(t, wch, "expected watcher channel, got nil") require.NoErrorf(t, wctx.w.Close(), "watch did not close successfully") wresp, ok := <-wch require.Falsef(t, ok, "read wch got %v; expected closed channel", wresp) } func TestWatch(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) t.Cleanup(func() { clus.Terminate(t) }) client := clus.Client(0) tcs := []struct { name string key string opts []clientv3.OpOption wantError error wantEvents []*clientv3.Event }{ { name: "Watch with negative revision", key: "/", opts: []clientv3.OpOption{clientv3.WithRev(-1)}, wantError: rpctypes.ErrCompacted, }, { name: "Watch with zero revision", key: "/", opts: []clientv3.OpOption{clientv3.WithRev(0)}, }, { name: "Watch with positive revision", key: "/", opts: []clientv3.OpOption{clientv3.WithRev(1)}, }, } ctx := t.Context() t.Log("Open watches") watches := make([]clientv3.WatchChan, len(tcs)) for i, tc := range tcs { watchCtx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() watches[i] = client.Watch(watchCtx, tc.key, tc.opts...) } t.Log("Validate") for i, tc := range tcs { t.Run(tc.name, func(t *testing.T) { events, err := collectEvents(ctx, watches[i]) if tc.wantError == nil { require.NoError(t, err) } else { require.ErrorContains(t, err, tc.wantError.Error()) } if diff := cmp.Diff(tc.wantEvents, events); diff != "" { t.Errorf("unexpected events (-want +got):\n%s", diff) } }) } } func collectEvents(ctx context.Context, watch clientv3.WatchChan) (events []*clientv3.Event, err error) { for { select { case resp, ok := <-watch: if !ok { return events, nil } err := resp.Err() if err != nil { return events, err } events = append(events, resp.Events...) case <-ctx.Done(): return events, ctx.Err() // Watch resync interval * 1.5 case <-time.After(150 * time.Millisecond): return events, nil } } } ================================================ FILE: tests/integration/cluster_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "fmt" "log" "math/rand" "os" "strconv" "strings" "testing" "time" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/integration" ) func init() { // open microsecond-level time log for integration test debugging log.SetFlags(log.Ltime | log.Lmicroseconds | log.Lshortfile) if t := os.Getenv("ETCD_ELECTION_TIMEOUT_TICKS"); t != "" { if i, err := strconv.ParseInt(t, 10, 64); err == nil { integration.ElectionTicks = int(i) } } } func TestClusterOf1(t *testing.T) { testCluster(t, 1) } func TestClusterOf3(t *testing.T) { testCluster(t, 3) } func testCluster(t *testing.T, size int) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: size}) defer c.Terminate(t) clusterMustProgress(t, c.Members) } func TestTLSClusterOf3(t *testing.T) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, PeerTLS: &integration.TestTLSInfo}) defer c.Terminate(t) clusterMustProgress(t, c.Members) } // TestTLSClusterOf3WithSpecificUsage tests that a cluster can progress when // using separate client and server certs when peering. This supports // certificate authorities that don't issue dual-usage certificates. func TestTLSClusterOf3WithSpecificUsage(t *testing.T) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, PeerTLS: &integration.TestTLSInfoWithSpecificUsage}) defer c.Terminate(t) clusterMustProgress(t, c.Members) } func TestDoubleClusterSizeOf1(t *testing.T) { testDoubleClusterSize(t, 1) } func TestDoubleClusterSizeOf3(t *testing.T) { testDoubleClusterSize(t, 3) } func testDoubleClusterSize(t *testing.T, size int) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: size, DisableStrictReconfigCheck: true}) defer c.Terminate(t) for i := 0; i < size; i++ { c.AddMember(t) } clusterMustProgress(t, c.Members) } func TestDoubleTLSClusterSizeOf3(t *testing.T) { integration.BeforeTest(t) cfg := &integration.ClusterConfig{ Size: 1, PeerTLS: &integration.TestTLSInfo, DisableStrictReconfigCheck: true, } c := integration.NewCluster(t, cfg) defer c.Terminate(t) for i := 0; i < 3; i++ { c.AddMember(t) } clusterMustProgress(t, c.Members) } func TestDecreaseClusterSizeOf3(t *testing.T) { testDecreaseClusterSize(t, 3) } func TestDecreaseClusterSizeOf5(t *testing.T) { testDecreaseClusterSize(t, 5) } func testDecreaseClusterSize(t *testing.T, size int) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: size, DisableStrictReconfigCheck: true}) defer c.Terminate(t) // TODO: remove the last but one member for i := 0; i < size-1; i++ { id := c.Members[len(c.Members)-1].Server.MemberID() // may hit second leader election on slow machines if err := c.RemoveMember(t, c.Members[0].Client, uint64(id)); err != nil { if strings.Contains(err.Error(), "no leader") { t.Logf("got leader error (%v)", err) i-- continue } t.Fatal(err) } c.WaitMembersForLeader(t, c.Members) } clusterMustProgress(t, c.Members) } func TestForceNewCluster(t *testing.T) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer c.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), integration.RequestTimeout) resp, err := c.Members[0].Client.Put(ctx, "/foo", "bar") require.NoErrorf(t, err, "unexpected create error") cancel() // ensure create has been applied in this machine ctx, cancel = context.WithTimeout(t.Context(), integration.RequestTimeout) watch := c.Members[0].Client.Watcher.Watch(ctx, "/foo", clientv3.WithRev(resp.Header.Revision-1)) for resp := range watch { if len(resp.Events) != 0 { break } require.NoErrorf(t, resp.Err(), "unexpected watch error") require.Falsef(t, resp.Canceled, "watch cancelled") } cancel() c.Members[0].Stop(t) c.Members[1].Terminate(t) c.Members[2].Terminate(t) c.Members[0].ForceNewCluster = true err = c.Members[0].Restart(t) require.NoErrorf(t, err, "unexpected ForceRestart error") c.WaitMembersForLeader(t, c.Members[:1]) // use new http client to init new connection // ensure force restart keep the old data, and new Cluster can make progress ctx, cancel = context.WithTimeout(t.Context(), integration.RequestTimeout) watch = c.Members[0].Client.Watcher.Watch(ctx, "/foo", clientv3.WithRev(resp.Header.Revision-1)) for resp := range watch { if len(resp.Events) != 0 { break } require.NoErrorf(t, resp.Err(), "unexpected watch error") require.Falsef(t, resp.Canceled, "watch cancelled") } cancel() clusterMustProgress(t, c.Members[:1]) } func TestAddMemberAfterClusterFullRotation(t *testing.T) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) defer c.Terminate(t) // remove all the previous three members and add in three new members. for i := 0; i < 3; i++ { err := c.RemoveMember(t, c.Members[0].Client, uint64(c.Members[1].Server.MemberID())) require.NoError(t, err) c.WaitMembersForLeader(t, c.Members) c.AddMember(t) c.WaitMembersForLeader(t, c.Members) } c.AddMember(t) c.WaitMembersForLeader(t, c.Members) clusterMustProgress(t, c.Members) } // TestIssue2681 ensures we can remove a member then add a new one back immediately. func TestIssue2681(t *testing.T) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 5, DisableStrictReconfigCheck: true}) defer c.Terminate(t) require.NoError(t, c.RemoveMember(t, c.Members[0].Client, uint64(c.Members[4].Server.MemberID()))) c.WaitMembersForLeader(t, c.Members) c.AddMember(t) c.WaitMembersForLeader(t, c.Members) clusterMustProgress(t, c.Members) } // TestIssue2746 ensures we can remove a member after a snapshot then add a new one back. func TestIssue2746(t *testing.T) { testIssue2746(t, 5) } // TestIssue2746WithThree tests with 3 nodes TestIssue2476 sometimes had a shutdown with an inflight snapshot. func TestIssue2746WithThree(t *testing.T) { testIssue2746(t, 3) } func testIssue2746(t *testing.T, members int) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: members, SnapshotCount: 10, DisableStrictReconfigCheck: true}) defer c.Terminate(t) // force a snapshot for i := 0; i < 20; i++ { clusterMustProgress(t, c.Members) } require.NoError(t, c.RemoveMember(t, c.Members[0].Client, uint64(c.Members[members-1].Server.MemberID()))) c.WaitMembersForLeader(t, c.Members) c.AddMember(t) c.WaitMembersForLeader(t, c.Members) clusterMustProgress(t, c.Members) } // TestIssue2904 ensures etcd will not panic when removing a just started member. func TestIssue2904(t *testing.T) { integration.BeforeTest(t) // start 1-member Cluster to ensure member 0 is the leader of the Cluster. c := integration.NewCluster(t, &integration.ClusterConfig{Size: 2, UseBridge: true, DisableStrictReconfigCheck: true}) defer c.Terminate(t) c.WaitLeader(t) c.AddMember(t) c.Members[2].Stop(t) // send remove member-1 request to the Cluster. ctx, cancel := context.WithTimeout(t.Context(), integration.RequestTimeout) // the proposal is not committed because member 1 is stopped, but the // proposal is appended to leader'Server raft log. c.Members[0].Client.MemberRemove(ctx, uint64(c.Members[2].Server.MemberID())) cancel() // restart member, and expect it to send UpdateAttributes request. // the log in the leader is like this: // [..., remove 1, ..., update attr 1, ...] c.Members[2].Restart(t) // when the member comes back, it ack the proposal to remove itself, // and apply it. <-c.Members[2].Server.StopNotify() // terminate removed member c.Members[2].Client.Close() c.Members[2].Terminate(t) c.Members = c.Members[:2] // wait member to be removed. c.WaitMembersMatch(t, c.ProtoMembers()) } // TestIssue3699 tests minority failure during cluster configuration; it was // deadlocking. func TestIssue3699(t *testing.T) { // start a Cluster of 3 nodes a, b, c integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true, DisableStrictReconfigCheck: true}) defer c.Terminate(t) // make node a unavailable c.Members[0].Stop(t) // add node d c.AddMember(t) t.Logf("Disturbing cluster till member:3 will become a leader") // electing node d as leader makes node a unable to participate leaderID := c.WaitMembersForLeader(t, c.Members) for leaderID != 3 { c.Members[leaderID].Stop(t) <-c.Members[leaderID].Server.StopNotify() // do not restart the killed member immediately. // the member will advance its election timeout after restart, // so it will have a better chance to become the leader again. time.Sleep(time.Duration(integration.ElectionTicks * int(config.TickDuration))) c.Members[leaderID].Restart(t) leaderID = c.WaitMembersForLeader(t, c.Members) } t.Logf("Finally elected member 3 as the leader.") t.Logf("Restarting member '0'...") // bring back node a // node a will remain useless as long as d is the leader. require.NoError(t, c.Members[0].Restart(t)) t.Logf("Restarted member '0'.") select { // waiting for ReadyNotify can take several seconds case <-time.After(10 * time.Second): t.Fatalf("waited too long for ready notification") case <-c.Members[0].Server.StopNotify(): t.Fatalf("should not be stopped") case <-c.Members[0].Server.ReadyNotify(): } // must WaitMembersForLeader so goroutines don't leak on terminate c.WaitLeader(t) t.Logf("Expecting successful put...") // try to participate in Cluster ctx, cancel := context.WithTimeout(t.Context(), integration.RequestTimeout) _, err := c.Members[0].Client.Put(ctx, "/foo", "bar") require.NoErrorf(t, err, "unexpected error on Put") cancel() } // TestRejectUnhealthyAdd ensures an unhealthy cluster rejects adding members. func TestRejectUnhealthyAdd(t *testing.T) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer c.Terminate(t) // make Cluster unhealthy and wait for downed peer c.Members[0].Stop(t) c.WaitLeader(t) // all attempts to add member should fail for i := 1; i < len(c.Members); i++ { err := c.AddMemberByURL(t, c.Members[i].Client, "unix://foo:12345") require.Errorf(t, err, "should have failed adding peer") // TODO: client should return descriptive error codes for internal errors if !strings.Contains(err.Error(), "unhealthy cluster") { t.Errorf("unexpected error (%v)", err) } } // make cluster healthy c.Members[0].Restart(t) c.WaitLeader(t) time.Sleep(2 * etcdserver.HealthInterval) // add member should succeed now that it'Server healthy var err error for i := 1; i < len(c.Members); i++ { if err = c.AddMemberByURL(t, c.Members[i].Client, "unix://foo:12345"); err == nil { break } } require.NoErrorf(t, err, "should have added peer to healthy Cluster (%v)", err) } // TestRejectUnhealthyRemove ensures an unhealthy cluster rejects removing members // if quorum will be lost. func TestRejectUnhealthyRemove(t *testing.T) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 5, UseBridge: true}) defer c.Terminate(t) // make cluster unhealthy and wait for downed peer; (3 up, 2 down) c.Members[0].Stop(t) c.Members[1].Stop(t) leader := c.WaitLeader(t) // reject remove active member since (3,2)-(1,0) => (2,2) lacks quorum err := c.RemoveMember(t, c.Members[leader].Client, uint64(c.Members[2].Server.MemberID())) require.Errorf(t, err, "should reject quorum breaking remove: %s", err) // TODO: client should return more descriptive error codes for internal errors if !strings.Contains(err.Error(), "unhealthy cluster") { t.Errorf("unexpected error (%v)", err) } // member stopped after launch; wait for missing heartbeats time.Sleep(time.Duration(integration.ElectionTicks * int(config.TickDuration))) // permit remove dead member since (3,2) - (0,1) => (3,1) has quorum err = c.RemoveMember(t, c.Members[2].Client, uint64(c.Members[0].Server.MemberID())) require.NoErrorf(t, err, "should accept removing down member") // bring cluster to (4,1) c.Members[0].Restart(t) // restarted member must be connected for a HealthInterval before remove is accepted time.Sleep((3 * etcdserver.HealthInterval) / 2) // accept remove member since (4,1)-(1,0) => (3,1) has quorum err = c.RemoveMember(t, c.Members[1].Client, uint64(c.Members[0].Server.MemberID())) require.NoErrorf(t, err, "expected to remove member, got error") } // TestRestartRemoved ensures that restarting removed member must exit // if 'initial-cluster-state' is set 'new' and old data directory still exists // (see https://github.com/etcd-io/etcd/issues/7512 for more). func TestRestartRemoved(t *testing.T) { integration.BeforeTest(t) // 1. start single-member Cluster c := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer c.Terminate(t) // 2. add a new member c.Cfg.DisableStrictReconfigCheck = true c.AddMember(t) c.WaitLeader(t) firstMember := c.Members[0] firstMember.KeepDataDirTerminate = true // 3. remove first member, shut down without deleting data err := c.RemoveMember(t, c.Members[1].Client, uint64(firstMember.Server.MemberID())) require.NoErrorf(t, err, "expected to remove member, got error") c.WaitLeader(t) // 4. restart first member with 'initial-cluster-state=new' // wrong config, expects exit within ReqTimeout firstMember.ServerConfig.NewCluster = false err = firstMember.Restart(t) require.NoErrorf(t, err, "unexpected ForceRestart error") defer func() { firstMember.Close() os.RemoveAll(firstMember.ServerConfig.DataDir) }() select { case <-firstMember.Server.StopNotify(): case <-time.After(time.Minute): t.Fatalf("removed member didn't exit within %v", time.Minute) } } // clusterMustProgress ensures that cluster can make progress. It creates // a random key first, and check the new key could be got from all client urls // of the cluster. func clusterMustProgress(t *testing.T, members []*integration.Member) { key := fmt.Sprintf("foo%d", rand.Int()) var ( err error resp *clientv3.PutResponse ) // retry in case of leader loss induced by slow CI for i := 0; i < 3; i++ { ctx, cancel := context.WithTimeout(t.Context(), integration.RequestTimeout) resp, err = members[0].Client.Put(ctx, key, "bar") cancel() if err == nil { break } t.Logf("failed to create key on #0 (%v)", err) } require.NoErrorf(t, err, "create on #0 error") for i, m := range members { mctx, mcancel := context.WithTimeout(t.Context(), integration.RequestTimeout) watch := m.Client.Watcher.Watch(mctx, key, clientv3.WithRev(resp.Header.Revision-1)) for resp := range watch { if len(resp.Events) != 0 { break } require.NoErrorf(t, resp.Err(), "#%d: watch error", i) require.Falsef(t, resp.Canceled, "#%d: watch: cancelled", i) } mcancel() } } func TestSpeedyTerminate(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) // Stop/Restart so requests will time out on lost leaders for i := 0; i < 3; i++ { clus.Members[i].Stop(t) clus.Members[i].Restart(t) } donec := make(chan struct{}) go func() { defer close(donec) clus.Terminate(t) }() select { case <-time.After(10 * time.Second): t.Fatalf("Cluster took too long to terminate") case <-donec: } } // TestConcurrentRemoveMember demonstrated a panic in mayRemoveMember with // concurrent calls to MemberRemove. To reliably reproduce the panic, a delay // needed to be injected in IsMemberExist, which is done using a failpoint. // After fixing the bug, IsMemberExist is no longer called by mayRemoveMember. func TestConcurrentRemoveMember(t *testing.T) { integration.BeforeTest(t, integration.WithFailpoint("afterIsMemberExist", `sleep("1s")`)) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer c.Terminate(t) addResp, err := c.Members[0].Client.MemberAddAsLearner(t.Context(), []string{"http://localhost:123"}) require.NoError(t, err) removeID := addResp.Member.ID done := make(chan struct{}) go func() { time.Sleep(time.Second / 2) c.Members[0].Client.MemberRemove(t.Context(), removeID) close(done) }() _, err = c.Members[0].Client.MemberRemove(t.Context(), removeID) require.NoError(t, err) <-done } func TestConcurrentMoveLeader(t *testing.T) { integration.BeforeTest(t, integration.WithFailpoint("afterIsMemberExist", `sleep("1s")`)) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer c.Terminate(t) addResp, err := c.Members[0].Client.MemberAddAsLearner(t.Context(), []string{"http://localhost:123"}) require.NoError(t, err) removeID := addResp.Member.ID done := make(chan struct{}) go func() { time.Sleep(time.Second / 2) c.Members[0].Client.MoveLeader(t.Context(), removeID) close(done) }() _, err = c.Members[0].Client.MemberRemove(t.Context(), removeID) require.NoError(t, err) <-done } ================================================ FILE: tests/integration/corrupt_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/storage/mvcc/testutil" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestPeriodicCheck(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) cc, err := clus.ClusterClient(t) require.NoError(t, err) ctx := t.Context() var totalRevisions int64 = 1210 var rev int64 for ; rev < totalRevisions; rev += testutil.CompactionCycle { testPeriodicCheck(ctx, t, cc, clus, rev, rev+testutil.CompactionCycle) } testPeriodicCheck(ctx, t, cc, clus, rev, rev+totalRevisions) alarmResponse, err := cc.AlarmList(ctx) require.NoErrorf(t, err, "error on alarm list") assert.Equal(t, []*etcdserverpb.AlarmMember(nil), alarmResponse.Alarms) } func testPeriodicCheck(ctx context.Context, t *testing.T, cc *clientv3.Client, clus *integration.Cluster, start, stop int64) { for i := start; i <= stop; i++ { if i%67 == 0 { _, err := cc.Delete(ctx, testutil.PickKey(i+83)) require.NoErrorf(t, err, "error on delete") } else { _, err := cc.Put(ctx, testutil.PickKey(i), fmt.Sprint(i)) require.NoErrorf(t, err, "error on put") } } err := clus.Members[0].Server.CorruptionChecker().PeriodicCheck() assert.NoErrorf(t, err, "error on periodic check (rev %v)", stop) } func TestPeriodicCheckDetectsCorruption(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) cc, err := clus.ClusterClient(t) require.NoError(t, err) ctx := t.Context() for i := 0; i < 10; i++ { _, err = cc.Put(ctx, testutil.PickKey(int64(i)), fmt.Sprint(i)) require.NoErrorf(t, err, "error on put") } err = clus.Members[0].Server.CorruptionChecker().PeriodicCheck() require.NoErrorf(t, err, "error on periodic check") clus.Members[0].Stop(t) clus.WaitLeader(t) err = testutil.CorruptBBolt(clus.Members[0].BackendPath()) require.NoError(t, err) err = clus.Members[0].Restart(t) require.NoError(t, err) time.Sleep(50 * time.Millisecond) leader := clus.WaitLeader(t) err = clus.Members[leader].Server.CorruptionChecker().PeriodicCheck() require.NoErrorf(t, err, "error on periodic check") time.Sleep(50 * time.Millisecond) alarmResponse, err := cc.AlarmList(ctx) require.NoErrorf(t, err, "error on alarm list") assert.Equal(t, []*etcdserverpb.AlarmMember{{Alarm: etcdserverpb.AlarmType_CORRUPT, MemberID: uint64(clus.Members[0].ID())}}, alarmResponse.Alarms) } func TestCompactHashCheck(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) cc, err := clus.ClusterClient(t) require.NoError(t, err) ctx := t.Context() var totalRevisions int64 = 1210 var rev int64 for ; rev < totalRevisions; rev += testutil.CompactionCycle { testCompactionHash(ctx, t, cc, clus, rev, rev+testutil.CompactionCycle) } testCompactionHash(ctx, t, cc, clus, rev, rev+totalRevisions) } func testCompactionHash(ctx context.Context, t *testing.T, cc *clientv3.Client, clus *integration.Cluster, start, stop int64) { for i := start; i <= stop; i++ { if i%67 == 0 { _, err := cc.Delete(ctx, testutil.PickKey(i+83)) require.NoErrorf(t, err, "error on delete") } else { _, err := cc.Put(ctx, testutil.PickKey(i), fmt.Sprint(i)) require.NoErrorf(t, err, "error on put") } } _, err := cc.Compact(ctx, stop) require.NoErrorf(t, err, "error on compact (rev %v)", stop) // Wait for compaction to be compacted time.Sleep(50 * time.Millisecond) clus.Members[0].Server.CorruptionChecker().CompactHashCheck() } func TestCompactHashCheckDetectCorruption(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) cc, err := clus.ClusterClient(t) require.NoError(t, err) ctx := t.Context() for i := 0; i < 10; i++ { _, err = cc.Put(ctx, testutil.PickKey(int64(i)), fmt.Sprint(i)) require.NoErrorf(t, err, "error on put") } clus.Members[0].Server.CorruptionChecker().CompactHashCheck() clus.Members[0].Stop(t) clus.WaitLeader(t) err = testutil.CorruptBBolt(clus.Members[0].BackendPath()) require.NoError(t, err) err = clus.Members[0].Restart(t) require.NoError(t, err) _, err = cc.Compact(ctx, 5) require.NoError(t, err) time.Sleep(50 * time.Millisecond) leader := clus.WaitLeader(t) clus.Members[leader].Server.CorruptionChecker().CompactHashCheck() time.Sleep(50 * time.Millisecond) alarmResponse, err := cc.AlarmList(ctx) require.NoErrorf(t, err, "error on alarm list") assert.Equal(t, []*etcdserverpb.AlarmMember{{Alarm: etcdserverpb.AlarmType_CORRUPT, MemberID: uint64(clus.Members[0].ID())}}, alarmResponse.Alarms) } func TestCompactHashCheckDetectMultipleCorruption(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 5}) defer clus.Terminate(t) cc, err := clus.ClusterClient(t) require.NoError(t, err) ctx := t.Context() for i := 0; i < 10; i++ { _, err = cc.Put(ctx, testutil.PickKey(int64(i)), fmt.Sprint(i)) require.NoErrorf(t, err, "error on put") } clus.Members[0].Server.CorruptionChecker().CompactHashCheck() clus.Members[0].Stop(t) clus.Members[1].Server.CorruptionChecker().CompactHashCheck() clus.Members[1].Stop(t) clus.WaitLeader(t) err = testutil.CorruptBBolt(clus.Members[0].BackendPath()) require.NoError(t, err) err = testutil.CorruptBBolt(clus.Members[1].BackendPath()) require.NoError(t, err) err = clus.Members[0].Restart(t) require.NoError(t, err) err = clus.Members[1].Restart(t) require.NoError(t, err) _, err = cc.Compact(ctx, 5) require.NoError(t, err) time.Sleep(50 * time.Millisecond) leader := clus.WaitLeader(t) clus.Members[leader].Server.CorruptionChecker().CompactHashCheck() time.Sleep(50 * time.Millisecond) alarmResponse, err := cc.AlarmList(ctx) require.NoErrorf(t, err, "error on alarm list") expectedAlarmMap := map[uint64]etcdserverpb.AlarmType{ uint64(clus.Members[0].ID()): etcdserverpb.AlarmType_CORRUPT, uint64(clus.Members[1].ID()): etcdserverpb.AlarmType_CORRUPT, } actualAlarmMap := make(map[uint64]etcdserverpb.AlarmType) for _, alarm := range alarmResponse.Alarms { actualAlarmMap[alarm.MemberID] = alarm.Alarm } require.Equal(t, expectedAlarmMap, actualAlarmMap) } ================================================ FILE: tests/integration/doc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /* Package integration implements tests built upon embedded etcd, and focus on etcd correctness. Features/goals of the integration tests: 1. test the whole code base except command-line parsing. 2. check internal data, including raft, store and etc. 3. based on goroutines, which is faster than process. 4. mainly tests user behavior and user-facing API. */ package integration ================================================ FILE: tests/integration/embed/embed_proxy_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build cluster_proxy // Package embed_test is an empty package that exists to keep the following test working: // # go test -tags=cluster_proxy ./integration/embed package embed_test ================================================ FILE: tests/integration/embed/embed_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build !cluster_proxy // Keep the test in a separate package from other tests such that // .setupLogging method does not race with other (previously running) servers (grpclog is global). package embed_test import ( "fmt" "net/url" "os" "path/filepath" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/tests/v3/framework/integration" "go.etcd.io/etcd/tests/v3/framework/testutils" ) var testTLSInfo = transport.TLSInfo{ KeyFile: testutils.MustAbsPath("../../fixtures/server.key.insecure"), CertFile: testutils.MustAbsPath("../../fixtures/server.crt"), TrustedCAFile: testutils.MustAbsPath("../../fixtures/ca.crt"), ClientCertAuth: true, } func TestEmbedEtcd(t *testing.T) { testutil.SkipTestIfShortMode(t, "Cannot start embedded cluster in --short tests") tests := []struct { cfg embed.Config werr string wpeers int wclients int }{ {werr: "advertise-client-urls is required"}, {werr: "should be at least"}, {werr: "is too long"}, {wpeers: 1, wclients: 1}, {wpeers: 2, wclients: 1}, {wpeers: 1, wclients: 2}, {werr: "expected IP"}, {werr: "expected IP"}, } urls := newEmbedURLs(false, 10) // setup defaults for i := range tests { tests[i].cfg = *embed.NewConfig() tests[i].cfg.Logger = "zap" tests[i].cfg.LogOutputs = []string{"/dev/null"} } setupEmbedCfg(&tests[0].cfg, []url.URL{urls[0]}, []url.URL{urls[1]}) tests[0].cfg.AdvertiseClientUrls = nil tests[1].cfg.TickMs = tests[2].cfg.ElectionMs - 1 tests[2].cfg.ElectionMs = 999999 setupEmbedCfg(&tests[3].cfg, []url.URL{urls[2]}, []url.URL{urls[3]}) setupEmbedCfg(&tests[4].cfg, []url.URL{urls[4]}, []url.URL{urls[5], urls[6]}) setupEmbedCfg(&tests[5].cfg, []url.URL{urls[7], urls[8]}, []url.URL{urls[9]}) dnsURL, _ := url.Parse("http://whatever.test:12345") tests[6].cfg.ListenClientUrls = []url.URL{*dnsURL} tests[7].cfg.ListenPeerUrls = []url.URL{*dnsURL} dir := filepath.Join(t.TempDir(), "embed-etcd") for i, tt := range tests { tests[i].cfg.Dir = dir e, err := embed.StartEtcd(&tests[i].cfg) if e != nil { <-e.Server.ReadyNotify() // wait for e.Server to join the cluster } if tt.werr != "" { if err == nil || !strings.Contains(err.Error(), tt.werr) { t.Errorf("%d: expected error with %q, got %v", i, tt.werr, err) } if e != nil { e.Close() } continue } if err != nil { t.Errorf("%d: expected success, got error %v", i, err) continue } if len(e.Peers) != tt.wpeers { t.Errorf("%d: expected %d peers, got %d", i, tt.wpeers, len(e.Peers)) } if len(e.Clients) != tt.wclients { t.Errorf("%d: expected %d clients, got %d", i, tt.wclients, len(e.Clients)) } e.Close() select { case err := <-e.Err(): if err != nil { t.Errorf("#%d: unexpected error on close (%v)", i, err) } } } } func TestEmbedEtcdGracefulStopSecure(t *testing.T) { testEmbedEtcdGracefulStop(t, true) } func TestEmbedEtcdGracefulStopInsecure(t *testing.T) { testEmbedEtcdGracefulStop(t, false) } // testEmbedEtcdGracefulStop ensures embedded server stops // cutting existing transports. func testEmbedEtcdGracefulStop(t *testing.T, secure bool) { testutil.SkipTestIfShortMode(t, "Cannot start embedded cluster in --short tests") cfg := embed.NewConfig() if secure { cfg.ClientTLSInfo = testTLSInfo cfg.PeerTLSInfo = testTLSInfo } urls := newEmbedURLs(secure, 2) setupEmbedCfg(cfg, []url.URL{urls[0]}, []url.URL{urls[1]}) cfg.Dir = filepath.Join(t.TempDir(), "embed-etcd") e, err := embed.StartEtcd(cfg) require.NoError(t, err) <-e.Server.ReadyNotify() // wait for e.Server to join the cluster clientCfg := clientv3.Config{ Endpoints: []string{urls[0].String()}, } if secure { clientCfg.TLS, err = testTLSInfo.ClientConfig() require.NoError(t, err) } cli, err := integration.NewClient(t, clientCfg) require.NoError(t, err) defer cli.Close() // open watch connection cli.Watch(t.Context(), "foo") donec := make(chan struct{}) go func() { e.Close() close(donec) }() select { case <-donec: case <-time.After(2*time.Second + e.Server.Cfg.ReqTimeout()): t.Fatalf("took too long to close server") } err = <-e.Err() require.NoError(t, err) } func newEmbedURLs(secure bool, n int) (urls []url.URL) { scheme := "unix" if secure { scheme = "unixs" } for i := 0; i < n; i++ { u, _ := url.Parse(fmt.Sprintf("%s://localhost:%d%06d", scheme, os.Getpid(), i)) urls = append(urls, *u) } return urls } func setupEmbedCfg(cfg *embed.Config, curls []url.URL, purls []url.URL) { cfg.Logger = "zap" cfg.LogOutputs = []string{"/dev/null"} cfg.ClusterState = "new" cfg.ListenClientUrls, cfg.AdvertiseClientUrls = curls, curls cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = purls, purls cfg.InitialCluster = "" for i := range purls { cfg.InitialCluster += ",default=" + purls[i].String() } cfg.InitialCluster = cfg.InitialCluster[1:] } func TestEmbedEtcdAutoCompactionRetentionRetained(t *testing.T) { cfg := embed.NewConfig() urls := newEmbedURLs(false, 2) setupEmbedCfg(cfg, []url.URL{urls[0]}, []url.URL{urls[1]}) cfg.Dir = filepath.Join(t.TempDir(), "embed-etcd") cfg.AutoCompactionRetention = "2" e, err := embed.StartEtcd(cfg) require.NoError(t, err) autoCompactionRetention := e.Server.Cfg.AutoCompactionRetention durationToCompare, _ := time.ParseDuration("2h0m0s") assert.Equal(t, durationToCompare, autoCompactionRetention) e.Close() } func TestEmbedEtcdStopDuringBootstrapping(t *testing.T) { integration.BeforeTest(t, integration.WithFailpoint("beforePublishing", `sleep("2s")`)) done := make(chan struct{}) go func() { defer close(done) cfg := embed.NewConfig() urls := newEmbedURLs(false, 2) setupEmbedCfg(cfg, []url.URL{urls[0]}, []url.URL{urls[1]}) cfg.Dir = filepath.Join(t.TempDir(), "embed-etcd") e, err := embed.StartEtcd(cfg) if err != nil { t.Errorf("Failed to start etcd, got error %v", err) } defer e.Close() go func() { time.Sleep(time.Second) e.Server.Stop() t.Log("Stopped server during bootstrapping") }() select { case <-e.Server.ReadyNotify(): t.Log("Server is ready!") case <-e.Server.StopNotify(): t.Log("Server is stopped") case <-time.After(20 * time.Second): e.Server.Stop() // trigger a shutdown t.Error("Server took too long to start!") } }() select { case <-done: case <-time.After(10 * time.Second): t.Error("timeout in bootstrapping etcd") } } ================================================ FILE: tests/integration/fixtures-expired/README ================================================ To generate bad certs 1. Manually set system time back to past 2. Run ./gencerts.sh ================================================ FILE: tests/integration/fixtures-expired/ca-csr.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "ca", "ca": { "expiry": "87600h" } } ================================================ FILE: tests/integration/fixtures-expired/ca.crt ================================================ -----BEGIN CERTIFICATE----- MIID0jCCArqgAwIBAgIUEKEIOO1O97Bz4car+7SHDxT5tB4wDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0wMDA0MTMxODU1MDBaFw0xMDA0MTExODU1 MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQDFeNJ9r2TFcJp9UHS42QN2NN1A96LQXxn/BirHzXdeTk6YEe0eloA91SJT BAae7aGdPMkpMyAAXheGPGHAbSde5dONYx2QE4nqWRl79v6kDbX6EmqwpzTOGD/T UfLXe65g6w6kaXcNZWiMdqfkUImke/WWM1qunsCKoGOXF8Jg1DLy7NSjqT2Kg1UP evJ5GOrWmIj5rEnEvW0ohR7mKV23xl5okVjrlzCi+arWDdl5RzE0I9x7vKNE0TKX NNHG9hMSJQ/ipXXXyMcahqGZXtkGvOpwpO3lpsGjo3WIUZMQW2FA3xR0nBC6Lt+0 d+7IXOy/LbzXpkcL8Ws5BZuLDSKLAgMBAAGjZjBkMA4GA1UdDwEB/wQEAwIBBjAS BgNVHRMBAf8ECDAGAQH/AgECMB0GA1UdDgQWBBT5Z7kwwTNntO1UsuMdUv/Yq3Ad cDAfBgNVHSMEGDAWgBT5Z7kwwTNntO1UsuMdUv/Yq3AdcDANBgkqhkiG9w0BAQsF AAOCAQEAkFF/fIVlcgj1uRL36wP4DgkOMCpU5+vwlDdHihDzRJHZqik+3+oNz7DD pRIURHMeeF+Wk5/GRQ/oGzKYotNLLzqCOggnLCxET6Hkb07vfve91HmYVOYix5pU GPW8+M3XyFTL3+2BnPpqPpJWpJ28g+N3eQjAG8rIbjXESdxrpJFKY22nMbtyS1rH dyzf3OO4S7LZiRQx0nuD9SZtX2vj5DyN8Am/zieSYm+GCtJsvIiDoB+Uhndnxxt0 FA0/89vGJ1gCo+Z6clzqBIbesRUBnLvPbUdpxhFAtjUKZhQv05IrE81/GP7F7kEr oODS2+D5WC6mKDO4v2k736OTw6HwOQ== -----END CERTIFICATE----- ================================================ FILE: tests/integration/fixtures-expired/gencert.json ================================================ { "signing": { "default": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "1h" } } } ================================================ FILE: tests/integration/fixtures-expired/gencerts.sh ================================================ #!/bin/bash set -euo pipefail if ! [[ "$0" =~ "./gencerts.sh" ]]; then echo "must be run from 'fixtures'" exit 255 fi if ! command -v cfssl; then echo "cfssl is not installed" echo 'use: bash -c "cd ../../../tools/mod; go install github.com/cloudflare/cfssl/cmd/cfssl"' exit 255 fi if ! command -v cfssljson; then echo "cfssljson is not installed" echo 'use: bash -c "cd ../../../tools/mod; go install github.com/cloudflare/cfssl/cmd/cfssljson"' exit 255 fi cfssl gencert --initca=true ./ca-csr.json | cfssljson --bare ./ca mv ca.pem ca.crt if command -v openssl >/dev/null; then openssl x509 -in ca.crt -noout -text fi # generate DNS: localhost, IP: 127.0.0.1, CN: example.com certificates cfssl gencert \ --ca ./ca.crt \ --ca-key ./ca-key.pem \ --config ./gencert.json \ ./server-ca-csr.json | cfssljson --bare ./server mv server.pem server.crt mv server-key.pem server.key.insecure # generate IP: 127.0.0.1, CN: example.com certificates cfssl gencert \ --ca ./ca.crt \ --ca-key ./ca-key.pem \ --config ./gencert.json \ ./server-ca-csr-ip.json | cfssljson --bare ./server-ip mv server-ip.pem server-ip.crt mv server-ip-key.pem server-ip.key.insecure if command -v openssl >/dev/null; then openssl x509 -in ./server.crt -text -noout openssl x509 -in ./server-ip.crt -text -noout fi rm -f *.csr *.pem *.stderr *.txt ================================================ FILE: tests/integration/fixtures-expired/server-ca-csr-ip.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "example.com", "hosts": [ "127.0.0.1" ] } ================================================ FILE: tests/integration/fixtures-expired/server-ca-csr.json ================================================ { "key": { "algo": "rsa", "size": 2048 }, "names": [ { "O": "etcd", "OU": "etcd Security", "L": "San Francisco", "ST": "California", "C": "USA" } ], "CN": "example.com", "hosts": [ "127.0.0.1", "localhost" ] } ================================================ FILE: tests/integration/fixtures-expired/server-ip.crt ================================================ -----BEGIN CERTIFICATE----- MIIEBzCCAu+gAwIBAgIUOc4vrxQ6OeHoGslhL7daP1Ye8ZYwDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0wMDA0MTMxODU1MDBaFw0wMDA0MTMxOTU1 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQDLUZuwXwiky2VvujM0EOP9LL85sx1dKLc/16hOl6qYRPOg PH7zsmXMVndsD2Fi9NDbhV9rVHfVNkCyZO/D81u52UEyr2uSFHOfIqLkFGKvcxhO FtOLA7wTjzHiYO1pFgqYBWzSfIyreYYo13tCYxHUlhn3ibqvCz9fimGsQmswhUiP yaC4C8iBICWNd4vrXHhtKb5pHHzUDFHkOxKF6VS9f7InKBy2yTr8ekgoEYyE3gtp ncoVbVlwxehChbZThFi0xsQc/kG/eoyGznKo9RUlUW+h3SEJR3bYizYP76ZwWXus nP5vgLmZ0wIi/689uTQbEAK438rK3xTSziPv6B51AgMBAAGjgZEwgY4wDgYDVR0P AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB Af8EAjAAMB0GA1UdDgQWBBR8bgC5SVrIrOAkjED8FB6OThk0IjAfBgNVHSMEGDAW gBT5Z7kwwTNntO1UsuMdUv/Yq3AdcDAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3 DQEBCwUAA4IBAQDEV4Ec8TpDXvFTJYXrpME5KnKvtq1fEv100jc88cmlGb/rygge MtisA1rYSaSEPMF0j7HoMtTwP90yrJCBTr7/vziAXCZU2H6bg24exRzqtMDpDhXg mvqkqvMVFem8ANIF3a+qXPY/pzjh4xrPuOw10TfG0bE576lAY/KbnY3UvXo6QL54 AMyimFhq8e9dJ7JnO3eaYmJv6oSjKjqNYSU+01UfxEJGNbx1IELMDlnVKX0Zmn9p YbUS3nrowKoVXpuca9KzS1pINgqVsztF5XJxzqlcDwERR/QcTKwUgQ0y0BBRqiGg WdtbyamFufvF8GPsNJ0KRHXSIRRXF7hbgiXd -----END CERTIFICATE----- ================================================ FILE: tests/integration/fixtures-expired/server-ip.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEogIBAAKCAQEAy1GbsF8IpMtlb7ozNBDj/Sy/ObMdXSi3P9eoTpeqmETzoDx+ 87JlzFZ3bA9hYvTQ24Vfa1R31TZAsmTvw/NbudlBMq9rkhRznyKi5BRir3MYThbT iwO8E48x4mDtaRYKmAVs0nyMq3mGKNd7QmMR1JYZ94m6rws/X4phrEJrMIVIj8mg uAvIgSAljXeL61x4bSm+aRx81AxR5DsShelUvX+yJygctsk6/HpIKBGMhN4LaZ3K FW1ZcMXoQoW2U4RYtMbEHP5Bv3qMhs5yqPUVJVFvod0hCUd22Is2D++mcFl7rJz+ b4C5mdMCIv+vPbk0GxACuN/Kyt8U0s4j7+gedQIDAQABAoIBAAjHl1+AWxEyr0ip 07g12oJ+QiutrmDtdyxMlbn/FqDIqXSL6DeBxp+SREnoSB5L0BEKq1opJZuRYi3R 6gCeK6HU3dngdVazh2KhzkLnFnPZFn2Ywr3IBYEat968rMPS7dYutcpJEpH9B2wQ EgSF3qk9ahWkXulcJPptMVaM77ACnZk6yYsPDPqjX/zsCXVga59QL0x1n2ai50er W7kthCj69zZP6crbnjyCUDjNdpDio7xurvvxs0k1KWmcN9QjdOyFXDFgTUnphIFX pEyVhu+LmLzFKc1WVQ4sIAt6ot9kpWt+cdaBVIWl2yCmqF4nbJ38DDG6wLXaZQd1 DgEL0YECgYEAzt7QukPfjgw5CKZguQVVn0LGYdHw47qHiusjABzYH4mokMHqR/r5 LIIRQ4JjB/vpxavj6B0e73tcfwbSzLSwsRI9/6Z27UVpXnpU5LY7+46d+ZXsQorE 8jeUX6ZQi65ujpFFKkftKlmq67XJtmSh2T+3dMqRXmFWVZThllBJcGUCgYEA+5rd gvZhaj9Rng1CwK3FoI/mp0BtSL+TE8/JbV0yA5X6NhXlts/ysafFZsj9RkR1NhXL ql8Bl9RxrV6mTIz6/76NC39ZUQUe5FZGv64rqjoFwOnv6ap1/8ntDFy29DgZ5Dqn flAtbbEyVG+VCwwhDgUT+FTNNS1eg18GStr6LNECgYASgo1anUgbhax0wa5V38xR e8AUcJyFQ+Ns4q03DV2pNMAIc9Fqr2IsQVcaG0iRJlE8hqzV0AU8mGUmWI30Exbc QS2a+mIZyOQst/VwoX2sfI5WDrwdGB2XLrHv/Qmn9euehhESP21RJMTOYm2yDD8P GUxo/tcTAtKexbuJn5VyoQKBgB7OH0DhmZvAlOWdCgc9P20hMURZBwhZLFDIqAjT 2EPIIRJuK+nuG/DUcb7b7OalixRMJtt9Nly4jhKD/ChzOmgFlI9L0EuzLM0YIyFk 2cPFxt6Pxef+DuR6fKN+1oegNstSwx8cAfPkNh1QbBcmLQXiaUeGWnmgTGoZQFP5 65eBAoGAfV98Mwka+VJ3hYNPL2ZHUXHnXw9Hnf5NnaGfgz7/Ucw3H88HsrIDIZgO NKSM3NVRIrweAx8/gDIrGqjXkvrwuCqXXYeS23gRteigUpoQrtGjBxIwtalT8K2O jI4vqz8SsNALtR8nehmBPzTj+t+rF5b1cMfyreHccoAa+0TbPac= -----END RSA PRIVATE KEY----- ================================================ FILE: tests/integration/fixtures-expired/server.crt ================================================ -----BEGIN CERTIFICATE----- MIIEEjCCAvqgAwIBAgIULscJmimDEFNvw3oQ5paqHc+V/w4wDQYJKoZIhvcNAQEL BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0wMDA0MTMxODU1MDBaFw0wMDA0MTMxOTU1 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA A4IBDwAwggEKAoIBAQC090lhTNQRzaNBROiSAt0Tv2KTEqGtus7bNJjAVEpRRz6X htuXfFf7jJoDqhz60/Xtyky+EoP+EH1sh6TwafZwl3PrCMDNL+Bvmm0BUHU00Tef afmFf1K7FpL/eYVQcavpNJ35sQRYjsw8JWQ0+Ge0qsyfEUSIqfzdNQgniBrITM7+ t7NWmCjm3awd0PWMFk10WnEudoWV8fb2TBSTE9gdx0wsafg2Xu7z9WUFCZWPtFRV 2qB91G8fGtZx32pxmi6WgKHIYBWceZ1IGaVsH/c+UWsxJvxoKQnRSuo8vfxHybLM wULDqFlNE7Z29KIDEmSwUXGeWwGUrud/VgQo0FBPAgMBAAGjgZwwgZkwDgYDVR0P AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB Af8EAjAAMB0GA1UdDgQWBBQqIZCfMH71R7RVCzGrap10FGf5ZjAfBgNVHSMEGDAW gBT5Z7kwwTNntO1UsuMdUv/Yq3AdcDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A AAEwDQYJKoZIhvcNAQELBQADggEBAKzFUSqROx2ERE06y6ZAjB7PeZuRFGIAZiSY QGG+4vQLAxNMqQN4Td7SXrOsFq/uffrqxlOPCVS94Kd2XRmsT6gCsMWXSr8zBQyA c//hWHU9jG9YSZ4s2IBqLTugsMUGxuq0ClEXzkrqzeJssHfJCEF+Peg39v/Bk/Hr iA/YDoQp7hgSdvwO8XH21HBab9nsYHvOIFivWdS4/w+au6QplwDC9a0R67tkNDnQ gxWvhA8SJ2HjumvZ0eOSZMYhOhXca52LwYBEM66600cKKvOcVAtIWAfx3MI7FY03 sCUu4iGbo61ceomM22hmZtPUEBzpVuwnaujmD7MMvr322Wu4QJY= -----END CERTIFICATE----- ================================================ FILE: tests/integration/fixtures-expired/server.key.insecure ================================================ -----BEGIN RSA PRIVATE KEY----- MIIEogIBAAKCAQEAtPdJYUzUEc2jQUTokgLdE79ikxKhrbrO2zSYwFRKUUc+l4bb l3xX+4yaA6oc+tP17cpMvhKD/hB9bIek8Gn2cJdz6wjAzS/gb5ptAVB1NNE3n2n5 hX9SuxaS/3mFUHGr6TSd+bEEWI7MPCVkNPhntKrMnxFEiKn83TUIJ4gayEzO/rez Vpgo5t2sHdD1jBZNdFpxLnaFlfH29kwUkxPYHcdMLGn4Nl7u8/VlBQmVj7RUVdqg fdRvHxrWcd9qcZouloChyGAVnHmdSBmlbB/3PlFrMSb8aCkJ0UrqPL38R8myzMFC w6hZTRO2dvSiAxJksFFxnlsBlK7nf1YEKNBQTwIDAQABAoIBAFny3E93v6VFwFLN 7Ie+0qJhK58M0L4or273msFmZDY4Il1w069dR+Ipxdfyc0sdlgzm0/RaAa+EBMOw PISfNrZKIXz+sc6LcJQofuv7UPa601nyc+suGTITC2fewCv3BEr7M1aL7SwTdmKi 90b4/ZsolmKuU5FWZPCSzoXPufg6lyyw/9AvD8gnjMPjyoPrG1AbLruogdD/gz62 SZEHAaY0jHdQZbobo6iOuGoOKfYHr39B2xVoJAjR+1PVxzuAnZYbpUhemzHMTW03 ikei3l6B/Qu/NdhW6CaR2dMetTVnbHwomm0cx2N4SNFFrKvnc1KQJR/b4RTxmVvd HQlVHqECgYEAzb9vxslb6vpbpGUu787X/SFGpHi5tSAeqoL2/TphrKMBh0EJIUSK tq2B021rzLjvVKdN2hSisa5EvBkNQnwkeF7Nvr0adPB13Xo26G45RmogUnH4UFrY RNTosMcab6VopFmcdFLxfNHx+hVKf0+l5mNsSN0MtxyM/9q92JF+fFECgYEA4SpY AcrldyZcmXF5ngFp24SHX0sTgjrbuq0VGk6HshVOYY/XFymlpoZLEpaRC5kKoZ7W YLVSE1BlJ79kmqPQ6Y2oB+TN2PALVMl8K1fwJxW/OfHEbq2tDylF5/jUS3noxU2w J2FjV4wyHoKCrVFGQjI+CWQBmvXaGsbEwQO6+p8CgYBfUz7afxiTOgOTmz2v5cm0 geJU+YoxHPyYS61bjd0LO0rN+5fbTgJmuOTZrGyxoU1hj1JGpCDs6az26TR3hUTw cBwrLzo+y9oQDzu5XLg0o57uE9fUgwKIgYx9uwHIkH53Bv2x92vjRPIzyAGIEsLu h0n4SFJH1HaPZC1pVZ+gwQKBgCCWbUhNIiq9bZdzmeNpVvXDV4hOKFOnyxdYZ354 MSFv/fkWxU1/5I6WTxUwn2trSeOcRnCWrXtIHmvDQn8zCFBVBSWnUrd7/lfWFVd8 kbBGcHelawWNs0dHdOue0rLdwPeVR9JbQPJxwusxflIxOhboiJv5UlYoENnhPKam sJAHAoGATyztNEOTtPk7Le4ZYzC2bR58vAPeiu6mzN37Vf4PGWJyLSzAqLnDyQ8c REFVsgawue5hKzHz+JBsc91CURWHlMcMQ1sjmMx0MGpNyjVlZIoHMxnIo/CGwGvI TSlyv2ErcTpiwC1gAw1G5dAp3fWASmiDxNX5UXnpVA2SMiCScY8= -----END RSA PRIVATE KEY----- ================================================ FILE: tests/integration/lazy_cluster.go ================================================ // Copyright 2020 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "log" "net/http" "sync" "time" "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/tests/v3/framework/integration" ) // Infrastructure to provision a single shared cluster for tests - only // when its needed. // // See ./tests/integration/clientv3/examples/main_test.go for canonical usage. // Please notice that the shared (LazyCluster's) state is preserved between // testcases, so left-over state might has cross-testcase effects. // Prefer dedicated clusters for substantial test-cases. type LazyCluster interface { // EndpointsHTTP - exposes connection points for http endpoints. // Calls to this method might initialize the cluster. EndpointsHTTP() []string // EndpointsGRPC - exposes connection points for client v3. // Calls to this method might initialize the cluster. EndpointsGRPC() []string // Cluster - calls to this method might initialize the cluster. Cluster() *integration.Cluster // Transport - call to this method might initialize the cluster. Transport() *http.Transport Terminate() TB() testutil.TB } type lazyCluster struct { cfg integration.ClusterConfig cluster *integration.Cluster transport *http.Transport once sync.Once tb testutil.TB closer func() } // NewLazyCluster returns a new test cluster handler that gets created on the // first call to GetEndpoints() or GetTransport() func NewLazyCluster() LazyCluster { return NewLazyClusterWithConfig(integration.ClusterConfig{Size: 1}) } // NewLazyClusterWithConfig returns a new test cluster handler that gets created // on the first call to GetEndpoints() or GetTransport() func NewLazyClusterWithConfig(cfg integration.ClusterConfig) LazyCluster { tb, closer := testutil.NewTestingTBProthesis("lazy_cluster") return &lazyCluster{cfg: cfg, tb: tb, closer: closer} } func (lc *lazyCluster) mustLazyInit() { lc.once.Do(func() { lc.tb.Logf("LazyIniting ...") var err error lc.transport, err = transport.NewTransport(transport.TLSInfo{}, time.Second) if err != nil { log.Fatal(err) } lc.cluster = integration.NewCluster(lc.tb, &lc.cfg) lc.tb.Logf("LazyIniting [Done]") }) } func (lc *lazyCluster) Terminate() { if lc != nil && lc.tb != nil { lc.tb.Logf("Terminating...") } if lc != nil && lc.cluster != nil { lc.cluster.Terminate(nil) lc.cluster = nil } if lc.closer != nil { lc.tb.Logf("Closer...") lc.closer() } } func (lc *lazyCluster) EndpointsHTTP() []string { return []string{lc.Cluster().Members[0].URL()} } func (lc *lazyCluster) EndpointsGRPC() []string { return lc.Cluster().Client(0).Endpoints() } func (lc *lazyCluster) Cluster() *integration.Cluster { lc.mustLazyInit() return lc.cluster } func (lc *lazyCluster) Transport() *http.Transport { lc.mustLazyInit() return lc.transport } func (lc *lazyCluster) TB() testutil.TB { return lc.tb } ================================================ FILE: tests/integration/main_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "os" "testing" "google.golang.org/grpc/grpclog" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func TestMain(m *testing.M) { testutil.MustTestMainWithLeakDetection(m) grpclog.SetLoggerV2(grpclog.NewLoggerV2(os.Stderr, os.Stderr, os.Stderr)) } ================================================ FILE: tests/integration/member_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestPauseMember(t *testing.T) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 5}) defer c.Terminate(t) for i := 0; i < 5; i++ { c.Members[i].Pause() membs := append([]*integration.Member{}, c.Members[:i]...) membs = append(membs, c.Members[i+1:]...) c.WaitMembersForLeader(t, membs) clusterMustProgress(t, membs) c.Members[i].Resume() } c.WaitMembersForLeader(t, c.Members) clusterMustProgress(t, c.Members) } func TestRestartMember(t *testing.T) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer c.Terminate(t) for i := 0; i < 3; i++ { c.Members[i].Stop(t) membs := append([]*integration.Member{}, c.Members[:i]...) membs = append(membs, c.Members[i+1:]...) c.WaitMembersForLeader(t, membs) clusterMustProgress(t, membs) err := c.Members[i].Restart(t) require.NoError(t, err) } c.WaitMembersForLeader(t, c.Members) clusterMustProgress(t, c.Members) } func TestLaunchDuplicateMemberShouldFail(t *testing.T) { integration.BeforeTest(t) size := 3 c := integration.NewCluster(t, &integration.ClusterConfig{Size: size}) m := c.Members[0].Clone(t) m.DataDir = t.TempDir() defer c.Terminate(t) if err := m.Launch(); err == nil { t.Errorf("unexpect successful launch") } else { t.Logf("launch failed as expected: %v", err) assert.Contains(t, err.Error(), "has already been bootstrapped") } } func TestSnapshotAndRestartMember(t *testing.T) { integration.BeforeTest(t) m := integration.MustNewMember(t, integration.MemberConfig{Name: "snapAndRestartTest", UseBridge: true}) m.SnapshotCount = 100 m.Launch() defer m.Terminate(t) defer m.Client.Close() m.WaitOK(t) var err error for i := 0; i < 120; i++ { ctx, cancel := context.WithTimeout(t.Context(), integration.RequestTimeout) key := fmt.Sprintf("foo%d", i) _, err = m.Client.Put(ctx, "/"+key, "bar") require.NoErrorf(t, err, "#%d: create on %s error", i, m.URL()) cancel() } m.Stop(t) m.Restart(t) m.WaitOK(t) for i := 0; i < 120; i++ { ctx, cancel := context.WithTimeout(t.Context(), integration.RequestTimeout) key := fmt.Sprintf("foo%d", i) resp, err := m.Client.Get(ctx, "/"+key) require.NoErrorf(t, err, "#%d: get on %s error", i, m.URL()) cancel() if len(resp.Kvs) != 1 || string(resp.Kvs[0].Value) != "bar" { t.Errorf("#%d: got = %v, want %v", i, resp.Kvs[0], "bar") } } } func TestRemoveMember(t *testing.T) { integration.BeforeTest(t) c := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true, BackendBatchInterval: 1000 * time.Second}) defer c.Terminate(t) // membership changes additionally require cluster to be stable for etcdserver.HealthInterval time.Sleep(etcdserver.HealthInterval) err := c.RemoveMember(t, c.Client(2), uint64(c.Members[0].ID())) require.NoError(t, err) checkMemberCount(t, c.Members[0], 2) checkMemberCount(t, c.Members[1], 2) } // TestRemoveMemberAndWALReplay ensures that etcd can properly handle // member removal followed by restart with WAL replay, ensuring no panics // occur when replaying already-applied removal operations. func TestRemoveMemberAndWALReplay(t *testing.T) { integration.BeforeTest(t) // Create a cluster with 3 member and a low snapshot count c := integration.NewCluster(t, &integration.ClusterConfig{ Size: 3, SnapshotCount: 10, UseBridge: true, DisableStrictReconfigCheck: true, }) defer c.Terminate(t) // Add some k/v to trigger snapshot for i := 0; i < 15; i++ { ctx, cancel := context.WithTimeout(t.Context(), integration.RequestTimeout) _, err := c.Members[0].Client.Put(ctx, fmt.Sprintf("k%d", i), fmt.Sprintf("v%d", i)) cancel() require.NoErrorf(t, err, "failed to put key-value") } // Record the ID of the member we'll remove memberToRemoveID := uint64(c.Members[2].Server.MemberID()) // Remove one member from the cluster err := c.RemoveMember(t, c.Members[0].Client, memberToRemoveID) require.NoErrorf(t, err, "failed to remove member") // Stop the remaining members c.Members[0].Stop(t) c.Members[1].Stop(t) // Restart one member - this would previously panic when loading // WAL entries that try to remove an already removed member err = c.Members[0].Restart(t) require.NoErrorf(t, err, "failed to restart member after removal") } func checkMemberCount(t *testing.T, m *integration.Member, expectedMemberCount int) { be := schema.NewMembershipBackend(m.Logger, m.Server.Backend()) membersFromBackend, _ := be.MustReadMembersFromBackend() if len(membersFromBackend) != expectedMemberCount { t.Errorf("Expect member count read from backend=%d, got %d", expectedMemberCount, len(membersFromBackend)) } membersResp, err := m.Client.MemberList(t.Context()) require.NoError(t, err) if len(membersResp.Members) != expectedMemberCount { t.Errorf("Expect len(MemberList)=%d, got %d", expectedMemberCount, len(membersResp.Members)) } } ================================================ FILE: tests/integration/metrics_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "fmt" "net/http" "strconv" "testing" "time" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/storage" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestMetricDbSizeBoot checks that the db size metric is set on boot. func TestMetricDbSizeBoot(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) v, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes") require.NoError(t, err) require.NotEqualf(t, "0", v, "expected non-zero, got %q", v) } func TestMetricDbSizeDefrag(t *testing.T) { testMetricDbSizeDefrag(t, "etcd") } // testMetricDbSizeDefrag checks that the db size metric is set after defrag. func testMetricDbSizeDefrag(t *testing.T, name string) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.Client(0)).KV mc := integration.ToGRPC(clus.Client(0)).Maintenance // expand the db size numPuts := 25 // large enough to write more than 1 page putreq := &pb.PutRequest{Key: []byte("k"), Value: make([]byte, 4096)} for i := 0; i < numPuts; i++ { time.Sleep(10 * time.Millisecond) // to execute multiple backend txn _, err := kvc.Put(t.Context(), putreq) require.NoError(t, err) } // wait for backend txn sync time.Sleep(500 * time.Millisecond) expected := numPuts * len(putreq.Value) beforeDefrag, err := clus.Members[0].Metric(name + "_mvcc_db_total_size_in_bytes") require.NoError(t, err) bv, err := strconv.Atoi(beforeDefrag) require.NoError(t, err) require.GreaterOrEqualf(t, bv, expected, "expected db size greater than %d, got %d", expected, bv) beforeDefragInUse, err := clus.Members[0].Metric("etcd_mvcc_db_total_size_in_use_in_bytes") require.NoError(t, err) biu, err := strconv.Atoi(beforeDefragInUse) require.NoError(t, err) require.GreaterOrEqualf(t, biu, expected, "expected db size in use is greater than %d, got %d", expected, biu) // clear out historical keys, in use bytes should free pages creq := &pb.CompactionRequest{Revision: int64(numPuts), Physical: true} _, kerr := kvc.Compact(t.Context(), creq) require.NoError(t, kerr) validateAfterCompactionInUse := func() error { // Put to move PendingPages to FreePages _, verr := kvc.Put(t.Context(), putreq) require.NoError(t, verr) time.Sleep(500 * time.Millisecond) afterCompactionInUse, verr := clus.Members[0].Metric("etcd_mvcc_db_total_size_in_use_in_bytes") require.NoError(t, verr) aciu, verr := strconv.Atoi(afterCompactionInUse) require.NoError(t, verr) if biu <= aciu { return fmt.Errorf("expected less than %d, got %d after compaction", biu, aciu) } return nil } // backend rollbacks read transaction asynchronously (PR #10523), // which causes the result to be flaky. Retry 3 times. maxRetry, retry := 3, 0 for { err = validateAfterCompactionInUse() if err == nil { break } retry++ require.Lessf(t, retry, maxRetry, "%v", err.Error()) } // defrag should give freed space back to fs mc.Defragment(t.Context(), &pb.DefragmentRequest{}) afterDefrag, err := clus.Members[0].Metric(name + "_mvcc_db_total_size_in_bytes") require.NoError(t, err) av, err := strconv.Atoi(afterDefrag) require.NoError(t, err) require.Greaterf(t, bv, av, "expected less than %d, got %d after defrag", bv, av) afterDefragInUse, err := clus.Members[0].Metric("etcd_mvcc_db_total_size_in_use_in_bytes") require.NoError(t, err) adiu, err := strconv.Atoi(afterDefragInUse) require.NoError(t, err) require.LessOrEqualf(t, adiu, av, "db size in use (%d) is expected less than db size (%d) after defrag", adiu, av) } func TestMetricQuotaBackendBytes(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) qs, err := clus.Members[0].Metric("etcd_server_quota_backend_bytes") require.NoError(t, err) qv, err := strconv.ParseFloat(qs, 64) require.NoError(t, err) require.Equalf(t, storage.DefaultQuotaBytes, int64(qv), "expected %d, got %f", storage.DefaultQuotaBytes, qv) } func TestMetricsHealth(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) tr, err := transport.NewTransport(transport.TLSInfo{}, 5*time.Second) require.NoError(t, err) u := clus.Members[0].ClientURLs[0] u.Path = "/health" resp, err := tr.RoundTrip(&http.Request{ Header: make(http.Header), Method: http.MethodGet, URL: &u, }) resp.Body.Close() require.NoError(t, err) hv, err := clus.Members[0].Metric("etcd_server_health_failures") require.NoError(t, err) require.Equalf(t, "0", hv, "expected '0' from etcd_server_health_failures, got %q", hv) } func TestMetricsRangeDurationSeconds(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) client := clus.RandClient() keys := []string{ "my-namespace/foobar", "my-namespace/foobar1", "namespace/foobar1", } for _, key := range keys { _, err := client.Put(t.Context(), key, "data") require.NoError(t, err) } _, err := client.Get(t.Context(), "", clientv3.WithFromKey()) require.NoError(t, err) rangeDurationSeconds, err := clus.Members[0].Metric("etcd_server_range_duration_seconds") require.NoError(t, err) require.NotEmptyf(t, rangeDurationSeconds, "expected a number from etcd_server_range_duration_seconds") rangeDuration, err := strconv.ParseFloat(rangeDurationSeconds, 64) require.NoErrorf(t, err, "failed to parse duration: %s", rangeDurationSeconds) maxRangeDuration := 600.0 require.GreaterOrEqualf(t, rangeDuration, 0.0, "expected etcd_server_range_duration_seconds to be between 0 and %f, got %f", maxRangeDuration, rangeDuration) require.LessOrEqualf(t, rangeDuration, maxRangeDuration, "expected etcd_server_range_duration_seconds to be between 0 and %f, got %f", maxRangeDuration, rangeDuration) } ================================================ FILE: tests/integration/network_partition_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "fmt" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestNetworkPartition5MembersLeaderInMinority(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 5}) defer clus.Terminate(t) leadIndex := clus.WaitLeader(t) // minority: leader, follower / majority: follower, follower, follower minority := []int{leadIndex, (leadIndex + 1) % 5} majority := []int{(leadIndex + 2) % 5, (leadIndex + 3) % 5, (leadIndex + 4) % 5} minorityMembers := getMembersByIndexSlice(clus, minority) majorityMembers := getMembersByIndexSlice(clus, majority) // network partition (bi-directional) injectPartition(t, minorityMembers, majorityMembers) // minority leader must be lost clus.WaitMembersNoLeader(minorityMembers) // wait extra election timeout time.Sleep(2 * majorityMembers[0].ElectionTimeout()) // new leader must be from majority clus.WaitMembersForLeader(t, majorityMembers) // recover network partition (bi-directional) recoverPartition(t, minorityMembers, majorityMembers) // write to majority first clusterMustProgress(t, append(majorityMembers, minorityMembers...)) } func TestNetworkPartition5MembersLeaderInMajority(t *testing.T) { // retry up to 3 times, in case of leader election on majority partition due to slow hardware var err error for i := 0; i < 3; i++ { if err = testNetworkPartition5MembersLeaderInMajority(t); err == nil { break } t.Logf("[%d] got %v", i, err) } require.NoErrorf(t, err, "failed after 3 tries (%v)", err) } func testNetworkPartition5MembersLeaderInMajority(t *testing.T) error { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 5}) defer clus.Terminate(t) leadIndex := clus.WaitLeader(t) // majority: leader, follower, follower / minority: follower, follower majority := []int{leadIndex, (leadIndex + 1) % 5, (leadIndex + 2) % 5} minority := []int{(leadIndex + 3) % 5, (leadIndex + 4) % 5} majorityMembers := getMembersByIndexSlice(clus, majority) minorityMembers := getMembersByIndexSlice(clus, minority) // network partition (bi-directional) injectPartition(t, majorityMembers, minorityMembers) // minority leader must be lost clus.WaitMembersNoLeader(minorityMembers) // wait extra election timeout time.Sleep(2 * majorityMembers[0].ElectionTimeout()) // leader must be hold in majority leadIndex2 := clus.WaitMembersForLeader(t, majorityMembers) leadID, leadID2 := clus.Members[leadIndex].Server.MemberID(), majorityMembers[leadIndex2].Server.MemberID() if leadID != leadID2 { return fmt.Errorf("unexpected leader change from %s, got %s", leadID, leadID2) } // recover network partition (bi-directional) recoverPartition(t, majorityMembers, minorityMembers) // write to majority first clusterMustProgress(t, append(majorityMembers, minorityMembers...)) return nil } func TestNetworkPartition4Members(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 4}) defer clus.Terminate(t) leadIndex := clus.WaitLeader(t) // groupA: leader, follower / groupB: follower, follower groupA := []int{leadIndex, (leadIndex + 1) % 4} groupB := []int{(leadIndex + 2) % 4, (leadIndex + 3) % 4} leaderPartition := getMembersByIndexSlice(clus, groupA) followerPartition := getMembersByIndexSlice(clus, groupB) // network partition (bi-directional) injectPartition(t, leaderPartition, followerPartition) // no group has quorum, so leader must be lost in all members clus.WaitNoLeader() // recover network partition (bi-directional) recoverPartition(t, leaderPartition, followerPartition) // need to wait since it recovered with no leader clus.WaitLeader(t) clusterMustProgress(t, clus.Members) } func getMembersByIndexSlice(clus *integration.Cluster, idxs []int) []*integration.Member { ms := make([]*integration.Member, len(idxs)) for i, idx := range idxs { ms[i] = clus.Members[idx] } return ms } func injectPartition(t *testing.T, src, others []*integration.Member) { for _, m := range src { m.InjectPartition(t, others...) } } func recoverPartition(t *testing.T, src, others []*integration.Member) { for _, m := range src { m.RecoverPartition(t, others...) } } ================================================ FILE: tests/integration/proxy/grpcproxy/cluster_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "net" "os" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" "google.golang.org/grpc" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/naming/endpoints" "go.etcd.io/etcd/server/v3/proxy/grpcproxy" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestClusterProxyMemberList(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lg := zaptest.NewLogger(t) serverEps := []string{clus.Members[0].GRPCURL} prefix := "test-prefix" hostname, _ := os.Hostname() cts := newClusterProxyServer(lg, serverEps, prefix, t) defer cts.close(t) cfg := clientv3.Config{ Endpoints: []string{cts.caddr}, DialTimeout: 5 * time.Second, } client, err := integration.NewClient(t, cfg) require.NoErrorf(t, err, "err %v, want nil", err) defer client.Close() // wait some time for register-loop to write keys time.Sleep(200 * time.Millisecond) var mresp *clientv3.MemberListResponse mresp, err = client.Cluster.MemberList(t.Context()) require.NoErrorf(t, err, "err %v, want nil", err) require.Lenf(t, mresp.Members, 1, "len(mresp.Members) expected 1, got %d (%+v)", len(mresp.Members), mresp.Members) require.Lenf(t, mresp.Members[0].ClientURLs, 1, "len(mresp.Members[0].ClientURLs) expected 1, got %d (%+v)", len(mresp.Members[0].ClientURLs), mresp.Members[0].ClientURLs[0]) assert.Contains(t, mresp.Members, &pb.Member{Name: hostname, ClientURLs: []string{cts.caddr}}) // test proxy member add newMemberAddr := "127.0.0.2:6789" grpcproxy.Register(lg, cts.c, prefix, newMemberAddr, 7) // wait some time for proxy update members time.Sleep(200 * time.Millisecond) // check add member succ mresp, err = client.Cluster.MemberList(t.Context()) require.NoErrorf(t, err, "err %v, want nil", err) require.Lenf(t, mresp.Members, 2, "len(mresp.Members) expected 2, got %d (%+v)", len(mresp.Members), mresp.Members) assert.Contains(t, mresp.Members, &pb.Member{Name: hostname, ClientURLs: []string{newMemberAddr}}) // test proxy member delete deregisterMember(cts.c, prefix, newMemberAddr, t) // wait some time for proxy update members time.Sleep(200 * time.Millisecond) // check delete member succ mresp, err = client.Cluster.MemberList(t.Context()) require.NoErrorf(t, err, "err %v, want nil", err) require.Lenf(t, mresp.Members, 1, "len(mresp.Members) expected 1, got %d (%+v)", len(mresp.Members), mresp.Members) assert.Contains(t, mresp.Members, &pb.Member{Name: hostname, ClientURLs: []string{cts.caddr}}) } type clusterproxyTestServer struct { cp pb.ClusterServer c *clientv3.Client server *grpc.Server l net.Listener donec <-chan struct{} caddr string } func (cts *clusterproxyTestServer) close(t *testing.T) { cts.server.Stop() cts.l.Close() cts.c.Close() select { case <-cts.donec: return case <-time.After(5 * time.Second): t.Fatalf("register-loop took too long to return") } } func newClusterProxyServer(lg *zap.Logger, endpoints []string, prefix string, t *testing.T) *clusterproxyTestServer { cfg := clientv3.Config{ Endpoints: endpoints, DialTimeout: 5 * time.Second, } client, err := integration.NewClient(t, cfg) require.NoError(t, err) cts := &clusterproxyTestServer{ c: client, } cts.l, err = net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) var opts []grpc.ServerOption cts.server = grpc.NewServer(opts...) servec := make(chan struct{}) go func() { <-servec cts.server.Serve(cts.l) }() grpcproxy.Register(lg, client, prefix, cts.l.Addr().String(), 7) cts.cp, cts.donec = grpcproxy.NewClusterProxy(lg, client, cts.l.Addr().String(), prefix) cts.caddr = cts.l.Addr().String() pb.RegisterClusterServer(cts.server, cts.cp) close(servec) // wait some time for free port 0 to be resolved time.Sleep(500 * time.Millisecond) return cts } func deregisterMember(c *clientv3.Client, prefix, addr string, t *testing.T) { em, err := endpoints.NewManager(c, prefix) require.NoErrorf(t, err, "new endpoint manager failed, err") err = em.DeleteEndpoint(c.Ctx(), prefix+"/"+addr) require.NoErrorf(t, err, "delete endpoint failed, err") } ================================================ FILE: tests/integration/proxy/grpcproxy/kv_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "net" "testing" "time" "github.com/stretchr/testify/require" "google.golang.org/grpc" pb "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/proxy/grpcproxy" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestKVProxyRange(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kvts := newKVProxyServer([]string{clus.Members[0].GRPCURL}, t) defer kvts.close() // create a client and try to get key from proxy. cfg := clientv3.Config{ Endpoints: []string{kvts.l.Addr().String()}, DialTimeout: 5 * time.Second, } client, err := integration.NewClient(t, cfg) require.NoErrorf(t, err, "err = %v, want nil", err) _, err = client.Get(t.Context(), "foo") require.NoErrorf(t, err, "err = %v, want nil", err) client.Close() } type kvproxyTestServer struct { kp pb.KVServer c *clientv3.Client server *grpc.Server l net.Listener } func (kts *kvproxyTestServer) close() { kts.server.Stop() kts.l.Close() kts.c.Close() } func newKVProxyServer(endpoints []string, t *testing.T) *kvproxyTestServer { cfg := clientv3.Config{ Endpoints: endpoints, DialTimeout: 5 * time.Second, } client, err := integration.NewClient(t, cfg) require.NoError(t, err) kvp, _ := grpcproxy.NewKvProxy(client) kvts := &kvproxyTestServer{ kp: kvp, c: client, } var opts []grpc.ServerOption kvts.server = grpc.NewServer(opts...) pb.RegisterKVServer(kvts.server, kvts.kp) kvts.l, err = net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) go kvts.server.Serve(kvts.l) return kvts } ================================================ FILE: tests/integration/proxy/grpcproxy/register_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grpcproxy import ( "testing" "time" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/naming/endpoints" "go.etcd.io/etcd/server/v3/proxy/grpcproxy" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestRegister(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) paddr := clus.Members[0].GRPCURL testPrefix := "test-name" wa := mustCreateWatcher(t, cli, testPrefix) donec := grpcproxy.Register(zaptest.NewLogger(t), cli, testPrefix, paddr, 5) ups := <-wa require.Lenf(t, ups, 1, "len(ups) expected 1, got %d (%v)", len(ups), ups) require.Equalf(t, ups[0].Endpoint.Addr, paddr, "ups[0].Addr expected %q, got %q", paddr, ups[0].Endpoint.Addr) cli.Close() clus.TakeClient(0) select { case <-donec: case <-time.After(5 * time.Second): t.Fatal("donec 'register' did not return in time") } } func mustCreateWatcher(t *testing.T, c *clientv3.Client, prefix string) endpoints.WatchChannel { em, err := endpoints.NewManager(c, prefix) require.NoErrorf(t, err, "failed to create endpoints.Manager") wc, err := em.NewWatchChannel(c.Ctx()) require.NoErrorf(t, err, "failed to resolve %q", prefix) return wc } ================================================ FILE: tests/integration/revision_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration_test import ( "context" "errors" "fmt" "strings" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/status" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestRevisionMonotonicWithLeaderPartitions(t *testing.T) { testRevisionMonotonicWithFailures(t, 12*time.Second, func(clus *integration.Cluster) { for i := 0; i < 5; i++ { leader := clus.WaitLeader(t) time.Sleep(time.Second) clus.Members[leader].InjectPartition(t, clus.Members[(leader+1)%3], clus.Members[(leader+2)%3]) time.Sleep(time.Second) clus.Members[leader].RecoverPartition(t, clus.Members[(leader+1)%3], clus.Members[(leader+2)%3]) } }) } func TestRevisionMonotonicWithPartitions(t *testing.T) { testRevisionMonotonicWithFailures(t, 11*time.Second, func(clus *integration.Cluster) { for i := 0; i < 5; i++ { time.Sleep(time.Second) clus.Members[i%3].InjectPartition(t, clus.Members[(i+1)%3], clus.Members[(i+2)%3]) time.Sleep(time.Second) clus.Members[i%3].RecoverPartition(t, clus.Members[(i+1)%3], clus.Members[(i+2)%3]) } }) } func TestRevisionMonotonicWithLeaderRestarts(t *testing.T) { testRevisionMonotonicWithFailures(t, 11*time.Second, func(clus *integration.Cluster) { for i := 0; i < 5; i++ { leader := clus.WaitLeader(t) time.Sleep(time.Second) clus.Members[leader].Stop(t) time.Sleep(time.Second) clus.Members[leader].Restart(t) } }) } func TestRevisionMonotonicWithRestarts(t *testing.T) { testRevisionMonotonicWithFailures(t, 11*time.Second, func(clus *integration.Cluster) { for i := 0; i < 5; i++ { time.Sleep(time.Second) clus.Members[i%3].Stop(t) time.Sleep(time.Second) clus.Members[i%3].Restart(t) } }) } func testRevisionMonotonicWithFailures(t *testing.T, testDuration time.Duration, injectFailures func(clus *integration.Cluster)) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), testDuration) defer cancel() wg := sync.WaitGroup{} for i := 0; i < 10; i++ { wg.Add(1) go func() { defer wg.Done() putWorker(ctx, t, clus) }() } for i := 0; i < 10; i++ { wg.Add(1) go func() { defer wg.Done() getWorker(ctx, t, clus) //nolint:testifylint }() } injectFailures(clus) wg.Wait() kv := clus.Client(0) resp, err := kv.Get(t.Context(), "foo") require.NoError(t, err) t.Logf("Revision %d", resp.Header.Revision) } func putWorker(ctx context.Context, t *testing.T, clus *integration.Cluster) { for i := 0; ; i++ { kv := clus.Client(i % 3) _, err := kv.Put(ctx, "foo", fmt.Sprintf("%d", i)) if errors.Is(err, context.DeadlineExceeded) { return } assert.NoError(t, silenceConnectionErrors(err)) } } func getWorker(ctx context.Context, t *testing.T, clus *integration.Cluster) { var prevRev int64 for i := 0; ; i++ { kv := clus.Client(i % 3) resp, err := kv.Get(ctx, "foo") if errors.Is(err, context.DeadlineExceeded) { return } require.NoError(t, silenceConnectionErrors(err)) if resp == nil { continue } require.LessOrEqualf(t, prevRev, resp.Header.Revision, "rev is less than previously observed revision, rev: %d, prevRev: %d", resp.Header.Revision, prevRev) prevRev = resp.Header.Revision } } func silenceConnectionErrors(err error) error { if err == nil { return nil } s := status.Convert(err) for _, msg := range connectionErrorMessages { if strings.Contains(s.Message(), msg) { return nil } } return err } var connectionErrorMessages = []string{ "context deadline exceeded", "etcdserver: request timed out", "error reading from server: EOF", "read: connection reset by peer", "use of closed network connection", } ================================================ FILE: tests/integration/snapshot/member_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package snapshot_test import ( "context" "fmt" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/testutil" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestSnapshotV3RestoreMultiMemberAdd ensures that multiple members // can boot into the same cluster after being restored from a same // snapshot file, and also be able to add another member to the cluster. func TestSnapshotV3RestoreMultiMemberAdd(t *testing.T) { integration.BeforeTest(t) kvs := []kv{{"foo1", "bar1"}, {"foo2", "bar2"}, {"foo3", "bar3"}} dbPath := createSnapshotFile(t, kvs) clusterN := 3 cURLs, pURLs, srvs := restoreCluster(t, clusterN, dbPath) defer func() { for i := 0; i < clusterN; i++ { srvs[i].Close() } }() // wait for health interval + leader election time.Sleep(etcdserver.HealthInterval + 2*time.Second) cli, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{cURLs[0].String()}}) require.NoError(t, err) defer cli.Close() urls := newEmbedURLs(t, 2) newCURLs, newPURLs := urls[:1], urls[1:] _, err = cli.MemberAdd(t.Context(), []string{newPURLs[0].String()}) require.NoError(t, err) // wait for membership reconfiguration apply time.Sleep(testutil.ApplyTimeout) cfg := integration.NewEmbedConfig(t, "3") cfg.InitialClusterToken = testClusterTkn cfg.ClusterState = "existing" cfg.ListenClientUrls, cfg.AdvertiseClientUrls = newCURLs, newCURLs cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = newPURLs, newPURLs cfg.InitialCluster = "" for i := 0; i < clusterN; i++ { cfg.InitialCluster += fmt.Sprintf(",%d=%s", i, pURLs[i].String()) } cfg.InitialCluster = cfg.InitialCluster[1:] cfg.InitialCluster += fmt.Sprintf(",%s=%s", cfg.Name, newPURLs[0].String()) srv, err := embed.StartEtcd(cfg) require.NoError(t, err) defer func() { srv.Close() }() select { case <-srv.Server.ReadyNotify(): case <-time.After(10 * time.Second): t.Fatalf("failed to start the newly added etcd member") } cli2, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{newCURLs[0].String()}}) require.NoError(t, err) defer cli2.Close() ctx, cancel := context.WithTimeout(t.Context(), testutil.RequestTimeout) mresp, err := cli2.MemberList(ctx) cancel() require.NoError(t, err) require.Lenf(t, mresp.Members, 4, "expected 4 members, got %+v", mresp) // make sure restored cluster has kept all data on recovery var gresp *clientv3.GetResponse ctx, cancel = context.WithTimeout(t.Context(), testutil.RequestTimeout) gresp, err = cli2.Get(ctx, "foo", clientv3.WithPrefix()) cancel() require.NoError(t, err) for i := range gresp.Kvs { require.Equalf(t, string(gresp.Kvs[i].Key), kvs[i].k, "#%d: key expected %s, got %s", i, kvs[i].k, gresp.Kvs[i].Key) require.Equalf(t, string(gresp.Kvs[i].Value), kvs[i].v, "#%d: value expected %s, got %s", i, kvs[i].v, gresp.Kvs[i].Value) } } ================================================ FILE: tests/integration/snapshot/v3_snapshot_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package snapshot_test import ( "context" "encoding/binary" "fmt" "net/url" "os" "path/filepath" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/client/pkg/v3/testutil" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/etcdutl/v3/snapshot" "go.etcd.io/etcd/pkg/v3/cpuutil" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/tests/v3/framework/integration" "go.etcd.io/etcd/tests/v3/framework/testutils" ) // TestSnapshotV3RestoreSingle tests single node cluster restoring // from a snapshot file. func TestSnapshotV3RestoreSingle(t *testing.T) { integration.BeforeTest(t) kvs := []kv{{"foo1", "bar1"}, {"foo2", "bar2"}, {"foo3", "bar3"}} dbPath := createSnapshotFile(t, kvs) clusterN := 1 urls := newEmbedURLs(t, clusterN*2) cURLs, pURLs := urls[:clusterN], urls[clusterN:] cfg := integration.NewEmbedConfig(t, "s1") cfg.InitialClusterToken = testClusterTkn cfg.ClusterState = "existing" cfg.ListenClientUrls, cfg.AdvertiseClientUrls = cURLs, cURLs cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = pURLs, pURLs cfg.InitialCluster = fmt.Sprintf("%s=%s", cfg.Name, pURLs[0].String()) sp := snapshot.NewV3(zaptest.NewLogger(t)) pss := make([]string, 0, len(pURLs)) for _, p := range pURLs { pss = append(pss, p.String()) } err := sp.Restore(snapshot.RestoreConfig{ SnapshotPath: dbPath, Name: cfg.Name, OutputDataDir: cfg.Dir, InitialCluster: cfg.InitialCluster, InitialClusterToken: cfg.InitialClusterToken, PeerURLs: pss, }) require.NoError(t, err) srv, err := embed.StartEtcd(cfg) require.NoError(t, err) defer func() { srv.Close() }() select { case <-srv.Server.ReadyNotify(): case <-time.After(3 * time.Second): t.Fatalf("failed to start restored etcd member") } var cli *clientv3.Client cli, err = integration.NewClient(t, clientv3.Config{Endpoints: []string{cfg.AdvertiseClientUrls[0].String()}}) require.NoError(t, err) defer cli.Close() for i := range kvs { var gresp *clientv3.GetResponse gresp, err = cli.Get(t.Context(), kvs[i].k) require.NoError(t, err) require.Equalf(t, string(gresp.Kvs[0].Value), kvs[i].v, "#%d: value expected %s, got %s", i, kvs[i].v, gresp.Kvs[0].Value) } } // TestSnapshotV3RestoreMulti ensures that multiple members // can boot into the same cluster after being restored from a same // snapshot file. func TestSnapshotV3RestoreMulti(t *testing.T) { integration.BeforeTest(t) kvs := []kv{{"foo1", "bar1"}, {"foo2", "bar2"}, {"foo3", "bar3"}} dbPath := createSnapshotFile(t, kvs) clusterN := 3 cURLs, _, srvs := restoreCluster(t, clusterN, dbPath) defer func() { for i := 0; i < clusterN; i++ { srvs[i].Close() } }() // wait for leader election time.Sleep(time.Second) for i := 0; i < clusterN; i++ { cli, err := integration.NewClient(t, clientv3.Config{Endpoints: []string{cURLs[i].String()}}) require.NoError(t, err) defer cli.Close() for i := range kvs { var gresp *clientv3.GetResponse gresp, err = cli.Get(t.Context(), kvs[i].k) require.NoError(t, err) require.Equalf(t, string(gresp.Kvs[0].Value), kvs[i].v, "#%d: value expected %s, got %s", i, kvs[i].v, gresp.Kvs[0].Value) } } } // TestCorruptedBackupFileCheck tests if we can correctly identify a corrupted backup file. func TestCorruptedBackupFileCheck(t *testing.T) { if cpuutil.ByteOrder() == binary.BigEndian { t.Skipf("skipping on big endian platforms since testdata/corrupted_backup.db is in little endian format") } dbPath := testutils.MustAbsPath("testdata/corrupted_backup.db") integration.BeforeTest(t) _, err := os.Stat(dbPath) require.NoErrorf(t, err, "test file [%s] does not exist: %v", dbPath, err) sp := snapshot.NewV3(zaptest.NewLogger(t)) _, err = sp.Status(dbPath) expectedErrKeywords := "snapshot file integrity check failed" /* example error message: snapshot file integrity check failed. 2 errors found. page 3: already freed page 4: unreachable unfreed */ if err == nil { t.Error("expected error due to corrupted snapshot file, got no error") } if !strings.Contains(err.Error(), expectedErrKeywords) { t.Errorf("expected error message to contain the following keywords:\n%s\n"+ "actual error message:\n%s", expectedErrKeywords, err.Error()) } } type kv struct { k, v string } // creates a snapshot file and returns the file path. func createSnapshotFile(t *testing.T, kvs []kv) string { testutil.SkipTestIfShortMode(t, "Snapshot creation tests are depending on embedded etcd server so are integration-level tests.") clusterN := 1 urls := newEmbedURLs(t, clusterN*2) cURLs, pURLs := urls[:clusterN], urls[clusterN:] cfg := integration.NewEmbedConfig(t, "default") cfg.ClusterState = "new" cfg.ListenClientUrls, cfg.AdvertiseClientUrls = cURLs, cURLs cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = pURLs, pURLs cfg.InitialCluster = fmt.Sprintf("%s=%s", cfg.Name, pURLs[0].String()) srv, err := embed.StartEtcd(cfg) require.NoError(t, err) defer func() { srv.Close() }() select { case <-srv.Server.ReadyNotify(): case <-time.After(3 * time.Second): t.Fatalf("failed to start embed.Etcd for creating snapshots") } ccfg := clientv3.Config{Endpoints: []string{cfg.AdvertiseClientUrls[0].String()}} cli, err := integration.NewClient(t, ccfg) require.NoError(t, err) defer cli.Close() for i := range kvs { ctx, cancel := context.WithTimeout(t.Context(), testutil.RequestTimeout) _, err = cli.Put(ctx, kvs[i].k, kvs[i].v) cancel() require.NoError(t, err) } sp := snapshot.NewV3(zaptest.NewLogger(t)) dpPath := filepath.Join(t.TempDir(), fmt.Sprintf("snapshot%d.db", time.Now().Nanosecond())) _, err = sp.Save(t.Context(), ccfg, dpPath) require.NoError(t, err) return dpPath } const testClusterTkn = "tkn" func restoreCluster(t *testing.T, clusterN int, dbPath string) ( cURLs []url.URL, pURLs []url.URL, srvs []*embed.Etcd, ) { urls := newEmbedURLs(t, clusterN*2) cURLs, pURLs = urls[:clusterN], urls[clusterN:] ics := "" for i := 0; i < clusterN; i++ { ics += fmt.Sprintf(",m%d=%s", i, pURLs[i].String()) } ics = ics[1:] cfgs := make([]*embed.Config, clusterN) for i := 0; i < clusterN; i++ { cfg := integration.NewEmbedConfig(t, fmt.Sprintf("m%d", i)) cfg.InitialClusterToken = testClusterTkn cfg.ClusterState = "existing" cfg.ListenClientUrls, cfg.AdvertiseClientUrls = []url.URL{cURLs[i]}, []url.URL{cURLs[i]} cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = []url.URL{pURLs[i]}, []url.URL{pURLs[i]} cfg.InitialCluster = ics sp := snapshot.NewV3( zaptest.NewLogger(t, zaptest.Level(zapcore.InfoLevel)).Named(cfg.Name).Named("sm")) err := sp.Restore(snapshot.RestoreConfig{ SnapshotPath: dbPath, Name: cfg.Name, OutputDataDir: cfg.Dir, PeerURLs: []string{pURLs[i].String()}, InitialCluster: ics, InitialClusterToken: cfg.InitialClusterToken, }) require.NoError(t, err) cfgs[i] = cfg } sch := make(chan *embed.Etcd, len(cfgs)) for i := range cfgs { go func(idx int) { srv, err := embed.StartEtcd(cfgs[idx]) if err != nil { t.Error(err) } <-srv.Server.ReadyNotify() sch <- srv }(i) } srvs = make([]*embed.Etcd, clusterN) for i := 0; i < clusterN; i++ { select { case srv := <-sch: srvs[i] = srv case <-time.After(5 * time.Second): t.Fatalf("#%d: failed to start embed.Etcd", i) } } return cURLs, pURLs, srvs } // TODO: TLS func newEmbedURLs(t testutil.TB, n int) (urls []url.URL) { urls = make([]url.URL, n) for i := 0; i < n; i++ { l := integration.NewLocalListener(t) defer l.Close() u, err := url.Parse(fmt.Sprintf("unix://%s", l.Addr())) require.NoError(t, err) urls[i] = *u } return urls } ================================================ FILE: tests/integration/testing_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration_test import ( "testing" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestBeforeTestWithoutLeakDetection(t *testing.T) { integration.BeforeTest(t, integration.WithoutGoLeakDetection(), integration.WithoutSkipInShort()) // Intentional leak that should get ignored go func() { }() } ================================================ FILE: tests/integration/tracing_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "fmt" "net" "strings" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel/propagation" sdktrace "go.opentelemetry.io/otel/sdk/trace" traceservice "go.opentelemetry.io/proto/otlp/collector/trace/v1" commonv1 "go.opentelemetry.io/proto/otlp/common/v1" v1 "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/grpc" "google.golang.org/protobuf/testing/protocmp" "go.etcd.io/etcd/client/pkg/v3/testutil" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestTracing ensures that distributed tracing is setup when the feature flag is enabled. func TestTracing(t *testing.T) { testutil.SkipTestIfShortMode(t, "Wal creation tests are depending on embedded etcd server so are integration-level tests.") for _, tc := range []struct { name string rpc func(context.Context, *clientv3.Client) error wantSpan *v1.Span }{ { name: "UnaryGet", rpc: func(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Get(ctx, "key") return err }, wantSpan: &v1.Span{ Name: "etcdserverpb.KV/Range", // Attributes are set outside Etcd in otelgrpc, so they are ignored here. }, }, { name: "UnaryGetWithCountOnly", rpc: func(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Get(ctx, "key", clientv3.WithCountOnly()) return err }, wantSpan: &v1.Span{ Name: "range", Attributes: []*commonv1.KeyValue{ { Key: "range_begin", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{StringValue: "key"}}, }, { Key: "range_end", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{StringValue: ""}}, }, { Key: "rev", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_IntValue{IntValue: 0}}, }, { Key: "limit", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_IntValue{IntValue: 0}}, }, { Key: "count_only", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_BoolValue{BoolValue: true}}, }, { Key: "keys_only", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_BoolValue{BoolValue: false}}, }, }, }, }, { name: "UnaryTxn", rpc: func(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Txn(ctx). If(clientv3.Compare(clientv3.ModRevision("cmp_key"), "=", 1)). Then(clientv3.OpPut("op_key", "val", clientv3.WithLease(1234)), clientv3.OpGet("other_key")). Commit() return err }, wantSpan: &v1.Span{ Name: "txn", Attributes: []*commonv1.KeyValue{ { Key: "compare_first_key", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{StringValue: "cmp_key"}}, }, { Key: "success_first_key", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{StringValue: "op_key"}}, }, { Key: "success_first_type", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{StringValue: "put"}}, }, { Key: "success_first_lease", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_IntValue{IntValue: 1234}}, }, { Key: "compare_len", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_IntValue{IntValue: 1}}, }, { Key: "success_len", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_IntValue{IntValue: 2}}, }, { Key: "failure_len", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_IntValue{IntValue: 0}}, }, { Key: "read_only", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_BoolValue{BoolValue: false}}, }, }, }, }, { name: "UnaryLeaseGrant", rpc: func(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Grant(ctx, 1_000_123) return err }, wantSpan: &v1.Span{ Name: "lease_grant", Attributes: []*commonv1.KeyValue{ { Key: "id", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_IntValue{IntValue: 0}}, }, { Key: "ttl", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_IntValue{IntValue: 1_000_123}}, }, }, }, }, { name: "UnaryLeaseRenew", rpc: func(ctx context.Context, cli *clientv3.Client) error { _, err := cli.KeepAliveOnce(ctx, 2345) if err != nil && strings.Contains(err.Error(), "requested lease not found") { // errors.Is does not work across gRPC bounduaries. return nil } return err }, wantSpan: &v1.Span{ Name: "lease_renew", Attributes: []*commonv1.KeyValue{ { Key: "id", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_IntValue{IntValue: 2345}}, }, }, }, }, { name: "UnaryLeaseRevoke", rpc: func(ctx context.Context, cli *clientv3.Client) error { _, err := cli.Revoke(ctx, 1234) if err != nil && strings.Contains(err.Error(), "requested lease not found") { // errors.Is does not work across gRPC bounduaries. return nil } return err }, wantSpan: &v1.Span{ Name: "lease_revoke", Attributes: []*commonv1.KeyValue{ { Key: "id", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_IntValue{IntValue: 1234}}, }, }, }, }, { name: "StreamWatch", rpc: func(ctx context.Context, cli *clientv3.Client) error { // Create a context with a reasonable timeout ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() // Create a watch channel watchChan := cli.Watch(ctx, "watch-key", clientv3.WithProgressNotify(), clientv3.WithRev(1)) // Put a value to trigger the watch _, err := cli.Put(ctx, "watch-key", "watch-value") if err != nil { return err } // Wait for watch event select { case watchResp := <-watchChan: return watchResp.Err() case <-time.After(5 * time.Second): return fmt.Errorf("Timed out waiting for watch event") } }, wantSpan: &v1.Span{ Name: "watch", Attributes: []*commonv1.KeyValue{ { Key: "key", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{StringValue: "watch-key"}}, }, { Key: "range_end", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{StringValue: ""}}, }, { Key: "start_rev", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_IntValue{IntValue: 1}}, }, { Key: "progress_notify", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_BoolValue{BoolValue: true}}, }, { Key: "prev_kv", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_BoolValue{BoolValue: false}}, }, { Key: "fragment", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_BoolValue{BoolValue: false}}, }, }, }, }, } { t.Run(tc.name, func(t *testing.T) { testRPCTracing(t, tc.wantSpan, tc.rpc) }) } } // testRPCTracing is a common test function for both Unary and Stream RPC tracing func testRPCTracing(t *testing.T, wantSpan *v1.Span, clientAction func(context.Context, *clientv3.Client) error) { // set up trace collector listener, err := net.Listen("tcp", "localhost:") require.NoError(t, err) traceFound := make(chan struct{}) defer close(traceFound) srv := grpc.NewServer() traceservice.RegisterTraceServiceServer(srv, &traceServer{ traceFound: traceFound, filterFunc: func(req *traceservice.ExportTraceServiceRequest) bool { for _, resourceSpans := range req.GetResourceSpans() { // Skip spans which weren't produced by test's gRPC client. matched := false for _, attr := range resourceSpans.GetResource().GetAttributes() { if attr.GetKey() == "service.name" && attr.GetValue().GetStringValue() == "integration-test-tracing" { matched = true break } } if !matched { continue } for _, scoped := range resourceSpans.GetScopeSpans() { for _, gotSpan := range scoped.GetSpans() { if gotSpan.GetName() != wantSpan.GetName() { continue } if len(wantSpan.GetAttributes()) == 0 { // Diff will compare only attributes and events when needed return true } if gotSpan.GetName() == "lease_grant" { // Ignore ID in lease grant which is not controlled by the client. for _, attr := range gotSpan.GetAttributes() { if attr.GetKey() == "id" { attr.Value.Value.(*commonv1.AnyValue_IntValue).IntValue = 0 } } } if diff := cmp.Diff(wantSpan, gotSpan, protocmp.Transform(), protocmp.IgnoreFields(&v1.Span{}, "end_time_unix_nano", "flags", "kind", "parent_span_id", "span_id", "start_time_unix_nano", "status", "trace_id", "events"), ); diff != "" { t.Errorf("Span mismatch (-want +got):\n%s", diff) } return true } } } return false }, }) go srv.Serve(listener) defer srv.Stop() cfg := integration.NewEmbedConfig(t, "default") cfg.EnableDistributedTracing = true cfg.DistributedTracingAddress = listener.Addr().String() cfg.DistributedTracingServiceName = "integration-test-tracing" cfg.DistributedTracingSamplingRatePerMillion = 100 // overridden later in the test // start an etcd instance with tracing enabled etcdSrv, err := embed.StartEtcd(cfg) require.NoError(t, err) defer etcdSrv.Close() select { case <-etcdSrv.Server.ReadyNotify(): case <-time.After(5 * time.Second): // default randomized election timeout is 1 to 2s, single node will fast-forward 900ms // change the timeout from 1 to 5 seconds to ensure de-flaking this test t.Fatalf("failed to start embed.Etcd for test") } // create a client that has tracing enabled tp := sdktrace.NewTracerProvider() defer tp.Shutdown(t.Context()) tracingOpts := []otelgrpc.Option{ otelgrpc.WithTracerProvider(tp), otelgrpc.WithPropagators( propagation.NewCompositeTextMapPropagator( propagation.TraceContext{}, propagation.Baggage{}, )), } dialOptions := []grpc.DialOption{ grpc.WithStatsHandler(otelgrpc.NewClientHandler(tracingOpts...)), } ccfg := clientv3.Config{DialOptions: dialOptions, Endpoints: []string{cfg.AdvertiseClientUrls[0].String()}} cli, err := integration.NewClient(t, ccfg) if err != nil { etcdSrv.Close() t.Fatal(err) } defer cli.Close() // Execute the client action (either Unary or Stream RPC) err = clientAction(t.Context(), cli) require.NoError(t, err) // Wait for a span to be recorded from our request select { case <-traceFound: t.Logf("Trace found") return case <-time.After(30 * time.Second): // default exporter has 5s scheduling delay t.Fatal("Timed out waiting for trace") } } // traceServer implements TracesServiceServer type traceServer struct { traceFound chan struct{} filterFunc func(req *traceservice.ExportTraceServiceRequest) bool traceservice.UnimplementedTraceServiceServer } func (t *traceServer) Export(ctx context.Context, req *traceservice.ExportTraceServiceRequest) (*traceservice.ExportTraceServiceResponse, error) { emptyValue := traceservice.ExportTraceServiceResponse{} if t.filterFunc(req) { select { case t.traceFound <- struct{}{}: default: // Channel already notified } } return &emptyValue, nil } ================================================ FILE: tests/integration/util_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "io" "os" "path/filepath" "go.etcd.io/etcd/client/pkg/v3/transport" ) // copyTLSFiles clones certs files to dst directory. func copyTLSFiles(ti transport.TLSInfo, dst string) (transport.TLSInfo, error) { ci := transport.TLSInfo{ KeyFile: filepath.Join(dst, "server-key.pem"), CertFile: filepath.Join(dst, "server.pem"), TrustedCAFile: filepath.Join(dst, "etcd-root-ca.pem"), ClientCertAuth: ti.ClientCertAuth, } if err := copyFile(ti.KeyFile, ci.KeyFile); err != nil { return transport.TLSInfo{}, err } if err := copyFile(ti.CertFile, ci.CertFile); err != nil { return transport.TLSInfo{}, err } if err := copyFile(ti.TrustedCAFile, ci.TrustedCAFile); err != nil { return transport.TLSInfo{}, err } return ci, nil } func copyFile(src, dst string) error { f, err := os.Open(src) if err != nil { return err } defer f.Close() w, err := os.Create(dst) if err != nil { return err } defer w.Close() if _, err = io.Copy(w, f); err != nil { return err } return w.Sync() } ================================================ FILE: tests/integration/utl_wal_version_test.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "testing" "time" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/testutil" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" framecfg "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestEtcdVersionFromWAL(t *testing.T) { testutil.SkipTestIfShortMode(t, "Wal creation tests are depending on embedded etcd server so are integration-level tests.") cfg := integration.NewEmbedConfig(t, "default") srv, err := embed.StartEtcd(cfg) require.NoError(t, err) select { case <-srv.Server.ReadyNotify(): case <-time.After(3 * time.Second): t.Fatalf("failed to start embed.Etcd for test") } // When the member becomes leader, it will update the cluster version // with the cluster's minimum version. As it's updated asynchronously, // it could not be updated in time before close. Wait for it to become // ready. if err = waitForClusterVersionReady(srv); err != nil { srv.Close() t.Fatalf("failed to wait for cluster version to become ready: %v", err) } ccfg := clientv3.Config{Endpoints: []string{cfg.AdvertiseClientUrls[0].String()}} cli, err := integration.NewClient(t, ccfg) if err != nil { srv.Close() t.Fatal(err) } // Once the cluster version has been updated, any entity's storage // version should be align with cluster version. ctx, cancel := context.WithTimeout(t.Context(), testutil.RequestTimeout) _, err = cli.AuthStatus(ctx) cancel() if err != nil { srv.Close() t.Fatalf("failed to get auth status: %v", err) } cli.Close() srv.Close() w, err := wal.Open(zap.NewNop(), cfg.Dir+"/member/wal", walpb.Snapshot{}) require.NoError(t, err) defer w.Close() walVersion, err := wal.ReadWALVersion(w) require.NoError(t, err) assert.Equal(t, &semver.Version{Major: 3, Minor: 5}, walVersion.MinimalEtcdVersion()) } func waitForClusterVersionReady(srv *embed.Etcd) error { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() for { select { case <-ctx.Done(): return ctx.Err() default: } if srv.Server.ClusterVersion() != nil { return nil } time.Sleep(framecfg.TickDuration) } } ================================================ FILE: tests/integration/v2store/main_test.go ================================================ // Copyright 2019 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store_test import ( "testing" "go.etcd.io/etcd/client/pkg/v3/testutil" ) func TestMain(m *testing.M) { //cfg := integration.ClusterConfig{Size: 1} //clus := integration.NewClusterV3(nil, &cfg) //endpoints = []string{clus.Client(0).Endpoints()[0]} // v := m.Run() //clus.Terminate(nil) //if err := testutil.CheckAfterTest(time.Second); err != nil { // fmt.Fprintf(os.Stderr, "%v", err) // os.Exit(1) //} testutil.MustTestMainWithLeakDetection(m) //if v == 0 && testutil.CheckLeakedGoroutine() { // os.Exit(1) //} //os.Exit(v) } ================================================ FILE: tests/integration/v2store/store_tag_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestStoreRecover ensures that the store can recover from a previously saved state. func TestStoreRecover(t *testing.T) { integration.BeforeTest(t) s := v2store.New() var eidx uint64 = 4 s.Create("/foo", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) s.Create("/foo/x", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) s.Update("/foo/x", "barbar", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) s.Create("/foo/y", false, "baz", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) b, err := s.Save() require.NoError(t, err) s2 := v2store.New() s2.Recovery(b) e, err := s.Get("/foo/x", false, false) assert.Equal(t, uint64(2), e.Node.CreatedIndex) assert.Equal(t, uint64(3), e.Node.ModifiedIndex) assert.Equal(t, eidx, e.EtcdIndex) require.NoError(t, err) assert.Equal(t, "barbar", *e.Node.Value) e, err = s.Get("/foo/y", false, false) assert.Equal(t, eidx, e.EtcdIndex) require.NoError(t, err) assert.Equal(t, "baz", *e.Node.Value) } ================================================ FILE: tests/integration/v2store/store_test.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package v2store_test import ( "fmt" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/server/v3/etcdserver/api/v2error" "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" ) type StoreCloser interface { v2store.Store Close() } func TestNewStoreWithNamespaces(t *testing.T) { s := v2store.New("/0", "/1") _, err := s.Get("/0", false, false) require.NoError(t, err) _, err = s.Get("/1", false, false) assert.NoError(t, err) } // TestStoreGetValue ensures that the store can retrieve an existing value. func TestStoreGetValue(t *testing.T) { s := v2store.New() s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) var eidx uint64 = 1 e, err := s.Get("/foo", false, false) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "get", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.Equal(t, "bar", *e.Node.Value) } // TestStoreGetSorted ensures that the store can retrieve a directory in sorted order. func TestStoreGetSorted(t *testing.T) { s := v2store.New() s.Create("/foo", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) s.Create("/foo/x", false, "0", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) s.Create("/foo/z", false, "0", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) s.Create("/foo/y", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) s.Create("/foo/y/a", false, "0", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) s.Create("/foo/y/b", false, "0", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) var eidx uint64 = 6 e, err := s.Get("/foo", true, true) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) var yNodes v2store.NodeExterns sortedStrings := []string{"/foo/x", "/foo/y", "/foo/z"} for i := range e.Node.Nodes { node := e.Node.Nodes[i] if node.Key != sortedStrings[i] { t.Errorf("expect key = %s, got key = %s", sortedStrings[i], node.Key) } if node.Key == "/foo/y" { yNodes = node.Nodes } } sortedStrings = []string{"/foo/y/a", "/foo/y/b"} for i := range yNodes { node := yNodes[i] if node.Key != sortedStrings[i] { t.Errorf("expect key = %s, got key = %s", sortedStrings[i], node.Key) } } } func TestSet(t *testing.T) { s := v2store.New() // Set /foo="" var eidx uint64 = 1 e, err := s.Set("/foo", false, "", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "set", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.False(t, e.Node.Dir) assert.Empty(t, *e.Node.Value) assert.Nil(t, e.Node.Nodes) assert.Nil(t, e.Node.Expiration) assert.Equal(t, int64(0), e.Node.TTL) assert.Equal(t, uint64(1), e.Node.ModifiedIndex) // Set /foo="bar" eidx = 2 e, err = s.Set("/foo", false, "bar", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "set", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.False(t, e.Node.Dir) assert.Equal(t, "bar", *e.Node.Value) assert.Nil(t, e.Node.Nodes) assert.Nil(t, e.Node.Expiration) assert.Equal(t, int64(0), e.Node.TTL) assert.Equal(t, uint64(2), e.Node.ModifiedIndex) // check prevNode require.NotNil(t, e.PrevNode) assert.Equal(t, "/foo", e.PrevNode.Key) assert.Empty(t, *e.PrevNode.Value) assert.Equal(t, uint64(1), e.PrevNode.ModifiedIndex) // Set /foo="baz" (for testing prevNode) eidx = 3 e, err = s.Set("/foo", false, "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "set", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.False(t, e.Node.Dir) assert.Equal(t, "baz", *e.Node.Value) assert.Nil(t, e.Node.Nodes) assert.Nil(t, e.Node.Expiration) assert.Equal(t, int64(0), e.Node.TTL) assert.Equal(t, uint64(3), e.Node.ModifiedIndex) // check prevNode require.NotNil(t, e.PrevNode) assert.Equal(t, "/foo", e.PrevNode.Key) assert.Equal(t, "bar", *e.PrevNode.Value) assert.Equal(t, uint64(2), e.PrevNode.ModifiedIndex) // Set /a/b/c/d="efg" eidx = 4 e, err = s.Set("/a/b/c/d", false, "efg", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "/a/b/c/d", e.Node.Key) assert.False(t, e.Node.Dir) assert.Equal(t, "efg", *e.Node.Value) assert.Nil(t, e.Node.Nodes) assert.Nil(t, e.Node.Expiration) assert.Equal(t, int64(0), e.Node.TTL) assert.Equal(t, uint64(4), e.Node.ModifiedIndex) // Set /dir as a directory eidx = 5 e, err = s.Set("/dir", true, "", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "set", e.Action) assert.Equal(t, "/dir", e.Node.Key) assert.True(t, e.Node.Dir) assert.Nil(t, e.Node.Value) assert.Nil(t, e.Node.Nodes) assert.Nil(t, e.Node.Expiration) assert.Equal(t, int64(0), e.Node.TTL) assert.Equal(t, uint64(5), e.Node.ModifiedIndex) } // TestStoreCreateValue ensures that the store can create a new key if it doesn't already exist. func TestStoreCreateValue(t *testing.T) { s := v2store.New() // Create /foo=bar var eidx uint64 = 1 e, err := s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "create", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.False(t, e.Node.Dir) assert.Equal(t, "bar", *e.Node.Value) assert.Nil(t, e.Node.Nodes) assert.Nil(t, e.Node.Expiration) assert.Equal(t, int64(0), e.Node.TTL) assert.Equal(t, uint64(1), e.Node.ModifiedIndex) // Create /empty="" eidx = 2 e, err = s.Create("/empty", false, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "create", e.Action) assert.Equal(t, "/empty", e.Node.Key) assert.False(t, e.Node.Dir) assert.Empty(t, *e.Node.Value) assert.Nil(t, e.Node.Nodes) assert.Nil(t, e.Node.Expiration) assert.Equal(t, int64(0), e.Node.TTL) assert.Equal(t, uint64(2), e.Node.ModifiedIndex) } // TestStoreCreateDirectory ensures that the store can create a new directory if it doesn't already exist. func TestStoreCreateDirectory(t *testing.T) { s := v2store.New() var eidx uint64 = 1 e, err := s.Create("/foo", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "create", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.True(t, e.Node.Dir) } // TestStoreCreateFailsIfExists ensure that the store fails to create a key if it already exists. func TestStoreCreateFailsIfExists(t *testing.T) { s := v2store.New() // create /foo as dir s.Create("/foo", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) // create /foo as dir again e, _err := s.Create("/foo", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) var err *v2error.Error require.ErrorAs(t, _err, &err) assert.Equal(t, v2error.EcodeNodeExist, err.ErrorCode) assert.Equal(t, "Key already exists", err.Message) assert.Equal(t, "/foo", err.Cause) assert.Equal(t, uint64(1), err.Index) assert.Nil(t, e) } // TestStoreUpdateValue ensures that the store can update a key if it already exists. func TestStoreUpdateValue(t *testing.T) { s := v2store.New() // create /foo=bar s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) // update /foo="bzr" var eidx uint64 = 2 e, err := s.Update("/foo", "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "update", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.False(t, e.Node.Dir) assert.Equal(t, "baz", *e.Node.Value) assert.Equal(t, int64(0), e.Node.TTL) assert.Equal(t, uint64(2), e.Node.ModifiedIndex) // check prevNode assert.Equal(t, "/foo", e.PrevNode.Key) assert.Equal(t, "bar", *e.PrevNode.Value) assert.Equal(t, int64(0), e.PrevNode.TTL) assert.Equal(t, uint64(1), e.PrevNode.ModifiedIndex) e, _ = s.Get("/foo", false, false) assert.Equal(t, "baz", *e.Node.Value) assert.Equal(t, eidx, e.EtcdIndex) // update /foo="" eidx = 3 e, err = s.Update("/foo", "", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "update", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.False(t, e.Node.Dir) assert.Empty(t, *e.Node.Value) assert.Equal(t, int64(0), e.Node.TTL) assert.Equal(t, uint64(3), e.Node.ModifiedIndex) // check prevNode assert.Equal(t, "/foo", e.PrevNode.Key) assert.Equal(t, "baz", *e.PrevNode.Value) assert.Equal(t, int64(0), e.PrevNode.TTL) assert.Equal(t, uint64(2), e.PrevNode.ModifiedIndex) e, _ = s.Get("/foo", false, false) assert.Equal(t, eidx, e.EtcdIndex) assert.Empty(t, *e.Node.Value) } // TestStoreUpdateFailsIfDirectory ensures that the store cannot update a directory. func TestStoreUpdateFailsIfDirectory(t *testing.T) { s := v2store.New() s.Create("/foo", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e, _err := s.Update("/foo", "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) var err *v2error.Error require.ErrorAs(t, _err, &err) assert.Equal(t, v2error.EcodeNotFile, err.ErrorCode) assert.Equal(t, "Not a file", err.Message) assert.Equal(t, "/foo", err.Cause) assert.Nil(t, e) } // TestStoreDeleteValue ensures that the store can delete a value. func TestStoreDeleteValue(t *testing.T) { s := v2store.New() var eidx uint64 = 2 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e, err := s.Delete("/foo", false, false) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "delete", e.Action) // check prevNode require.NotNil(t, e.PrevNode) assert.Equal(t, "/foo", e.PrevNode.Key) assert.Equal(t, "bar", *e.PrevNode.Value) } // TestStoreDeleteDirectory ensures that the store can delete a directory if recursive is specified. func TestStoreDeleteDirectory(t *testing.T) { s := v2store.New() // create directory /foo var eidx uint64 = 2 s.Create("/foo", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) // delete /foo with dir = true and recursive = false // this should succeed, since the directory is empty e, err := s.Delete("/foo", true, false) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "delete", e.Action) // check prevNode require.NotNil(t, e.PrevNode) assert.Equal(t, "/foo", e.PrevNode.Key) assert.True(t, e.PrevNode.Dir) // create directory /foo and directory /foo/bar _, err = s.Create("/foo/bar", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) // delete /foo with dir = true and recursive = false // this should fail, since the directory is not empty _, err = s.Delete("/foo", true, false) require.Error(t, err) // delete /foo with dir=false and recursive = true // this should succeed, since recursive implies dir=true // and recursively delete should be able to delete all // items under the given directory e, err = s.Delete("/foo", false, true) require.NoError(t, err) assert.Equal(t, "delete", e.Action) } // TestStoreDeleteDirectoryFailsIfNonRecursiveAndDir ensures that the // store cannot delete a directory if both of recursive and dir are not specified. func TestStoreDeleteDirectoryFailsIfNonRecursiveAndDir(t *testing.T) { s := v2store.New() s.Create("/foo", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e, _err := s.Delete("/foo", false, false) var err *v2error.Error require.ErrorAs(t, _err, &err) assert.Equal(t, v2error.EcodeNotFile, err.ErrorCode) assert.Equal(t, "Not a file", err.Message) assert.Nil(t, e) } func TestRootRdOnly(t *testing.T) { s := v2store.New("/0") for _, tt := range []string{"/", "/0"} { _, err := s.Set(tt, true, "", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.Error(t, err) _, err = s.Delete(tt, true, true) require.Error(t, err) _, err = s.Create(tt, true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.Error(t, err) _, err = s.Update(tt, "", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.Error(t, err) _, err = s.CompareAndSwap(tt, "", 0, "", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.Error(t, err) } } func TestStoreCompareAndDeletePrevValue(t *testing.T) { s := v2store.New() var eidx uint64 = 2 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e, err := s.CompareAndDelete("/foo", "bar", 0) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "compareAndDelete", e.Action) assert.Equal(t, "/foo", e.Node.Key) // check prevNode require.NotNil(t, e.PrevNode) assert.Equal(t, "/foo", e.PrevNode.Key) assert.Equal(t, "bar", *e.PrevNode.Value) assert.Equal(t, uint64(1), e.PrevNode.ModifiedIndex) assert.Equal(t, uint64(1), e.PrevNode.CreatedIndex) } func TestStoreCompareAndDeletePrevValueFailsIfNotMatch(t *testing.T) { s := v2store.New() var eidx uint64 = 1 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e, _err := s.CompareAndDelete("/foo", "baz", 0) var err *v2error.Error require.ErrorAs(t, _err, &err) assert.Equal(t, v2error.EcodeTestFailed, err.ErrorCode) assert.Equal(t, "Compare failed", err.Message) assert.Nil(t, e) e, _ = s.Get("/foo", false, false) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "bar", *e.Node.Value) } func TestStoreCompareAndDeletePrevIndex(t *testing.T) { s := v2store.New() var eidx uint64 = 2 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e, err := s.CompareAndDelete("/foo", "", 1) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "compareAndDelete", e.Action) // check prevNode require.NotNil(t, e.PrevNode) assert.Equal(t, "/foo", e.PrevNode.Key) assert.Equal(t, "bar", *e.PrevNode.Value) assert.Equal(t, uint64(1), e.PrevNode.ModifiedIndex) assert.Equal(t, uint64(1), e.PrevNode.CreatedIndex) } func TestStoreCompareAndDeletePrevIndexFailsIfNotMatch(t *testing.T) { s := v2store.New() var eidx uint64 = 1 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e, _err := s.CompareAndDelete("/foo", "", 100) require.Error(t, _err) var err *v2error.Error require.ErrorAs(t, _err, &err) assert.Equal(t, v2error.EcodeTestFailed, err.ErrorCode) assert.Equal(t, "Compare failed", err.Message) assert.Nil(t, e) e, _ = s.Get("/foo", false, false) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "bar", *e.Node.Value) } // TestStoreCompareAndDeleteDirectoryFail ensures that the store cannot delete a directory. func TestStoreCompareAndDeleteDirectoryFail(t *testing.T) { s := v2store.New() s.Create("/foo", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) _, _err := s.CompareAndDelete("/foo", "", 0) require.Error(t, _err) var err *v2error.Error require.ErrorAs(t, _err, &err) assert.Equal(t, v2error.EcodeNotFile, err.ErrorCode) } // TestStoreCompareAndSwapPrevValue ensures that the store can conditionally // update a key if it has a previous value. func TestStoreCompareAndSwapPrevValue(t *testing.T) { s := v2store.New() var eidx uint64 = 2 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e, err := s.CompareAndSwap("/foo", "bar", 0, "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "compareAndSwap", e.Action) assert.Equal(t, "baz", *e.Node.Value) // check prevNode require.NotNil(t, e.PrevNode) assert.Equal(t, "/foo", e.PrevNode.Key) assert.Equal(t, "bar", *e.PrevNode.Value) assert.Equal(t, uint64(1), e.PrevNode.ModifiedIndex) assert.Equal(t, uint64(1), e.PrevNode.CreatedIndex) e, _ = s.Get("/foo", false, false) assert.Equal(t, "baz", *e.Node.Value) } // TestStoreCompareAndSwapPrevValueFailsIfNotMatch ensure that the store cannot // conditionally update a key if it has the wrong previous value. func TestStoreCompareAndSwapPrevValueFailsIfNotMatch(t *testing.T) { s := v2store.New() var eidx uint64 = 1 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e, _err := s.CompareAndSwap("/foo", "wrong_value", 0, "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) var err *v2error.Error require.ErrorAs(t, _err, &err) assert.Equal(t, v2error.EcodeTestFailed, err.ErrorCode) assert.Equal(t, "Compare failed", err.Message) assert.Nil(t, e) e, _ = s.Get("/foo", false, false) assert.Equal(t, "bar", *e.Node.Value) assert.Equal(t, eidx, e.EtcdIndex) } // TestStoreCompareAndSwapPrevIndex ensures that the store can conditionally // update a key if it has a previous index. func TestStoreCompareAndSwapPrevIndex(t *testing.T) { s := v2store.New() var eidx uint64 = 2 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e, err := s.CompareAndSwap("/foo", "", 1, "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) require.NoError(t, err) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "compareAndSwap", e.Action) assert.Equal(t, "baz", *e.Node.Value) // check prevNode require.NotNil(t, e.PrevNode) assert.Equal(t, "/foo", e.PrevNode.Key) assert.Equal(t, "bar", *e.PrevNode.Value) assert.Equal(t, uint64(1), e.PrevNode.ModifiedIndex) assert.Equal(t, uint64(1), e.PrevNode.CreatedIndex) e, _ = s.Get("/foo", false, false) assert.Equal(t, "baz", *e.Node.Value) assert.Equal(t, eidx, e.EtcdIndex) } // TestStoreCompareAndSwapPrevIndexFailsIfNotMatch ensures that the store cannot // conditionally update a key if it has the wrong previous index. func TestStoreCompareAndSwapPrevIndexFailsIfNotMatch(t *testing.T) { s := v2store.New() var eidx uint64 = 1 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e, _err := s.CompareAndSwap("/foo", "", 100, "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) var err *v2error.Error require.ErrorAs(t, _err, &err) assert.Equal(t, v2error.EcodeTestFailed, err.ErrorCode) assert.Equal(t, "Compare failed", err.Message) assert.Nil(t, e) e, _ = s.Get("/foo", false, false) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "bar", *e.Node.Value) } // TestStoreWatchCreate ensures that the store can watch for key creation. func TestStoreWatchCreate(t *testing.T) { s := v2store.New() var eidx uint64 w, _ := s.Watch("/foo", false, false, 0) c := w.EventChan() assert.Equal(t, eidx, w.StartIndex()) s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) eidx = 1 e := timeoutSelect(t, c) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "create", e.Action) assert.Equal(t, "/foo", e.Node.Key) select { case e = <-w.EventChan(): assert.Nil(t, e) case <-time.After(100 * time.Millisecond): } } // TestStoreWatchRecursiveCreate ensures that the store // can watch for recursive key creation. func TestStoreWatchRecursiveCreate(t *testing.T) { s := v2store.New() var eidx uint64 w, err := s.Watch("/foo", true, false, 0) require.NoError(t, err) assert.Equal(t, eidx, w.StartIndex()) eidx = 1 s.Create("/foo/bar", false, "baz", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e := timeoutSelect(t, w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "create", e.Action) assert.Equal(t, "/foo/bar", e.Node.Key) } // TestStoreWatchUpdate ensures that the store can watch for key updates. func TestStoreWatchUpdate(t *testing.T) { s := v2store.New() var eidx uint64 = 1 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) w, _ := s.Watch("/foo", false, false, 0) assert.Equal(t, eidx, w.StartIndex()) eidx = 2 s.Update("/foo", "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e := timeoutSelect(t, w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "update", e.Action) assert.Equal(t, "/foo", e.Node.Key) } // TestStoreWatchRecursiveUpdate ensures that the store can watch for recursive key updates. func TestStoreWatchRecursiveUpdate(t *testing.T) { s := v2store.New() var eidx uint64 = 1 s.Create("/foo/bar", false, "baz", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) w, err := s.Watch("/foo", true, false, 0) require.NoError(t, err) assert.Equal(t, eidx, w.StartIndex()) eidx = 2 s.Update("/foo/bar", "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e := timeoutSelect(t, w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "update", e.Action) assert.Equal(t, "/foo/bar", e.Node.Key) } // TestStoreWatchDelete ensures that the store can watch for key deletions. func TestStoreWatchDelete(t *testing.T) { s := v2store.New() var eidx uint64 = 1 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) w, _ := s.Watch("/foo", false, false, 0) assert.Equal(t, eidx, w.StartIndex()) eidx = 2 s.Delete("/foo", false, false) e := timeoutSelect(t, w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "delete", e.Action) assert.Equal(t, "/foo", e.Node.Key) } // TestStoreWatchRecursiveDelete ensures that the store can watch for recursive key deletions. func TestStoreWatchRecursiveDelete(t *testing.T) { s := v2store.New() var eidx uint64 = 1 s.Create("/foo/bar", false, "baz", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) w, err := s.Watch("/foo", true, false, 0) require.NoError(t, err) assert.Equal(t, eidx, w.StartIndex()) eidx = 2 s.Delete("/foo/bar", false, false) e := timeoutSelect(t, w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "delete", e.Action) assert.Equal(t, "/foo/bar", e.Node.Key) } // TestStoreWatchCompareAndSwap ensures that the store can watch for CAS updates. func TestStoreWatchCompareAndSwap(t *testing.T) { s := v2store.New() var eidx uint64 = 1 s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) w, _ := s.Watch("/foo", false, false, 0) assert.Equal(t, eidx, w.StartIndex()) eidx = 2 s.CompareAndSwap("/foo", "bar", 0, "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e := timeoutSelect(t, w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "compareAndSwap", e.Action) assert.Equal(t, "/foo", e.Node.Key) } // TestStoreWatchRecursiveCompareAndSwap ensures that the // store can watch for recursive CAS updates. func TestStoreWatchRecursiveCompareAndSwap(t *testing.T) { s := v2store.New() var eidx uint64 = 1 s.Create("/foo/bar", false, "baz", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) w, _ := s.Watch("/foo", true, false, 0) assert.Equal(t, eidx, w.StartIndex()) eidx = 2 s.CompareAndSwap("/foo/bar", "baz", 0, "bat", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e := timeoutSelect(t, w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "compareAndSwap", e.Action) assert.Equal(t, "/foo/bar", e.Node.Key) } // TestStoreWatchStream ensures that the store can watch in streaming mode. func TestStoreWatchStream(t *testing.T) { s := v2store.New() var eidx uint64 = 1 w, _ := s.Watch("/foo", false, true, 0) // first modification s.Create("/foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e := timeoutSelect(t, w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "create", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.Equal(t, "bar", *e.Node.Value) select { case e = <-w.EventChan(): assert.Nil(t, e) case <-time.After(100 * time.Millisecond): } // second modification eidx = 2 s.Update("/foo", "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e = timeoutSelect(t, w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "update", e.Action) assert.Equal(t, "/foo", e.Node.Key) assert.Equal(t, "baz", *e.Node.Value) select { case e = <-w.EventChan(): assert.Nil(t, e) case <-time.After(100 * time.Millisecond): } } // TestStoreWatchCreateWithHiddenKey ensure that the store can // watch for hidden keys as long as it's an exact path match. func TestStoreWatchCreateWithHiddenKey(t *testing.T) { s := v2store.New() var eidx uint64 = 1 w, _ := s.Watch("/_foo", false, false, 0) s.Create("/_foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e := timeoutSelect(t, w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "create", e.Action) assert.Equal(t, "/_foo", e.Node.Key) select { case e = <-w.EventChan(): assert.Nil(t, e) case <-time.After(100 * time.Millisecond): } } // TestStoreWatchRecursiveCreateWithHiddenKey ensures that the store doesn't // see hidden key creates without an exact path match in recursive mode. func TestStoreWatchRecursiveCreateWithHiddenKey(t *testing.T) { s := v2store.New() w, _ := s.Watch("/foo", true, false, 0) s.Create("/foo/_bar", false, "baz", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e := nbselect(w.EventChan()) assert.Nil(t, e) w, _ = s.Watch("/foo", true, false, 0) s.Create("/foo/_baz", true, "", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) select { case e = <-w.EventChan(): assert.Nil(t, e) case <-time.After(100 * time.Millisecond): } s.Create("/foo/_baz/quux", false, "quux", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) select { case e = <-w.EventChan(): assert.Nil(t, e) case <-time.After(100 * time.Millisecond): } } // TestStoreWatchUpdateWithHiddenKey ensures that the store // doesn't see hidden key updates. func TestStoreWatchUpdateWithHiddenKey(t *testing.T) { s := v2store.New() s.Create("/_foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) w, _ := s.Watch("/_foo", false, false, 0) s.Update("/_foo", "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e := timeoutSelect(t, w.EventChan()) assert.Equal(t, "update", e.Action) assert.Equal(t, "/_foo", e.Node.Key) e = nbselect(w.EventChan()) assert.Nil(t, e) } // TestStoreWatchRecursiveUpdateWithHiddenKey ensures that the store doesn't // see hidden key updates without an exact path match in recursive mode. func TestStoreWatchRecursiveUpdateWithHiddenKey(t *testing.T) { s := v2store.New() s.Create("/foo/_bar", false, "baz", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) w, _ := s.Watch("/foo", true, false, 0) s.Update("/foo/_bar", "baz", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e := nbselect(w.EventChan()) assert.Nil(t, e) } // TestStoreWatchDeleteWithHiddenKey ensures that the store can watch for key deletions. func TestStoreWatchDeleteWithHiddenKey(t *testing.T) { s := v2store.New() var eidx uint64 = 2 s.Create("/_foo", false, "bar", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) w, _ := s.Watch("/_foo", false, false, 0) s.Delete("/_foo", false, false) e := timeoutSelect(t, w.EventChan()) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "delete", e.Action) assert.Equal(t, "/_foo", e.Node.Key) e = nbselect(w.EventChan()) assert.Nil(t, e) } // TestStoreWatchRecursiveDeleteWithHiddenKey ensures that the store doesn't see // hidden key deletes without an exact path match in recursive mode. func TestStoreWatchRecursiveDeleteWithHiddenKey(t *testing.T) { s := v2store.New() s.Create("/foo/_bar", false, "baz", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) w, _ := s.Watch("/foo", true, false, 0) s.Delete("/foo/_bar", false, false) e := nbselect(w.EventChan()) assert.Nil(t, e) } // TestStoreWatchRecursiveCreateDeeperThanHiddenKey ensures that the store does see // hidden key creates if watching deeper than a hidden key in recursive mode. func TestStoreWatchRecursiveCreateDeeperThanHiddenKey(t *testing.T) { s := v2store.New() var eidx uint64 = 1 w, _ := s.Watch("/_foo/bar", true, false, 0) s.Create("/_foo/bar/baz", false, "baz", false, v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) e := timeoutSelect(t, w.EventChan()) require.NotNil(t, e) assert.Equal(t, eidx, e.EtcdIndex) assert.Equal(t, "create", e.Action) assert.Equal(t, "/_foo/bar/baz", e.Node.Key) } // TestStoreWatchSlowConsumer ensures that slow consumers are handled properly. // // Since Watcher.EventChan() has a buffer of size 100 we can only queue 100 // event per watcher. If the consumer cannot consume the event on time and // another event arrives, the channel is closed and event is discarded. // This test ensures that after closing the channel, the store can continue // to operate correctly. func TestStoreWatchSlowConsumer(t *testing.T) { s := v2store.New() s.Watch("/foo", true, true, 0) // stream must be true // Fill watch channel with 100 events for i := 1; i <= 100; i++ { s.Set("/foo", false, fmt.Sprint(i), v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) // ok } // assert.Equal(t, s.WatcherHub.count, int64(1)) s.Set("/foo", false, "101", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) // ok // remove watcher // assert.Equal(t, s.WatcherHub.count, int64(0)) s.Set("/foo", false, "102", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) // must not panic } // Performs a non-blocking select on an event channel. func nbselect(c <-chan *v2store.Event) *v2store.Event { select { case e := <-c: return e default: return nil } } // Performs a non-blocking select on an event channel. func timeoutSelect(t *testing.T, c <-chan *v2store.Event) *v2store.Event { select { case e := <-c: return e case <-time.After(time.Second): t.Errorf("timed out waiting on event") return nil } } ================================================ FILE: tests/integration/v3_alarm_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "os" "path/filepath" "sync" "testing" "time" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease/leasepb" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestV3StorageQuotaApply tests the V3 server respects quotas during apply func TestV3StorageQuotaApply(t *testing.T) { integration.BeforeTest(t) quotasize := int64(16 * os.Getpagesize()) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 2}) defer clus.Terminate(t) kvc1 := integration.ToGRPC(clus.Client(1)).KV // Set a quota on one node clus.Members[0].QuotaBackendBytes = quotasize clus.Members[0].Stop(t) clus.Members[0].Restart(t) clus.WaitMembersForLeader(t, clus.Members) kvc0 := integration.ToGRPC(clus.Client(0)).KV waitForRestart(t, kvc0) key := []byte("abc") // test small put still works smallbuf := make([]byte, 1024) _, serr := kvc0.Put(t.Context(), &pb.PutRequest{Key: key, Value: smallbuf}) require.NoError(t, serr) // test big put bigbuf := make([]byte, quotasize) _, err := kvc1.Put(t.Context(), &pb.PutRequest{Key: key, Value: bigbuf}) require.NoError(t, err) // quorum get should work regardless of whether alarm is raised _, err = kvc0.Range(t.Context(), &pb.RangeRequest{Key: []byte("foo")}) require.NoError(t, err) // wait until alarm is raised for sure-- poll the alarms stopc := time.After(5 * time.Second) for { req := &pb.AlarmRequest{Action: pb.AlarmRequest_GET} resp, aerr := clus.Members[0].Server.Alarm(t.Context(), req) require.NoError(t, aerr) if len(resp.Alarms) != 0 { break } select { case <-stopc: t.Fatalf("timed out waiting for alarm") case <-time.After(10 * time.Millisecond): } } // txn with non-mutating Ops should go through when NOSPACE alarm is raised _, err = kvc0.Txn(t.Context(), &pb.TxnRequest{ Compare: []*pb.Compare{ { Key: key, Result: pb.Compare_EQUAL, Target: pb.Compare_CREATE, TargetUnion: &pb.Compare_CreateRevision{CreateRevision: 0}, }, }, Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestDeleteRange{ RequestDeleteRange: &pb.DeleteRangeRequest{ Key: key, }, }, }, }, }) require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), integration.RequestWaitTimeout) defer cancel() // small quota machine should reject put _, err = kvc0.Put(ctx, &pb.PutRequest{Key: key, Value: smallbuf}) require.Errorf(t, err, "past-quota instance should reject put") // large quota machine should reject put _, err = kvc1.Put(ctx, &pb.PutRequest{Key: key, Value: smallbuf}) require.Errorf(t, err, "past-quota instance should reject put") // reset large quota node to ensure alarm persisted clus.Members[1].Stop(t) clus.Members[1].Restart(t) clus.WaitMembersForLeader(t, clus.Members) _, err = kvc1.Put(t.Context(), &pb.PutRequest{Key: key, Value: smallbuf}) require.Errorf(t, err, "alarmed instance should reject put after reset") } // TestV3AlarmDeactivate ensures that space alarms can be deactivated so puts go through. func TestV3AlarmDeactivate(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV mt := integration.ToGRPC(clus.RandClient()).Maintenance alarmReq := &pb.AlarmRequest{ MemberID: 123, Action: pb.AlarmRequest_ACTIVATE, Alarm: pb.AlarmType_NOSPACE, } _, err := mt.Alarm(t.Context(), alarmReq) require.NoError(t, err) key := []byte("abc") smallbuf := make([]byte, 512) _, err = kvc.Put(t.Context(), &pb.PutRequest{Key: key, Value: smallbuf}) if err == nil && !eqErrGRPC(err, rpctypes.ErrGRPCNoSpace) { t.Fatalf("put got %v, expected %v", err, rpctypes.ErrGRPCNoSpace) } alarmReq.Action = pb.AlarmRequest_DEACTIVATE _, err = mt.Alarm(t.Context(), alarmReq) require.NoError(t, err) _, err = kvc.Put(t.Context(), &pb.PutRequest{Key: key, Value: smallbuf}) require.NoError(t, err) } func TestV3CorruptAlarm(t *testing.T) { integration.BeforeTest(t) lg := zaptest.NewLogger(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) var wg sync.WaitGroup wg.Add(10) for i := 0; i < 10; i++ { go func() { defer wg.Done() if _, err := clus.Client(0).Put(t.Context(), "k", "v"); err != nil { t.Error(err) } }() } wg.Wait() // Corrupt member 0 by modifying backend offline. clus.Members[0].Stop(t) fp := filepath.Join(clus.Members[0].DataDir, "member", "snap", "db") be := backend.NewDefaultBackend(lg, fp) s := mvcc.NewStore(lg, be, nil, mvcc.StoreConfig{}) // NOTE: cluster_proxy mode with namespacing won't set 'k', but namespace/'k'. s.Put([]byte("abc"), []byte("def"), 0) s.Put([]byte("xyz"), []byte("123"), 0) s.Compact(traceutil.TODO(), 5) s.Commit() s.Close() be.Close() clus.Members[1].WaitOK(t) clus.Members[2].WaitOK(t) time.Sleep(time.Second * 2) // Wait for cluster so Puts succeed in case member 0 was the leader. _, err := clus.Client(1).Get(t.Context(), "k") require.NoError(t, err) _, err = clus.Client(1).Put(t.Context(), "xyz", "321") require.NoError(t, err) _, err = clus.Client(1).Put(t.Context(), "abc", "fed") require.NoError(t, err) // Restart with corruption checking enabled. clus.Members[1].Stop(t) clus.Members[2].Stop(t) for _, m := range clus.Members { m.CorruptCheckTime = time.Second m.Restart(t) } clus.WaitLeader(t) time.Sleep(time.Second * 2) clus.Members[0].WaitStarted(t) resp0, err0 := clus.Client(0).Get(t.Context(), "abc") require.NoError(t, err0) clus.Members[1].WaitStarted(t) resp1, err1 := clus.Client(1).Get(t.Context(), "abc") require.NoError(t, err1) require.NotEqualf(t, resp0.Kvs[0].ModRevision, resp1.Kvs[0].ModRevision, "matching ModRevision values") for i := 0; i < 5; i++ { presp, perr := clus.Client(0).Put(t.Context(), "abc", "aaa") if perr != nil { if eqErrGRPC(perr, rpctypes.ErrCorrupt) { return } t.Fatalf("expected %v, got %+v (%v)", rpctypes.ErrCorrupt, presp, perr) } time.Sleep(time.Second) } t.Fatalf("expected error %v after %s", rpctypes.ErrCorrupt, 5*time.Second) } func TestV3CorruptAlarmWithLeaseCorrupted(t *testing.T) { integration.BeforeTest(t) lg := zaptest.NewLogger(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ CorruptCheckTime: time.Second, Size: 3, SnapshotCount: 10, SnapshotCatchUpEntries: 5, DisableStrictReconfigCheck: true, }) defer clus.Terminate(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() lresp, err := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant(ctx, &pb.LeaseGrantRequest{ID: 1, TTL: 60}) if err != nil { t.Errorf("could not create lease 1 (%v)", err) } if lresp.ID != 1 { t.Errorf("got id %v, wanted id %v", lresp.ID, 1) } putr := &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar"), Lease: lresp.ID} // Trigger snapshot from the leader to new member for i := 0; i < 15; i++ { _, err = integration.ToGRPC(clus.RandClient()).KV.Put(ctx, putr) if err != nil { t.Errorf("#%d: couldn't put key (%v)", i, err) } } require.NoError(t, clus.RemoveMember(t, clus.Client(1), uint64(clus.Members[2].ID()))) clus.WaitMembersForLeader(t, clus.Members) clus.AddMember(t) clus.WaitMembersForLeader(t, clus.Members) // Wait for new member to catch up integration.WaitClientV3(t, clus.Members[2].Client) // Corrupt member 2 by modifying backend lease bucket offline. clus.Members[2].Stop(t) fp := filepath.Join(clus.Members[2].DataDir, "member", "snap", "db") bcfg := backend.DefaultBackendConfig(lg) bcfg.Path = fp be := backend.New(bcfg) olpb := leasepb.Lease{ID: int64(1), TTL: 60} tx := be.BatchTx() schema.UnsafeDeleteLease(tx, &olpb) lpb := leasepb.Lease{ID: int64(2), TTL: 60} schema.MustUnsafePutLease(tx, &lpb) tx.Commit() require.NoError(t, be.Close()) require.NoError(t, clus.Members[2].Restart(t)) clus.Members[1].WaitOK(t) clus.Members[2].WaitOK(t) // Revoke lease should remove key except the member with corruption _, err = integration.ToGRPC(clus.Members[0].Client).Lease.LeaseRevoke(ctx, &pb.LeaseRevokeRequest{ID: lresp.ID}) require.NoError(t, err) resp0, err0 := clus.Members[1].Client.KV.Get(t.Context(), "foo") require.NoError(t, err0) resp1, err1 := clus.Members[2].Client.KV.Get(t.Context(), "foo") require.NoError(t, err1) require.NotEqualf(t, resp0.Header.Revision, resp1.Header.Revision, "matching Revision values") // Wait for CorruptCheckTime time.Sleep(time.Second) presp, perr := clus.Client(0).Put(t.Context(), "abc", "aaa") if perr != nil { if eqErrGRPC(perr, rpctypes.ErrCorrupt) { return } t.Fatalf("expected %v, got %+v (%v)", rpctypes.ErrCorrupt, presp, perr) } } ================================================ FILE: tests/integration/v3_auth_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "fmt" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/authpb" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestV3AuthEmptyUserGet ensures that a get with an empty user will return an empty user error. func TestV3AuthEmptyUserGet(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() api := integration.ToGRPC(clus.Client(0)) authSetupRoot(t, api.Auth) _, err := api.KV.Range(ctx, &pb.RangeRequest{Key: []byte("abc")}) require.Truef(t, eqErrGRPC(err, rpctypes.ErrUserEmpty), "got %v, expected %v", err, rpctypes.ErrUserEmpty) } // TestV3AuthEmptyUserPut ensures that a put with an empty user will return an empty user error, // and the consistent_index should be moved forward even the apply-->Put fails. func TestV3AuthEmptyUserPut(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 1, SnapshotCount: 3, }) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() api := integration.ToGRPC(clus.Client(0)) authSetupRoot(t, api.Auth) // The SnapshotCount is 3, so there must be at least 3 new snapshot files being created. // The VERIFY logic will check whether the consistent_index >= last snapshot index on // cluster terminating. for i := 0; i < 10; i++ { _, err := api.KV.Put(ctx, &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}) require.Truef(t, eqErrGRPC(err, rpctypes.ErrUserEmpty), "got %v, expected %v", err, rpctypes.ErrUserEmpty) } } // TestV3AuthTokenWithDisable tests that auth won't crash if // given a valid token when authentication is disabled func TestV3AuthTokenWithDisable(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) authSetupRoot(t, integration.ToGRPC(clus.Client(0)).Auth) c, cerr := integration.NewClient(t, clientv3.Config{Endpoints: clus.Client(0).Endpoints(), Username: "root", Password: "123"}) require.NoError(t, cerr) defer c.Close() rctx, cancel := context.WithCancel(t.Context()) donec := make(chan struct{}) go func() { defer close(donec) for rctx.Err() == nil { c.Put(rctx, "abc", "def") } }() time.Sleep(10 * time.Millisecond) _, err := c.AuthDisable(t.Context()) require.NoError(t, err) time.Sleep(10 * time.Millisecond) cancel() <-donec } func TestV3AuthRevision(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) api := integration.ToGRPC(clus.Client(0)) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) presp, perr := api.KV.Put(ctx, &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}) cancel() require.NoError(t, perr) rev := presp.Header.Revision ctx, cancel = context.WithTimeout(t.Context(), 5*time.Second) aresp, aerr := api.Auth.UserAdd(ctx, &pb.AuthUserAddRequest{Name: "root", Password: "123", Options: &authpb.UserAddOptions{NoPassword: false}}) cancel() require.NoError(t, aerr) require.Equalf(t, aresp.Header.Revision, rev, "revision expected %d, got %d", rev, aresp.Header.Revision) } // TestV3AuthWithLeaseRevokeWithRoot ensures that granted leases // with root user be revoked after TTL. func TestV3AuthWithLeaseRevokeWithRoot(t *testing.T) { testV3AuthWithLeaseRevokeWithRoot(t, integration.ClusterConfig{Size: 1}) } // TestV3AuthWithLeaseRevokeWithRootJWT creates a lease with a JWT-token enabled cluster. // And tests if server is able to revoke expiry lease item. func TestV3AuthWithLeaseRevokeWithRootJWT(t *testing.T) { testV3AuthWithLeaseRevokeWithRoot(t, integration.ClusterConfig{Size: 1, AuthToken: integration.DefaultTokenJWT}) } func testV3AuthWithLeaseRevokeWithRoot(t *testing.T, ccfg integration.ClusterConfig) { integration.BeforeTest(t) clus := integration.NewCluster(t, &ccfg) defer clus.Terminate(t) api := integration.ToGRPC(clus.Client(0)) authSetupRoot(t, api.Auth) rootc, cerr := integration.NewClient(t, clientv3.Config{ Endpoints: clus.Client(0).Endpoints(), Username: "root", Password: "123", }) require.NoError(t, cerr) defer rootc.Close() leaseResp, err := rootc.Grant(t.Context(), 2) require.NoError(t, err) leaseID := leaseResp.ID _, err = rootc.Put(t.Context(), "foo", "bar", clientv3.WithLease(leaseID)) require.NoError(t, err) // wait for lease expire time.Sleep(3 * time.Second) tresp, terr := rootc.TimeToLive( t.Context(), leaseID, clientv3.WithAttachedKeys(), ) if terr != nil { t.Error(terr) } if len(tresp.Keys) > 0 || tresp.GrantedTTL != 0 { t.Errorf("lease %016x should have been revoked, got %+v", leaseID, tresp) } if tresp.TTL != -1 { t.Errorf("lease %016x should have been expired, got %+v", leaseID, tresp) } } type user struct { name string password string role string key string end string } func TestV3AuthWithLeaseRevoke(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) users := []user{ { name: "user1", password: "user1-123", role: "role1", key: "k1", end: "k2", }, } authSetupUsers(t, integration.ToGRPC(clus.Client(0)).Auth, users) authSetupRoot(t, integration.ToGRPC(clus.Client(0)).Auth) rootc, cerr := integration.NewClient(t, clientv3.Config{Endpoints: clus.Client(0).Endpoints(), Username: "root", Password: "123"}) require.NoError(t, cerr) defer rootc.Close() leaseResp, err := rootc.Grant(t.Context(), 90) require.NoError(t, err) leaseID := leaseResp.ID // permission of k3 isn't granted to user1 _, err = rootc.Put(t.Context(), "k3", "val", clientv3.WithLease(leaseID)) require.NoError(t, err) userc, cerr := integration.NewClient(t, clientv3.Config{Endpoints: clus.Client(0).Endpoints(), Username: "user1", Password: "user1-123"}) require.NoError(t, cerr) defer userc.Close() _, err = userc.Revoke(t.Context(), leaseID) if err == nil { t.Fatal("revoking from user1 should be failed with permission denied") } } func TestV3AuthWithLeaseRenew(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) users := []user{ { name: "test-user", password: "test-user-123", role: "test-role", // test-user can only write keys in [k1, k3), i.e. k1 and k2. key: "k1", end: "k3", }, } authSetupUsers(t, integration.ToGRPC(clus.Client(0)).Auth, users) authSetupRoot(t, integration.ToGRPC(clus.Client(0)).Auth) rootCli, cerr := integration.NewClient(t, clientv3.Config{ Endpoints: clus.Client(0).Endpoints(), Username: "root", Password: "123", }) require.NoError(t, cerr) defer rootCli.Close() testUserClis := []*clientv3.Client{} for i := 0; i < len(clus.Members); i++ { testUserCli, err := integration.NewClient(t, clientv3.Config{ Endpoints: clus.Client(i).Endpoints(), Username: "test-user", Password: "test-user-123", }) require.NoError(t, err) defer testUserCli.Close() testUserClis = append(testUserClis, testUserCli) } anonCli, cerr := integration.NewClient(t, clientv3.Config{ Endpoints: clus.Client(0).Endpoints(), }) require.NoError(t, cerr) defer anonCli.Close() leaseResp, err := rootCli.Grant(t.Context(), 90) require.NoError(t, err) leaseID := leaseResp.ID _, err = rootCli.Put(t.Context(), "k1", "val", clientv3.WithLease(leaseID)) require.NoError(t, err) _, err = rootCli.Put(t.Context(), "k3", "val", clientv3.WithLease(leaseID)) require.NoError(t, err) _, err = anonCli.KeepAliveOnce(t.Context(), leaseID) require.ErrorContainsf(t, err, "etcdserver: user name is empty", "should reject renew") _, err = rootCli.KeepAliveOnce(t.Context(), leaseID) require.NoError(t, err) for _, testUserCli := range testUserClis { _, err = testUserCli.KeepAliveOnce(t.Context(), leaseID) require.ErrorContainsf(t, err, "etcdserver: permission denied", "[%v] should reject renew", testUserCli.Endpoints()) } leaseResp, err = rootCli.Grant(t.Context(), 90) require.NoError(t, err) leaseID = leaseResp.ID _, err = rootCli.Put(t.Context(), "k1", "val", clientv3.WithLease(leaseID)) require.NoError(t, err) _, err = rootCli.Put(t.Context(), "k2", "val", clientv3.WithLease(leaseID)) require.NoError(t, err) for _, testUserCli := range testUserClis { _, err = testUserCli.KeepAliveOnce(t.Context(), leaseID) require.NoErrorf(t, err, "[%v] should accept renew", testUserCli.Endpoints()) } } func TestV3AuthWithLeaseAttach(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) users := []user{ { name: "user1", password: "user1-123", role: "role1", key: "k1", end: "k3", }, { name: "user2", password: "user2-123", role: "role2", key: "k2", end: "k4", }, } authSetupUsers(t, integration.ToGRPC(clus.Client(0)).Auth, users) authSetupRoot(t, integration.ToGRPC(clus.Client(0)).Auth) user1c, cerr := integration.NewClient(t, clientv3.Config{Endpoints: clus.Client(0).Endpoints(), Username: "user1", Password: "user1-123"}) require.NoError(t, cerr) defer user1c.Close() user2c, cerr := integration.NewClient(t, clientv3.Config{Endpoints: clus.Client(0).Endpoints(), Username: "user2", Password: "user2-123"}) require.NoError(t, cerr) defer user2c.Close() leaseResp, err := user1c.Grant(t.Context(), 90) require.NoError(t, err) leaseID := leaseResp.ID // permission of k2 is also granted to user2 _, err = user1c.Put(t.Context(), "k2", "val", clientv3.WithLease(leaseID)) require.NoError(t, err) _, err = user2c.Revoke(t.Context(), leaseID) require.NoError(t, err) leaseResp, err = user1c.Grant(t.Context(), 90) require.NoError(t, err) leaseID = leaseResp.ID // permission of k1 isn't granted to user2 _, err = user1c.Put(t.Context(), "k1", "val", clientv3.WithLease(leaseID)) require.NoError(t, err) _, err = user2c.Revoke(t.Context(), leaseID) if err == nil { t.Fatal("revoking from user2 should be failed with permission denied") } } func authSetupUsers(t *testing.T, auth pb.AuthClient, users []user) { for _, user := range users { _, err := auth.UserAdd(t.Context(), &pb.AuthUserAddRequest{Name: user.name, Password: user.password, Options: &authpb.UserAddOptions{NoPassword: false}}) require.NoError(t, err) _, err = auth.RoleAdd(t.Context(), &pb.AuthRoleAddRequest{Name: user.role}) require.NoError(t, err) _, err = auth.UserGrantRole(t.Context(), &pb.AuthUserGrantRoleRequest{User: user.name, Role: user.role}) require.NoError(t, err) if len(user.key) == 0 { continue } perm := &authpb.Permission{ PermType: authpb.Permission_READWRITE, Key: []byte(user.key), RangeEnd: []byte(user.end), } _, err = auth.RoleGrantPermission(t.Context(), &pb.AuthRoleGrantPermissionRequest{Name: user.role, Perm: perm}) require.NoError(t, err) } } func authSetupRoot(t *testing.T, auth pb.AuthClient) { root := []user{ { name: "root", password: "123", role: "root", key: "", }, } authSetupUsers(t, auth, root) _, err := auth.AuthEnable(t.Context(), &pb.AuthEnableRequest{}) require.NoError(t, err) } func TestV3AuthNonAuthorizedRPCs(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) nonAuthedKV := clus.Client(0).KV key := "foo" val := "bar" _, err := nonAuthedKV.Put(t.Context(), key, val) require.NoErrorf(t, err, "couldn't put key (%v)", err) authSetupRoot(t, integration.ToGRPC(clus.Client(0)).Auth) respput, err := nonAuthedKV.Put(t.Context(), key, val) require.Truef(t, eqErrGRPC(err, rpctypes.ErrGRPCUserEmpty), "could put key (%v), it should cause an error of permission denied", respput) } func TestV3AuthNestedTxnPermissionDenied(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) users := []user{ { name: "user1", password: "user1-123", role: "role1", key: "foo", end: "zoo", }, } authSetupUsers(t, integration.ToGRPC(clus.Client(0)).Auth, users) authSetupRoot(t, integration.ToGRPC(clus.Client(0)).Auth) rootc, err := integration.NewClient(t, clientv3.Config{ Endpoints: clus.Client(0).Endpoints(), Username: "root", Password: "123", }) require.NoError(t, err) defer rootc.Close() userc, err := integration.NewClient(t, clientv3.Config{ Endpoints: clus.Client(0).Endpoints(), Username: "user1", Password: "user1-123", }) require.NoError(t, err) defer userc.Close() _, err = rootc.Put(t.Context(), "boo", "bar") require.NoError(t, err) txn := &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{ Success: []*pb.RequestOp{ { Request: &pb.RequestOp_RequestDeleteRange{ RequestDeleteRange: &pb.DeleteRangeRequest{ Key: []byte("boo"), }, }, }, }, }, }, }, }, } _, err = integration.ToGRPC(userc).KV.Txn(t.Context(), txn) require.Error(t, err) require.Truef(t, eqErrGRPC(err, rpctypes.ErrGRPCPermissionDenied), "got %v, expected %v", err, rpctypes.ErrGRPCPermissionDenied) resp, err := rootc.Get(t.Context(), "boo") require.NoError(t, err) require.Len(t, resp.Kvs, 1) require.Equal(t, resp.Kvs[0].Value, []byte("bar")) } func TestV3AuthOldRevConcurrent(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) authSetupRoot(t, integration.ToGRPC(clus.Client(0)).Auth) c, cerr := integration.NewClient(t, clientv3.Config{ Endpoints: clus.Client(0).Endpoints(), DialTimeout: 5 * time.Second, Username: "root", Password: "123", }) require.NoError(t, cerr) defer c.Close() var wg sync.WaitGroup f := func(i int) { defer wg.Done() role, user := fmt.Sprintf("test-role-%d", i), fmt.Sprintf("test-user-%d", i) _, err := c.RoleAdd(t.Context(), role) require.NoError(t, err) _, err = c.RoleGrantPermission(t.Context(), role, "\x00", clientv3.GetPrefixRangeEnd(""), clientv3.PermissionType(clientv3.PermReadWrite)) require.NoError(t, err) _, err = c.UserAdd(t.Context(), user, "123") require.NoError(t, err) _, err = c.Put(t.Context(), "a", "b") assert.NoError(t, err) } // needs concurrency to trigger numRoles := 2 wg.Add(numRoles) for i := 0; i < numRoles; i++ { go f(i) } wg.Wait() } func TestV3AuthWatchErrorAndWatchId0(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() users := []user{ { name: "user1", password: "user1-123", role: "role1", key: "k1", end: "k2", }, } authSetupUsers(t, integration.ToGRPC(clus.Client(0)).Auth, users) authSetupRoot(t, integration.ToGRPC(clus.Client(0)).Auth) c, cerr := integration.NewClient(t, clientv3.Config{Endpoints: clus.Client(0).Endpoints(), Username: "user1", Password: "user1-123"}) require.NoError(t, cerr) defer c.Close() watchStartCh, watchEndCh := make(chan any), make(chan any) go func() { wChan := c.Watch(ctx, "k1", clientv3.WithRev(1)) watchStartCh <- struct{}{} watchResponse := <-wChan t.Logf("watch response from k1: %v", watchResponse) assert.NotEmpty(t, watchResponse.Events) watchEndCh <- struct{}{} }() // Chan for making sure that the above goroutine invokes Watch() // So the above Watch() can get watch ID = 0 <-watchStartCh wChan := c.Watch(ctx, "non-allowed-key", clientv3.WithRev(1)) watchResponse := <-wChan require.Error(t, watchResponse.Err()) // permission denied _, err := c.Put(ctx, "k1", "val") require.NoErrorf(t, err, "Unexpected error from Put: %v", err) <-watchEndCh } ================================================ FILE: tests/integration/v3_election_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "fmt" "sync" "testing" "time" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestElectionWait tests if followers can correctly wait for elections. func TestElectionWait(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) leaders := 3 followers := 3 var clients []*clientv3.Client newClient := integration.MakeMultiNodeClients(t, clus, &clients) defer func() { integration.CloseClients(t, clients) }() electedc := make(chan string) var nextc []chan struct{} // wait for all elections donec := make(chan struct{}) for i := 0; i < followers; i++ { nextc = append(nextc, make(chan struct{})) go func(ch chan struct{}) { for j := 0; j < leaders; j++ { session, err := concurrency.NewSession(newClient()) if err != nil { t.Error(err) } b := concurrency.NewElection(session, "test-election") cctx, cancel := context.WithCancel(t.Context()) defer cancel() s, ok := <-b.Observe(cctx) if !ok { t.Errorf("could not observe election; channel closed") } electedc <- string(s.Kvs[0].Value) // wait for next election round <-ch session.Orphan() } donec <- struct{}{} }(nextc[i]) } // elect some leaders for i := 0; i < leaders; i++ { go func() { session, err := concurrency.NewSession(newClient()) if err != nil { t.Error(err) } defer session.Orphan() e := concurrency.NewElection(session, "test-election") ev := fmt.Sprintf("electval-%v", time.Now().UnixNano()) if err := e.Campaign(t.Context(), ev); err != nil { t.Errorf("failed volunteer (%v)", err) } // wait for followers to accept leadership for j := 0; j < followers; j++ { s := <-electedc if s != ev { t.Errorf("wrong election value got %s, wanted %s", s, ev) } } // let next leader take over if err := e.Resign(t.Context()); err != nil { t.Errorf("failed resign (%v)", err) } // tell followers to start listening for next leader for j := 0; j < followers; j++ { nextc[j] <- struct{}{} } }() } // wait on followers for i := 0; i < followers; i++ { <-donec } } // TestElectionFailover tests that an election will func TestElectionFailover(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) cctx, cancel := context.WithCancel(t.Context()) defer cancel() ss := make([]*concurrency.Session, 3) for i := 0; i < 3; i++ { var err error ss[i], err = concurrency.NewSession(clus.Client(i)) if err != nil { t.Error(err) } defer ss[i].Orphan() } // first leader (elected) e := concurrency.NewElection(ss[0], "test-election") err := e.Campaign(t.Context(), "foo") require.NoErrorf(t, err, "failed volunteer") // check first leader resp, ok := <-e.Observe(cctx) require.Truef(t, ok, "could not wait for first election; channel closed") s := string(resp.Kvs[0].Value) require.Equalf(t, "foo", s, "wrong election result. got %s, wanted foo", s) // next leader electedErrC := make(chan error, 1) go func() { ee := concurrency.NewElection(ss[1], "test-election") eer := ee.Campaign(t.Context(), "bar") electedErrC <- eer // If eer != nil, the test will fail by calling t.Fatal(eer) }() // invoke leader failover err = ss[0].Close() require.NoError(t, err) // check new leader e = concurrency.NewElection(ss[2], "test-election") resp, ok = <-e.Observe(cctx) require.Truef(t, ok, "could not wait for second election; channel closed") s = string(resp.Kvs[0].Value) require.Equalf(t, "bar", s, "wrong election result. got %s, wanted bar", s) // leader must ack election (otherwise, Campaign may see closed conn) eer := <-electedErrC require.NoError(t, eer) } // TestElectionSessionRecampaign ensures that campaigning twice on the same election // with the same lock will Proclaim instead of deadlocking. func TestElectionSessionRecampaign(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.RandClient() session, err := concurrency.NewSession(cli) if err != nil { t.Error(err) } defer session.Orphan() e := concurrency.NewElection(session, "test-elect") err = e.Campaign(t.Context(), "abc") require.NoError(t, err) e2 := concurrency.NewElection(session, "test-elect") err = e2.Campaign(t.Context(), "def") require.NoError(t, err) ctx, cancel := context.WithCancel(t.Context()) defer cancel() if resp := <-e.Observe(ctx); len(resp.Kvs) == 0 || string(resp.Kvs[0].Value) != "def" { t.Fatalf("expected value=%q, got response %v", "def", resp) } } // TestElectionOnPrefixOfExistingKey checks that a single // candidate can be elected on a new key that is a prefix // of an existing key. To wit, check for regression // of bug #6278. https://github.com/etcd-io/etcd/issues/6278 func TestElectionOnPrefixOfExistingKey(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.RandClient() _, err := cli.Put(t.Context(), "testa", "value") require.NoError(t, err) s, serr := concurrency.NewSession(cli) require.NoError(t, serr) e := concurrency.NewElection(s, "test") ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) err = e.Campaign(ctx, "abc") cancel() // after 5 seconds, deadlock results in // 'context deadline exceeded' here. require.NoError(t, err) } // TestElectionOnSessionRestart tests that a quick restart of leader (resulting // in a new session with the same lease id) does not result in loss of // leadership. func TestElectionOnSessionRestart(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.RandClient() session, err := concurrency.NewSession(cli) require.NoError(t, err) e := concurrency.NewElection(session, "test-elect") require.NoError(t, e.Campaign(t.Context(), "abc")) // ensure leader is not lost to waiter on fail-over waitSession, werr := concurrency.NewSession(cli) require.NoError(t, werr) defer waitSession.Orphan() waitCtx, waitCancel := context.WithTimeout(t.Context(), 5*time.Second) defer waitCancel() go concurrency.NewElection(waitSession, "test-elect").Campaign(waitCtx, "123") // simulate restart by reusing the lease from the old session newSession, nerr := concurrency.NewSession(cli, concurrency.WithLease(session.Lease())) require.NoError(t, nerr) defer newSession.Orphan() newElection := concurrency.NewElection(newSession, "test-elect") require.NoError(t, newElection.Campaign(t.Context(), "def")) ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() if resp := <-newElection.Observe(ctx); len(resp.Kvs) == 0 || string(resp.Kvs[0].Value) != "def" { t.Errorf("expected value=%q, got response %v", "def", resp) } } // TestElectionObserveCompacted checks that observe can tolerate // a leader key with a modrev less than the compaction revision. func TestElectionObserveCompacted(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) session, err := concurrency.NewSession(cli) require.NoError(t, err) defer session.Orphan() e := concurrency.NewElection(session, "test-elect") require.NoError(t, e.Campaign(t.Context(), "abc")) presp, perr := cli.Put(t.Context(), "foo", "bar") require.NoError(t, perr) _, cerr := cli.Compact(t.Context(), presp.Header.Revision) require.NoError(t, cerr) v, ok := <-e.Observe(t.Context()) if !ok { t.Fatal("failed to observe on compacted revision") } require.Equalf(t, "abc", string(v.Kvs[0].Value), `expected leader value "abc", got %q`, string(v.Kvs[0].Value)) } // TestElectionWithAuthEnabled verifies the election interface when auth is enabled. // Refer to the discussion in https://github.com/etcd-io/etcd/issues/17502 func TestElectionWithAuthEnabled(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) users := []user{ { name: "user1", password: "123", role: "role1", key: "/foo1", // prefix /foo1 end: "/foo2", }, { name: "user2", password: "456", role: "role2", key: "/bar1", // prefix /bar1 end: "/bar2", }, } t.Log("Setting rbac info and enable auth.") authSetupUsers(t, integration.ToGRPC(clus.Client(0)).Auth, users) authSetupRoot(t, integration.ToGRPC(clus.Client(0)).Auth) c1, c1err := integration.NewClient(t, clientv3.Config{Endpoints: clus.Client(0).Endpoints(), Username: "user1", Password: "123"}) require.NoError(t, c1err) defer c1.Close() c2, c2err := integration.NewClient(t, clientv3.Config{Endpoints: clus.Client(0).Endpoints(), Username: "user2", Password: "456"}) require.NoError(t, c2err) defer c2.Close() campaigns := []struct { name string c *clientv3.Client pfx string sleepTime time.Duration // time to sleep before campaigning }{ { name: "client1 first campaign", c: c1, pfx: "/foo1/a", }, { name: "client1 second campaign", c: c1, pfx: "/foo1/a", }, { name: "client2 first campaign", c: c2, pfx: "/bar1/b", sleepTime: 5 * time.Second, }, { name: "client2 second campaign", c: c2, pfx: "/bar1/b", sleepTime: 6 * time.Second, }, } t.Log("Starting to campaign with multiple users.") var wg sync.WaitGroup errC := make(chan error, 8) doneC := make(chan error) for _, campaign := range campaigns { campaign := campaign wg.Add(1) go func() { defer wg.Done() if campaign.sleepTime > 0 { time.Sleep(campaign.sleepTime) } s, serr := concurrency.NewSession(campaign.c, concurrency.WithTTL(10)) if serr != nil { errC <- fmt.Errorf("[NewSession] %s: %w", campaign.name, serr) } s.Orphan() e := concurrency.NewElection(s, campaign.pfx) eerr := e.Campaign(t.Context(), "whatever") if eerr != nil { errC <- fmt.Errorf("[Campaign] %s: %w", campaign.name, eerr) } }() } go func() { t.Log("Waiting for all goroutines to finish.") defer close(doneC) wg.Wait() }() select { case err := <-errC: t.Fatalf("Error: %v", err) case <-doneC: t.Log("All goroutine done!") case <-time.After(30 * time.Second): t.Fatal("Timed out") } } ================================================ FILE: tests/integration/v3_failover_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "bytes" "context" "crypto/tls" "errors" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" clientv3test "go.etcd.io/etcd/tests/v3/integration/clientv3" ) func TestFailover(t *testing.T) { cases := []struct { name string testFunc func(*testing.T, *tls.Config, *integration.Cluster) (*clientv3.Client, error) }{ { name: "create client before the first server down", testFunc: createClientBeforeServerDown, }, { name: "create client after the first server down", testFunc: createClientAfterServerDown, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { t.Logf("Starting test [%s]", tc.name) integration.BeforeTest(t) // Launch an etcd cluster with 3 members t.Logf("Launching an etcd cluster with 3 members [%s]", tc.name) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, ClientTLS: &integration.TestTLSInfo}) defer clus.Terminate(t) cc, err := integration.TestTLSInfo.ClientConfig() require.NoError(t, err) // Create an etcd client before or after first server down t.Logf("Creating an etcd client [%s]", tc.name) cli, err := tc.testFunc(t, cc, clus) require.NoErrorf(t, err, "Failed to create client") defer cli.Close() // Sanity test t.Logf("Running sanity test [%s]", tc.name) key, val := "key1", "val1" putWithRetries(t, cli, key, val, 10) getWithRetries(t, cli, key, val, 10) t.Logf("Test done [%s]", tc.name) }) } } func createClientBeforeServerDown(t *testing.T, cc *tls.Config, clus *integration.Cluster) (*clientv3.Client, error) { cli, err := createClient(t, cc, clus) if err != nil { return nil, err } clus.Members[0].Close() return cli, nil } func createClientAfterServerDown(t *testing.T, cc *tls.Config, clus *integration.Cluster) (*clientv3.Client, error) { clus.Members[0].Close() return createClient(t, cc, clus) } func createClient(t *testing.T, cc *tls.Config, clus *integration.Cluster) (*clientv3.Client, error) { cli, err := integration.NewClient(t, clientv3.Config{ Endpoints: clus.Endpoints(), DialTimeout: 5 * time.Second, TLS: cc, }) if err != nil { return nil, err } return cli, nil } func putWithRetries(t *testing.T, cli *clientv3.Client, key, val string, retryCount int) { for retryCount > 0 { // put data test err := func() error { t.Log("Sanity test, putting data") ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) defer cancel() if _, putErr := cli.Put(ctx, key, val); putErr != nil { t.Logf("Failed to put data (%v)", putErr) return putErr } return nil }() if err != nil { retryCount-- if shouldRetry(err) { continue } t.Fatal(err) } break } } func getWithRetries(t *testing.T, cli *clientv3.Client, key, val string, retryCount int) { for retryCount > 0 { // get data test err := func() error { t.Log("Sanity test, getting data") ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) defer cancel() resp, getErr := cli.Get(ctx, key) if getErr != nil { t.Logf("Failed to get key (%v)", getErr) return getErr } require.Lenf(t, resp.Kvs, 1, "Expected 1 key, got %d", len(resp.Kvs)) require.Truef(t, bytes.Equal([]byte(val), resp.Kvs[0].Value), "Unexpected value, expected: %s, got: %s", val, resp.Kvs[0].Value) return nil }() if err != nil { retryCount-- if shouldRetry(err) { continue } t.Fatal(err) } break } } func shouldRetry(err error) bool { if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || errors.Is(err, rpctypes.ErrTimeout) || errors.Is(err, rpctypes.ErrTimeoutDueToLeaderFail) { return true } return false } ================================================ FILE: tests/integration/v3_grpc_inflight_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "sync" "testing" "time" "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestV3MaintenanceDefragmentInflightRange ensures inflight range requests // does not panic the mvcc backend while defragment is running. func TestV3MaintenanceDefragmentInflightRange(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.RandClient() kvc := integration.ToGRPC(cli).KV _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}) require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), time.Second) donec := make(chan struct{}) go func() { defer close(donec) kvc.Range(ctx, &pb.RangeRequest{Key: []byte("foo")}) }() mvc := integration.ToGRPC(cli).Maintenance mvc.Defragment(t.Context(), &pb.DefragmentRequest{}) cancel() <-donec } // TestV3KVInflightRangeRequests ensures that inflight requests // (sent before server shutdown) are gracefully handled by server-side. // They are either finished or canceled, but never crash the backend. // See https://github.com/etcd-io/etcd/issues/7322 for more detail. func TestV3KVInflightRangeRequests(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) cli := clus.RandClient() kvc := integration.ToGRPC(cli).KV _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}) require.NoError(t, err) ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) reqN := 10 // use 500+ for fast machine var wg sync.WaitGroup wg.Add(reqN) for i := 0; i < reqN; i++ { go func() { defer wg.Done() _, err := kvc.Range(ctx, &pb.RangeRequest{Key: []byte("foo"), Serializable: true}, grpc.WaitForReady(true)) if err != nil { errCode := status.Convert(err).Code() errDesc := rpctypes.ErrorDesc(err) if err != nil && !(errDesc == context.Canceled.Error() || errCode == codes.Canceled || errCode == codes.Unavailable) { t.Errorf("inflight request should be canceled with '%v' or code Canceled or Unavailable, got '%v' with code '%s'", context.Canceled.Error(), errDesc, errCode) } } }() } clus.Members[0].Stop(t) cancel() wg.Wait() } ================================================ FILE: tests/integration/v3_grpc_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "bytes" "context" "errors" "fmt" "math/rand" "os" "reflect" "strings" "testing" "time" "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestV3PutOverwrite puts a key with the v3 api to a random Cluster member, // overwrites it, then checks that the change was applied. func TestV3PutOverwrite(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV key := []byte("foo") reqput := &pb.PutRequest{Key: key, Value: []byte("bar"), PrevKv: true} respput, err := kvc.Put(t.Context(), reqput) require.NoErrorf(t, err, "couldn't put key") // overwrite reqput.Value = []byte("baz") respput2, err := kvc.Put(t.Context(), reqput) require.NoErrorf(t, err, "couldn't put key") require.Greaterf(t, respput2.Header.Revision, respput.Header.Revision, "expected newer revision on overwrite, got %v <= %v", respput2.Header.Revision, respput.Header.Revision) if pkv := respput2.PrevKv; pkv == nil || string(pkv.Value) != "bar" { t.Fatalf("expected PrevKv=bar, got response %+v", respput2) } reqrange := &pb.RangeRequest{Key: key} resprange, err := kvc.Range(t.Context(), reqrange) require.NoErrorf(t, err, "couldn't get key") require.Lenf(t, resprange.Kvs, 1, "expected 1 key, got %v", len(resprange.Kvs)) kv := resprange.Kvs[0] if kv.ModRevision <= kv.CreateRevision { t.Errorf("expected modRev > createRev, got %d <= %d", kv.ModRevision, kv.CreateRevision) } if !reflect.DeepEqual(reqput.Value, kv.Value) { t.Errorf("expected value %v, got %v", reqput.Value, kv.Value) } } // TestV3PutRestart checks if a put after an unrelated member restart succeeds func TestV3PutRestart(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) kvIdx := rand.Intn(3) kvc := integration.ToGRPC(clus.Client(kvIdx)).KV stopIdx := kvIdx for stopIdx == kvIdx { stopIdx = rand.Intn(3) } clus.Client(stopIdx).Close() clus.Members[stopIdx].Stop(t) clus.Members[stopIdx].Restart(t) c, cerr := integration.NewClientV3(clus.Members[stopIdx]) require.NoErrorf(t, cerr, "cannot create client") clus.Members[stopIdx].ServerClient = c ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() reqput := &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")} _, err := kvc.Put(ctx, reqput) if err != nil && errors.Is(err, ctx.Err()) { t.Fatalf("expected grpc error, got local ctx error (%v)", err) } } // TestV3CompactCurrentRev ensures keys are present when compacting on current revision. func TestV3CompactCurrentRev(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV preq := &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")} for i := 0; i < 3; i++ { _, err := kvc.Put(t.Context(), preq) require.NoErrorf(t, err, "couldn't put key") } // get key to add to proxy cache, if any _, err := kvc.Range(t.Context(), &pb.RangeRequest{Key: []byte("foo")}) require.NoError(t, err) // compact on current revision _, err = kvc.Compact(t.Context(), &pb.CompactionRequest{Revision: 4}) require.NoErrorf(t, err, "couldn't compact kv space") // key still exists when linearized? _, err = kvc.Range(t.Context(), &pb.RangeRequest{Key: []byte("foo")}) require.NoErrorf(t, err, "couldn't get key after compaction") // key still exists when serialized? _, err = kvc.Range(t.Context(), &pb.RangeRequest{Key: []byte("foo"), Serializable: true}) require.NoErrorf(t, err, "couldn't get serialized key after compaction") } // TestV3HashKV ensures that multiple calls of HashKV on same node return same hash and compact rev. func TestV3HashKV(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV mvc := integration.ToGRPC(clus.RandClient()).Maintenance for i := 0; i < 10; i++ { resp, err := kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte(fmt.Sprintf("bar%d", i))}) require.NoError(t, err) rev := resp.Header.Revision hresp, err := mvc.HashKV(t.Context(), &pb.HashKVRequest{Revision: 0}) require.NoError(t, err) if rev != hresp.Header.Revision { t.Fatalf("Put rev %v != HashKV rev %v", rev, hresp.Header.Revision) } prevHash := hresp.Hash prevCompactRev := hresp.CompactRevision for i := 0; i < 10; i++ { hresp, err := mvc.HashKV(t.Context(), &pb.HashKVRequest{Revision: 0}) require.NoError(t, err) if rev != hresp.Header.Revision { t.Fatalf("Put rev %v != HashKV rev %v", rev, hresp.Header.Revision) } if prevHash != hresp.Hash { t.Fatalf("prevHash %v != Hash %v", prevHash, hresp.Hash) } if prevCompactRev != hresp.CompactRevision { t.Fatalf("prevCompactRev %v != CompactRevision %v", prevHash, hresp.Hash) } prevHash = hresp.Hash prevCompactRev = hresp.CompactRevision } } } func TestV3TxnTooManyOps(t *testing.T) { integration.BeforeTest(t) maxTxnOps := uint(128) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, MaxTxnOps: maxTxnOps}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV // unique keys i := new(int) keyf := func() []byte { *i++ return []byte(fmt.Sprintf("key-%d", i)) } addCompareOps := func(txn *pb.TxnRequest) { txn.Compare = append(txn.Compare, &pb.Compare{ Result: pb.Compare_GREATER, Target: pb.Compare_CREATE, Key: keyf(), }) } addSuccessOps := func(txn *pb.TxnRequest) { txn.Success = append(txn.Success, &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Key: keyf(), Value: []byte("bar"), }, }, }) } addFailureOps := func(txn *pb.TxnRequest) { txn.Failure = append(txn.Failure, &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Key: keyf(), Value: []byte("bar"), }, }, }) } addTxnOps := func(txn *pb.TxnRequest) { newTxn := &pb.TxnRequest{} addSuccessOps(newTxn) txn.Success = append(txn.Success, &pb.RequestOp{ Request: &pb.RequestOp_RequestTxn{ RequestTxn: newTxn, }, }, ) } tests := []func(txn *pb.TxnRequest){ addCompareOps, addSuccessOps, addFailureOps, addTxnOps, } for i, tt := range tests { txn := &pb.TxnRequest{} for j := 0; j < int(maxTxnOps+1); j++ { tt(txn) } _, err := kvc.Txn(t.Context(), txn) if !eqErrGRPC(err, rpctypes.ErrGRPCTooManyOps) { t.Errorf("#%d: err = %v, want %v", i, err, rpctypes.ErrGRPCTooManyOps) } } } func TestV3TxnDuplicateKeys(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) putreq := &pb.RequestOp{Request: &pb.RequestOp_RequestPut{RequestPut: &pb.PutRequest{Key: []byte("abc"), Value: []byte("def")}}} delKeyReq := &pb.RequestOp{ Request: &pb.RequestOp_RequestDeleteRange{ RequestDeleteRange: &pb.DeleteRangeRequest{ Key: []byte("abc"), }, }, } delInRangeReq := &pb.RequestOp{ Request: &pb.RequestOp_RequestDeleteRange{ RequestDeleteRange: &pb.DeleteRangeRequest{ Key: []byte("a"), RangeEnd: []byte("b"), }, }, } delOutOfRangeReq := &pb.RequestOp{ Request: &pb.RequestOp_RequestDeleteRange{ RequestDeleteRange: &pb.DeleteRangeRequest{ Key: []byte("abb"), RangeEnd: []byte("abc"), }, }, } txnDelReq := &pb.RequestOp{ Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{Success: []*pb.RequestOp{delInRangeReq}}, }, } txnDelReqTwoSide := &pb.RequestOp{ Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{ Success: []*pb.RequestOp{delInRangeReq}, Failure: []*pb.RequestOp{delInRangeReq}, }, }, } txnPutReq := &pb.RequestOp{ Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{Success: []*pb.RequestOp{putreq}}, }, } txnPutReqTwoSide := &pb.RequestOp{ Request: &pb.RequestOp_RequestTxn{ RequestTxn: &pb.TxnRequest{ Success: []*pb.RequestOp{putreq}, Failure: []*pb.RequestOp{putreq}, }, }, } kvc := integration.ToGRPC(clus.RandClient()).KV tests := []struct { txnSuccess []*pb.RequestOp werr error }{ { txnSuccess: []*pb.RequestOp{putreq, putreq}, werr: rpctypes.ErrGRPCDuplicateKey, }, { txnSuccess: []*pb.RequestOp{putreq, delKeyReq}, werr: rpctypes.ErrGRPCDuplicateKey, }, { txnSuccess: []*pb.RequestOp{putreq, delInRangeReq}, werr: rpctypes.ErrGRPCDuplicateKey, }, // Then(Put(a), Then(Del(a))) { txnSuccess: []*pb.RequestOp{putreq, txnDelReq}, werr: rpctypes.ErrGRPCDuplicateKey, }, // Then(Del(a), Then(Put(a))) { txnSuccess: []*pb.RequestOp{delInRangeReq, txnPutReq}, werr: rpctypes.ErrGRPCDuplicateKey, }, // Then((Then(Put(a)), Else(Put(a))), (Then(Put(a)), Else(Put(a))) { txnSuccess: []*pb.RequestOp{txnPutReqTwoSide, txnPutReqTwoSide}, werr: rpctypes.ErrGRPCDuplicateKey, }, // Then(Del(x), (Then(Put(a)), Else(Put(a)))) { txnSuccess: []*pb.RequestOp{delOutOfRangeReq, txnPutReqTwoSide}, werr: nil, }, // Then(Then(Del(a)), (Then(Del(a)), Else(Del(a)))) { txnSuccess: []*pb.RequestOp{txnDelReq, txnDelReqTwoSide}, werr: nil, }, { txnSuccess: []*pb.RequestOp{delKeyReq, delInRangeReq, delKeyReq, delInRangeReq}, werr: nil, }, { txnSuccess: []*pb.RequestOp{putreq, delOutOfRangeReq}, werr: nil, }, } for i, tt := range tests { txn := &pb.TxnRequest{Success: tt.txnSuccess} _, err := kvc.Txn(t.Context(), txn) if !eqErrGRPC(err, tt.werr) { t.Errorf("#%d: err = %v, want %v", i, err, tt.werr) } } } // TestV3TxnRevision tests that the transaction header revision is set as expected. func TestV3TxnRevision(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV pr := &pb.PutRequest{Key: []byte("abc"), Value: []byte("def")} presp, err := kvc.Put(t.Context(), pr) require.NoError(t, err) txnget := &pb.RequestOp{Request: &pb.RequestOp_RequestRange{RequestRange: &pb.RangeRequest{Key: []byte("abc")}}} txn := &pb.TxnRequest{Success: []*pb.RequestOp{txnget}} tresp, err := kvc.Txn(t.Context(), txn) require.NoError(t, err) // did not update revision if presp.Header.Revision != tresp.Header.Revision { t.Fatalf("got rev %d, wanted rev %d", tresp.Header.Revision, presp.Header.Revision) } txndr := &pb.RequestOp{Request: &pb.RequestOp_RequestDeleteRange{RequestDeleteRange: &pb.DeleteRangeRequest{Key: []byte("def")}}} txn = &pb.TxnRequest{Success: []*pb.RequestOp{txndr}} tresp, err = kvc.Txn(t.Context(), txn) require.NoError(t, err) // did not update revision if presp.Header.Revision != tresp.Header.Revision { t.Fatalf("got rev %d, wanted rev %d", tresp.Header.Revision, presp.Header.Revision) } txnput := &pb.RequestOp{Request: &pb.RequestOp_RequestPut{RequestPut: &pb.PutRequest{Key: []byte("abc"), Value: []byte("123")}}} txn = &pb.TxnRequest{Success: []*pb.RequestOp{txnput}} tresp, err = kvc.Txn(t.Context(), txn) require.NoError(t, err) // updated revision if tresp.Header.Revision != presp.Header.Revision+1 { t.Fatalf("got rev %d, wanted rev %d", tresp.Header.Revision, presp.Header.Revision+1) } } // TestV3TxnCmpHeaderRev tests that the txn header revision is set as expected // when compared to the Succeeded field in the txn response. func TestV3TxnCmpHeaderRev(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV for i := 0; i < 10; i++ { // Concurrently put a key with a txn comparing on it. revc := make(chan int64, 1) errCh := make(chan error, 1) go func() { defer close(revc) pr := &pb.PutRequest{Key: []byte("k"), Value: []byte("v")} presp, err := kvc.Put(t.Context(), pr) errCh <- err if err != nil { return } revc <- presp.Header.Revision }() // The read-only txn uses the optimized readindex server path. txnget := &pb.RequestOp{Request: &pb.RequestOp_RequestRange{ RequestRange: &pb.RangeRequest{Key: []byte("k")}, }} txn := &pb.TxnRequest{Success: []*pb.RequestOp{txnget}} // i = 0 /\ Succeeded => put followed txn cmp := &pb.Compare{ Result: pb.Compare_EQUAL, Target: pb.Compare_VERSION, Key: []byte("k"), TargetUnion: &pb.Compare_Version{Version: int64(i)}, } txn.Compare = append(txn.Compare, cmp) tresp, err := kvc.Txn(t.Context(), txn) require.NoError(t, err) prev := <-revc err = <-errCh require.NoError(t, err) // put followed txn; should eval to false if prev > tresp.Header.Revision && !tresp.Succeeded { t.Errorf("#%d: got else but put rev %d followed txn rev (%+v)", i, prev, tresp) } // txn follows put; should eval to true if tresp.Header.Revision >= prev && tresp.Succeeded { t.Errorf("#%d: got then but put rev %d preceded txn (%+v)", i, prev, tresp) } } } // TestV3TxnRangeCompare tests range comparisons in txns func TestV3TxnRangeCompare(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) // put keys, named by expected revision for _, k := range []string{"/a/2", "/a/3", "/a/4", "/f/5"} { _, err := clus.Client(0).Put(t.Context(), k, "x") require.NoError(t, err) } tests := []struct { cmp pb.Compare wSuccess bool }{ { // >= /a/; all create revs fit pb.Compare{ Key: []byte("/a/"), RangeEnd: []byte{0}, Target: pb.Compare_CREATE, Result: pb.Compare_LESS, TargetUnion: &pb.Compare_CreateRevision{CreateRevision: 6}, }, true, }, { // >= /a/; one create rev doesn't fit pb.Compare{ Key: []byte("/a/"), RangeEnd: []byte{0}, Target: pb.Compare_CREATE, Result: pb.Compare_LESS, TargetUnion: &pb.Compare_CreateRevision{CreateRevision: 5}, }, false, }, { // prefix /a/*; all create revs fit pb.Compare{ Key: []byte("/a/"), RangeEnd: []byte("/a0"), Target: pb.Compare_CREATE, Result: pb.Compare_LESS, TargetUnion: &pb.Compare_CreateRevision{CreateRevision: 5}, }, true, }, { // prefix /a/*; one create rev doesn't fit pb.Compare{ Key: []byte("/a/"), RangeEnd: []byte("/a0"), Target: pb.Compare_CREATE, Result: pb.Compare_LESS, TargetUnion: &pb.Compare_CreateRevision{CreateRevision: 4}, }, false, }, { // does not exist, does not succeed pb.Compare{ Key: []byte("/b/"), RangeEnd: []byte("/b0"), Target: pb.Compare_VALUE, Result: pb.Compare_EQUAL, TargetUnion: &pb.Compare_Value{Value: []byte("x")}, }, false, }, { // all keys are leased pb.Compare{ Key: []byte("/a/"), RangeEnd: []byte("/a0"), Target: pb.Compare_LEASE, Result: pb.Compare_GREATER, TargetUnion: &pb.Compare_Lease{Lease: 0}, }, false, }, { // no keys are leased pb.Compare{ Key: []byte("/a/"), RangeEnd: []byte("/a0"), Target: pb.Compare_LEASE, Result: pb.Compare_EQUAL, TargetUnion: &pb.Compare_Lease{Lease: 0}, }, true, }, } kvc := integration.ToGRPC(clus.Client(0)).KV for i, tt := range tests { txn := &pb.TxnRequest{} txn.Compare = append(txn.Compare, &tt.cmp) tresp, err := kvc.Txn(t.Context(), txn) require.NoError(t, err) if tt.wSuccess != tresp.Succeeded { t.Errorf("#%d: expected %v, got %v", i, tt.wSuccess, tresp.Succeeded) } } } // TestV3TxnNestedPath tests nested txns follow paths as expected. func TestV3TxnNestedPath(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV cmpTrue := &pb.Compare{ Result: pb.Compare_EQUAL, Target: pb.Compare_VERSION, Key: []byte("k"), TargetUnion: &pb.Compare_Version{Version: int64(0)}, } cmpFalse := &pb.Compare{ Result: pb.Compare_EQUAL, Target: pb.Compare_VERSION, Key: []byte("k"), TargetUnion: &pb.Compare_Version{Version: int64(1)}, } // generate random path to eval txns topTxn := &pb.TxnRequest{} txn := topTxn txnPath := make([]bool, 10) for i := range txnPath { nextTxn := &pb.TxnRequest{} op := &pb.RequestOp{Request: &pb.RequestOp_RequestTxn{RequestTxn: nextTxn}} txnPath[i] = rand.Intn(2) == 0 if txnPath[i] { txn.Compare = append(txn.Compare, cmpTrue) txn.Success = append(txn.Success, op) } else { txn.Compare = append(txn.Compare, cmpFalse) txn.Failure = append(txn.Failure, op) } txn = nextTxn } tresp, err := kvc.Txn(t.Context(), topTxn) require.NoError(t, err) curTxnResp := tresp for i := range txnPath { if curTxnResp.Succeeded != txnPath[i] { t.Fatalf("expected path %+v, got response %+v", txnPath, *tresp) } curTxnResp = curTxnResp.Responses[0].Response.(*pb.ResponseOp_ResponseTxn).ResponseTxn } } // TestV3PutIgnoreValue ensures that writes with ignore_value overwrites with previous key-value pair. func TestV3PutIgnoreValue(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV key, val := []byte("foo"), []byte("bar") putReq := pb.PutRequest{Key: key, Value: val} // create lease lc := integration.ToGRPC(clus.RandClient()).Lease lresp, err := lc.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err) require.Empty(t, lresp.Error) tests := []struct { putFunc func() error putErr error wleaseID int64 }{ { // put failure for non-existent key func() error { preq := putReq preq.IgnoreValue = true _, err := kvc.Put(t.Context(), &preq) return err }, rpctypes.ErrGRPCKeyNotFound, 0, }, { // txn failure for non-existent key func() error { preq := putReq preq.Value = nil preq.IgnoreValue = true txn := &pb.TxnRequest{} txn.Success = append(txn.Success, &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{RequestPut: &preq}, }) _, err := kvc.Txn(t.Context(), txn) return err }, rpctypes.ErrGRPCKeyNotFound, 0, }, { // put success func() error { _, err := kvc.Put(t.Context(), &putReq) return err }, nil, 0, }, { // txn success, attach lease func() error { preq := putReq preq.Value = nil preq.Lease = lresp.ID preq.IgnoreValue = true txn := &pb.TxnRequest{} txn.Success = append(txn.Success, &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{RequestPut: &preq}, }) _, err := kvc.Txn(t.Context(), txn) return err }, nil, lresp.ID, }, { // non-empty value with ignore_value should error func() error { preq := putReq preq.IgnoreValue = true _, err := kvc.Put(t.Context(), &preq) return err }, rpctypes.ErrGRPCValueProvided, 0, }, { // overwrite with previous value, ensure no prev-kv is returned and lease is detached func() error { preq := putReq preq.Value = nil preq.IgnoreValue = true presp, err := kvc.Put(t.Context(), &preq) if err != nil { return err } if presp.PrevKv != nil && len(presp.PrevKv.Key) != 0 { return fmt.Errorf("unexexpected previous key-value %v", presp.PrevKv) } return nil }, nil, 0, }, { // revoke lease, ensure detached key doesn't get deleted func() error { _, err := lc.LeaseRevoke(t.Context(), &pb.LeaseRevokeRequest{ID: lresp.ID}) return err }, nil, 0, }, } for i, tt := range tests { if err := tt.putFunc(); !eqErrGRPC(err, tt.putErr) { t.Fatalf("#%d: err expected %v, got %v", i, tt.putErr, err) } if tt.putErr != nil { continue } rr, err := kvc.Range(t.Context(), &pb.RangeRequest{Key: key}) if err != nil { t.Fatalf("#%d: %v", i, err) } if len(rr.Kvs) != 1 { t.Fatalf("#%d: len(rr.KVs) expected 1, got %d", i, len(rr.Kvs)) } if !bytes.Equal(rr.Kvs[0].Value, val) { t.Fatalf("#%d: value expected %q, got %q", i, val, rr.Kvs[0].Value) } if rr.Kvs[0].Lease != tt.wleaseID { t.Fatalf("#%d: lease ID expected %d, got %d", i, tt.wleaseID, rr.Kvs[0].Lease) } } } // TestV3PutIgnoreLease ensures that writes with ignore_lease uses previous lease for the key overwrites. func TestV3PutIgnoreLease(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV // create lease lc := integration.ToGRPC(clus.RandClient()).Lease lresp, err := lc.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err) require.Empty(t, lresp.Error) key, val, val1 := []byte("zoo"), []byte("bar"), []byte("bar1") putReq := pb.PutRequest{Key: key, Value: val} tests := []struct { putFunc func() error putErr error wleaseID int64 wvalue []byte }{ { // put failure for non-existent key func() error { preq := putReq preq.IgnoreLease = true _, err := kvc.Put(t.Context(), &preq) return err }, rpctypes.ErrGRPCKeyNotFound, 0, nil, }, { // txn failure for non-existent key func() error { preq := putReq preq.IgnoreLease = true txn := &pb.TxnRequest{} txn.Success = append(txn.Success, &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{RequestPut: &preq}, }) _, err := kvc.Txn(t.Context(), txn) return err }, rpctypes.ErrGRPCKeyNotFound, 0, nil, }, { // put success func() error { preq := putReq preq.Lease = lresp.ID _, err := kvc.Put(t.Context(), &preq) return err }, nil, lresp.ID, val, }, { // txn success, modify value using 'ignore_lease' and ensure lease is not detached func() error { preq := putReq preq.Value = val1 preq.IgnoreLease = true txn := &pb.TxnRequest{} txn.Success = append(txn.Success, &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{RequestPut: &preq}, }) _, err := kvc.Txn(t.Context(), txn) return err }, nil, lresp.ID, val1, }, { // non-empty lease with ignore_lease should error func() error { preq := putReq preq.Lease = lresp.ID preq.IgnoreLease = true _, err := kvc.Put(t.Context(), &preq) return err }, rpctypes.ErrGRPCLeaseProvided, 0, nil, }, { // overwrite with previous value, ensure no prev-kv is returned and lease is detached func() error { presp, err := kvc.Put(t.Context(), &putReq) if err != nil { return err } if presp.PrevKv != nil && len(presp.PrevKv.Key) != 0 { return fmt.Errorf("unexexpected previous key-value %v", presp.PrevKv) } return nil }, nil, 0, val, }, { // revoke lease, ensure detached key doesn't get deleted func() error { _, err := lc.LeaseRevoke(t.Context(), &pb.LeaseRevokeRequest{ID: lresp.ID}) return err }, nil, 0, val, }, } for i, tt := range tests { if err := tt.putFunc(); !eqErrGRPC(err, tt.putErr) { t.Fatalf("#%d: err expected %v, got %v", i, tt.putErr, err) } if tt.putErr != nil { continue } rr, err := kvc.Range(t.Context(), &pb.RangeRequest{Key: key}) if err != nil { t.Fatalf("#%d: %v", i, err) } if len(rr.Kvs) != 1 { t.Fatalf("#%d: len(rr.KVs) expected 1, got %d", i, len(rr.Kvs)) } if !bytes.Equal(rr.Kvs[0].Value, tt.wvalue) { t.Fatalf("#%d: value expected %q, got %q", i, val, rr.Kvs[0].Value) } if rr.Kvs[0].Lease != tt.wleaseID { t.Fatalf("#%d: lease ID expected %d, got %d", i, tt.wleaseID, rr.Kvs[0].Lease) } } } // TestV3PutMissingLease ensures that a Put on a key with a bogus lease fails. func TestV3PutMissingLease(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV key := []byte("foo") preq := &pb.PutRequest{Key: key, Lease: 123456} tests := []func(){ // put case func() { if presp, err := kvc.Put(t.Context(), preq); err == nil { t.Errorf("succeeded put key. req: %v. resp: %v", preq, presp) } }, // txn success case func() { txn := &pb.TxnRequest{} txn.Success = append(txn.Success, &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: preq, }, }) if tresp, err := kvc.Txn(t.Context(), txn); err == nil { t.Errorf("succeeded txn success. req: %v. resp: %v", txn, tresp) } }, // txn failure case func() { txn := &pb.TxnRequest{} txn.Failure = append(txn.Failure, &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: preq, }, }) cmp := &pb.Compare{ Result: pb.Compare_GREATER, Target: pb.Compare_CREATE, Key: []byte("bar"), } txn.Compare = append(txn.Compare, cmp) if tresp, err := kvc.Txn(t.Context(), txn); err == nil { t.Errorf("succeeded txn failure. req: %v. resp: %v", txn, tresp) } }, // ignore bad lease in failure on success txn func() { txn := &pb.TxnRequest{} rreq := &pb.RangeRequest{Key: []byte("bar")} txn.Success = append(txn.Success, &pb.RequestOp{ Request: &pb.RequestOp_RequestRange{ RequestRange: rreq, }, }) txn.Failure = append(txn.Failure, &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: preq, }, }) if tresp, err := kvc.Txn(t.Context(), txn); err != nil { t.Errorf("failed good txn. req: %v. resp: %v", txn, tresp) } }, } for i, f := range tests { f() // key shouldn't have been stored rreq := &pb.RangeRequest{Key: key} rresp, err := kvc.Range(t.Context(), rreq) if err != nil { t.Errorf("#%d. could not rangereq (%v)", i, err) } else if len(rresp.Kvs) != 0 { t.Errorf("#%d. expected no keys, got %v", i, rresp) } } } // TestV3DeleteRange tests various edge cases in the DeleteRange API. func TestV3DeleteRange(t *testing.T) { integration.BeforeTest(t) tests := []struct { name string keySet []string begin string end string prevKV bool wantSet [][]byte deleted int64 }{ { "delete middle", []string{"foo", "foo/abc", "fop"}, "foo/", "fop", false, [][]byte{[]byte("foo"), []byte("fop")}, 1, }, { "no delete", []string{"foo", "foo/abc", "fop"}, "foo/", "foo/", false, [][]byte{[]byte("foo"), []byte("foo/abc"), []byte("fop")}, 0, }, { "delete first", []string{"foo", "foo/abc", "fop"}, "fo", "fop", false, [][]byte{[]byte("fop")}, 2, }, { "delete tail", []string{"foo", "foo/abc", "fop"}, "foo/", "fos", false, [][]byte{[]byte("foo")}, 2, }, { "delete exact", []string{"foo", "foo/abc", "fop"}, "foo/abc", "", false, [][]byte{[]byte("foo"), []byte("fop")}, 1, }, { "delete none [x,x)", []string{"foo"}, "foo", "foo", false, [][]byte{[]byte("foo")}, 0, }, { "delete middle with preserveKVs set", []string{"foo", "foo/abc", "fop"}, "foo/", "fop", true, [][]byte{[]byte("foo"), []byte("fop")}, 1, }, } for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) kvc := integration.ToGRPC(clus.RandClient()).KV defer clus.Terminate(t) ks := tt.keySet for j := range ks { reqput := &pb.PutRequest{Key: []byte(ks[j]), Value: []byte{}} _, err := kvc.Put(t.Context(), reqput) if err != nil { t.Fatalf("couldn't put key (%v)", err) } } dreq := &pb.DeleteRangeRequest{ Key: []byte(tt.begin), RangeEnd: []byte(tt.end), PrevKv: tt.prevKV, } dresp, err := kvc.DeleteRange(t.Context(), dreq) if err != nil { t.Fatalf("couldn't delete range on test %d (%v)", i, err) } if tt.deleted != dresp.Deleted { t.Errorf("expected %d on test %v, got %d", tt.deleted, i, dresp.Deleted) } if tt.prevKV { if len(dresp.PrevKvs) != int(dresp.Deleted) { t.Errorf("preserve %d keys, want %d", len(dresp.PrevKvs), dresp.Deleted) } } rreq := &pb.RangeRequest{Key: []byte{0x0}, RangeEnd: []byte{0xff}} rresp, err := kvc.Range(t.Context(), rreq) if err != nil { t.Errorf("couldn't get range on test %v (%v)", i, err) } if dresp.Header.Revision != rresp.Header.Revision { t.Errorf("expected revision %v, got %v", dresp.Header.Revision, rresp.Header.Revision) } var keys [][]byte for j := range rresp.Kvs { keys = append(keys, rresp.Kvs[j].Key) } if !reflect.DeepEqual(tt.wantSet, keys) { t.Errorf("expected %v on test %v, got %v", tt.wantSet, i, keys) } }) } } // TestV3TxnInvalidRange tests that invalid ranges are rejected in txns. func TestV3TxnInvalidRange(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV preq := &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")} for i := 0; i < 3; i++ { _, err := kvc.Put(t.Context(), preq) if err != nil { t.Fatalf("couldn't put key (%v)", err) } } _, err := kvc.Compact(t.Context(), &pb.CompactionRequest{Revision: 2}) if err != nil { t.Fatalf("couldn't compact kv space (%v)", err) } // future rev txn := &pb.TxnRequest{} txn.Success = append(txn.Success, &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: preq, }, }) rreq := &pb.RangeRequest{Key: []byte("foo"), Revision: 100} txn.Success = append(txn.Success, &pb.RequestOp{ Request: &pb.RequestOp_RequestRange{ RequestRange: rreq, }, }) if _, err := kvc.Txn(t.Context(), txn); !eqErrGRPC(err, rpctypes.ErrGRPCFutureRev) { t.Errorf("err = %v, want %v", err, rpctypes.ErrGRPCFutureRev) } // compacted rev tv, _ := txn.Success[1].Request.(*pb.RequestOp_RequestRange) tv.RequestRange.Revision = 1 if _, err := kvc.Txn(t.Context(), txn); !eqErrGRPC(err, rpctypes.ErrGRPCCompacted) { t.Errorf("err = %v, want %v", err, rpctypes.ErrGRPCCompacted) } } func TestV3TooLargeRequest(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.RandClient()).KV // 2MB request value largeV := make([]byte, 2*1024*1024) preq := &pb.PutRequest{Key: []byte("foo"), Value: largeV} _, err := kvc.Put(t.Context(), preq) if !eqErrGRPC(err, rpctypes.ErrGRPCRequestTooLarge) { t.Errorf("err = %v, want %v", err, rpctypes.ErrGRPCRequestTooLarge) } } // TestV3Hash tests hash. func TestV3Hash(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) cli := clus.RandClient() kvc := integration.ToGRPC(cli).KV m := integration.ToGRPC(cli).Maintenance preq := &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")} for i := 0; i < 3; i++ { _, err := kvc.Put(t.Context(), preq) if err != nil { t.Fatalf("couldn't put key (%v)", err) } } resp, err := m.Hash(t.Context(), &pb.HashRequest{}) if err != nil || resp.Hash == 0 { t.Fatalf("couldn't hash (%v, hash %d)", err, resp.Hash) } } // TestV3HashRestart ensures that hash stays the same after restart. func TestV3HashRestart(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) cli := clus.RandClient() resp, err := integration.ToGRPC(cli).Maintenance.Hash(t.Context(), &pb.HashRequest{}) if err != nil || resp.Hash == 0 { t.Fatalf("couldn't hash (%v, hash %d)", err, resp.Hash) } hash1 := resp.Hash clus.Members[0].Stop(t) clus.Members[0].Restart(t) clus.WaitMembersForLeader(t, clus.Members) kvc := integration.ToGRPC(clus.Client(0)).KV waitForRestart(t, kvc) cli = clus.RandClient() resp, err = integration.ToGRPC(cli).Maintenance.Hash(t.Context(), &pb.HashRequest{}) if err != nil || resp.Hash == 0 { t.Fatalf("couldn't hash (%v, hash %d)", err, resp.Hash) } hash2 := resp.Hash if hash1 != hash2 { t.Fatalf("hash expected %d, got %d", hash1, hash2) } } // TestV3StorageQuotaAPI tests the V3 server respects quotas at the API layer func TestV3StorageQuotaAPI(t *testing.T) { integration.BeforeTest(t) quotasize := int64(16 * os.Getpagesize()) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) // Set a quota on one node clus.Members[0].QuotaBackendBytes = quotasize clus.Members[0].Stop(t) clus.Members[0].Restart(t) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.Client(0)).KV waitForRestart(t, kvc) key := []byte("abc") // test small put that fits in quota smallbuf := make([]byte, 512) _, err := kvc.Put(t.Context(), &pb.PutRequest{Key: key, Value: smallbuf}) require.NoError(t, err) // test big put bigbuf := make([]byte, quotasize) _, err = kvc.Put(t.Context(), &pb.PutRequest{Key: key, Value: bigbuf}) if !eqErrGRPC(err, rpctypes.ErrGRPCNoSpace) { t.Fatalf("big put got %v, expected %v", err, rpctypes.ErrGRPCNoSpace) } // test big txn puttxn := &pb.RequestOp{ Request: &pb.RequestOp_RequestPut{ RequestPut: &pb.PutRequest{ Key: key, Value: bigbuf, }, }, } txnreq := &pb.TxnRequest{} txnreq.Success = append(txnreq.Success, puttxn) _, txnerr := kvc.Txn(t.Context(), txnreq) if !eqErrGRPC(txnerr, rpctypes.ErrGRPCNoSpace) { t.Fatalf("big txn got %v, expected %v", err, rpctypes.ErrGRPCNoSpace) } } func TestV3RangeRequest(t *testing.T) { integration.BeforeTest(t) tests := []struct { name string putKeys []string reqs []pb.RangeRequest wresps [][]string wmores []bool wcounts []int64 }{ { "single key", []string{"foo", "bar"}, []pb.RangeRequest{ // exists {Key: []byte("foo")}, // doesn't exist {Key: []byte("baz")}, }, [][]string{ {"foo"}, {}, }, []bool{false, false}, []int64{1, 0}, }, { "multi-key", []string{"a", "b", "c", "d", "e"}, []pb.RangeRequest{ // all in range {Key: []byte("a"), RangeEnd: []byte("z")}, // [b, d) {Key: []byte("b"), RangeEnd: []byte("d")}, // out of range {Key: []byte("f"), RangeEnd: []byte("z")}, // [c,c) = empty {Key: []byte("c"), RangeEnd: []byte("c")}, // [d, b) = empty {Key: []byte("d"), RangeEnd: []byte("b")}, // ["\0", "\0") => all in range {Key: []byte{0}, RangeEnd: []byte{0}}, }, [][]string{ {"a", "b", "c", "d", "e"}, {"b", "c"}, {}, {}, {}, {"a", "b", "c", "d", "e"}, }, []bool{false, false, false, false, false, false}, []int64{5, 2, 0, 0, 0, 5}, }, { "revision", []string{"a", "b", "c", "d", "e"}, []pb.RangeRequest{ {Key: []byte("a"), RangeEnd: []byte("z"), Revision: 0}, {Key: []byte("a"), RangeEnd: []byte("z"), Revision: 1}, {Key: []byte("a"), RangeEnd: []byte("z"), Revision: 2}, {Key: []byte("a"), RangeEnd: []byte("z"), Revision: 3}, }, [][]string{ {"a", "b", "c", "d", "e"}, {}, {"a"}, {"a", "b"}, }, []bool{false, false, false, false}, []int64{5, 0, 1, 2}, }, { "limit", []string{"a", "b", "c"}, []pb.RangeRequest{ // more {Key: []byte("a"), RangeEnd: []byte("z"), Limit: 1}, // half {Key: []byte("a"), RangeEnd: []byte("z"), Limit: 2}, // no more {Key: []byte("a"), RangeEnd: []byte("z"), Limit: 3}, // limit over {Key: []byte("a"), RangeEnd: []byte("z"), Limit: 4}, }, [][]string{ {"a"}, {"a", "b"}, {"a", "b", "c"}, {"a", "b", "c"}, }, []bool{true, true, false, false}, []int64{3, 3, 3, 3}, }, { "sort", []string{"b", "a", "c", "d", "c"}, []pb.RangeRequest{ { Key: []byte("a"), RangeEnd: []byte("z"), Limit: 1, SortOrder: pb.RangeRequest_ASCEND, SortTarget: pb.RangeRequest_KEY, }, { Key: []byte("a"), RangeEnd: []byte("z"), Limit: 1, SortOrder: pb.RangeRequest_DESCEND, SortTarget: pb.RangeRequest_KEY, }, { Key: []byte("a"), RangeEnd: []byte("z"), Limit: 1, SortOrder: pb.RangeRequest_ASCEND, SortTarget: pb.RangeRequest_CREATE, }, { Key: []byte("a"), RangeEnd: []byte("z"), Limit: 1, SortOrder: pb.RangeRequest_DESCEND, SortTarget: pb.RangeRequest_MOD, }, { Key: []byte("z"), RangeEnd: []byte("z"), Limit: 1, SortOrder: pb.RangeRequest_DESCEND, SortTarget: pb.RangeRequest_CREATE, }, { // sort ASCEND by default Key: []byte("a"), RangeEnd: []byte("z"), Limit: 10, SortOrder: pb.RangeRequest_NONE, SortTarget: pb.RangeRequest_CREATE, }, }, [][]string{ {"a"}, {"d"}, {"b"}, {"c"}, {}, {"b", "a", "c", "d"}, }, []bool{true, true, true, true, false, false}, []int64{4, 4, 4, 4, 0, 4}, }, { "min/max mod rev", []string{"rev2", "rev3", "rev4", "rev5", "rev6"}, []pb.RangeRequest{ { Key: []byte{0}, RangeEnd: []byte{0}, MinModRevision: 3, }, { Key: []byte{0}, RangeEnd: []byte{0}, MaxModRevision: 3, }, { Key: []byte{0}, RangeEnd: []byte{0}, MinModRevision: 3, MaxModRevision: 5, }, { Key: []byte{0}, RangeEnd: []byte{0}, MaxModRevision: 10, }, }, [][]string{ {"rev3", "rev4", "rev5", "rev6"}, {"rev2", "rev3"}, {"rev3", "rev4", "rev5"}, {"rev2", "rev3", "rev4", "rev5", "rev6"}, }, []bool{false, false, false, false}, []int64{5, 5, 5, 5}, }, { "min/max create rev", []string{"rev2", "rev3", "rev2", "rev2", "rev6", "rev3"}, []pb.RangeRequest{ { Key: []byte{0}, RangeEnd: []byte{0}, MinCreateRevision: 3, }, { Key: []byte{0}, RangeEnd: []byte{0}, MaxCreateRevision: 3, }, { Key: []byte{0}, RangeEnd: []byte{0}, MinCreateRevision: 3, MaxCreateRevision: 5, }, { Key: []byte{0}, RangeEnd: []byte{0}, MaxCreateRevision: 10, }, }, [][]string{ {"rev3", "rev6"}, {"rev2", "rev3"}, {"rev3"}, {"rev2", "rev3", "rev6"}, }, []bool{false, false, false, false}, []int64{3, 3, 3, 3}, }, } for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) for _, k := range tt.putKeys { kvc := integration.ToGRPC(clus.RandClient()).KV req := &pb.PutRequest{Key: []byte(k), Value: []byte("bar")} if _, err := kvc.Put(t.Context(), req); err != nil { t.Fatalf("#%d: couldn't put key (%v)", i, err) } } for j, req := range tt.reqs { kvc := integration.ToGRPC(clus.RandClient()).KV resp, err := kvc.Range(t.Context(), &req) if err != nil { t.Errorf("#%d.%d: Range error: %v", i, j, err) continue } if len(resp.Kvs) != len(tt.wresps[j]) { t.Errorf("#%d.%d: bad len(resp.Kvs). got = %d, want = %d, ", i, j, len(resp.Kvs), len(tt.wresps[j])) continue } for k, wKey := range tt.wresps[j] { respKey := string(resp.Kvs[k].Key) if respKey != wKey { t.Errorf("#%d.%d: key[%d]. got = %v, want = %v, ", i, j, k, respKey, wKey) } } if resp.More != tt.wmores[j] { t.Errorf("#%d.%d: bad more. got = %v, want = %v, ", i, j, resp.More, tt.wmores[j]) } if resp.GetCount() != tt.wcounts[j] { t.Errorf("#%d.%d: bad count. got = %v, want = %v, ", i, j, resp.GetCount(), tt.wcounts[j]) } wrev := int64(len(tt.putKeys) + 1) if resp.Header.Revision != wrev { t.Errorf("#%d.%d: bad header revision. got = %d. want = %d", i, j, resp.Header.Revision, wrev) } } }) } } // TestTLSGRPCRejectInsecureClient checks that connection is rejected if server is TLS but not client. func TestTLSGRPCRejectInsecureClient(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, ClientTLS: &integration.TestTLSInfo}) defer clus.Terminate(t) // nil out TLS field so client will use an insecure connection clus.Members[0].ClientTLSInfo = nil client, err := integration.NewClientV3(clus.Members[0]) if err != nil && !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("unexpected error (%v)", err) } else if client == nil { // Ideally, no client would be returned. However, grpc will // return a connection without trying to handshake first so // the connection appears OK. return } defer client.Close() donec := make(chan error, 1) go func() { ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) reqput := &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")} _, perr := integration.ToGRPC(client).KV.Put(ctx, reqput) cancel() donec <- perr }() if perr := <-donec; perr == nil { t.Fatalf("expected client error on put") } } // TestTLSGRPCRejectSecureClient checks that connection is rejected if client is TLS but not server. func TestTLSGRPCRejectSecureClient(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) clus.Members[0].ClientTLSInfo = &integration.TestTLSInfo clus.Members[0].GRPCURL = strings.Replace(clus.Members[0].GRPCURL, "http://", "https://", 1) client, err := integration.NewClientV3(clus.Members[0]) if client != nil || err == nil { client.Close() t.Fatalf("expected no client") } else if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("unexpected error (%v)", err) } } // TestTLSGRPCAcceptSecureAll checks that connection is accepted if both client and server are TLS func TestTLSGRPCAcceptSecureAll(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, ClientTLS: &integration.TestTLSInfo}) defer clus.Terminate(t) client, err := integration.NewClientV3(clus.Members[0]) if err != nil { t.Fatalf("expected tls client (%v)", err) } defer client.Close() reqput := &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")} if _, err := integration.ToGRPC(client).KV.Put(t.Context(), reqput); err != nil { t.Fatalf("unexpected error on put over tls (%v)", err) } } // TestTLSReloadAtomicReplace ensures server reloads expired/valid certs // when all certs are atomically replaced by directory renaming. // And expects server to reject client requests, and vice versa. func TestTLSReloadAtomicReplace(t *testing.T) { tmpDir := t.TempDir() os.RemoveAll(tmpDir) certsDir := t.TempDir() certsDirExp := t.TempDir() cloneFunc := func() transport.TLSInfo { tlsInfo, terr := copyTLSFiles(integration.TestTLSInfo, certsDir) require.NoError(t, terr) _, err := copyTLSFiles(integration.TestTLSInfoExpired, certsDirExp) require.NoError(t, err) return tlsInfo } replaceFunc := func() { err := os.Rename(certsDir, tmpDir) require.NoError(t, err) err = os.Rename(certsDirExp, certsDir) require.NoError(t, err) // after rename, // 'certsDir' contains expired certs // 'tmpDir' contains valid certs // 'certsDirExp' does not exist } revertFunc := func() { err := os.Rename(tmpDir, certsDirExp) require.NoError(t, err) err = os.Rename(certsDir, tmpDir) require.NoError(t, err) err = os.Rename(certsDirExp, certsDir) require.NoError(t, err) } testTLSReload(t, cloneFunc, replaceFunc, revertFunc, false) } // TestTLSReloadCopy ensures server reloads expired/valid certs // when new certs are copied over, one by one. And expects server // to reject client requests, and vice versa. func TestTLSReloadCopy(t *testing.T) { certsDir := t.TempDir() cloneFunc := func() transport.TLSInfo { tlsInfo, terr := copyTLSFiles(integration.TestTLSInfo, certsDir) require.NoError(t, terr) return tlsInfo } replaceFunc := func() { _, err := copyTLSFiles(integration.TestTLSInfoExpired, certsDir) require.NoError(t, err) } revertFunc := func() { _, err := copyTLSFiles(integration.TestTLSInfo, certsDir) require.NoError(t, err) } testTLSReload(t, cloneFunc, replaceFunc, revertFunc, false) } // TestTLSReloadCopyIPOnly ensures server reloads expired/valid certs // when new certs are copied over, one by one. And expects server // to reject client requests, and vice versa. func TestTLSReloadCopyIPOnly(t *testing.T) { certsDir := t.TempDir() cloneFunc := func() transport.TLSInfo { tlsInfo, terr := copyTLSFiles(integration.TestTLSInfoIP, certsDir) require.NoError(t, terr) return tlsInfo } replaceFunc := func() { _, err := copyTLSFiles(integration.TestTLSInfoExpiredIP, certsDir) require.NoError(t, err) } revertFunc := func() { _, err := copyTLSFiles(integration.TestTLSInfoIP, certsDir) require.NoError(t, err) } testTLSReload(t, cloneFunc, replaceFunc, revertFunc, true) } func testTLSReload( t *testing.T, cloneFunc func() transport.TLSInfo, replaceFunc func(), revertFunc func(), useIP bool, ) { integration.BeforeTest(t) // 1. separate copies for TLS assets modification tlsInfo := cloneFunc() // 2. start cluster with valid certs clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 1, PeerTLS: &tlsInfo, ClientTLS: &tlsInfo, UseIP: useIP, }) defer clus.Terminate(t) // 3. concurrent client dialing while certs become expired errc := make(chan error, 1) go func() { for { cc, err := tlsInfo.ClientConfig() if err != nil { // errors in 'go/src/crypto/tls/tls.go' // tls: private key does not match public key // tls: failed to find any PEM data in key input // tls: failed to find any PEM data in certificate input // Or 'does not exist', 'not found', etc t.Log(err) continue } cli, cerr := integration.NewClient(t, clientv3.Config{ Endpoints: []string{clus.Members[0].GRPCURL}, DialTimeout: time.Second, TLS: cc, }) if cerr != nil { errc <- cerr return } cli.Close() } }() // 4. replace certs with expired ones replaceFunc() // 5. expect dial time-out when loading expired certs select { case gerr := <-errc: if !errors.Is(gerr, context.DeadlineExceeded) { t.Fatalf("expected %v, got %v", context.DeadlineExceeded, gerr) } case <-time.After(5 * time.Second): t.Fatal("failed to receive dial timeout error") } // 6. replace expired certs back with valid ones revertFunc() // 7. new requests should trigger listener to reload valid certs tls, terr := tlsInfo.ClientConfig() require.NoError(t, terr) cl, cerr := integration.NewClient(t, clientv3.Config{ Endpoints: []string{clus.Members[0].GRPCURL}, DialTimeout: 5 * time.Second, TLS: tls, }) if cerr != nil { t.Fatalf("expected no error, got %v", cerr) } cl.Close() } func TestGRPCRequireLeader(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) clus.Members[1].Stop(t) clus.Members[2].Stop(t) client, err := integration.NewClientV3(clus.Members[0]) if err != nil { t.Fatalf("cannot create client: %v", err) } defer client.Close() // wait for election timeout, then member[0] will not have a leader. time.Sleep(time.Duration(3*integration.ElectionTicks) * config.TickDuration) md := metadata.Pairs(rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader) ctx := metadata.NewOutgoingContext(t.Context(), md) reqput := &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")} if _, err := integration.ToGRPC(client).KV.Put(ctx, reqput); rpctypes.ErrorDesc(err) != rpctypes.ErrNoLeader.Error() { t.Errorf("err = %v, want %v", err, rpctypes.ErrNoLeader) } } func TestGRPCStreamRequireLeader(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) client, err := integration.NewClientV3(clus.Members[0]) if err != nil { t.Fatalf("failed to create client (%v)", err) } defer client.Close() wAPI := integration.ToGRPC(client).Watch md := metadata.Pairs(rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader) ctx := metadata.NewOutgoingContext(t.Context(), md) wStream, err := wAPI.Watch(ctx) if err != nil { t.Fatalf("wAPI.Watch error: %v", err) } clus.Members[1].Stop(t) clus.Members[2].Stop(t) // existing stream should be rejected _, err = wStream.Recv() if rpctypes.ErrorDesc(err) != rpctypes.ErrNoLeader.Error() { t.Errorf("err = %v, want %v", err, rpctypes.ErrNoLeader) } // new stream should also be rejected wStream, err = wAPI.Watch(ctx) if err != nil { t.Fatalf("wAPI.Watch error: %v", err) } _, err = wStream.Recv() if rpctypes.ErrorDesc(err) != rpctypes.ErrNoLeader.Error() { t.Errorf("err = %v, want %v", err, rpctypes.ErrNoLeader) } clus.Members[1].Restart(t) clus.Members[2].Restart(t) clus.WaitMembersForLeader(t, clus.Members) time.Sleep(time.Duration(2*integration.ElectionTicks) * config.TickDuration) // new stream should also be OK now after we restarted the other members wStream, err = wAPI.Watch(ctx) if err != nil { t.Fatalf("wAPI.Watch error: %v", err) } wreq := &pb.WatchRequest{ RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{Key: []byte("foo")}, }, } err = wStream.Send(wreq) if err != nil { t.Errorf("err = %v, want nil", err) } } // TestV3LargeRequests ensures that configurable MaxRequestBytes works as intended. func TestV3LargeRequests(t *testing.T) { integration.BeforeTest(t) tests := []struct { maxRequestBytes uint valueSize int expectError error }{ // don't set to 0. use 0 as the default. {256, 1024, rpctypes.ErrGRPCRequestTooLarge}, {10 * 1024 * 1024, 9 * 1024 * 1024, nil}, {10 * 1024 * 1024, 10 * 1024 * 1024, rpctypes.ErrGRPCRequestTooLarge}, {10 * 1024 * 1024, 10*1024*1024 + 5, rpctypes.ErrGRPCRequestTooLarge}, } for i, test := range tests { t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, MaxRequestBytes: test.maxRequestBytes}) defer clus.Terminate(t) kvcli := integration.ToGRPC(clus.Client(0)).KV reqput := &pb.PutRequest{Key: []byte("foo"), Value: make([]byte, test.valueSize)} _, err := kvcli.Put(t.Context(), reqput) if !eqErrGRPC(err, test.expectError) { t.Errorf("#%d: expected error %v, got %v", i, test.expectError, err) } // request went through, expect large response back from server if test.expectError == nil { reqget := &pb.RangeRequest{Key: []byte("foo")} // limit receive call size with original value + gRPC overhead bytes _, err = kvcli.Range(t.Context(), reqget, grpc.MaxCallRecvMsgSize(test.valueSize+512*1024)) if err != nil { t.Errorf("#%d: range expected no error, got %v", i, err) } } }) } } // TestV3AdditionalGRPCOptions ensures that configurable GRPCAdditionalServerOptions works as intended. func TestV3AdditionalGRPCOptions(t *testing.T) { integration.BeforeTest(t) tests := []struct { name string maxRequestBytes uint grpcOpts []grpc.ServerOption valueSize int expectError error }{ { name: "requests will get a gRPC error because it's larger than gRPC MaxRecvMsgSize", maxRequestBytes: 8 * 1024 * 1024, grpcOpts: nil, valueSize: 9 * 1024 * 1024, expectError: status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max"), }, { name: "requests will get an etcd custom gRPC error because it's larger than MaxRequestBytes", maxRequestBytes: 8 * 1024 * 1024, grpcOpts: []grpc.ServerOption{grpc.MaxRecvMsgSize(10 * 1024 * 1024)}, valueSize: 9 * 1024 * 1024, expectError: rpctypes.ErrGRPCRequestTooLarge, }, { name: "requests size is smaller than MaxRequestBytes but larger than MaxRecvMsgSize", maxRequestBytes: 8 * 1024 * 1024, grpcOpts: []grpc.ServerOption{grpc.MaxRecvMsgSize(4 * 1024 * 1024)}, valueSize: 6 * 1024 * 1024, expectError: status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max"), }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { clus := integration.NewCluster(t, &integration.ClusterConfig{ Size: 1, MaxRequestBytes: test.maxRequestBytes, ClientMaxCallSendMsgSize: 12 * 1024 * 1024, GRPCAdditionalServerOptions: test.grpcOpts, }) defer clus.Terminate(t) kvcli := integration.ToGRPC(clus.Client(0)).KV reqput := &pb.PutRequest{Key: []byte("foo"), Value: make([]byte, test.valueSize)} if _, err := kvcli.Put(t.Context(), reqput); err != nil { var etcdErr rpctypes.EtcdError if errors.As(err, &etcdErr) { if err.Error() != status.Convert(test.expectError).Message() { t.Errorf("expected %v, got %v", status.Convert(test.expectError).Message(), err.Error()) } } else if !strings.HasPrefix(err.Error(), test.expectError.Error()) { t.Errorf("expected error starting with '%s', got '%s'", test.expectError.Error(), err.Error()) } } // request went through, expect large response back from server if test.expectError == nil { reqget := &pb.RangeRequest{Key: []byte("foo")} // limit receive call size with original value + gRPC overhead bytes _, err := kvcli.Range(t.Context(), reqget, grpc.MaxCallRecvMsgSize(test.valueSize+512*1024)) if err != nil { t.Errorf("range expected no error, got %v", err) } } }) } } func eqErrGRPC(err1 error, err2 error) bool { return !(err1 == nil && err2 != nil) || err1.Error() == err2.Error() } // waitForRestart tries a range request until the client's server responds. // This is mainly a stop-gap function until grpcproxy's KVClient adapter // (and by extension, clientv3) supports grpc.CallOption pass-through so // FailFast=false works with Put. func waitForRestart(t *testing.T, kvc pb.KVClient) { req := &pb.RangeRequest{Key: []byte("_"), Serializable: true} // TODO: Remove retry loop once the new grpc load balancer provides retry. var err error for i := 0; i < 10; i++ { if _, err = kvc.Range(t.Context(), req, grpc.WaitForReady(true)); err != nil { if status, ok := status.FromError(err); ok && status.Code() == codes.Unavailable { time.Sleep(time.Millisecond * 250) } else { t.Fatal(err) } } } if err != nil { t.Fatalf("timed out waiting for restart: %v", err) } } ================================================ FILE: tests/integration/v3_kv_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "testing" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/namespace" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestKVWithEmptyValue ensures that a get/delete with an empty value, and with WithFromKey/WithPrefix function will return an empty error. func TestKVWithEmptyValue(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) client := clus.RandClient() _, err := client.Put(t.Context(), "my-namespace/foobar", "data") require.NoError(t, err) _, err = client.Put(t.Context(), "my-namespace/foobar1", "data") require.NoError(t, err) _, err = client.Put(t.Context(), "namespace/foobar1", "data") require.NoError(t, err) // Range over all keys. resp, err := client.Get(t.Context(), "", clientv3.WithFromKey()) require.NoError(t, err) for _, kv := range resp.Kvs { t.Log(string(kv.Key), "=", string(kv.Value)) } // Range over all keys in a namespace. client.KV = namespace.NewKV(client.KV, "my-namespace/") resp, err = client.Get(t.Context(), "", clientv3.WithFromKey()) require.NoError(t, err) for _, kv := range resp.Kvs { t.Log(string(kv.Key), "=", string(kv.Value)) } // Remove all keys without WithFromKey/WithPrefix func _, err = client.Delete(t.Context(), "") // fatal error duo to without WithFromKey/WithPrefix func called. require.Error(t, err) respDel, err := client.Delete(t.Context(), "", clientv3.WithFromKey()) // fatal error duo to with WithFromKey/WithPrefix func called. require.NoError(t, err) t.Logf("delete keys:%d", respDel.Deleted) } ================================================ FILE: tests/integration/v3_leadership_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "fmt" "strings" "testing" "time" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestMoveLeader(t *testing.T) { testMoveLeader(t, true) } func TestMoveLeaderService(t *testing.T) { testMoveLeader(t, false) } func testMoveLeader(t *testing.T, auto bool) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) oldLeadIdx := clus.WaitLeader(t) oldLeadID := uint64(clus.Members[oldLeadIdx].Server.MemberID()) // ensure followers go through leader transition while leadership transfer idc := make(chan uint64) stopc := make(chan struct{}) defer close(stopc) for i := range clus.Members { if oldLeadIdx != i { go func(m *integration.Member) { select { case idc <- integration.CheckLeaderTransition(m, oldLeadID): case <-stopc: } }(clus.Members[i]) } } target := uint64(clus.Members[(oldLeadIdx+1)%3].Server.MemberID()) if auto { err := clus.Members[oldLeadIdx].Server.TryTransferLeadershipOnShutdown() require.NoError(t, err) } else { mvc := integration.ToGRPC(clus.Client(oldLeadIdx)).Maintenance _, err := mvc.MoveLeader(t.Context(), &pb.MoveLeaderRequest{TargetID: target}) require.NoError(t, err) } // wait until leader transitions have happened var newLeadIDs [2]uint64 for i := range newLeadIDs { select { case newLeadIDs[i] = <-idc: case <-time.After(time.Second): t.Fatal("timed out waiting for leader transition") } } // remaining members must agree on the same leader if newLeadIDs[0] != newLeadIDs[1] { t.Fatalf("expected same new leader %d == %d", newLeadIDs[0], newLeadIDs[1]) } // new leader must be different than the old leader if oldLeadID == newLeadIDs[0] { t.Fatalf("expected old leader %d != new leader %d", oldLeadID, newLeadIDs[0]) } // if move-leader were used, new leader must match transferee if !auto { if newLeadIDs[0] != target { t.Fatalf("expected new leader %d != target %d", newLeadIDs[0], target) } } } // TestMoveLeaderError ensures that request to non-leader fail. func TestMoveLeaderError(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) oldLeadIdx := clus.WaitLeader(t) followerIdx := (oldLeadIdx + 1) % 3 target := uint64(clus.Members[(oldLeadIdx+2)%3].Server.MemberID()) mvc := integration.ToGRPC(clus.Client(followerIdx)).Maintenance _, err := mvc.MoveLeader(t.Context(), &pb.MoveLeaderRequest{TargetID: target}) if !eqErrGRPC(err, rpctypes.ErrGRPCNotLeader) { t.Errorf("err = %v, want %v", err, rpctypes.ErrGRPCNotLeader) } } // TestMoveLeaderToLearnerError ensures that leader transfer to learner member will fail. func TestMoveLeaderToLearnerError(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) defer clus.Terminate(t) // we have to add and launch learner member after initial cluster was created, because // bootstrapping a cluster with learner member is not supported. clus.AddAndLaunchLearnerMember(t) learners, err := clus.GetLearnerMembers() if err != nil { t.Fatalf("failed to get the learner members in Cluster: %v", err) } if len(learners) != 1 { t.Fatalf("added 1 learner to Cluster, got %d", len(learners)) } learnerID := learners[0].ID leaderIdx := clus.WaitLeader(t) cli := clus.Client(leaderIdx) _, err = cli.MoveLeader(t.Context(), learnerID) if err == nil { t.Fatalf("expecting leader transfer to learner to fail, got no error") } expectedErrKeywords := "bad leader transferee" if !strings.Contains(err.Error(), expectedErrKeywords) { t.Errorf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error()) } } // TestTransferLeadershipWithLearner ensures TryTransferLeadershipOnShutdown does not timeout due to learner is // automatically picked by leader as transferee. func TestTransferLeadershipWithLearner(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) clus.AddAndLaunchLearnerMember(t) learners, err := clus.GetLearnerMembers() if err != nil { t.Fatalf("failed to get the learner members in Cluster: %v", err) } if len(learners) != 1 { t.Fatalf("added 1 learner to Cluster, got %d", len(learners)) } leaderIdx := clus.WaitLeader(t) errCh := make(chan error, 1) go func() { // note that this cluster has 1 leader and 1 learner. TryTransferLeadershipOnShutdown should return nil. // Leadership transfer is skipped in cluster with 1 voting member. errCh <- clus.Members[leaderIdx].Server.TryTransferLeadershipOnShutdown() }() select { case err := <-errCh: if err != nil { t.Errorf("got error during leadership transfer: %v", err) } case <-time.After(5 * time.Second): t.Error("timed out waiting for leader transition") } } func TestFirstCommitNotification(t *testing.T) { integration.BeforeTest(t) ctx := t.Context() clusterSize := 3 cluster := integration.NewCluster(t, &integration.ClusterConfig{Size: clusterSize}) defer cluster.Terminate(t) oldLeaderIdx := cluster.WaitLeader(t) oldLeaderClient := cluster.Client(oldLeaderIdx) newLeaderIdx := (oldLeaderIdx + 1) % clusterSize newLeaderID := uint64(cluster.Members[newLeaderIdx].ID()) notifiers := make(map[int]<-chan struct{}, clusterSize) for i, clusterMember := range cluster.Members { notifiers[i] = clusterMember.Server.FirstCommitInTermNotify() } _, err := oldLeaderClient.MoveLeader(t.Context(), newLeaderID) if err != nil { t.Errorf("got error during leadership transfer: %v", err) } t.Logf("Leadership transferred.") t.Logf("Submitting write to make sure empty and 'foo' index entry was already flushed") cli := cluster.RandClient() if _, err = cli.Put(ctx, "foo", "bar"); err != nil { t.Fatalf("Failed to put kv pair.") } // It's guaranteed now that leader contains the 'foo'->'bar' index entry. leaderAppliedIndex := cluster.Members[newLeaderIdx].Server.AppliedIndex() ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() group, groupContext := errgroup.WithContext(ctx) for i, notifier := range notifiers { member := cluster.Members[i] notifier := notifier group.Go(func() error { return checkFirstCommitNotification(groupContext, t, member, leaderAppliedIndex, notifier) }) } err = group.Wait() if err != nil { t.Error(err) } } func checkFirstCommitNotification( ctx context.Context, tb testing.TB, member *integration.Member, leaderAppliedIndex uint64, notifier <-chan struct{}, ) error { // wait until server applies all the changes of leader for member.Server.AppliedIndex() < leaderAppliedIndex { tb.Logf("member.Server.AppliedIndex():%v <= leaderAppliedIndex:%v", member.Server.AppliedIndex(), leaderAppliedIndex) select { case <-ctx.Done(): return ctx.Err() default: time.Sleep(100 * time.Millisecond) } } select { case msg, ok := <-notifier: if ok { return fmt.Errorf( "member with ID %d got message via notifier, msg: %v", member.ID(), msg, ) } default: tb.Logf("member.Server.AppliedIndex():%v >= leaderAppliedIndex:%v", member.Server.AppliedIndex(), leaderAppliedIndex) return fmt.Errorf( "notification was not triggered, member ID: %d", member.ID(), ) } return nil } ================================================ FILE: tests/integration/v3_lease_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "errors" "fmt" "math" "strconv" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" "go.etcd.io/etcd/client/pkg/v3/testutil" clientv3 "go.etcd.io/etcd/client/v3" framecfg "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/integration" gofail "go.etcd.io/gofail/runtime" ) // TestV3LeasePromote ensures the newly elected leader can promote itself // to the primary lessor, refresh the leases and start to manage leases. // TODO: use customized clock to make this test go faster? func TestV3LeasePromote(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, UseBridge: true}) defer clus.Terminate(t) // create lease lresp, err := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 3}) ttl := time.Duration(lresp.TTL) * time.Second afterGrant := time.Now() require.NoError(t, err) require.Empty(t, lresp.Error) // wait until the lease is going to expire. time.Sleep(time.Until(afterGrant.Add(ttl - time.Second))) // kill the current leader, all leases should be refreshed. toStop := clus.WaitMembersForLeader(t, clus.Members) beforeStop := time.Now() clus.Members[toStop].Stop(t) var toWait []*integration.Member for i, m := range clus.Members { if i != toStop { toWait = append(toWait, m) } } clus.WaitMembersForLeader(t, toWait) clus.Members[toStop].Restart(t) clus.WaitMembersForLeader(t, clus.Members) afterReelect := time.Now() // ensure lease is refreshed by waiting for a "long" time. // it was going to expire anyway. time.Sleep(time.Until(beforeStop.Add(ttl - time.Second))) if !leaseExist(t, clus, lresp.ID) { t.Error("unexpected lease not exists") } // wait until the renewed lease is expected to expire. time.Sleep(time.Until(afterReelect.Add(ttl))) // wait for up to 10 seconds for lease to expire. expiredCondition := func() (bool, error) { return !leaseExist(t, clus, lresp.ID), nil } expired, err := testutil.Poll(100*time.Millisecond, 10*time.Second, expiredCondition) if err != nil { t.Error(err) } if !expired { t.Error("unexpected lease exists") } } // TestV3LeaseRevoke ensures a key is deleted once its lease is revoked. func TestV3LeaseRevoke(t *testing.T) { integration.BeforeTest(t) testLeaseRemoveLeasedKey(t, func(clus *integration.Cluster, leaseID int64) error { lc := integration.ToGRPC(clus.RandClient()).Lease _, err := lc.LeaseRevoke(t.Context(), &pb.LeaseRevokeRequest{ID: leaseID}) return err }) } // TestV3LeaseGrantByID ensures leases may be created by a given id. func TestV3LeaseGrantByID(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) // create fixed lease lresp, err := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant( t.Context(), &pb.LeaseGrantRequest{ID: 1, TTL: 1}) if err != nil { t.Errorf("could not create lease 1 (%v)", err) } if lresp.ID != 1 { t.Errorf("got id %v, wanted id %v", lresp.ID, 1) } // create duplicate fixed lease _, err = integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant( t.Context(), &pb.LeaseGrantRequest{ID: 1, TTL: 1}) if !eqErrGRPC(err, rpctypes.ErrGRPCLeaseExist) { t.Error(err) } // create fresh fixed lease lresp, err = integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant( t.Context(), &pb.LeaseGrantRequest{ID: 2, TTL: 1}) if err != nil { t.Errorf("could not create lease 2 (%v)", err) } if lresp.ID != 2 { t.Errorf("got id %v, wanted id %v", lresp.ID, 2) } } // TestV3LeaseNegativeID ensures restarted member lessor can recover negative leaseID from backend. // // When the negative leaseID is used for lease revoke, all etcd nodes will remove the lease // and delete associated keys to ensure kv store data consistency // // It ensures issue 12535 is fixed by PR 13676 func TestV3LeaseNegativeID(t *testing.T) { tcs := []struct { leaseID int64 k []byte v []byte }{ { leaseID: -1, // int64 -1 is 2^64 -1 in uint64 k: []byte("foo"), v: []byte("bar"), }, { leaseID: math.MaxInt64, k: []byte("bar"), v: []byte("foo"), }, { leaseID: math.MinInt64, k: []byte("hello"), v: []byte("world"), }, } for _, tc := range tcs { t.Run(fmt.Sprintf("test with lease ID %16x", tc.leaseID), func(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() cc := clus.RandClient() lresp, err := integration.ToGRPC(cc).Lease.LeaseGrant(ctx, &pb.LeaseGrantRequest{ID: tc.leaseID, TTL: 300}) if err != nil { t.Errorf("could not create lease %d (%v)", tc.leaseID, err) } if lresp.ID != tc.leaseID { t.Errorf("got id %v, wanted id %v", lresp.ID, tc.leaseID) } putr := &pb.PutRequest{Key: tc.k, Value: tc.v, Lease: tc.leaseID} _, err = integration.ToGRPC(cc).KV.Put(ctx, putr) if err != nil { t.Errorf("couldn't put key (%v)", err) } // wait for backend Commit time.Sleep(100 * time.Millisecond) // restore lessor from db file clus.Members[2].Stop(t) err = clus.Members[2].Restart(t) require.NoError(t, err) // revoke lease should remove key integration.WaitClientV3(t, clus.Members[2].Client) _, err = integration.ToGRPC(clus.RandClient()).Lease.LeaseRevoke(ctx, &pb.LeaseRevokeRequest{ID: tc.leaseID}) if err != nil { t.Errorf("could not revoke lease %d (%v)", tc.leaseID, err) } var revision int64 for _, m := range clus.Members { getr := &pb.RangeRequest{Key: tc.k} getresp, err := integration.ToGRPC(m.Client).KV.Range(ctx, getr) require.NoError(t, err) if revision == 0 { revision = getresp.Header.Revision } if revision != getresp.Header.Revision { t.Errorf("expect revision %d, but got %d", revision, getresp.Header.Revision) } if len(getresp.Kvs) != 0 { t.Errorf("lease removed but key remains") } } }) } } // TestV3LeaseExpire ensures a key is deleted once a key expires. func TestV3LeaseExpire(t *testing.T) { integration.BeforeTest(t) testLeaseRemoveLeasedKey(t, func(clus *integration.Cluster, leaseID int64) error { // let lease lapse; wait for deleted key ctx, cancel := context.WithCancel(t.Context()) defer cancel() wStream, err := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) if err != nil { return err } wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), StartRevision: 1, }, }} if err := wStream.Send(wreq); err != nil { return err } if _, err := wStream.Recv(); err != nil { // the 'created' message return err } if _, err := wStream.Recv(); err != nil { // the 'put' message return err } errc := make(chan error, 1) go func() { resp, err := wStream.Recv() switch { case err != nil: errc <- err case len(resp.Events) != 1: fallthrough case resp.Events[0].Type != mvccpb.Event_DELETE: errc <- fmt.Errorf("expected key delete, got %v", resp) default: errc <- nil } }() select { case <-time.After(15 * time.Second): return fmt.Errorf("lease expiration too slow") case err := <-errc: return err } }) } // TestV3LeaseKeepAlive ensures keepalive keeps the lease alive. func TestV3LeaseKeepAlive(t *testing.T) { integration.BeforeTest(t) testLeaseRemoveLeasedKey(t, func(clus *integration.Cluster, leaseID int64) error { lc := integration.ToGRPC(clus.RandClient()).Lease lreq := &pb.LeaseKeepAliveRequest{ID: leaseID} ctx, cancel := context.WithCancel(t.Context()) defer cancel() lac, err := lc.LeaseKeepAlive(ctx) if err != nil { return err } defer lac.CloseSend() // renew long enough so lease would've expired otherwise for i := 0; i < 3; i++ { if err = lac.Send(lreq); err != nil { return err } lresp, rxerr := lac.Recv() if rxerr != nil { return rxerr } if lresp.ID != leaseID { return fmt.Errorf("expected lease ID %v, got %v", leaseID, lresp.ID) } time.Sleep(time.Duration(lresp.TTL/2) * time.Second) } _, err = lc.LeaseRevoke(t.Context(), &pb.LeaseRevokeRequest{ID: leaseID}) return err }) } // TestV3LeaseKeepAliveForwardingCatchError ensures the server properly generates error // codes while the follower server is forwarding LeaseKeepAlive request to the leader. func TestV3LeaseKeepAliveForwardingCatchError(t *testing.T) { integration.BeforeTest(t) // Longer than leaseHandler.ServeHTTP()'s default timeout duration sleepDuration := 8 * time.Second t.Run("forwarding succeeds", func(t *testing.T) { leader, follower, _ := setupLeaseForwardingCluster(t) leaderClient := integration.ToGRPC(leader.Client).Lease grantResp, err := leaderClient.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err) leaseID := grantResp.ID keepAliveClient, err := integration.ToGRPC(follower.Client).Lease.LeaseKeepAlive(t.Context()) require.NoError(t, err) defer keepAliveClient.CloseSend() require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID})) resp, err := keepAliveClient.Recv() require.NoError(t, err) require.Equal(t, leaseID, resp.ID) require.Positive(t, resp.TTL) }) t.Run("client cancels while forwarding", func(t *testing.T) { integration.SkipIfNoGoFail(t) leader, follower, _ := setupLeaseForwardingCluster(t) leaderClient := integration.ToGRPC(leader.Client).Lease grantResp, err := leaderClient.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err) leaseID := grantResp.ID ctx, cancel := context.WithCancel(t.Context()) keepAliveClient, err := integration.ToGRPC(follower.Client).Lease.LeaseKeepAlive(ctx) require.NoError(t, err) defer keepAliveClient.CloseSend() require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID})) _, err = keepAliveClient.Recv() require.NoError(t, err) sleepBeforeServingLeaseRenew(t, sleepDuration) // Use server metrics to verify behavior since client.Recv() always returns Canceled // after cancel() regardless of the actual server response. prevCanceledCount := getLeaseKeepAliveMetric(t, follower, "Canceled") prevUnavailableCount := getLeaseKeepAliveMetric(t, follower, "Unavailable") require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID})) time.Sleep(50 * time.Millisecond) cancel() // Client sees Canceled (gRPC returns this immediately after cancel()) _, err = keepAliveClient.Recv() require.Equal(t, codes.Canceled, status.Code(err)) require.Eventually(t, func() bool { return getLeaseKeepAliveMetric(t, follower, "Canceled") == prevCanceledCount+1 }, 3*time.Second, 100*time.Millisecond) require.Equal(t, prevUnavailableCount, getLeaseKeepAliveMetric(t, follower, "Unavailable")) }) t.Run("forwarding times out", func(t *testing.T) { integration.SkipIfNoGoFail(t) leader, follower, _ := setupLeaseForwardingCluster(t) leaderClient := integration.ToGRPC(leader.Client).Lease grantResp, err := leaderClient.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err) leaseID := grantResp.ID keepAliveClient, err := integration.ToGRPC(follower.Client).Lease.LeaseKeepAlive(t.Context()) require.NoError(t, err) defer keepAliveClient.CloseSend() require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID})) _, err = keepAliveClient.Recv() require.NoError(t, err) sleepBeforeServingLeaseRenew(t, sleepDuration) prevUnavailableCount := getLeaseKeepAliveMetric(t, follower, "Unavailable") require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID})) _, err = keepAliveClient.Recv() require.Equal(t, rpctypes.ErrGRPCTimeout, err) require.Eventually(t, func() bool { return getLeaseKeepAliveMetric(t, follower, "Unavailable") == prevUnavailableCount+1 }, 3*time.Second, 100*time.Millisecond) }) // Client set up with WithRequireLeader() will receive NoLeader error right after // monitorLeader() detects leader missing and cancels the server stream with ErrGRPCNoLeader. t.Run("catches NoLeader error with WithRequireLeader", func(t *testing.T) { leader, follower, anotherFollower := setupLeaseForwardingCluster(t) followerClient := integration.ToGRPC(follower.Client).Lease prevUnavailableCount := getLeaseKeepAliveMetric(t, follower, "Unavailable") leader.Stop(t) anotherFollower.Stop(t) keepAliveClient, err := followerClient.LeaseKeepAlive(clientv3.WithRequireLeader(t.Context())) require.NoError(t, err) _, err = keepAliveClient.Recv() require.Equal(t, rpctypes.ErrNoLeader.Error(), rpctypes.ErrorDesc(err)) // Skip metric check in proxy mode - metrics are recorded on the proxy, not the etcd server. if !integration.ThroughProxy { require.Equal(t, prevUnavailableCount+1, getLeaseKeepAliveMetric(t, follower, "Unavailable")) } }) // Client receives NoLeader error after the waitLeader() timed out in LeaseRenew(). t.Run("catches NoLeader error without WithRequireLeader", func(t *testing.T) { leader, follower, anotherFollower := setupLeaseForwardingCluster(t) leaderClient := integration.ToGRPC(leader.Client).Lease followerClient := integration.ToGRPC(follower.Client).Lease grantResp, err := leaderClient.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err) leaseID := grantResp.ID keepAliveClient, err := followerClient.LeaseKeepAlive(t.Context()) require.NoError(t, err) defer keepAliveClient.CloseSend() require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID})) _, err = keepAliveClient.Recv() require.NoError(t, err) prevUnavailableCount := getLeaseKeepAliveMetric(t, follower, "Unavailable") leader.Stop(t) anotherFollower.Stop(t) require.NoError(t, keepAliveClient.Send(&pb.LeaseKeepAliveRequest{ID: leaseID})) _, err = keepAliveClient.Recv() if integration.ThroughProxy { // Known limitation: grpcproxy doesn't propagate NoLeader error without // WithRequireLeader. The keepAliveLoop in server/proxy/grpcproxy/lease.go // discards errors and only calls cancel(), resulting in context.Canceled. // TODO: Consider fixing grpcproxy to properly propagate errors. require.ErrorIs(t, err, context.Canceled) } else { require.Equal(t, rpctypes.ErrNoLeader.Error(), rpctypes.ErrorDesc(err)) require.Equal(t, prevUnavailableCount+1, getLeaseKeepAliveMetric(t, follower, "Unavailable")) } }) } func setupLeaseForwardingCluster(t *testing.T) (*integration.Member, *integration.Member, *integration.Member) { t.Helper() cluster := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) t.Cleanup(func() { cluster.Terminate(t) }) leaderIdx := cluster.WaitLeader(t) return cluster.Members[leaderIdx], cluster.Members[(leaderIdx+1)%3], cluster.Members[(leaderIdx+2)%3] } func getLeaseKeepAliveMetric(t *testing.T, member *integration.Member, grpcCode string) int64 { t.Helper() metricVal, err := member.Metric( "grpc_server_handled_total", `grpc_method="LeaseKeepAlive"`, fmt.Sprintf(`grpc_code="%v"`, grpcCode), ) require.NoError(t, err) count, err := strconv.ParseInt(metricVal, 10, 32) require.NoError(t, err) return count } func sleepBeforeServingLeaseRenew(t *testing.T, duration time.Duration) { t.Helper() failpointName := "beforeServeHTTPLeaseRenew" require.NoError(t, gofail.Enable(failpointName, fmt.Sprintf(`sleep("%s")`, duration))) t.Cleanup(func() { terr := gofail.Disable(failpointName) if terr != nil && !errors.Is(terr, gofail.ErrDisabled) { t.Fatalf("Failed to disable failpoint %v, got error: %v", failpointName, terr) } }) } // TestV3LeaseCheckpoint ensures a lease checkpoint results in a remaining TTL being persisted // across leader elections. func TestV3LeaseCheckpoint(t *testing.T) { tcs := []struct { name string checkpointingEnabled bool ttl time.Duration checkpointingInterval time.Duration leaderChanges int clusterSize int expectTTLIsGT time.Duration expectTTLIsLT time.Duration }{ { name: "Checkpointing disabled, lease TTL is reset", ttl: 300 * time.Second, leaderChanges: 1, clusterSize: 3, expectTTLIsGT: 298 * time.Second, }, { name: "Checkpointing enabled 10s, lease TTL is preserved after leader change", ttl: 300 * time.Second, checkpointingEnabled: true, checkpointingInterval: 10 * time.Second, leaderChanges: 1, clusterSize: 3, expectTTLIsLT: 290 * time.Second, }, { name: "Checkpointing enabled 10s, lease TTL is preserved after cluster restart", ttl: 300 * time.Second, checkpointingEnabled: true, checkpointingInterval: 10 * time.Second, leaderChanges: 1, clusterSize: 1, expectTTLIsLT: 290 * time.Second, }, { // Checking if checkpointing continues after the first leader change. name: "Checkpointing enabled 10s, lease TTL is preserved after 2 leader changes", ttl: 300 * time.Second, checkpointingEnabled: true, checkpointingInterval: 10 * time.Second, leaderChanges: 2, clusterSize: 3, expectTTLIsLT: 280 * time.Second, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { integration.BeforeTest(t) config := &integration.ClusterConfig{ Size: tc.clusterSize, EnableLeaseCheckpoint: tc.checkpointingEnabled, LeaseCheckpointInterval: tc.checkpointingInterval, } clus := integration.NewCluster(t, config) defer clus.Terminate(t) // create lease ctx, cancel := context.WithCancel(t.Context()) defer cancel() c := integration.ToGRPC(clus.RandClient()) lresp, err := c.Lease.LeaseGrant(ctx, &pb.LeaseGrantRequest{TTL: int64(tc.ttl.Seconds())}) require.NoError(t, err) for i := 0; i < tc.leaderChanges; i++ { // wait for a checkpoint to occur time.Sleep(tc.checkpointingInterval + 1*time.Second) // Force a leader election leaderID := clus.WaitLeader(t) leader := clus.Members[leaderID] leader.Stop(t) time.Sleep(time.Duration(3*integration.ElectionTicks) * framecfg.TickDuration) leader.Restart(t) } newLeaderID := clus.WaitLeader(t) c2 := integration.ToGRPC(clus.Client(newLeaderID)) time.Sleep(250 * time.Millisecond) // Check the TTL of the new leader var ttlresp *pb.LeaseTimeToLiveResponse for i := 0; i < 10; i++ { if ttlresp, err = c2.Lease.LeaseTimeToLive(ctx, &pb.LeaseTimeToLiveRequest{ID: lresp.ID}); err != nil { if status, ok := status.FromError(err); ok && status.Code() == codes.Unavailable { time.Sleep(time.Millisecond * 250) } else { t.Fatal(err) } } } if tc.expectTTLIsGT != 0 && time.Duration(ttlresp.TTL)*time.Second < tc.expectTTLIsGT { t.Errorf("Expected lease ttl (%v) to be >= than (%v)", time.Duration(ttlresp.TTL)*time.Second, tc.expectTTLIsGT) } if tc.expectTTLIsLT != 0 && time.Duration(ttlresp.TTL)*time.Second > tc.expectTTLIsLT { t.Errorf("Expected lease ttl (%v) to be lower than (%v)", time.Duration(ttlresp.TTL)*time.Second, tc.expectTTLIsLT) } }) } } // TestV3LeaseExists creates a lease on a random client and confirms it exists in the cluster. func TestV3LeaseExists(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) // create lease ctx0, cancel0 := context.WithCancel(t.Context()) defer cancel0() lresp, err := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant( ctx0, &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err) require.Empty(t, lresp.Error) if !leaseExist(t, clus, lresp.ID) { t.Error("unexpected lease not exists") } } // TestV3LeaseLeases creates leases and confirms list RPC fetches created ones. func TestV3LeaseLeases(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) ctx0, cancel0 := context.WithCancel(t.Context()) defer cancel0() // create leases var ids []int64 for i := 0; i < 5; i++ { lresp, err := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant( ctx0, &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err) require.Empty(t, lresp.Error) ids = append(ids, lresp.ID) } lresp, err := integration.ToGRPC(clus.RandClient()).Lease.LeaseLeases( t.Context(), &pb.LeaseLeasesRequest{}) require.NoError(t, err) for i := range lresp.Leases { if lresp.Leases[i].ID != ids[i] { t.Fatalf("#%d: lease ID expected %d, got %d", i, ids[i], lresp.Leases[i].ID) } } } // TestV3LeaseRenewStress keeps creating lease and renewing it immediately to ensure the renewal goes through. // it was oberserved that the immediate lease renewal after granting a lease from follower resulted lease not found. // related issue https://github.com/etcd-io/etcd/issues/6978 func TestV3LeaseRenewStress(t *testing.T) { testLeaseStress(t, stressLeaseRenew, false) } // TestV3LeaseRenewStressWithClusterClient is similar to TestV3LeaseRenewStress, // but it uses a cluster client instead of a specific member's client. // The related issue is https://github.com/etcd-io/etcd/issues/13675. func TestV3LeaseRenewStressWithClusterClient(t *testing.T) { testLeaseStress(t, stressLeaseRenew, true) } // TestV3LeaseTimeToLiveStress keeps creating lease and retrieving it immediately to ensure the lease can be retrieved. // it was oberserved that the immediate lease retrieval after granting a lease from follower resulted lease not found. // related issue https://github.com/etcd-io/etcd/issues/6978 func TestV3LeaseTimeToLiveStress(t *testing.T) { testLeaseStress(t, stressLeaseTimeToLive, false) } // TestV3LeaseTimeToLiveStressWithClusterClient is similar to TestV3LeaseTimeToLiveStress, // but it uses a cluster client instead of a specific member's client. // The related issue is https://github.com/etcd-io/etcd/issues/13675. func TestV3LeaseTimeToLiveStressWithClusterClient(t *testing.T) { testLeaseStress(t, stressLeaseTimeToLive, true) } func testLeaseStress(t *testing.T, stresser func(context.Context, pb.LeaseClient) error, useClusterClient bool) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() errc := make(chan error) if useClusterClient { clusterClient, err := clus.ClusterClient(t) require.NoError(t, err) for i := 0; i < 300; i++ { go func() { errc <- stresser(ctx, integration.ToGRPC(clusterClient).Lease) }() } } else { for i := 0; i < 100; i++ { for j := 0; j < 3; j++ { go func(i int) { errc <- stresser(ctx, integration.ToGRPC(clus.Client(i)).Lease) }(j) } } } for i := 0; i < 300; i++ { err := <-errc require.NoError(t, err) } } func stressLeaseRenew(tctx context.Context, lc pb.LeaseClient) (reterr error) { defer func() { if tctx.Err() != nil { reterr = nil } }() lac, err := lc.LeaseKeepAlive(tctx) if err != nil { return err } for tctx.Err() == nil { resp, gerr := lc.LeaseGrant(tctx, &pb.LeaseGrantRequest{TTL: 60}) if gerr != nil { continue } err = lac.Send(&pb.LeaseKeepAliveRequest{ID: resp.ID}) if err != nil { continue } rresp, rxerr := lac.Recv() if rxerr != nil { continue } if rresp.TTL == 0 { return errors.New("TTL shouldn't be 0 so soon") } } return nil } func stressLeaseTimeToLive(tctx context.Context, lc pb.LeaseClient) (reterr error) { defer func() { if tctx.Err() != nil { reterr = nil } }() for tctx.Err() == nil { resp, gerr := lc.LeaseGrant(tctx, &pb.LeaseGrantRequest{TTL: 60}) if gerr != nil { continue } _, kerr := lc.LeaseTimeToLive(tctx, &pb.LeaseTimeToLiveRequest{ID: resp.ID}) if errors.Is(rpctypes.Error(kerr), rpctypes.ErrLeaseNotFound) { return kerr } } return nil } func TestV3PutOnNonExistLease(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() badLeaseID := int64(0x12345678) putr := &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar"), Lease: badLeaseID} _, err := integration.ToGRPC(clus.RandClient()).KV.Put(ctx, putr) if !eqErrGRPC(err, rpctypes.ErrGRPCLeaseNotFound) { t.Errorf("err = %v, want %v", err, rpctypes.ErrGRPCLeaseNotFound) } } // TestV3GetNonExistLease ensures client retrieving nonexistent lease on a follower doesn't result node panic // related issue https://github.com/etcd-io/etcd/issues/6537 func TestV3GetNonExistLease(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) ctx, cancel := context.WithCancel(t.Context()) defer cancel() lc := integration.ToGRPC(clus.RandClient()).Lease lresp, err := lc.LeaseGrant(ctx, &pb.LeaseGrantRequest{TTL: 10}) if err != nil { t.Errorf("failed to create lease %v", err) } _, err = lc.LeaseRevoke(t.Context(), &pb.LeaseRevokeRequest{ID: lresp.ID}) require.NoError(t, err) leaseTTLr := &pb.LeaseTimeToLiveRequest{ ID: lresp.ID, Keys: true, } for _, m := range clus.Members { // quorum-read to ensure revoke completes before TimeToLive _, err := integration.ToGRPC(m.Client).KV.Range(ctx, &pb.RangeRequest{Key: []byte("_")}) require.NoError(t, err) resp, err := integration.ToGRPC(m.Client).Lease.LeaseTimeToLive(ctx, leaseTTLr) if err != nil { t.Fatalf("expected non nil error, but go %v", err) } if resp.TTL != -1 { t.Fatalf("expected TTL to be -1, but got %v", resp.TTL) } } } // TestV3LeaseSwitch tests a key can be switched from one lease to another. func TestV3LeaseSwitch(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) key := "foo" // create lease ctx, cancel := context.WithCancel(t.Context()) defer cancel() lresp1, err1 := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant(ctx, &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err1) lresp2, err2 := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant(ctx, &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err2) // attach key on lease1 then switch it to lease2 put1 := &pb.PutRequest{Key: []byte(key), Lease: lresp1.ID} _, err := integration.ToGRPC(clus.RandClient()).KV.Put(ctx, put1) require.NoError(t, err) put2 := &pb.PutRequest{Key: []byte(key), Lease: lresp2.ID} _, err = integration.ToGRPC(clus.RandClient()).KV.Put(ctx, put2) require.NoError(t, err) // revoke lease1 should not remove key _, err = integration.ToGRPC(clus.RandClient()).Lease.LeaseRevoke(ctx, &pb.LeaseRevokeRequest{ID: lresp1.ID}) require.NoError(t, err) rreq := &pb.RangeRequest{Key: []byte("foo")} rresp, err := integration.ToGRPC(clus.RandClient()).KV.Range(t.Context(), rreq) require.NoError(t, err) if len(rresp.Kvs) != 1 { t.Fatalf("unexpect removal of key") } // revoke lease2 should remove key _, err = integration.ToGRPC(clus.RandClient()).Lease.LeaseRevoke(ctx, &pb.LeaseRevokeRequest{ID: lresp2.ID}) require.NoError(t, err) rresp, err = integration.ToGRPC(clus.RandClient()).KV.Range(t.Context(), rreq) require.NoError(t, err) if len(rresp.Kvs) != 0 { t.Fatalf("lease removed but key remains") } } // TestV3LeaseFailover ensures the old leader drops lease keepalive requests within // election timeout after it loses its quorum. And the new leader extends the TTL of // the lease to at least TTL + election timeout. func TestV3LeaseFailover(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) toIsolate := clus.WaitMembersForLeader(t, clus.Members) lc := integration.ToGRPC(clus.Client(toIsolate)).Lease // create lease lresp, err := lc.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 5}) require.NoError(t, err) require.Empty(t, lresp.Error) // isolate the current leader with its followers. clus.Members[toIsolate].Pause() lreq := &pb.LeaseKeepAliveRequest{ID: lresp.ID} md := metadata.Pairs(rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader) mctx := metadata.NewOutgoingContext(t.Context(), md) ctx, cancel := context.WithCancel(mctx) defer cancel() lac, err := lc.LeaseKeepAlive(ctx) require.NoError(t, err) // send keep alive to old leader until the old leader starts // to drop lease request. expectedExp := time.Now().Add(5 * time.Second) for { if err = lac.Send(lreq); err != nil { break } lkresp, rxerr := lac.Recv() if rxerr != nil { break } expectedExp = time.Now().Add(time.Duration(lkresp.TTL) * time.Second) time.Sleep(time.Duration(lkresp.TTL/2) * time.Second) } clus.Members[toIsolate].Resume() clus.WaitMembersForLeader(t, clus.Members) // lease should not expire at the last received expire deadline. time.Sleep(time.Until(expectedExp) - 500*time.Millisecond) if !leaseExist(t, clus, lresp.ID) { t.Error("unexpected lease not exists") } } const fiveMinTTL int64 = 300 // TestV3LeaseRecoverAndRevoke ensures that revoking a lease after restart deletes the attached key. func TestV3LeaseRecoverAndRevoke(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.Client(0)).KV lsc := integration.ToGRPC(clus.Client(0)).Lease lresp, err := lsc.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: fiveMinTTL}) require.NoError(t, err) require.Empty(t, lresp.Error) _, err = kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar"), Lease: lresp.ID}) require.NoError(t, err) // restart server and ensure lease still exists clus.Members[0].Stop(t) clus.Members[0].Restart(t) clus.WaitMembersForLeader(t, clus.Members) // overwrite old client with newly dialed connection // otherwise, error with "grpc: RPC failed fast due to transport failure" nc, err := integration.NewClientV3(clus.Members[0]) require.NoError(t, err) kvc = integration.ToGRPC(nc).KV lsc = integration.ToGRPC(nc).Lease defer nc.Close() // revoke should delete the key _, err = lsc.LeaseRevoke(t.Context(), &pb.LeaseRevokeRequest{ID: lresp.ID}) require.NoError(t, err) rresp, err := kvc.Range(t.Context(), &pb.RangeRequest{Key: []byte("foo")}) require.NoError(t, err) if len(rresp.Kvs) != 0 { t.Fatalf("lease removed but key remains") } } // TestV3LeaseRevokeAndRecover ensures that revoked key stays deleted after restart. func TestV3LeaseRevokeAndRecover(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.Client(0)).KV lsc := integration.ToGRPC(clus.Client(0)).Lease lresp, err := lsc.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: fiveMinTTL}) require.NoError(t, err) require.Empty(t, lresp.Error) _, err = kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar"), Lease: lresp.ID}) require.NoError(t, err) // revoke should delete the key _, err = lsc.LeaseRevoke(t.Context(), &pb.LeaseRevokeRequest{ID: lresp.ID}) require.NoError(t, err) // restart server and ensure revoked key doesn't exist clus.Members[0].Stop(t) clus.Members[0].Restart(t) clus.WaitMembersForLeader(t, clus.Members) // overwrite old client with newly dialed connection // otherwise, error with "grpc: RPC failed fast due to transport failure" nc, err := integration.NewClientV3(clus.Members[0]) require.NoError(t, err) kvc = integration.ToGRPC(nc).KV defer nc.Close() rresp, err := kvc.Range(t.Context(), &pb.RangeRequest{Key: []byte("foo")}) require.NoError(t, err) if len(rresp.Kvs) != 0 { t.Fatalf("lease removed but key remains") } } // TestV3LeaseRecoverKeyWithDetachedLease ensures that revoking a detached lease after restart // does not delete the key. func TestV3LeaseRecoverKeyWithDetachedLease(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.Client(0)).KV lsc := integration.ToGRPC(clus.Client(0)).Lease lresp, err := lsc.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: fiveMinTTL}) require.NoError(t, err) require.Empty(t, lresp.Error) _, err = kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar"), Lease: lresp.ID}) require.NoError(t, err) // overwrite lease with none _, err = kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}) require.NoError(t, err) // restart server and ensure lease still exists clus.Members[0].Stop(t) clus.Members[0].Restart(t) clus.WaitMembersForLeader(t, clus.Members) // overwrite old client with newly dialed connection // otherwise, error with "grpc: RPC failed fast due to transport failure" nc, err := integration.NewClientV3(clus.Members[0]) require.NoError(t, err) kvc = integration.ToGRPC(nc).KV lsc = integration.ToGRPC(nc).Lease defer nc.Close() // revoke the detached lease _, err = lsc.LeaseRevoke(t.Context(), &pb.LeaseRevokeRequest{ID: lresp.ID}) require.NoError(t, err) rresp, err := kvc.Range(t.Context(), &pb.RangeRequest{Key: []byte("foo")}) require.NoError(t, err) if len(rresp.Kvs) != 1 { t.Fatalf("only detached lease removed, key should remain") } } func TestV3LeaseRecoverKeyWithMultipleLease(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) defer clus.Terminate(t) kvc := integration.ToGRPC(clus.Client(0)).KV lsc := integration.ToGRPC(clus.Client(0)).Lease var leaseIDs []int64 for i := 0; i < 2; i++ { lresp, err := lsc.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: fiveMinTTL}) require.NoError(t, err) require.Empty(t, lresp.Error) leaseIDs = append(leaseIDs, lresp.ID) _, err = kvc.Put(t.Context(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar"), Lease: lresp.ID}) require.NoError(t, err) } // restart server and ensure lease still exists clus.Members[0].Stop(t) clus.Members[0].Restart(t) clus.WaitMembersForLeader(t, clus.Members) for i, leaseID := range leaseIDs { if !leaseExist(t, clus, leaseID) { t.Errorf("#%d: unexpected lease not exists", i) } } // overwrite old client with newly dialed connection // otherwise, error with "grpc: RPC failed fast due to transport failure" nc, err := integration.NewClientV3(clus.Members[0]) require.NoError(t, err) kvc = integration.ToGRPC(nc).KV lsc = integration.ToGRPC(nc).Lease defer nc.Close() // revoke the old lease _, err = lsc.LeaseRevoke(t.Context(), &pb.LeaseRevokeRequest{ID: leaseIDs[0]}) require.NoError(t, err) // key should still exist rresp, err := kvc.Range(t.Context(), &pb.RangeRequest{Key: []byte("foo")}) require.NoError(t, err) if len(rresp.Kvs) != 1 { t.Fatalf("only detached lease removed, key should remain") } // revoke the latest lease _, err = lsc.LeaseRevoke(t.Context(), &pb.LeaseRevokeRequest{ID: leaseIDs[1]}) require.NoError(t, err) rresp, err = kvc.Range(t.Context(), &pb.RangeRequest{Key: []byte("foo")}) require.NoError(t, err) if len(rresp.Kvs) != 0 { t.Fatalf("lease removed but key remains") } } func TestV3LeaseTimeToLiveWithLeaderChanged(t *testing.T) { t.Run("normal", func(subT *testing.T) { testV3LeaseTimeToLiveWithLeaderChanged(subT, "beforeLookupWhenLeaseTimeToLive") }) t.Run("forward", func(subT *testing.T) { testV3LeaseTimeToLiveWithLeaderChanged(subT, "beforeLookupWhenForwardLeaseTimeToLive") }) } func testV3LeaseTimeToLiveWithLeaderChanged(t *testing.T, fpName string) { integration.SkipIfNoGoFail(t) integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() oldLeadIdx := clus.WaitLeader(t) followerIdx := (oldLeadIdx + 1) % 3 followerMemberID := clus.Members[followerIdx].ID() oldLeadC := clus.Client(oldLeadIdx) leaseResp, err := oldLeadC.Grant(ctx, 100) require.NoError(t, err) require.NoError(t, gofail.Enable(fpName, `sleep("3s")`)) t.Cleanup(func() { terr := gofail.Disable(fpName) if terr != nil && !errors.Is(terr, gofail.ErrDisabled) { t.Fatalf("failed to disable %s: %v", fpName, terr) } }) readyCh := make(chan struct{}) errCh := make(chan error, 1) var targetC *clientv3.Client switch fpName { case "beforeLookupWhenLeaseTimeToLive": targetC = oldLeadC case "beforeLookupWhenForwardLeaseTimeToLive": targetC = clus.Client((oldLeadIdx + 2) % 3) default: t.Fatalf("unsupported %s failpoint", fpName) } go func() { <-readyCh time.Sleep(1 * time.Second) _, merr := oldLeadC.MoveLeader(ctx, uint64(followerMemberID)) assert.NoError(t, gofail.Disable(fpName)) errCh <- merr }() close(readyCh) ttlResp, err := targetC.TimeToLive(ctx, leaseResp.ID) require.NoError(t, err) require.GreaterOrEqual(t, int64(100), ttlResp.TTL) require.NoError(t, <-errCh) } // acquireLeaseAndKey creates a new lease and creates an attached key. func acquireLeaseAndKey(clus *integration.Cluster, key string) (int64, error) { // create lease lresp, err := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant( context.TODO(), &pb.LeaseGrantRequest{TTL: 1}) if err != nil { return 0, err } if lresp.Error != "" { return 0, errors.New(lresp.Error) } // attach to key put := &pb.PutRequest{Key: []byte(key), Lease: lresp.ID} if _, err := integration.ToGRPC(clus.RandClient()).KV.Put(context.TODO(), put); err != nil { return 0, err } return lresp.ID, nil } // testLeaseRemoveLeasedKey performs some action while holding a lease with an // attached key "foo", then confirms the key is gone. func testLeaseRemoveLeasedKey(t *testing.T, act func(*integration.Cluster, int64) error) { clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) leaseID, err := acquireLeaseAndKey(clus, "foo") require.NoError(t, err) err = act(clus, leaseID) require.NoError(t, err) // confirm no key rreq := &pb.RangeRequest{Key: []byte("foo")} rresp, err := integration.ToGRPC(clus.RandClient()).KV.Range(t.Context(), rreq) require.NoError(t, err) if len(rresp.Kvs) != 0 { t.Fatalf("lease removed but key remains") } } func leaseExist(t *testing.T, clus *integration.Cluster, leaseID int64) bool { l := integration.ToGRPC(clus.RandClient()).Lease _, err := l.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{ID: leaseID, TTL: 5}) if err == nil { _, err = l.LeaseRevoke(t.Context(), &pb.LeaseRevokeRequest{ID: leaseID}) if err != nil { t.Fatalf("failed to check lease %v", err) } return false } if eqErrGRPC(err, rpctypes.ErrGRPCLeaseExist) { return true } t.Fatalf("unexpecter error %v", err) return true } ================================================ FILE: tests/integration/v3_stm_test.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "fmt" "math/rand" "strconv" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestSTMConflict tests that conflicts are retried. func TestSTMConflict(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) etcdc := clus.RandClient() keys := make([]string, 5) for i := 0; i < len(keys); i++ { keys[i] = fmt.Sprintf("foo-%d", i) if _, err := etcdc.Put(t.Context(), keys[i], "100"); err != nil { t.Fatalf("could not make key (%v)", err) } } errc := make(chan error) for i := range keys { curEtcdc := clus.RandClient() srcKey := keys[i] applyf := func(stm concurrency.STM) { src := stm.Get(srcKey) // must be different key to avoid double-adding dstKey := srcKey for dstKey == srcKey { dstKey = keys[rand.Intn(len(keys))] } dst := stm.Get(dstKey) srcV, _ := strconv.ParseInt(src, 10, 64) dstV, _ := strconv.ParseInt(dst, 10, 64) if srcV == 0 { // can't rand.Intn on 0, so skip this transaction return } xfer := int64(rand.Intn(int(srcV)) / 2) stm.Put(srcKey, fmt.Sprintf("%d", srcV-xfer)) stm.Put(dstKey, fmt.Sprintf("%d", dstV+xfer)) } go func() { iso := concurrency.WithIsolation(concurrency.RepeatableReads) _, err := concurrency.NewSTM(curEtcdc, func(stm concurrency.STM) error { applyf(stm) return nil }, iso, ) errc <- err }() } // wait for txns for range keys { if err := <-errc; err != nil { t.Fatalf("apply failed (%v)", err) } } // ensure sum matches initial sum sum := 0 for _, oldkey := range keys { rk, err := etcdc.Get(t.Context(), oldkey) if err != nil { t.Fatalf("couldn't fetch key %s (%v)", oldkey, err) } v, _ := strconv.ParseInt(string(rk.Kvs[0].Value), 10, 64) sum += int(v) } if sum != len(keys)*100 { t.Fatalf("bad sum. got %d, expected %d", sum, len(keys)*100) } } // TestSTMPutNewKey confirms a STM put on a new key is visible after commit. func TestSTMPutNewKey(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) etcdc := clus.RandClient() applyf := func(stm concurrency.STM) error { stm.Put("foo", "bar") return nil } iso := concurrency.WithIsolation(concurrency.RepeatableReads) if _, err := concurrency.NewSTM(etcdc, applyf, iso); err != nil { t.Fatalf("error on stm txn (%v)", err) } resp, err := etcdc.Get(t.Context(), "foo") if err != nil { t.Fatalf("error fetching key (%v)", err) } if string(resp.Kvs[0].Value) != "bar" { t.Fatalf("bad value. got %+v, expected 'bar' value", resp) } } // TestSTMAbort tests that an aborted txn does not modify any keys. func TestSTMAbort(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) etcdc := clus.RandClient() ctx, cancel := context.WithCancel(t.Context()) applyf := func(stm concurrency.STM) error { stm.Put("foo", "baz") cancel() stm.Put("foo", "bap") return nil } iso := concurrency.WithIsolation(concurrency.RepeatableReads) sctx := concurrency.WithAbortContext(ctx) if _, err := concurrency.NewSTM(etcdc, applyf, iso, sctx); err == nil { t.Fatalf("no error on stm txn") } resp, err := etcdc.Get(t.Context(), "foo") if err != nil { t.Fatalf("error fetching key (%v)", err) } if len(resp.Kvs) != 0 { t.Fatalf("bad value. got %+v, expected nothing", resp) } } // TestSTMSerialize tests that serialization is honored when serializable. func TestSTMSerialize(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) defer clus.Terminate(t) etcdc := clus.RandClient() // set up initial keys keys := make([]string, 5) for i := 0; i < len(keys); i++ { keys[i] = fmt.Sprintf("foo-%d", i) } // update keys in full batches updatec := make(chan struct{}) go func() { defer close(updatec) for i := 0; i < 5; i++ { s := fmt.Sprintf("%d", i) var ops []v3.Op for _, k := range keys { ops = append(ops, v3.OpPut(k, s)) } if _, err := etcdc.Txn(t.Context()).Then(ops...).Commit(); err != nil { t.Errorf("couldn't put keys (%v)", err) } updatec <- struct{}{} } }() // read all keys in txn, make sure all values match errc := make(chan error) for range updatec { curEtcdc := clus.RandClient() applyf := func(stm concurrency.STM) error { var vs []string for i := range keys { vs = append(vs, stm.Get(keys[i])) } for i := range vs { if vs[0] != vs[i] { return fmt.Errorf("got vs[%d] = %v, want %v", i, vs[i], vs[0]) } } return nil } go func() { iso := concurrency.WithIsolation(concurrency.Serializable) _, err := concurrency.NewSTM(curEtcdc, applyf, iso) errc <- err }() } for i := 0; i < 5; i++ { if err := <-errc; err != nil { t.Error(err) } } } // TestSTMApplyOnConcurrentDeletion ensures that concurrent key deletion // fails the first GET revision comparison within STM; trigger retry. func TestSTMApplyOnConcurrentDeletion(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) etcdc := clus.RandClient() _, err := etcdc.Put(t.Context(), "foo", "bar") require.NoError(t, err) donec, readyc := make(chan struct{}), make(chan struct{}) go func() { <-readyc _, derr := etcdc.Delete(t.Context(), "foo") assert.NoError(t, derr) close(donec) }() try := 0 applyf := func(stm concurrency.STM) error { try++ stm.Get("foo") if try == 1 { // trigger delete to make GET rev comparison outdated close(readyc) <-donec } stm.Put("foo2", "bar2") return nil } iso := concurrency.WithIsolation(concurrency.RepeatableReads) _, err = concurrency.NewSTM(etcdc, applyf, iso) require.NoErrorf(t, err, "error on stm txn") if try != 2 { t.Fatalf("STM apply expected to run twice, got %d", try) } resp, err := etcdc.Get(t.Context(), "foo2") if err != nil { t.Fatalf("error fetching key (%v)", err) } if string(resp.Kvs[0].Value) != "bar2" { t.Fatalf("bad value. got %+v, expected 'bar2' value", resp) } } func TestSTMSerializableSnapshotPut(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) cli := clus.Client(0) // key with lower create/mod revision than keys being updated _, err := cli.Put(t.Context(), "a", "0") require.NoError(t, err) tries := 0 applyf := func(stm concurrency.STM) error { if tries > 2 { return fmt.Errorf("too many retries") } tries++ stm.Get("a") stm.Put("b", "1") return nil } iso := concurrency.WithIsolation(concurrency.SerializableSnapshot) _, err = concurrency.NewSTM(cli, applyf, iso) require.NoError(t, err) _, err = concurrency.NewSTM(cli, applyf, iso) require.NoError(t, err) resp, err := cli.Get(t.Context(), "b") require.NoError(t, err) if resp.Kvs[0].Version != 2 { t.Fatalf("bad version. got %+v, expected version 2", resp) } } ================================================ FILE: tests/integration/v3_tls_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "context" "crypto/tls" "errors" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/integration" ) func TestTLSClientCipherSuitesValid(t *testing.T) { testTLSCipherSuites(t, true) } func TestTLSClientCipherSuitesMismatch(t *testing.T) { testTLSCipherSuites(t, false) } // testTLSCipherSuites ensures mismatching client-side cipher suite // fail TLS handshake with the server. func testTLSCipherSuites(t *testing.T, valid bool) { integration.BeforeTest(t) cipherSuites := []uint16{ tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, } srvTLS, cliTLS := integration.TestTLSInfo, integration.TestTLSInfo if valid { srvTLS.CipherSuites, cliTLS.CipherSuites = cipherSuites, cipherSuites } else { srvTLS.CipherSuites, cliTLS.CipherSuites = cipherSuites[:2], cipherSuites[2:] } // go1.13 enables TLS 1.3 by default // and in TLS 1.3, cipher suites are not configurable, // so setting Max TLS version to TLS 1.2 to test cipher config. srvTLS.MaxVersion = tls.VersionTLS12 cliTLS.MaxVersion = tls.VersionTLS12 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, ClientTLS: &srvTLS}) defer clus.Terminate(t) cc, err := cliTLS.ClientConfig() require.NoError(t, err) cli, cerr := integration.NewClient(t, clientv3.Config{ Endpoints: []string{clus.Members[0].GRPCURL}, DialTimeout: time.Second, TLS: cc, }) if cli != nil { cli.Close() } if !valid && !errors.Is(cerr, context.DeadlineExceeded) { t.Fatalf("expected %v with TLS handshake failure, got %v", context.DeadlineExceeded, cerr) } if valid && cerr != nil { t.Fatalf("expected TLS handshake success, got %v", cerr) } } func TestTLSMinMaxVersion(t *testing.T) { integration.BeforeTest(t) tests := []struct { name string minVersion uint16 maxVersion uint16 expectError bool }{ { name: "Connect with default TLS version should succeed", minVersion: 0, maxVersion: 0, }, { name: "Connect with TLS 1.2 only should fail", minVersion: tls.VersionTLS12, maxVersion: tls.VersionTLS12, expectError: true, }, { name: "Connect with TLS 1.2 and 1.3 should succeed", minVersion: tls.VersionTLS12, maxVersion: tls.VersionTLS13, }, { name: "Connect with TLS 1.3 only should succeed", minVersion: tls.VersionTLS13, maxVersion: tls.VersionTLS13, }, } // Configure server to support TLS 1.3 only. srvTLS := integration.TestTLSInfo srvTLS.MinVersion = tls.VersionTLS13 srvTLS.MaxVersion = tls.VersionTLS13 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, ClientTLS: &srvTLS}) defer clus.Terminate(t) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cc, err := integration.TestTLSInfo.ClientConfig() require.NoError(t, err) cc.MinVersion = tt.minVersion cc.MaxVersion = tt.maxVersion cli, cerr := integration.NewClient(t, clientv3.Config{ Endpoints: []string{clus.Members[0].GRPCURL}, DialTimeout: time.Second, TLS: cc, }) if cerr != nil { assert.Truef(t, tt.expectError, "got TLS handshake error while expecting success: %v", cerr) assert.ErrorIs(t, cerr, context.DeadlineExceeded) return } cli.Close() }) } } ================================================ FILE: tests/integration/v3election_grpc_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "fmt" "testing" "time" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" epb "go.etcd.io/etcd/server/v3/etcdserver/api/v3election/v3electionpb" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestV3ElectionCampaign checks that Campaign will not give // simultaneous leadership to multiple campaigners. func TestV3ElectionCampaign(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lease1, err1 := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err1) lease2, err2 := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err2) lc := integration.ToGRPC(clus.Client(0)).Election req1 := &epb.CampaignRequest{Name: []byte("foo"), Lease: lease1.ID, Value: []byte("abc")} l1, lerr1 := lc.Campaign(t.Context(), req1) require.NoError(t, lerr1) campaignc := make(chan struct{}) go func() { defer close(campaignc) req2 := &epb.CampaignRequest{Name: []byte("foo"), Lease: lease2.ID, Value: []byte("def")} l2, lerr2 := lc.Campaign(t.Context(), req2) if lerr2 != nil { t.Error(lerr2) } if l1.Header.Revision >= l2.Header.Revision { t.Errorf("expected l1 revision < l2 revision, got %d >= %d", l1.Header.Revision, l2.Header.Revision) } }() select { case <-time.After(200 * time.Millisecond): case <-campaignc: t.Fatalf("got leadership before resign") } _, uerr := lc.Resign(t.Context(), &epb.ResignRequest{Leader: l1.Leader}) require.NoError(t, uerr) select { case <-time.After(200 * time.Millisecond): t.Fatalf("campaigner unelected after resign") case <-campaignc: } lval, lverr := lc.Leader(t.Context(), &epb.LeaderRequest{Name: []byte("foo")}) require.NoError(t, lverr) if string(lval.Kv.Value) != "def" { t.Fatalf("got election value %q, expected %q", string(lval.Kv.Value), "def") } } // TestV3ElectionObserve checks that an Observe stream receives // proclamations from different leaders uninterrupted. func TestV3ElectionObserve(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lc := integration.ToGRPC(clus.Client(0)).Election // observe leadership events observec := make(chan struct{}, 1) go func() { defer close(observec) s, err := lc.Observe(t.Context(), &epb.LeaderRequest{Name: []byte("foo")}) observec <- struct{}{} if err != nil { t.Error(err) } for i := 0; i < 10; i++ { resp, rerr := s.Recv() if rerr != nil { t.Error(rerr) } respV := 0 fmt.Sscanf(string(resp.Kv.Value), "%d", &respV) // leader transitions should not go backwards if respV < i { t.Errorf(`got observe value %q, expected >= "%d"`, string(resp.Kv.Value), i) } i = respV } }() select { case <-observec: case <-time.After(time.Second): t.Fatalf("observe stream took too long to start") } lease1, err1 := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err1) c1, cerr1 := lc.Campaign(t.Context(), &epb.CampaignRequest{Name: []byte("foo"), Lease: lease1.ID, Value: []byte("0")}) require.NoError(t, cerr1) // overlap other leader so it waits on resign leader2c := make(chan struct{}) go func() { defer close(leader2c) lease2, err2 := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) if err2 != nil { t.Error(err2) } c2, cerr2 := lc.Campaign(t.Context(), &epb.CampaignRequest{Name: []byte("foo"), Lease: lease2.ID, Value: []byte("5")}) if cerr2 != nil { t.Error(cerr2) } for i := 6; i < 10; i++ { v := []byte(fmt.Sprintf("%d", i)) req := &epb.ProclaimRequest{Leader: c2.Leader, Value: v} if _, err := lc.Proclaim(t.Context(), req); err != nil { t.Error(err) } } }() for i := 1; i < 5; i++ { v := []byte(fmt.Sprintf("%d", i)) req := &epb.ProclaimRequest{Leader: c1.Leader, Value: v} _, err := lc.Proclaim(t.Context(), req) require.NoError(t, err) } // start second leader lc.Resign(t.Context(), &epb.ResignRequest{Leader: c1.Leader}) select { case <-observec: case <-time.After(time.Second): t.Fatalf("observe did not observe all events in time") } <-leader2c } ================================================ FILE: tests/integration/v3lock_grpc_test.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package integration import ( "testing" "time" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" lockpb "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb" "go.etcd.io/etcd/tests/v3/framework/integration" ) // TestV3LockLockWaiter tests that a client will wait for a lock, then acquire it // once it is unlocked. func TestV3LockLockWaiter(t *testing.T) { integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) lease1, err1 := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err1) lease2, err2 := integration.ToGRPC(clus.RandClient()).Lease.LeaseGrant(t.Context(), &pb.LeaseGrantRequest{TTL: 30}) require.NoError(t, err2) lc := integration.ToGRPC(clus.Client(0)).Lock l1, lerr1 := lc.Lock(t.Context(), &lockpb.LockRequest{Name: []byte("foo"), Lease: lease1.ID}) require.NoError(t, lerr1) lockc := make(chan struct{}) go func() { l2, lerr2 := lc.Lock(t.Context(), &lockpb.LockRequest{Name: []byte("foo"), Lease: lease2.ID}) if lerr2 != nil { t.Error(lerr2) } if l1.Header.Revision >= l2.Header.Revision { t.Errorf("expected l1 revision < l2 revision, got %d >= %d", l1.Header.Revision, l2.Header.Revision) } close(lockc) }() select { case <-time.After(200 * time.Millisecond): case <-lockc: t.Fatalf("locked before unlock") } _, uerr := lc.Unlock(t.Context(), &lockpb.UnlockRequest{Key: l1.Key}) require.NoError(t, uerr) select { case <-time.After(200 * time.Millisecond): t.Fatalf("waiter did not lock after unlock") case <-lockc: } } ================================================ FILE: tests/robustness/Makefile ================================================ REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel) .PHONY: test-robustness-reports test-robustness-reports: export GOTOOLCHAIN := go$(shell cat $(REPOSITORY_ROOT)/.go-version) test-robustness-reports: cd $(REPOSITORY_ROOT)/tests && go test ./robustness/validate -v --count 1 --run TestDataReports # Test main and previous release branches # Note that executing make at the top-level repository needs a change in the # directory. So, instead of calling just $(MAKE) or make, use this # $(TOPLEVEL_MAKE) variable. TOPLEVEL_MAKE := $(MAKE) --directory=$(REPOSITORY_ROOT) .PHONY: test-robustness-main test-robustness-main: /tmp/etcd-main-failpoints/bin /tmp/etcd-release-3.6-failpoints/bin GO_TEST_FLAGS="$${GO_TEST_FLAGS} --bin-dir=/tmp/etcd-main-failpoints/bin --bin-last-release=/tmp/etcd-release-3.6-failpoints/bin/etcd" $(TOPLEVEL_MAKE) test-robustness .PHONY: test-robustness-release-3.6 test-robustness-release-3.6: /tmp/etcd-release-3.6-failpoints/bin /tmp/etcd-release-3.5-failpoints/bin GO_TEST_FLAGS="$${GO_TEST_FLAGS} --bin-dir=/tmp/etcd-release-3.6-failpoints/bin --bin-last-release=/tmp/etcd-release-3.5-failpoints/bin/etcd" $(TOPLEVEL_MAKE) test-robustness .PHONY: test-robustness-release-3.5 test-robustness-release-3.5: /tmp/etcd-release-3.5-failpoints/bin /tmp/etcd-release-3.4-failpoints/bin GO_TEST_FLAGS="$${GO_TEST_FLAGS} --bin-dir=/tmp/etcd-release-3.5-failpoints/bin --bin-last-release=/tmp/etcd-release-3.4-failpoints/bin/etcd" $(TOPLEVEL_MAKE) test-robustness .PHONY: test-robustness-release-3.4 test-robustness-release-3.4: /tmp/etcd-release-3.4-failpoints/bin GO_TEST_FLAGS="$${GO_TEST_FLAGS} --bin-dir=/tmp/etcd-release-3.4-failpoints/bin" $(TOPLEVEL_MAKE) test-robustness # Reproduce historical issues .PHONY: test-robustness-issue14370 test-robustness-issue14370: /tmp/etcd-v3.5.4-failpoints/bin GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue14370 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.4-failpoints/bin' $(TOPLEVEL_MAKE) test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue13766 test-robustness-issue13766: /tmp/etcd-v3.5.2-failpoints/bin GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue13766 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.2-failpoints/bin' $(TOPLEVEL_MAKE) test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue14685 test-robustness-issue14685: /tmp/etcd-v3.5.5-failpoints/bin GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue14685 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.5-failpoints/bin' $(TOPLEVEL_MAKE) test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue15220 test-robustness-issue15220: /tmp/etcd-v3.5.7-failpoints/bin GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue15220 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.7-failpoints/bin' $(TOPLEVEL_MAKE) test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue15271 test-robustness-issue15271: /tmp/etcd-v3.5.7-failpoints/bin GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue15271 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.7-failpoints/bin' $(TOPLEVEL_MAKE) test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue17529 test-robustness-issue17529: /tmp/etcd-v3.5.12-beforeSendWatchResponse/bin GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue17529 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.12-beforeSendWatchResponse/bin' $(TOPLEVEL_MAKE) test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue17780 test-robustness-issue17780: /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/bin GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue17780 --count 200 --failfast --bin-dir=/tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/bin' $(TOPLEVEL_MAKE) test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue18089 test-robustness-issue18089: /tmp/etcd-v3.5.12-beforeSendWatchResponse/bin GO_TEST_FLAGS='-v -run=TestRobustnessRegression/Issue18089 -count 100 -failfast --bin-dir=/tmp/etcd-v3.5.12-beforeSendWatchResponse/bin' $(TOPLEVEL_MAKE) test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue19179 test-robustness-issue19179: /tmp/etcd-v3.5.17-failpoints/bin GO_TEST_FLAGS='-v -run=TestRobustnessRegression/Issue19179 -count 200 -failfast --bin-dir=/tmp/etcd-v3.5.17-failpoints/bin' $(TOPLEVEL_MAKE) test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" .PHONY: test-robustness-issue20221 test-robustness-issue20221: /tmp/etcd-bc47e771-failpoints/bin GO_TEST_FLAGS='-v -run=TestRobustnessRegression/Issue20221 -count 100 -failfast --bin-dir=/tmp/etcd-bc47e771-failpoints/bin' $(TOPLEVEL_MAKE) test-robustness && \ echo "Failed to reproduce" || echo "Successful reproduction" # Etcd API usage by Kubernetes .PHONY: k8s-coverage k8s-coverage: $(TOPLEVEL_MAKE) build @echo "Running k8s coverage script" ETCD_REPO="$(REPOSITORY_ROOT)" "$(REPOSITORY_ROOT)/tests/robustness/coverage/collect_kind_traces.sh" # Failpoints GOPATH = $(shell go env GOPATH) GOFAIL_VERSION = $(shell cd $(REPOSITORY_ROOT)/tools/mod && go list -m -f {{.Version}} go.etcd.io/gofail) .PHONY:install-gofail install-gofail: $(GOPATH)/bin/gofail .PHONY: gofail-enable gofail-enable: $(GOPATH)/bin/gofail $(GOPATH)/bin/gofail enable server/etcdserver/ server/lease server/lease/leasehttp server/storage/backend/ server/storage/mvcc/ server/storage/wal/ server/etcdserver/api/v3rpc/ server/etcdserver/api/membership/ server/etcdserver/api/rafthttp/ cd $(REPOSITORY_ROOT)/server && go get go.etcd.io/gofail@${GOFAIL_VERSION} cd $(REPOSITORY_ROOT)/etcdutl && go get go.etcd.io/gofail@${GOFAIL_VERSION} cd $(REPOSITORY_ROOT)/etcdctl && go get go.etcd.io/gofail@${GOFAIL_VERSION} cd $(REPOSITORY_ROOT)/tests && go get go.etcd.io/gofail@${GOFAIL_VERSION} .PHONY: gofail-disable gofail-disable: $(GOPATH)/bin/gofail $(GOPATH)/bin/gofail disable server/etcdserver/ server/lease server/lease/leasehttp server/storage/backend/ server/storage/mvcc/ server/storage/wal/ server/etcdserver/api/v3rpc/ server/etcdserver/api/membership/ server/etcdserver/api/rafthttp/ cd $(REPOSITORY_ROOT)/server && go mod tidy cd $(REPOSITORY_ROOT)/etcdutl && go mod tidy cd $(REPOSITORY_ROOT)/etcdctl && go mod tidy cd $(REPOSITORY_ROOT)/tests && go mod tidy $(GOPATH)/bin/gofail: $(REPOSITORY_ROOT)/tools/mod/go.mod $(REPOSITORY_ROOT)/tools/mod/go.sum go install go.etcd.io/gofail@${GOFAIL_VERSION} # Build main and previous releases for robustness tests /tmp/etcd-main-failpoints/bin: $(GOPATH)/bin/gofail rm -rf /tmp/etcd-main-failpoints/ mkdir -p /tmp/etcd-main-failpoints/ cd /tmp/etcd-main-failpoints/; \ git clone --depth 1 --branch main https://github.com/etcd-io/etcd.git .; \ $(MAKE) gofail-enable; \ $(MAKE) build; /tmp/etcd-v3.6.0-failpoints/bin: $(GOPATH)/bin/gofail rm -rf /tmp/etcd-v3.6.0-failpoints/ mkdir -p /tmp/etcd-v3.6.0-failpoints/ cd /tmp/etcd-v3.6.0-failpoints/; \ git clone --depth 1 --branch main https://github.com/etcd-io/etcd.git .; \ $(MAKE) gofail-enable; \ $(MAKE) build; /tmp/etcd-bc47e771-failpoints/bin: $(GOPATH)/bin/gofail rm -rf /tmp/etcd-bc47e771-failpoints/ mkdir -p /tmp/etcd-bc47e771-failpoints/ cd /tmp/etcd-bc47e771-failpoints/; \ git init; \ git remote add origin https://github.com/etcd-io/etcd.git; \ git fetch --depth 1 origin bc47e7711664ec5557c5ae528d0d02175ea6e166; \ git checkout FETCH_HEAD; \ $(MAKE) gofail-enable; \ $(MAKE) build; /tmp/etcd-release-3.6-failpoints/bin: $(GOPATH)/bin/gofail rm -rf /tmp/etcd-release-3.6-failpoints/ mkdir -p /tmp/etcd-release-3.6-failpoints/ cd /tmp/etcd-release-3.6-failpoints/; \ git clone --depth 1 --branch release-3.6 https://github.com/etcd-io/etcd.git .; \ $(MAKE) gofail-enable; \ $(MAKE) build; /tmp/etcd-v3.5.2-failpoints/bin: /tmp/etcd-v3.5.4-failpoints/bin: /tmp/etcd-v3.5.5-failpoints/bin: /tmp/etcd-v3.5.%-failpoints/bin: $(GOPATH)/bin/gofail rm -rf /tmp/etcd-v3.5.$*-failpoints/ mkdir -p /tmp/etcd-v3.5.$*-failpoints/ cd /tmp/etcd-v3.5.$*-failpoints/; \ git clone --depth 1 --branch v3.5.$* https://github.com/etcd-io/etcd.git .; \ go get go.etcd.io/gofail@${GOFAIL_VERSION}; \ (cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ (cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ (cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ (cd tools/mod; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ FAILPOINTS=true ./build; /tmp/etcd-v3.5.12-beforeSendWatchResponse/bin: $(GOPATH)/bin/gofail rm -rf /tmp/etcd-v3.5.12-beforeSendWatchResponse/ mkdir -p /tmp/etcd-v3.5.12-beforeSendWatchResponse/ git clone --depth 1 --branch v3.5.12 https://github.com/etcd-io/etcd.git /tmp/etcd-v3.5.12-beforeSendWatchResponse/ cp -r ./tests/robustness/patches/beforeSendWatchResponse /tmp/etcd-v3.5.12-beforeSendWatchResponse/ cd /tmp/etcd-v3.5.12-beforeSendWatchResponse/; \ patch -l server/etcdserver/api/v3rpc/watch.go ./beforeSendWatchResponse/watch.patch; \ patch -l build.sh ./beforeSendWatchResponse/build.patch; \ go get go.etcd.io/gofail@${GOFAIL_VERSION}; \ (cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ (cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ (cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ (cd tools/mod; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ FAILPOINTS=true ./build; /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/bin: $(GOPATH)/bin/gofail rm -rf /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/ mkdir -p /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/ git clone --depth 1 --branch v3.5.13 https://github.com/etcd-io/etcd.git /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/ cp -r ./tests/robustness/patches/compactBeforeSetFinishedCompact /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/ cd /tmp/etcd-v3.5.13-compactBeforeSetFinishedCompact/; \ patch -l server/mvcc/kvstore_compaction.go ./compactBeforeSetFinishedCompact/kvstore_compaction.patch; \ go get go.etcd.io/gofail@${GOFAIL_VERSION}; \ (cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ (cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ (cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ (cd tools/mod; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ FAILPOINTS=true ./build; /tmp/etcd-release-3.5-failpoints/bin: $(GOPATH)/bin/gofail rm -rf /tmp/etcd-release-3.5-failpoints/ mkdir -p /tmp/etcd-release-3.5-failpoints/ cd /tmp/etcd-release-3.5-failpoints/; \ git clone --depth 1 --branch release-3.5 https://github.com/etcd-io/etcd.git .; \ go get go.etcd.io/gofail@${GOFAIL_VERSION}; \ (cd tools/mod; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ FAILPOINTS=true ./build; /tmp/etcd-v3.4.23-failpoints/bin: /tmp/etcd-v3.4.%-failpoints/bin: $(GOPATH)/bin/gofail rm -rf /tmp/etcd-v3.4.$*-failpoints/ mkdir -p /tmp/etcd-v3.4.$*-failpoints/ cd /tmp/etcd-v3.4.$*-failpoints/; \ git clone --depth 1 --branch v3.4.$* https://github.com/etcd-io/etcd.git .; \ go get go.etcd.io/gofail@${GOFAIL_VERSION}; \ (cd tools/mod; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ FAILPOINTS=true ./build; /tmp/etcd-release-3.4-failpoints/bin: $(GOPATH)/bin/gofail rm -rf /tmp/etcd-release-3.4-failpoints/ mkdir -p /tmp/etcd-release-3.4-failpoints/ cd /tmp/etcd-release-3.4-failpoints/; \ git clone --depth 1 --branch release-3.4 https://github.com/etcd-io/etcd.git .; \ go get go.etcd.io/gofail@${GOFAIL_VERSION}; \ (cd tools/mod; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \ FAILPOINTS=true ./build; ================================================ FILE: tests/robustness/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/robustness-testing ================================================ FILE: tests/robustness/README.md ================================================ # etcd Robustness Testing This document describes the robustness testing framework for etcd, a distributed key-value store. The purpose of these tests is to rigorously validate that etcd maintains its [KV API guarantees] and [watch API guarantees] under a wide range of conditions and failures. [KV API guarantees]: https://etcd.io/docs/v3.6/learning/api_guarantees/#kv-apis [watch API guarantees]: https://etcd.io/docs/v3.6/learning/api_guarantees/#watch-apis ## Robustness vs Antithesis tests [Antithesis] runs the robustness tests inside their [deterministic simulation testing](https://antithesis.com/resources/deterministic_simulation_testing/) environment and [fault injection](https://antithesis.com/docs/environment/fault_injection/). For more details on Antithesis integration, see the [antithesis directory](../antithesis). [Antithesis]: https://antithesis.com/ ## Robustness track record | Correctness / Consistency issue | Report | Introduced in | Discovered by | Last reproduction commit | Reproduction Script | | ----------------------------------------------------------------- | -------- | ------------------ | --------------------------------------------------------- | -----------------------------| --------------------------------- | | Inconsistent revision caused by crash during high load [#13766] | Mar 2022 | v3.5 | User | Load not high enough | `make test-robustness-issue13766` | | Single node cluster can lose a write on crash [#14370] | Aug 2022 | v3.4 or earlier | User | [a438759] from Jan 3, 2026 | `make test-robustness-issue14370` | | Enabling auth can lead to inconsistency [#14571] | Oct 2022 | v3.4 or earlier | User | Authorization is not covered | | | Inconsistent revision caused by crash during defrag [#14685] | Nov 2022 | v3.5 | Robustness, after covering defragmentation | [a438759] from Jan 3, 2026 | `make test-robustness-issue14685` | | Watch progress notification not synced with stream [#15220] | Jan 2023 | v3.4 or earlier | User | [a438759] from Jan 3, 2026 | `make test-robustness-issue15220` | | Watch traveling back in time after network partition [#15271] | Feb 2023 | v3.4 or earlier | Robustness, after covering network partitions | | `make test-robustness-issue15271` | | Duplicated watch event due to bug in TXN caching [#17247] | Jan 2024 | main branch | Robustness, prevented regression on main branch | | | | Watch events lost during stream starvation [#17529] | Mar 2024 | v3.4 or earlier | User | [c272ade] from May 30, 2025 | `make test-robustness-issue17529` | | Revision decreasing caused by crash during compaction [#17780] | Apr 2024 | v3.4 or earlier | Robustness, after covering compaction | | | | Watch dropping an event when compacting on delete [#18089] | May 2024 | v3.4 or earlier | Robustness, after covering compaction | [a438759] from Jan 3, 2026 | `make test-robustness-issue18089` | | Panic when two snapshots are received in a short period [#18055] | May 2024 | v3.4 or earlier | Robustness | | | | Inconsistency when reading compacted revision in TXN [#18667] | Oct 2024 | v3.4 or earlier | User | | | | Missing delete event on watch opened on same revision as compaction [#19179] | Jan 2025 | v3.4 or earlier | Robustness, after covering compaction | | `make test-robustness-issue19179` | | Watch on future revision returns notifications [#20221] | Jun 2025 | v3.4 or earlier | Robustness, after covering connection to multiple members | | `make test-robustness-issue20221` | | Watch on future revision returns old events [#20221] | Jun 2025 | v3.4 or earlier | Antithesis, after covering connection to multiple members | | | | Panic from db page expected to be 5 [#20271] | Jul 2025 | v3.4 or earlier | Antithesis | | | | Stale reads caused by process pausing [#20418] | Jul 2025 | v3.5.0 and v3.4.20 | Antithesis | | | [c272ade]: https://github.com/etcd-io/etcd/tree/c272adec29afaa69f08b7458422c53b8978c7af1 [a438759]: https://github.com/etcd-io/etcd/tree/a438759bf0bcafce851fae1a84a8511452b6b704 [#13766]: https://github.com/etcd-io/etcd/issues/13766 [#14370]: https://github.com/etcd-io/etcd/issues/14370 [#14571]: https://github.com/etcd-io/etcd/issues/14571 [#14685]: https://github.com/etcd-io/etcd/pull/14685 [#15220]: https://github.com/etcd-io/etcd/issues/15220 [#15271]: https://github.com/etcd-io/etcd/issues/15271 [#17247]: https://github.com/etcd-io/etcd/issues/17247 [#17529]: https://github.com/etcd-io/etcd/issues/17529 [#17780]: https://github.com/etcd-io/etcd/issues/17780 [#18089]: https://github.com/etcd-io/etcd/issues/18089 [#18667]: https://github.com/etcd-io/etcd/issues/18667 [#19179]: https://github.com/etcd-io/etcd/issues/19179 [#20221]: https://github.com/etcd-io/etcd/issues/20221 [#18055]: https://github.com/etcd-io/etcd/issues/18055 [#20271]: https://github.com/etcd-io/etcd/issues/20271 [#20418]: https://github.com/etcd-io/etcd/issues/20418 ## Maintaining Bug Reproducibility During Non-Trivial Changes When performing large non-trivial changes to the robustness testing framework, it is critical to ensure that we do not lose the ability to reproduce previously discovered bugs. The track record table above documents known correctness issues, and many include specific reproduction commands (e.g., `make test-robustness-issue14370`). To prevent regressions, we must ensure that the latest version of the robustness framework remains capable of reproducing old bugs. We manually track this capability in the "Last reproduction commit" column. **Best Practices:** * **Establish Baseline:** Before starting a large non-trivial change, run all reproducible test cases listed in the track record table. * **Verify Reproducibility:** After completing the change, verify that all previously reproducible bugs can still be detected. * **Update Tracking:** Refresh the "Last reproduction commit" column with commit hash and it's creation date to confirm the new framework version works. * **Update Commands:** If the change affects test execution, update the reproduction commands accordingly. * **Gate Completion:** Consider the change incomplete until all regression tests continue to catch their target bugs. This ensures that improvements to the testing framework do not inadvertently reduce our ability to detect known failure modes. ## How Robustness Tests Work Robustness tests compare the etcd cluster behavior against a simplified model of its expected behavior. These tests cover various scenarios, including: * **Different etcd cluster setups:** Cluster sizes, configurations, and deployment topologies. * **Client traffic types:** Variety of key-value operations (puts, ranges, transactions) and watch patterns. * **Failures:** Network partitions, node crashes, disk failures, and other disruptions. **Test Procedure:** 1. **Cluster Creation:** A new etcd cluster is created with the specified configuration. 2. **Traffic and Failures:** Client traffic is generated and sent to the cluster while failures are injected. 3. **History Collection:** All client operations and their results are recorded. 4. **Validation:** The collected history is validated against the etcd model and a set of validators to ensure consistency and correctness. 5. **Report Generation:** If a failure is detected then a detailed report is generated to help diagnose the issue. This report includes information about the client operations and etcd data directories. ## Key Concepts ### Distributed System Terminology * **Consensus:** A process where nodes in a distributed system agree on a single data value. Etcd uses the Raft algorithm to achieve consensus. * **Strict vs Eventual consistency:** * **Strict Consistency:** All components see the same data at the same time after an update. * **Eventual Consistency:** Components may temporarily see different data after an update but converge to the same view eventually. * **Consistency Models ()** * **Single-Object Consistency Models:** * **Sequential Consistency:** A strong single-object model. Operations appear to take place in some total order, consistent with the order of operations on each individual process. * **Linearizable Consistency:** The strongest single-object model. Operations appear to happen instantly and in order, consistent with real-time ordering. * **Transactional Consistency Models** * **Serializable Consistency:** A transactional model guaranteeing that transactions appear to occur in some total order. Operations within a transaction are atomic and do not interleave with other transactions. It's a multi-object property, applying to the entire system, not just individual objects. * **Strict Serializable Consistency:** The strongest transactional model. Combines the total order of serializability with the real-time ordering constraints of linearizability. Etcd provides strict serializability for KV operations and eventual consistency for Watch. #### Etcd Guarantees * **Key-value API operations** * **Watch API guarantees** ### Kubernetes Integration * **[Implicit Kubernetes-ETCD Contract]:** Defines how Kubernetes uses etcd to store cluster state. * **ResourceVersion:** A string used by Kubernetes to track resource versions, corresponding to etcd revisions. * **Sharding resource types:** Kubernetes treats each resource type as a totally independent entity. It allows sharding each resource type into a separate etcd cluster. [Implicit Kubernetes-ETCD Contract]: https://docs.google.com/document/d/1NUZDiJeiIH5vo_FMaTWf0JtrQKCx0kpEaIIuPoj9P6A/edit?usp=sharing ## Running locally 1. Build etcd with failpoints ```bash make gofail-enable make build make gofail-disable ``` 2. Run the tests ```bash make test-robustness ``` Optionally, you can pass environment variables: * `GO_TEST_FLAGS` - to pass additional arguments to `go test`. It is recommended to run tests multiple times with failfast enabled. this can be done by setting `GO_TEST_FLAGS='--count=100 --failfast'`. * `EXPECT_DEBUG=true` - to get logs from the cluster. * `RESULTS_DIR` - to change the location where the results report will be saved. * `PERSIST_RESULTS` - to persist the results report of the test. By default this will not be persisted in the case of a successful run. * `TRACING_SERVER_ADDR` - to export Open Telemetry traces from test runs to the collector running at given address, for example: `localhost:4317` ## Re-evaluate existing report Robustness test validation is constantly changing and improving. Errors in the etcd model could be causing false positives, which makes the ability to re-evaluate the reports after we fix the issue important. > Note: Robustness test report format is not stable, and it's expected that not all old reports can be re-evaluated using the newest version. 1. Identify the location of the robustness test report. > Note: By default robustness test report is only generated for failed test. * **For local runs:** this would be by identifying log line, in the following example that would be `/tmp/TestRobustnessExploratory_Etcd_HighTraffic_ClusterOfSize1`: ```text logger.go:146: 2024-04-08T09:45:27.734+0200 INFO Saving robustness test report {"path": "/tmp/TestRobustnessExploratory_Etcd_HighTraffic_ClusterOfSize1"} ``` * **For remote runs on CI:** you need to go to the [Prow Dashboard](https://testgrid.k8s.io/sig-etcd-robustness#Summary), go to a build, download one of the Artifacts (`artifacts/results.zip`), and extract it locally. ![Prow job run page](readme-images/prow_job.png) ![Prow job artifacts run page](readme-images/prow_job_artifacts_page.png) ![Prow job artifacts run page artifacts dir](readme-images/prow_job_artifacts_dir_page.png) Each directory will be prefixed by `TestRobustness` each containing a robustness test report. ![artifact archive](readme-images/artifact_archive.png) Pick one of the directories within the archive corresponding to the failed test scenario. The largest directory by size usually corresponds to the failed scenario. If you are not sure, you may check which scenario failed in the test logs. 2. Copy the robustness report directory into the `testdata` directory. The `testdata` directory can contain multiple robustness test reports. The name of the report directory doesn't matter, as long as it's unique to prevent clashing with reports already present in `testdata` directory. For example, the path for `history.html` file could look like `$REPO_ROOT/tests/robustness/testdata/v3.5_failure_24_April/history.html`. 3. Run `make test-robustness-reports` to validate all reports in the `testdata` directory. ## Analysing failure If robustness tests fail, we want to analyse the report to confirm if the issue is on etcd side. The location of the directory with the report is mentioned in the `Saving robustness test report` log. Logs from report generation should look like: ```text logger.go:146: 2024-05-08T10:42:54.429+0200 INFO Saving robustness test report {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550"} logger.go:146: 2024-05-08T10:42:54.429+0200 INFO Saving member data dir {"member": "TestRobustnessRegressionIssue14370-test-0", "path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/server-TestRobustnessRegressionIssue14370-test-0"} logger.go:146: 2024-05-08T10:42:54.430+0200 INFO no watch operations for client, skip persisting {"client-id": 1} logger.go:146: 2024-05-08T10:42:54.430+0200 INFO Saving operation history {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/client-1/operations.json"} logger.go:146: 2024-05-08T10:42:54.430+0200 INFO Saving watch operations {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/client-2/watch.json"} logger.go:146: 2024-05-08T10:42:54.431+0200 INFO no KV operations for client, skip persisting {"client-id": 2} logger.go:146: 2024-05-08T10:42:54.431+0200 INFO no watch operations for client, skip persisting {"client-id": 3} logger.go:146: 2024-05-08T10:42:54.431+0200 INFO Saving operation history {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/client-3/operations.json"} logger.go:146: 2024-05-08T10:42:54.433+0200 INFO no watch operations for client, skip persisting {"client-id": 4} logger.go:146: 2024-05-08T10:42:54.433+0200 INFO Saving operation history {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/client-4/operations.json"} logger.go:146: 2024-05-08T10:42:54.434+0200 INFO no watch operations for client, skip persisting {"client-id": 5} logger.go:146: 2024-05-08T10:42:54.434+0200 INFO Saving operation history {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/client-5/operations.json"} logger.go:146: 2024-05-08T10:42:54.435+0200 INFO no watch operations for client, skip persisting {"client-id": 6} logger.go:146: 2024-05-08T10:42:54.435+0200 INFO Saving operation history {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/client-6/operations.json"} logger.go:146: 2024-05-08T10:42:54.437+0200 INFO no watch operations for client, skip persisting {"client-id": 7} logger.go:146: 2024-05-08T10:42:54.437+0200 INFO Saving operation history {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/client-7/operations.json"} logger.go:146: 2024-05-08T10:42:54.438+0200 INFO no watch operations for client, skip persisting {"client-id": 8} logger.go:146: 2024-05-08T10:42:54.438+0200 INFO Saving operation history {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/client-8/operations.json"} logger.go:146: 2024-05-08T10:42:54.439+0200 INFO no watch operations for client, skip persisting {"client-id": 9} logger.go:146: 2024-05-08T10:42:54.439+0200 INFO Saving operation history {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/client-9/operations.json"} logger.go:146: 2024-05-08T10:42:54.440+0200 INFO no watch operations for client, skip persisting {"client-id": 10} logger.go:146: 2024-05-08T10:42:54.440+0200 INFO Saving operation history {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/client-10/operations.json"} logger.go:146: 2024-05-08T10:42:54.441+0200 INFO Saving visualization {"path": "/tmp/TestRobustnessRegression_Issue14370/1715157774429416550/history.html"} ``` The report follows the hierarchy: * `server-*` - etcd server data directories, can be used to verify disk/memory corruption. * `member` * `wal` - Write Ahead Log (WAL) directory, that can be analysed using `etcd-dump-logs` command line tool available in `tools` directory. * `snap` - Snapshot directory, includes the bbolt database file `db`, that can be analysed using `etcd-dump-db` command line tool available in `tools` directory. * `client-*` - Client request and response dumps in json format. * `watch.json` - Watch requests and responses, can be used to validate [watch API guarantees]. * `operations.json` - KV operation history * `history.html` - Visualization of KV operation history, can be used to validate [KV API guarantees]. ### Example analysis of a linearization issue Let's reproduce and analyse robustness test report for issue [#14370]. To reproduce the issue by yourself run `make test-robustness-issue14370`. After a couple of tries robustness tests should fail with a log `Linearization illegal` and save the report locally. Example: ```text logger.go:146: 2025-08-01T22:54:26.550+0900 INFO Validating linearizable operations {"timeout": "5m0s"} logger.go:146: 2025-08-01T22:54:26.755+0900 ERROR Linearization illegal {"duration": "205.05225ms"} logger.go:146: 2025-08-01T22:54:26.755+0900 INFO Skipping other validations as linearization failed main_test.go:122: linearization: illegal logger.go:146: 2025-08-01T22:54:26.756+0900 INFO Saving robustness test report {"path": "/tmp/TestRobustnessRegression_Issue14370/1754056466755991000"} ... logger.go:146: 2025-08-01T22:54:26.850+0900 INFO Saving visualization {"path": "/tmp/TestRobustnessRegression_Issue14370/1754056466755991000/history.html"} logger.go:146: 2025-08-01T22:54:26.878+0900 INFO killing server... {"name": "TestRobustnessRegressionIssue14370-test-0"} logger.go:146: 2025-08-01T22:54:26.878+0900 INFO stopping server... {"name": "TestRobustnessRegressionIssue14370-test-0"} logger.go:146: 2025-08-01T22:54:26.886+0900 INFO stopped server. {"name": "TestRobustnessRegressionIssue14370-test-0"} ``` Linearization issues are easiest to analyse via history visualization. Open `/tmp/TestRobustnessRegression_Issue14370/1754056466755991000/history.html` file in your browser. Jump to the error in linearization by clicking `[ jump to first error ]` on the top of the page. You should see a graph similar to the one on the image below. ![issue14370](readme-images/issue14370.png) The last correct request (connected with the grey line) is a `Put` request that succeeded and got revision `168`. All following requests are invalid (connected with red line) as they have revision `167`. Etcd guarantees that revision is non-decreasing, so this shows a bug in etcd as there is no way revision should decrease. This is consistent with the root cause of [#14370] as it was an issue with the process crash causing the last write to be lost. ### Example analysis of a watch issue Let's reproduce and analyse robustness test report for issue [#15271]. To reproduce the issue by yourself run `make test-robustness-issue15271`. After a couple of tries robustness tests should fail with a logs `Broke watch guarantee` and save report locally. Example: ```text logger.go:146: 2024-05-08T10:50:11.301+0200 INFO Validating linearizable operations {"timeout": "5m0s"} logger.go:146: 2024-05-08T10:50:15.754+0200 INFO Linearization success {"duration": "4.453346487s"} logger.go:146: 2024-05-08T10:50:15.754+0200 INFO Validating watch logger.go:146: 2024-05-08T10:50:15.849+0200 ERROR Broke watch guarantee {"guarantee": "ordered", "client": 4, "revision": 3} validate.go:45: Failed validating watch history, err: broke Ordered - events are ordered by revision; an event will never appear on a watch if it precedes an event in time that has already been posted logger.go:146: 2024-05-08T10:50:15.849+0200 INFO Validating serializable operations logger.go:146: 2024-05-08T10:50:15.866+0200 INFO Saving robustness test report {"path": "/tmp/TestRobustnessRegression_Issue15271/1715158215866033806"} ``` Watch issues are easiest to analyse by reading the recorded watch history. Watch history is recorded for each client separated in different subdirectory under `/tmp/TestRobustnessRegression_Issue15271/1715158215866033806`. Open `watch.json` for the client mentioned in the log `Broke watch guarantee`. For client `4` that broke the watch guarantee open `/tmp/TestRobustnessRegression_Issue15271/1715158215866033806/client-4/watch.json`. Each line consists of json blob corresponding to a single watch request sent by the client. Look for events with `Revision` equal to revision mentioned in the first log with `Broke watch guarantee`, in this case, look for `"Revision":3,`. You should see watch responses where the `Revision` decreases like ones below: ```text {"Events":[{"Type":"put-operation","Key":"key5","Value":{"Value":"793","Hash":0},"Revision":799,"IsCreate":false,"PrevValue":null}],"IsProgressNotify":false,"Revision":799,"Time":3202907249,"Error":""} {"Events":[{"Type":"put-operation","Key":"key4","Value":{"Value":"1","Hash":0},"Revision":3,"IsCreate":true,"PrevValue":null}, ... ``` Up to the first response, the `Revision` of events only increased up to a value of `799`. However, the following line includes an event with `Revision` equal `3`. If you follow the `revision` throughout the file you should notice that watch replayed revisions for a second time. This is incorrect and breaks `Ordered` [watch API guarantees]. This is consistent with the root cause of [#15271] where the member reconnecting to cluster will resend revisions. ================================================ FILE: tests/robustness/client/client.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package client import ( "context" "errors" "fmt" "sync" "time" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/model" "go.etcd.io/etcd/tests/v3/robustness/report" ) // RecordingClient provides a semi-etcd client (different interface than // clientv3.Client) that records all the requests and responses made. Doesn't // allow for concurrent requests to conform to model.AppendableHistory requirements. type RecordingClient struct { ID int client *clientv3.Client // using baseTime time-measuring operation to get monotonic clock reading // see https://github.com/golang/go/blob/master/src/time/time.go#L17 baseTime time.Time watchMux sync.Mutex watchOperations []model.WatchOperation // mux ensures order of request appending. kvMux sync.Mutex kvOperations *model.AppendableHistory } var _ clientv3.KV = (*RecordingClient)(nil) type TimedWatchEvent struct { model.WatchEvent Time time.Duration } func NewRecordingClient(endpoints []string, ids identity.Provider, baseTime time.Time) (*RecordingClient, error) { cc, err := clientv3.New(clientv3.Config{ Endpoints: endpoints, Logger: zap.NewNop(), DialKeepAliveTime: 10 * time.Second, DialKeepAliveTimeout: 100 * time.Millisecond, }) if err != nil { return nil, err } return &RecordingClient{ ID: ids.NewClientID(), client: cc, kvOperations: model.NewAppendableHistory(ids), baseTime: baseTime, }, nil } func (c *RecordingClient) Close() error { return c.client.Close() } func (c *RecordingClient) Report() report.ClientReport { return report.ClientReport{ ClientID: c.ID, KeyValue: c.kvOperations.History.Operations(), Watch: c.watchOperations, } } func (c *RecordingClient) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) { panic("not implemented") } func (c *RecordingClient) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { op := clientv3.OpGet(key, opts...) return c.Range(ctx, key, string(op.RangeBytes()), op.Rev(), op.Limit()) } func (c *RecordingClient) Range(ctx context.Context, start, end string, revision, limit int64) (*clientv3.GetResponse, error) { ops := []clientv3.OpOption{} if end != "" { ops = append(ops, clientv3.WithRange(end)) } if revision != 0 { ops = append(ops, clientv3.WithRev(revision)) } if limit != 0 { ops = append(ops, clientv3.WithLimit(limit)) } c.kvMux.Lock() defer c.kvMux.Unlock() callTime := time.Since(c.baseTime) resp, err := c.client.Get(ctx, start, ops...) returnTime := time.Since(c.baseTime) c.kvOperations.AppendRange(start, end, revision, limit, callTime, returnTime, resp, err) return resp, err } func (c *RecordingClient) Put(ctx context.Context, key, value string, _ ...clientv3.OpOption) (*clientv3.PutResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() callTime := time.Since(c.baseTime) resp, err := c.client.Put(ctx, key, value) returnTime := time.Since(c.baseTime) c.kvOperations.AppendPut(key, value, callTime, returnTime, resp, err) return resp, err } func (c *RecordingClient) Delete(ctx context.Context, key string, _ ...clientv3.OpOption) (*clientv3.DeleteResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() callTime := time.Since(c.baseTime) resp, err := c.client.Delete(ctx, key) returnTime := time.Since(c.baseTime) c.kvOperations.AppendDelete(key, callTime, returnTime, resp, err) return resp, err } type wrappedTxn struct { txn clientv3.Txn conditions []clientv3.Cmp onSuccess []clientv3.Op onFailure []clientv3.Op c *RecordingClient } var _ clientv3.Txn = (*wrappedTxn)(nil) func (w *wrappedTxn) If(cs ...clientv3.Cmp) clientv3.Txn { w.conditions = append(w.conditions, cs...) w.txn = w.txn.If(cs...) return w } func (w *wrappedTxn) Then(ops ...clientv3.Op) clientv3.Txn { w.onSuccess = append(w.onSuccess, ops...) w.txn = w.txn.Then(ops...) return w } func (w *wrappedTxn) Else(ops ...clientv3.Op) clientv3.Txn { w.onFailure = append(w.onFailure, ops...) w.txn = w.txn.Else(ops...) return w } func (w *wrappedTxn) Commit() (*clientv3.TxnResponse, error) { w.c.kvMux.Lock() defer w.c.kvMux.Unlock() callTime := time.Since(w.c.baseTime) resp, err := w.txn.Commit() returnTime := time.Since(w.c.baseTime) w.c.kvOperations.AppendTxn(w.conditions, w.onSuccess, w.onFailure, callTime, returnTime, resp, err) return resp, err } func (c *RecordingClient) Txn(ctx context.Context) clientv3.Txn { return &wrappedTxn{txn: c.client.Txn(ctx), c: c} } func (c *RecordingClient) LeaseGrant(ctx context.Context, ttl int64) (*clientv3.LeaseGrantResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() callTime := time.Since(c.baseTime) resp, err := c.client.Lease.Grant(ctx, ttl) returnTime := time.Since(c.baseTime) c.kvOperations.AppendLeaseGrant(callTime, returnTime, resp, err) return resp, err } func (c *RecordingClient) LeaseRevoke(ctx context.Context, leaseID int64) (*clientv3.LeaseRevokeResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() callTime := time.Since(c.baseTime) resp, err := c.client.Lease.Revoke(ctx, clientv3.LeaseID(leaseID)) returnTime := time.Since(c.baseTime) c.kvOperations.AppendLeaseRevoke(leaseID, callTime, returnTime, resp, err) return resp, err } func (c *RecordingClient) PutWithLease(ctx context.Context, key string, value string, leaseID int64) (*clientv3.PutResponse, error) { opts := clientv3.WithLease(clientv3.LeaseID(leaseID)) c.kvMux.Lock() defer c.kvMux.Unlock() callTime := time.Since(c.baseTime) resp, err := c.client.Put(ctx, key, value, opts) returnTime := time.Since(c.baseTime) c.kvOperations.AppendPutWithLease(key, value, leaseID, callTime, returnTime, resp, err) return resp, err } func (c *RecordingClient) Defragment(ctx context.Context) (*clientv3.DefragmentResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() callTime := time.Since(c.baseTime) resp, err := c.client.Defragment(ctx, c.client.Endpoints()[0]) returnTime := time.Since(c.baseTime) c.kvOperations.AppendDefragment(callTime, returnTime, resp, err) return resp, err } func (c *RecordingClient) Compact(ctx context.Context, rev int64, _ ...clientv3.CompactOption) (*clientv3.CompactResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() callTime := time.Since(c.baseTime) resp, err := c.client.Compact(ctx, rev) returnTime := time.Since(c.baseTime) c.kvOperations.AppendCompact(rev, callTime, returnTime, resp, err) return resp, err } func (c *RecordingClient) MemberList(ctx context.Context, opts ...clientv3.OpOption) (*clientv3.MemberListResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() resp, err := c.client.MemberList(ctx, opts...) return resp, err } func (c *RecordingClient) MemberAdd(ctx context.Context, peerAddrs []string) (*clientv3.MemberAddResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() resp, err := c.client.MemberAdd(ctx, peerAddrs) return resp, err } func (c *RecordingClient) MemberAddAsLearner(ctx context.Context, peerAddrs []string) (*clientv3.MemberAddResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() resp, err := c.client.MemberAddAsLearner(ctx, peerAddrs) return resp, err } func (c *RecordingClient) MemberRemove(ctx context.Context, id uint64) (*clientv3.MemberRemoveResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() resp, err := c.client.MemberRemove(ctx, id) return resp, err } func (c *RecordingClient) MemberUpdate(ctx context.Context, id uint64, peerAddrs []string) (*clientv3.MemberUpdateResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() resp, err := c.client.MemberUpdate(ctx, id, peerAddrs) return resp, err } func (c *RecordingClient) MemberPromote(ctx context.Context, id uint64) (*clientv3.MemberPromoteResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() resp, err := c.client.MemberPromote(ctx, id) return resp, err } func (c *RecordingClient) Status(ctx context.Context, endpoint string) (*clientv3.StatusResponse, error) { c.kvMux.Lock() defer c.kvMux.Unlock() resp, err := c.client.Status(ctx, endpoint) return resp, err } func (c *RecordingClient) Endpoints() []string { return c.client.Endpoints() } func (c *RecordingClient) Watch(ctx context.Context, key string, rev int64, withPrefix bool, withProgressNotify bool, withPrevKV bool) clientv3.WatchChan { request := model.WatchRequest{ Key: key, Revision: rev, WithPrefix: withPrefix, WithProgressNotify: withProgressNotify, WithPrevKV: withPrevKV, } return c.watch(ctx, request) } func (c *RecordingClient) watch(ctx context.Context, request model.WatchRequest) clientv3.WatchChan { ops := []clientv3.OpOption{} if request.WithPrefix { ops = append(ops, clientv3.WithPrefix()) } if request.Revision != 0 { ops = append(ops, clientv3.WithRev(request.Revision)) } if request.WithProgressNotify { ops = append(ops, clientv3.WithProgressNotify()) } if request.WithPrevKV { ops = append(ops, clientv3.WithPrevKV()) } respCh := make(chan clientv3.WatchResponse) responses := []model.WatchResponse{} c.watchMux.Lock() c.watchOperations = append(c.watchOperations, model.WatchOperation{ Request: request, Responses: responses, }) index := len(c.watchOperations) - 1 c.watchMux.Unlock() go func() { defer close(respCh) for r := range c.client.Watch(ctx, request.Key, ops...) { responses = append(responses, ToWatchResponse(r, c.baseTime)) c.watchMux.Lock() c.watchOperations[index].Responses = responses c.watchMux.Unlock() select { case respCh <- r: case <-ctx.Done(): return } } }() return respCh } func (c *RecordingClient) RequestProgress(ctx context.Context) error { return c.client.RequestProgress(ctx) } func ToWatchResponse(r clientv3.WatchResponse, baseTime time.Time) model.WatchResponse { // using time.Since time-measuring operation to get monotonic clock reading // see https://github.com/golang/go/blob/master/src/time/time.go#L17 resp := model.WatchResponse{Time: time.Since(baseTime)} for _, event := range r.Events { resp.Events = append(resp.Events, toWatchEvent(*event)) } resp.IsProgressNotify = r.IsProgressNotify() resp.Revision = r.Header.Revision err := r.Err() if err != nil { resp.Error = r.Err().Error() } return resp } func toWatchEvent(event clientv3.Event) (watch model.WatchEvent) { watch.Revision = event.Kv.ModRevision watch.Key = string(event.Kv.Key) watch.Value = model.ToValueOrHash(string(event.Kv.Value)) if event.PrevKv != nil { watch.PrevValue = &model.ValueRevision{ Value: model.ToValueOrHash(string(event.PrevKv.Value)), ModRevision: event.PrevKv.ModRevision, Version: event.PrevKv.Version, } } watch.IsCreate = event.IsCreate() switch event.Type { case mvccpb.Event_PUT: watch.Type = model.PutOperation case mvccpb.Event_DELETE: watch.Type = model.DeleteOperation default: panic(fmt.Sprintf("Unexpected event type: %s", event.Type)) } return watch } type ClientSet struct { idProvider identity.Provider baseTime time.Time mux sync.Mutex closed bool clients []*RecordingClient reports []report.ClientReport } func NewSet(ids identity.Provider, baseTime time.Time) *ClientSet { return &ClientSet{ idProvider: ids, baseTime: baseTime, clients: []*RecordingClient{}, } } func (cs *ClientSet) NewClient(endpoints []string) (*RecordingClient, error) { cs.mux.Lock() defer cs.mux.Unlock() if cs.closed { return nil, errors.New("the clientset is already closed") } cli, err := NewRecordingClient(endpoints, cs.idProvider, cs.baseTime) if err != nil { return nil, err } cs.clients = append(cs.clients, cli) return cli, nil } func (cs *ClientSet) Reports() []report.ClientReport { cs.mux.Lock() defer cs.mux.Unlock() if !cs.closed { cs.close() } if cs.reports == nil { reports := cs.generateReports() cs.reports = reports } return cs.reports } func (cs *ClientSet) Close() { cs.mux.Lock() defer cs.mux.Unlock() cs.close() } func (cs *ClientSet) close() { if cs.closed { return } for _, c := range cs.clients { c.Close() } cs.closed = true } func (cs *ClientSet) generateReports() []report.ClientReport { reports := make([]report.ClientReport, 0, len(cs.clients)) for _, c := range cs.clients { reports = append(reports, c.Report()) } return reports } func (cs *ClientSet) IdentityProvider() identity.Provider { return cs.idProvider } func (cs *ClientSet) BaseTime() time.Time { return cs.baseTime } ================================================ FILE: tests/robustness/client/kvhash.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package client import ( "context" "fmt" "time" "go.uber.org/zap" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func CheckEndOfTestHashKV(ctx context.Context, clus *e2e.EtcdProcessCluster) error { c, err := clientv3.New(clientv3.Config{ Endpoints: clus.EndpointsGRPC(), Logger: zap.NewNop(), DialKeepAliveTime: 10 * time.Second, DialKeepAliveTimeout: 100 * time.Millisecond, }) if err != nil { return err } defer c.Close() hashKV, err := c.HashKV(ctx, clus.EndpointsGRPC()[0], 0) if err != nil { return err } rev := hashKV.Header.Revision hashKVs := make([]*clientv3.HashKVResponse, 0) hashKVs = append(hashKVs, hashKV) for _, member := range clus.Procs { hashKV, err := c.HashKV(ctx, member.EndpointsGRPC()[0], rev) if err != nil { return err } if hashKV.Header.Revision != rev { return fmt.Errorf("latest revision at the end of the test between nodes should be the same. Want %v, get %v", rev, hashKV.Header.Revision) } hashKVs = append(hashKVs, hashKV) } for i := 1; i < len(hashKVs); i++ { if hashKVs[i-1].CompactRevision != hashKVs[i].CompactRevision { return fmt.Errorf("compactRevision mismatch, node %v has %+v, node %v has %+v", hashKVs[i-1].Header.MemberId, hashKVs[i-1], hashKVs[i].Header.MemberId, hashKVs[i]) } if hashKVs[i-1].Hash != hashKVs[i].Hash { return fmt.Errorf("hash mismatch, node %v has %+v, node %v has %+v", hashKVs[i-1].Header.MemberId, hashKVs[i-1], hashKVs[i].Header.MemberId, hashKVs[i]) } } return nil } ================================================ FILE: tests/robustness/client/watch.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package client import ( "context" "errors" "fmt" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) type CollectClusterWatchEventsParam struct { Lg *zap.Logger Endpoints []string MaxRevisionChan <-chan int64 Cfg WatchConfig ClientSet *ClientSet } func CollectClusterWatchEvents(ctx context.Context, param CollectClusterWatchEventsParam) error { var g errgroup.Group memberMaxRevisionChans := make([]chan int64, len(param.Endpoints)) for i, endpoint := range param.Endpoints { memberMaxRevisionChan := make(chan int64, 1) memberMaxRevisionChans[i] = memberMaxRevisionChan g.Go(func() error { c, err := param.ClientSet.NewClient([]string{endpoint}) if err != nil { return err } defer c.Close() return watchUntilRevision(ctx, param.Lg, c, memberMaxRevisionChan, param.Cfg) }) } g.Go(func() error { maxRevision := <-param.MaxRevisionChan for _, memberChan := range memberMaxRevisionChans { memberChan <- maxRevision } return nil }) return g.Wait() } type WatchConfig struct { RequestProgress bool } // watchUntilRevision watches all changes until context is canceled, it has observed the revision provided via maxRevisionChan or maxRevisionChan was closed. func watchUntilRevision(ctx context.Context, lg *zap.Logger, c *RecordingClient, maxRevisionChan <-chan int64, cfg WatchConfig) error { var maxRevision int64 var lastRevision int64 = 1 var closing bool ctx, cancel := context.WithCancel(ctx) defer cancel() resetWatch: for { if closing { if maxRevision == 0 { return errors.New("client didn't collect all events, max revision not set") } if lastRevision < maxRevision { return fmt.Errorf("client didn't collect all events, got: %d, expected: %d", lastRevision, maxRevision) } return nil } watch := c.Watch(ctx, "", lastRevision+1, true, true, false) for { select { case revision, ok := <-maxRevisionChan: if ok { maxRevision = revision if lastRevision >= maxRevision { closing = true cancel() } } else { // Only cancel if maxRevision was never set. if maxRevision == 0 { closing = true cancel() } } case resp, ok := <-watch: if !ok { lg.Info("Watch channel closed") continue resetWatch } if cfg.RequestProgress { c.RequestProgress(ctx) } if resp.Err() != nil { if resp.Canceled { if resp.CompactRevision > lastRevision { lastRevision = resp.CompactRevision } continue resetWatch } return fmt.Errorf("watch stream received error: %w", resp.Err()) } if len(resp.Events) > 0 { lastRevision = resp.Events[len(resp.Events)-1].Kv.ModRevision } if maxRevision != 0 && lastRevision >= maxRevision { closing = true cancel() } } } } } ================================================ FILE: tests/robustness/coverage/README.md ================================================ ## Overview Go tests in this directory analyze the usage of Etcd API based on collected traces from a Kubernetes cluster. They output information on: 1. Number of calls per gRPC method used by Kubernetes 1. Key pattern 1. Arguments provided to KV, Watch and Lease methods This information can be used to track the coverage of k8s-etcd contract. ### Manual test execution At first we will manually set up the cluster, run e2e tests, download traces and then execute the test. 1. Customize and set the environment variables used by the code snippets below: ```shell # Used for patches, building kind nodes, and running e2e tests. export KUBERNETES_REPO="$(go env GOPATH)/src/k8s.io/kubernetes" # Used when creating kind cluster and running e2e tests. export KUBECONFIG="${KUBERNETES_REPO}/kind-with-tracing-config" ``` 1. Set up [KIND cluster](https://kind.sigs.k8s.io/docs/user/quick-start/#installation), exercise Kubernetes API, export traces to [Jaeger](https://www.jaegertracing.io/) and then download them: ```shell make k8s-coverage ``` 1. Run Go test ```shell go test -v -timeout 60s go.etcd.io/etcd/tests/v3/robustness/coverage ``` 1. Clean up the environment ```shell kind delete cluster --name kind-with-external-etcd docker network rm kind-with-external-etcd ``` ### Manual trace collection from robustness tests 1. Run [Jaeger](https://www.jaegertracing.io/) container: ```shell docker run --rm --name jaeger \ -p 16686:16686 \ -p 4317:4317 \ jaegertracing/jaeger:2.6.0 --set=extensions.jaeger_storage.backends.some_storage.memory.max_traces=20000000 ``` 1. Run robustness tests. For example: ```shell env \ TRACING_SERVER_ADDR=localhost:4317 \ GO_TEST_FLAGS='--timeout 10m --count=1 -v --run "^TestRobustness.*/Kubernetes.*"' \ make test-robustness ``` 1. Download traces and put them into `tests/robustness/coverage/testdata` directory in Etcd git repository: ```shell curl -v --get --retry 10 --retry-connrefused -o testdata/demo_traces.json \ -H "Content-Type: application/json" \ --data-urlencode "query.start_time_min=$(date --date="5 days ago" -Ins)" \ --data-urlencode "query.start_time_max=$(date -Ins)" \ --data-urlencode "query.service_name=etcd" \ "http://localhost:16686/api/v3/traces" ``` 1. Run Go test ```shell go test -v -timeout 60s go.etcd.io/etcd/tests/v3/robustness/coverage ``` It will show the coverage of Kubernetes-Etcd surface by robustness tests.:w ### Automated test execution Work on improving these tests is tracked in [#20182](https://github.com/etcd-io/etcd/issues/20182) and periodic runs in [#20642](https://github.com/etcd-io/etcd/issues/20642). ================================================ FILE: tests/robustness/coverage/apiserver-shared-conf/tracing.yaml ================================================ --- apiVersion: apiserver.config.k8s.io/v1beta1 kind: TracingConfiguration endpoint: 192.168.32.1:4317 samplingRatePerMillion: 1000000 ================================================ FILE: tests/robustness/coverage/collect_kind_traces.sh ================================================ #!/usr/bin/env bash # Copyright 2025 The etcd Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This script runs KIND (Kubernetes IN Docker) cluster to collect traces with # Jaeger for analysis in robustness/coverage tests. # It is based on instructions from tests/robustness/coverage/README.md set -o errexit set -o nounset set -o pipefail # 1. Customize and set the environment variables export KUBERNETES_REPO="${KUBERNETES_REPO:-$(go env GOPATH)/src/k8s.io/kubernetes}" export ETCD_REPO="${ETCD_REPO:-$(go env GOPATH)/src/go.etcd.io/etcd}" export KUBECONFIG="${KUBERNETES_REPO}/kind-with-tracing-config" echo "Using KUBERNETES_REPO: ${KUBERNETES_REPO}" echo "Using ETCD_REPO: ${KUBERNETES_REPO}" echo "Using KUBECONFIG: ${KUBECONFIG}" echo "Applying patches to Kubernetes repo..." PATCHES_DIR="${ETCD_REPO}/tests/robustness/coverage/patches/kubernetes" pushd "${KUBERNETES_REPO}" git apply --reverse --check "${PATCHES_DIR}/"* || git apply --recount "${PATCHES_DIR}/"* echo "Building KIND node image..." kind build node-image popd echo "Creating Docker network..." docker network create kind-with-external-etcd \ --driver bridge \ --gateway "192.168.32.1" \ --subnet "192.168.32.0/24" || echo "Docker network already exists." rm_docker_network() { docker network rm kind-with-external-etcd } trap "rm_docker_network" EXIT SIGINT echo "Starting Jaeger container..." docker stop jaeger && docker wait jaeger || echo docker run --rm --detach --name jaeger \ -p 16686:16686 \ -p 4317:4317 \ jaegertracing/jaeger:2.6.0 --set=extensions.jaeger_storage.backends.some_storage.memory.max_traces=20000000 stop_jaeger() { docker stop jaeger rm_docker_network } trap "stop_jaeger" EXIT SIGINT echo "Building and starting etcd..." pushd "${ETCD_REPO}" mkdir -p "${KUBERNETES_REPO}/third_party/etcd" DATA_DIR="$(mktemp -d -p "${DATA_DIR:-/tmp}")" export DATA_DIR cp "./bin/etcd" "${KUBERNETES_REPO}/third_party/etcd/etcd" "./bin/etcd" --watch-progress-notify-interval=5s \ --data-dir "${DATA_DIR}" \ --listen-client-urls "http://192.168.32.1:2379" \ --advertise-client-urls "http://192.168.32.1:2379" \ --enable-distributed-tracing \ --distributed-tracing-address="192.168.32.1:4317" \ --distributed-tracing-service-name="etcd" \ --distributed-tracing-sampling-rate=1000000 & ETCD_PID=$! echo "etcd started with PID: ${ETCD_PID}" popd stop_etcd() { kill "${ETCD_PID}" rm -rf "${DATA_DIR}" stop_jaeger } trap "stop_etcd" EXIT SIGINT echo "Creating KIND cluster..." export KIND_EXPERIMENTAL_DOCKER_NETWORK=kind-with-external-etcd kind delete cluster --name kind-with-external-etcd delete_kind_cluster() { kind delete cluster --name kind-with-external-etcd stop_etcd } trap "delete_kind_cluster" EXIT SIGINT pushd "${ETCD_REPO}/tests/robustness/coverage" kind create cluster --config kind-with-tracing.yaml --name kind-with-external-etcd --image kindest/node:latest popd echo "Running Kubernetes e2e tests..." pushd "${KUBERNETES_REPO}" make WHAT="test/e2e/e2e.test" ./_output/bin/e2e.test \ -context kind-kind-with-external-etcd \ -ginkgo.focus="\[sig-apps\].*Conformance" \ -num-nodes 2 || echo "[sig-apps] Conformance tests failed. Ignoring..." echo "Running Kubernetes cmd tests..." ./build/run.sh ./hack/jenkins/test-cmd-dockerized.sh || echo "Command tests failed. Ignoring..." popd echo "Downloading traces..." curl -v --get --retry 10 --retry-connrefused -o "${ETCD_REPO}/tests/robustness/coverage/testdata/traces.json" \ -H "Content-Type: application/json" \ --data-urlencode "query.start_time_min=$(date --date="5 days ago" -Ins)" \ --data-urlencode "query.start_time_max=$(date --date="2 minutes ago" -Ins)" \ --data-urlencode "query.service_name=etcd" \ "http://192.168.32.1:16686/api/v3/traces" echo "Cleaning up..." set +o errexit delete_kind_cluster echo "Done." ================================================ FILE: tests/robustness/coverage/contract_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package coverage_test import ( "iter" "strings" "testing" tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" ) func contract(spansByID map[string]*tracev1.Span, span *tracev1.Span) (string, bool) { for span := range walk(spansByID, span) { for _, event := range span.GetEvents() { name := event.GetName() if strings.HasPrefix(name, "contract.") { return name, true } } } return "", false } // walk iterates over the chain of spans starting from span and then going up // the tree to parent span if such exists. func walk(spansByID map[string]*tracev1.Span, span *tracev1.Span) iter.Seq[*tracev1.Span] { return func(yield func(*tracev1.Span) bool) { for node := span; node != nil; node = spansByID[string(node.GetParentSpanId())] { if !yield(node) { return } } } } func TestContract(t *testing.T) { spansByID := map[string]*tracev1.Span{ "child": { ParentSpanId: []byte("middle"), }, "middle": { ParentSpanId: []byte("parent"), Events: []*tracev1.Span_Event{ {Name: "wrong.contract"}, }, }, "parent": { ParentSpanId: []byte("grandparent"), Events: []*tracev1.Span_Event{ {Name: "contract.OptimisticPut"}, }, }, "grandparent": { Events: []*tracev1.Span_Event{ {Name: "contract.Get"}, }, }, "outside_delete": { Events: []*tracev1.Span_Event{ {Name: "otherEvent"}, {Name: "contract.OptimisticDelete"}, }, }, "outside_invalid": { Events: []*tracev1.Span_Event{ {Name: "contractWrong"}, }, }, } for _, tc := range []struct { name string source string want string found bool }{ { name: "ok_chain", source: "child", want: "contract.OptimisticPut", found: true, }, { name: "ok_single", source: "outside_delete", want: "contract.OptimisticDelete", found: true, }, { name: "not_in_contract_chain", source: "outside_invalid", want: "", found: false, }, } { t.Run(tc.name, func(t *testing.T) { got, found := contract(spansByID, spansByID[tc.source]) if found != tc.found { t.Errorf("got=%v, want=%v", found, tc.found) } if got != tc.want { t.Errorf("got=%v, want=%v", got, tc.want) } }) } } ================================================ FILE: tests/robustness/coverage/coverage_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package coverage_test import ( "bytes" "encoding/json" "fmt" "os" "path/filepath" "strconv" "strings" "testing" "github.com/olekukonko/tablewriter" "github.com/olekukonko/tablewriter/tw" traceservice "go.opentelemetry.io/proto/otlp/collector/trace/v1" tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/encoding/protojson" ) type column struct { name string // matcher encodes column with boolean values. matcher Matcher } type method struct { name string // matcher should be a tight filter that returns true only if the method was // used to make a call to Etcd (that produced the matching trace). matcher Matcher } type refOp struct { // args splits traces into buckets based on Matcher or Modeler results. args []column // methods tries to associate Matched with method name. methods []method keyAttrName string } type Row struct { Method, Pattern, Args string } const notMatched byte = ' ' type leaseRow struct { SumUses int SumTTL int Calls int } var referenceUsageOfWatchAndKV = map[string]refOp{ "etcdserverpb.KV/Range": { args: []column{ {name: "limit", matcher: isLimitSet}, {name: "rangeEnd", matcher: isRangeEndSet}, {name: "rev", matcher: isRevisionSet}, {name: "countOnly", matcher: isCountOnly}, {name: "keysOnly", matcher: isKeysOnly}, }, keyAttrName: "range_begin", methods: []method{ { name: "Healthcheck", matcher: keyIsEqualStr("range_begin", "/registry/health"), }, { name: "Compaction", matcher: keyIsEqualStr("range_begin", "compact_rev_key"), }, { name: "Get", matcher: andMatcher( keyIsEqualInt("limit", 1), notMatcher(isRangeEndSet), ), }, { name: "Count", matcher: andMatcher( isCountOnly, isRangeEndSet, notMatcher(orMatcher(isKeysOnly, isLimitSet, isRevisionSet)), ), }, { name: "Keys", matcher: andMatcher( isKeysOnly, isRangeEndSet, notMatcher(orMatcher(isCountOnly, isLimitSet, isRevisionSet)), ), }, { name: "List", matcher: andMatcher( isRangeEndSet, notMatcher(orMatcher(isKeysOnly, isCountOnly)), ), }, { name: "GetCurrentRevision", matcher: andMatcher( keyIsEqualInt("limit", 1), isRangeEndSet, notMatcher(orMatcher(isKeysOnly, isCountOnly, isRevisionSet)), ), }, }, }, "etcdserverpb.KV/Txn": { args: []column{ {name: "getOnFailure", matcher: keyIsEqualInt("failure_len", 1)}, {name: "readOnly", matcher: isReadOnly}, {name: "lease", matcher: intAttrSet("success_first_lease")}, }, keyAttrName: "compare_first_key", methods: []method{ { name: "Compaction", matcher: keyIsEqualStr("compare_first_key", "compact_rev_key"), }, { name: "OptimisticPut", matcher: andMatcher( keyIsEqualInt("compare_len", 1), keyIsEqualInt("success_len", 1), keyIsEqualStr("success_first_type", "put"), notMatcher(isReadOnly), ), }, { name: "OptimisticDelete", matcher: andMatcher( keyIsEqualInt("compare_len", 1), keyIsEqualInt("success_len", 1), keyIsEqualStr("success_first_type", "delete_range"), notMatcher(isReadOnly), ), }, }, }, "etcdserverpb.KV/Compact": { args: []column{ {name: "rev", matcher: isRevisionSet}, {name: "physical", matcher: boolAttrSet("is_physical")}, }, methods: []method{ {name: "Compact", matcher: all}, }, }, "etcdserverpb.Watch/Watch": { args: []column{ {name: "range_end", matcher: isRangeEndSet}, {name: "start_rev", matcher: intAttrSet("start_rev")}, {name: "prev_kv", matcher: boolAttrSet("prev_kv")}, {name: "fragment", matcher: boolAttrSet("fragment")}, {name: "progress_notify", matcher: boolAttrSet("progress_notify")}, }, keyAttrName: "key", methods: []method{ {name: "Compaction", matcher: keyIsEqualStr("key", "compact_rev_key")}, {name: "Watch", matcher: andMatcher( isRangeEndSet, intAttrSet("start_rev"), boolAttrSet("prev_kv"), notMatcher(boolAttrSet("fragment")), )}, }, }, } func TestInterfaceUse(t *testing.T) { files, err := os.ReadDir("testdata") if err != nil { t.Fatal(err) } for _, file := range files { filename := file.Name() if filename == ".gitignore" { continue } t.Run(filename, func(t *testing.T) { testInterfaceUse(t, filename) }) } } func testInterfaceUse(t *testing.T, filename string) { b, err := os.ReadFile(filepath.Join("testdata", filename)) if err != nil { t.Fatalf("read test data: %v", err) } var dump Dump err = json.Unmarshal(b, &dump) if err != nil { t.Fatalf("unmarshal testdata %s: %v", filename, err) } if dump.Result == nil { t.Fatalf("missing result data") } spansByID := spansMap(t, dump.Result.GetResourceSpans()) callsByOperationName, countsByGRPC := callsMap(spansByID) t.Logf("\n%s", printableCallTable(countsByGRPC)) spansByLeaseID := leaseMap(callsByOperationName) t.Run("only_expected_methods_were_called", func(t *testing.T) { expectedEtcdMethodsCalled := map[string]bool{ // All calls should go through etcd-k8s interface "etcdserverpb.KV/Range": true, "etcdserverpb.KV/Txn": true, // Not part of the contract interface (yet) "etcdserverpb.Watch/Watch": true, // Compaction should move to using internal Etcd mechanism // Discussed in https://github.com/kubernetes/kubernetes/issues/80513 "etcdserverpb.KV/Compact": true, // Used to manage masterleases and events "etcdserverpb.Lease/LeaseGrant": true, // Used to expose database size on apiserver's metrics endpoint "etcdserverpb.Maintenance/Status": true, } for opName, count := range countsByGRPC { if _, ok := expectedEtcdMethodsCalled[opName]; !ok { t.Errorf("unexpected %d calls to method: %s", count, opName) } } }) for op, td := range referenceUsageOfWatchAndKV { t.Run(op, func(t *testing.T) { if _, ok := countsByGRPC[op]; !ok { t.Fatalf("expected %q method to be called at least once", op) } // tracesWithNoMethod ensures that we print error only once when a // new call pattern is found. tracesWithNoMethod := make(map[string]bool) callCounts := make(map[Row]int) contractCallCounts := make(map[Row]int) for _, span := range callsByOperationName[op] { args := columnsToArgs(span, td.args) pattern, pFound := extractPattern(span, td.keyAttrName) if !pFound && !tracesWithNoMethod[args] { t.Errorf("New key pattern detected: %s", pattern) tracesWithNoMethod[args] = true } method, mFound := extractMethod(td.methods, span) if !mFound && !tracesWithNoMethod[args] { t.Errorf("New call pattern detected: %s(key=%s,%s)", op, pattern, argsToDescription(args, td.args)) tracesWithNoMethod[args] = true } _, cFound := contract(spansByID, span) if cFound { contractCallCounts[Row{method, pattern, args}]++ } callCounts[Row{method, pattern, args}]++ } t.Logf("\n%s", printableMatcherTable(td.args, callCounts, contractCallCounts)) }) } t.Run("etcdserverpb.Lease/LeaseGrant", func(t *testing.T) { op := "etcdserverpb.Lease/LeaseGrant" if _, ok := countsByGRPC[op]; !ok { t.Fatalf("expected %q method to be called at least once", op) } callCounts := make(map[string]leaseRow) for _, span := range callsByOperationName[op] { leaseID, lFound := intAttr(span, "id") if !lFound { t.Errorf("Lease without ID: %v", span) continue } txns := spansByLeaseID[leaseID] pattern, pFound := extractPatternFromTxns(txns) if !pFound { t.Errorf("New key pattern detected: %s", pattern) continue } row := callCounts[pattern] ttl, found := intAttr(span, "ttl") if !found { t.Errorf("Lease without TTL: %v", span) continue } row.SumTTL += ttl row.SumUses += len(txns) row.Calls++ callCounts[pattern] = row } t.Logf("\n%s", printableLeaseTable(callCounts)) }) } func callsMap(spansByID map[string]*tracev1.Span) (map[string][]*tracev1.Span, map[string]int) { // Add to map only spans that are direct children of Etcd grpc spans. callsByOperationName := make(map[string][]*tracev1.Span) grpcCounts := make(map[string]int) for _, span := range spansByID { if isEtcdGRPC(span) { grpcCounts[span.GetName()]++ continue } parent, ok := spansByID[string(span.GetParentSpanId())] if !ok || !isEtcdGRPC(parent) { continue } opName := parent.GetName() callsByOperationName[opName] = append(callsByOperationName[opName], span) } return callsByOperationName, grpcCounts } func spansMap(t *testing.T, traces []*tracev1.ResourceSpans) map[string]*tracev1.Span { t.Helper() // Mark all traces with at least one span recorded in apiserver. inApiserver := make(map[string]bool) for _, trace := range traces { sn, sFound := serviceName(trace) if !sFound { t.Fatalf("resource span without service.name: %+v", trace) } if sn != "apiserver" { continue } for _, scopeSpan := range trace.GetScopeSpans() { for _, span := range scopeSpan.GetSpans() { inApiserver[string(span.GetTraceId())] = true } } } if len(inApiserver) == 0 { t.Logf("WARN: no records of traces from the apiserver") } // Map traces by their span ID. spansByID := make(map[string]*tracev1.Span) skipped := 0 for _, trace := range traces { for _, scopeSpan := range trace.GetScopeSpans() { for _, span := range scopeSpan.GetSpans() { if len(inApiserver) > 0 && !inApiserver[string(span.GetTraceId())] { skipped++ continue } id := string(span.GetSpanId()) if id == "" { t.Fatalf("span without id: %+v", span) } spansByID[id] = span } } } if skipped > 0 { t.Logf("WARN: skipped %d spans without traces in apiserver", skipped) } return spansByID } func leaseMap(callsByOperationName map[string][]*tracev1.Span) map[int][]*tracev1.Span { ret := make(map[int][]*tracev1.Span) for _, leaseSpan := range callsByOperationName["etcdserverpb.Lease/LeaseGrant"] { leaseID, lFound := intAttr(leaseSpan, "id") if !lFound { continue } ret[leaseID] = nil } for _, txnSpan := range callsByOperationName["etcdserverpb.KV/Txn"] { leaseID, lFound := intAttr(txnSpan, "success_first_lease") if !lFound { continue } ret[leaseID] = append(ret[leaseID], txnSpan) } return ret } func extractPatternFromTxns(txns []*tracev1.Span) (string, bool) { patterns := make(map[string]int) for _, txn := range txns { key, kFound := strAttr(txn, "success_first_key") if kFound { p, pFound := pattern(key) if !pFound { return key, false } patterns[p]++ } } if len(patterns) > 1 { return "multiple key patterns", false } for p := range patterns { return p, true } return "no pattern found", false } func extractPattern(span *tracev1.Span, key string) (string, bool) { if key == "" { return "", true } k, found := strAttr(span, key) if !found { return "", false } return pattern(k) } func columnsToArgs(span *tracev1.Span, cols []column) string { acc := make([]byte, len(cols)) for i, col := range cols { if col.matcher(span) { acc[i] = 'X' } else { acc[i] = notMatched } } return string(acc) } func argsToDescription(matched string, cols []column) string { ret := make([]string, len(cols)) for i, col := range cols { ret[i] = fmt.Sprintf("%s=%v", col.name, matched[i] != notMatched) } return strings.Join(ret, ",") } func extractMethod(methodToMatched []method, span *tracev1.Span) (string, bool) { for _, mm := range methodToMatched { if mm.matcher(span) { return mm.name, true } } return "", false } type Traces struct { traceservice.ExportTraceServiceRequest } func (t *Traces) UnmarshalJSON(b []byte) error { return protojson.Unmarshal(b, &t.ExportTraceServiceRequest) } type Dump struct { Result *Traces `json:"result"` } func printableCallTable(callsByOperationName map[string]int) string { buf := new(bytes.Buffer) cfgBuilder := tablewriter.NewConfigBuilder().WithRowAlignment(tw.AlignRight) table := tablewriter.NewTable(buf, tablewriter.WithConfig(cfgBuilder.Build())) table.Header("method", "calls", "percent") totalCalls := 0 for _, c := range callsByOperationName { totalCalls += c } for opName, callCount := range callsByOperationName { table.Append(opName, callCount, fmt.Sprintf("%.2f%%", float64(callCount*100)/float64(totalCalls))) } table.Footer("total", totalCalls, "100.00%") table.Render() return buf.String() } func printableMatcherTable(cols []column, res map[Row]int, contract map[Row]int) string { keys := sortPatternTable(res) buf := new(bytes.Buffer) width := 2 + len(cols) + 3 alignment := make([]tw.Align, width) alignment[1] = tw.AlignLeft cfgBuilder := tablewriter.NewConfigBuilder(). WithRowAlignment(tw.AlignRight). Row().Alignment().WithPerColumn(alignment).Build() table := tablewriter.NewTable(buf, tablewriter.WithConfig(cfgBuilder.Build())) hdr := make([]string, width) hdr[0] = "method" hdr[1] = "pattern" for i, col := range cols { hdr[i+2] = col.name } hdr[len(hdr)-3] = "contract" hdr[len(hdr)-2] = "calls" hdr[len(hdr)-1] = "percent" table.Header(hdr) totalCalls := 0 for _, c := range res { totalCalls += c } totalContractCalls := 0 for _, c := range contract { totalContractCalls += c } footer := make([]int, len(cols)) for _, r := range keys { callCount := res[r] rowPrefix := make([]string, len(cols)) for i := range cols { rowPrefix[i] = string(r.Args[i]) if r.Args[i] != notMatched { footer[i] += callCount } } table.Append(append( []string{r.Method, r.Pattern}, append(rowPrefix, fmt.Sprintf("%.2f%%", float64(contract[r]*100)/float64(callCount)), strconv.Itoa(callCount), fmt.Sprintf("%.2f%%", float64(callCount*100)/float64(totalCalls)), )...)) } footerStr := make([]string, len(cols)) for i := range footer { footerStr[i] = strconv.Itoa(footer[i]) } table.Footer(append([]string{"", ""}, append( footerStr, strconv.Itoa(totalContractCalls), strconv.Itoa(totalCalls), "100.00%", )...)) table.Render() return buf.String() } func printableLeaseTable(callCounts map[string]leaseRow) string { hdr := []string{"pattern", "avg uses", "avg ttl", "calls", "percent"} buf := new(bytes.Buffer) alignment := make([]tw.Align, len(hdr)) alignment[0] = tw.AlignLeft cfgBuilder := tablewriter.NewConfigBuilder(). WithRowAlignment(tw.AlignRight). Row().Alignment().WithPerColumn(alignment).Build() table := tablewriter.NewTable(buf, tablewriter.WithConfig(cfgBuilder.Build())) table.Header(hdr) totalCalls := 0 for _, row := range callCounts { totalCalls += row.Calls } for pattern, row := range callCounts { table.Append( pattern, fmt.Sprintf("%.1f", float64(row.SumUses)/float64(row.Calls)), fmt.Sprintf("%.1f", float64(row.SumTTL)/float64(row.Calls)), strconv.Itoa(row.Calls), fmt.Sprintf("%.2f%%", float64(row.Calls*100)/float64(totalCalls)), ) } table.Footer([]string{3: strconv.Itoa(totalCalls), 4: "100.00%"}) table.Render() return buf.String() } ================================================ FILE: tests/robustness/coverage/key_pattern_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package coverage_test import ( "strings" "testing" ) func pattern(key string) (string, bool) { if key == "/registry/health" || key == "compact_rev_key" { return key, true } key, valid := strings.CutPrefix(key, "/registry") if !valid { return key, false } slash := "" if strings.HasSuffix(key, "/") { slash = "/" } parts := strings.Split(strings.Trim(key, "/"), "/") if len(parts[0]) == 0 { return key, false } apiResource, shift := resourceOrCRD(parts[0]) parts = parts[shift:] switch len(parts) { case 0: // Listing all resources. return "/registry/" + apiResource + slash, true case 1: if slash == "" { // Get on a non-namespaced resource. return "/registry/" + apiResource + "/{name}", true } // Listing all resources in a namespace. return "/registry/" + apiResource + "/{namespace}" + slash, true case 2: if slash == "" { // Get on a namespaced resource. return "/registry/" + apiResource + "/{namespace}/{name}", true } } return key, false } func resourceOrCRD(s string) (string, int) { if strings.Contains(s, ".") { // Group name from the Custom Resource Definition. // Names without dots are technically valid DNS name, but against best practices: // https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.33/#customresourcedefinitionspec-v1-apiextensions-k8s-io return "{api-group}/{resource}", 2 } switch s { case "services": // services have subresources specs and endpoints. return "services/{subresource}", 2 case "masterleases": // masterleases are not a regular resource. return "masterleases", 1 case "events": // events have watch cache disabled by default and use Etcd leases. return "events", 1 } return "{resource}", 1 } func TestPatternValid(t *testing.T) { for _, tc := range []struct { key string want string }{ { key: "compact_rev_key", want: "compact_rev_key", }, { key: "/registry/health", want: "/registry/health", }, { key: "/registry/resource/namespace/object", want: "/registry/{resource}/{namespace}/{name}", }, { key: "/registry/resource/namespace_list/", want: "/registry/{resource}/{namespace}/", }, { key: "/registry/services/specs/namespace/object", want: "/registry/services/{subresource}/{namespace}/{name}", }, { key: "/registry/services/endpoints", want: "/registry/services/{subresource}", }, { key: "/registry/services/specs/namespace_list/", want: "/registry/services/{subresource}/{namespace}/", }, { key: "/registry/resource/object_get", want: "/registry/{resource}/{name}", }, { key: "/registry/resource_rev", want: "/registry/{resource}", }, { key: "/registry/resource_list/", want: "/registry/{resource}/", }, { key: "/registry/crd.io/resource/namespace/object_get", want: "/registry/{api-group}/{resource}/{namespace}/{name}", }, { key: "/registry/crd.io/resource/namespace_list/", want: "/registry/{api-group}/{resource}/{namespace}/", }, { key: "/registry/crd.io/resource/object_get", want: "/registry/{api-group}/{resource}/{name}", }, { key: "/registry/crd.io/resource_rev", want: "/registry/{api-group}/{resource}", }, { key: "/registry/crd.io/resource_list/", want: "/registry/{api-group}/{resource}/", }, } { name := strings.Trim(strings.ReplaceAll(tc.key, "/", "_"), "_") t.Run(name, func(t *testing.T) { got, valid := pattern(tc.key) if !valid { t.Fatalf("key=%q, want=valid, got=invalid", tc.key) } if got != tc.want { t.Fatalf("key=%q, want=%s, got=%s", tc.key, tc.want, got) } }) } } func TestPatternInvalid(t *testing.T) { for _, tc := range []string{ "/other-registry", "/registry//", } { name := strings.Trim(strings.ReplaceAll(tc, "/", "_"), "_") t.Run(name, func(t *testing.T) { got, valid := pattern(tc) if valid { t.Fatalf("key=%q, want=invalid, got=valid, pattern=%q", tc, got) } }) } } ================================================ FILE: tests/robustness/coverage/kind-with-tracing.yaml ================================================ --- kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 featureGates: "APIServerTracing": true nodes: - role: control-plane extraMounts: - hostPath: apiserver-shared-conf containerPath: /apiserver-shared-conf readOnly: true kubeadmConfigPatches: - | kind: ClusterConfiguration etcd: external: endpoints: - http://192.168.32.1:2379 apiServer: extraArgs: tracing-config-file: "/apiserver-shared-conf/tracing.yaml" extraVolumes: - name: tracing hostPath: "/apiserver-shared-conf" mountPath: "/apiserver-shared-conf" - role: worker - role: worker ================================================ FILE: tests/robustness/coverage/matchers_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package coverage_test import ( "strings" tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" ) // Matcher returns true if span passes the filter. type Matcher func(span *tracev1.Span) bool func serviceName(trace *tracev1.ResourceSpans) (string, bool) { for _, attr := range trace.GetResource().GetAttributes() { if attr.GetKey() == "service.name" { return attr.GetValue().GetStringValue(), true } } return "", false } func isEtcdGRPC(span *tracev1.Span) bool { return strings.HasPrefix(span.GetName(), "etcdserverpb.") && span.GetKind() == tracev1.Span_SPAN_KIND_SERVER } //nolint:unparam func keyIsEqualInt(key string, want int) Matcher { return func(span *tracev1.Span) bool { got, found := intAttr(span, key) return found && got == want } } var ( isLimitSet = intAttrSet("limit") isRevisionSet = intAttrSet("rev") ) func intAttrSet(key string) Matcher { return func(span *tracev1.Span) bool { val, found := intAttr(span, key) return found && val > 0 } } func intAttr(span *tracev1.Span, key string) (int, bool) { for _, attr := range span.GetAttributes() { if attr.GetKey() == key { return int(attr.GetValue().GetIntValue()), true } } return 0, false } func keyIsEqualStr(key string, want string) Matcher { return func(span *tracev1.Span) bool { got, found := strAttr(span, key) return found && got == want } } func isRangeEndSet(span *tracev1.Span) bool { rangeEnd, found := strAttr(span, "range_end") return found && len(rangeEnd) > 0 } func strAttr(span *tracev1.Span, key string) (string, bool) { for _, attr := range span.GetAttributes() { if attr.GetKey() == key { return attr.Value.GetStringValue(), true } } return "", false } var ( isReadOnly = boolAttrSet("read_only") isKeysOnly = boolAttrSet("keys_only") isCountOnly = boolAttrSet("count_only") ) func boolAttrSet(key string) Matcher { return func(span *tracev1.Span) bool { val, found := boolAttr(span, key) return found && val } } func boolAttr(span *tracev1.Span, key string) (bool, bool) { for _, attr := range span.GetAttributes() { if attr.GetKey() == key { return attr.Value.GetBoolValue(), true } } return false, false } func orMatcher(l ...Matcher) Matcher { return func(span *tracev1.Span) bool { for _, m := range l { if m(span) { return true } } return false } } func andMatcher(l ...Matcher) Matcher { return func(span *tracev1.Span) bool { for _, m := range l { if !m(span) { return false } } return true } } func notMatcher(m Matcher) Matcher { return func(span *tracev1.Span) bool { return !m(span) } } func all(_ *tracev1.Span) bool { return true } ================================================ FILE: tests/robustness/coverage/patches/kubernetes/0001-Add-Open-Telemetry-trace-event-when-passing-through-.patch ================================================ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Aleksander Mistewicz Date: Wed, 13 Aug 2025 13:45:20 +0200 Subject: [PATCH 1/4] Add Open Telemetry trace event when passing through contract interface Signed-off-by: Aleksander Mistewicz diff --git a/vendor/go.etcd.io/etcd/client/v3/kubernetes/client.go b/vendor/go.etcd.io/etcd/client/v3/kubernetes/client.go index 11f2a456447..0efab3711d7 100644 --- a/vendor/go.etcd.io/etcd/client/v3/kubernetes/client.go +++ b/vendor/go.etcd.io/etcd/client/v3/kubernetes/client.go @@ -21,6 +21,8 @@ import ( pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // New creates Client from config. @@ -45,6 +47,10 @@ type Client struct { var _ Interface = (*Client)(nil) func (k Client) Get(ctx context.Context, key string, opts GetOptions) (resp GetResponse, err error) { + trace.SpanFromContext(ctx).AddEvent("contract.Get", trace.WithAttributes( + attribute.String("key", key), + attribute.Int64("rev", opts.Revision), + )) rangeResp, err := k.KV.Get(ctx, key, clientv3.WithRev(opts.Revision), clientv3.WithLimit(1)) if err != nil { return resp, err @@ -57,6 +63,10 @@ func (k Client) Get(ctx context.Context, key string, opts GetOptions) (resp GetR } func (k Client) List(ctx context.Context, prefix string, opts ListOptions) (resp ListResponse, err error) { + trace.SpanFromContext(ctx).AddEvent("contract.List", trace.WithAttributes( + attribute.String("prefix", prefix), + attribute.Int64("rev", opts.Revision), + )) rangeStart := prefix if opts.Continue != "" { rangeStart = opts.Continue @@ -73,6 +83,9 @@ func (k Client) List(ctx context.Context, prefix string, opts ListOptions) (resp } func (k Client) Count(ctx context.Context, prefix string, _ CountOptions) (int64, error) { + trace.SpanFromContext(ctx).AddEvent("contract.Count", trace.WithAttributes( + attribute.String("prefix", prefix), + )) resp, err := k.KV.Get(ctx, prefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) if err != nil { return 0, err @@ -81,6 +94,10 @@ func (k Client) Count(ctx context.Context, prefix string, _ CountOptions) (int64 } func (k Client) OptimisticPut(ctx context.Context, key string, value []byte, expectedRevision int64, opts PutOptions) (resp PutResponse, err error) { + trace.SpanFromContext(ctx).AddEvent("contract.OptimisticPut", trace.WithAttributes( + attribute.String("key", key), + attribute.Int64("expected_rev", expectedRevision), + )) txn := k.KV.Txn(ctx).If( clientv3.Compare(clientv3.ModRevision(key), "=", expectedRevision), ).Then( @@ -107,6 +124,10 @@ func (k Client) OptimisticPut(ctx context.Context, key string, value []byte, exp } func (k Client) OptimisticDelete(ctx context.Context, key string, expectedRevision int64, opts DeleteOptions) (resp DeleteResponse, err error) { + trace.SpanFromContext(ctx).AddEvent("contract.OptimisticDelete", trace.WithAttributes( + attribute.String("key", key), + attribute.Int64("expected_rev", expectedRevision), + )) txn := k.KV.Txn(ctx).If( clientv3.Compare(clientv3.ModRevision(key), "=", expectedRevision), ).Then( ================================================ FILE: tests/robustness/coverage/patches/kubernetes/0002-Add-kubernetesEtcdContractTracker.patch ================================================ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Aleksander Mistewicz Date: Mon, 18 Aug 2025 12:32:16 +0200 Subject: [PATCH 2/4] Add kubernetesEtcdContractTracker This decorator will make it easier to ensure that all calls to the underlying storage that go through the contract interface produce spans. TracerProvider needs to be passed to the contract tracker to ensure that all spans will be correctly exported. Signed-off-by: Aleksander Mistewicz diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/kubernetes_contract_tracker.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/kubernetes_contract_tracker.go new file mode 100644 index 00000000000..c16eecb20fa --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/kubernetes_contract_tracker.go @@ -0,0 +1,73 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd3 + +import ( + "context" + + "go.etcd.io/etcd/client/v3/kubernetes" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +const instrumentationScope = "k8s.io/apiserver/pkg/storage/etcd3" + +func NewKubernetesEtcdContractTracker(delegate kubernetes.Interface, tp trace.TracerProvider) kubernetes.Interface { + return &kubernetesEtcdContractTracker{Interface: delegate, tracer: tp.Tracer(instrumentationScope)} +} + +type kubernetesEtcdContractTracker struct { + kubernetes.Interface + + tracer trace.Tracer +} + +func (k *kubernetesEtcdContractTracker) Get(ctx context.Context, key string, opts kubernetes.GetOptions) (kubernetes.GetResponse, error) { + ctx, span := k.tracer.Start(ctx, "Get kubernetesEtcdContract", + trace.WithAttributes(attribute.String("key", key), attribute.Int("rev", int(opts.Revision)))) + defer span.End() + return k.Interface.Get(ctx, key, opts) +} + +func (k *kubernetesEtcdContractTracker) List(ctx context.Context, prefix string, opts kubernetes.ListOptions) (kubernetes.ListResponse, error) { + ctx, span := k.tracer.Start(ctx, "List kubernetesEtcdContract", + trace.WithAttributes(attribute.String("key", prefix), attribute.Int("rev", int(opts.Revision)), attribute.Int("limit", int(opts.Limit)))) + defer span.End() + return k.Interface.List(ctx, prefix, opts) +} + +func (k *kubernetesEtcdContractTracker) Count(ctx context.Context, prefix string, opts kubernetes.CountOptions) (int64, error) { + ctx, span := k.tracer.Start(ctx, "Count kubernetesEtcdContract", + trace.WithNewRoot(), // Count is called periodically from the same context, so it would show up as a single trace otherwise + trace.WithAttributes(attribute.String("key", prefix))) + defer span.End() + return k.Interface.Count(ctx, prefix, opts) +} + +func (k *kubernetesEtcdContractTracker) OptimisticPut(ctx context.Context, key string, value []byte, expectedRevision int64, opts kubernetes.PutOptions) (kubernetes.PutResponse, error) { + ctx, span := k.tracer.Start(ctx, "OptimisticPut kubernetesEtcdContract", + trace.WithAttributes(attribute.String("key", key), attribute.Int("rev", int(expectedRevision)), attribute.Int("lease", int(opts.LeaseID)), attribute.Bool("get_on_failure", opts.GetOnFailure))) + defer span.End() + return k.Interface.OptimisticPut(ctx, key, value, expectedRevision, opts) +} + +func (k *kubernetesEtcdContractTracker) OptimisticDelete(ctx context.Context, key string, expectedRevision int64, opts kubernetes.DeleteOptions) (kubernetes.DeleteResponse, error) { + ctx, span := k.tracer.Start(ctx, "OptimisticDelete kubernetesEtcdContract", + trace.WithAttributes(attribute.String("key", key), attribute.Int("rev", int(expectedRevision)), attribute.Bool("get_on_failure", opts.GetOnFailure))) + defer span.End() + return k.Interface.OptimisticDelete(ctx, key, expectedRevision, opts) +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go index 6b60601a784..235850a44f2 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go @@ -354,8 +354,16 @@ var newETCD3Client = func(c storagebackend.TransportConfig) (*kubernetes.Client, TLS: tlsConfig, Logger: etcd3ClientLogger, } + client, err := kubernetes.New(cfg) + if err != nil { + return nil, err + } + if c.TracerProvider != nil { + // Decorate the Kubernetes instance so all events added later will belong to short-lived Spans. + client.Kubernetes = etcd3.NewKubernetesEtcdContractTracker(client, c.TracerProvider) + } - return kubernetes.New(cfg) + return client, nil } type runningCompactor struct { ================================================ FILE: tests/robustness/coverage/patches/kubernetes/0003-Add-1m-timeout-to-Watch-to-ensure-Spans-are-exported.patch ================================================ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Aleksander Mistewicz Date: Fri, 22 Aug 2025 14:18:07 +0200 Subject: [PATCH 3/4] Add 1m timeout to Watch to ensure Spans are exported Signed-off-by: Aleksander Mistewicz diff --git a/vendor/go.etcd.io/etcd/client/v3/watch.go b/vendor/go.etcd.io/etcd/client/v3/watch.go index a46f98b8e28..ac820cabbb1 100644 --- a/vendor/go.etcd.io/etcd/client/v3/watch.go +++ b/vendor/go.etcd.io/etcd/client/v3/watch.go @@ -273,7 +273,7 @@ func (vc *valCtx) Done() <-chan struct{} { return valCtxCh } func (vc *valCtx) Err() error { return nil } func (w *watcher) newWatcherGRPCStream(inctx context.Context) *watchGRPCStream { - ctx, cancel := context.WithCancel(&valCtx{inctx}) + ctx, cancel := context.WithTimeout(&valCtx{inctx}, time.Minute) wgs := &watchGRPCStream{ owner: w, remote: w.remote, ================================================ FILE: tests/robustness/coverage/patches/kubernetes/0004-Configure-cmd-tests.patch ================================================ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Aleksander Mistewicz Date: Mon, 4 Aug 2025 15:56:48 +0200 Subject: [PATCH 4/4] Configure cmd tests diff --git a/hack/lib/etcd.sh b/hack/lib/etcd.sh index 15c4e59bba9..a22e8d04775 100755 --- a/hack/lib/etcd.sh +++ b/hack/lib/etcd.sh @@ -25,6 +25,8 @@ ETCD_PORT=${ETCD_PORT:-2379} ETCD_LOGLEVEL=${ETCD_LOGLEVEL:-warn} export KUBE_INTEGRATION_ETCD_URL="http://${ETCD_HOST}:${ETCD_PORT}" +export PATH="${PATH}:/go/src/k8s.io/kubernetes/third_party/etcd" + kube::etcd::validate() { # validate if in path command -v etcd >/dev/null || { @@ -84,8 +86,8 @@ kube::etcd::start() { else ETCD_LOGFILE=${ETCD_LOGFILE:-"/dev/null"} fi - kube::log::info "etcd --advertise-client-urls ${KUBE_INTEGRATION_ETCD_URL} --data-dir ${ETCD_DIR} --listen-client-urls http://${ETCD_HOST}:${ETCD_PORT} --listen-peer-urls http://localhost:0 --log-level=${ETCD_LOGLEVEL} 2> \"${ETCD_LOGFILE}\" >/dev/null" - etcd --advertise-client-urls "${KUBE_INTEGRATION_ETCD_URL}" --data-dir "${ETCD_DIR}" --listen-client-urls "${KUBE_INTEGRATION_ETCD_URL}" --listen-peer-urls "http://localhost:0" --log-level="${ETCD_LOGLEVEL}" 2> "${ETCD_LOGFILE}" >/dev/null & + kube::log::info "etcd --advertise-client-urls ${KUBE_INTEGRATION_ETCD_URL} --data-dir ${ETCD_DIR} --listen-client-urls http://${ETCD_HOST}:${ETCD_PORT} --listen-peer-urls http://localhost:0 --log-level=${ETCD_LOGLEVEL} --enable-distributed-tracing --distributed-tracing-address=\"192.168.32.1:4317\" --distributed-tracing-service-name=\"etcd\" --distributed-tracing-sampling-rate=1000000 2> \"${ETCD_LOGFILE}\" >/dev/null" + etcd --advertise-client-urls "${KUBE_INTEGRATION_ETCD_URL}" --data-dir "${ETCD_DIR}" --listen-client-urls "${KUBE_INTEGRATION_ETCD_URL}" --listen-peer-urls "http://localhost:0" --log-level="${ETCD_LOGLEVEL}" --enable-distributed-tracing --distributed-tracing-address="192.168.32.1:4317" --distributed-tracing-service-name="etcd" --distributed-tracing-sampling-rate=1000000 2> "${ETCD_LOGFILE}" >/dev/null & ETCD_PID=$! echo "Waiting for etcd to come up." diff --git a/hack/make-rules/test-cmd.sh b/hack/make-rules/test-cmd.sh index 7d9fb1db65c..be3b95341bb 100755 --- a/hack/make-rules/test-cmd.sh +++ b/hack/make-rules/test-cmd.sh @@ -69,6 +69,13 @@ function run_kube_apiserver() { VERSION_OVERRIDE="--version=$("${THIS_PLATFORM_BIN}/kube-apiserver" --version | awk '{print $2}')${CUSTOM_VERSION_SUFFIX:-}" fi + cat < "/tmp/kube-tracing-file" +apiVersion: apiserver.config.k8s.io/v1beta1 +kind: TracingConfiguration +endpoint: 192.168.32.1:4317 +samplingRatePerMillion: 1000000 +EOF + "${THIS_PLATFORM_BIN}/kube-apiserver" \ ${VERSION_OVERRIDE:+"${VERSION_OVERRIDE}"} \ --bind-address="127.0.0.1" \ @@ -84,6 +91,7 @@ function run_kube_apiserver() { --service-account-issuer="https://kubernetes.default.svc" \ --service-account-signing-key-file="${SERVICE_ACCOUNT_KEY}" \ --storage-media-type="${KUBE_TEST_API_STORAGE_TYPE-}" \ + --tracing-config-file="/tmp/kube-tracing-file" \ --cert-dir="${TMPDIR:-/tmp/}" \ --service-cluster-ip-range="10.0.0.0/24" \ --client-ca-file=hack/testdata/ca/ca.crt \ ================================================ FILE: tests/robustness/coverage/sort_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package coverage_test import ( "maps" "slices" "strings" "testing" "github.com/google/go-cmp/cmp" ) func sortPatternTable(res map[Row]int) []Row { keys := slices.Collect(maps.Keys(res)) slices.SortFunc(keys, func(a, b Row) int { if a.Pattern != b.Pattern { return strings.Compare(a.Pattern, b.Pattern) } if a.Method != b.Method { return strings.Compare(a.Method, b.Method) } return strings.Compare(a.Args, b.Args) }) return keys } func TestSortPatternTable(t *testing.T) { want := []Row{ {Pattern: "/registry/events/", Method: "", Args: ""}, {Pattern: "/registry/events/{namespace}/", Method: "List", Args: ""}, {Pattern: "/registry/events/{namespace}/{name}", Method: "Get", Args: ""}, {Pattern: "/registry/health", Method: "Healthcheck", Args: ""}, {Pattern: "/registry/masterleases/", Method: "List", Args: ""}, {Pattern: "/registry/masterleases/{name}", Method: "Get", Args: ""}, {Pattern: "/registry/{api-group}/{resource}/", Method: "List", Args: "XX"}, {Pattern: "/registry/{api-group}/{resource}/", Method: "List", Args: "XXX"}, {Pattern: "/registry/{api-group}/{resource}/{name}", Method: "Get", Args: ""}, {Pattern: "/registry/{resource}", Method: "Get", Args: ""}, {Pattern: "/registry/{resource}/", Method: "List", Args: ""}, {Pattern: "/registry/{resource}/{namespace}/{name}", Method: "Get", Args: ""}, {Pattern: "/registry/{resource}/{name}", Method: "Get", Args: ""}, {Pattern: "compact_rev_key", Method: "Compaction", Args: ""}, } res := make(map[Row]int, len(want)) for _, r := range want { res[r] = 0 } got := sortPatternTable(res) if diff := cmp.Diff(want, got); diff != "" { t.Logf("\n+%#v", got) t.Errorf("Sort mismatch (-want +got):\n%s", diff) } } ================================================ FILE: tests/robustness/coverage/testdata/.gitignore ================================================ * !.gitignore ================================================ FILE: tests/robustness/failpoint/cluster.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package failpoint import ( "context" "fmt" "math/rand" "os" "strings" "testing" "time" "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/fileutil" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/report" "go.etcd.io/etcd/tests/v3/robustness/traffic" ) var ( MemberReplace Failpoint = memberReplace{} MemberDowngrade Failpoint = memberDowngrade{} MemberDowngradeUpgrade Failpoint = memberDowngradeUpgrade{} ) type memberReplace struct{} func (f memberReplace) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { memberID := uint64(rand.Int() % len(clus.Procs)) member := clus.Procs[memberID] endpoints := []string{clus.Procs[(int(memberID)+1)%len(clus.Procs)].EndpointsGRPC()[0]} cc, err := clientv3.New(clientv3.Config{ Endpoints: endpoints, Logger: zap.NewNop(), DialKeepAliveTime: 10 * time.Second, DialKeepAliveTimeout: 100 * time.Millisecond, }) if err != nil { return nil, err } defer cc.Close() memberID, found, err := getID(ctx, cc, member.Config().Name) if err != nil { return nil, err } require.Truef(t, found, "Member not found") // Need to wait health interval for cluster to accept member changes time.Sleep(etcdserver.HealthInterval) lg.Info("Removing member", zap.String("member", member.Config().Name)) _, err = cc.MemberRemove(ctx, memberID) if err != nil { return nil, err } _, found, err = getID(ctx, cc, member.Config().Name) if err != nil { return nil, err } require.Falsef(t, found, "Expected member to be removed") for member.IsRunning() { err = member.Kill() if err != nil { lg.Info("Sending kill signal failed", zap.Error(err)) } err = member.Wait(ctx) if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { lg.Info("Failed to kill the process", zap.Error(err)) return nil, fmt.Errorf("failed to kill the process within %s, err: %w", triggerTimeout, err) } } lg.Info("Removing member data", zap.String("member", member.Config().Name)) err = os.RemoveAll(member.Config().DataDirPath) if err != nil { return nil, err } lg.Info("Adding member back", zap.String("member", member.Config().Name)) removedMemberPeerURL := member.Config().PeerURL.String() for { select { case <-ctx.Done(): return nil, ctx.Err() default: } reqCtx, cancel := context.WithTimeout(ctx, time.Second) _, err = cc.MemberAdd(reqCtx, []string{removedMemberPeerURL}) cancel() if err == nil { break } } err = patchArgs(member.Config().Args, "initial-cluster-state", "existing") if err != nil { return nil, err } lg.Info("Starting member", zap.String("member", member.Config().Name)) err = member.Start(ctx) if err != nil { return nil, err } for { select { case <-ctx.Done(): return nil, ctx.Err() default: } _, found, err := getID(ctx, cc, member.Config().Name) if err != nil { continue } if found { break } } return nil, nil } func (f memberReplace) Name() string { return "MemberReplace" } func (f memberReplace) Available(config e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess, profile traffic.Profile) bool { // a lower etcd version may not be able to join a cluster with higher cluster version. return config.ClusterSize > 1 && (config.Version == e2e.QuorumLastVersion || member.Config().ExecPath == e2e.BinPath.Etcd) } type memberDowngrade struct{} func (f memberDowngrade) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { currentVersion, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd) if err != nil { return nil, err } lastVersion, err := e2e.GetVersionFromBinary(e2e.BinPath.EtcdLastRelease) if err != nil { return nil, err } numberOfMembersToDowngrade := rand.Int()%len(clus.Procs) + 1 member := clus.Procs[0] endpoints := []string{member.EndpointsGRPC()[0]} cc, err := clientv3.New(clientv3.Config{ Endpoints: endpoints, Logger: zap.NewNop(), DialKeepAliveTime: 10 * time.Second, DialKeepAliveTimeout: 100 * time.Millisecond, }) if err != nil { return nil, err } defer cc.Close() // Need to wait health interval for cluster to accept changes time.Sleep(etcdserver.HealthInterval) e2e.DowngradeEnable(t, clus, lastVersion) err = e2e.DowngradeUpgradeMembers(t, lg, clus, numberOfMembersToDowngrade, true, currentVersion, lastVersion) time.Sleep(etcdserver.HealthInterval) return nil, err } func (f memberDowngrade) Name() string { return "MemberDowngrade" } func (f memberDowngrade) Available(config e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess, profile traffic.Profile) bool { if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { return false } // only run memberDowngrade test if no snapshot would be sent between members. // see https://github.com/etcd-io/etcd/issues/19147 for context. if config.ServerConfig.SnapshotCatchUpEntries < etcdserver.DefaultSnapshotCatchUpEntries { return false } v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd) if err != nil { panic("Failed checking etcd version binary") } v3_6 := semver.Version{Major: 3, Minor: 6} // only current version cluster can be downgraded. return v.Compare(v3_6) >= 0 && (config.Version == e2e.CurrentVersion && member.Config().ExecPath == e2e.BinPath.Etcd) } type memberDowngradeUpgrade struct{} func (f memberDowngradeUpgrade) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { currentVersion, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd) if err != nil { return nil, err } lastVersion, err := e2e.GetVersionFromBinary(e2e.BinPath.EtcdLastRelease) if err != nil { return nil, err } member := clus.Procs[0] endpoints := []string{member.EndpointsGRPC()[0]} cc, err := clientv3.New(clientv3.Config{ Endpoints: endpoints, Logger: zap.NewNop(), DialKeepAliveTime: 10 * time.Second, DialKeepAliveTimeout: 100 * time.Millisecond, }) if err != nil { return nil, err } defer cc.Close() e2e.DowngradeEnable(t, clus, lastVersion) // downgrade all members first err = e2e.DowngradeUpgradeMembers(t, lg, clus, len(clus.Procs), true, currentVersion, lastVersion) if err != nil { return nil, err } // NOTE: By default, the leader can cancel the downgrade once all members // have reached the target version. However, determining the final stable // cluster version after an upgrade can be challenging. To ensure stability, // we should wait for the leader to cancel the downgrade process. e2e.AssertProcessLogs(t, clus.Procs[clus.WaitLeader(t)], "the cluster has been downgraded") // partial upgrade the cluster numberOfMembersToUpgrade := rand.Int()%len(clus.Procs) + 1 err = e2e.DowngradeUpgradeMembers(t, lg, clus, numberOfMembersToUpgrade, false, lastVersion, currentVersion) time.Sleep(etcdserver.HealthInterval) return nil, err } func (f memberDowngradeUpgrade) Name() string { return "MemberDowngradeUpgrade" } func (f memberDowngradeUpgrade) Available(config e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess, profile traffic.Profile) bool { if !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { return false } // only run memberDowngrade test if no snapshot would be sent between members. // see https://github.com/etcd-io/etcd/issues/19147 for context. if config.ServerConfig.SnapshotCatchUpEntries < etcdserver.DefaultSnapshotCatchUpEntries { return false } v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd) if err != nil { panic("Failed checking etcd version binary") } v3_6 := semver.Version{Major: 3, Minor: 6} // only current version cluster can be downgraded. return v.Compare(v3_6) >= 0 && (config.Version == e2e.CurrentVersion && member.Config().ExecPath == e2e.BinPath.Etcd) } func (f memberDowngradeUpgrade) Timeout() time.Duration { return 120 * time.Second } func getID(ctx context.Context, cc *clientv3.Client, name string) (id uint64, found bool, err error) { // Ensure linearized MemberList by first making a linearized Get request from the same member. // This is required for v3.4 support as it doesn't support linearized MemberList https://github.com/etcd-io/etcd/issues/18929 // TODO: Remove preceding Get when v3.4 is no longer supported. getResp, err := cc.Get(ctx, "linearized-list-before-member-list") if err != nil { return 0, false, err } resp, err := cc.MemberList(ctx) if err != nil { return 0, false, err } if getResp.Header.MemberId != resp.Header.MemberId { return 0, false, fmt.Errorf("expected Get and MemberList to be sent to the same member, got: %d and %d", getResp.Header.MemberId, resp.Header.MemberId) } for _, member := range resp.Members { if name == member.Name { return member.ID, true, nil } } return 0, false, nil } func patchArgs(args []string, flag, newValue string) error { for i, arg := range args { if strings.Contains(arg, flag) { args[i] = fmt.Sprintf("--%s=%s", flag, newValue) return nil } } return fmt.Errorf("--%s flag not found", flag) } ================================================ FILE: tests/robustness/failpoint/failpoint.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package failpoint import ( "context" "fmt" "math/rand" "testing" "time" "go.uber.org/zap" healthpb "google.golang.org/grpc/health/grpc_health_v1" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/report" "go.etcd.io/etcd/tests/v3/robustness/traffic" ) const ( triggerTimeout = time.Minute ) var allFailpoints = []Failpoint{ KillFailpoint, BeforeCommitPanic, AfterCommitPanic, RaftBeforeSavePanic, RaftAfterSavePanic, DefragBeforeCopyPanic, DefragBeforeRenamePanic, BackendBeforePreCommitHookPanic, BackendAfterPreCommitHookPanic, BackendBeforeStartDBTxnPanic, BackendAfterStartDBTxnPanic, BackendBeforeWritebackBufPanic, BackendAfterWritebackBufPanic, CompactBeforeCommitScheduledCompactPanic, CompactAfterCommitScheduledCompactPanic, CompactBeforeSetFinishedCompactPanic, CompactAfterSetFinishedCompactPanic, CompactBeforeCommitBatchPanic, CompactAfterCommitBatchPanic, RaftBeforeLeaderSendPanic, BlackholePeerNetwork, DelayPeerNetwork, RaftBeforeFollowerSendPanic, RaftBeforeApplySnapPanic, RaftAfterApplySnapPanic, RaftAfterWALReleasePanic, RaftBeforeSaveSnapPanic, RaftAfterSaveSnapPanic, BlackholeUntilSnapshot, BeforeApplyOneConfChangeSleep, MemberReplace, MemberDowngrade, MemberDowngradeUpgrade, DropPeerNetwork, RaftBeforeSaveSleep, RaftAfterSaveSleep, ApplyBeforeOpenSnapshot, SleepBeforeSendWatchResponse, } func PickRandom(clus *e2e.EtcdProcessCluster, profile traffic.Profile) (Failpoint, error) { availableFailpoints := make([]Failpoint, 0, len(allFailpoints)) for _, failpoint := range allFailpoints { err := Validate(clus, failpoint, profile) if err != nil { continue } availableFailpoints = append(availableFailpoints, failpoint) } if len(availableFailpoints) == 0 { return nil, fmt.Errorf("no available failpoints") } return availableFailpoints[rand.Int()%len(availableFailpoints)], nil } func Validate(clus *e2e.EtcdProcessCluster, failpoint Failpoint, profile traffic.Profile) error { for _, proc := range clus.Procs { if !failpoint.Available(*clus.Cfg, proc, profile) { return fmt.Errorf("failpoint %q not available on %s", failpoint.Name(), proc.Config().Name) } } return nil } func Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, failpoint Failpoint, baseTime time.Time, ids identity.Provider) (*report.FailpointReport, error) { timeout := triggerTimeout if timeoutObj, ok := failpoint.(TimeoutInterface); ok { timeout = timeoutObj.Timeout() } ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() var err error if err = verifyClusterHealth(ctx, t, clus); err != nil { return nil, fmt.Errorf("failed to verify cluster health before failpoint injection, err: %w", err) } lg.Info("Triggering failpoint", zap.String("failpoint", failpoint.Name())) start := time.Since(baseTime) clientReport, err := failpoint.Inject(ctx, t, lg, clus, baseTime, ids) if err != nil { lg.Error("Failed to trigger failpoint", zap.String("failpoint", failpoint.Name()), zap.Error(err)) return nil, fmt.Errorf("failed triggering failpoint, err: %w", err) } if err = verifyClusterHealth(ctx, t, clus); err != nil { return nil, fmt.Errorf("failed to verify cluster health after failpoint injection, err: %w", err) } lg.Info("Finished triggering failpoint", zap.String("failpoint", failpoint.Name())) end := time.Since(baseTime) return &report.FailpointReport{ FailpointInjection: report.FailpointInjection{ Start: start, End: end, Name: failpoint.Name(), }, Client: clientReport, }, nil } func verifyClusterHealth(ctx context.Context, _ *testing.T, clus *e2e.EtcdProcessCluster) error { for i := 0; i < len(clus.Procs); i++ { clusterClient, err := clientv3.New(clientv3.Config{ Endpoints: clus.Procs[i].EndpointsGRPC(), Logger: zap.NewNop(), DialKeepAliveTime: 10 * time.Second, DialKeepAliveTimeout: 100 * time.Millisecond, }) if err != nil { return fmt.Errorf("Error creating client for cluster %s: %w", clus.Procs[i].Config().Name, err) } defer clusterClient.Close() cli := healthpb.NewHealthClient(clusterClient.ActiveConnection()) resp, err := cli.Check(ctx, &healthpb.HealthCheckRequest{}) if err != nil { return fmt.Errorf("Error checking member %s health: %w", clus.Procs[i].Config().Name, err) } if resp.Status != healthpb.HealthCheckResponse_SERVING { return fmt.Errorf("Member %s health status expected %s, got %s", clus.Procs[i].Config().Name, healthpb.HealthCheckResponse_SERVING, resp.Status) } } return nil } type Failpoint interface { Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) Name() string AvailabilityChecker } type AvailabilityChecker interface { Available(e2e.EtcdProcessClusterConfig, e2e.EtcdProcess, traffic.Profile) bool } type TimeoutInterface interface { Timeout() time.Duration } ================================================ FILE: tests/robustness/failpoint/gofail.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package failpoint import ( "context" "fmt" "math/rand" "strings" "testing" "time" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/report" "go.etcd.io/etcd/tests/v3/robustness/traffic" ) var ( DefragBeforeCopyPanic Failpoint = goPanicFailpoint{"defragBeforeCopy", triggerDefrag{}, AnyMember} DefragBeforeRenamePanic Failpoint = goPanicFailpoint{"defragBeforeRename", triggerDefrag{}, AnyMember} BeforeCommitPanic Failpoint = goPanicFailpoint{"beforeCommit", nil, AnyMember} AfterCommitPanic Failpoint = goPanicFailpoint{"afterCommit", nil, AnyMember} RaftBeforeSavePanic Failpoint = goPanicFailpoint{"raftBeforeSave", nil, AnyMember} RaftAfterSavePanic Failpoint = goPanicFailpoint{"raftAfterSave", nil, AnyMember} BackendBeforePreCommitHookPanic Failpoint = goPanicFailpoint{"commitBeforePreCommitHook", nil, AnyMember} BackendAfterPreCommitHookPanic Failpoint = goPanicFailpoint{"commitAfterPreCommitHook", nil, AnyMember} BackendBeforeStartDBTxnPanic Failpoint = goPanicFailpoint{"beforeStartDBTxn", nil, AnyMember} BackendAfterStartDBTxnPanic Failpoint = goPanicFailpoint{"afterStartDBTxn", nil, AnyMember} BackendBeforeWritebackBufPanic Failpoint = goPanicFailpoint{"beforeWritebackBuf", nil, AnyMember} BackendAfterWritebackBufPanic Failpoint = goPanicFailpoint{"afterWritebackBuf", nil, AnyMember} CompactBeforeCommitScheduledCompactPanic Failpoint = goPanicFailpoint{"compactBeforeCommitScheduledCompact", triggerCompact{}, AnyMember} CompactAfterCommitScheduledCompactPanic Failpoint = goPanicFailpoint{"compactAfterCommitScheduledCompact", triggerCompact{}, AnyMember} CompactBeforeSetFinishedCompactPanic Failpoint = goPanicFailpoint{"compactBeforeSetFinishedCompact", triggerCompact{}, AnyMember} BatchCompactBeforeSetFinishedCompactPanic Failpoint = goPanicFailpoint{"compactBeforeSetFinishedCompact", triggerCompact{multiBatchCompaction: true}, AnyMember} CompactAfterSetFinishedCompactPanic Failpoint = goPanicFailpoint{"compactAfterSetFinishedCompact", triggerCompact{}, AnyMember} CompactBeforeCommitBatchPanic Failpoint = goPanicFailpoint{"compactBeforeCommitBatch", triggerCompact{multiBatchCompaction: true}, AnyMember} CompactAfterCommitBatchPanic Failpoint = goPanicFailpoint{"compactAfterCommitBatch", triggerCompact{multiBatchCompaction: true}, AnyMember} RaftBeforeLeaderSendPanic Failpoint = goPanicFailpoint{"raftBeforeLeaderSend", nil, Leader} RaftBeforeFollowerSendPanic Failpoint = goPanicFailpoint{"raftBeforeFollowerSend", nil, Follower} RaftBeforeApplySnapPanic Failpoint = goPanicFailpoint{"raftBeforeApplySnap", triggerBlackhole{waitTillSnapshot: true}, Follower} RaftAfterApplySnapPanic Failpoint = goPanicFailpoint{"raftAfterApplySnap", triggerBlackhole{waitTillSnapshot: true}, Follower} RaftAfterWALReleasePanic Failpoint = goPanicFailpoint{"raftAfterWALRelease", triggerBlackhole{waitTillSnapshot: true}, Follower} RaftBeforeSaveSnapPanic Failpoint = goPanicFailpoint{"raftBeforeSaveSnap", triggerBlackhole{waitTillSnapshot: true}, Follower} RaftAfterSaveSnapPanic Failpoint = goPanicFailpoint{"raftAfterSaveSnap", triggerBlackhole{waitTillSnapshot: true}, Follower} ApplyBeforeOpenSnapshot Failpoint = goPanicFailpoint{"applyBeforeOpenSnapshot", triggerBlackhole{waitTillSnapshot: true}, Follower} BeforeApplyOneConfChangeSleep Failpoint = killAndGofailSleep{"beforeApplyOneConfChange", time.Second} RaftBeforeSaveSleep Failpoint = gofailSleepAndDeactivate{"raftBeforeSave", time.Second} RaftAfterSaveSleep Failpoint = gofailSleepAndDeactivate{"raftAfterSave", time.Second} SleepBeforeSendWatchResponse Failpoint = gofailSleepAndDeactivate{"beforeSendWatchResponse", time.Second} ) type goPanicFailpoint struct { failpoint string trigger trigger target failpointTarget } type failpointTarget string const ( AnyMember failpointTarget = "AnyMember" Leader failpointTarget = "Leader" Follower failpointTarget = "Follower" ) func (f goPanicFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) (reports []report.ClientReport, err error) { member := f.pickMember(t, clus) for member.IsRunning() { select { case <-ctx.Done(): return reports, ctx.Err() default: } lg.Info("Setting up gofailpoint", zap.String("failpoint", f.Name())) err = member.Failpoints().SetupHTTP(ctx, f.failpoint, "panic") if err != nil { lg.Info("goFailpoint setup failed", zap.String("failpoint", f.Name()), zap.Error(err)) continue } break } if f.trigger != nil { for member.IsRunning() { select { case <-ctx.Done(): return reports, ctx.Err() default: } var r []report.ClientReport lg.Info("Triggering gofailpoint", zap.String("failpoint", f.Name())) r, err = f.trigger.Trigger(ctx, t, member, clus, baseTime, ids) if err != nil { lg.Info("gofailpoint trigger failed", zap.String("failpoint", f.Name()), zap.Error(err)) continue } if r != nil { reports = append(reports, r...) } break } } lg.Info("Waiting for member to exit", zap.String("member", member.Config().Name)) err = member.Wait(ctx) if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { lg.Info("Member didn't exit as expected", zap.String("member", member.Config().Name), zap.Error(err)) return reports, fmt.Errorf("member didn't exit as expected: %w", err) } lg.Info("Member exited as expected", zap.String("member", member.Config().Name)) if lazyfs := member.LazyFS(); lazyfs != nil { lg.Info("Removing data that was not fsynced") err := lazyfs.ClearCache(ctx) if err != nil { return reports, err } } return reports, member.Start(ctx) } func (f goPanicFailpoint) pickMember(t *testing.T, clus *e2e.EtcdProcessCluster) e2e.EtcdProcess { switch f.target { case AnyMember: return clus.Procs[rand.Int()%len(clus.Procs)] case Leader: return clus.Procs[clus.WaitLeader(t)] case Follower: return clus.Procs[(clus.WaitLeader(t)+1)%len(clus.Procs)] default: panic("unknown target") } } func (f goPanicFailpoint) Available(config e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess, profile traffic.Profile) bool { if f.target == Follower && config.ClusterSize == 1 { return false } if f.trigger != nil && !f.trigger.Available(config, member, profile) { return false } memberFailpoints := member.Failpoints() if memberFailpoints == nil { return false } return memberFailpoints.Available(f.failpoint) } func (f goPanicFailpoint) Name() string { return fmt.Sprintf("%s=panic", f.failpoint) } type killAndGofailSleep struct { failpoint string time time.Duration } func (f killAndGofailSleep) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { member := clus.Procs[rand.Int()%len(clus.Procs)] for member.IsRunning() { err := member.Kill() if err != nil { lg.Info("Sending kill signal failed", zap.Error(err)) } err = member.Wait(ctx) if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { lg.Info("Failed to kill the process", zap.Error(err)) return nil, fmt.Errorf("failed to kill the process within %s, err: %w", triggerTimeout, err) } } lg.Info("Setting up goFailpoint", zap.String("failpoint", f.Name())) err := member.Failpoints().SetupEnv(f.failpoint, fmt.Sprintf(`sleep(%q)`, f.time)) if err != nil { return nil, err } err = member.Start(ctx) if err != nil { return nil, err } // TODO: Check gofail status (https://github.com/etcd-io/gofail/pull/47) and wait for sleep to be executed at least once. return nil, nil } func (f killAndGofailSleep) Name() string { return fmt.Sprintf("%s=sleep", f.failpoint) } func (f killAndGofailSleep) Available(config e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess, profile traffic.Profile) bool { if config.ClusterSize == 1 { return false } memberFailpoints := member.Failpoints() if memberFailpoints == nil { return false } return memberFailpoints.Available(f.failpoint) } type gofailSleepAndDeactivate struct { failpoint string time time.Duration } func (f gofailSleepAndDeactivate) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { member := clus.Procs[rand.Int()%len(clus.Procs)] lg.Info("Setting up gofailpoint", zap.String("failpoint", f.Name())) err := member.Failpoints().SetupHTTP(ctx, f.failpoint, fmt.Sprintf(`sleep(%q)`, f.time)) if err != nil { lg.Info("goFailpoint setup failed", zap.String("failpoint", f.Name()), zap.Error(err)) return nil, fmt.Errorf("goFailpoint %s setup failed, err:%w", f.Name(), err) } time.Sleep(f.time) lg.Info("Deactivating gofailpoint", zap.String("failpoint", f.Name())) err = member.Failpoints().DeactivateHTTP(ctx, f.failpoint) if err != nil { lg.Info("goFailpoint deactivate failed", zap.String("failpoint", f.Name()), zap.Error(err)) return nil, fmt.Errorf("goFailpoint %s deactivate failed, err: %w", f.Name(), err) } return nil, nil } func (f gofailSleepAndDeactivate) Name() string { return fmt.Sprintf("%s=sleep", f.failpoint) } func (f gofailSleepAndDeactivate) Available(config e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess, profile traffic.Profile) bool { memberFailpoints := member.Failpoints() if memberFailpoints == nil { return false } return memberFailpoints.Available(f.failpoint) } ================================================ FILE: tests/robustness/failpoint/kill.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package failpoint import ( "context" "fmt" "math/rand" "strings" "testing" "time" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/report" "go.etcd.io/etcd/tests/v3/robustness/traffic" ) var KillFailpoint Failpoint = killFailpoint{} type killFailpoint struct{} func (f killFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { member := clus.Procs[rand.Int()%len(clus.Procs)] for member.IsRunning() { err := member.Kill() if err != nil { lg.Info("Sending kill signal failed", zap.Error(err)) } err = member.Wait(ctx) if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { lg.Info("Failed to kill the process", zap.Error(err)) return nil, fmt.Errorf("failed to kill the process within %s, err: %w", triggerTimeout, err) } } if lazyfs := member.LazyFS(); lazyfs != nil { lg.Info("Removing data that was not fsynced") err := lazyfs.ClearCache(ctx) if err != nil { return nil, err } } err := member.Start(ctx) if err != nil { return nil, err } return nil, nil } func (f killFailpoint) Name() string { return "Kill" } func (f killFailpoint) Available(e2e.EtcdProcessClusterConfig, e2e.EtcdProcess, traffic.Profile) bool { return true } ================================================ FILE: tests/robustness/failpoint/network.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package failpoint import ( "context" "math/rand" "testing" "time" "go.uber.org/zap" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/report" "go.etcd.io/etcd/tests/v3/robustness/traffic" ) var ( BlackholePeerNetwork Failpoint = blackholePeerNetworkFailpoint{triggerBlackhole{waitTillSnapshot: false}} BlackholeUntilSnapshot Failpoint = blackholePeerNetworkFailpoint{triggerBlackhole{waitTillSnapshot: true}} DelayPeerNetwork Failpoint = delayPeerNetworkFailpoint{duration: time.Second, baseLatency: 75 * time.Millisecond, randomizedLatency: 50 * time.Millisecond} DropPeerNetwork Failpoint = dropPeerNetworkFailpoint{duration: time.Second, dropProbabilityPercent: 50} ) type blackholePeerNetworkFailpoint struct { triggerBlackhole } func (f blackholePeerNetworkFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { member := clus.Procs[rand.Int()%len(clus.Procs)] return f.Trigger(ctx, t, member, clus, baseTime, ids) } func (f blackholePeerNetworkFailpoint) Name() string { return "blackholePeerNetwork" } type triggerBlackhole struct { waitTillSnapshot bool } func (tb triggerBlackhole) Trigger(ctx context.Context, t *testing.T, member e2e.EtcdProcess, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { return nil, Blackhole(ctx, t, member, clus, tb.waitTillSnapshot) } func (tb triggerBlackhole) Available(config e2e.EtcdProcessClusterConfig, process e2e.EtcdProcess, profile traffic.Profile) bool { // Avoid triggering failpoint if waiting for failpoint would take too long to fit into timeout. // Number of required entries for snapshot depends on etcd configuration. if tb.waitTillSnapshot && (entriesToGuaranteeSnapshot(config) > 200 || !e2e.CouldSetSnapshotCatchupEntries(process.Config().ExecPath)) { return false } return config.ClusterSize > 1 && process.PeerProxy() != nil } func Blackhole(ctx context.Context, t *testing.T, member e2e.EtcdProcess, clus *e2e.EtcdProcessCluster, shouldWaitTillSnapshot bool) error { proxy := member.PeerProxy() // Blackholing will cause peers to not be able to use streamWriters registered with member // but peer traffic is still possible because member has 'pipeline' with peers // TODO: find a way to stop all traffic t.Logf("Blackholing traffic from and to member %q", member.Config().Name) proxy.BlackholeTx() proxy.BlackholeRx() defer func() { t.Logf("Traffic restored from and to member %q", member.Config().Name) proxy.UnblackholeTx() proxy.UnblackholeRx() }() if shouldWaitTillSnapshot { return waitTillSnapshot(ctx, t, clus, member) } time.Sleep(time.Second) return nil } func waitTillSnapshot(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessCluster, blackholedMember e2e.EtcdProcess) error { var endpoints []string for _, ep := range clus.EndpointsGRPC() { if ep == blackholedMember.Config().ClientURL { continue } endpoints = append(endpoints, ep) } clusterClient, err := clientv3.New(clientv3.Config{ Endpoints: endpoints, Logger: zap.NewNop(), DialKeepAliveTime: 10 * time.Second, DialKeepAliveTimeout: 100 * time.Millisecond, }) if err != nil { return err } defer clusterClient.Close() blackholedMemberClient, err := clientv3.New(clientv3.Config{ Endpoints: []string{blackholedMember.Config().ClientURL}, Logger: zap.NewNop(), DialKeepAliveTime: 10 * time.Second, DialKeepAliveTimeout: 100 * time.Millisecond, }) if err != nil { return err } defer blackholedMemberClient.Close() for { select { case <-ctx.Done(): return ctx.Err() default: } // Have to refresh blackholedMemberRevision. It can still increase as blackholedMember processes changes that are received but not yet applied. blackholedMemberRevision, err := latestRevisionForEndpoint(ctx, blackholedMemberClient) if err != nil { return err } clusterRevision, err := latestRevisionForEndpoint(ctx, clusterClient) if err != nil { return err } minEntriesToGuarantee := entriesToGuaranteeSnapshot(*clus.Cfg) t.Logf("clusterRevision: %d, blackholedMemberRevision: %d, minEntriesToGuarantee: %d", clusterRevision, blackholedMemberRevision, minEntriesToGuarantee) // Blackholed member has to be sufficiently behind to trigger snapshot transfer. if clusterRevision-blackholedMemberRevision > int64(minEntriesToGuarantee) { break } time.Sleep(100 * time.Millisecond) } return nil } func entriesToGuaranteeSnapshot(config e2e.EtcdProcessClusterConfig) uint64 { // Need to make sure leader compacted latest revBlackholedMem inside EtcdServer.snapshot. // That's why we wait for clus.Cfg.SnapshotCount (to trigger snapshot) + clus.Cfg.SnapshotCatchUpEntries (EtcdServer.snapshot compaction offset) return config.ServerConfig.SnapshotCount + config.ServerConfig.SnapshotCatchUpEntries } // latestRevisionForEndpoint gets latest revision of the first endpoint in Client.Endpoints list func latestRevisionForEndpoint(ctx context.Context, c *clientv3.Client) (int64, error) { resp, err := c.Status(ctx, c.Endpoints()[0]) if err != nil { return 0, err } return resp.Header.Revision, err } type delayPeerNetworkFailpoint struct { duration time.Duration baseLatency time.Duration randomizedLatency time.Duration } func (f delayPeerNetworkFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { member := clus.Procs[rand.Int()%len(clus.Procs)] proxy := member.PeerProxy() proxy.DelayRx(f.baseLatency, f.randomizedLatency) proxy.DelayTx(f.baseLatency, f.randomizedLatency) lg.Info("Delaying traffic from and to member", zap.String("member", member.Config().Name), zap.Duration("baseLatency", f.baseLatency), zap.Duration("randomizedLatency", f.randomizedLatency)) time.Sleep(f.duration) lg.Info("Traffic delay removed", zap.String("member", member.Config().Name)) proxy.UndelayRx() proxy.UndelayTx() return nil, nil } func (f delayPeerNetworkFailpoint) Name() string { return "delayPeerNetwork" } func (f delayPeerNetworkFailpoint) Available(config e2e.EtcdProcessClusterConfig, clus e2e.EtcdProcess, profile traffic.Profile) bool { return config.ClusterSize > 1 && clus.PeerProxy() != nil } type dropPeerNetworkFailpoint struct { duration time.Duration dropProbabilityPercent int } func (f dropPeerNetworkFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { member := clus.Procs[rand.Int()%len(clus.Procs)] proxy := member.PeerProxy() proxy.ModifyRx(f.modifyPacket) proxy.ModifyTx(f.modifyPacket) lg.Info("Dropping traffic from and to member", zap.String("member", member.Config().Name), zap.Int("probability", f.dropProbabilityPercent)) time.Sleep(f.duration) lg.Info("Traffic drop removed", zap.String("member", member.Config().Name)) proxy.UnmodifyRx() proxy.UnmodifyTx() return nil, nil } func (f dropPeerNetworkFailpoint) modifyPacket(data []byte) []byte { if rand.Intn(100) < f.dropProbabilityPercent { return nil } return data } func (f dropPeerNetworkFailpoint) Name() string { return "dropPeerNetwork" } func (f dropPeerNetworkFailpoint) Available(config e2e.EtcdProcessClusterConfig, clus e2e.EtcdProcess, profile traffic.Profile) bool { return config.ClusterSize > 1 && clus.PeerProxy() != nil } ================================================ FILE: tests/robustness/failpoint/trigger.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package failpoint import ( "context" "fmt" "strings" "testing" "time" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/client" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/report" "go.etcd.io/etcd/tests/v3/robustness/traffic" ) type trigger interface { Trigger(ctx context.Context, t *testing.T, member e2e.EtcdProcess, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) AvailabilityChecker } type triggerDefrag struct{} func (t triggerDefrag) Trigger(ctx context.Context, _ *testing.T, member e2e.EtcdProcess, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { cc, err := client.NewRecordingClient(member.EndpointsGRPC(), ids, baseTime) if err != nil { return nil, fmt.Errorf("failed creating client: %w", err) } defer cc.Close() _, err = cc.Defragment(ctx) if err != nil && !connectionError(err) { return nil, err } return nil, nil } func (t triggerDefrag) Available(e2e.EtcdProcessClusterConfig, e2e.EtcdProcess, traffic.Profile) bool { return true } type triggerCompact struct { multiBatchCompaction bool } func (t triggerCompact) Trigger(ctx context.Context, _ *testing.T, member e2e.EtcdProcess, clus *e2e.EtcdProcessCluster, baseTime time.Time, ids identity.Provider) ([]report.ClientReport, error) { ctx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() cc, err := client.NewRecordingClient(member.EndpointsGRPC(), ids, baseTime) if err != nil { return nil, fmt.Errorf("failed creating client: %w", err) } defer cc.Close() var rev int64 for { var resp *clientv3.GetResponse resp, err = cc.Get(ctx, "/", clientv3.WithRev(0)) if err != nil { return nil, fmt.Errorf("failed to get revision: %w", err) } rev = resp.Header.Revision if !t.multiBatchCompaction || rev > int64(clus.Cfg.ServerConfig.CompactionBatchLimit) { break } time.Sleep(50 * time.Millisecond) } _, err = cc.Compact(ctx, rev) reports := []report.ClientReport{cc.Report()} if err != nil && !connectionError(err) { return reports, fmt.Errorf("failed to compact: %w", err) } return reports, nil } func (t triggerCompact) Available(config e2e.EtcdProcessClusterConfig, _ e2e.EtcdProcess, profile traffic.Profile) bool { if profile.Compaction != nil { return false } // Since the introduction of compaction into traffic, injecting compaction failpoints started interfering with the peer proxy. // TODO: Re-enable the peer proxy for compact failpoints when we confirm the root cause. if config.PeerProxy { return false } // For multiBatchCompaction we need to guarantee that there are enough revisions between two compaction requests. // With addition of compaction requests to traffic this might be hard if -compaction-batch-limit is too high. if t.multiBatchCompaction { return config.ServerConfig.CompactionBatchLimit <= 10 } return true } func connectionError(err error) bool { return strings.Contains(err.Error(), "error reading from server: EOF") || strings.HasSuffix(err.Error(), "read: connection reset by peer") } ================================================ FILE: tests/robustness/identity/id.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package identity import "sync/atomic" type Provider interface { // NewStreamID returns an integer starting from zero to make it render nicely by the porcupine visualization. NewStreamID() int // NewRequestID returns a unique identification used to make write requests unique. NewRequestID() int // NewClientID returns a unique identification for client and their reports. NewClientID() int } func NewIDProvider() Provider { return &atomicProvider{} } type atomicProvider struct { streamID atomic.Int64 requestID atomic.Int64 clientID atomic.Int64 } func (id *atomicProvider) NewStreamID() int { return int(id.streamID.Add(1) - 1) } func (id *atomicProvider) NewRequestID() int { return int(id.requestID.Add(1)) } func (id *atomicProvider) NewClientID() int { return int(id.clientID.Add(1)) } ================================================ FILE: tests/robustness/identity/lease_ids.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package identity import ( "sync" ) type LeaseIDStorage interface { LeaseID(int) int64 AddLeaseID(int, int64) RemoveLeaseID(int) } func NewLeaseIDStorage() LeaseIDStorage { return &atomicClientID2LeaseIDMapper{m: map[int]int64{}} } type atomicClientID2LeaseIDMapper struct { sync.RWMutex // m is used to store clientId to leaseId mapping. m map[int]int64 } func (lm *atomicClientID2LeaseIDMapper) LeaseID(clientID int) int64 { lm.RLock() defer lm.RUnlock() return lm.m[clientID] } func (lm *atomicClientID2LeaseIDMapper) AddLeaseID(clientID int, leaseID int64) { lm.Lock() defer lm.Unlock() lm.m[clientID] = leaseID } func (lm *atomicClientID2LeaseIDMapper) RemoveLeaseID(clientID int) { lm.Lock() defer lm.Unlock() delete(lm.m, clientID) } ================================================ FILE: tests/robustness/main_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package robustness import ( "context" "fmt" "math/rand" "os" "path/filepath" "slices" "strings" "testing" "time" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" "golang.org/x/sync/errgroup" "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/client" "go.etcd.io/etcd/tests/v3/robustness/failpoint" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/report" "go.etcd.io/etcd/tests/v3/robustness/scenarios" "go.etcd.io/etcd/tests/v3/robustness/traffic" "go.etcd.io/etcd/tests/v3/robustness/validate" ) var testRunner = framework.E2eTestRunner var ( WaitBeforeFailpoint = time.Second WaitJitter = time.Millisecond * 200 WaitAfterFailpoint = time.Second ) func TestMain(m *testing.M) { testRunner.TestMain(m) } func TestRobustnessExploratory(t *testing.T) { testRunner.BeforeTest(t) for _, s := range scenarios.Exploratory(t) { t.Run(s.Name, func(t *testing.T) { lg := zaptest.NewLogger(t) s.Cluster.Logger = lg ctx := t.Context() c, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithConfig(&s.Cluster)) require.NoError(t, err) defer forcestopCluster(c) s.Failpoint, err = failpoint.PickRandom(c, s.Profile) require.NoError(t, err) t.Run(s.Failpoint.Name(), func(t *testing.T) { testRobustness(ctx, t, lg, s, c) }) }) } } func TestRobustnessRegression(t *testing.T) { testRunner.BeforeTest(t) for _, s := range scenarios.Regression(t) { t.Run(s.Name, func(t *testing.T) { lg := zaptest.NewLogger(t) s.Cluster.Logger = lg ctx := t.Context() c, err := e2e.NewEtcdProcessCluster(ctx, t, e2e.WithConfig(&s.Cluster)) require.NoError(t, err) defer forcestopCluster(c) testRobustness(ctx, t, lg, s, c) }) } } func testRobustness(ctx context.Context, t *testing.T, lg *zap.Logger, s scenarios.TestScenario, c *e2e.EtcdProcessCluster) { serverDataPaths := report.ServerDataPaths(c) r := report.TestReport{ Logger: lg, ServersDataPath: serverDataPaths, Traffic: &report.TrafficDetail{ExpectUniqueRevision: s.Traffic.ExpectUniqueRevision()}, } // t.Failed() returns false during panicking. We need to forcibly // save data on panicking. // Refer to: https://github.com/golang/go/issues/49929 panicked := true defer func() { _, persistResults := os.LookupEnv("PERSIST_RESULTS") shouldReport := t.Failed() || panicked || persistResults if shouldReport { path := testResultsDirectory(t) if err := r.Report(path); err != nil { t.Error(err) } } }() r.Client = runScenario(ctx, t, s, lg, c) persistedRequests, err := report.PersistedRequestsCluster(lg, c) if err != nil { t.Error(err) } validateConfig := validate.Config{ExpectRevisionUnique: s.Traffic.ExpectUniqueRevision()} result := validate.ValidateAndReturnVisualize(lg, validateConfig, r.Client, persistedRequests, 5*time.Minute) r.Visualize = result.Linearization.Visualize err = result.Error() if err != nil { t.Error(err) } panicked = false } func runScenario(ctx context.Context, t *testing.T, s scenarios.TestScenario, lg *zap.Logger, clus *e2e.EtcdProcessCluster) (reports []report.ClientReport) { ctx, cancel := context.WithCancel(ctx) defer cancel() g := errgroup.Group{} var failpointClientReport []report.ClientReport failpointInjected := make(chan report.FailpointInjection, 1) // using baseTime time-measuring operation to get monotonic clock reading // see https://github.com/golang/go/blob/master/src/time/time.go#L17 baseTime := time.Now() ids := identity.NewIDProvider() g.Go(func() error { defer close(failpointInjected) // Give some time for traffic to reach qps target before injecting failpoint. time.Sleep(randomizeTime(WaitBeforeFailpoint, WaitJitter)) fr, err := failpoint.Inject(ctx, t, lg, clus, s.Failpoint, baseTime, ids) if err != nil { t.Error(err) cancel() } // Give some time for traffic to reach qps target after injecting failpoint. time.Sleep(randomizeTime(WaitAfterFailpoint, WaitJitter)) if fr != nil { failpointInjected <- fr.FailpointInjection failpointClientReport = fr.Client } return nil }) trafficSet := client.NewSet(ids, baseTime) defer trafficSet.Close() maxRevisionChan := make(chan int64, 1) g.Go(func() error { defer close(maxRevisionChan) operationReport := traffic.SimulateTraffic(ctx, t, lg, clus, s.Profile, s.Traffic, failpointInjected, trafficSet) maxRevision := report.OperationsMaxRevision(operationReport) maxRevisionChan <- maxRevision lg.Info("Finished simulating Traffic", zap.Int64("max-revision", maxRevision)) return nil }) watchSet := client.NewSet(ids, baseTime) defer watchSet.Close() g.Go(func() error { endpoints := processEndpoints(clus) err := client.CollectClusterWatchEvents(ctx, client.CollectClusterWatchEventsParam{ Lg: lg, Endpoints: endpoints, MaxRevisionChan: maxRevisionChan, Cfg: s.Watch, ClientSet: watchSet, }) return err }) err := g.Wait() if err != nil { t.Error(err) } err = client.CheckEndOfTestHashKV(ctx, clus) if err != nil { t.Error(err) } return slices.Concat(trafficSet.Reports(), watchSet.Reports(), failpointClientReport) } func randomizeTime(base time.Duration, jitter time.Duration) time.Duration { return base - jitter + time.Duration(rand.Int63n(int64(jitter)*2)) } // forcestopCluster stops the etcd member with signal kill. func forcestopCluster(clus *e2e.EtcdProcessCluster) error { for _, member := range clus.Procs { member.Kill() } return clus.ConcurrentStop() } func testResultsDirectory(t *testing.T) string { resultsDirectory, ok := os.LookupEnv("RESULTS_DIR") if !ok { resultsDirectory = "/tmp/" } resultsDirectory, err := filepath.Abs(resultsDirectory) if err != nil { panic(err) } path, err := filepath.Abs(filepath.Join( resultsDirectory, strings.ReplaceAll(t.Name(), "/", "_"), fmt.Sprintf("%v", time.Now().UnixNano()))) require.NoError(t, err) err = os.RemoveAll(path) require.NoError(t, err) err = os.MkdirAll(path, 0o700) require.NoError(t, err) return path } func processEndpoints(clus *e2e.EtcdProcessCluster) []string { endpoints := make([]string, 0, len(clus.Procs)) for _, proc := range clus.Procs { endpoints = append(endpoints, proc.EndpointsGRPC()[0]) } return endpoints } ================================================ FILE: tests/robustness/model/describe.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "fmt" "maps" "slices" "sort" "strings" "go.etcd.io/etcd/client/pkg/v3/types" clientv3 "go.etcd.io/etcd/client/v3" ) func describeEtcdResponse(request EtcdRequest, response MaybeEtcdResponse) string { if response.Error != "" { return fmt.Sprintf("err: %q", response.Error) } if response.ClientError != "" { return fmt.Sprintf("err: %q", response.ClientError) } if response.Persisted { if response.PersistedRevision != 0 { return fmt.Sprintf("unknown, rev: %d", response.PersistedRevision) } return "unknown" } switch request.Type { case Range: return fmt.Sprintf("%s, rev: %d", describeRangeResponse(request.Range.RangeOptions, *response.Range), response.Revision) case Txn: return fmt.Sprintf("%s, rev: %d", describeTxnResponse(request.Txn, response.Txn), response.Revision) case LeaseGrant, LeaseRevoke: return fmt.Sprintf("ok, rev: %d", response.Revision) case Compact, Defragment: return "ok" default: return fmt.Sprintf("", request.Type) } } func describeEtcdRequest(request EtcdRequest) string { switch request.Type { case Range: return describeRangeRequest(request.Range.RangeOptions, request.Range.Revision) case Txn: guaranteedTxnDescription := describeGuaranteedTxn(request.Txn) if guaranteedTxnDescription != "" { return guaranteedTxnDescription } onSuccess := describeEtcdOperations(request.Txn.OperationsOnSuccess) if len(request.Txn.Conditions) != 0 { if len(request.Txn.OperationsOnFailure) == 0 { return fmt.Sprintf("if(%s).then(%s)", describeEtcdConditions(request.Txn.Conditions), onSuccess) } onFailure := describeEtcdOperations(request.Txn.OperationsOnFailure) return fmt.Sprintf("if(%s).then(%s).else(%s)", describeEtcdConditions(request.Txn.Conditions), onSuccess, onFailure) } return onSuccess case LeaseGrant: return fmt.Sprintf("leaseGrant(%d)", request.LeaseGrant.LeaseID) case LeaseRevoke: return fmt.Sprintf("leaseRevoke(%d)", request.LeaseRevoke.LeaseID) case Defragment: return "defragment()" case Compact: return fmt.Sprintf("compact(%d)", request.Compact.Revision) default: return fmt.Sprintf("", request.Type) } } func describeEtcdState(state EtcdState) string { descHTML := make([]string, 0) descHTML = append(descHTML, fmt.Sprintf("

state, rev: %d, compactRev: %d

", state.Revision, state.CompactRevision)) if len(state.KeyValues) > 0 { descHTML = append(descHTML, "keys:
    ") keys := slices.Collect(maps.Keys(state.KeyValues)) sort.Strings(keys) for _, key := range keys { descHTML = append(descHTML, fmt.Sprintf("
  • %s - ", key)) value := state.KeyValues[key] if value.Value.Value != "" { descHTML = append(descHTML, fmt.Sprintf("val: %q, ", value.Value.Value)) } if value.Value.Hash != 0 { descHTML = append(descHTML, fmt.Sprintf("hash: %d, ", value.Value.Hash)) } lease := state.KeyLeases[key] if lease != 0 { descHTML = append(descHTML, fmt.Sprintf("lease: %d, ", lease)) } descHTML = append(descHTML, fmt.Sprintf("mod: %d, ver: %d
  • ", value.ModRevision, value.Version)) } descHTML = append(descHTML, "
") } if len(state.Leases) > 0 { descHTML = append(descHTML, "leases:
    ") leases := slices.Collect(maps.Keys(state.Leases)) slices.Sort(leases) for _, lease := range leases { descHTML = append(descHTML, fmt.Sprintf("
  • %d
  • ", lease)) } descHTML = append(descHTML, "
") } return strings.Join(descHTML, "") } func describeGuaranteedTxn(txn *TxnRequest) string { if len(txn.Conditions) != 1 || len(txn.OperationsOnSuccess) != 1 || len(txn.OperationsOnFailure) > 1 { return "" } switch txn.OperationsOnSuccess[0].Type { case PutOperation: if txn.Conditions[0].Key != txn.OperationsOnSuccess[0].Put.Key || (len(txn.OperationsOnFailure) == 1 && txn.Conditions[0].Key != txn.OperationsOnFailure[0].Range.Start) { return "" } if txn.Conditions[0].ExpectedVersion > 0 { return "" } if txn.Conditions[0].ExpectedRevision == 0 { return fmt.Sprintf("guaranteedCreate(%q, %s)", txn.Conditions[0].Key, describeValueOrHash(txn.OperationsOnSuccess[0].Put.Value)) } return fmt.Sprintf("guaranteedUpdate(%q, %s, mod_rev=%d)", txn.Conditions[0].Key, describeValueOrHash(txn.OperationsOnSuccess[0].Put.Value), txn.Conditions[0].ExpectedRevision) case DeleteOperation: if txn.Conditions[0].Key != txn.OperationsOnSuccess[0].Delete.Key || (len(txn.OperationsOnFailure) == 1 && txn.Conditions[0].Key != txn.OperationsOnFailure[0].Range.Start) { return "" } return fmt.Sprintf("guaranteedDelete(%q, mod_rev=%d)", txn.Conditions[0].Key, txn.Conditions[0].ExpectedRevision) } return "" } func describeEtcdConditions(conds []EtcdCondition) string { opsDescription := make([]string, len(conds)) for i, cond := range conds { if cond.ExpectedVersion > 0 { opsDescription[i] = fmt.Sprintf("ver(%s)==%d", cond.Key, cond.ExpectedVersion) } else { opsDescription[i] = fmt.Sprintf("mod_rev(%s)==%d", cond.Key, cond.ExpectedRevision) } } return strings.Join(opsDescription, " && ") } func describeEtcdOperations(ops []EtcdOperation) string { opsDescription := make([]string, len(ops)) for i := range ops { opsDescription[i] = describeEtcdOperation(ops[i]) } return strings.Join(opsDescription, ", ") } func describeTxnResponse(request *TxnRequest, response *TxnResponse) string { respDescription := make([]string, len(response.Results)) for i, result := range response.Results { if response.Failure { respDescription[i] = describeEtcdOperationResponse(request.OperationsOnFailure[i], result) } else { respDescription[i] = describeEtcdOperationResponse(request.OperationsOnSuccess[i], result) } } description := strings.Join(respDescription, ", ") if len(request.Conditions) == 0 { return description } if response.Failure { return fmt.Sprintf("failure(%s)", description) } return fmt.Sprintf("success(%s)", description) } func describeEtcdOperation(op EtcdOperation) string { switch op.Type { case RangeOperation: return describeRangeRequest(op.Range, 0) case PutOperation: if op.Put.LeaseID != 0 { return fmt.Sprintf("put(%q, %s, %d)", op.Put.Key, describeValueOrHash(op.Put.Value), op.Put.LeaseID) } return fmt.Sprintf("put(%q, %s)", op.Put.Key, describeValueOrHash(op.Put.Value)) case DeleteOperation: return fmt.Sprintf("delete(%q)", op.Delete.Key) default: return fmt.Sprintf("", op.Type) } } func describeRangeRequest(opts RangeOptions, revision int64) string { kwargs := []string{} if revision != 0 { kwargs = append(kwargs, fmt.Sprintf("rev=%d", revision)) } if opts.Limit != 0 { kwargs = append(kwargs, fmt.Sprintf("limit=%d", opts.Limit)) } kwargsString := strings.Join(kwargs, ", ") if kwargsString != "" { kwargsString = ", " + kwargsString } switch { case opts.End == "": return fmt.Sprintf("get(%q%s)", opts.Start, kwargsString) case opts.End == clientv3.GetPrefixRangeEnd(opts.Start): return fmt.Sprintf("list(%q%s)", opts.Start, kwargsString) case strings.HasSuffix(opts.Start, "\x00") && strings.HasSuffix(opts.End, "0") && strings.HasPrefix(opts.Start, opts.End[:len(opts.End)-1]): return fmt.Sprintf("list[continued](%q%s)", strings.TrimRight(opts.Start, "\x00"), kwargsString) default: return fmt.Sprintf("range(%q..%q%s)", opts.Start, opts.End, kwargsString) } } func describeEtcdOperationResponse(op EtcdOperation, resp EtcdOperationResult) string { switch op.Type { case RangeOperation: return describeRangeResponse(op.Range, resp.RangeResponse) case PutOperation: return "ok" case DeleteOperation: return fmt.Sprintf("deleted: %d", resp.Deleted) default: return fmt.Sprintf("", op.Type) } } func describeRangeResponse(request RangeOptions, response RangeResponse) string { if request.End != "" { kvs := make([]string, len(response.KVs)) for i, kv := range response.KVs { kvs[i] = describeValueOrHash(kv.Value) } return fmt.Sprintf("[%s], count: %d", strings.Join(kvs, ","), response.Count) } if len(response.KVs) == 0 { return "nil" } return describeValueOrHash(response.KVs[0].Value) } func DescribeOperationMetadata(response MaybeEtcdResponse) string { if response.MemberID != 0 { return fmt.Sprintf("memberID: %s", types.ID(response.MemberID).String()) } return "" } func describeValueOrHash(value ValueOrHash) string { if value.Hash != 0 { return fmt.Sprintf("hash: %d", value.Hash) } if value.Value == "" { return "nil" } return fmt.Sprintf("%q", value.Value) } ================================================ FILE: tests/robustness/model/describe_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "errors" "testing" "github.com/stretchr/testify/assert" "go.etcd.io/etcd/api/v3/mvccpb" ) func TestModelDescribe(t *testing.T) { tcs := []struct { req EtcdRequest resp MaybeEtcdResponse expectDescribe string }{ { req: getRequest("key1"), resp: emptyGetResponse(1), expectDescribe: `get("key1") -> nil, rev: 1`, }, { req: getRequest("key2"), resp: getResponse("key", "2", 2, 2), expectDescribe: `get("key2") -> "2", rev: 2`, }, { req: getRequest("key2b"), resp: getResponse("key2b", "01234567890123456789", 2, 2), expectDescribe: `get("key2b") -> hash: 2945867837, rev: 2`, }, { req: putRequest("key3", "3"), resp: putResponse(3), expectDescribe: `put("key3", "3") -> ok, rev: 3`, }, { req: putWithLeaseRequest("key3b", "3b", 3), resp: putResponse(3), expectDescribe: `put("key3b", "3b", 3) -> ok, rev: 3`, }, { req: putRequest("key3c", "01234567890123456789"), resp: putResponse(3), expectDescribe: `put("key3c", hash: 2945867837) -> ok, rev: 3`, }, { req: putRequest("key4", "4"), resp: failedResponse(errors.New("failed")), expectDescribe: `put("key4", "4") -> err: "failed"`, }, { req: putRequest("key4b", "4b"), resp: partialResponse(42), expectDescribe: `put("key4b", "4b") -> unknown, rev: 42`, }, { req: deleteRequest("key5"), resp: deleteResponse(1, 5), expectDescribe: `delete("key5") -> deleted: 1, rev: 5`, }, { req: deleteRequest("key6"), resp: failedResponse(errors.New("failed")), expectDescribe: `delete("key6") -> err: "failed"`, }, { req: compareRevisionAndPutRequest("key7", 7, "77"), resp: txnEmptyResponse(false, 7), expectDescribe: `guaranteedUpdate("key7", "77", mod_rev=7) -> failure(), rev: 7`, }, { req: compareRevisionAndPutRequest("key8", 8, "88"), resp: txnPutResponse(true, 8), expectDescribe: `guaranteedUpdate("key8", "88", mod_rev=8) -> success(ok), rev: 8`, }, { req: compareRevisionAndPutRequest("key8", 0, "89"), resp: txnPutResponse(true, 8), expectDescribe: `guaranteedCreate("key8", "89") -> success(ok), rev: 8`, }, { req: compareRevisionAndPutRequest("key9", 9, "99"), resp: failedResponse(errors.New("failed")), expectDescribe: `guaranteedUpdate("key9", "99", mod_rev=9) -> err: "failed"`, }, { req: txnRequest([]EtcdCondition{{Key: "key9b", ExpectedRevision: 9}}, []EtcdOperation{{Type: PutOperation, Put: PutOptions{Key: "key9b", Value: ValueOrHash{Value: "991"}}}}, []EtcdOperation{{Type: RangeOperation, Range: RangeOptions{Start: "key9b"}}}), resp: txnResponse([]EtcdOperationResult{{}}, true, 10), expectDescribe: `guaranteedUpdate("key9b", "991", mod_rev=9) -> success(ok), rev: 10`, }, { req: txnRequest([]EtcdCondition{{Key: "key9c", ExpectedRevision: 9}}, []EtcdOperation{{Type: PutOperation, Put: PutOptions{Key: "key9c", Value: ValueOrHash{Value: "992"}}}}, []EtcdOperation{{Type: RangeOperation, Range: RangeOptions{Start: "key9c"}}}), resp: txnResponse([]EtcdOperationResult{{RangeResponse: RangeResponse{KVs: []KeyValue{{Key: "key9c", ValueRevision: ValueRevision{Value: ValueOrHash{Value: "993"}, ModRevision: 10}}}}}}, false, 10), expectDescribe: `guaranteedUpdate("key9c", "992", mod_rev=9) -> failure("993"), rev: 10`, }, { req: txnRequest(nil, []EtcdOperation{{Type: RangeOperation, Range: RangeOptions{Start: "10"}}, {Type: PutOperation, Put: PutOptions{Key: "11", Value: ValueOrHash{Value: "111"}}}, {Type: DeleteOperation, Delete: DeleteOptions{Key: "12"}}}, nil), resp: txnResponse([]EtcdOperationResult{{RangeResponse: RangeResponse{KVs: []KeyValue{{ValueRevision: ValueRevision{Value: ValueOrHash{Value: "110"}}}}}}, {}, {Deleted: 1}}, true, 10), expectDescribe: `get("10"), put("11", "111"), delete("12") -> "110", ok, deleted: 1, rev: 10`, }, { req: txnRequest([]EtcdCondition{{Key: "key11", ExpectedRevision: 11}}, []EtcdOperation{{Type: PutOperation, Put: PutOptions{Key: "key11", Value: ValueOrHash{Value: "11"}}}}, []EtcdOperation{{Type: RangeOperation, Range: RangeOptions{Start: "key12"}}}), resp: txnResponse([]EtcdOperationResult{{}}, true, 11), expectDescribe: `if(mod_rev(key11)==11).then(put("key11", "11")).else(get("key12")) -> success(ok), rev: 11`, }, { req: txnRequest([]EtcdCondition{{Key: "key11", ExpectedRevision: 11}}, []EtcdOperation{{Type: PutOperation, Put: PutOptions{Key: "key12", Value: ValueOrHash{Value: "11"}}}}, nil), resp: txnResponse([]EtcdOperationResult{{}}, true, 11), expectDescribe: `if(mod_rev(key11)==11).then(put("key12", "11")) -> success(ok), rev: 11`, }, { req: defragmentRequest(), resp: defragmentResponse(), expectDescribe: `defragment() -> ok`, }, { req: listRequest("key11", 0), resp: rangeResponse(nil, 0, 11), expectDescribe: `list("key11") -> [], count: 0, rev: 11`, }, { req: listRequest("key12", 0), resp: rangeResponse([]*mvccpb.KeyValue{{Value: []byte("12")}}, 2, 12), expectDescribe: `list("key12") -> ["12"], count: 2, rev: 12`, }, { req: listRequest("key13", 0), resp: rangeResponse([]*mvccpb.KeyValue{{Value: []byte("01234567890123456789")}}, 1, 13), expectDescribe: `list("key13") -> [hash: 2945867837], count: 1, rev: 13`, }, { req: listRequest("key14", 14), resp: rangeResponse(nil, 0, 14), expectDescribe: `list("key14", limit=14) -> [], count: 0, rev: 14`, }, { req: staleListRequest("key15", 0, 15), resp: rangeResponse(nil, 0, 15), expectDescribe: `list("key15", rev=15) -> [], count: 0, rev: 15`, }, { req: staleListRequest("key15", 2, 15), resp: rangeResponse(nil, 0, 15), expectDescribe: `list("key15", rev=15, limit=2) -> [], count: 0, rev: 15`, }, { req: rangeRequest("key16", "key16b", 0), resp: rangeResponse(nil, 0, 16), expectDescribe: `range("key16".."key16b") -> [], count: 0, rev: 16`, }, { req: rangeRequest("key16", "key16b", 2), resp: rangeResponse(nil, 0, 16), expectDescribe: `range("key16".."key16b", limit=2) -> [], count: 0, rev: 16`, }, } for _, tc := range tcs { assert.Equal(t, tc.expectDescribe, NonDeterministicModel.DescribeOperation(tc.req, tc.resp)) } } func TestDescribeOperationMetadata(t *testing.T) { tcs := []struct { resp MaybeEtcdResponse expectDescribe string }{ { resp: MaybeEtcdResponse{}, expectDescribe: "", }, { resp: MaybeEtcdResponse{EtcdResponse: EtcdResponse{MemberID: 1}}, expectDescribe: "memberID: 1", }, { resp: MaybeEtcdResponse{EtcdResponse: EtcdResponse{MemberID: 100}}, expectDescribe: "memberID: 64", }, { resp: MaybeEtcdResponse{EtcdResponse: EtcdResponse{MemberID: 255}}, expectDescribe: "memberID: ff", }, } for _, tc := range tcs { assert.Equal(t, tc.expectDescribe, DescribeOperationMetadata(tc.resp)) } } ================================================ FILE: tests/robustness/model/deterministic.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "encoding/json" "fmt" "html" "maps" "reflect" "sort" "github.com/anishathalye/porcupine" "go.etcd.io/etcd/server/v3/storage/mvcc" ) // DeterministicModel assumes a deterministic execution of etcd requests. All // requests that the client called were executed and persisted by etcd. This // assumption is good for simulating etcd behavior (aka writing a fake), but not // for validating correctness as requests might be lost or interrupted. It // requires perfect knowledge of what happened to a request, which is not possible // in real systems. // // Model can still respond with an error or partial response. // - Error for etcd known errors, like future revision or compacted revision. // - Incomplete response when the request is correct, but the model doesn't have all // the data to provide a full response. For example, stale reads as the model doesn't store // the whole change history as real etcd does. var DeterministicModel = porcupine.Model{ Init: func() any { return freshEtcdState() }, Step: func(st any, in any, out any) (bool, any) { return st.(EtcdState).apply(in.(EtcdRequest), out.(EtcdResponse)) }, Equal: func(st1, st2 any) bool { return st1.(EtcdState).Equal(st2.(EtcdState)) }, DescribeOperation: func(in, out any) string { return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), MaybeEtcdResponse{EtcdResponse: out.(EtcdResponse)})) }, DescribeOperationMetadata: func(info any) string { if info == nil { return "" } return DescribeOperationMetadata(MaybeEtcdResponse{EtcdResponse: info.(EtcdResponse)}) }, DescribeState: func(st any) string { data, err := json.MarshalIndent(st, "", " ") if err != nil { panic(err) } return "
" + html.EscapeString(string(data)) + "
" }, } type EtcdState struct { Revision int64 `json:",omitempty"` CompactRevision int64 `json:",omitempty"` KeyValues map[string]ValueRevision `json:",omitempty"` KeyLeases map[string]int64 `json:",omitempty"` Leases map[int64]EtcdLease `json:",omitempty"` } func (s EtcdState) Equal(other EtcdState) bool { if s.Revision != other.Revision { return false } if s.CompactRevision != other.CompactRevision { return false } if !reflect.DeepEqual(s.KeyValues, other.KeyValues) { return false } if !reflect.DeepEqual(s.KeyLeases, other.KeyLeases) { return false } return reflect.DeepEqual(s.Leases, other.Leases) } func (s EtcdState) apply(request EtcdRequest, response EtcdResponse) (bool, EtcdState) { newState, modelResponse := s.Step(request) return Match(MaybeEtcdResponse{EtcdResponse: response}, modelResponse), newState } func (s EtcdState) DeepCopy() EtcdState { newState := EtcdState{ Revision: s.Revision, CompactRevision: s.CompactRevision, } newState.KeyValues = maps.Clone(s.KeyValues) newState.KeyLeases = maps.Clone(s.KeyLeases) newLeases := map[int64]EtcdLease{} for key, val := range s.Leases { newLeases[key] = val.DeepCopy() } newState.Leases = newLeases return newState } func freshEtcdState() EtcdState { return EtcdState{ Revision: 1, // Start from CompactRevision equal -1 as etcd allows client to compact revision 0 for some reason. CompactRevision: -1, KeyValues: map[string]ValueRevision{}, KeyLeases: map[string]int64{}, Leases: map[int64]EtcdLease{}, } } // Step handles a successful request, returning updated state and response it would generate. func (s EtcdState) Step(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { switch request.Type { case Range: return s.stepRange(request) case Txn: return s.stepTxn(request) case LeaseGrant: return s.stepLeaseGrant(request) case LeaseRevoke: return s.stepLeaseRevoke(request) case Defragment: return s.stepDefragment() case Compact: return s.stepCompact(request) default: panic(fmt.Sprintf("Unknown request type: %v", request.Type)) } } func (s EtcdState) stepRange(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { if request.Range.Revision == 0 || request.Range.Revision == s.Revision { resp := s.getRange(request.Range.RangeOptions) return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Range: &resp, Revision: s.Revision}} } if request.Range.Revision > s.Revision { return s, MaybeEtcdResponse{Error: ErrEtcdFutureRev.Error()} } if request.Range.Revision < s.CompactRevision { return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{ClientError: mvcc.ErrCompacted.Error()}} } return s, MaybeEtcdResponse{Persisted: true, PersistedRevision: s.Revision} } func (s EtcdState) stepTxn(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { // TODO: Avoid copying when TXN only has read operations newState := s.DeepCopy() failure := false for _, cond := range request.Txn.Conditions { val := newState.KeyValues[cond.Key] if cond.ExpectedVersion > 0 { if val.Version != cond.ExpectedVersion { failure = true break } } else if val.ModRevision != cond.ExpectedRevision { failure = true break } } operations := request.Txn.OperationsOnSuccess if failure { operations = request.Txn.OperationsOnFailure } opResp := make([]EtcdOperationResult, len(operations)) increaseRevision := false for i, op := range operations { switch op.Type { case RangeOperation: opResp[i] = EtcdOperationResult{ RangeResponse: newState.getRange(op.Range), } case PutOperation: _, leaseExists := newState.Leases[op.Put.LeaseID] if op.Put.LeaseID != 0 && !leaseExists { break } ver := int64(1) if val, exists := newState.KeyValues[op.Put.Key]; exists && val.Version > 0 { ver = val.Version + 1 } newState.KeyValues[op.Put.Key] = ValueRevision{ Value: op.Put.Value, ModRevision: newState.Revision + 1, Version: ver, } increaseRevision = true newState = detachFromOldLease(newState, op.Put.Key) if leaseExists { newState = attachToNewLease(newState, op.Put.LeaseID, op.Put.Key) } case DeleteOperation: if _, ok := newState.KeyValues[op.Delete.Key]; ok { delete(newState.KeyValues, op.Delete.Key) increaseRevision = true newState = detachFromOldLease(newState, op.Delete.Key) opResp[i].Deleted = 1 } default: panic("unsupported operation") } } if increaseRevision { newState.Revision++ } return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Failure: failure, Results: opResp}, Revision: newState.Revision}} } func (s EtcdState) stepLeaseGrant(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { newState := s.DeepCopy() // Empty LeaseID means the request failed and client didn't get response. Ignore it as client cannot use lease without knowing its id. if request.LeaseGrant.LeaseID == 0 { return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: newState.Revision, LeaseGrant: &LeaseGrantResponse{}}} } lease := EtcdLease{ LeaseID: request.LeaseGrant.LeaseID, Keys: map[string]struct{}{}, } newState.Leases[request.LeaseGrant.LeaseID] = lease return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: newState.Revision, LeaseGrant: &LeaseGrantResponse{}}} } func (s EtcdState) stepLeaseRevoke(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { newState := s.DeepCopy() // Delete the keys attached to the lease keyDeleted := false for key := range newState.Leases[request.LeaseRevoke.LeaseID].Keys { // same as delete. if _, ok := newState.KeyValues[key]; ok { if !keyDeleted { keyDeleted = true } delete(newState.KeyValues, key) delete(newState.KeyLeases, key) } } // delete the lease delete(newState.Leases, request.LeaseRevoke.LeaseID) if keyDeleted { newState.Revision++ } return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: newState.Revision, LeaseRevoke: &LeaseRevokeResponse{}}} } func (s EtcdState) stepDefragment() (EtcdState, MaybeEtcdResponse) { return s, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: RevisionForNonLinearizableResponse}} } func (s EtcdState) stepCompact(request EtcdRequest) (EtcdState, MaybeEtcdResponse) { newState := s.DeepCopy() if request.Compact.Revision <= newState.CompactRevision { return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{ClientError: mvcc.ErrCompacted.Error()}} } if request.Compact.Revision > newState.Revision { return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{ClientError: mvcc.ErrFutureRev.Error()}} } newState.CompactRevision = request.Compact.Revision return newState, MaybeEtcdResponse{EtcdResponse: EtcdResponse{Compact: &CompactResponse{}, Revision: RevisionForNonLinearizableResponse}} } func (s EtcdState) getRange(options RangeOptions) RangeResponse { response := RangeResponse{ KVs: []KeyValue{}, } if options.End != "" { var count int64 for k, v := range s.KeyValues { if k >= options.Start && k < options.End { response.KVs = append(response.KVs, KeyValue{Key: k, ValueRevision: v}) count++ } } sort.Slice(response.KVs, func(j, k int) bool { return response.KVs[j].Key < response.KVs[k].Key }) if options.Limit != 0 && count > options.Limit { response.KVs = response.KVs[:options.Limit] } response.Count = count } else { value, ok := s.KeyValues[options.Start] if ok { response.KVs = append(response.KVs, KeyValue{ Key: options.Start, ValueRevision: value, }) response.Count = 1 } } return response } func detachFromOldLease(s EtcdState, key string) EtcdState { if oldLeaseID, ok := s.KeyLeases[key]; ok { delete(s.Leases[oldLeaseID].Keys, key) delete(s.KeyLeases, key) } return s } func attachToNewLease(s EtcdState, leaseID int64, key string) EtcdState { s.KeyLeases[key] = leaseID s.Leases[leaseID].Keys[key] = leased return s } ================================================ FILE: tests/robustness/model/deterministic_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "encoding/json" "testing" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/mvccpb" ) func TestModelDeterministic(t *testing.T) { for _, tc := range commonTestScenarios { tc := tc t.Run(tc.name, func(t *testing.T) { state := DeterministicModel.Init() for _, op := range tc.operations { ok, newState := DeterministicModel.Step(state, op.req, op.resp.EtcdResponse) if op.expectFailure == ok { t.Logf("state: %v", state) t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.expectFailure, ok, DeterministicModel.DescribeOperation(op.req, op.resp.EtcdResponse)) var loadedState EtcdState err := json.Unmarshal([]byte(state.(string)), &loadedState) require.NoErrorf(t, err, "Failed to load state") _, resp := loadedState.Step(op.req) t.Errorf("Response diff: %s", cmp.Diff(op.resp, resp)) break } if ok { state = newState t.Logf("state: %v", state) } } }) } } type modelTestCase struct { name string operations []testOperation } type testOperation struct { req EtcdRequest resp MaybeEtcdResponse expectFailure bool } var commonTestScenarios = []modelTestCase{ { name: "Get response data should match put", operations: []testOperation{ {req: putRequest("key1", "11"), resp: putResponse(2)}, {req: putRequest("key2", "12"), resp: putResponse(3)}, {req: getRequest("key1"), resp: getResponse("key1", "11", 2, 2), expectFailure: true}, {req: getRequest("key1"), resp: getResponse("key1", "12", 2, 2), expectFailure: true}, {req: getRequest("key1"), resp: getResponse("key1", "12", 3, 3), expectFailure: true}, {req: getRequest("key1"), resp: getResponse("key1", "11", 2, 3)}, {req: getRequest("key2"), resp: getResponse("key2", "11", 3, 3), expectFailure: true}, {req: getRequest("key2"), resp: getResponse("key2", "12", 2, 2), expectFailure: true}, {req: getRequest("key2"), resp: getResponse("key2", "11", 2, 2), expectFailure: true}, {req: getRequest("key2"), resp: getResponse("key2", "12", 3, 3)}, }, }, { name: "Range response data should match put", operations: []testOperation{ {req: putRequest("key1", "1"), resp: putResponse(2)}, {req: putRequest("key2", "2"), resp: putResponse(3)}, {req: listRequest("key", 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2, Version: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3, Version: 1}}, 2, 3)}, {req: listRequest("key", 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2, Version: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3, Version: 1}}, 2, 3)}, }, }, { name: "Range limit should reduce number of kvs, but maintain count", operations: []testOperation{ {req: putRequest("key1", "1"), resp: putResponse(2)}, {req: putRequest("key2", "2"), resp: putResponse(3)}, {req: putRequest("key3", "3"), resp: putResponse(4)}, {req: listRequest("key", 0), resp: rangeResponse([]*mvccpb.KeyValue{ {Key: []byte("key1"), Value: []byte("1"), ModRevision: 2, Version: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3, Version: 1}, {Key: []byte("key3"), Value: []byte("3"), ModRevision: 4, Version: 1}, }, 3, 4)}, {req: listRequest("key", 4), resp: rangeResponse([]*mvccpb.KeyValue{ {Key: []byte("key1"), Value: []byte("1"), ModRevision: 2, Version: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3, Version: 1}, {Key: []byte("key3"), Value: []byte("3"), ModRevision: 4, Version: 1}, }, 3, 4)}, {req: listRequest("key", 3), resp: rangeResponse([]*mvccpb.KeyValue{ {Key: []byte("key1"), Value: []byte("1"), ModRevision: 2, Version: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3, Version: 1}, {Key: []byte("key3"), Value: []byte("3"), ModRevision: 4, Version: 1}, }, 3, 4)}, {req: listRequest("key", 2), resp: rangeResponse([]*mvccpb.KeyValue{ {Key: []byte("key1"), Value: []byte("1"), ModRevision: 2, Version: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3, Version: 1}, }, 3, 4)}, {req: listRequest("key", 1), resp: rangeResponse([]*mvccpb.KeyValue{ {Key: []byte("key1"), Value: []byte("1"), ModRevision: 2, Version: 1}, }, 3, 4)}, }, }, { name: "Range response should be ordered by key", operations: []testOperation{ {req: putRequest("key3", "3"), resp: putResponse(2)}, {req: putRequest("key2", "1"), resp: putResponse(3)}, {req: putRequest("key1", "2"), resp: putResponse(4)}, {req: listRequest("key", 0), resp: rangeResponse([]*mvccpb.KeyValue{ {Key: []byte("key1"), Value: []byte("2"), ModRevision: 4, Version: 1}, {Key: []byte("key2"), Value: []byte("1"), ModRevision: 3, Version: 1}, {Key: []byte("key3"), Value: []byte("3"), ModRevision: 2, Version: 1}, }, 3, 4)}, {req: listRequest("key", 0), resp: rangeResponse([]*mvccpb.KeyValue{ {Key: []byte("key2"), Value: []byte("1"), ModRevision: 3, Version: 1}, {Key: []byte("key1"), Value: []byte("2"), ModRevision: 4, Version: 1}, {Key: []byte("key3"), Value: []byte("3"), ModRevision: 2, Version: 1}, }, 3, 4), expectFailure: true}, {req: listRequest("key", 0), resp: rangeResponse([]*mvccpb.KeyValue{ {Key: []byte("key3"), Value: []byte("3"), ModRevision: 2, Version: 1}, {Key: []byte("key2"), Value: []byte("1"), ModRevision: 3, Version: 1}, {Key: []byte("key1"), Value: []byte("2"), ModRevision: 4, Version: 1}, }, 3, 4), expectFailure: true}, }, }, { name: "Range response data should match large put", operations: []testOperation{ {req: putRequest("key", "012345678901234567890"), resp: putResponse(2)}, {req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 2, 2), expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 2, 2)}, {req: putRequest("key", "123456789012345678901"), resp: putResponse(3)}, {req: getRequest("key"), resp: getResponseWithVer("key", "123456789012345678901", 3, 2, 3)}, {req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 3, 3), expectFailure: true}, }, }, { name: "Stale Get doesn't need to match put if asking about old revision", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: staleGetRequest("key", 1), resp: getResponse("key", "2", 2, 2)}, {req: staleGetRequest("key", 1), resp: getResponse("key", "1", 2, 2)}, }, }, { name: "Stale Get need to match put if asking about matching revision", operations: []testOperation{ {req: putRequest("key1", "1"), resp: putResponse(2)}, {req: staleGetRequest("key1", 2), resp: getResponse("key1", "1", 3, 2), expectFailure: true}, {req: staleGetRequest("key1", 2), resp: getResponse("key1", "1", 2, 3), expectFailure: true}, {req: staleGetRequest("key1", 2), resp: getResponse("key1", "2", 2, 2), expectFailure: true}, {req: staleGetRequest("key1", 2), resp: getResponse("key1", "1", 2, 2)}, }, }, { name: "Stale Get need to have a proper response revision", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: staleGetRequest("key", 2), resp: getResponse("key", "1", 2, 3), expectFailure: true}, {req: staleGetRequest("key", 2), resp: getResponse("key", "1", 2, 2)}, {req: putRequest("key", "2"), resp: putResponse(3)}, {req: staleGetRequest("key", 2), resp: getResponse("key", "1", 2, 3)}, }, }, { name: "Put must increase revision by 1", operations: []testOperation{ {req: getRequest("key"), resp: emptyGetResponse(1)}, {req: putRequest("key", "1"), resp: putResponse(1), expectFailure: true}, {req: putRequest("key", "1"), resp: putResponse(3), expectFailure: true}, {req: putRequest("key", "1"), resp: putResponse(2)}, }, }, { name: "Delete only increases revision on success", operations: []testOperation{ {req: putRequest("key1", "11"), resp: putResponse(2)}, {req: putRequest("key2", "12"), resp: putResponse(3)}, {req: deleteRequest("key1"), resp: deleteResponse(1, 3), expectFailure: true}, {req: deleteRequest("key1"), resp: deleteResponse(1, 4)}, {req: deleteRequest("key1"), resp: deleteResponse(0, 5), expectFailure: true}, {req: deleteRequest("key1"), resp: deleteResponse(0, 4)}, }, }, { name: "Delete not existing key", operations: []testOperation{ {req: getRequest("key"), resp: emptyGetResponse(1)}, {req: deleteRequest("key"), resp: deleteResponse(1, 2), expectFailure: true}, {req: deleteRequest("key"), resp: deleteResponse(0, 1)}, }, }, { name: "Delete clears value", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: deleteRequest("key"), resp: deleteResponse(1, 3)}, {req: getRequest("key"), resp: getResponse("key", "1", 2, 2), expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "1", 3, 3), expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "1", 2, 3), expectFailure: true}, {req: getRequest("key"), resp: emptyGetResponse(3)}, }, }, { name: "Txn executes onSuccess if revision matches expected", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: compareRevisionAndPutRequest("key", 2, "2"), resp: compareRevisionAndPutResponse(true, 2), expectFailure: true}, {req: compareRevisionAndPutRequest("key", 2, "2"), resp: compareRevisionAndPutResponse(false, 3), expectFailure: true}, {req: compareRevisionAndPutRequest("key", 2, "2"), resp: compareRevisionAndPutResponse(false, 2), expectFailure: true}, {req: compareRevisionAndPutRequest("key", 2, "2"), resp: compareRevisionAndPutResponse(true, 3)}, {req: getRequest("key"), resp: getResponse("key", "1", 2, 2), expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "1", 2, 3), expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "1", 3, 3), expectFailure: true}, {req: getRequest("key"), resp: getResponseWithVer("key", "2", 2, 2, 2), expectFailure: true}, {req: getRequest("key"), resp: getResponseWithVer("key", "2", 3, 2, 3)}, }, }, { name: "Txn can expect on key not existing", operations: []testOperation{ {req: getRequest("key1"), resp: emptyGetResponse(1)}, {req: compareRevisionAndPutRequest("key1", 0, "2"), resp: compareRevisionAndPutResponse(true, 2)}, {req: compareRevisionAndPutRequest("key1", 0, "3"), resp: compareRevisionAndPutResponse(true, 3), expectFailure: true}, {req: txnRequestSingleOperation(compareRevision("key1", 0), putOperation("key1", "4"), putOperation("key1", "5")), resp: txnPutResponse(false, 3)}, {req: getRequest("key1"), resp: getResponseWithVer("key1", "5", 3, 2, 3)}, {req: compareRevisionAndPutRequest("key2", 0, "6"), resp: compareRevisionAndPutResponse(true, 4)}, }, }, { name: "Txn executes onFailure if revision doesn't match expected", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnPutResponse(false, 3), expectFailure: true}, {req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnEmptyResponse(false, 3), expectFailure: true}, {req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 3), expectFailure: true}, {req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnPutResponse(true, 2), expectFailure: true}, {req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 2)}, {req: txnRequestSingleOperation(compareRevision("key", 3), nil, putOperation("key", "2")), resp: txnPutResponse(false, 3)}, }, }, { name: "Put with valid lease id should succeed. Put with invalid lease id should fail", operations: []testOperation{ {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)}, {req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3), expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "2", 2, 2)}, }, }, { name: "Lease grant returns current revision", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(2)}, }, }, { name: "Put with valid lease id should succeed. Put with expired lease id should fail", operations: []testOperation{ {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)}, {req: getRequest("key"), resp: getResponse("key", "2", 2, 2)}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)}, {req: putWithLeaseRequest("key", "4", 1), resp: putResponse(4), expectFailure: true}, {req: getRequest("key"), resp: emptyGetResponse(3)}, }, }, { name: "Revoke should increment the revision", operations: []testOperation{ {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)}, {req: getRequest("key"), resp: emptyGetResponse(3)}, }, }, { name: "Put following a PutWithLease will detach the key from the lease", operations: []testOperation{ {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)}, {req: putRequest("key", "3"), resp: putResponse(3)}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)}, {req: getRequest("key"), resp: getResponseWithVer("key", "3", 3, 2, 3)}, }, }, { name: "Change lease. Revoking older lease should not increment revision", operations: []testOperation{ {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: leaseGrantRequest(2), resp: leaseGrantResponse(1)}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)}, {req: putWithLeaseRequest("key", "3", 2), resp: putResponse(3)}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)}, {req: getRequest("key"), resp: getResponseWithVer("key", "3", 3, 2, 3)}, {req: leaseRevokeRequest(2), resp: leaseRevokeResponse(4)}, {req: getRequest("key"), resp: emptyGetResponse(4)}, }, }, { name: "Update key with same lease", operations: []testOperation{ {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)}, {req: putWithLeaseRequest("key", "3", 1), resp: putResponse(3)}, {req: getRequest("key"), resp: getResponseWithVer("key", "3", 3, 2, 3)}, }, }, { name: "Deleting a leased key - revoke should not increment revision", operations: []testOperation{ {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: putWithLeaseRequest("key", "2", 1), resp: putResponse(2)}, {req: deleteRequest("key"), resp: deleteResponse(1, 3)}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(4), expectFailure: true}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)}, }, }, { name: "Lease a few keys - revoke should increment revision only once", operations: []testOperation{ {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2)}, {req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3)}, {req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4)}, {req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5)}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(6)}, }, }, { name: "Lease some keys then delete some of them. Revoke should increment revision since some keys were still leased", operations: []testOperation{ {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2)}, {req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3)}, {req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4)}, {req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5)}, {req: deleteRequest("key1"), resp: deleteResponse(1, 6)}, {req: deleteRequest("key3"), resp: deleteResponse(1, 7)}, {req: deleteRequest("key4"), resp: deleteResponse(1, 8)}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9)}, {req: deleteRequest("key2"), resp: deleteResponse(0, 9)}, {req: getRequest("key1"), resp: emptyGetResponse(9)}, {req: getRequest("key2"), resp: emptyGetResponse(9)}, {req: getRequest("key3"), resp: emptyGetResponse(9)}, {req: getRequest("key4"), resp: emptyGetResponse(9)}, }, }, { name: "Lease some keys then delete all of them. Revoke should not increment", operations: []testOperation{ {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: putWithLeaseRequest("key1", "1", 1), resp: putResponse(2)}, {req: putWithLeaseRequest("key2", "2", 1), resp: putResponse(3)}, {req: putWithLeaseRequest("key3", "3", 1), resp: putResponse(4)}, {req: putWithLeaseRequest("key4", "4", 1), resp: putResponse(5)}, {req: deleteRequest("key1"), resp: deleteResponse(1, 6)}, {req: deleteRequest("key2"), resp: deleteResponse(1, 7)}, {req: deleteRequest("key3"), resp: deleteResponse(1, 8)}, {req: deleteRequest("key4"), resp: deleteResponse(1, 9)}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(9)}, }, }, { name: "All request types", operations: []testOperation{ {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2)}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)}, {req: putRequest("key", "4"), resp: putResponse(4)}, {req: getRequest("key"), resp: getResponse("key", "4", 4, 4)}, {req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5)}, {req: deleteRequest("key"), resp: deleteResponse(1, 6)}, {req: defragmentRequest(), resp: defragmentResponse()}, }, }, { name: "Defragment success between all other request types", operations: []testOperation{ {req: defragmentRequest(), resp: defragmentResponse()}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: defragmentRequest(), resp: defragmentResponse()}, {req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2)}, {req: defragmentRequest(), resp: defragmentResponse()}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)}, {req: defragmentRequest(), resp: defragmentResponse()}, {req: putRequest("key", "4"), resp: putResponse(4)}, {req: defragmentRequest(), resp: defragmentResponse()}, {req: getRequest("key"), resp: getResponse("key", "4", 4, 4)}, {req: defragmentRequest(), resp: defragmentResponse()}, {req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5)}, {req: defragmentRequest(), resp: defragmentResponse()}, {req: deleteRequest("key"), resp: deleteResponse(1, 6)}, {req: defragmentRequest(), resp: defragmentResponse()}, }, }, } ================================================ FILE: tests/robustness/model/history.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "fmt" "strings" "time" "github.com/anishathalye/porcupine" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/tests/v3/robustness/identity" ) // AppendableHistory allows collecting the history of sequential operations. // // Ensures that the operation history is compatible with the porcupine library by preventing concurrent requests from sharing the // same stream id. For failed requests, we don't know their return time, so we generate a new stream id. // // Appending needs to be done in order of operation execution time (start, end time). // Operation time should be calculated as time.Since a common base time to ensure that Go monotonic time is used. // More in https://github.com/golang/go/blob/96add980ad27faed627f26ef1ab09e8fe45d6bd1/src/time/time.go#L10. type AppendableHistory struct { // streamID for the next operation. Used for porcupine.Operation.ClientId as porcupine assumes no concurrent requests. streamID int // If needed a new streamId is requested from idProvider. idProvider identity.Provider // lastOperation holds the last operation of each stream. lastOperation map[int]*porcupine.Operation History } func NewAppendableHistory(ids identity.Provider) *AppendableHistory { return &AppendableHistory{ streamID: ids.NewStreamID(), idProvider: ids, lastOperation: make(map[int]*porcupine.Operation), History: History{ operations: []porcupine.Operation{}, }, } } func (h *AppendableHistory) AppendRange(startKey, endKey string, revision, limit int64, start, end time.Duration, resp *clientv3.GetResponse, err error) { request := staleRangeRequest(startKey, endKey, limit, revision) if err != nil { h.appendFailed(request, start, end, err) return } var respRevision int64 var respMemberID uint64 if resp != nil && resp.Header != nil { respRevision = resp.Header.Revision respMemberID = resp.Header.MemberId } h.appendSuccessful(request, start, end, rangeResponseWithMemberID(resp.Kvs, resp.Count, respRevision, MemberID(respMemberID))) } func (h *AppendableHistory) AppendPut(key, value string, start, end time.Duration, resp *clientv3.PutResponse, err error) { request := putRequest(key, value) if err != nil { h.appendFailed(request, start, end, err) return } var revision int64 var memberID MemberID if resp != nil && resp.Header != nil { revision = resp.Header.Revision memberID = MemberID(resp.Header.MemberId) } h.appendSuccessful(request, start, end, putResponseWithMemberID(revision, memberID)) } func (h *AppendableHistory) AppendPutWithLease(key, value string, leaseID int64, start, end time.Duration, resp *clientv3.PutResponse, err error) { request := putWithLeaseRequest(key, value, leaseID) if err != nil { h.appendFailed(request, start, end, err) return } var revision int64 var memberID MemberID if resp != nil && resp.Header != nil { revision = resp.Header.Revision memberID = MemberID(resp.Header.MemberId) } h.appendSuccessful(request, start, end, putResponseWithMemberID(revision, memberID)) } func (h *AppendableHistory) AppendLeaseGrant(start, end time.Duration, resp *clientv3.LeaseGrantResponse, err error) { var leaseID int64 if resp != nil { leaseID = int64(resp.ID) } request := leaseGrantRequest(leaseID) if err != nil { h.appendFailed(request, start, end, err) return } var revision int64 var memberID MemberID if resp != nil && resp.ResponseHeader != nil { revision = resp.ResponseHeader.Revision memberID = MemberID(resp.ResponseHeader.MemberId) } h.appendSuccessful(request, start, end, leaseGrantResponseWithMemberID(revision, memberID)) } func (h *AppendableHistory) AppendLeaseRevoke(id int64, start, end time.Duration, resp *clientv3.LeaseRevokeResponse, err error) { request := leaseRevokeRequest(id) if err != nil { h.appendFailed(request, start, end, err) return } var revision int64 var memberID MemberID if resp != nil && resp.Header != nil { revision = resp.Header.Revision memberID = MemberID(resp.Header.MemberId) } h.appendSuccessful(request, start, end, leaseRevokeResponseWithMemberID(revision, memberID)) } func (h *AppendableHistory) AppendDelete(key string, start, end time.Duration, resp *clientv3.DeleteResponse, err error) { request := deleteRequest(key) if err != nil { h.appendFailed(request, start, end, err) return } var revision int64 var memberID MemberID var deleted int64 if resp != nil && resp.Header != nil { revision = resp.Header.Revision memberID = MemberID(resp.Header.MemberId) deleted = resp.Deleted } h.appendSuccessful(request, start, end, deleteResponseWithMemberID(deleted, revision, memberID)) } func (h *AppendableHistory) AppendTxn(cmp []clientv3.Cmp, clientOnSuccessOps, clientOnFailure []clientv3.Op, start, end time.Duration, resp *clientv3.TxnResponse, err error) { conds := []EtcdCondition{} for _, cmp := range cmp { conds = append(conds, toEtcdCondition(cmp)) } modelOnSuccess := []EtcdOperation{} for _, op := range clientOnSuccessOps { modelOnSuccess = append(modelOnSuccess, toEtcdOperation(op)) } modelOnFailure := []EtcdOperation{} for _, op := range clientOnFailure { modelOnFailure = append(modelOnFailure, toEtcdOperation(op)) } request := txnRequest(conds, modelOnSuccess, modelOnFailure) if err != nil { h.appendFailed(request, start, end, err) return } var revision int64 var memberID MemberID if resp != nil && resp.Header != nil { revision = resp.Header.Revision memberID = MemberID(resp.Header.MemberId) } results := []EtcdOperationResult{} for _, resp := range resp.Responses { results = append(results, toEtcdOperationResult(resp)) } h.appendSuccessful(request, start, end, txnResponseWithMemberID(results, resp.Succeeded, revision, memberID)) } func (h *AppendableHistory) appendClientError(request EtcdRequest, start, end time.Duration, err error) { h.appendSuccessful(request, start, end, MaybeEtcdResponse{ EtcdResponse: EtcdResponse{ClientError: err.Error()}, }) } func (h *AppendableHistory) appendSuccessful(request EtcdRequest, start, end time.Duration, response MaybeEtcdResponse) { op := porcupine.Operation{ ClientId: h.streamID, Input: request, Call: start.Nanoseconds(), Output: response, Return: end.Nanoseconds(), Metadata: response, } h.append(op) } func toEtcdCondition(cmp clientv3.Cmp) (cond EtcdCondition) { switch { case cmp.Result == etcdserverpb.Compare_EQUAL && cmp.Target == etcdserverpb.Compare_MOD: cond.Key = string(cmp.KeyBytes()) cond.ExpectedRevision = cmp.TargetUnion.(*etcdserverpb.Compare_ModRevision).ModRevision case cmp.Result == etcdserverpb.Compare_EQUAL && cmp.Target == etcdserverpb.Compare_VERSION: cond.ExpectedVersion = cmp.TargetUnion.(*etcdserverpb.Compare_Version).Version cond.Key = string(cmp.KeyBytes()) default: panic(fmt.Sprintf("Compare not supported, target: %q, result: %q", cmp.Target, cmp.Result)) } return cond } func toEtcdOperation(option clientv3.Op) (op EtcdOperation) { switch { case option.IsGet(): op.Type = RangeOperation op.Range = RangeOptions{ Start: string(option.KeyBytes()), End: string(option.RangeBytes()), } case option.IsPut(): op.Type = PutOperation op.Put = PutOptions{ Key: string(option.KeyBytes()), Value: ValueOrHash{Value: string(option.ValueBytes())}, } case option.IsDelete(): op.Type = DeleteOperation op.Delete = DeleteOptions{ Key: string(option.KeyBytes()), } default: panic("Unsupported operation") } return op } func toEtcdOperationResult(resp *etcdserverpb.ResponseOp) EtcdOperationResult { switch { case resp.GetResponseRange() != nil: getResp := resp.GetResponseRange() kvs := make([]KeyValue, len(getResp.Kvs)) for i, kv := range getResp.Kvs { kvs[i] = KeyValue{ Key: string(kv.Key), ValueRevision: ValueRevision{ Value: ToValueOrHash(string(kv.Value)), ModRevision: kv.ModRevision, Version: kv.Version, }, } } return EtcdOperationResult{ RangeResponse: RangeResponse{ KVs: kvs, Count: getResp.Count, }, } case resp.GetResponsePut() != nil: return EtcdOperationResult{} case resp.GetResponseDeleteRange() != nil: return EtcdOperationResult{ Deleted: resp.GetResponseDeleteRange().Deleted, } default: panic("Unsupported operation") } } func (h *AppendableHistory) AppendDefragment(start, end time.Duration, resp *clientv3.DefragmentResponse, err error) { request := defragmentRequest() if err != nil { h.appendFailed(request, start, end, err) return } h.appendSuccessful(request, start, end, defragmentResponse()) } func (h *AppendableHistory) AppendCompact(rev int64, start, end time.Duration, resp *clientv3.CompactResponse, err error) { request := compactRequest(rev) if err != nil { if strings.Contains(err.Error(), mvcc.ErrCompacted.Error()) { h.appendClientError(request, start, end, mvcc.ErrCompacted) return } if strings.Contains(err.Error(), mvcc.ErrFutureRev.Error()) { h.appendClientError(request, start, end, mvcc.ErrFutureRev) return } h.appendFailed(request, start, end, err) return } h.appendSuccessful(request, start, end, compactResponse()) } func (h *AppendableHistory) appendFailed(request EtcdRequest, start, end time.Duration, err error) { resp := failedResponse(err) op := porcupine.Operation{ ClientId: h.streamID, Input: request, Call: start.Nanoseconds(), Output: resp, Return: end.Nanoseconds(), Metadata: resp, } isRead := request.IsRead() if !isRead { // Operations of single client needs to be sequential. // As we don't know return time of failed operations, all new writes need to be done with new stream id. h.streamID = h.idProvider.NewStreamID() } h.append(op) } func (h *AppendableHistory) append(op porcupine.Operation) { if op.Call >= op.Return { panic(fmt.Sprintf("Invalid operation, call(%d) >= return(%d)", op.Call, op.Return)) } if prev, ok := h.lastOperation[op.ClientId]; ok { if op.Call <= prev.Call { panic(fmt.Sprintf("Out of order append, new.call(%d) <= prev.call(%d)", op.Call, prev.Call)) } if op.Call <= prev.Return { panic(fmt.Sprintf("Overlapping operations, new.call(%d) <= prev.return(%d)", op.Call, prev.Return)) } } h.lastOperation[op.ClientId] = &op h.operations = append(h.operations, op) } func getRequest(key string) EtcdRequest { return rangeRequest(key, "", 0) } func staleGetRequest(key string, revision int64) EtcdRequest { return staleRangeRequest(key, "", 0, revision) } func rangeRequest(start, end string, limit int64) EtcdRequest { return staleRangeRequest(start, end, limit, 0) } func listRequest(key string, limit int64) EtcdRequest { return staleListRequest(key, limit, 0) } func staleListRequest(key string, limit, revision int64) EtcdRequest { return staleRangeRequest(key, clientv3.GetPrefixRangeEnd(key), limit, revision) } func staleRangeRequest(start, end string, limit, revision int64) EtcdRequest { return EtcdRequest{Type: Range, Range: &RangeRequest{RangeOptions: RangeOptions{Start: start, End: end, Limit: limit}, Revision: revision}} } func emptyGetResponse(revision int64) MaybeEtcdResponse { return rangeResponse([]*mvccpb.KeyValue{}, 0, revision) } func getResponse(key, value string, modRevision, revision int64) MaybeEtcdResponse { return getResponseWithVer(key, value, modRevision, 1, revision) } func getResponseWithVer(key, value string, modRevision, ver, revision int64) MaybeEtcdResponse { return rangeResponse([]*mvccpb.KeyValue{{Key: []byte(key), Value: []byte(value), ModRevision: modRevision, Version: ver}}, 1, revision) } func rangeResponse(kvs []*mvccpb.KeyValue, count int64, revision int64) MaybeEtcdResponse { return rangeResponseWithMemberID(kvs, count, revision, 0) } func rangeResponseWithMemberID(kvs []*mvccpb.KeyValue, count int64, revision int64, memberID MemberID) MaybeEtcdResponse { result := RangeResponse{KVs: make([]KeyValue, len(kvs)), Count: count} for i, kv := range kvs { result.KVs[i] = KeyValue{ Key: string(kv.Key), ValueRevision: ValueRevision{ Value: ToValueOrHash(string(kv.Value)), ModRevision: kv.ModRevision, Version: kv.Version, }, } } return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Range: &result, Revision: revision, MemberID: memberID}} } func failedResponse(err error) MaybeEtcdResponse { return MaybeEtcdResponse{Error: err.Error()} } func partialResponse(revision int64) MaybeEtcdResponse { return MaybeEtcdResponse{Persisted: true, PersistedRevision: revision} } func putRequest(key, value string) EtcdRequest { return EtcdRequest{Type: Txn, Txn: &TxnRequest{OperationsOnSuccess: []EtcdOperation{{Type: PutOperation, Put: PutOptions{Key: key, Value: ToValueOrHash(value)}}}}} } func putResponse(revision int64) MaybeEtcdResponse { return putResponseWithMemberID(revision, 0) } func putResponseWithMemberID(revision int64, memberID MemberID) MaybeEtcdResponse { return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{}}}, Revision: revision, MemberID: memberID}} } func deleteRequest(key string) EtcdRequest { return EtcdRequest{Type: Txn, Txn: &TxnRequest{OperationsOnSuccess: []EtcdOperation{{Type: DeleteOperation, Delete: DeleteOptions{Key: key}}}}} } func deleteResponse(deleted int64, revision int64) MaybeEtcdResponse { return deleteResponseWithMemberID(deleted, revision, 0) } func deleteResponseWithMemberID(deleted int64, revision int64, memberID MemberID) MaybeEtcdResponse { return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: []EtcdOperationResult{{Deleted: deleted}}}, Revision: revision, MemberID: memberID}} } func compareRevisionAndPutRequest(key string, expectedRevision int64, value string) EtcdRequest { return txnRequestSingleOperation(compareRevision(key, expectedRevision), putOperation(key, value), nil) } func compareRevisionAndPutResponse(succeeded bool, revision int64) MaybeEtcdResponse { if succeeded { return txnPutResponse(succeeded, revision) } return txnEmptyResponse(succeeded, revision) } func compareRevision(key string, expectedRevision int64) *EtcdCondition { return &EtcdCondition{Key: key, ExpectedRevision: expectedRevision} } func putOperation(key, value string) *EtcdOperation { return &EtcdOperation{Type: PutOperation, Put: PutOptions{Key: key, Value: ToValueOrHash(value)}} } func txnRequestSingleOperation(cond *EtcdCondition, onSuccess, onFailure *EtcdOperation) EtcdRequest { var conds []EtcdCondition if cond != nil { conds = []EtcdCondition{*cond} } var onSuccess2 []EtcdOperation if onSuccess != nil { onSuccess2 = []EtcdOperation{*onSuccess} } var onFailure2 []EtcdOperation if onFailure != nil { onFailure2 = []EtcdOperation{*onFailure} } return txnRequest(conds, onSuccess2, onFailure2) } func txnRequest(conds []EtcdCondition, onSuccess, onFailure []EtcdOperation) EtcdRequest { return EtcdRequest{Type: Txn, Txn: &TxnRequest{Conditions: conds, OperationsOnSuccess: onSuccess, OperationsOnFailure: onFailure}} } func txnPutResponse(succeeded bool, revision int64) MaybeEtcdResponse { return txnResponse([]EtcdOperationResult{{}}, succeeded, revision) } func txnEmptyResponse(succeeded bool, revision int64) MaybeEtcdResponse { return txnResponse([]EtcdOperationResult{}, succeeded, revision) } func txnResponse(result []EtcdOperationResult, succeeded bool, revision int64) MaybeEtcdResponse { return txnResponseWithMemberID(result, succeeded, revision, 0) } func txnResponseWithMemberID(result []EtcdOperationResult, succeeded bool, revision int64, memberID MemberID) MaybeEtcdResponse { return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{Results: result, Failure: !succeeded}, Revision: revision, MemberID: memberID}} } func putWithLeaseRequest(key, value string, leaseID int64) EtcdRequest { return EtcdRequest{Type: Txn, Txn: &TxnRequest{OperationsOnSuccess: []EtcdOperation{{Type: PutOperation, Put: PutOptions{Key: key, Value: ToValueOrHash(value), LeaseID: leaseID}}}}} } func leaseGrantRequest(leaseID int64) EtcdRequest { return EtcdRequest{Type: LeaseGrant, LeaseGrant: &LeaseGrantRequest{LeaseID: leaseID}} } func leaseGrantResponse(revision int64) MaybeEtcdResponse { return leaseGrantResponseWithMemberID(revision, 0) } func leaseGrantResponseWithMemberID(revision int64, memberID MemberID) MaybeEtcdResponse { return MaybeEtcdResponse{EtcdResponse: EtcdResponse{LeaseGrant: &LeaseGrantResponse{}, Revision: revision, MemberID: memberID}} } func leaseRevokeRequest(leaseID int64) EtcdRequest { return EtcdRequest{Type: LeaseRevoke, LeaseRevoke: &LeaseRevokeRequest{LeaseID: leaseID}} } func leaseRevokeResponse(revision int64) MaybeEtcdResponse { return leaseRevokeResponseWithMemberID(revision, 0) } func leaseRevokeResponseWithMemberID(revision int64, memberID MemberID) MaybeEtcdResponse { return MaybeEtcdResponse{EtcdResponse: EtcdResponse{LeaseRevoke: &LeaseRevokeResponse{}, Revision: revision, MemberID: memberID}} } func defragmentRequest() EtcdRequest { return EtcdRequest{Type: Defragment, Defragment: &DefragmentRequest{}} } func defragmentResponse() MaybeEtcdResponse { return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Defragment: &DefragmentResponse{}, Revision: RevisionForNonLinearizableResponse}} } func compactRequest(rev int64) EtcdRequest { return EtcdRequest{Type: Compact, Compact: &CompactRequest{Revision: rev}} } func compactResponse() MaybeEtcdResponse { return MaybeEtcdResponse{EtcdResponse: EtcdResponse{Compact: &CompactResponse{}, Revision: RevisionForNonLinearizableResponse}} } type History struct { operations []porcupine.Operation } func (h History) Len() int { return len(h.operations) } func (h History) Operations() []porcupine.Operation { operations := make([]porcupine.Operation, 0, len(h.operations)) return append(operations, h.operations...) } func (h History) MaxRevision() int64 { var maxRevision int64 for _, op := range h.operations { revision := op.Output.(MaybeEtcdResponse).Revision if revision > maxRevision { maxRevision = revision } } return maxRevision } ================================================ FILE: tests/robustness/model/history_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "testing" "github.com/anishathalye/porcupine" "github.com/stretchr/testify/assert" "go.etcd.io/etcd/tests/v3/robustness/identity" ) func TestHistoryAppendSuccess(t *testing.T) { tcs := []struct { name string operations []porcupine.Operation }{ { name: "append non-overlapping operations for the same clientId", operations: []porcupine.Operation{ { ClientId: 1, Input: nil, Call: 100, Output: nil, Return: 200, }, { ClientId: 2, Input: nil, Call: 1, Output: nil, Return: 2, }, }, }, { name: "append overlapping operations in different ClientId", operations: []porcupine.Operation{ { ClientId: 1, Input: nil, Call: 1, Output: nil, Return: 100, }, { ClientId: 2, Input: nil, Call: 1, Output: nil, Return: 2, }, { ClientId: 3, Input: nil, Call: 10, Output: nil, Return: 20, }, { ClientId: 1, Input: nil, Call: 101, Output: nil, Return: 200, }, { ClientId: 2, Input: nil, Call: 3, Output: nil, Return: 4, }, { ClientId: 3, Input: nil, Call: 30, Output: nil, Return: 40, }, }, }, } for _, tc := range tcs { h := NewAppendableHistory(identity.NewIDProvider()) for _, operation := range tc.operations[:len(tc.operations)-1] { h.append(operation) } } } func TestHistoryAppendFailure(t *testing.T) { tcs := []struct { name string operations []porcupine.Operation expectError string }{ { name: "append operation with call time > return time", operations: []porcupine.Operation{ { ClientId: 1, Input: nil, Call: 2, Output: nil, Return: 1, }, }, expectError: "Invalid operation, call(2) >= return(1)", }, { name: "out of order append in the same ClientId", operations: []porcupine.Operation{ { ClientId: 1, Input: nil, Call: 10, Output: nil, Return: 100, }, { ClientId: 1, Input: nil, Call: 1, Output: nil, Return: 10, }, }, expectError: "Out of order append, new.call(1) <= prev.call(10)", }, { name: "out of order append in one of the ClientIds", operations: []porcupine.Operation{ { ClientId: 1, Input: nil, Call: 10, Output: nil, Return: 100, }, { ClientId: 1, Input: nil, Call: 101, Output: nil, Return: 200, }, { ClientId: 2, Input: nil, Call: 10, Output: nil, Return: 100, }, { ClientId: 2, Input: nil, Call: 1, Output: nil, Return: 10, }, }, expectError: "Out of order append, new.call(1) <= prev.call(10)", }, { name: "append overlapping operations in the same ClientId", operations: []porcupine.Operation{ { ClientId: 1, Input: nil, Call: 1, Output: nil, Return: 100, }, { ClientId: 1, Input: nil, Call: 10, Output: nil, Return: 20, }, }, expectError: "Overlapping operations, new.call(10) <= prev.return(100)", }, { name: "append overlapping operations in one of the ClientIds", operations: []porcupine.Operation{ { ClientId: 1, Input: nil, Call: 1, Output: nil, Return: 100, }, { ClientId: 1, Input: nil, Call: 101, Output: nil, Return: 200, }, { ClientId: 2, Input: nil, Call: 1, Output: nil, Return: 100, }, { ClientId: 2, Input: nil, Call: 10, Output: nil, Return: 20, }, }, expectError: "Overlapping operations, new.call(10) <= prev.return(100)", }, } for _, tc := range tcs { h := NewAppendableHistory(identity.NewIDProvider()) for _, operation := range tc.operations[:len(tc.operations)-1] { h.append(operation) } assert.PanicsWithValue(t, tc.expectError, func() { h.append(tc.operations[len(tc.operations)-1]) }) } } ================================================ FILE: tests/robustness/model/non_deterministic.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "cmp" "fmt" "reflect" "slices" "strings" "github.com/anishathalye/porcupine" ) // NonDeterministicModel extends DeterministicModel to allow for clients with imperfect knowledge of the request's destiny. // An unknown/error response doesn't inform whether the request was persisted or not, so the model // considers both cases. This is represented as multiple, equally possible deterministic states. // Failed requests fork the possible states, while successful requests merge and filter them. var NonDeterministicModel = porcupine.Model{ Init: func() any { return nonDeterministicState{freshEtcdState()} }, Step: func(st any, in any, out any) (bool, any) { return st.(nonDeterministicState).apply(in.(EtcdRequest), out.(MaybeEtcdResponse)) }, Equal: func(st1, st2 any) bool { return st1.(nonDeterministicState).Equal(st2.(nonDeterministicState)) }, DescribeOperation: func(in, out any) string { return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), out.(MaybeEtcdResponse))) }, DescribeOperationMetadata: func(info any) string { if info == nil { return "" } return DescribeOperationMetadata(info.(MaybeEtcdResponse)) }, DescribeState: func(st any) string { etcdStates := st.(nonDeterministicState) desc := make([]string, 0, len(etcdStates)) slices.SortFunc(etcdStates, func(i, j EtcdState) int { if c := cmp.Compare(i.Revision, j.Revision); c != 0 { return c } return cmp.Compare(i.CompactRevision, j.CompactRevision) }) for i, s := range etcdStates { // Describe just 3 first states before truncating if i >= 3 { desc = append(desc, "...truncated...") break } desc = append(desc, describeEtcdState(s)) } return strings.Join(desc, "\n") }, } type nonDeterministicState []EtcdState func (states nonDeterministicState) Equal(other nonDeterministicState) bool { if len(states) != len(other) { return false } otherMatched := make([]bool, len(other)) for _, sItem := range states { foundMatchInOther := false for j, otherItem := range other { if !otherMatched[j] && sItem.Equal(otherItem) { otherMatched[j] = true foundMatchInOther = true break } } if !foundMatchInOther { return false } } return true } func (states nonDeterministicState) apply(request EtcdRequest, response MaybeEtcdResponse) (bool, nonDeterministicState) { var newStates nonDeterministicState switch { case response.Error != "": newStates = states.applyFailedRequest(request) case response.Persisted && response.PersistedRevision == 0: newStates = states.applyPersistedRequest(request) case response.Persisted && response.PersistedRevision != 0: newStates = states.applyPersistedRequestWithRevision(request, response.PersistedRevision) default: newStates = states.applyRequestWithResponse(request, response.EtcdResponse) } return len(newStates) > 0, newStates } // applyFailedRequest returns both the original states and states with applied request. It considers both cases, request was persisted and request was lost. func (states nonDeterministicState) applyFailedRequest(request EtcdRequest) nonDeterministicState { newStates := make(nonDeterministicState, 0, len(states)*2) for _, s := range states { newStates = append(newStates, s) newState, _ := s.Step(request) if !reflect.DeepEqual(newState, s) { newStates = append(newStates, newState) } } return newStates } // applyPersistedRequest applies request to all possible states. func (states nonDeterministicState) applyPersistedRequest(request EtcdRequest) nonDeterministicState { newStates := make(nonDeterministicState, 0, len(states)) for _, s := range states { newState, _ := s.Step(request) newStates = append(newStates, newState) } return newStates } // applyPersistedRequestWithRevision applies request to all possible states, but leaves only states that would return proper revision. func (states nonDeterministicState) applyPersistedRequestWithRevision(request EtcdRequest, responseRevision int64) nonDeterministicState { newStates := make(nonDeterministicState, 0, len(states)) for _, s := range states { newState, modelResponse := s.Step(request) if modelResponse.Revision == responseRevision { newStates = append(newStates, newState) } } return newStates } // applyRequestWithResponse applies request to all possible states, but leaves only state that would return proper response. func (states nonDeterministicState) applyRequestWithResponse(request EtcdRequest, response EtcdResponse) nonDeterministicState { newStates := make(nonDeterministicState, 0, len(states)) for _, s := range states { newState, modelResponse := s.Step(request) if Match(modelResponse, MaybeEtcdResponse{EtcdResponse: response}) { newStates = append(newStates, newState) } } return newStates } ================================================ FILE: tests/robustness/model/non_deterministic_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "encoding/json" "errors" "testing" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/mvccpb" ) func TestModelNonDeterministic(t *testing.T) { nonDeterministicTestScenarios := append(commonTestScenarios, []modelTestCase{ { name: "First Put request fails, but is persisted", operations: []testOperation{ {req: putRequest("key1", "1"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key2", "2"), resp: putResponse(3)}, {req: listRequest("key", 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2, Version: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3, Version: 1}}, 2, 3)}, }, }, { name: "First Put request fails, and is lost", operations: []testOperation{ {req: putRequest("key1", "1"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key2", "2"), resp: putResponse(2)}, {req: listRequest("key", 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2, Version: 1}}, 1, 2)}, }, }, { name: "Put can fail and be lost before get", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))}, {req: getRequest("key"), resp: getResponse("key", "1", 2, 2)}, {req: getRequest("key"), resp: getResponse("key", "2", 2, 2), expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "1", 2, 3), expectFailure: true}, {req: getRequest("key"), resp: getResponse("key", "2", 2, 3), expectFailure: true}, }, }, { name: "Put can fail and be lost before put", operations: []testOperation{ {req: getRequest("key"), resp: emptyGetResponse(1)}, {req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "3"), resp: putResponse(2)}, }, }, { name: "Put can fail and be lost before delete", operations: []testOperation{ {req: deleteRequest("key"), resp: deleteResponse(0, 1)}, {req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: deleteResponse(0, 1)}, }, }, { name: "Put can fail and be lost before txn", operations: []testOperation{ // Txn failure {req: getRequest("key"), resp: emptyGetResponse(1)}, {req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 2, "3"), resp: compareRevisionAndPutResponse(false, 1)}, // Txn success {req: putRequest("key", "2"), resp: putResponse(2)}, {req: putRequest("key", "4"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 2, "5"), resp: compareRevisionAndPutResponse(true, 3)}, }, }, { name: "Put can fail but be persisted and increase revision before get", operations: []testOperation{ // One failed request, one persisted. {req: putRequest("key", "1"), resp: putResponse(2)}, {req: putRequest("key", "2"), resp: failedResponse(errors.New("failed"))}, {req: getRequest("key"), resp: getResponseWithVer("key", "3", 3, 2, 3), expectFailure: true}, {req: getRequest("key"), resp: getResponseWithVer("key", "3", 2, 2, 3), expectFailure: true}, {req: getRequest("key"), resp: getResponseWithVer("key", "2", 2, 2, 2), expectFailure: true}, {req: getRequest("key"), resp: getResponseWithVer("key", "2", 3, 2, 3)}, // Two failed request, two persisted. {req: putRequest("key", "3"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "4"), resp: failedResponse(errors.New("failed"))}, {req: getRequest("key"), resp: getResponseWithVer("key", "4", 5, 4, 5)}, }, }, { name: "Put can fail but be persisted and increase revision before delete", operations: []testOperation{ // One failed request, one persisted. {req: deleteRequest("key"), resp: deleteResponse(0, 1)}, {req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: deleteResponse(1, 1), expectFailure: true}, {req: deleteRequest("key"), resp: deleteResponse(1, 2), expectFailure: true}, {req: deleteRequest("key"), resp: deleteResponse(1, 3)}, // Two failed request, two persisted. {req: putRequest("key", "4"), resp: putResponse(4)}, {req: putRequest("key", "5"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "6"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: deleteResponse(1, 7)}, // Two failed request, one persisted. {req: putRequest("key", "8"), resp: putResponse(8)}, {req: putRequest("key", "9"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "10"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: deleteResponse(1, 10)}, }, }, { name: "Put can fail but be persisted before txn", operations: []testOperation{ // Txn success {req: getRequest("key"), resp: emptyGetResponse(1)}, {req: putRequest("key", "2"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 2, ""), resp: compareRevisionAndPutResponse(true, 2), expectFailure: true}, {req: compareRevisionAndPutRequest("key", 2, ""), resp: compareRevisionAndPutResponse(true, 3)}, // Txn failure {req: putRequest("key", "4"), resp: putResponse(4)}, {req: compareRevisionAndPutRequest("key", 5, ""), resp: compareRevisionAndPutResponse(false, 4)}, {req: putRequest("key", "5"), resp: failedResponse(errors.New("failed"))}, {req: getRequest("key"), resp: getResponseWithVer("key", "5", 5, 4, 5)}, }, }, { name: "Delete can fail and be lost before get", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: getRequest("key"), resp: getResponse("key", "1", 2, 2)}, {req: getRequest("key"), resp: emptyGetResponse(3), expectFailure: true}, {req: getRequest("key"), resp: emptyGetResponse(3), expectFailure: true}, {req: getRequest("key"), resp: emptyGetResponse(2), expectFailure: true}, }, }, { name: "Delete can fail and be lost before delete", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: deleteResponse(1, 2), expectFailure: true}, {req: deleteRequest("key"), resp: deleteResponse(1, 3)}, }, }, { name: "Delete can fail and be lost before put", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "1"), resp: putResponse(3)}, }, }, { name: "Delete can fail but be persisted before get", operations: []testOperation{ // One failed request, one persisted. {req: putRequest("key", "1"), resp: putResponse(2)}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: getRequest("key"), resp: emptyGetResponse(3)}, // Two failed request, one persisted. {req: putRequest("key", "3"), resp: putResponse(4)}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: getRequest("key"), resp: emptyGetResponse(5)}, }, }, { name: "Delete can fail but be persisted before put", operations: []testOperation{ // One failed request, one persisted. {req: putRequest("key", "1"), resp: putResponse(2)}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "3"), resp: putResponse(4)}, // Two failed request, one persisted. {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "5"), resp: putResponse(6)}, }, }, { name: "Delete can fail but be persisted before delete", operations: []testOperation{ // One failed request, one persisted. {req: putRequest("key", "1"), resp: putResponse(2)}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: deleteResponse(0, 3)}, {req: putRequest("key", "3"), resp: putResponse(4)}, // Two failed request, one persisted. {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: deleteResponse(0, 5)}, }, }, { name: "Delete can fail but be persisted before txn", operations: []testOperation{ // Txn success {req: putRequest("key", "1"), resp: putResponse(2)}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 0, "3"), resp: compareRevisionAndPutResponse(true, 4)}, // Txn failure {req: putRequest("key", "4"), resp: putResponse(5)}, {req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 5, "5"), resp: compareRevisionAndPutResponse(false, 6)}, }, }, { name: "Txn can fail and be lost before get", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))}, {req: getRequest("key"), resp: getResponse("key", "1", 2, 2)}, {req: getRequest("key"), resp: getResponse("key", "2", 3, 3), expectFailure: true}, }, }, { name: "Txn can fail and be lost before delete", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: deleteResponse(1, 3)}, }, }, { name: "Txn can fail and be lost before put", operations: []testOperation{ {req: putRequest("key", "1"), resp: putResponse(2)}, {req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "3"), resp: putResponse(3)}, }, }, { name: "Txn can fail but be persisted before get", operations: []testOperation{ // One failed request, one persisted. {req: putRequest("key", "1"), resp: putResponse(2)}, {req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))}, {req: getRequest("key"), resp: getResponseWithVer("key", "2", 2, 2, 2), expectFailure: true}, {req: getRequest("key"), resp: getResponseWithVer("key", "2", 3, 2, 3)}, // Two failed request, two persisted. {req: putRequest("key", "3"), resp: putResponse(4)}, {req: compareRevisionAndPutRequest("key", 4, "4"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 5, "5"), resp: failedResponse(errors.New("failed"))}, {req: getRequest("key"), resp: getResponseWithVer("key", "5", 6, 5, 6)}, }, }, { name: "Txn can fail but be persisted before put", operations: []testOperation{ // One failed request, one persisted. {req: putRequest("key", "1"), resp: putResponse(2)}, {req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "3"), resp: putResponse(4)}, // Two failed request, two persisted. {req: putRequest("key", "4"), resp: putResponse(5)}, {req: compareRevisionAndPutRequest("key", 5, "5"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 6, "6"), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "7"), resp: putResponse(8)}, }, }, { name: "Txn can fail but be persisted before delete", operations: []testOperation{ // One failed request, one persisted. {req: putRequest("key", "1"), resp: putResponse(2)}, {req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: deleteResponse(1, 4)}, // Two failed request, two persisted. {req: putRequest("key", "4"), resp: putResponse(5)}, {req: compareRevisionAndPutRequest("key", 5, "5"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 6, "6"), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: deleteResponse(1, 8)}, }, }, { name: "Txn can fail but be persisted before txn", operations: []testOperation{ // One failed request, one persisted with success. {req: putRequest("key", "1"), resp: putResponse(2)}, {req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 3, "3"), resp: compareRevisionAndPutResponse(true, 4)}, // Two failed request, two persisted with success. {req: putRequest("key", "4"), resp: putResponse(5)}, {req: compareRevisionAndPutRequest("key", 5, "5"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 6, "6"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 7, "7"), resp: compareRevisionAndPutResponse(true, 8)}, // One failed request, one persisted with failure. {req: putRequest("key", "8"), resp: putResponse(9)}, {req: compareRevisionAndPutRequest("key", 9, "9"), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 9, "10"), resp: compareRevisionAndPutResponse(false, 10)}, }, }, { name: "Defragment failures between all other request types", operations: []testOperation{ {req: defragmentRequest(), resp: failedResponse(errors.New("failed"))}, {req: leaseGrantRequest(1), resp: leaseGrantResponse(1)}, {req: defragmentRequest(), resp: failedResponse(errors.New("failed"))}, {req: putWithLeaseRequest("key", "1", 1), resp: putResponse(2)}, {req: defragmentRequest(), resp: failedResponse(errors.New("failed"))}, {req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3)}, {req: defragmentRequest(), resp: failedResponse(errors.New("failed"))}, {req: putRequest("key", "4"), resp: putResponse(4)}, {req: defragmentRequest(), resp: failedResponse(errors.New("failed"))}, {req: getRequest("key"), resp: getResponse("key", "4", 4, 4)}, {req: defragmentRequest(), resp: failedResponse(errors.New("failed"))}, {req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5)}, {req: defragmentRequest(), resp: failedResponse(errors.New("failed"))}, {req: deleteRequest("key"), resp: deleteResponse(1, 6)}, {req: defragmentRequest(), resp: failedResponse(errors.New("failed"))}, }, }, }...) for _, tc := range nonDeterministicTestScenarios { tc := tc t.Run(tc.name, func(t *testing.T) { state := NonDeterministicModel.Init() for _, op := range tc.operations { ok, newState := NonDeterministicModel.Step(state, op.req, op.resp) if ok != !op.expectFailure { t.Logf("state: %v", state) t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.expectFailure, ok, NonDeterministicModel.DescribeOperation(op.req, op.resp)) var loadedState nonDeterministicState err := json.Unmarshal([]byte(state.(string)), &loadedState) require.NoErrorf(t, err, "Failed to load state") for i, s := range loadedState { _, resp := s.Step(op.req) t.Errorf("For state %d, response diff: %s", i, cmp.Diff(op.resp, resp)) } break } if ok { state = newState t.Logf("state: %v", state) } } }) } } func TestModelResponseMatch(t *testing.T) { tcs := []struct { resp1 MaybeEtcdResponse resp2 MaybeEtcdResponse expectMatch bool }{ { resp1: getResponse("key", "a", 1, 1), resp2: getResponse("key", "a", 1, 1), expectMatch: true, }, { resp1: getResponse("key", "a", 1, 1), resp2: getResponse("key", "b", 1, 1), expectMatch: false, }, { resp1: getResponse("key", "a", 1, 1), resp2: getResponse("key", "a", 2, 1), expectMatch: false, }, { resp1: getResponse("key", "a", 1, 1), resp2: getResponse("key", "a", 1, 2), expectMatch: false, }, { resp1: getResponse("key", "a", 1, 1), resp2: failedResponse(errors.New("failed request")), expectMatch: false, }, { resp1: getResponse("key", "a", 1, 1), resp2: partialResponse(1), expectMatch: true, }, { resp1: getResponse("key", "a", 1, 1), resp2: partialResponse(2), expectMatch: false, }, { resp1: putResponse(3), resp2: putResponse(3), expectMatch: true, }, { resp1: putResponse(3), resp2: putResponse(4), expectMatch: false, }, { resp1: putResponse(3), resp2: failedResponse(errors.New("failed request")), expectMatch: false, }, { resp1: putResponse(3), resp2: partialResponse(3), expectMatch: true, }, { resp1: putResponse(3), resp2: partialResponse(1), expectMatch: false, }, { resp1: putResponse(3), resp2: partialResponse(0), expectMatch: true, }, { resp1: deleteResponse(1, 5), resp2: deleteResponse(1, 5), expectMatch: true, }, { resp1: deleteResponse(1, 5), resp2: deleteResponse(0, 5), expectMatch: false, }, { resp1: deleteResponse(1, 5), resp2: deleteResponse(1, 6), expectMatch: false, }, { resp1: deleteResponse(1, 5), resp2: failedResponse(errors.New("failed request")), expectMatch: false, }, { resp1: deleteResponse(1, 5), resp2: partialResponse(5), expectMatch: true, }, { resp1: deleteResponse(0, 5), resp2: partialResponse(4), expectMatch: false, }, { resp1: deleteResponse(0, 5), resp2: partialResponse(0), expectMatch: true, }, { resp1: deleteResponse(1, 5), resp2: partialResponse(0), expectMatch: true, }, { resp1: deleteResponse(0, 5), resp2: partialResponse(2), expectMatch: false, }, { resp1: compareRevisionAndPutResponse(false, 7), resp2: compareRevisionAndPutResponse(false, 7), expectMatch: true, }, { resp1: compareRevisionAndPutResponse(true, 7), resp2: compareRevisionAndPutResponse(false, 7), expectMatch: false, }, { resp1: compareRevisionAndPutResponse(false, 7), resp2: compareRevisionAndPutResponse(false, 8), expectMatch: false, }, { resp1: compareRevisionAndPutResponse(false, 7), resp2: failedResponse(errors.New("failed request")), expectMatch: false, }, { resp1: compareRevisionAndPutResponse(true, 7), resp2: partialResponse(7), expectMatch: true, }, { resp1: compareRevisionAndPutResponse(false, 7), resp2: partialResponse(7), expectMatch: true, }, { resp1: compareRevisionAndPutResponse(true, 7), resp2: partialResponse(4), expectMatch: false, }, { resp1: compareRevisionAndPutResponse(false, 7), resp2: partialResponse(3), expectMatch: false, }, { resp1: compareRevisionAndPutResponse(false, 7), resp2: partialResponse(0), expectMatch: true, }, { resp1: MaybeEtcdResponse{EtcdResponse: EtcdResponse{Revision: 1, Txn: &TxnResponse{Failure: false, Results: []EtcdOperationResult{{Deleted: 1}}}}}, resp2: failedResponse(errors.New("failed request")), expectMatch: false, }, { resp1: failedResponse(errors.New("failed request 1")), resp2: failedResponse(errors.New("failed request 2")), expectMatch: false, }, { resp1: failedResponse(errors.New("failed request")), resp2: failedResponse(errors.New("failed request")), expectMatch: true, }, { resp1: putResponse(2), resp2: MaybeEtcdResponse{Persisted: true}, expectMatch: true, }, { resp1: putResponse(2), resp2: MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}, expectMatch: true, }, { resp1: putResponse(2), resp2: MaybeEtcdResponse{Persisted: true, PersistedRevision: 3}, expectMatch: false, }, { resp1: putResponseWithMemberID(2, 1), resp2: putResponseWithMemberID(2, 2), expectMatch: true, }, { resp1: failedResponse(errors.New("failed request")), resp2: MaybeEtcdResponse{Persisted: true}, expectMatch: true, }, { resp1: failedResponse(errors.New("failed request")), resp2: MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}, expectMatch: true, }, { resp1: MaybeEtcdResponse{Persisted: true}, resp2: MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}, expectMatch: true, }, { resp1: MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}, resp2: MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}, expectMatch: true, }, { resp1: MaybeEtcdResponse{Persisted: true, PersistedRevision: 1}, resp2: MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}, expectMatch: false, }, } for i, tc := range tcs { assert.Equalf(t, tc.expectMatch, Match(tc.resp1, tc.resp2), "%d %+v %+v", i, tc.resp1, tc.resp2) } } ================================================ FILE: tests/robustness/model/replay.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "fmt" "sort" "strings" "github.com/anishathalye/porcupine" ) func NewReplayFromOperations(ops []porcupine.Operation) *EtcdReplay { requests := []EtcdRequest{} for _, op := range ops { requests = append(requests, op.Input.(EtcdRequest)) } return NewReplay(requests) } func NewReplay(persistedRequests []EtcdRequest) *EtcdReplay { state := freshEtcdState() // Padding for index 0 and 1, so the index matches the revision. revisionToEtcdState := []EtcdState{state, state} var events []PersistedEvent for _, request := range persistedRequests { newState, response := state.Step(request) if state.Revision != newState.Revision { revisionToEtcdState = append(revisionToEtcdState, newState) } events = append(events, toWatchEvents(&state, request, response)...) state = newState } return &EtcdReplay{ revisionToEtcdState: revisionToEtcdState, Events: events, } } type EtcdReplay struct { revisionToEtcdState []EtcdState Events []PersistedEvent } func (r *EtcdReplay) StateForRevision(revision int64) (EtcdState, error) { if int(revision) >= len(r.revisionToEtcdState) { return EtcdState{}, fmt.Errorf("requested revision %d, higher than observed in replay %d", revision, len(r.revisionToEtcdState)-1) } return r.revisionToEtcdState[revision], nil } func (r *EtcdReplay) EventsForWatch(watch WatchRequest) (events []PersistedEvent) { for _, e := range r.Events { if e.Revision < watch.Revision || !e.Match(watch) { continue } events = append(events, e) } return events } func toWatchEvents(prevState *EtcdState, request EtcdRequest, response MaybeEtcdResponse) (events []PersistedEvent) { if response.Error != "" { return events } switch request.Type { case Txn: var ops []EtcdOperation if response.Txn.Failure { ops = request.Txn.OperationsOnFailure } else { ops = request.Txn.OperationsOnSuccess } for i, op := range ops { switch op.Type { case RangeOperation: case DeleteOperation: e := PersistedEvent{ Event: Event{ Type: op.Type, Key: op.Delete.Key, }, Revision: response.Revision, } if response.Txn.Results[i].Deleted != 0 { events = append(events, e) } case PutOperation: _, leaseExists := prevState.Leases[op.Put.LeaseID] if op.Put.LeaseID != 0 && !leaseExists { break } e := PersistedEvent{ Event: Event{ Type: op.Type, Key: op.Put.Key, Value: op.Put.Value, }, Revision: response.Revision, } if _, ok := prevState.KeyValues[op.Put.Key]; !ok { e.IsCreate = true } events = append(events, e) default: panic(fmt.Sprintf("unsupported operation type: %v", op)) } } case LeaseRevoke: deletedKeys := []string{} for key := range prevState.Leases[request.LeaseRevoke.LeaseID].Keys { if _, ok := prevState.KeyValues[key]; ok { deletedKeys = append(deletedKeys, key) } } sort.Strings(deletedKeys) for _, key := range deletedKeys { e := PersistedEvent{ Event: Event{ Type: DeleteOperation, Key: key, }, Revision: response.Revision, } events = append(events, e) } } return events } type WatchEvent struct { PersistedEvent `json:",omitempty"` PrevValue *ValueRevision `json:",omitempty"` } type PersistedEvent struct { Event `json:",omitempty"` Revision int64 `json:",omitempty"` IsCreate bool `json:",omitempty"` } type Event struct { Type OperationType `json:",omitempty"` Key string `json:",omitempty"` Value ValueOrHash `json:",omitempty"` } func (e Event) Match(request WatchRequest) bool { if request.WithPrefix { return strings.HasPrefix(e.Key, request.Key) } return e.Key == request.Key } type WatchRequest struct { Key string Revision int64 WithPrefix bool WithProgressNotify bool WithPrevKV bool } ================================================ FILE: tests/robustness/model/types.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "encoding/json" "errors" "hash/fnv" "maps" "reflect" "slices" "go.etcd.io/etcd/client/pkg/v3/types" ) // RevisionForNonLinearizableResponse is a fake revision value used to // separate responses for requests that are not linearizable, // thus we need to ignore it in model when checking linearization. const RevisionForNonLinearizableResponse = -1 type RequestType string const ( Range RequestType = "range" Txn RequestType = "txn" LeaseGrant RequestType = "leaseGrant" LeaseRevoke RequestType = "leaseRevoke" Defragment RequestType = "defragment" Compact RequestType = "compact" ) type EtcdRequest struct { Type RequestType LeaseGrant *LeaseGrantRequest LeaseRevoke *LeaseRevokeRequest Range *RangeRequest Txn *TxnRequest Defragment *DefragmentRequest Compact *CompactRequest } func (r *EtcdRequest) IsRead() bool { if r.Type == Range { return true } if r.Type != Txn { return false } for _, op := range append(r.Txn.OperationsOnSuccess, r.Txn.OperationsOnFailure...) { if op.Type != RangeOperation { return false } } return true } type RangeRequest struct { RangeOptions Revision int64 } type RangeOptions struct { Start string End string Limit int64 } type PutOptions struct { Key string Value ValueOrHash LeaseID int64 } type DeleteOptions struct { Key string } type TxnRequest struct { Conditions []EtcdCondition OperationsOnSuccess []EtcdOperation OperationsOnFailure []EtcdOperation } func (txn *TxnRequest) AllOperations() []EtcdOperation { return slices.Concat(txn.OperationsOnSuccess, txn.OperationsOnFailure) } type EtcdCondition struct { Key string ExpectedRevision int64 ExpectedVersion int64 } type EtcdOperation struct { Type OperationType Range RangeOptions Put PutOptions Delete DeleteOptions } type OperationType string const ( RangeOperation OperationType = "range-operation" PutOperation OperationType = "put-operation" DeleteOperation OperationType = "delete-operation" ) type LeaseGrantRequest struct { LeaseID int64 } type LeaseRevokeRequest struct { LeaseID int64 } type DefragmentRequest struct{} // MaybeEtcdResponse extends EtcdResponse to include partial information about responses to a request. // Possible response state information: // * Normal response. Client observed response. Only EtcdResponse is set. // * Persisted. Client didn't observe response, but we know it was persisted by etcd. Only Persisted is set // * Persisted with Revision. Client didn't observe a response, but we know that it was persisted, and its revision. Both Persisted and PersistedRevision is set. // * Error response. Client observed error, but we don't know if it was persisted. Only Error is set. type MaybeEtcdResponse struct { EtcdResponse Persisted bool PersistedRevision int64 Error string } var ErrEtcdFutureRev = errors.New("future rev") type MemberID uint64 // MarshalJSON implements the json.Marshaler interface for MemberID. // It marshals the MemberID as a hexadecimal string. func (m MemberID) MarshalJSON() ([]byte, error) { return json.Marshal(types.ID(m).String()) } // UnmarshalJSON implements the json.Unmarshaler interface for MemberID. // It unmarshals the MemberID from a hexadecimal string. func (m *MemberID) UnmarshalJSON(data []byte) error { var s string if err := json.Unmarshal(data, &s); err != nil { return err } id, err := types.IDFromString(s) if err != nil { return err } *m = MemberID(id) return nil } type EtcdResponse struct { Txn *TxnResponse Range *RangeResponse LeaseGrant *LeaseGrantResponse LeaseRevoke *LeaseRevokeResponse Defragment *DefragmentResponse Compact *CompactResponse ClientError string Revision int64 MemberID MemberID `json:"member-id,omitempty"` } func Match(r1, r2 MaybeEtcdResponse) bool { r1Revision := r1.Revision if r1.Persisted { r1Revision = r1.PersistedRevision } r2Revision := r2.Revision if r2.Persisted { r2Revision = r2.PersistedRevision } r1.EtcdResponse.MemberID = 0 r2.EtcdResponse.MemberID = 0 return (r1.Persisted && r1.PersistedRevision == 0) || (r2.Persisted && r2.PersistedRevision == 0) || ((r1.Persisted || r2.Persisted) && (r1.Error != "" || r2.Error != "" || r1Revision == r2Revision)) || reflect.DeepEqual(r1, r2) } type TxnResponse struct { Failure bool Results []EtcdOperationResult } type RangeResponse struct { KVs []KeyValue Count int64 } type LeaseGrantResponse struct { LeaseID int64 } type ( LeaseRevokeResponse struct{} DefragmentResponse struct{} ) type EtcdOperationResult struct { RangeResponse Deleted int64 } type KeyValue struct { Key string ValueRevision } var leased = struct{}{} type EtcdLease struct { LeaseID int64 Keys map[string]struct{} } func (el EtcdLease) DeepCopy() EtcdLease { return EtcdLease{ LeaseID: el.LeaseID, Keys: maps.Clone(el.Keys), } } type ValueRevision struct { Value ValueOrHash `json:",omitempty"` ModRevision int64 `json:",omitempty"` Version int64 `json:",omitempty"` } type ValueOrHash struct { Value string `json:",omitempty"` Hash uint32 `json:",omitempty"` } func ToValueOrHash(value string) ValueOrHash { v := ValueOrHash{} if len(value) < 20 { v.Value = value } else { h := fnv.New32a() h.Write([]byte(value)) v.Hash = h.Sum32() } return v } type CompactResponse struct{} type CompactRequest struct { Revision int64 } ================================================ FILE: tests/robustness/model/types_test.go ================================================ // Copyright 2026 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import ( "encoding/json" "testing" "github.com/stretchr/testify/require" "go.etcd.io/etcd/client/pkg/v3/types" ) func TestMemberIDMatchesTypesID(t *testing.T) { raw := uint64(13770331908272521436) mID := MemberID(raw) jsonBytes, err := json.Marshal(mID) require.NoError(t, err) jsonStr := string(jsonBytes) tID := types.ID(raw) expectedHex := tID.String() require.JSONEq(t, "\""+expectedHex+"\"", jsonStr) } func TestMemberIDMatchesTypesIDUnmarshal(t *testing.T) { raw := uint64(13770331908272521999) mID := MemberID(raw) jsonBytes, err := json.Marshal(mID) require.NoError(t, err) tID := types.ID(raw) expectedHex := "\"" + tID.String() + "\"" require.JSONEq(t, expectedHex, string(jsonBytes)) var parsedID MemberID err = json.Unmarshal(jsonBytes, &parsedID) require.NoError(t, err) require.Equal(t, mID, parsedID) } ================================================ FILE: tests/robustness/model/watch.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package model import "time" type WatchOperation struct { Request WatchRequest `json:",omitempty"` Responses []WatchResponse `json:",omitempty"` } type WatchResponse struct { Events []WatchEvent `json:",omitempty"` IsProgressNotify bool `json:",omitempty"` Revision int64 `json:",omitempty"` Time time.Duration `json:",omitempty"` Error string `json:",omitempty"` } ================================================ FILE: tests/robustness/options/cluster_options.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package options import ( "math/rand" "time" "go.etcd.io/etcd/tests/v3/framework/e2e" ) var internalRand = rand.New(rand.NewSource(time.Now().UnixNano())) type ClusterOptions []e2e.EPClusterOption // WithClusterOptionGroups takes an array of EPClusterOption arrays and randomly picks one EPClusterOption array when constructing the config. // This function is mainly used to group strongly coupled config options together so that we can dynamically test different groups of options. func WithClusterOptionGroups(input ...ClusterOptions) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { optsPicked := input[internalRand.Intn(len(input))] for _, opt := range optsPicked { opt(c) } } } // WithSubsetOptions randomly selects a subset of input options and applies the subset to the cluster config. func WithSubsetOptions(input ...e2e.EPClusterOption) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { // selects a random subsetLen (0 to len(input)) elements from the input array. subsetLen := internalRand.Intn(len(input) + 1) perm := internalRand.Perm(len(input)) for i := 0; i < subsetLen; i++ { opt := input[perm[i]] opt(c) } } } ================================================ FILE: tests/robustness/options/cluster_options_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package options import ( "math/rand" "testing" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/tests/v3/framework/e2e" ) func mockRand(source rand.Source) func() { tmp := internalRand internalRand = rand.New(source) return func() { internalRand = tmp } } func TestWithClusterOptionGroups(t *testing.T) { restore := mockRand(rand.NewSource(1)) defer restore() tickOptions1 := ClusterOptions{WithTickMs(101), WithElectionMs(1001)} tickOptions2 := ClusterOptions{WithTickMs(202), WithElectionMs(2002)} tickOptions3 := ClusterOptions{WithTickMs(303), WithElectionMs(3003)} opts := ClusterOptions{ WithSnapshotCount(100, 150, 200), WithClusterOptionGroups(tickOptions1, tickOptions2, tickOptions3), WithSnapshotCatchUpEntries(100), } expectedServerConfigs := []embed.Config{ {SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001}, {SnapshotCount: 100, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002}, {SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002}, {SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001}, {SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 101, ElectionMs: 1001}, {SnapshotCount: 150, SnapshotCatchUpEntries: 100, TickMs: 202, ElectionMs: 2002}, } for i, tt := range expectedServerConfigs { cluster := *e2e.NewConfig(opts...) if cluster.ServerConfig.SnapshotCount != tt.SnapshotCount { t.Errorf("Test case %d: SnapshotCount = %v, want %v\n", i, cluster.ServerConfig.SnapshotCount, tt.SnapshotCount) } if cluster.ServerConfig.SnapshotCatchUpEntries != tt.SnapshotCatchUpEntries { t.Errorf("Test case %d: SnapshotCatchUpEntries = %v, want %v\n", i, cluster.ServerConfig.SnapshotCatchUpEntries, tt.SnapshotCatchUpEntries) } if cluster.ServerConfig.TickMs != tt.TickMs { t.Errorf("Test case %d: TickMs = %v, want %v\n", i, cluster.ServerConfig.TickMs, tt.TickMs) } if cluster.ServerConfig.ElectionMs != tt.ElectionMs { t.Errorf("Test case %d: ElectionMs = %v, want %v\n", i, cluster.ServerConfig.ElectionMs, tt.ElectionMs) } } } func TestWithOptionsSubset(t *testing.T) { restore := mockRand(rand.NewSource(1)) defer restore() tickOptions := ClusterOptions{WithTickMs(50), WithElectionMs(500)} opts := ClusterOptions{ WithSnapshotCatchUpEntries(100), WithSubsetOptions(WithSnapshotCount(100, 150, 200), WithClusterOptionGroups(tickOptions)), } expectedServerConfigs := []embed.Config{ {SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000}, {SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000}, {SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000}, // both SnapshotCount and TickMs&ElectionMs are not default values. {SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500}, {SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000}, // only TickMs&ElectionMs are not default values. {SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500}, // both SnapshotCount and TickMs&ElectionMs are not default values. {SnapshotCount: 200, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500}, // both SnapshotCount and TickMs&ElectionMs are not default values. {SnapshotCount: 10000, SnapshotCatchUpEntries: 100, TickMs: 50, ElectionMs: 500}, // only SnapshotCount is not default value. {SnapshotCount: 100, SnapshotCatchUpEntries: 100, TickMs: 100, ElectionMs: 1000}, } for i, tt := range expectedServerConfigs { cluster := *e2e.NewConfig(opts...) if cluster.ServerConfig.SnapshotCount != tt.SnapshotCount { t.Errorf("Test case %d: SnapshotCount = %v, want %v\n", i, cluster.ServerConfig.SnapshotCount, tt.SnapshotCount) } if cluster.ServerConfig.SnapshotCatchUpEntries != tt.SnapshotCatchUpEntries { t.Errorf("Test case %d: SnapshotCatchUpEntries = %v, want %v\n", i, cluster.ServerConfig.SnapshotCatchUpEntries, tt.SnapshotCatchUpEntries) } if cluster.ServerConfig.TickMs != tt.TickMs { t.Errorf("Test case %d: TickMs = %v, want %v\n", i, cluster.ServerConfig.TickMs, tt.TickMs) } if cluster.ServerConfig.ElectionMs != tt.ElectionMs { t.Errorf("Test case %d: ElectionMs = %v, want %v\n", i, cluster.ServerConfig.ElectionMs, tt.ElectionMs) } } } ================================================ FILE: tests/robustness/options/server_config_options.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package options import ( "time" e2e "go.etcd.io/etcd/tests/v3/framework/e2e" ) func WithSnapshotCount(input ...uint64) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { c.ServerConfig.SnapshotCount = input[internalRand.Intn(len(input))] } } func WithCompactionBatchLimit(input ...int) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { c.ServerConfig.CompactionBatchLimit = input[internalRand.Intn(len(input))] } } func WithSnapshotCatchUpEntries(input ...uint64) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { c.ServerConfig.SnapshotCatchUpEntries = input[internalRand.Intn(len(input))] } } func WithTickMs(input ...uint) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { c.ServerConfig.TickMs = input[internalRand.Intn(len(input))] } } func WithElectionMs(input ...uint) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { c.ServerConfig.ElectionMs = input[internalRand.Intn(len(input))] } } func WithWatchProgressNotifyInterval(input ...time.Duration) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { c.ServerConfig.WatchProgressNotifyInterval = input[internalRand.Intn(len(input))] } } func WithVersion(input ...e2e.ClusterVersion) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { c.Version = input[internalRand.Intn(len(input))] } } func WithInitialLeaderIndex(input ...int) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { c.InitialLeaderIndex = input[internalRand.Intn(len(input))] } } ================================================ FILE: tests/robustness/patches/beforeSendWatchResponse/build.patch ================================================ @@ -25,7 +26,7 @@ GOFAIL_VERSION=$(cd tools/mod && go list -m -f {{.Version}} go.etcd.io/gofail) toggle_failpoints() { mode="$1" if command -v gofail >/dev/null 2>&1; then - run gofail "$mode" server/etcdserver/ server/mvcc/ server/wal/ server/mvcc/backend/ + run gofail "$mode" server/etcdserver/ server/mvcc/ server/wal/ server/mvcc/backend/ server/etcdserver/api/v3rpc/ if [[ "$mode" == "enable" ]]; then go get go.etcd.io/gofail@${GOFAIL_VERSION} cd ./server && go get go.etcd.io/gofail@${GOFAIL_VERSION} ================================================ FILE: tests/robustness/patches/beforeSendWatchResponse/watch.patch ================================================ diff --git a/server/etcdserver/api/v3rpc/watch.go b/server/etcdserver/api/v3rpc/watch.go index cd834aa..e6aaf2b 100644 --- a/server/etcdserver/api/v3rpc/watch.go +++ b/server/etcdserver/api/v3rpc/watch.go @@ -460,6 +460,7 @@ func (sws *serverWatchStream) sendLoop() { sws.mu.RUnlock() var serr error + // gofail: var beforeSendWatchResponse struct{} if !fragmented && !ok { serr = sws.gRPCStream.Send(wr) } else { ================================================ FILE: tests/robustness/patches/compactBeforeSetFinishedCompact/kvstore_compaction.patch ================================================ From 6b034466aa0ac2b46fe01fb5bd2233946f46d453 Mon Sep 17 00:00:00 2001 From: Wei Fu Date: Wed, 24 Apr 2024 12:14:27 +0800 Subject: [PATCH] server/mvcc: introduce compactBeforeSetFinishedCompact failpoint Signed-off-by: Wei Fu --- server/mvcc/kvstore_compaction.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/mvcc/kvstore_compaction.go b/server/mvcc/kvstore_compaction.go index c7d343d5c..89defbd9e 100644 --- a/server/mvcc/kvstore_compaction.go +++ b/server/mvcc/kvstore_compaction.go @@ -59,6 +59,7 @@ func (s *store) scheduleCompaction(compactMainRev, prevCompactRev int64) (KeyVal } if len(keys) < s.cfg.CompactionBatchLimit { + // gofail: var compactBeforeSetFinishedCompact struct{} rbytes := make([]byte, 8+1+8) revToBytes(revision{main: compactMainRev}, rbytes) tx.UnsafePut(buckets.Meta, finishedCompactKeyName, rbytes) -- 2.34.1 ================================================ FILE: tests/robustness/random/random.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package random import ( "math/rand" "strings" ) func RandString(size int) string { data := strings.Builder{} data.Grow(size) for i := 0; i < size; i++ { data.WriteByte(byte(int('a') + rand.Intn(26))) } return data.String() } func RandRange(start, end int64) int64 { return rand.Int63n(end-start) + start } type ChoiceWeight[T any] struct { Choice T Weight int } func PickRandom[T any](choices []ChoiceWeight[T]) T { sum := 0 for _, op := range choices { sum += op.Weight } roll := rand.Int() % sum for _, op := range choices { if roll < op.Weight { return op.Choice } roll -= op.Weight } panic("unexpected") } ================================================ FILE: tests/robustness/report/client.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package report import ( "encoding/json" "fmt" "os" "path/filepath" "sort" "strconv" "strings" "github.com/anishathalye/porcupine" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/robustness/model" ) type ClientReport struct { ClientID int KeyValue []porcupine.Operation Watch []model.WatchOperation } func (r ClientReport) WatchEventCount() int { count := 0 for _, op := range r.Watch { for _, resp := range op.Responses { count += len(resp.Events) } } return count } func persistClientReports(lg *zap.Logger, path string, reports []ClientReport) error { sort.Slice(reports, func(i, j int) bool { return reports[i].ClientID < reports[j].ClientID }) for _, r := range reports { clientDir := filepath.Join(path, fmt.Sprintf("client-%d", r.ClientID)) if err := os.MkdirAll(clientDir, 0o700); err != nil { return err } if len(r.Watch) != 0 { if err := persistWatchOperations(lg, filepath.Join(clientDir, "watch.json"), r.Watch); err != nil { return err } } else { lg.Info("no watch operations for client, skip persisting", zap.Int("client-id", r.ClientID)) } if len(r.KeyValue) != 0 { if err := persistKeyValueOperations(lg, filepath.Join(clientDir, "operations.json"), r.KeyValue); err != nil { return err } } else { lg.Info("no KV operations for client, skip persisting", zap.Int("client-id", r.ClientID)) } } return nil } func LoadClientReports(path string) ([]ClientReport, error) { files, err := os.ReadDir(path) if err != nil { return nil, err } reports := []ClientReport{} for _, file := range files { if file.IsDir() && strings.HasPrefix(file.Name(), "client-") { idString := strings.Replace(file.Name(), "client-", "", 1) id, err := strconv.Atoi(idString) if err != nil { return nil, fmt.Errorf("failed to extract clientID from directory: %q", file.Name()) } r, err := loadClientReport(filepath.Join(path, file.Name())) if err != nil { return nil, err } r.ClientID = id reports = append(reports, r) } } sort.Slice(reports, func(i, j int) bool { return reports[i].ClientID < reports[j].ClientID }) return reports, nil } func loadClientReport(path string) (report ClientReport, err error) { report.Watch, err = loadWatchOperations(filepath.Join(path, "watch.json")) if err != nil { return report, err } report.KeyValue, err = loadKeyValueOperations(filepath.Join(path, "operations.json")) if err != nil { return report, err } return report, nil } func loadWatchOperations(path string) (operations []model.WatchOperation, err error) { _, err = os.Stat(path) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, fmt.Errorf("failed to open watch operation file: %q, err: %w", path, err) } file, err := os.OpenFile(path, os.O_RDONLY, 0o755) if err != nil { return nil, fmt.Errorf("failed to open watch operation file: %q, err: %w", path, err) } defer file.Close() decoder := json.NewDecoder(file) for decoder.More() { var watch model.WatchOperation err = decoder.Decode(&watch) if err != nil { return nil, fmt.Errorf("failed to decode watch operation, err: %w", err) } operations = append(operations, watch) } return operations, nil } func loadKeyValueOperations(path string) (operations []porcupine.Operation, err error) { _, err = os.Stat(path) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, fmt.Errorf("failed to open KV operation file: %q, err: %w", path, err) } file, err := os.OpenFile(path, os.O_RDONLY, 0o755) if err != nil { return nil, fmt.Errorf("failed to open KV operation file: %q, err: %w", path, err) } defer file.Close() decoder := json.NewDecoder(file) for decoder.More() { var operation struct { ClientID int Input model.EtcdRequest Call int64 Output model.MaybeEtcdResponse Return int64 } err = decoder.Decode(&operation) if err != nil { return nil, fmt.Errorf("failed to decode KV operation, err: %w", err) } operations = append(operations, porcupine.Operation{ ClientId: operation.ClientID, Input: operation.Input, Call: operation.Call, Output: operation.Output, Return: operation.Return, Metadata: operation.Output, }) } return operations, nil } func persistWatchOperations(lg *zap.Logger, path string, responses []model.WatchOperation) error { lg.Info("Saving watch operations", zap.String("path", path)) file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o755) if err != nil { return fmt.Errorf("failed to save watch operations: %w", err) } defer file.Close() for _, resp := range responses { data, err := json.MarshalIndent(resp, "", " ") if err != nil { return fmt.Errorf("failed to encode operation: %w", err) } file.Write(data) file.WriteString("\n") } return nil } func persistKeyValueOperations(lg *zap.Logger, path string, operations []porcupine.Operation) error { lg.Info("Saving operation history", zap.String("path", path)) file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o755) if err != nil { return fmt.Errorf("Failed to save KV operation history: %w", err) } defer file.Close() for _, op := range operations { data, err := json.MarshalIndent(op, "", " ") if err != nil { return fmt.Errorf("Failed to encode KV operation: %w", err) } file.Write(data) file.WriteString("\n") } return nil } func OperationsMaxRevision(reports []ClientReport) int64 { var maxRevision int64 for _, r := range reports { for _, op := range r.KeyValue { resp := op.Output.(model.MaybeEtcdResponse) if resp.Revision > maxRevision { maxRevision = resp.Revision } } } return maxRevision } ================================================ FILE: tests/robustness/report/client_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package report import ( "errors" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/model" ) func TestPersistLoadClientReports(t *testing.T) { h := model.NewAppendableHistory(identity.NewIDProvider()) baseTime := time.Now() start := time.Since(baseTime) time.Sleep(time.Nanosecond) stop := time.Since(baseTime) h.AppendRange("key", "", 0, 0, start, stop, &clientv3.GetResponse{Header: &etcdserverpb.ResponseHeader{Revision: 2}, Count: 2, Kvs: []*mvccpb.KeyValue{{ Key: []byte("key"), ModRevision: 2, Value: []byte("value"), }}}, nil) start = time.Since(baseTime) time.Sleep(time.Nanosecond) stop = time.Since(baseTime) h.AppendPut("key1", "1", start, stop, &clientv3.PutResponse{Header: &etcdserverpb.ResponseHeader{Revision: 2}}, nil) start = time.Since(baseTime) time.Sleep(time.Nanosecond) stop = time.Since(baseTime) h.AppendPut("key", "value", start, stop, nil, errors.New("failed")) start = time.Since(baseTime) time.Sleep(time.Nanosecond) stop = time.Since(baseTime) h.AppendPutWithLease("key1", "1", 1, start, stop, &clientv3.PutResponse{Header: &etcdserverpb.ResponseHeader{Revision: 2}}, nil) start = time.Since(baseTime) time.Sleep(time.Nanosecond) stop = time.Since(baseTime) h.AppendLeaseGrant(start, stop, &clientv3.LeaseGrantResponse{ID: 1, ResponseHeader: &etcdserverpb.ResponseHeader{Revision: 2}}, nil) start = time.Since(baseTime) time.Sleep(time.Nanosecond) stop = time.Since(baseTime) h.AppendLeaseRevoke(1, start, stop, &clientv3.LeaseRevokeResponse{Header: &etcdserverpb.ResponseHeader{Revision: 2}}, nil) start = time.Since(baseTime) time.Sleep(time.Nanosecond) stop = time.Since(baseTime) h.AppendDelete("key", start, stop, &clientv3.DeleteResponse{Deleted: 1, Header: &etcdserverpb.ResponseHeader{Revision: 3}}, nil) start = time.Since(baseTime) time.Sleep(time.Nanosecond) stop = time.Since(baseTime) h.AppendTxn([]clientv3.Cmp{clientv3.Compare(clientv3.ModRevision("key"), "=", 2)}, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{clientv3.OpDelete("key")}, start, stop, &clientv3.TxnResponse{Header: &etcdserverpb.ResponseHeader{Revision: 2}}, nil) start = time.Since(baseTime) time.Sleep(time.Nanosecond) stop = time.Since(baseTime) h.AppendDefragment(start, stop, &clientv3.DefragmentResponse{Header: &etcdserverpb.ResponseHeader{Revision: 2}}, nil) watch := model.WatchOperation{ Request: model.WatchRequest{ Key: "key", Revision: 0, WithPrefix: true, WithProgressNotify: false, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ { PersistedEvent: model.PersistedEvent{ Event: model.Event{ Type: model.PutOperation, Key: "key1", Value: model.ToValueOrHash("1"), }, Revision: 2, }, }, { PersistedEvent: model.PersistedEvent{ Event: model.Event{ Type: model.DeleteOperation, Key: "key2", }, Revision: 3, }, }, }, IsProgressNotify: false, Revision: 3, Time: 100, }, }, } reports := []ClientReport{ { ClientID: 1, KeyValue: h.Operations(), Watch: []model.WatchOperation{watch}, }, { ClientID: 2, KeyValue: nil, Watch: []model.WatchOperation{watch}, }, } path := t.TempDir() err := persistClientReports(zaptest.NewLogger(t), path, reports) require.NoError(t, err) got, err := LoadClientReports(path) require.NoError(t, err) if diff := cmp.Diff(reports, got, cmpopts.EquateEmpty()); diff != "" { t.Errorf("Reports don't match after persist and load, %s", diff) } } ================================================ FILE: tests/robustness/report/failpoint.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package report import ( "time" ) type FailpointReport struct { FailpointInjection Client []ClientReport } type FailpointInjection struct { Start, End time.Duration Name string } ================================================ FILE: tests/robustness/report/report.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package report import ( "encoding/json" "fmt" "os" "path" "path/filepath" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/framework/e2e" ) type TestReport struct { Logger *zap.Logger Cluster *e2e.EtcdProcessCluster ServersDataPath map[string]string Client []ClientReport Visualize func(lg *zap.Logger, path string) error Traffic *TrafficDetail } func (r *TestReport) Report(path string) error { r.Logger.Info("Saving robustness test report", zap.String("path", path)) err := os.RemoveAll(path) if err != nil { r.Logger.Error("Failed to remove report dir", zap.Error(err)) } for server, dataPath := range r.ServersDataPath { serverReportPath := filepath.Join(path, fmt.Sprintf("server-%s", server)) r.Logger.Info("Saving member data dir", zap.String("member", server), zap.String("data-dir", dataPath), zap.String("path", serverReportPath)) if err := os.CopyFS(serverReportPath, os.DirFS(dataPath)); err != nil { return err } } if r.Client != nil { if err := persistClientReports(r.Logger, path, r.Client); err != nil { return err } } if r.Traffic != nil { if err := persistTrafficDetail(r.Logger, path, *r.Traffic); err != nil { return err } } if r.Visualize != nil { if err := r.Visualize(r.Logger, filepath.Join(path, "history.html")); err != nil { return err } } return nil } func ServerDataPaths(c *e2e.EtcdProcessCluster) map[string]string { dataPaths := make(map[string]string) for _, member := range c.Procs { dataPaths[member.Config().Name] = memberDataDir(member) } return dataPaths } func memberDataDir(member e2e.EtcdProcess) string { lazyFS := member.LazyFS() if lazyFS != nil { return filepath.Join(lazyFS.LazyFSDir, "data") } return member.Config().DataDirPath } type TrafficDetail struct { ExpectUniqueRevision bool `json:"expectuniquerevision,omitempty"` } const trafficDetailFileName = "traffic.json" func persistTrafficDetail(lg *zap.Logger, p string, td TrafficDetail) error { lg.Info("Saving traffic configuration details", zap.String("path", path.Join(p, trafficDetailFileName))) b, err := json.Marshal(td) if err != nil { return nil } return os.WriteFile(filepath.Join(p, trafficDetailFileName), b, 0o644) } func LoadTrafficDetail(p string) (TrafficDetail, error) { var detail TrafficDetail b, err := os.ReadFile(filepath.Join(p, trafficDetailFileName)) if err != nil { return TrafficDetail{}, err } err = json.Unmarshal(b, &detail) return detail, err } ================================================ FILE: tests/robustness/report/wal.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package report import ( "errors" "fmt" "io" "math" "os" "path/filepath" "reflect" "strings" "go.uber.org/zap" pb "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/server/v3/storage/datadir" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/model" "go.etcd.io/raft/v3/raftpb" ) func LoadClusterPersistedRequests(lg *zap.Logger, path string) ([]model.EtcdRequest, error) { files, err := os.ReadDir(path) if err != nil { return nil, err } dataDirs := []string{} for _, file := range files { if file.IsDir() && strings.HasPrefix(file.Name(), "server-") { dataDirs = append(dataDirs, filepath.Join(path, file.Name())) } } return PersistedRequests(lg, dataDirs) } func PersistedRequestsCluster(lg *zap.Logger, cluster *e2e.EtcdProcessCluster) ([]model.EtcdRequest, error) { dataDirs := []string{} for _, proc := range cluster.Procs { dataDirs = append(dataDirs, memberDataDir(proc)) } return PersistedRequests(lg, dataDirs) } func PersistedRequests(lg *zap.Logger, dataDirs []string) ([]model.EtcdRequest, error) { if len(dataDirs) == 0 { return nil, errors.New("no data dirs") } entriesPersistedInWAL := make([][]raftpb.Entry, len(dataDirs)) var minCommitIndex uint64 = math.MaxUint64 for i, dir := range dataDirs { state, entries, err := ReadWAL(lg, dir) if err != nil { lg.Error("Failed to read WAL", zap.Error(err), zap.String("data-dir", dir)) continue } minCommitIndex = min(minCommitIndex, state.Commit) entriesPersistedInWAL[i] = entries } entries, err := mergeMembersEntries(minCommitIndex, entriesPersistedInWAL) if err != nil { return nil, err } persistedRequests := make([]model.EtcdRequest, 0, len(entries)) for _, e := range entries { if e.Type != raftpb.EntryNormal { continue } request, err := parseEntryNormal(e) if err != nil { return nil, err } if request != nil { persistedRequests = append(persistedRequests, *request) } } return persistedRequests, nil } func mergeMembersEntries(minCommitIndex uint64, memberEntries [][]raftpb.Entry) ([]raftpb.Entry, error) { for _, entries := range memberEntries { var lastIndex uint64 for _, e := range entries { if e.Index <= lastIndex { return nil, fmt.Errorf("raft index should increase, got: %d, previous: %d", e.Index, lastIndex) } lastIndex = e.Index } } memberIndices := make([]int, len(memberEntries)) mergedHistory := []raftpb.Entry{} var raftIndex uint64 for { // Find entry with raftIndex. raftIndex++ entriesLeft := false for i, entries := range memberEntries { memberIndex := memberIndices[i] for memberIndex < len(entries) && entries[memberIndex].Index < raftIndex { memberIndex++ } if memberIndex < len(entries) { entriesLeft = true } memberIndices[i] = memberIndex } if !entriesLeft { break } // Entries collects votes from matching entries. votes := make([]int, len(memberEntries)) for i := 0; i < len(memberEntries); i++ { if len(memberEntries[i]) <= memberIndices[i] { continue } entry1 := memberEntries[i][memberIndices[i]] if entry1.Index != raftIndex { continue } for j := i; j < len(memberEntries); j++ { if i == j { votes[i]++ continue } if len(memberEntries[j]) <= memberIndices[j] { continue } entry2 := memberEntries[j][memberIndices[j]] if entry2.Index != raftIndex { continue } if reflect.DeepEqual(entry1, entry2) { votes[i]++ votes[j]++ } } } // Select entry with most votes topVotes := 0 for _, vote := range votes { if vote > topVotes { topVotes = vote } } if topVotes == 0 { return nil, fmt.Errorf("no entry for raft index %d", raftIndex) } var entryWithMostVotes *raftpb.Entry for i, vote := range votes { if vote != topVotes { continue } entry := memberEntries[i][memberIndices[i]] if entryWithMostVotes == nil { entryWithMostVotes = &entry continue } if entryWithMostVotes.Term != entry.Term && entry.Index > minCommitIndex { if entryWithMostVotes.Term < entry.Term { entryWithMostVotes = &entry } continue } if !reflect.DeepEqual(*entryWithMostVotes, entry) { return nil, fmt.Errorf("mismatching entries on raft index %d, mostVotes: %+v, other: %+v", raftIndex, *entryWithMostVotes, entry) } } mergedHistory = append(mergedHistory, *entryWithMostVotes) } if len(mergedHistory) == 0 { return nil, errors.New("no WAL entries matched") } return mergedHistory, nil } func ReadWAL(lg *zap.Logger, dataDir string) (state raftpb.HardState, ents []raftpb.Entry, err error) { walDir := datadir.ToWALDir(dataDir) repaired := false for { state, ents, err = ReadAllWALEntries(lg, walDir) if err != nil { // we can only repair ErrUnexpectedEOF and we never repair twice. if repaired || !errors.Is(err, io.ErrUnexpectedEOF) { return state, nil, fmt.Errorf("failed to read WAL, cannot be repaired, err: %w", err) } if !wal.Repair(lg, walDir) { return state, nil, fmt.Errorf("failed to repair WAL, err: %w", err) } lg.Info("repaired WAL", zap.Error(err)) repaired = true continue } return state, ents, nil } } func parseEntryNormal(ent raftpb.Entry) (*model.EtcdRequest, error) { var raftReq pb.InternalRaftRequest if len(ent.Data) == 0 { return nil, nil } if err := raftReq.Unmarshal(ent.Data); err != nil { // PR https://github.com/etcd-io/etcd/pull/21263 removed v2 requests // in etcd v3.7. However, robustness always uses the latest protobuf // definitions to parse WAL entries generated by all previous versions. // etcd v3.4 and v3.5 generate v2 requests during bootstrap, which the // v3.7 protobuf can no longer parse. As a result, robustness fails when // parsing those WAL entries. We intentionally ignore this error here. // See https://github.com/etcd-io/etcd/pull/21263#discussion_r2776042340 if strings.Contains(err.Error(), "proto: wrong wireType") { return nil, nil } return nil, err } switch { case raftReq.Put != nil: op := model.PutOptions{ Key: string(raftReq.Put.Key), Value: model.ToValueOrHash(string(raftReq.Put.Value)), LeaseID: raftReq.Put.Lease, } request := model.EtcdRequest{ Type: model.Txn, Txn: &model.TxnRequest{ OperationsOnSuccess: []model.EtcdOperation{ {Type: model.PutOperation, Put: op}, }, }, } return &request, nil case raftReq.DeleteRange != nil: op := model.DeleteOptions{Key: string(raftReq.DeleteRange.Key)} request := model.EtcdRequest{ Type: model.Txn, Txn: &model.TxnRequest{ OperationsOnSuccess: []model.EtcdOperation{ {Type: model.DeleteOperation, Delete: op}, }, }, } return &request, nil case raftReq.LeaseRevoke != nil: return &model.EtcdRequest{ Type: model.LeaseRevoke, LeaseRevoke: &model.LeaseRevokeRequest{LeaseID: raftReq.LeaseRevoke.ID}, }, nil case raftReq.LeaseGrant != nil: return &model.EtcdRequest{ Type: model.LeaseGrant, LeaseGrant: &model.LeaseGrantRequest{LeaseID: raftReq.LeaseGrant.ID}, }, nil case raftReq.ClusterMemberAttrSet != nil: return nil, nil case raftReq.ClusterVersionSet != nil: return nil, nil case raftReq.DowngradeInfoSet != nil: return nil, nil case raftReq.Compaction != nil: request := model.EtcdRequest{ Type: model.Compact, Compact: &model.CompactRequest{Revision: raftReq.Compaction.Revision}, } return &request, nil case raftReq.Txn != nil: txn := model.TxnRequest{ Conditions: []model.EtcdCondition{}, OperationsOnSuccess: []model.EtcdOperation{}, OperationsOnFailure: []model.EtcdOperation{}, } for _, cmp := range raftReq.Txn.Compare { switch { case cmp.Result == pb.Compare_EQUAL && cmp.Target == pb.Compare_VERSION: txn.Conditions = append(txn.Conditions, model.EtcdCondition{ Key: string(cmp.Key), ExpectedVersion: cmp.GetVersion(), }) case cmp.Result == pb.Compare_EQUAL && cmp.Target == pb.Compare_MOD: txn.Conditions = append(txn.Conditions, model.EtcdCondition{ Key: string(cmp.Key), ExpectedRevision: cmp.GetModRevision(), }) default: panic(fmt.Sprintf("unsupported condition: %+v", cmp)) } } for _, op := range raftReq.Txn.Success { txn.OperationsOnSuccess = append(txn.OperationsOnSuccess, toEtcdOperation(op)) } for _, op := range raftReq.Txn.Failure { txn.OperationsOnFailure = append(txn.OperationsOnFailure, toEtcdOperation(op)) } request := model.EtcdRequest{ Type: model.Txn, Txn: &txn, } return &request, nil default: panic(fmt.Sprintf("Unhandled raft request: %+v", raftReq)) } } func toEtcdOperation(op *pb.RequestOp) (operation model.EtcdOperation) { switch { case op.GetRequestRange() != nil: rangeOp := op.GetRequestRange() operation = model.EtcdOperation{ Type: model.RangeOperation, Range: model.RangeOptions{ Start: string(rangeOp.Key), End: string(rangeOp.RangeEnd), Limit: rangeOp.Limit, }, } case op.GetRequestPut() != nil: putOp := op.GetRequestPut() operation = model.EtcdOperation{ Type: model.PutOperation, Put: model.PutOptions{ Key: string(putOp.Key), Value: model.ToValueOrHash(string(putOp.Value)), }, } case op.GetRequestDeleteRange() != nil: deleteOp := op.GetRequestDeleteRange() operation = model.EtcdOperation{ Type: model.DeleteOperation, Delete: model.DeleteOptions{ Key: string(deleteOp.Key), }, } default: panic(fmt.Sprintf("Unknown op type %v", op)) } return operation } func ReadAllWALEntries(lg *zap.Logger, dirpath string) (state raftpb.HardState, ents []raftpb.Entry, err error) { names, err := fileutil.ReadDir(dirpath) if err != nil { return state, nil, err } files := make([]fileutil.FileReader, 0, len(names)) for _, name := range names { if !strings.HasSuffix(name, ".wal") { continue } p := filepath.Join(dirpath, name) var f *os.File f, err = os.OpenFile(p, os.O_RDONLY, fileutil.PrivateFileMode) if err != nil { return state, nil, fmt.Errorf("os.OpenFile failed (%q): %w", p, err) } defer f.Close() files = append(files, fileutil.NewFileReader(f)) } rec := &walpb.Record{} decoder := wal.NewDecoder(files...) for err = decoder.Decode(rec); err == nil; err = decoder.Decode(rec) { switch rec.GetType() { case wal.EntryType: e := wal.MustUnmarshalEntry(rec.Data) i := len(ents) for ; i > 0 && ents[i-1].Index >= e.Index; i-- { } // The line below is potentially overriding some 'uncommitted' entries. ents = append(ents[:i], e) case wal.StateType: state = wal.MustUnmarshalState(rec.Data) case wal.MetadataType: case wal.CrcType: crc := decoder.LastCRC() // current crc of decoder must match the crc of the record. // do no need to match 0 crc, since the decoder is a new one at this case. if crc != 0 && rec.Validate(crc) != nil { state.Reset() return state, nil, wal.ErrCRCMismatch } decoder.UpdateCRC(rec.GetCrc()) case wal.SnapshotType: default: return state, nil, fmt.Errorf("unexpected block type %d", rec.Type) } } if err != nil && !errors.Is(err, io.EOF) { return state, nil, err } return state, ents, nil } ================================================ FILE: tests/robustness/report/wal_test.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package report import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) func TestMergeMemberEntries(t *testing.T) { tcs := []struct { name string minCommitIndex uint64 memberEntries [][]raftpb.Entry expectErr string expectEntries []raftpb.Entry }{ { name: "Error when empty data dir", memberEntries: [][]raftpb.Entry{}, expectErr: "no WAL entries matched", }, { name: "Success when no entries", memberEntries: [][]raftpb.Entry{ {}, }, expectErr: "no WAL entries matched", }, { name: "Error when one member cluster didn't observe the index", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, }, expectErr: "no entry for raft index 2", }, { name: "Error when entries index unordered", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 3, Data: []byte("c")}, raftpb.Entry{Index: 1, Data: []byte("a")}, }, }, expectErr: "raft index should increase, got: 1, previous: 3", }, { name: "Error when entries index duplicated", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 1, Data: []byte("a")}, }, }, expectErr: "raft index should increase, got: 1, previous: 1", }, { name: "Success when one member cluster", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, }, expectEntries: []raftpb.Entry{ {Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, }, }, { name: "Success when three members agree on entries", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, }, expectEntries: []raftpb.Entry{ {Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, }, }, { name: "Success when three members have no entries", memberEntries: [][]raftpb.Entry{ {}, {}, {}, }, expectErr: "no WAL entries matched", }, { name: "Success when one member has no entries in three node cluster", memberEntries: [][]raftpb.Entry{ {}, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, }, expectEntries: []raftpb.Entry{ {Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, }, }, { name: "Success if two members have no entries in three node cluster", memberEntries: [][]raftpb.Entry{ {}, {}, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, }, expectEntries: []raftpb.Entry{ {Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, }, }, { name: "Success if members didn't observe the whole history", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, }, { raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 3, Data: []byte("c")}, }, }, expectEntries: []raftpb.Entry{ {Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, }, }, { name: "Success if members observed only one part of history", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Data: []byte("a")}, }, { raftpb.Entry{Index: 2, Data: []byte("b")}, }, { raftpb.Entry{Index: 3, Data: []byte("c")}, }, }, expectEntries: []raftpb.Entry{ {Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, }, }, { name: "Error when in three member cluster if no members observed index", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, }, expectErr: "no entry for raft index 2", }, { name: "Success if only one member observed history", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, {}, {}, }, expectEntries: []raftpb.Entry{ {Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, }, }, { name: "Success when one member observed different last entry", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("x")}, }, }, expectEntries: []raftpb.Entry{ {Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, }, }, { name: "Error when one member didn't observe whole history and others observed different last entry", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, }, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("x")}, }, }, expectErr: "mismatching entries on raft index 3", }, { name: "Error when three members observed different last entry", memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("x")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("y")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Data: []byte("z")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, }, expectErr: "mismatching entries on raft index 2", }, { name: "Error when one member observed empty history and others differ on last entry", memberEntries: [][]raftpb.Entry{ {}, { raftpb.Entry{Index: 1, Data: []byte("x")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, { raftpb.Entry{Index: 1, Data: []byte("y")}, raftpb.Entry{Index: 2, Data: []byte("b")}, raftpb.Entry{Index: 3, Data: []byte("c")}, }, }, expectErr: "mismatching entries on raft index 1", }, { name: "Error if entries mismatch on index before minCommitIndex", minCommitIndex: 2, memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Term: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Term: 1, Data: []byte("b")}, }, { raftpb.Entry{Index: 1, Term: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Term: 2, Data: []byte("c")}, }, }, expectErr: "mismatching entries on raft index 2", }, { name: "Select entry with higher term if they conflict on uncommitted index", minCommitIndex: 1, memberEntries: [][]raftpb.Entry{ { raftpb.Entry{Index: 1, Term: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Term: 1, Data: []byte("b")}, }, { raftpb.Entry{Index: 1, Term: 1, Data: []byte("a")}, raftpb.Entry{Index: 2, Term: 2, Data: []byte("x")}, }, }, expectEntries: []raftpb.Entry{ {Index: 1, Term: 1, Data: []byte("a")}, {Index: 2, Term: 2, Data: []byte("x")}, }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { entries, err := mergeMembersEntries(tc.minCommitIndex, tc.memberEntries) if tc.expectErr == "" { require.NoError(t, err) } else { require.ErrorContains(t, err, tc.expectErr) } require.Equal(t, tc.expectEntries, entries) }) } } func TestWriteReadWAL(t *testing.T) { type batch struct { state *raftpb.HardState entries []raftpb.Entry snapshot *walpb.Snapshot } type want struct { wantState raftpb.HardState wantEntries []raftpb.Entry wantError string } tcs := []struct { name string operations []batch readAt walpb.Snapshot walReadAll want readAllEntries want }{ { name: "single batch", operations: []batch{ { state: &raftpb.HardState{Commit: 5}, entries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, }, walReadAll: want{ wantState: raftpb.HardState{Commit: 5}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, readAllEntries: want{ wantState: raftpb.HardState{Commit: 5}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, }, { name: "multiple committed batches", operations: []batch{ { state: &raftpb.HardState{Commit: 2}, entries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}}, }, { state: &raftpb.HardState{Commit: 4}, entries: []raftpb.Entry{{Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("d")}}, }, { state: &raftpb.HardState{Commit: 5}, entries: []raftpb.Entry{{Index: 5, Data: []byte("e")}}, }, }, walReadAll: want{ wantState: raftpb.HardState{Commit: 5}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, readAllEntries: want{ wantState: raftpb.HardState{Commit: 5}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, }, { name: "uncommitted ovewritten entries", operations: []batch{ { state: &raftpb.HardState{Commit: 1}, entries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("a")}}, }, { state: &raftpb.HardState{Commit: 3}, entries: []raftpb.Entry{{Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("b")}, {Index: 4, Data: []byte("b")}}, }, { state: &raftpb.HardState{Commit: 4}, entries: []raftpb.Entry{{Index: 4, Data: []byte("c")}, {Index: 5, Data: []byte("c")}}, }, }, walReadAll: want{ wantState: raftpb.HardState{Commit: 4}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("b")}, {Index: 4, Data: []byte("c")}, {Index: 5, Data: []byte("c")}}, }, readAllEntries: want{ wantState: raftpb.HardState{Commit: 4}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("b")}, {Index: 4, Data: []byte("c")}, {Index: 5, Data: []byte("c")}}, }, }, { name: "entries in bad order", operations: []batch{ { state: &raftpb.HardState{Commit: 2}, entries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}}, }, { state: &raftpb.HardState{Commit: 6}, entries: []raftpb.Entry{{Index: 5, Data: []byte("e")}, {Index: 6, Data: []byte("f")}}, }, { state: &raftpb.HardState{Commit: 4}, entries: []raftpb.Entry{{Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("d")}}, }, }, walReadAll: want{ wantError: "slice bounds out of range", wantState: raftpb.HardState{Commit: 2}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}}, }, readAllEntries: want{ wantState: raftpb.HardState{Commit: 4}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("d")}}, }, }, { name: "read before snapshot", operations: []batch{ { state: &raftpb.HardState{Commit: 1}, entries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}}, }, { snapshot: &walpb.Snapshot{Index: new(uint64(3)), Term: new(uint64(0)), ConfState: &raftpb.ConfState{}}, }, { state: &raftpb.HardState{Commit: 5}, entries: []raftpb.Entry{{Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, }, walReadAll: want{ wantError: "slice bounds out of range", wantState: raftpb.HardState{Commit: 1}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}}, }, readAllEntries: want{ wantState: raftpb.HardState{Commit: 5}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, }, { name: "read at snapshot", operations: []batch{ { state: &raftpb.HardState{Commit: 1}, entries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}}, }, { snapshot: &walpb.Snapshot{Index: new(uint64(3)), Term: new(uint64(0)), ConfState: &raftpb.ConfState{}}, }, { state: &raftpb.HardState{Commit: 5}, entries: []raftpb.Entry{{Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, }, readAt: walpb.Snapshot{Index: new(uint64(3))}, walReadAll: want{ wantState: raftpb.HardState{Commit: 5}, wantEntries: []raftpb.Entry{{Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, readAllEntries: want{ wantState: raftpb.HardState{Commit: 5}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, }, { name: "uncommitted entries before snapshot", operations: []batch{ { state: &raftpb.HardState{Commit: 1}, entries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}}, }, { state: &raftpb.HardState{Commit: 3}, entries: []raftpb.Entry{{Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("d")}}, }, { snapshot: &walpb.Snapshot{Index: new(uint64(3)), Term: new(uint64(0)), ConfState: &raftpb.ConfState{}}, }, { state: &raftpb.HardState{Commit: 4}, entries: []raftpb.Entry{{Index: 4, Data: []byte("e")}, {Index: 5, Data: []byte("f")}}, }, }, walReadAll: want{ wantState: raftpb.HardState{Commit: 4}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("e")}, {Index: 5, Data: []byte("f")}}, }, readAllEntries: want{ wantState: raftpb.HardState{Commit: 4}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("e")}, {Index: 5, Data: []byte("f")}}, }, }, { name: "entries preceding snapshot", operations: []batch{ { snapshot: &walpb.Snapshot{Index: new(uint64(4)), Term: new(uint64(0)), ConfState: &raftpb.ConfState{}}, }, { state: &raftpb.HardState{Commit: 2}, entries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}}, }, { state: &raftpb.HardState{Commit: 4}, entries: []raftpb.Entry{{Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("d")}}, }, { state: &raftpb.HardState{Commit: 6}, entries: []raftpb.Entry{{Index: 5, Data: []byte("e")}, {Index: 6, Data: []byte("f")}}, }, }, readAt: walpb.Snapshot{Index: new(uint64(4))}, walReadAll: want{ wantState: raftpb.HardState{Commit: 6}, wantEntries: []raftpb.Entry{{Index: 5, Data: []byte("e")}, {Index: 6, Data: []byte("f")}}, }, readAllEntries: want{ wantState: raftpb.HardState{Commit: 6}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 3, Data: []byte("c")}, {Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}, {Index: 6, Data: []byte("f")}}, }, }, { name: "read after snapshot", operations: []batch{ { state: &raftpb.HardState{Commit: 1}, entries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}}, }, { snapshot: &walpb.Snapshot{Index: new(uint64(3)), Term: new(uint64(0)), ConfState: &raftpb.ConfState{}}, }, { state: &raftpb.HardState{Commit: 5}, entries: []raftpb.Entry{{Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, }, readAt: walpb.Snapshot{Index: new(uint64(4))}, walReadAll: want{ wantError: "snapshot not found", wantState: raftpb.HardState{Commit: 5}, wantEntries: []raftpb.Entry{{Index: 5, Data: []byte("e")}}, }, readAllEntries: want{ wantState: raftpb.HardState{Commit: 5}, wantEntries: []raftpb.Entry{{Index: 1, Data: []byte("a")}, {Index: 2, Data: []byte("b")}, {Index: 4, Data: []byte("d")}, {Index: 5, Data: []byte("e")}}, }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { dir := t.TempDir() lg := zaptest.NewLogger(t) w, err := wal.Create(lg, dir, nil) require.NoError(t, err) for _, op := range tc.operations { if op.state != nil { err = w.Save(*op.state, op.entries) require.NoError(t, err) } if op.snapshot != nil { err = w.SaveSnapshot(*op.snapshot) require.NoError(t, err) } } w.Close() w2, err := wal.OpenForRead(lg, dir, tc.readAt) require.NoError(t, err) defer w2.Close() t.Run("wal.ReadAll", func(t *testing.T) { _, state, entries, err := w2.ReadAll() if tc.walReadAll.wantError != "" { require.ErrorContains(t, err, tc.walReadAll.wantError) } else { require.NoError(t, err) } assert.Equal(t, tc.walReadAll.wantState, state) assert.Equal(t, tc.walReadAll.wantEntries, entries) }) t.Run("ReadAllEntries", func(t *testing.T) { state, entries, err := ReadAllWALEntries(lg, dir) if tc.readAllEntries.wantError != "" { require.ErrorContains(t, err, tc.walReadAll.wantError) } else { require.NoError(t, err) } assert.Equal(t, tc.readAllEntries.wantState, state) assert.Equal(t, tc.readAllEntries.wantEntries, entries) }) }) } } ================================================ FILE: tests/robustness/scenarios/scenarios.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package scenarios import ( "os" "path/filepath" "testing" "time" "github.com/stretchr/testify/require" "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/client" "go.etcd.io/etcd/tests/v3/robustness/failpoint" "go.etcd.io/etcd/tests/v3/robustness/options" "go.etcd.io/etcd/tests/v3/robustness/random" "go.etcd.io/etcd/tests/v3/robustness/traffic" ) type TrafficProfile struct { Name string Traffic traffic.Traffic Profile traffic.Profile } var trafficProfiles = []TrafficProfile{ { Name: "EtcdHighTraffic", Traffic: traffic.EtcdPut, Profile: traffic.Profile{ KeyValue: &traffic.KeyValueHigh, Watch: &traffic.WatchDefault, Compaction: &traffic.CompactionDefault, }, }, { Name: "EtcdTrafficDeleteLeases", Traffic: traffic.EtcdPutDeleteLease, Profile: traffic.Profile{ KeyValue: &traffic.KeyValueMedium, Watch: &traffic.WatchDefault, Compaction: &traffic.CompactionDefault, }, }, { Name: "KubernetesHighTraffic", Traffic: traffic.Kubernetes, Profile: traffic.Profile{ KeyValue: &traffic.KeyValueHigh, Watch: &traffic.WatchDefault, Compaction: &traffic.CompactionDefault, }, }, { Name: "KubernetesLowTraffic", Traffic: traffic.Kubernetes, Profile: traffic.Profile{ KeyValue: &traffic.KeyValueMedium, Watch: &traffic.WatchDefault, Compaction: &traffic.CompactionDefault, }, }, } type TestScenario struct { Name string Failpoint failpoint.Failpoint Cluster e2e.EtcdProcessClusterConfig Traffic traffic.Traffic Profile traffic.Profile Watch client.WatchConfig } func Exploratory(_ *testing.T) []TestScenario { randomizableOptions := []e2e.EPClusterOption{ options.WithClusterOptionGroups( options.ClusterOptions{options.WithTickMs(29), options.WithElectionMs(271)}, options.ClusterOptions{options.WithTickMs(101), options.WithElectionMs(521)}, options.ClusterOptions{options.WithTickMs(100), options.WithElectionMs(2000)}), } mixedVersionOptionChoices := []random.ChoiceWeight[options.ClusterOptions]{ // 60% with all members of current version {Choice: options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, Weight: 60}, // 10% with 2 members of current version, 1 member last version, leader is current version {Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10}, // 10% with 2 members of current version, 1 member last version, leader is last version {Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10}, // 10% with 2 members of last version, 1 member current version, leader is last version {Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10}, // 10% with 2 members of last version, 1 member current version, leader is current version {Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10}, } mixedVersionOption := options.WithClusterOptionGroups(random.PickRandom[options.ClusterOptions](mixedVersionOptionChoices)) baseOptions := []e2e.EPClusterOption{ options.WithSnapshotCount(50, 100, 1000), options.WithSubsetOptions(randomizableOptions...), e2e.WithGoFailEnabled(true), // Set a low minimal compaction batch limit to allow for triggering multi batch compaction failpoints. options.WithCompactionBatchLimit(10, 100, 1000), e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond), } if addr := os.Getenv("TRACING_SERVER_ADDR"); addr != "" { baseOptions = append(baseOptions, e2e.WithEnableDistributedTracing(addr)) } if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) { baseOptions = append(baseOptions, options.WithSnapshotCatchUpEntries(100, etcdserver.DefaultSnapshotCatchUpEntries)) } scenarios := []TestScenario{} for _, tp := range trafficProfiles { name := filepath.Join(tp.Name, "ClusterOfSize1") clusterOfSize1Options := baseOptions clusterOfSize1Options = append(clusterOfSize1Options, e2e.WithClusterSize(1)) scenarios = append(scenarios, TestScenario{ Name: name, Traffic: tp.Traffic, Profile: tp.Profile, Cluster: *e2e.NewConfig(clusterOfSize1Options...), }) } for _, tp := range trafficProfiles { name := filepath.Join(tp.Name, "ClusterOfSize3") clusterOfSize3Options := baseOptions clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithIsPeerTLS(true)) clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithPeerProxy(true)) if fileutil.Exist(e2e.BinPath.EtcdLastRelease) { clusterOfSize3Options = append(clusterOfSize3Options, mixedVersionOption) } scenarios = append(scenarios, TestScenario{ Name: name, Traffic: tp.Traffic, Profile: tp.Profile, Cluster: *e2e.NewConfig(clusterOfSize3Options...), }) } if e2e.BinPath.LazyFSAvailable() { newScenarios := scenarios for _, s := range scenarios { // LazyFS increases the load on the CPU, so we run it with a more lightweight case. if s.Profile.KeyValue.MinimalQPS <= 100 && s.Cluster.ClusterSize == 1 { lazyfsCluster := s.Cluster lazyfsCluster.LazyFSEnabled = true profileWithoutCompaction := s.Profile profileWithoutCompaction.Compaction = nil newScenarios = append(newScenarios, TestScenario{ Name: filepath.Join(s.Name, "LazyFS"), Failpoint: s.Failpoint, Cluster: lazyfsCluster, Traffic: s.Traffic, Profile: profileWithoutCompaction, Watch: s.Watch, }) } } scenarios = newScenarios } return scenarios } func Regression(t *testing.T) []TestScenario { v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd) require.NoErrorf(t, err, "Failed checking etcd version binary, binary: %q", e2e.BinPath.Etcd) scenarios := []TestScenario{} scenarios = append(scenarios, TestScenario{ Name: "Issue14370", Failpoint: failpoint.RaftBeforeSavePanic, Profile: traffic.Profile{ KeyValue: &traffic.KeyValueMedium, Compaction: &traffic.CompactionDefault, }, Traffic: traffic.EtcdPutDeleteLease, Cluster: *e2e.NewConfig( e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), ), }) scenarios = append(scenarios, TestScenario{ Name: "Issue14685", Failpoint: failpoint.DefragBeforeCopyPanic, Profile: traffic.Profile{ KeyValue: &traffic.KeyValueMedium, Compaction: &traffic.CompactionDefault, }, Traffic: traffic.EtcdPutDeleteLease, Cluster: *e2e.NewConfig( e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), ), }) scenarios = append(scenarios, TestScenario{ Name: "Issue13766", Failpoint: failpoint.KillFailpoint, Profile: traffic.Profile{ KeyValue: &traffic.KeyValueHigh, Compaction: &traffic.CompactionDefault, }, Traffic: traffic.EtcdPut, Cluster: *e2e.NewConfig( e2e.WithSnapshotCount(100), ), }) scenarios = append(scenarios, TestScenario{ Name: "Issue15220", Watch: client.WatchConfig{ RequestProgress: true, }, Profile: traffic.Profile{ KeyValue: &traffic.KeyValueMedium, Watch: &traffic.WatchDefault, Compaction: &traffic.CompactionDefault, }, Traffic: traffic.EtcdPutDeleteLease, Failpoint: failpoint.KillFailpoint, Cluster: *e2e.NewConfig( e2e.WithClusterSize(1), ), }) scenarios = append(scenarios, TestScenario{ Name: "Issue17529", Profile: traffic.Profile{ KeyValue: &traffic.KeyValueHigh, Watch: &traffic.WatchDefault, Compaction: &traffic.CompactionDefault, }, Traffic: traffic.Kubernetes, Failpoint: failpoint.SleepBeforeSendWatchResponse, Cluster: *e2e.NewConfig( e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), options.WithSnapshotCount(100), ), }) scenarios = append(scenarios, TestScenario{ Name: "Issue17780", Profile: traffic.Profile{ KeyValue: &traffic.KeyValueMedium, }, Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic, Traffic: traffic.Kubernetes, Cluster: *e2e.NewConfig( e2e.WithClusterSize(1), e2e.WithCompactionBatchLimit(300), e2e.WithSnapshotCount(1000), e2e.WithGoFailEnabled(true), ), }) // NOTE: // // 1. All keys have only two revisions: creation and tombstone. With // a small compaction batch limit, it's easy to separate a key's two // revisions into different batch runs. If the compaction revision is a // tombstone and the creation revision was deleted in a previous // compaction run, we may encounter issue 19179. // // 2. It can be easily reproduced when using a lower QPS with a lower // burstable value. A higher QPS can generate more new keys than // expected, making it difficult to determine an optimal compaction // batch limit within a larger key space. scenarios = append(scenarios, TestScenario{ Name: "Issue19179", Profile: traffic.Profile{ KeyValue: &traffic.KeyValueVeryLow, Watch: &traffic.WatchDefault, }, Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic, Traffic: traffic.KubernetesCreateDelete, Cluster: *e2e.NewConfig( e2e.WithClusterSize(1), e2e.WithCompactionBatchLimit(50), e2e.WithSnapshotCount(1000), e2e.WithGoFailEnabled(true), ), }) scenarios = append(scenarios, TestScenario{ Name: "Issue18089", Profile: traffic.Profile{ KeyValue: &traffic.KeyValueMedium, Compaction: &traffic.CompactionFrequent, // Use frequent compaction for high reproduce rate Watch: &traffic.WatchDefault, }, Failpoint: failpoint.SleepBeforeSendWatchResponse, Traffic: traffic.EtcdDelete, Cluster: *e2e.NewConfig( e2e.WithClusterSize(1), e2e.WithGoFailEnabled(true), ), }) scenarios = append(scenarios, TestScenario{ Name: "Issue20221", Failpoint: failpoint.BlackholeUntilSnapshot, Watch: client.WatchConfig{ RequestProgress: true, }, Profile: traffic.Profile{ KeyValue: &traffic.KeyValueHigh, Watch: &traffic.Watch{ MemberClientCount: 6, ClusterClientCount: 2, RevisionOffsetRange: traffic.Range{Min: 50, Max: 100}, }, }, Traffic: traffic.EtcdPut, Cluster: *e2e.NewConfig( e2e.WithSnapshotCount(10), e2e.WithPeerProxy(true), e2e.WithIsPeerTLS(true), e2e.WithWatchProcessNotifyInterval(10*time.Millisecond), e2e.WithSnapshotCatchUpEntries(10), ), }) if v.Compare(version.V3_5) >= 0 { opts := []e2e.EPClusterOption{ e2e.WithSnapshotCount(100), e2e.WithPeerProxy(true), e2e.WithIsPeerTLS(true), } if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) { opts = append(opts, e2e.WithSnapshotCatchUpEntries(100)) } scenarios = append(scenarios, TestScenario{ Name: "Issue15271", Failpoint: failpoint.BlackholeUntilSnapshot, Profile: traffic.Profile{ KeyValue: &traffic.KeyValueHigh, Compaction: &traffic.CompactionDefault, Watch: &traffic.WatchDefault, }, Traffic: traffic.EtcdPut, Cluster: *e2e.NewConfig(opts...), }) } return scenarios } ================================================ FILE: tests/robustness/testdata/.gitignore ================================================ * !.gitignore ================================================ FILE: tests/robustness/traffic/etcd.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package traffic import ( "context" "fmt" "math/rand" "time" "golang.org/x/time/rate" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/robustness/client" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/model" "go.etcd.io/etcd/tests/v3/robustness/random" ) var ( EtcdPutDeleteLease Traffic = etcdTraffic{ keyCount: 10, leaseTTL: DefaultLeaseTTL, // Please keep the sum of weights equal 100. requests: []random.ChoiceWeight[etcdRequestType]{ {Choice: Get, Weight: 15}, {Choice: List, Weight: 15}, {Choice: StaleGet, Weight: 10}, {Choice: StaleList, Weight: 10}, {Choice: Delete, Weight: 5}, {Choice: MultiOpTxn, Weight: 10}, {Choice: PutWithLease, Weight: 5}, {Choice: LeaseRevoke, Weight: 5}, {Choice: CompareAndSet, Weight: 5}, {Choice: Put, Weight: 20}, }, } EtcdPut Traffic = etcdTraffic{ keyCount: 10, leaseTTL: DefaultLeaseTTL, // Please keep the sum of weights equal 100. requests: []random.ChoiceWeight[etcdRequestType]{ {Choice: Get, Weight: 15}, {Choice: List, Weight: 15}, {Choice: StaleGet, Weight: 10}, {Choice: StaleList, Weight: 10}, {Choice: MultiOpTxn, Weight: 10}, {Choice: Put, Weight: 40}, }, } EtcdDelete Traffic = etcdTraffic{ keyCount: 10, leaseTTL: DefaultLeaseTTL, // Please keep the sum of weights equal 100. requests: []random.ChoiceWeight[etcdRequestType]{ {Choice: Put, Weight: 50}, {Choice: Delete, Weight: 50}, }, } ) type etcdTraffic struct { keyCount int requests []random.ChoiceWeight[etcdRequestType] leaseTTL int64 } func (t etcdTraffic) ExpectUniqueRevision() bool { return false } type etcdRequestType string const ( Get etcdRequestType = "get" StaleGet etcdRequestType = "staleGet" List etcdRequestType = "list" StaleList etcdRequestType = "staleList" Put etcdRequestType = "put" Delete etcdRequestType = "delete" MultiOpTxn etcdRequestType = "multiOpTxn" PutWithLease etcdRequestType = "putWithLease" LeaseRevoke etcdRequestType = "leaseRevoke" CompareAndSet etcdRequestType = "compareAndSet" Defragment etcdRequestType = "defragment" ) func (t etcdTraffic) Name() string { return "Etcd" } func (t etcdTraffic) RunKeyValueLoop(ctx context.Context, p RunTrafficLoopParam) { lastOperationSucceeded := true var lastRev int64 var requestType etcdRequestType client := etcdTrafficClient{ etcdTraffic: t, keyStore: p.KeyStore, client: p.Client, limiter: p.QPSLimiter, idProvider: p.IDs, leaseStorage: p.LeaseIDStorage, } for { select { case <-ctx.Done(): return case <-p.Finish: return default: } shouldReturn := false // Avoid multiple failed writes in a row if lastOperationSucceeded { choices := t.requests if shouldReturn = p.NonUniqueRequestConcurrencyLimiter.Take(); !shouldReturn { choices = filterOutNonUniqueEtcdWrites(choices) } requestType = random.PickRandom(choices) } else { requestType = List } rev, err := client.Request(ctx, requestType, lastRev) if shouldReturn { p.NonUniqueRequestConcurrencyLimiter.Return() } lastOperationSucceeded = err == nil if err != nil { continue } if rev != 0 { lastRev = rev } p.QPSLimiter.Wait(ctx) } } func (t etcdTraffic) RunWatchLoop(ctx context.Context, p RunWatchLoopParam) { runWatchLoop(ctx, p, watchLoopConfig{ key: p.KeyStore.GetPrefix(), }) } func (t etcdTraffic) RunCompactLoop(ctx context.Context, param RunCompactLoopParam) { var lastRev int64 = 2 ticker := time.NewTicker(param.Period) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-param.Finish: return case <-ticker.C: } statusCtx, cancel := context.WithTimeout(ctx, RequestTimeout) resp, err := param.Client.Status(statusCtx, param.Client.Endpoints()[0]) cancel() if err != nil { continue } // Range allows for both revision has been compacted and future revision errors compactRev := random.RandRange(lastRev, resp.Header.Revision+5) _, err = param.Client.Compact(ctx, compactRev) if err != nil { continue } lastRev = compactRev } } func filterOutNonUniqueEtcdWrites(choices []random.ChoiceWeight[etcdRequestType]) (resp []random.ChoiceWeight[etcdRequestType]) { for _, choice := range choices { if choice.Choice != Delete && choice.Choice != LeaseRevoke { resp = append(resp, choice) } } return resp } type etcdTrafficClient struct { etcdTraffic keyStore *keyStore client *client.RecordingClient limiter *rate.Limiter idProvider identity.Provider leaseStorage identity.LeaseIDStorage } func (c etcdTrafficClient) Request(ctx context.Context, request etcdRequestType, lastRev int64) (rev int64, err error) { opCtx, cancel := context.WithTimeout(ctx, RequestTimeout) defer cancel() var limit int64 switch request { case StaleGet: var resp *clientv3.GetResponse resp, err = c.client.Get(opCtx, c.keyStore.GetKey(), clientv3.WithRev(lastRev)) if err == nil { rev = resp.Header.Revision } case Get: var resp *clientv3.GetResponse resp, err = c.client.Get(opCtx, c.keyStore.GetKey(), clientv3.WithRev(0)) if err == nil { rev = resp.Header.Revision } case List: var resp *clientv3.GetResponse resp, err = c.client.Range(opCtx, c.keyStore.GetPrefix(), clientv3.GetPrefixRangeEnd(c.keyStore.GetPrefix()), 0, limit) if resp != nil { c.keyStore.SyncKeys(resp) rev = resp.Header.Revision } case StaleList: var resp *clientv3.GetResponse resp, err = c.client.Range(opCtx, c.keyStore.GetPrefix(), clientv3.GetPrefixRangeEnd(c.keyStore.GetPrefix()), lastRev, limit) if resp != nil { rev = resp.Header.Revision } case Put: var resp *clientv3.PutResponse resp, err = c.client.Put(opCtx, c.keyStore.GetKey(), fmt.Sprintf("%d", c.idProvider.NewRequestID())) if resp != nil { rev = resp.Header.Revision } case Delete: var resp *clientv3.DeleteResponse resp, err = c.client.Delete(opCtx, c.keyStore.GetKeyForDelete()) if resp != nil { rev = resp.Header.Revision } case MultiOpTxn: var resp *clientv3.TxnResponse resp, err = c.client.Txn(opCtx).Then( c.pickMultiTxnOps(c.keyStore)..., ).Commit() if resp != nil { rev = resp.Header.Revision } case CompareAndSet: var kv *mvccpb.KeyValue key := c.keyStore.GetKey() var resp *clientv3.GetResponse resp, err = c.client.Get(opCtx, key, clientv3.WithRev(0)) if err == nil { rev = resp.Header.Revision if len(resp.Kvs) == 1 { kv = resp.Kvs[0] } c.limiter.Wait(ctx) var expectedRevision int64 if kv != nil { expectedRevision = kv.ModRevision } txnCtx, txnCancel := context.WithTimeout(ctx, RequestTimeout) var resp *clientv3.TxnResponse resp, err = c.client.Txn(txnCtx).If( clientv3.Compare(clientv3.ModRevision(key), "=", expectedRevision), ).Then( clientv3.OpPut(key, fmt.Sprintf("%d", c.idProvider.NewRequestID())), ).Commit() txnCancel() if resp != nil { rev = resp.Header.Revision } } case PutWithLease: leaseID := c.leaseStorage.LeaseID(c.client.ID) if leaseID == 0 { var resp *clientv3.LeaseGrantResponse resp, err = c.client.LeaseGrant(opCtx, c.leaseTTL) if resp != nil { leaseID = int64(resp.ID) rev = resp.ResponseHeader.Revision } if err == nil { c.leaseStorage.AddLeaseID(c.client.ID, leaseID) c.limiter.Wait(ctx) } } if leaseID != 0 { putCtx, putCancel := context.WithTimeout(ctx, RequestTimeout) var resp *clientv3.PutResponse resp, err = c.client.PutWithLease(putCtx, c.keyStore.GetKey(), fmt.Sprintf("%d", c.idProvider.NewRequestID()), leaseID) putCancel() if resp != nil { rev = resp.Header.Revision } } case LeaseRevoke: leaseID := c.leaseStorage.LeaseID(c.client.ID) if leaseID != 0 { var resp *clientv3.LeaseRevokeResponse resp, err = c.client.LeaseRevoke(opCtx, leaseID) // if LeaseRevoke has failed, do not remove the mapping. if err == nil { c.leaseStorage.RemoveLeaseID(c.client.ID) } if resp != nil { rev = resp.Header.Revision } } case Defragment: var resp *clientv3.DefragmentResponse resp, err = c.client.Defragment(opCtx) if resp != nil { rev = resp.Header.Revision } default: panic("invalid choice") } return rev, err } func (c etcdTrafficClient) pickMultiTxnOps(keyStore *keyStore) (ops []clientv3.Op) { opTypes := make([]model.OperationType, 4) atLeastOnePut := false for i := 0; i < MultiOpTxnOpCount; i++ { opTypes[i] = c.pickOperationType() if opTypes[i] == model.PutOperation { atLeastOnePut = true } } // Ensure at least one put to make operation unique if !atLeastOnePut { opTypes[0] = model.PutOperation } keys := keyStore.GetKeysForMultiTxnOps(opTypes) for i, opType := range opTypes { key := keys[i] switch opType { case model.RangeOperation: ops = append(ops, clientv3.OpGet(key)) case model.PutOperation: value := fmt.Sprintf("%d", c.idProvider.NewRequestID()) ops = append(ops, clientv3.OpPut(key, value)) case model.DeleteOperation: ops = append(ops, clientv3.OpDelete(key)) default: panic("unsupported choice type") } } return ops } func (t etcdTraffic) pickOperationType() model.OperationType { roll := rand.Int() % 100 if roll < 10 { return model.DeleteOperation } if roll < 50 { return model.RangeOperation } return model.PutOperation } ================================================ FILE: tests/robustness/traffic/key_store.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package traffic import ( "fmt" "math/rand" "slices" "sort" "strconv" "sync" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/robustness/model" ) // Stores the key pool to use for operations. This allows keys used in Put and Delete operations to be more unique by probabilistically swapping out used keys. type keyStore struct { mu sync.Mutex counter int keys []string keyPrefix string latestRevision int64 } func NewKeyStore(size int, keyPrefix string) *keyStore { k := &keyStore{ keys: make([]string, size), counter: 0, keyPrefix: keyPrefix, } // Fill with default values i.e. key0-key9 for ; k.counter < len(k.keys); k.counter++ { k.keys[k.counter] = fmt.Sprintf("%s%d", k.keyPrefix, k.counter) } return k } func (k *keyStore) GetKey() string { k.mu.Lock() defer k.mu.Unlock() useKey := k.keys[rand.Intn(len(k.keys))] return useKey } func (k *keyStore) GetKeyForDelete() string { k.mu.Lock() defer k.mu.Unlock() useKeyIndex := rand.Intn(len(k.keys)) useKey := k.keys[useKeyIndex] k.replaceKey(useKeyIndex) return useKey } func (k *keyStore) GetKeysForMultiTxnOps(ops []model.OperationType) []string { k.mu.Lock() defer k.mu.Unlock() numOps := len(ops) if numOps > len(k.keys) { panic("GetKeysForMultiTxnOps: number of operations is more than the key pool size") } keys := make([]string, numOps) permutedKeyIndexes := rand.Perm(len(k.keys)) for i, op := range ops { keys[i] = k.keys[permutedKeyIndexes[i]] if op == model.DeleteOperation { k.replaceKey(permutedKeyIndexes[i]) } } return keys } func (k *keyStore) GetPrefix() string { k.mu.Lock() defer k.mu.Unlock() return k.keyPrefix } // SyncKeys reconciles our local key store with the keys currently in etcd. // // SyncKeys resolves this by: // 1. Getting the list request result of all the keys. // 2. Adding any keys that exist in etcd but are missing in the key store. // 3. Maintaining the key store size by removing the higher numbered keys. // // Notice that higher numbered keys will eventually be added // again into the keystore, so it is safe to temporarily remove them from the // key store. func (k *keyStore) SyncKeys(resp *clientv3.GetResponse) { k.mu.Lock() defer k.mu.Unlock() if resp.Header.GetRevision() < k.latestRevision { return } k.latestRevision = resp.Header.GetRevision() listKeys := make([]string, len(resp.Kvs)) for i, kv := range resp.Kvs { listKeys[i] = string(kv.Key) } for _, key := range k.keys { if !slices.Contains(listKeys, key) { listKeys = append(listKeys, key) } } sort.Slice(listKeys, func(i, j int) bool { keyNumI := k.getKeyNum(listKeys[i]) keyNumJ := k.getKeyNum(listKeys[j]) return keyNumI < keyNumJ }) k.keys = listKeys[:len(k.keys)] lastKeyNum := k.getKeyNum(k.keys[len(k.keys)-1]) k.counter = lastKeyNum + 1 } func (k *keyStore) getKeyNum(key string) int { if len(key) < len(k.keyPrefix) { return 0 } numStr := key[len(k.keyPrefix):] num, _ := strconv.Atoi(numStr) return num } func (k *keyStore) replaceKey(index int) { if rand.Intn(100) < 90 { k.keys[index] = fmt.Sprintf("%s%d", k.keyPrefix, k.counter) k.counter++ } } ================================================ FILE: tests/robustness/traffic/kubernetes.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package traffic import ( "context" "errors" "fmt" "math/rand" "strconv" "sync" "time" "golang.org/x/sync/errgroup" "golang.org/x/time/rate" "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/kubernetes" "go.etcd.io/etcd/pkg/v3/stringutil" "go.etcd.io/etcd/tests/v3/robustness/client" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/random" ) var ( Kubernetes Traffic = kubernetesTraffic{ averageKeyCount: 10, resource: "pods", namespace: "default", // Please keep the sum of weights equal 100. readChoices: []random.ChoiceWeight[KubernetesRequestType]{ {Choice: KubernetesGet, Weight: 5}, {Choice: KubernetesGetStale, Weight: 2}, {Choice: KubernetesGetRev, Weight: 8}, {Choice: KubernetesListStale, Weight: 5}, {Choice: KubernetesListAndWatch, Weight: 80}, }, // Please keep the sum of weights equal 100. writeChoices: []random.ChoiceWeight[KubernetesRequestType]{ {Choice: KubernetesUpdate, Weight: 90}, {Choice: KubernetesDelete, Weight: 5}, {Choice: KubernetesCreate, Weight: 5}, }, } KubernetesCreateDelete Traffic = kubernetesTraffic{ averageKeyCount: 10, resource: "pods", namespace: "default", // Please keep the sum of weights equal 100. readChoices: []random.ChoiceWeight[KubernetesRequestType]{ {Choice: KubernetesGet, Weight: 5}, {Choice: KubernetesGetStale, Weight: 2}, {Choice: KubernetesGetRev, Weight: 8}, {Choice: KubernetesListStale, Weight: 5}, {Choice: KubernetesListAndWatch, Weight: 80}, }, // Please keep the sum of weights equal 100. writeChoices: []random.ChoiceWeight[KubernetesRequestType]{ {Choice: KubernetesDelete, Weight: 40}, {Choice: KubernetesCreate, Weight: 60}, }, } ) type kubernetesTraffic struct { averageKeyCount int resource string namespace string readChoices []random.ChoiceWeight[KubernetesRequestType] writeChoices []random.ChoiceWeight[KubernetesRequestType] } func (t kubernetesTraffic) ExpectUniqueRevision() bool { return true } func (t kubernetesTraffic) RunKeyValueLoop(ctx context.Context, p RunTrafficLoopParam) { kc := kubernetes.Client{Client: &clientv3.Client{KV: p.Client}} s := p.Storage keyPrefix := "/registry/" + t.resource + "/" g := errgroup.Group{} g.Go(func() error { for { select { case <-ctx.Done(): return ctx.Err() case <-p.Finish: return nil default: } err := t.Read(ctx, p.Client, s, p.QPSLimiter, keyPrefix) if err != nil { continue } } }) g.Go(func() error { lastWriteFailed := false for { select { case <-ctx.Done(): return ctx.Err() case <-p.Finish: return nil default: } // Avoid multiple failed writes in a row if lastWriteFailed { _, err := t.List(ctx, kc, s, p.QPSLimiter, keyPrefix, t.averageKeyCount, 0) if err != nil { continue } } err := t.Write(ctx, kc, p.IDs, s, p.QPSLimiter, p.NonUniqueRequestConcurrencyLimiter) lastWriteFailed = err != nil if err != nil { continue } } }) g.Wait() } func (t kubernetesTraffic) RunWatchLoop(ctx context.Context, p RunWatchLoopParam) { runWatchLoop(ctx, p, watchLoopConfig{ key: "/registry/" + t.resource + "/", requireLeader: true, }) } func (t kubernetesTraffic) Read(ctx context.Context, c *client.RecordingClient, s *storage, limiter *rate.Limiter, keyPrefix string) error { kc := kubernetes.Client{Client: &clientv3.Client{KV: c}} op := random.PickRandom(t.readChoices) switch op { case KubernetesGet: key, unusedRev := s.PickRandom() if unusedRev == 0 { return errors.New("storage empty") } return t.Get(ctx, kc, s, limiter, key, 0) case KubernetesGetStale: key, rev := s.KeyWithUnrelatedRev() return t.Get(ctx, kc, s, limiter, key, rev) case KubernetesGetRev: return t.Get(ctx, kc, s, limiter, "/registry/"+t.resource, 0) case KubernetesListStale: _, rev := s.PickRandom() _, err := t.List(ctx, kc, s, limiter, keyPrefix, t.averageKeyCount, rev) return err case KubernetesListAndWatch: rev, err := t.List(ctx, kc, s, limiter, keyPrefix, t.averageKeyCount, 0) if err != nil { return err } t.Watch(ctx, c, s, limiter, keyPrefix, rev+1) return nil default: panic(fmt.Sprintf("invalid choice: %q", op)) } } func (t kubernetesTraffic) Get(ctx context.Context, kc kubernetes.Interface, s *storage, limiter *rate.Limiter, key string, rev int64) error { _, err := kc.Get(ctx, key, kubernetes.GetOptions{Revision: rev}) limiter.Wait(ctx) return err } func (t kubernetesTraffic) List(ctx context.Context, kc kubernetes.Interface, s *storage, limiter *rate.Limiter, keyPrefix string, limit int, revision int64) (rev int64, err error) { hasMore := true var kvs []*mvccpb.KeyValue var cont string for hasMore { readCtx, cancel := context.WithTimeout(ctx, RequestTimeout) resp, err := kc.List(readCtx, keyPrefix, kubernetes.ListOptions{Continue: cont, Revision: revision, Limit: int64(limit)}) cancel() if err != nil { return 0, err } limiter.Wait(ctx) kvs = append(kvs, resp.Kvs...) if revision == 0 { revision = resp.Revision } hasMore = resp.Count > int64(len(resp.Kvs)) if hasMore { cont = string(kvs[len(kvs)-1].Key) + "\x00" } } s.Reset(revision, kvs) return revision, nil } func (t kubernetesTraffic) Write(ctx context.Context, kc kubernetes.Interface, ids identity.Provider, s *storage, limiter *rate.Limiter, nonUniqueWriteLimiter ConcurrencyLimiter) (err error) { writeCtx, cancel := context.WithTimeout(ctx, RequestTimeout) defer cancel() count := s.Count() if count < t.averageKeyCount/2 { _, err = kc.OptimisticPut(writeCtx, t.generateKey(), []byte(fmt.Sprintf("%d", ids.NewRequestID())), 0, kubernetes.PutOptions{}) } else { key, rev := s.PickRandom() if rev == 0 { return errors.New("storage empty") } if count > t.averageKeyCount*3/2 && nonUniqueWriteLimiter.Take() { _, err = kc.OptimisticDelete(writeCtx, key, rev, kubernetes.DeleteOptions{GetOnFailure: true}) nonUniqueWriteLimiter.Return() } else { shouldReturn := false choices := t.writeChoices if shouldReturn = nonUniqueWriteLimiter.Take(); !shouldReturn { choices = filterOutNonUniqueKubernetesWrites(t.writeChoices) } op := random.PickRandom(choices) switch op { case KubernetesDelete: _, err = kc.OptimisticDelete(writeCtx, key, rev, kubernetes.DeleteOptions{GetOnFailure: true}) case KubernetesUpdate: _, err = kc.OptimisticPut(writeCtx, key, []byte(fmt.Sprintf("%d", ids.NewRequestID())), rev, kubernetes.PutOptions{GetOnFailure: true}) case KubernetesCreate: _, err = kc.OptimisticPut(writeCtx, t.generateKey(), []byte(fmt.Sprintf("%d", ids.NewRequestID())), 0, kubernetes.PutOptions{}) default: panic(fmt.Sprintf("invalid choice: %q", op)) } if shouldReturn { nonUniqueWriteLimiter.Return() } } } if err != nil { return err } limiter.Wait(ctx) return nil } func filterOutNonUniqueKubernetesWrites(choices []random.ChoiceWeight[KubernetesRequestType]) (resp []random.ChoiceWeight[KubernetesRequestType]) { for _, choice := range choices { if choice.Choice != KubernetesDelete { resp = append(resp, choice) } } return resp } func (t kubernetesTraffic) Watch(ctx context.Context, c *client.RecordingClient, s *storage, limiter *rate.Limiter, keyPrefix string, revision int64) { watchCtx, cancel := context.WithTimeout(ctx, WatchTimeout) defer cancel() // Kubernetes issues Watch requests by requiring a leader to exist // in the cluster: // https://github.com/kubernetes/kubernetes/blob/2016fab3085562b4132e6d3774b6ded5ba9939fd/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go#L872 watchCtx = clientv3.WithRequireLeader(watchCtx) for e := range c.Watch(watchCtx, keyPrefix, revision, true, true, true) { s.Update(e) } limiter.Wait(ctx) } func (t kubernetesTraffic) generateKey() string { return fmt.Sprintf("/registry/%s/%s/%s", t.resource, t.namespace, stringutil.RandString(5)) } func (t kubernetesTraffic) RunCompactLoop(ctx context.Context, param RunCompactLoopParam) { // Based on https://github.com/kubernetes/apiserver/blob/7dd4904f1896e11244ba3c5a59797697709de6b6/pkg/storage/etcd3/compact.go#L112-L127 var compactTime int64 var rev int64 var err error for { select { case <-time.After(param.Period): case <-ctx.Done(): return case <-param.Finish: return } compactTime, rev, err = compact(ctx, param.Client, compactTime, rev) if err != nil { continue } } } // Based on https://github.com/kubernetes/apiserver/blob/7dd4904f1896e11244ba3c5a59797697709de6b6/pkg/storage/etcd3/compact.go#L30 const ( compactRevKey = "compact_rev_key" ) func compact(ctx context.Context, client *client.RecordingClient, t, rev int64) (int64, int64, error) { // Based on https://github.com/kubernetes/apiserver/blob/7dd4904f1896e11244ba3c5a59797697709de6b6/pkg/storage/etcd3/compact.go#L133-L162 resp, err := client.Txn(ctx). If(clientv3.Compare(clientv3.Version(compactRevKey), "=", t)). Then(clientv3.OpPut(compactRevKey, strconv.FormatInt(rev, 10))). Else(clientv3.OpGet(compactRevKey)). Commit() if err != nil { return t, rev, err } curRev := resp.Header.Revision if !resp.Succeeded { curTime := resp.Responses[0].GetResponseRange().Kvs[0].Version return curTime, curRev, nil } curTime := t + 1 if rev == 0 { return curTime, curRev, nil } if _, err = client.Compact(ctx, rev); err != nil { return curTime, curRev, err } return curTime, curRev, nil } type KubernetesRequestType string const ( KubernetesDelete KubernetesRequestType = "delete" KubernetesUpdate KubernetesRequestType = "update" KubernetesCreate KubernetesRequestType = "create" KubernetesGet KubernetesRequestType = "get" KubernetesGetStale KubernetesRequestType = "get_stale" KubernetesGetRev KubernetesRequestType = "get_rev" KubernetesListStale KubernetesRequestType = "list_stale" KubernetesListAndWatch KubernetesRequestType = "list_watch" ) type storage struct { mux sync.RWMutex keyRevision map[string]int64 revision int64 } // NewKubernetesStorage creates a new storage instance for tracking Kubernetes resource revisions. // This storage is used by Kubernetes traffic to maintain consistency in read and write operations. func NewKubernetesStorage() *storage { return &storage{ keyRevision: map[string]int64{}, } } func (s *storage) Update(resp clientv3.WatchResponse) { s.mux.Lock() defer s.mux.Unlock() for _, e := range resp.Events { if e.Kv.ModRevision < s.revision { continue } s.revision = e.Kv.ModRevision switch e.Type { case mvccpb.Event_PUT: s.keyRevision[string(e.Kv.Key)] = e.Kv.ModRevision case mvccpb.Event_DELETE: delete(s.keyRevision, string(e.Kv.Key)) } } } func (s *storage) Reset(revision int64, kvs []*mvccpb.KeyValue) { s.mux.Lock() defer s.mux.Unlock() if revision <= s.revision { return } s.keyRevision = make(map[string]int64, len(kvs)) for _, kv := range kvs { s.keyRevision[string(kv.Key)] = kv.ModRevision } s.revision = revision } func (s *storage) Count() int { s.mux.RLock() defer s.mux.RUnlock() return len(s.keyRevision) } func (s *storage) PickRandom() (key string, rev int64) { s.mux.RLock() defer s.mux.RUnlock() return s.pickRandomLocked() } func (s *storage) pickRandomLocked() (key string, rev int64) { l := len(s.keyRevision) if l == 0 { return "", 0 } n := rand.Intn(l) i := 0 for k, v := range s.keyRevision { if i == n { return k, v } i++ } return "", 0 } func (s *storage) KeyWithUnrelatedRev() (key string, rev int64) { s.mux.RLock() defer s.mux.RUnlock() l := len(s.keyRevision) if l == 0 { return "", 0 } key, _ = s.pickRandomLocked() _, rev = s.pickRandomLocked() return key, rev } ================================================ FILE: tests/robustness/traffic/limiter.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package traffic func NewConcurrencyLimiter(size int) ConcurrencyLimiter { return &concurrencyLimiter{ ch: make(chan struct{}, size), } } type ConcurrencyLimiter interface { Take() bool Return() } type concurrencyLimiter struct { ch chan struct{} } func (c *concurrencyLimiter) Take() bool { select { case c.ch <- struct{}{}: return true default: return false } } func (c *concurrencyLimiter) Return() { select { case <-c.ch: default: panic("Call to Return() without a successful Take") } } ================================================ FILE: tests/robustness/traffic/limiter_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package traffic import ( "sync/atomic" "testing" "github.com/stretchr/testify/assert" "golang.org/x/sync/errgroup" ) func TestLimiter(t *testing.T) { limiter := NewConcurrencyLimiter(3) counter := &atomic.Int64{} g := errgroup.Group{} for i := 0; i < 10; i++ { g.Go(func() error { if limiter.Take() { counter.Add(1) } return nil }) } g.Wait() assert.Equal(t, 3, int(counter.Load())) assert.False(t, limiter.Take()) limiter.Return() counter.Store(0) for i := 0; i < 10; i++ { g.Go(func() error { if limiter.Take() { counter.Add(1) } return nil }) } g.Wait() assert.Equal(t, 1, int(counter.Load())) assert.False(t, limiter.Take()) limiter.Return() limiter.Return() limiter.Return() assert.Panics(t, func() { limiter.Return() }) } ================================================ FILE: tests/robustness/traffic/traffic.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package traffic import ( "context" "sync" "testing" "time" "github.com/stretchr/testify/require" "go.uber.org/zap" "golang.org/x/time/rate" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/client" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/model" "go.etcd.io/etcd/tests/v3/robustness/random" "go.etcd.io/etcd/tests/v3/robustness/report" "go.etcd.io/etcd/tests/v3/robustness/validate" ) type Range struct { Min int64 Max int64 } func (r Range) Rand() int64 { if r.Min == r.Max { return r.Min } return random.RandRange(r.Min, r.Max+1) } var ( DefaultLeaseTTL int64 = 7200 RequestTimeout = 200 * time.Millisecond WatchTimeout = 500 * time.Millisecond MultiOpTxnOpCount = 4 MinimalCompactionPeriod = 100 * time.Millisecond KeyValueVeryLow = KeyValue{ MinimalQPS: 50, MaximalQPS: 100, BurstableQPS: 100, MemberClientCount: 6, ClusterClientCount: 2, MaxNonUniqueRequestConcurrency: 3, } KeyValueMedium = KeyValue{ MinimalQPS: 100, MaximalQPS: 200, BurstableQPS: 1000, MemberClientCount: 6, ClusterClientCount: 2, MaxNonUniqueRequestConcurrency: 3, } KeyValueHigh = KeyValue{ MinimalQPS: 100, MaximalQPS: 1000, BurstableQPS: 1000, MemberClientCount: 6, ClusterClientCount: 2, MaxNonUniqueRequestConcurrency: 3, } WatchDefault = Watch{ MemberClientCount: 6, ClusterClientCount: 2, RevisionOffsetRange: Range{Min: -100, Max: 100}, } CompactionDefault = Compaction{ Period: 200 * time.Millisecond, } CompactionFrequent = Compaction{ Period: 100 * time.Millisecond, } ) func SimulateTraffic(ctx context.Context, t *testing.T, lg *zap.Logger, clus *e2e.EtcdProcessCluster, profile Profile, traffic Traffic, failpointInjected <-chan report.FailpointInjection, clientSet *client.ClientSet) []report.ClientReport { endpoints := clus.EndpointsGRPC() lm := identity.NewLeaseIDStorage() // Use the highest MaximalQPS of all traffic profiles as burst otherwise actual traffic may be accidentally limited limiter := rate.NewLimiter(rate.Limit(profile.KeyValue.MaximalQPS), profile.KeyValue.BurstableQPS) err := CheckEmptyDatabaseAtStart(ctx, lg, endpoints, clientSet) require.NoError(t, err) wg := sync.WaitGroup{} nonUniqueWriteLimiter := NewConcurrencyLimiter(profile.KeyValue.MaxNonUniqueRequestConcurrency) finish := make(chan struct{}) keyStore := NewKeyStore(10, "key") kubernetesStorage := NewKubernetesStorage() lg.Info("Start traffic") startTime := time.Since(clientSet.BaseTime()) for i := range profile.KeyValue.MemberClientCount { wg.Add(1) c, nerr := clientSet.NewClient([]string{endpoints[i%len(endpoints)]}) require.NoError(t, nerr) go func(c *client.RecordingClient) { defer wg.Done() defer c.Close() traffic.RunKeyValueLoop(ctx, RunTrafficLoopParam{ Client: c, QPSLimiter: limiter, IDs: clientSet.IdentityProvider(), LeaseIDStorage: lm, NonUniqueRequestConcurrencyLimiter: nonUniqueWriteLimiter, KeyStore: keyStore, Storage: kubernetesStorage, Finish: finish, }) }(c) } for range profile.KeyValue.ClusterClientCount { wg.Add(1) c, nerr := clientSet.NewClient(endpoints) require.NoError(t, nerr) go func(c *client.RecordingClient) { defer wg.Done() defer c.Close() traffic.RunKeyValueLoop(ctx, RunTrafficLoopParam{ Client: c, QPSLimiter: limiter, IDs: clientSet.IdentityProvider(), LeaseIDStorage: lm, NonUniqueRequestConcurrencyLimiter: nonUniqueWriteLimiter, KeyStore: keyStore, Storage: kubernetesStorage, Finish: finish, }) }(c) } if profile.Watch != nil { for i := range profile.Watch.MemberClientCount { wg.Add(1) c, nerr := clientSet.NewClient([]string{endpoints[i%len(endpoints)]}) require.NoError(t, nerr) go func(c *client.RecordingClient) { defer wg.Done() defer c.Close() traffic.RunWatchLoop(ctx, RunWatchLoopParam{ Config: *profile.Watch, Client: c, QPSLimiter: limiter, KeyStore: keyStore, Storage: kubernetesStorage, Finish: finish, Logger: lg, }) }(c) } for range profile.Watch.ClusterClientCount { wg.Add(1) c, nerr := clientSet.NewClient(endpoints) require.NoError(t, nerr) go func(c *client.RecordingClient) { defer wg.Done() defer c.Close() traffic.RunWatchLoop(ctx, RunWatchLoopParam{ Config: *profile.Watch, Client: c, QPSLimiter: limiter, KeyStore: keyStore, Storage: kubernetesStorage, Finish: finish, Logger: lg, }) }(c) } } if profile.Compaction != nil { wg.Add(1) c, nerr := clientSet.NewClient(endpoints) if nerr != nil { t.Fatal(nerr) } if profile.Compaction.Period < MinimalCompactionPeriod { t.Fatalf("Compaction period %v below minimal %v", profile.Compaction.Period, MinimalCompactionPeriod) } go func(c *client.RecordingClient) { defer wg.Done() defer c.Close() traffic.RunCompactLoop(ctx, RunCompactLoopParam{ Client: c, Period: profile.Compaction.Period, Finish: finish, }) }(c) } var fr *report.FailpointInjection select { case frp, ok := <-failpointInjected: require.Truef(t, ok, "Failed to collect failpoint report") fr = &frp case <-ctx.Done(): t.Fatalf("Traffic finished before failure was injected: %s", ctx.Err()) } close(finish) wg.Wait() lg.Info("Finished traffic") endTime := time.Since(clientSet.BaseTime()) time.Sleep(time.Second) // Ensure that last operation succeeds cc, err := clientSet.NewClient(endpoints) require.NoError(t, err) defer cc.Close() _, err = cc.Put(ctx, "tombstone", "true") require.NoErrorf(t, err, "Last operation failed, validation requires last operation to succeed") reports := clientSet.Reports() totalStats := CalculateStats(reports, startTime, endTime) beforeFailpointStats := CalculateStats(reports, startTime, fr.Start) duringFailpointStats := CalculateStats(reports, fr.Start, fr.End) afterFailpointStats := CalculateStats(reports, fr.End, endTime) lg.Info("Reporting complete traffic", zap.Int("successes", totalStats.Successes), zap.Int("failures", totalStats.Failures), zap.Float64("successRate", totalStats.SuccessRate()), zap.Duration("period", totalStats.Period), zap.Float64("qps", totalStats.QPS())) lg.Info("Reporting traffic before failure injection", zap.Int("successes", beforeFailpointStats.Successes), zap.Int("failures", beforeFailpointStats.Failures), zap.Float64("successRate", beforeFailpointStats.SuccessRate()), zap.Duration("period", beforeFailpointStats.Period), zap.Float64("qps", beforeFailpointStats.QPS())) lg.Info("Reporting traffic during failure injection", zap.Int("successes", duringFailpointStats.Successes), zap.Int("failures", duringFailpointStats.Failures), zap.Float64("successRate", duringFailpointStats.SuccessRate()), zap.Duration("period", duringFailpointStats.Period), zap.Float64("qps", duringFailpointStats.QPS())) lg.Info("Reporting traffic after failure injection", zap.Int("successes", afterFailpointStats.Successes), zap.Int("failures", afterFailpointStats.Failures), zap.Float64("successRate", afterFailpointStats.SuccessRate()), zap.Duration("period", afterFailpointStats.Period), zap.Float64("qps", afterFailpointStats.QPS())) watchTotal := CalculateWatchStats(reports, startTime, endTime) lg.Info("Reporting complete watch", zap.Int("requests", watchTotal.Requests), zap.Int("events", watchTotal.Events), zap.Float64("eventsQPS", watchTotal.EventsQPS()), zap.Int("progressNotifies", watchTotal.ProgressNotifies), zap.Int("immediateClosures", watchTotal.ImmediateClosures), zap.Duration("period", watchTotal.Period), zap.Duration("avgDuration", watchTotal.AvgDuration())) if beforeFailpointStats.QPS() < profile.KeyValue.MinimalQPS { t.Errorf("Requiring minimal %f qps before failpoint injection for test results to be reliable, got %f qps", profile.KeyValue.MinimalQPS, beforeFailpointStats.QPS()) } // TODO: Validate QPS post failpoint injection to ensure that we sufficiently cover the period when the cluster recovers. return reports } func CalculateWatchStats(reports []report.ClientReport, start, end time.Duration) (ws watchStats) { ws.Period = end - start if ws.Period <= 0 { return ws } for _, r := range reports { for _, w := range r.Watch { var ( firstInWindow time.Duration lastInWindow time.Duration haveInWindow bool noEventsYetInWindow = true closedCounted = false ) for _, resp := range w.Responses { if resp.Time < start || resp.Time > end { continue } if !haveInWindow { firstInWindow = resp.Time haveInWindow = true } lastInWindow = resp.Time if resp.IsProgressNotify { ws.ProgressNotifies++ } if len(resp.Events) > 0 { ws.Events += len(resp.Events) noEventsYetInWindow = false } if resp.Error != "" && noEventsYetInWindow && !closedCounted { ws.ImmediateClosures++ closedCounted = true } } if haveInWindow { ws.Requests++ if lastInWindow > firstInWindow { ws.SumDuration += lastInWindow - firstInWindow ws.DurationsCount++ } } } } return ws } type watchStats struct { Period time.Duration Requests int Events int ProgressNotifies int ImmediateClosures int SumDuration time.Duration DurationsCount int } func (ws *watchStats) AvgDuration() time.Duration { if ws.DurationsCount == 0 { return 0 } return ws.SumDuration / time.Duration(ws.DurationsCount) } func (ws *watchStats) EventsQPS() float64 { if ws.Period <= 0 { return 0 } return float64(ws.Events) / ws.Period.Seconds() } func CalculateStats(reports []report.ClientReport, start, end time.Duration) (ts trafficStats) { ts.Period = end - start for _, r := range reports { for _, op := range r.KeyValue { if op.Call < start.Nanoseconds() || op.Call > end.Nanoseconds() { continue } resp := op.Output.(model.MaybeEtcdResponse) if resp.Error == "" { ts.Successes++ } else { ts.Failures++ } } } return ts } type trafficStats struct { Successes, Failures int Period time.Duration } func (ts *trafficStats) SuccessRate() float64 { return float64(ts.Successes) / float64(ts.Successes+ts.Failures) } func (ts *trafficStats) QPS() float64 { return float64(ts.Successes) / ts.Period.Seconds() } type Profile struct { KeyValue *KeyValue Watch *Watch Compaction *Compaction } type KeyValue struct { MinimalQPS float64 MaximalQPS float64 BurstableQPS int MaxNonUniqueRequestConcurrency int MemberClientCount int ClusterClientCount int } type Watch struct { MemberClientCount int ClusterClientCount int RevisionOffsetRange Range } type Compaction struct { Period time.Duration } type RunTrafficLoopParam struct { Client *client.RecordingClient QPSLimiter *rate.Limiter IDs identity.Provider LeaseIDStorage identity.LeaseIDStorage NonUniqueRequestConcurrencyLimiter ConcurrencyLimiter KeyStore *keyStore Storage *storage Finish <-chan struct{} } type RunCompactLoopParam struct { Client *client.RecordingClient Period time.Duration Finish <-chan struct{} } type RunWatchLoopParam struct { Config Watch Client *client.RecordingClient QPSLimiter *rate.Limiter // TODO: merge 2 key stores into 1 KeyStore *keyStore Storage *storage Finish <-chan struct{} Logger *zap.Logger } type Traffic interface { RunKeyValueLoop(ctx context.Context, param RunTrafficLoopParam) RunWatchLoop(ctx context.Context, param RunWatchLoopParam) RunCompactLoop(ctx context.Context, param RunCompactLoopParam) ExpectUniqueRevision() bool } func runWatchLoop(ctx context.Context, p RunWatchLoopParam, cfg watchLoopConfig) { for { select { case <-ctx.Done(): return case <-p.Finish: return default: } err := p.QPSLimiter.Wait(ctx) if err != nil { return } // Client.get may fail when the blackhole is injected. // We suppress logging for these expected failures to avoid polluting the logs. _ = runWatch(ctx, p, cfg) } } func runWatch(ctx context.Context, p RunWatchLoopParam, cfg watchLoopConfig) error { getCtx, getCancel := context.WithTimeout(ctx, RequestTimeout) defer getCancel() resp, err := p.Client.Get(getCtx, cfg.key) if err != nil { return err } rev := resp.Header.Revision + p.Config.RevisionOffsetRange.Rand() watchCtx, watchCancel := context.WithTimeout(ctx, WatchTimeout) defer watchCancel() if cfg.requireLeader { watchCtx = clientv3.WithRequireLeader(watchCtx) } w := p.Client.Watch(watchCtx, cfg.key, rev, true, true, true) for { select { case <-ctx.Done(): return nil case <-p.Finish: return nil case _, ok := <-w: if !ok { return nil } } } } type watchLoopConfig struct { key string requireLeader bool } func CheckEmptyDatabaseAtStart(ctx context.Context, lg *zap.Logger, endpoints []string, cs *client.ClientSet) error { c, err := cs.NewClient(endpoints) if err != nil { return err } defer c.Close() for { rCtx, cancel := context.WithTimeout(ctx, RequestTimeout) resp, err := c.Get(rCtx, "key") cancel() if err != nil { lg.Warn("Failed to check if database empty at start, retrying", zap.Error(err)) continue } if resp.Header.Revision != 1 { return validate.ErrNotEmptyDatabase } break } return nil } ================================================ FILE: tests/robustness/validate/operations.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package validate import ( "errors" "time" "github.com/anishathalye/porcupine" "github.com/google/go-cmp/cmp" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/robustness/model" ) var ( errRespNotMatched = errors.New("response didn't match expected") errFutureRevRespRequested = errors.New("request about a future rev with response") ) func validateLinearizableOperationsAndVisualize(lg *zap.Logger, operations []porcupine.Operation, timeout time.Duration) LinearizationResult { lg.Info("Validating linearizable operations", zap.Duration("timeout", timeout)) start := time.Now() check, info := porcupine.CheckOperationsVerbose(model.NonDeterministicModel, operations, timeout) result := LinearizationResult{ Info: info, Model: model.NonDeterministicModel, } switch check { case porcupine.Ok: result.Status = Success lg.Info("Linearization success", zap.Duration("duration", time.Since(start))) case porcupine.Unknown: result.Status = Failure result.Message = "timed out" result.Timeout = true lg.Error("Linearization timed out", zap.Duration("duration", time.Since(start))) case porcupine.Illegal: result.Status = Failure result.Message = "illegal" lg.Error("Linearization illegal", zap.Duration("duration", time.Since(start))) default: result.Status = Failure result.Message = "unknown" } return result } func validateSerializableOperations(lg *zap.Logger, operations []porcupine.Operation, replay *model.EtcdReplay) Result { lg.Info("Validating serializable operations") start := time.Now() err := validateSerializableOperationsError(lg, operations, replay) if err != nil { lg.Error("Serializable validation failed", zap.Duration("duration", time.Since(start)), zap.Error(err)) } else { lg.Info("Serializable validation success", zap.Duration("duration", time.Since(start))) } return ResultFromError(err) } func validateSerializableOperationsError(lg *zap.Logger, operations []porcupine.Operation, replay *model.EtcdReplay) (lastErr error) { for _, read := range operations { request := read.Input.(model.EtcdRequest) response := read.Output.(model.MaybeEtcdResponse) err := validateSerializableRead(lg, replay, request, response) if err != nil { lastErr = err } } return lastErr } func validateSerializableRead(lg *zap.Logger, replay *model.EtcdReplay, request model.EtcdRequest, response model.MaybeEtcdResponse) error { if response.Persisted || response.Error != "" { return nil } state, err := replay.StateForRevision(request.Range.Revision) if err != nil { if response.Error == model.ErrEtcdFutureRev.Error() { return nil } lg.Error("Failed validating serializable operation", zap.Any("request", request), zap.Any("response", response)) return errFutureRevRespRequested } _, expectResp := state.Step(request) if diff := cmp.Diff(response.EtcdResponse.Range, expectResp.Range); diff != "" { lg.Error("Failed validating serializable operation", zap.Any("request", request), zap.String("diff", diff)) return errRespNotMatched } return nil } ================================================ FILE: tests/robustness/validate/operations_test.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //nolint:unparam package validate import ( "fmt" "math/rand/v2" "testing" "time" "github.com/anishathalye/porcupine" "go.uber.org/zap" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/tests/v3/robustness/model" ) func TestValidateSerializableOperations(t *testing.T) { tcs := []struct { name string persistedRequests []model.EtcdRequest operations []porcupine.Operation expectError string }{ { name: "Success", persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, operations: []porcupine.Operation{ { Input: rangeRequest("a", "z", 1, 0), Output: rangeResponse(0), }, { Input: rangeRequest("a", "z", 2, 0), Output: rangeResponse(1, keyValueRevision("a", "1", 2)), }, { Input: rangeRequest("a", "z", 3, 0), Output: rangeResponse(2, keyValueRevision("a", "1", 2), keyValueRevision("b", "2", 3), ), }, { Input: rangeRequest("a", "z", 4, 0), Output: rangeResponse(3, keyValueRevision("a", "1", 2), keyValueRevision("b", "2", 3), keyValueRevision("c", "3", 4), ), }, { Input: rangeRequest("a", "z", 4, 3), Output: rangeResponse(3, keyValueRevision("a", "1", 2), keyValueRevision("b", "2", 3), keyValueRevision("c", "3", 4), ), }, { Input: rangeRequest("a", "z", 4, 4), Output: rangeResponse(3, keyValueRevision("a", "1", 2), keyValueRevision("b", "2", 3), keyValueRevision("c", "3", 4), ), }, { Input: rangeRequest("a", "z", 4, 2), Output: rangeResponse(3, keyValueRevision("a", "1", 2), keyValueRevision("b", "2", 3), ), }, { Input: rangeRequest("b\x00", "z", 4, 2), Output: rangeResponse(1, keyValueRevision("c", "3", 4), ), }, { Input: rangeRequest("b", "", 4, 0), Output: rangeResponse(1, keyValueRevision("b", "2", 3), ), }, { Input: rangeRequest("b", "", 2, 0), Output: rangeResponse(0), }, }, }, { name: "Invalid order", persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, operations: []porcupine.Operation{ { Input: rangeRequest("a", "z", 4, 0), Output: rangeResponse(3, keyValueRevision("c", "3", 4), keyValueRevision("b", "2", 3), keyValueRevision("a", "1", 2), ), }, }, expectError: errRespNotMatched.Error(), }, { name: "Invalid count", persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, operations: []porcupine.Operation{ { Input: rangeRequest("a", "z", 1, 0), Output: rangeResponse(1), }, }, expectError: errRespNotMatched.Error(), }, { name: "Invalid keys", persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, operations: []porcupine.Operation{ { Input: rangeRequest("a", "z", 2, 0), Output: rangeResponse(3, keyValueRevision("b", "2", 3), ), }, }, expectError: errRespNotMatched.Error(), }, { name: "Invalid revision", persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, operations: []porcupine.Operation{ { Input: rangeRequest("a", "z", 2, 0), Output: rangeResponse(3, keyValueRevision("a", "1", 2), keyValueRevision("b", "2", 3), ), }, }, expectError: errRespNotMatched.Error(), }, { name: "Error", persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, operations: []porcupine.Operation{ { Input: rangeRequest("a", "z", 2, 0), Output: errorResponse(model.ErrEtcdFutureRev), }, { Input: rangeRequest("a", "z", 2, 0), Output: errorResponse(fmt.Errorf("timeout")), }, }, }, { name: "Future rev returned", persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, operations: []porcupine.Operation{ { Input: rangeRequest("a", "z", 6, 0), Output: errorResponse(model.ErrEtcdFutureRev), }, }, }, { name: "Future rev success", persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, operations: []porcupine.Operation{ { Input: rangeRequest("a", "z", 6, 0), Output: rangeResponse(0), }, }, expectError: errFutureRevRespRequested.Error(), }, { name: "Future rev failure", persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, operations: []porcupine.Operation{ { Input: rangeRequest("a", "z", 6, 0), Output: errorResponse(fmt.Errorf("timeout")), }, }, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { replay := model.NewReplay(tc.persistedRequests) result := validateSerializableOperations(zaptest.NewLogger(t), tc.operations, replay) if result.Message != tc.expectError { t.Errorf("validateSerializableOperations(...), got: %q, want: %q", result.Message, tc.expectError) } }) } } func rangeRequest(start, end string, rev, limit int64) model.EtcdRequest { return model.EtcdRequest{ Type: model.Range, Range: &model.RangeRequest{ RangeOptions: model.RangeOptions{ Start: start, End: end, Limit: limit, }, Revision: rev, }, } } func rangeResponse(count int64, kvs ...model.KeyValue) model.MaybeEtcdResponse { if kvs == nil { kvs = []model.KeyValue{} } return model.MaybeEtcdResponse{ EtcdResponse: model.EtcdResponse{ Range: &model.RangeResponse{ KVs: kvs, Count: count, }, }, } } func errorResponse(err error) model.MaybeEtcdResponse { return model.MaybeEtcdResponse{ Error: err.Error(), } } func keyValueRevision(key, value string, rev int64) model.KeyValue { return model.KeyValue{ Key: key, ValueRevision: model.ValueRevision{ Value: model.ToValueOrHash(value), ModRevision: rev, Version: 1, }, } } func BenchmarkValidateLinearizableOperations(b *testing.B) { lg := zap.NewNop() b.Run("SequentialSuccessPuts", func(b *testing.B) { history := sequentialSuccessPuts(7000, 2) shuffles := shuffleHistory(history, b.N) b.ResetTimer() validateShuffles(b, lg, shuffles, time.Second) }) b.Run("SequentialFailedPuts", func(b *testing.B) { history := sequentialFailedPuts(14, 1) shuffles := shuffleHistory(history, b.N) b.ResetTimer() validateShuffles(b, lg, shuffles, time.Second) }) b.Run("ConcurrentFailedPutsWithRead", func(b *testing.B) { history := concurrentFailedPutsWithRead(b, 13) shuffles := shuffleHistory(history, b.N) b.ResetTimer() validateShuffles(b, lg, shuffles, time.Second) }) b.Run("BacktrackingHeavy", func(b *testing.B) { history := backtrackingHeavy(b) shuffles := shuffleHistory(history, b.N) b.ResetTimer() for i := 0; i < len(shuffles); i++ { validateLinearizableOperationsAndVisualize(lg, shuffles[i], time.Second) } }) } func sequentialSuccessPuts(count int, startRevision int64) []porcupine.Operation { ops := []porcupine.Operation{} for i := 0; i < count; i++ { ops = append(ops, porcupine.Operation{ ClientId: i, Input: putRequest("key", "value"), Output: txnResponse(startRevision+int64(i), model.EtcdOperationResult{}), Call: int64(i * 2), Return: int64(i*2 + 1), }) } return ops } func concurrentFailedPutsWithRead(b *testing.B, concurrencyCount int) []porcupine.Operation { ops := []porcupine.Operation{} for i := 0; i < concurrencyCount; i++ { ops = append(ops, porcupine.Operation{ ClientId: i, Input: putRequest("key", "value"), Output: errorResponse(fmt.Errorf("timeout")), Call: int64(i), Return: int64(i) + int64(concurrencyCount), }) } replay := model.NewReplayFromOperations(ops) state, err := replay.StateForRevision(int64(concurrencyCount) + 1) if err != nil { b.Fatal(err) } request := rangeRequest("key", "kez", 0, 0) _, resp := state.Step(request) ops = append(ops, porcupine.Operation{ ClientId: 0, Input: request, Output: resp, Call: int64(concurrencyCount) + 1, Return: int64(concurrencyCount) + 2, }) return ops } func sequentialFailedPuts(count int, keyCount int) []porcupine.Operation { ops := []porcupine.Operation{} for i := 0; i < count; i++ { key := "key0" if keyCount > 1 { key = fmt.Sprintf("key%d", i%keyCount) } ops = append(ops, porcupine.Operation{ ClientId: i, Input: putRequest(key, "value"), Output: errorResponse(fmt.Errorf("timeout")), Call: int64(i * 2), Return: int64(i*2 + 1), }) } return ops } func backtrackingHeavy(b *testing.B) (ops []porcupine.Operation) { for i := 0; i < 30; i++ { ops = append(ops, porcupine.Operation{ ClientId: -1, Input: putRequest(fmt.Sprintf("key%d", i+1000), "value"), Output: txnResponse(int64(i+2), model.EtcdOperationResult{}), Call: int64(i), Return: int64(i) + 1, }) } startTime := int64(1000) failedPuts := 4 for i := 0; i < failedPuts; i++ { ops = append(ops, porcupine.Operation{ ClientId: i, Input: putRequest(fmt.Sprintf("key%d", i), "value"), Output: errorResponse(fmt.Errorf("timeout")), Call: startTime + int64(i), Return: startTime + 1000 + int64(i), }) } replay := model.NewReplayFromOperations(ops) state, err := replay.StateForRevision(int64(30 + 1)) if err != nil { b.Fatal(err) } concurrentReads := 3 for i := 0; i < concurrentReads; i++ { request := rangeRequest(fmt.Sprintf("key%d", i), "", 0, 0) _, resp := state.Step(request) ops = append(ops, porcupine.Operation{ ClientId: failedPuts + i, Input: request, Output: resp, Call: startTime + 1100, Return: startTime + 2100, }) } ops = append(ops, porcupine.Operation{ ClientId: 99, Input: rangeRequest("key0", "", 0, 0), Output: rangeResponse(0, keyValueRevision("key0", "wrong", 9999)), Call: startTime + 3000, Return: startTime + 4000, }) return ops } func shuffleHistory(history []porcupine.Operation, shuffleCount int) [][]porcupine.Operation { shuffles := make([][]porcupine.Operation, shuffleCount) for i := 0; i < shuffleCount; i++ { historyCopy := make([]porcupine.Operation, len(history)) copy(historyCopy, history) rand.Shuffle(len(historyCopy), func(i, j int) { historyCopy[i], historyCopy[j] = historyCopy[j], historyCopy[i] }) shuffles[i] = historyCopy } return shuffles } func validateShuffles(b *testing.B, lg *zap.Logger, shuffles [][]porcupine.Operation, duration time.Duration) { for i := 0; i < len(shuffles); i++ { result := validateLinearizableOperationsAndVisualize(lg, shuffles[i], duration) if err := result.Error(); err != nil { b.Fatalf("Not linearizable: %v", err) } } } ================================================ FILE: tests/robustness/validate/patch_history.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package validate import ( "fmt" "math" "github.com/anishathalye/porcupine" "go.etcd.io/etcd/tests/v3/robustness/model" "go.etcd.io/etcd/tests/v3/robustness/report" ) type patchArgs struct { returnTime int64 clientCount int64 persistedCount int64 revision int64 } func patchLinearizableOperations(operations []porcupine.Operation, reports []report.ClientReport, persistedRequests []model.EtcdRequest) []porcupine.Operation { putRevision := watchRevisions(reports) persistedPutCount := countPersistedPuts(persistedRequests) clientPutCount := countClientPuts(reports) persistedDeleteCount := countPersistedDeletes(persistedRequests) clientDeleteCount := countClientDeletes(reports) persistedCompactCount := countPersistedCompacts(persistedRequests) clientCompactCount := countClientCompacts(reports) putReturnTime, delReturnTime, compactReturnTime := uniqueOperationReturnTime(operations, persistedRequests, clientPutCount, clientDeleteCount, clientCompactCount) putArgs := make(map[model.PutOptions]patchArgs) for opts, c := range clientPutCount { putArgs[opts] = patchArgs{ clientCount: c, persistedCount: persistedPutCount[opts], returnTime: putReturnTime[opts], revision: putRevision[opts], } } delArgs := make(map[model.DeleteOptions]patchArgs) for opts, c := range clientDeleteCount { delArgs[opts] = patchArgs{ clientCount: c, persistedCount: persistedDeleteCount[opts], returnTime: delReturnTime[opts], } } compactArgs := make(map[model.CompactRequest]patchArgs) for opts, c := range clientCompactCount { compactArgs[opts] = patchArgs{ clientCount: c, persistedCount: persistedCompactCount[opts], returnTime: compactReturnTime[opts], } } return patchOperations( operations, putArgs, delArgs, compactArgs, ) } func watchRevisions(reports []report.ClientReport) map[model.PutOptions]int64 { putRevisions := map[model.PutOptions]int64{} for _, client := range reports { for _, watch := range client.Watch { for _, resp := range watch.Responses { for _, event := range resp.Events { switch event.Type { case model.RangeOperation: case model.PutOperation: kv := model.PutOptions{Key: event.Key, Value: event.Value} putRevisions[kv] = event.Revision case model.DeleteOperation: // Delete events are also created by leaseRevoke request. default: panic(fmt.Sprintf("unknown event type %q", event.Type)) } } } } } return putRevisions } func patchOperations( operations []porcupine.Operation, putArgs map[model.PutOptions]patchArgs, delArgs map[model.DeleteOptions]patchArgs, compactArgs map[model.CompactRequest]patchArgs, ) []porcupine.Operation { newOperations := make([]porcupine.Operation, 0, len(operations)) for _, op := range operations { request := op.Input.(model.EtcdRequest) resp := op.Output.(model.MaybeEtcdResponse) if request.Type == model.Compact { kv := model.CompactRequest{Revision: request.Compact.Revision} if arg, ok := compactArgs[kv]; ok && arg.clientCount == 1 { if arg.persistedCount == 0 && resp.Error != "" { // the failed compact should be dropped continue } if arg.returnTime > 0 { op.Return = min(op.Return, arg.returnTime) } } newOperations = append(newOperations, op) continue } if resp.Error == "" || request.Type != model.Txn { // Cannot patch those requests. newOperations = append(newOperations, op) continue } var txnRevision int64 var persisted bool for _, etcdOp := range request.Txn.AllOperations() { switch etcdOp.Type { case model.PutOperation: kv := model.PutOptions{Key: etcdOp.Put.Key, Value: etcdOp.Put.Value} arg, ok := putArgs[kv] if !ok { continue } if arg.persistedCount > 0 { persisted = true } if arg.clientCount != 1 { continue } if arg.revision > 0 { txnRevision = arg.revision } if arg.returnTime > 0 { op.Return = min(op.Return, arg.returnTime) } case model.DeleteOperation: kv := model.DeleteOptions{Key: etcdOp.Delete.Key} arg, ok := delArgs[kv] if !ok { continue } if arg.persistedCount > 0 { persisted = true } if arg.clientCount != 1 { continue } if arg.revision > 0 { txnRevision = arg.revision } if arg.returnTime > 0 { op.Return = min(op.Return, arg.returnTime) } case model.RangeOperation: default: panic(fmt.Sprintf("unknown operation type %q", etcdOp.Type)) } } if isUniqueTxn(request.Txn, putArgs, delArgs) { if !persisted { // Remove non persisted operations continue } if txnRevision != 0 { op.Output = model.MaybeEtcdResponse{Persisted: true, PersistedRevision: txnRevision} } else { op.Output = model.MaybeEtcdResponse{Persisted: true} } } // Leave operation as it is as we cannot discard it. newOperations = append(newOperations, op) } return newOperations } func isUniqueTxn(request *model.TxnRequest, putArgs map[model.PutOptions]patchArgs, delArgs map[model.DeleteOptions]patchArgs) bool { return isUniqueOps(request.OperationsOnSuccess, putArgs, delArgs) && isUniqueOps(request.OperationsOnFailure, putArgs, delArgs) } func isUniqueOps(ops []model.EtcdOperation, putArgs map[model.PutOptions]patchArgs, delArgs map[model.DeleteOptions]patchArgs) bool { return hasUniqueWriteOperation(ops, putArgs, delArgs) || !hasWriteOperation(ops) } func hasWriteOperation(ops []model.EtcdOperation) bool { for _, etcdOp := range ops { if etcdOp.Type == model.PutOperation || etcdOp.Type == model.DeleteOperation { return true } } return false } func hasUniqueWriteOperation(ops []model.EtcdOperation, putArgs map[model.PutOptions]patchArgs, delArgs map[model.DeleteOptions]patchArgs) bool { for _, operation := range ops { switch operation.Type { case model.PutOperation: kv := model.PutOptions{Key: operation.Put.Key, Value: operation.Put.Value} if arg, ok := putArgs[kv]; ok && arg.clientCount == 1 { return true } case model.DeleteOperation: kv := model.DeleteOptions{Key: operation.Delete.Key} if arg, ok := delArgs[kv]; ok && arg.clientCount == 1 { return true } case model.RangeOperation: default: panic(fmt.Sprintf("unknown operation type %q", operation.Type)) } } return false } func uniqueOperationReturnTime( allOperations []porcupine.Operation, persistedRequests []model.EtcdRequest, clientPutCount map[model.PutOptions]int64, clientDeleteCount map[model.DeleteOptions]int64, clientCompactCount map[model.CompactRequest]int64, ) ( map[model.PutOptions]int64, map[model.DeleteOptions]int64, map[model.CompactRequest]int64, ) { putTimes := map[model.PutOptions]int64{} delTimes := map[model.DeleteOptions]int64{} compactTimes := map[model.CompactRequest]int64{} var lastReturnTime int64 for _, op := range allOperations { request := op.Input.(model.EtcdRequest) switch request.Type { case model.Txn: for _, etcdOp := range request.Txn.AllOperations() { switch etcdOp.Type { case model.PutOperation: kv := model.PutOptions{Key: etcdOp.Put.Key, Value: etcdOp.Put.Value} if clientPutCount[kv] > 1 { continue } if returnTime, ok := putTimes[kv]; !ok || returnTime > op.Return { putTimes[kv] = op.Return } case model.DeleteOperation: kv := model.DeleteOptions{Key: etcdOp.Delete.Key} if clientDeleteCount[kv] > 1 { continue } if returnTime, ok := delTimes[kv]; !ok || returnTime > op.Return { delTimes[kv] = op.Return } } } case model.Range: case model.LeaseGrant: case model.LeaseRevoke: case model.Compact: if clientCompactCount[*request.Compact] > 1 { continue } if returnTime, ok := compactTimes[*request.Compact]; !ok || returnTime > op.Return { compactTimes[*request.Compact] = op.Return } case model.Defragment: default: panic(fmt.Sprintf("Unknown request type: %q", request.Type)) } if op.Return > lastReturnTime { lastReturnTime = op.Return } } for i := len(persistedRequests) - 1; i >= 0; i-- { request := persistedRequests[i] switch request.Type { case model.Txn: if lastReturnTime != math.MaxInt64 { lastReturnTime-- } for _, op := range request.Txn.AllOperations() { switch op.Type { case model.PutOperation: kv := model.PutOptions{Key: op.Put.Key, Value: op.Put.Value} if clientPutCount[kv] > 1 { continue } if returnTime, ok := putTimes[kv]; ok { lastReturnTime = min(returnTime, lastReturnTime) putTimes[kv] = lastReturnTime } case model.DeleteOperation: kv := model.DeleteOptions{Key: op.Delete.Key} if clientDeleteCount[kv] > 1 { continue } if returnTime, ok := delTimes[kv]; ok { lastReturnTime = min(returnTime, lastReturnTime) delTimes[kv] = lastReturnTime } } } case model.LeaseGrant: case model.LeaseRevoke: case model.Compact: if lastReturnTime != math.MaxInt64 { lastReturnTime-- } if clientCompactCount[*request.Compact] > 1 { continue } if returnTime, ok := compactTimes[*request.Compact]; ok { lastReturnTime = min(returnTime, lastReturnTime) compactTimes[*request.Compact] = lastReturnTime } default: panic(fmt.Sprintf("Unknown request type: %q", request.Type)) } } return putTimes, delTimes, compactTimes } func countClientPuts(reports []report.ClientReport) map[model.PutOptions]int64 { counter := map[model.PutOptions]int64{} for _, client := range reports { for _, op := range client.KeyValue { request := op.Input.(model.EtcdRequest) countPuts(counter, request) } } return counter } func countPersistedPuts(requests []model.EtcdRequest) map[model.PutOptions]int64 { counter := map[model.PutOptions]int64{} for _, request := range requests { countPuts(counter, request) } return counter } func countPuts(counter map[model.PutOptions]int64, request model.EtcdRequest) { switch request.Type { case model.Txn: for _, operation := range request.Txn.AllOperations() { switch operation.Type { case model.PutOperation: kv := model.PutOptions{Key: operation.Put.Key, Value: operation.Put.Value} counter[kv]++ case model.DeleteOperation: case model.RangeOperation: default: panic(fmt.Sprintf("unknown operation type %q", operation.Type)) } } case model.LeaseGrant: case model.LeaseRevoke: case model.Compact: case model.Defragment: case model.Range: default: panic(fmt.Sprintf("unknown request type %q", request.Type)) } } func countClientDeletes(reports []report.ClientReport) map[model.DeleteOptions]int64 { counter := map[model.DeleteOptions]int64{} for _, client := range reports { for _, op := range client.KeyValue { request := op.Input.(model.EtcdRequest) countDeletes(counter, request) } } return counter } func countPersistedDeletes(requests []model.EtcdRequest) map[model.DeleteOptions]int64 { counter := map[model.DeleteOptions]int64{} for _, req := range requests { countDeletes(counter, req) } return counter } func countDeletes(counter map[model.DeleteOptions]int64, request model.EtcdRequest) { if request.Type != model.Txn { return } for _, operation := range request.Txn.AllOperations() { if operation.Type == model.DeleteOperation { counter[operation.Delete]++ } } } func countClientCompacts(reports []report.ClientReport) map[model.CompactRequest]int64 { counter := map[model.CompactRequest]int64{} for _, client := range reports { for _, op := range client.KeyValue { request := op.Input.(model.EtcdRequest) countCompacts(counter, request) } } return counter } func countPersistedCompacts(requests []model.EtcdRequest) map[model.CompactRequest]int64 { counter := map[model.CompactRequest]int64{} for _, req := range requests { countCompacts(counter, req) } return counter } func countCompacts(counter map[model.CompactRequest]int64, request model.EtcdRequest) { if request.Type == model.Compact { counter[*request.Compact]++ } } ================================================ FILE: tests/robustness/validate/patch_history_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //nolint:unparam package validate import ( "errors" "math" "testing" "time" "github.com/anishathalye/porcupine" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/robustness/identity" "go.etcd.io/etcd/tests/v3/robustness/model" "go.etcd.io/etcd/tests/v3/robustness/report" ) const infinite = math.MaxInt64 func TestPatchHistory(t *testing.T) { for _, tc := range []struct { name string historyFunc func(h *model.AppendableHistory) persistedRequest []model.EtcdRequest watchOperations []model.WatchOperation expectedRemainingOperations []porcupine.Operation }{ { name: "successful range remains", historyFunc: func(h *model.AppendableHistory) { h.AppendRange("key", "", 0, 0, 100, 200, &clientv3.GetResponse{}, nil) }, expectedRemainingOperations: []porcupine.Operation{ {Return: 200, Output: rangeResponse(0)}, }, }, { name: "successful put remains", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key", "value", 100, 200, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put remains if there is a matching event, return time untouched", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key", "value", 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed put remains if there is a matching event, uniqueness allows for return time to be based on next persisted request", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key1", "value", 100, 200, nil, errors.New("failed")) h.AppendPut("key2", "value", 300, 400, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ putRequest("key1", "value"), putRequest("key2", "value"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: 399, Output: model.MaybeEtcdResponse{Persisted: true}}, {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put remains if there is a matching persisted request, uniqueness allows for revision to be based on watch", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key", "value", 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value"), }, watchOperations: watchResponse(300, putEvent("key", "value", 2)), expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}}, }, }, { name: "failed put remains if there is a matching persisted request, lack of uniqueness causes time to be untouched regardless of persisted event and watch", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key", "value", 1, 2, nil, errors.New("failed")) h.AppendPut("key", "value", 3, 4, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value"), putRequest("key", "value"), }, watchOperations: watchResponse(3, putEvent("key", "value", 2), putEvent("key", "value", 3)), expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, {Return: 4, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put is dropped if event has different key", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key2", "value", 100, 200, &clientv3.PutResponse{}, nil) h.AppendPut("key1", "value", 300, 400, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key2", "value"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put is dropped if event has different value", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key", "value2", 100, 200, &clientv3.PutResponse{}, nil) h.AppendPut("key", "value1", 300, 400, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value2"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put with lease remains if there is a matching event, return time untouched", historyFunc: func(h *model.AppendableHistory) { h.AppendPutWithLease("key", "value", 123, 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequestWithLease("key", "value", 123), }, expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed put with lease remains if there is a matching event, uniqueness allows return time to be based on next persisted request", historyFunc: func(h *model.AppendableHistory) { h.AppendPutWithLease("key1", "value", 123, 100, 200, nil, errors.New("failed")) h.AppendPutWithLease("key2", "value", 234, 300, 400, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ putRequestWithLease("key1", "value", 123), putRequestWithLease("key2", "value", 234), }, expectedRemainingOperations: []porcupine.Operation{ {Return: 399, Output: model.MaybeEtcdResponse{Persisted: true}}, {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put with lease remains if there is a matching event, uniqueness allows for revision to be based on watch", historyFunc: func(h *model.AppendableHistory) { h.AppendPutWithLease("key", "value", 123, 1, 2, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequestWithLease("key", "value", 123), }, watchOperations: watchResponse(3, putEvent("key", "value", 2)), expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true, PersistedRevision: 2}}, }, }, { name: "failed put with lease remains if there is a matching persisted request, lack of uniqueness causes time to be untouched regardless of persisted event and watch", historyFunc: func(h *model.AppendableHistory) { h.AppendPutWithLease("key", "value", 123, 1, 2, nil, errors.New("failed")) h.AppendPutWithLease("key", "value", 321, 3, 4, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ putRequestWithLease("key", "value", 123), putRequestWithLease("key", "value", 321), }, watchOperations: watchResponse(3, putEvent("key", "value", 2), putEvent("key", "value", 3)), expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, {Return: 4, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put is dropped", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key", "value", 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed put with lease is dropped", historyFunc: func(h *model.AppendableHistory) { h.AppendPutWithLease("key", "value", 123, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "successful delete remains", historyFunc: func(h *model.AppendableHistory) { h.AppendDelete("key", 100, 200, &clientv3.DeleteResponse{}, nil) }, expectedRemainingOperations: []porcupine.Operation{ {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed delete with lease is dropped", historyFunc: func(h *model.AppendableHistory) { h.AppendDelete("key", 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed delete remains, if there is a persisted request", historyFunc: func(h *model.AppendableHistory) { h.AppendDelete("key", 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ deleteRequest("key"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed delete remains, if there is a persisted request, revision is not patched based on watch due to leaseRevoke also triggering delete watch events", historyFunc: func(h *model.AppendableHistory) { h.AppendDelete("key", 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ deleteRequest("key"), }, watchOperations: watchResponse(3, deleteEvent("key", 2)), expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed delete remains, if there is a matching persisted request, uniqueness of this operation and following operation allows patching based on following operation", historyFunc: func(h *model.AppendableHistory) { h.AppendDelete("key", 100, 200, nil, errors.New("failed")) h.AppendPut("key", "value", 300, 400, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ deleteRequest("key"), putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: 399, Output: model.MaybeEtcdResponse{Persisted: true}}, {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed delete remains if there is a matching persisted request, lack of uniqueness of this operation prevents patching based on following operation", historyFunc: func(h *model.AppendableHistory) { h.AppendDelete("key", 100, 200, nil, errors.New("failed")) h.AppendPut("key", "value", 300, 400, &clientv3.PutResponse{}, nil) h.AppendDelete("key", 500, 600, &clientv3.DeleteResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ deleteRequest("key"), putRequest("key", "value"), deleteRequest("key"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed empty txn is dropped", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn put is dropped", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn put remains if there is a matching event", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed txn delete remains", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{clientv3.OpDelete("key")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ deleteRequest("key"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "successful txn put/delete remains", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{clientv3.OpDelete("key")}, 100, 200, &clientv3.TxnResponse{Succeeded: true}, nil) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ // It's successful, so it remains as-is with its original response {Return: 200, Output: txnResponse(0)}, }, }, { name: "failed txn empty/delete is dropped", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{clientv3.OpDelete("key")}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn delete/put remains", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{clientv3.OpDelete("key")}, []clientv3.Op{clientv3.OpPut("key", "value")}, 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ deleteRequest("key"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed txn empty/put is dropped", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{clientv3.OpPut("key", "value")}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn empty/put remains if there is a matching event", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed put remains if there is a matching persisted request, uniqueness of this operation and following operation allows patching based on following operation", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key1", "value1", 300, 400, nil, errors.New("failed")) h.AppendPut("key2", "value2", 500, 600, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ putRequest("key1", "value1"), putRequest("key2", "value2"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: 599, Output: model.MaybeEtcdResponse{Persisted: true}}, {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put remains if there is a matching persisted request, lack of uniqueness of this operation prevents patching based on following operation", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key1", "value1", 100, 200, &clientv3.PutResponse{}, nil) h.AppendPut("key1", "value1", 300, 400, nil, errors.New("failed")) h.AppendPut("key2", "value2", 500, 600, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ putRequest("key1", "value1"), putRequest("key1", "value1"), putRequest("key2", "value2"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed put remains if there is a matching persisted request, lack of uniqueness of following operation prevents patching based on following operation", historyFunc: func(h *model.AppendableHistory) { h.AppendPut("key2", "value2", 100, 200, &clientv3.PutResponse{}, nil) h.AppendPut("key1", "value1", 300, 400, nil, errors.New("failed")) h.AppendPut("key2", "value2", 500, 600, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ putRequest("key2", "value2"), putRequest("key1", "value1"), putRequest("key2", "value2"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: 200, Output: txnResponse(0, model.EtcdOperationResult{})}, // TODO: We can infer that failed operation finished before last operation matching following. {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed txn empty/delete remains", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{clientv3.OpDelete("key")}, 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{ deleteRequest("key"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Persisted: true}}, }, }, { name: "failed txn put&delete is dropped", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value1"), clientv3.OpDelete("key")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn empty/put&delete is dropped", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{}, []clientv3.Op{clientv3.OpPut("key", "value1"), clientv3.OpDelete("key")}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed txn put&delete/put&delete is dropped", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{clientv3.OpPut("key", "value1"), clientv3.OpDelete("key")}, []clientv3.Op{clientv3.OpPut("key", "value2"), clientv3.OpDelete("key")}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "failed delete remains, time untouched due to non-uniqueness", historyFunc: func(h *model.AppendableHistory) { h.AppendDelete("key", 100, 200, nil, errors.New("failed")) h.AppendDelete("key", 300, 400, &clientv3.DeleteResponse{}, nil) h.AppendPut("key", "value", 500, 600, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ deleteRequest("key"), deleteRequest("key"), putRequest("key", "value"), }, watchOperations: watchResponse(250, deleteEvent("key", 2)), expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed txn delete is dropped when not persisted", historyFunc: func(h *model.AppendableHistory) { h.AppendTxn(nil, []clientv3.Op{clientv3.OpDelete("key")}, []clientv3.Op{}, 100, 200, nil, errors.New("failed")) }, expectedRemainingOperations: []porcupine.Operation{}, }, { name: "successful compact with unique revision keeps original return time", historyFunc: func(h *model.AppendableHistory) { h.AppendCompact(5, 100, 200, &clientv3.CompactResponse{}, nil) h.AppendPut("key", "value", 300, 400, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ compactRequest(5), putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: 200, Output: model.MaybeEtcdResponse{EtcdResponse: model.EtcdResponse{Revision: -1, Compact: &model.CompactResponse{}}}}, {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed compact with unique revision is patched with return time", historyFunc: func(h *model.AppendableHistory) { h.AppendCompact(5, 100, 200, nil, errors.New("failed")) h.AppendPut("key", "value", 300, 400, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ compactRequest(5), putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: 399, Output: model.MaybeEtcdResponse{Error: "failed"}}, {Return: 400, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed compact with non-unique revision remains unchanged", historyFunc: func(h *model.AppendableHistory) { h.AppendCompact(5, 100, 200, nil, errors.New("failed")) h.AppendCompact(5, 300, 400, &clientv3.CompactResponse{}, nil) h.AppendPut("key", "value", 500, 600, &clientv3.PutResponse{}, nil) }, persistedRequest: []model.EtcdRequest{ compactRequest(5), compactRequest(5), putRequest("key", "value"), }, expectedRemainingOperations: []porcupine.Operation{ {Return: infinite, Output: model.MaybeEtcdResponse{Error: "failed"}}, {Return: 400, Output: model.MaybeEtcdResponse{EtcdResponse: model.EtcdResponse{Revision: -1, Compact: &model.CompactResponse{}}}}, {Return: 600, Output: txnResponse(0, model.EtcdOperationResult{})}, }, }, { name: "failed compact with unique revision is dropped when not persisted", historyFunc: func(h *model.AppendableHistory) { h.AppendCompact(5, 100, 200, nil, errors.New("failed")) }, persistedRequest: []model.EtcdRequest{}, expectedRemainingOperations: []porcupine.Operation{}, }, } { t.Run(tc.name, func(t *testing.T) { history := model.NewAppendableHistory(identity.NewIDProvider()) tc.historyFunc(history) reports := []report.ClientReport{ { ClientID: 0, KeyValue: history.History.Operations(), Watch: tc.watchOperations, }, } operations, _, _ := prepareAndCategorizeOperations(reports) patched := patchLinearizableOperations(operations, reports, tc.persistedRequest) if diff := cmp.Diff(tc.expectedRemainingOperations, patched, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(porcupine.Operation{}, "Input", "Call", "ClientId", "Metadata"), ); diff != "" { t.Errorf("Response didn't match expected, diff:\n%s", diff) } }) } } func txnResponse(rev int64, result ...model.EtcdOperationResult) model.MaybeEtcdResponse { return model.MaybeEtcdResponse{EtcdResponse: model.EtcdResponse{Revision: rev, Txn: &model.TxnResponse{Results: result}}} } func watchResponse(responseTime int64, events ...model.WatchEvent) []model.WatchOperation { return []model.WatchOperation{ { Responses: []model.WatchResponse{ { Time: time.Duration(responseTime), Events: events, }, }, }, } } func putEvent(key, value string, revision int64) model.WatchEvent { return model.WatchEvent{ PersistedEvent: model.PersistedEvent{ Event: model.Event{ Type: model.PutOperation, Key: key, Value: model.ToValueOrHash(value), }, Revision: revision, }, } } func deleteEvent(key string, revision int64) model.WatchEvent { return model.WatchEvent{ PersistedEvent: model.PersistedEvent{ Event: model.Event{ Type: model.DeleteOperation, Key: key, }, Revision: revision, }, } } func compactRequest(revision int64) model.EtcdRequest { return model.EtcdRequest{ Type: model.Compact, Compact: &model.CompactRequest{ Revision: revision, }, } } ================================================ FILE: tests/robustness/validate/result.go ================================================ // Copyright 2025 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package validate import ( "errors" "fmt" "github.com/anishathalye/porcupine" "go.uber.org/zap" ) type RobustnessResult struct { Assumptions Result Linearization LinearizationResult Watch Result Serializable Result } type Result struct { Status ResultStatus Message string } type ResultStatus string var ( Unknown ResultStatus Success ResultStatus = "Success" Failure ResultStatus = "Failure" ) func (r RobustnessResult) Error() error { if err := r.Assumptions.Error(); err != nil { return fmt.Errorf("assumptions: %w", err) } if err := r.Linearization.Error(); err != nil { return fmt.Errorf("linearization: %w", err) } if err := r.Watch.Error(); err != nil { return fmt.Errorf("watch: %w", err) } if err := r.Serializable.Error(); err != nil { return fmt.Errorf("serializable: %w", err) } return nil } func ResultFromError(err error) Result { if err != nil { return Result{ Status: Failure, Message: err.Error(), } } return Result{ Status: Success, } } func (r Result) Error() error { if r.Status == Failure { if r.Message != "" { return errors.New(r.Message) } return errors.New("failure") } return nil } type LinearizationResult struct { Info porcupine.LinearizationInfo Model porcupine.Model Result Timeout bool } func (r *LinearizationResult) Visualize(lg *zap.Logger, path string) error { lg.Info("Saving visualization", zap.String("path", path)) err := porcupine.VisualizePath(r.Model, r.Info, path) if err != nil { return fmt.Errorf("failed to visualize, err: %w", err) } return nil } func (r *LinearizationResult) AddToVisualization(serializable []porcupine.Operation) { annotations := []porcupine.Annotation{} for _, op := range serializable { annotations = append(annotations, porcupine.Annotation{ ClientId: op.ClientId, Start: op.Call, End: op.Return, Description: r.Model.DescribeOperation(op.Input, op.Output), Details: r.Model.DescribeOperationMetadata(op.Metadata), }) } r.Info.AddAnnotations(annotations) } ================================================ FILE: tests/robustness/validate/validate.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package validate import ( "errors" "fmt" "math" "time" "github.com/anishathalye/porcupine" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/robustness/model" "go.etcd.io/etcd/tests/v3/robustness/report" ) var ErrNotEmptyDatabase = errors.New("non empty database at start, required by model used for linearizability validation") func ValidateAndReturnVisualize(lg *zap.Logger, cfg Config, reports []report.ClientReport, persistedRequests []model.EtcdRequest, timeout time.Duration) (result RobustnessResult) { result.Assumptions = ResultFromError(checkValidationAssumptions(reports)) if result.Assumptions.Error() != nil { return result } linearizableOperations, serializableOperations, operationsForVisualization := prepareAndCategorizeOperations(reports) // We are passing in the original reports and linearizableOperations with modified return time. // The reason is that linearizableOperations are those dedicated for linearization, which requires them to have returnTime set to infinity as required by pourcupine. // As for the report, the original report is used so the consumer doesn't need to track what patching was done or not. if len(persistedRequests) != 0 { linearizableOperations = patchLinearizableOperations(linearizableOperations, reports, persistedRequests) } result.Linearization = validateLinearizableOperationsAndVisualize(lg, linearizableOperations, timeout) result.Linearization.AddToVisualization(operationsForVisualization) // Skip other validations if model is not linearizable, as they are expected to fail too and obfuscate the logs. if result.Linearization.Error() != nil { lg.Info("Skipping other validations as linearization failed") return result } if len(persistedRequests) == 0 { lg.Info("Skipping other validations as persisted requests were empty") return result } replay := model.NewReplay(persistedRequests) result.Watch = validateWatch(lg, cfg, reports, replay) result.Serializable = validateSerializableOperations(lg, serializableOperations, replay) return result } type Config struct { ExpectRevisionUnique bool } func prepareAndCategorizeOperations(reports []report.ClientReport) (linearizable, serializable, forVisualization []porcupine.Operation) { for _, report := range reports { for _, op := range report.KeyValue { request := op.Input.(model.EtcdRequest) response := op.Output.(model.MaybeEtcdResponse) if isSerializable(request, response) { serializable = append(serializable, op) } // Operations that will not be linearized need to be added separately to the visualization. if !isLinearizable(request, response) { forVisualization = append(forVisualization, op) continue } // For linearization, we set the return time of failed requests to MaxInt64. // Failed requests can still be persisted, however we don't know when the request has taken effect. if response.Error != "" { op.Return = math.MaxInt64 } linearizable = append(linearizable, op) } } return linearizable, serializable, forVisualization } func isLinearizable(request model.EtcdRequest, response model.MaybeEtcdResponse) bool { // Cannot test response for request without side effect. if request.IsRead() && response.Error != "" { return false } // Defragment is not linearizable if request.Type == model.Defragment { return false } return true } func isSerializable(request model.EtcdRequest, response model.MaybeEtcdResponse) bool { // Cannot test response for request without side effect. if request.IsRead() && response.Error != "" { return false } // Test range requests about stale revision if request.Type == model.Range && request.Range.Revision != 0 { return true } return false } func checkValidationAssumptions(reports []report.ClientReport) error { err := validateEmptyDatabaseAtStart(reports) if err != nil { return err } err = validateNonConcurrentClientRequests(reports) if err != nil { return err } return nil } func validateEmptyDatabaseAtStart(reports []report.ClientReport) error { if len(reports) == 0 { return nil } for _, r := range reports { for _, op := range r.KeyValue { request := op.Input.(model.EtcdRequest) response := op.Output.(model.MaybeEtcdResponse) if response.Revision == 1 && request.IsRead() { return nil } } } return ErrNotEmptyDatabase } func validateNonConcurrentClientRequests(reports []report.ClientReport) error { lastClientRequestReturn := map[int]int64{} for _, r := range reports { for _, op := range r.KeyValue { lastRequest := lastClientRequestReturn[op.ClientId] if op.Call <= lastRequest { return fmt.Errorf("client %d has concurrent request, required for operation linearization", op.ClientId) } if op.Return <= op.Call { return fmt.Errorf("operation %v ends before it starts, required for operation linearization", op) } lastClientRequestReturn[op.ClientId] = op.Return } } return nil } ================================================ FILE: tests/robustness/validate/validate_test.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //nolint:unparam package validate import ( "os" "path/filepath" "testing" "time" "github.com/anishathalye/porcupine" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/tests/v3/framework/testutils" "go.etcd.io/etcd/tests/v3/robustness/model" "go.etcd.io/etcd/tests/v3/robustness/report" ) func TestDataReports(t *testing.T) { testdataPath := testutils.MustAbsPath("../testdata/") files, err := os.ReadDir(testdataPath) require.NoError(t, err) for _, file := range files { if !file.IsDir() { continue } t.Run(file.Name(), func(t *testing.T) { lg := zaptest.NewLogger(t) path := filepath.Join(testdataPath, file.Name()) reports, err := report.LoadClientReports(path) require.NoError(t, err) persistedRequests, err := report.LoadClusterPersistedRequests(lg, path) if err != nil { t.Error(err) } result := ValidateAndReturnVisualize(zaptest.NewLogger(t), Config{}, reports, persistedRequests, 5*time.Minute) err = result.Error() if err != nil { t.Error(err) } err = result.Linearization.Visualize(lg, filepath.Join(path, "history.html")) if err != nil { t.Error(err) } }) } } func TestValidateAndReturnVisualize(t *testing.T) { tcs := []struct { name string reports []report.ClientReport persistedRequests []model.EtcdRequest config Config expectError string }{ { name: "Success with no persisted requests", reports: []report.ClientReport{ { KeyValue: []porcupine.Operation{ { ClientId: 0, Input: getRequest("key"), Call: 100, Output: getResponse(1), Return: 200, }, }, }, }, expectError: "", }, { name: "Failure with not empty database", reports: []report.ClientReport{ { KeyValue: []porcupine.Operation{ { ClientId: 0, Input: getRequest("key"), Call: 100, // Empty database should have revision 1 Output: getResponse(2), Return: 200, }, }, }, }, expectError: "non empty database at start, required by model used for linearizability validation", }, { name: "Success", reports: []report.ClientReport{ { KeyValue: []porcupine.Operation{ { ClientId: 0, Input: getRequest("key"), Call: 100, Output: getResponse(1), Return: 200, }, { ClientId: 0, Input: putRequest("key", "value"), Call: 300, Output: txnResponse(2, model.EtcdOperationResult{}), Return: 400, }, }, Watch: []model.WatchOperation{ { Request: model.WatchRequest{Key: "key"}, Responses: []model.WatchResponse{ {Events: []model.WatchEvent{watchEvent(2, true, model.PutOperation, "key", "value")}}, }, }, }, }, }, persistedRequests: []model.EtcdRequest{putRequest("key", "value")}, expectError: "", }, { name: "Failure of watch", reports: []report.ClientReport{ { KeyValue: []porcupine.Operation{ { ClientId: 0, Input: getRequest("key"), Call: 100, Output: getResponse(1), Return: 200, }, { ClientId: 0, Input: putRequest("key", "value"), Call: 300, Output: txnResponse(2, model.EtcdOperationResult{}), Return: 400, }, }, Watch: []model.WatchOperation{ { Request: model.WatchRequest{Key: "key"}, Responses: []model.WatchResponse{ {Events: []model.WatchEvent{watchEvent(2, true, model.PutOperation, "key", "value2")}}, }, }, }, }, }, persistedRequests: []model.EtcdRequest{putRequest("key", "value")}, expectError: "watch: broke Reliable", }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { lg := zaptest.NewLogger(t) result := ValidateAndReturnVisualize(lg, Config{}, tc.reports, tc.persistedRequests, 5*time.Second) if tc.expectError != "" { require.ErrorContains(t, result.Error(), tc.expectError) } else { require.NoError(t, result.Error()) } err := result.Linearization.Visualize(lg, filepath.Join(t.TempDir(), "history.html")) require.NoError(t, err) }) } } func watchEvent(rev int64, isCreate bool, eventType model.OperationType, key, value string) model.WatchEvent { return model.WatchEvent{PersistedEvent: model.PersistedEvent{Revision: rev, IsCreate: isCreate, Event: model.Event{Type: eventType, Key: key, Value: model.ToValueOrHash(value)}}} } func TestValidateWatch(t *testing.T) { tcs := []struct { name string config Config reports []report.ClientReport persistedRequests []model.EtcdRequest expectError string }{ { name: "Ordered, Unique - ordered unique events in one response - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 3, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), }, }, { name: "Ordered, Unique - unique ordered events in separate response - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, { Events: []model.WatchEvent{ putWatchEvent("b", "2", 3, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), }, }, { name: "Ordered - unordered events in one response - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("b", "2", 3, true), putWatchEvent("a", "1", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), }, expectError: errBrokeOrdered.Error(), }, { name: "Ordered - unordered events in separate response - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("b", "2", 3, true), }, }, { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), }, expectError: errBrokeOrdered.Error(), }, { name: "Ordered - unordered events in separate watch - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "b", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("b", "2", 3, true), }, }, }, }, { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), }, }, { name: "Unique - duplicated events in one response - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("a", "2", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), }, expectError: errBrokeUnique.Error(), }, { name: "Unique - duplicated events in separate responses - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, { Events: []model.WatchEvent{ putWatchEvent("a", "2", 2, true), }, }, }, }, }, }, }, expectError: errBrokeUnique.Error(), persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), }, }, { name: "Unique - duplicated events in watch requests - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, }, }, { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), }, }, { name: "Unique, Atomic - duplicated revision in one response - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ { Type: model.Txn, LeaseGrant: nil, LeaseRevoke: nil, Range: nil, Txn: &model.TxnRequest{ Conditions: nil, OperationsOnSuccess: []model.EtcdOperation{ { Type: model.PutOperation, Put: model.PutOptions{ Key: "a", Value: model.ToValueOrHash("1"), }, }, { Type: model.PutOperation, Put: model.PutOptions{ Key: "b", Value: model.ToValueOrHash("2"), }, }, }, OperationsOnFailure: nil, }, Defragment: nil, }, }, }, { name: "Unique - duplicated revision in separate watch request - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, }, }, { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), }, }, { name: "Unique revision - duplicated revision in one response - fail", config: Config{ExpectRevisionUnique: true}, reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), }, expectError: errBrokeUnique.Error(), }, { name: "Atomic - duplicated revision in one response - fail", config: Config{ExpectRevisionUnique: true}, reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), }, expectError: errBrokeUnique.Error(), }, { name: "Atomic - revision in separate responses - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, { Events: []model.WatchEvent{ putWatchEvent("b", "2", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), }, expectError: errBrokeAtomic.Error(), }, { name: "Resumable, Reliable, Bookmarkable - all events with watch revision and bookmark - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Revision: 2, WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 3, true), putWatchEvent("c", "3", 4, true), }, }, { Revision: 4, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, }, { name: "Resumable, Reliable, Bookmarkable - all events with only bookmarks - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Revision: 1, IsProgressNotify: true, }, { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 3, true), putWatchEvent("c", "3", 4, true), }, }, { Revision: 4, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, }, { name: "Resumable, Reliable, Bookmarkable - empty events without revision - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Revision: 4, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, }, { name: "Resumable, Reliable, Bookmarkable - empty events with watch revision - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Revision: 2, WithPrefix: true, }, Responses: []model.WatchResponse{ { Revision: 4, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, expectError: errBrokeReliable.Error(), }, { name: "Resumable, Reliable, Bookmarkable - unmatched events with watch revision - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "d", Revision: 2, }, Responses: []model.WatchResponse{ { Revision: 2, IsProgressNotify: true, }, { Revision: 3, IsProgressNotify: true, }, { Revision: 3, IsProgressNotify: true, }, { Revision: 4, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, }, { name: "Resumable, Reliable, Bookmarkable - empty events between progress notifies - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Revision: 1, IsProgressNotify: true, }, { Revision: 4, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, expectError: errBrokeReliable.Error(), }, { name: "Resumable, Reliable, Bookmarkable - unmatched events between progress notifies - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "d", }, Responses: []model.WatchResponse{ { Revision: 2, IsProgressNotify: true, }, { Revision: 3, IsProgressNotify: true, }, { Revision: 3, IsProgressNotify: true, }, { Revision: 4, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, }, { name: "Bookmarkable - revision non decreasing - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Revision: 1, IsProgressNotify: true, }, { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, { Revision: 2, IsProgressNotify: true, }, { Events: []model.WatchEvent{ putWatchEvent("b", "2", 3, true), }, }, { Revision: 3, IsProgressNotify: true, }, { Revision: 3, IsProgressNotify: true, }, { Events: []model.WatchEvent{ putWatchEvent("c", "3", 4, true), }, }, { Revision: 4, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, }, { name: "Bookmarkable - event precedes progress - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 3, true), putWatchEvent("c", "3", 4, true), }, }, { Revision: 3, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, expectError: errBrokeBookmarkable.Error(), }, { name: "Bookmarkable - progress precedes event - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 3, true), }, }, { Revision: 4, IsProgressNotify: true, }, { Events: []model.WatchEvent{ putWatchEvent("c", "3", 4, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, expectError: errBrokeBookmarkable.Error(), }, { name: "Bookmarkable - progress precedes other progress - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { IsProgressNotify: true, Revision: 2, }, { IsProgressNotify: true, Revision: 1, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{}, expectError: errBrokeBookmarkable.Error(), }, { name: "Bookmarkable - progress notification lower than watch request - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Revision: 3, WithPrefix: true, }, Responses: []model.WatchResponse{ { IsProgressNotify: true, Revision: 2, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), }, }, { name: "Bookmarkable - empty event history - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { IsProgressNotify: true, Revision: 1, }, { IsProgressNotify: true, Revision: 1, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{}, }, { name: "Reliable - missing event before bookmark - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 3, true), }, }, { Revision: 4, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, expectError: errBrokeReliable.Error(), }, { name: "Reliable - missing event matching watch before bookmark - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, { Revision: 4, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("a", "2"), putRequest("c", "3"), }, expectError: errBrokeReliable.Error(), }, { name: "Reliable - missing event matching watch with prefix before bookmark - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("aa", "1", 2, true), }, }, { Revision: 4, IsProgressNotify: true, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("aa", "1"), putRequest("ab", "2"), putRequest("cc", "3"), }, expectError: errBrokeReliable.Error(), }, { name: "Reliable - all events history - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 3, true), putWatchEvent("c", "3", 4, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, expectError: "", }, { name: "Reliable - single revision - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), }, expectError: "", }, { name: "Reliable - single revision with watch revision - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, Revision: 2, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), }, expectError: "", }, { name: "Reliable - missing single revision with watch revision - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, Revision: 2, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{}, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), }, expectError: "", }, { name: "Reliable - single revision with progress notify - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), }, }, { IsProgressNotify: true, Revision: 2, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), }, expectError: "", }, { name: "Reliable - single revision missing with progress notify - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, Revision: 2, }, Responses: []model.WatchResponse{ { IsProgressNotify: true, Revision: 2, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), }, expectError: errBrokeReliable.Error(), }, { name: "Reliable - missing middle event - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("c", "3", 4, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, expectError: errBrokeReliable.Error(), }, { name: "Reliable - middle event doesn't match request - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("a", "3", 4, false), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("ab", "2"), putRequest("a", "3"), }, }, { name: "Reliable - middle event doesn't match request with prefix - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("aa", "1", 2, true), putWatchEvent("ac", "3", 4, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("aa", "1"), putRequest("bb", "2"), putRequest("ac", "3"), }, }, { name: "Reliable, Resumable - missing first event - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("b", "2", 3, true), putWatchEvent("c", "3", 4, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, }, { name: "Reliable - missing last event - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 3, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, }, { name: "Reliable - ignore empty last error response - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 3, true), putWatchEvent("c", "3", 4, true), }, }, { Revision: 5, Error: "etcdserver: mvcc: required revision has been compacted", }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, }, { name: "Resumable - watch revision from middle event - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, Revision: 3, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("b", "2", 3, true), putWatchEvent("c", "3", 4, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, }, { name: "Resumable - watch key from middle event - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "b", Revision: 2, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("b", "2", 3, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, }, { name: "Resumable - watch key with prefix from middle event - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "b", Revision: 2, WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("bb", "2", 3, true), putWatchEvent("bc", "3", 4, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("bb", "2"), putRequest("bc", "3"), }, }, { name: "Resumable - missing first matching event - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, Revision: 3, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("c", "3", 4, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("c", "3"), }, expectError: errBrokeResumable.Error(), }, { name: "Resumable - missing first matching event with prefix - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "b", Revision: 2, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("b", "3", 4, false), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), putRequest("b", "3"), }, expectError: errBrokeResumable.Error(), }, { name: "Resumable - missing first matching event with prefix - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "b", WithPrefix: true, Revision: 2, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("bc", "3", 4, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("bb", "2"), putRequest("bc", "3"), }, expectError: errBrokeResumable.Error(), }, { name: "IsCreate - correct IsCreate values - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("a", "2", 3, false), deleteWatchEvent("a", 4), putWatchEvent("a", "4", 5, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("a", "2"), deleteRequest("a"), putRequest("a", "4"), }, }, { name: "IsCreate - second put marked as created - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("a", "2", 3, true), deleteWatchEvent("a", 4), putWatchEvent("a", "4", 5, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("a", "2"), deleteRequest("a"), putRequest("a", "4"), }, expectError: errBrokeIsCreate.Error(), }, { name: "IsCreate - put after delete marked as not created - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("a", "2", 3, false), deleteWatchEvent("a", 4), putWatchEvent("a", "4", 5, false), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("a", "2"), deleteRequest("a"), putRequest("a", "4"), }, expectError: errBrokeIsCreate.Error(), }, { name: "PrevKV - no previous values - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", WithPrevKV: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("a", "2", 3, false), deleteWatchEvent("a", 4), putWatchEvent("a", "4", 5, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("a", "2"), deleteRequest("a"), putRequest("a", "4"), }, }, { name: "PrevKV - all previous values - pass", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", WithPrevKV: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEventWithPrevKVV("a", "2", 3, false, "1", 2, 1), deleteWatchEventWithPrevKVV("a", 4, "2", 3, 2), putWatchEvent("a", "4", 5, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("a", "2"), deleteRequest("a"), putRequest("a", "4"), }, }, { name: "PrevKV - mismatch value on put - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", WithPrevKV: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEventWithPrevKV("a", "2", 3, false, "2", 2), deleteWatchEventWithPrevKV("a", 4, "2", 3), putWatchEvent("a", "4", 5, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("a", "2"), deleteRequest("a"), putRequest("a", "4"), }, expectError: errBrokePrevKV.Error(), }, { name: "PrevKV - mismatch revision on put - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", WithPrevKV: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEventWithPrevKV("a", "2", 3, false, "1", 3), deleteWatchEventWithPrevKV("a", 4, "2", 3), putWatchEvent("a", "4", 5, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("a", "2"), deleteRequest("a"), putRequest("a", "4"), }, expectError: errBrokePrevKV.Error(), }, { name: "PrevKV - mismatch value on delete - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", WithPrevKV: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEventWithPrevKV("a", "2", 3, false, "1", 2), deleteWatchEventWithPrevKV("a", 4, "1", 3), putWatchEvent("a", "4", 5, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("a", "2"), deleteRequest("a"), putRequest("a", "4"), }, expectError: errBrokePrevKV.Error(), }, { name: "PrevKV - mismatch revision on delete - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ WithPrefix: true, WithPrevKV: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEventWithPrevKV("a", "2", 3, false, "1", 2), deleteWatchEventWithPrevKV("a", 4, "2", 2), putWatchEvent("a", "4", 5, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("a", "2"), deleteRequest("a"), putRequest("a", "4"), }, expectError: errBrokePrevKV.Error(), }, { name: "Filter - event not matching the watch - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("a", "1", 2, true), putWatchEvent("b", "2", 3, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("a", "1"), putRequest("b", "2"), }, expectError: errBrokeFilter.Error(), }, { name: "Filter - event not matching the watch with prefix - fail", reports: []report.ClientReport{ { Watch: []model.WatchOperation{ { Request: model.WatchRequest{ Key: "a", WithPrefix: true, }, Responses: []model.WatchResponse{ { Events: []model.WatchEvent{ putWatchEvent("aa", "1", 2, true), putWatchEvent("bb", "2", 3, true), putWatchEvent("ac", "3", 4, true), }, }, }, }, }, }, }, persistedRequests: []model.EtcdRequest{ putRequest("aa", "1"), putRequest("bb", "2"), putRequest("ac", "3"), }, expectError: errBrokeFilter.Error(), }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { replay := model.NewReplay(tc.persistedRequests) result := validateWatch(zaptest.NewLogger(t), tc.config, tc.reports, replay) if result.Message != tc.expectError { t.Errorf("validateWatch(...), got: %q, want: %q", result.Message, tc.expectError) } }) } } func putWatchEvent(key, value string, rev int64, isCreate bool) model.WatchEvent { return model.WatchEvent{ PersistedEvent: putPersistedEvent(key, value, rev, isCreate), } } func deleteWatchEvent(key string, rev int64) model.WatchEvent { return model.WatchEvent{ PersistedEvent: deletePersistedEvent(key, rev), } } func putWatchEventWithPrevKV(key, value string, rev int64, isCreate bool, prevValue string, modRev int64) model.WatchEvent { return putWatchEventWithPrevKVV(key, value, rev, isCreate, prevValue, modRev, 0) } func putWatchEventWithPrevKVV(key, value string, rev int64, isCreate bool, prevValue string, modRev, ver int64) model.WatchEvent { return model.WatchEvent{ PersistedEvent: putPersistedEvent(key, value, rev, isCreate), PrevValue: &model.ValueRevision{ Value: model.ToValueOrHash(prevValue), ModRevision: modRev, Version: ver, }, } } func deleteWatchEventWithPrevKV(key string, rev int64, prevValue string, modRev int64) model.WatchEvent { return deleteWatchEventWithPrevKVV(key, rev, prevValue, modRev, 0) } func deleteWatchEventWithPrevKVV(key string, rev int64, prevValue string, modRev, ver int64) model.WatchEvent { return model.WatchEvent{ PersistedEvent: deletePersistedEvent(key, rev), PrevValue: &model.ValueRevision{ Value: model.ToValueOrHash(prevValue), ModRevision: modRev, Version: ver, }, } } func putPersistedEvent(key, value string, rev int64, isCreate bool) model.PersistedEvent { return model.PersistedEvent{ Event: model.Event{ Type: model.PutOperation, Key: key, Value: model.ToValueOrHash(value), }, Revision: rev, IsCreate: isCreate, } } func deletePersistedEvent(key string, rev int64) model.PersistedEvent { return model.PersistedEvent{ Event: model.Event{ Type: model.DeleteOperation, Key: key, }, Revision: rev, } } func getRequest(key string) model.EtcdRequest { return model.EtcdRequest{ Type: model.Range, Range: &model.RangeRequest{ RangeOptions: model.RangeOptions{ Start: "key", }, }, } } func getResponse(rev int64) model.MaybeEtcdResponse { return model.MaybeEtcdResponse{EtcdResponse: model.EtcdResponse{Revision: rev, Range: &model.RangeResponse{KVs: []model.KeyValue{}}}} } func putRequest(key, value string) model.EtcdRequest { return model.EtcdRequest{ Type: model.Txn, LeaseGrant: nil, LeaseRevoke: nil, Range: nil, Txn: &model.TxnRequest{ Conditions: nil, OperationsOnSuccess: []model.EtcdOperation{ { Type: model.PutOperation, Put: model.PutOptions{ Key: key, Value: model.ToValueOrHash(value), }, }, }, OperationsOnFailure: nil, }, Defragment: nil, } } func putRequestWithLease(key, value string, leaseID int64) model.EtcdRequest { req := putRequest(key, value) req.Txn.OperationsOnSuccess[0].Put.LeaseID = leaseID return req } func deleteRequest(key string) model.EtcdRequest { return model.EtcdRequest{ Type: model.Txn, LeaseGrant: nil, LeaseRevoke: nil, Range: nil, Txn: &model.TxnRequest{ Conditions: nil, OperationsOnSuccess: []model.EtcdOperation{ { Type: model.DeleteOperation, Delete: model.DeleteOptions{ Key: key, }, }, }, OperationsOnFailure: nil, }, Defragment: nil, } } ================================================ FILE: tests/robustness/validate/watch.go ================================================ // Copyright 2023 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package validate import ( "errors" "fmt" "reflect" "time" "github.com/google/go-cmp/cmp" "go.uber.org/zap" "go.etcd.io/etcd/tests/v3/robustness/model" "go.etcd.io/etcd/tests/v3/robustness/report" ) var ( errBrokeBookmarkable = errors.New("broke Bookmarkable - Progress notification events guarantee that all events up to a revision have already been delivered") errBrokeOrdered = errors.New("broke Ordered - events are ordered by revision; an event will never appear on a watch if it precedes an event in time that has already been posted") errBrokeUnique = errors.New("broke Unique - an event will never appear on a watch twice") errBrokeAtomic = errors.New("broke Atomic - a list of events is guaranteed to encompass complete revisions; updates in the same revision over multiple keys will not be split over several lists of events") errBrokeReliable = errors.New("broke Reliable - a sequence of events will never drop any subsequence of events; if there are events ordered in time as a < b < c, then if the watch receives events a and c, it is guaranteed to receive b") errBrokeResumable = errors.New("broke Resumable - A broken watch can be resumed by establishing a new watch starting after the last revision received in a watch event before the break, so long as the revision is in the history window") errBrokePrevKV = errors.New("incorrect event prevValue") errBrokeIsCreate = errors.New("incorrect event IsCreate") errBrokeFilter = errors.New("event not matching watch filter") ) func validateWatch(lg *zap.Logger, cfg Config, reports []report.ClientReport, replay *model.EtcdReplay) Result { lg.Info("Validating watch") start := time.Now() err := validateWatchError(lg, cfg, reports, replay) if err != nil { lg.Error("Watch validation failed", zap.Duration("duration", time.Since(start)), zap.Error(err)) } else { lg.Info("Watch validation success", zap.Duration("duration", time.Since(start))) } return ResultFromError(err) } func validateWatchError(lg *zap.Logger, cfg Config, reports []report.ClientReport, replay *model.EtcdReplay) error { // Validate etcd watch properties defined in https://etcd.io/docs/v3.6/learning/api_guarantees/#watch-apis for _, r := range reports { err := validateFilter(lg, r) if err != nil { return err } err = validateOrdered(lg, r) if err != nil { return err } err = validateUnique(lg, cfg.ExpectRevisionUnique, r) if err != nil { return err } err = validateAtomic(lg, r) if err != nil { return err } err = validateBookmarkable(lg, r) if err != nil { return err } err = validateResumable(lg, replay, r) if err != nil { return err } err = validateIsCreate(lg, replay, r) if err != nil { return err } err = validateReliable(lg, replay, r) if err != nil { return err } err = validatePrevKV(lg, replay, r) if err != nil { return err } } return nil } func validateFilter(lg *zap.Logger, report report.ClientReport) (err error) { for _, watch := range report.Watch { for _, resp := range watch.Responses { for _, event := range resp.Events { if !event.Match(watch.Request) { lg.Error("event not matching event filter", zap.Int("client", report.ClientID), zap.Any("request", watch.Request), zap.Any("event", event)) err = errBrokeFilter } } } } return err } func validateBookmarkable(lg *zap.Logger, report report.ClientReport) (err error) { for _, op := range report.Watch { var lastProgressNotifyRevision int64 var lastEventRevision int64 for _, resp := range op.Responses { for _, event := range resp.Events { if event.Revision <= lastProgressNotifyRevision { lg.Error("Broke watch guarantee", zap.String("guarantee", "bookmarkable"), zap.Int("client", report.ClientID), zap.Int64("revision", event.Revision)) err = errBrokeBookmarkable } lastEventRevision = event.Revision } if resp.IsProgressNotify { if resp.Revision < lastProgressNotifyRevision { lg.Error("Broke watch guarantee", zap.String("guarantee", "bookmarkable"), zap.Int("client", report.ClientID), zap.Int64("revision", resp.Revision)) err = errBrokeBookmarkable } if resp.Revision < lastEventRevision { lg.Error("Broke watch guarantee", zap.String("guarantee", "bookmarkable"), zap.Int("client", report.ClientID), zap.Int64("revision", resp.Revision)) err = errBrokeBookmarkable } lastProgressNotifyRevision = resp.Revision } } } return err } func validateOrdered(lg *zap.Logger, report report.ClientReport) (err error) { for _, op := range report.Watch { var lastEventRevision int64 = 1 for _, resp := range op.Responses { for _, event := range resp.Events { if event.Revision < lastEventRevision { lg.Error("Broke watch guarantee", zap.String("guarantee", "ordered"), zap.Int("client", report.ClientID), zap.Int64("revision", event.Revision)) err = errBrokeOrdered } lastEventRevision = event.Revision } } } return err } func validateUnique(lg *zap.Logger, expectUniqueRevision bool, report report.ClientReport) (err error) { for _, op := range report.Watch { uniqueOperations := map[any]struct{}{} for _, resp := range op.Responses { for _, event := range resp.Events { var key any if expectUniqueRevision { key = event.Revision } else { key = struct { revision int64 key string }{event.Revision, event.Key} } if _, found := uniqueOperations[key]; found { lg.Error("Broke watch guarantee", zap.String("guarantee", "unique"), zap.Int("client", report.ClientID), zap.String("key", event.Key), zap.Int64("revision", event.Revision)) err = errBrokeUnique } uniqueOperations[key] = struct{}{} } } } return err } func validateAtomic(lg *zap.Logger, report report.ClientReport) (err error) { for _, op := range report.Watch { var lastEventRevision int64 = 1 for _, resp := range op.Responses { if len(resp.Events) > 0 { if resp.Events[0].Revision == lastEventRevision { lg.Error("Broke watch guarantee", zap.String("guarantee", "atomic"), zap.Int("client", report.ClientID), zap.Int64("revision", resp.Events[0].Revision)) err = errBrokeAtomic } lastEventRevision = resp.Events[len(resp.Events)-1].Revision } } } return err } func validateReliable(lg *zap.Logger, replay *model.EtcdReplay, report report.ClientReport) (err error) { for _, watch := range report.Watch { firstRev := firstExpectedRevision(watch) lastRev := lastRevision(watch) events := replay.EventsForWatch(watch.Request) wantEvents := []model.PersistedEvent{} if firstRev != 0 { for _, e := range events { if e.Revision < firstRev { continue } if e.Revision > lastRev { break } if e.Match(watch.Request) { wantEvents = append(wantEvents, e) } } } gotEvents := make([]model.PersistedEvent, 0) for _, resp := range watch.Responses { for _, event := range resp.Events { gotEvents = append(gotEvents, event.PersistedEvent) } } if !reflect.DeepEqual(wantEvents, gotEvents) { lg.Error("Broke watch guarantee", zap.String("guarantee", "reliable"), zap.Int("client", report.ClientID)) // Directly print to console to avoid escaping newline. fmt.Print(cmp.Diff(wantEvents, gotEvents)) err = errBrokeReliable } } return err } func validateResumable(lg *zap.Logger, replay *model.EtcdReplay, report report.ClientReport) (err error) { for _, watch := range report.Watch { if watch.Request.Revision == 0 { continue } events := replay.EventsForWatch(watch.Request) index := 0 for index < len(events) && (events[index].Revision < watch.Request.Revision || !events[index].Match(watch.Request)) { index++ } if index == len(events) { continue } firstEvent := firstWatchEvent(watch) // If watch is resumable, first event it gets should the first event that happened after the requested revision. if firstEvent != nil && events[index] != firstEvent.PersistedEvent { lg.Error("Broke watch guarantee", zap.String("guarantee", "resumable"), zap.Int("client", report.ClientID), zap.Any("request", watch.Request), zap.Any("got-event", *firstEvent), zap.Any("want-event", events[index])) err = errBrokeResumable } } return err } // validatePrevKV ensures that a watch response (if configured with WithPrevKV()) returns // the appropriate response. func validatePrevKV(lg *zap.Logger, replay *model.EtcdReplay, report report.ClientReport) (err error) { for _, op := range report.Watch { if !op.Request.WithPrevKV { continue } for _, resp := range op.Responses { for _, event := range resp.Events { // Get state just before the current event. state, err2 := replay.StateForRevision(event.Revision - 1) if err2 != nil { panic(err2) } // TODO(MadhavJivrajani): check if compaction has been run as part // of failpoint injection. If compaction has run, prevKV can be nil // even if it is not a create event. // // Considering that Kubernetes opens watches to etcd using WithPrevKV() // option, ideally we would want to explicitly check the condition that // Kubernetes does while parsing events received from etcd: // https://github.com/kubernetes/kubernetes/blob/a9e4f5b7862e84c4152eabe2e960f3f6fb9a4867/staging/src/k8s.io/apiserver/pkg/storage/etcd3/event.go#L59 // i.e. prevKV is nil iff the event is a create event, we cannot reliably // check that without knowing if compaction has run. // We allow PrevValue to be nil since in the face of compaction, etcd does not // guarantee its presence. if event.PrevValue != nil && *event.PrevValue != state.KeyValues[event.Key] { lg.Error("Incorrect event prevValue field", zap.Int("client", report.ClientID), zap.Any("event", event), zap.Any("previousValue", state.KeyValues[event.Key])) err = errBrokePrevKV } } } } return err } func validateIsCreate(lg *zap.Logger, replay *model.EtcdReplay, report report.ClientReport) (err error) { for _, op := range report.Watch { for _, resp := range op.Responses { for _, event := range resp.Events { // Get state just before the current event. state, err2 := replay.StateForRevision(event.Revision - 1) if err2 != nil { panic(err2) } // A create event will not have an entry in our history and a non-create // event *should* have an entry in our history. if _, prevKeyExists := state.KeyValues[event.Key]; event.IsCreate == prevKeyExists { lg.Error("Incorrect event IsCreate field", zap.Int("client", report.ClientID), zap.Any("event", event)) err = errBrokeIsCreate } } } } return err } func firstExpectedRevision(op model.WatchOperation) int64 { if op.Request.Revision != 0 { return op.Request.Revision } if len(op.Responses) > 0 { firstResp := op.Responses[0] if firstResp.IsProgressNotify { return firstResp.Revision + 1 } if len(firstResp.Events) > 0 { return firstResp.Events[0].Revision } } return 0 } func lastRevision(op model.WatchOperation) int64 { for i := len(op.Responses) - 1; i >= 0; i-- { resp := op.Responses[i] if resp.IsProgressNotify { return resp.Revision } if len(resp.Events) > 0 { lastEvent := resp.Events[len(resp.Events)-1] return lastEvent.Revision } } return 0 } func firstWatchEvent(op model.WatchOperation) *model.WatchEvent { for _, resp := range op.Responses { for _, event := range resp.Events { return &event } } return nil } ================================================ FILE: tests/semaphore.test.bash ================================================ #!/usr/bin/env bash # Placeholder until SemaphoreCI is disabled # E2e tests were migrated to GitHub Actions ================================================ FILE: tools/.golangci.yaml ================================================ --- version: "2" linters: default: none enable: # please keep this alphabetized - errorlint - goheader - govet - ineffassign - misspell - nakedret - revive - staticcheck - testifylint - thelper - unconvert # Remove unnecessary type conversions - unparam - unused - usestdlibvars - usetesting - whitespace settings: goheader: values: regexp: ORIGINAL_YEAR: 20[1-2][0-9] template: |- Copyright {{ORIGINAL_YEAR}} The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. govet: enable: - shadow nakedret: # Align with https://github.com/alexkohler/nakedret/blob/v1.0.2/cmd/nakedret/main.go#L10 max-func-lines: 5 revive: confidence: 0.8 rules: - name: blank-imports - name: context-as-argument - name: context-keys-type - name: dot-imports - name: early-return arguments: - preserveScope - name: error-return - name: error-naming - name: error-strings - name: errorf - name: if-return - name: increment-decrement - name: indent-error-flow - name: package-comments - name: range - name: receiver-naming - name: superfluous-else arguments: - preserveScope - name: time-naming - name: unnecessary-stmt - name: use-any - name: var-declaration - name: var-naming arguments: # The following is the configuration for var-naming rule, the first element is the allow list and the second element is the deny list. - [] # AllowList: leave it empty to use the default (empty, too). This means that we're not relaxing the rule in any way, i.e. elementId will raise a violation, it should be elementID, refer to the next line to see the list of denied initialisms. # DenyList: Add GRPC and WAL to strict the rule not allowing instances like Wal or Grpc. The default values are located at commonInitialisms, refer to: https://github.com/mgechev/revive/blob/v1.3.7/lint/utils.go#L93-L133. - - GRPC - WAL # Disable checks on package names that collide with Go standard library packages. See example error: # var-naming: avoid package names that conflict with Go standard library package names (revive) - - skip-package-name-collision-with-go-std: true - name: exported disabled: true - name: unexported-return disabled: true staticcheck: checks: - all - -SA2002 # TODO(fix) Called testing.T.FailNow or SkipNow in a goroutine, which isn’t allowed - -QF1001 # TODO(fix) Apply De Morgan’s law - -QF1002 # TODO(fix) Convert untagged switch to tagged switch - -QF1003 # TODO(fix) Convert if/else-if chain to tagged switch - -QF1004 # TODO(fix) Use strings.ReplaceAll instead of strings.Replace with n == -1 - -QF1006 # TODO(fix) Lift if+break into loop condition - -QF1008 # TODO(fix) Omit embedded fields from selector expression - -QF1009 # TODO(fix) Use time.Time.Equal instead of == operator - -QF1012 # TODO(fix) Use fmt.Fprintf(x, ...) instead of x.Write(fmt.Sprintf(...)) - -ST1003 # TODO(fix) Poorly chosen identifier - -ST1005 # TODO(fix) Drop unnecessary use of the blank identifier - ST1019 # Importing the same package multiple times. testifylint: enable-all: true formatter: # Require f-assertions (e.g. assert.Equalf) if a message is passed to the assertion, even if # there is no variable-length variables, i.e. require require.NoErrorf for both cases below: # require.NoErrorf(t, err, "whatever message") # require.NoErrorf(t, err, "whatever message: %v", v) # # Note from golang programming perspective, we still prefer non-f-functions (i.e. fmt.Print) # to f-functions (i.e. fmt.Printf) when there is no variable-length parameters. It's accepted # to always require f-functions for stretchr/testify, but not for golang standard lib. # Also refer to https://github.com/etcd-io/etcd/pull/18741#issuecomment-2422395914 require-f-funcs: true thelper: test: first: false begin: false fuzz: first: false begin: false benchmark: first: false begin: false tb: first: false begin: false usetesting: os-mkdir-temp: false exclusions: generated: lax presets: - comments - common-false-positives - legacy - std-error-handling rules: - linters: - ineffassign path: conversion\.go - linters: - staticcheck text: S1000 - linters: - revive text: 'var-naming: avoid meaningless package names' paths: - ^zz_generated.* - third_party$ - builtin$ - examples$ issues: max-same-issues: 0 formatters: enable: - gci - gofmt - goimports settings: # please keep this alphabetized gci: sections: - standard - default - prefix(go.etcd.io) goimports: local-prefixes: - go.etcd.io # Put imports beginning with prefix after 3rd-party packages. exclusions: generated: lax paths: - ^zz_generated.* - third_party$ - builtin$ - examples$ ================================================ FILE: tools/.markdownlint.jsonc ================================================ // Example markdownlint configuration with all properties set to their default value { // Default state for all rules "default": true, // Path to configuration file to extend "extends": null, // MD001/heading-increment : Heading levels should only increment by one level at a time : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md001.md "MD001": true, // MD003/heading-style : Heading style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md003.md "MD003": { // Heading style "style": "consistent" }, // MD004/ul-style : Unordered list style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md004.md "MD004": { // List style "style": "consistent" }, // MD005/list-indent : Inconsistent indentation for list items at the same level : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md005.md "MD005": true, // MD007/ul-indent : Unordered list indentation : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md007.md "MD007": { // Spaces for indent "indent": 2, // Whether to indent the first level of the list "start_indented": false, // Spaces for first level indent (when start_indented is set) "start_indent": 2 }, // MD009/no-trailing-spaces : Trailing spaces : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md009.md "MD009": { // Spaces for line break "br_spaces": 2, // Allow spaces for empty lines in list items "list_item_empty_lines": false, // Include unnecessary breaks "strict": false }, // MD010/no-hard-tabs : Hard tabs : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md010.md "MD010": { // Include code blocks "code_blocks": true, // Fenced code languages to ignore "ignore_code_languages": [], // Number of spaces for each hard tab "spaces_per_tab": 1 }, // MD011/no-reversed-links : Reversed link syntax : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md011.md "MD011": true, // MD012/no-multiple-blanks : Multiple consecutive blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md012.md "MD012": { // Consecutive blank lines "maximum": 1 }, "MD013": false, // MD014/commands-show-output : Dollar signs used before commands without showing output : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md014.md "MD014": true, // MD018/no-missing-space-atx : No space after hash on atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md018.md "MD018": true, // MD019/no-multiple-space-atx : Multiple spaces after hash on atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md019.md "MD019": true, // MD020/no-missing-space-closed-atx : No space inside hashes on closed atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md020.md "MD020": true, // MD021/no-multiple-space-closed-atx : Multiple spaces inside hashes on closed atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md021.md "MD021": true, // MD022/blanks-around-headings : Headings should be surrounded by blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md022.md "MD022": { // Blank lines above heading "lines_above": 1, // Blank lines below heading "lines_below": 1 }, // MD023/heading-start-left : Headings must start at the beginning of the line : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md023.md "MD023": true, // MD024/no-duplicate-heading : Multiple headings with the same content : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md024.md "MD024": { // Only check sibling headings "siblings_only": false }, // MD025/single-title/single-h1 : Multiple top-level headings in the same document : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md025.md "MD025": { // Heading level "level": 1, // RegExp for matching title in front matter "front_matter_title": "^\\s*title\\s*[:=]" }, // MD026/no-trailing-punctuation : Trailing punctuation in heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md026.md "MD026": { // Punctuation characters "punctuation": ".,;:!。,;:!" }, // MD027/no-multiple-space-blockquote : Multiple spaces after blockquote symbol : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md027.md "MD027": true, // MD028/no-blanks-blockquote : Blank line inside blockquote : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md028.md "MD028": true, // MD029/ol-prefix : Ordered list item prefix : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md029.md "MD029": { // List style "style": "one_or_ordered" }, // MD030/list-marker-space : Spaces after list markers : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md030.md "MD030": { // Spaces for single-line unordered list items "ul_single": 1, // Spaces for single-line ordered list items "ol_single": 1, // Spaces for multi-line unordered list items "ul_multi": 1, // Spaces for multi-line ordered list items "ol_multi": 1 }, // MD031/blanks-around-fences : Fenced code blocks should be surrounded by blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md031.md "MD031": { // Include list items "list_items": true }, // MD032/blanks-around-lists : Lists should be surrounded by blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md032.md "MD032": true, // MD033/no-inline-html : Inline HTML : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md033.md "MD033": { // Allowed elements "allowed_elements": [] }, // MD034/no-bare-urls : Bare URL used : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md034.md "MD034": true, // MD035/hr-style : Horizontal rule style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md035.md "MD035": { // Horizontal rule style "style": "consistent" }, // MD036/no-emphasis-as-heading : Emphasis used instead of a heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md036.md "MD036": { // Punctuation characters "punctuation": ".,;:!?。,;:!?" }, // MD037/no-space-in-emphasis : Spaces inside emphasis markers : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md037.md "MD037": true, // MD038/no-space-in-code : Spaces inside code span elements : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md038.md "MD038": true, // MD039/no-space-in-links : Spaces inside link text : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md039.md "MD039": true, // MD040/fenced-code-language : Fenced code blocks should have a language specified : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md040.md "MD040": { // List of languages "allowed_languages": [], // Require language only "language_only": false }, // MD041/first-line-heading/first-line-h1 : First line in a file should be a top-level heading : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md041.md "MD041": false, // MD042/no-empty-links : No empty links : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md042.md "MD042": true, // MD043/required-headings : Required heading structure : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md043.md "MD043": false, // MD044/proper-names : Proper names should have the correct capitalization : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md044.md "MD044": { // List of proper names "names": [], // Include code blocks "code_blocks": true, // Include HTML elements "html_elements": true }, // MD045/no-alt-text : Images should have alternate text (alt text) : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md045.md "MD045": true, // MD046/code-block-style : Code block style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md046.md "MD046": { // Block style "style": "consistent" }, // MD047/single-trailing-newline : Files should end with a single newline character : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md047.md "MD047": true, // MD048/code-fence-style : Code fence style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md048.md "MD048": { // Code fence style "style": "consistent" }, // MD049/emphasis-style : Emphasis style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md049.md "MD049": { // Emphasis style "style": "consistent" }, // MD050/strong-style : Strong style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md050.md "MD050": { // Strong style "style": "consistent" }, // MD051/link-fragments : Link fragments should be valid : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md051.md "MD051": { // Ignore case of fragments "ignore_case": false }, // MD052/reference-links-images : Reference links and images should use a label that is defined : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md052.md "MD052": { // Include shortcut syntax "shortcut_syntax": false }, // MD053/link-image-reference-definitions : Link and image reference definitions should be needed : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md053.md "MD053": { // Ignored definitions "ignored_definitions": [ "//" ] }, // MD054/link-image-style : Link and image style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md054.md "MD054": { // Allow autolinks "autolink": true, // Allow inline links and images "inline": true, // Allow full reference links and images "full": true, // Allow collapsed reference links and images "collapsed": true, // Allow shortcut reference links and images "shortcut": true, // Allow URLs as inline links "url_inline": true }, // MD055/table-pipe-style : Table pipe style : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md055.md "MD055": { // Table pipe style "style": "consistent" }, // MD056/table-column-count : Table column count : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md056.md "MD056": true, // MD058/blanks-around-tables : Tables should be surrounded by blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.37.4/doc/md058.md "MD058": true } ================================================ FILE: tools/.yamlfmt ================================================ formatter: type: basic include_document_start: true retain_line_breaks: true ================================================ FILE: tools/.yamllint ================================================ --- extends: default rules: line-length: disable truthy: disable comments: disable ================================================ FILE: tools/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/tooling ================================================ FILE: tools/benchmark/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/performance ================================================ FILE: tools/benchmark/README.md ================================================ # etcd/tools/benchmark `etcd/tools/benchmark` is the official benchmarking tool for etcd clusters. ## Installation Install the tool by running the following command from the etcd source directory. ``` $ go install -v ./tools/benchmark ``` The installation will place executables in the $GOPATH/bin. If $GOPATH environment variable is not set, the tool will be installed into the $HOME/go/bin. You can also find out the installed location by running the following command from the etcd source directory. Make sure that $PATH is set accordingly in your environment. ``` $ go list -f "{{.Target}}" ./tools/benchmark ``` Alternatively, instead of installing the tool, you can use it by simply running the following command from the etcd source directory. ``` $ go run ./tools/benchmark ``` ## Usage The following command should output the usage per the latest development. ``` $ benchmark --help ``` ================================================ FILE: tools/benchmark/cmd/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package cmd implements individual benchmark commands for the benchmark utility. package cmd ================================================ FILE: tools/benchmark/cmd/lease.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "context" "fmt" "time" "github.com/cheggaaa/pb/v3" "github.com/spf13/cobra" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/report" ) var leaseKeepaliveCmd = &cobra.Command{ Use: "lease-keepalive", Short: "Benchmark lease keepalive", Run: leaseKeepaliveFunc, } var leaseKeepaliveTotal int func init() { RootCmd.AddCommand(leaseKeepaliveCmd) leaseKeepaliveCmd.Flags().IntVar(&leaseKeepaliveTotal, "total", 10000, "Total number of lease keepalive requests") } func leaseKeepaliveFunc(cmd *cobra.Command, _ []string) { requests := make(chan struct{}) clients := mustCreateClients(totalClients, totalConns) bar = pb.New(leaseKeepaliveTotal) bar.Start() r := newReport(cmd.Name()) for i := range clients { wg.Add(1) go func(c v3.Lease) { defer wg.Done() resp, err := c.Grant(context.Background(), 100) if err != nil { panic(err) } for range requests { st := time.Now() _, err := c.KeepAliveOnce(context.TODO(), resp.ID) r.Results() <- report.Result{Err: err, Start: st, End: time.Now()} bar.Increment() } }(clients[i]) } wg.Go(func() { for i := 0; i < leaseKeepaliveTotal; i++ { requests <- struct{}{} } close(requests) }) rc := r.Run() wg.Wait() close(r.Results()) bar.Finish() fmt.Printf("%s", <-rc) } ================================================ FILE: tools/benchmark/cmd/mvcc-put.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "crypto/rand" "fmt" "os" "runtime/pprof" "time" "github.com/spf13/cobra" "go.etcd.io/etcd/pkg/v3/report" "go.etcd.io/etcd/pkg/v3/traceutil" "go.etcd.io/etcd/server/v3/lease" ) // mvccPutCmd represents a storage put performance benchmarking tool var mvccPutCmd = &cobra.Command{ Use: "put", Short: "Benchmark put performance of storage", Run: mvccPutFunc, } var ( mvccTotalRequests int storageKeySize int valueSize int txn bool nrTxnOps int ) func init() { mvccCmd.AddCommand(mvccPutCmd) mvccPutCmd.Flags().IntVar(&mvccTotalRequests, "total", 100, "a total number of keys to put") mvccPutCmd.Flags().IntVar(&storageKeySize, "key-size", 64, "a size of key (Byte)") mvccPutCmd.Flags().IntVar(&valueSize, "value-size", 64, "a size of value (Byte)") mvccPutCmd.Flags().BoolVar(&txn, "txn", false, "put a key in transaction or not") mvccPutCmd.Flags().IntVar(&nrTxnOps, "txn-ops", 1, "a number of keys to put per transaction") // TODO: after the PR https://github.com/spf13/cobra/pull/220 is merged, the below pprof related flags should be moved to RootCmd mvccPutCmd.Flags().StringVar(&cpuProfPath, "cpuprofile", "", "the path of file for storing cpu profile result") mvccPutCmd.Flags().StringVar(&memProfPath, "memprofile", "", "the path of file for storing heap profile result") } func createBytesSlice(bytesN, sliceN int) [][]byte { rs := make([][]byte, sliceN) for i := range rs { rs[i] = make([]byte, bytesN) if _, err := rand.Read(rs[i]); err != nil { panic(err) } } return rs } func mvccPutFunc(cmd *cobra.Command, _ []string) { if cpuProfPath != "" { f, err := os.Create(cpuProfPath) if err != nil { fmt.Fprintln(os.Stderr, "Failed to create a file for storing cpu profile result: ", err) os.Exit(1) } defer f.Close() err = pprof.StartCPUProfile(f) if err != nil { fmt.Fprintln(os.Stderr, "Failed to start cpu profile: ", err) os.Exit(1) } defer pprof.StopCPUProfile() } if memProfPath != "" { f, err := os.Create(memProfPath) if err != nil { fmt.Fprintln(os.Stderr, "Failed to create a file for storing heap profile result: ", err) os.Exit(1) } defer f.Close() defer func() { err := pprof.WriteHeapProfile(f) if err != nil { fmt.Fprintln(os.Stderr, "Failed to write heap profile result: ", err) // can do nothing for handling the error } }() } keys := createBytesSlice(storageKeySize, mvccTotalRequests*nrTxnOps) vals := createBytesSlice(valueSize, mvccTotalRequests*nrTxnOps) weight := float64(nrTxnOps) r := newWeightedReport(cmd.Name()) rrc := r.Results() rc := r.Run() if txn { for i := 0; i < mvccTotalRequests; i++ { st := time.Now() tw := s.Write(traceutil.TODO()) for j := i; j < i+nrTxnOps; j++ { tw.Put(keys[j], vals[j], lease.NoLease) } tw.End() rrc <- report.Result{Start: st, End: time.Now(), Weight: weight} } } else { for i := 0; i < mvccTotalRequests; i++ { st := time.Now() s.Put(keys[i], vals[i], lease.NoLease) rrc <- report.Result{Start: st, End: time.Now()} } } close(r.Results()) fmt.Printf("%s", <-rc) } ================================================ FILE: tools/benchmark/cmd/mvcc.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "os" "time" "github.com/spf13/cobra" "go.uber.org/zap" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" ) var ( batchInterval int batchLimit int s mvcc.KV ) func initMVCC() { bcfg := backend.DefaultBackendConfig(zap.NewNop()) bcfg.Path, bcfg.BatchInterval, bcfg.BatchLimit = "mvcc-bench", time.Duration(batchInterval)*time.Millisecond, batchLimit be := backend.New(bcfg) s = mvcc.NewStore(zap.NewExample(), be, &lease.FakeLessor{}, mvcc.StoreConfig{}) os.Remove("mvcc-bench") // boltDB has an opened fd, so removing the file is ok } // mvccCmd represents the MVCC storage benchmarking tools var mvccCmd = &cobra.Command{ Use: "mvcc", Short: "Benchmark mvcc", Long: `storage subcommand is a set of various benchmark tools for MVCC storage subsystem of etcd. Actual benchmarks are implemented as its subcommands.`, PersistentPreRun: mvccPreRun, } func init() { RootCmd.AddCommand(mvccCmd) mvccCmd.PersistentFlags().IntVar(&batchInterval, "batch-interval", 100, "Interval of batching (milliseconds)") mvccCmd.PersistentFlags().IntVar(&batchLimit, "batch-limit", 10000, "A limit of batched transaction") } func mvccPreRun(_ *cobra.Command, _ []string) { initMVCC() } ================================================ FILE: tools/benchmark/cmd/put.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "context" "encoding/binary" "fmt" "math" "math/rand" "os" "strings" "time" "github.com/cheggaaa/pb/v3" "github.com/dustin/go-humanize" "github.com/spf13/cobra" "golang.org/x/time/rate" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/report" ) // putCmd represents the put command var putCmd = &cobra.Command{ Use: "put", Short: "Benchmark put", Run: putFunc, } var ( keySize int valSize int putTotal int putRate int keySpaceSize int seqKeys bool compactInterval time.Duration compactIndexDelta int64 checkHashkv bool ) func init() { RootCmd.AddCommand(putCmd) putCmd.Flags().IntVar(&keySize, "key-size", 8, "Key size of put request") putCmd.Flags().IntVar(&valSize, "val-size", 8, "Value size of put request") putCmd.Flags().IntVar(&putRate, "rate", 0, "Maximum puts per second (0 is no limit)") putCmd.Flags().IntVar(&putTotal, "total", 10000, "Total number of put requests") putCmd.Flags().IntVar(&keySpaceSize, "key-space-size", 1, "Maximum possible keys") putCmd.Flags().BoolVar(&seqKeys, "sequential-keys", false, "Use sequential keys") putCmd.Flags().DurationVar(&compactInterval, "compact-interval", 0, `Interval to compact database (do not duplicate this with etcd's 'auto-compaction-retention' flag) (e.g. --compact-interval=5m compacts every 5-minute)`) putCmd.Flags().Int64Var(&compactIndexDelta, "compact-index-delta", 1000, "Delta between current revision and compact revision (e.g. current revision 10000, compact at 9000)") putCmd.Flags().BoolVar(&checkHashkv, "check-hashkv", false, "'true' to check hashkv") } func putFunc(cmd *cobra.Command, _ []string) { if keySpaceSize <= 0 { fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", keySpaceSize) os.Exit(1) } requests := make(chan v3.Op, totalClients) if putRate == 0 { putRate = math.MaxInt32 } limit := rate.NewLimiter(rate.Limit(putRate), 1) clients := mustCreateClients(totalClients, totalConns) k, v := make([]byte, keySize), string(mustRandBytes(valSize)) bar = pb.New(putTotal) bar.Start() r := newReport(cmd.Name()) for i := range clients { wg.Add(1) go func(c *v3.Client) { defer wg.Done() for op := range requests { limit.Wait(context.Background()) st := time.Now() _, err := c.Do(context.Background(), op) r.Results() <- report.Result{Err: err, Start: st, End: time.Now()} bar.Increment() } }(clients[i]) } go func() { for i := 0; i < putTotal; i++ { if seqKeys { binary.PutVarint(k, int64(i%keySpaceSize)) } else { binary.PutVarint(k, int64(rand.Intn(keySpaceSize))) } requests <- v3.OpPut(string(k), v) } close(requests) }() if compactInterval > 0 { go func() { for { time.Sleep(compactInterval) compactKV(clients) } }() } rc := r.Run() wg.Wait() close(r.Results()) bar.Finish() fmt.Println(<-rc) if checkHashkv { hashKV(cmd, clients) } } func compactKV(clients []*v3.Client) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) resp, err := clients[0].KV.Get(ctx, "foo") cancel() if err != nil { panic(err) } revToCompact := max(0, resp.Header.Revision-compactIndexDelta) ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) _, err = clients[0].KV.Compact(ctx, revToCompact) cancel() if err != nil { panic(err) } } func hashKV(cmd *cobra.Command, clients []*v3.Client) { eps, err := cmd.Flags().GetStringSlice("endpoints") if err != nil { panic(err) } for i, ip := range eps { eps[i] = strings.TrimSpace(ip) } host := eps[0] st := time.Now() rh, err := clients[0].HashKV(context.Background(), host, 0) if err != nil { fmt.Fprintf(os.Stderr, "Failed to get the hashkv of endpoint %s (%v)\n", host, err) panic(err) } rt, err := clients[0].Status(context.Background(), host) if err != nil { fmt.Fprintf(os.Stderr, "Failed to get the status of endpoint %s (%v)\n", host, err) panic(err) } rs := "HashKV Summary:\n" rs += fmt.Sprintf("\tHashKV: %d\n", rh.Hash) rs += fmt.Sprintf("\tEndpoint: %s\n", host) rs += fmt.Sprintf("\tTime taken to get hashkv: %v\n", time.Since(st)) rs += fmt.Sprintf("\tDB size: %s", humanize.Bytes(uint64(rt.DbSize))) fmt.Println(rs) } ================================================ FILE: tools/benchmark/cmd/range.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "context" "fmt" "math" "os" "time" "github.com/cheggaaa/pb/v3" "github.com/spf13/cobra" "golang.org/x/time/rate" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/report" ) // rangeCmd represents the range command var rangeCmd = &cobra.Command{ Use: "range key [end-range]", Short: "Benchmark range", Run: rangeFunc, } var ( rangeRate int rangeTotal int rangeConsistency string rangeLimit int64 rangeCountOnly bool ) func init() { RootCmd.AddCommand(rangeCmd) rangeCmd.Flags().IntVar(&rangeRate, "rate", 0, "Maximum range requests per second (0 is no limit)") rangeCmd.Flags().IntVar(&rangeTotal, "total", 10000, "Total number of range requests") rangeCmd.Flags().StringVar(&rangeConsistency, "consistency", "l", "Linearizable(l) or Serializable(s)") rangeCmd.Flags().Int64Var(&rangeLimit, "limit", 0, "Maximum number of results to return from range request (0 is no limit)") rangeCmd.Flags().BoolVar(&rangeCountOnly, "count-only", false, "Only returns the count of keys") } func rangeFunc(cmd *cobra.Command, args []string) { if len(args) == 0 || len(args) > 2 { fmt.Fprintln(os.Stderr, cmd.Usage()) os.Exit(1) } k := args[0] end := "" if len(args) == 2 { end = args[1] } if rangeConsistency == "l" { fmt.Println("bench with linearizable range") } else if rangeConsistency == "s" { fmt.Println("bench with serializable range") } else { fmt.Fprintln(os.Stderr, cmd.Usage()) os.Exit(1) } if rangeRate == 0 { rangeRate = math.MaxInt32 } limit := rate.NewLimiter(rate.Limit(rangeRate), 1) requests := make(chan v3.Op, totalClients) clients := mustCreateClients(totalClients, totalConns) bar = pb.New(rangeTotal) bar.Start() r := newReport(cmd.Name()) for i := range clients { wg.Add(1) go func(c *v3.Client) { defer wg.Done() for op := range requests { limit.Wait(context.Background()) st := time.Now() _, err := c.Do(context.Background(), op) r.Results() <- report.Result{Err: err, Start: st, End: time.Now()} bar.Increment() } }(clients[i]) } go func() { for i := 0; i < rangeTotal; i++ { opts := []v3.OpOption{v3.WithRange(end), v3.WithLimit(rangeLimit)} if rangeCountOnly { opts = append(opts, v3.WithCountOnly()) } if rangeConsistency == "s" { opts = append(opts, v3.WithSerializable()) } op := v3.OpGet(k, opts...) requests <- op } close(requests) }() rc := r.Run() wg.Wait() close(r.Results()) bar.Finish() fmt.Printf("%s", <-rc) } ================================================ FILE: tools/benchmark/cmd/root.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "sync" "time" "github.com/cheggaaa/pb/v3" "github.com/spf13/cobra" "go.etcd.io/etcd/client/pkg/v3/transport" ) // This represents the base command when called without any subcommands var RootCmd = &cobra.Command{ Use: "benchmark", Short: "A low-level benchmark tool for etcd3", Long: `benchmark is a low-level benchmark tool for etcd3. It uses gRPC client directly and does not depend on etcd client library. `, } var ( endpoints []string totalConns uint totalClients uint precise bool sample bool bar *pb.ProgressBar wg sync.WaitGroup tls transport.TLSInfo cpuProfPath string memProfPath string user string dialTimeout time.Duration autoSyncInterval time.Duration generatePerfReport bool ) func init() { RootCmd.PersistentFlags().StringSliceVar(&endpoints, "endpoints", []string{"127.0.0.1:2379"}, "gRPC endpoints") RootCmd.PersistentFlags().UintVar(&totalConns, "conns", 1, "Total number of gRPC connections") RootCmd.PersistentFlags().UintVar(&totalClients, "clients", 1, "Total number of gRPC clients") RootCmd.PersistentFlags().BoolVar(&precise, "precise", false, "use full floating point precision") RootCmd.PersistentFlags().BoolVar(&sample, "sample", false, "'true' to sample requests for every second") RootCmd.PersistentFlags().StringVar(&tls.CertFile, "cert", "", "identify HTTPS client using this SSL certificate file") RootCmd.PersistentFlags().StringVar(&tls.KeyFile, "key", "", "identify HTTPS client using this SSL key file") RootCmd.PersistentFlags().StringVar(&tls.TrustedCAFile, "cacert", "", "verify certificates of HTTPS-enabled servers using this CA bundle") RootCmd.PersistentFlags().BoolVar(&tls.InsecureSkipVerify, "insecure-skip-tls-verify", false, "skip server certificate verification") RootCmd.PersistentFlags().StringVar(&user, "user", "", "provide username[:password] and prompt if password is not supplied.") RootCmd.PersistentFlags().DurationVar(&dialTimeout, "dial-timeout", 0, "dial timeout for client connections") RootCmd.PersistentFlags().DurationVar(&autoSyncInterval, "auto-sync-interval", time.Duration(0), "AutoSyncInterval is the interval to update endpoints with its latest members") RootCmd.PersistentFlags().BoolVar(&generatePerfReport, "report-perfdash", false, "Generate benchmark report in perfdash format") } ================================================ FILE: tools/benchmark/cmd/stm.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "context" "encoding/binary" "fmt" "math" "math/rand" "os" "time" "github.com/cheggaaa/pb/v3" "github.com/spf13/cobra" "golang.org/x/time/rate" v3 "go.etcd.io/etcd/client/v3" v3sync "go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/pkg/v3/report" "go.etcd.io/etcd/server/v3/etcdserver/api/v3lock/v3lockpb" ) // stmCmd represents the STM benchmark command var stmCmd = &cobra.Command{ Use: "stm", Short: "Benchmark STM", Run: stmFunc, } type stmApply func(v3sync.STM) error var ( stmIsolation string stmIso v3sync.Isolation stmTotal int stmKeysPerTxn int stmKeyCount int stmValSize int stmWritePercent int stmLocker string stmRate int ) func init() { RootCmd.AddCommand(stmCmd) stmCmd.Flags().StringVar(&stmIsolation, "isolation", "r", "Read Committed (c), Repeatable Reads (r), Serializable (s), or Snapshot (ss)") stmCmd.Flags().IntVar(&stmKeyCount, "keys", 1, "Total unique keys accessible by the benchmark") stmCmd.Flags().IntVar(&stmTotal, "total", 10000, "Total number of completed STM transactions") stmCmd.Flags().IntVar(&stmKeysPerTxn, "keys-per-txn", 1, "Number of keys to access per transaction") stmCmd.Flags().IntVar(&stmWritePercent, "txn-wr-percent", 50, "Percentage of keys to overwrite per transaction") stmCmd.Flags().StringVar(&stmLocker, "stm-locker", "stm", "Wrap STM transaction with a custom locking mechanism (stm, lock-client, lock-rpc)") stmCmd.Flags().IntVar(&stmValSize, "val-size", 8, "Value size of each STM put request") stmCmd.Flags().IntVar(&stmRate, "rate", 0, "Maximum STM transactions per second (0 is no limit)") } func stmFunc(cmd *cobra.Command, _ []string) { if stmKeyCount <= 0 { fmt.Fprintf(os.Stderr, "expected positive --keys, got (%v)", stmKeyCount) os.Exit(1) } if stmWritePercent < 0 || stmWritePercent > 100 { fmt.Fprintf(os.Stderr, "expected [0, 100] --txn-wr-percent, got (%v)", stmWritePercent) os.Exit(1) } if stmKeysPerTxn < 0 || stmKeysPerTxn > stmKeyCount { fmt.Fprintf(os.Stderr, "expected --keys-per-txn between 0 and %v, got (%v)", stmKeyCount, stmKeysPerTxn) os.Exit(1) } switch stmIsolation { case "c": stmIso = v3sync.ReadCommitted case "r": stmIso = v3sync.RepeatableReads case "s": stmIso = v3sync.Serializable case "ss": stmIso = v3sync.SerializableSnapshot default: fmt.Fprintln(os.Stderr, cmd.Usage()) os.Exit(1) } if stmRate == 0 { stmRate = math.MaxInt32 } limit := rate.NewLimiter(rate.Limit(stmRate), 1) requests := make(chan stmApply, totalClients) clients := mustCreateClients(totalClients, totalConns) bar = pb.New(stmTotal) bar.Start() r := newReport(cmd.Name()) for i := range clients { wg.Add(1) go doSTM(clients[i], requests, r.Results()) } go func() { for i := 0; i < stmTotal; i++ { kset := make(map[string]struct{}) for len(kset) != stmKeysPerTxn { k := make([]byte, 16) binary.PutVarint(k, int64(rand.Intn(stmKeyCount))) s := string(k) kset[s] = struct{}{} } applyf := func(s v3sync.STM) error { limit.Wait(context.Background()) wrs := int(float32(len(kset)*stmWritePercent) / 100.0) for k := range kset { s.Get(k) if wrs > 0 { s.Put(k, string(mustRandBytes(stmValSize))) wrs-- } } return nil } requests <- applyf } close(requests) }() rc := r.Run() wg.Wait() close(r.Results()) bar.Finish() fmt.Printf("%s", <-rc) } func doSTM(client *v3.Client, requests <-chan stmApply, results chan<- report.Result) { defer wg.Done() lock, unlock := func() error { return nil }, func() error { return nil } switch stmLocker { case "lock-client": s, err := v3sync.NewSession(client) if err != nil { panic(err) } defer s.Close() m := v3sync.NewMutex(s, "stmlock") lock = func() error { return m.Lock(context.TODO()) } unlock = func() error { return m.Unlock(context.TODO()) } case "lock-rpc": var lockKey []byte s, err := v3sync.NewSession(client) if err != nil { panic(err) } defer s.Close() lc := v3lockpb.NewLockClient(client.ActiveConnection()) lock = func() error { req := &v3lockpb.LockRequest{Name: []byte("stmlock"), Lease: int64(s.Lease())} resp, err := lc.Lock(context.TODO(), req) if resp != nil { lockKey = resp.Key } return err } unlock = func() error { req := &v3lockpb.UnlockRequest{Key: lockKey} _, err := lc.Unlock(context.TODO(), req) return err } case "stm": default: fmt.Fprintf(os.Stderr, "unexpected stm locker %q\n", stmLocker) os.Exit(1) } for applyf := range requests { st := time.Now() if lerr := lock(); lerr != nil { panic(lerr) } _, err := v3sync.NewSTM(client, applyf, v3sync.WithIsolation(stmIso)) if lerr := unlock(); lerr != nil { panic(lerr) } results <- report.Result{Err: err, Start: st, End: time.Now()} bar.Increment() } } ================================================ FILE: tools/benchmark/cmd/txn_mixed.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "context" "encoding/binary" "fmt" "math" "math/rand" "os" "time" "github.com/cheggaaa/pb/v3" "github.com/spf13/cobra" "golang.org/x/time/rate" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/report" ) // mixeTxnCmd represents the mixedTxn command var mixedTxnCmd = &cobra.Command{ Use: "txn-mixed key [end-range]", Short: "Benchmark a mixed load of txn-put & txn-range.", Run: mixedTxnFunc, } var ( mixedTxnTotal int mixedTxnRate int mixedTxnReadWriteRatio float64 mixedTxnRangeLimit int64 mixedTxnEndKey string writeOpsTotal uint64 readOpsTotal uint64 ) func init() { RootCmd.AddCommand(mixedTxnCmd) mixedTxnCmd.Flags().IntVar(&keySize, "key-size", 8, "Key size of mixed txn") mixedTxnCmd.Flags().IntVar(&valSize, "val-size", 8, "Value size of mixed txn") mixedTxnCmd.Flags().IntVar(&mixedTxnRate, "rate", 0, "Maximum txns per second (0 is no limit)") mixedTxnCmd.Flags().IntVar(&mixedTxnTotal, "total", 10000, "Total number of txn requests") mixedTxnCmd.Flags().StringVar(&mixedTxnEndKey, "end-key", "", "Read operation range end key. By default, we do full range query with the default limit of 1000.") mixedTxnCmd.Flags().Int64Var(&mixedTxnRangeLimit, "limit", 1000, "Read operation range result limit") mixedTxnCmd.Flags().IntVar(&keySpaceSize, "key-space-size", 1, "Maximum possible keys") mixedTxnCmd.Flags().StringVar(&rangeConsistency, "consistency", "l", "Linearizable(l) or Serializable(s)") mixedTxnCmd.Flags().Float64Var(&mixedTxnReadWriteRatio, "rw-ratio", 1, "Read/write ops ratio") } type request struct { isWrite bool op v3.Op } func mixedTxnFunc(cmd *cobra.Command, _ []string) { if keySpaceSize <= 0 { fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", keySpaceSize) os.Exit(1) } if rangeConsistency == "l" { fmt.Println("bench with linearizable range") } else if rangeConsistency == "s" { fmt.Println("bench with serializable range") } else { fmt.Fprintln(os.Stderr, cmd.Usage()) os.Exit(1) } requests := make(chan request, totalClients) if mixedTxnRate == 0 { mixedTxnRate = math.MaxInt32 } limit := rate.NewLimiter(rate.Limit(mixedTxnRate), 1) clients := mustCreateClients(totalClients, totalConns) k, v := make([]byte, keySize), string(mustRandBytes(valSize)) bar = pb.New(mixedTxnTotal) bar.Start() reportRead := newReport(cmd.Name() + "-read") reportWrite := newReport(cmd.Name() + "-write") for i := range clients { wg.Add(1) go func(c *v3.Client) { defer wg.Done() for req := range requests { limit.Wait(context.Background()) st := time.Now() _, err := c.Txn(context.TODO()).Then(req.op).Commit() if req.isWrite { reportWrite.Results() <- report.Result{Err: err, Start: st, End: time.Now()} } else { reportRead.Results() <- report.Result{Err: err, Start: st, End: time.Now()} } bar.Increment() } }(clients[i]) } go func() { for i := 0; i < mixedTxnTotal; i++ { var req request if rand.Float64() < mixedTxnReadWriteRatio/(1+mixedTxnReadWriteRatio) { opts := []v3.OpOption{v3.WithRange(mixedTxnEndKey)} if rangeConsistency == "s" { opts = append(opts, v3.WithSerializable()) } opts = append(opts, v3.WithPrefix(), v3.WithLimit(mixedTxnRangeLimit)) req.op = v3.OpGet("", opts...) req.isWrite = false readOpsTotal++ } else { binary.PutVarint(k, int64(i%keySpaceSize)) req.op = v3.OpPut(string(k), v) req.isWrite = true writeOpsTotal++ } requests <- req } close(requests) }() rcRead := reportRead.Run() rcWrite := reportWrite.Run() wg.Wait() close(reportRead.Results()) close(reportWrite.Results()) bar.Finish() fmt.Printf("Total Read Ops: %d\nDetails:", readOpsTotal) fmt.Println(<-rcRead) fmt.Printf("Total Write Ops: %d\nDetails:", writeOpsTotal) fmt.Println(<-rcWrite) } ================================================ FILE: tools/benchmark/cmd/txn_put.go ================================================ // Copyright 2017 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "context" "encoding/binary" "fmt" "math" "os" "time" "github.com/cheggaaa/pb/v3" "github.com/spf13/cobra" "golang.org/x/time/rate" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/report" ) // txnPutCmd represents the txnPut command var txnPutCmd = &cobra.Command{ Use: "txn-put", Short: "Benchmark txn-put", Run: txnPutFunc, } var ( txnPutTotal int txnPutRate int txnPutOpsPerTxn int ) func init() { RootCmd.AddCommand(txnPutCmd) txnPutCmd.Flags().IntVar(&keySize, "key-size", 8, "Key size of txn put") txnPutCmd.Flags().IntVar(&valSize, "val-size", 8, "Value size of txn put") txnPutCmd.Flags().IntVar(&txnPutOpsPerTxn, "txn-ops", 1, "Number of puts per txn") txnPutCmd.Flags().IntVar(&txnPutRate, "rate", 0, "Maximum txns per second (0 is no limit)") txnPutCmd.Flags().IntVar(&txnPutTotal, "total", 10000, "Total number of txn requests") txnPutCmd.Flags().IntVar(&keySpaceSize, "key-space-size", 1, "Maximum possible keys") } func txnPutFunc(cmd *cobra.Command, _ []string) { if keySpaceSize <= 0 { fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", keySpaceSize) os.Exit(1) } if txnPutOpsPerTxn > keySpaceSize { fmt.Fprintf(os.Stderr, "expected --txn-ops no larger than --key-space-size, "+ "got txn-ops(%v) key-space-size(%v)\n", txnPutOpsPerTxn, keySpaceSize) os.Exit(1) } requests := make(chan []v3.Op, totalClients) if txnPutRate == 0 { txnPutRate = math.MaxInt32 } limit := rate.NewLimiter(rate.Limit(txnPutRate), 1) clients := mustCreateClients(totalClients, totalConns) k, v := make([]byte, keySize), string(mustRandBytes(valSize)) bar = pb.New(txnPutTotal) bar.Start() r := newReport(cmd.Name()) for i := range clients { wg.Add(1) go func(c *v3.Client) { defer wg.Done() for ops := range requests { limit.Wait(context.Background()) st := time.Now() _, err := c.Txn(context.TODO()).Then(ops...).Commit() r.Results() <- report.Result{Err: err, Start: st, End: time.Now()} bar.Increment() } }(clients[i]) } go func() { for i := 0; i < txnPutTotal; i++ { ops := make([]v3.Op, txnPutOpsPerTxn) for j := 0; j < txnPutOpsPerTxn; j++ { binary.PutVarint(k, int64(((i*txnPutOpsPerTxn)+j)%keySpaceSize)) ops[j] = v3.OpPut(string(k), v) } requests <- ops } close(requests) }() rc := r.Run() wg.Wait() close(r.Results()) bar.Finish() fmt.Println(<-rc) } ================================================ FILE: tools/benchmark/cmd/util.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "crypto/rand" "fmt" "os" "strings" "github.com/bgentry/speakeasy" "google.golang.org/grpc/grpclog" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/report" ) var ( // cache the username and password for multiple connections globalUserName string globalPassword string ) func getUsernamePassword(usernameFlag string) (string, string, error) { if globalUserName != "" && globalPassword != "" { return globalUserName, globalPassword, nil } var ok bool globalUserName, globalPassword, ok = strings.Cut(usernameFlag, ":") if !ok { // Prompt for the password. password, err := speakeasy.Ask("Password: ") if err != nil { return "", "", err } globalUserName = usernameFlag globalPassword = password } return globalUserName, globalPassword, nil } func mustCreateConn() *clientv3.Client { cfg := clientv3.Config{ AutoSyncInterval: autoSyncInterval, Endpoints: endpoints, DialTimeout: dialTimeout, } if !tls.Empty() || tls.TrustedCAFile != "" { cfgtls, err := tls.ClientConfig() if err != nil { fmt.Fprintf(os.Stderr, "bad tls config: %v\n", err) os.Exit(1) } cfg.TLS = cfgtls } if len(user) != 0 { username, password, err := getUsernamePassword(user) if err != nil { fmt.Fprintf(os.Stderr, "bad user information: %s %v\n", user, err) os.Exit(1) } cfg.Username = username cfg.Password = password } client, err := clientv3.New(cfg) grpclog.SetLoggerV2(grpclog.NewLoggerV2(os.Stderr, os.Stderr, os.Stderr)) if err != nil { fmt.Fprintf(os.Stderr, "dial error: %v\n", err) os.Exit(1) } return client } func mustCreateClients(totalClients, totalConns uint) []*clientv3.Client { conns := make([]*clientv3.Client, totalConns) for i := range conns { conns[i] = mustCreateConn() } clients := make([]*clientv3.Client, totalClients) for i := range clients { clients[i] = conns[i%int(totalConns)] } return clients } func mustRandBytes(n int) []byte { rb := make([]byte, n) _, err := rand.Read(rb) if err != nil { fmt.Fprintf(os.Stderr, "failed to generate value: %v\n", err) os.Exit(1) } return rb } func newReport(benchmarkOp string) report.Report { p := "%4.4f" if precise { p = "%g" } if sample { return report.NewReportSample(p, benchmarkOp, generatePerfReport) } return report.NewReport(p, benchmarkOp, generatePerfReport) } func newWeightedReport(benchmarkOp string) report.Report { p := "%4.4f" if precise { p = "%g" } if sample { return report.NewReportSample(p, benchmarkOp, generatePerfReport) } return report.NewWeightedReport(report.NewReport(p, benchmarkOp, generatePerfReport), p, benchmarkOp, generatePerfReport) } ================================================ FILE: tools/benchmark/cmd/watch.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "context" "encoding/binary" "fmt" "math/rand" "os" "sync/atomic" "time" "github.com/cheggaaa/pb/v3" "github.com/spf13/cobra" "golang.org/x/time/rate" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/report" ) // watchCmd represents the watch command var watchCmd = &cobra.Command{ Use: "watch", Short: "Benchmark watch", Long: `Benchmark watch tests the performance of processing watch requests and sending events to watchers. It tests the sending performance by changing the value of the watched keys with concurrent put requests. During the test, each watcher watches (--total/--watchers) keys (a watcher might watch on the same key multiple times if --watched-key-total is small). Each key is watched by (--total/--watched-key-total) watchers. `, Run: watchFunc, } var ( watchStreams int watchWatchesPerStream int watchedKeyTotal int watchPutRate int watchPutTotal int watchKeySize int watchKeySpaceSize int watchSeqKeys bool ) type watchedKeys struct { watched []string numWatchers map[string]int watches []clientv3.WatchChan // ctx to control all watches ctx context.Context cancel context.CancelFunc } func init() { RootCmd.AddCommand(watchCmd) watchCmd.Flags().IntVar(&watchStreams, "streams", 10, "Total watch streams") watchCmd.Flags().IntVar(&watchWatchesPerStream, "watch-per-stream", 100, "Total watchers per stream") watchCmd.Flags().IntVar(&watchedKeyTotal, "watched-key-total", 1, "Total number of keys to be watched") watchCmd.Flags().IntVar(&watchPutRate, "put-rate", 0, "Number of keys to put per second") watchCmd.Flags().IntVar(&watchPutTotal, "put-total", 1000, "Number of put requests") watchCmd.Flags().IntVar(&watchKeySize, "key-size", 32, "Key size of watch request") watchCmd.Flags().IntVar(&watchKeySpaceSize, "key-space-size", 1, "Maximum possible keys") watchCmd.Flags().BoolVar(&watchSeqKeys, "sequential-keys", false, "Use sequential keys") } func watchFunc(_ *cobra.Command, _ []string) { if watchKeySpaceSize <= 0 { fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", watchKeySpaceSize) os.Exit(1) } grpcConns := int(totalClients) if totalClients > totalConns { grpcConns = int(totalConns) } wantedConns := 1 + (watchStreams / 100) if grpcConns < wantedConns { fmt.Fprintf(os.Stderr, "warning: grpc limits 100 streams per client connection, have %d but need %d\n", grpcConns, wantedConns) } clients := mustCreateClients(totalClients, totalConns) wk := newWatchedKeys() benchMakeWatches(clients, wk) benchPutWatches(clients, wk) } func benchMakeWatches(clients []*clientv3.Client, wk *watchedKeys) { streams := make([]clientv3.Watcher, watchStreams) for i := range streams { streams[i] = clientv3.NewWatcher(clients[i%len(clients)]) } keyc := make(chan string, watchStreams) bar = pb.New(watchStreams * watchWatchesPerStream) bar.Start() r := newReport("watch-make") rch := r.Results() wg.Add(len(streams) + 1) wc := make(chan []clientv3.WatchChan, len(streams)) for _, s := range streams { go func(s clientv3.Watcher) { defer wg.Done() var ws []clientv3.WatchChan for i := 0; i < watchWatchesPerStream; i++ { k := <-keyc st := time.Now() wch := s.Watch(wk.ctx, k) rch <- report.Result{Start: st, End: time.Now()} ws = append(ws, wch) bar.Increment() } wc <- ws }(s) } go func() { defer func() { close(keyc) wg.Done() }() for i := 0; i < watchStreams*watchWatchesPerStream; i++ { key := wk.watched[i%len(wk.watched)] keyc <- key wk.numWatchers[key]++ } }() rc := r.Run() wg.Wait() bar.Finish() close(r.Results()) fmt.Printf("Watch creation summary:\n%s", <-rc) for i := 0; i < len(streams); i++ { wk.watches = append(wk.watches, (<-wc)...) } } func newWatchedKeys() *watchedKeys { watched := make([]string, watchedKeyTotal) for i := range watched { k := make([]byte, watchKeySize) if watchSeqKeys { binary.PutVarint(k, int64(i%watchKeySpaceSize)) } else { binary.PutVarint(k, int64(rand.Intn(watchKeySpaceSize))) } watched[i] = string(k) } ctx, cancel := context.WithCancel(context.TODO()) return &watchedKeys{ watched: watched, numWatchers: make(map[string]int), ctx: ctx, cancel: cancel, } } func benchPutWatches(clients []*clientv3.Client, wk *watchedKeys) { eventsTotal := 0 for i := 0; i < watchPutTotal; i++ { eventsTotal += wk.numWatchers[wk.watched[i%len(wk.watched)]] } bar = pb.New(eventsTotal) bar.Start() r := newReport("watch-put") wg.Add(len(wk.watches)) nrRxed := int32(eventsTotal) for _, w := range wk.watches { go func(wc clientv3.WatchChan) { defer wg.Done() recvWatchChan(wc, r.Results(), &nrRxed) wk.cancel() }(w) } putreqc := make(chan clientv3.Op, len(clients)) go func() { defer close(putreqc) for i := 0; i < watchPutTotal; i++ { putreqc <- clientv3.OpPut(wk.watched[i%(len(wk.watched))], "data") } }() watchPutLimit := rate.Inf if watchPutRate > 0 { watchPutLimit = rate.Limit(watchPutRate) } limit := rate.NewLimiter(watchPutLimit, 1) for _, cc := range clients { go func(c *clientv3.Client) { for op := range putreqc { if err := limit.Wait(context.TODO()); err != nil { panic(err) } if _, err := c.Do(context.TODO(), op); err != nil { panic(err) } } }(cc) } rc := r.Run() wg.Wait() bar.Finish() close(r.Results()) fmt.Printf("Watch events received summary:\n%s", <-rc) } func recvWatchChan(wch clientv3.WatchChan, results chan<- report.Result, nrRxed *int32) { for r := range wch { st := time.Now() for range r.Events { results <- report.Result{Start: st, End: time.Now()} bar.Increment() if atomic.AddInt32(nrRxed, -1) <= 0 { return } } } } ================================================ FILE: tools/benchmark/cmd/watch_get.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "context" "fmt" "sync" "time" "github.com/cheggaaa/pb/v3" "github.com/spf13/cobra" v3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/report" ) // watchGetCmd represents the watch command var watchGetCmd = &cobra.Command{ Use: "watch-get", Short: "Benchmark watch with get", Long: `Benchmark for serialized key gets with many unsynced watchers`, Run: watchGetFunc, } var ( watchGetTotalWatchers int watchGetTotalStreams int watchEvents int firstWatch sync.Once ) func init() { RootCmd.AddCommand(watchGetCmd) watchGetCmd.Flags().IntVar(&watchGetTotalWatchers, "watchers", 10000, "Total number of watchers") watchGetCmd.Flags().IntVar(&watchGetTotalStreams, "streams", 1, "Total number of watcher streams") watchGetCmd.Flags().IntVar(&watchEvents, "events", 8, "Number of events per watcher") } func watchGetFunc(cmd *cobra.Command, _ []string) { clients := mustCreateClients(totalClients, totalConns) getClient := mustCreateClients(1, 1) // setup keys for watchers watchRev := int64(0) for i := 0; i < watchEvents; i++ { v := fmt.Sprintf("%d", i) resp, err := clients[0].Put(context.TODO(), "watchkey", v) if err != nil { panic(err) } if i == 0 { watchRev = resp.Header.Revision } } streams := make([]v3.Watcher, watchGetTotalStreams) for i := range streams { streams[i] = v3.NewWatcher(clients[i%len(clients)]) } bar = pb.New(watchGetTotalWatchers * watchEvents) bar.Start() // report from trying to do serialized gets with concurrent watchers r := newReport(cmd.Name()) ctx, cancel := context.WithCancel(context.TODO()) f := func() { defer close(r.Results()) for { st := time.Now() _, err := getClient[0].Get(ctx, "abc", v3.WithSerializable()) if ctx.Err() != nil { break } r.Results() <- report.Result{Err: err, Start: st, End: time.Now()} } } wg.Add(watchGetTotalWatchers) for i := 0; i < watchGetTotalWatchers; i++ { go doUnsyncWatch(streams[i%len(streams)], watchRev, f) } rc := r.Run() wg.Wait() cancel() bar.Finish() fmt.Printf("Get during watch summary:\n%s", <-rc) } func doUnsyncWatch(stream v3.Watcher, rev int64, f func()) { defer wg.Done() wch := stream.Watch(context.TODO(), "watchkey", v3.WithRev(rev)) if wch == nil { panic("could not open watch channel") } firstWatch.Do(func() { go f() }) i := 0 for i < watchEvents { wev := <-wch i += len(wev.Events) bar.Add(len(wev.Events)) } } ================================================ FILE: tools/benchmark/cmd/watch_latency.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "context" "fmt" "os" "time" "github.com/cheggaaa/pb/v3" "github.com/spf13/cobra" "golang.org/x/time/rate" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/report" ) // watchLatencyCmd represents the watch latency command var watchLatencyCmd = &cobra.Command{ Use: "watch-latency", Short: "Benchmark watch latency", Long: `Benchmarks the latency for watches by measuring the latency between writing to a key and receiving the associated watch response.`, Run: watchLatencyFunc, } var ( watchLPutTotal int watchLPutRate int watchLKeySize int watchLValueSize int watchLStreams int watchLWatchersPerStream int watchLPrevKV bool ) func init() { RootCmd.AddCommand(watchLatencyCmd) watchLatencyCmd.Flags().IntVar(&watchLStreams, "streams", 10, "Total watch streams") watchLatencyCmd.Flags().IntVar(&watchLWatchersPerStream, "watchers-per-stream", 10, "Total watchers per stream") watchLatencyCmd.Flags().BoolVar(&watchLPrevKV, "prevkv", false, "PrevKV enabled on watch requests") watchLatencyCmd.Flags().IntVar(&watchLPutTotal, "put-total", 1000, "Total number of put requests") watchLatencyCmd.Flags().IntVar(&watchLPutRate, "put-rate", 100, "Number of keys to put per second") watchLatencyCmd.Flags().IntVar(&watchLKeySize, "key-size", 32, "Key size of watch response") watchLatencyCmd.Flags().IntVar(&watchLValueSize, "val-size", 32, "Value size of watch response") } func watchLatencyFunc(cmd *cobra.Command, _ []string) { key := string(mustRandBytes(watchLKeySize)) value := string(mustRandBytes(watchLValueSize)) wchs := setupWatchChannels(key) putClient := mustCreateConn() bar = pb.New(watchLPutTotal * len(wchs)) bar.Start() limiter := rate.NewLimiter(rate.Limit(watchLPutRate), watchLPutRate) putTimes := make([]time.Time, watchLPutTotal) eventTimes := make([][]time.Time, len(wchs)) for i, wch := range wchs { eventTimes[i] = make([]time.Time, watchLPutTotal) wg.Go(func() { eventCount := 0 for eventCount < watchLPutTotal { resp := <-wch for range resp.Events { eventTimes[i][eventCount] = time.Now() eventCount++ bar.Increment() } } }) } putReport := newReport(cmd.Name() + "-put") putReportResults := putReport.Run() watchReport := newReport(cmd.Name() + "-watch") watchReportResults := watchReport.Run() for i := 0; i < watchLPutTotal; i++ { // limit key put as per reqRate if err := limiter.Wait(context.TODO()); err != nil { break } start := time.Now() if _, err := putClient.Put(context.TODO(), key, value); err != nil { fmt.Fprintf(os.Stderr, "Failed to Put for watch latency benchmark: %v\n", err) os.Exit(1) } end := time.Now() putReport.Results() <- report.Result{Start: start, End: end} putTimes[i] = end } wg.Wait() close(putReport.Results()) bar.Finish() fmt.Printf("\nPut summary:\n%s", <-putReportResults) for i := 0; i < len(wchs); i++ { for j := 0; j < watchLPutTotal; j++ { start := putTimes[j] end := eventTimes[i][j] if end.Before(start) { start = end } watchReport.Results() <- report.Result{Start: start, End: end} } } close(watchReport.Results()) fmt.Printf("\nWatch events summary:\n%s", <-watchReportResults) } func setupWatchChannels(key string) []clientv3.WatchChan { clients := mustCreateClients(totalClients, totalConns) streams := make([]clientv3.Watcher, watchLStreams) for i := range streams { streams[i] = clientv3.NewWatcher(clients[i%len(clients)]) } opts := []clientv3.OpOption{} if watchLPrevKV { opts = append(opts, clientv3.WithPrevKV()) } wchs := make([]clientv3.WatchChan, len(streams)*watchLWatchersPerStream) for i := 0; i < len(streams); i++ { for j := 0; j < watchLWatchersPerStream; j++ { wchs[i*watchLWatchersPerStream+j] = streams[i].Watch(context.TODO(), key, opts...) } } return wchs } ================================================ FILE: tools/benchmark/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // benchmark is a program for benchmarking etcd v3 API performance. package main ================================================ FILE: tools/benchmark/main.go ================================================ // Copyright 2015 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "fmt" "os" "go.etcd.io/etcd/v3/tools/benchmark/cmd" ) func main() { if err := cmd.RootCmd.Execute(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(-1) } } ================================================ FILE: tools/check-grpc-experimental/allowlist.txt ================================================ # Allowlist for experimental gRPC APIs # Format: PackageName.Symbol # Remove items from this list as they are migrated or stabilized. grpc.NewContextWithServerTransportStream grpc.ServeHTTP grpc.WithResolvers resolver.Address resolver.BuildOptions resolver.Builder resolver.ClientConn resolver.Endpoint resolver.ParseServiceConfig resolver.ResolveNowOptions resolver.Resolver resolver.State resolver.Target resolver.URL resolver.UpdateState serviceconfig.Err serviceconfig.ParseResult ================================================ FILE: tools/check-grpc-experimental/doc.go ================================================ // Copyright 2026 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // check-grpc-experimental checks for experimental gRPC APIs in etcd. package main ================================================ FILE: tools/check-grpc-experimental/main.go ================================================ // Copyright 2026 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "bufio" "flag" "fmt" "go/ast" "go/parser" "go/token" "go/types" "log" "os" "path/filepath" "regexp" "strings" "sync" "golang.org/x/tools/go/packages" ) var ( debugMode = flag.Bool("debug", false, "enable verbose debug logging") allowListFile = flag.String("allow-list", "", "path to a file containing allowed APIs (one per line)") ) // Map to store allowed signatures (e.g., "grpc.WithResolvers" -> true) var allowList = make(map[string]bool) func main() { flag.Parse() patterns := flag.Args() if len(patterns) == 0 { patterns = []string{"./..."} } if *allowListFile != "" { if err := loadAllowList(*allowListFile); err != nil { log.Fatalf("Failed to load allow list: %v", err) } } // Load source with type info. cfg := &packages.Config{ Mode: packages.NeedName | packages.NeedFiles | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedImports | packages.NeedDeps, Tests: true, } if *debugMode { log.Println("Loading packages...") } pkgs, err := packages.Load(cfg, patterns...) if err != nil { log.Fatalf("failed to load packages: %v", err) } if n := packages.PrintErrors(pkgs); n > 0 { os.Exit(1) } if *debugMode { log.Printf("Loaded %d packages. Scanning for gRPC usage...", len(pkgs)) } foundExperimental := false for _, pkg := range pkgs { for _, file := range pkg.Syntax { ast.Inspect(file, func(n ast.Node) bool { sel, ok := n.(*ast.SelectorExpr) if !ok { return true } obj := pkg.TypesInfo.Uses[sel.Sel] if obj == nil || obj.Pkg() == nil { return true } // Strict filter for gRPC if !strings.Contains(obj.Pkg().Path(), "google.golang.org/grpc") { return true } // Check Allowlist // Construct the signature: PackageName.Symbol (e.g. "grpc.WithResolvers", "resolver.Address") signature := obj.Pkg().Name() + "." + obj.Name() if allowList[signature] { if *debugMode { log.Printf("Ignoring allowed usage: %s", signature) } return true } if *debugMode { log.Printf("Checking reference: %s", signature) } if isExperimental(pkg.Fset, obj) { pos := pkg.Fset.Position(sel.Pos()) fmt.Printf("%s:%d:%d: usage of experimental gRPC API: %s\n", pos.Filename, pos.Line, pos.Column, signature) foundExperimental = true } return true }) } } if foundExperimental { os.Exit(1) } } var ( experimentalRegex = regexp.MustCompile(`(?i)(#\s*Experimental|All APIs in this package are experimental|This API is EXPERIMENTAL|is currently experimental)`) ) func loadAllowList(fpath string) error { f, err := os.Open(fpath) if err != nil { return err } defer f.Close() scanner := bufio.NewScanner(f) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } allowList[line] = true } return scanner.Err() } func isExperimental(mainFset *token.FileSet, obj types.Object) bool { pos := obj.Pos() if !pos.IsValid() { return false } // Get the absolute path to the dependency file position := mainFset.Position(pos) filename := position.Filename // Skip if it's not a Go file (e.g. built-in types might have no file) if !strings.HasSuffix(filename, ".go") { return false } if *debugMode { // Log checking definition log.Printf(" -> Definition found at: %s:%d", filename, position.Line) } return checkFileForExperimental(filename, position.Line) } // Cached file structure type cachedFile struct { f *ast.File fset *token.FileSet src []byte hasDocs bool } var ( cacheMu sync.RWMutex advancedCache = make(map[string]*cachedFile) ) func getParsedFile(filename string) (*cachedFile, error) { cacheMu.RLock() if v, ok := advancedCache[filename]; ok { cacheMu.RUnlock() return v, nil } cacheMu.RUnlock() // Parse the file fset := token.NewFileSet() // We read the file content manually to help with debugging if needed src, err := os.ReadFile(filename) if err != nil { if *debugMode { log.Printf("ERROR reading file %s: %v", filename, err) } return nil, err } f, err := parser.ParseFile(fset, filename, src, parser.ParseComments|parser.SkipObjectResolution) if err != nil { return nil, err } res := &cachedFile{f: f, fset: fset, src: src, hasDocs: f.Doc != nil} cacheMu.Lock() advancedCache[filename] = res cacheMu.Unlock() return res, nil } func checkFileForExperimental(filename string, targetLine int) bool { cf, err := getParsedFile(filename) if err != nil { return false } // check package-level comments if cf.f.Doc != nil { if experimentalRegex.MatchString(cf.f.Doc.Text()) { if *debugMode { log.Printf(" -> [MATCH] Package experimental (doc in file): %s", filename) } return true } } // check package-level comment in doc.go // If the current file didn't have the experimental tag, check if a doc.go exists in the same folder dir := filepath.Dir(filename) docPath := filepath.Join(dir, "doc.go") // Only check doc.go if we aren't already looking at it if docPath != filename { if docContent, err := os.ReadFile(docPath); err == nil { if experimentalRegex.Match(docContent) { if *debugMode { log.Printf(" -> [MATCH] Package experimental (found in doc.go): %s", docPath) } return true } } } // check specific object comments found := false ast.Inspect(cf.f, func(n ast.Node) bool { if found { return false } if n == nil { return true } // Helper to check a comment group checkDoc := func(doc *ast.CommentGroup, name string) { if doc != nil && experimentalRegex.MatchString(doc.Text()) { found = true if *debugMode { log.Printf(" -> [MATCH] Object experimental: %s", name) } } } switch decl := n.(type) { case *ast.FuncDecl: // Match if the target line is within the function declaration lines // Actually, we want the definition line exactly, or close to it. start := cf.fset.Position(decl.Pos()).Line // The object.Pos() points to the name, not the 'func' keyword, usually. namePos := cf.fset.Position(decl.Name.Pos()).Line if namePos == targetLine || start == targetLine { checkDoc(decl.Doc, decl.Name.Name) } case *ast.GenDecl: // GenDecl covers `type X struct`, `var X`, `const X` // The GenDecl doc applies to all specs inside it usually. // If the GenDecl itself starts on the line (e.g. `type ( ...`) // or if it contains our line. start := cf.fset.Position(decl.Pos()).Line end := cf.fset.Position(decl.End()).Line if targetLine >= start && targetLine <= end { // Check the top-level GenDecl doc (e.g. "// Experimental\n var ( ... )") if decl.Doc != nil && experimentalRegex.MatchString(decl.Doc.Text()) { found = true if *debugMode { log.Printf(" -> [MATCH] GenDecl experimental block around line %d", targetLine) } return false } // Check individual specs for _, spec := range decl.Specs { switch s := spec.(type) { case *ast.TypeSpec: if cf.fset.Position(s.Name.Pos()).Line == targetLine { checkDoc(s.Doc, s.Name.Name) } case *ast.ValueSpec: // var/const for _, name := range s.Names { if cf.fset.Position(name.Pos()).Line == targetLine { checkDoc(s.Doc, name.Name) } } } } } } return true }) return found } ================================================ FILE: tools/etcd-dump-db/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/debugging ================================================ FILE: tools/etcd-dump-db/README.md ================================================ # etcd-dump-db `etcd-dump-db` inspects etcd db files. > ⚠️ **Deprecated**: This tool has been deprecated. > Please use [etcdutl](https://github.com/etcd-io/etcd/tree/main/etcdutl) instead. ## Installation Install the tool by running the following command from the etcd source directory. ``` $ go install -v ./tools/etcd-dump-db ``` The installation will place executables in the $GOPATH/bin. If $GOPATH environment variable is not set, the tool will be installed into the $HOME/go/bin. You can also find out the installed location by running the following command from the etcd source directory. Make sure that $PATH is set accordingly in your environment. ``` $ go list -f "{{.Target}}" ./tools/etcd-dump-db ``` Alternatively, instead of installing the tool, you can use it by simply running the following command from the etcd source directory. ``` $ go run ./tools/etcd-dump-db ``` ## Usage The following command should output the usage per the latest development. ``` $ etcd-dump-db --help ``` An example of usage detail is provided below. ``` Usage: etcd-dump-db [command] Available Commands: list-bucket bucket lists all buckets. iterate-bucket iterate-bucket lists key-value pairs in reverse order. hash hash computes the hash of db file. Flags: -h, --help[=false]: help for etcd-dump-db Use "etcd-dump-db [command] --help" for more information about a command. ``` #### list-bucket [data dir or db file path] Lists all buckets. ``` $ etcd-dump-db list-bucket agent01/agent.etcd alarm auth authRoles authUsers cluster key lease members members_removed meta ``` #### hash [data dir or db file path] Computes the hash of db file. ``` $ etcd-dump-db hash agent01/agent.etcd db path: agent01/agent.etcd/member/snap/db Hash: 3700260467 $ etcd-dump-db hash agent02/agent.etcd db path: agent02/agent.etcd/member/snap/db Hash: 3700260467 $ etcd-dump-db hash agent03/agent.etcd db path: agent03/agent.etcd/member/snap/db Hash: 3700260467 ``` #### iterate-bucket [data dir or db file path] Lists key-value pairs in reverse order. ``` $ etcd-dump-db iterate-bucket agent03/agent.etcd key --limit 3 key="\x00\x00\x00\x00\x005@x_\x00\x00\x00\x00\x00\x00\x00\tt", value="\n\x153640412599896088633_9" key="\x00\x00\x00\x00\x005@x_\x00\x00\x00\x00\x00\x00\x00\bt", value="\n\x153640412599896088633_8" key="\x00\x00\x00\x00\x005@x_\x00\x00\x00\x00\x00\x00\x00\at", value="\n\x153640412599896088633_7" ``` #### scan-keys [data dir or db file path] Scans all the key-value pairs starting from a specific revision in the key space. It works even the db is corrupted. ``` $ ./etcd-dump-db scan-keys ~/tmp/etcd/778/db.db 16589739 2>/dev/null | grep "/registry/configmaps/istio-system/istio-namespace-controller-election" pageID=1306, index=5/5, rev={Revision:{Main:16589739 Sub:0} tombstone:false}, value=[key "/registry/configmaps/istio-system/istio-namespace-controller-election" | val "k8s\x00\n\x0f\n\x02v1\x12\tConfigMap\x12\xeb\x03\n\xe8\x03\n#istio-namespace-controller-election\x12\x00\x1a\fistio-system\"\x00*$bb696087-260d-4167-bf06-17d3361f9b5f2\x008\x00B\b\b\x9e\xbe\xed\xb5\x06\x10\x00b\xe6\x01\n(control-plane.alpha.kubernetes.io/leader\x12\xb9\x01{\"holderIdentity\":\"istiod-d56968787-txq2d\",\"holderKey\":\"default\",\"leaseDurationSeconds\":30,\"acquireTime\":\"2024-08-13T13:26:54Z\",\"renewTime\":\"2024-08-27T06:16:13Z\",\"leaderTransitions\":0}\x8a\x01\x90\x01\n\x0fpilot-discovery\x12\x06Update\x1a\x02v1\"\b\b\xad\u07b5\xb6\x06\x10\x002\bFieldsV1:[\nY{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:control-plane.alpha.kubernetes.io/leader\":{}}}}B\x00\x1a\x00\"\x00" | created 9612546 | mod 16589739 | ver 157604] pageID=4737, index=4/4, rev={Revision:{Main:16589786 Sub:0} tombstone:false}, value=[key "/registry/configmaps/istio-system/istio-namespace-controller-election" | val "k8s\x00\n\x0f\n\x02v1\x12\tConfigMap\x12\xeb\x03\n\xe8\x03\n#istio-namespace-controller-election\x12\x00\x1a\fistio-system\"\x00*$bb696087-260d-4167-bf06-17d3361f9b5f2\x008\x00B\b\b\x9e\xbe\xed\xb5\x06\x10\x00b\xe6\x01\n(control-plane.alpha.kubernetes.io/leader\x12\xb9\x01{\"holderIdentity\":\"istiod-d56968787-txq2d\",\"holderKey\":\"default\",\"leaseDurationSeconds\":30,\"acquireTime\":\"2024-08-13T13:26:54Z\",\"renewTime\":\"2024-08-27T06:16:21Z\",\"leaderTransitions\":0}\x8a\x01\x90\x01\n\x0fpilot-discovery\x12\x06Update\x1a\x02v1\"\b\b\xb5\u07b5\xb6\x06\x10\x002\bFieldsV1:[\nY{\"f:metadata\":{\"f:annotations\":{\".\":{},\"f:control-plane.alpha.kubernetes.io/leader\":{}}}}B\x00\x1a\x00\"\x00" | created 9612546 | mod 16589786 | ver 157605] ``` ================================================ FILE: tools/etcd-dump-db/backend.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "encoding/binary" "fmt" "path/filepath" "go.uber.org/zap" bolt "go.etcd.io/bbolt" "go.etcd.io/etcd/api/v3/authpb" "go.etcd.io/etcd/api/v3/mvccpb" "go.etcd.io/etcd/server/v3/lease/leasepb" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" ) func snapDir(dataDir string) string { return filepath.Join(dataDir, "member", "snap") } func getBuckets(dbPath string) (buckets []string, err error) { db, derr := bolt.Open(dbPath, 0o600, &bolt.Options{Timeout: flockTimeout}) if derr != nil { return nil, fmt.Errorf("failed to open bolt DB %w", derr) } defer db.Close() err = db.View(func(tx *bolt.Tx) error { return tx.ForEach(func(b []byte, _ *bolt.Bucket) error { buckets = append(buckets, string(b)) return nil }) }) return buckets, err } // TODO: import directly from packages, rather than copy&paste type decoder func(k, v []byte) // key is the bucket name, and value is the function to decode K/V in the bucket. var decoders = map[string]decoder{ "key": keyDecoder, "lease": leaseDecoder, "auth": authDecoder, "authRoles": authRolesDecoder, "authUsers": authUsersDecoder, "meta": metaDecoder, } func defaultDecoder(k, v []byte) { fmt.Printf("key=%q, value=%q\n", k, v) } func keyDecoder(k, v []byte) { rev := mvcc.BytesToBucketKey(k) var kv mvccpb.KeyValue if err := kv.Unmarshal(v); err != nil { panic(err) } fmt.Printf("rev=%+v, value=[key %q | val %q | created %d | mod %d | ver %d]\n", rev, string(kv.Key), string(kv.Value), kv.CreateRevision, kv.ModRevision, kv.Version) } func bytesToLeaseID(bytes []byte) int64 { if len(bytes) != 8 { panic(fmt.Errorf("lease ID must be 8-byte")) } return int64(binary.BigEndian.Uint64(bytes)) } func leaseDecoder(k, v []byte) { leaseID := bytesToLeaseID(k) var lpb leasepb.Lease if err := lpb.Unmarshal(v); err != nil { panic(err) } fmt.Printf("lease ID=%016x, TTL=%ds, remaining TTL=%ds\n", leaseID, lpb.TTL, lpb.RemainingTTL) } func authDecoder(k, v []byte) { if string(k) == "authRevision" { rev := binary.BigEndian.Uint64(v) fmt.Printf("key=%q, value=%v\n", k, rev) } else { fmt.Printf("key=%q, value=%v\n", k, v) } } func authRolesDecoder(_, v []byte) { role := &authpb.Role{} err := role.Unmarshal(v) if err != nil { panic(err) } fmt.Printf("role=%q, keyPermission=%v\n", string(role.Name), role.KeyPermission) } func authUsersDecoder(_, v []byte) { user := &authpb.User{} err := user.Unmarshal(v) if err != nil { panic(err) } fmt.Printf("user=%q, roles=%q, option=%v\n", user.Name, user.Roles, user.Options) } func metaDecoder(k, v []byte) { if string(k) == string(schema.MetaConsistentIndexKeyName) || string(k) == string(schema.MetaTermKeyName) { fmt.Printf("key=%q, value=%v\n", k, binary.BigEndian.Uint64(v)) } else if string(k) == string(schema.ScheduledCompactKeyName) || string(k) == string(schema.FinishedCompactKeyName) { rev := mvcc.BytesToRev(v) fmt.Printf("key=%q, value=%v\n", k, rev) } else { defaultDecoder(k, v) } } func iterateBucket(dbPath, bucket string, limit uint64, decode bool) (err error) { db, err := bolt.Open(dbPath, 0o600, &bolt.Options{Timeout: flockTimeout}) if err != nil { return fmt.Errorf("failed to open bolt DB %w", err) } defer db.Close() err = db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(bucket)) if b == nil { return fmt.Errorf("got nil bucket for %s", bucket) } c := b.Cursor() // iterate in reverse order (use First() and Next() for ascending order) for k, v := c.Last(); k != nil; k, v = c.Prev() { // TODO: remove sensitive information // (https://github.com/etcd-io/etcd/issues/7620) if dec, ok := decoders[bucket]; decode && ok { dec(k, v) } else { defaultDecoder(k, v) } limit-- if limit == 0 { break } } return nil }) return err } func getHash(dbPath string) (hash uint32, err error) { b := backend.NewDefaultBackend(zap.NewNop(), dbPath) return b.Hash(schema.DefaultIgnores) } // TODO: revert by revision and find specified hash value // currently, it's hard because lease is in separate bucket // and does not modify revision ================================================ FILE: tools/etcd-dump-db/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // etcd-dump-db inspects etcd db files. package main ================================================ FILE: tools/etcd-dump-db/main.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "fmt" "log" "os" "path/filepath" "strconv" "strings" "time" "github.com/spf13/cobra" "go.etcd.io/etcd/client/pkg/v3/fileutil" ) var ( rootCommand = &cobra.Command{ Use: "etcd-dump-db", Short: "etcd-dump-db inspects etcd db files.", } listBucketCommand = &cobra.Command{ Use: "list-bucket [data dir or db file path]", Short: "bucket lists all buckets.", Run: listBucketCommandFunc, } iterateBucketCommand = &cobra.Command{ Use: "iterate-bucket [data dir or db file path] [bucket name]", Short: "iterate-bucket lists key-value pairs in reverse order.", Run: iterateBucketCommandFunc, } scanKeySpaceCommand = &cobra.Command{ Use: "scan-keys [data dir or db file path] [start revision]", Short: "scan-keys scans all the key-value pairs starting from a specific revision in the key space.", Run: scanKeysCommandFunc, } getHashCommand = &cobra.Command{ Use: "hash [data dir or db file path]", Short: "hash computes the hash of db file.", Run: getHashCommandFunc, } ) var ( flockTimeout time.Duration iterateBucketLimit uint64 iterateBucketDecode bool ) func init() { rootCommand.PersistentFlags().DurationVar(&flockTimeout, "timeout", 10*time.Second, "time to wait to obtain a file lock on db file, 0 to block indefinitely") iterateBucketCommand.PersistentFlags().Uint64Var(&iterateBucketLimit, "limit", 0, "max number of key-value pairs to iterate (0< to iterate all)") iterateBucketCommand.PersistentFlags().BoolVar(&iterateBucketDecode, "decode", false, "true to decode Protocol Buffer encoded data") rootCommand.AddCommand(listBucketCommand) rootCommand.AddCommand(iterateBucketCommand) rootCommand.AddCommand(scanKeySpaceCommand) rootCommand.AddCommand(getHashCommand) } func main() { if err := rootCommand.Execute(); err != nil { fmt.Fprintln(os.Stdout, err) os.Exit(1) } } func listBucketCommandFunc(_ *cobra.Command, args []string) { if len(args) < 1 { log.Fatalf("Must provide at least 1 argument (got %v)", args) } dp := args[0] if !strings.HasSuffix(dp, "db") { dp = filepath.Join(snapDir(dp), "db") } if !fileutil.Exist(dp) { log.Fatalf("%q does not exist", dp) } bts, err := getBuckets(dp) if err != nil { log.Fatal(err) } for _, b := range bts { fmt.Println(b) } } func iterateBucketCommandFunc(_ *cobra.Command, args []string) { if len(args) != 2 { log.Fatalf("Must provide 2 arguments (got %v)", args) } dp := args[0] if !strings.HasSuffix(dp, "db") { dp = filepath.Join(snapDir(dp), "db") } if !fileutil.Exist(dp) { log.Fatalf("%q does not exist", dp) } bucket := args[1] err := iterateBucket(dp, bucket, iterateBucketLimit, iterateBucketDecode) if err != nil { log.Fatal(err) } } func scanKeysCommandFunc(_ *cobra.Command, args []string) { if len(args) != 2 { log.Fatalf("Must provide 2 arguments (got %v)", args) } dp := args[0] if !strings.HasSuffix(dp, "db") { dp = filepath.Join(snapDir(dp), "db") } if !fileutil.Exist(dp) { log.Fatalf("%q does not exist", dp) } startRev, err := strconv.ParseInt(args[1], 10, 64) if err != nil { log.Fatal(err) } err = scanKeys(dp, startRev) if err != nil { log.Fatal(err) } } func getHashCommandFunc(_ *cobra.Command, args []string) { if len(args) < 1 { log.Fatalf("Must provide at least 1 argument (got %v)", args) } dp := args[0] if !strings.HasSuffix(dp, "db") { dp = filepath.Join(snapDir(dp), "db") } if !fileutil.Exist(dp) { log.Fatalf("%q does not exist", dp) } hash, err := getHash(dp) if err != nil { log.Fatal(err) } fmt.Printf("db path: %s\nHash: %d\n", dp, hash) } ================================================ FILE: tools/etcd-dump-db/meta.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import "unsafe" const magic uint32 = 0xED0CDAED type inBucket struct { root uint64 // page id of the bucket's root-level page sequence uint64 // monotonically incrementing, used by NextSequence() } type meta struct { magic uint32 version uint32 pageSize uint32 flags uint32 root inBucket freelist uint64 pgid uint64 txid uint64 checksum uint64 } func loadPageMeta(buf []byte) *meta { return (*meta)(unsafe.Pointer(&buf[pageHeaderSize])) } ================================================ FILE: tools/etcd-dump-db/page.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import "unsafe" const ( pageHeaderSize = unsafe.Sizeof(page{}) leafPageElementSize = unsafe.Sizeof(leafPageElement{}) pageMaxAllocSize = 0xFFFFFFF ) const ( leafPageFlag = 0x02 ) type page struct { id uint64 flags uint16 count uint16 overflow uint32 } func (p *page) isLeafPage() bool { return p.flags == leafPageFlag } func loadPage(buf []byte) *page { return (*page)(unsafe.Pointer(&buf[0])) } // leafPageElement retrieves the leaf node by index func (p *page) leafPageElement(index uint16) *leafPageElement { return (*leafPageElement)(unsafeIndex(unsafe.Pointer(p), unsafe.Sizeof(*p), leafPageElementSize, int(index))) } // leafPageElement represents a node on a leaf page. type leafPageElement struct { flags uint32 pos uint32 ksize uint32 vsize uint32 } // Key returns a byte slice of the node key. func (n *leafPageElement) key() []byte { i := int(n.pos) j := i + int(n.ksize) return unsafeByteSlice(unsafe.Pointer(n), 0, i, j) } // Value returns a byte slice of the node value. func (n *leafPageElement) value() []byte { i := int(n.pos) + int(n.ksize) j := i + int(n.vsize) return unsafeByteSlice(unsafe.Pointer(n), 0, i, j) } ================================================ FILE: tools/etcd-dump-db/scan.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "fmt" "io" "os" "go.etcd.io/etcd/server/v3/storage/mvcc" ) func scanKeys(dbPath string, startRev int64) error { pgSize, hwm, err := readPageAndHWMSize(dbPath) if err != nil { return fmt.Errorf("failed to read page and HWM size: %w", err) } for pageID := uint64(2); pageID < hwm; { p, _, err := readPage(dbPath, pgSize, pageID) if err != nil { fmt.Fprintf(os.Stderr, "Reading page %d failed: %v. Continuing...\n", pageID, err) pageID++ continue } if !p.isLeafPage() { pageID++ continue } for i := uint16(0); i < p.count; i++ { e := p.leafPageElement(i) rev, err := bytesToBucketKey(e.key()) if err != nil { if exceptionCheck(e.key()) { break } fmt.Fprintf(os.Stderr, "Decoding revision failed, pageID: %d, index: %d, key: %x, error: %v\n", pageID, i, string(e.key()), err) continue } if startRev != 0 && rev.Main < startRev { continue } fmt.Printf("pageID=%d, index=%d/%d, ", pageID, i, p.count-1) keyDecoder(e.key(), e.value()) } pageID += uint64(p.overflow) + 1 } return nil } func bytesToBucketKey(key []byte) (rev mvcc.BucketKey, err error) { defer func() { if r := recover(); r != nil { err = fmt.Errorf("BytesToBucketKey failed: %v", r) } }() rev = mvcc.BytesToBucketKey(key) return rev, err } func readPageAndHWMSize(dbPath string) (uint64, uint64, error) { f, err := os.Open(dbPath) if err != nil { return 0, 0, err } defer f.Close() // read 4KB chunk buf := make([]byte, 4096) if _, err := io.ReadFull(f, buf); err != nil { return 0, 0, err } m := loadPageMeta(buf) if m.magic != magic { return 0, 0, fmt.Errorf("the Meta Page has wrong (unexpected) magic") } return uint64(m.pageSize), m.pgid, nil } func readPage(dbPath string, pageSize uint64, pageID uint64) (*page, []byte, error) { f, err := os.Open(dbPath) if err != nil { return nil, nil, err } defer f.Close() buf := make([]byte, pageSize) if _, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { return nil, nil, err } p := loadPage(buf) if p.id != pageID { return nil, nil, fmt.Errorf("unexpected page id: %d, wanted: %d", p.id, pageID) } if p.overflow == 0 { return p, buf, nil } buf = make([]byte, (uint64(p.overflow)+1)*pageSize) if _, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { return nil, nil, err } p = loadPage(buf) if p.id != pageID { return nil, nil, fmt.Errorf("unexpected page id: %d, wanted: %d", p.id, pageID) } return p, buf, nil } func exceptionCheck(key []byte) bool { whiteKeyList := map[string]struct{}{ "alarm": {}, "auth": {}, "authRoles": {}, "authUsers": {}, "cluster": {}, "key": {}, "lease": {}, "members": {}, "members_removed": {}, "meta": {}, } _, ok := whiteKeyList[string(key)] return ok } ================================================ FILE: tools/etcd-dump-db/utils.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "unsafe" ) func unsafeAdd(base unsafe.Pointer, offset uintptr) unsafe.Pointer { return unsafe.Pointer(uintptr(base) + offset) } func unsafeIndex(base unsafe.Pointer, offset uintptr, elemsz uintptr, n int) unsafe.Pointer { return unsafe.Pointer(uintptr(base) + offset + uintptr(n)*elemsz) } func unsafeByteSlice(base unsafe.Pointer, offset uintptr, i, j int) []byte { // See: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices // // This memory is not allocated from C, but it is unmanaged by Go's // garbage collector and should behave similarly, and the compiler // should produce similar code. Note that this conversion allows a // subslice to begin after the base address, with an optional offset, // while the URL above does not cover this case and only slices from // index 0. However, the wiki never says that the address must be to // the beginning of a C allocation (or even that malloc was used at // all), so this is believed to be correct. return (*[pageMaxAllocSize]byte)(unsafeAdd(base, offset))[i:j:j] } ================================================ FILE: tools/etcd-dump-logs/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/debugging ================================================ FILE: tools/etcd-dump-logs/README.md ================================================ # etcd-dump-logs `etcd-dump-logs` dumps the log from data directory. ## Installation Install the tool by running the following command from the etcd source directory. ``` $ go install -v ./tools/etcd-dump-logs ``` The installation will place executables in the $GOPATH/bin. If $GOPATH environment variable is not set, the tool will be installed into the $HOME/go/bin. You can also find out the installed location by running the following command from the etcd source directory. Make sure that $PATH is set accordingly in your environment. ``` $ go list -f "{{.Target}}" ./tools/etcd-dump-logs ``` Alternatively, instead of installing the tool, you can use it by simply running the following command from the etcd source directory. ``` $ go run ./tools/etcd-dump-logs ``` ## Usage The following command should output the usage per the latest development. ``` $ etcd-dump-logs --help ``` An example of usage detail is provided below. ``` Usage: etcd-dump-logs [data dir] * Data dir is where the snapshots and WAL logs are located. The structure of the data dir should look like this: - data_dir/member - data_dir/member/snap - data_dir/member/wal - data_dir/member/wal/0000000000000000-0000000000000000.wal Flags: -wal-dir string If set, dumps WAL from the informed path, rather than following the standard 'data_dir/member/wal/' location -entry-type string If set, filters output by entry type. Must be one or more than one of: ConfigChange, Normal, Request, InternalRaftRequest, IRRRange, IRRPut, IRRDeleteRange, IRRTxn, IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke -start-index uint The index to start dumping (inclusive) If unspecified, dumps from the index of the last snapshot. -end-index uint The index to stop dumping (exclusive) -start-snap string The base name of snapshot file to start dumping -stream-decoder string The name and arguments of an executable decoding tool, the executable must process hex encoded lines of binary input (from etcd-dump-logs) and output a hex encoded line of binary for each input line ``` #### etcd-dump-logs -entry-type [data dir] Filter entries by type from WAL log. ``` $ etcd-dump-logs -entry-type IRRTxn /tmp/datadir Snapshot: empty Start dupmping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: lastIndex=34 term index type data 7 13 norm ID:8 txn: > failure: > > Entry types (IRRTxn) count is : 1 $ etcd-dump-logs -entry-type ConfigChange,IRRCompaction /tmp/datadir Snapshot: empty Start dupmping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: lastIndex=34 term index type data 1 1 conf method=ConfChangeAddNode id=2 2 2 conf method=ConfChangeRemoveNode id=2 2 3 conf method=ConfChangeUpdateNode id=2 2 4 conf method=ConfChangeAddLearnerNode id=3 8 14 norm ID:9 compaction: Entry types (ConfigChange,IRRCompaction) count is : 5 ``` #### etcd-dump-logs -stream-decoder [data dir] Decode each entry based on logic in the passed decoder. Decoder status and decoded data are listed in separated tab/columns in the output. For parsing purpose, the output from decoder are expected to be in format of "|". Please refer to [decoder_correctoutputformat.sh] as an example. However, if the decoder output format is not as expected, "decoder_status" will be "decoder output format is not right, print output anyway", and all output from decoder will be considered as "decoded_data" ``` $ etcd-dump-logs -stream-decoder decoder_correctoutputformat.sh /tmp/datadir Snapshot: empty Start dupmping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: lastIndex=34 term index type data decoder_status decoded_data 1 1 conf method=ConfChangeAddNode id=2 ERROR jhjaajjjahjbbbjj 3 2 norm noop OK jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 3 3 norm method=QGET path="/path1" OK jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 7 4 norm ID:8 txn: > failure: > > OK jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb 8 5 norm ID:9 compaction: ERROR jhjicajbajja 9 6 norm ID:10 lease_grant: ERROR jhjadbjdjhjaajja 12 7 norm ID:13 auth_enable:<> ERROR jhjdcbcejj 27 8 norm ??? ERROR cf Entry types () count is : 8 ``` ``` $ etcd-dump-logs -stream-decoder decoder_wrongoutputformat.sh /tmp/datadir Snapshot: empty Start dupmping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: lastIndex=34 term index type data decoder_status decoded_data 1 1 conf method=ConfChangeAddNode id=2 decoder output format is not right, print output anyway jhjaajjjahjbbbjj 3 2 norm noop decoder output format is not right, print output anyway jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 3 3 norm method=QGET path="/path1" decoder output format is not right, print output anyway jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 7 4 norm ID:8 txn: > failure: > > decoder output format is not right, print output anyway jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb 8 5 norm ID:9 compaction: decoder output format is not right, print output anyway jhjicajbajja 9 6 norm ID:10 lease_grant: decoder output format is not right, print output anyway jhjadbjdjhjaajja 12 7 norm ID:13 auth_enable:<> decoder output format is not right, print output anyway jhjdcbcejj 27 8 norm ??? decoder output format is not right, print output anyway cf Entry types () count is : 8 ``` #### etcd-dump-logs -start-index [data dir] Only shows WAL log entries after the specified start-index number, inclusively. ``` $ etcd-dump-logs -start-index 31 /tmp/datadir Start dumping log entries from index 31. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: lastIndex=34 term index type data 25 31 norm ID:26 auth_role_get: 26 32 norm ID:27 auth_role_grant_permission: > 27 33 norm ID:28 auth_role_revoke_permission: 27 34 norm ??? Entry types () count is : 4 ``` #### etcd-dump-logs -start-index -end-index [data dir] Only shows WAL log entries from the specified start-index number (inclusively) to the specified end-index number (exclusively). ``` $ etcd-dump-logs -start-index 930 -end-index 932 /tmp/datadir Start dumping log entries from index 930. WAL metadata: nodeID=0 clusterID=0 term=5 commitIndex=2448 vote=0 WAL entries: 2 lastIndex=931 term index type data 3 930 norm header: put: 3 931 norm header: put: Entry types (Normal,ConfigChange) count is : 2 ``` [decoder_correctoutputformat.sh]: ./testdecoder/decoder_correctoutputformat.sh ================================================ FILE: tools/etcd-dump-logs/doc.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // etcd-dump-logs is a program for analyzing etcd server write ahead logs. package main ================================================ FILE: tools/etcd-dump-logs/etcd-dump-log_test.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "os" "os/exec" "path" "path/filepath" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/authpb" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/raft/v3/raftpb" ) func TestEtcdDumpLogEntryType(t *testing.T) { // directory where the command is binDir, err := os.Getwd() require.NoError(t, err) // TODO(ptabor): The test does not run by default from ./scripts/test.sh. dumpLogsBinary := path.Join(binDir + "/etcd-dump-logs") if !fileutil.Exist(dumpLogsBinary) { t.Skipf("%q does not exist", dumpLogsBinary) } decoderCorrectOutputFormat := filepath.Join(binDir, "/testdecoder/decoder_correctoutputformat.sh") decoderWrongOutputFormat := filepath.Join(binDir, "/testdecoder/decoder_wrongoutputformat.sh") p := t.TempDir() mustCreateWALLog(t, p) argtests := []struct { name string args []string fileExpected string }{ {"no entry-type", []string{p}, "expectedoutput/listAll.output"}, {"confchange entry-type", []string{"-entry-type", "ConfigChange", p}, "expectedoutput/listConfigChange.output"}, {"normal entry-type", []string{"-entry-type", "Normal", p}, "expectedoutput/listNormal.output"}, {"request entry-type", []string{"-entry-type", "Request", p}, "expectedoutput/listRequest.output"}, {"internalRaftRequest entry-type", []string{"-entry-type", "InternalRaftRequest", p}, "expectedoutput/listInternalRaftRequest.output"}, {"range entry-type", []string{"-entry-type", "IRRRange", p}, "expectedoutput/listIRRRange.output"}, {"put entry-type", []string{"-entry-type", "IRRPut", p}, "expectedoutput/listIRRPut.output"}, {"del entry-type", []string{"-entry-type", "IRRDeleteRange", p}, "expectedoutput/listIRRDeleteRange.output"}, {"txn entry-type", []string{"-entry-type", "IRRTxn", p}, "expectedoutput/listIRRTxn.output"}, {"compaction entry-type", []string{"-entry-type", "IRRCompaction", p}, "expectedoutput/listIRRCompaction.output"}, {"lease grant entry-type", []string{"-entry-type", "IRRLeaseGrant", p}, "expectedoutput/listIRRLeaseGrant.output"}, {"lease revoke entry-type", []string{"-entry-type", "IRRLeaseRevoke", p}, "expectedoutput/listIRRLeaseRevoke.output"}, {"confchange and txn entry-type", []string{"-entry-type", "ConfigChange,IRRCompaction", p}, "expectedoutput/listConfigChangeIRRCompaction.output"}, {"decoder_correctoutputformat", []string{"-stream-decoder", decoderCorrectOutputFormat, p}, "expectedoutput/decoder_correctoutputformat.output"}, {"decoder_wrongoutputformat", []string{"-stream-decoder", decoderWrongOutputFormat, p}, "expectedoutput/decoder_wrongoutputformat.output"}, } for _, argtest := range argtests { t.Run(argtest.name, func(t *testing.T) { cmd := exec.Command(dumpLogsBinary, argtest.args...) actual, err := cmd.CombinedOutput() require.NoError(t, err) expected, err := os.ReadFile(path.Join(binDir, argtest.fileExpected)) require.NoError(t, err) assert.Equal(t, string(expected), string(actual)) // The output files contains a lot of trailing whitespaces... difficult to diagnose without printing them explicitly. // TODO(ptabor): Get rid of the whitespaces both in code and the test-files. assert.Equal(t, strings.ReplaceAll(string(expected), " ", "_"), strings.ReplaceAll(string(actual), " ", "_")) }) } } func mustCreateWALLog(t *testing.T, path string) { memberdir := filepath.Join(path, "member") err := os.Mkdir(memberdir, 0o744) require.NoError(t, err) waldir := walDir(path) snapdir := snapDir(path) w, err := wal.Create(zaptest.NewLogger(t), waldir, nil) require.NoError(t, err) err = os.Mkdir(snapdir, 0o744) require.NoError(t, err) ents := make([]raftpb.Entry, 0) // append entries into wal log appendConfigChangeEnts(&ents) appendNormalIRREnts(&ents) appendUnknownNormalEnts(&ents) // force commit newly appended entries err = w.Save(raftpb.HardState{}, ents) require.NoError(t, err) w.Close() } func appendConfigChangeEnts(ents *[]raftpb.Entry) { configChangeData := []raftpb.ConfChange{ {ID: 1, Type: raftpb.ConfChangeAddNode, NodeID: 2, Context: []byte("")}, {ID: 2, Type: raftpb.ConfChangeRemoveNode, NodeID: 2, Context: []byte("")}, {ID: 3, Type: raftpb.ConfChangeUpdateNode, NodeID: 2, Context: []byte("")}, {ID: 4, Type: raftpb.ConfChangeAddLearnerNode, NodeID: 3, Context: []byte("")}, } configChangeEntries := []raftpb.Entry{ {Term: 1, Index: 1, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[0])}, {Term: 2, Index: 2, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[1])}, {Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[2])}, {Term: 2, Index: 4, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[3])}, } *ents = append(*ents, configChangeEntries...) } func appendNormalIRREnts(ents *[]raftpb.Entry) { irrrange := &etcdserverpb.RangeRequest{Key: []byte("1"), RangeEnd: []byte("hi"), Limit: 6, Revision: 1, SortOrder: 1, SortTarget: 0, Serializable: false, KeysOnly: false, CountOnly: false, MinModRevision: 0, MaxModRevision: 20000, MinCreateRevision: 0, MaxCreateRevision: 20000} irrput := &etcdserverpb.PutRequest{Key: []byte("foo1"), Value: []byte("bar1"), Lease: 1, PrevKv: false, IgnoreValue: false, IgnoreLease: true} irrdeleterange := &etcdserverpb.DeleteRangeRequest{Key: []byte("0"), RangeEnd: []byte("9"), PrevKv: true} delInRangeReq := &etcdserverpb.RequestOp{ Request: &etcdserverpb.RequestOp_RequestDeleteRange{ RequestDeleteRange: &etcdserverpb.DeleteRangeRequest{ Key: []byte("a"), RangeEnd: []byte("b"), }, }, } irrtxn := &etcdserverpb.TxnRequest{Success: []*etcdserverpb.RequestOp{delInRangeReq}, Failure: []*etcdserverpb.RequestOp{delInRangeReq}} irrcompaction := &etcdserverpb.CompactionRequest{Revision: 0, Physical: true} irrleasegrant := &etcdserverpb.LeaseGrantRequest{TTL: 1, ID: 1} irrleaserevoke := &etcdserverpb.LeaseRevokeRequest{ID: 2} irralarm := &etcdserverpb.AlarmRequest{Action: 3, MemberID: 4, Alarm: 5} irrauthenable := &etcdserverpb.AuthEnableRequest{} irrauthdisable := &etcdserverpb.AuthDisableRequest{} irrauthenticate := &etcdserverpb.InternalAuthenticateRequest{Name: "myname", Password: "password", SimpleToken: "token"} irrauthuseradd := &etcdserverpb.AuthUserAddRequest{Name: "name1", Password: "pass1", Options: &authpb.UserAddOptions{NoPassword: false}} irrauthuserdelete := &etcdserverpb.AuthUserDeleteRequest{Name: "name1"} irrauthuserget := &etcdserverpb.AuthUserGetRequest{Name: "name1"} irrauthuserchangepassword := &etcdserverpb.AuthUserChangePasswordRequest{Name: "name1", Password: "pass2"} irrauthusergrantrole := &etcdserverpb.AuthUserGrantRoleRequest{User: "user1", Role: "role1"} irrauthuserrevokerole := &etcdserverpb.AuthUserRevokeRoleRequest{Name: "user2", Role: "role2"} irrauthuserlist := &etcdserverpb.AuthUserListRequest{} irrauthrolelist := &etcdserverpb.AuthRoleListRequest{} irrauthroleadd := &etcdserverpb.AuthRoleAddRequest{Name: "role2"} irrauthroledelete := &etcdserverpb.AuthRoleDeleteRequest{Role: "role1"} irrauthroleget := &etcdserverpb.AuthRoleGetRequest{Role: "role3"} perm := &authpb.Permission{ PermType: authpb.Permission_WRITE, Key: []byte("Keys"), RangeEnd: []byte("RangeEnd"), } irrauthrolegrantpermission := &etcdserverpb.AuthRoleGrantPermissionRequest{Name: "role3", Perm: perm} irrauthrolerevokepermission := &etcdserverpb.AuthRoleRevokePermissionRequest{Role: "role3", Key: []byte("key"), RangeEnd: []byte("rangeend")} irrs := []etcdserverpb.InternalRaftRequest{ {ID: 5, Range: irrrange}, {ID: 6, Put: irrput}, {ID: 7, DeleteRange: irrdeleterange}, {ID: 8, Txn: irrtxn}, {ID: 9, Compaction: irrcompaction}, {ID: 10, LeaseGrant: irrleasegrant}, {ID: 11, LeaseRevoke: irrleaserevoke}, {ID: 12, Alarm: irralarm}, {ID: 13, AuthEnable: irrauthenable}, {ID: 14, AuthDisable: irrauthdisable}, {ID: 15, Authenticate: irrauthenticate}, {ID: 16, AuthUserAdd: irrauthuseradd}, {ID: 17, AuthUserDelete: irrauthuserdelete}, {ID: 18, AuthUserGet: irrauthuserget}, {ID: 19, AuthUserChangePassword: irrauthuserchangepassword}, {ID: 20, AuthUserGrantRole: irrauthusergrantrole}, {ID: 21, AuthUserRevokeRole: irrauthuserrevokerole}, {ID: 22, AuthUserList: irrauthuserlist}, {ID: 23, AuthRoleList: irrauthrolelist}, {ID: 24, AuthRoleAdd: irrauthroleadd}, {ID: 25, AuthRoleDelete: irrauthroledelete}, {ID: 26, AuthRoleGet: irrauthroleget}, {ID: 27, AuthRoleGrantPermission: irrauthrolegrantpermission}, {ID: 28, AuthRoleRevokePermission: irrauthrolerevokepermission}, } for i, irr := range irrs { var currentry raftpb.Entry currentry.Term = uint64(i + 4) currentry.Index = uint64(i + 10) currentry.Type = raftpb.EntryNormal currentry.Data = pbutil.MustMarshal(&irr) *ents = append(*ents, currentry) } } func appendUnknownNormalEnts(ents *[]raftpb.Entry) { var currentry raftpb.Entry currentry.Term = 27 currentry.Index = 34 currentry.Type = raftpb.EntryNormal currentry.Data = []byte("?") *ents = append(*ents, currentry) } ================================================ FILE: tools/etcd-dump-logs/expectedoutput/decoder_correctoutputformat.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data decoder_status decoded_data 1 1 conf method=ConfChangeAddNode id=2 ERROR jhjaajjjahjbbbjj 2 2 conf method=ConfChangeRemoveNode id=2 ERROR jhjbajjaahjbbbjj 2 3 conf method=ConfChangeUpdateNode id=2 ERROR jhjcajjbahjbbbjj 2 4 conf method=ConfChangeAddLearnerNode id=3 ERROR jhjdajjcahjcbbjj 3 5 norm noop OK jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 3 6 norm method=QGET path="/path1" OK jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 3 7 norm method=SYNC time="1970-01-01 00:00:00.000000001 +0000 UTC" OK jhjbabjdeceidedcaajfbfgjfagdfhcbbbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 3 8 norm method=DELETE path="/path3" OK jhjcabjfdddedcdeeddeaajfbfgjfagdfhccbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjadhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" OK jhjdabjfebdadedddfddaaafjabfgjfagdfhcdbfgcgegjfegbfcfffefgbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 4 10 norm ID:5 range: OK jhjeaaaejajacaabjbfhfiahjfbjjabhjaehajicjafhajicja 5 11 norm ID:6 put: OK jhjfbbajjajdffffffcaabjdfbfagbcaahjacjja 6 12 norm ID:7 delete_range: OK jhjgbajhjajacjabjaciahja 7 13 norm ID:8 txn: > failure: > > OK jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb 8 14 norm ID:9 compaction: ERROR jhjicajbajja 9 15 norm ID:10 lease_grant: ERROR jhjadbjdjhjaajja 10 16 norm ID:11 lease_revoke: ERROR jhjbdajbjhjb 11 17 norm ID:12 alarm: OK jhjcebjfjhjcajjdahje 12 18 norm ID:13 auth_enable:<> ERROR jhjdcbcejj 13 19 norm ID:14 auth_disable:<> ERROR jhjeiacfjj 14 20 norm ID:15 authenticate: OK jhjfabcfaijajffdgifefafdfeabjhgjfagcgcggffgbfdaajegdfffbfefe 15 21 norm ID:16 auth_user_add: > OK jhajebddajjajefefafdfecaabjegjfagcgccaaajj 16 22 norm ID:17 auth_user_delete: OK jhaaeaddjgjajefefafdfeca 17 23 norm ID:18 auth_user_get: OK jhabfbddjgjajefefafdfeca 18 24 norm ID:19 auth_user_change_password:" > OK jhacfaddjejajefefafdfecaabjegjfagcgccb 19 25 norm ID:20 auth_user_grant_role: OK jhadhbdejejajegegcfegbcaabjegbfffcfeca 20 26 norm ID:21 auth_user_revoke_role: OK jhaehadejejajegegcfegbcbabjegbfffcfecb 21 27 norm ID:22 auth_user_list:<> ERROR jhafibdejj 22 28 norm ID:23 auth_role_list:<> ERROR jhagiadejj 23 29 norm ID:24 auth_role_add: OK jhahhbdbjgjajegbfffcfecb 24 30 norm ID:25 auth_role_delete: OK jhaihadbjgjajegbfffcfeca 25 31 norm ID:26 auth_role_get: OK jhaaibdbjgjajegbfffcfecc 26 32 norm ID:27 auth_role_grant_permission: > OK jhabiadbabjajegbfffcfeccababjhjaabjddbfegigcaajhebfafefgfedefefd 27 33 norm ID:28 auth_role_revoke_permission: OK jhacabdbafjajegbfffcfeccabjcfbfegiaajhgbfafefgfefefefd 27 34 norm ??? ERROR cf Entry types (Normal,ConfigChange) count is : 34 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/decoder_wrongoutputformat.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data decoder_status decoded_data 1 1 conf method=ConfChangeAddNode id=2 decoder output format is not right, print output anyway jhjaajjjahjbbbjj 2 2 conf method=ConfChangeRemoveNode id=2 decoder output format is not right, print output anyway jhjbajjaahjbbbjj 2 3 conf method=ConfChangeUpdateNode id=2 decoder output format is not right, print output anyway jhjcajjbahjbbbjj 2 4 conf method=ConfChangeAddLearnerNode id=3 decoder output format is not right, print output anyway jhjdajjcahjcbbjj 3 5 norm noop decoder output format is not right, print output anyway jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 3 6 norm method=QGET path="/path1" decoder output format is not right, print output anyway jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 3 7 norm method=SYNC time="1970-01-01 00:00:00.000000001 +0000 UTC" decoder output format is not right, print output anyway jhjbabjdeceidedcaajfbfgjfagdfhcbbbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 3 8 norm method=DELETE path="/path3" decoder output format is not right, print output anyway jhjcabjfdddedcdeeddeaajfbfgjfagdfhccbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjadhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" decoder output format is not right, print output anyway jhjdabjfebdadedddfddaaafjabfgjfagdfhcdbfgcgegjfegbfcfffefgbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj 4 10 norm ID:5 range: decoder output format is not right, print output anyway jhjeaaaejajacaabjbfhfiahjfbjjabhjaehajicjafhajicja 5 11 norm ID:6 put: decoder output format is not right, print output anyway jhjfbbajjajdffffffcaabjdfbfagbcaahjacjja 6 12 norm ID:7 delete_range: decoder output format is not right, print output anyway jhjgbajhjajacjabjaciahja 7 13 norm ID:8 txn: > failure: > > decoder output format is not right, print output anyway jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb 8 14 norm ID:9 compaction: decoder output format is not right, print output anyway jhjicajbajja 9 15 norm ID:10 lease_grant: decoder output format is not right, print output anyway jhjadbjdjhjaajja 10 16 norm ID:11 lease_revoke: decoder output format is not right, print output anyway jhjbdajbjhjb 11 17 norm ID:12 alarm: decoder output format is not right, print output anyway jhjcebjfjhjcajjdahje 12 18 norm ID:13 auth_enable:<> decoder output format is not right, print output anyway jhjdcbcejj 13 19 norm ID:14 auth_disable:<> decoder output format is not right, print output anyway jhjeiacfjj 14 20 norm ID:15 authenticate: decoder output format is not right, print output anyway jhjfabcfaijajffdgifefafdfeabjhgjfagcgcggffgbfdaajegdfffbfefe 15 21 norm ID:16 auth_user_add: > decoder output format is not right, print output anyway jhajebddajjajefefafdfecaabjegjfagcgccaaajj 16 22 norm ID:17 auth_user_delete: decoder output format is not right, print output anyway jhaaeaddjgjajefefafdfeca 17 23 norm ID:18 auth_user_get: decoder output format is not right, print output anyway jhabfbddjgjajefefafdfeca 18 24 norm ID:19 auth_user_change_password:" > decoder output format is not right, print output anyway jhacfaddjejajefefafdfecaabjegjfagcgccb 19 25 norm ID:20 auth_user_grant_role: decoder output format is not right, print output anyway jhadhbdejejajegegcfegbcaabjegbfffcfeca 20 26 norm ID:21 auth_user_revoke_role: decoder output format is not right, print output anyway jhaehadejejajegegcfegbcbabjegbfffcfecb 21 27 norm ID:22 auth_user_list:<> decoder output format is not right, print output anyway jhafibdejj 22 28 norm ID:23 auth_role_list:<> decoder output format is not right, print output anyway jhagiadejj 23 29 norm ID:24 auth_role_add: decoder output format is not right, print output anyway jhahhbdbjgjajegbfffcfecb 24 30 norm ID:25 auth_role_delete: decoder output format is not right, print output anyway jhaihadbjgjajegbfffcfeca 25 31 norm ID:26 auth_role_get: decoder output format is not right, print output anyway jhaaibdbjgjajegbfffcfecc 26 32 norm ID:27 auth_role_grant_permission: > decoder output format is not right, print output anyway jhabiadbabjajegbfffcfeccababjhjaabjddbfegigcaajhebfafefgfedefefd 27 33 norm ID:28 auth_role_revoke_permission: decoder output format is not right, print output anyway jhacabdbafjajegbfffcfeccabjcfbfegiaajhgbfafefgfefefefd 27 34 norm ??? decoder output format is not right, print output anyway cf Entry types (Normal,ConfigChange) count is : 34 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listAll.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 1 1 conf method=ConfChangeAddNode id=2 2 2 conf method=ConfChangeRemoveNode id=2 2 3 conf method=ConfChangeUpdateNode id=2 2 4 conf method=ConfChangeAddLearnerNode id=3 3 5 norm noop 3 6 norm method=QGET path="/path1" 3 7 norm method=SYNC time="1970-01-01 00:00:00.000000001 +0000 UTC" 3 8 norm method=DELETE path="/path3" 3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" 4 10 norm ID:5 range: 5 11 norm ID:6 put: 6 12 norm ID:7 delete_range: 7 13 norm ID:8 txn: > failure: > > 8 14 norm ID:9 compaction: 9 15 norm ID:10 lease_grant: 10 16 norm ID:11 lease_revoke: 11 17 norm ID:12 alarm: 12 18 norm ID:13 auth_enable:<> 13 19 norm ID:14 auth_disable:<> 14 20 norm ID:15 authenticate: 15 21 norm ID:16 auth_user_add: > 16 22 norm ID:17 auth_user_delete: 17 23 norm ID:18 auth_user_get: 18 24 norm ID:19 auth_user_change_password:" > 19 25 norm ID:20 auth_user_grant_role: 20 26 norm ID:21 auth_user_revoke_role: 21 27 norm ID:22 auth_user_list:<> 22 28 norm ID:23 auth_role_list:<> 23 29 norm ID:24 auth_role_add: 24 30 norm ID:25 auth_role_delete: 25 31 norm ID:26 auth_role_get: 26 32 norm ID:27 auth_role_grant_permission: > 27 33 norm ID:28 auth_role_revoke_permission: 27 34 norm ??? Entry types (Normal,ConfigChange) count is : 34 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listConfigChange.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 1 1 conf method=ConfChangeAddNode id=2 2 2 conf method=ConfChangeRemoveNode id=2 2 3 conf method=ConfChangeUpdateNode id=2 2 4 conf method=ConfChangeAddLearnerNode id=3 Entry types (ConfigChange) count is : 4 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRCompaction.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 1 1 conf method=ConfChangeAddNode id=2 2 2 conf method=ConfChangeRemoveNode id=2 2 3 conf method=ConfChangeUpdateNode id=2 2 4 conf method=ConfChangeAddLearnerNode id=3 8 14 norm ID:9 compaction: Entry types (ConfigChange,IRRCompaction) count is : 5 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listIRRCompaction.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 8 14 norm ID:9 compaction: Entry types (IRRCompaction) count is : 1 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listIRRDeleteRange.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 6 12 norm ID:7 delete_range: Entry types (IRRDeleteRange) count is : 1 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listIRRLeaseGrant.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 9 15 norm ID:10 lease_grant: Entry types (IRRLeaseGrant) count is : 1 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listIRRLeaseRevoke.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 10 16 norm ID:11 lease_revoke: Entry types (IRRLeaseRevoke) count is : 1 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listIRRPut.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 5 11 norm ID:6 put: Entry types (IRRPut) count is : 1 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listIRRRange.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 4 10 norm ID:5 range: Entry types (IRRRange) count is : 1 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listIRRTxn.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 7 13 norm ID:8 txn: > failure: > > Entry types (IRRTxn) count is : 1 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listInternalRaftRequest.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 4 10 norm ID:5 range: 5 11 norm ID:6 put: 6 12 norm ID:7 delete_range: 7 13 norm ID:8 txn: > failure: > > 8 14 norm ID:9 compaction: 9 15 norm ID:10 lease_grant: 10 16 norm ID:11 lease_revoke: 11 17 norm ID:12 alarm: 12 18 norm ID:13 auth_enable:<> 13 19 norm ID:14 auth_disable:<> 14 20 norm ID:15 authenticate: 15 21 norm ID:16 auth_user_add: > 16 22 norm ID:17 auth_user_delete: 17 23 norm ID:18 auth_user_get: 18 24 norm ID:19 auth_user_change_password:" > 19 25 norm ID:20 auth_user_grant_role: 20 26 norm ID:21 auth_user_revoke_role: 21 27 norm ID:22 auth_user_list:<> 22 28 norm ID:23 auth_role_list:<> 23 29 norm ID:24 auth_role_add: 24 30 norm ID:25 auth_role_delete: 25 31 norm ID:26 auth_role_get: 26 32 norm ID:27 auth_role_grant_permission: > 27 33 norm ID:28 auth_role_revoke_permission: Entry types (InternalRaftRequest) count is : 24 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listNormal.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 3 5 norm noop 3 6 norm method=QGET path="/path1" 3 7 norm method=SYNC time="1970-01-01 00:00:00.000000001 +0000 UTC" 3 8 norm method=DELETE path="/path3" 3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" 4 10 norm ID:5 range: 5 11 norm ID:6 put: 6 12 norm ID:7 delete_range: 7 13 norm ID:8 txn: > failure: > > 8 14 norm ID:9 compaction: 9 15 norm ID:10 lease_grant: 10 16 norm ID:11 lease_revoke: 11 17 norm ID:12 alarm: 12 18 norm ID:13 auth_enable:<> 13 19 norm ID:14 auth_disable:<> 14 20 norm ID:15 authenticate: 15 21 norm ID:16 auth_user_add: > 16 22 norm ID:17 auth_user_delete: 17 23 norm ID:18 auth_user_get: 18 24 norm ID:19 auth_user_change_password:" > 19 25 norm ID:20 auth_user_grant_role: 20 26 norm ID:21 auth_user_revoke_role: 21 27 norm ID:22 auth_user_list:<> 22 28 norm ID:23 auth_role_list:<> 23 29 norm ID:24 auth_role_add: 24 30 norm ID:25 auth_role_delete: 25 31 norm ID:26 auth_role_get: 26 32 norm ID:27 auth_role_grant_permission: > 27 33 norm ID:28 auth_role_revoke_permission: 27 34 norm ??? Entry types (Normal) count is : 30 ================================================ FILE: tools/etcd-dump-logs/expectedoutput/listRequest.output ================================================ Snapshot: empty Start dumping log entries from snapshot. WAL metadata: nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 WAL entries: 34 lastIndex=34 term index type data 3 5 norm noop 3 6 norm method=QGET path="/path1" 3 7 norm method=SYNC time="1970-01-01 00:00:00.000000001 +0000 UTC" 3 8 norm method=DELETE path="/path3" 3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" Entry types (Request) count is : 5 ================================================ FILE: tools/etcd-dump-logs/main.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "bufio" "encoding/hex" "encoding/json" "errors" "flag" "fmt" "io" "log" "math" "os" "os/exec" "path/filepath" "strings" "go.uber.org/zap" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/etcdserver/api/snap" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) const ( defaultEntryTypes string = "Normal,ConfigChange" methodSync string = "SYNC" methodQGet string = "QGET" methodDelete string = "DELETE" methodRandom string = "RANDOM" ) func main() { snapfile := flag.String("start-snap", "", "The base name of snapshot file to start dumping") waldir := flag.String("wal-dir", "", "If set, dumps WAL from the informed path, rather than following the standard 'data_dir/member/wal/' location") startIndex := flag.Uint64("start-index", 0, "The index to start dumping (inclusive). If unspecified, dumps from the index of the last snapshot.") endIndex := flag.Uint64("end-index", math.MaxUint64, "The index to stop dumping (exclusive)") // Default entry types are Normal and ConfigChange entrytype := flag.String("entry-type", defaultEntryTypes, `If set, filters output by entry type. Must be one or more than one of: ConfigChange, Normal, Request, InternalRaftRequest, IRRRange, IRRPut, IRRDeleteRange, IRRTxn, IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke, IRRLeaseCheckpoint`) streamdecoder := flag.String("stream-decoder", "", `The name of an executable decoding tool, the executable must process hex encoded lines of binary input (from etcd-dump-logs) and output a hex encoded line of binary for each input line`) raw := flag.Bool("raw", false, "Read the logs in the low-level form") flag.Parse() lg := zap.NewExample() if len(flag.Args()) != 1 { log.Fatalf("Must provide data-dir argument (got %+v)", flag.Args()) } dataDir := flag.Args()[0] if *snapfile != "" && *startIndex != 0 { log.Fatal("start-snap and start-index flags cannot be used together.") } startFromIndex := false flag.Visit(func(f *flag.Flag) { if f.Name == "start-index" { startFromIndex = true } }) if !*raw { ents := readUsingReadAll(lg, startFromIndex, startIndex, endIndex, snapfile, dataDir, waldir) fmt.Printf("WAL entries: %d\n", len(ents)) if len(ents) > 0 { fmt.Printf("lastIndex=%d\n", ents[len(ents)-1].Index) } fmt.Printf("%4s\t%10s\ttype\tdata", "term", "index") if *streamdecoder != "" { fmt.Print("\tdecoder_status\tdecoded_data") } fmt.Println() listEntriesType(*entrytype, *streamdecoder, ents) } else { if *snapfile != "" || *entrytype != defaultEntryTypes || *streamdecoder != "" { log.Fatalf("Flags --entry-type, --stream-decoder, --entrytype not supported in the RAW mode.") } wd := *waldir if wd == "" { wd = walDir(dataDir) } readRaw(startIndex, wd, os.Stdout) } } func readUsingReadAll(lg *zap.Logger, startFromIndex bool, startIndex *uint64, endIndex *uint64, snapfile *string, dataDir string, waldir *string) []raftpb.Entry { var ( walsnap walpb.Snapshot snapshot *raftpb.Snapshot err error ) endAtIndex := *endIndex < math.MaxUint64 if startFromIndex { fmt.Printf("Start dumping log entries from index %d.\n", *startIndex) // ReadAll() reads entries from the index after walsnap.Index, so we need to move walsnap.Index back one. if *startIndex > 0 { *startIndex-- } index := *startIndex walsnap.Index = &index } else { if *snapfile == "" { ss := snap.New(lg, snapDir(dataDir)) snapshot, err = ss.Load() } else { snapshot, err = snap.Read(lg, filepath.Join(snapDir(dataDir), *snapfile)) } switch { case err == nil: walsnap.Index, walsnap.Term = new(snapshot.Metadata.Index), new(snapshot.Metadata.Term) nodes := genIDSlice(snapshot.Metadata.ConfState.Voters) confStateJSON, merr := json.Marshal(snapshot.Metadata.ConfState) if merr != nil { confStateJSON = []byte(fmt.Sprintf("confstate err: %v", merr)) } fmt.Printf("Snapshot:\nterm=%d index=%d nodes=%s confstate=%s\n", walsnap.Term, walsnap.Index, nodes, confStateJSON) case errors.Is(err, snap.ErrNoSnapshot): fmt.Print("Snapshot:\nempty\n") default: log.Fatalf("Failed loading snapshot: %v", err) } fmt.Println("Start dumping log entries from snapshot.") } wd := *waldir if wd == "" { wd = walDir(dataDir) } w, err := wal.OpenForRead(zap.NewExample(), wd, walsnap) if err != nil { log.Fatalf("Failed opening WAL: %v", err) } wmetadata, state, ents, err := w.ReadAll() w.Close() if err != nil && (!startFromIndex || !errors.Is(err, wal.ErrSnapshotNotFound)) { // ReadAll might return ErrSliceOutOfRange and the first series of entries if the server is offline for a while and receives a snapshot from leader. // It is ok to ignore ErrSliceOutOfRange if just requesting a specific range of entries if !endAtIndex || !errors.Is(err, wal.ErrSliceOutOfRange) { log.Fatalf("Failed reading WAL: %v", err) } log.Printf("Failed reading all WAL: %v", err) } id, cid := parseWALMetadata(wmetadata) vid := types.ID(state.Vote) fmt.Printf("WAL metadata:\nnodeID=%s clusterID=%s term=%d commitIndex=%d vote=%s\n", id, cid, state.Term, state.Commit, vid) if endAtIndex { entries := make([]raftpb.Entry, 0) for _, e := range ents { // WAL might contain entries with e.Index >= *endIndex from prev term, then e.Index < *endIndex in the next term. // We cannot break when e.Index >= *endIndex. if e.Index >= *endIndex { continue } entries = append(entries, e) } return entries } return ents } func walDir(dataDir string) string { return filepath.Join(dataDir, "member", "wal") } func snapDir(dataDir string) string { return filepath.Join(dataDir, "member", "snap") } func parseWALMetadata(b []byte) (id, cid types.ID) { var metadata etcdserverpb.Metadata pbutil.MustUnmarshal(&metadata, b) id = types.ID(metadata.GetNodeID()) cid = types.ID(metadata.GetClusterID()) return id, cid } func genIDSlice(a []uint64) []types.ID { ids := make([]types.ID, len(a)) for i, id := range a { ids[i] = types.ID(id) } return ids } type EntryFilter func(e raftpb.Entry) (bool, string) // The 9 pass functions below takes the raftpb.Entry and return if the entry should be printed and the type of entry, // the type of the entry will used in the following print function func passConfChange(entry raftpb.Entry) (bool, string) { return entry.Type == raftpb.EntryConfChange, "ConfigChange" } func passInternalRaftRequest(entry raftpb.Entry) (bool, string) { var rr etcdserverpb.InternalRaftRequest return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil, "InternalRaftRequest" } func passUnknownNormal(entry raftpb.Entry) (bool, string) { var rr2 etcdserverpb.InternalRaftRequest return (entry.Type == raftpb.EntryNormal) && (rr2.Unmarshal(entry.Data) != nil), "UnknownNormal" } func passIRRRange(entry raftpb.Entry) (bool, string) { var rr etcdserverpb.InternalRaftRequest return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.Range != nil, "InternalRaftRequest" } func passIRRPut(entry raftpb.Entry) (bool, string) { var rr etcdserverpb.InternalRaftRequest return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.Put != nil, "InternalRaftRequest" } func passIRRDeleteRange(entry raftpb.Entry) (bool, string) { var rr etcdserverpb.InternalRaftRequest return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.DeleteRange != nil, "InternalRaftRequest" } func passIRRTxn(entry raftpb.Entry) (bool, string) { var rr etcdserverpb.InternalRaftRequest return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.Txn != nil, "InternalRaftRequest" } func passIRRCompaction(entry raftpb.Entry) (bool, string) { var rr etcdserverpb.InternalRaftRequest return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.Compaction != nil, "InternalRaftRequest" } func passIRRLeaseGrant(entry raftpb.Entry) (bool, string) { var rr etcdserverpb.InternalRaftRequest return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.LeaseGrant != nil, "InternalRaftRequest" } func passIRRLeaseRevoke(entry raftpb.Entry) (bool, string) { var rr etcdserverpb.InternalRaftRequest return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.LeaseRevoke != nil, "InternalRaftRequest" } func passIRRLeaseCheckpoint(entry raftpb.Entry) (bool, string) { var rr etcdserverpb.InternalRaftRequest return entry.Type == raftpb.EntryNormal && rr.Unmarshal(entry.Data) == nil && rr.LeaseCheckpoint != nil, "InternalRaftRequest" } func passRequest(entry raftpb.Entry) (bool, string) { var rr2 etcdserverpb.InternalRaftRequest return entry.Type == raftpb.EntryNormal && rr2.Unmarshal(entry.Data) != nil, "Request" } type EntryPrinter func(e raftpb.Entry) // The 4 print functions below print the entry format based on there types // printInternalRaftRequest is used to print entry information for IRRRange, IRRPut, // IRRDeleteRange and IRRTxn entries func printInternalRaftRequest(entry raftpb.Entry) { var rr etcdserverpb.InternalRaftRequest if err := rr.Unmarshal(entry.Data); err == nil { // Ensure we don't log user password if rr.AuthUserChangePassword != nil && rr.AuthUserChangePassword.Password != "" { rr.AuthUserChangePassword.Password = "" } fmt.Printf("%4d\t%10d\tnorm\t%s", entry.Term, entry.Index, rr.String()) } } func printUnknownNormal(entry raftpb.Entry) { fmt.Printf("%4d\t%10d\tnorm\t???", entry.Term, entry.Index) } func printConfChange(entry raftpb.Entry) { fmt.Printf("%4d\t%10d", entry.Term, entry.Index) fmt.Print("\tconf") var r raftpb.ConfChange if err := r.Unmarshal(entry.Data); err != nil { fmt.Print("\t???") } else { fmt.Printf("\tmethod=%s id=%s", r.Type, types.ID(r.NodeID)) } } // evaluateEntrytypeFlag evaluates entry-type flag and choose proper filter/filters to filter entries func evaluateEntrytypeFlag(entrytype string) []EntryFilter { var entrytypelist []string if entrytype != "" { entrytypelist = strings.Split(entrytype, ",") } validRequest := map[string][]EntryFilter{ "ConfigChange": {passConfChange}, "Normal": {passInternalRaftRequest, passRequest, passUnknownNormal}, "Request": {passRequest}, "InternalRaftRequest": {passInternalRaftRequest}, "IRRRange": {passIRRRange}, "IRRPut": {passIRRPut}, "IRRDeleteRange": {passIRRDeleteRange}, "IRRTxn": {passIRRTxn}, "IRRCompaction": {passIRRCompaction}, "IRRLeaseGrant": {passIRRLeaseGrant}, "IRRLeaseRevoke": {passIRRLeaseRevoke}, "IRRLeaseCheckpoint": {passIRRLeaseCheckpoint}, } filters := make([]EntryFilter, 0) for _, et := range entrytypelist { if f, ok := validRequest[et]; ok { filters = append(filters, f...) } else { log.Printf(`[%+v] is not a valid entry-type, ignored. Please set entry-type to one or more of the following: ConfigChange, Normal, Request, InternalRaftRequest, IRRRange, IRRPut, IRRDeleteRange, IRRTxn, IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke, IRRLeaseCheckpoint`, et) } } return filters } // listEntriesType filters and prints entries based on the entry-type flag, func listEntriesType(entrytype string, streamdecoder string, ents []raftpb.Entry) { entryFilters := evaluateEntrytypeFlag(entrytype) printerMap := map[string]EntryPrinter{ "InternalRaftRequest": printInternalRaftRequest, "ConfigChange": printConfChange, "UnknownNormal": printUnknownNormal, } var stderr strings.Builder args := strings.Split(streamdecoder, " ") cmd := exec.Command(args[0], args[1:]...) stdin, err := cmd.StdinPipe() if err != nil { log.Panic(err) } stdout, err := cmd.StdoutPipe() if err != nil { log.Panic(err) } cmd.Stderr = &stderr if streamdecoder != "" { err = cmd.Start() if err != nil { log.Panic(err) } } cnt := 0 for _, e := range ents { passed := false currtype := "" for _, filter := range entryFilters { passed, currtype = filter(e) if passed { cnt++ break } } if passed { printer := printerMap[currtype] printer(e) if streamdecoder == "" { fmt.Println() continue } // if decoder is set, pass the e.Data to stdin and read the stdout from decoder io.WriteString(stdin, hex.EncodeToString(e.Data)) io.WriteString(stdin, "\n") outputReader := bufio.NewReader(stdout) decoderoutput, currerr := outputReader.ReadString('\n') if currerr != nil { fmt.Println(currerr) return } decoderStatus, decodedData := parseDecoderOutput(decoderoutput) fmt.Printf("\t%s\t%s", decoderStatus, decodedData) } } stdin.Close() err = cmd.Wait() if streamdecoder != "" { if err != nil { log.Panic(err) } if stderr.String() != "" { os.Stderr.WriteString("decoder stderr: " + stderr.String()) } } fmt.Printf("\nEntry types (%s) count is : %d\n", entrytype, cnt) } func parseDecoderOutput(decoderoutput string) (string, string) { var decoderStatus string var decodedData string output := strings.Split(decoderoutput, "|") switch len(output) { case 1: decoderStatus = "decoder output format is not right, print output anyway" decodedData = decoderoutput case 2: decoderStatus = output[0] decodedData = output[1] default: decoderStatus = output[0] + "(*WARNING: data might contain deliminator used by etcd-dump-logs)" decodedData = strings.Join(output[1:], "") } return decoderStatus, decodedData } ================================================ FILE: tools/etcd-dump-logs/raw.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "errors" "fmt" "io" "log" "os" "path/filepath" "go.etcd.io/etcd/api/v3/etcdserverpb" "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/pkg/v3/pbutil" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" "go.etcd.io/raft/v3/raftpb" ) func readRaw(fromIndex *uint64, waldir string, out io.Writer) { var walReaders []fileutil.FileReader dirEntry, err := os.ReadDir(waldir) if err != nil { log.Fatalf("Error: Failed to read directory '%s' error:%v", waldir, err) } for _, e := range dirEntry { finfo, err := e.Info() if err != nil { log.Fatalf("Error: failed to get fileInfo of file: %s, error: %v", e.Name(), err) } if filepath.Ext(finfo.Name()) != ".wal" { log.Printf("Warning: Ignoring not .wal file: %s", finfo.Name()) continue } f, err := os.Open(filepath.Join(waldir, finfo.Name())) if err != nil { log.Printf("Error: Failed to read file: %s . error:%v", finfo.Name(), err) } walReaders = append(walReaders, fileutil.NewFileReader(f)) } decoder := wal.NewDecoderAdvanced(true, walReaders...) // The variable is used to not pollute log with multiple continuous crc errors. crcDesync := false for { rec := walpb.Record{} err := decoder.Decode(&rec) if err == nil || errors.Is(err, walpb.ErrCRCMismatch) { if err != nil && !crcDesync { log.Printf("Error: Reading entry failed with CRC error: %c", err) crcDesync = true } printRec(&rec, fromIndex, out) if rec.GetType() == wal.CrcType { decoder.UpdateCRC(rec.GetCrc()) crcDesync = false } continue } if errors.Is(err, io.EOF) { fmt.Fprintf(out, "EOF: All entries were processed.\n") break } else if errors.Is(err, io.ErrUnexpectedEOF) { fmt.Fprintf(out, "ErrUnexpectedEOF: The last record might be corrupted, error: %v.\n", err) break } log.Printf("Error: Reading failed: %v", err) break } } func printRec(rec *walpb.Record, fromIndex *uint64, out io.Writer) { switch rec.GetType() { case wal.MetadataType: var metadata etcdserverpb.Metadata pbutil.MustUnmarshal(&metadata, rec.Data) fmt.Fprintf(out, "Metadata: %s\n", metadata.String()) case wal.CrcType: fmt.Fprintf(out, "CRC: %d\n", rec.GetCrc()) case wal.EntryType: e := wal.MustUnmarshalEntry(rec.Data) if fromIndex == nil || e.Index >= *fromIndex { fmt.Fprintf(out, "Entry: %s\n", e.String()) } case wal.SnapshotType: var snap walpb.Snapshot pbutil.MustUnmarshal(&snap, rec.Data) if fromIndex == nil || snap.GetIndex() >= *fromIndex { fmt.Fprintf(out, "Snapshot: %s\n", snap.String()) } case wal.StateType: var state raftpb.HardState pbutil.MustUnmarshal(&state, rec.Data) if fromIndex == nil || state.Commit >= *fromIndex { fmt.Fprintf(out, "HardState: %s\n", state.String()) } default: log.Printf("Unexpected WAL log type: %d", rec.GetType()) } } ================================================ FILE: tools/etcd-dump-logs/raw_test.go ================================================ // Copyright 2022 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "bytes" "testing" "github.com/stretchr/testify/assert" ) func Test_readRaw(t *testing.T) { path := t.TempDir() mustCreateWALLog(t, path) var out bytes.Buffer readRaw(nil, walDir(path), &out) assert.Equal(t, `CRC: 0 Metadata: Snapshot: index:0 term:0 Entry: Term:1 Index:1 Type:EntryConfChange Data:"\010\001\020\000\030\002\"\000" Entry: Term:2 Index:2 Type:EntryConfChange Data:"\010\002\020\001\030\002\"\000" Entry: Term:2 Index:3 Type:EntryConfChange Data:"\010\003\020\002\030\002\"\000" Entry: Term:2 Index:4 Type:EntryConfChange Data:"\010\004\020\003\030\003\"\000" Entry: Term:4 Index:10 Data:"\010\005\032\025\n\0011\022\002hi\030\006 \001(\001X\240\234\001h\240\234\001" Entry: Term:5 Index:11 Data:"\010\006\"\020\n\004foo1\022\004bar1\030\0010\001" Entry: Term:6 Index:12 Data:"\010\007*\010\n\0010\022\0019\030\001" Entry: Term:7 Index:13 Data:"\010\0102\024\022\010\032\006\n\001a\022\001b\032\010\032\006\n\001a\022\001b" Entry: Term:8 Index:14 Data:"\010\t:\002\020\001" Entry: Term:9 Index:15 Data:"\010\nB\004\010\001\020\001" Entry: Term:10 Index:16 Data:"\010\013J\002\010\002" Entry: Term:11 Index:17 Data:"\010\014R\006\010\003\020\004\030\005" Entry: Term:12 Index:18 Data:"\010\r\302>\000" Entry: Term:13 Index:19 Data:"\010\016\232?\000" Entry: Term:14 Index:20 Data:"\010\017\242?\031\n\006myname\022\010password\032\005token" Entry: Term:15 Index:21 Data:"\010\020\342D\020\n\005name1\022\005pass1\032\000" Entry: Term:16 Index:22 Data:"\010\021\352D\007\n\005name1" Entry: Term:17 Index:23 Data:"\010\022\362D\007\n\005name1" Entry: Term:18 Index:24 Data:"\010\023\372D\016\n\005name1\022\005pass2" Entry: Term:19 Index:25 Data:"\010\024\202E\016\n\005user1\022\005role1" Entry: Term:20 Index:26 Data:"\010\025\212E\016\n\005user2\022\005role2" Entry: Term:21 Index:27 Data:"\010\026\222E\000" Entry: Term:22 Index:28 Data:"\010\027\232E\000" Entry: Term:23 Index:29 Data:"\010\030\202K\007\n\005role2" Entry: Term:24 Index:30 Data:"\010\031\212K\007\n\005role1" Entry: Term:25 Index:31 Data:"\010\032\222K\007\n\005role3" Entry: Term:26 Index:32 Data:"\010\033\232K\033\n\005role3\022\022\010\001\022\004Keys\032\010RangeEnd" Entry: Term:27 Index:33 Data:"\010\034\242K\026\n\005role3\022\003key\032\010rangeend" Entry: Term:27 Index:34 Data:"?" EOF: All entries were processed. `, out.String()) } ================================================ FILE: tools/etcd-dump-logs/testdecoder/decoder_correctoutputformat.sh ================================================ #!/bin/bash while read line do LEN=$(echo ${#line}) if [ $LEN -ge 20 ]; then echo "OK|$line" | tr 1234567890 abcdefghij else echo "ERROR|$line" | tr 1234567890 abcdefghij fi done < "${1:-/dev/stdin}" ================================================ FILE: tools/etcd-dump-logs/testdecoder/decoder_wrongoutputformat.sh ================================================ #!/bin/bash while read line do echo "$line" | tr 1234567890 abcdefghij done < "${1:-/dev/stdin}" ================================================ FILE: tools/etcd-dump-metrics/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/observability ================================================ FILE: tools/etcd-dump-metrics/README.md ================================================ # etcd-dump-metrics `etcd-dump-metrics` provides metrics for the latest main branch, a given endpoint, or version. ## Installation Install the tool by running the following command from the etcd source directory. ``` $ go install -v ./tools/etcd-dump-metrics ``` The installation will place executables in the $GOPATH/bin. If $GOPATH environment variable is not set, the tool will be installed into the $HOME/go/bin. You can also find out the installed location by running the following command from the etcd source directory. Make sure that $PATH is set accordingly in your environment. ``` $ go list -f "{{.Target}}" ./tools/etcd-dump-metrics ``` Alternatively, instead of installing the tool, you can use it by simply running the following command from the etcd source directory. ``` $ go run ./tools/etcd-dump-metrics ``` ## Usage The following command should output the usage per the latest development. ``` $ etcd-dump-metrics --help ``` An example of usage detail is provided below. ### For the latest main branch ``` $ etcd-dump-metrics ``` ### For the provided endpoint ``` $ goreman start $ etcd-dump-metrics --addr http://localhost:2379/metrics ``` ### Download specific version to temporary directory to fetch metrics ``` $ etcd-dump-metrics --debug --download-ver v3.5.3 $ etcd-dump-metrics --download-ver v3.5.3 ``` ================================================ FILE: tools/etcd-dump-metrics/etcd.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "context" "fmt" "net/url" "os" "strings" "time" "go.uber.org/zap" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/server/v3/embed" ) func newEmbedURLs(n int) (urls []url.URL) { urls = make([]url.URL, n) for i := 0; i < n; i++ { u, _ := url.Parse(fmt.Sprintf("unix://localhost:%d%06d", os.Getpid(), i)) urls[i] = *u } return urls } func setupEmbedCfg(cfg *embed.Config, curls, purls, ics []url.URL) { cfg.Logger = "zap" cfg.LogOutputs = []string{"/dev/null"} // []string{"stderr"} to enable server logging var err error cfg.Dir, err = os.MkdirTemp(os.TempDir(), fmt.Sprintf("%016X", time.Now().UnixNano())) if err != nil { panic(err) } os.RemoveAll(cfg.Dir) cfg.ClusterState = "new" cfg.ListenClientUrls, cfg.AdvertiseClientUrls = curls, curls cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = purls, purls cfg.InitialCluster = "" for i := range ics { cfg.InitialCluster += fmt.Sprintf(",%d=%s", i, ics[i].String()) } cfg.InitialCluster = cfg.InitialCluster[1:] } func getCommand(exec, name, dir, cURL, pURL, cluster string) (args []string) { if !strings.Contains(exec, "etcd") { panic(fmt.Errorf("%q doesn't seem like etcd binary", exec)) } return []string{ exec, "--name", name, "--data-dir", dir, "--listen-client-urls", cURL, "--advertise-client-urls", cURL, "--listen-peer-urls", pURL, "--initial-advertise-peer-urls", pURL, "--initial-cluster", cluster, "--initial-cluster-token=tkn", "--initial-cluster-state=new", } } func write(ep string) { cli, err := clientv3.New(clientv3.Config{Endpoints: []string{strings.Replace(ep, "/metrics", "", 1)}}) if err != nil { lg.Panic("failed to create client", zap.Error(err)) } defer cli.Close() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() _, err = cli.Put(ctx, "____test", "") if err != nil { lg.Panic("failed to write test key", zap.Error(err)) } _, err = cli.Get(ctx, "____test") if err != nil { lg.Panic("failed to read test key", zap.Error(err)) } _, err = cli.Delete(ctx, "____test") if err != nil { lg.Panic("failed to delete test key", zap.Error(err)) } cli.Watch(ctx, "____test", clientv3.WithCreatedNotify()) } ================================================ FILE: tools/etcd-dump-metrics/install_darwin.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build darwin package main import ( "fmt" "io" "net/http" "os" "os/exec" "path/filepath" "go.etcd.io/etcd/client/pkg/v3/fileutil" ) const downloadURL = `https://storage.googleapis.com/etcd/%s/etcd-%s-darwin-amd64.zip` func install(ver, dir string) (string, error) { ep := fmt.Sprintf(downloadURL, ver, ver) resp, err := http.Get(ep) if err != nil { return "", err } defer resp.Body.Close() d, err := io.ReadAll(resp.Body) if err != nil { return "", err } zipPath := filepath.Join(dir, "etcd.zip") if err = os.WriteFile(zipPath, d, fileutil.PrivateFileMode); err != nil { return "", err } if err = exec.Command("bash", "-c", fmt.Sprintf("unzip %s -d %s", zipPath, dir)).Run(); err != nil { return "", err } bp1 := filepath.Join(dir, fmt.Sprintf("etcd-%s-darwin-amd64", ver), "etcd") bp2 := filepath.Join(dir, "etcd") return bp2, os.Rename(bp1, bp2) } ================================================ FILE: tools/etcd-dump-metrics/install_linux.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build linux package main import ( "fmt" "io" "net/http" "os" "os/exec" "path/filepath" "go.etcd.io/etcd/client/pkg/v3/fileutil" ) const downloadURL = `https://storage.googleapis.com/etcd/%s/etcd-%s-linux-amd64.tar.gz` func install(ver, dir string) (string, error) { ep := fmt.Sprintf(downloadURL, ver, ver) resp, err := http.Get(ep) if err != nil { return "", err } defer resp.Body.Close() d, err := io.ReadAll(resp.Body) if err != nil { return "", err } tarPath := filepath.Join(dir, "etcd.tar.gz") if err = os.WriteFile(tarPath, d, fileutil.PrivateFileMode); err != nil { return "", err } // parametrizes to prevent attackers from adding arbitrary OS commands if err = exec.Command("tar", "xzvf", tarPath, "-C", dir, "--strip-components=1").Run(); err != nil { return "", err } return filepath.Join(dir, "etcd"), nil } ================================================ FILE: tools/etcd-dump-metrics/install_windows.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build windows package main import "errors" func install(ver, dir string) (string, error) { return "", errors.New("windows install is not supported yet") } ================================================ FILE: tools/etcd-dump-metrics/main.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // etcd-dump-metrics automates etcd Prometheus metrics documentation. package main import ( "flag" "fmt" "net/url" "os" "os/exec" "path/filepath" "time" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/server/v3/embed" ) var lg *zap.Logger func init() { var err error lg, err = logutil.CreateDefaultZapLogger(zap.InfoLevel) if err != nil { panic(err) } } func main() { addr := flag.String("addr", "", "etcd metrics URL to fetch from (empty to use current git branch)") downloadVer := flag.String("download-ver", "", "etcd binary version to download and fetch metrics from") debug := flag.Bool("debug", false, "true to enable debug logging") flag.Parse() if *addr != "" && *downloadVer != "" { panic("specify either 'addr' or 'download-ver'") } if *debug { var err error lg, err = logutil.CreateDefaultZapLogger(zap.DebugLevel) if err != nil { panic(err) } } ep := *addr if ep == "" { if *downloadVer != "" { ver := *downloadVer // download release binary to temporary directory d, err := os.MkdirTemp(os.TempDir(), ver) if err != nil { panic(err) } defer os.RemoveAll(d) var bp string bp, err = install(ver, d) if err != nil { panic(err) } // set up 2-node cluster locally ep = "http://localhost:2379/metrics" cluster := "s1=http://localhost:2380,s2=http://localhost:22380" d1 := filepath.Join(d, "s1") d2 := filepath.Join(d, "s2") os.RemoveAll(d1) os.RemoveAll(d2) type run struct { err error cmd *exec.Cmd } rc := make(chan run) cs1 := getCommand(bp, "s1", d1, "http://localhost:2379", "http://localhost:2380", cluster) cmd1 := exec.Command(cs1[0], cs1[1:]...) go func() { if *debug { cmd1.Stderr = os.Stderr } if cerr := cmd1.Start(); cerr != nil { lg.Warn("failed to start first process", zap.Error(cerr)) rc <- run{err: cerr} return } lg.Debug("started first process") rc <- run{cmd: cmd1} }() cs2 := getCommand(bp, "s2", d2, "http://localhost:22379", "http://localhost:22380", cluster) cmd2 := exec.Command(cs2[0], cs2[1:]...) go func() { if *debug { cmd2.Stderr = os.Stderr } if cerr := cmd2.Start(); cerr != nil { lg.Warn("failed to start second process", zap.Error(cerr)) rc <- run{err: cerr} return } lg.Debug("started second process") rc <- run{cmd: cmd2} }() rc1 := <-rc if rc1.err != nil { panic(rc1.err) } rc2 := <-rc if rc2.err != nil { panic(rc2.err) } defer func() { lg.Debug("killing processes") rc1.cmd.Process.Kill() rc2.cmd.Process.Kill() rc1.cmd.Wait() rc2.cmd.Wait() lg.Debug("killed processes") }() // give enough time for peer-to-peer metrics lg.Debug("waiting") time.Sleep(7 * time.Second) lg.Debug("started 2-node etcd cluster") } else { // fetch metrics from embedded etcd uss := newEmbedURLs(4) ep = uss[0].String() + "/metrics" cfgs := []*embed.Config{embed.NewConfig(), embed.NewConfig()} cfgs[0].Name, cfgs[1].Name = "0", "1" setupEmbedCfg(cfgs[0], []url.URL{uss[0]}, []url.URL{uss[1]}, []url.URL{uss[1], uss[3]}) setupEmbedCfg(cfgs[1], []url.URL{uss[2]}, []url.URL{uss[3]}, []url.URL{uss[1], uss[3]}) type embedAndError struct { ec *embed.Etcd err error } ech := make(chan embedAndError) for _, cfg := range cfgs { go func(c *embed.Config) { e, err := embed.StartEtcd(c) if err != nil { ech <- embedAndError{err: err} return } <-e.Server.ReadyNotify() ech <- embedAndError{ec: e} }(cfg) } for range cfgs { ev := <-ech if ev.err != nil { lg.Panic("failed to start embedded etcd", zap.Error(ev.err)) } defer ev.ec.Close() } // give enough time for peer-to-peer metrics lg.Debug("waiting") time.Sleep(7 * time.Second) lg.Debug("started 2-node embedded etcd cluster") } } // send client requests to populate gRPC client-side metrics // TODO: enable default metrics initialization in v3.1 and v3.2 write(ep) lg.Debug("fetching metrics", zap.String("endpoint", ep)) fmt.Println(getMetrics(ep)) } ================================================ FILE: tools/etcd-dump-metrics/metrics.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "fmt" "io" "net/http" "sort" "strings" "time" "go.uber.org/zap" "go.etcd.io/etcd/client/pkg/v3/transport" ) func fetchMetrics(ep string) (lines []string, err error) { tr, err := transport.NewTimeoutTransport(transport.TLSInfo{}, time.Second, time.Second, time.Second) if err != nil { return nil, err } cli := &http.Client{Transport: tr} resp, err := cli.Get(ep) if err != nil { return nil, err } defer resp.Body.Close() b, rerr := io.ReadAll(resp.Body) if rerr != nil { return nil, rerr } lines = strings.Split(string(b), "\n") return lines, nil } func getMetrics(ep string) (m metricSlice) { lines, err := fetchMetrics(ep) if err != nil { lg.Panic("failed to fetch metrics", zap.Error(err)) } mss := parse(lines) sort.Sort(metricSlice(mss)) return mss } func (mss metricSlice) String() (s string) { ver := "unknown" for i, v := range mss { if strings.HasPrefix(v.name, "etcd_server_version") { ver = v.metrics[0] } s += v.String() if i != len(mss)-1 { s += "\n\n" } } return "# server version: " + ver + "\n\n" + s } type metricSlice []metric func (mss metricSlice) Len() int { return len(mss) } func (mss metricSlice) Less(i, j int) bool { return mss[i].name < mss[j].name } func (mss metricSlice) Swap(i, j int) { mss[i], mss[j] = mss[j], mss[i] } type metric struct { // raw data for debugging purposes raw []string // metrics name name string // metrics description desc string // metrics type tp string // aggregates of "grpc_server_handled_total" grpcCodes []string // keep fist 1 and last 4 if histogram or summary // otherwise, keep only 1 metrics []string } func (m metric) String() (s string) { s += fmt.Sprintf("# name: %q\n", m.name) s += fmt.Sprintf("# description: %q\n", m.desc) s += fmt.Sprintf("# type: %q\n", m.tp) if len(m.grpcCodes) > 0 { s += "# gRPC codes: \n" for _, c := range m.grpcCodes { s += fmt.Sprintf("# - %q\n", c) } } s += strings.Join(m.metrics, "\n") return s } func parse(lines []string) (mss []metric) { m := metric{raw: make([]string, 0), metrics: make([]string, 0)} for _, line := range lines { if strings.HasPrefix(line, "# HELP ") { // add previous metric and initialize if m.name != "" { mss = append(mss, m) } m = metric{raw: make([]string, 0), metrics: make([]string, 0)} m.raw = append(m.raw, line) ss := strings.Split(strings.Replace(line, "# HELP ", "", 1), " ") m.name, m.desc = ss[0], strings.Join(ss[1:], " ") continue } if strings.HasPrefix(line, "# TYPE ") { m.raw = append(m.raw, line) m.tp = strings.Split(strings.Replace(line, "# TYPE "+m.tp, "", 1), " ")[1] continue } m.raw = append(m.raw, line) m.metrics = append(m.metrics, strings.Split(line, " ")[0]) } if m.name != "" { mss = append(mss, m) } // aggregate for i := range mss { /* munge data for: etcd_network_active_peers{Local="c6c9b5143b47d146",Remote="fbdddd08d7e1608b"} etcd_network_peer_sent_bytes_total{To="c6c9b5143b47d146"} etcd_network_peer_received_bytes_total{From="0"} etcd_network_peer_received_bytes_total{From="fd422379fda50e48"} etcd_network_peer_round_trip_time_seconds_bucket{To="91bc3c398fb3c146",le="0.0001"} etcd_network_peer_round_trip_time_seconds_bucket{To="fd422379fda50e48",le="0.8192"} etcd_network_peer_round_trip_time_seconds_bucket{To="fd422379fda50e48",le="+Inf"} etcd_network_peer_round_trip_time_seconds_sum{To="fd422379fda50e48"} etcd_network_peer_round_trip_time_seconds_count{To="fd422379fda50e48"} */ if mss[i].name == "etcd_network_active_peers" { mss[i].metrics = []string{`etcd_network_active_peers{Local="LOCAL_NODE_ID",Remote="REMOTE_PEER_NODE_ID"}`} } if mss[i].name == "etcd_network_peer_sent_bytes_total" { mss[i].metrics = []string{`etcd_network_peer_sent_bytes_total{To="REMOTE_PEER_NODE_ID"}`} } if mss[i].name == "etcd_network_peer_received_bytes_total" { mss[i].metrics = []string{`etcd_network_peer_received_bytes_total{From="REMOTE_PEER_NODE_ID"}`} } if mss[i].tp == "histogram" || mss[i].tp == "summary" { if mss[i].name == "etcd_network_peer_round_trip_time_seconds" { for j := range mss[i].metrics { l := mss[i].metrics[j] if strings.Contains(l, `To="`) && strings.Contains(l, `le="`) { k1 := strings.Index(l, `To="`) k2 := strings.Index(l, `",le="`) mss[i].metrics[j] = l[:k1+4] + "REMOTE_PEER_NODE_ID" + l[k2:] } if strings.HasPrefix(l, "etcd_network_peer_round_trip_time_seconds_sum") { mss[i].metrics[j] = `etcd_network_peer_round_trip_time_seconds_sum{To="REMOTE_PEER_NODE_ID"}` } if strings.HasPrefix(l, "etcd_network_peer_round_trip_time_seconds_count") { mss[i].metrics[j] = `etcd_network_peer_round_trip_time_seconds_count{To="REMOTE_PEER_NODE_ID"}` } } mss[i].metrics = aggSort(mss[i].metrics) } } // aggregate gRPC RPC metrics if mss[i].name == "grpc_server_handled_total" { pfx := `grpc_server_handled_total{grpc_code="` codes, metrics := make(map[string]struct{}), make(map[string]struct{}) for _, v := range mss[i].metrics { v2 := strings.Replace(v, pfx, "", 1) idx := strings.Index(v2, `",grpc_method="`) code := v2[:idx] v2 = v2[idx:] codes[code] = struct{}{} v2 = pfx + "CODE" + v2 metrics[v2] = struct{}{} } mss[i].grpcCodes = sortMap(codes) mss[i].metrics = sortMap(metrics) } } return mss } ================================================ FILE: tools/etcd-dump-metrics/utils.go ================================================ // Copyright 2018 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "maps" "slices" ) func aggSort(ss []string) (sorted []string) { dup := slices.Clone(ss) slices.Sort(dup) return slices.Compact(dup) } func sortMap(set map[string]struct{}) (sorted []string) { return slices.Sorted(maps.Keys(set)) } ================================================ FILE: tools/local-tester/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/testing ================================================ FILE: tools/local-tester/Procfile ================================================ # Use goreman to run `go get github.com/mattn/goreman` # peer bridges pbridge1: tools/local-tester/bridge.sh 127.0.0.1:11111 127.0.0.1:12380 pbridge2: tools/local-tester/bridge.sh 127.0.0.1:22222 127.0.0.1:22380 pbridge3: tools/local-tester/bridge.sh 127.0.0.1:33333 127.0.0.1:32380 # client bridges cbridge1: tools/local-tester/bridge.sh 127.0.0.1:2379 127.0.0.1:11119 cbridge2: tools/local-tester/bridge.sh 127.0.0.1:22379 127.0.0.1:22229 cbridge3: tools/local-tester/bridge.sh 127.0.0.1:32379 127.0.0.1:33339 faults: tools/local-tester/faults.sh stress-put: tools/benchmark/benchmark --endpoints=127.0.0.1:2379,127.0.0.1:22379,127.0.0.1:32379 --clients=27 --conns=3 put --sequential-keys --key-space-size=100000 --total=100000 etcd1: GOFAIL_HTTP="127.0.0.1:11180" bin/etcd --name infra1 --snapshot-count=1000 --listen-client-urls http://127.0.0.1:11119 --advertise-client-urls http://127.0.0.1:2379 --listen-peer-urls http://127.0.0.1:12380 --initial-advertise-peer-urls http://127.0.0.1:11111 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:11111,infra2=http://127.0.0.1:22222,infra3=http://127.0.0.1:33333' --initial-cluster-state new --enable-pprof etcd2: GOFAIL_HTTP="127.0.0.1:22280" bin/etcd --name infra2 --snapshot-count=1000 --listen-client-urls http://127.0.0.1:22229 --advertise-client-urls http://127.0.0.1:22379 --listen-peer-urls http://127.0.0.1:22380 --initial-advertise-peer-urls http://127.0.0.1:22222 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:11111,infra2=http://127.0.0.1:22222,infra3=http://127.0.0.1:33333' --initial-cluster-state new --enable-pprof etcd3: GOFAIL_HTTP="127.0.0.1:33380" bin/etcd --name infra3 --snapshot-count=1000 --listen-client-urls http://127.0.0.1:33339 --advertise-client-urls http://127.0.0.1:32379 --listen-peer-urls http://127.0.0.1:32380 --initial-advertise-peer-urls http://127.0.0.1:33333 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:11111,infra2=http://127.0.0.1:22222,infra3=http://127.0.0.1:33333' --initial-cluster-state new --enable-pprof # in future, use proxy to listen on 2379 #proxy: bin/etcd --name infra-proxy1 --proxy=on --listen-client-urls http://127.0.0.1:2378 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --enable-pprof ================================================ FILE: tools/local-tester/README.md ================================================ # etcd local-tester > [!WARNING] > etcd-local-tester is now deprecated in favor of our much more comprehensive [robustness testing suite](https://github.com/etcd-io/etcd/tree/main/tests/robustness). In a future etcd release this historic tool will be removed as it is no longer maintained. The etcd local-tester runs a fault injected cluster using local processes. It sets up an etcd cluster with unreliable network bridges on its peer and client interfaces. The cluster runs with a constant stream of `Put` requests to simulate client usage. A fault injection script periodically kills cluster members and disrupts bridge connectivity. # Requirements local-tester depends on `goreman` to manage its processes and `bash` to run fault injection. # Building local-tester needs `etcd`, `benchmark`, and `bridge` binaries. To build these binaries, run the following from the etcd repository root: ```sh ./scripts/build.sh pushd tools/benchmark/ && go build && popd pushd tools/local-tester/bridge && go build && popd ``` # Running The fault injected cluster is invoked with `goreman`: ```sh goreman -f tools/local-tester/Procfile start ``` ================================================ FILE: tools/local-tester/bridge/bridge.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package main is the entry point for the local tester network bridge. // Deprecated: etcd local tester is now deprecated. Use the etcd robustness // testing suite instead to validate etcd behaviour under failure conditions. package main import ( "flag" "fmt" "io" "log" "math/rand" "net" "sync" "time" ) type bridgeConn struct { in net.Conn out net.Conn d dispatcher } func newBridgeConn(in net.Conn, d dispatcher) (*bridgeConn, error) { out, err := net.Dial("tcp", flag.Args()[1]) if err != nil { in.Close() return nil, err } return &bridgeConn{in, out, d}, nil } func (b *bridgeConn) String() string { return fmt.Sprintf("%v <-> %v", b.in.RemoteAddr(), b.out.RemoteAddr()) } func (b *bridgeConn) Close() { b.in.Close() b.out.Close() } func bridge(b *bridgeConn) { log.Println("bridging", b.String()) go b.d.Copy(b.out, makeFetch(b.in)) b.d.Copy(b.in, makeFetch(b.out)) } func delayBridge(b *bridgeConn, txDelay, rxDelay time.Duration) { go b.d.Copy(b.out, makeFetchDelay(makeFetch(b.in), txDelay)) b.d.Copy(b.in, makeFetchDelay(makeFetch(b.out), rxDelay)) } func timeBridge(b *bridgeConn) { go func() { t := time.Duration(rand.Intn(5)+1) * time.Second time.Sleep(t) log.Printf("killing connection %s after %v\n", b.String(), t) b.Close() }() bridge(b) } func blackhole(b *bridgeConn) { log.Println("blackholing connection", b.String()) io.Copy(io.Discard, b.in) b.Close() } func readRemoteOnly(b *bridgeConn) { log.Println("one way (<-)", b.String()) b.d.Copy(b.in, makeFetch(b.out)) } func writeRemoteOnly(b *bridgeConn) { log.Println("one way (->)", b.String()) b.d.Copy(b.out, makeFetch(b.in)) } func corruptReceive(b *bridgeConn) { log.Println("corruptReceive", b.String()) go b.d.Copy(b.in, makeFetchCorrupt(makeFetch(b.out))) b.d.Copy(b.out, makeFetch(b.in)) } func corruptSend(b *bridgeConn) { log.Println("corruptSend", b.String()) go b.d.Copy(b.out, makeFetchCorrupt(makeFetch(b.in))) b.d.Copy(b.in, makeFetch(b.out)) } func makeFetch(c io.Reader) fetchFunc { return func() ([]byte, error) { b := make([]byte, 4096) n, err := c.Read(b) if err != nil { return nil, err } return b[:n], nil } } func makeFetchCorrupt(f func() ([]byte, error)) fetchFunc { return func() ([]byte, error) { b, err := f() if err != nil { return nil, err } // corrupt one byte approximately every 16K for i := 0; i < len(b); i++ { if rand.Intn(16*1024) == 0 { b[i] = b[i] + 1 } } return b, nil } } func makeFetchRand(f func() ([]byte, error)) fetchFunc { return func() ([]byte, error) { if rand.Intn(10) == 0 { return nil, fmt.Errorf("fetchRand: done") } b, err := f() if err != nil { return nil, err } return b, nil } } func makeFetchDelay(f fetchFunc, delay time.Duration) fetchFunc { return func() ([]byte, error) { b, err := f() if err != nil { return nil, err } time.Sleep(delay) return b, nil } } func randomBlackhole(b *bridgeConn) { log.Println("random blackhole: connection", b.String()) var wg sync.WaitGroup wg.Add(2) go func() { b.d.Copy(b.in, makeFetchRand(makeFetch(b.out))) wg.Done() }() go func() { b.d.Copy(b.out, makeFetchRand(makeFetch(b.in))) wg.Done() }() wg.Wait() b.Close() } type config struct { delayAccept bool resetListen bool connFaultRate float64 immediateClose bool blackhole bool timeClose bool writeRemoteOnly bool readRemoteOnly bool randomBlackhole bool corruptSend bool corruptReceive bool reorder bool txDelay string rxDelay string } type ( acceptFaultFunc func() connFaultFunc func(*bridgeConn) ) func main() { var cfg config flag.BoolVar(&cfg.delayAccept, "delay-accept", false, "delays accepting new connections") flag.BoolVar(&cfg.resetListen, "reset-listen", false, "resets the listening port") flag.Float64Var(&cfg.connFaultRate, "conn-fault-rate", 0.0, "rate of faulty connections") flag.BoolVar(&cfg.immediateClose, "immediate-close", false, "close after accept") flag.BoolVar(&cfg.blackhole, "blackhole", false, "reads nothing, writes go nowhere") flag.BoolVar(&cfg.timeClose, "time-close", false, "close after random time") flag.BoolVar(&cfg.writeRemoteOnly, "write-remote-only", false, "only write, no read") flag.BoolVar(&cfg.readRemoteOnly, "read-remote-only", false, "only read, no write") flag.BoolVar(&cfg.randomBlackhole, "random-blackhole", false, "blackhole after data xfer") flag.BoolVar(&cfg.corruptReceive, "corrupt-receive", false, "corrupt packets received from destination") flag.BoolVar(&cfg.corruptSend, "corrupt-send", false, "corrupt packets sent to destination") flag.BoolVar(&cfg.reorder, "reorder", false, "reorder packet delivery") flag.StringVar(&cfg.txDelay, "tx-delay", "0", "duration to delay client transmission to server") flag.StringVar(&cfg.rxDelay, "rx-delay", "0", "duration to delay client receive from server") flag.Parse() lAddr := flag.Args()[0] fwdAddr := flag.Args()[1] log.Println("listening on ", lAddr) log.Println("forwarding to ", fwdAddr) l, err := net.Listen("tcp", lAddr) if err != nil { log.Fatal(err) } defer l.Close() acceptFaults := []acceptFaultFunc{func() {}} if cfg.delayAccept { f := func() { log.Println("delaying accept") time.Sleep(3 * time.Second) } acceptFaults = append(acceptFaults, f) } if cfg.resetListen { f := func() { log.Println("reset listen port") l.Close() newListener, err := net.Listen("tcp", lAddr) if err != nil { log.Fatal(err) } l = newListener } acceptFaults = append(acceptFaults, f) } connFaults := []connFaultFunc{func(b *bridgeConn) { bridge(b) }} if cfg.immediateClose { f := func(b *bridgeConn) { log.Printf("terminating connection %s immediately", b.String()) b.Close() } connFaults = append(connFaults, f) } if cfg.blackhole { connFaults = append(connFaults, blackhole) } if cfg.timeClose { connFaults = append(connFaults, timeBridge) } if cfg.writeRemoteOnly { connFaults = append(connFaults, writeRemoteOnly) } if cfg.readRemoteOnly { connFaults = append(connFaults, readRemoteOnly) } if cfg.randomBlackhole { connFaults = append(connFaults, randomBlackhole) } if cfg.corruptSend { connFaults = append(connFaults, corruptSend) } if cfg.corruptReceive { connFaults = append(connFaults, corruptReceive) } txd, txdErr := time.ParseDuration(cfg.txDelay) if txdErr != nil { log.Fatal(txdErr) } rxd, rxdErr := time.ParseDuration(cfg.rxDelay) if rxdErr != nil { log.Fatal(rxdErr) } if txd != 0 || rxd != 0 { f := func(b *bridgeConn) { delayBridge(b, txd, rxd) } connFaults = append(connFaults, f) } if len(connFaults) > 1 && cfg.connFaultRate == 0 { log.Fatal("connection faults defined but conn-fault-rate=0") } var disp dispatcher if cfg.reorder { disp = newDispatcherPool() } else { disp = newDispatcherImmediate() } for { acceptFaults[rand.Intn(len(acceptFaults))]() conn, err := l.Accept() if err != nil { log.Fatal(err) } r := rand.Intn(len(connFaults)) if rand.Intn(100) >= int(100.0*cfg.connFaultRate) { r = 0 } bc, err := newBridgeConn(conn, disp) if err != nil { log.Printf("oops %v", err) continue } go connFaults[r](bc) } } ================================================ FILE: tools/local-tester/bridge/dispatch.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Deprecated: etcd local tester is now deprecated. Use the etcd robustness // testing suite instead to validate etcd behaviour under failure conditions. package main import ( "io" "math/rand" "sync" "time" ) var ( // dispatchPoolDelay is the time to wait before flushing all buffered packets dispatchPoolDelay = 100 * time.Millisecond // dispatchPacketBytes is how many bytes to send until choosing a new connection dispatchPacketBytes = 32 ) type dispatcher interface { // Copy works like io.Copy using buffers provided by fetchFunc Copy(io.Writer, fetchFunc) error } type fetchFunc func() ([]byte, error) type dispatcherPool struct { // mu protects the dispatch packet queue 'q' mu sync.Mutex q []dispatchPacket } type dispatchPacket struct { buf []byte out io.Writer } func newDispatcherPool() dispatcher { d := &dispatcherPool{} go d.writeLoop() return d } func (d *dispatcherPool) writeLoop() { for { time.Sleep(dispatchPoolDelay) d.flush() } } func (d *dispatcherPool) flush() { d.mu.Lock() pkts := d.q d.q = nil d.mu.Unlock() if len(pkts) == 0 { return } // sort by sockets; preserve the packet ordering within a socket pktmap := make(map[io.Writer][]dispatchPacket) var outs []io.Writer for _, pkt := range pkts { opkts, ok := pktmap[pkt.out] if !ok { outs = append(outs, pkt.out) } pktmap[pkt.out] = append(opkts, pkt) } // send all packets in pkts for len(outs) != 0 { // randomize writer on every write r := rand.Intn(len(outs)) rpkts := pktmap[outs[r]] rpkts[0].out.Write(rpkts[0].buf) // dequeue packet rpkts = rpkts[1:] if len(rpkts) == 0 { delete(pktmap, outs[r]) outs = append(outs[:r], outs[r+1:]...) } else { pktmap[outs[r]] = rpkts } } } func (d *dispatcherPool) Copy(w io.Writer, f fetchFunc) error { for { b, err := f() if err != nil { return err } var pkts []dispatchPacket for len(b) > 0 { pkt := b if len(b) > dispatchPacketBytes { pkt = pkt[:dispatchPacketBytes] b = b[dispatchPacketBytes:] } else { b = nil } pkts = append(pkts, dispatchPacket{pkt, w}) } d.mu.Lock() d.q = append(d.q, pkts...) d.mu.Unlock() } } type dispatcherImmediate struct{} func newDispatcherImmediate() dispatcher { return &dispatcherImmediate{} } func (d *dispatcherImmediate) Copy(w io.Writer, f fetchFunc) error { for { b, err := f() if err != nil { return err } if _, err := w.Write(b); err != nil { return err } } } ================================================ FILE: tools/local-tester/bridge.sh ================================================ #!/bin/sh echo "Warning: etcd-local-tester is now deprecated in favor of our robustness testing suite and will be removed in a future release." exec tools/local-tester/bridge/bridge \ -delay-accept \ -reset-listen \ -conn-fault-rate=0.25 \ -immediate-close \ -blackhole \ -time-close \ -write-remote-only \ -read-remote-only \ -random-blackhole \ -corrupt-receive \ -corrupt-send \ -reorder \ $@ ================================================ FILE: tools/local-tester/faults.sh ================================================ #!/bin/bash PROCFILE="tools/local-tester/Procfile" HTTPFAIL=(127.0.0.1:11180 127.0.0.1:22280 127.0.0.1:33380) function wait_time { expr $RANDOM % 10 + 1 } function cycle { for a; do echo "cycling $a" goreman -f $PROCFILE run stop $a || echo "could not stop $a" sleep `wait_time`s goreman -f $PROCFILE run restart $a || echo "could not restart $a" done } function cycle_members { cycle etcd1 etcd2 etcd3 } function cycle_pbridge { cycle pbridge1 pbridge2 pbridge3 } function cycle_cbridge { cycle cbridge1 cbridge2 cbridge3 } function cycle_stresser { cycle stress-put } function kill_maj { idx="etcd"`expr $RANDOM % 3 + 1` idx2="$idx" while [ "$idx" == "$idx2" ]; do idx2="etcd"`expr $RANDOM % 3 + 1` done echo "kill majority $idx $idx2" goreman -f $PROCFILE run stop $idx || echo "could not stop $idx" goreman -f $PROCFILE run stop $idx2 || echo "could not stop $idx2" sleep `wait_time`s goreman -f $PROCFILE run restart $idx || echo "could not restart $idx" goreman -f $PROCFILE run restart $idx2 || echo "could not restart $idx2" } function kill_all { for a in etcd1 etcd2 etcd3; do goreman -f $PROCFILE run stop $a || echo "could not stop $a" done sleep `wait_time`s for a in etcd1 etcd2 etcd3; do goreman -f $PROCFILE run restart $a || echo "could not restart $a" done } function rand_fp { echo "$FAILPOINTS" | sed `expr $RANDOM % $NUMFPS + 1`"q;d" } # fp_activate function fp_activate { curl "$1"/"$2" -XPUT -d "$3" >/dev/null 2>&1 } function fp_rand_single { fp=`rand_fp` fp_activate ${HTTPFAIL[`expr $RANDOM % ${#HTTPFAIL[@]}`]} $fp 'panic("'$fp'")' sleep `wait_time`s } function fp_rand_all { fp=`rand_fp` for a in `seq ${#HTTPFAIL[@]}`; do fp_activate ${HTTPFAIL[$a]} "$fp" 'panic("'$fp'")'; done sleep `wait_time`s } function fp_all_rand_fire { for fp in $FAILPOINTS; do for url in "${HTTPFAIL[@]}"; do fp_activate "$url" "$fp" '0.5%panic("0.5%'$fp'")' done done } function choose { fault=${FAULTS[`expr $RANDOM % ${#FAULTS[@]}`]} echo $fault $fault || echo "failed: $fault" } sleep 2s FAULTS=(cycle_members kill_maj kill_all cycle_pbridge cycle_cbridge cycle_stresser) # add failpoint faults if available FAILPOINTS=`curl http://"${HTTPFAIL[0]}" 2>/dev/null | cut -f1 -d'=' | grep -v "^$"` NUMFPS=`echo $(echo "$FAILPOINTS" | wc -l)` if [ "$NUMFPS" != "0" ]; then FAULTS+=(fp_rand_single) FAULTS+=(fp_rand_all) fi while [ 1 ]; do choose # start any nodes that have been killed by failpoints for a in etcd1 etcd2 etcd3; do goreman -f $PROCFILE run start $a; done fp_all_rand_fire done ================================================ FILE: tools/mod/doc.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // As this directory implements the pattern for tracking tool dependencies as documented here: // https://go.dev/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module, it doesn't // contain any valid go source code in the directory directly. This would break scripts for // unit testing, golangci-lint, and coverage calculation. // // Thus, to ensure tools to run normally, we've added this empty file. package mod ================================================ FILE: tools/mod/go.mod ================================================ module go.etcd.io/etcd/tools/v3 go 1.26 toolchain go1.26.1 require ( github.com/alexfalkowski/gocovmerge v1.11.0 github.com/appscodelabs/license-bill-of-materials v0.0.0-20220707232035-6018e0c5287c github.com/cloudflare/cfssl v1.6.5 github.com/gogo/protobuf v1.3.2 github.com/golangci/golangci-lint/v2 v2.11.1 github.com/google/yamlfmt v0.21.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 github.com/ryancurrah/gomodguard v1.4.1 go.etcd.io/gofail v0.2.0 go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee golang.org/x/tools v0.42.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1 google.golang.org/protobuf v1.36.11 gotest.tools/gotestsum v1.13.0 gotest.tools/v3 v3.5.2 honnef.co/go/tools v0.7.0 ) require ( 4d63.com/gocheckcompilerdirectives v1.3.0 // indirect 4d63.com/gochecknoglobals v0.2.2 // indirect codeberg.org/chavacava/garif v0.2.0 // indirect codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect dev.gaijin.team/go/golib v0.6.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/4meepo/tagalign v1.4.3 // indirect github.com/Abirdcfly/dupword v0.1.7 // indirect github.com/AdminBenni/iota-mixing v1.0.0 // indirect github.com/AlwxSin/noinlineerr v1.0.5 // indirect github.com/Antonboom/errname v1.1.1 // indirect github.com/Antonboom/nilnil v1.1.1 // indirect github.com/Antonboom/testifylint v1.6.4 // indirect github.com/BurntSushi/toml v1.6.0 // indirect github.com/Djarvur/go-err113 v0.1.1 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/MirrexOne/unqueryvet v1.5.4 // indirect github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect github.com/alecthomas/chroma/v2 v2.23.1 // indirect github.com/alecthomas/go-check-sumtype v0.3.1 // indirect github.com/alexkohler/nakedret/v2 v2.0.6 // indirect github.com/alexkohler/prealloc v1.1.0 // indirect github.com/alfatraining/structtag v1.0.0 // indirect github.com/alingse/asasalint v0.0.11 // indirect github.com/alingse/nilnesserr v0.2.0 // indirect github.com/ashanbrown/forbidigo/v2 v2.3.0 // indirect github.com/ashanbrown/makezero/v2 v2.1.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitfield/gotestdox v0.2.2 // indirect github.com/bkielbasa/cyclop v1.2.3 // indirect github.com/blizzy78/varnamelen v0.8.0 // indirect github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect github.com/bombsimon/wsl/v4 v4.7.0 // indirect github.com/bombsimon/wsl/v5 v5.6.0 // indirect github.com/breml/bidichk v0.3.3 // indirect github.com/breml/errchkjson v0.4.1 // indirect github.com/butuzov/ireturn v0.4.0 // indirect github.com/butuzov/mirror v1.3.0 // indirect github.com/catenacyber/perfsprint v0.10.1 // indirect github.com/ccojocar/zxcvbn-go v1.0.4 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charithe/durationcheck v0.0.11 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/lipgloss v1.1.0 // indirect github.com/charmbracelet/x/ansi v0.10.1 // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/ckaznocha/intrange v0.3.1 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect github.com/curioswitch/go-reassign v0.3.0 // indirect github.com/daixiang0/gci v0.13.7 // indirect github.com/dave/dst v0.27.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dnephin/pflag v1.0.7 // indirect github.com/ettle/strcase v0.2.0 // indirect github.com/fatih/color v1.18.0 // indirect github.com/fatih/structtag v1.2.0 // indirect github.com/firefart/nonamedreturns v1.0.6 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect github.com/ghostiam/protogetter v0.3.20 // indirect github.com/go-critic/go-critic v0.14.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/go-toolsmith/astcast v1.1.0 // indirect github.com/go-toolsmith/astcopy v1.1.0 // indirect github.com/go-toolsmith/astequal v1.2.0 // indirect github.com/go-toolsmith/astfmt v1.1.0 // indirect github.com/go-toolsmith/astp v1.1.0 // indirect github.com/go-toolsmith/strparse v1.1.0 // indirect github.com/go-toolsmith/typep v1.1.0 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/godoc-lint/godoc-lint v0.11.2 // indirect github.com/gofrs/flock v0.13.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golangci/asciicheck v0.5.0 // indirect github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 // indirect github.com/golangci/go-printf-func-name v0.1.1 // indirect github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d // indirect github.com/golangci/golines v0.15.0 // indirect github.com/golangci/misspell v0.8.0 // indirect github.com/golangci/plugin-module-register v0.1.2 // indirect github.com/golangci/revgrep v0.8.0 // indirect github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect github.com/google/certificate-transparency-go v1.1.8 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gordonklaus/ineffassign v0.2.0 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect github.com/gostaticanalysis/comment v1.5.0 // indirect github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect github.com/gostaticanalysis/nilerr v0.1.2 // indirect github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect github.com/hashicorp/go-version v1.8.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jgautheron/goconst v1.8.2 // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect github.com/jjti/go-spancheck v0.6.5 // indirect github.com/jmhodges/clock v1.2.0 // indirect github.com/jmoiron/sqlx v1.4.0 // indirect github.com/julz/importas v0.2.0 // indirect github.com/karamaru-alpha/copyloopvar v1.2.2 // indirect github.com/kisielk/errcheck v1.10.0 // indirect github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46 // indirect github.com/kkHAIKE/contextcheck v1.1.6 // indirect github.com/kulti/thelper v0.7.1 // indirect github.com/kunwardeep/paralleltest v1.0.15 // indirect github.com/lasiar/canonicalheader v1.1.2 // indirect github.com/ldez/exptostd v0.4.5 // indirect github.com/ldez/gomoddirectives v0.8.0 // indirect github.com/ldez/grignotin v0.10.1 // indirect github.com/ldez/structtags v0.6.1 // indirect github.com/ldez/tagliatelle v0.7.2 // indirect github.com/ldez/usetesting v0.5.0 // indirect github.com/leonklingele/grouper v1.1.2 // indirect github.com/lib/pq v1.11.2 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/macabu/inamedparam v0.2.0 // indirect github.com/manuelarte/embeddedstructfieldcheck v0.4.0 // indirect github.com/manuelarte/funcorder v0.5.0 // indirect github.com/maratori/testableexamples v1.0.1 // indirect github.com/maratori/testpackage v1.1.2 // indirect github.com/matoous/godox v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/mattn/go-sqlite3 v1.14.24 // indirect github.com/mgechev/revive v1.15.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moricho/tparallel v0.3.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nakabonne/nestif v0.3.1 // indirect github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.23.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/quasilyte/go-ruleguard v0.4.5 // indirect github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect github.com/raeperd/recvcheck v0.2.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/sashamelentyev/interfacebloat v1.1.0 // indirect github.com/sashamelentyev/usestdlibvars v1.29.0 // indirect github.com/securego/gosec/v2 v2.24.7 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/sivchari/containedctx v1.0.3 // indirect github.com/sonatard/noctx v0.5.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/sourcegraph/go-diff v0.7.0 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.7.1 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/spf13/viper v1.20.0 // indirect github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.3.1 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/stretchr/testify v1.11.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tetafro/godot v1.5.4 // indirect github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 // indirect github.com/timonwong/loggercheck v0.11.0 // indirect github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect github.com/ultraware/funlen v0.2.0 // indirect github.com/ultraware/whitespace v0.2.0 // indirect github.com/uudashr/gocognit v1.2.1 // indirect github.com/uudashr/iface v1.4.1 // indirect github.com/weppos/publicsuffix-go v0.30.3-0.20240510084413-5f1d03393b3d // indirect github.com/xen0n/gosmopolitan v1.3.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yagipy/maintidx v1.0.0 // indirect github.com/yeya24/promlinter v0.3.0 // indirect github.com/ykadowak/zerologlint v0.1.5 // indirect github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect github.com/zmap/zlint/v3 v3.6.0 // indirect gitlab.com/bosi/decorder v0.4.2 // indirect go-simpler.org/musttag v0.14.0 // indirect go-simpler.org/sloglint v0.11.1 // indirect go.augendre.info/arangolint v0.4.0 // indirect go.augendre.info/fatcontext v0.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect golang.org/x/term v0.40.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/grpc v1.79.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect mvdan.cc/gofumpt v0.9.2 // indirect mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) ================================================ FILE: tools/mod/go.sum ================================================ 4d63.com/gocheckcompilerdirectives v1.3.0 h1:Ew5y5CtcAAQeTVKUVFrE7EwHMrTO6BggtEj8BZSjZ3A= 4d63.com/gocheckcompilerdirectives v1.3.0/go.mod h1:ofsJ4zx2QAuIP/NO/NAh1ig6R1Fb18/GI7RVMwz7kAY= 4d63.com/gochecknoglobals v0.2.2 h1:H1vdnwnMaZdQW/N+NrkT1SZMTBmcwHe9Vq8lJcYYTtU= 4d63.com/gochecknoglobals v0.2.2/go.mod h1:lLxwTQjL5eIesRbvnzIP3jZtG140FnTdz+AlMa+ogt0= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= codeberg.org/chavacava/garif v0.2.0 h1:F0tVjhYbuOCnvNcU3YSpO6b3Waw6Bimy4K0mM8y6MfY= codeberg.org/chavacava/garif v0.2.0/go.mod h1:P2BPbVbT4QcvLZrORc2T29szK3xEOlnl0GiPTJmEqBQ= codeberg.org/polyfloyd/go-errorlint v1.9.0 h1:VkdEEmA1VBpH6ecQoMR4LdphVI3fA4RrCh2an7YmodI= codeberg.org/polyfloyd/go-errorlint v1.9.0/go.mod h1:GPRRu2LzVijNn4YkrZYJfatQIdS+TrcK8rL5Xs24qw8= dev.gaijin.team/go/exhaustruct/v4 v4.0.0 h1:873r7aNneqoBB3IaFIzhvt2RFYTuHgmMjoKfwODoI1Y= dev.gaijin.team/go/exhaustruct/v4 v4.0.0/go.mod h1:aZ/k2o4Y05aMJtiux15x8iXaumE88YdiB0Ai4fXOzPI= dev.gaijin.team/go/golib v0.6.0 h1:v6nnznFTs4bppib/NyU1PQxobwDHwCXXl15P7DV5Zgo= dev.gaijin.team/go/golib v0.6.0/go.mod h1:uY1mShx8Z/aNHWDyAkZTkX+uCi5PdX7KsG1eDQa2AVE= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/4meepo/tagalign v1.4.3 h1:Bnu7jGWwbfpAie2vyl63Zup5KuRv21olsPIha53BJr8= github.com/4meepo/tagalign v1.4.3/go.mod h1:00WwRjiuSbrRJnSVeGWPLp2epS5Q/l4UEy0apLLS37c= github.com/Abirdcfly/dupword v0.1.7 h1:2j8sInznrje4I0CMisSL6ipEBkeJUJAmK1/lfoNGWrQ= github.com/Abirdcfly/dupword v0.1.7/go.mod h1:K0DkBeOebJ4VyOICFdppB23Q0YMOgVafM0zYW0n9lF4= github.com/AdminBenni/iota-mixing v1.0.0 h1:Os6lpjG2dp/AE5fYBPAA1zfa2qMdCAWwPMCgpwKq7wo= github.com/AdminBenni/iota-mixing v1.0.0/go.mod h1:i4+tpAaB+qMVIV9OK3m4/DAynOd5bQFaOu+2AhtBCNY= github.com/AlwxSin/noinlineerr v1.0.5 h1:RUjt63wk1AYWTXtVXbSqemlbVTb23JOSRiNsshj7TbY= github.com/AlwxSin/noinlineerr v1.0.5/go.mod h1:+QgkkoYrMH7RHvcdxdlI7vYYEdgeoFOVjU9sUhw/rQc= github.com/Antonboom/errname v1.1.1 h1:bllB7mlIbTVzO9jmSWVWLjxTEbGBVQ1Ff/ClQgtPw9Q= github.com/Antonboom/errname v1.1.1/go.mod h1:gjhe24xoxXp0ScLtHzjiXp0Exi1RFLKJb0bVBtWKCWQ= github.com/Antonboom/nilnil v1.1.1 h1:9Mdr6BYd8WHCDngQnNVV0b554xyisFioEKi30sksufQ= github.com/Antonboom/nilnil v1.1.1/go.mod h1:yCyAmSw3doopbOWhJlVci+HuyNRuHJKIv6V2oYQa8II= github.com/Antonboom/testifylint v1.6.4 h1:gs9fUEy+egzxkEbq9P4cpcMB6/G0DYdMeiFS87UiqmQ= github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g= github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/MirrexOne/unqueryvet v1.5.4 h1:38QOxShO7JmMWT+eCdDMbcUgGCOeJphVkzzRgyLJgsQ= github.com/MirrexOne/unqueryvet v1.5.4/go.mod h1:fs9Zq6eh1LRIhsDIsxf9PONVUjYdFHdtkHIgZdJnyPU= github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma/v2 v2.23.1 h1:nv2AVZdTyClGbVQkIzlDm/rnhk1E9bU9nXwmZ/Vk/iY= github.com/alecthomas/chroma/v2 v2.23.1/go.mod h1:NqVhfBR0lte5Ouh3DcthuUCTUpDC9cxBOfyMbMQPs3o= github.com/alecthomas/go-check-sumtype v0.3.1 h1:u9aUvbGINJxLVXiFvHUlPEaD7VDULsrxJb4Aq31NLkU= github.com/alecthomas/go-check-sumtype v0.3.1/go.mod h1:A8TSiN3UPRw3laIgWEUOHHLPa6/r9MtoigdlP5h3K/E= github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alexfalkowski/gocovmerge v1.11.0 h1:mHYRBKEBHxjTWveV6RCAnCAhF6l1evO9JYboZRvZmuU= github.com/alexfalkowski/gocovmerge v1.11.0/go.mod h1:gkfERgPiozeXq7FlJBQDmTeXo8FnrNxZwabB4uSGZ3M= github.com/alexkohler/nakedret/v2 v2.0.6 h1:ME3Qef1/KIKr3kWX3nti3hhgNxw6aqN5pZmQiFSsuzQ= github.com/alexkohler/nakedret/v2 v2.0.6/go.mod h1:l3RKju/IzOMQHmsEvXwkqMDzHHvurNQfAgE1eVmT40Q= github.com/alexkohler/prealloc v1.1.0 h1:cKGRBqlXw5iyQGLYhrXrDlcHxugXpTq4tQ5c91wkf8M= github.com/alexkohler/prealloc v1.1.0/go.mod h1:fT39Jge3bQrfA7nPMDngUfvUbQGQeJyGQnR+913SCig= github.com/alfatraining/structtag v1.0.0 h1:2qmcUqNcCoyVJ0up879K614L9PazjBSFruTB0GOFjCc= github.com/alfatraining/structtag v1.0.0/go.mod h1:p3Xi5SwzTi+Ryj64DqjLWz7XurHxbGsq6y3ubePJPus= github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw= github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEWd/w= github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= github.com/appscodelabs/license-bill-of-materials v0.0.0-20220707232035-6018e0c5287c h1:xv0ICJ4AO52aNZ+vI2KFUYZBMh7dHvROixZ1vzMMfu8= github.com/appscodelabs/license-bill-of-materials v0.0.0-20220707232035-6018e0c5287c/go.mod h1:Y5/1I+0gnnhHKyX4z65mgaGTJ08tnz9WUgkoymA/cws= github.com/ashanbrown/forbidigo/v2 v2.3.0 h1:OZZDOchCgsX5gvToVtEBoV2UWbFfI6RKQTir2UZzSxo= github.com/ashanbrown/forbidigo/v2 v2.3.0/go.mod h1:5p6VmsG5/1xx3E785W9fouMxIOkvY2rRV9nMdWadd6c= github.com/ashanbrown/makezero/v2 v2.1.0 h1:snuKYMbqosNokUKm+R6/+vOPs8yVAi46La7Ck6QYSaE= github.com/ashanbrown/makezero/v2 v2.1.0/go.mod h1:aEGT/9q3S8DHeE57C88z2a6xydvgx8J5hgXIGWgo0MY= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE= github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY= github.com/bkielbasa/cyclop v1.2.3 h1:faIVMIGDIANuGPWH031CZJTi2ymOQBULs9H21HSMa5w= github.com/bkielbasa/cyclop v1.2.3/go.mod h1:kHTwA9Q0uZqOADdupvcFJQtp/ksSnytRMe8ztxG8Fuo= github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38= github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bombsimon/wsl/v4 v4.7.0 h1:1Ilm9JBPRczjyUs6hvOPKvd7VL1Q++PL8M0SXBDf+jQ= github.com/bombsimon/wsl/v4 v4.7.0/go.mod h1:uV/+6BkffuzSAVYD+yGyld1AChO7/EuLrCF/8xTiapg= github.com/bombsimon/wsl/v5 v5.6.0 h1:4z+/sBqC5vUmSp1O0mS+czxwH9+LKXtCWtHH9rZGQL8= github.com/bombsimon/wsl/v5 v5.6.0/go.mod h1:Uqt2EfrMj2NV8UGoN1f1Y3m0NpUVCsUdrNCdet+8LvU= github.com/breml/bidichk v0.3.3 h1:WSM67ztRusf1sMoqH6/c4OBCUlRVTKq+CbSeo0R17sE= github.com/breml/bidichk v0.3.3/go.mod h1:ISbsut8OnjB367j5NseXEGGgO/th206dVa427kR8YTE= github.com/breml/errchkjson v0.4.1 h1:keFSS8D7A2T0haP9kzZTi7o26r7kE3vymjZNeNDRDwg= github.com/breml/errchkjson v0.4.1/go.mod h1:a23OvR6Qvcl7DG/Z4o0el6BRAjKnaReoPQFciAl9U3s= github.com/butuzov/ireturn v0.4.0 h1:+s76bF/PfeKEdbG8b54aCocxXmi0wvYdOVsWxVO7n8E= github.com/butuzov/ireturn v0.4.0/go.mod h1:ghI0FrCmap8pDWZwfPisFD1vEc56VKH4NpQUxDHta70= github.com/butuzov/mirror v1.3.0 h1:HdWCXzmwlQHdVhwvsfBb2Au0r3HyINry3bDWLYXiKoc= github.com/butuzov/mirror v1.3.0/go.mod h1:AEij0Z8YMALaq4yQj9CPPVYOyJQyiexpQEQgihajRfI= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/catenacyber/perfsprint v0.10.1 h1:u7Riei30bk46XsG8nknMhKLXG9BcXz3+3tl/WpKm0PQ= github.com/catenacyber/perfsprint v0.10.1/go.mod h1:DJTGsi/Zufpuus6XPGJyKOTMELe347o6akPvWG9Zcsc= github.com/ccojocar/zxcvbn-go v1.0.4 h1:FWnCIRMXPj43ukfX000kvBZvV6raSxakYr1nzyNrUcc= github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charithe/durationcheck v0.0.11 h1:g1/EX1eIiKS57NTWsYtHDZ/APfeXKhye1DidBcABctk= github.com/charithe/durationcheck v0.0.11/go.mod h1:x5iZaixRNl8ctbM+3B2RrPG5t856TxRyVQEnbIEM2X4= github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= github.com/charmbracelet/x/ansi v0.10.1 h1:rL3Koar5XvX0pHGfovN03f5cxLbCF2YvLeyz7D2jVDQ= github.com/charmbracelet/x/ansi v0.10.1/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE= github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8= github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/ckaznocha/intrange v0.3.1 h1:j1onQyXvHUsPWujDH6WIjhyH26gkRt/txNlV7LspvJs= github.com/ckaznocha/intrange v0.3.1/go.mod h1:QVepyz1AkUoFQkpEqksSYpNpUo3c5W7nWh/s6SHIJJk= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cloudflare/cfssl v1.6.5 h1:46zpNkm6dlNkMZH/wMW22ejih6gIaJbzL2du6vD7ZeI= github.com/cloudflare/cfssl v1.6.5/go.mod h1:Bk1si7sq8h2+yVEDrFJiz3d7Aw+pfjjJSZVaD+Taky4= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/curioswitch/go-reassign v0.3.0 h1:dh3kpQHuADL3cobV/sSGETA8DOv457dwl+fbBAhrQPs= github.com/curioswitch/go-reassign v0.3.0/go.mod h1:nApPCCTtqLJN/s8HfItCcKV0jIPwluBOvZP+dsJGA88= github.com/daixiang0/gci v0.13.7 h1:+0bG5eK9vlI08J+J/NWGbWPTNiXPG4WhNLJOkSxWITQ= github.com/daixiang0/gci v0.13.7/go.mod h1:812WVN6JLFY9S6Tv76twqmNqevN0pa3SX3nih0brVzQ= github.com/dave/dst v0.27.3 h1:P1HPoMza3cMEquVf9kKy8yXsFirry4zEnWOdYPOoIzY= github.com/dave/dst v0.27.3/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEyc= github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo= github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q= github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/firefart/nonamedreturns v1.0.6 h1:vmiBcKV/3EqKY3ZiPxCINmpS431OcE1S47AQUwhrg8E= github.com/firefart/nonamedreturns v1.0.6/go.mod h1:R8NisJnSIpvPWheCq0mNRXJok6D8h7fagJTF8EMEwCo= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0= github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= github.com/go-critic/go-critic v0.14.3 h1:5R1qH2iFeo4I/RJU8vTezdqs08Egi4u5p6vOESA0pog= github.com/go-critic/go-critic v0.14.3/go.mod h1:xwntfW6SYAd7h1OqDzmN6hBX/JxsEKl5up/Y2bsxgVQ= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU= github.com/go-toolsmith/astcopy v1.1.0 h1:YGwBN0WM+ekI/6SS6+52zLDEf8Yvp3n2seZITCUBt5s= github.com/go-toolsmith/astcopy v1.1.0/go.mod h1:hXM6gan18VA1T/daUEHCFcYiW8Ai1tIwIzHY6srfEAw= github.com/go-toolsmith/astequal v1.0.3/go.mod h1:9Ai4UglvtR+4up+bAD4+hCj7iTo4m/OXVTSLnCyTAx4= github.com/go-toolsmith/astequal v1.1.0/go.mod h1:sedf7VIdCL22LD8qIvv7Nn9MuWJruQA/ysswh64lffQ= github.com/go-toolsmith/astequal v1.2.0 h1:3Fs3CYZ1k9Vo4FzFhwwewC3CHISHDnVUPC4x0bI2+Cw= github.com/go-toolsmith/astequal v1.2.0/go.mod h1:c8NZ3+kSFtFY/8lPso4v8LuJjdJiUFVnSuU3s0qrrDY= github.com/go-toolsmith/astfmt v1.1.0 h1:iJVPDPp6/7AaeLJEruMsBUlOYCmvg0MoCfJprsOmcco= github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlNMV634mhwuQ4= github.com/go-toolsmith/astp v1.1.0 h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA= github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA= github.com/go-toolsmith/pkgload v1.2.2 h1:0CtmHq/02QhxcF7E9N5LIFcYFsMR5rdovfqTtRKkgIk= github.com/go-toolsmith/pkgload v1.2.2/go.mod h1:R2hxLNRKuAsiXCo2i5J6ZQPhnPMOVtU+f0arbFPWCus= github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw= github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus= github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-xmlfmt/xmlfmt v1.1.3 h1:t8Ey3Uy7jDSEisW2K3somuMKIpzktkWptA0iFCnRUWY= github.com/go-xmlfmt/xmlfmt v1.1.3/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godoc-lint/godoc-lint v0.11.2 h1:Bp0FkJWoSdNsBikdNgIcgtaoo+xz6I/Y9s5WSBQUeeM= github.com/godoc-lint/godoc-lint v0.11.2/go.mod h1:iVpGdL1JCikNH2gGeAn3Hh+AgN5Gx/I/cxV+91L41jo= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golangci/asciicheck v0.5.0 h1:jczN/BorERZwK8oiFBOGvlGPknhvq0bjnysTj4nUfo0= github.com/golangci/asciicheck v0.5.0/go.mod h1:5RMNAInbNFw2krqN6ibBxN/zfRFa9S6tA1nPdM0l8qQ= github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 h1:WUvBfQL6EW/40l6OmeSBYQJNSif4O11+bmWEz+C7FYw= github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32/go.mod h1:NUw9Zr2Sy7+HxzdjIULge71wI6yEg1lWQr7Evcu8K0E= github.com/golangci/go-printf-func-name v0.1.1 h1:hIYTFJqAGp1iwoIfsNTpoq1xZAarogrvjO9AfiW3B4U= github.com/golangci/go-printf-func-name v0.1.1/go.mod h1:Es64MpWEZbh0UBtTAICOZiB+miW53w/K9Or/4QogJss= github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d h1:viFft9sS/dxoYY0aiOTsLKO2aZQAPT4nlQCsimGcSGE= github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d/go.mod h1:ivJ9QDg0XucIkmwhzCDsqcnxxlDStoTl89jDMIoNxKY= github.com/golangci/golangci-lint/v2 v2.11.1 h1:aGbjflzzKNIdOoq/NawrhFjYpkNY4WzPSeIp2zBbzG8= github.com/golangci/golangci-lint/v2 v2.11.1/go.mod h1:wexdFBIQNhHNhDe1oqzlGFE5dYUqlfccWJKWjoWF1GI= github.com/golangci/golines v0.15.0 h1:Qnph25g8Y1c5fdo1X7GaRDGgnMHgnxh4Gk4VfPTtRx0= github.com/golangci/golines v0.15.0/go.mod h1:AZjXd23tbHMpowhtnGlj9KCNsysj72aeZVVHnVcZx10= github.com/golangci/misspell v0.8.0 h1:qvxQhiE2/5z+BVRo1kwYA8yGz+lOlu5Jfvtx2b04Jbg= github.com/golangci/misspell v0.8.0/go.mod h1:WZyyI2P3hxPY2UVHs3cS8YcllAeyfquQcKfdeE9AFVg= github.com/golangci/plugin-module-register v0.1.2 h1:e5WM6PO6NIAEcij3B053CohVp3HIYbzSuP53UAYgOpg= github.com/golangci/plugin-module-register v0.1.2/go.mod h1:1+QGTsKBvAIvPvoY/os+G5eoqxWn70HYDm2uvUyGuVw= github.com/golangci/revgrep v0.8.0 h1:EZBctwbVd0aMeRnNUsFogoyayvKHyxlV3CdUA46FX2s= github.com/golangci/revgrep v0.8.0/go.mod h1:U4R/s9dlXZsg8uJmaR1GrloUr14D7qDl8gi2iPXJH8k= github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e h1:ai0EfmVYE2bRA5htgAG9r7s3tHsfjIhN98WshBTJ9jM= github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e/go.mod h1:Vrn4B5oR9qRwM+f54koyeH3yzphlecwERs0el27Fr/s= github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e h1:gD6P7NEo7Eqtt0ssnqSJNNndxe69DOQ24A5h7+i3KpM= github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e/go.mod h1:h+wZwLjUTJnm/P2rwlbJdRPZXOzaT36/FwnPnY2inzc= github.com/google/certificate-transparency-go v1.1.8 h1:LGYKkgZF7satzgTak9R4yzfJXEeYVAjV6/EAEJOf1to= github.com/google/certificate-transparency-go v1.1.8/go.mod h1:bV/o8r0TBKRf1X//iiiSgWrvII4d7/8OiA+3vG26gI8= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/yamlfmt v0.21.0 h1:9FKApQkDpMKgBjwLFytBHUCgqnQgxaQnci0uiESfbzs= github.com/google/yamlfmt v0.21.0/go.mod h1:q6FYExB+Ueu7jZDjKECJk+EaeDXJzJ6Ne0dxx69GWfI= github.com/gordonklaus/ineffassign v0.2.0 h1:Uths4KnmwxNJNzq87fwQQDDnbNb7De00VOk9Nu0TySs= github.com/gordonklaus/ineffassign v0.2.0/go.mod h1:TIpymnagPSexySzs7F9FnO1XFTy8IT3a59vmZp5Y9Lw= github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk= github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc= github.com/gostaticanalysis/comment v1.4.2/go.mod h1:KLUTGDv6HOCotCH8h2erHKmpci2ZoR8VPu34YA2uzdM= github.com/gostaticanalysis/comment v1.5.0 h1:X82FLl+TswsUMpMh17srGRuKaaXprTaytmEpgnKIDu8= github.com/gostaticanalysis/comment v1.5.0/go.mod h1:V6eb3gpCv9GNVqb6amXzEUX3jXLVK/AdA+IrAMSqvEc= github.com/gostaticanalysis/forcetypeassert v0.2.0 h1:uSnWrrUEYDr86OCxWa4/Tp2jeYDlogZiZHzGkWFefTk= github.com/gostaticanalysis/forcetypeassert v0.2.0/go.mod h1:M5iPavzE9pPqWyeiVXSFghQjljW1+l/Uke3PXHS6ILY= github.com/gostaticanalysis/nilerr v0.1.2 h1:S6nk8a9N8g062nsx63kUkF6AzbHGw7zzyHMcpu52xQU= github.com/gostaticanalysis/nilerr v0.1.2/go.mod h1:A19UHhoY3y8ahoL7YKz6sdjDtduwTSI4CsymaC2htPA= github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= github.com/gostaticanalysis/testutil v0.5.0 h1:Dq4wT1DdTwTGCQQv3rl3IvD5Ld0E6HiY+3Zh0sUGqw8= github.com/gostaticanalysis/testutil v0.5.0/go.mod h1:OLQSbuM6zw2EvCcXTz1lVq5unyoNft372msDY0nY5Hs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/hashicorp/go-immutable-radix/v2 v2.1.0 h1:CUW5RYIcysz+D3B+l1mDeXrQ7fUvGGCwJfdASSzbrfo= github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jgautheron/goconst v1.8.2 h1:y0XF7X8CikZ93fSNT6WBTb/NElBu9IjaY7CCYQrCMX4= github.com/jgautheron/goconst v1.8.2/go.mod h1:A0oxgBCHy55NQn6sYpO7UdnA9p+h7cPtoOZUmvNIako= github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs= github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c= github.com/jjti/go-spancheck v0.6.5 h1:lmi7pKxa37oKYIMScialXUK6hP3iY5F1gu+mLBPgYB8= github.com/jjti/go-spancheck v0.6.5/go.mod h1:aEogkeatBrbYsyW6y5TgDfihCulDYciL1B7rG2vSsrU= github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs= github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/julz/importas v0.2.0 h1:y+MJN/UdL63QbFJHws9BVC5RpA2iq0kpjrFajTGivjQ= github.com/julz/importas v0.2.0/go.mod h1:pThlt589EnCYtMnmhmRYY/qn9lCf/frPOK+WMx3xiJY= github.com/karamaru-alpha/copyloopvar v1.2.2 h1:yfNQvP9YaGQR7VaWLYcfZUlRP2eo2vhExWKxD/fP6q0= github.com/karamaru-alpha/copyloopvar v1.2.2/go.mod h1:oY4rGZqZ879JkJMtX3RRkcXRkmUvH0x35ykgaKgsgJY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.10.0 h1:Lvs/YAHP24YKg08LA8oDw2z9fJVme090RAXd90S+rrw= github.com/kisielk/errcheck v1.10.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46 h1:veS9QfglfvqAw2e+eeNT/SbGySq8ajECXJ9e4fPoLhY= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kulti/thelper v0.7.1 h1:fI8QITAoFVLx+y+vSyuLBP+rcVIB8jKooNSCT2EiI98= github.com/kulti/thelper v0.7.1/go.mod h1:NsMjfQEy6sd+9Kfw8kCP61W1I0nerGSYSFnGaxQkcbs= github.com/kunwardeep/paralleltest v1.0.15 h1:ZMk4Qt306tHIgKISHWFJAO1IDQJLc6uDyJMLyncOb6w= github.com/kunwardeep/paralleltest v1.0.15/go.mod h1:di4moFqtfz3ToSKxhNjhOZL+696QtJGCFe132CbBLGk= github.com/lasiar/canonicalheader v1.1.2 h1:vZ5uqwvDbyJCnMhmFYimgMZnJMjwljN5VGY0VKbMXb4= github.com/lasiar/canonicalheader v1.1.2/go.mod h1:qJCeLFS0G/QlLQ506T+Fk/fWMa2VmBUiEI2cuMK4djI= github.com/ldez/exptostd v0.4.5 h1:kv2ZGUVI6VwRfp/+bcQ6Nbx0ghFWcGIKInkG/oFn1aQ= github.com/ldez/exptostd v0.4.5/go.mod h1:QRjHRMXJrCTIm9WxVNH6VW7oN7KrGSht69bIRwvdFsM= github.com/ldez/gomoddirectives v0.8.0 h1:JqIuTtgvFC2RdH1s357vrE23WJF2cpDCPFgA/TWDGpk= github.com/ldez/gomoddirectives v0.8.0/go.mod h1:jutzamvZR4XYJLr0d5Honycp4Gy6GEg2mS9+2YX3F1Q= github.com/ldez/grignotin v0.10.1 h1:keYi9rYsgbvqAZGI1liek5c+jv9UUjbvdj3Tbn5fn4o= github.com/ldez/grignotin v0.10.1/go.mod h1:UlDbXFCARrXbWGNGP3S5vsysNXAPhnSuBufpTEbwOas= github.com/ldez/structtags v0.6.1 h1:bUooFLbXx41tW8SvkfwfFkkjPYvFFs59AAMgVg6DUBk= github.com/ldez/structtags v0.6.1/go.mod h1:YDxVSgDy/MON6ariaxLF2X09bh19qL7MtGBN5MrvbdY= github.com/ldez/tagliatelle v0.7.2 h1:KuOlL70/fu9paxuxbeqlicJnCspCRjH0x8FW+NfgYUk= github.com/ldez/tagliatelle v0.7.2/go.mod h1:PtGgm163ZplJfZMZ2sf5nhUT170rSuPgBimoyYtdaSI= github.com/ldez/usetesting v0.5.0 h1:3/QtzZObBKLy1F4F8jLuKJiKBjjVFi1IavpoWbmqLwc= github.com/ldez/usetesting v0.5.0/go.mod h1:Spnb4Qppf8JTuRgblLrEWb7IE6rDmUpGvxY3iRrzvDQ= github.com/leonklingele/grouper v1.1.2 h1:o1ARBDLOmmasUaNDesWqWCIFH3u7hoFlM84YrjT3mIY= github.com/leonklingele/grouper v1.1.2/go.mod h1:6D0M/HVkhs2yRKRFZUoGjeDy7EZTfFBE9gl4kjmIGkA= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/macabu/inamedparam v0.2.0 h1:VyPYpOc10nkhI2qeNUdh3Zket4fcZjEWe35poddBCpE= github.com/macabu/inamedparam v0.2.0/go.mod h1:+Pee9/YfGe5LJ62pYXqB89lJ+0k5bsR8Wgz/C0Zlq3U= github.com/manuelarte/embeddedstructfieldcheck v0.4.0 h1:3mAIyaGRtjK6EO9E73JlXLtiy7ha80b2ZVGyacxgfww= github.com/manuelarte/embeddedstructfieldcheck v0.4.0/go.mod h1:z8dFSyXqp+fC6NLDSljRJeNQJJDWnY7RoWFzV3PC6UM= github.com/manuelarte/funcorder v0.5.0 h1:llMuHXXbg7tD0i/LNw8vGnkDTHFpTnWqKPI85Rknc+8= github.com/manuelarte/funcorder v0.5.0/go.mod h1:Yt3CiUQthSBMBxjShjdXMexmzpP8YGvGLjrxJNkO2hA= github.com/maratori/testableexamples v1.0.1 h1:HfOQXs+XgfeRBJ+Wz0XfH+FHnoY9TVqL6Fcevpzy4q8= github.com/maratori/testableexamples v1.0.1/go.mod h1:XE2F/nQs7B9N08JgyRmdGjYVGqxWwClLPCGSQhXQSrQ= github.com/maratori/testpackage v1.1.2 h1:ffDSh+AgqluCLMXhM19f/cpvQAKygKAJXFl9aUjmbqs= github.com/maratori/testpackage v1.1.2/go.mod h1:8F24GdVDFW5Ew43Et02jamrVMNXLUNaOynhDssITGfc= github.com/matoous/godox v1.1.0 h1:W5mqwbyWrwZv6OQ5Z1a/DHGMOvXYCBP3+Ht7KMoJhq4= github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mgechev/revive v1.15.0 h1:vJ0HzSBzfNyPbHKolgiFjHxLek9KUijhqh42yGoqZ8Q= github.com/mgechev/revive v1.15.0/go.mod h1:LlAKO3QQe9OJ0pVZzI2GPa8CbXGZ/9lNpCGvK4T/a8A= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moricho/tparallel v0.3.2 h1:odr8aZVFA3NZrNybggMkYO3rgPRcqjeQUlBBFVxKHTI= github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U= github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg= github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= github.com/nunnatsa/ginkgolinter v0.23.0 h1:x3o4DGYOWbBMP/VdNQKgSj+25aJKx2Pe6lHr8gBcgf8= github.com/nunnatsa/ginkgolinter v0.23.0/go.mod h1:9qN1+0akwXEccwV1CAcCDfcoBlWXHB+ML9884pL4SZ4= github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA= github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/quasilyte/go-ruleguard v0.4.5 h1:AGY0tiOT5hJX9BTdx/xBdoCubQUAE2grkqY2lSwvZcA= github.com/quasilyte/go-ruleguard v0.4.5/go.mod h1:Vl05zJ538vcEEwu16V/Hdu7IYZWyKSwIy4c88Ro1kRE= github.com/quasilyte/go-ruleguard/dsl v0.3.23 h1:lxjt5B6ZCiBeeNO8/oQsegE6fLeCzuMRoVWSkXC4uvY= github.com/quasilyte/go-ruleguard/dsl v0.3.23/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= github.com/quasilyte/gogrep v0.5.0/go.mod h1:Cm9lpz9NZjEoL1tgZ2OgeUKPIxL1meE7eo60Z6Sk+Ng= github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl980XxGFEZSS6KlBGIV0diGdySzxATTWoqaU= github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= github.com/raeperd/recvcheck v0.2.0 h1:GnU+NsbiCqdC2XX5+vMZzP+jAJC5fht7rcVTAhX74UI= github.com/raeperd/recvcheck v0.2.0/go.mod h1:n04eYkwIR0JbgD73wT8wL4JjPC3wm0nFtzBnWNocnYU= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryancurrah/gomodguard v1.4.1 h1:eWC8eUMNZ/wM/PWuZBv7JxxqT5fiIKSIyTvjb7Elr+g= github.com/ryancurrah/gomodguard v1.4.1/go.mod h1:qnMJwV1hX9m+YJseXEBhd2s90+1Xn6x9dLz11ualI1I= github.com/ryanrolds/sqlclosecheck v0.5.1 h1:dibWW826u0P8jNLsLN+En7+RqWWTYrjCB9fJfSfdyCU= github.com/ryanrolds/sqlclosecheck v0.5.1/go.mod h1:2g3dUjoS6AL4huFdv6wn55WpLIDjY7ZgUR4J8HOO/XQ= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/sanposhiho/wastedassign/v2 v2.1.0 h1:crurBF7fJKIORrV85u9UUpePDYGWnwvv3+A96WvwXT0= github.com/sanposhiho/wastedassign/v2 v2.1.0/go.mod h1:+oSmSC+9bQ+VUAxA66nBb0Z7N8CK7mscKTDYC6aIek4= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tMEOsumirXcOJqAw= github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= github.com/sashamelentyev/usestdlibvars v1.29.0 h1:8J0MoRrw4/NAXtjQqTHrbW9NN+3iMf7Knkq057v4XOQ= github.com/sashamelentyev/usestdlibvars v1.29.0/go.mod h1:8PpnjHMk5VdeWlVb4wCdrB8PNbLqZ3wBZTZWkrpZZL8= github.com/securego/gosec/v2 v2.24.7 h1:3k5yJnrhT1TTdsG0ZsnenlfCcT+7Y/+zeCPHbL7QAn8= github.com/securego/gosec/v2 v2.24.7/go.mod h1:AdDJbjcG/XxFgVv7pW19vMNYlFM6+Q6Qy3t6lWAUcEY= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+Wwfd0XE= github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4= github.com/sonatard/noctx v0.5.0 h1:e/jdaqAsuWVOKQ0P6NWiIdDNHmHT5SwuuSfojFjzwrw= github.com/sonatard/noctx v0.5.0/go.mod h1:64XdbzFb18XL4LporKXp8poqZtPKbCrqQ402CV+kJas= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0= github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY= github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stbenjam/no-sprintf-host-port v0.3.1 h1:AyX7+dxI4IdLBPtDbsGAyqiTSLpCP9hWRrXQDU4Cm/g= github.com/stbenjam/no-sprintf-host-port v0.3.1/go.mod h1:ODbZesTCHMVKthBHskvUUexdcNHAQRXk9NpSsL8p/HQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tenntenn/modver v1.0.1 h1:2klLppGhDgzJrScMpkj9Ujy3rXPUspSjAcev9tSEBgA= github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpRQGxTSkNYKJ51yaw6ChIqO+Je8UqsTKN/cDag= github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= github.com/tetafro/godot v1.5.4 h1:u1ww+gqpRLiIA16yF2PV1CV1n/X3zhyezbNXC3E14Sg= github.com/tetafro/godot v1.5.4/go.mod h1:eOkMrVQurDui411nBY2FA05EYH01r14LuWY/NrVDVcU= github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 h1:9LPGD+jzxMlnk5r6+hJnar67cgpDIz/iyD+rfl5r2Vk= github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= github.com/timonwong/loggercheck v0.11.0 h1:jdaMpYBl+Uq9mWPXv1r8jc5fC3gyXx4/WGwTnnNKn4M= github.com/timonwong/loggercheck v0.11.0/go.mod h1:HEAWU8djynujaAVX7QI65Myb8qgfcZ1uKbdpg3ZzKl8= github.com/tomarrell/wrapcheck/v2 v2.12.0 h1:H/qQ1aNWz/eeIhxKAFvkfIA+N7YDvq6TWVFL27Of9is= github.com/tomarrell/wrapcheck/v2 v2.12.0/go.mod h1:AQhQuZd0p7b6rfW+vUwHm5OMCGgp63moQ9Qr/0BpIWo= github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw= github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLkI= github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA= github.com/ultraware/whitespace v0.2.0 h1:TYowo2m9Nfj1baEQBjuHzvMRbp19i+RCcRYrSWoFa+g= github.com/ultraware/whitespace v0.2.0/go.mod h1:XcP1RLD81eV4BW8UhQlpaR+SDc2givTvyI8a586WjW8= github.com/uudashr/gocognit v1.2.1 h1:CSJynt5txTnORn/DkhiB4mZjwPuifyASC8/6Q0I/QS4= github.com/uudashr/gocognit v1.2.1/go.mod h1:acaubQc6xYlXFEMb9nWX2dYBzJ/bIjEkc1zzvyIZg5Q= github.com/uudashr/iface v1.4.1 h1:J16Xl1wyNX9ofhpHmQ9h9gk5rnv2A6lX/2+APLTo0zU= github.com/uudashr/iface v1.4.1/go.mod h1:pbeBPlbuU2qkNDn0mmfrxP2X+wjPMIQAy+r1MBXSXtg= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ= github.com/weppos/publicsuffix-go v0.30.3-0.20240510084413-5f1d03393b3d h1:q80YKUcDWRNvvQcziH63e3ammTWARwrhohBCunHaYAg= github.com/weppos/publicsuffix-go v0.30.3-0.20240510084413-5f1d03393b3d/go.mod h1:vLdXKydr/OJssAXmjY0XBgLXUfivBMrNRIBljgtqCnw= github.com/xen0n/gosmopolitan v1.3.0 h1:zAZI1zefvo7gcpbCOrPSHJZJYA9ZgLfJqtKzZ5pHqQM= github.com/xen0n/gosmopolitan v1.3.0/go.mod h1:rckfr5T6o4lBtM1ga7mLGKZmLxswUoH1zxHgNXOsEt4= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM= github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs= github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4= github.com/ykadowak/zerologlint v0.1.5 h1:Gy/fMz1dFQN9JZTPjv1hxEk+sRWm05row04Yoolgdiw= github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk= github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c h1:U1b4THKcgOpJ+kILupuznNwPiURtwVW3e9alJvji9+s= github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c/go.mod h1:GSDpFDD4TASObxvfZfvpZZ3OWHIUHMlhVWlkOe4ewVk= github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= github.com/zmap/zlint/v3 v3.6.0 h1:vTEaDRtYN0d/1Ax60T+ypvbLQUHwHxbvYRnUMVr35ug= github.com/zmap/zlint/v3 v3.6.0/go.mod h1:NVgiIWssgzp0bNl8P4Gz94NHV2ep/4Jyj9V69uTmZyg= gitlab.com/bosi/decorder v0.4.2 h1:qbQaV3zgwnBZ4zPMhGLW4KZe7A7NwxEhJx39R3shffo= gitlab.com/bosi/decorder v0.4.2/go.mod h1:muuhHoaJkA9QLcYHq4Mj8FJUwDZ+EirSHRiaTcTf6T8= go-simpler.org/assert v0.9.0 h1:PfpmcSvL7yAnWyChSjOz6Sp6m9j5lyK8Ok9pEL31YkQ= go-simpler.org/assert v0.9.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28= go-simpler.org/musttag v0.14.0 h1:XGySZATqQYSEV3/YTy+iX+aofbZZllJaqwFWs+RTtSo= go-simpler.org/musttag v0.14.0/go.mod h1:uP8EymctQjJ4Z1kUnjX0u2l60WfUdQxCwSNKzE1JEOE= go-simpler.org/sloglint v0.11.1 h1:xRbPepLT/MHPTCA6TS/wNfZrDzkGvCCqUv4Bdwc3H7s= go-simpler.org/sloglint v0.11.1/go.mod h1:2PowwiCOK8mjiF+0KGifVOT8ZsCNiFzvfyJeJOIt8MQ= go.augendre.info/arangolint v0.4.0 h1:xSCZjRoS93nXazBSg5d0OGCi9APPLNMmmLrC995tR50= go.augendre.info/arangolint v0.4.0/go.mod h1:l+f/b4plABuFISuKnTGD4RioXiCCgghv2xqst/xOvAA= go.augendre.info/fatcontext v0.9.0 h1:Gt5jGD4Zcj8CDMVzjOJITlSb9cEch54hjRRlN3qDojE= go.augendre.info/fatcontext v0.9.0/go.mod h1:L94brOAT1OOUNue6ph/2HnwxoNlds9aXDF2FcUntbNw= go.etcd.io/gofail v0.2.0 h1:p19drv16FKK345a09a1iubchlw/vmRuksmRzgBIGjcA= go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o= go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 h1:QQiUXlqz+d96jyNG71NE+IGTgOK6Xlhdx+PzvfbLHlQ= go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116/go.mod h1:F9kog+iVAuvPJucb1dkYcDcbV0g4uyGEHllTP5NrXiw= go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee h1:9s5V0M58uCy51LgP6SUjROx7Ofqf8lGmeD/cCLaoagI= go.etcd.io/raft/v3 v3.6.0-beta.0.0.20260116184858-6d944ca211ee/go.mod h1:VteWcRz3UV3TOpfex1x8jgPKAyjRXLKw3j8RdK3UAps= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 h1:qWFG1Dj7TBjOjOvhEOkmyGPVoquqUKnIU0lEVLp8xyk= golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358/go.mod h1:4Mzdyp/6jzw9auFDJ3OMF5qksa7UvPnzKqTVGcb04ms= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 h1:bTLqdHv7xrGlFbvf5/TXNxy/iUwwdkjhqQTJDjW7aj0= golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1 h1:/WILD1UcXj/ujCxgoL/DvRgt2CP3txG8+FwkUbb9110= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1/go.mod h1:YNKnb2OAApgYn2oYY47Rn7alMr1zWjb2U8Q0aoGWiNc= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/gotestsum v1.13.0 h1:+Lh454O9mu9AMG1APV4o0y7oDYKyik/3kBOiCqiEpRo= gotest.tools/gotestsum v1.13.0/go.mod h1:7f0NS5hFb0dWr4NtcsAsF0y1kzjEFfAil0HiBQJE03Q= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= mvdan.cc/gofumpt v0.9.2 h1:zsEMWL8SVKGHNztrx6uZrXdp7AX8r421Vvp23sz7ik4= mvdan.cc/gofumpt v0.9.2/go.mod h1:iB7Hn+ai8lPvofHd9ZFGVg2GOr8sBUw1QUWjNbmIL/s= mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 h1:ssMzja7PDPJV8FStj7hq9IKiuiKhgz9ErWw+m68e7DI= mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15/go.mod h1:4M5MMXl2kW6fivUT6yRGpLLPNfuGtU2Z0cPvFquGDYU= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= ================================================ FILE: tools/mod/install_all.sh ================================================ #!/usr/bin/env bash set -euo pipefail cd ./tools/mod || exit 2 go list --tags tools -f '{{ join .Imports "\n" }}' | xargs go install ================================================ FILE: tools/mod/libs.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build libs // This file implements that pattern: // https://go.dev/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module // for etcd. Thanks to this file 'go mod tidy' does not removes dependencies. package libs import ( _ "github.com/gogo/protobuf/proto" ) ================================================ FILE: tools/mod/tools.go ================================================ // Copyright 2016 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build tools // This file implements that pattern: // https://go.dev/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module // for etcd. Thanks to this file 'go mod tidy' does not removes dependencies. package tools import ( _ "github.com/alexfalkowski/gocovmerge" _ "github.com/appscodelabs/license-bill-of-materials" _ "github.com/cloudflare/cfssl/cmd/cfssl" _ "github.com/cloudflare/cfssl/cmd/cfssljson" _ "github.com/golangci/golangci-lint/v2/cmd/golangci-lint" _ "github.com/google/yamlfmt/cmd/yamlfmt" _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway" _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2" _ "github.com/ryancurrah/gomodguard/cmd/gomodguard" _ "golang.org/x/tools/cmd/goimports" _ "google.golang.org/grpc/cmd/protoc-gen-go-grpc" _ "google.golang.org/protobuf/cmd/protoc-gen-go" _ "gotest.tools/gotestsum" _ "gotest.tools/v3" _ "honnef.co/go/tools/cmd/staticcheck" _ "go.etcd.io/gofail" _ "go.etcd.io/protodoc" _ "go.etcd.io/raft/v3" ) ================================================ FILE: tools/proto-annotations/cmd/etcd_version.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "fmt" "io" "slices" "sort" "strings" "github.com/coreos/go-semver/semver" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/reflect/protoregistry" "go.etcd.io/etcd/server/v3/storage/wal" ) // externalPackages that are not expected to have etcd version annotation. var externalPackages = []string{ "io.prometheus.client", "grpc.binarylog.v1", "google.protobuf", "google.rpc", "google.api", "raftpb", "grpc.gateway.protoc_gen_swagger.options", "grpc.gateway.protoc_gen_openapiv2.options", } // printEtcdVersion writes etcd_version proto annotation to stdout and returns any errors encountered when reading annotation. func printEtcdVersion() []error { var errs []error annotations, err := allEtcdVersionAnnotations() if err != nil { errs = append(errs, err) return errs } sort.Slice(annotations, func(i, j int) bool { return annotations[i].fullName < annotations[j].fullName }) output := &strings.Builder{} for _, a := range annotations { newErrs := a.Validate() if len(newErrs) == 0 { err := a.PrintLine(output) if err != nil { errs = append(errs, err) return errs } } errs = append(errs, newErrs...) } if len(errs) == 0 { fmt.Print(output) } return errs } func allEtcdVersionAnnotations() (annotations []etcdVersionAnnotation, err error) { var fileAnnotations []etcdVersionAnnotation protoregistry.GlobalFiles.RangeFiles(func(file protoreflect.FileDescriptor) bool { pkg := string(file.Package()) if slices.Contains(externalPackages, pkg) { return true } fileAnnotations, err = fileEtcdVersionAnnotations(file) if err != nil { return false } annotations = append(annotations, fileAnnotations...) return true }) return annotations, err } func fileEtcdVersionAnnotations(file protoreflect.FileDescriptor) (annotations []etcdVersionAnnotation, err error) { err = wal.VisitFileDescriptor(file, func(path protoreflect.FullName, ver *semver.Version) error { a := etcdVersionAnnotation{fullName: path, version: ver} annotations = append(annotations, a) return nil }) return annotations, err } type etcdVersionAnnotation struct { fullName protoreflect.FullName version *semver.Version } func (a etcdVersionAnnotation) Validate() (errs []error) { if a.version == nil { return nil } if a.version.Major == 0 { errs = append(errs, fmt.Errorf("%s: etcd_version major version should not be zero", a.fullName)) } if a.version.Patch != 0 { errs = append(errs, fmt.Errorf("%s: etcd_version patch version should be zero", a.fullName)) } if a.version.PreRelease != "" { errs = append(errs, fmt.Errorf("%s: etcd_version should not be prerelease", a.fullName)) } if a.version.Metadata != "" { errs = append(errs, fmt.Errorf("%s: etcd_version should not have metadata", a.fullName)) } return errs } func (a etcdVersionAnnotation) PrintLine(out io.Writer) error { if a.version == nil { _, err := fmt.Fprintf(out, "%s: \"\"\n", a.fullName) return err } _, err := fmt.Fprintf(out, "%s: \"%d.%d\"\n", a.fullName, a.version.Major, a.version.Minor) return err } ================================================ FILE: tools/proto-annotations/cmd/root.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "fmt" "os" "github.com/spf13/cobra" ) const ( EtcdVersionAnnotation = "etcd_version" ) func RootCmd() *cobra.Command { var annotation string cmd := &cobra.Command{ Use: "proto-annotation", Short: "Proto-annotations prints a dump of annotations used by all protobuf definitions used by Etcd.", Long: `Tool used to extract values of a specific proto annotation used by protobuf definitions used by Etcd. Created to ensure that all newly introduced proto definitions have a etcd_version_* annotation, by analysing diffs between generated by this tool. Proto annotations is printed to stdout in format: : "" For example: ''' etcdserverpb.Member: "3.0" etcdserverpb.Member.ID: "" etcdserverpb.Member.clientURLs: "" etcdserverpb.Member.isLearner: "3.4" etcdserverpb.Member.name: "" etcdserverpb.Member.peerURLs: "" ''' Any errors in proto will be printed to stderr. `, RunE: func(cmd *cobra.Command, args []string) error { return runProtoAnnotation(annotation) }, } cmd.Flags().StringVar(&annotation, "annotation", "", "Specify what proto annotation to read. Options: etcd_version") cmd.MarkFlagRequired("annotation") return cmd } func runProtoAnnotation(annotation string) error { var errs []error switch annotation { case EtcdVersionAnnotation: errs = printEtcdVersion() default: return fmt.Errorf("unknown annotation %q. Options: %q", annotation, EtcdVersionAnnotation) } if len(errs) != 0 { for _, err := range errs { fmt.Fprintln(os.Stderr, err) } return fmt.Errorf("failed reading anotation") } return nil } ================================================ FILE: tools/proto-annotations/main.go ================================================ // Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "fmt" "os" "go.etcd.io/etcd/v3/tools/proto-annotations/cmd" ) func main() { if err := cmd.RootCmd().Execute(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } ================================================ FILE: tools/testgrid-analysis/OWNERS ================================================ # See the OWNERS docs at https://go.k8s.io/owners labels: - area/testing ================================================ FILE: tools/testgrid-analysis/cmd/data.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "fmt" "io" "net/http" "os" "strings" "time" apipb "github.com/GoogleCloudPlatform/testgrid/pb/api/v1" statuspb "github.com/GoogleCloudPlatform/testgrid/pb/test_status" "google.golang.org/protobuf/encoding/protojson" ) var ( validTestStatuses = []statuspb.TestStatus{statuspb.TestStatus_PASS, statuspb.TestStatus_FAIL, statuspb.TestStatus_FLAKY} failureTestStatuses = []statuspb.TestStatus{statuspb.TestStatus_FAIL, statuspb.TestStatus_FLAKY} validTestStatusesInt = intStatusSet(validTestStatuses) failureTestStatusesInt = intStatusSet(failureTestStatuses) skippedTestStatuses = make(map[int32]struct{}) ) type TestResultSummary struct { Name string FullName string TotalRuns, FailedRuns int FailureRate float32 FailureLogs []string IssueBody string } func fetchTestResultSummaries(dashboard, tab string) []*TestResultSummary { // Fetch test data rowsURL := fmt.Sprintf("http://testgrid-data.k8s.io/api/v1/dashboards/%s/tabs/%s/rows", dashboard, tab) headersURL := fmt.Sprintf("http://testgrid-data.k8s.io/api/v1/dashboards/%s/tabs/%s/headers", dashboard, tab) var testData apipb.ListRowsResponse var headerData apipb.ListHeadersResponse protojson.Unmarshal(fetchJSON(rowsURL), &testData) protojson.Unmarshal(fetchJSON(headersURL), &headerData) var allTests []string for _, row := range testData.Rows { allTests = append(allTests, row.Name) } summaries := []*TestResultSummary{} // Process rows for _, row := range testData.Rows { t := processRow(dashboard, tab, row, allTests, headerData.Headers) summaries = append(summaries, t) } return summaries } func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests []string, headers []*apipb.ListHeadersResponse_Header) *TestResultSummary { t := TestResultSummary{Name: shortenTestName(row.Name), FullName: row.Name} // we do not want to create issues for a parent test. if isParentTest(row.Name, allTests) { return &t } if !strings.HasPrefix(row.Name, "go.etcd.io") { return &t } earliestTimeToConsider := time.Now().AddDate(0, 0, -1*maxDays) total := 0 failed := 0 logs := []string{} for i, cell := range row.Cells { // ignore tests with status not in the validTestStatuses // cell result codes are listed in https://github.com/GoogleCloudPlatform/testgrid/blob/main/pb/test_status/test_status.proto if _, ok := validTestStatusesInt[cell.Result]; !ok { if cell.Result != 0 { skippedTestStatuses[cell.Result] = struct{}{} } continue } header := headers[i] if maxDays > 0 && header.Started.AsTime().Before(earliestTimeToConsider) { continue } total++ if _, ok := failureTestStatusesInt[cell.Result]; ok { failed++ // markdown table format of | commit | log | logs = append(logs, fmt.Sprintf("| %s | %s | https://prow.k8s.io/view/gs/kubernetes-jenkins/logs/%s/%s |", strings.Join(header.Extra, ","), header.Started.AsTime().String(), tab, header.Build)) } if maxRuns > 0 && total >= maxRuns { break } } t.FailedRuns = failed t.TotalRuns = total t.FailureLogs = logs t.FailureRate = float32(failed) / float32(total) if t.FailedRuns > 0 { dashboardURL := fmt.Sprintf("[%s](https://testgrid.k8s.io/%s#%s)", tab, dashboard, tab) t.IssueBody = fmt.Sprintf("## %s Test: %s \nTest failed %.1f%% (%d/%d) of the time\n\nfailure logs are:\n| commit | started | log |\n| --- | --- | --- |\n%s\n", dashboardURL, t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns, strings.Join(t.FailureLogs, "\n")) t.IssueBody += "\nPlease follow the [instructions in the contributing guide](https://github.com/etcd-io/etcd/blob/main/CONTRIBUTING.md#check-for-flaky-tests) to reproduce the issue.\n" fmt.Printf("%s failed %.1f%% (%d/%d) of the time\n", t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns) } return &t } // isParentTest checks if a test is a rollup of some child tests. func isParentTest(test string, allTests []string) bool { for _, t := range allTests { if t != test && strings.HasPrefix(t, test+"/") { return true } } return false } func fetchJSON(url string) []byte { resp, err := http.Get(url) if err != nil { fmt.Println("Error fetching test data:", err) os.Exit(1) } defer resp.Body.Close() testBody, _ := io.ReadAll(resp.Body) return testBody } // intStatusSet converts a list of statuspb.TestStatus into a set of int. func intStatusSet(statuses []statuspb.TestStatus) map[int32]struct{} { s := make(map[int32]struct{}) for _, status := range statuses { s[int32(status)] = struct{}{} } return s } func shortenTestName(fullname string) string { parts := strings.Split(fullname, ".") return parts[len(parts)-1] } ================================================ FILE: tools/testgrid-analysis/cmd/flaky.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "fmt" "github.com/spf13/cobra" ) // flakyCmd represents the flaky command var flakyCmd = &cobra.Command{ Use: "flaky", Short: "detect flaky tests", Long: `detect flaky tests within the dashobard#tab, and create GitHub issues if desired.`, Run: flakyFunc, } var ( flakyThreshold float32 minRuns int maxRuns int maxDays int createGithubIssue bool githubOwner string githubRepo string lineSep = "-------------------------------------------------------------" ) func init() { rootCmd.AddCommand(flakyCmd) flakyCmd.Flags().BoolVar(&createGithubIssue, "create-issue", false, "create Github issue for each flaky test") flakyCmd.Flags().Float32Var(&flakyThreshold, "flaky-threshold", 0.1, "fraction threshold of test failures for a test to be considered flaky") flakyCmd.Flags().IntVar(&minRuns, "min-runs", 20, "minimum test runs for a test to be included in flaky analysis") flakyCmd.Flags().IntVar(&maxRuns, "max-runs", 0, "maximum test runs for a test to be included in flaky analysis, 0 to include all") flakyCmd.Flags().IntVar(&maxDays, "max-days", 0, "maximum days of results before today to be included in flaky analysis, 0 to include all") flakyCmd.Flags().StringVar(&githubOwner, "github-owner", "etcd-io", "the github organization to create the issue for") flakyCmd.Flags().StringVar(&githubRepo, "github-repo", "etcd", "the github repo to create the issue for") } func flakyFunc(cmd *cobra.Command, args []string) { fmt.Printf("flaky called, for %s#%s, createGithubIssue=%v, githubRepo=%s/%s, flakyThreshold=%f, minRuns=%d\n", dashboard, tab, createGithubIssue, githubOwner, githubRepo, flakyThreshold, minRuns) allTests := fetchTestResultSummaries(dashboard, tab) flakyTests := []*TestResultSummary{} for _, t := range allTests { if t.TotalRuns >= minRuns && t.FailureRate >= flakyThreshold { flakyTests = append(flakyTests, t) } } fmt.Println(lineSep) fmt.Printf("Detected total %d flaky tests above the %.0f%% threshold for %s#%s\n", len(flakyTests), flakyThreshold*100, dashboard, tab) fmt.Println(lineSep) if len(flakyTests) == 0 { return } for _, t := range flakyTests { fmt.Println(lineSep) fmt.Println(t.IssueBody) fmt.Println(lineSep) } if createGithubIssue { createIssues(flakyTests, []string{"type/flake"}) } } ================================================ FILE: tools/testgrid-analysis/cmd/github.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "context" "fmt" "os" "strings" "github.com/google/go-github/v60/github" ) func createIssues(tests []*TestResultSummary, labels []string) { openIssues := getOpenIssues(labels) for _, t := range tests { createIssueIfNonExist(t, openIssues, append(labels, "help wanted")) } } func getOpenIssues(labels []string) []*github.Issue { client := github.NewClient(nil).WithAuthToken(os.Getenv("GITHUB_TOKEN")) ctx := context.Background() // list open issues with label type/flake issueOpt := &github.IssueListByRepoOptions{ Labels: labels, ListOptions: github.ListOptions{PerPage: 100}, } allIssues := []*github.Issue{} for { issues, resp, err := client.Issues.ListByRepo(ctx, githubOwner, githubRepo, issueOpt) if err != nil { panic(err) } allIssues = append(allIssues, issues...) if resp.NextPage == 0 { break } issueOpt.Page = resp.NextPage } fmt.Printf("There are %d issues open with label %v\n", len(allIssues), labels) return allIssues } func createIssueIfNonExist(t *TestResultSummary, issues []*github.Issue, labels []string) { // check if there is already an open issue regarding this test for _, issue := range issues { if strings.Contains(*issue.Title, t.Name) { fmt.Printf("%s is already open for test %s\n\n", issue.GetHTMLURL(), t.Name) return } } fmt.Printf("Opening new issue for %s\n", t.Name) client := github.NewClient(nil).WithAuthToken(os.Getenv("GITHUB_TOKEN")) ctx := context.Background() req := &github.IssueRequest{ Title: github.String(fmt.Sprintf("Flaky test %s", t.Name)), Body: &t.IssueBody, Labels: &labels, } issue, _, err := client.Issues.Create(ctx, githubOwner, githubRepo, req) if err != nil { panic(err) } fmt.Printf("New issue %s created for %s\n\n", issue.GetHTMLURL(), t.Name) } ================================================ FILE: tools/testgrid-analysis/cmd/root.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cmd import ( "os" "github.com/spf13/cobra" ) var ( dashboard string tab string ) var rootCmd = &cobra.Command{ Use: "testgrid-analysis", Short: "testgrid-analysis", Long: `testgrid-analysis analyzes the testgrid test results of sig-etcd.`, } func Execute() { err := rootCmd.Execute() if err != nil { os.Exit(1) } } func init() { rootCmd.PersistentFlags().StringVar(&dashboard, "dashboard", "sig-etcd-periodics", "testgrid dashboard to retrieve data from") rootCmd.PersistentFlags().StringVar(&tab, "tab", "ci-etcd-e2e-amd64", "testgrid tab within the dashboard to retrieve data from") } ================================================ FILE: tools/testgrid-analysis/go.mod ================================================ module go.etcd.io/etcd/tools/testgrid-analysis/v3 go 1.26 toolchain go1.26.1 require ( github.com/GoogleCloudPlatform/testgrid v0.0.173 github.com/google/go-github/v60 v60.0.0 github.com/spf13/cobra v1.10.2 google.golang.org/protobuf v1.36.11 ) require ( github.com/google/go-querystring v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.10 // indirect go.opentelemetry.io/otel v1.42.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect google.golang.org/grpc v1.79.2 // indirect ) ================================================ FILE: tools/testgrid-analysis/go.sum ================================================ bitbucket.org/creachadair/stringset v0.0.11/go.mod h1:wh0BHewFe+j0HrzWz7KcGbSNpFzWwnpmgPRlB57U5jU= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps= cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= cloud.google.com/go/aiplatform v1.48.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= cloud.google.com/go/analytics v0.21.3/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= cloud.google.com/go/baremetalsolution v1.1.1/go.mod h1:D1AV6xwOksJMV4OSlWHtWuFNZZYujJknMAP4Qa27QIA= cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= cloud.google.com/go/batch v1.3.1/go.mod h1:VguXeQKXIYaeeIYbuozUmBR13AfL4SJP7IltNPS+A4A= cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= cloud.google.com/go/beyondcorp v0.6.1/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= cloud.google.com/go/beyondcorp v1.0.0/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= cloud.google.com/go/bigquery v1.53.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= cloud.google.com/go/cloudbuild v1.13.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= cloud.google.com/go/cloudtasks v1.12.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= cloud.google.com/go/contactcenterinsights v1.10.0/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= cloud.google.com/go/container v1.24.0/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= cloud.google.com/go/dataplex v1.9.0/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= cloud.google.com/go/dataproc/v2 v2.0.1/go.mod h1:7Ez3KRHdFGcfY7GcevBbvozX+zyWGcwLJvvAMwCaoZ4= cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= cloud.google.com/go/datastore v1.12.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= cloud.google.com/go/datastream v1.10.0/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= cloud.google.com/go/deploy v1.13.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= cloud.google.com/go/dialogflow v1.40.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= cloud.google.com/go/documentai v1.22.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= cloud.google.com/go/eventarc v1.13.0/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= cloud.google.com/go/gaming v1.10.1/go.mod h1:XQQvtfP8Rb9Rxnxm5wFVpAp9zCQkJi2bLIb7iHGwB3s= cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= cloud.google.com/go/gkebackup v1.3.0/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= cloud.google.com/go/gkemulticloud v1.0.0/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= cloud.google.com/go/kms v1.11.0/go.mod h1:hwdiYC0xjnWsKQQCQQmIQnS9asjYVSK6jtXm+zFqXLM= cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/longrunning v0.4.2/go.mod h1:OHrnaYyLUV6oqwh0xiS7e5sLQhP1m0QU9R+WhGDMgIQ= cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= cloud.google.com/go/maps v1.3.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= cloud.google.com/go/maps v1.4.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= cloud.google.com/go/metastore v1.12.0/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= cloud.google.com/go/orgpolicy v1.11.0/go.mod h1:2RK748+FtVvnfuynxBzdnyu7sygtoZa1za/0ZfpOs1M= cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= cloud.google.com/go/osconfig v1.12.0/go.mod h1:8f/PaYzoS3JMVfdfTubkowZYGmAhUCjjwnjqWI7NVBc= cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0= cloud.google.com/go/policytroubleshooter v1.8.0/go.mod h1:tmn5Ir5EToWe384EuboTcVQT7nTag2+DuH3uHmKd1HU= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= cloud.google.com/go/run v1.2.0/go.mod h1:36V1IlDzQ0XxbQjUx6IYbw8H3TJnWvhii963WW3B/bo= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= cloud.google.com/go/servicedirectory v1.11.0/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= cloud.google.com/go/speech v1.19.0/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= cloud.google.com/go/translate v1.8.2/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= cloud.google.com/go/video v1.19.0/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= cloud.google.com/go/vmwareengine v1.0.0/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GoogleCloudPlatform/testgrid v0.0.173 h1:NyHqWe5gJW9G22VjwTBYpQniMfgQFKNm5OpLaRop97s= github.com/GoogleCloudPlatform/testgrid v0.0.173/go.mod h1:lOKP2QzzzIDB4D0nJs1BcNMzJErjrlTNqG3vsCddx8c= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v60 v60.0.0 h1:oLG98PsLauFvvu4D/YPxq374jhSxFYdzQGNCyONLfn8= github.com/google/go-github/v60 v60.0.0/go.mod h1:ByhX2dP9XT9o/ll2yXAu2VD8l5eNVg8hD4Cr0S/LmQk= github.com/google/go-pkcs11 v0.2.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjYK+5E= google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y= google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108= google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230720185612-659f7aaaa771/go.mod h1:3QoBVwTHkXbY1oRGzlhwhOykfcATQN43LJ6iT8Wy8kE= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8s.io/api v0.27.4/go.mod h1:O3smaaX15NfxjzILfiln1D8Z3+gEYpjEpiNA/1EVK1Y= k8s.io/apimachinery v0.27.4/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI= modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA= modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0= modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= ================================================ FILE: tools/testgrid-analysis/main.go ================================================ // Copyright 2024 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import "go.etcd.io/etcd/tools/testgrid-analysis/v3/cmd" func main() { cmd.Execute() }